diff --git a/go.mod b/go.mod index 2c02b7c..02b1002 100644 --- a/go.mod +++ b/go.mod @@ -3,38 +3,36 @@ module main go 1.19 require ( - github.com/cockroachdb/errors v1.9.0 - github.com/cockroachdb/pebble v0.0.0-20221103174942-7486bbcf2861 + github.com/cockroachdb/errors v1.9.1 + github.com/cockroachdb/pebble v0.0.0-20230517202617-f6eaf9a696e6 github.com/go-redis/redis/v8 v8.11.5 github.com/hjson/hjson-go v3.3.0+incompatible - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.2 ) require ( - github.com/DataDog/zstd v1.5.2 // indirect + github.com/DataDog/zstd v1.5.5 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.1.2 // indirect - github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect - github.com/cockroachdb/redact v1.1.3 // indirect - github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect - github.com/codahale/hdrhistogram v0.9.0 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/redact v1.1.4 // indirect github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect - github.com/getsentry/sentry-go v0.14.0 // indirect + github.com/getsentry/sentry-go v0.21.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect - github.com/klauspost/compress v1.15.12 // indirect + github.com/klauspost/compress v1.16.5 // indirect github.com/kr/pretty v0.3.1 // indirect github.com/kr/text v0.2.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/pkg/errors v0.9.1 // indirect - github.com/prometheus/client_golang v1.13.1 // indirect - github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect - github.com/rogpeppe/go-internal v1.9.0 // indirect - golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 // indirect - golang.org/x/sys v0.1.0 // indirect - golang.org/x/text v0.4.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + github.com/prometheus/client_golang v1.15.1 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.43.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect + github.com/rogpeppe/go-internal v1.10.0 // indirect + golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect ) diff --git a/go.sum b/go.sum index b603a0c..2f361ba 100644 --- a/go.sum +++ b/go.sum @@ -1,99 +1,34 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= -cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= -cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= -cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= -cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= -cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= -cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= -cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= -cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= -cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= -cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= -cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= -cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= -cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= -cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= -cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= -cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= -cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= -cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= -cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= -cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= -cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= -cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= -cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= -cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/CloudyKit/fastprinter v0.0.0-20170127035650-74b38d55f37a/go.mod h1:EFZQ978U7x8IRnstaskI3IysnWY5Ao3QgZUKOXlsAdw= github.com/CloudyKit/fastprinter v0.0.0-20200109182630-33d98a066a53/go.mod h1:+3IMCy2vIlbG1XG/0ggNQv0SvxCAIpPM5b1nCz56Xno= -github.com/CloudyKit/jet v2.1.3-0.20180809161101-62edd43e4f88+incompatible/go.mod h1:HPYO+50pSWkPoj9Q/eq0aRGByCL6ScRlUmiEX5Zgm+w= github.com/CloudyKit/jet/v3 v3.0.0/go.mod h1:HKQPgSJmdK8hdoAbKUUWajkHyHo4RaU5rMdUywE7VMo= -github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= -github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= -github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= -github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.5 h1:oWf5W7GtOLgp6bciQYDmhHHjdhYkALu6S/5Ni9ZgSvQ= +github.com/DataDog/zstd v1.5.5/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Joker/hpp v1.0.0/go.mod h1:8x5n+M1Hp5hC0g8okX3sR3vFQwynaX/UgSOM9MeBKzY= -github.com/Joker/jade v1.0.1-0.20190614124447-d475f43051e7/go.mod h1:6E6s8o2AE4KhCrqr6GRJjdC/gNfTdxkIXvuGZZda2VM= github.com/Shopify/goreferrer v0.0.0-20181106222321-ec9c9a553398/go.mod h1:a1uqRtAwp2Xwc6WNPJEufxJ7fx3npB4UV/JOLmbu5I0= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/aymerick/raymond v2.0.3-0.20180322193309-b565731e1464+incompatible/go.mod h1:osfaiScAUVup+UC9Nfq76eWqDhXlp+4UYaA8uhTBO6g= -github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= -github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= -github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= -github.com/cockroachdb/datadriven v1.0.1-0.20211007161720-b558070c3be0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= -github.com/cockroachdb/datadriven v1.0.1-0.20220214170620-9913f5bc19b7/go.mod h1:hi0MtSY3AYDQNDi83kDkMH5/yqM/CsIrsOITkSoH7KI= -github.com/cockroachdb/errors v1.6.1/go.mod h1:tm6FTP5G81vwJ5lC0SizQo374JNCOPrHyXGitRJoDqM= -github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= -github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= -github.com/cockroachdb/errors v1.8.8/go.mod h1:z6VnEL3hZ/2ONZEvG7S5Ym0bU2AqPcEKnIiA1wbsSu0= -github.com/cockroachdb/errors v1.9.0 h1:B48dYem5SlAY7iU8AKsgedb4gH6mo+bDkbtLIvM/a88= -github.com/cockroachdb/errors v1.9.0/go.mod h1:vaNcEYYqbIqB5JhKBhFV9CneUqeuEbB2OYJBK4GBNYQ= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f h1:6jduT9Hfc0njg5jJ1DdKCFPdMBrp/mdZfCpa5h+WM74= +github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20220513193540-b8c9a560bed5 h1:+v7nOUh8/HqhcK5lzgE0cLqNDrMlscfuNCzo8xiwILg= -github.com/cockroachdb/pebble v0.0.0-20220513193540-b8c9a560bed5/go.mod h1:buxOO9GBtOcq1DiXDpIPYrmxY020K2A8lOrwno5FetU= -github.com/cockroachdb/pebble v0.0.0-20221103174942-7486bbcf2861 h1:YarvrMA6pv032mYkBakTCkzqKpatcAZrdIknHJv+yVg= -github.com/cockroachdb/pebble v0.0.0-20221103174942-7486bbcf2861/go.mod h1:qf9bLis2yy1XyNYD01wvIHPabuC1STzQsvGibYVsom4= -github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= -github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v0.0.0-20230517202617-f6eaf9a696e6 h1:qmPygPODBjUWt7dT3kR118Yn7y0Ol6ULetf78jj/fNQ= +github.com/cockroachdb/pebble v0.0.0-20230517202617-f6eaf9a696e6/go.mod h1:TkdVsGYRqtULUppt2RbC+YaKtTHnHoWa2apfFrSKABw= github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T2OTKmB4acZcyKaMtRnY5Y44NuXGX2GFJ1w= -github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/codahale/hdrhistogram v0.9.0 h1:9GjrtRI+mLEFPtTfR/AZhcxp+Ii8NZYWq5104FbZQY0= -github.com/codahale/hdrhistogram v0.9.0/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/cockroachdb/redact v1.1.4 h1:Y0XrVI2FAyofNyGveodTN//qdpPtFKcKTeCBsK3AHAQ= +github.com/cockroachdb/redact v1.1.4/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -104,7 +39,6 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= -github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= @@ -112,71 +46,40 @@ github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25Kn github.com/eknkc/amber v0.0.0-20171010120322-cdade1c07385/go.mod h1:0vRUJqYpeSZifjYj7uP3BG/gKcuzL9xWVV/Y+cK33KM= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= -github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/etcd-io/bbolt v1.3.3/go.mod h1:ZF2nL25h33cCyBtcyWeZ2/I3HQOfTP+0PIEvHjkjCrw= github.com/fasthttp-contrib/websocket v0.0.0-20160511215533-1f3b11f56072/go.mod h1:duJ4Jxv5lDcvg4QuQr0oowTf7dz4/CR8NtyCooz9HL8= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= -github.com/flosch/pongo2 v0.0.0-20190707114632-bbf5a6c351f4/go.mod h1:T9YF2M40nIgbVgp3rreNmTged+9HrbNTIQf1PsaIiTA= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= -github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/getsentry/sentry-go v0.12.0/go.mod h1:NSap0JBYWzHND8oMbyi0+XZhUalc1TBdRL1M71JZW2c= -github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70= -github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I= -github.com/ghemawat/stream v0.0.0-20171120220530-696b145b53b9/go.mod h1:106OIgooyS7OzLDOpUGgm9fA3bQENb/cFSyyBmMoJDs= +github.com/getsentry/sentry-go v0.21.0 h1:c9l5F1nPF30JIppulk4veau90PK6Smu3abgVtVQWon4= +github.com/getsentry/sentry-go v0.21.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= -github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= -github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= -github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= -github.com/go-kit/log v0.2.0/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= -github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= -github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gogo/googleapis v0.0.0-20180223154316-0cd9801be74a/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= -github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= -github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= -github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -184,114 +87,62 @@ github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:W github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= -github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= -github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= -github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= -github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= -github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= -github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= -github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hjson/hjson-go v3.1.0+incompatible h1:DY/9yE8ey8Zv22bY+mHV1uk2yRy0h8tKhZ77hEdi0Aw= -github.com/hjson/hjson-go v3.1.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/hjson/hjson-go v3.3.0+incompatible h1:Rqr+Ya+0aCJMjaE4s8E9YKvuJLuLVpEvz4ONum52vnI= github.com/hjson/hjson-go v3.3.0+incompatible/go.mod h1:qsetwF8NlsTsOTwZTApNlTCerV+b2GjYRRcIk4JMFio= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hydrogen18/memlistener v0.0.0-20141126152155-54553eb933fb/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= github.com/hydrogen18/memlistener v0.0.0-20200120041712-dcc25e7acd91/go.mod h1:qEIFzExnS6016fRpRfxrExeVn2gbClQA99gQhnIcdhE= -github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imkira/go-interpol v1.1.0/go.mod h1:z0h2/2T3XF8kyEPpRgJ3kmNv+C43p+I/CoI+jC3w2iA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/iris-contrib/blackfriday v2.0.0+incompatible/go.mod h1:UzZ2bDEoaSGPbkg6SAB4att1aAwTmVIx/5gCVqeyUdI= github.com/iris-contrib/go.uuid v2.0.0+incompatible/go.mod h1:iz2lgM/1UnEf1kP0L/+fafWORmlnuysV2EMP8MW+qe0= -github.com/iris-contrib/i18n v0.0.0-20171121225848-987a633949d0/go.mod h1:pMCz62A0xJL6I+umB2YTlFRwWXaDFA0jy+5HzGiJjqI= github.com/iris-contrib/jade v1.1.3/go.mod h1:H/geBymxJhShH5kecoiOCSssPX7QWYH7UaeZTSWddIk= github.com/iris-contrib/pongo2 v0.0.1/go.mod h1:Ssh+00+3GAZqSQb30AvBRNxBx7rf0GqwkjqxNd0u65g= github.com/iris-contrib/schema v0.0.1/go.mod h1:urYA3uvUNG1TIIjOSCzHr9/LmbQo8LrOcOqfqxa4hXw= -github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= -github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= -github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= -github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= -github.com/kataras/golog v0.0.9/go.mod h1:12HJgwBIZFNGL0EJnMRhmvGA0PQGx8VFwrZtM4CqbAk= github.com/kataras/golog v0.0.10/go.mod h1:yJ8YKCmyL+nWjERB90Qwn+bdyBZsaQwU3bTVFgkFIp8= -github.com/kataras/iris/v12 v12.0.1/go.mod h1:udK4vLQKkdDqMGJJVd/msuMtN6hpYJhg/lSzuxjhO+U= github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYbq3UhfoFmE= -github.com/kataras/neffos v0.0.10/go.mod h1:ZYmJC07hQPW67eKuzlfY7SO3bC0mw83A3j6im82hfqw= github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= -github.com/kataras/pio v0.0.0-20190103105442-ea782b38602d/go.mod h1:NV88laa9UiiDuX9AhMbDPkGYSPugBOV6yTZB1l2K9Z0= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= -github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.8.2/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.9.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.7/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= -github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= -github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v1.2.1/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g= github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= @@ -304,11 +155,8 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= -github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= -github.com/mediocregopher/mediocre-go-lib v0.0.0-20181029021733-cb65787f37ed/go.mod h1:dSsfyI2zABAdhcbvkXqgxOxrCsbYeHCPgrZkku60dSg= -github.com/mediocregopher/radix/v3 v3.3.0/go.mod h1:EmfVyvspXz1uZEyPBMyGK+kjWiKQGvsUt6O3Pj+LDCQ= github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -317,81 +165,47 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/moul/http2curl v1.0.0/go.mod h1:8UbvGypXm98wA/IqH45anm5Y2Z6ep6O31QGOAZ3H0fQ= -github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= -github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/nats.go v1.8.1/go.mod h1:BrFz9vVn0fU3AcH9Vn4Kd7W0NpJ651tD5omQ3M8LwxM= github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.0.2/go.mod h1:dab7URMsZm6Z/jp9Z5UGa87Uutgc2mVpXLC4B7TDb/4= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/ginkgo v1.13.0/go.mod h1:+REjRxOmWfHCjfv9TTWB1jD1Frx4XydAD3zm1lskyM0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= -github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= -github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.13.1 h1:3gMjIY2+/hzmqhtUC/aQNYldJA6DtH3CgQvwS+02K1c= -github.com/prometheus/client_golang v1.13.1/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= -github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= -github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_golang v1.15.1 h1:8tXpTmJbyH5lydzFPoxSIJ0J46jdh3tylbvM1xCv0LI= +github.com/prometheus/client_golang v1.15.1/go.mod h1:e9yaBhRPU2pPNsZwE+JdQl0KEt1N9XgF6zxWmaC0xOk= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= -github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.43.0 h1:iq+BVjvYLei5f27wiuNiB1DN6DYQkp1c8Bx0Vykh5us= +github.com/prometheus/common v0.43.0/go.mod h1:NCvr5cQIh3Y/gy73/RdVtC9r8xxrxwJnB+2lB3BxrFc= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/schollz/closestmatch v2.1.0+incompatible/go.mod h1:RtP1ddjLong6gTkbtmuhtR2uUrrJOpYzYRvbcPAid+g= -github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= -github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= -github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= -github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= +github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= @@ -401,14 +215,12 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -427,21 +239,11 @@ github.com/yalp/jsonpath v0.0.0-20180802001716-5cc68e5049a0/go.mod h1:/LWChgwKmv github.com/yudai/gojsondiff v1.0.0/go.mod h1:AY32+k2cwILAkW1fbgxQ5mUmMiZFgLIV+FBNExI05xg= github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82/go.mod h1:lgjkn3NuSvDfVJdfcVVdX+jpBxNmX4rDAzaS45IcYoM= github.com/yudai/pp v2.0.1+incompatible/go.mod h1:PuxR/8QJ7cyCkFp/aUDS+JY727OFEZkTdatxwunjIkc= -github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= -github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= -go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191227163750-53104e6ec876/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -449,299 +251,107 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= -golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= -golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= -golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= -golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= -golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20200513190911-00229845015e h1:rMqLP+9XLy+LdbCXHjJHAmTfXCr93W7oruWA6Hq1Alc= -golang.org/x/exp v0.0.0-20200513190911-00229845015e/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= -golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 h1:QfTh0HpN6hlw6D3vu8DAwC8pBIwikq0AI1evdm+FksE= -golang.org/x/exp v0.0.0-20221031165847-c99f073a8326/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= -golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= +golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= -golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= -golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= -golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= -golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= -golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= -golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= -golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190327091125-710a502c58a2/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= -golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= -golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211008194852-3b03d305991f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= -golang.org/x/net v0.0.0-20221002022538-bcab6841153b h1:6e93nYa3hNqAvLr0pD4PN1fFS+gKzp2zAXqrnTCstqU= +golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210909193231-528a39cd75f3/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181221001348-537d06c36207/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190327201419-c70d86f8b7cf/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= -golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= -golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= -golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= -golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= -google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= -google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= -google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= -google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= -google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= -google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= -google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180518175338-11a468237815/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= -google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= -google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= -google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= -google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= -google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= -google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= -google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= -google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= -google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -751,13 +361,11 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= -google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -769,24 +377,12 @@ gopkg.in/ini.v1 v1.51.1/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= -gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20191120175047-4206685974f2/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= -honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= -honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= -rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/vendor/github.com/DataDog/zstd/ZSTD_LICENSE b/vendor/github.com/DataDog/zstd/ZSTD_LICENSE index a793a80..7580028 100644 --- a/vendor/github.com/DataDog/zstd/ZSTD_LICENSE +++ b/vendor/github.com/DataDog/zstd/ZSTD_LICENSE @@ -2,7 +2,7 @@ BSD License For Zstandard software -Copyright (c) 2016-present, Facebook, Inc. All rights reserved. +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: @@ -14,9 +14,9 @@ are permitted provided that the following conditions are met: this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. - * Neither the name Facebook nor the names of its contributors may be used to - endorse or promote products derived from this software without specific - prior written permission. + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED diff --git a/vendor/github.com/DataDog/zstd/allocations.h b/vendor/github.com/DataDog/zstd/allocations.h new file mode 100644 index 0000000..5289f24 --- /dev/null +++ b/vendor/github.com/DataDog/zstd/allocations.h @@ -0,0 +1,58 @@ +#ifndef USE_EXTERNAL_ZSTD +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +/* This file provides custom allocation primitives + */ + +#define ZSTD_DEPS_NEED_MALLOC +#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ + +#include "mem.h" /* MEM_STATIC */ +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" /* ZSTD_customMem */ + +#ifndef ZSTD_ALLOCATIONS_H +#define ZSTD_ALLOCATIONS_H + +/* custom memory allocation functions */ + +MEM_STATIC void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) + return customMem.customAlloc(customMem.opaque, size); + return ZSTD_malloc(size); +} + +MEM_STATIC void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) +{ + if (customMem.customAlloc) { + /* calloc implemented as malloc+memset; + * not as efficient as calloc, but next best guess for custom malloc */ + void* const ptr = customMem.customAlloc(customMem.opaque, size); + ZSTD_memset(ptr, 0, size); + return ptr; + } + return ZSTD_calloc(1, size); +} + +MEM_STATIC void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) +{ + if (ptr!=NULL) { + if (customMem.customFree) + customMem.customFree(customMem.opaque, ptr); + else + ZSTD_free(ptr); + } +} + +#endif /* ZSTD_ALLOCATIONS_H */ + +#endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/bits.h b/vendor/github.com/DataDog/zstd/bits.h new file mode 100644 index 0000000..59856e4 --- /dev/null +++ b/vendor/github.com/DataDog/zstd/bits.h @@ -0,0 +1,203 @@ +#ifndef USE_EXTERNAL_ZSTD +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_BITS_H +#define ZSTD_BITS_H + +#include "mem.h" + +MEM_STATIC unsigned ZSTD_countTrailingZeros32_fallback(U32 val) +{ + assert(val != 0); + { + static const U32 DeBruijnBytePos[32] = {0, 1, 28, 2, 29, 14, 24, 3, + 30, 22, 20, 15, 25, 17, 4, 8, + 31, 27, 13, 23, 21, 19, 16, 7, + 26, 12, 18, 6, 11, 5, 10, 9}; + return DeBruijnBytePos[((U32) ((val & -(S32) val) * 0x077CB531U)) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_ctz(val); +# else + return ZSTD_countTrailingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32_fallback(U32 val) { + assert(val != 0); + { + static const U32 DeBruijnClz[32] = {0, 9, 1, 10, 13, 21, 2, 29, + 11, 14, 16, 18, 22, 25, 3, 30, + 8, 12, 20, 28, 15, 17, 24, 7, + 19, 27, 23, 6, 26, 5, 4, 31}; + val |= val >> 1; + val |= val >> 2; + val |= val >> 4; + val |= val >> 8; + val |= val >> 16; + return 31 - DeBruijnClz[(val * 0x07C4ACDDU) >> 27]; + } +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros32(U32 val) +{ + assert(val != 0); +# if defined(_MSC_VER) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u32(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse(&r, val); + return (unsigned)(31 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)__builtin_clz(val); +# else + return ZSTD_countLeadingZeros32_fallback(val); +# endif +} + +MEM_STATIC unsigned ZSTD_countTrailingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_tzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanForward64(&r, val); + return (unsigned)r; + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) && defined(__LP64__) + return (unsigned)__builtin_ctzll(val); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (leastSignificantWord == 0) { + return 32 + ZSTD_countTrailingZeros32(mostSignificantWord); + } else { + return ZSTD_countTrailingZeros32(leastSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_countLeadingZeros64(U64 val) +{ + assert(val != 0); +# if defined(_MSC_VER) && defined(_WIN64) +# if STATIC_BMI2 == 1 + return (unsigned)_lzcnt_u64(val); +# else + if (val != 0) { + unsigned long r; + _BitScanReverse64(&r, val); + return (unsigned)(63 - r); + } else { + /* Should not reach this code path */ + __assume(0); + } +# endif +# elif defined(__GNUC__) && (__GNUC__ >= 4) + return (unsigned)(__builtin_clzll(val)); +# else + { + U32 mostSignificantWord = (U32)(val >> 32); + U32 leastSignificantWord = (U32)val; + if (mostSignificantWord == 0) { + return 32 + ZSTD_countLeadingZeros32(leastSignificantWord); + } else { + return ZSTD_countLeadingZeros32(mostSignificantWord); + } + } +# endif +} + +MEM_STATIC unsigned ZSTD_NbCommonBytes(size_t val) +{ + if (MEM_isLittleEndian()) { + if (MEM_64bits()) { + return ZSTD_countTrailingZeros64((U64)val) >> 3; + } else { + return ZSTD_countTrailingZeros32((U32)val) >> 3; + } + } else { /* Big Endian CPU */ + if (MEM_64bits()) { + return ZSTD_countLeadingZeros64((U64)val) >> 3; + } else { + return ZSTD_countLeadingZeros32((U32)val) >> 3; + } + } +} + +MEM_STATIC unsigned ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ +{ + assert(val != 0); + return 31 - ZSTD_countLeadingZeros32(val); +} + +/* ZSTD_rotateRight_*(): + * Rotates a bitfield to the right by "count" bits. + * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts + */ +MEM_STATIC +U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { + assert(count < 64); + count &= 0x3F; /* for fickle pattern recognition */ + return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); +} + +MEM_STATIC +U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { + assert(count < 32); + count &= 0x1F; /* for fickle pattern recognition */ + return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); +} + +MEM_STATIC +U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { + assert(count < 16); + count &= 0x0F; /* for fickle pattern recognition */ + return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +} + +#endif /* ZSTD_BITS_H */ + +#endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/bitstream.h b/vendor/github.com/DataDog/zstd/bitstream.h index 585507c..9d92f7b 100644 --- a/vendor/github.com/DataDog/zstd/bitstream.h +++ b/vendor/github.com/DataDog/zstd/bitstream.h @@ -2,7 +2,7 @@ /* ****************************************************************** * bitstream * Part of FSE library - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -31,14 +31,15 @@ extern "C" { #include "compiler.h" /* UNLIKELY() */ #include "debug.h" /* assert(), DEBUGLOG(), RAWLOG() */ #include "error_private.h" /* error codes and messages */ +#include "bits.h" /* ZSTD_highbit32 */ /*========================================= * Target specific =========================================*/ #ifndef ZSTD_NO_INTRINSICS -# if defined(__BMI__) && defined(__GNUC__) -# include /* support for bextr (experimental) */ +# if (defined(__BMI__) || defined(__BMI2__)) && defined(__GNUC__) +# include /* support for bextr (experimental)/bzhi */ # elif defined(__ICCARM__) # include # endif @@ -133,48 +134,6 @@ MEM_STATIC void BIT_flushBitsFast(BIT_CStream_t* bitC); MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits); /* faster, but works only if nbBits >= 1 */ - - -/*-************************************************************** -* Internal functions -****************************************************************/ -MEM_STATIC unsigned BIT_highbit32 (U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val) ^ 31; -# else - if (val != 0) { - unsigned long r; - _BitScanReverse(&r, val); - return (unsigned)r; - } else { - /* Should not reach this code path */ - __assume(0); - } -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* Use GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const unsigned DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, - 11, 14, 16, 18, 22, 25, 3, 30, - 8, 12, 20, 28, 15, 17, 24, 7, - 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[ (U32) (v * 0x07C4ACDDU) >> 27]; -# endif - } -} - /*===== Local Constants =====*/ static const unsigned BIT_mask[] = { 0, 1, 3, 7, 0xF, 0x1F, @@ -204,6 +163,16 @@ MEM_STATIC size_t BIT_initCStream(BIT_CStream_t* bitC, return 0; } +MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) +{ +#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 && !defined(ZSTD_NO_INTRINSICS) + return _bzhi_u64(bitContainer, nbBits); +#else + assert(nbBits < BIT_MASK_SIZE); + return bitContainer & BIT_mask[nbBits]; +#endif +} + /*! BIT_addBits() : * can add up to 31 bits into `bitC`. * Note : does not check for register overflow ! */ @@ -213,7 +182,7 @@ MEM_STATIC void BIT_addBits(BIT_CStream_t* bitC, DEBUG_STATIC_ASSERT(BIT_MASK_SIZE == 32); assert(nbBits < BIT_MASK_SIZE); assert(nbBits + bitC->bitPos < sizeof(bitC->bitContainer) * 8); - bitC->bitContainer |= (value & BIT_mask[nbBits]) << bitC->bitPos; + bitC->bitContainer |= BIT_getLowerBits(value, nbBits) << bitC->bitPos; bitC->bitPos += nbBits; } @@ -292,7 +261,7 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si bitD->ptr = (const char*)srcBuffer + srcSize - sizeof(bitD->bitContainer); bitD->bitContainer = MEM_readLEST(bitD->ptr); { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; /* ensures bitsConsumed is always set */ if (lastByte == 0) return ERROR(GENERIC); /* endMark not present */ } } else { bitD->ptr = bitD->start; @@ -320,7 +289,7 @@ MEM_STATIC size_t BIT_initDStream(BIT_DStream_t* bitD, const void* srcBuffer, si default: break; } { BYTE const lastByte = ((const BYTE*)srcBuffer)[srcSize-1]; - bitD->bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + bitD->bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; if (lastByte == 0) return ERROR(corruption_detected); /* endMark not present */ } bitD->bitsConsumed += (U32)(sizeof(bitD->bitContainer) - srcSize)*8; @@ -351,16 +320,6 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getMiddleBits(size_t bitContainer, U32 c #endif } -MEM_STATIC FORCE_INLINE_ATTR size_t BIT_getLowerBits(size_t bitContainer, U32 const nbBits) -{ -#if defined(STATIC_BMI2) && STATIC_BMI2 == 1 - return _bzhi_u64(bitContainer, nbBits); -#else - assert(nbBits < BIT_MASK_SIZE); - return bitContainer & BIT_mask[nbBits]; -#endif -} - /*! BIT_lookBits() : * Provides next n bits from local register. * local register is not modified. @@ -407,7 +366,7 @@ MEM_STATIC FORCE_INLINE_ATTR size_t BIT_readBits(BIT_DStream_t* bitD, unsigned n } /*! BIT_readBitsFast() : - * unsafe version; only works only if nbBits >= 1 */ + * unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, unsigned nbBits) { size_t const value = BIT_lookBitsFast(bitD, nbBits); @@ -438,7 +397,7 @@ MEM_STATIC BIT_DStream_status BIT_reloadDStreamFast(BIT_DStream_t* bitD) * This function is safe, it guarantees it will not read beyond src buffer. * @return : status of `BIT_DStream_t` internal register. * when status == BIT_DStream_unfinished, internal register is filled with at least 25 or 57 bits */ -MEM_STATIC BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) +MEM_STATIC FORCE_INLINE_ATTR BIT_DStream_status BIT_reloadDStream(BIT_DStream_t* bitD) { if (bitD->bitsConsumed > (sizeof(bitD->bitContainer)*8)) /* overflow detected, like end of stream */ return BIT_DStream_overflow; diff --git a/vendor/github.com/DataDog/zstd/clevels.h b/vendor/github.com/DataDog/zstd/clevels.h index d0c6205..9215306 100644 --- a/vendor/github.com/DataDog/zstd/clevels.h +++ b/vendor/github.com/DataDog/zstd/clevels.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/compiler.h b/vendor/github.com/DataDog/zstd/compiler.h index 1f25768..ffd66ca 100644 --- a/vendor/github.com/DataDog/zstd/compiler.h +++ b/vendor/github.com/DataDog/zstd/compiler.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -166,6 +166,12 @@ #define UNLIKELY(x) (x) #endif +#if __has_builtin(__builtin_unreachable) || (defined(__GNUC__) && (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 5))) +# define ZSTD_UNREACHABLE { assert(0), __builtin_unreachable(); } +#else +# define ZSTD_UNREACHABLE { assert(0); } +#endif + /* disable warnings */ #ifdef _MSC_VER /* Visual Studio */ # include /* For Visual 2005 */ @@ -182,6 +188,8 @@ # ifdef __AVX2__ //MSVC does not have a BMI2 specific flag, but every CPU that supports AVX2 also supports BMI2 # define STATIC_BMI2 1 # endif +# elif defined(__BMI2__) && defined(__x86_64__) && defined(__GNUC__) +# define STATIC_BMI2 1 # endif #endif @@ -274,7 +282,18 @@ * Sanitizer *****************************************************************/ -#if ZSTD_MEMORY_SANITIZER +/* Issue #3240 reports an ASAN failure on an llvm-mingw build. Out of an + * abundance of caution, disable our custom poisoning on mingw. */ +#ifdef __MINGW32__ +#ifndef ZSTD_ASAN_DONT_POISON_WORKSPACE +#define ZSTD_ASAN_DONT_POISON_WORKSPACE 1 +#endif +#ifndef ZSTD_MSAN_DONT_POISON_WORKSPACE +#define ZSTD_MSAN_DONT_POISON_WORKSPACE 1 +#endif +#endif + +#if ZSTD_MEMORY_SANITIZER && !defined(ZSTD_MSAN_DONT_POISON_WORKSPACE) /* Not all platforms that support msan provide sanitizers/msan_interface.h. * We therefore declare the functions we need ourselves, rather than trying to * include the header file... */ @@ -293,9 +312,13 @@ void __msan_poison(const volatile void *a, size_t size); /* Returns the offset of the first (at least partially) poisoned byte in the memory range, or -1 if the whole range is good. */ intptr_t __msan_test_shadow(const volatile void *x, size_t size); + +/* Print shadow and origin for the memory range to stderr in a human-readable + format. */ +void __msan_print_shadow(const volatile void *x, size_t size); #endif -#if ZSTD_ADDRESS_SANITIZER +#if ZSTD_ADDRESS_SANITIZER && !defined(ZSTD_ASAN_DONT_POISON_WORKSPACE) /* Not all platforms that support asan provide sanitizers/asan_interface.h. * We therefore declare the functions we need ourselves, rather than trying to * include the header file... */ diff --git a/vendor/github.com/DataDog/zstd/cover.c b/vendor/github.com/DataDog/zstd/cover.c index 472ac81..330e33a 100644 --- a/vendor/github.com/DataDog/zstd/cover.c +++ b/vendor/github.com/DataDog/zstd/cover.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -35,6 +35,7 @@ #include "pool.h" #include "threading.h" #include "zstd_internal.h" /* includes zstd.h */ +#include "bits.h" /* ZSTD_highbit32 */ #include "zdict.h" #include "cover.h" @@ -542,7 +543,7 @@ static void COVER_ctx_destroy(COVER_ctx_t *ctx) { /** * Prepare a context for dictionary building. - * The context is only dependent on the parameter `d` and can used multiple + * The context is only dependent on the parameter `d` and can be used multiple * times. * Returns 0 on success or error code on error. * The context must be destroyed with `COVER_ctx_destroy()`. @@ -647,7 +648,7 @@ static size_t COVER_ctx_init(COVER_ctx_t *ctx, const void *samplesBuffer, void COVER_warnOnSmallCorpus(size_t maxDictSize, size_t nbDmers, int displayLevel) { - const double ratio = (double)nbDmers / maxDictSize; + const double ratio = (double)nbDmers / (double)maxDictSize; if (ratio >= 10) { return; } @@ -951,9 +952,17 @@ void COVER_best_finish(COVER_best_t *best, ZDICT_cover_params_t parameters, } } +static COVER_dictSelection_t setDictSelection(BYTE* buf, size_t s, size_t csz) +{ + COVER_dictSelection_t ds; + ds.dictContent = buf; + ds.dictSize = s; + ds.totalCompressedSize = csz; + return ds; +} + COVER_dictSelection_t COVER_dictSelectionError(size_t error) { - COVER_dictSelection_t selection = { NULL, 0, error }; - return selection; + return setDictSelection(NULL, 0, error); } unsigned COVER_dictSelectionIsError(COVER_dictSelection_t selection) { @@ -1006,9 +1015,8 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe } if (params.shrinkDict == 0) { - COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize }; free(candidateDictBuffer); - return selection; + return setDictSelection(largestDictbuffer, dictContentSize, totalCompressedSize); } largestDict = dictContentSize; @@ -1040,20 +1048,16 @@ COVER_dictSelection_t COVER_selectDict(BYTE* customDictContent, size_t dictBuffe return COVER_dictSelectionError(totalCompressedSize); } - if (totalCompressedSize <= largestCompressed * regressionTolerance) { - COVER_dictSelection_t selection = { candidateDictBuffer, dictContentSize, totalCompressedSize }; + if ((double)totalCompressedSize <= (double)largestCompressed * regressionTolerance) { free(largestDictbuffer); - return selection; + return setDictSelection( candidateDictBuffer, dictContentSize, totalCompressedSize ); } dictContentSize *= 2; } dictContentSize = largestDict; totalCompressedSize = largestCompressed; - { - COVER_dictSelection_t selection = { largestDictbuffer, dictContentSize, totalCompressedSize }; - free(candidateDictBuffer); - return selection; - } + free(candidateDictBuffer); + return setDictSelection( largestDictbuffer, dictContentSize, totalCompressedSize ); } /** diff --git a/vendor/github.com/DataDog/zstd/cover.h b/vendor/github.com/DataDog/zstd/cover.h index 47d31e6..5c739da 100644 --- a/vendor/github.com/DataDog/zstd/cover.h +++ b/vendor/github.com/DataDog/zstd/cover.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/cpu.h b/vendor/github.com/DataDog/zstd/cpu.h index 1ad2e85..22b646e 100644 --- a/vendor/github.com/DataDog/zstd/cpu.h +++ b/vendor/github.com/DataDog/zstd/cpu.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/debug.c b/vendor/github.com/DataDog/zstd/debug.c index d83d083..d1dfc35 100644 --- a/vendor/github.com/DataDog/zstd/debug.c +++ b/vendor/github.com/DataDog/zstd/debug.c @@ -2,7 +2,7 @@ /* ****************************************************************** * debug * Part of FSE library - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy diff --git a/vendor/github.com/DataDog/zstd/debug.h b/vendor/github.com/DataDog/zstd/debug.h index 109a790..110d55f 100644 --- a/vendor/github.com/DataDog/zstd/debug.h +++ b/vendor/github.com/DataDog/zstd/debug.h @@ -2,7 +2,7 @@ /* ****************************************************************** * debug * Part of FSE library - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy diff --git a/vendor/github.com/DataDog/zstd/entropy_common.c b/vendor/github.com/DataDog/zstd/entropy_common.c index cdcf91d..1bfbb52 100644 --- a/vendor/github.com/DataDog/zstd/entropy_common.c +++ b/vendor/github.com/DataDog/zstd/entropy_common.c @@ -1,7 +1,7 @@ #ifndef USE_EXTERNAL_ZSTD /* ****************************************************************** * Common functions of New Generation Entropy library - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -20,8 +20,8 @@ #include "error_private.h" /* ERR_*, ERROR */ #define FSE_STATIC_LINKING_ONLY /* FSE_MIN_TABLELOG */ #include "fse.h" -#define HUF_STATIC_LINKING_ONLY /* HUF_TABLELOG_ABSOLUTEMAX */ #include "huf.h" +#include "bits.h" /* ZSDT_highbit32, ZSTD_countTrailingZeros32 */ /*=== Version ===*/ @@ -39,34 +39,6 @@ const char* HUF_getErrorName(size_t code) { return ERR_getErrorName(code); } /*-************************************************************** * FSE NCount encoding-decoding ****************************************************************/ -static U32 FSE_ctz(U32 val) -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ - if (val != 0) { - unsigned long r; - _BitScanForward(&r, val); - return (unsigned)r; - } else { - /* Should not reach this code path */ - __assume(0); - } -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_ctz(val); -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return __CTZ(val); -# else /* Software version */ - U32 count = 0; - while ((val & 1) == 0) { - val >>= 1; - ++count; - } - return count; -# endif - } -} - FORCE_INLINE_TEMPLATE size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigned* tableLogPtr, const void* headerBuffer, size_t hbSize) @@ -114,7 +86,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * repeat. * Avoid UB by setting the high bit to 1. */ - int repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + int repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; while (repeats >= 12) { charnum += 3 * 12; if (LIKELY(ip <= iend-7)) { @@ -125,7 +97,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne ip = iend - 4; } bitStream = MEM_readLE32(ip) >> bitCount; - repeats = FSE_ctz(~bitStream | 0x80000000) >> 1; + repeats = ZSTD_countTrailingZeros32(~bitStream | 0x80000000) >> 1; } charnum += 3 * repeats; bitStream >>= 2 * repeats; @@ -190,7 +162,7 @@ size_t FSE_readNCount_body(short* normalizedCounter, unsigned* maxSVPtr, unsigne * know that threshold > 1. */ if (remaining <= 1) break; - nbBits = BIT_highbit32(remaining) + 1; + nbBits = ZSTD_highbit32(remaining) + 1; threshold = 1 << (nbBits - 1); } if (charnum >= maxSV1) break; @@ -265,7 +237,7 @@ size_t HUF_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, const void* src, size_t srcSize) { U32 wksp[HUF_READ_STATS_WORKSPACE_SIZE_U32]; - return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* bmi2 */ 0); + return HUF_readStats_wksp(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, wksp, sizeof(wksp), /* flags */ 0); } FORCE_INLINE_TEMPLATE size_t @@ -313,14 +285,14 @@ HUF_readStats_body(BYTE* huffWeight, size_t hwSize, U32* rankStats, if (weightTotal == 0) return ERROR(corruption_detected); /* get last non-null symbol weight (implied, total must be 2^n) */ - { U32 const tableLog = BIT_highbit32(weightTotal) + 1; + { U32 const tableLog = ZSTD_highbit32(weightTotal) + 1; if (tableLog > HUF_TABLELOG_MAX) return ERROR(corruption_detected); *tableLogPtr = tableLog; /* determine last weight */ { U32 const total = 1 << tableLog; U32 const rest = total - weightTotal; - U32 const verif = 1 << BIT_highbit32(rest); - U32 const lastWeight = BIT_highbit32(rest) + 1; + U32 const verif = 1 << ZSTD_highbit32(rest); + U32 const lastWeight = ZSTD_highbit32(rest) + 1; if (verif != rest) return ERROR(corruption_detected); /* last value must be a clean power of 2 */ huffWeight[oSize] = (BYTE)lastWeight; rankStats[lastWeight]++; @@ -357,14 +329,14 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, - int bmi2) + int flags) { #if DYNAMIC_BMI2 - if (bmi2) { + if (flags & HUF_flags_bmi2) { return HUF_readStats_body_bmi2(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } #endif - (void)bmi2; + (void)flags; return HUF_readStats_body_default(huffWeight, hwSize, rankStats, nbSymbolsPtr, tableLogPtr, src, srcSize, workSpace, wkspSize); } diff --git a/vendor/github.com/DataDog/zstd/error_private.c b/vendor/github.com/DataDog/zstd/error_private.c index 5cca2c5..803cc7b 100644 --- a/vendor/github.com/DataDog/zstd/error_private.c +++ b/vendor/github.com/DataDog/zstd/error_private.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -28,9 +28,11 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(version_unsupported): return "Version not supported"; case PREFIX(frameParameter_unsupported): return "Unsupported frame parameter"; case PREFIX(frameParameter_windowTooLarge): return "Frame requires too much memory for decoding"; - case PREFIX(corruption_detected): return "Corrupted block detected"; + case PREFIX(corruption_detected): return "Data corruption detected"; case PREFIX(checksum_wrong): return "Restored data doesn't match checksum"; + case PREFIX(literals_headerWrong): return "Header of Literals' block doesn't respect format specification"; case PREFIX(parameter_unsupported): return "Unsupported parameter"; + case PREFIX(parameter_combination_unsupported): return "Unsupported combination of parameters"; case PREFIX(parameter_outOfBound): return "Parameter is out of bound"; case PREFIX(init_missing): return "Context should be init first"; case PREFIX(memory_allocation): return "Allocation error : not enough memory"; @@ -39,17 +41,22 @@ const char* ERR_getErrorString(ERR_enum code) case PREFIX(tableLog_tooLarge): return "tableLog requires too much memory : unsupported"; case PREFIX(maxSymbolValue_tooLarge): return "Unsupported max Symbol Value : too large"; case PREFIX(maxSymbolValue_tooSmall): return "Specified maxSymbolValue is too small"; + case PREFIX(stabilityCondition_notRespected): return "pledged buffer stability condition is not respected"; case PREFIX(dictionary_corrupted): return "Dictionary is corrupted"; case PREFIX(dictionary_wrong): return "Dictionary mismatch"; case PREFIX(dictionaryCreation_failed): return "Cannot create Dictionary from provided samples"; case PREFIX(dstSize_tooSmall): return "Destination buffer is too small"; case PREFIX(srcSize_wrong): return "Src size is incorrect"; case PREFIX(dstBuffer_null): return "Operation on NULL destination buffer"; + case PREFIX(noForwardProgress_destFull): return "Operation made no progress over multiple calls, due to output buffer being full"; + case PREFIX(noForwardProgress_inputEmpty): return "Operation made no progress over multiple calls, due to input being empty"; /* following error codes are not stable and may be removed or changed in a future version */ case PREFIX(frameIndex_tooLarge): return "Frame index is too large"; case PREFIX(seekableIO): return "An I/O error occurred when reading/seeking"; case PREFIX(dstBuffer_wrong): return "Destination buffer is wrong"; case PREFIX(srcBuffer_wrong): return "Source buffer is wrong"; + case PREFIX(sequenceProducer_failed): return "Block-level external sequence producer returned an error code"; + case PREFIX(externalSequences_invalid): return "External sequences are not valid"; case PREFIX(maxCode): default: return notErrorCode; } diff --git a/vendor/github.com/DataDog/zstd/error_private.h b/vendor/github.com/DataDog/zstd/error_private.h index 12ca15f..05ddbfa 100644 --- a/vendor/github.com/DataDog/zstd/error_private.h +++ b/vendor/github.com/DataDog/zstd/error_private.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/fastcover.c b/vendor/github.com/DataDog/zstd/fastcover.c index 0c5297f..147ccbf 100644 --- a/vendor/github.com/DataDog/zstd/fastcover.c +++ b/vendor/github.com/DataDog/zstd/fastcover.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -305,7 +305,7 @@ FASTCOVER_computeFrequency(U32* freqs, const FASTCOVER_ctx_t* ctx) /** * Prepare a context for dictionary building. - * The context is only dependent on the parameter `d` and can used multiple + * The context is only dependent on the parameter `d` and can be used multiple * times. * Returns 0 on success or error code on error. * The context must be destroyed with `FASTCOVER_ctx_destroy()`. diff --git a/vendor/github.com/DataDog/zstd/fse.h b/vendor/github.com/DataDog/zstd/fse.h index caec0a5..5d6e4c4 100644 --- a/vendor/github.com/DataDog/zstd/fse.h +++ b/vendor/github.com/DataDog/zstd/fse.h @@ -2,7 +2,7 @@ /* ****************************************************************** * FSE : Finite State Entropy codec * Public Prototypes declaration - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -54,34 +54,6 @@ extern "C" { FSE_PUBLIC_API unsigned FSE_versionNumber(void); /**< library version number; to be used when checking dll version */ -/*-**************************************** -* FSE simple functions -******************************************/ -/*! FSE_compress() : - Compress content of buffer 'src', of size 'srcSize', into destination buffer 'dst'. - 'dst' buffer must be already allocated. Compression runs faster is dstCapacity >= FSE_compressBound(srcSize). - @return : size of compressed data (<= dstCapacity). - Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression instead. - if FSE_isError(return), compression failed (more details using FSE_getErrorName()) -*/ -FSE_PUBLIC_API size_t FSE_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/*! FSE_decompress(): - Decompress FSE data from buffer 'cSrc', of size 'cSrcSize', - into already allocated destination buffer 'dst', of size 'dstCapacity'. - @return : size of regenerated data (<= maxDstSize), - or an error code, which can be tested using FSE_isError() . - - ** Important ** : FSE_decompress() does not decompress non-compressible nor RLE data !!! - Why ? : making this distinction requires a header. - Header management is intentionally delegated to the user layer, which can better manage special cases. -*/ -FSE_PUBLIC_API size_t FSE_decompress(void* dst, size_t dstCapacity, - const void* cSrc, size_t cSrcSize); - - /*-***************************************** * Tool functions ******************************************/ @@ -92,20 +64,6 @@ FSE_PUBLIC_API unsigned FSE_isError(size_t code); /* tells if a return FSE_PUBLIC_API const char* FSE_getErrorName(size_t code); /* provides error code string (useful for debugging) */ -/*-***************************************** -* FSE advanced functions -******************************************/ -/*! FSE_compress2() : - Same as FSE_compress(), but allows the selection of 'maxSymbolValue' and 'tableLog' - Both parameters can be defined as '0' to mean : use default value - @return : size of compressed data - Special values : if return == 0, srcData is not compressible => Nothing is stored within cSrc !!! - if return == 1, srcData is a single byte symbol * srcSize times. Use RLE compression. - if FSE_isError(return), it's an error code. -*/ -FSE_PUBLIC_API size_t FSE_compress2 (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); - - /*-***************************************** * FSE detailed API ******************************************/ @@ -165,8 +123,6 @@ FSE_PUBLIC_API size_t FSE_writeNCount (void* buffer, size_t bufferSize, /*! Constructor and Destructor of FSE_CTable. Note that FSE_CTable size depends on 'tableLog' and 'maxSymbolValue' */ typedef unsigned FSE_CTable; /* don't allocate that. It's only meant to be more restrictive than void* */ -FSE_PUBLIC_API FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog); -FSE_PUBLIC_API void FSE_freeCTable (FSE_CTable* ct); /*! FSE_buildCTable(): Builds `ct`, which must be already allocated, using FSE_createCTable(). @@ -242,23 +198,7 @@ FSE_PUBLIC_API size_t FSE_readNCount_bmi2(short* normalizedCounter, unsigned* maxSymbolValuePtr, unsigned* tableLogPtr, const void* rBuffer, size_t rBuffSize, int bmi2); -/*! Constructor and Destructor of FSE_DTable. - Note that its size depends on 'tableLog' */ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be more restrictive than void* */ -FSE_PUBLIC_API FSE_DTable* FSE_createDTable(unsigned tableLog); -FSE_PUBLIC_API void FSE_freeDTable(FSE_DTable* dt); - -/*! FSE_buildDTable(): - Builds 'dt', which must be already allocated, using FSE_createDTable(). - return : 0, or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_buildDTable (FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog); - -/*! FSE_decompress_usingDTable(): - Decompress compressed source `cSrc` of size `cSrcSize` using `dt` - into `dst` which must be already allocated. - @return : size of regenerated data (necessarily <= `dstCapacity`), - or an errorCode, which can be tested using FSE_isError() */ -FSE_PUBLIC_API size_t FSE_decompress_usingDTable(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, const FSE_DTable* dt); /*! Tutorial : @@ -321,16 +261,6 @@ If there is an error, the function will return an error code, which can be teste unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus); /**< same as FSE_optimalTableLog(), which used `minus==2` */ -/* FSE_compress_wksp() : - * Same as FSE_compress2(), but using an externally allocated scratch buffer (`workSpace`). - * FSE_COMPRESS_WKSP_SIZE_U32() provides the minimum size required for `workSpace` as a table of FSE_CTable. - */ -#define FSE_COMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) ( FSE_CTABLE_SIZE_U32(maxTableLog, maxSymbolValue) + ((maxTableLog > 12) ? (1 << (maxTableLog - 2)) : 1024) ) -size_t FSE_compress_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); - -size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits); -/**< build a fake FSE_CTable, designed for a flat distribution, where each symbol uses nbBits */ - size_t FSE_buildCTable_rle (FSE_CTable* ct, unsigned char symbolValue); /**< build a fake FSE_CTable, designed to compress always the same symbolValue */ @@ -348,19 +278,11 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, const short* normalizedCounter, unsi FSE_PUBLIC_API size_t FSE_buildDTable_wksp(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< Same as FSE_buildDTable(), using an externally allocated `workspace` produced with `FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxSymbolValue)` */ -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits); -/**< build a fake FSE_DTable, designed to read a flat distribution where each symbol uses nbBits */ - -size_t FSE_buildDTable_rle (FSE_DTable* dt, unsigned char symbolValue); -/**< build a fake FSE_DTable, designed to always generate the same symbolValue */ - -#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) +#define FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) (FSE_DTABLE_SIZE_U32(maxTableLog) + 1 + FSE_BUILD_DTABLE_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) + (FSE_MAX_SYMBOL_VALUE + 1) / 2 + 1) #define FSE_DECOMPRESS_WKSP_SIZE(maxTableLog, maxSymbolValue) (FSE_DECOMPRESS_WKSP_SIZE_U32(maxTableLog, maxSymbolValue) * sizeof(unsigned)) -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize); -/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)` */ - size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize, int bmi2); -/**< Same as FSE_decompress_wksp() but with dynamic BMI2 support. Pass 1 if your CPU supports BMI2 or 0 if it doesn't. */ +/**< same as FSE_decompress(), using an externally allocated `workSpace` produced with `FSE_DECOMPRESS_WKSP_SIZE_U32(maxLog, maxSymbolValue)`. + * Set bmi2 to 1 if your CPU supports BMI2 or 0 if it doesn't */ typedef enum { FSE_repeat_none, /**< Cannot use the previous table */ @@ -556,7 +478,7 @@ MEM_STATIC void FSE_flushCState(BIT_CStream_t* bitC, const FSE_CState_t* statePt /* FSE_getMaxNbBits() : * Approximate maximum cost of a symbol, in bits. - * Fractional get rounded up (i.e : a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) + * Fractional get rounded up (i.e. a symbol with a normalized frequency of 3 gives the same result as a frequency of 2) * note 1 : assume symbolValue is valid (<= maxSymbolValue) * note 2 : if freq[symbolValue]==0, @return a fake cost of tableLog+1 bits */ MEM_STATIC U32 FSE_getMaxNbBits(const void* symbolTTPtr, U32 symbolValue) diff --git a/vendor/github.com/DataDog/zstd/fse_compress.c b/vendor/github.com/DataDog/zstd/fse_compress.c index ec3ab43..41f56e0 100644 --- a/vendor/github.com/DataDog/zstd/fse_compress.c +++ b/vendor/github.com/DataDog/zstd/fse_compress.c @@ -1,7 +1,7 @@ #ifndef USE_EXTERNAL_ZSTD /* ****************************************************************** * FSE : Finite State Entropy encoder - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -27,6 +27,7 @@ #define ZSTD_DEPS_NEED_MALLOC #define ZSTD_DEPS_NEED_MATH64 #include "zstd_deps.h" /* ZSTD_malloc, ZSTD_free, ZSTD_memcpy, ZSTD_memset */ +#include "bits.h" /* ZSTD_highbit32 */ /* ************************************************************** @@ -91,7 +92,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, assert(tableLog < 16); /* required for threshold strategy to work */ /* For explanations on how to distribute symbol values over the table : - * http://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ + * https://fastcompression.blogspot.fr/2014/02/fse-distributing-symbol-values.html */ #ifdef __clang_analyzer__ ZSTD_memset(tableSymbol, 0, sizeof(*tableSymbol) * tableSize); /* useless initialization, just to keep scan-build happy */ @@ -192,7 +193,7 @@ size_t FSE_buildCTable_wksp(FSE_CTable* ct, break; default : assert(normalizedCounter[s] > 1); - { U32 const maxBitsOut = tableLog - BIT_highbit32 ((U32)normalizedCounter[s]-1); + { U32 const maxBitsOut = tableLog - ZSTD_highbit32 ((U32)normalizedCounter[s]-1); U32 const minStatePlus = (U32)normalizedCounter[s] << maxBitsOut; symbolTT[s].deltaNbBits = (maxBitsOut << 16) - minStatePlus; symbolTT[s].deltaFindState = (int)(total - (unsigned)normalizedCounter[s]); @@ -343,21 +344,11 @@ size_t FSE_writeNCount (void* buffer, size_t bufferSize, * FSE Compression Code ****************************************************************/ -FSE_CTable* FSE_createCTable (unsigned maxSymbolValue, unsigned tableLog) -{ - size_t size; - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - size = FSE_CTABLE_SIZE_U32 (tableLog, maxSymbolValue) * sizeof(U32); - return (FSE_CTable*)ZSTD_malloc(size); -} - -void FSE_freeCTable (FSE_CTable* ct) { ZSTD_free(ct); } - /* provides the minimum logSize to safely represent a distribution */ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) { - U32 minBitsSrc = BIT_highbit32((U32)(srcSize)) + 1; - U32 minBitsSymbols = BIT_highbit32(maxSymbolValue) + 2; + U32 minBitsSrc = ZSTD_highbit32((U32)(srcSize)) + 1; + U32 minBitsSymbols = ZSTD_highbit32(maxSymbolValue) + 2; U32 minBits = minBitsSrc < minBitsSymbols ? minBitsSrc : minBitsSymbols; assert(srcSize > 1); /* Not supported, RLE should be used instead */ return minBits; @@ -365,7 +356,7 @@ static unsigned FSE_minTableLog(size_t srcSize, unsigned maxSymbolValue) unsigned FSE_optimalTableLog_internal(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, unsigned minus) { - U32 maxBitsSrc = BIT_highbit32((U32)(srcSize - 1)) - minus; + U32 maxBitsSrc = ZSTD_highbit32((U32)(srcSize - 1)) - minus; U32 tableLog = maxTableLog; U32 minBits = FSE_minTableLog(srcSize, maxSymbolValue); assert(srcSize > 1); /* Not supported, RLE should be used instead */ @@ -533,40 +524,6 @@ size_t FSE_normalizeCount (short* normalizedCounter, unsigned tableLog, return tableLog; } - -/* fake FSE_CTable, for raw (uncompressed) input */ -size_t FSE_buildCTable_raw (FSE_CTable* ct, unsigned nbBits) -{ - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSymbolValue = tableMask; - void* const ptr = ct; - U16* const tableU16 = ( (U16*) ptr) + 2; - void* const FSCT = ((U32*)ptr) + 1 /* header */ + (tableSize>>1); /* assumption : tableLog >= 1 */ - FSE_symbolCompressionTransform* const symbolTT = (FSE_symbolCompressionTransform*) (FSCT); - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* header */ - tableU16[-2] = (U16) nbBits; - tableU16[-1] = (U16) maxSymbolValue; - - /* Build table */ - for (s=0; s not compressible */ - if (maxCount < (srcSize >> 7)) return 0; /* Heuristic : not compressible enough */ - } - - tableLog = FSE_optimalTableLog(tableLog, srcSize, maxSymbolValue); - CHECK_F( FSE_normalizeCount(norm, tableLog, count, srcSize, maxSymbolValue, /* useLowProbCount */ srcSize >= 2048) ); - - /* Write table description header */ - { CHECK_V_F(nc_err, FSE_writeNCount(op, oend-op, norm, maxSymbolValue, tableLog) ); - op += nc_err; - } - - /* Compress */ - CHECK_F( FSE_buildCTable_wksp(CTable, norm, maxSymbolValue, tableLog, scratchBuffer, scratchBufferSize) ); - { CHECK_V_F(cSize, FSE_compress_usingCTable(op, oend - op, src, srcSize, CTable) ); - if (cSize == 0) return 0; /* not enough space for compressed data */ - op += cSize; - } - - /* check compressibility */ - if ( (size_t)(op-ostart) >= srcSize-1 ) return 0; - - return op-ostart; -} - -typedef struct { - FSE_CTable CTable_max[FSE_CTABLE_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; - union { - U32 hist_wksp[HIST_WKSP_SIZE_U32]; - BYTE scratchBuffer[1 << FSE_MAX_TABLELOG]; - } workspace; -} fseWkspMax_t; - -size_t FSE_compress2 (void* dst, size_t dstCapacity, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog) -{ - fseWkspMax_t scratchBuffer; - DEBUG_STATIC_ASSERT(sizeof(scratchBuffer) >= FSE_COMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)); /* compilation failures here means scratchBuffer is not large enough */ - if (tableLog > FSE_MAX_TABLELOG) return ERROR(tableLog_tooLarge); - return FSE_compress_wksp(dst, dstCapacity, src, srcSize, maxSymbolValue, tableLog, &scratchBuffer, sizeof(scratchBuffer)); -} - -size_t FSE_compress (void* dst, size_t dstCapacity, const void* src, size_t srcSize) -{ - return FSE_compress2(dst, dstCapacity, src, srcSize, FSE_MAX_SYMBOL_VALUE, FSE_DEFAULT_TABLELOG); -} -#endif - #endif /* FSE_COMMONDEFS_ONLY */ #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/fse_decompress.c b/vendor/github.com/DataDog/zstd/fse_decompress.c index 867251c..e6aafae 100644 --- a/vendor/github.com/DataDog/zstd/fse_decompress.c +++ b/vendor/github.com/DataDog/zstd/fse_decompress.c @@ -1,7 +1,7 @@ #ifndef USE_EXTERNAL_ZSTD /* ****************************************************************** * FSE : Finite State Entropy decoder - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -25,6 +25,7 @@ #include "error_private.h" #define ZSTD_DEPS_NEED_MALLOC #include "zstd_deps.h" +#include "bits.h" /* ZSTD_highbit32 */ /* ************************************************************** @@ -56,19 +57,6 @@ #define FSE_FUNCTION_NAME(X,Y) FSE_CAT(X,Y) #define FSE_TYPE_NAME(X,Y) FSE_CAT(X,Y) - -/* Function templates */ -FSE_DTable* FSE_createDTable (unsigned tableLog) -{ - if (tableLog > FSE_TABLELOG_ABSOLUTE_MAX) tableLog = FSE_TABLELOG_ABSOLUTE_MAX; - return (FSE_DTable*)ZSTD_malloc( FSE_DTABLE_SIZE_U32(tableLog) * sizeof (U32) ); -} - -void FSE_freeDTable (FSE_DTable* dt) -{ - ZSTD_free(dt); -} - static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize) { void* const tdPtr = dt+1; /* because *dt is unsigned, 32-bits aligned on 32-bits */ @@ -128,10 +116,10 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -167,7 +155,7 @@ static size_t FSE_buildDTable_internal(FSE_DTable* dt, const short* normalizedCo for (u=0; utableLog = 0; - DTableH->fastMode = 0; - - cell->newState = 0; - cell->symbol = symbolValue; - cell->nbBits = 0; - - return 0; -} - - -size_t FSE_buildDTable_raw (FSE_DTable* dt, unsigned nbBits) -{ - void* ptr = dt; - FSE_DTableHeader* const DTableH = (FSE_DTableHeader*)ptr; - void* dPtr = dt + 1; - FSE_decode_t* const dinfo = (FSE_decode_t*)dPtr; - const unsigned tableSize = 1 << nbBits; - const unsigned tableMask = tableSize - 1; - const unsigned maxSV1 = tableMask+1; - unsigned s; - - /* Sanity checks */ - if (nbBits < 1) return ERROR(GENERIC); /* min size */ - - /* Build Decoding Table */ - DTableH->tableLog = (U16)nbBits; - DTableH->fastMode = 1; - for (s=0; sfastMode; - - /* select fast mode (static) */ - if (fastMode) return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 1); - return FSE_decompress_usingDTable_generic(dst, originalSize, cSrc, cSrcSize, dt, 0); -} - - -size_t FSE_decompress_wksp(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize, unsigned maxLog, void* workSpace, size_t wkspSize) -{ - return FSE_decompress_wksp_bmi2(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize, /* bmi2 */ 0); -} - typedef struct { short ncount[FSE_MAX_SYMBOL_VALUE + 1]; FSE_DTable dtable[1]; /* Dynamically sized */ @@ -343,7 +268,8 @@ FORCE_INLINE_TEMPLATE size_t FSE_decompress_wksp_body( } if (FSE_DECOMPRESS_WKSP_SIZE(tableLog, maxSymbolValue) > wkspSize) return ERROR(tableLog_tooLarge); - workSpace = wksp->dtable + FSE_DTABLE_SIZE_U32(tableLog); + assert(sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog) <= wkspSize); + workSpace = (BYTE*)workSpace + sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); wkspSize -= sizeof(*wksp) + FSE_DTABLE_SIZE(tableLog); CHECK_F( FSE_buildDTable_internal(wksp->dtable, wksp->ncount, maxSymbolValue, tableLog, workSpace, wkspSize) ); @@ -383,24 +309,6 @@ size_t FSE_decompress_wksp_bmi2(void* dst, size_t dstCapacity, const void* cSrc, return FSE_decompress_wksp_body_default(dst, dstCapacity, cSrc, cSrcSize, maxLog, workSpace, wkspSize); } - -typedef FSE_DTable DTable_max_t[FSE_DTABLE_SIZE_U32(FSE_MAX_TABLELOG)]; - -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -size_t FSE_buildDTable(FSE_DTable* dt, const short* normalizedCounter, unsigned maxSymbolValue, unsigned tableLog) { - U32 wksp[FSE_BUILD_DTABLE_WKSP_SIZE_U32(FSE_TABLELOG_ABSOLUTE_MAX, FSE_MAX_SYMBOL_VALUE)]; - return FSE_buildDTable_wksp(dt, normalizedCounter, maxSymbolValue, tableLog, wksp, sizeof(wksp)); -} - -size_t FSE_decompress(void* dst, size_t dstCapacity, const void* cSrc, size_t cSrcSize) -{ - /* Static analyzer seems unable to understand this table will be properly initialized later */ - U32 wksp[FSE_DECOMPRESS_WKSP_SIZE_U32(FSE_MAX_TABLELOG, FSE_MAX_SYMBOL_VALUE)]; - return FSE_decompress_wksp(dst, dstCapacity, cSrc, cSrcSize, FSE_MAX_TABLELOG, wksp, sizeof(wksp)); -} -#endif - - #endif /* FSE_COMMONDEFS_ONLY */ #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/hist.c b/vendor/github.com/DataDog/zstd/hist.c index 7515f1d..2d6eae8 100644 --- a/vendor/github.com/DataDog/zstd/hist.c +++ b/vendor/github.com/DataDog/zstd/hist.c @@ -2,7 +2,7 @@ /* ****************************************************************** * hist : Histogram functions * part of Finite State Entropy project - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy diff --git a/vendor/github.com/DataDog/zstd/hist.h b/vendor/github.com/DataDog/zstd/hist.h index 0da8748..410accc 100644 --- a/vendor/github.com/DataDog/zstd/hist.h +++ b/vendor/github.com/DataDog/zstd/hist.h @@ -2,7 +2,7 @@ /* ****************************************************************** * hist : Histogram functions * part of Finite State Entropy project - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE source repository : https://github.com/Cyan4973/FiniteStateEntropy diff --git a/vendor/github.com/DataDog/zstd/huf.h b/vendor/github.com/DataDog/zstd/huf.h index b522cff..ad9a485 100644 --- a/vendor/github.com/DataDog/zstd/huf.h +++ b/vendor/github.com/DataDog/zstd/huf.h @@ -2,7 +2,7 @@ /* ****************************************************************** * huff0 huffman codec, * part of Finite State Entropy library - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - Source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -22,100 +22,23 @@ extern "C" { /* *** Dependencies *** */ #include "zstd_deps.h" /* size_t */ - - -/* *** library symbols visibility *** */ -/* Note : when linking with -fvisibility=hidden on gcc, or by default on Visual, - * HUF symbols remain "private" (internal symbols for library only). - * Set macro FSE_DLL_EXPORT to 1 if you want HUF symbols visible on DLL interface */ -#if defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) && defined(__GNUC__) && (__GNUC__ >= 4) -# define HUF_PUBLIC_API __attribute__ ((visibility ("default"))) -#elif defined(FSE_DLL_EXPORT) && (FSE_DLL_EXPORT==1) /* Visual expected */ -# define HUF_PUBLIC_API __declspec(dllexport) -#elif defined(FSE_DLL_IMPORT) && (FSE_DLL_IMPORT==1) -# define HUF_PUBLIC_API __declspec(dllimport) /* not required, just to generate faster code (saves a function pointer load from IAT and an indirect jump) */ -#else -# define HUF_PUBLIC_API -#endif - - -/* ========================== */ -/* *** simple functions *** */ -/* ========================== */ - -/** HUF_compress() : - * Compress content from buffer 'src', of size 'srcSize', into buffer 'dst'. - * 'dst' buffer must be already allocated. - * Compression runs faster if `dstCapacity` >= HUF_compressBound(srcSize). - * `srcSize` must be <= `HUF_BLOCKSIZE_MAX` == 128 KB. - * @return : size of compressed data (<= `dstCapacity`). - * Special values : if return == 0, srcData is not compressible => Nothing is stored within dst !!! - * if HUF_isError(return), compression failed (more details using HUF_getErrorName()) - */ -HUF_PUBLIC_API size_t HUF_compress(void* dst, size_t dstCapacity, - const void* src, size_t srcSize); - -/** HUF_decompress() : - * Decompress HUF data from buffer 'cSrc', of size 'cSrcSize', - * into already allocated buffer 'dst', of minimum size 'dstSize'. - * `originalSize` : **must** be the ***exact*** size of original (uncompressed) data. - * Note : in contrast with FSE, HUF_decompress can regenerate - * RLE (cSrcSize==1) and uncompressed (cSrcSize==dstSize) data, - * because it knows size to regenerate (originalSize). - * @return : size of regenerated data (== originalSize), - * or an error code, which can be tested using HUF_isError() - */ -HUF_PUBLIC_API size_t HUF_decompress(void* dst, size_t originalSize, - const void* cSrc, size_t cSrcSize); - - -/* *** Tool functions *** */ -#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ -HUF_PUBLIC_API size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ - -/* Error Management */ -HUF_PUBLIC_API unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ -HUF_PUBLIC_API const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ - - -/* *** Advanced function *** */ - -/** HUF_compress2() : - * Same as HUF_compress(), but offers control over `maxSymbolValue` and `tableLog`. - * `maxSymbolValue` must be <= HUF_SYMBOLVALUE_MAX . - * `tableLog` must be `<= HUF_TABLELOG_MAX` . */ -HUF_PUBLIC_API size_t HUF_compress2 (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog); - -/** HUF_compress4X_wksp() : - * Same as HUF_compress2(), but uses externally allocated `workSpace`. - * `workspace` must be at least as large as HUF_WORKSPACE_SIZE */ -#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) -#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) -HUF_PUBLIC_API size_t HUF_compress4X_wksp (void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned tableLog, - void* workSpace, size_t wkspSize); - -#endif /* HUF_H_298734234 */ - -/* ****************************************************************** - * WARNING !! - * The following section contains advanced and experimental definitions - * which shall never be used in the context of a dynamic library, - * because they are not guaranteed to remain stable in the future. - * Only consider them in association with static linking. - * *****************************************************************/ -#if defined(HUF_STATIC_LINKING_ONLY) && !defined(HUF_H_HUF_STATIC_LINKING_ONLY) -#define HUF_H_HUF_STATIC_LINKING_ONLY - -/* *** Dependencies *** */ -#include "mem.h" /* U32 */ +#include "mem.h" /* U32 */ #define FSE_STATIC_LINKING_ONLY #include "fse.h" +/* *** Tool functions *** */ +#define HUF_BLOCKSIZE_MAX (128 * 1024) /**< maximum input size for a single block compressed with HUF_compress */ +size_t HUF_compressBound(size_t size); /**< maximum compressed size (worst case) */ + +/* Error Management */ +unsigned HUF_isError(size_t code); /**< tells if a return value is an error code */ +const char* HUF_getErrorName(size_t code); /**< provides error code string (useful for debugging) */ + + +#define HUF_WORKSPACE_SIZE ((8 << 10) + 512 /* sorting scratch space */) +#define HUF_WORKSPACE_SIZE_U64 (HUF_WORKSPACE_SIZE / sizeof(U64)) + /* *** Constants *** */ #define HUF_TABLELOG_MAX 12 /* max runtime value of tableLog (due to static allocation); can be modified up to HUF_TABLELOG_ABSOLUTEMAX */ #define HUF_TABLELOG_DEFAULT 11 /* default tableLog value when none specified */ @@ -155,25 +78,49 @@ typedef U32 HUF_DTable; /* **************************************** * Advanced decompression functions ******************************************/ -size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -#endif -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< decodes RLE and uncompressed */ -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< considers RLE and uncompressed as errors */ -size_t HUF_decompress4X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif +/** + * Huffman flags bitset. + * For all flags, 0 is the default value. + */ +typedef enum { + /** + * If compiled with DYNAMIC_BMI2: Set flag only if the CPU supports BMI2 at runtime. + * Otherwise: Ignored. + */ + HUF_flags_bmi2 = (1 << 0), + /** + * If set: Test possible table depths to find the one that produces the smallest header + encoded size. + * If unset: Use heuristic to find the table depth. + */ + HUF_flags_optimalDepth = (1 << 1), + /** + * If set: If the previous table can encode the input, always reuse the previous table. + * If unset: If the previous table can encode the input, reuse the previous table if it results in a smaller output. + */ + HUF_flags_preferRepeat = (1 << 2), + /** + * If set: Sample the input and check if the sample is uncompressible, if it is then don't attempt to compress. + * If unset: Always histogram the entire input. + */ + HUF_flags_suspectUncompressible = (1 << 3), + /** + * If set: Don't use assembly implementations + * If unset: Allow using assembly implementations + */ + HUF_flags_disableAsm = (1 << 4), + /** + * If set: Don't use the fast decoding loop, always use the fallback decoding loop. + * If unset: Use the fast decoding loop when possible. + */ + HUF_flags_disableFast = (1 << 5) +} HUF_flags_e; /* **************************************** * HUF detailed API * ****************************************/ +#define HUF_OPTIMAL_DEPTH_THRESHOLD ZSTD_btultra /*! HUF_compress() does the following: * 1. count symbol occurrence from source[] into table count[] using FSE_count() (exposed within "fse.h") @@ -186,12 +133,12 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, * For example, it's possible to compress several blocks using the same 'CTable', * or to save and regenerate 'CTable' using external methods. */ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue); -size_t HUF_buildCTable (HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits); /* @return : maxNbBits; CTable and count can overlap. In which case, CTable will overwrite count content */ -size_t HUF_writeCTable (void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog); +unsigned HUF_minTableLog(unsigned symbolCardinality); +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue); +unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue, void* workSpace, + size_t wkspSize, HUF_CElt* table, const unsigned* count, int flags); /* table is used as scratch space for building and testing tables, not a return value */ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog, void* workspace, size_t workspaceSize); -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); -size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2); +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); size_t HUF_estimateCompressedSize(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); int HUF_validateCTable(const HUF_CElt* CTable, const unsigned* count, unsigned maxSymbolValue); @@ -200,6 +147,7 @@ typedef enum { HUF_repeat_check, /**< Can use the previous table but it must be checked. Note : The previous table must have been constructed by HUF_compress{1, 4}X_repeat */ HUF_repeat_valid /**< Can use the previous table and it is assumed to be valid */ } HUF_repeat; + /** HUF_compress4X_repeat() : * Same as HUF_compress4X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. @@ -210,13 +158,13 @@ size_t HUF_compress4X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible); + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); /** HUF_buildCTable_wksp() : * Same as HUF_buildCTable(), but using externally allocated scratch buffer. * `workSpace` must be aligned on 4-bytes boundaries, and its size must be >= HUF_CTABLE_WORKSPACE_SIZE. */ -#define HUF_CTABLE_WORKSPACE_SIZE_U32 (2*HUF_SYMBOLVALUE_MAX +1 +1) +#define HUF_CTABLE_WORKSPACE_SIZE_U32 ((4 * (HUF_SYMBOLVALUE_MAX + 1)) + 192) #define HUF_CTABLE_WORKSPACE_SIZE (HUF_CTABLE_WORKSPACE_SIZE_U32 * sizeof(unsigned)) size_t HUF_buildCTable_wksp (HUF_CElt* tree, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, @@ -242,7 +190,7 @@ size_t HUF_readStats_wksp(BYTE* huffWeight, size_t hwSize, U32* rankStats, U32* nbSymbolsPtr, U32* tableLogPtr, const void* src, size_t srcSize, void* workspace, size_t wkspSize, - int bmi2); + int flags); /** HUF_readCTable() : * Loading a CTable saved with HUF_writeCTable() */ @@ -280,32 +228,12 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize); #define HUF_DECOMPRESS_WORKSPACE_SIZE ((2 << 10) + (1 << 9)) #define HUF_DECOMPRESS_WORKSPACE_SIZE_U32 (HUF_DECOMPRESS_WORKSPACE_SIZE / sizeof(U32)) -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1 (HUF_DTable* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX1_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_readDTableX2 (HUF_DTable* DTable, const void* src, size_t srcSize); -size_t HUF_readDTableX2_wksp (HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize); -#endif - -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress4X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif - /* ====================== */ /* single stream variants */ /* ====================== */ -size_t HUF_compress1X (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog); -size_t HUF_compress1X_wksp (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize); /**< `workSpace` must be a table of at least HUF_WORKSPACE_SIZE_U64 U64 */ -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable); -size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2); +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags); /** HUF_compress1X_repeat() : * Same as HUF_compress1X_wksp(), but considers using hufTable if *repeat != HUF_repeat_none. * If it uses hufTable it does not modify hufTable or repeat. @@ -316,49 +244,30 @@ size_t HUF_compress1X_repeat(void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned tableLog, void* workSpace, size_t wkspSize, /**< `workSpace` must be aligned on 4-bytes boundaries, `wkspSize` must be >= HUF_WORKSPACE_SIZE */ - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible); + HUF_CElt* hufTable, HUF_repeat* repeat, int flags); -size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* single-symbol decoder */ +size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /* double-symbol decoder */ -#endif - -size_t HUF_decompress1X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); -size_t HUF_decompress1X_DCtx_wksp (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< single-symbol decoder */ -size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< single-symbol decoder */ -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); /**< double-symbols decoder */ -size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize); /**< double-symbols decoder */ -#endif - -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); /**< automatic selection of sing or double symbol decoder, based on DTable */ -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); -#endif -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress1X2_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable); +size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); /**< double-symbols decoder */ #endif /* BMI2 variants. * If the CPU has BMI2 support, pass bmi2=1, otherwise pass bmi2=0. */ -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #endif -size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2); -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags); +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags); #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); #endif #ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2); +size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags); #endif -#endif /* HUF_STATIC_LINKING_ONLY */ +#endif /* HUF_H_298734234 */ #if defined (__cplusplus) } diff --git a/vendor/github.com/DataDog/zstd/huf_compress.c b/vendor/github.com/DataDog/zstd/huf_compress.c index 2633ffd..7fb3c96 100644 --- a/vendor/github.com/DataDog/zstd/huf_compress.c +++ b/vendor/github.com/DataDog/zstd/huf_compress.c @@ -1,7 +1,7 @@ #ifndef USE_EXTERNAL_ZSTD /* ****************************************************************** * Huffman encoder, part of New Generation Entropy library - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -30,9 +30,9 @@ #include "hist.h" #define FSE_STATIC_LINKING_ONLY /* FSE_optimalTableLog_internal */ #include "fse.h" /* header compression */ -#define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "error_private.h" +#include "bits.h" /* ZSTD_highbit32 */ /* ************************************************************** @@ -43,13 +43,67 @@ /* ************************************************************** -* Utils +* Required declarations ****************************************************************/ -unsigned HUF_optimalTableLog(unsigned maxTableLog, size_t srcSize, unsigned maxSymbolValue) +typedef struct nodeElt_s { + U32 count; + U16 parent; + BYTE byte; + BYTE nbBits; +} nodeElt; + + +/* ************************************************************** +* Debug Traces +****************************************************************/ + +#if DEBUGLEVEL >= 2 + +static size_t showU32(const U32* arr, size_t size) { - return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); + size_t u; + for (u=0; u= sizeof(HUF_WriteCTableWksp)); + /* check conditions */ if (workspaceSize < sizeof(HUF_WriteCTableWksp)) return ERROR(GENERIC); if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) return ERROR(maxSymbolValue_tooLarge); @@ -208,16 +267,6 @@ size_t HUF_writeCTable_wksp(void* dst, size_t maxDstSize, return ((maxSymbolValue+1)/2) + 1; } -/*! HUF_writeCTable() : - `CTable` : Huffman tree to save, using huf representation. - @return : size of saved CTable */ -size_t HUF_writeCTable (void* dst, size_t maxDstSize, - const HUF_CElt* CTable, unsigned maxSymbolValue, unsigned huffLog) -{ - HUF_WriteCTableWksp wksp; - return HUF_writeCTable_wksp(dst, maxDstSize, CTable, maxSymbolValue, huffLog, &wksp, sizeof(wksp)); -} - size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void* src, size_t srcSize, unsigned* hasZeroWeights) { @@ -273,68 +322,64 @@ size_t HUF_readCTable (HUF_CElt* CTable, unsigned* maxSymbolValuePtr, const void U32 HUF_getNbBitsFromCTable(HUF_CElt const* CTable, U32 symbolValue) { - const HUF_CElt* ct = CTable + 1; + const HUF_CElt* const ct = CTable + 1; assert(symbolValue <= HUF_SYMBOLVALUE_MAX); return (U32)HUF_getNbBits(ct[symbolValue]); } -typedef struct nodeElt_s { - U32 count; - U16 parent; - BYTE byte; - BYTE nbBits; -} nodeElt; - /** * HUF_setMaxHeight(): - * Enforces maxNbBits on the Huffman tree described in huffNode. + * Try to enforce @targetNbBits on the Huffman tree described in @huffNode. * - * It sets all nodes with nbBits > maxNbBits to be maxNbBits. Then it adjusts - * the tree to so that it is a valid canonical Huffman tree. + * It attempts to convert all nodes with nbBits > @targetNbBits + * to employ @targetNbBits instead. Then it adjusts the tree + * so that it remains a valid canonical Huffman tree. * * @pre The sum of the ranks of each symbol == 2^largestBits, * where largestBits == huffNode[lastNonNull].nbBits. * @post The sum of the ranks of each symbol == 2^largestBits, - * where largestBits is the return value <= maxNbBits. + * where largestBits is the return value (expected <= targetNbBits). * - * @param huffNode The Huffman tree modified in place to enforce maxNbBits. + * @param huffNode The Huffman tree modified in place to enforce targetNbBits. + * It's presumed sorted, from most frequent to rarest symbol. * @param lastNonNull The symbol with the lowest count in the Huffman tree. - * @param maxNbBits The maximum allowed number of bits, which the Huffman tree + * @param targetNbBits The allowed number of bits, which the Huffman tree * may not respect. After this function the Huffman tree will - * respect maxNbBits. - * @return The maximum number of bits of the Huffman tree after adjustment, - * necessarily no more than maxNbBits. + * respect targetNbBits. + * @return The maximum number of bits of the Huffman tree after adjustment. */ -static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) +static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 targetNbBits) { const U32 largestBits = huffNode[lastNonNull].nbBits; - /* early exit : no elt > maxNbBits, so the tree is already valid. */ - if (largestBits <= maxNbBits) return largestBits; + /* early exit : no elt > targetNbBits, so the tree is already valid. */ + if (largestBits <= targetNbBits) return largestBits; + + DEBUGLOG(5, "HUF_setMaxHeight (targetNbBits = %u)", targetNbBits); /* there are several too large elements (at least >= 2) */ { int totalCost = 0; - const U32 baseCost = 1 << (largestBits - maxNbBits); + const U32 baseCost = 1 << (largestBits - targetNbBits); int n = (int)lastNonNull; - /* Adjust any ranks > maxNbBits to maxNbBits. + /* Adjust any ranks > targetNbBits to targetNbBits. * Compute totalCost, which is how far the sum of the ranks is * we are over 2^largestBits after adjust the offending ranks. */ - while (huffNode[n].nbBits > maxNbBits) { + while (huffNode[n].nbBits > targetNbBits) { totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)); - huffNode[n].nbBits = (BYTE)maxNbBits; + huffNode[n].nbBits = (BYTE)targetNbBits; n--; } - /* n stops at huffNode[n].nbBits <= maxNbBits */ - assert(huffNode[n].nbBits <= maxNbBits); - /* n end at index of smallest symbol using < maxNbBits */ - while (huffNode[n].nbBits == maxNbBits) --n; + /* n stops at huffNode[n].nbBits <= targetNbBits */ + assert(huffNode[n].nbBits <= targetNbBits); + /* n end at index of smallest symbol using < targetNbBits */ + while (huffNode[n].nbBits == targetNbBits) --n; - /* renorm totalCost from 2^largestBits to 2^maxNbBits + /* renorm totalCost from 2^largestBits to 2^targetNbBits * note : totalCost is necessarily a multiple of baseCost */ - assert((totalCost & (baseCost - 1)) == 0); - totalCost >>= (largestBits - maxNbBits); + assert(((U32)totalCost & (baseCost - 1)) == 0); + totalCost >>= (largestBits - targetNbBits); assert(totalCost > 0); /* repay normalized cost */ @@ -343,19 +388,19 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) /* Get pos of last (smallest = lowest cum. count) symbol per rank */ ZSTD_memset(rankLast, 0xF0, sizeof(rankLast)); - { U32 currentNbBits = maxNbBits; + { U32 currentNbBits = targetNbBits; int pos; for (pos=n ; pos >= 0; pos--) { if (huffNode[pos].nbBits >= currentNbBits) continue; - currentNbBits = huffNode[pos].nbBits; /* < maxNbBits */ - rankLast[maxNbBits-currentNbBits] = (U32)pos; + currentNbBits = huffNode[pos].nbBits; /* < targetNbBits */ + rankLast[targetNbBits-currentNbBits] = (U32)pos; } } while (totalCost > 0) { /* Try to reduce the next power of 2 above totalCost because we * gain back half the rank. */ - U32 nBitsToDecrease = BIT_highbit32((U32)totalCost) + 1; + U32 nBitsToDecrease = ZSTD_highbit32((U32)totalCost) + 1; for ( ; nBitsToDecrease > 1; nBitsToDecrease--) { U32 const highPos = rankLast[nBitsToDecrease]; U32 const lowPos = rankLast[nBitsToDecrease-1]; @@ -395,7 +440,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) rankLast[nBitsToDecrease] = noSymbol; else { rankLast[nBitsToDecrease]--; - if (huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease) + if (huffNode[rankLast[nBitsToDecrease]].nbBits != targetNbBits-nBitsToDecrease) rankLast[nBitsToDecrease] = noSymbol; /* this rank is now empty */ } } /* while (totalCost > 0) */ @@ -407,11 +452,11 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) * TODO. */ while (totalCost < 0) { /* Sometimes, cost correction overshoot */ - /* special case : no rank 1 symbol (using maxNbBits-1); - * let's create one from largest rank 0 (using maxNbBits). + /* special case : no rank 1 symbol (using targetNbBits-1); + * let's create one from largest rank 0 (using targetNbBits). */ if (rankLast[1] == noSymbol) { - while (huffNode[n].nbBits == maxNbBits) n--; + while (huffNode[n].nbBits == targetNbBits) n--; huffNode[n+1].nbBits--; assert(n >= 0); rankLast[1] = (U32)(n+1); @@ -425,7 +470,7 @@ static U32 HUF_setMaxHeight(nodeElt* huffNode, U32 lastNonNull, U32 maxNbBits) } /* repay normalized cost */ } /* there are several too large elements (at least >= 2) */ - return maxNbBits; + return targetNbBits; } typedef struct { @@ -433,7 +478,7 @@ typedef struct { U16 curr; } rankPos; -typedef nodeElt huffNodeTable[HUF_CTABLE_WORKSPACE_SIZE_U32]; +typedef nodeElt huffNodeTable[2 * (HUF_SYMBOLVALUE_MAX + 1)]; /* Number of buckets available for HUF_sort() */ #define RANK_POSITION_TABLE_SIZE 192 @@ -452,8 +497,8 @@ typedef struct { * Let buckets 166 to 192 represent all remaining counts up to RANK_POSITION_MAX_COUNT_LOG using log2 bucketing. */ #define RANK_POSITION_MAX_COUNT_LOG 32 -#define RANK_POSITION_LOG_BUCKETS_BEGIN (RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */ -#define RANK_POSITION_DISTINCT_COUNT_CUTOFF RANK_POSITION_LOG_BUCKETS_BEGIN + BIT_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */ +#define RANK_POSITION_LOG_BUCKETS_BEGIN ((RANK_POSITION_TABLE_SIZE - 1) - RANK_POSITION_MAX_COUNT_LOG - 1 /* == 158 */) +#define RANK_POSITION_DISTINCT_COUNT_CUTOFF (RANK_POSITION_LOG_BUCKETS_BEGIN + ZSTD_highbit32(RANK_POSITION_LOG_BUCKETS_BEGIN) /* == 166 */) /* Return the appropriate bucket index for a given count. See definition of * RANK_POSITION_DISTINCT_COUNT_CUTOFF for explanation of bucketing strategy. @@ -461,7 +506,7 @@ typedef struct { static U32 HUF_getIndex(U32 const count) { return (count < RANK_POSITION_DISTINCT_COUNT_CUTOFF) ? count - : BIT_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN; + : ZSTD_highbit32(count) + RANK_POSITION_LOG_BUCKETS_BEGIN; } /* Helper swap function for HUF_quickSortPartition() */ @@ -584,7 +629,7 @@ static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSy /* Sort each bucket. */ for (n = RANK_POSITION_DISTINCT_COUNT_CUTOFF; n < RANK_POSITION_TABLE_SIZE - 1; ++n) { - U32 const bucketSize = rankPosition[n].curr-rankPosition[n].base; + int const bucketSize = rankPosition[n].curr - rankPosition[n].base; U32 const bucketStartIdx = rankPosition[n].base; if (bucketSize > 1) { assert(bucketStartIdx < maxSymbolValue1); @@ -595,6 +640,7 @@ static void HUF_sort(nodeElt huffNode[], const unsigned count[], U32 const maxSy assert(HUF_isSorted(huffNode, maxSymbolValue1)); } + /** HUF_buildCTable_wksp() : * Same as HUF_buildCTable(), but using externally allocated scratch buffer. * `workSpace` must be aligned on 4-bytes boundaries, and be at least as large as sizeof(HUF_buildCTable_wksp_tables). @@ -615,6 +661,7 @@ static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) int lowS, lowN; int nodeNb = STARTNODE; int n, nodeRoot; + DEBUGLOG(5, "HUF_buildTree (alphabet size = %u)", maxSymbolValue + 1); /* init for parents */ nonNullRank = (int)maxSymbolValue; while(huffNode[nonNullRank].count == 0) nonNullRank--; @@ -641,6 +688,8 @@ static int HUF_buildTree(nodeElt* huffNode, U32 maxSymbolValue) for (n=0; n<=nonNullRank; n++) huffNode[n].nbBits = huffNode[ huffNode[n].parent ].nbBits + 1; + DEBUGLOG(6, "Initial distribution of bits completed (%zu sorted symbols)", showHNodeBits(huffNode, maxSymbolValue+1)); + return nonNullRank; } @@ -678,28 +727,36 @@ static void HUF_buildCTableFromTree(HUF_CElt* CTable, nodeElt const* huffNode, i CTable[0] = maxNbBits; } -size_t HUF_buildCTable_wksp (HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, void* workSpace, size_t wkspSize) +size_t +HUF_buildCTable_wksp(HUF_CElt* CTable, const unsigned* count, U32 maxSymbolValue, U32 maxNbBits, + void* workSpace, size_t wkspSize) { - HUF_buildCTable_wksp_tables* const wksp_tables = (HUF_buildCTable_wksp_tables*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(U32)); + HUF_buildCTable_wksp_tables* const wksp_tables = + (HUF_buildCTable_wksp_tables*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(U32)); nodeElt* const huffNode0 = wksp_tables->huffNodeTbl; nodeElt* const huffNode = huffNode0+1; int nonNullRank; + HUF_STATIC_ASSERT(HUF_CTABLE_WORKSPACE_SIZE == sizeof(HUF_buildCTable_wksp_tables)); + + DEBUGLOG(5, "HUF_buildCTable_wksp (alphabet size = %u)", maxSymbolValue+1); + /* safety checks */ if (wkspSize < sizeof(HUF_buildCTable_wksp_tables)) - return ERROR(workSpace_tooSmall); + return ERROR(workSpace_tooSmall); if (maxNbBits == 0) maxNbBits = HUF_TABLELOG_DEFAULT; if (maxSymbolValue > HUF_SYMBOLVALUE_MAX) - return ERROR(maxSymbolValue_tooLarge); + return ERROR(maxSymbolValue_tooLarge); ZSTD_memset(huffNode0, 0, sizeof(huffNodeTable)); /* sort, decreasing order */ HUF_sort(huffNode, count, maxSymbolValue, wksp_tables->rankPosition); + DEBUGLOG(6, "sorted symbols completed (%zu symbols)", showHNodeSymbols(huffNode, maxSymbolValue+1)); /* build tree */ nonNullRank = HUF_buildTree(huffNode, maxSymbolValue); - /* enforce maxTableLog */ + /* determine and enforce maxTableLog */ maxNbBits = HUF_setMaxHeight(huffNode, (U32)nonNullRank, maxNbBits); if (maxNbBits > HUF_TABLELOG_MAX) return ERROR(GENERIC); /* check fit into table */ @@ -808,7 +865,7 @@ FORCE_INLINE_TEMPLATE void HUF_addBits(HUF_CStream_t* bitC, HUF_CElt elt, int id #if DEBUGLEVEL >= 1 { size_t const nbBits = HUF_getNbBits(elt); - size_t const dirtyBits = nbBits == 0 ? 0 : BIT_highbit32((U32)nbBits) + 1; + size_t const dirtyBits = nbBits == 0 ? 0 : ZSTD_highbit32((U32)nbBits) + 1; (void)dirtyBits; /* Middle bits are 0. */ assert(((elt >> dirtyBits) << (dirtyBits + nbBits)) == 0); @@ -888,7 +945,7 @@ static size_t HUF_closeCStream(HUF_CStream_t* bitC) { size_t const nbBits = bitC->bitPos[0] & 0xFF; if (bitC->ptr >= bitC->endPtr) return 0; /* overflow detected */ - return (bitC->ptr - bitC->startPtr) + (nbBits > 0); + return (size_t)(bitC->ptr - bitC->startPtr) + (nbBits > 0); } } @@ -1049,9 +1106,9 @@ HUF_compress1X_usingCTable_internal_default(void* dst, size_t dstSize, static size_t HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, const void* src, size_t srcSize, - const HUF_CElt* CTable, const int bmi2) + const HUF_CElt* CTable, const int flags) { - if (bmi2) { + if (flags & HUF_flags_bmi2) { return HUF_compress1X_usingCTable_internal_bmi2(dst, dstSize, src, srcSize, CTable); } return HUF_compress1X_usingCTable_internal_default(dst, dstSize, src, srcSize, CTable); @@ -1062,28 +1119,23 @@ HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, static size_t HUF_compress1X_usingCTable_internal(void* dst, size_t dstSize, const void* src, size_t srcSize, - const HUF_CElt* CTable, const int bmi2) + const HUF_CElt* CTable, const int flags) { - (void)bmi2; + (void)flags; return HUF_compress1X_usingCTable_internal_body(dst, dstSize, src, srcSize, CTable); } #endif -size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +size_t HUF_compress1X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) { - return HUF_compress1X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); -} - -size_t HUF_compress1X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2) -{ - return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2); + return HUF_compress1X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); } static size_t HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, const void* src, size_t srcSize, - const HUF_CElt* CTable, int bmi2) + const HUF_CElt* CTable, int flags) { size_t const segmentSize = (srcSize+3)/4; /* first 3 segments */ const BYTE* ip = (const BYTE*) src; @@ -1097,7 +1149,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, op += 6; /* jumpTable */ assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); if (cSize == 0 || cSize > 65535) return 0; MEM_writeLE16(ostart, (U16)cSize); op += cSize; @@ -1105,7 +1157,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, ip += segmentSize; assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); if (cSize == 0 || cSize > 65535) return 0; MEM_writeLE16(ostart+2, (U16)cSize); op += cSize; @@ -1113,7 +1165,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, ip += segmentSize; assert(op <= oend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, bmi2) ); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, segmentSize, CTable, flags) ); if (cSize == 0 || cSize > 65535) return 0; MEM_writeLE16(ostart+4, (U16)cSize); op += cSize; @@ -1122,7 +1174,7 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, ip += segmentSize; assert(op <= oend); assert(ip <= iend); - { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, bmi2) ); + { CHECK_V_F(cSize, HUF_compress1X_usingCTable_internal(op, (size_t)(oend-op), ip, (size_t)(iend-ip), CTable, flags) ); if (cSize == 0 || cSize > 65535) return 0; op += cSize; } @@ -1130,14 +1182,9 @@ HUF_compress4X_usingCTable_internal(void* dst, size_t dstSize, return (size_t)(op-ostart); } -size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable) +size_t HUF_compress4X_usingCTable(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int flags) { - return HUF_compress4X_usingCTable_bmi2(dst, dstSize, src, srcSize, CTable, /* bmi2 */ 0); -} - -size_t HUF_compress4X_usingCTable_bmi2(void* dst, size_t dstSize, const void* src, size_t srcSize, const HUF_CElt* CTable, int bmi2) -{ - return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, bmi2); + return HUF_compress4X_usingCTable_internal(dst, dstSize, src, srcSize, CTable, flags); } typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; @@ -1145,11 +1192,11 @@ typedef enum { HUF_singleStream, HUF_fourStreams } HUF_nbStreams_e; static size_t HUF_compressCTable_internal( BYTE* const ostart, BYTE* op, BYTE* const oend, const void* src, size_t srcSize, - HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int bmi2) + HUF_nbStreams_e nbStreams, const HUF_CElt* CTable, const int flags) { size_t const cSize = (nbStreams==HUF_singleStream) ? - HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2) : - HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, bmi2); + HUF_compress1X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags) : + HUF_compress4X_usingCTable_internal(op, (size_t)(oend - op), src, srcSize, CTable, flags); if (HUF_isError(cSize)) { return cSize; } if (cSize==0) { return 0; } /* uncompressible */ op += cSize; @@ -1172,6 +1219,79 @@ typedef struct { #define SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE 4096 #define SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO 10 /* Must be >= 2 */ +unsigned HUF_cardinality(const unsigned* count, unsigned maxSymbolValue) +{ + unsigned cardinality = 0; + unsigned i; + + for (i = 0; i < maxSymbolValue + 1; i++) { + if (count[i] != 0) cardinality += 1; + } + + return cardinality; +} + +unsigned HUF_minTableLog(unsigned symbolCardinality) +{ + U32 minBitsSymbols = ZSTD_highbit32(symbolCardinality) + 1; + return minBitsSymbols; +} + +unsigned HUF_optimalTableLog( + unsigned maxTableLog, + size_t srcSize, + unsigned maxSymbolValue, + void* workSpace, size_t wkspSize, + HUF_CElt* table, + const unsigned* count, + int flags) +{ + assert(srcSize > 1); /* Not supported, RLE should be used instead */ + assert(wkspSize >= sizeof(HUF_buildCTable_wksp_tables)); + + if (!(flags & HUF_flags_optimalDepth)) { + /* cheap evaluation, based on FSE */ + return FSE_optimalTableLog_internal(maxTableLog, srcSize, maxSymbolValue, 1); + } + + { BYTE* dst = (BYTE*)workSpace + sizeof(HUF_WriteCTableWksp); + size_t dstSize = wkspSize - sizeof(HUF_WriteCTableWksp); + size_t maxBits, hSize, newSize; + const unsigned symbolCardinality = HUF_cardinality(count, maxSymbolValue); + const unsigned minTableLog = HUF_minTableLog(symbolCardinality); + size_t optSize = ((size_t) ~0) - 1; + unsigned optLog = maxTableLog, optLogGuess; + + DEBUGLOG(6, "HUF_optimalTableLog: probing huf depth (srcSize=%zu)", srcSize); + + /* Search until size increases */ + for (optLogGuess = minTableLog; optLogGuess <= maxTableLog; optLogGuess++) { + DEBUGLOG(7, "checking for huffLog=%u", optLogGuess); + maxBits = HUF_buildCTable_wksp(table, count, maxSymbolValue, optLogGuess, workSpace, wkspSize); + if (ERR_isError(maxBits)) continue; + + if (maxBits < optLogGuess && optLogGuess > minTableLog) break; + + hSize = HUF_writeCTable_wksp(dst, dstSize, table, maxSymbolValue, (U32)maxBits, workSpace, wkspSize); + + if (ERR_isError(hSize)) continue; + + newSize = HUF_estimateCompressedSize(table, count, maxSymbolValue) + hSize; + + if (newSize > optSize + 1) { + break; + } + + if (newSize < optSize) { + optSize = newSize; + optLog = optLogGuess; + } + } + assert(optLog <= HUF_TABLELOG_MAX); + return optLog; + } +} + /* HUF_compress_internal() : * `workSpace_align4` must be aligned on 4-bytes boundaries, * and occupies the same space as a table of HUF_WORKSPACE_SIZE_U64 unsigned */ @@ -1181,14 +1301,14 @@ HUF_compress_internal (void* dst, size_t dstSize, unsigned maxSymbolValue, unsigned huffLog, HUF_nbStreams_e nbStreams, void* workSpace, size_t wkspSize, - HUF_CElt* oldHufTable, HUF_repeat* repeat, int preferRepeat, - const int bmi2, unsigned suspectUncompressible) + HUF_CElt* oldHufTable, HUF_repeat* repeat, int flags) { HUF_compress_tables_t* const table = (HUF_compress_tables_t*)HUF_alignUpWorkspace(workSpace, &wkspSize, ZSTD_ALIGNOF(size_t)); BYTE* const ostart = (BYTE*)dst; BYTE* const oend = ostart + dstSize; BYTE* op = ostart; + DEBUGLOG(5, "HUF_compress_internal (srcSize=%zu)", srcSize); HUF_STATIC_ASSERT(sizeof(*table) + HUF_WORKSPACE_MAX_ALIGNMENT <= HUF_WORKSPACE_SIZE); /* checks & inits */ @@ -1202,16 +1322,17 @@ HUF_compress_internal (void* dst, size_t dstSize, if (!huffLog) huffLog = HUF_TABLELOG_DEFAULT; /* Heuristic : If old table is valid, use it for small inputs */ - if (preferRepeat && repeat && *repeat == HUF_repeat_valid) { + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat == HUF_repeat_valid) { return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - nbStreams, oldHufTable, bmi2); + nbStreams, oldHufTable, flags); } /* If uncompressible data is suspected, do a smaller sampling first */ DEBUG_STATIC_ASSERT(SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO >= 2); - if (suspectUncompressible && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) { + if ((flags & HUF_flags_suspectUncompressible) && srcSize >= (SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE * SUSPECT_INCOMPRESSIBLE_SAMPLE_RATIO)) { size_t largestTotal = 0; + DEBUGLOG(5, "input suspected incompressible : sampling to check"); { unsigned maxSymbolValueBegin = maxSymbolValue; CHECK_V_F(largestBegin, HIST_count_simple (table->count, &maxSymbolValueBegin, (const BYTE*)src, SUSPECT_INCOMPRESSIBLE_SAMPLE_SIZE) ); largestTotal += largestBegin; @@ -1228,6 +1349,7 @@ HUF_compress_internal (void* dst, size_t dstSize, if (largest == srcSize) { *ostart = ((const BYTE*)src)[0]; return 1; } /* single symbol, rle */ if (largest <= (srcSize >> 7)+4) return 0; /* heuristic : probably not compressible enough */ } + DEBUGLOG(6, "histogram detail completed (%zu symbols)", showU32(table->count, maxSymbolValue+1)); /* Check validity of previous table */ if ( repeat @@ -1236,19 +1358,20 @@ HUF_compress_internal (void* dst, size_t dstSize, *repeat = HUF_repeat_none; } /* Heuristic : use existing table for small inputs */ - if (preferRepeat && repeat && *repeat != HUF_repeat_none) { + if ((flags & HUF_flags_preferRepeat) && repeat && *repeat != HUF_repeat_none) { return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - nbStreams, oldHufTable, bmi2); + nbStreams, oldHufTable, flags); } /* Build Huffman Tree */ - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, &table->wksps, sizeof(table->wksps), table->CTable, table->count, flags); { size_t const maxBits = HUF_buildCTable_wksp(table->CTable, table->count, maxSymbolValue, huffLog, &table->wksps.buildCTable_wksp, sizeof(table->wksps.buildCTable_wksp)); CHECK_F(maxBits); huffLog = (U32)maxBits; + DEBUGLOG(6, "bit distribution completed (%zu symbols)", showCTableBits(table->CTable + 1, maxSymbolValue+1)); } /* Zero unused symbols in CTable, so we can check it for validity */ { @@ -1267,7 +1390,7 @@ HUF_compress_internal (void* dst, size_t dstSize, if (oldSize <= hSize + newSize || hSize + 12 >= srcSize) { return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - nbStreams, oldHufTable, bmi2); + nbStreams, oldHufTable, flags); } } /* Use the new huffman table */ @@ -1279,46 +1402,20 @@ HUF_compress_internal (void* dst, size_t dstSize, } return HUF_compressCTable_internal(ostart, op, oend, src, srcSize, - nbStreams, table->CTable, bmi2); -} - - -size_t HUF_compress1X_wksp (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, HUF_singleStream, - workSpace, wkspSize, - NULL, NULL, 0, 0 /*bmi2*/, 0); + nbStreams, table->CTable, flags); } size_t HUF_compress1X_repeat (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void* workSpace, size_t wkspSize, - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, - int bmi2, unsigned suspectUncompressible) + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) { + DEBUGLOG(5, "HUF_compress1X_repeat (srcSize = %zu)", srcSize); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, HUF_singleStream, workSpace, wkspSize, hufTable, - repeat, preferRepeat, bmi2, suspectUncompressible); -} - -/* HUF_compress4X_repeat(): - * compress input using 4 streams. - * provide workspace to generate compression tables */ -size_t HUF_compress4X_wksp (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog, - void* workSpace, size_t wkspSize) -{ - return HUF_compress_internal(dst, dstSize, src, srcSize, - maxSymbolValue, huffLog, HUF_fourStreams, - workSpace, wkspSize, - NULL, NULL, 0, 0 /*bmi2*/, 0); + repeat, flags); } /* HUF_compress4X_repeat(): @@ -1329,45 +1426,13 @@ size_t HUF_compress4X_repeat (void* dst, size_t dstSize, const void* src, size_t srcSize, unsigned maxSymbolValue, unsigned huffLog, void* workSpace, size_t wkspSize, - HUF_CElt* hufTable, HUF_repeat* repeat, int preferRepeat, int bmi2, unsigned suspectUncompressible) + HUF_CElt* hufTable, HUF_repeat* repeat, int flags) { + DEBUGLOG(5, "HUF_compress4X_repeat (srcSize = %zu)", srcSize); return HUF_compress_internal(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, HUF_fourStreams, workSpace, wkspSize, - hufTable, repeat, preferRepeat, bmi2, suspectUncompressible); + hufTable, repeat, flags); } -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -/** HUF_buildCTable() : - * @return : maxNbBits - * Note : count is used before tree is written, so they can safely overlap - */ -size_t HUF_buildCTable (HUF_CElt* tree, const unsigned* count, unsigned maxSymbolValue, unsigned maxNbBits) -{ - HUF_buildCTable_wksp_tables workspace; - return HUF_buildCTable_wksp(tree, count, maxSymbolValue, maxNbBits, &workspace, sizeof(workspace)); -} - -size_t HUF_compress1X (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - U64 workSpace[HUF_WORKSPACE_SIZE_U64]; - return HUF_compress1X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); -} - -size_t HUF_compress2 (void* dst, size_t dstSize, - const void* src, size_t srcSize, - unsigned maxSymbolValue, unsigned huffLog) -{ - U64 workSpace[HUF_WORKSPACE_SIZE_U64]; - return HUF_compress4X_wksp(dst, dstSize, src, srcSize, maxSymbolValue, huffLog, workSpace, sizeof(workSpace)); -} - -size_t HUF_compress (void* dst, size_t maxDstSize, const void* src, size_t srcSize) -{ - return HUF_compress2(dst, maxDstSize, src, srcSize, 255, HUF_TABLELOG_DEFAULT); -} -#endif - #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/huf_decompress.c b/vendor/github.com/DataDog/zstd/huf_decompress.c index 6cb07c4..da4d184 100644 --- a/vendor/github.com/DataDog/zstd/huf_decompress.c +++ b/vendor/github.com/DataDog/zstd/huf_decompress.c @@ -2,7 +2,7 @@ /* ****************************************************************** * huff0 huffman decoder, * part of Finite State Entropy library - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : * - FSE+HUF source repository : https://github.com/Cyan4973/FiniteStateEntropy @@ -20,10 +20,10 @@ #include "compiler.h" #include "bitstream.h" /* BIT_* */ #include "fse.h" /* to compress headers */ -#define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "error_private.h" #include "zstd_internal.h" +#include "bits.h" /* ZSTD_highbit32, ZSTD_countTrailingZeros64 */ /* ************************************************************** * Constants @@ -44,10 +44,14 @@ #error "Cannot force the use of the X1 and X2 decoders at the same time!" #endif -#if ZSTD_ENABLE_ASM_X86_64_BMI2 && DYNAMIC_BMI2 -# define HUF_ASM_X86_64_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE +/* When DYNAMIC_BMI2 is enabled, fast decoders are only called when bmi2 is + * supported at runtime, so we can add the BMI2 target attribute. + * When it is disabled, we will still get BMI2 if it is enabled statically. + */ +#if DYNAMIC_BMI2 +# define HUF_FAST_BMI2_ATTRS BMI2_TARGET_ATTRIBUTE #else -# define HUF_ASM_X86_64_BMI2_ATTRS +# define HUF_FAST_BMI2_ATTRS #endif #ifdef __cplusplus @@ -57,18 +61,12 @@ #endif #define HUF_ASM_DECL HUF_EXTERN_C -#if DYNAMIC_BMI2 || (ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) +#if DYNAMIC_BMI2 # define HUF_NEED_BMI2_FUNCTION 1 #else # define HUF_NEED_BMI2_FUNCTION 0 #endif -#if !(ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__)) -# define HUF_NEED_DEFAULT_FUNCTION 1 -#else -# define HUF_NEED_DEFAULT_FUNCTION 0 -#endif - /* ************************************************************** * Error Management ****************************************************************/ @@ -85,6 +83,11 @@ /* ************************************************************** * BMI2 Variant Wrappers ****************************************************************/ +typedef size_t (*HUF_DecompressUsingDTableFn)(void *dst, size_t dstSize, + const void *cSrc, + size_t cSrcSize, + const HUF_DTable *DTable); + #if DYNAMIC_BMI2 #define HUF_DGEN(fn) \ @@ -106,9 +109,9 @@ } \ \ static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ { \ - if (bmi2) { \ + if (flags & HUF_flags_bmi2) { \ return fn##_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); \ } \ return fn##_default(dst, dstSize, cSrc, cSrcSize, DTable); \ @@ -118,9 +121,9 @@ #define HUF_DGEN(fn) \ static size_t fn(void* dst, size_t dstSize, void const* cSrc, \ - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) \ + size_t cSrcSize, HUF_DTable const* DTable, int flags) \ { \ - (void)bmi2; \ + (void)flags; \ return fn##_body(dst, dstSize, cSrc, cSrcSize, DTable); \ } @@ -139,15 +142,28 @@ static DTableDesc HUF_getDTableDesc(const HUF_DTable* table) return dtd; } -#if ZSTD_ENABLE_ASM_X86_64_BMI2 - -static size_t HUF_initDStream(BYTE const* ip) { +static size_t HUF_initFastDStream(BYTE const* ip) { BYTE const lastByte = ip[7]; - size_t const bitsConsumed = lastByte ? 8 - BIT_highbit32(lastByte) : 0; + size_t const bitsConsumed = lastByte ? 8 - ZSTD_highbit32(lastByte) : 0; size_t const value = MEM_readLEST(ip) | 1; assert(bitsConsumed <= 8); + assert(sizeof(size_t) == 8); return value << bitsConsumed; } + + +/** + * The input/output arguments to the Huffman fast decoding loop: + * + * ip [in/out] - The input pointers, must be updated to reflect what is consumed. + * op [in/out] - The output pointers, must be updated to reflect what is written. + * bits [in/out] - The bitstream containers, must be updated to reflect the current state. + * dt [in] - The decoding table. + * ilimit [in] - The input limit, stop when any input pointer is below ilimit. + * oend [in] - The end of the output stream. op[3] must not cross oend. + * iend [in] - The end of each input stream. ip[i] may cross iend[i], + * as long as it is above ilimit, but that indicates corruption. + */ typedef struct { BYTE const* ip[4]; BYTE* op[4]; @@ -156,15 +172,17 @@ typedef struct { BYTE const* ilimit; BYTE* oend; BYTE const* iend[4]; -} HUF_DecompressAsmArgs; +} HUF_DecompressFastArgs; + +typedef void (*HUF_DecompressFastLoopFn)(HUF_DecompressFastArgs*); /** - * Initializes args for the asm decoding loop. - * @returns 0 on success - * 1 if the fallback implementation should be used. + * Initializes args for the fast decoding loop. + * @returns 1 on success + * 0 if the fallback implementation should be used. * Or an error code on failure. */ -static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) +static size_t HUF_DecompressFastArgs_init(HUF_DecompressFastArgs* args, void* dst, size_t dstSize, void const* src, size_t srcSize, const HUF_DTable* DTable) { void const* dt = DTable + 1; U32 const dtLog = HUF_getDTableDesc(DTable).tableLog; @@ -173,9 +191,11 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, BYTE* const oend = (BYTE*)dst + dstSize; - /* The following condition is false on x32 platform, - * but HUF_asm is not compatible with this ABI */ - if (!(MEM_isLittleEndian() && !MEM_32bits())) return 1; + /* The fast decoding loop assumes 64-bit little-endian. + * This condition is false on x32. + */ + if (!MEM_isLittleEndian() || MEM_32bits()) + return 0; /* strict minimum : jump table + 1 byte per stream */ if (srcSize < 10) @@ -186,7 +206,7 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, * On small inputs we don't have enough data to trigger the fast loop, so use the old decoder. */ if (dtLog != HUF_DECODER_FAST_TABLELOG) - return 1; + return 0; /* Read the jump table. */ { @@ -200,13 +220,13 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, args->iend[2] = args->iend[1] + length2; args->iend[3] = args->iend[2] + length3; - /* HUF_initDStream() requires this, and this small of an input + /* HUF_initFastDStream() requires this, and this small of an input * won't benefit from the ASM loop anyways. * length1 must be >= 16 so that ip[0] >= ilimit before the loop * starts. */ if (length1 < 16 || length2 < 8 || length3 < 8 || length4 < 8) - return 1; + return 0; if (length4 > srcSize) return ERROR(corruption_detected); /* overflow */ } /* ip[] contains the position that is currently loaded into bits[]. */ @@ -223,7 +243,7 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, /* No point to call the ASM loop for tiny outputs. */ if (args->op[3] >= oend) - return 1; + return 0; /* bits[] is the bit container. * It is read from the MSB down to the LSB. @@ -232,10 +252,10 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, * set, so that CountTrailingZeros(bits[]) can be used * to count how many bits we've consumed. */ - args->bits[0] = HUF_initDStream(args->ip[0]); - args->bits[1] = HUF_initDStream(args->ip[1]); - args->bits[2] = HUF_initDStream(args->ip[2]); - args->bits[3] = HUF_initDStream(args->ip[3]); + args->bits[0] = HUF_initFastDStream(args->ip[0]); + args->bits[1] = HUF_initFastDStream(args->ip[1]); + args->bits[2] = HUF_initFastDStream(args->ip[2]); + args->bits[3] = HUF_initFastDStream(args->ip[3]); /* If ip[] >= ilimit, it is guaranteed to be safe to * reload bits[]. It may be beyond its section, but is @@ -246,10 +266,10 @@ static size_t HUF_DecompressAsmArgs_init(HUF_DecompressAsmArgs* args, void* dst, args->oend = oend; args->dt = dt; - return 0; + return 1; } -static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs const* args, int stream, BYTE* segmentEnd) +static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressFastArgs const* args, int stream, BYTE* segmentEnd) { /* Validate that we haven't overwritten. */ if (args->op[stream] > segmentEnd) @@ -263,15 +283,15 @@ static size_t HUF_initRemainingDStream(BIT_DStream_t* bit, HUF_DecompressAsmArgs return ERROR(corruption_detected); /* Construct the BIT_DStream_t. */ - bit->bitContainer = MEM_readLE64(args->ip[stream]); - bit->bitsConsumed = ZSTD_countTrailingZeros((size_t)args->bits[stream]); + assert(sizeof(size_t) == 8); + bit->bitContainer = MEM_readLEST(args->ip[stream]); + bit->bitsConsumed = ZSTD_countTrailingZeros64(args->bits[stream]); bit->start = (const char*)args->iend[0]; bit->limitPtr = bit->start + sizeof(size_t); bit->ptr = (const char*)args->ip[stream]; return 0; } -#endif #ifndef HUF_FORCE_DECOMPRESS_X2 @@ -288,10 +308,11 @@ typedef struct { BYTE nbBits; BYTE byte; } HUF_DEltX1; /* single-symbol decodi static U64 HUF_DEltX1_set4(BYTE symbol, BYTE nbBits) { U64 D4; if (MEM_isLittleEndian()) { - D4 = (symbol << 8) + nbBits; + D4 = (U64)((symbol << 8) + nbBits); } else { - D4 = symbol + (nbBits << 8); + D4 = (U64)(symbol + (nbBits << 8)); } + assert(D4 < (1U << 16)); D4 *= 0x0001000100010001ULL; return D4; } @@ -334,13 +355,7 @@ typedef struct { BYTE huffWeight[HUF_SYMBOLVALUE_MAX + 1]; } HUF_ReadDTableX1_Workspace; - -size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize) -{ - return HUF_readDTableX1_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); -} - -size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int bmi2) +size_t HUF_readDTableX1_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, void* workSpace, size_t wkspSize, int flags) { U32 tableLog = 0; U32 nbSymbols = 0; @@ -355,7 +370,7 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr DEBUG_STATIC_ASSERT(sizeof(DTableDesc) == sizeof(HUF_DTable)); /* ZSTD_memset(huffWeight, 0, sizeof(huffWeight)); */ /* is not necessary, even though some analyzer complain ... */ - iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), bmi2); + iSize = HUF_readStats_wksp(wksp->huffWeight, HUF_SYMBOLVALUE_MAX + 1, wksp->rankVal, &nbSymbols, &tableLog, src, srcSize, wksp->statsWksp, sizeof(wksp->statsWksp), flags); if (HUF_isError(iSize)) return iSize; @@ -382,9 +397,8 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr * rankStart[0] is not filled because there are no entries in the table for * weight 0. */ - { - int n; - int nextRankStart = 0; + { int n; + U32 nextRankStart = 0; int const unroll = 4; int const nLimit = (int)nbSymbols - unroll + 1; for (n=0; n<(int)tableLog+1; n++) { @@ -411,10 +425,9 @@ size_t HUF_readDTableX1_wksp_bmi2(HUF_DTable* DTable, const void* src, size_t sr * We can switch based on the length to a different inner loop which is * optimized for that particular case. */ - { - U32 w; - int symbol=wksp->rankVal[0]; - int rankStart=0; + { U32 w; + int symbol = wksp->rankVal[0]; + int rankStart = 0; for (w=1; wrankVal[w]; int const length = (1 << w) >> 1; @@ -524,7 +537,7 @@ HUF_decodeStreamX1(BYTE* p, BIT_DStream_t* const bitDPtr, BYTE* const pEnd, cons while (p < pEnd) HUF_DECODE_SYMBOLX1_0(p, bitDPtr); - return pEnd-pStart; + return (size_t)(pEnd-pStart); } FORCE_INLINE_TEMPLATE size_t @@ -550,6 +563,10 @@ HUF_decompress1X1_usingDTable_internal_body( return dstSize; } +/* HUF_decompress4X1_usingDTable_internal_body(): + * Conditions : + * @dstSize >= 6 + */ FORCE_INLINE_TEMPLATE size_t HUF_decompress4X1_usingDTable_internal_body( void* dst, size_t dstSize, @@ -593,6 +610,7 @@ HUF_decompress4X1_usingDTable_internal_body( if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); @@ -655,38 +673,142 @@ size_t HUF_decompress4X1_usingDTable_internal_bmi2(void* dst, size_t dstSize, vo } #endif -#if HUF_NEED_DEFAULT_FUNCTION static size_t HUF_decompress4X1_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, size_t cSrcSize, HUF_DTable const* DTable) { return HUF_decompress4X1_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); } -#endif #if ZSTD_ENABLE_ASM_X86_64_BMI2 -HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN; +HUF_ASM_DECL void HUF_decompress4X1_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; -static HUF_ASM_X86_64_BMI2_ATTRS +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X1_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + U16 const* const dtable = (U16 const*)args->dt; + BYTE* const oend = args->oend; + BYTE const* const ilimit = args->ilimit; + + /* Copy the arguments to local variables */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + int symbol; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= (stream == 3 ? oend : op[stream + 1])); + assert(ip[stream] >= ilimit); + } +#endif + /* Compute olimit */ + { + /* Each iteration produces 5 output symbols per stream */ + size_t const oiters = (size_t)(oend - op[3]) / 5; + /* Each iteration consumes up to 11 bits * 5 = 55 bits < 7 bytes + * per stream. + */ + size_t const iiters = (size_t)(ip[0] - ilimit) / 7; + /* We can safely run iters iterations before running bounds checks */ + size_t const iters = MIN(oiters, iiters); + size_t const symbols = iters * 5; + + /* We can simply check that op[3] < olimit, instead of checking all + * of our bounds, since we can't hit the other bounds until we've run + * iters iterations, which only happens when op[3] == olimit. + */ + olimit = op[3] + symbols; + + /* Exit fast decoding loop once we get close to the end. */ + if (op[3] + 20 > olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + + do { + /* Decode 5 symbols in each of the 4 streams */ + for (symbol = 0; symbol < 5; ++symbol) { + for (stream = 0; stream < 4; ++stream) { + int const index = (int)(bits[stream] >> 53); + int const entry = (int)dtable[index]; + bits[stream] <<= (entry & 63); + op[stream][symbol] = (BYTE)((entry >> 8) & 0xFF); + } + } + /* Reload the bitstreams */ + for (stream = 0; stream < 4; ++stream) { + int const ctz = ZSTD_countTrailingZeros64(bits[stream]); + int const nbBits = ctz & 7; + int const nbBytes = ctz >> 3; + op[stream] += 5; + ip[stream] -= nbBytes; + bits[stream] = MEM_read64(ip[stream]) | 1; + bits[stream] <<= nbBits; + } + } while (op[3] < olimit); + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + +/** + * @returns @p dstSize on success (>= 6) + * 0 if the fallback implementation should be used + * An error if an error occurred + */ +static HUF_FAST_BMI2_ATTRS size_t -HUF_decompress4X1_usingDTable_internal_bmi2_asm( +HUF_decompress4X1_usingDTable_internal_fast( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { void const* dt = DTable + 1; const BYTE* const iend = (const BYTE*)cSrc + 6; BYTE* const oend = (BYTE*)dst + dstSize; - HUF_DecompressAsmArgs args; - { - size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); - FORWARD_IF_ERROR(ret, "Failed to init asm args"); - if (ret != 0) - return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + HUF_DecompressFastArgs args; + { size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + FORWARD_IF_ERROR(ret, "Failed to init fast loop args"); + if (ret == 0) + return 0; } assert(args.ip[0] >= args.ilimit); - HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop(&args); + loopFn(&args); /* Our loop guarantees that ip[] >= ilimit and that we haven't * overwritten any op[]. @@ -699,8 +821,7 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm( (void)iend; /* finish bit streams one by one. */ - { - size_t const segmentSize = (dstSize+3) / 4; + { size_t const segmentSize = (dstSize+3) / 4; BYTE* segmentEnd = (BYTE*)dst; int i; for (i = 0; i < 4; ++i) { @@ -717,97 +838,59 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm( } /* decoded size */ + assert(dstSize != 0); return dstSize; } -#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */ - -typedef size_t (*HUF_decompress_usingDTable_t)(void *dst, size_t dstSize, - const void *cSrc, - size_t cSrcSize, - const HUF_DTable *DTable); HUF_DGEN(HUF_decompress1X1_usingDTable_internal) static size_t HUF_decompress4X1_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) + size_t cSrcSize, HUF_DTable const* DTable, int flags) { + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X1_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X1_usingDTable_internal_fast_c_loop; + #if DYNAMIC_BMI2 - if (bmi2) { + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X1_usingDTable_internal_bmi2; # if ZSTD_ENABLE_ASM_X86_64_BMI2 - return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); -# else - return HUF_decompress4X1_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } # endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } -#else - (void)bmi2; #endif #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) - return HUF_decompress4X1_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); -#else - return HUF_decompress4X1_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable); + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X1_usingDTable_internal_fast_asm_loop; + } #endif + + if (!(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X1_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } - -size_t HUF_decompress1X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - -size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, +static size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) + void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; - size_t const hSize = HUF_readDTableX1_wksp(DCtx, cSrc, cSrcSize, workSpace, wkspSize); + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); + return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); } - -size_t HUF_decompress4X1_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 0) return ERROR(GENERIC); - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - -static size_t HUF_decompress4X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) -{ - const BYTE* ip = (const BYTE*) cSrc; - - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); - if (HUF_isError(hSize)) return hSize; - if (hSize >= cSrcSize) return ERROR(srcSize_wrong); - ip += hSize; cSrcSize -= hSize; - - return HUF_decompress4X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); -} - -size_t HUF_decompress4X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, 0); -} - - #endif /* HUF_FORCE_DECOMPRESS_X2 */ @@ -990,7 +1073,7 @@ static void HUF_fillDTableX2Level2(HUF_DEltX2* DTable, U32 targetLog, const U32 static void HUF_fillDTableX2(HUF_DEltX2* DTable, const U32 targetLog, const sortedSymbol_t* sortedList, - const U32* rankStart, rankVal_t rankValOrigin, const U32 maxWeight, + const U32* rankStart, rankValCol_t* rankValOrigin, const U32 maxWeight, const U32 nbBitsBaseline) { U32* const rankVal = rankValOrigin[0]; @@ -1045,14 +1128,7 @@ typedef struct { size_t HUF_readDTableX2_wksp(HUF_DTable* DTable, const void* src, size_t srcSize, - void* workSpace, size_t wkspSize) -{ - return HUF_readDTableX2_wksp_bmi2(DTable, src, srcSize, workSpace, wkspSize, /* bmi2 */ 0); -} - -size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, - const void* src, size_t srcSize, - void* workSpace, size_t wkspSize, int bmi2) + void* workSpace, size_t wkspSize, int flags) { U32 tableLog, maxW, nbSymbols; DTableDesc dtd = HUF_getDTableDesc(DTable); @@ -1074,7 +1150,7 @@ size_t HUF_readDTableX2_wksp_bmi2(HUF_DTable* DTable, if (maxTableLog > HUF_TABLELOG_MAX) return ERROR(tableLog_tooLarge); /* ZSTD_memset(weightList, 0, sizeof(weightList)); */ /* is not necessary, even though some analyzer complain ... */ - iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), bmi2); + iSize = HUF_readStats_wksp(wksp->weightList, HUF_SYMBOLVALUE_MAX + 1, wksp->rankStats, &nbSymbols, &tableLog, src, srcSize, wksp->calleeWksp, sizeof(wksp->calleeWksp), flags); if (HUF_isError(iSize)) return iSize; /* check result */ @@ -1245,6 +1321,11 @@ HUF_decompress1X2_usingDTable_internal_body( /* decoded size */ return dstSize; } + +/* HUF_decompress4X2_usingDTable_internal_body(): + * Conditions: + * @dstSize >= 6 + */ FORCE_INLINE_TEMPLATE size_t HUF_decompress4X2_usingDTable_internal_body( void* dst, size_t dstSize, @@ -1285,8 +1366,9 @@ HUF_decompress4X2_usingDTable_internal_body( DTableDesc const dtd = HUF_getDTableDesc(DTable); U32 const dtLog = dtd.tableLog; - if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ - if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + if (length4 > cSrcSize) return ERROR(corruption_detected); /* overflow */ + if (opStart4 > oend) return ERROR(corruption_detected); /* overflow */ + if (dstSize < 6) return ERROR(corruption_detected); /* stream 4-split doesn't work */ CHECK_F( BIT_initDStream(&bitD1, istart1, length1) ); CHECK_F( BIT_initDStream(&bitD2, istart2, length2) ); CHECK_F( BIT_initDStream(&bitD3, istart3, length3) ); @@ -1371,36 +1453,177 @@ size_t HUF_decompress4X2_usingDTable_internal_bmi2(void* dst, size_t dstSize, vo } #endif -#if HUF_NEED_DEFAULT_FUNCTION static size_t HUF_decompress4X2_usingDTable_internal_default(void* dst, size_t dstSize, void const* cSrc, size_t cSrcSize, HUF_DTable const* DTable) { return HUF_decompress4X2_usingDTable_internal_body(dst, dstSize, cSrc, cSrcSize, DTable); } -#endif #if ZSTD_ENABLE_ASM_X86_64_BMI2 -HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(HUF_DecompressAsmArgs* args) ZSTDLIB_HIDDEN; +HUF_ASM_DECL void HUF_decompress4X2_usingDTable_internal_fast_asm_loop(HUF_DecompressFastArgs* args) ZSTDLIB_HIDDEN; -static HUF_ASM_X86_64_BMI2_ATTRS size_t -HUF_decompress4X2_usingDTable_internal_bmi2_asm( +#endif + +static HUF_FAST_BMI2_ATTRS +void HUF_decompress4X2_usingDTable_internal_fast_c_loop(HUF_DecompressFastArgs* args) +{ + U64 bits[4]; + BYTE const* ip[4]; + BYTE* op[4]; + BYTE* oend[4]; + HUF_DEltX2 const* const dtable = (HUF_DEltX2 const*)args->dt; + BYTE const* const ilimit = args->ilimit; + + /* Copy the arguments to local registers. */ + ZSTD_memcpy(&bits, &args->bits, sizeof(bits)); + ZSTD_memcpy((void*)(&ip), &args->ip, sizeof(ip)); + ZSTD_memcpy(&op, &args->op, sizeof(op)); + + oend[0] = op[1]; + oend[1] = op[2]; + oend[2] = op[3]; + oend[3] = args->oend; + + assert(MEM_isLittleEndian()); + assert(!MEM_32bits()); + + for (;;) { + BYTE* olimit; + int stream; + int symbol; + + /* Assert loop preconditions */ +#ifndef NDEBUG + for (stream = 0; stream < 4; ++stream) { + assert(op[stream] <= oend[stream]); + assert(ip[stream] >= ilimit); + } +#endif + /* Compute olimit */ + { + /* Each loop does 5 table lookups for each of the 4 streams. + * Each table lookup consumes up to 11 bits of input, and produces + * up to 2 bytes of output. + */ + /* We can consume up to 7 bytes of input per iteration per stream. + * We also know that each input pointer is >= ip[0]. So we can run + * iters loops before running out of input. + */ + size_t iters = (size_t)(ip[0] - ilimit) / 7; + /* Each iteration can produce up to 10 bytes of output per stream. + * Each output stream my advance at different rates. So take the + * minimum number of safe iterations among all the output streams. + */ + for (stream = 0; stream < 4; ++stream) { + size_t const oiters = (size_t)(oend[stream] - op[stream]) / 10; + iters = MIN(iters, oiters); + } + + /* Each iteration produces at least 5 output symbols. So until + * op[3] crosses olimit, we know we haven't executed iters + * iterations yet. This saves us maintaining an iters counter, + * at the expense of computing the remaining # of iterations + * more frequently. + */ + olimit = op[3] + (iters * 5); + + /* Exit the fast decoding loop if we are too close to the end. */ + if (op[3] + 10 > olimit) + break; + + /* Exit the decoding loop if any input pointer has crossed the + * previous one. This indicates corruption, and a precondition + * to our loop is that ip[i] >= ip[0]. + */ + for (stream = 1; stream < 4; ++stream) { + if (ip[stream] < ip[stream - 1]) + goto _out; + } + } + +#ifndef NDEBUG + for (stream = 1; stream < 4; ++stream) { + assert(ip[stream] >= ip[stream - 1]); + } +#endif + + do { + /* Do 5 table lookups for each of the first 3 streams */ + for (symbol = 0; symbol < 5; ++symbol) { + for (stream = 0; stream < 3; ++stream) { + int const index = (int)(bits[stream] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[stream], entry.sequence); + bits[stream] <<= (entry.nbBits); + op[stream] += (entry.length); + } + } + /* Do 1 table lookup from the final stream */ + { + int const index = (int)(bits[3] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[3], entry.sequence); + bits[3] <<= (entry.nbBits); + op[3] += (entry.length); + } + /* Do 4 table lookups from the final stream & reload bitstreams */ + for (stream = 0; stream < 4; ++stream) { + /* Do a table lookup from the final stream. + * This is interleaved with the reloading to reduce register + * pressure. This shouldn't be necessary, but compilers can + * struggle with codegen with high register pressure. + */ + { + int const index = (int)(bits[3] >> 53); + HUF_DEltX2 const entry = dtable[index]; + MEM_write16(op[3], entry.sequence); + bits[3] <<= (entry.nbBits); + op[3] += (entry.length); + } + /* Reload the bistreams. The final bitstream must be reloaded + * after the 5th symbol was decoded. + */ + { + int const ctz = ZSTD_countTrailingZeros64(bits[stream]); + int const nbBits = ctz & 7; + int const nbBytes = ctz >> 3; + ip[stream] -= nbBytes; + bits[stream] = MEM_read64(ip[stream]) | 1; + bits[stream] <<= nbBits; + } + } + } while (op[3] < olimit); + } + +_out: + + /* Save the final values of each of the state variables back to args. */ + ZSTD_memcpy(&args->bits, &bits, sizeof(bits)); + ZSTD_memcpy((void*)(&args->ip), &ip, sizeof(ip)); + ZSTD_memcpy(&args->op, &op, sizeof(op)); +} + + +static HUF_FAST_BMI2_ATTRS size_t +HUF_decompress4X2_usingDTable_internal_fast( void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) { + const HUF_DTable* DTable, + HUF_DecompressFastLoopFn loopFn) { void const* dt = DTable + 1; const BYTE* const iend = (const BYTE*)cSrc + 6; BYTE* const oend = (BYTE*)dst + dstSize; - HUF_DecompressAsmArgs args; + HUF_DecompressFastArgs args; { - size_t const ret = HUF_DecompressAsmArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); + size_t const ret = HUF_DecompressFastArgs_init(&args, dst, dstSize, cSrc, cSrcSize, DTable); FORWARD_IF_ERROR(ret, "Failed to init asm args"); - if (ret != 0) - return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + if (ret == 0) + return 0; } assert(args.ip[0] >= args.ilimit); - HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop(&args); + loopFn(&args); /* note : op4 already verified within main loop */ assert(args.ip[0] >= iend); @@ -1431,91 +1654,72 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm( /* decoded size */ return dstSize; } -#endif /* ZSTD_ENABLE_ASM_X86_64_BMI2 */ static size_t HUF_decompress4X2_usingDTable_internal(void* dst, size_t dstSize, void const* cSrc, - size_t cSrcSize, HUF_DTable const* DTable, int bmi2) + size_t cSrcSize, HUF_DTable const* DTable, int flags) { + HUF_DecompressUsingDTableFn fallbackFn = HUF_decompress4X2_usingDTable_internal_default; + HUF_DecompressFastLoopFn loopFn = HUF_decompress4X2_usingDTable_internal_fast_c_loop; + #if DYNAMIC_BMI2 - if (bmi2) { + if (flags & HUF_flags_bmi2) { + fallbackFn = HUF_decompress4X2_usingDTable_internal_bmi2; # if ZSTD_ENABLE_ASM_X86_64_BMI2 - return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); -# else - return HUF_decompress4X2_usingDTable_internal_bmi2(dst, dstSize, cSrc, cSrcSize, DTable); + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } # endif + } else { + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } -#else - (void)bmi2; #endif #if ZSTD_ENABLE_ASM_X86_64_BMI2 && defined(__BMI2__) - return HUF_decompress4X2_usingDTable_internal_bmi2_asm(dst, dstSize, cSrc, cSrcSize, DTable); -#else - return HUF_decompress4X2_usingDTable_internal_default(dst, dstSize, cSrc, cSrcSize, DTable); + if (!(flags & HUF_flags_disableAsm)) { + loopFn = HUF_decompress4X2_usingDTable_internal_fast_asm_loop; + } #endif + + if (!(flags & HUF_flags_disableFast)) { + size_t const ret = HUF_decompress4X2_usingDTable_internal_fast(dst, dstSize, cSrc, cSrcSize, DTable, loopFn); + if (ret != 0) + return ret; + } + return fallbackFn(dst, dstSize, cSrc, cSrcSize, DTable); } HUF_DGEN(HUF_decompress1X2_usingDTable_internal) -size_t HUF_decompress1X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - size_t HUF_decompress1X2_DCtx_wksp(HUF_DTable* DCtx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) + void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; size_t const hSize = HUF_readDTableX2_wksp(DCtx, cSrc, cSrcSize, - workSpace, wkspSize); + workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, /* bmi2 */ 0); + return HUF_decompress1X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, DCtx, flags); } - -size_t HUF_decompress4X2_usingDTable( - void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc dtd = HUF_getDTableDesc(DTable); - if (dtd.tableType != 1) return ERROR(GENERIC); - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -} - -static size_t HUF_decompress4X2_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, +static size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize, int bmi2) + void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; size_t hSize = HUF_readDTableX2_wksp(dctx, cSrc, cSrcSize, - workSpace, wkspSize); + workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); + return HUF_decompress4X2_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); } -size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) -{ - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, /* bmi2 */ 0); -} - - #endif /* HUF_FORCE_DECOMPRESS_X1 */ @@ -1523,44 +1727,6 @@ size_t HUF_decompress4X2_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, /* Universal decompression selectors */ /* ***********************************/ -size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif -} - -size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, - const void* cSrc, size_t cSrcSize, - const HUF_DTable* DTable) -{ - DTableDesc const dtd = HUF_getDTableDesc(DTable); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)dtd; - assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)dtd; - assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, /* bmi2 */ 0); -#endif -} - #if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) typedef struct { U32 tableTime; U32 decode256Time; } algo_time_t; @@ -1615,36 +1781,9 @@ U32 HUF_selectDecoder (size_t dstSize, size_t cSrcSize) #endif } - -size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, - size_t dstSize, const void* cSrc, - size_t cSrcSize, void* workSpace, - size_t wkspSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize == 0) return ERROR(corruption_detected); - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#else - return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize): - HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize); -#endif - } -} - size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, - void* workSpace, size_t wkspSize) + void* workSpace, size_t wkspSize, int flags) { /* validation checks */ if (dstSize == 0) return ERROR(dstSize_tooSmall); @@ -1657,71 +1796,71 @@ size_t HUF_decompress1X_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, (void)algoNb; assert(algoNb == 0); return HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); + cSrcSize, workSpace, wkspSize, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)algoNb; assert(algoNb == 1); return HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); + cSrcSize, workSpace, wkspSize, flags); #else return algoNb ? HUF_decompress1X2_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize): + cSrcSize, workSpace, wkspSize, flags): HUF_decompress1X1_DCtx_wksp(dctx, dst, dstSize, cSrc, - cSrcSize, workSpace, wkspSize); + cSrcSize, workSpace, wkspSize, flags); #endif } } -size_t HUF_decompress1X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +size_t HUF_decompress1X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) { DTableDesc const dtd = HUF_getDTableDesc(DTable); #if defined(HUF_FORCE_DECOMPRESS_X1) (void)dtd; assert(dtd.tableType == 0); - return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)dtd; assert(dtd.tableType == 1); - return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #else - return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return dtd.tableType ? HUF_decompress1X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress1X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #endif } #ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress1X1_DCtx_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +size_t HUF_decompress1X1_DCtx_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) { const BYTE* ip = (const BYTE*) cSrc; - size_t const hSize = HUF_readDTableX1_wksp_bmi2(dctx, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + size_t const hSize = HUF_readDTableX1_wksp(dctx, cSrc, cSrcSize, workSpace, wkspSize, flags); if (HUF_isError(hSize)) return hSize; if (hSize >= cSrcSize) return ERROR(srcSize_wrong); ip += hSize; cSrcSize -= hSize; - return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, bmi2); + return HUF_decompress1X1_usingDTable_internal(dst, dstSize, ip, cSrcSize, dctx, flags); } #endif -size_t HUF_decompress4X_usingDTable_bmi2(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int bmi2) +size_t HUF_decompress4X_usingDTable(void* dst, size_t maxDstSize, const void* cSrc, size_t cSrcSize, const HUF_DTable* DTable, int flags) { DTableDesc const dtd = HUF_getDTableDesc(DTable); #if defined(HUF_FORCE_DECOMPRESS_X1) (void)dtd; assert(dtd.tableType == 0); - return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)dtd; assert(dtd.tableType == 1); - return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #else - return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2) : - HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, bmi2); + return dtd.tableType ? HUF_decompress4X2_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags) : + HUF_decompress4X1_usingDTable_internal(dst, maxDstSize, cSrc, cSrcSize, DTable, flags); #endif } -size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int bmi2) +size_t HUF_decompress4X_hufOnly_wksp(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize, void* workSpace, size_t wkspSize, int flags) { /* validation checks */ if (dstSize == 0) return ERROR(dstSize_tooSmall); @@ -1731,162 +1870,16 @@ size_t HUF_decompress4X_hufOnly_wksp_bmi2(HUF_DTable* dctx, void* dst, size_t ds #if defined(HUF_FORCE_DECOMPRESS_X1) (void)algoNb; assert(algoNb == 0); - return HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); #elif defined(HUF_FORCE_DECOMPRESS_X2) (void)algoNb; assert(algoNb == 1); - return HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); #else - return algoNb ? HUF_decompress4X2_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2) : - HUF_decompress4X1_DCtx_wksp_bmi2(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, bmi2); + return algoNb ? HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags) : + HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, workSpace, wkspSize, flags); #endif } } -#ifndef ZSTD_NO_UNUSED_FUNCTIONS -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_readDTableX1(HUF_DTable* DTable, const void* src, size_t srcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX1_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X1_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X1_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X1_DCtx (DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_readDTableX2(HUF_DTable* DTable, const void* src, size_t srcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_readDTableX2_wksp(DTable, src, srcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X2_DCtx(HUF_DTable* DCtx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X2_DCtx_wksp(DCtx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress1X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -#ifndef HUF_FORCE_DECOMPRESS_X2 -size_t HUF_decompress4X1_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X1_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -size_t HUF_decompress4X1 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX1(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X1_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -#ifndef HUF_FORCE_DECOMPRESS_X1 -size_t HUF_decompress4X2_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X2_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress4X2 (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - HUF_CREATE_STATIC_DTABLEX2(DTable, HUF_TABLELOG_MAX); - return HUF_decompress4X2_DCtx(DTable, dst, dstSize, cSrc, cSrcSize); -} -#endif - -typedef size_t (*decompressionAlgo)(void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize); - -size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ -#if !defined(HUF_FORCE_DECOMPRESS_X1) && !defined(HUF_FORCE_DECOMPRESS_X2) - static const decompressionAlgo decompress[2] = { HUF_decompress4X1, HUF_decompress4X2 }; -#endif - - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1(dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2(dst, dstSize, cSrc, cSrcSize); -#else - return decompress[algoNb](dst, dstSize, cSrc, cSrcSize); -#endif - } -} - -size_t HUF_decompress4X_DCtx (HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - /* validation checks */ - if (dstSize == 0) return ERROR(dstSize_tooSmall); - if (cSrcSize > dstSize) return ERROR(corruption_detected); /* invalid */ - if (cSrcSize == dstSize) { ZSTD_memcpy(dst, cSrc, dstSize); return dstSize; } /* not compressed */ - if (cSrcSize == 1) { ZSTD_memset(dst, *(const BYTE*)cSrc, dstSize); return dstSize; } /* RLE */ - - { U32 const algoNb = HUF_selectDecoder(dstSize, cSrcSize); -#if defined(HUF_FORCE_DECOMPRESS_X1) - (void)algoNb; - assert(algoNb == 0); - return HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); -#elif defined(HUF_FORCE_DECOMPRESS_X2) - (void)algoNb; - assert(algoNb == 1); - return HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize); -#else - return algoNb ? HUF_decompress4X2_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) : - HUF_decompress4X1_DCtx(dctx, dst, dstSize, cSrc, cSrcSize) ; -#endif - } -} - -size_t HUF_decompress4X_hufOnly(HUF_DTable* dctx, void* dst, size_t dstSize, const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress4X_hufOnly_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} - -size_t HUF_decompress1X_DCtx(HUF_DTable* dctx, void* dst, size_t dstSize, - const void* cSrc, size_t cSrcSize) -{ - U32 workSpace[HUF_DECOMPRESS_WORKSPACE_SIZE_U32]; - return HUF_decompress1X_DCtx_wksp(dctx, dst, dstSize, cSrc, cSrcSize, - workSpace, sizeof(workSpace)); -} -#endif - #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/huf_decompress_amd64.S b/vendor/github.com/DataDog/zstd/huf_decompress_amd64.S index 1e8fe4d..e5dc9a6 100644 --- a/vendor/github.com/DataDog/zstd/huf_decompress_amd64.S +++ b/vendor/github.com/DataDog/zstd/huf_decompress_amd64.S @@ -1,5 +1,5 @@ /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -30,14 +30,14 @@ * TODO: Support Windows calling convention. */ -ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop) -ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop) -ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop) -ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop) -.global HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop -.global HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop -.global _HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop -.global _HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X1_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(HUF_decompress4X2_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X2_usingDTable_internal_fast_asm_loop) +ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_fast_asm_loop) +.global HUF_decompress4X1_usingDTable_internal_fast_asm_loop +.global HUF_decompress4X2_usingDTable_internal_fast_asm_loop +.global _HUF_decompress4X1_usingDTable_internal_fast_asm_loop +.global _HUF_decompress4X2_usingDTable_internal_fast_asm_loop .text /* Sets up register mappings for clarity. @@ -95,8 +95,9 @@ ZSTD_HIDE_ASM_FUNCTION(_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop) /* Define both _HUF_* & HUF_* symbols because MacOS * C symbols are prefixed with '_' & Linux symbols aren't. */ -_HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop: -HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop: +_HUF_decompress4X1_usingDTable_internal_fast_asm_loop: +HUF_decompress4X1_usingDTable_internal_fast_asm_loop: + ZSTD_CET_ENDBRANCH /* Save all registers - even if they are callee saved for simplicity. */ push %rax push %rbx @@ -350,8 +351,9 @@ HUF_decompress4X1_usingDTable_internal_bmi2_asm_loop: pop %rax ret -_HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop: -HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop: +_HUF_decompress4X2_usingDTable_internal_fast_asm_loop: +HUF_decompress4X2_usingDTable_internal_fast_asm_loop: + ZSTD_CET_ENDBRANCH /* Save all registers - even if they are callee saved for simplicity. */ push %rax push %rbx @@ -427,41 +429,30 @@ HUF_decompress4X2_usingDTable_internal_bmi2_asm_loop: /* r15 = (ip0 - ilimit) / 7 */ movq %rdx, %r15 - movabsq $-3689348814741910323, %rdx - movq 8(%rsp), %rax /* rax = oend0 */ - subq %op0, %rax /* rax = oend0 - op0 */ - mulq %rdx - shrq $3, %rdx /* rdx = rax / 10 */ + /* r15 = min(r15, min(oend0 - op0, oend1 - op1, oend2 - op2, oend3 - op3) / 10) */ + movq 8(%rsp), %rax /* rax = oend0 */ + subq %op0, %rax /* rax = oend0 - op0 */ + movq 16(%rsp), %rdx /* rdx = oend1 */ + subq %op1, %rdx /* rdx = oend1 - op1 */ - /* r15 = min(%rdx, %r15) */ - cmpq %rdx, %r15 - cmova %rdx, %r15 + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ - movabsq $-3689348814741910323, %rdx - movq 16(%rsp), %rax /* rax = oend1 */ - subq %op1, %rax /* rax = oend1 - op1 */ - mulq %rdx - shrq $3, %rdx /* rdx = rax / 10 */ - - /* r15 = min(%rdx, %r15) */ - cmpq %rdx, %r15 - cmova %rdx, %r15 - - movabsq $-3689348814741910323, %rdx movq 24(%rsp), %rax /* rax = oend2 */ subq %op2, %rax /* rax = oend2 - op2 */ - mulq %rdx - shrq $3, %rdx /* rdx = rax / 10 */ - /* r15 = min(%rdx, %r15) */ - cmpq %rdx, %r15 - cmova %rdx, %r15 + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ - movabsq $-3689348814741910323, %rdx movq 32(%rsp), %rax /* rax = oend3 */ subq %op3, %rax /* rax = oend3 - op3 */ + + cmpq %rax, %rdx + cmova %rax, %rdx /* rdx = min(%rdx, %rax) */ + + movabsq $-3689348814741910323, %rax mulq %rdx - shrq $3, %rdx /* rdx = rax / 10 */ + shrq $3, %rdx /* rdx = rdx / 10 */ /* r15 = min(%rdx, %r15) */ cmpq %rdx, %r15 diff --git a/vendor/github.com/DataDog/zstd/mem.h b/vendor/github.com/DataDog/zstd/mem.h index 78703f7..a6007c9 100644 --- a/vendor/github.com/DataDog/zstd/mem.h +++ b/vendor/github.com/DataDog/zstd/mem.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -134,21 +134,15 @@ MEM_STATIC size_t MEM_swapST(size_t in); /*-************************************************************** * Memory I/O Implementation *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (i.e., not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. +/* MEM_FORCE_MEMORY_ACCESS : For accessing unaligned memory: + * Method 0 : always use `memcpy()`. Safe and portable. + * Method 1 : Use compiler extension to set unaligned access. * Method 2 : direct access. This method is portable but violate C standard. * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (i.e. GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) + * Default : method 1 if supported, else method 0 */ #ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) +# ifdef __GNUC__ # define MEM_FORCE_MEMORY_ACCESS 1 # endif #endif @@ -191,30 +185,19 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } #elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -#if defined(_MSC_VER) || (defined(__INTEL_COMPILER) && defined(WIN32)) - __pragma( pack(push, 1) ) - typedef struct { U16 v; } unalign16; - typedef struct { U32 v; } unalign32; - typedef struct { U64 v; } unalign64; - typedef struct { size_t v; } unalignArch; - __pragma( pack(pop) ) -#else - typedef struct { U16 v; } __attribute__((packed)) unalign16; - typedef struct { U32 v; } __attribute__((packed)) unalign32; - typedef struct { U64 v; } __attribute__((packed)) unalign64; - typedef struct { size_t v; } __attribute__((packed)) unalignArch; -#endif +typedef __attribute__((aligned(1))) U16 unalign16; +typedef __attribute__((aligned(1))) U32 unalign32; +typedef __attribute__((aligned(1))) U64 unalign64; +typedef __attribute__((aligned(1))) size_t unalignArch; -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign16*)ptr)->v; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign32*)ptr)->v; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign64*)ptr)->v; } -MEM_STATIC size_t MEM_readST(const void* ptr) { return ((const unalignArch*)ptr)->v; } +MEM_STATIC U16 MEM_read16(const void* ptr) { return *(const unalign16*)ptr; } +MEM_STATIC U32 MEM_read32(const void* ptr) { return *(const unalign32*)ptr; } +MEM_STATIC U64 MEM_read64(const void* ptr) { return *(const unalign64*)ptr; } +MEM_STATIC size_t MEM_readST(const void* ptr) { return *(const unalignArch*)ptr; } -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign16*)memPtr)->v = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign32*)memPtr)->v = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign64*)memPtr)->v = value; } +MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(unalign16*)memPtr = value; } +MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(unalign32*)memPtr = value; } +MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(unalign64*)memPtr = value; } #else @@ -258,6 +241,14 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) #endif /* MEM_FORCE_MEMORY_ACCESS */ +MEM_STATIC U32 MEM_swap32_fallback(U32 in) +{ + return ((in << 24) & 0xff000000 ) | + ((in << 8) & 0x00ff0000 ) | + ((in >> 8) & 0x0000ff00 ) | + ((in >> 24) & 0x000000ff ); +} + MEM_STATIC U32 MEM_swap32(U32 in) { #if defined(_MSC_VER) /* Visual Studio */ @@ -266,13 +257,22 @@ MEM_STATIC U32 MEM_swap32(U32 in) || (defined(__clang__) && __has_builtin(__builtin_bswap32)) return __builtin_bswap32(in); #else - return ((in << 24) & 0xff000000 ) | - ((in << 8) & 0x00ff0000 ) | - ((in >> 8) & 0x0000ff00 ) | - ((in >> 24) & 0x000000ff ); + return MEM_swap32_fallback(in); #endif } +MEM_STATIC U64 MEM_swap64_fallback(U64 in) +{ + return ((in << 56) & 0xff00000000000000ULL) | + ((in << 40) & 0x00ff000000000000ULL) | + ((in << 24) & 0x0000ff0000000000ULL) | + ((in << 8) & 0x000000ff00000000ULL) | + ((in >> 8) & 0x00000000ff000000ULL) | + ((in >> 24) & 0x0000000000ff0000ULL) | + ((in >> 40) & 0x000000000000ff00ULL) | + ((in >> 56) & 0x00000000000000ffULL); +} + MEM_STATIC U64 MEM_swap64(U64 in) { #if defined(_MSC_VER) /* Visual Studio */ @@ -281,14 +281,7 @@ MEM_STATIC U64 MEM_swap64(U64 in) || (defined(__clang__) && __has_builtin(__builtin_bswap64)) return __builtin_bswap64(in); #else - return ((in << 56) & 0xff00000000000000ULL) | - ((in << 40) & 0x00ff000000000000ULL) | - ((in << 24) & 0x0000ff0000000000ULL) | - ((in << 8) & 0x000000ff00000000ULL) | - ((in >> 8) & 0x00000000ff000000ULL) | - ((in >> 24) & 0x0000000000ff0000ULL) | - ((in >> 40) & 0x000000000000ff00ULL) | - ((in >> 56) & 0x00000000000000ffULL); + return MEM_swap64_fallback(in); #endif } diff --git a/vendor/github.com/DataDog/zstd/pool.c b/vendor/github.com/DataDog/zstd/pool.c index c1ef32a..d067ac3 100644 --- a/vendor/github.com/DataDog/zstd/pool.c +++ b/vendor/github.com/DataDog/zstd/pool.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -11,9 +11,9 @@ /* ====== Dependencies ======= */ +#include "allocations.h" /* ZSTD_customCalloc, ZSTD_customFree */ #include "zstd_deps.h" /* size_t */ #include "debug.h" /* assert */ -#include "zstd_internal.h" /* ZSTD_customMalloc, ZSTD_customFree */ #include "pool.h" /* ====== Compiler specifics ====== */ @@ -97,9 +97,7 @@ static void* POOL_thread(void* opaque) { /* If the intended queue size was 0, signal after finishing job */ ZSTD_pthread_mutex_lock(&ctx->queueMutex); ctx->numThreadsBusy--; - if (ctx->queueSize == 1) { - ZSTD_pthread_cond_signal(&ctx->queuePushCond); - } + ZSTD_pthread_cond_signal(&ctx->queuePushCond); ZSTD_pthread_mutex_unlock(&ctx->queueMutex); } } /* for (;;) */ @@ -129,7 +127,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, * empty and full queues. */ ctx->queueSize = queueSize + 1; - ctx->queue = (POOL_job*)ZSTD_customMalloc(ctx->queueSize * sizeof(POOL_job), customMem); + ctx->queue = (POOL_job*)ZSTD_customCalloc(ctx->queueSize * sizeof(POOL_job), customMem); ctx->queueHead = 0; ctx->queueTail = 0; ctx->numThreadsBusy = 0; @@ -143,7 +141,7 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, } ctx->shutdown = 0; /* Allocate space for the thread handles */ - ctx->threads = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); + ctx->threads = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), customMem); ctx->threadCapacity = 0; ctx->customMem = customMem; /* Check for errors */ @@ -176,7 +174,7 @@ static void POOL_join(POOL_ctx* ctx) { /* Join all of the threads */ { size_t i; for (i = 0; i < ctx->threadCapacity; ++i) { - ZSTD_pthread_join(ctx->threads[i], NULL); /* note : could fail */ + ZSTD_pthread_join(ctx->threads[i]); /* note : could fail */ } } } @@ -191,6 +189,17 @@ void POOL_free(POOL_ctx *ctx) { ZSTD_customFree(ctx, ctx->customMem); } +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx) { + ZSTD_pthread_mutex_lock(&ctx->queueMutex); + while(!ctx->queueEmpty || ctx->numThreadsBusy > 0) { + ZSTD_pthread_cond_wait(&ctx->queuePushCond, &ctx->queueMutex); + } + ZSTD_pthread_mutex_unlock(&ctx->queueMutex); +} + void ZSTD_freeThreadPool (ZSTD_threadPool* pool) { POOL_free (pool); } @@ -212,7 +221,7 @@ static int POOL_resize_internal(POOL_ctx* ctx, size_t numThreads) return 0; } /* numThreads > threadCapacity */ - { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customMalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); + { ZSTD_pthread_t* const threadPool = (ZSTD_pthread_t*)ZSTD_customCalloc(numThreads * sizeof(ZSTD_pthread_t), ctx->customMem); if (!threadPool) return 1; /* replace existing thread pool */ ZSTD_memcpy(threadPool, ctx->threads, ctx->threadCapacity * sizeof(*threadPool)); @@ -263,7 +272,9 @@ static int isQueueFull(POOL_ctx const* ctx) { static void POOL_add_internal(POOL_ctx* ctx, POOL_function function, void *opaque) { - POOL_job const job = {function, opaque}; + POOL_job job; + job.function = function; + job.opaque = opaque; assert(ctx != NULL); if (ctx->shutdown) return; @@ -331,6 +342,11 @@ void POOL_free(POOL_ctx* ctx) { (void)ctx; } +void POOL_joinJobs(POOL_ctx* ctx){ + assert(!ctx || ctx == &g_poolCtx); + (void)ctx; +} + int POOL_resize(POOL_ctx* ctx, size_t numThreads) { (void)ctx; (void)numThreads; return 0; diff --git a/vendor/github.com/DataDog/zstd/pool.h b/vendor/github.com/DataDog/zstd/pool.h index f873352..7dcf495 100644 --- a/vendor/github.com/DataDog/zstd/pool.h +++ b/vendor/github.com/DataDog/zstd/pool.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -39,6 +39,12 @@ POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize, */ void POOL_free(POOL_ctx* ctx); + +/*! POOL_joinJobs() : + * Waits for all queued jobs to finish executing. + */ +void POOL_joinJobs(POOL_ctx* ctx); + /*! POOL_resize() : * Expands or shrinks pool's number of threads. * This is more efficient than releasing + creating a new context, diff --git a/vendor/github.com/DataDog/zstd/portability_macros.h b/vendor/github.com/DataDog/zstd/portability_macros.h index e65a09b..ecf5453 100644 --- a/vendor/github.com/DataDog/zstd/portability_macros.h +++ b/vendor/github.com/DataDog/zstd/portability_macros.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -13,7 +13,7 @@ #define ZSTD_PORTABILITY_MACROS_H /** - * This header file contains macro defintions to support portability. + * This header file contains macro definitions to support portability. * This header is shared between C and ASM code, so it MUST only * contain macro definitions. It MUST not contain any C code. * @@ -89,7 +89,7 @@ #endif /** - * Only enable assembly for GNUC comptabile compilers, + * Only enable assembly for GNUC compatible compilers, * because other platforms may not support GAS assembly syntax. * * Only enable assembly for Linux / MacOS, other platforms may @@ -135,6 +135,25 @@ # define ZSTD_ENABLE_ASM_X86_64_BMI2 0 #endif +/* + * For x86 ELF targets, add .note.gnu.property section for Intel CET in + * assembly sources when CET is enabled. + * + * Additionally, any function that may be called indirectly must begin + * with ZSTD_CET_ENDBRANCH. + */ +#if defined(__ELF__) && (defined(__x86_64__) || defined(__i386__)) \ + && defined(__has_include) +# if __has_include() +# include +# define ZSTD_CET_ENDBRANCH _CET_ENDBR +# endif +#endif + +#ifndef ZSTD_CET_ENDBRANCH +# define ZSTD_CET_ENDBRANCH +#endif + #endif /* ZSTD_PORTABILITY_MACROS_H */ #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/threading.c b/vendor/github.com/DataDog/zstd/threading.c index 722f56d..df3c7dd 100644 --- a/vendor/github.com/DataDog/zstd/threading.c +++ b/vendor/github.com/DataDog/zstd/threading.c @@ -24,8 +24,7 @@ int g_ZSTD_threading_useless_symbol; #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) /** - * Windows minimalist Pthread Wrapper, based on : - * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + * Windows minimalist Pthread Wrapper */ @@ -36,37 +35,92 @@ int g_ZSTD_threading_useless_symbol; /* === Implementation === */ +typedef struct { + void* (*start_routine)(void*); + void* arg; + int initialized; + ZSTD_pthread_cond_t initialized_cond; + ZSTD_pthread_mutex_t initialized_mutex; +} ZSTD_thread_params_t; + static unsigned __stdcall worker(void *arg) { - ZSTD_pthread_t* const thread = (ZSTD_pthread_t*) arg; - thread->arg = thread->start_routine(thread->arg); + void* (*start_routine)(void*); + void* thread_arg; + + /* Initialized thread_arg and start_routine and signal main thread that we don't need it + * to wait any longer. + */ + { + ZSTD_thread_params_t* thread_param = (ZSTD_thread_params_t*)arg; + thread_arg = thread_param->arg; + start_routine = thread_param->start_routine; + + /* Signal main thread that we are running and do not depend on its memory anymore */ + ZSTD_pthread_mutex_lock(&thread_param->initialized_mutex); + thread_param->initialized = 1; + ZSTD_pthread_cond_signal(&thread_param->initialized_cond); + ZSTD_pthread_mutex_unlock(&thread_param->initialized_mutex); + } + + start_routine(thread_arg); + return 0; } int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, void* (*start_routine) (void*), void* arg) { + ZSTD_thread_params_t thread_param; (void)unused; - thread->arg = arg; - thread->start_routine = start_routine; - thread->handle = (HANDLE) _beginthreadex(NULL, 0, worker, thread, 0, NULL); - if (!thread->handle) + thread_param.start_routine = start_routine; + thread_param.arg = arg; + thread_param.initialized = 0; + *thread = NULL; + + /* Setup thread initialization synchronization */ + if(ZSTD_pthread_cond_init(&thread_param.initialized_cond, NULL)) { + /* Should never happen on Windows */ + return -1; + } + if(ZSTD_pthread_mutex_init(&thread_param.initialized_mutex, NULL)) { + /* Should never happen on Windows */ + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + return -1; + } + + /* Spawn thread */ + *thread = (HANDLE)_beginthreadex(NULL, 0, worker, &thread_param, 0, NULL); + if (!thread) { + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); return errno; - else - return 0; + } + + /* Wait for thread to be initialized */ + ZSTD_pthread_mutex_lock(&thread_param.initialized_mutex); + while(!thread_param.initialized) { + ZSTD_pthread_cond_wait(&thread_param.initialized_cond, &thread_param.initialized_mutex); + } + ZSTD_pthread_mutex_unlock(&thread_param.initialized_mutex); + ZSTD_pthread_mutex_destroy(&thread_param.initialized_mutex); + ZSTD_pthread_cond_destroy(&thread_param.initialized_cond); + + return 0; } -int ZSTD_pthread_join(ZSTD_pthread_t thread, void **value_ptr) +int ZSTD_pthread_join(ZSTD_pthread_t thread) { DWORD result; - if (!thread.handle) return 0; + if (!thread) return 0; + + result = WaitForSingleObject(thread, INFINITE); + CloseHandle(thread); - result = WaitForSingleObject(thread.handle, INFINITE); switch (result) { case WAIT_OBJECT_0: - if (value_ptr) *value_ptr = thread.arg; return 0; case WAIT_ABANDONED: return EINVAL; diff --git a/vendor/github.com/DataDog/zstd/threading.h b/vendor/github.com/DataDog/zstd/threading.h index 1b09b0c..b921659 100644 --- a/vendor/github.com/DataDog/zstd/threading.h +++ b/vendor/github.com/DataDog/zstd/threading.h @@ -24,8 +24,7 @@ extern "C" { #if defined(ZSTD_MULTITHREAD) && defined(_WIN32) /** - * Windows minimalist Pthread Wrapper, based on : - * http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + * Windows minimalist Pthread Wrapper */ #ifdef WINVER # undef WINVER @@ -63,16 +62,12 @@ extern "C" { #define ZSTD_pthread_cond_broadcast(a) WakeAllConditionVariable((a)) /* ZSTD_pthread_create() and ZSTD_pthread_join() */ -typedef struct { - HANDLE handle; - void* (*start_routine)(void*); - void* arg; -} ZSTD_pthread_t; +typedef HANDLE ZSTD_pthread_t; int ZSTD_pthread_create(ZSTD_pthread_t* thread, const void* unused, void* (*start_routine) (void*), void* arg); -int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); +int ZSTD_pthread_join(ZSTD_pthread_t thread); /** * add here more wrappers as required @@ -100,7 +95,7 @@ int ZSTD_pthread_join(ZSTD_pthread_t thread, void** value_ptr); #define ZSTD_pthread_t pthread_t #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) -#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) #else /* DEBUGLEVEL >= 1 */ @@ -125,7 +120,7 @@ int ZSTD_pthread_cond_destroy(ZSTD_pthread_cond_t* cond); #define ZSTD_pthread_t pthread_t #define ZSTD_pthread_create(a, b, c, d) pthread_create((a), (b), (c), (d)) -#define ZSTD_pthread_join(a, b) pthread_join((a),(b)) +#define ZSTD_pthread_join(a) pthread_join((a),NULL) #endif diff --git a/vendor/github.com/DataDog/zstd/xxhash.c b/vendor/github.com/DataDog/zstd/xxhash.c index b3bf79f..71df219 100644 --- a/vendor/github.com/DataDog/zstd/xxhash.c +++ b/vendor/github.com/DataDog/zstd/xxhash.c @@ -1,10 +1,10 @@ #ifndef USE_EXTERNAL_ZSTD /* * xxHash - Fast Hash algorithm - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : - * - xxHash homepage: http://www.xxhash.com + * - xxHash homepage: https://cyan4973.github.io/xxHash/ * - xxHash source repository : https://github.com/Cyan4973/xxHash * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/xxhash.h b/vendor/github.com/DataDog/zstd/xxhash.h index 62770b4..8675d59 100644 --- a/vendor/github.com/DataDog/zstd/xxhash.h +++ b/vendor/github.com/DataDog/zstd/xxhash.h @@ -1,10 +1,10 @@ #ifndef USE_EXTERNAL_ZSTD /* * xxHash - Fast Hash algorithm - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * * You can contact the author at : - * - xxHash homepage: http://www.xxhash.com + * - xxHash homepage: https://cyan4973.github.io/xxHash/ * - xxHash source repository : https://github.com/Cyan4973/xxHash * * This source code is licensed under both the BSD-style license (found in the @@ -1315,7 +1315,7 @@ XXH3_128bits_reset_withSecretandSeed(XXH3_state_t* statePtr, * care, as what works on one compiler/platform/optimization level may cause * another to read garbage data or even crash. * - * See http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. + * See https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html for details. * * Prefer these methods in priority order (0 > 3 > 1 > 2) */ @@ -1535,7 +1535,7 @@ static void* XXH_memcpy(void* dest, const void* src, size_t size) { return ZSTD_ * @brief Used to prevent unwanted optimizations for @p var. * * It uses an empty GCC inline assembly statement with a register constraint - * which forces @p var into a general purpose register (eg eax, ebx, ecx + * which forces @p var into a general purpose register (e.g. eax, ebx, ecx * on x86) and marks it as modified. * * This is used in a few places to avoid unwanted autovectorization (e.g. @@ -1656,7 +1656,7 @@ static xxh_u32 XXH_read32(const void* ptr) /* * Portable and safe solution. Generally efficient. - * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html */ static xxh_u32 XXH_read32(const void* memPtr) { @@ -2297,7 +2297,7 @@ static xxh_u64 XXH_read64(const void* ptr) /* * Portable and safe solution. Generally efficient. - * see: http://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html + * see: https://fastcompression.blogspot.com/2015/08/accessing-unaligned-memory.html */ static xxh_u64 XXH_read64(const void* memPtr) { @@ -2810,7 +2810,7 @@ enum XXH_VECTOR_TYPE /* fake enum */ { * @ingroup tuning * @brief Selects the minimum alignment for XXH3's accumulators. * - * When using SIMD, this should match the alignment reqired for said vector + * When using SIMD, this should match the alignment required for said vector * type, so, for example, 32 for AVX2. * * Default: Auto detected. @@ -3027,7 +3027,7 @@ enum XXH_VECTOR_TYPE /* fake enum */ { * have more than 2 NEON (F0/F1) micro-ops. If you are only using NEON instructions, * you are only using 2/3 of the CPU bandwidth. * - * This is even more noticable on the more advanced cores like the A76 which + * This is even more noticeable on the more advanced cores like the A76 which * can dispatch 8 micro-ops per cycle, but still only 2 NEON micro-ops at once. * * Therefore, @ref XXH3_NEON_LANES lanes will be processed using NEON, and the diff --git a/vendor/github.com/DataDog/zstd/zbuff.h b/vendor/github.com/DataDog/zstd/zbuff.h index 53f0e4b..969f8bd 100644 --- a/vendor/github.com/DataDog/zstd/zbuff.h +++ b/vendor/github.com/DataDog/zstd/zbuff.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zbuff_common.c b/vendor/github.com/DataDog/zstd/zbuff_common.c index 2eafb33..367ba5e 100644 --- a/vendor/github.com/DataDog/zstd/zbuff_common.c +++ b/vendor/github.com/DataDog/zstd/zbuff_common.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zbuff_compress.c b/vendor/github.com/DataDog/zstd/zbuff_compress.c index 0a2f3c5..aada93b 100644 --- a/vendor/github.com/DataDog/zstd/zbuff_compress.c +++ b/vendor/github.com/DataDog/zstd/zbuff_compress.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zbuff_decompress.c b/vendor/github.com/DataDog/zstd/zbuff_decompress.c index 3ccf7ae..49e9a47 100644 --- a/vendor/github.com/DataDog/zstd/zbuff_decompress.c +++ b/vendor/github.com/DataDog/zstd/zbuff_decompress.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -14,6 +14,8 @@ /* ************************************* * Dependencies ***************************************/ +#define ZSTD_DISABLE_DEPRECATE_WARNINGS /* suppress warning on ZSTD_initDStream_usingDict */ +#include "zstd.h" /* ZSTD_CStream, ZSTD_DStream, ZSTDLIB_API */ #define ZBUFF_STATIC_LINKING_ONLY #include "zbuff.h" diff --git a/vendor/github.com/DataDog/zstd/zdict.c b/vendor/github.com/DataDog/zstd/zdict.c index 6ff1378..175865d 100644 --- a/vendor/github.com/DataDog/zstd/zdict.c +++ b/vendor/github.com/DataDog/zstd/zdict.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -45,7 +45,6 @@ #ifndef ZDICT_STATIC_LINKING_ONLY # define ZDICT_STATIC_LINKING_ONLY #endif -#define HUF_STATIC_LINKING_ONLY #include "mem.h" /* read */ #include "fse.h" /* FSE_normalizeCount, FSE_writeNCount */ @@ -55,6 +54,7 @@ #include "zstd_compress_internal.h" /* ZSTD_loadCEntropy() */ #include "zdict.h" #include "divsufsort.h" +#include "bits.h" /* ZSTD_NbCommonBytes */ /*-************************************* @@ -131,85 +131,6 @@ size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictSize) /*-******************************************************** * Dictionary training functions **********************************************************/ -static unsigned ZDICT_NbCommonBytes (size_t val) -{ - if (MEM_isLittleEndian()) { - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) - if (val != 0) { - unsigned long r; - _BitScanForward64(&r, (U64)val); - return (unsigned)(r >> 3); - } else { - /* Should not reach this code path */ - __assume(0); - } -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (unsigned)(__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, 0, 3, 1, 3, 1, 4, 2, 7, 0, 2, 3, 6, 1, 5, 3, 5, 1, 3, 4, 4, 2, 5, 6, 7, 7, 0, 1, 2, 3, 3, 4, 6, 2, 6, 5, 5, 3, 4, 5, 6, 7, 1, 2, 4, 6, 4, 4, 5, 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - if (val != 0) { - unsigned long r; - _BitScanForward(&r, (U32)val); - return (unsigned)(r >> 3); - } else { - /* Should not reach this code path */ - __assume(0); - } -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (unsigned)(__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, 3, 2, 2, 1, 3, 2, 0, 1, 3, 3, 1, 2, 2, 2, 2, 0, 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else { /* Big Endian CPU */ - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) - if (val != 0) { - unsigned long r; - _BitScanReverse64(&r, val); - return (unsigned)(r >> 3); - } else { - /* Should not reach this code path */ - __assume(0); - } -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (unsigned)(__builtin_clzll(val) >> 3); -# else - unsigned r; - const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ - if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - if (val != 0) { - unsigned long r; - _BitScanReverse(&r, (unsigned long)val); - return (unsigned)(r >> 3); - } else { - /* Should not reach this code path */ - __assume(0); - } -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (unsigned)(__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } } -} - - /*! ZDICT_count() : Count the nb of common bytes between 2 pointers. Note : this function presumes end of buffer followed by noisy guard band. @@ -224,7 +145,7 @@ static size_t ZDICT_count(const void* pIn, const void* pMatch) pMatch = (const char*)pMatch+sizeof(size_t); continue; } - pIn = (const char*)pIn+ZDICT_NbCommonBytes(diff); + pIn = (const char*)pIn+ZSTD_NbCommonBytes(diff); return (size_t)((const char*)pIn - pStart); } } @@ -452,7 +373,7 @@ static U32 ZDICT_tryMerge(dictItem* table, dictItem elt, U32 eltNbToSkip, const elt = table[u]; /* sort : improve rank */ while ((u>1) && (table[u-1].savings < elt.savings)) - table[u] = table[u-1], u--; + table[u] = table[u-1], u--; table[u] = elt; return u; } } @@ -603,7 +524,7 @@ static size_t ZDICT_trainBuffer_legacy(dictItem* dictList, U32 dictListSize, if (solution.length==0) { cursor++; continue; } ZDICT_insertDictItem(dictList, dictListSize, solution, buffer); cursor += solution.length; - DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / bufferSize * 100); + DISPLAYUPDATE(2, "\r%4.2f %% \r", (double)cursor / (double)bufferSize * 100.0); } } _cleanup: @@ -646,11 +567,11 @@ static void ZDICT_countEStats(EStats_ress_t esr, const ZSTD_parameters* params, size_t cSize; if (srcSize > blockSizeMax) srcSize = blockSizeMax; /* protection vs large samples */ - { size_t const errorCode = ZSTD_compressBegin_usingCDict(esr.zc, esr.dict); + { size_t const errorCode = ZSTD_compressBegin_usingCDict_deprecated(esr.zc, esr.dict); if (ZSTD_isError(errorCode)) { DISPLAYLEVEL(1, "warning : ZSTD_compressBegin_usingCDict failed \n"); return; } } - cSize = ZSTD_compressBlock(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); + cSize = ZSTD_compressBlock_deprecated(esr.zc, esr.workPlace, ZSTD_BLOCKSIZE_MAX, src, srcSize); if (ZSTD_isError(cSize)) { DISPLAYLEVEL(3, "warning : could not compress sample size %u \n", (unsigned)srcSize); return; } if (cSize) { /* if == 0; block is not compressible */ @@ -755,6 +676,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, size_t const totalSrcSize = ZDICT_totalSampleSize(fileSizes, nbFiles); size_t const averageSampleSize = totalSrcSize / (nbFiles + !nbFiles); BYTE* dstPtr = (BYTE*)dstBuffer; + U32 wksp[HUF_CTABLE_WORKSPACE_SIZE_U32]; /* init */ DEBUGLOG(4, "ZDICT_analyzeEntropy"); @@ -795,7 +717,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, } } /* analyze, build stats, starting with literals */ - { size_t maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog); + { size_t maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp)); if (HUF_isError(maxNbBits)) { eSize = maxNbBits; DISPLAYLEVEL(1, " HUF_buildCTable error \n"); @@ -804,7 +726,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, if (maxNbBits==8) { /* not compressible : will fail on HUF_writeCTable() */ DISPLAYLEVEL(2, "warning : pathological dataset : literals are not compressible : samples are noisy or too regular \n"); ZDICT_flatLit(countLit); /* replace distribution by a fake "mostly flat but still compressible" distribution, that HUF_writeCTable() can encode */ - maxNbBits = HUF_buildCTable (hufTable, countLit, 255, huffLog); + maxNbBits = HUF_buildCTable_wksp(hufTable, countLit, 255, huffLog, wksp, sizeof(wksp)); assert(maxNbBits==9); } huffLog = (U32)maxNbBits; @@ -845,7 +767,7 @@ static size_t ZDICT_analyzeEntropy(void* dstBuffer, size_t maxDstSize, llLog = (U32)errorCode; /* write result to buffer */ - { size_t const hhSize = HUF_writeCTable(dstPtr, maxDstSize, hufTable, 255, huffLog); + { size_t const hhSize = HUF_writeCTable_wksp(dstPtr, maxDstSize, hufTable, 255, huffLog, wksp, sizeof(wksp)); if (HUF_isError(hhSize)) { eSize = hhSize; DISPLAYLEVEL(1, "HUF_writeCTable error \n"); diff --git a/vendor/github.com/DataDog/zstd/zdict.h b/vendor/github.com/DataDog/zstd/zdict.h index ebc6c07..5f4af1e 100644 --- a/vendor/github.com/DataDog/zstd/zdict.h +++ b/vendor/github.com/DataDog/zstd/zdict.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -9,32 +9,43 @@ * You may select, at your option, one of the above-listed licenses. */ -#ifndef DICTBUILDER_H_001 -#define DICTBUILDER_H_001 - #if defined (__cplusplus) extern "C" { #endif +#ifndef ZSTD_ZDICT_H +#define ZSTD_ZDICT_H /*====== Dependencies ======*/ #include /* size_t */ /* ===== ZDICTLIB_API : control library symbols visibility ===== */ -#ifndef ZDICTLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZDICTLIB_VISIBILITY __attribute__ ((visibility ("default"))) +#ifndef ZDICTLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZDICTLIB_VISIBILITY +# define ZDICTLIB_VISIBLE ZDICTLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_VISIBLE __attribute__ ((visibility ("default"))) # else -# define ZDICTLIB_VISIBILITY +# define ZDICTLIB_VISIBLE # endif #endif + +#ifndef ZDICTLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZDICTLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZDICTLIB_HIDDEN +# endif +#endif + #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBILITY +# define ZDICTLIB_API __declspec(dllexport) ZDICTLIB_VISIBLE #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +# define ZDICTLIB_API __declspec(dllimport) ZDICTLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define ZDICTLIB_API ZDICTLIB_VISIBILITY +# define ZDICTLIB_API ZDICTLIB_VISIBLE #endif /******************************************************************************* @@ -111,8 +122,8 @@ extern "C" { * The zstd CLI defaults to a 110KB dictionary. You likely don't need a * dictionary larger than that. But, most use cases can get away with a * smaller dictionary. The advanced dictionary builders can automatically - * shrink the dictionary for you, and select a the smallest size that - * doesn't hurt compression ratio too much. See the `shrinkDict` parameter. + * shrink the dictionary for you, and select the smallest size that doesn't + * hurt compression ratio too much. See the `shrinkDict` parameter. * A smaller dictionary can save memory, and potentially speed up * compression. * @@ -202,9 +213,9 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer(void* dictBuffer, size_t dictBufferCap const size_t* samplesSizes, unsigned nbSamples); typedef struct { - int compressionLevel; /*< optimize for a specific zstd compression level; 0 means default */ - unsigned notificationLevel; /*< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ - unsigned dictID; /*< force dictID value; 0 means auto mode (32-bits random value) + int compressionLevel; /**< optimize for a specific zstd compression level; 0 means default */ + unsigned notificationLevel; /**< Write log to stderr; 0 = none (default); 1 = errors; 2 = progression; 3 = details; 4 = debug; */ + unsigned dictID; /**< force dictID value; 0 means auto mode (32-bits random value) * NOTE: The zstd format reserves some dictionary IDs for future use. * You may use them in private settings, but be warned that they * may be used by zstd in a public dictionary registry in the future. @@ -261,9 +272,21 @@ ZDICTLIB_API size_t ZDICT_getDictHeaderSize(const void* dictBuffer, size_t dictS ZDICTLIB_API unsigned ZDICT_isError(size_t errorCode); ZDICTLIB_API const char* ZDICT_getErrorName(size_t errorCode); +#endif /* ZSTD_ZDICT_H */ +#if defined(ZDICT_STATIC_LINKING_ONLY) && !defined(ZSTD_ZDICT_H_STATIC) +#define ZSTD_ZDICT_H_STATIC -#ifdef ZDICT_STATIC_LINKING_ONLY +/* This can be overridden externally to hide static symbols. */ +#ifndef ZDICTLIB_STATIC_API +# if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllexport) ZDICTLIB_VISIBLE +# elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) +# define ZDICTLIB_STATIC_API __declspec(dllimport) ZDICTLIB_VISIBLE +# else +# define ZDICTLIB_STATIC_API ZDICTLIB_VISIBLE +# endif +#endif /* ==================================================================================== * The definitions in this section are considered experimental. @@ -319,7 +342,7 @@ typedef struct { * In general, it's recommended to provide a few thousands samples, though this can vary a lot. * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. */ -ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_cover( void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_cover_params_t parameters); @@ -341,7 +364,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_cover( * See ZDICT_trainFromBuffer() for details on failure modes. * Note: ZDICT_optimizeTrainFromBuffer_cover() requires about 8 bytes of memory for each input byte and additionally another 5 bytes of memory for each byte of memory for each thread. */ -ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_cover( void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_cover_params_t* parameters); @@ -362,7 +385,7 @@ ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_cover( * In general, it's recommended to provide a few thousands samples, though this can vary a lot. * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. */ -ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, size_t dictBufferCapacity, const void *samplesBuffer, const size_t *samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t parameters); @@ -385,7 +408,7 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_fastCover(void *dictBuffer, * See ZDICT_trainFromBuffer() for details on failure modes. * Note: ZDICT_optimizeTrainFromBuffer_fastCover() requires about 6 * 2^f bytes of memory for each thread. */ -ZDICTLIB_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, +ZDICTLIB_STATIC_API size_t ZDICT_optimizeTrainFromBuffer_fastCover(void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_fastCover_params_t* parameters); @@ -410,7 +433,7 @@ typedef struct { * It's recommended that total size of all samples be about ~x100 times the target size of dictionary. * Note: ZDICT_trainFromBuffer_legacy() will send notifications into stderr if instructed to, using notificationLevel>0. */ -ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( +ZDICTLIB_STATIC_API size_t ZDICT_trainFromBuffer_legacy( void* dictBuffer, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples, ZDICT_legacy_params_t parameters); @@ -422,34 +445,33 @@ ZDICTLIB_API size_t ZDICT_trainFromBuffer_legacy( or _CRT_SECURE_NO_WARNINGS in Visual. Otherwise, it's also possible to manually define ZDICT_DISABLE_DEPRECATE_WARNINGS */ #ifdef ZDICT_DISABLE_DEPRECATE_WARNINGS -# define ZDICT_DEPRECATED(message) ZDICTLIB_API /* disable deprecation warnings */ +# define ZDICT_DEPRECATED(message) /* disable deprecation warnings */ #else # define ZDICT_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) # if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ -# define ZDICT_DEPRECATED(message) [[deprecated(message)]] ZDICTLIB_API +# define ZDICT_DEPRECATED(message) [[deprecated(message)]] # elif defined(__clang__) || (ZDICT_GCC_VERSION >= 405) -# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated(message))) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated(message))) # elif (ZDICT_GCC_VERSION >= 301) -# define ZDICT_DEPRECATED(message) ZDICTLIB_API __attribute__((deprecated)) +# define ZDICT_DEPRECATED(message) __attribute__((deprecated)) # elif defined(_MSC_VER) -# define ZDICT_DEPRECATED(message) ZDICTLIB_API __declspec(deprecated(message)) +# define ZDICT_DEPRECATED(message) __declspec(deprecated(message)) # else # pragma message("WARNING: You need to implement ZDICT_DEPRECATED for this compiler") -# define ZDICT_DEPRECATED(message) ZDICTLIB_API +# define ZDICT_DEPRECATED(message) # endif #endif /* ZDICT_DISABLE_DEPRECATE_WARNINGS */ ZDICT_DEPRECATED("use ZDICT_finalizeDictionary() instead") +ZDICTLIB_STATIC_API size_t ZDICT_addEntropyTablesFromBuffer(void* dictBuffer, size_t dictContentSize, size_t dictBufferCapacity, const void* samplesBuffer, const size_t* samplesSizes, unsigned nbSamples); -#endif /* ZDICT_STATIC_LINKING_ONLY */ +#endif /* ZSTD_ZDICT_H_STATIC */ #if defined (__cplusplus) } #endif -#endif /* DICTBUILDER_H_001 */ - #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/zstd.go b/vendor/github.com/DataDog/zstd/zstd.go index f792116..2cf5c61 100644 --- a/vendor/github.com/DataDog/zstd/zstd.go +++ b/vendor/github.com/DataDog/zstd/zstd.go @@ -4,7 +4,7 @@ package zstd // support decoding of "legacy" zstd payloads from versions [0.4, 0.8], matching the // default configuration of the zstd command line tool: // https://github.com/facebook/zstd/blob/dev/programs/README.md -#cgo CFLAGS: -DZSTD_LEGACY_SUPPORT=4 +#cgo CFLAGS: -DZSTD_LEGACY_SUPPORT=4 -DZSTD_MULTITHREAD=1 #include "zstd.h" */ @@ -36,7 +36,7 @@ const ( // decompressed, err := zstd.Decompress(dst, src) decompressSizeBufferLimit = 1000 * 1000 - zstdFrameHeaderSizeMax = 18 // From zstd.h. Since it's experimental API, hardcoding it + zstdFrameHeaderSizeMin = 2 // From zstd.h. Since it's experimental API, hardcoding it ) // CompressBound returns the worst case size needed for a destination buffer, @@ -67,11 +67,14 @@ func decompressSizeHint(src []byte) int { } hint := upperBound - if len(src) >= zstdFrameHeaderSizeMax { + if len(src) >= zstdFrameHeaderSizeMin { hint = int(C.ZSTD_getFrameContentSize(unsafe.Pointer(&src[0]), C.size_t(len(src)))) if hint < 0 { // On error, just use upperBound hint = upperBound } + if hint == 0 { // When compressing the empty slice, we need an output of at least 1 to pass down to the C lib + hint = 1 + } } // Take the minimum of both diff --git a/vendor/github.com/DataDog/zstd/zstd.h b/vendor/github.com/DataDog/zstd/zstd.h index 89966c8..c26acee 100644 --- a/vendor/github.com/DataDog/zstd/zstd.h +++ b/vendor/github.com/DataDog/zstd/zstd.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -15,21 +15,31 @@ extern "C" { #ifndef ZSTD_H_235446 #define ZSTD_H_235446 -/* ====== Dependency ======*/ +/* ====== Dependencies ======*/ #include /* INT_MAX */ #include /* size_t */ /* ===== ZSTDLIB_API : control library symbols visibility ===== */ #ifndef ZSTDLIB_VISIBLE -# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) + /* Backwards compatibility with old macro name */ +# ifdef ZSTDLIB_VISIBILITY +# define ZSTDLIB_VISIBLE ZSTDLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) # define ZSTDLIB_VISIBLE __attribute__ ((visibility ("default"))) -# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) # else # define ZSTDLIB_VISIBLE +# endif +#endif + +#ifndef ZSTDLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else # define ZSTDLIB_HIDDEN # endif #endif + #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) # define ZSTDLIB_API __declspec(dllexport) ZSTDLIB_VISIBLE #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) @@ -38,6 +48,28 @@ extern "C" { # define ZSTDLIB_API ZSTDLIB_VISIBLE #endif +/* Deprecation warnings : + * Should these warnings be a problem, it is generally possible to disable them, + * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. + * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. + */ +#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS +# define ZSTD_DEPRECATED(message) /* disable deprecation warnings */ +#else +# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ +# define ZSTD_DEPRECATED(message) [[deprecated(message)]] +# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated(message))) +# elif defined(__GNUC__) && (__GNUC__ >= 3) +# define ZSTD_DEPRECATED(message) __attribute__((deprecated)) +# elif defined(_MSC_VER) +# define ZSTD_DEPRECATED(message) __declspec(deprecated(message)) +# else +# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") +# define ZSTD_DEPRECATED(message) +# endif +#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ + /******************************************************************************* Introduction @@ -75,7 +107,7 @@ extern "C" { /*------ Version ------*/ #define ZSTD_VERSION_MAJOR 1 #define ZSTD_VERSION_MINOR 5 -#define ZSTD_VERSION_RELEASE 2 +#define ZSTD_VERSION_RELEASE 5 #define ZSTD_VERSION_NUMBER (ZSTD_VERSION_MAJOR *100*100 + ZSTD_VERSION_MINOR *100 + ZSTD_VERSION_RELEASE) /*! ZSTD_versionNumber() : @@ -117,7 +149,8 @@ ZSTDLIB_API const char* ZSTD_versionString(void); ***************************************/ /*! ZSTD_compress() : * Compresses `src` content as a single zstd compressed frame into already allocated `dst`. - * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data. * @return : compressed size written into `dst` (<= `dstCapacity), * or an error code if it fails (which can be tested using ZSTD_isError()). */ ZSTDLIB_API size_t ZSTD_compress( void* dst, size_t dstCapacity, @@ -166,7 +199,9 @@ ZSTDLIB_API unsigned long long ZSTD_getFrameContentSize(const void *src, size_t * "empty", "unknown" and "error" results to the same return value (0), * while ZSTD_getFrameContentSize() gives them separate return values. * @return : decompressed size of `src` frame content _if known and not empty_, 0 otherwise. */ -ZSTDLIB_API unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); +ZSTD_DEPRECATED("Replaced by ZSTD_getFrameContentSize") +ZSTDLIB_API +unsigned long long ZSTD_getDecompressedSize(const void* src, size_t srcSize); /*! ZSTD_findFrameCompressedSize() : Requires v1.4.0+ * `src` should point to the start of a ZSTD frame or skippable frame. @@ -178,8 +213,30 @@ ZSTDLIB_API size_t ZSTD_findFrameCompressedSize(const void* src, size_t srcSize) /*====== Helper functions ======*/ -#define ZSTD_COMPRESSBOUND(srcSize) ((srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ -ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ +/* ZSTD_compressBound() : + * maximum compressed size in worst case single-pass scenario. + * When invoking `ZSTD_compress()` or any other one-pass compression function, + * it's recommended to provide @dstCapacity >= ZSTD_compressBound(srcSize) + * as it eliminates one potential failure scenario, + * aka not enough room in dst buffer to write the compressed frame. + * Note : ZSTD_compressBound() itself can fail, if @srcSize > ZSTD_MAX_INPUT_SIZE . + * In which case, ZSTD_compressBound() will return an error code + * which can be tested using ZSTD_isError(). + * + * ZSTD_COMPRESSBOUND() : + * same as ZSTD_compressBound(), but as a macro. + * It can be used to produce constants, which can be useful for static allocation, + * for example to size a static array on stack. + * Will produce constant value 0 if srcSize too large. + */ +#define ZSTD_MAX_INPUT_SIZE ((sizeof(size_t)==8) ? 0xFF00FF00FF00FF00LLU : 0xFF00FF00U) +#define ZSTD_COMPRESSBOUND(srcSize) (((size_t)(srcSize) >= ZSTD_MAX_INPUT_SIZE) ? 0 : (srcSize) + ((srcSize)>>8) + (((srcSize) < (128<<10)) ? (((128<<10) - (srcSize)) >> 11) /* margin, from 64 to 0 */ : 0)) /* this formula ensures that bound(A) + bound(B) <= bound(A+B) as long as A and B >= 128 KB */ +ZSTDLIB_API size_t ZSTD_compressBound(size_t srcSize); /*!< maximum compressed size in worst case single-pass scenario */ +/* ZSTD_isError() : + * Most ZSTD_* functions returning a size_t value can be tested for error, + * using ZSTD_isError(). + * @return 1 if error, 0 otherwise + */ ZSTDLIB_API unsigned ZSTD_isError(size_t code); /*!< tells if a `size_t` function result is an error code */ ZSTDLIB_API const char* ZSTD_getErrorName(size_t code); /*!< provides readable string from an error code */ ZSTDLIB_API int ZSTD_minCLevel(void); /*!< minimum negative compression level allowed, requires v1.4.0+ */ @@ -422,6 +479,9 @@ typedef enum { * ZSTD_c_validateSequences * ZSTD_c_useBlockSplitter * ZSTD_c_useRowMatchFinder + * ZSTD_c_prefetchCDictTables + * ZSTD_c_enableSeqProducerFallback + * ZSTD_c_maxBlockSize * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly; * also, the enums values themselves are unstable and can still change. @@ -440,7 +500,11 @@ typedef enum { ZSTD_c_experimentalParam12=1009, ZSTD_c_experimentalParam13=1010, ZSTD_c_experimentalParam14=1011, - ZSTD_c_experimentalParam15=1012 + ZSTD_c_experimentalParam15=1012, + ZSTD_c_experimentalParam16=1013, + ZSTD_c_experimentalParam17=1014, + ZSTD_c_experimentalParam18=1015, + ZSTD_c_experimentalParam19=1016 } ZSTD_cParameter; typedef struct { @@ -503,7 +567,7 @@ typedef enum { * They will be used to compress next frame. * Resetting session never fails. * - The parameters : changes all parameters back to "default". - * This removes any reference to any dictionary too. + * This also removes any reference to any dictionary or external sequence producer. * Parameters can only be changed between 2 sessions (i.e. no compression is currently ongoing) * otherwise the reset fails, and function returns an error value (which can be tested using ZSTD_isError()) * - Both : similar to resetting the session, followed by resetting parameters. @@ -516,7 +580,8 @@ ZSTDLIB_API size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset); * Should cctx hold data from a previously unfinished frame, everything about it is forgotten. * - Compression parameters are pushed into CCtx before starting compression, using ZSTD_CCtx_set*() * - The function is always blocking, returns when compression is completed. - * Hint : compression runs faster if `dstCapacity` >= `ZSTD_compressBound(srcSize)`. + * NOTE: Providing `dstCapacity >= ZSTD_compressBound(srcSize)` guarantees that zstd will have + * enough space to successfully compress the data, though it is possible it fails for other reasons. * @return : compressed size written into `dst` (<= `dstCapacity), * or an error code if it fails (which can be tested using ZSTD_isError()). */ @@ -553,13 +618,15 @@ typedef enum { * ZSTD_d_stableOutBuffer * ZSTD_d_forceIgnoreChecksum * ZSTD_d_refMultipleDDicts + * ZSTD_d_disableHuffmanAssembly * Because they are not stable, it's necessary to define ZSTD_STATIC_LINKING_ONLY to access them. * note : never ever use experimentalParam? names directly */ ZSTD_d_experimentalParam1=1000, ZSTD_d_experimentalParam2=1001, ZSTD_d_experimentalParam3=1002, - ZSTD_d_experimentalParam4=1003 + ZSTD_d_experimentalParam4=1003, + ZSTD_d_experimentalParam5=1004 } ZSTD_dParameter; @@ -738,8 +805,6 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output * This following is a legacy streaming API, available since v1.0+ . * It can be replaced by ZSTD_CCtx_reset() and ZSTD_compressStream2(). * It is redundant, but remains fully supported. - * Streaming in combination with advanced parameters and dictionary compression - * can only be used through the new API. ******************************************************************************/ /*! @@ -748,6 +813,9 @@ ZSTDLIB_API size_t ZSTD_CStreamOutSize(void); /**< recommended size for output * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); * ZSTD_CCtx_refCDict(zcs, NULL); // clear the dictionary (if any) * ZSTD_CCtx_setParameter(zcs, ZSTD_c_compressionLevel, compressionLevel); + * + * Note that ZSTD_initCStream() clears any previously set dictionary. Use the new API + * to compress with a dictionary. */ ZSTDLIB_API size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel); /*! @@ -798,13 +866,31 @@ ZSTDLIB_API size_t ZSTD_freeDStream(ZSTD_DStream* zds); /* accept NULL pointer /*===== Streaming decompression functions =====*/ -/* This function is redundant with the advanced API and equivalent to: +/*! ZSTD_initDStream() : + * Initialize/reset DStream state for new decompression operation. + * Call before new decompression operation using same DStream. * + * Note : This function is redundant with the advanced API and equivalent to: * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); * ZSTD_DCtx_refDDict(zds, NULL); */ ZSTDLIB_API size_t ZSTD_initDStream(ZSTD_DStream* zds); +/*! ZSTD_decompressStream() : + * Streaming decompression function. + * Call repetitively to consume full input updating it as necessary. + * Function will update both input and output `pos` fields exposing current state via these fields: + * - `input.pos < input.size`, some input remaining and caller should provide remaining input + * on the next call. + * - `output.pos < output.size`, decoder finished and flushed all remaining buffers. + * - `output.pos == output.size`, potentially uncflushed data present in the internal buffers, + * call ZSTD_decompressStream() again to flush remaining data to output. + * Note : with no additional input, amount of data flushed <= ZSTD_BLOCKSIZE_MAX. + * + * @return : 0 when a frame is completely decoded and fully flushed, + * or an error code, which can be tested using ZSTD_isError(), + * or any other value > 0, which means there is some decoding or flushing to do to complete current frame. + */ ZSTDLIB_API size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inBuffer* input); ZSTDLIB_API size_t ZSTD_DStreamInSize(void); /*!< recommended size for input buffer */ @@ -923,7 +1009,7 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict); * If @return == 0, the dictID could not be decoded. * This could for one of the following reasons : * - The frame does not require a dictionary to be decoded (most common case). - * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden information. + * - The frame was built with dictID intentionally removed. Whatever dictionary is necessary is a hidden piece of information. * Note : this use case also happens when using a non-conformant dictionary. * - `srcSize` is too small, and as a result, the frame header could not be decoded (only possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`). * - This is not a Zstandard frame. @@ -935,9 +1021,11 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); * Advanced dictionary and prefix API (Requires v1.4.0+) * * This API allows dictionaries to be used with ZSTD_compress2(), - * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). Dictionaries are sticky, and - * only reset with the context is reset with ZSTD_reset_parameters or - * ZSTD_reset_session_and_parameters. Prefixes are single-use. + * ZSTD_compressStream2(), and ZSTD_decompressDCtx(). + * Dictionaries are sticky, they remain valid when same context is re-used, + * they only reset when the context is reset + * with ZSTD_reset_parameters or ZSTD_reset_session_and_parameters. + * In contrast, Prefixes are single-use. ******************************************************************************/ @@ -947,8 +1035,9 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Special: Loading a NULL (or 0-size) dictionary invalidates previous dictionary, * meaning "return to no-dictionary mode". - * Note 1 : Dictionary is sticky, it will be used for all future compressed frames. - * To return to "no-dictionary" situation, load a NULL dictionary (or reset parameters). + * Note 1 : Dictionary is sticky, it will be used for all future compressed frames, + * until parameters are reset, a new dictionary is loaded, or the dictionary + * is explicitly invalidated by loading a NULL dictionary. * Note 2 : Loading a dictionary involves building tables. * It's also a CPU consuming operation, with non-negligible impact on latency. * Tables are dependent on compression parameters, and for this reason, @@ -957,11 +1046,15 @@ ZSTDLIB_API unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize); * Use experimental ZSTD_CCtx_loadDictionary_byReference() to reference content instead. * In such a case, dictionary buffer must outlive its users. * Note 4 : Use ZSTD_CCtx_loadDictionary_advanced() - * to precisely select how dictionary content must be interpreted. */ + * to precisely select how dictionary content must be interpreted. + * Note 5 : This method does not benefit from LDM (long distance mode). + * If you want to employ LDM on some large dictionary content, + * prefer employing ZSTD_CCtx_refPrefix() described below. + */ ZSTDLIB_API size_t ZSTD_CCtx_loadDictionary(ZSTD_CCtx* cctx, const void* dict, size_t dictSize); /*! ZSTD_CCtx_refCDict() : Requires v1.4.0+ - * Reference a prepared dictionary, to be used for all next compressed frames. + * Reference a prepared dictionary, to be used for all future compressed frames. * Note that compression parameters are enforced from within CDict, * and supersede any compression parameter previously set within CCtx. * The parameters ignored are labelled as "superseded-by-cdict" in the ZSTD_cParameter enum docs. @@ -980,6 +1073,7 @@ ZSTDLIB_API size_t ZSTD_CCtx_refCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); * Decompression will need same prefix to properly regenerate data. * Compressing with a prefix is similar in outcome as performing a diff and compressing it, * but performs much faster, especially during decompression (compression speed is tunable with compression level). + * This method is compatible with LDM (long distance mode). * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Special: Adding any prefix (including NULL) invalidates any previous prefix or dictionary * Note 1 : Prefix buffer is referenced. It **must** outlive compression. @@ -996,9 +1090,9 @@ ZSTDLIB_API size_t ZSTD_CCtx_refPrefix(ZSTD_CCtx* cctx, const void* prefix, size_t prefixSize); /*! ZSTD_DCtx_loadDictionary() : Requires v1.4.0+ - * Create an internal DDict from dict buffer, - * to be used to decompress next frames. - * The dictionary remains valid for all future frames, until explicitly invalidated. + * Create an internal DDict from dict buffer, to be used to decompress all future frames. + * The dictionary remains valid for all future frames, until explicitly invalidated, or + * a new dictionary is loaded. * @result : 0, or an error code (which can be tested with ZSTD_isError()). * Special : Adding a NULL (or 0-size) dictionary invalidates any previous dictionary, * meaning "return to no-dictionary mode". @@ -1022,9 +1116,10 @@ ZSTDLIB_API size_t ZSTD_DCtx_loadDictionary(ZSTD_DCtx* dctx, const void* dict, s * The memory for the table is allocated on the first call to refDDict, and can be * freed with ZSTD_freeDCtx(). * + * If called with ZSTD_d_refMultipleDDicts disabled (the default), only one dictionary + * will be managed, and referencing a dictionary effectively "discards" any previous one. + * * @result : 0, or an error code (which can be tested with ZSTD_isError()). - * Note 1 : Currently, only one dictionary can be managed. - * Referencing a new dictionary effectively "discards" any previous one. * Special: referencing a NULL DDict means "return to no-dictionary mode". * Note 2 : DDict is just referenced, its lifetime must outlive its usage from DCtx. */ @@ -1087,28 +1182,6 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); # endif #endif -/* Deprecation warnings : - * Should these warnings be a problem, it is generally possible to disable them, - * typically with -Wno-deprecated-declarations for gcc or _CRT_SECURE_NO_WARNINGS in Visual. - * Otherwise, it's also possible to define ZSTD_DISABLE_DEPRECATE_WARNINGS. - */ -#ifdef ZSTD_DISABLE_DEPRECATE_WARNINGS -# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API /* disable deprecation warnings */ -#else -# if defined (__cplusplus) && (__cplusplus >= 201402) /* C++14 or greater */ -# define ZSTD_DEPRECATED(message) [[deprecated(message)]] ZSTDLIB_STATIC_API -# elif (defined(GNUC) && (GNUC > 4 || (GNUC == 4 && GNUC_MINOR >= 5))) || defined(__clang__) -# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated(message))) -# elif defined(__GNUC__) && (__GNUC__ >= 3) -# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __attribute__((deprecated)) -# elif defined(_MSC_VER) -# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API __declspec(deprecated(message)) -# else -# pragma message("WARNING: You need to implement ZSTD_DEPRECATED for this compiler") -# define ZSTD_DEPRECATED(message) ZSTDLIB_STATIC_API -# endif -#endif /* ZSTD_DISABLE_DEPRECATE_WARNINGS */ - /**************************************************************************************** * experimental API (static linking only) **************************************************************************************** @@ -1143,6 +1216,7 @@ ZSTDLIB_API size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict); #define ZSTD_TARGETLENGTH_MIN 0 /* note : comparing this constant to an unsigned results in a tautological test */ #define ZSTD_STRATEGY_MIN ZSTD_fast #define ZSTD_STRATEGY_MAX ZSTD_btultra2 +#define ZSTD_BLOCKSIZE_MAX_MIN (1 << 10) /* The minimum valid max blocksize. Maximum blocksizes smaller than this make compressBound() inaccurate. */ #define ZSTD_OVERLAPLOG_MIN 0 @@ -1323,7 +1397,7 @@ typedef enum { } ZSTD_paramSwitch_e; /*************************************** -* Frame size functions +* Frame header and size functions ***************************************/ /*! ZSTD_findDecompressedSize() : @@ -1370,33 +1444,113 @@ ZSTDLIB_STATIC_API unsigned long long ZSTD_decompressBound(const void* src, size * or an error code (if srcSize is too small) */ ZSTDLIB_STATIC_API size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize); +typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; +typedef struct { + unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ + unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ + unsigned blockSizeMax; + ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ + unsigned headerSize; + unsigned dictID; + unsigned checksumFlag; + unsigned _reserved1; + unsigned _reserved2; +} ZSTD_frameHeader; + +/*! ZSTD_getFrameHeader() : + * decode Frame Header, or requires larger `srcSize`. + * @return : 0, `zfhPtr` is correctly filled, + * >0, `srcSize` is too small, value is wanted `srcSize` amount, + * or an error code, which can be tested using ZSTD_isError() */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ +/*! ZSTD_getFrameHeader_advanced() : + * same as ZSTD_getFrameHeader(), + * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ +ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); + +/*! ZSTD_decompressionMargin() : + * Zstd supports in-place decompression, where the input and output buffers overlap. + * In this case, the output buffer must be at least (Margin + Output_Size) bytes large, + * and the input buffer must be at the end of the output buffer. + * + * _______________________ Output Buffer ________________________ + * | | + * | ____ Input Buffer ____| + * | | | + * v v v + * |---------------------------------------|-----------|----------| + * ^ ^ ^ + * |___________________ Output_Size ___________________|_ Margin _| + * + * NOTE: See also ZSTD_DECOMPRESSION_MARGIN(). + * NOTE: This applies only to single-pass decompression through ZSTD_decompress() or + * ZSTD_decompressDCtx(). + * NOTE: This function supports multi-frame input. + * + * @param src The compressed frame(s) + * @param srcSize The size of the compressed frame(s) + * @returns The decompression margin or an error that can be checked with ZSTD_isError(). + */ +ZSTDLIB_STATIC_API size_t ZSTD_decompressionMargin(const void* src, size_t srcSize); + +/*! ZSTD_DECOMPRESS_MARGIN() : + * Similar to ZSTD_decompressionMargin(), but instead of computing the margin from + * the compressed frame, compute it from the original size and the blockSizeLog. + * See ZSTD_decompressionMargin() for details. + * + * WARNING: This macro does not support multi-frame input, the input must be a single + * zstd frame. If you need that support use the function, or implement it yourself. + * + * @param originalSize The original uncompressed size of the data. + * @param blockSize The block size == MIN(windowSize, ZSTD_BLOCKSIZE_MAX). + * Unless you explicitly set the windowLog smaller than + * ZSTD_BLOCKSIZELOG_MAX you can just use ZSTD_BLOCKSIZE_MAX. + */ +#define ZSTD_DECOMPRESSION_MARGIN(originalSize, blockSize) ((size_t)( \ + ZSTD_FRAMEHEADERSIZE_MAX /* Frame header */ + \ + 4 /* checksum */ + \ + ((originalSize) == 0 ? 0 : 3 * (((originalSize) + (blockSize) - 1) / blockSize)) /* 3 bytes per block */ + \ + (blockSize) /* One block of margin */ \ + )) + typedef enum { ZSTD_sf_noBlockDelimiters = 0, /* Representation of ZSTD_Sequence has no block delimiters, sequences only */ ZSTD_sf_explicitBlockDelimiters = 1 /* Representation of ZSTD_Sequence contains explicit block delimiters */ } ZSTD_sequenceFormat_e; +/*! ZSTD_sequenceBound() : + * `srcSize` : size of the input buffer + * @return : upper-bound for the number of sequences that can be generated + * from a buffer of srcSize bytes + * + * note : returns number of sequences - to get bytes, multiply by sizeof(ZSTD_Sequence). + */ +ZSTDLIB_STATIC_API size_t ZSTD_sequenceBound(size_t srcSize); + /*! ZSTD_generateSequences() : - * Generate sequences using ZSTD_compress2, given a source buffer. + * Generate sequences using ZSTD_compress2(), given a source buffer. * * Each block will end with a dummy sequence * with offset == 0, matchLength == 0, and litLength == length of last literals. * litLength may be == 0, and if so, then the sequence of (of: 0 ml: 0 ll: 0) * simply acts as a block delimiter. * - * zc can be used to insert custom compression params. - * This function invokes ZSTD_compress2 + * @zc can be used to insert custom compression params. + * This function invokes ZSTD_compress2(). * * The output of this function can be fed into ZSTD_compressSequences() with CCtx * setting of ZSTD_c_blockDelimiters as ZSTD_sf_explicitBlockDelimiters * @return : number of sequences generated */ -ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, - size_t outSeqsSize, const void* src, size_t srcSize); +ZSTDLIB_STATIC_API size_t +ZSTD_generateSequences( ZSTD_CCtx* zc, + ZSTD_Sequence* outSeqs, size_t outSeqsSize, + const void* src, size_t srcSize); /*! ZSTD_mergeBlockDelimiters() : * Given an array of ZSTD_Sequence, remove all sequences that represent block delimiters/last literals - * by merging them into into the literals of the next sequence. + * by merging them into the literals of the next sequence. * * As such, the final generated result has no explicit representation of block boundaries, * and the final last literals segment is not represented in the sequences. @@ -1408,7 +1562,9 @@ ZSTDLIB_STATIC_API size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* o ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, size_t seqsSize); /*! ZSTD_compressSequences() : - * Compress an array of ZSTD_Sequence, generated from the original source buffer, into dst. + * Compress an array of ZSTD_Sequence, associated with @src buffer, into dst. + * @src contains the entire input (not just the literals). + * If @srcSize > sum(sequence.length), the remaining bytes are considered all literals * If a dictionary is included, then the cctx should reference the dict. (see: ZSTD_CCtx_refCDict(), ZSTD_CCtx_loadDictionary(), etc.) * The entire source is compressed into a single frame. * @@ -1433,17 +1589,18 @@ ZSTDLIB_STATIC_API size_t ZSTD_mergeBlockDelimiters(ZSTD_Sequence* sequences, si * Note: Repcodes are, as of now, always re-calculated within this function, so ZSTD_Sequence::rep is unused. * Note 2: Once we integrate ability to ingest repcodes, the explicit block delims mode must respect those repcodes exactly, * and cannot emit an RLE block that disagrees with the repcode history - * @return : final compressed size or a ZSTD error. + * @return : final compressed size, or a ZSTD error code. */ -ZSTDLIB_STATIC_API size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstSize, - const ZSTD_Sequence* inSeqs, size_t inSeqsSize, - const void* src, size_t srcSize); +ZSTDLIB_STATIC_API size_t +ZSTD_compressSequences( ZSTD_CCtx* cctx, void* dst, size_t dstSize, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, + const void* src, size_t srcSize); /*! ZSTD_writeSkippableFrame() : * Generates a zstd skippable frame containing data given by src, and writes it to dst buffer. * - * Skippable frames begin with a a 4-byte magic number. There are 16 possible choices of magic number, + * Skippable frames begin with a 4-byte magic number. There are 16 possible choices of magic number, * ranging from ZSTD_MAGIC_SKIPPABLE_START to ZSTD_MAGIC_SKIPPABLE_START+15. * As such, the parameter magicVariant controls the exact skippable frame magic number variant used, so * the magic number used will be ZSTD_MAGIC_SKIPPABLE_START + magicVariant. @@ -1501,8 +1658,11 @@ ZSTDLIB_API unsigned ZSTD_isSkippableFrame(const void* buffer, size_t size); * and ZSTD_estimateCCtxSize_usingCCtxParams(), which can be used in tandem with ZSTD_CCtxParams_setParameter(). * Both can be used to estimate memory using custom compression parameters and arbitrary srcSize limits. * - * Note 2 : only single-threaded compression is supported. + * Note : only single-threaded compression is supported. * ZSTD_estimateCCtxSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * + * Note 2 : ZSTD_estimateCCtxSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. */ ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize(int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams); @@ -1521,7 +1681,12 @@ ZSTDLIB_STATIC_API size_t ZSTD_estimateDCtxSize(void); * or deducted from a valid frame Header, using ZSTD_estimateDStreamSize_fromFrame(); * Note : if streaming is init with function ZSTD_init?Stream_usingDict(), * an internal ?Dict will be created, which additional size is not estimated here. - * In this case, get total size by adding ZSTD_estimate?DictSize */ + * In this case, get total size by adding ZSTD_estimate?DictSize + * Note 2 : only single-threaded compression is supported. + * ZSTD_estimateCStreamSize_usingCCtxParams() will return an error code if ZSTD_c_nbWorkers is >= 1. + * Note 3 : ZSTD_estimateCStreamSize* functions are not compatible with the Block-Level Sequence Producer API at this time. + * Size estimates assume that no external sequence producer is registered. + */ ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize(int compressionLevel); ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCParams(ZSTD_compressionParameters cParams); ZSTDLIB_STATIC_API size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params); @@ -1671,22 +1836,45 @@ ZSTDLIB_STATIC_API size_t ZSTD_checkCParams(ZSTD_compressionParameters params); * This function never fails (wide contract) */ ZSTDLIB_STATIC_API ZSTD_compressionParameters ZSTD_adjustCParams(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize); +/*! ZSTD_CCtx_setCParams() : + * Set all parameters provided within @p cparams into the working @p cctx. + * Note : if modifying parameters during compression (MT mode only), + * note that changes to the .windowLog parameter will be ignored. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + * On failure, no parameters are updated. + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams); + +/*! ZSTD_CCtx_setFParams() : + * Set all parameters provided within @p fparams into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams); + +/*! ZSTD_CCtx_setParams() : + * Set all parameters provided within @p params into the working @p cctx. + * @return 0 on success, or an error code (can be checked with ZSTD_isError()). + */ +ZSTDLIB_STATIC_API size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params); + /*! ZSTD_compress_advanced() : * Note : this function is now DEPRECATED. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_setParameter() and other parameter setters. * This prototype will generate compilation warnings. */ ZSTD_DEPRECATED("use ZSTD_compress2") +ZSTDLIB_STATIC_API size_t ZSTD_compress_advanced(ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - const void* dict,size_t dictSize, - ZSTD_parameters params); + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + const void* dict,size_t dictSize, + ZSTD_parameters params); /*! ZSTD_compress_usingCDict_advanced() : * Note : this function is now DEPRECATED. * It can be replaced by ZSTD_compress2(), in combination with ZSTD_CCtx_loadDictionary() and other parameter setters. * This prototype will generate compilation warnings. */ ZSTD_DEPRECATED("use ZSTD_compress2 with ZSTD_CCtx_loadDictionary") +ZSTDLIB_STATIC_API size_t ZSTD_compress_usingCDict_advanced(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -1830,13 +2018,16 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo * Experimental parameter. * Default is 0 == disabled. Set to 1 to enable. * - * Tells the compressor that the ZSTD_inBuffer will ALWAYS be the same - * between calls, except for the modifications that zstd makes to pos (the - * caller must not modify pos). This is checked by the compressor, and - * compression will fail if it ever changes. This means the only flush - * mode that makes sense is ZSTD_e_end, so zstd will error if ZSTD_e_end - * is not used. The data in the ZSTD_inBuffer in the range [src, src + pos) - * MUST not be modified during compression or you will get data corruption. + * Tells the compressor that input data presented with ZSTD_inBuffer + * will ALWAYS be the same between calls. + * Technically, the @src pointer must never be changed, + * and the @pos field can only be updated by zstd. + * However, it's possible to increase the @size field, + * allowing scenarios where more data can be appended after compressions starts. + * These conditions are checked by the compressor, + * and compression will fail if they are not respected. + * Also, data in the ZSTD_inBuffer within the range [src, src + pos) + * MUST not be modified during compression or it will result in data corruption. * * When this flag is enabled zstd won't allocate an input window buffer, * because the user guarantees it can reference the ZSTD_inBuffer until @@ -1844,18 +2035,15 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo * large enough to fit a block (see ZSTD_c_stableOutBuffer). This will also * avoid the memcpy() from the input buffer to the input window buffer. * - * NOTE: ZSTD_compressStream2() will error if ZSTD_e_end is not used. - * That means this flag cannot be used with ZSTD_compressStream(). - * * NOTE: So long as the ZSTD_inBuffer always points to valid memory, using * this flag is ALWAYS memory safe, and will never access out-of-bounds - * memory. However, compression WILL fail if you violate the preconditions. + * memory. However, compression WILL fail if conditions are not respected. * - * WARNING: The data in the ZSTD_inBuffer in the range [dst, dst + pos) MUST - * not be modified during compression or you will get data corruption. This - * is because zstd needs to reference data in the ZSTD_inBuffer to find + * WARNING: The data in the ZSTD_inBuffer in the range [src, src + pos) MUST + * not be modified during compression or it will result in data corruption. + * This is because zstd needs to reference data in the ZSTD_inBuffer to find * matches. Normally zstd maintains its own window buffer for this purpose, - * but passing this flag tells zstd to use the user provided buffer. + * but passing this flag tells zstd to rely on user provided buffer instead. */ #define ZSTD_c_stableInBuffer ZSTD_c_experimentalParam9 @@ -1900,7 +2088,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo * Without validation, providing a sequence that does not conform to the zstd spec will cause * undefined behavior, and may produce a corrupted block. * - * With validation enabled, a if sequence is invalid (see doc/zstd_compression_format.md for + * With validation enabled, if sequence is invalid (see doc/zstd_compression_format.md for * specifics regarding offset/matchlength requirements) then the function will bail out and * return an error. * @@ -1950,6 +2138,79 @@ ZSTDLIB_STATIC_API size_t ZSTD_CCtx_refPrefix_advanced(ZSTD_CCtx* cctx, const vo */ #define ZSTD_c_deterministicRefPrefix ZSTD_c_experimentalParam15 +/* ZSTD_c_prefetchCDictTables + * Controlled with ZSTD_paramSwitch_e enum. Default is ZSTD_ps_auto. + * + * In some situations, zstd uses CDict tables in-place rather than copying them + * into the working context. (See docs on ZSTD_dictAttachPref_e above for details). + * In such situations, compression speed is seriously impacted when CDict tables are + * "cold" (outside CPU cache). This parameter instructs zstd to prefetch CDict tables + * when they are used in-place. + * + * For sufficiently small inputs, the cost of the prefetch will outweigh the benefit. + * For sufficiently large inputs, zstd will by default memcpy() CDict tables + * into the working context, so there is no need to prefetch. This parameter is + * targeted at a middle range of input sizes, where a prefetch is cheap enough to be + * useful but memcpy() is too expensive. The exact range of input sizes where this + * makes sense is best determined by careful experimentation. + * + * Note: for this parameter, ZSTD_ps_auto is currently equivalent to ZSTD_ps_disable, + * but in the future zstd may conditionally enable this feature via an auto-detection + * heuristic for cold CDicts. + * Use ZSTD_ps_disable to opt out of prefetching under any circumstances. + */ +#define ZSTD_c_prefetchCDictTables ZSTD_c_experimentalParam16 + +/* ZSTD_c_enableSeqProducerFallback + * Allowed values are 0 (disable) and 1 (enable). The default setting is 0. + * + * Controls whether zstd will fall back to an internal sequence producer if an + * external sequence producer is registered and returns an error code. This fallback + * is block-by-block: the internal sequence producer will only be called for blocks + * where the external sequence producer returns an error code. Fallback parsing will + * follow any other cParam settings, such as compression level, the same as in a + * normal (fully-internal) compression operation. + * + * The user is strongly encouraged to read the full Block-Level Sequence Producer API + * documentation (below) before setting this parameter. */ +#define ZSTD_c_enableSeqProducerFallback ZSTD_c_experimentalParam17 + +/* ZSTD_c_maxBlockSize + * Allowed values are between 1KB and ZSTD_BLOCKSIZE_MAX (128KB). + * The default is ZSTD_BLOCKSIZE_MAX, and setting to 0 will set to the default. + * + * This parameter can be used to set an upper bound on the blocksize + * that overrides the default ZSTD_BLOCKSIZE_MAX. It cannot be used to set upper + * bounds greater than ZSTD_BLOCKSIZE_MAX or bounds lower than 1KB (will make + * compressBound() inaccurate). Only currently meant to be used for testing. + * + */ +#define ZSTD_c_maxBlockSize ZSTD_c_experimentalParam18 + +/* ZSTD_c_searchForExternalRepcodes + * This parameter affects how zstd parses external sequences, such as sequences + * provided through the compressSequences() API or from an external block-level + * sequence producer. + * + * If set to ZSTD_ps_enable, the library will check for repeated offsets in + * external sequences, even if those repcodes are not explicitly indicated in + * the "rep" field. Note that this is the only way to exploit repcode matches + * while using compressSequences() or an external sequence producer, since zstd + * currently ignores the "rep" field of external sequences. + * + * If set to ZSTD_ps_disable, the library will not exploit repeated offsets in + * external sequences, regardless of whether the "rep" field has been set. This + * reduces sequence compression overhead by about 25% while sacrificing some + * compression ratio. + * + * The default value is ZSTD_ps_auto, for which the library will enable/disable + * based on compression level. + * + * Note: for now, this param only has an effect if ZSTD_c_blockDelimiters is + * set to ZSTD_sf_explicitBlockDelimiters. That may change in the future. + */ +#define ZSTD_c_searchForExternalRepcodes ZSTD_c_experimentalParam19 + /*! ZSTD_CCtx_getParameter() : * Get the requested compression parameter value, selected by enum ZSTD_cParameter, * and store it into int* value. @@ -2106,7 +2367,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete * in the range [dst, dst + pos) MUST not be modified during decompression * or you will get data corruption. * - * When this flags is enabled zstd won't allocate an output buffer, because + * When this flag is enabled zstd won't allocate an output buffer, because * it can write directly to the ZSTD_outBuffer, but it will still allocate * an input buffer large enough to fit any compressed block. This will also * avoid the memcpy() from the internal output buffer to the ZSTD_outBuffer. @@ -2159,6 +2420,17 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete */ #define ZSTD_d_refMultipleDDicts ZSTD_d_experimentalParam4 +/* ZSTD_d_disableHuffmanAssembly + * Set to 1 to disable the Huffman assembly implementation. + * The default value is 0, which allows zstd to use the Huffman assembly + * implementation if available. + * + * This parameter can be used to disable Huffman assembly at runtime. + * If you want to disable it at compile time you can define the macro + * ZSTD_DISABLE_ASM. + */ +#define ZSTD_d_disableHuffmanAssembly ZSTD_d_experimentalParam5 + /*! ZSTD_DCtx_setFormat() : * This function is REDUNDANT. Prefer ZSTD_DCtx_setParameter(). @@ -2167,6 +2439,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParamete * such ZSTD_f_zstd1_magicless for example. * @return : 0, or an error code (which can be tested using ZSTD_isError()). */ ZSTD_DEPRECATED("use ZSTD_DCtx_setParameter() instead") +ZSTDLIB_STATIC_API size_t ZSTD_DCtx_setFormat(ZSTD_DCtx* dctx, ZSTD_format_e format); /*! ZSTD_decompressStream_simpleArgs() : @@ -2203,6 +2476,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_decompressStream_simpleArgs ( * This prototype will generate compilation warnings. */ ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, int compressionLevel, unsigned long long pledgedSrcSize); @@ -2220,17 +2494,15 @@ size_t ZSTD_initCStream_srcSize(ZSTD_CStream* zcs, * This prototype will generate compilation warnings. */ ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, const void* dict, size_t dictSize, int compressionLevel); /*! ZSTD_initCStream_advanced() : - * This function is DEPRECATED, and is approximately equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd parameter and leave the rest as-is. - * for ((param, value) : params) { - * ZSTD_CCtx_setParameter(zcs, param, value); - * } + * ZSTD_CCtx_setParams(zcs, params); * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_loadDictionary(zcs, dict, dictSize); * @@ -2240,6 +2512,7 @@ size_t ZSTD_initCStream_usingDict(ZSTD_CStream* zcs, * This prototype will generate compilation warnings. */ ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, const void* dict, size_t dictSize, ZSTD_parameters params, @@ -2254,15 +2527,13 @@ size_t ZSTD_initCStream_advanced(ZSTD_CStream* zcs, * This prototype will generate compilation warnings. */ ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); /*! ZSTD_initCStream_usingCDict_advanced() : - * This function is DEPRECATED, and is approximately equivalent to: + * This function is DEPRECATED, and is equivalent to: * ZSTD_CCtx_reset(zcs, ZSTD_reset_session_only); - * // Pseudocode: Set each zstd frame parameter and leave the rest as-is. - * for ((fParam, value) : fParams) { - * ZSTD_CCtx_setParameter(zcs, fParam, value); - * } + * ZSTD_CCtx_setFParams(zcs, fParams); * ZSTD_CCtx_setPledgedSrcSize(zcs, pledgedSrcSize); * ZSTD_CCtx_refCDict(zcs, cdict); * @@ -2272,6 +2543,7 @@ size_t ZSTD_initCStream_usingCDict(ZSTD_CStream* zcs, const ZSTD_CDict* cdict); * This prototype will generate compilation warnings. */ ZSTD_DEPRECATED("use ZSTD_CCtx_reset and ZSTD_CCtx_refCDict, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams, @@ -2296,6 +2568,7 @@ size_t ZSTD_initCStream_usingCDict_advanced(ZSTD_CStream* zcs, * This prototype will generate compilation warnings. */ ZSTD_DEPRECATED("use ZSTD_CCtx_reset, see zstd.h for detailed instructions") +ZSTDLIB_STATIC_API size_t ZSTD_resetCStream(ZSTD_CStream* zcs, unsigned long long pledgedSrcSize); @@ -2341,8 +2614,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_toFlushNow(ZSTD_CCtx* cctx); * ZSTD_DCtx_loadDictionary(zds, dict, dictSize); * * note: no dictionary will be used if dict == NULL or dictSize < 8 - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_loadDictionary, see zstd.h for detailed instructions") ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t dictSize); /*! @@ -2352,8 +2625,8 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const vo * ZSTD_DCtx_refDDict(zds, ddict); * * note : ddict is referenced, it must outlive decompression session - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset + ZSTD_DCtx_refDDict, see zstd.h for detailed instructions") ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const ZSTD_DDict* ddict); /*! @@ -2362,17 +2635,185 @@ ZSTDLIB_STATIC_API size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* zds, const Z * ZSTD_DCtx_reset(zds, ZSTD_reset_session_only); * * re-use decompression parameters from previous init; saves dictionary loading - * Note : this prototype will be marked as deprecated and generate compilation warnings on reaching v1.5.x */ +ZSTD_DEPRECATED("use ZSTD_DCtx_reset, see zstd.h for detailed instructions") ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); +/* ********************* BLOCK-LEVEL SEQUENCE PRODUCER API ********************* + * + * *** OVERVIEW *** + * The Block-Level Sequence Producer API allows users to provide their own custom + * sequence producer which libzstd invokes to process each block. The produced list + * of sequences (literals and matches) is then post-processed by libzstd to produce + * valid compressed blocks. + * + * This block-level offload API is a more granular complement of the existing + * frame-level offload API compressSequences() (introduced in v1.5.1). It offers + * an easier migration story for applications already integrated with libzstd: the + * user application continues to invoke the same compression functions + * ZSTD_compress2() or ZSTD_compressStream2() as usual, and transparently benefits + * from the specific advantages of the external sequence producer. For example, + * the sequence producer could be tuned to take advantage of known characteristics + * of the input, to offer better speed / ratio, or could leverage hardware + * acceleration not available within libzstd itself. + * + * See contrib/externalSequenceProducer for an example program employing the + * Block-Level Sequence Producer API. + * + * *** USAGE *** + * The user is responsible for implementing a function of type + * ZSTD_sequenceProducer_F. For each block, zstd will pass the following + * arguments to the user-provided function: + * + * - sequenceProducerState: a pointer to a user-managed state for the sequence + * producer. + * + * - outSeqs, outSeqsCapacity: an output buffer for the sequence producer. + * outSeqsCapacity is guaranteed >= ZSTD_sequenceBound(srcSize). The memory + * backing outSeqs is managed by the CCtx. + * + * - src, srcSize: an input buffer for the sequence producer to parse. + * srcSize is guaranteed to be <= ZSTD_BLOCKSIZE_MAX. + * + * - dict, dictSize: a history buffer, which may be empty, which the sequence + * producer may reference as it parses the src buffer. Currently, zstd will + * always pass dictSize == 0 into external sequence producers, but this will + * change in the future. + * + * - compressionLevel: a signed integer representing the zstd compression level + * set by the user for the current operation. The sequence producer may choose + * to use this information to change its compression strategy and speed/ratio + * tradeoff. Note: the compression level does not reflect zstd parameters set + * through the advanced API. + * + * - windowSize: a size_t representing the maximum allowed offset for external + * sequences. Note that sequence offsets are sometimes allowed to exceed the + * windowSize if a dictionary is present, see doc/zstd_compression_format.md + * for details. + * + * The user-provided function shall return a size_t representing the number of + * sequences written to outSeqs. This return value will be treated as an error + * code if it is greater than outSeqsCapacity. The return value must be non-zero + * if srcSize is non-zero. The ZSTD_SEQUENCE_PRODUCER_ERROR macro is provided + * for convenience, but any value greater than outSeqsCapacity will be treated as + * an error code. + * + * If the user-provided function does not return an error code, the sequences + * written to outSeqs must be a valid parse of the src buffer. Data corruption may + * occur if the parse is not valid. A parse is defined to be valid if the + * following conditions hold: + * - The sum of matchLengths and literalLengths must equal srcSize. + * - All sequences in the parse, except for the final sequence, must have + * matchLength >= ZSTD_MINMATCH_MIN. The final sequence must have + * matchLength >= ZSTD_MINMATCH_MIN or matchLength == 0. + * - All offsets must respect the windowSize parameter as specified in + * doc/zstd_compression_format.md. + * - If the final sequence has matchLength == 0, it must also have offset == 0. + * + * zstd will only validate these conditions (and fail compression if they do not + * hold) if the ZSTD_c_validateSequences cParam is enabled. Note that sequence + * validation has a performance cost. + * + * If the user-provided function returns an error, zstd will either fall back + * to an internal sequence producer or fail the compression operation. The user can + * choose between the two behaviors by setting the ZSTD_c_enableSeqProducerFallback + * cParam. Fallback compression will follow any other cParam settings, such as + * compression level, the same as in a normal compression operation. + * + * The user shall instruct zstd to use a particular ZSTD_sequenceProducer_F + * function by calling + * ZSTD_registerSequenceProducer(cctx, + * sequenceProducerState, + * sequenceProducer) + * This setting will persist until the next parameter reset of the CCtx. + * + * The sequenceProducerState must be initialized by the user before calling + * ZSTD_registerSequenceProducer(). The user is responsible for destroying the + * sequenceProducerState. + * + * *** LIMITATIONS *** + * This API is compatible with all zstd compression APIs which respect advanced parameters. + * However, there are three limitations: + * + * First, the ZSTD_c_enableLongDistanceMatching cParam is not currently supported. + * COMPRESSION WILL FAIL if it is enabled and the user tries to compress with a block-level + * external sequence producer. + * - Note that ZSTD_c_enableLongDistanceMatching is auto-enabled by default in some + * cases (see its documentation for details). Users must explicitly set + * ZSTD_c_enableLongDistanceMatching to ZSTD_ps_disable in such cases if an external + * sequence producer is registered. + * - As of this writing, ZSTD_c_enableLongDistanceMatching is disabled by default + * whenever ZSTD_c_windowLog < 128MB, but that's subject to change. Users should + * check the docs on ZSTD_c_enableLongDistanceMatching whenever the Block-Level Sequence + * Producer API is used in conjunction with advanced settings (like ZSTD_c_windowLog). + * + * Second, history buffers are not currently supported. Concretely, zstd will always pass + * dictSize == 0 to the external sequence producer (for now). This has two implications: + * - Dictionaries are not currently supported. Compression will *not* fail if the user + * references a dictionary, but the dictionary won't have any effect. + * - Stream history is not currently supported. All advanced compression APIs, including + * streaming APIs, work with external sequence producers, but each block is treated as + * an independent chunk without history from previous blocks. + * + * Third, multi-threading within a single compression is not currently supported. In other words, + * COMPRESSION WILL FAIL if ZSTD_c_nbWorkers > 0 and an external sequence producer is registered. + * Multi-threading across compressions is fine: simply create one CCtx per thread. + * + * Long-term, we plan to overcome all three limitations. There is no technical blocker to + * overcoming them. It is purely a question of engineering effort. + */ + +#define ZSTD_SEQUENCE_PRODUCER_ERROR ((size_t)(-1)) + +typedef size_t ZSTD_sequenceProducer_F ( + void* sequenceProducerState, + ZSTD_Sequence* outSeqs, size_t outSeqsCapacity, + const void* src, size_t srcSize, + const void* dict, size_t dictSize, + int compressionLevel, + size_t windowSize +); + +/*! ZSTD_registerSequenceProducer() : + * Instruct zstd to use a block-level external sequence producer function. + * + * The sequenceProducerState must be initialized by the caller, and the caller is + * responsible for managing its lifetime. This parameter is sticky across + * compressions. It will remain set until the user explicitly resets compression + * parameters. + * + * Sequence producer registration is considered to be an "advanced parameter", + * part of the "advanced API". This means it will only have an effect on compression + * APIs which respect advanced parameters, such as compress2() and compressStream2(). + * Older compression APIs such as compressCCtx(), which predate the introduction of + * "advanced parameters", will ignore any external sequence producer setting. + * + * The sequence producer can be "cleared" by registering a NULL function pointer. This + * removes all limitations described above in the "LIMITATIONS" section of the API docs. + * + * The user is strongly encouraged to read the full API documentation (above) before + * calling this function. */ +ZSTDLIB_STATIC_API void +ZSTD_registerSequenceProducer( + ZSTD_CCtx* cctx, + void* sequenceProducerState, + ZSTD_sequenceProducer_F* sequenceProducer +); + + /********************************************************************* -* Buffer-less and synchronous inner streaming functions +* Buffer-less and synchronous inner streaming functions (DEPRECATED) * -* This is an advanced API, giving full control over buffer management, for users which need direct control over memory. -* But it's also a complex one, with several restrictions, documented below. -* Prefer normal streaming API for an easier experience. +* This API is deprecated, and will be removed in a future version. +* It allows streaming (de)compression with user allocated buffers. +* However, it is hard to use, and not as well tested as the rest of +* our API. +* +* Please use the normal streaming API instead: ZSTD_compressStream2, +* and ZSTD_decompressStream. +* If there is functionality that you need, but it doesn't provide, +* please open an issue on our GitHub. ********************************************************************* */ /** @@ -2384,7 +2825,6 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); Start by initializing a context. Use ZSTD_compressBegin(), or ZSTD_compressBegin_usingDict() for dictionary compression. - It's also possible to duplicate a reference context which has already been initialized, using ZSTD_copyCCtx() Then, consume your input using ZSTD_compressContinue(). There are some important considerations to keep in mind when using this advanced function : @@ -2406,18 +2846,28 @@ ZSTDLIB_STATIC_API size_t ZSTD_resetDStream(ZSTD_DStream* zds); */ /*===== Buffer-less streaming compression functions =====*/ +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") ZSTDLIB_STATIC_API size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); /**< note: fails if cdict==NULL */ -ZSTDLIB_STATIC_API size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ +ZSTD_DEPRECATED("This function will likely be removed in a future release. It is misleading and has very limited utility.") +ZSTDLIB_STATIC_API +size_t ZSTD_copyCCtx(ZSTD_CCtx* cctx, const ZSTD_CCtx* preparedCCtx, unsigned long long pledgedSrcSize); /**< note: if pledgedSrcSize is not known, use ZSTD_CONTENTSIZE_UNKNOWN */ + +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") ZSTDLIB_STATIC_API size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The buffer-less API is deprecated in favor of the normal streaming API. See docs.") ZSTDLIB_STATIC_API size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); /* The ZSTD_compressBegin_advanced() and ZSTD_compressBegin_usingCDict_advanced() are now DEPRECATED and will generate a compiler warning */ ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, ZSTD_parameters params, unsigned long long pledgedSrcSize); /**< pledgedSrcSize : If srcSize is not known at init time, use ZSTD_CONTENTSIZE_UNKNOWN */ ZSTD_DEPRECATED("use advanced API to access custom parameters") +ZSTDLIB_STATIC_API size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_CDict* const cdict, ZSTD_frameParameters const fParams, unsigned long long const pledgedSrcSize); /* compression parameters are already set within cdict. pledgedSrcSize must be correct. If srcSize is not known, use macro ZSTD_CONTENTSIZE_UNKNOWN */ /** Buffer-less streaming decompression (synchronous mode) @@ -2430,8 +2880,8 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_ Frame header is extracted from the beginning of compressed frame, so providing only the frame's beginning is enough. Data fragment must be large enough to ensure successful decoding. `ZSTD_frameHeaderSize_max` bytes is guaranteed to always be large enough. - @result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. - >0 : `srcSize` is too small, please provide at least @result bytes on next attempt. + result : 0 : successful decoding, the `ZSTD_frameHeader` structure is correctly filled. + >0 : `srcSize` is too small, please provide at least result bytes on next attempt. errorCode, which can be tested using ZSTD_isError(). It fills a ZSTD_frameHeader structure with important information to correctly decode the frame, @@ -2450,7 +2900,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_ The most memory efficient way is to use a round buffer of sufficient size. Sufficient size is determined by invoking ZSTD_decodingBufferSize_min(), - which can @return an error code if required value is too large for current system (in 32-bits mode). + which can return an error code if required value is too large for current system (in 32-bits mode). In a round buffer methodology, ZSTD_decompressContinue() decompresses each block next to previous one, up to the moment there is not enough room left in the buffer to guarantee decoding another full block, which maximum size is provided in `ZSTD_frameHeader` structure, field `blockSizeMax`. @@ -2470,7 +2920,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_ ZSTD_nextSrcSizeToDecompress() tells how many bytes to provide as 'srcSize' to ZSTD_decompressContinue(). ZSTD_decompressContinue() requires this _exact_ amount of bytes, or it will fail. - @result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). + result of ZSTD_decompressContinue() is the number of bytes regenerated within 'dst' (necessarily <= dstCapacity). It can be zero : it just means ZSTD_decompressContinue() has decoded some metadata item. It can also be an error code, which can be tested with ZSTD_isError(). @@ -2493,27 +2943,7 @@ size_t ZSTD_compressBegin_usingCDict_advanced(ZSTD_CCtx* const cctx, const ZSTD_ */ /*===== Buffer-less streaming decompression functions =====*/ -typedef enum { ZSTD_frame, ZSTD_skippableFrame } ZSTD_frameType_e; -typedef struct { - unsigned long long frameContentSize; /* if == ZSTD_CONTENTSIZE_UNKNOWN, it means this field is not available. 0 means "empty" */ - unsigned long long windowSize; /* can be very large, up to <= frameContentSize */ - unsigned blockSizeMax; - ZSTD_frameType_e frameType; /* if == ZSTD_skippableFrame, frameContentSize is the size of skippable content */ - unsigned headerSize; - unsigned dictID; - unsigned checksumFlag; -} ZSTD_frameHeader; -/*! ZSTD_getFrameHeader() : - * decode Frame Header, or requires larger `srcSize`. - * @return : 0, `zfhPtr` is correctly filled, - * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ -ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize); /**< doesn't consume input */ -/*! ZSTD_getFrameHeader_advanced() : - * same as ZSTD_getFrameHeader(), - * with added capability to select a format (like ZSTD_f_zstd1_magicless) */ -ZSTDLIB_STATIC_API size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format); ZSTDLIB_STATIC_API size_t ZSTD_decodingBufferSize_min(unsigned long long windowSize, unsigned long long frameContentSize); /**< when frame content size is not known, pass in frameContentSize == ZSTD_CONTENTSIZE_UNKNOWN */ ZSTDLIB_STATIC_API size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx); @@ -2524,6 +2954,7 @@ ZSTDLIB_STATIC_API size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx); ZSTDLIB_STATIC_API size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); /* misc */ +ZSTD_DEPRECATED("This function will likely be removed in the next minor release. It is misleading and has very limited utility.") ZSTDLIB_STATIC_API void ZSTD_copyDCtx(ZSTD_DCtx* dctx, const ZSTD_DCtx* preparedDCtx); typedef enum { ZSTDnit_frameHeader, ZSTDnit_blockHeader, ZSTDnit_block, ZSTDnit_lastBlock, ZSTDnit_checksum, ZSTDnit_skippableFrame } ZSTD_nextInputType_e; ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); @@ -2531,11 +2962,23 @@ ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); -/* ============================ */ -/** Block level API */ -/* ============================ */ +/* ========================================= */ +/** Block level API (DEPRECATED) */ +/* ========================================= */ /*! + + This API is deprecated in favor of the regular compression API. + You can get the frame header down to 2 bytes by setting: + - ZSTD_c_format = ZSTD_f_zstd1_magicless + - ZSTD_c_contentSizeFlag = 0 + - ZSTD_c_checksumFlag = 0 + - ZSTD_c_dictIDFlag = 0 + + This API is not as well tested as our normal API, so we recommend not using it. + We will be removing it in a future version. If the normal API doesn't provide + the functionality you need, please open a GitHub issue. + Block functions produce and decode raw zstd blocks, without frame metadata. Frame metadata cost is typically ~12 bytes, which can be non-negligible for very small blocks (< 100 bytes). But users will have to take in charge needed metadata to regenerate data, such as compressed and content sizes. @@ -2546,7 +2989,6 @@ ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); - It is necessary to init context before starting + compression : any ZSTD_compressBegin*() variant, including with dictionary + decompression : any ZSTD_decompressBegin*() variant, including with dictionary - + copyCCtx() and copyDCtx() can be used too - Block size is limited, it must be <= ZSTD_getBlockSize() <= ZSTD_BLOCKSIZE_MAX == 128 KB + If input is larger than a block size, it's necessary to split input data into multiple blocks + For inputs larger than a single block, consider using regular ZSTD_compress() instead. @@ -2563,12 +3005,15 @@ ZSTDLIB_STATIC_API ZSTD_nextInputType_e ZSTD_nextInputType(ZSTD_DCtx* dctx); */ /*===== Raw zstd block functions =====*/ +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") ZSTDLIB_STATIC_API size_t ZSTD_getBlockSize (const ZSTD_CCtx* cctx); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") ZSTDLIB_STATIC_API size_t ZSTD_compressBlock (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") ZSTDLIB_STATIC_API size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); +ZSTD_DEPRECATED("The block API is deprecated in favor of the normal compression API. See docs.") ZSTDLIB_STATIC_API size_t ZSTD_insertBlock (ZSTD_DCtx* dctx, const void* blockStart, size_t blockSize); /**< insert uncompressed block into `dctx` history. Useful for multi-blocks decompression. */ - #endif /* ZSTD_H_ZSTD_STATIC_LINKING_ONLY */ #if defined (__cplusplus) diff --git a/vendor/github.com/DataDog/zstd/zstd_common.c b/vendor/github.com/DataDog/zstd/zstd_common.c index a5f380d..656c46c 100644 --- a/vendor/github.com/DataDog/zstd/zstd_common.c +++ b/vendor/github.com/DataDog/zstd/zstd_common.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -15,7 +15,6 @@ * Dependencies ***************************************/ #define ZSTD_DEPS_NEED_MALLOC -#include "zstd_deps.h" /* ZSTD_malloc, ZSTD_calloc, ZSTD_free, ZSTD_memset */ #include "error_private.h" #include "zstd_internal.h" @@ -49,38 +48,4 @@ ZSTD_ErrorCode ZSTD_getErrorCode(size_t code) { return ERR_getErrorCode(code); } * provides error code string from enum */ const char* ZSTD_getErrorString(ZSTD_ErrorCode code) { return ERR_getErrorString(code); } - - -/*=************************************************************** -* Custom allocator -****************************************************************/ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem) -{ - if (customMem.customAlloc) - return customMem.customAlloc(customMem.opaque, size); - return ZSTD_malloc(size); -} - -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem) -{ - if (customMem.customAlloc) { - /* calloc implemented as malloc+memset; - * not as efficient as calloc, but next best guess for custom malloc */ - void* const ptr = customMem.customAlloc(customMem.opaque, size); - ZSTD_memset(ptr, 0, size); - return ptr; - } - return ZSTD_calloc(1, size); -} - -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem) -{ - if (ptr!=NULL) { - if (customMem.customFree) - customMem.customFree(customMem.opaque, ptr); - else - ZSTD_free(ptr); - } -} - #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/zstd_compress.c b/vendor/github.com/DataDog/zstd/zstd_compress.c index 3cca764..728514a 100644 --- a/vendor/github.com/DataDog/zstd/zstd_compress.c +++ b/vendor/github.com/DataDog/zstd/zstd_compress.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -12,12 +12,12 @@ /*-************************************* * Dependencies ***************************************/ +#include "allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ #include "zstd_deps.h" /* INT_MAX, ZSTD_memset, ZSTD_memcpy */ #include "mem.h" #include "hist.h" /* HIST_countFast_wksp */ #define FSE_STATIC_LINKING_ONLY /* FSE_encodeSymbol */ #include "fse.h" -#define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_compress_internal.h" #include "zstd_compress_sequences.h" @@ -28,6 +28,7 @@ #include "zstd_opt.h" #include "zstd_ldm.h" #include "zstd_compress_superblock.h" +#include "bits.h" /* ZSTD_highbit32, ZSTD_rotateRight_U64 */ /* *************************************************************** * Tuning parameters @@ -59,14 +60,17 @@ * Helper functions ***************************************/ /* ZSTD_compressBound() - * Note that the result from this function is only compatible with the "normal" - * full-block strategy. - * When there are a lot of small blocks due to frequent flush in streaming mode - * the overhead of headers can make the compressed data to be larger than the - * return value of ZSTD_compressBound(). + * Note that the result from this function is only valid for + * the one-pass compression functions. + * When employing the streaming mode, + * if flushes are frequently altering the size of blocks, + * the overhead from block headers can make the compressed data larger + * than the return value of ZSTD_compressBound(). */ size_t ZSTD_compressBound(size_t srcSize) { - return ZSTD_COMPRESSBOUND(srcSize); + size_t const r = ZSTD_COMPRESSBOUND(srcSize); + if (r==0) return ERROR(srcSize_wrong); + return r; } @@ -178,12 +182,9 @@ size_t ZSTD_freeCCtx(ZSTD_CCtx* cctx) if (cctx==NULL) return 0; /* support free on NULL */ RETURN_ERROR_IF(cctx->staticSize, memory_allocation, "not compatible with static CCtx"); - { - int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); + { int cctxInWorkspace = ZSTD_cwksp_owns_buffer(&cctx->workspace, cctx); ZSTD_freeCCtxContent(cctx); - if (!cctxInWorkspace) { - ZSTD_customFree(cctx, cctx->customMem); - } + if (!cctxInWorkspace) ZSTD_customFree(cctx, cctx->customMem); } return 0; } @@ -268,9 +269,9 @@ static int ZSTD_allocateChainTable(const ZSTD_strategy strategy, return forDDSDict || ((strategy != ZSTD_fast) && !ZSTD_rowMatchFinderUsed(strategy, useRowMatchFinder)); } -/* Returns 1 if compression parameters are such that we should +/* Returns ZSTD_ps_enable if compression parameters are such that we should * enable long distance matching (wlog >= 27, strategy >= btopt). - * Returns 0 otherwise. + * Returns ZSTD_ps_disable otherwise. */ static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode, const ZSTD_compressionParameters* const cParams) { @@ -278,6 +279,34 @@ static ZSTD_paramSwitch_e ZSTD_resolveEnableLdm(ZSTD_paramSwitch_e mode, return (cParams->strategy >= ZSTD_btopt && cParams->windowLog >= 27) ? ZSTD_ps_enable : ZSTD_ps_disable; } +static int ZSTD_resolveExternalSequenceValidation(int mode) { + return mode; +} + +/* Resolves maxBlockSize to the default if no value is present. */ +static size_t ZSTD_resolveMaxBlockSize(size_t maxBlockSize) { + if (maxBlockSize == 0) { + return ZSTD_BLOCKSIZE_MAX; + } else { + return maxBlockSize; + } +} + +static ZSTD_paramSwitch_e ZSTD_resolveExternalRepcodeSearch(ZSTD_paramSwitch_e value, int cLevel) { + if (value != ZSTD_ps_auto) return value; + if (cLevel < 10) { + return ZSTD_ps_disable; + } else { + return ZSTD_ps_enable; + } +} + +/* Returns 1 if compression parameters are such that CDict hashtable and chaintable indices are tagged. + * If so, the tags need to be removed in ZSTD_resetCCtx_byCopyingCDict. */ +static int ZSTD_CDictIndicesAreTagged(const ZSTD_compressionParameters* const cParams) { + return cParams->strategy == ZSTD_fast || cParams->strategy == ZSTD_dfast; +} + static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( ZSTD_compressionParameters cParams) { @@ -295,6 +324,10 @@ static ZSTD_CCtx_params ZSTD_makeCCtxParamsFromCParams( } cctxParams.useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams.useBlockSplitter, &cParams); cctxParams.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams.useRowMatchFinder, &cParams); + cctxParams.validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams.validateSequences); + cctxParams.maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams.maxBlockSize); + cctxParams.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams.searchForExternalRepcodes, + cctxParams.compressionLevel); assert(!ZSTD_checkCParams(cParams)); return cctxParams; } @@ -340,10 +373,13 @@ size_t ZSTD_CCtxParams_init(ZSTD_CCtx_params* cctxParams, int compressionLevel) #define ZSTD_NO_CLEVEL 0 /** - * Initializes the cctxParams from params and compressionLevel. + * Initializes `cctxParams` from `params` and `compressionLevel`. * @param compressionLevel If params are derived from a compression level then that compression level, otherwise ZSTD_NO_CLEVEL. */ -static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_parameters const* params, int compressionLevel) +static void +ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, + const ZSTD_parameters* params, + int compressionLevel) { assert(!ZSTD_checkCParams(params->cParams)); ZSTD_memset(cctxParams, 0, sizeof(*cctxParams)); @@ -356,6 +392,9 @@ static void ZSTD_CCtxParams_init_internal(ZSTD_CCtx_params* cctxParams, ZSTD_par cctxParams->useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(cctxParams->useRowMatchFinder, ¶ms->cParams); cctxParams->useBlockSplitter = ZSTD_resolveBlockSplitterMode(cctxParams->useBlockSplitter, ¶ms->cParams); cctxParams->ldmParams.enableLdm = ZSTD_resolveEnableLdm(cctxParams->ldmParams.enableLdm, ¶ms->cParams); + cctxParams->validateSequences = ZSTD_resolveExternalSequenceValidation(cctxParams->validateSequences); + cctxParams->maxBlockSize = ZSTD_resolveMaxBlockSize(cctxParams->maxBlockSize); + cctxParams->searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(cctxParams->searchForExternalRepcodes, compressionLevel); DEBUGLOG(4, "ZSTD_CCtxParams_init_internal: useRowMatchFinder=%d, useBlockSplitter=%d ldm=%d", cctxParams->useRowMatchFinder, cctxParams->useBlockSplitter, cctxParams->ldmParams.enableLdm); } @@ -370,7 +409,7 @@ size_t ZSTD_CCtxParams_init_advanced(ZSTD_CCtx_params* cctxParams, ZSTD_paramete /** * Sets cctxParams' cParams and fParams from params, but otherwise leaves them alone. - * @param param Validated zstd parameters. + * @param params Validated zstd parameters. */ static void ZSTD_CCtxParams_setZstdParams( ZSTD_CCtx_params* cctxParams, const ZSTD_parameters* params) @@ -479,8 +518,8 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) return bounds; case ZSTD_c_enableLongDistanceMatching: - bounds.lowerBound = 0; - bounds.upperBound = 1; + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; return bounds; case ZSTD_c_ldmHashLog: @@ -573,6 +612,26 @@ ZSTD_bounds ZSTD_cParam_getBounds(ZSTD_cParameter param) bounds.upperBound = 1; return bounds; + case ZSTD_c_prefetchCDictTables: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + + case ZSTD_c_enableSeqProducerFallback: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + + case ZSTD_c_maxBlockSize: + bounds.lowerBound = ZSTD_BLOCKSIZE_MAX_MIN; + bounds.upperBound = ZSTD_BLOCKSIZE_MAX; + return bounds; + + case ZSTD_c_searchForExternalRepcodes: + bounds.lowerBound = (int)ZSTD_ps_auto; + bounds.upperBound = (int)ZSTD_ps_disable; + return bounds; + default: bounds.error = ERROR(parameter_unsupported); return bounds; @@ -637,6 +696,10 @@ static int ZSTD_isUpdateAuthorized(ZSTD_cParameter param) case ZSTD_c_useBlockSplitter: case ZSTD_c_useRowMatchFinder: case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_searchForExternalRepcodes: default: return 0; } @@ -649,7 +712,7 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) if (ZSTD_isUpdateAuthorized(param)) { cctx->cParamsChanged = 1; } else { - RETURN_ERROR(stage_wrong, "can only set params in ctx init stage"); + RETURN_ERROR(stage_wrong, "can only set params in cctx init stage"); } } switch(param) @@ -692,6 +755,10 @@ size_t ZSTD_CCtx_setParameter(ZSTD_CCtx* cctx, ZSTD_cParameter param, int value) case ZSTD_c_useBlockSplitter: case ZSTD_c_useRowMatchFinder: case ZSTD_c_deterministicRefPrefix: + case ZSTD_c_prefetchCDictTables: + case ZSTD_c_enableSeqProducerFallback: + case ZSTD_c_maxBlockSize: + case ZSTD_c_searchForExternalRepcodes: break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); @@ -747,12 +814,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, case ZSTD_c_minMatch : if (value!=0) /* 0 => use default */ BOUNDCHECK(ZSTD_c_minMatch, value); - CCtxParams->cParams.minMatch = value; + CCtxParams->cParams.minMatch = (U32)value; return CCtxParams->cParams.minMatch; case ZSTD_c_targetLength : BOUNDCHECK(ZSTD_c_targetLength, value); - CCtxParams->cParams.targetLength = value; + CCtxParams->cParams.targetLength = (U32)value; return CCtxParams->cParams.targetLength; case ZSTD_c_strategy : @@ -765,12 +832,12 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, /* Content size written in frame header _when known_ (default:1) */ DEBUGLOG(4, "set content size flag = %u", (value!=0)); CCtxParams->fParams.contentSizeFlag = value != 0; - return CCtxParams->fParams.contentSizeFlag; + return (size_t)CCtxParams->fParams.contentSizeFlag; case ZSTD_c_checksumFlag : /* A 32-bits content checksum will be calculated and written at end of frame (default:0) */ CCtxParams->fParams.checksumFlag = value != 0; - return CCtxParams->fParams.checksumFlag; + return (size_t)CCtxParams->fParams.checksumFlag; case ZSTD_c_dictIDFlag : /* When applicable, dictionary's dictID is provided in frame header (default:1) */ DEBUGLOG(4, "set dictIDFlag = %u", (value!=0)); @@ -779,18 +846,18 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, case ZSTD_c_forceMaxWindow : CCtxParams->forceWindow = (value != 0); - return CCtxParams->forceWindow; + return (size_t)CCtxParams->forceWindow; case ZSTD_c_forceAttachDict : { const ZSTD_dictAttachPref_e pref = (ZSTD_dictAttachPref_e)value; - BOUNDCHECK(ZSTD_c_forceAttachDict, pref); + BOUNDCHECK(ZSTD_c_forceAttachDict, (int)pref); CCtxParams->attachDictPref = pref; return CCtxParams->attachDictPref; } case ZSTD_c_literalCompressionMode : { const ZSTD_paramSwitch_e lcm = (ZSTD_paramSwitch_e)value; - BOUNDCHECK(ZSTD_c_literalCompressionMode, lcm); + BOUNDCHECK(ZSTD_c_literalCompressionMode, (int)lcm); CCtxParams->literalCompressionMode = lcm; return CCtxParams->literalCompressionMode; } @@ -841,47 +908,48 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, case ZSTD_c_enableDedicatedDictSearch : CCtxParams->enableDedicatedDictSearch = (value!=0); - return CCtxParams->enableDedicatedDictSearch; + return (size_t)CCtxParams->enableDedicatedDictSearch; case ZSTD_c_enableLongDistanceMatching : + BOUNDCHECK(ZSTD_c_enableLongDistanceMatching, value); CCtxParams->ldmParams.enableLdm = (ZSTD_paramSwitch_e)value; return CCtxParams->ldmParams.enableLdm; case ZSTD_c_ldmHashLog : if (value!=0) /* 0 ==> auto */ BOUNDCHECK(ZSTD_c_ldmHashLog, value); - CCtxParams->ldmParams.hashLog = value; + CCtxParams->ldmParams.hashLog = (U32)value; return CCtxParams->ldmParams.hashLog; case ZSTD_c_ldmMinMatch : if (value!=0) /* 0 ==> default */ BOUNDCHECK(ZSTD_c_ldmMinMatch, value); - CCtxParams->ldmParams.minMatchLength = value; + CCtxParams->ldmParams.minMatchLength = (U32)value; return CCtxParams->ldmParams.minMatchLength; case ZSTD_c_ldmBucketSizeLog : if (value!=0) /* 0 ==> default */ BOUNDCHECK(ZSTD_c_ldmBucketSizeLog, value); - CCtxParams->ldmParams.bucketSizeLog = value; + CCtxParams->ldmParams.bucketSizeLog = (U32)value; return CCtxParams->ldmParams.bucketSizeLog; case ZSTD_c_ldmHashRateLog : if (value!=0) /* 0 ==> default */ BOUNDCHECK(ZSTD_c_ldmHashRateLog, value); - CCtxParams->ldmParams.hashRateLog = value; + CCtxParams->ldmParams.hashRateLog = (U32)value; return CCtxParams->ldmParams.hashRateLog; case ZSTD_c_targetCBlockSize : if (value!=0) /* 0 ==> default */ BOUNDCHECK(ZSTD_c_targetCBlockSize, value); - CCtxParams->targetCBlockSize = value; + CCtxParams->targetCBlockSize = (U32)value; return CCtxParams->targetCBlockSize; case ZSTD_c_srcSizeHint : if (value!=0) /* 0 ==> default */ BOUNDCHECK(ZSTD_c_srcSizeHint, value); CCtxParams->srcSizeHint = value; - return CCtxParams->srcSizeHint; + return (size_t)CCtxParams->srcSizeHint; case ZSTD_c_stableInBuffer: BOUNDCHECK(ZSTD_c_stableInBuffer, value); @@ -918,6 +986,27 @@ size_t ZSTD_CCtxParams_setParameter(ZSTD_CCtx_params* CCtxParams, CCtxParams->deterministicRefPrefix = !!value; return CCtxParams->deterministicRefPrefix; + case ZSTD_c_prefetchCDictTables: + BOUNDCHECK(ZSTD_c_prefetchCDictTables, value); + CCtxParams->prefetchCDictTables = (ZSTD_paramSwitch_e)value; + return CCtxParams->prefetchCDictTables; + + case ZSTD_c_enableSeqProducerFallback: + BOUNDCHECK(ZSTD_c_enableSeqProducerFallback, value); + CCtxParams->enableMatchFinderFallback = value; + return CCtxParams->enableMatchFinderFallback; + + case ZSTD_c_maxBlockSize: + if (value!=0) /* 0 ==> default */ + BOUNDCHECK(ZSTD_c_maxBlockSize, value); + CCtxParams->maxBlockSize = value; + return CCtxParams->maxBlockSize; + + case ZSTD_c_searchForExternalRepcodes: + BOUNDCHECK(ZSTD_c_searchForExternalRepcodes, value); + CCtxParams->searchForExternalRepcodes = (ZSTD_paramSwitch_e)value; + return CCtxParams->searchForExternalRepcodes; + default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } } @@ -1050,6 +1139,18 @@ size_t ZSTD_CCtxParams_getParameter( case ZSTD_c_deterministicRefPrefix: *value = (int)CCtxParams->deterministicRefPrefix; break; + case ZSTD_c_prefetchCDictTables: + *value = (int)CCtxParams->prefetchCDictTables; + break; + case ZSTD_c_enableSeqProducerFallback: + *value = CCtxParams->enableMatchFinderFallback; + break; + case ZSTD_c_maxBlockSize: + *value = (int)CCtxParams->maxBlockSize; + break; + case ZSTD_c_searchForExternalRepcodes: + *value = (int)CCtxParams->searchForExternalRepcodes; + break; default: RETURN_ERROR(parameter_unsupported, "unknown parameter"); } return 0; @@ -1076,9 +1177,47 @@ size_t ZSTD_CCtx_setParametersUsingCCtxParams( return 0; } +size_t ZSTD_CCtx_setCParams(ZSTD_CCtx* cctx, ZSTD_compressionParameters cparams) +{ + ZSTD_STATIC_ASSERT(sizeof(cparams) == 7 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setCParams"); + /* only update if all parameters are valid */ + FORWARD_IF_ERROR(ZSTD_checkCParams(cparams), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_windowLog, cparams.windowLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_chainLog, cparams.chainLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_hashLog, cparams.hashLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_searchLog, cparams.searchLog), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_minMatch, cparams.minMatch), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_targetLength, cparams.targetLength), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_strategy, cparams.strategy), ""); + return 0; +} + +size_t ZSTD_CCtx_setFParams(ZSTD_CCtx* cctx, ZSTD_frameParameters fparams) +{ + ZSTD_STATIC_ASSERT(sizeof(fparams) == 3 * 4 /* all params are listed below */); + DEBUGLOG(4, "ZSTD_CCtx_setFParams"); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_contentSizeFlag, fparams.contentSizeFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_checksumFlag, fparams.checksumFlag != 0), ""); + FORWARD_IF_ERROR(ZSTD_CCtx_setParameter(cctx, ZSTD_c_dictIDFlag, fparams.noDictIDFlag == 0), ""); + return 0; +} + +size_t ZSTD_CCtx_setParams(ZSTD_CCtx* cctx, ZSTD_parameters params) +{ + DEBUGLOG(4, "ZSTD_CCtx_setParams"); + /* First check cParams, because we want to update all or none. */ + FORWARD_IF_ERROR(ZSTD_checkCParams(params.cParams), ""); + /* Next set fParams, because this could fail if the cctx isn't in init stage. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setFParams(cctx, params.fParams), ""); + /* Finally set cParams, which should succeed. */ + FORWARD_IF_ERROR(ZSTD_CCtx_setCParams(cctx, params.cParams), ""); + return 0; +} + size_t ZSTD_CCtx_setPledgedSrcSize(ZSTD_CCtx* cctx, unsigned long long pledgedSrcSize) { - DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %u bytes", (U32)pledgedSrcSize); + DEBUGLOG(4, "ZSTD_CCtx_setPledgedSrcSize to %llu bytes", pledgedSrcSize); RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, "Can't set pledgedSrcSize when not in init stage."); cctx->pledgedSrcSizePlusOne = pledgedSrcSize+1; @@ -1094,9 +1233,9 @@ static void ZSTD_dedicatedDictSearch_revertCParams( ZSTD_compressionParameters* cParams); /** - * Initializes the local dict using the requested parameters. - * NOTE: This does not use the pledged src size, because it may be used for more - * than one compression. + * Initializes the local dictionary using requested parameters. + * NOTE: Initialization does not employ the pledged src size, + * because the dictionary may be used for multiple compressions. */ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) { @@ -1109,8 +1248,8 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) return 0; } if (dl->cdict != NULL) { - assert(cctx->cdict == dl->cdict); /* Local dictionary already initialized. */ + assert(cctx->cdict == dl->cdict); return 0; } assert(dl->dictSize > 0); @@ -1130,26 +1269,30 @@ static size_t ZSTD_initLocalDict(ZSTD_CCtx* cctx) } size_t ZSTD_CCtx_loadDictionary_advanced( - ZSTD_CCtx* cctx, const void* dict, size_t dictSize, - ZSTD_dictLoadMethod_e dictLoadMethod, ZSTD_dictContentType_e dictContentType) + ZSTD_CCtx* cctx, + const void* dict, size_t dictSize, + ZSTD_dictLoadMethod_e dictLoadMethod, + ZSTD_dictContentType_e dictContentType) { - RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, - "Can't load a dictionary when ctx is not in init stage."); DEBUGLOG(4, "ZSTD_CCtx_loadDictionary_advanced (size: %u)", (U32)dictSize); - ZSTD_clearAllDicts(cctx); /* in case one already exists */ - if (dict == NULL || dictSize == 0) /* no dictionary mode */ + RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, + "Can't load a dictionary when cctx is not in init stage."); + ZSTD_clearAllDicts(cctx); /* erase any previously set dictionary */ + if (dict == NULL || dictSize == 0) /* no dictionary */ return 0; if (dictLoadMethod == ZSTD_dlm_byRef) { cctx->localDict.dict = dict; } else { + /* copy dictionary content inside CCtx to own its lifetime */ void* dictBuffer; RETURN_ERROR_IF(cctx->staticSize, memory_allocation, - "no malloc for static CCtx"); + "static CCtx can't allocate for an internal copy of dictionary"); dictBuffer = ZSTD_customMalloc(dictSize, cctx->customMem); - RETURN_ERROR_IF(!dictBuffer, memory_allocation, "NULL pointer!"); + RETURN_ERROR_IF(dictBuffer==NULL, memory_allocation, + "allocation failed for dictionary content"); ZSTD_memcpy(dictBuffer, dict, dictSize); - cctx->localDict.dictBuffer = dictBuffer; - cctx->localDict.dict = dictBuffer; + cctx->localDict.dictBuffer = dictBuffer; /* owned ptr to free */ + cctx->localDict.dict = dictBuffer; /* read-only reference */ } cctx->localDict.dictSize = dictSize; cctx->localDict.dictContentType = dictContentType; @@ -1219,8 +1362,9 @@ size_t ZSTD_CCtx_reset(ZSTD_CCtx* cctx, ZSTD_ResetDirective reset) if ( (reset == ZSTD_reset_parameters) || (reset == ZSTD_reset_session_and_parameters) ) { RETURN_ERROR_IF(cctx->streamStage != zcss_init, stage_wrong, - "Can't reset parameters only when not in init stage."); + "Reset parameters is only possible during init stage."); ZSTD_clearAllDicts(cctx); + ZSTD_memset(&cctx->externalMatchCtx, 0, sizeof(cctx->externalMatchCtx)); return ZSTD_CCtxParams_reset(&cctx->requestedParams); } return 0; @@ -1317,7 +1461,8 @@ static ZSTD_compressionParameters ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, unsigned long long srcSize, size_t dictSize, - ZSTD_cParamMode_e mode) + ZSTD_cParamMode_e mode, + ZSTD_paramSwitch_e useRowMatchFinder) { const U64 minSrcSize = 513; /* (1<<9) + 1 */ const U64 maxWindowResize = 1ULL << (ZSTD_WINDOWLOG_MAX-1); @@ -1351,8 +1496,8 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, } /* resize windowLog if input is small enough, to use less memory */ - if ( (srcSize < maxWindowResize) - && (dictSize < maxWindowResize) ) { + if ( (srcSize <= maxWindowResize) + && (dictSize <= maxWindowResize) ) { U32 const tSize = (U32)(srcSize + dictSize); static U32 const hashSizeMin = 1 << ZSTD_HASHLOG_MIN; U32 const srcLog = (tSize < hashSizeMin) ? ZSTD_HASHLOG_MIN : @@ -1370,6 +1515,42 @@ ZSTD_adjustCParams_internal(ZSTD_compressionParameters cPar, if (cPar.windowLog < ZSTD_WINDOWLOG_ABSOLUTEMIN) cPar.windowLog = ZSTD_WINDOWLOG_ABSOLUTEMIN; /* minimum wlog required for valid frame header */ + /* We can't use more than 32 bits of hash in total, so that means that we require: + * (hashLog + 8) <= 32 && (chainLog + 8) <= 32 + */ + if (mode == ZSTD_cpm_createCDict && ZSTD_CDictIndicesAreTagged(&cPar)) { + U32 const maxShortCacheHashLog = 32 - ZSTD_SHORT_CACHE_TAG_BITS; + if (cPar.hashLog > maxShortCacheHashLog) { + cPar.hashLog = maxShortCacheHashLog; + } + if (cPar.chainLog > maxShortCacheHashLog) { + cPar.chainLog = maxShortCacheHashLog; + } + } + + + /* At this point, we aren't 100% sure if we are using the row match finder. + * Unless it is explicitly disabled, conservatively assume that it is enabled. + * In this case it will only be disabled for small sources, so shrinking the + * hash log a little bit shouldn't result in any ratio loss. + */ + if (useRowMatchFinder == ZSTD_ps_auto) + useRowMatchFinder = ZSTD_ps_enable; + + /* We can't hash more than 32-bits in total. So that means that we require: + * (hashLog - rowLog + 8) <= 32 + */ + if (ZSTD_rowMatchFinderUsed(cPar.strategy, useRowMatchFinder)) { + /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cPar.searchLog, 6); + U32 const maxRowHashLog = 32 - ZSTD_ROW_HASH_TAG_BITS; + U32 const maxHashLog = maxRowHashLog + rowLog; + assert(cPar.hashLog >= rowLog); + if (cPar.hashLog > maxHashLog) { + cPar.hashLog = maxHashLog; + } + } + return cPar; } @@ -1380,7 +1561,7 @@ ZSTD_adjustCParams(ZSTD_compressionParameters cPar, { cPar = ZSTD_clampCParams(cPar); /* resulting cPar is necessarily valid (all parameters within range) */ if (srcSize == 0) srcSize = ZSTD_CONTENTSIZE_UNKNOWN; - return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown); + return ZSTD_adjustCParams_internal(cPar, srcSize, dictSize, ZSTD_cpm_unknown, ZSTD_ps_auto); } static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, unsigned long long srcSizeHint, size_t dictSize, ZSTD_cParamMode_e mode); @@ -1411,7 +1592,7 @@ ZSTD_compressionParameters ZSTD_getCParamsFromCCtxParams( ZSTD_overrideCParams(&cParams, &CCtxParams->cParams); assert(!ZSTD_checkCParams(cParams)); /* srcSizeHint == 0 means 0 */ - return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode); + return ZSTD_adjustCParams_internal(cParams, srcSizeHint, dictSize, mode, CCtxParams->useRowMatchFinder); } static size_t @@ -1440,7 +1621,7 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_match_t)) + ZSTD_cwksp_aligned_alloc_size((ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); size_t const lazyAdditionalSpace = ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder) - ? ZSTD_cwksp_aligned_alloc_size(hSize*sizeof(U16)) + ? ZSTD_cwksp_aligned_alloc_size(hSize) : 0; size_t const optSpace = (forCCtx && (cParams->strategy >= ZSTD_btopt)) ? optPotentialSpace @@ -1456,6 +1637,13 @@ ZSTD_sizeof_matchState(const ZSTD_compressionParameters* const cParams, return tableSpace + optSpace + slackSpace + lazyAdditionalSpace; } +/* Helper function for calculating memory requirements. + * Gives a tighter bound than ZSTD_sequenceBound() by taking minMatch into account. */ +static size_t ZSTD_maxNbSeq(size_t blockSize, unsigned minMatch, int useSequenceProducer) { + U32 const divider = (minMatch==3 || useSequenceProducer) ? 3 : 4; + return blockSize / divider; +} + static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( const ZSTD_compressionParameters* cParams, const ldmParams_t* ldmParams, @@ -1463,12 +1651,13 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( const ZSTD_paramSwitch_e useRowMatchFinder, const size_t buffInSize, const size_t buffOutSize, - const U64 pledgedSrcSize) + const U64 pledgedSrcSize, + int useSequenceProducer, + size_t maxBlockSize) { size_t const windowSize = (size_t) BOUNDED(1ULL, 1ULL << cParams->windowLog, pledgedSrcSize); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - U32 const divider = (cParams->minMatch==3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(maxBlockSize), windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, cParams->minMatch, useSequenceProducer); size_t const tokenSpace = ZSTD_cwksp_alloc_size(WILDCOPY_OVERLENGTH + blockSize) + ZSTD_cwksp_aligned_alloc_size(maxNbSeq * sizeof(seqDef)) + 3 * ZSTD_cwksp_alloc_size(maxNbSeq * sizeof(BYTE)); @@ -1487,6 +1676,11 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( size_t const cctxSpace = isStatic ? ZSTD_cwksp_alloc_size(sizeof(ZSTD_CCtx)) : 0; + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + size_t const externalSeqSpace = useSequenceProducer + ? ZSTD_cwksp_aligned_alloc_size(maxNbExternalSeq * sizeof(ZSTD_Sequence)) + : 0; + size_t const neededSpace = cctxSpace + entropySpace + @@ -1495,7 +1689,8 @@ static size_t ZSTD_estimateCCtxSize_usingCCtxParams_internal( ldmSeqSpace + matchStateSize + tokenSpace + - bufferSpace; + bufferSpace + + externalSeqSpace; DEBUGLOG(5, "estimate workspace : %u", (U32)neededSpace); return neededSpace; @@ -1513,7 +1708,7 @@ size_t ZSTD_estimateCCtxSize_usingCCtxParams(const ZSTD_CCtx_params* params) * be needed. However, we still allocate two 0-sized buffers, which can * take space under ASAN. */ return ZSTD_estimateCCtxSize_usingCCtxParams_internal( - &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN); + &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, 0, 0, ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize); } size_t ZSTD_estimateCCtxSize_usingCParams(ZSTD_compressionParameters cParams) @@ -1563,7 +1758,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) RETURN_ERROR_IF(params->nbWorkers > 0, GENERIC, "Estimate CCtx size is supported for single-threaded compression only."); { ZSTD_compressionParameters const cParams = ZSTD_getCParamsFromCCtxParams(params, ZSTD_CONTENTSIZE_UNKNOWN, 0, ZSTD_cpm_noAttachDict); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, (size_t)1 << cParams.windowLog); + size_t const blockSize = MIN(ZSTD_resolveMaxBlockSize(params->maxBlockSize), (size_t)1 << cParams.windowLog); size_t const inBuffSize = (params->inBufferMode == ZSTD_bm_buffered) ? ((size_t)1 << cParams.windowLog) + blockSize : 0; @@ -1574,7 +1769,7 @@ size_t ZSTD_estimateCStreamSize_usingCCtxParams(const ZSTD_CCtx_params* params) return ZSTD_estimateCCtxSize_usingCCtxParams_internal( &cParams, ¶ms->ldmParams, 1, useRowMatchFinder, inBuffSize, outBuffSize, - ZSTD_CONTENTSIZE_UNKNOWN); + ZSTD_CONTENTSIZE_UNKNOWN, params->useSequenceProducer, params->maxBlockSize); } } @@ -1717,6 +1912,19 @@ typedef enum { ZSTD_resetTarget_CCtx } ZSTD_resetTarget_e; +/* Mixes bits in a 64 bits in a value, based on XXH3_rrmxmx */ +static U64 ZSTD_bitmix(U64 val, U64 len) { + val ^= ZSTD_rotateRight_U64(val, 49) ^ ZSTD_rotateRight_U64(val, 24); + val *= 0x9FB21C651E98DF25ULL; + val ^= (val >> 35) + len ; + val *= 0x9FB21C651E98DF25ULL; + return val ^ (val >> 28); +} + +/* Mixes in the hashSalt and hashSaltEntropy to create a new hashSalt */ +static void ZSTD_advanceHashSalt(ZSTD_matchState_t* ms) { + ms->hashSalt = ZSTD_bitmix(ms->hashSalt, 8) ^ ZSTD_bitmix((U64) ms->hashSaltEntropy, 4); +} static size_t ZSTD_reset_matchState(ZSTD_matchState_t* ms, @@ -1744,6 +1952,7 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, } ms->hashLog3 = hashLog3; + ms->lazySkipping = 0; ZSTD_invalidateMatchState(ms); @@ -1765,6 +1974,27 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ZSTD_cwksp_clean_tables(ws); } + if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { + /* Row match finder needs an additional table of hashes ("tags") */ + size_t const tagTableSize = hSize; + /* We want to generate a new salt in case we reset a Cctx, but we always want to use + * 0 when we reset a Cdict */ + if(forWho == ZSTD_resetTarget_CCtx) { + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned_init_once(ws, tagTableSize); + ZSTD_advanceHashSalt(ms); + } else { + /* When we are not salting we want to always memset the memory */ + ms->tagTable = (BYTE*) ZSTD_cwksp_reserve_aligned(ws, tagTableSize); + ZSTD_memset(ms->tagTable, 0, tagTableSize); + ms->hashSalt = 0; + } + { /* Switch to 32-entry rows if searchLog is 5 (or more) */ + U32 const rowLog = BOUNDED(4, cParams->searchLog, 6); + assert(cParams->hashLog >= rowLog); + ms->rowHashLog = cParams->hashLog - rowLog; + } + } + /* opt parser space */ if ((forWho == ZSTD_resetTarget_CCtx) && (cParams->strategy >= ZSTD_btopt)) { DEBUGLOG(4, "reserving optimal parser space"); @@ -1776,19 +2006,6 @@ ZSTD_reset_matchState(ZSTD_matchState_t* ms, ms->opt.priceTable = (ZSTD_optimal_t*)ZSTD_cwksp_reserve_aligned(ws, (ZSTD_OPT_NUM+1) * sizeof(ZSTD_optimal_t)); } - if (ZSTD_rowMatchFinderUsed(cParams->strategy, useRowMatchFinder)) { - { /* Row match finder needs an additional table of hashes ("tags") */ - size_t const tagTableSize = hSize*sizeof(U16); - ms->tagTable = (U16*)ZSTD_cwksp_reserve_aligned(ws, tagTableSize); - if (ms->tagTable) ZSTD_memset(ms->tagTable, 0, tagTableSize); - } - { /* Switch to 32-entry rows if searchLog is 5 (or more) */ - U32 const rowLog = BOUNDED(4, cParams->searchLog, 6); - assert(cParams->hashLog >= rowLog); - ms->rowHashLog = cParams->hashLog - rowLog; - } - } - ms->cParams = *cParams; RETURN_ERROR_IF(ZSTD_cwksp_reserve_failed(ws), memory_allocation, @@ -1848,6 +2065,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, assert(params->useRowMatchFinder != ZSTD_ps_auto); assert(params->useBlockSplitter != ZSTD_ps_auto); assert(params->ldmParams.enableLdm != ZSTD_ps_auto); + assert(params->maxBlockSize != 0); if (params->ldmParams.enableLdm == ZSTD_ps_enable) { /* Adjust long distance matching parameters */ ZSTD_ldm_adjustParameters(&zc->appliedParams.ldmParams, ¶ms->cParams); @@ -1856,9 +2074,8 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, } { size_t const windowSize = MAX(1, (size_t)MIN(((U64)1 << params->cParams.windowLog), pledgedSrcSize)); - size_t const blockSize = MIN(ZSTD_BLOCKSIZE_MAX, windowSize); - U32 const divider = (params->cParams.minMatch==3) ? 3 : 4; - size_t const maxNbSeq = blockSize / divider; + size_t const blockSize = MIN(params->maxBlockSize, windowSize); + size_t const maxNbSeq = ZSTD_maxNbSeq(blockSize, params->cParams.minMatch, params->useSequenceProducer); size_t const buffOutSize = (zbuff == ZSTDb_buffered && params->outBufferMode == ZSTD_bm_buffered) ? ZSTD_compressBound(blockSize) + 1 : 0; @@ -1875,7 +2092,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, size_t const neededSpace = ZSTD_estimateCCtxSize_usingCCtxParams_internal( ¶ms->cParams, ¶ms->ldmParams, zc->staticSize != 0, params->useRowMatchFinder, - buffInSize, buffOutSize, pledgedSrcSize); + buffInSize, buffOutSize, pledgedSrcSize, params->useSequenceProducer, params->maxBlockSize); int resizeWorkspace; FORWARD_IF_ERROR(neededSpace, "cctx size estimate failed!"); @@ -1918,6 +2135,7 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, /* init params */ zc->blockState.matchState.cParams = params->cParams; + zc->blockState.matchState.prefetchCDictTables = params->prefetchCDictTables == ZSTD_ps_enable; zc->pledgedSrcSizePlusOne = pledgedSrcSize+1; zc->consumedSrcSize = 0; zc->producedCSize = 0; @@ -1934,13 +2152,46 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, ZSTD_reset_compressedBlockState(zc->blockState.prevCBlock); + FORWARD_IF_ERROR(ZSTD_reset_matchState( + &zc->blockState.matchState, + ws, + ¶ms->cParams, + params->useRowMatchFinder, + crp, + needsIndexReset, + ZSTD_resetTarget_CCtx), ""); + + zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); + + /* ldm hash table */ + if (params->ldmParams.enableLdm == ZSTD_ps_enable) { + /* TODO: avoid memset? */ + size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; + zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); + ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); + zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); + zc->maxNbLdmSequences = maxNbLdmSeq; + + ZSTD_window_init(&zc->ldmState.window); + zc->ldmState.loadedDictEnd = 0; + } + + /* reserve space for block-level external sequences */ + if (params->useSequenceProducer) { + size_t const maxNbExternalSeq = ZSTD_sequenceBound(blockSize); + zc->externalMatchCtx.seqBufferCapacity = maxNbExternalSeq; + zc->externalMatchCtx.seqBuffer = + (ZSTD_Sequence*)ZSTD_cwksp_reserve_aligned(ws, maxNbExternalSeq * sizeof(ZSTD_Sequence)); + } + + /* buffers */ + /* ZSTD_wildcopy() is used to copy into the literals buffer, * so we have to oversize the buffer by WILDCOPY_OVERLENGTH bytes. */ zc->seqStore.litStart = ZSTD_cwksp_reserve_buffer(ws, blockSize + WILDCOPY_OVERLENGTH); zc->seqStore.maxNbLit = blockSize; - /* buffers */ zc->bufferedPolicy = zbuff; zc->inBuffSize = buffInSize; zc->inBuff = (char*)ZSTD_cwksp_reserve_buffer(ws, buffInSize); @@ -1963,32 +2214,9 @@ static size_t ZSTD_resetCCtx_internal(ZSTD_CCtx* zc, zc->seqStore.llCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); zc->seqStore.mlCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); zc->seqStore.ofCode = ZSTD_cwksp_reserve_buffer(ws, maxNbSeq * sizeof(BYTE)); - zc->seqStore.sequencesStart = (seqDef*)ZSTD_cwksp_reserve_aligned(ws, maxNbSeq * sizeof(seqDef)); - - FORWARD_IF_ERROR(ZSTD_reset_matchState( - &zc->blockState.matchState, - ws, - ¶ms->cParams, - params->useRowMatchFinder, - crp, - needsIndexReset, - ZSTD_resetTarget_CCtx), ""); - - /* ldm hash table */ - if (params->ldmParams.enableLdm == ZSTD_ps_enable) { - /* TODO: avoid memset? */ - size_t const ldmHSize = ((size_t)1) << params->ldmParams.hashLog; - zc->ldmState.hashTable = (ldmEntry_t*)ZSTD_cwksp_reserve_aligned(ws, ldmHSize * sizeof(ldmEntry_t)); - ZSTD_memset(zc->ldmState.hashTable, 0, ldmHSize * sizeof(ldmEntry_t)); - zc->ldmSequences = (rawSeq*)ZSTD_cwksp_reserve_aligned(ws, maxNbLdmSeq * sizeof(rawSeq)); - zc->maxNbLdmSequences = maxNbLdmSeq; - - ZSTD_window_init(&zc->ldmState.window); - zc->ldmState.loadedDictEnd = 0; - } DEBUGLOG(3, "wksp: finished allocating, %zd bytes remain available", ZSTD_cwksp_available_space(ws)); - assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace, resizeWorkspace)); + assert(ZSTD_cwksp_estimated_space_within_bounds(ws, neededSpace)); zc->initialized = 1; @@ -2060,7 +2288,8 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, } params.cParams = ZSTD_adjustCParams_internal(adjusted_cdict_cParams, pledgedSrcSize, - cdict->dictContentSize, ZSTD_cpm_attachDict); + cdict->dictContentSize, ZSTD_cpm_attachDict, + params.useRowMatchFinder); params.cParams.windowLog = windowLog; params.useRowMatchFinder = cdict->useRowMatchFinder; /* cdict overrides */ FORWARD_IF_ERROR(ZSTD_resetCCtx_internal(cctx, ¶ms, pledgedSrcSize, @@ -2099,6 +2328,22 @@ ZSTD_resetCCtx_byAttachingCDict(ZSTD_CCtx* cctx, return 0; } +static void ZSTD_copyCDictTableIntoCCtx(U32* dst, U32 const* src, size_t tableSize, + ZSTD_compressionParameters const* cParams) { + if (ZSTD_CDictIndicesAreTagged(cParams)){ + /* Remove tags from the CDict table if they are present. + * See docs on "short cache" in zstd_compress_internal.h for context. */ + size_t i; + for (i = 0; i < tableSize; i++) { + U32 const taggedIndex = src[i]; + U32 const index = taggedIndex >> ZSTD_SHORT_CACHE_TAG_BITS; + dst[i] = index; + } + } else { + ZSTD_memcpy(dst, src, tableSize * sizeof(U32)); + } +} + static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, ZSTD_CCtx_params params, @@ -2134,21 +2379,23 @@ static size_t ZSTD_resetCCtx_byCopyingCDict(ZSTD_CCtx* cctx, : 0; size_t const hSize = (size_t)1 << cdict_cParams->hashLog; - ZSTD_memcpy(cctx->blockState.matchState.hashTable, - cdict->matchState.hashTable, - hSize * sizeof(U32)); + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.hashTable, + cdict->matchState.hashTable, + hSize, cdict_cParams); + /* Do not copy cdict's chainTable if cctx has parameters such that it would not use chainTable */ if (ZSTD_allocateChainTable(cctx->appliedParams.cParams.strategy, cctx->appliedParams.useRowMatchFinder, 0 /* forDDSDict */)) { - ZSTD_memcpy(cctx->blockState.matchState.chainTable, - cdict->matchState.chainTable, - chainSize * sizeof(U32)); + ZSTD_copyCDictTableIntoCCtx(cctx->blockState.matchState.chainTable, + cdict->matchState.chainTable, + chainSize, cdict_cParams); } /* copy tag table */ if (ZSTD_rowMatchFinderUsed(cdict_cParams->strategy, cdict->useRowMatchFinder)) { - size_t const tagTableSize = hSize*sizeof(U16); + size_t const tagTableSize = hSize; ZSTD_memcpy(cctx->blockState.matchState.tagTable, - cdict->matchState.tagTable, - tagTableSize); + cdict->matchState.tagTable, + tagTableSize); + cctx->blockState.matchState.hashSalt = cdict->matchState.hashSalt; } } @@ -2227,6 +2474,7 @@ static size_t ZSTD_copyCCtx_internal(ZSTD_CCtx* dstCCtx, params.useBlockSplitter = srcCCtx->appliedParams.useBlockSplitter; params.ldmParams = srcCCtx->appliedParams.ldmParams; params.fParams = fParams; + params.maxBlockSize = srcCCtx->appliedParams.maxBlockSize; ZSTD_resetCCtx_internal(dstCCtx, ¶ms, pledgedSrcSize, /* loadedDictSize */ 0, ZSTDcrp_leaveDirty, zbuff); @@ -2386,7 +2634,7 @@ static void ZSTD_reduceIndex (ZSTD_matchState_t* ms, ZSTD_CCtx_params const* par /* See doc/zstd_compression_format.md for detailed format description */ -void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr) { const seqDef* const sequences = seqStorePtr->sequencesStart; BYTE* const llCodeTable = seqStorePtr->llCode; @@ -2394,18 +2642,24 @@ void ZSTD_seqToCodes(const seqStore_t* seqStorePtr) BYTE* const mlCodeTable = seqStorePtr->mlCode; U32 const nbSeq = (U32)(seqStorePtr->sequences - seqStorePtr->sequencesStart); U32 u; + int longOffsets = 0; assert(nbSeq <= seqStorePtr->maxNbSeq); for (u=0; u= STREAM_ACCUMULATOR_MIN)); + if (MEM_32bits() && ofCode >= STREAM_ACCUMULATOR_MIN) + longOffsets = 1; } if (seqStorePtr->longLengthType==ZSTD_llt_literalLength) llCodeTable[seqStorePtr->longLengthPos] = MaxLL; if (seqStorePtr->longLengthType==ZSTD_llt_matchLength) mlCodeTable[seqStorePtr->longLengthPos] = MaxML; + return longOffsets; } /* ZSTD_useTargetCBlockSize(): @@ -2439,6 +2693,7 @@ typedef struct { U32 MLtype; size_t size; size_t lastCountSize; /* Accounts for bug in 1.3.4. More detail in ZSTD_entropyCompressSeqStore_internal() */ + int longOffsets; } ZSTD_symbolEncodingTypeStats_t; /* ZSTD_buildSequencesStatistics(): @@ -2449,11 +2704,13 @@ typedef struct { * entropyWkspSize must be of size at least ENTROPY_WORKSPACE_SIZE - (MaxSeq + 1)*sizeof(U32) */ static ZSTD_symbolEncodingTypeStats_t -ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, - const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, - BYTE* dst, const BYTE* const dstEnd, - ZSTD_strategy strategy, unsigned* countWorkspace, - void* entropyWorkspace, size_t entropyWkspSize) { +ZSTD_buildSequencesStatistics( + const seqStore_t* seqStorePtr, size_t nbSeq, + const ZSTD_fseCTables_t* prevEntropy, ZSTD_fseCTables_t* nextEntropy, + BYTE* dst, const BYTE* const dstEnd, + ZSTD_strategy strategy, unsigned* countWorkspace, + void* entropyWorkspace, size_t entropyWkspSize) +{ BYTE* const ostart = dst; const BYTE* const oend = dstEnd; BYTE* op = ostart; @@ -2467,7 +2724,7 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, stats.lastCountSize = 0; /* convert length/distances into codes */ - ZSTD_seqToCodes(seqStorePtr); + stats.longOffsets = ZSTD_seqToCodes(seqStorePtr); assert(op <= oend); assert(nbSeq != 0); /* ZSTD_selectEncodingType() divides by nbSeq */ /* build CTable for Literal Lengths */ @@ -2572,22 +2829,22 @@ ZSTD_buildSequencesStatistics(seqStore_t* seqStorePtr, size_t nbSeq, */ #define SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO 20 MEM_STATIC size_t -ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - void* entropyWorkspace, size_t entropyWkspSize, - const int bmi2) +ZSTD_entropyCompressSeqStore_internal( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + void* entropyWorkspace, size_t entropyWkspSize, + const int bmi2) { - const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; ZSTD_strategy const strategy = cctxParams->cParams.strategy; unsigned* count = (unsigned*)entropyWorkspace; FSE_CTable* CTable_LitLength = nextEntropy->fse.litlengthCTable; FSE_CTable* CTable_OffsetBits = nextEntropy->fse.offcodeCTable; FSE_CTable* CTable_MatchLength = nextEntropy->fse.matchlengthCTable; const seqDef* const sequences = seqStorePtr->sequencesStart; - const size_t nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + const size_t nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); const BYTE* const ofCodeTable = seqStorePtr->ofCode; const BYTE* const llCodeTable = seqStorePtr->llCode; const BYTE* const mlCodeTable = seqStorePtr->mlCode; @@ -2595,29 +2852,31 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, BYTE* const oend = ostart + dstCapacity; BYTE* op = ostart; size_t lastCountSize; + int longOffsets = 0; entropyWorkspace = count + (MaxSeq + 1); entropyWkspSize -= (MaxSeq + 1) * sizeof(*count); - DEBUGLOG(4, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu)", nbSeq); + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore_internal (nbSeq=%zu, dstCapacity=%zu)", nbSeq, dstCapacity); ZSTD_STATIC_ASSERT(HUF_WORKSPACE_SIZE >= (1<= HUF_WORKSPACE_SIZE); /* Compress literals */ { const BYTE* const literals = seqStorePtr->litStart; - size_t const numSequences = seqStorePtr->sequences - seqStorePtr->sequencesStart; - size_t const numLiterals = seqStorePtr->lit - seqStorePtr->litStart; + size_t const numSequences = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); + size_t const numLiterals = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); /* Base suspicion of uncompressibility on ratio of literals to sequences */ unsigned const suspectUncompressible = (numSequences == 0) || (numLiterals / numSequences >= SUSPECT_UNCOMPRESSIBLE_LITERAL_RATIO); size_t const litSize = (size_t)(seqStorePtr->lit - literals); + size_t const cSize = ZSTD_compressLiterals( - &prevEntropy->huf, &nextEntropy->huf, - cctxParams->cParams.strategy, - ZSTD_literalsCompressionIsDisabled(cctxParams), op, dstCapacity, literals, litSize, entropyWorkspace, entropyWkspSize, - bmi2, suspectUncompressible); + &prevEntropy->huf, &nextEntropy->huf, + cctxParams->cParams.strategy, + ZSTD_literalsCompressionIsDisabled(cctxParams), + suspectUncompressible, bmi2); FORWARD_IF_ERROR(cSize, "ZSTD_compressLiterals failed"); assert(cSize <= dstCapacity); op += cSize; @@ -2643,11 +2902,10 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, ZSTD_memcpy(&nextEntropy->fse, &prevEntropy->fse, sizeof(prevEntropy->fse)); return (size_t)(op - ostart); } - { - ZSTD_symbolEncodingTypeStats_t stats; - BYTE* seqHead = op++; + { BYTE* const seqHead = op++; /* build stats for sequences */ - stats = ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, + const ZSTD_symbolEncodingTypeStats_t stats = + ZSTD_buildSequencesStatistics(seqStorePtr, nbSeq, &prevEntropy->fse, &nextEntropy->fse, op, oend, strategy, count, @@ -2656,6 +2914,7 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, *seqHead = (BYTE)((stats.LLtype<<6) + (stats.Offtype<<4) + (stats.MLtype<<2)); lastCountSize = stats.lastCountSize; op += stats.size; + longOffsets = stats.longOffsets; } { size_t const bitstreamSize = ZSTD_encodeSequences( @@ -2690,14 +2949,15 @@ ZSTD_entropyCompressSeqStore_internal(seqStore_t* seqStorePtr, } MEM_STATIC size_t -ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - size_t srcSize, - void* entropyWorkspace, size_t entropyWkspSize, - int bmi2) +ZSTD_entropyCompressSeqStore( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + size_t srcSize, + void* entropyWorkspace, size_t entropyWkspSize, + int bmi2) { size_t const cSize = ZSTD_entropyCompressSeqStore_internal( seqStorePtr, prevEntropy, nextEntropy, cctxParams, @@ -2707,15 +2967,21 @@ ZSTD_entropyCompressSeqStore(seqStore_t* seqStorePtr, /* When srcSize <= dstCapacity, there is enough space to write a raw uncompressed block. * Since we ran out of space, block must be not compressible, so fall back to raw uncompressed block. */ - if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) + if ((cSize == ERROR(dstSize_tooSmall)) & (srcSize <= dstCapacity)) { + DEBUGLOG(4, "not enough dstCapacity (%zu) for ZSTD_entropyCompressSeqStore_internal()=> do not compress block", dstCapacity); return 0; /* block not compressed */ + } FORWARD_IF_ERROR(cSize, "ZSTD_entropyCompressSeqStore_internal failed"); /* Check compressibility */ { size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, cctxParams->cParams.strategy); if (cSize >= maxCSize) return 0; /* block not compressed */ } - DEBUGLOG(4, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); + DEBUGLOG(5, "ZSTD_entropyCompressSeqStore() cSize: %zu", cSize); + /* libzstd decoder before > v1.5.4 is not compatible with compressed blocks of size ZSTD_BLOCKSIZE_MAX exactly. + * This restriction is indirectly already fulfilled by respecting ZSTD_minGain() condition above. + */ + assert(cSize < ZSTD_BLOCKSIZE_MAX); return cSize; } @@ -2810,6 +3076,72 @@ void ZSTD_resetSeqStore(seqStore_t* ssPtr) ssPtr->longLengthType = ZSTD_llt_none; } +/* ZSTD_postProcessSequenceProducerResult() : + * Validates and post-processes sequences obtained through the external matchfinder API: + * - Checks whether nbExternalSeqs represents an error condition. + * - Appends a block delimiter to outSeqs if one is not already present. + * See zstd.h for context regarding block delimiters. + * Returns the number of sequences after post-processing, or an error code. */ +static size_t ZSTD_postProcessSequenceProducerResult( + ZSTD_Sequence* outSeqs, size_t nbExternalSeqs, size_t outSeqsCapacity, size_t srcSize +) { + RETURN_ERROR_IF( + nbExternalSeqs > outSeqsCapacity, + sequenceProducer_failed, + "External sequence producer returned error code %lu", + (unsigned long)nbExternalSeqs + ); + + RETURN_ERROR_IF( + nbExternalSeqs == 0 && srcSize > 0, + sequenceProducer_failed, + "Got zero sequences from external sequence producer for a non-empty src buffer!" + ); + + if (srcSize == 0) { + ZSTD_memset(&outSeqs[0], 0, sizeof(ZSTD_Sequence)); + return 1; + } + + { + ZSTD_Sequence const lastSeq = outSeqs[nbExternalSeqs - 1]; + + /* We can return early if lastSeq is already a block delimiter. */ + if (lastSeq.offset == 0 && lastSeq.matchLength == 0) { + return nbExternalSeqs; + } + + /* This error condition is only possible if the external matchfinder + * produced an invalid parse, by definition of ZSTD_sequenceBound(). */ + RETURN_ERROR_IF( + nbExternalSeqs == outSeqsCapacity, + sequenceProducer_failed, + "nbExternalSeqs == outSeqsCapacity but lastSeq is not a block delimiter!" + ); + + /* lastSeq is not a block delimiter, so we need to append one. */ + ZSTD_memset(&outSeqs[nbExternalSeqs], 0, sizeof(ZSTD_Sequence)); + return nbExternalSeqs + 1; + } +} + +/* ZSTD_fastSequenceLengthSum() : + * Returns sum(litLen) + sum(matchLen) + lastLits for *seqBuf*. + * Similar to another function in zstd_compress.c (determine_blockSize), + * except it doesn't check for a block delimiter to end summation. + * Removing the early exit allows the compiler to auto-vectorize (https://godbolt.org/z/cY1cajz9P). + * This function can be deleted and replaced by determine_blockSize after we resolve issue #3456. */ +static size_t ZSTD_fastSequenceLengthSum(ZSTD_Sequence const* seqBuf, size_t seqBufSize) { + size_t matchLenSum, litLenSum, i; + matchLenSum = 0; + litLenSum = 0; + for (i = 0; i < seqBufSize; i++) { + litLenSum += seqBuf[i].litLength; + matchLenSum += seqBuf[i].matchLength; + } + return litLenSum + matchLenSum; +} + typedef enum { ZSTDbss_compress, ZSTDbss_noCompress } ZSTD_buildSeqStore_e; static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) @@ -2819,7 +3151,9 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) assert(srcSize <= ZSTD_BLOCKSIZE_MAX); /* Assert that we have correctly flushed the ctx params into the ms's copy */ ZSTD_assertEqualCParams(zc->appliedParams.cParams, ms->cParams); - if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (srcSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { if (zc->appliedParams.cParams.strategy >= ZSTD_btopt) { ZSTD_ldm_skipRawSeqStoreBytes(&zc->externSeqStore, srcSize); } else { @@ -2855,6 +3189,15 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) } if (zc->externSeqStore.pos < zc->externSeqStore.size) { assert(zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_disable); + + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + zc->appliedParams.useSequenceProducer, + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); + /* Updates ldmSeqStore.pos */ lastLLSize = ZSTD_ldm_blockCompress(&zc->externSeqStore, @@ -2866,6 +3209,14 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) } else if (zc->appliedParams.ldmParams.enableLdm == ZSTD_ps_enable) { rawSeqStore_t ldmSeqStore = kNullRawSeqStore; + /* External matchfinder + LDM is technically possible, just not implemented yet. + * We need to revisit soon and implement it. */ + RETURN_ERROR_IF( + zc->appliedParams.useSequenceProducer, + parameter_combination_unsupported, + "Long-distance matching with external sequence producer enabled is not currently supported." + ); + ldmSeqStore.seq = zc->ldmSequences; ldmSeqStore.capacity = zc->maxNbLdmSequences; /* Updates ldmSeqStore.size */ @@ -2880,7 +3231,68 @@ static size_t ZSTD_buildSeqStore(ZSTD_CCtx* zc, const void* src, size_t srcSize) zc->appliedParams.useRowMatchFinder, src, srcSize); assert(ldmSeqStore.pos == ldmSeqStore.size); - } else { /* not long range mode */ + } else if (zc->appliedParams.useSequenceProducer) { + assert( + zc->externalMatchCtx.seqBufferCapacity >= ZSTD_sequenceBound(srcSize) + ); + assert(zc->externalMatchCtx.mFinder != NULL); + + { U32 const windowSize = (U32)1 << zc->appliedParams.cParams.windowLog; + + size_t const nbExternalSeqs = (zc->externalMatchCtx.mFinder)( + zc->externalMatchCtx.mState, + zc->externalMatchCtx.seqBuffer, + zc->externalMatchCtx.seqBufferCapacity, + src, srcSize, + NULL, 0, /* dict and dictSize, currently not supported */ + zc->appliedParams.compressionLevel, + windowSize + ); + + size_t const nbPostProcessedSeqs = ZSTD_postProcessSequenceProducerResult( + zc->externalMatchCtx.seqBuffer, + nbExternalSeqs, + zc->externalMatchCtx.seqBufferCapacity, + srcSize + ); + + /* Return early if there is no error, since we don't need to worry about last literals */ + if (!ZSTD_isError(nbPostProcessedSeqs)) { + ZSTD_sequencePosition seqPos = {0,0,0}; + size_t const seqLenSum = ZSTD_fastSequenceLengthSum(zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs); + RETURN_ERROR_IF(seqLenSum > srcSize, externalSequences_invalid, "External sequences imply too large a block!"); + FORWARD_IF_ERROR( + ZSTD_copySequencesToSeqStoreExplicitBlockDelim( + zc, &seqPos, + zc->externalMatchCtx.seqBuffer, nbPostProcessedSeqs, + src, srcSize, + zc->appliedParams.searchForExternalRepcodes + ), + "Failed to copy external sequences to seqStore!" + ); + ms->ldmSeqStore = NULL; + DEBUGLOG(5, "Copied %lu sequences from external sequence producer to internal seqStore.", (unsigned long)nbExternalSeqs); + return ZSTDbss_compress; + } + + /* Propagate the error if fallback is disabled */ + if (!zc->appliedParams.enableMatchFinderFallback) { + return nbPostProcessedSeqs; + } + + /* Fallback to software matchfinder */ + { ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, + zc->appliedParams.useRowMatchFinder, + dictMode); + ms->ldmSeqStore = NULL; + DEBUGLOG( + 5, + "External sequence producer returned error code %lu. Falling back to internal parser.", + (unsigned long)nbExternalSeqs + ); + lastLLSize = blockCompressor(ms, &zc->seqStore, zc->blockState.nextCBlock->rep, src, srcSize); + } } + } else { /* not long range mode and no external matchfinder */ ZSTD_blockCompressor const blockCompressor = ZSTD_selectBlockCompressor(zc->appliedParams.cParams.strategy, zc->appliedParams.useRowMatchFinder, dictMode); @@ -2941,7 +3353,7 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) /* seqStoreSeqs[i].offset == offCode+1, and ZSTD_updateRep() expects offCode so we provide seqStoreSeqs[i].offset - 1 */ ZSTD_updateRep(updatedRepcodes.rep, - seqStoreSeqs[i].offBase - 1, + seqStoreSeqs[i].offBase, seqStoreSeqs[i].litLength == 0); literalsRead += outSeqs[i].litLength; } @@ -2957,6 +3369,10 @@ static void ZSTD_copyBlockSequences(ZSTD_CCtx* zc) zc->seqCollector.seqIndex += seqStoreSeqSize; } +size_t ZSTD_sequenceBound(size_t srcSize) { + return (srcSize / ZSTD_MINMATCH_MIN) + 1; +} + size_t ZSTD_generateSequences(ZSTD_CCtx* zc, ZSTD_Sequence* outSeqs, size_t outSeqsSize, const void* src, size_t srcSize) { @@ -3002,19 +3418,17 @@ static int ZSTD_isRLE(const BYTE* src, size_t length) { const size_t unrollMask = unrollSize - 1; const size_t prefixLength = length & unrollMask; size_t i; - size_t u; if (length == 1) return 1; /* Check if prefix is RLE first before using unrolled loop */ if (prefixLength && ZSTD_count(ip+1, ip, ip+prefixLength) != prefixLength-1) { return 0; } for (i = prefixLength; i != length; i += unrollSize) { + size_t u; for (u = 0; u < unrollSize; u += sizeof(size_t)) { if (MEM_readST(ip + i + u) != valueST) { return 0; - } - } - } + } } } return 1; } @@ -3030,7 +3444,8 @@ static int ZSTD_maybeRLE(seqStore_t const* seqStore) return nbSeqs < 4 && nbLits < 10; } -static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs) +static void +ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* const bs) { ZSTD_compressedBlockState_t* const tmp = bs->prevCBlock; bs->prevCBlock = bs->nextCBlock; @@ -3038,7 +3453,9 @@ static void ZSTD_blockState_confirmRepcodesAndEntropyTables(ZSTD_blockState_t* c } /* Writes the block header */ -static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) { +static void +writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastBlock) +{ U32 const cBlockHeader = cSize == 1 ? lastBlock + (((U32)bt_rle)<<1) + (U32)(blockSize << 3) : lastBlock + (((U32)bt_compressed)<<1) + (U32)(cSize << 3); @@ -3051,13 +3468,16 @@ static void writeBlockHeader(void* op, size_t cSize, size_t blockSize, U32 lastB * Stores literals block type (raw, rle, compressed, repeat) and * huffman description table to hufMetadata. * Requires ENTROPY_WORKSPACE_SIZE workspace - * @return : size of huffman description table or error code */ -static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, - const ZSTD_hufCTables_t* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_hufCTablesMetadata_t* hufMetadata, - const int literalsCompressionIsDisabled, - void* workspace, size_t wkspSize) + * @return : size of huffman description table, or an error code + */ +static size_t +ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_hufCTablesMetadata_t* hufMetadata, + const int literalsCompressionIsDisabled, + void* workspace, size_t wkspSize, + int hufFlags) { BYTE* const wkspStart = (BYTE*)workspace; BYTE* const wkspEnd = wkspStart + wkspSize; @@ -3065,9 +3485,9 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi unsigned* const countWksp = (unsigned*)workspace; const size_t countWkspSize = (HUF_SYMBOLVALUE_MAX + 1) * sizeof(unsigned); BYTE* const nodeWksp = countWkspStart + countWkspSize; - const size_t nodeWkspSize = wkspEnd-nodeWksp; + const size_t nodeWkspSize = (size_t)(wkspEnd - nodeWksp); unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; - unsigned huffLog = HUF_TABLELOG_DEFAULT; + unsigned huffLog = LitHufLog; HUF_repeat repeat = prevHuf->repeatMode; DEBUGLOG(5, "ZSTD_buildBlockEntropyStats_literals (srcSize=%zu)", srcSize); @@ -3082,73 +3502,77 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi /* small ? don't even attempt compression (speed opt) */ #ifndef COMPRESS_LITERALS_SIZE_MIN -#define COMPRESS_LITERALS_SIZE_MIN 63 +# define COMPRESS_LITERALS_SIZE_MIN 63 /* heuristic */ #endif { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; if (srcSize <= minLitSize) { DEBUGLOG(5, "set_basic - too small"); hufMetadata->hType = set_basic; return 0; - } - } + } } /* Scan input and build symbol stats */ - { size_t const largest = HIST_count_wksp (countWksp, &maxSymbolValue, (const BYTE*)src, srcSize, workspace, wkspSize); + { size_t const largest = + HIST_count_wksp (countWksp, &maxSymbolValue, + (const BYTE*)src, srcSize, + workspace, wkspSize); FORWARD_IF_ERROR(largest, "HIST_count_wksp failed"); if (largest == srcSize) { + /* only one literal symbol */ DEBUGLOG(5, "set_rle"); hufMetadata->hType = set_rle; return 0; } if (largest <= (srcSize >> 7)+4) { + /* heuristic: likely not compressible */ DEBUGLOG(5, "set_basic - no gain"); hufMetadata->hType = set_basic; return 0; - } - } + } } /* Validate the previous Huffman table */ - if (repeat == HUF_repeat_check && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { + if (repeat == HUF_repeat_check + && !HUF_validateCTable((HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue)) { repeat = HUF_repeat_none; } /* Build Huffman Tree */ ZSTD_memset(nextHuf->CTable, 0, sizeof(nextHuf->CTable)); - huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue); + huffLog = HUF_optimalTableLog(huffLog, srcSize, maxSymbolValue, nodeWksp, nodeWkspSize, nextHuf->CTable, countWksp, hufFlags); + assert(huffLog <= LitHufLog); { size_t const maxBits = HUF_buildCTable_wksp((HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue, huffLog, nodeWksp, nodeWkspSize); FORWARD_IF_ERROR(maxBits, "HUF_buildCTable_wksp"); huffLog = (U32)maxBits; - { /* Build and write the CTable */ - size_t const newCSize = HUF_estimateCompressedSize( - (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); - size_t const hSize = HUF_writeCTable_wksp( - hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), - (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, - nodeWksp, nodeWkspSize); - /* Check against repeating the previous CTable */ - if (repeat != HUF_repeat_none) { - size_t const oldCSize = HUF_estimateCompressedSize( - (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); - if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { - DEBUGLOG(5, "set_repeat - smaller"); - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - hufMetadata->hType = set_repeat; - return 0; - } - } - if (newCSize + hSize >= srcSize) { - DEBUGLOG(5, "set_basic - no gains"); + } + { /* Build and write the CTable */ + size_t const newCSize = HUF_estimateCompressedSize( + (HUF_CElt*)nextHuf->CTable, countWksp, maxSymbolValue); + size_t const hSize = HUF_writeCTable_wksp( + hufMetadata->hufDesBuffer, sizeof(hufMetadata->hufDesBuffer), + (HUF_CElt*)nextHuf->CTable, maxSymbolValue, huffLog, + nodeWksp, nodeWkspSize); + /* Check against repeating the previous CTable */ + if (repeat != HUF_repeat_none) { + size_t const oldCSize = HUF_estimateCompressedSize( + (HUF_CElt const*)prevHuf->CTable, countWksp, maxSymbolValue); + if (oldCSize < srcSize && (oldCSize <= hSize + newCSize || hSize + 12 >= srcSize)) { + DEBUGLOG(5, "set_repeat - smaller"); ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - hufMetadata->hType = set_basic; + hufMetadata->hType = set_repeat; return 0; - } - DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); - hufMetadata->hType = set_compressed; - nextHuf->repeatMode = HUF_repeat_check; - return hSize; + } } + if (newCSize + hSize >= srcSize) { + DEBUGLOG(5, "set_basic - no gains"); + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + hufMetadata->hType = set_basic; + return 0; } + DEBUGLOG(5, "set_compressed (hSize=%u)", (U32)hSize); + hufMetadata->hType = set_compressed; + nextHuf->repeatMode = HUF_repeat_check; + return hSize; } } @@ -3158,8 +3582,9 @@ static size_t ZSTD_buildBlockEntropyStats_literals(void* const src, size_t srcSi * and updates nextEntropy to the appropriate repeatMode. */ static ZSTD_symbolEncodingTypeStats_t -ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) { - ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0}; +ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) +{ + ZSTD_symbolEncodingTypeStats_t stats = {set_basic, set_basic, set_basic, 0, 0, 0}; nextEntropy->litlength_repeatMode = FSE_repeat_none; nextEntropy->offcode_repeatMode = FSE_repeat_none; nextEntropy->matchlength_repeatMode = FSE_repeat_none; @@ -3170,16 +3595,18 @@ ZSTD_buildDummySequencesStatistics(ZSTD_fseCTables_t* nextEntropy) { * Builds entropy for the sequences. * Stores symbol compression modes and fse table to fseMetadata. * Requires ENTROPY_WORKSPACE_SIZE wksp. - * @return : size of fse tables or error code */ -static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr, - const ZSTD_fseCTables_t* prevEntropy, - ZSTD_fseCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - ZSTD_fseCTablesMetadata_t* fseMetadata, - void* workspace, size_t wkspSize) + * @return : size of fse tables or error code */ +static size_t +ZSTD_buildBlockEntropyStats_sequences( + const seqStore_t* seqStorePtr, + const ZSTD_fseCTables_t* prevEntropy, + ZSTD_fseCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize) { ZSTD_strategy const strategy = cctxParams->cParams.strategy; - size_t const nbSeq = seqStorePtr->sequences - seqStorePtr->sequencesStart; + size_t const nbSeq = (size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart); BYTE* const ostart = fseMetadata->fseTablesBuffer; BYTE* const oend = ostart + sizeof(fseMetadata->fseTablesBuffer); BYTE* op = ostart; @@ -3206,23 +3633,28 @@ static size_t ZSTD_buildBlockEntropyStats_sequences(seqStore_t* seqStorePtr, /** ZSTD_buildBlockEntropyStats() : * Builds entropy for the block. * Requires workspace size ENTROPY_WORKSPACE_SIZE - * - * @return : 0 on success or error code + * @return : 0 on success, or an error code + * Note : also employed in superblock */ -size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - ZSTD_entropyCTablesMetadata_t* entropyMetadata, - void* workspace, size_t wkspSize) +size_t ZSTD_buildBlockEntropyStats( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize) { - size_t const litSize = seqStorePtr->lit - seqStorePtr->litStart; + size_t const litSize = (size_t)(seqStorePtr->lit - seqStorePtr->litStart); + int const huf_useOptDepth = (cctxParams->cParams.strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD); + int const hufFlags = huf_useOptDepth ? HUF_flags_optimalDepth : 0; + entropyMetadata->hufMetadata.hufDesSize = ZSTD_buildBlockEntropyStats_literals(seqStorePtr->litStart, litSize, &prevEntropy->huf, &nextEntropy->huf, &entropyMetadata->hufMetadata, ZSTD_literalsCompressionIsDisabled(cctxParams), - workspace, wkspSize); + workspace, wkspSize, hufFlags); + FORWARD_IF_ERROR(entropyMetadata->hufMetadata.hufDesSize, "ZSTD_buildBlockEntropyStats_literals failed"); entropyMetadata->fseMetadata.fseTablesSize = ZSTD_buildBlockEntropyStats_sequences(seqStorePtr, @@ -3235,11 +3667,12 @@ size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr, } /* Returns the size estimate for the literals section (header + content) of a block */ -static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize, - const ZSTD_hufCTables_t* huf, - const ZSTD_hufCTablesMetadata_t* hufMetadata, - void* workspace, size_t wkspSize, - int writeEntropy) +static size_t +ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSize, + const ZSTD_hufCTables_t* huf, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) { unsigned* const countWksp = (unsigned*)workspace; unsigned maxSymbolValue = HUF_SYMBOLVALUE_MAX; @@ -3261,12 +3694,13 @@ static size_t ZSTD_estimateBlockSize_literal(const BYTE* literals, size_t litSiz } /* Returns the size estimate for the FSE-compressed symbols (of, ml, ll) of a block */ -static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type, - const BYTE* codeTable, size_t nbSeq, unsigned maxCode, - const FSE_CTable* fseCTable, - const U8* additionalBits, - short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, - void* workspace, size_t wkspSize) +static size_t +ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type, + const BYTE* codeTable, size_t nbSeq, unsigned maxCode, + const FSE_CTable* fseCTable, + const U8* additionalBits, + short const* defaultNorm, U32 defaultNormLog, U32 defaultMax, + void* workspace, size_t wkspSize) { unsigned* const countWksp = (unsigned*)workspace; const BYTE* ctp = codeTable; @@ -3298,99 +3732,107 @@ static size_t ZSTD_estimateBlockSize_symbolType(symbolEncodingType_e type, } /* Returns the size estimate for the sequences section (header + content) of a block */ -static size_t ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable, - const BYTE* llCodeTable, - const BYTE* mlCodeTable, - size_t nbSeq, - const ZSTD_fseCTables_t* fseTables, - const ZSTD_fseCTablesMetadata_t* fseMetadata, - void* workspace, size_t wkspSize, - int writeEntropy) +static size_t +ZSTD_estimateBlockSize_sequences(const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + void* workspace, size_t wkspSize, + int writeEntropy) { size_t sequencesSectionHeaderSize = 1 /* seqHead */ + 1 /* min seqSize size */ + (nbSeq >= 128) + (nbSeq >= LONGNBSEQ); size_t cSeqSizeEstimate = 0; cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->ofType, ofCodeTable, nbSeq, MaxOff, - fseTables->offcodeCTable, NULL, - OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, - workspace, wkspSize); + fseTables->offcodeCTable, NULL, + OF_defaultNorm, OF_defaultNormLog, DefaultMaxOff, + workspace, wkspSize); cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->llType, llCodeTable, nbSeq, MaxLL, - fseTables->litlengthCTable, LL_bits, - LL_defaultNorm, LL_defaultNormLog, MaxLL, - workspace, wkspSize); + fseTables->litlengthCTable, LL_bits, + LL_defaultNorm, LL_defaultNormLog, MaxLL, + workspace, wkspSize); cSeqSizeEstimate += ZSTD_estimateBlockSize_symbolType(fseMetadata->mlType, mlCodeTable, nbSeq, MaxML, - fseTables->matchlengthCTable, ML_bits, - ML_defaultNorm, ML_defaultNormLog, MaxML, - workspace, wkspSize); + fseTables->matchlengthCTable, ML_bits, + ML_defaultNorm, ML_defaultNormLog, MaxML, + workspace, wkspSize); if (writeEntropy) cSeqSizeEstimate += fseMetadata->fseTablesSize; return cSeqSizeEstimate + sequencesSectionHeaderSize; } /* Returns the size estimate for a given stream of literals, of, ll, ml */ -static size_t ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize, - const BYTE* ofCodeTable, - const BYTE* llCodeTable, - const BYTE* mlCodeTable, - size_t nbSeq, - const ZSTD_entropyCTables_t* entropy, - const ZSTD_entropyCTablesMetadata_t* entropyMetadata, - void* workspace, size_t wkspSize, - int writeLitEntropy, int writeSeqEntropy) { +static size_t +ZSTD_estimateBlockSize(const BYTE* literals, size_t litSize, + const BYTE* ofCodeTable, + const BYTE* llCodeTable, + const BYTE* mlCodeTable, + size_t nbSeq, + const ZSTD_entropyCTables_t* entropy, + const ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize, + int writeLitEntropy, int writeSeqEntropy) +{ size_t const literalsSize = ZSTD_estimateBlockSize_literal(literals, litSize, - &entropy->huf, &entropyMetadata->hufMetadata, - workspace, wkspSize, writeLitEntropy); + &entropy->huf, &entropyMetadata->hufMetadata, + workspace, wkspSize, writeLitEntropy); size_t const seqSize = ZSTD_estimateBlockSize_sequences(ofCodeTable, llCodeTable, mlCodeTable, - nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, - workspace, wkspSize, writeSeqEntropy); + nbSeq, &entropy->fse, &entropyMetadata->fseMetadata, + workspace, wkspSize, writeSeqEntropy); return seqSize + literalsSize + ZSTD_blockHeaderSize; } /* Builds entropy statistics and uses them for blocksize estimation. * - * Returns the estimated compressed size of the seqStore, or a zstd error. + * @return: estimated compressed size of the seqStore, or a zstd error. */ -static size_t ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc) { - ZSTD_entropyCTablesMetadata_t* entropyMetadata = &zc->blockSplitCtx.entropyMetadata; +static size_t +ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(seqStore_t* seqStore, ZSTD_CCtx* zc) +{ + ZSTD_entropyCTablesMetadata_t* const entropyMetadata = &zc->blockSplitCtx.entropyMetadata; DEBUGLOG(6, "ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize()"); FORWARD_IF_ERROR(ZSTD_buildBlockEntropyStats(seqStore, &zc->blockState.prevCBlock->entropy, &zc->blockState.nextCBlock->entropy, &zc->appliedParams, entropyMetadata, - zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */), ""); - return ZSTD_estimateBlockSize(seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE), ""); + return ZSTD_estimateBlockSize( + seqStore->litStart, (size_t)(seqStore->lit - seqStore->litStart), seqStore->ofCode, seqStore->llCode, seqStore->mlCode, (size_t)(seqStore->sequences - seqStore->sequencesStart), - &zc->blockState.nextCBlock->entropy, entropyMetadata, zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE, + &zc->blockState.nextCBlock->entropy, + entropyMetadata, + zc->entropyWorkspace, ENTROPY_WORKSPACE_SIZE, (int)(entropyMetadata->hufMetadata.hType == set_compressed), 1); } /* Returns literals bytes represented in a seqStore */ -static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) { +static size_t ZSTD_countSeqStoreLiteralsBytes(const seqStore_t* const seqStore) +{ size_t literalsBytes = 0; - size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); size_t i; for (i = 0; i < nbSeqs; ++i) { - seqDef seq = seqStore->sequencesStart[i]; + seqDef const seq = seqStore->sequencesStart[i]; literalsBytes += seq.litLength; if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_literalLength) { literalsBytes += 0x10000; - } - } + } } return literalsBytes; } /* Returns match bytes represented in a seqStore */ -static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) { +static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) +{ size_t matchBytes = 0; - size_t const nbSeqs = seqStore->sequences - seqStore->sequencesStart; + size_t const nbSeqs = (size_t)(seqStore->sequences - seqStore->sequencesStart); size_t i; for (i = 0; i < nbSeqs; ++i) { seqDef seq = seqStore->sequencesStart[i]; matchBytes += seq.mlBase + MINMATCH; if (i == seqStore->longLengthPos && seqStore->longLengthType == ZSTD_llt_matchLength) { matchBytes += 0x10000; - } - } + } } return matchBytes; } @@ -3399,15 +3841,12 @@ static size_t ZSTD_countSeqStoreMatchBytes(const seqStore_t* const seqStore) { */ static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore, const seqStore_t* originalSeqStore, - size_t startIdx, size_t endIdx) { - BYTE* const litEnd = originalSeqStore->lit; - size_t literalsBytes; - size_t literalsBytesPreceding = 0; - + size_t startIdx, size_t endIdx) +{ *resultSeqStore = *originalSeqStore; if (startIdx > 0) { resultSeqStore->sequences = originalSeqStore->sequencesStart + startIdx; - literalsBytesPreceding = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + resultSeqStore->litStart += ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); } /* Move longLengthPos into the correct position if necessary */ @@ -3420,13 +3859,12 @@ static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore, } resultSeqStore->sequencesStart = originalSeqStore->sequencesStart + startIdx; resultSeqStore->sequences = originalSeqStore->sequencesStart + endIdx; - literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); - resultSeqStore->litStart += literalsBytesPreceding; if (endIdx == (size_t)(originalSeqStore->sequences - originalSeqStore->sequencesStart)) { /* This accounts for possible last literals if the derived chunk reaches the end of the block */ - resultSeqStore->lit = litEnd; + assert(resultSeqStore->lit == originalSeqStore->lit); } else { - resultSeqStore->lit = resultSeqStore->litStart+literalsBytes; + size_t const literalsBytes = ZSTD_countSeqStoreLiteralsBytes(resultSeqStore); + resultSeqStore->lit = resultSeqStore->litStart + literalsBytes; } resultSeqStore->llCode += startIdx; resultSeqStore->mlCode += startIdx; @@ -3434,20 +3872,26 @@ static void ZSTD_deriveSeqStoreChunk(seqStore_t* resultSeqStore, } /** - * Returns the raw offset represented by the combination of offCode, ll0, and repcode history. - * offCode must represent a repcode in the numeric representation of ZSTD_storeSeq(). + * Returns the raw offset represented by the combination of offBase, ll0, and repcode history. + * offBase must represent a repcode in the numeric representation of ZSTD_storeSeq(). */ static U32 -ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offCode, const U32 ll0) +ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offBase, const U32 ll0) { - U32 const adjustedOffCode = STORED_REPCODE(offCode) - 1 + ll0; /* [ 0 - 3 ] */ - assert(STORED_IS_REPCODE(offCode)); - if (adjustedOffCode == ZSTD_REP_NUM) { - /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 */ - assert(rep[0] > 0); + U32 const adjustedRepCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; /* [ 0 - 3 ] */ + assert(OFFBASE_IS_REPCODE(offBase)); + if (adjustedRepCode == ZSTD_REP_NUM) { + assert(ll0); + /* litlength == 0 and offCode == 2 implies selection of first repcode - 1 + * This is only valid if it results in a valid offset value, aka > 0. + * Note : it may happen that `rep[0]==1` in exceptional circumstances. + * In which case this function will return 0, which is an invalid offset. + * It's not an issue though, since this value will be + * compared and discarded within ZSTD_seqStore_resolveOffCodes(). + */ return rep[0] - 1; } - return rep[adjustedOffCode]; + return rep[adjustedRepCode]; } /** @@ -3463,30 +3907,33 @@ ZSTD_resolveRepcodeToRawOffset(const U32 rep[ZSTD_REP_NUM], const U32 offCode, c * 1-3 : repcode 1-3 * 4+ : real_offset+3 */ -static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes, - seqStore_t* const seqStore, U32 const nbSeq) { +static void +ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_t* const cRepcodes, + const seqStore_t* const seqStore, U32 const nbSeq) +{ U32 idx = 0; + U32 const longLitLenIdx = seqStore->longLengthType == ZSTD_llt_literalLength ? seqStore->longLengthPos : nbSeq; for (; idx < nbSeq; ++idx) { seqDef* const seq = seqStore->sequencesStart + idx; - U32 const ll0 = (seq->litLength == 0); - U32 const offCode = OFFBASE_TO_STORED(seq->offBase); - assert(seq->offBase > 0); - if (STORED_IS_REPCODE(offCode)) { - U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offCode, ll0); - U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offCode, ll0); + U32 const ll0 = (seq->litLength == 0) && (idx != longLitLenIdx); + U32 const offBase = seq->offBase; + assert(offBase > 0); + if (OFFBASE_IS_REPCODE(offBase)) { + U32 const dRawOffset = ZSTD_resolveRepcodeToRawOffset(dRepcodes->rep, offBase, ll0); + U32 const cRawOffset = ZSTD_resolveRepcodeToRawOffset(cRepcodes->rep, offBase, ll0); /* Adjust simulated decompression repcode history if we come across a mismatch. Replace * the repcode with the offset it actually references, determined by the compression * repcode history. */ if (dRawOffset != cRawOffset) { - seq->offBase = cRawOffset + ZSTD_REP_NUM; + seq->offBase = OFFSET_TO_OFFBASE(cRawOffset); } } /* Compression repcode history is always updated with values directly from the unmodified seqStore. * Decompression repcode history may use modified seq->offset value taken from compression repcode history. */ - ZSTD_updateRep(dRepcodes->rep, OFFBASE_TO_STORED(seq->offBase), ll0); - ZSTD_updateRep(cRepcodes->rep, offCode, ll0); + ZSTD_updateRep(dRepcodes->rep, seq->offBase, ll0); + ZSTD_updateRep(cRepcodes->rep, offBase, ll0); } } @@ -3496,10 +3943,11 @@ static void ZSTD_seqStore_resolveOffCodes(repcodes_t* const dRepcodes, repcodes_ * Returns the total size of that block (including header) or a ZSTD error code. */ static size_t -ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, seqStore_t* const seqStore, +ZSTD_compressSeqStore_singleBlock(ZSTD_CCtx* zc, + const seqStore_t* const seqStore, repcodes_t* const dRep, repcodes_t* const cRep, void* dst, size_t dstCapacity, - const void* src, size_t srcSize, + const void* src, size_t srcSize, U32 lastBlock, U32 isPartition) { const U32 rleMaxLength = 25; @@ -3573,45 +4021,49 @@ typedef struct { /* Helper function to perform the recursive search for block splits. * Estimates the cost of seqStore prior to split, and estimates the cost of splitting the sequences in half. - * If advantageous to split, then we recurse down the two sub-blocks. If not, or if an error occurred in estimation, then - * we do not recurse. + * If advantageous to split, then we recurse down the two sub-blocks. + * If not, or if an error occurred in estimation, then we do not recurse. * - * Note: The recursion depth is capped by a heuristic minimum number of sequences, defined by MIN_SEQUENCES_BLOCK_SPLITTING. + * Note: The recursion depth is capped by a heuristic minimum number of sequences, + * defined by MIN_SEQUENCES_BLOCK_SPLITTING. * In theory, this means the absolute largest recursion depth is 10 == log2(maxNbSeqInBlock/MIN_SEQUENCES_BLOCK_SPLITTING). * In practice, recursion depth usually doesn't go beyond 4. * - * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS. At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize + * Furthermore, the number of splits is capped by ZSTD_MAX_NB_BLOCK_SPLITS. + * At ZSTD_MAX_NB_BLOCK_SPLITS == 196 with the current existing blockSize * maximum of 128 KB, this value is actually impossible to reach. */ static void ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t endIdx, ZSTD_CCtx* zc, const seqStore_t* origSeqStore) { - seqStore_t* fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk; - seqStore_t* firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore; - seqStore_t* secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore; + seqStore_t* const fullSeqStoreChunk = &zc->blockSplitCtx.fullSeqStoreChunk; + seqStore_t* const firstHalfSeqStore = &zc->blockSplitCtx.firstHalfSeqStore; + seqStore_t* const secondHalfSeqStore = &zc->blockSplitCtx.secondHalfSeqStore; size_t estimatedOriginalSize; size_t estimatedFirstHalfSize; size_t estimatedSecondHalfSize; size_t midIdx = (startIdx + endIdx)/2; + DEBUGLOG(5, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx); + assert(endIdx >= startIdx); if (endIdx - startIdx < MIN_SEQUENCES_BLOCK_SPLITTING || splits->idx >= ZSTD_MAX_NB_BLOCK_SPLITS) { - DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences"); + DEBUGLOG(6, "ZSTD_deriveBlockSplitsHelper: Too few sequences (%zu)", endIdx - startIdx); return; } - DEBUGLOG(4, "ZSTD_deriveBlockSplitsHelper: startIdx=%zu endIdx=%zu", startIdx, endIdx); ZSTD_deriveSeqStoreChunk(fullSeqStoreChunk, origSeqStore, startIdx, endIdx); ZSTD_deriveSeqStoreChunk(firstHalfSeqStore, origSeqStore, startIdx, midIdx); ZSTD_deriveSeqStoreChunk(secondHalfSeqStore, origSeqStore, midIdx, endIdx); estimatedOriginalSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(fullSeqStoreChunk, zc); estimatedFirstHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(firstHalfSeqStore, zc); estimatedSecondHalfSize = ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(secondHalfSeqStore, zc); - DEBUGLOG(4, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu", + DEBUGLOG(5, "Estimated original block size: %zu -- First half split: %zu -- Second half split: %zu", estimatedOriginalSize, estimatedFirstHalfSize, estimatedSecondHalfSize); if (ZSTD_isError(estimatedOriginalSize) || ZSTD_isError(estimatedFirstHalfSize) || ZSTD_isError(estimatedSecondHalfSize)) { return; } if (estimatedFirstHalfSize + estimatedSecondHalfSize < estimatedOriginalSize) { + DEBUGLOG(5, "split decided at seqNb:%zu", midIdx); ZSTD_deriveBlockSplitsHelper(splits, startIdx, midIdx, zc, origSeqStore); splits->splitLocations[splits->idx] = (U32)midIdx; splits->idx++; @@ -3619,14 +4071,18 @@ ZSTD_deriveBlockSplitsHelper(seqStoreSplits* splits, size_t startIdx, size_t end } } -/* Base recursive function. Populates a table with intra-block partition indices that can improve compression ratio. +/* Base recursive function. + * Populates a table with intra-block partition indices that can improve compression ratio. * - * Returns the number of splits made (which equals the size of the partition table - 1). + * @return: number of splits made (which equals the size of the partition table - 1). */ -static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) { - seqStoreSplits splits = {partitions, 0}; +static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) +{ + seqStoreSplits splits; + splits.splitLocations = partitions; + splits.idx = 0; if (nbSeq <= 4) { - DEBUGLOG(4, "ZSTD_deriveBlockSplits: Too few sequences to split"); + DEBUGLOG(5, "ZSTD_deriveBlockSplits: Too few sequences to split (%u <= 4)", nbSeq); /* Refuse to try and split anything with less than 4 sequences */ return 0; } @@ -3642,18 +4098,20 @@ static size_t ZSTD_deriveBlockSplits(ZSTD_CCtx* zc, U32 partitions[], U32 nbSeq) * Returns combined size of all blocks (which includes headers), or a ZSTD error code. */ static size_t -ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, - const void* src, size_t blockSize, U32 lastBlock, U32 nbSeq) +ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, + void* dst, size_t dstCapacity, + const void* src, size_t blockSize, + U32 lastBlock, U32 nbSeq) { size_t cSize = 0; const BYTE* ip = (const BYTE*)src; BYTE* op = (BYTE*)dst; size_t i = 0; size_t srcBytesTotal = 0; - U32* partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */ - seqStore_t* nextSeqStore = &zc->blockSplitCtx.nextSeqStore; - seqStore_t* currSeqStore = &zc->blockSplitCtx.currSeqStore; - size_t numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); + U32* const partitions = zc->blockSplitCtx.partitions; /* size == ZSTD_MAX_NB_BLOCK_SPLITS */ + seqStore_t* const nextSeqStore = &zc->blockSplitCtx.nextSeqStore; + seqStore_t* const currSeqStore = &zc->blockSplitCtx.currSeqStore; + size_t const numSplits = ZSTD_deriveBlockSplits(zc, partitions, nbSeq); /* If a block is split and some partitions are emitted as RLE/uncompressed, then repcode history * may become invalid. In order to reconcile potentially invalid repcodes, we keep track of two @@ -3675,30 +4133,31 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac ZSTD_memcpy(cRep.rep, zc->blockState.prevCBlock->rep, sizeof(repcodes_t)); ZSTD_memset(nextSeqStore, 0, sizeof(seqStore_t)); - DEBUGLOG(4, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", + DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal (dstCapacity=%u, dictLimit=%u, nextToUpdate=%u)", (unsigned)dstCapacity, (unsigned)zc->blockState.matchState.window.dictLimit, (unsigned)zc->blockState.matchState.nextToUpdate); if (numSplits == 0) { - size_t cSizeSingleBlock = ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, - &dRep, &cRep, - op, dstCapacity, - ip, blockSize, - lastBlock, 0 /* isPartition */); + size_t cSizeSingleBlock = + ZSTD_compressSeqStore_singleBlock(zc, &zc->seqStore, + &dRep, &cRep, + op, dstCapacity, + ip, blockSize, + lastBlock, 0 /* isPartition */); FORWARD_IF_ERROR(cSizeSingleBlock, "Compressing single block from splitBlock_internal() failed!"); DEBUGLOG(5, "ZSTD_compressBlock_splitBlock_internal: No splits"); - assert(cSizeSingleBlock <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize); + assert(zc->blockSize <= ZSTD_BLOCKSIZE_MAX); + assert(cSizeSingleBlock <= zc->blockSize + ZSTD_blockHeaderSize); return cSizeSingleBlock; } ZSTD_deriveSeqStoreChunk(currSeqStore, &zc->seqStore, 0, partitions[0]); for (i = 0; i <= numSplits; ++i) { - size_t srcBytes; size_t cSizeChunk; U32 const lastPartition = (i == numSplits); U32 lastBlockEntireSrc = 0; - srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore); + size_t srcBytes = ZSTD_countSeqStoreLiteralsBytes(currSeqStore) + ZSTD_countSeqStoreMatchBytes(currSeqStore); srcBytesTotal += srcBytes; if (lastPartition) { /* This is the final partition, need to account for possible last literals */ @@ -3713,7 +4172,8 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac op, dstCapacity, ip, srcBytes, lastBlockEntireSrc, 1 /* isPartition */); - DEBUGLOG(5, "Estimated size: %zu actual size: %zu", ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk); + DEBUGLOG(5, "Estimated size: %zu vs %zu : actual size", + ZSTD_buildEntropyStatisticsAndEstimateSubBlockSize(currSeqStore, zc), cSizeChunk); FORWARD_IF_ERROR(cSizeChunk, "Compressing chunk failed!"); ip += srcBytes; @@ -3721,10 +4181,10 @@ ZSTD_compressBlock_splitBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapac dstCapacity -= cSizeChunk; cSize += cSizeChunk; *currSeqStore = *nextSeqStore; - assert(cSizeChunk <= ZSTD_BLOCKSIZE_MAX + ZSTD_blockHeaderSize); + assert(cSizeChunk <= zc->blockSize + ZSTD_blockHeaderSize); } - /* cRep and dRep may have diverged during the compression. If so, we use the dRep repcodes - * for the next block. + /* cRep and dRep may have diverged during the compression. + * If so, we use the dRep repcodes for the next block. */ ZSTD_memcpy(zc->blockState.prevCBlock->rep, dRep.rep, sizeof(repcodes_t)); return cSize; @@ -3735,8 +4195,6 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) { - const BYTE* ip = (const BYTE*)src; - BYTE* op = (BYTE*)dst; U32 nbSeq; size_t cSize; DEBUGLOG(4, "ZSTD_compressBlock_splitBlock"); @@ -3747,7 +4205,7 @@ ZSTD_compressBlock_splitBlock(ZSTD_CCtx* zc, if (bss == ZSTDbss_noCompress) { if (zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode == FSE_repeat_valid) zc->blockState.prevCBlock->entropy.fse.offcode_repeatMode = FSE_repeat_check; - cSize = ZSTD_noCompressBlock(op, dstCapacity, ip, srcSize, lastBlock); + cSize = ZSTD_noCompressBlock(dst, dstCapacity, src, srcSize, lastBlock); FORWARD_IF_ERROR(cSize, "ZSTD_noCompressBlock failed"); DEBUGLOG(4, "ZSTD_compressBlock_splitBlock: Nocompress block"); return cSize; @@ -3765,9 +4223,9 @@ ZSTD_compressBlock_internal(ZSTD_CCtx* zc, void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 frame) { - /* This the upper bound for the length of an rle block. - * This isn't the actual upper bound. Finding the real threshold - * needs further investigation. + /* This is an estimated upper bound for the length of an rle block. + * This isn't the actual upper bound. + * Finding the real threshold needs further investigation. */ const U32 rleMaxLength = 25; size_t cSize; @@ -3859,10 +4317,11 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, * * cSize >= blockBound(srcSize): We have expanded the block too much so * emit an uncompressed block. */ - { - size_t const cSize = ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); + { size_t const cSize = + ZSTD_compressSuperBlock(zc, dst, dstCapacity, src, srcSize, lastBlock); if (cSize != ERROR(dstSize_tooSmall)) { - size_t const maxCSize = srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); + size_t const maxCSize = + srcSize - ZSTD_minGain(srcSize, zc->appliedParams.cParams.strategy); FORWARD_IF_ERROR(cSize, "ZSTD_compressSuperBlock failed"); if (cSize != 0 && cSize < maxCSize + ZSTD_blockHeaderSize) { ZSTD_blockState_confirmRepcodesAndEntropyTables(&zc->blockState); @@ -3870,7 +4329,7 @@ static size_t ZSTD_compressBlock_targetCBlockSize_body(ZSTD_CCtx* zc, } } } - } + } /* if (bss == ZSTDbss_compress)*/ DEBUGLOG(6, "Resorting to ZSTD_noCompressBlock()"); /* Superblock compression failed, attempt to emit a single no compress block. @@ -3928,7 +4387,7 @@ static void ZSTD_overflowCorrectIfNeeded(ZSTD_matchState_t* ms, * All blocks will be terminated, all input will be consumed. * Function will issue an error if there is not enough `dstCapacity` to hold the compressed content. * Frame is supposed already started (header already produced) -* @return : compressed size, or an error code +* @return : compressed size, or an error code */ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, @@ -3952,7 +4411,9 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, ZSTD_matchState_t* const ms = &cctx->blockState.matchState; U32 const lastBlock = lastFrameChunk & (blockSize >= remaining); - RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE, + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize + MIN_CBLOCK_SIZE + 1, dstSize_tooSmall, "not enough space to store compressed block"); if (remaining < blockSize) blockSize = remaining; @@ -3991,7 +4452,7 @@ static size_t ZSTD_compress_frameChunk(ZSTD_CCtx* cctx, MEM_writeLE24(op, cBlockHeader); cSize += ZSTD_blockHeaderSize; } - } + } /* if (ZSTD_useTargetCBlockSize(&cctx->appliedParams))*/ ip += blockSize; @@ -4170,31 +4631,51 @@ static size_t ZSTD_compressContinue_internal (ZSTD_CCtx* cctx, } } -size_t ZSTD_compressContinue (ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressContinue (srcSize=%u)", (unsigned)srcSize); return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 1 /* frame mode */, 0 /* last chunk */); } +/* NOTE: Must just wrap ZSTD_compressContinue_public() */ +size_t ZSTD_compressContinue(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressContinue_public(cctx, dst, dstCapacity, src, srcSize); +} -size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +static size_t ZSTD_getBlockSize_deprecated(const ZSTD_CCtx* cctx) { ZSTD_compressionParameters const cParams = cctx->appliedParams.cParams; assert(!ZSTD_checkCParams(cParams)); - return MIN (ZSTD_BLOCKSIZE_MAX, (U32)1 << cParams.windowLog); + return MIN(cctx->appliedParams.maxBlockSize, (size_t)1 << cParams.windowLog); } -size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +/* NOTE: Must just wrap ZSTD_getBlockSize_deprecated() */ +size_t ZSTD_getBlockSize(const ZSTD_CCtx* cctx) +{ + return ZSTD_getBlockSize_deprecated(cctx); +} + +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) { DEBUGLOG(5, "ZSTD_compressBlock: srcSize = %u", (unsigned)srcSize); - { size_t const blockSizeMax = ZSTD_getBlockSize(cctx); + { size_t const blockSizeMax = ZSTD_getBlockSize_deprecated(cctx); RETURN_ERROR_IF(srcSize > blockSizeMax, srcSize_wrong, "input is larger than a block"); } return ZSTD_compressContinue_internal(cctx, dst, dstCapacity, src, srcSize, 0 /* frame mode */, 0 /* last chunk */); } +/* NOTE: Must just wrap ZSTD_compressBlock_deprecated() */ +size_t ZSTD_compressBlock(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize) +{ + return ZSTD_compressBlock_deprecated(cctx, dst, dstCapacity, src, srcSize); +} + /*! ZSTD_loadDictionaryContent() : * @return : 0, or an error code */ @@ -4203,25 +4684,36 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, ZSTD_cwksp* ws, ZSTD_CCtx_params const* params, const void* src, size_t srcSize, - ZSTD_dictTableLoadMethod_e dtlm) + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) { const BYTE* ip = (const BYTE*) src; const BYTE* const iend = ip + srcSize; int const loadLdmDict = params->ldmParams.enableLdm == ZSTD_ps_enable && ls != NULL; - /* Assert that we the ms params match the params we're being given */ + /* Assert that the ms params match the params we're being given */ ZSTD_assertEqualCParams(params->cParams, ms->cParams); - if (srcSize > ZSTD_CHUNKSIZE_MAX) { + { /* Ensure large dictionaries can't cause index overflow */ + /* Allow the dictionary to set indices up to exactly ZSTD_CURRENT_MAX. * Dictionaries right at the edge will immediately trigger overflow * correction, but I don't want to insert extra constraints here. */ - U32 const maxDictSize = ZSTD_CURRENT_MAX - 1; - /* We must have cleared our windows when our source is this large. */ - assert(ZSTD_window_isEmpty(ms->window)); - if (loadLdmDict) - assert(ZSTD_window_isEmpty(ls->window)); + U32 maxDictSize = ZSTD_CURRENT_MAX - ZSTD_WINDOW_START_INDEX; + + int const CDictTaggedIndices = ZSTD_CDictIndicesAreTagged(¶ms->cParams); + if (CDictTaggedIndices && tfp == ZSTD_tfp_forCDict) { + /* Some dictionary matchfinders in zstd use "short cache", + * which treats the lower ZSTD_SHORT_CACHE_TAG_BITS of each + * CDict hashtable entry as a tag rather than as part of an index. + * When short cache is used, we need to truncate the dictionary + * so that its indices don't overlap with the tag. */ + U32 const shortCacheMaxDictSize = (1u << (32 - ZSTD_SHORT_CACHE_TAG_BITS)) - ZSTD_WINDOW_START_INDEX; + maxDictSize = MIN(maxDictSize, shortCacheMaxDictSize); + assert(!loadLdmDict); + } + /* If the dictionary is too large, only load the suffix of the dictionary. */ if (srcSize > maxDictSize) { ip = iend - maxDictSize; @@ -4230,30 +4722,46 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, } } - DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder); + if (srcSize > ZSTD_CHUNKSIZE_MAX) { + /* We must have cleared our windows when our source is this large. */ + assert(ZSTD_window_isEmpty(ms->window)); + if (loadLdmDict) assert(ZSTD_window_isEmpty(ls->window)); + } ZSTD_window_update(&ms->window, src, srcSize, /* forceNonContiguous */ 0); - ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); - ms->forceNonContiguous = params->deterministicRefPrefix; - if (loadLdmDict) { + DEBUGLOG(4, "ZSTD_loadDictionaryContent(): useRowMatchFinder=%d", (int)params->useRowMatchFinder); + + if (loadLdmDict) { /* Load the entire dict into LDM matchfinders. */ ZSTD_window_update(&ls->window, src, srcSize, /* forceNonContiguous */ 0); ls->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ls->window.base); + ZSTD_ldm_fillHashTable(ls, ip, iend, ¶ms->ldmParams); } + /* If the dict is larger than we can reasonably index in our tables, only load the suffix. */ + if (params->cParams.strategy < ZSTD_btultra) { + U32 maxDictSize = 8U << MIN(MAX(params->cParams.hashLog, params->cParams.chainLog), 28); + if (srcSize > maxDictSize) { + ip = iend - maxDictSize; + src = ip; + srcSize = maxDictSize; + } + } + + ms->nextToUpdate = (U32)(ip - ms->window.base); + ms->loadedDictEnd = params->forceWindow ? 0 : (U32)(iend - ms->window.base); + ms->forceNonContiguous = params->deterministicRefPrefix; + if (srcSize <= HASH_READ_SIZE) return 0; ZSTD_overflowCorrectIfNeeded(ms, ws, params, ip, iend); - if (loadLdmDict) - ZSTD_ldm_fillHashTable(ls, ip, iend, ¶ms->ldmParams); - switch(params->cParams.strategy) { case ZSTD_fast: - ZSTD_fillHashTable(ms, iend, dtlm); + ZSTD_fillHashTable(ms, iend, dtlm, tfp); break; case ZSTD_dfast: - ZSTD_fillDoubleHashTable(ms, iend, dtlm); + ZSTD_fillDoubleHashTable(ms, iend, dtlm, tfp); break; case ZSTD_greedy: @@ -4266,7 +4774,7 @@ static size_t ZSTD_loadDictionaryContent(ZSTD_matchState_t* ms, } else { assert(params->useRowMatchFinder != ZSTD_ps_auto); if (params->useRowMatchFinder == ZSTD_ps_enable) { - size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog) * sizeof(U16); + size_t const tagTableSize = ((size_t)1 << params->cParams.hashLog); ZSTD_memset(ms->tagTable, 0, tagTableSize); ZSTD_row_update(ms, iend-HASH_READ_SIZE); DEBUGLOG(4, "Using row-based hash table for lazy dict"); @@ -4419,6 +4927,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, ZSTD_CCtx_params const* params, const void* dict, size_t dictSize, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, void* workspace) { const BYTE* dictPtr = (const BYTE*)dict; @@ -4437,7 +4946,7 @@ static size_t ZSTD_loadZstdDictionary(ZSTD_compressedBlockState_t* bs, { size_t const dictContentSize = (size_t)(dictEnd - dictPtr); FORWARD_IF_ERROR(ZSTD_loadDictionaryContent( - ms, NULL, ws, params, dictPtr, dictContentSize, dtlm), ""); + ms, NULL, ws, params, dictPtr, dictContentSize, dtlm, tfp), ""); } return dictID; } @@ -4453,6 +4962,7 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, const void* dict, size_t dictSize, ZSTD_dictContentType_e dictContentType, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp, void* workspace) { DEBUGLOG(4, "ZSTD_compress_insertDictionary (dictSize=%u)", (U32)dictSize); @@ -4465,13 +4975,13 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, /* dict restricted modes */ if (dictContentType == ZSTD_dct_rawContent) - return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm); + return ZSTD_loadDictionaryContent(ms, ls, ws, params, dict, dictSize, dtlm, tfp); if (MEM_readLE32(dict) != ZSTD_MAGIC_DICTIONARY) { if (dictContentType == ZSTD_dct_auto) { DEBUGLOG(4, "raw content dictionary detected"); return ZSTD_loadDictionaryContent( - ms, ls, ws, params, dict, dictSize, dtlm); + ms, ls, ws, params, dict, dictSize, dtlm, tfp); } RETURN_ERROR_IF(dictContentType == ZSTD_dct_fullDict, dictionary_wrong, ""); assert(0); /* impossible */ @@ -4479,13 +4989,14 @@ ZSTD_compress_insertDictionary(ZSTD_compressedBlockState_t* bs, /* dict as full zstd dictionary */ return ZSTD_loadZstdDictionary( - bs, ms, ws, params, dict, dictSize, dtlm, workspace); + bs, ms, ws, params, dict, dictSize, dtlm, tfp, workspace); } #define ZSTD_USE_CDICT_PARAMS_SRCSIZE_CUTOFF (128 KB) #define ZSTD_USE_CDICT_PARAMS_DICTSIZE_MULTIPLIER (6ULL) /*! ZSTD_compressBegin_internal() : + * Assumption : either @dict OR @cdict (or none) is non-NULL, never both * @return : 0, or an error code */ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, @@ -4521,11 +5032,11 @@ static size_t ZSTD_compressBegin_internal(ZSTD_CCtx* cctx, cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, cdict->dictContent, cdict->dictContentSize, cdict->dictContentType, dtlm, - cctx->entropyWorkspace) + ZSTD_tfp_forCCtx, cctx->entropyWorkspace) : ZSTD_compress_insertDictionary( cctx->blockState.prevCBlock, &cctx->blockState.matchState, &cctx->ldmState, &cctx->workspace, &cctx->appliedParams, dict, dictSize, - dictContentType, dtlm, cctx->entropyWorkspace); + dictContentType, dtlm, ZSTD_tfp_forCCtx, cctx->entropyWorkspace); FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); assert(dictID <= UINT_MAX); cctx->dictID = (U32)dictID; @@ -4566,11 +5077,11 @@ size_t ZSTD_compressBegin_advanced(ZSTD_CCtx* cctx, &cctxParams, pledgedSrcSize); } -size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +static size_t +ZSTD_compressBegin_usingDict_deprecated(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) { ZSTD_CCtx_params cctxParams; - { - ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); + { ZSTD_parameters const params = ZSTD_getParams_internal(compressionLevel, ZSTD_CONTENTSIZE_UNKNOWN, dictSize, ZSTD_cpm_noAttachDict); ZSTD_CCtxParams_init_internal(&cctxParams, ¶ms, (compressionLevel == 0) ? ZSTD_CLEVEL_DEFAULT : compressionLevel); } DEBUGLOG(4, "ZSTD_compressBegin_usingDict (dictSize=%u)", (unsigned)dictSize); @@ -4578,9 +5089,15 @@ size_t ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t di &cctxParams, ZSTD_CONTENTSIZE_UNKNOWN, ZSTDb_not_buffered); } +size_t +ZSTD_compressBegin_usingDict(ZSTD_CCtx* cctx, const void* dict, size_t dictSize, int compressionLevel) +{ + return ZSTD_compressBegin_usingDict_deprecated(cctx, dict, dictSize, compressionLevel); +} + size_t ZSTD_compressBegin(ZSTD_CCtx* cctx, int compressionLevel) { - return ZSTD_compressBegin_usingDict(cctx, NULL, 0, compressionLevel); + return ZSTD_compressBegin_usingDict_deprecated(cctx, NULL, 0, compressionLevel); } @@ -4650,9 +5167,9 @@ void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize) #endif } -size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { size_t endResult; size_t const cSize = ZSTD_compressContinue_internal(cctx, @@ -4676,6 +5193,14 @@ size_t ZSTD_compressEnd (ZSTD_CCtx* cctx, return cSize + endResult; } +/* NOTE: Must just wrap ZSTD_compressEnd_public() */ +size_t ZSTD_compressEnd(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); +} + size_t ZSTD_compress_advanced (ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize, @@ -4704,7 +5229,7 @@ size_t ZSTD_compress_advanced_internal( FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, dict, dictSize, ZSTD_dct_auto, ZSTD_dtlm_fast, NULL, params, srcSize, ZSTDb_not_buffered) , ""); - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); } size_t ZSTD_compress_usingDict(ZSTD_CCtx* cctx, @@ -4829,7 +5354,7 @@ static size_t ZSTD_initCDict_internal( { size_t const dictID = ZSTD_compress_insertDictionary( &cdict->cBlockState, &cdict->matchState, NULL, &cdict->workspace, ¶ms, cdict->dictContent, cdict->dictContentSize, - dictContentType, ZSTD_dtlm_full, cdict->entropyWorkspace); + dictContentType, ZSTD_dtlm_full, ZSTD_tfp_forCDict, cdict->entropyWorkspace); FORWARD_IF_ERROR(dictID, "ZSTD_compress_insertDictionary failed"); assert(dictID <= (size_t)(U32)-1); cdict->dictID = (U32)dictID; @@ -5026,6 +5551,7 @@ const ZSTD_CDict* ZSTD_initStaticCDict( params.cParams = cParams; params.useRowMatchFinder = useRowMatchFinder; cdict->useRowMatchFinder = useRowMatchFinder; + cdict->compressionLevel = ZSTD_NO_CLEVEL; if (ZSTD_isError( ZSTD_initCDict_internal(cdict, dict, dictSize, @@ -5105,12 +5631,17 @@ size_t ZSTD_compressBegin_usingCDict_advanced( /* ZSTD_compressBegin_usingCDict() : * cdict must be != NULL */ -size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) { ZSTD_frameParameters const fParams = { 0 /*content*/, 0 /*checksum*/, 0 /*noDictID*/ }; return ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, ZSTD_CONTENTSIZE_UNKNOWN); } +size_t ZSTD_compressBegin_usingCDict(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict) +{ + return ZSTD_compressBegin_usingCDict_deprecated(cctx, cdict); +} + /*! ZSTD_compress_usingCDict_internal(): * Implementation of various ZSTD_compress_usingCDict* functions. */ @@ -5120,7 +5651,7 @@ static size_t ZSTD_compress_usingCDict_internal(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict, ZSTD_frameParameters fParams) { FORWARD_IF_ERROR(ZSTD_compressBegin_usingCDict_internal(cctx, cdict, fParams, srcSize), ""); /* will check if cdict != NULL */ - return ZSTD_compressEnd(cctx, dst, dstCapacity, src, srcSize); + return ZSTD_compressEnd_public(cctx, dst, dstCapacity, src, srcSize); } /*! ZSTD_compress_usingCDict_advanced(): @@ -5317,30 +5848,41 @@ size_t ZSTD_initCStream(ZSTD_CStream* zcs, int compressionLevel) static size_t ZSTD_nextInputSizeHint(const ZSTD_CCtx* cctx) { - size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; - if (hintInSize==0) hintInSize = cctx->blockSize; - return hintInSize; + if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { + return cctx->blockSize - cctx->stableIn_notConsumed; + } + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_buffered); + { size_t hintInSize = cctx->inBuffTarget - cctx->inBuffPos; + if (hintInSize==0) hintInSize = cctx->blockSize; + return hintInSize; + } } /** ZSTD_compressStream_generic(): * internal function for all *compressStream*() variants - * non-static, because can be called from zstdmt_compress.c - * @return : hint size for next input */ + * @return : hint size for next input to complete ongoing block */ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuffer* input, ZSTD_EndDirective const flushMode) { - const char* const istart = (const char*)input->src; - const char* const iend = input->size != 0 ? istart + input->size : istart; - const char* ip = input->pos != 0 ? istart + input->pos : istart; - char* const ostart = (char*)output->dst; - char* const oend = output->size != 0 ? ostart + output->size : ostart; - char* op = output->pos != 0 ? ostart + output->pos : ostart; + const char* const istart = (assert(input != NULL), (const char*)input->src); + const char* const iend = (istart != NULL) ? istart + input->size : istart; + const char* ip = (istart != NULL) ? istart + input->pos : istart; + char* const ostart = (assert(output != NULL), (char*)output->dst); + char* const oend = (ostart != NULL) ? ostart + output->size : ostart; + char* op = (ostart != NULL) ? ostart + output->pos : ostart; U32 someMoreWork = 1; /* check expectations */ - DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%u", (unsigned)flushMode); + DEBUGLOG(5, "ZSTD_compressStream_generic, flush=%i, srcSize = %zu", (int)flushMode, input->size - input->pos); + assert(zcs != NULL); + if (zcs->appliedParams.inBufferMode == ZSTD_bm_stable) { + assert(input->pos >= zcs->stableIn_notConsumed); + input->pos -= zcs->stableIn_notConsumed; + ip -= zcs->stableIn_notConsumed; + zcs->stableIn_notConsumed = 0; + } if (zcs->appliedParams.inBufferMode == ZSTD_bm_buffered) { assert(zcs->inBuff != NULL); assert(zcs->inBuffSize > 0); @@ -5349,8 +5891,10 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, assert(zcs->outBuff != NULL); assert(zcs->outBuffSize > 0); } - assert(output->pos <= output->size); + if (input->src == NULL) assert(input->size == 0); assert(input->pos <= input->size); + if (output->dst == NULL) assert(output->size == 0); + assert(output->pos <= output->size); assert((U32)flushMode <= (U32)ZSTD_e_end); while (someMoreWork) { @@ -5365,7 +5909,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) /* OR we are allowed to return dstSizeTooSmall */ && (zcs->inBuffPos == 0) ) { /* shortcut to compression pass directly into output buffer */ - size_t const cSize = ZSTD_compressEnd(zcs, + size_t const cSize = ZSTD_compressEnd_public(zcs, op, oend-op, ip, iend-ip); DEBUGLOG(4, "ZSTD_compressEnd : cSize=%u", (unsigned)cSize); FORWARD_IF_ERROR(cSize, "ZSTD_compressEnd failed"); @@ -5382,8 +5926,7 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, zcs->inBuff + zcs->inBuffPos, toLoad, ip, iend-ip); zcs->inBuffPos += loaded; - if (loaded != 0) - ip += loaded; + if (ip) ip += loaded; if ( (flushMode == ZSTD_e_continue) && (zcs->inBuffPos < zcs->inBuffTarget) ) { /* not enough input to fill full block : stop here */ @@ -5394,6 +5937,20 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, /* empty */ someMoreWork = 0; break; } + } else { + assert(zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + if ( (flushMode == ZSTD_e_continue) + && ( (size_t)(iend - ip) < zcs->blockSize) ) { + /* can't compress a full block : stop here */ + zcs->stableIn_notConsumed = (size_t)(iend - ip); + ip = iend; /* pretend to have consumed input */ + someMoreWork = 0; break; + } + if ( (flushMode == ZSTD_e_flush) + && (ip == iend) ) { + /* empty */ + someMoreWork = 0; break; + } } /* compress current block (note : this stage cannot be stopped in the middle) */ DEBUGLOG(5, "stream compression stage (flushMode==%u)", flushMode); @@ -5401,9 +5958,8 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, void* cDst; size_t cSize; size_t oSize = oend-op; - size_t const iSize = inputBuffered - ? zcs->inBuffPos - zcs->inToCompress - : MIN((size_t)(iend - ip), zcs->blockSize); + size_t const iSize = inputBuffered ? zcs->inBuffPos - zcs->inToCompress + : MIN((size_t)(iend - ip), zcs->blockSize); if (oSize >= ZSTD_compressBound(iSize) || zcs->appliedParams.outBufferMode == ZSTD_bm_stable) cDst = op; /* compress into output buffer, to skip flush stage */ else @@ -5411,9 +5967,9 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, if (inputBuffered) { unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip==iend); cSize = lastBlock ? - ZSTD_compressEnd(zcs, cDst, oSize, + ZSTD_compressEnd_public(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize) : - ZSTD_compressContinue(zcs, cDst, oSize, + ZSTD_compressContinue_public(zcs, cDst, oSize, zcs->inBuff + zcs->inToCompress, iSize); FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); zcs->frameEnded = lastBlock; @@ -5426,19 +5982,16 @@ static size_t ZSTD_compressStream_generic(ZSTD_CStream* zcs, if (!lastBlock) assert(zcs->inBuffTarget <= zcs->inBuffSize); zcs->inToCompress = zcs->inBuffPos; - } else { - unsigned const lastBlock = (ip + iSize == iend); - assert(flushMode == ZSTD_e_end /* Already validated */); + } else { /* !inputBuffered, hence ZSTD_bm_stable */ + unsigned const lastBlock = (flushMode == ZSTD_e_end) && (ip + iSize == iend); cSize = lastBlock ? - ZSTD_compressEnd(zcs, cDst, oSize, ip, iSize) : - ZSTD_compressContinue(zcs, cDst, oSize, ip, iSize); + ZSTD_compressEnd_public(zcs, cDst, oSize, ip, iSize) : + ZSTD_compressContinue_public(zcs, cDst, oSize, ip, iSize); /* Consume the input prior to error checking to mirror buffered mode. */ - if (iSize > 0) - ip += iSize; + if (ip) ip += iSize; FORWARD_IF_ERROR(cSize, "%s", lastBlock ? "ZSTD_compressEnd failed" : "ZSTD_compressContinue failed"); zcs->frameEnded = lastBlock; - if (lastBlock) - assert(ip == iend); + if (lastBlock) assert(ip == iend); } if (cDst == op) { /* no need to flush */ op += cSize; @@ -5514,8 +6067,10 @@ size_t ZSTD_compressStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output, ZSTD_inBuf /* After a compression call set the expected input/output buffer. * This is validated at the start of the next compression call. */ -static void ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, ZSTD_outBuffer const* output, ZSTD_inBuffer const* input) +static void +ZSTD_setBufferExpectations(ZSTD_CCtx* cctx, const ZSTD_outBuffer* output, const ZSTD_inBuffer* input) { + DEBUGLOG(5, "ZSTD_setBufferExpectations (for advanced stable in/out modes)"); if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { cctx->expectedInBuffer = *input; } @@ -5534,22 +6089,22 @@ static size_t ZSTD_checkBufferStability(ZSTD_CCtx const* cctx, { if (cctx->appliedParams.inBufferMode == ZSTD_bm_stable) { ZSTD_inBuffer const expect = cctx->expectedInBuffer; - if (expect.src != input->src || expect.pos != input->pos || expect.size != input->size) - RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer enabled but input differs!"); - if (endOp != ZSTD_e_end) - RETURN_ERROR(srcBuffer_wrong, "ZSTD_c_stableInBuffer can only be used with ZSTD_e_end!"); + if (expect.src != input->src || expect.pos != input->pos) + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableInBuffer enabled but input differs!"); } + (void)endOp; if (cctx->appliedParams.outBufferMode == ZSTD_bm_stable) { size_t const outBufferSize = output->size - output->pos; if (cctx->expectedOutBufferSize != outBufferSize) - RETURN_ERROR(dstBuffer_wrong, "ZSTD_c_stableOutBuffer enabled but output size differs!"); + RETURN_ERROR(stabilityCondition_notRespected, "ZSTD_c_stableOutBuffer enabled but output size differs!"); } return 0; } static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, ZSTD_EndDirective endOp, - size_t inSize) { + size_t inSize) +{ ZSTD_CCtx_params params = cctx->requestedParams; ZSTD_prefixDict const prefixDict = cctx->prefixDict; FORWARD_IF_ERROR( ZSTD_initLocalDict(cctx) , ""); /* Init the local dict if present. */ @@ -5563,9 +6118,9 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, params.compressionLevel = cctx->cdict->compressionLevel; } DEBUGLOG(4, "ZSTD_compressStream2 : transparent init stage"); - if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-fix pledgedSrcSize */ - { - size_t const dictSize = prefixDict.dict + if (endOp == ZSTD_e_end) cctx->pledgedSrcSizePlusOne = inSize + 1; /* auto-determine pledgedSrcSize */ + + { size_t const dictSize = prefixDict.dict ? prefixDict.dictSize : (cctx->cdict ? cctx->cdict->dictContentSize : 0); ZSTD_cParamMode_e const mode = ZSTD_getCParamMode(cctx->cdict, ¶ms, cctx->pledgedSrcSizePlusOne - 1); @@ -5577,8 +6132,18 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, params.useBlockSplitter = ZSTD_resolveBlockSplitterMode(params.useBlockSplitter, ¶ms.cParams); params.ldmParams.enableLdm = ZSTD_resolveEnableLdm(params.ldmParams.enableLdm, ¶ms.cParams); params.useRowMatchFinder = ZSTD_resolveRowMatchFinderMode(params.useRowMatchFinder, ¶ms.cParams); + params.validateSequences = ZSTD_resolveExternalSequenceValidation(params.validateSequences); + params.maxBlockSize = ZSTD_resolveMaxBlockSize(params.maxBlockSize); + params.searchForExternalRepcodes = ZSTD_resolveExternalRepcodeSearch(params.searchForExternalRepcodes, params.compressionLevel); #ifdef ZSTD_MULTITHREAD + /* If external matchfinder is enabled, make sure to fail before checking job size (for consistency) */ + RETURN_ERROR_IF( + params.useSequenceProducer == 1 && params.nbWorkers >= 1, + parameter_combination_unsupported, + "External sequence producer isn't supported with nbWorkers >= 1" + ); + if ((cctx->pledgedSrcSizePlusOne-1) <= ZSTDMT_JOBSIZE_MIN) { params.nbWorkers = 0; /* do not invoke multi-threading when src size is too small */ } @@ -5606,7 +6171,7 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, cctx->streamStage = zcss_load; cctx->appliedParams = params; } else -#endif +#endif /* ZSTD_MULTITHREAD */ { U64 const pledgedSrcSize = cctx->pledgedSrcSizePlusOne - 1; assert(!ZSTD_isError(ZSTD_checkCParams(params.cParams))); FORWARD_IF_ERROR( ZSTD_compressBegin_internal(cctx, @@ -5632,6 +6197,8 @@ static size_t ZSTD_CCtx_init_compressStream2(ZSTD_CCtx* cctx, return 0; } +/* @return provides a minimum amount of data remaining to be flushed from internal buffers + */ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_outBuffer* output, ZSTD_inBuffer* input, @@ -5646,8 +6213,27 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, /* transparent initialization stage */ if (cctx->streamStage == zcss_init) { - FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, input->size), "CompressStream2 initialization failed"); - ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ + size_t const inputSize = input->size - input->pos; /* no obligation to start from pos==0 */ + size_t const totalInputSize = inputSize + cctx->stableIn_notConsumed; + if ( (cctx->requestedParams.inBufferMode == ZSTD_bm_stable) /* input is presumed stable, across invocations */ + && (endOp == ZSTD_e_continue) /* no flush requested, more input to come */ + && (totalInputSize < ZSTD_BLOCKSIZE_MAX) ) { /* not even reached one block yet */ + if (cctx->stableIn_notConsumed) { /* not the first time */ + /* check stable source guarantees */ + RETURN_ERROR_IF(input->src != cctx->expectedInBuffer.src, stabilityCondition_notRespected, "stableInBuffer condition not respected: wrong src pointer"); + RETURN_ERROR_IF(input->pos != cctx->expectedInBuffer.size, stabilityCondition_notRespected, "stableInBuffer condition not respected: externally modified pos"); + } + /* pretend input was consumed, to give a sense forward progress */ + input->pos = input->size; + /* save stable inBuffer, for later control, and flush/end */ + cctx->expectedInBuffer = *input; + /* but actually input wasn't consumed, so keep track of position from where compression shall resume */ + cctx->stableIn_notConsumed += inputSize; + /* don't initialize yet, wait for the first block of flush() order, for better parameters adaptation */ + return ZSTD_FRAMEHEADERSIZE_MIN(cctx->requestedParams.format); /* at least some header to produce */ + } + FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, endOp, totalInputSize), "compressStream2 initialization failed"); + ZSTD_setBufferExpectations(cctx, output, input); /* Set initial buffer expectations now that we've initialized */ } /* end of transparent initialization stage */ @@ -5660,6 +6246,13 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTDMT_updateCParams_whileCompressing(cctx->mtctx, &cctx->requestedParams); cctx->cParamsChanged = 0; } + if (cctx->stableIn_notConsumed) { + assert(cctx->appliedParams.inBufferMode == ZSTD_bm_stable); + /* some early data was skipped - make it available for consumption */ + assert(input->pos >= cctx->stableIn_notConsumed); + input->pos -= cctx->stableIn_notConsumed; + cctx->stableIn_notConsumed = 0; + } for (;;) { size_t const ipos = input->pos; size_t const opos = output->pos; @@ -5698,7 +6291,7 @@ size_t ZSTD_compressStream2( ZSTD_CCtx* cctx, ZSTD_setBufferExpectations(cctx, output, input); return flushMin; } -#endif +#endif /* ZSTD_MULTITHREAD */ FORWARD_IF_ERROR( ZSTD_compressStream_generic(cctx, output, input, endOp) , ""); DEBUGLOG(5, "completed ZSTD_compressStream2"); ZSTD_setBufferExpectations(cctx, output, input); @@ -5711,13 +6304,20 @@ size_t ZSTD_compressStream2_simpleArgs ( const void* src, size_t srcSize, size_t* srcPos, ZSTD_EndDirective endOp) { - ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; - ZSTD_inBuffer input = { src, srcSize, *srcPos }; + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; /* ZSTD_compressStream2() will check validity of dstPos and srcPos */ - size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); - *dstPos = output.pos; - *srcPos = input.pos; - return cErr; + { size_t const cErr = ZSTD_compressStream2(cctx, &output, &input, endOp); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } } size_t ZSTD_compress2(ZSTD_CCtx* cctx, @@ -5740,6 +6340,7 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, /* Reset to the original values. */ cctx->requestedParams.inBufferMode = originalInBufferMode; cctx->requestedParams.outBufferMode = originalOutBufferMode; + FORWARD_IF_ERROR(result, "ZSTD_compressStream2_simpleArgs failed"); if (result != 0) { /* compression not completed, due to lack of output space */ assert(oPos == dstCapacity); @@ -5750,64 +6351,61 @@ size_t ZSTD_compress2(ZSTD_CCtx* cctx, } } -typedef struct { - U32 idx; /* Index in array of ZSTD_Sequence */ - U32 posInSequence; /* Position within sequence at idx */ - size_t posInSrc; /* Number of bytes given by sequences provided so far */ -} ZSTD_sequencePosition; - /* ZSTD_validateSequence() : * @offCode : is presumed to follow format required by ZSTD_storeSeq() * @returns a ZSTD error code if sequence is not valid */ static size_t -ZSTD_validateSequence(U32 offCode, U32 matchLength, - size_t posInSrc, U32 windowLog, size_t dictSize) +ZSTD_validateSequence(U32 offCode, U32 matchLength, U32 minMatch, + size_t posInSrc, U32 windowLog, size_t dictSize, int useSequenceProducer) { - U32 const windowSize = 1 << windowLog; - /* posInSrc represents the amount of data the the decoder would decode up to this point. + U32 const windowSize = 1u << windowLog; + /* posInSrc represents the amount of data the decoder would decode up to this point. * As long as the amount of data decoded is less than or equal to window size, offsets may be * larger than the total length of output decoded in order to reference the dict, even larger than * window size. After output surpasses windowSize, we're limited to windowSize offsets again. */ size_t const offsetBound = posInSrc > windowSize ? (size_t)windowSize : posInSrc + (size_t)dictSize; - RETURN_ERROR_IF(offCode > STORE_OFFSET(offsetBound), corruption_detected, "Offset too large!"); - RETURN_ERROR_IF(matchLength < MINMATCH, corruption_detected, "Matchlength too small"); + size_t const matchLenLowerBound = (minMatch == 3 || useSequenceProducer) ? 3 : 4; + RETURN_ERROR_IF(offCode > OFFSET_TO_OFFBASE(offsetBound), externalSequences_invalid, "Offset too large!"); + /* Validate maxNbSeq is large enough for the given matchLength and minMatch */ + RETURN_ERROR_IF(matchLength < matchLenLowerBound, externalSequences_invalid, "Matchlength too small for the minMatch"); return 0; } /* Returns an offset code, given a sequence's raw offset, the ongoing repcode array, and whether litLength == 0 */ -static U32 ZSTD_finalizeOffCode(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) +static U32 ZSTD_finalizeOffBase(U32 rawOffset, const U32 rep[ZSTD_REP_NUM], U32 ll0) { - U32 offCode = STORE_OFFSET(rawOffset); + U32 offBase = OFFSET_TO_OFFBASE(rawOffset); if (!ll0 && rawOffset == rep[0]) { - offCode = STORE_REPCODE_1; + offBase = REPCODE1_TO_OFFBASE; } else if (rawOffset == rep[1]) { - offCode = STORE_REPCODE(2 - ll0); + offBase = REPCODE_TO_OFFBASE(2 - ll0); } else if (rawOffset == rep[2]) { - offCode = STORE_REPCODE(3 - ll0); + offBase = REPCODE_TO_OFFBASE(3 - ll0); } else if (ll0 && rawOffset == rep[0] - 1) { - offCode = STORE_REPCODE_3; + offBase = REPCODE3_TO_OFFBASE; } - return offCode; + return offBase; } -/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of - * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. - */ -static size_t +size_t ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, - const void* src, size_t blockSize) + const void* src, size_t blockSize, + ZSTD_paramSwitch_e externalRepSearch) { U32 idx = seqPos->idx; + U32 const startIdx = idx; BYTE const* ip = (BYTE const*)(src); const BYTE* const iend = ip + blockSize; repcodes_t updatedRepcodes; U32 dictSize; + DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreExplicitBlockDelim (blockSize = %zu)", blockSize); + if (cctx->cdict) { dictSize = (U32)cctx->cdict->dictContentSize; } else if (cctx->prefixDict.dict) { @@ -5816,25 +6414,55 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, dictSize = 0; } ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); - for (; (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0) && idx < inSeqsSize; ++idx) { + for (; idx < inSeqsSize && (inSeqs[idx].matchLength != 0 || inSeqs[idx].offset != 0); ++idx) { U32 const litLength = inSeqs[idx].litLength; - U32 const ll0 = (litLength == 0); U32 const matchLength = inSeqs[idx].matchLength; - U32 const offCode = ZSTD_finalizeOffCode(inSeqs[idx].offset, updatedRepcodes.rep, ll0); - ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + U32 offBase; - DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); + if (externalRepSearch == ZSTD_ps_disable) { + offBase = OFFSET_TO_OFFBASE(inSeqs[idx].offset); + } else { + U32 const ll0 = (litLength == 0); + offBase = ZSTD_finalizeOffBase(inSeqs[idx].offset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); + } + + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); if (cctx->appliedParams.validateSequences) { seqPos->posInSrc += litLength + matchLength; - FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, - cctx->appliedParams.cParams.windowLog, dictSize), + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer), "Sequence validation failed"); } - RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); - ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); ip += matchLength + litLength; } + + /* If we skipped repcode search while parsing, we need to update repcodes now */ + assert(externalRepSearch != ZSTD_ps_auto); + assert(idx >= startIdx); + if (externalRepSearch == ZSTD_ps_disable && idx != startIdx) { + U32* const rep = updatedRepcodes.rep; + U32 lastSeqIdx = idx - 1; /* index of last non-block-delimiter sequence */ + + if (lastSeqIdx >= startIdx + 2) { + rep[2] = inSeqs[lastSeqIdx - 2].offset; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else if (lastSeqIdx == startIdx + 1) { + rep[2] = rep[0]; + rep[1] = inSeqs[lastSeqIdx - 1].offset; + rep[0] = inSeqs[lastSeqIdx].offset; + } else { + assert(lastSeqIdx == startIdx); + rep[2] = rep[1]; + rep[1] = rep[0]; + rep[0] = inSeqs[lastSeqIdx].offset; + } + } + ZSTD_memcpy(cctx->blockState.nextCBlock->rep, updatedRepcodes.rep, sizeof(repcodes_t)); if (inSeqs[idx].litLength) { @@ -5843,26 +6471,15 @@ ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, ip += inSeqs[idx].litLength; seqPos->posInSrc += inSeqs[idx].litLength; } - RETURN_ERROR_IF(ip != iend, corruption_detected, "Blocksize doesn't agree with block delimiter!"); + RETURN_ERROR_IF(ip != iend, externalSequences_invalid, "Blocksize doesn't agree with block delimiter!"); seqPos->idx = idx+1; return 0; } -/* Returns the number of bytes to move the current read position back by. Only non-zero - * if we ended up splitting a sequence. Otherwise, it may return a ZSTD error if something - * went wrong. - * - * This function will attempt to scan through blockSize bytes represented by the sequences - * in inSeqs, storing any (partial) sequences. - * - * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to - * avoid splitting a match, or to avoid splitting a match such that it would produce a match - * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. - */ -static size_t +size_t ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, - const void* src, size_t blockSize) + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch) { U32 idx = seqPos->idx; U32 startPosInSequence = seqPos->posInSequence; @@ -5874,6 +6491,9 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* U32 bytesAdjustment = 0; U32 finalMatchSplit = 0; + /* TODO(embg) support fast parsing mode in noBlockDelim mode */ + (void)externalRepSearch; + if (cctx->cdict) { dictSize = cctx->cdict->dictContentSize; } else if (cctx->prefixDict.dict) { @@ -5881,7 +6501,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* } else { dictSize = 0; } - DEBUGLOG(5, "ZSTD_copySequencesToSeqStore: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); + DEBUGLOG(5, "ZSTD_copySequencesToSeqStoreNoBlockDelim: idx: %u PIS: %u blockSize: %zu", idx, startPosInSequence, blockSize); DEBUGLOG(5, "Start seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); ZSTD_memcpy(updatedRepcodes.rep, cctx->blockState.prevCBlock->rep, sizeof(repcodes_t)); while (endPosInSequence && idx < inSeqsSize && !finalMatchSplit) { @@ -5889,7 +6509,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* U32 litLength = currSeq.litLength; U32 matchLength = currSeq.matchLength; U32 const rawOffset = currSeq.offset; - U32 offCode; + U32 offBase; /* Modify the sequence depending on where endPosInSequence lies */ if (endPosInSequence >= currSeq.litLength + currSeq.matchLength) { @@ -5903,7 +6523,6 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* /* Move to the next sequence */ endPosInSequence -= currSeq.litLength + currSeq.matchLength; startPosInSequence = 0; - idx++; } else { /* This is the final (partial) sequence we're adding from inSeqs, and endPosInSequence does not reach the end of the match. So, we have to split the sequence */ @@ -5943,21 +6562,23 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* } /* Check if this offset can be represented with a repcode */ { U32 const ll0 = (litLength == 0); - offCode = ZSTD_finalizeOffCode(rawOffset, updatedRepcodes.rep, ll0); - ZSTD_updateRep(updatedRepcodes.rep, offCode, ll0); + offBase = ZSTD_finalizeOffBase(rawOffset, updatedRepcodes.rep, ll0); + ZSTD_updateRep(updatedRepcodes.rep, offBase, ll0); } if (cctx->appliedParams.validateSequences) { seqPos->posInSrc += litLength + matchLength; - FORWARD_IF_ERROR(ZSTD_validateSequence(offCode, matchLength, seqPos->posInSrc, - cctx->appliedParams.cParams.windowLog, dictSize), + FORWARD_IF_ERROR(ZSTD_validateSequence(offBase, matchLength, cctx->appliedParams.cParams.minMatch, seqPos->posInSrc, + cctx->appliedParams.cParams.windowLog, dictSize, cctx->appliedParams.useSequenceProducer), "Sequence validation failed"); } - DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offCode, matchLength, litLength); - RETURN_ERROR_IF(idx - seqPos->idx > cctx->seqStore.maxNbSeq, memory_allocation, + DEBUGLOG(6, "Storing sequence: (of: %u, ml: %u, ll: %u)", offBase, matchLength, litLength); + RETURN_ERROR_IF(idx - seqPos->idx >= cctx->seqStore.maxNbSeq, externalSequences_invalid, "Not enough memory allocated. Try adjusting ZSTD_c_minMatch."); - ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offCode, matchLength); + ZSTD_storeSeq(&cctx->seqStore, litLength, ip, iend, offBase, matchLength); ip += matchLength + litLength; + if (!finalMatchSplit) + idx++; /* Next Sequence */ } DEBUGLOG(5, "Ending seq: idx: %u (of: %u ml: %u ll: %u)", idx, inSeqs[idx].offset, inSeqs[idx].matchLength, inSeqs[idx].litLength); assert(idx == inSeqsSize || endPosInSequence <= inSeqs[idx].litLength + inSeqs[idx].matchLength); @@ -5980,7 +6601,7 @@ ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* typedef size_t (*ZSTD_sequenceCopier) (ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, - const void* src, size_t blockSize); + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) { ZSTD_sequenceCopier sequenceCopier = NULL; @@ -5994,6 +6615,57 @@ static ZSTD_sequenceCopier ZSTD_selectSequenceCopier(ZSTD_sequenceFormat_e mode) return sequenceCopier; } +/* Discover the size of next block by searching for the delimiter. + * Note that a block delimiter **must** exist in this mode, + * otherwise it's an input error. + * The block size retrieved will be later compared to ensure it remains within bounds */ +static size_t +blockSize_explicitDelimiter(const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos) +{ + int end = 0; + size_t blockSize = 0; + size_t spos = seqPos.idx; + DEBUGLOG(6, "blockSize_explicitDelimiter : seq %zu / %zu", spos, inSeqsSize); + assert(spos <= inSeqsSize); + while (spos < inSeqsSize) { + end = (inSeqs[spos].offset == 0); + blockSize += inSeqs[spos].litLength + inSeqs[spos].matchLength; + if (end) { + if (inSeqs[spos].matchLength != 0) + RETURN_ERROR(externalSequences_invalid, "delimiter format error : both matchlength and offset must be == 0"); + break; + } + spos++; + } + if (!end) + RETURN_ERROR(externalSequences_invalid, "Reached end of sequences without finding a block delimiter"); + return blockSize; +} + +/* More a "target" block size */ +static size_t blockSize_noDelimiter(size_t blockSize, size_t remaining) +{ + int const lastBlock = (remaining <= blockSize); + return lastBlock ? remaining : blockSize; +} + +static size_t determine_blockSize(ZSTD_sequenceFormat_e mode, + size_t blockSize, size_t remaining, + const ZSTD_Sequence* inSeqs, size_t inSeqsSize, ZSTD_sequencePosition seqPos) +{ + DEBUGLOG(6, "determine_blockSize : remainingSize = %zu", remaining); + if (mode == ZSTD_sf_noBlockDelimiters) + return blockSize_noDelimiter(blockSize, remaining); + { size_t const explicitBlockSize = blockSize_explicitDelimiter(inSeqs, inSeqsSize, seqPos); + FORWARD_IF_ERROR(explicitBlockSize, "Error while determining block size with explicit delimiters"); + if (explicitBlockSize > blockSize) + RETURN_ERROR(externalSequences_invalid, "sequences incorrectly define a too large block"); + if (explicitBlockSize > remaining) + RETURN_ERROR(externalSequences_invalid, "sequences define a frame longer than source"); + return explicitBlockSize; + } +} + /* Compress, block-by-block, all of the sequences given. * * Returns the cumulative size of all compressed blocks (including their headers), @@ -6006,9 +6678,6 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, const void* src, size_t srcSize) { size_t cSize = 0; - U32 lastBlock; - size_t blockSize; - size_t compressedSeqsSize; size_t remaining = srcSize; ZSTD_sequencePosition seqPos = {0, 0, 0}; @@ -6028,22 +6697,29 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, } while (remaining) { + size_t compressedSeqsSize; size_t cBlockSize; size_t additionalByteAdjustment; - lastBlock = remaining <= cctx->blockSize; - blockSize = lastBlock ? (U32)remaining : (U32)cctx->blockSize; + size_t blockSize = determine_blockSize(cctx->appliedParams.blockDelimiters, + cctx->blockSize, remaining, + inSeqs, inSeqsSize, seqPos); + U32 const lastBlock = (blockSize == remaining); + FORWARD_IF_ERROR(blockSize, "Error while trying to determine block size"); + assert(blockSize <= remaining); ZSTD_resetSeqStore(&cctx->seqStore); - DEBUGLOG(4, "Working on new block. Blocksize: %zu", blockSize); + DEBUGLOG(5, "Working on new block. Blocksize: %zu (total:%zu)", blockSize, (ip - (const BYTE*)src) + blockSize); - additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize); + additionalByteAdjustment = sequenceCopier(cctx, &seqPos, inSeqs, inSeqsSize, ip, blockSize, cctx->appliedParams.searchForExternalRepcodes); FORWARD_IF_ERROR(additionalByteAdjustment, "Bad sequence copy"); blockSize -= additionalByteAdjustment; /* If blocks are too small, emit as a nocompress block */ - if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1) { + /* TODO: See 3090. We reduced MIN_CBLOCK_SIZE from 3 to 2 so to compensate we are adding + * additional 1. We need to revisit and change this logic to be more consistent */ + if (blockSize < MIN_CBLOCK_SIZE+ZSTD_blockHeaderSize+1+1) { cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); - DEBUGLOG(4, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); + DEBUGLOG(5, "Block too small, writing out nocompress block: cSize: %zu", cBlockSize); cSize += cBlockSize; ip += blockSize; op += cBlockSize; @@ -6052,6 +6728,7 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, continue; } + RETURN_ERROR_IF(dstCapacity < ZSTD_blockHeaderSize, dstSize_tooSmall, "not enough dstCapacity to write a new compressed block"); compressedSeqsSize = ZSTD_entropyCompressSeqStore(&cctx->seqStore, &cctx->blockState.prevCBlock->entropy, &cctx->blockState.nextCBlock->entropy, &cctx->appliedParams, @@ -6060,11 +6737,11 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, cctx->entropyWorkspace, ENTROPY_WORKSPACE_SIZE /* statically allocated in resetCCtx */, cctx->bmi2); FORWARD_IF_ERROR(compressedSeqsSize, "Compressing sequences of block failed"); - DEBUGLOG(4, "Compressed sequences size: %zu", compressedSeqsSize); + DEBUGLOG(5, "Compressed sequences size: %zu", compressedSeqsSize); if (!cctx->isFirstBlock && ZSTD_maybeRLE(&cctx->seqStore) && - ZSTD_isRLE((BYTE const*)src, srcSize)) { + ZSTD_isRLE(ip, blockSize)) { /* We don't want to emit our first block as a RLE even if it qualifies because * doing so will cause the decoder (cli only) to throw a "should consume all input error." * This is only an issue for zstd <= v1.4.3 @@ -6075,12 +6752,12 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, if (compressedSeqsSize == 0) { /* ZSTD_noCompressBlock writes the block header as well */ cBlockSize = ZSTD_noCompressBlock(op, dstCapacity, ip, blockSize, lastBlock); - FORWARD_IF_ERROR(cBlockSize, "Nocompress block failed"); - DEBUGLOG(4, "Writing out nocompress block, size: %zu", cBlockSize); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_noCompressBlock failed"); + DEBUGLOG(5, "Writing out nocompress block, size: %zu", cBlockSize); } else if (compressedSeqsSize == 1) { cBlockSize = ZSTD_rleCompressBlock(op, dstCapacity, *ip, blockSize, lastBlock); - FORWARD_IF_ERROR(cBlockSize, "RLE compress block failed"); - DEBUGLOG(4, "Writing out RLE block, size: %zu", cBlockSize); + FORWARD_IF_ERROR(cBlockSize, "ZSTD_rleCompressBlock failed"); + DEBUGLOG(5, "Writing out RLE block, size: %zu", cBlockSize); } else { U32 cBlockHeader; /* Error checking and repcodes update */ @@ -6092,11 +6769,10 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, cBlockHeader = lastBlock + (((U32)bt_compressed)<<1) + (U32)(compressedSeqsSize << 3); MEM_writeLE24(op, cBlockHeader); cBlockSize = ZSTD_blockHeaderSize + compressedSeqsSize; - DEBUGLOG(4, "Writing out compressed block, size: %zu", cBlockSize); + DEBUGLOG(5, "Writing out compressed block, size: %zu", cBlockSize); } cSize += cBlockSize; - DEBUGLOG(4, "cSize running total: %zu", cSize); if (lastBlock) { break; @@ -6107,12 +6783,15 @@ ZSTD_compressSequences_internal(ZSTD_CCtx* cctx, dstCapacity -= cBlockSize; cctx->isFirstBlock = 0; } + DEBUGLOG(5, "cSize running total: %zu (remaining dstCapacity=%zu)", cSize, dstCapacity); } + DEBUGLOG(4, "cSize final total: %zu", cSize); return cSize; } -size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapacity, +size_t ZSTD_compressSequences(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, const ZSTD_Sequence* inSeqs, size_t inSeqsSize, const void* src, size_t srcSize) { @@ -6122,7 +6801,7 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapaci size_t frameHeaderSize = 0; /* Transparent initialization stage, same as compressStream2() */ - DEBUGLOG(3, "ZSTD_compressSequences()"); + DEBUGLOG(4, "ZSTD_compressSequences (dstCapacity=%zu)", dstCapacity); assert(cctx != NULL); FORWARD_IF_ERROR(ZSTD_CCtx_init_compressStream2(cctx, ZSTD_e_end, srcSize), "CCtx initialization failed"); /* Begin writing output, starting with frame header */ @@ -6150,26 +6829,34 @@ size_t ZSTD_compressSequences(ZSTD_CCtx* const cctx, void* dst, size_t dstCapaci cSize += 4; } - DEBUGLOG(3, "Final compressed size: %zu", cSize); + DEBUGLOG(4, "Final compressed size: %zu", cSize); return cSize; } /*====== Finalize ======*/ +static ZSTD_inBuffer inBuffer_forEndFlush(const ZSTD_CStream* zcs) +{ + const ZSTD_inBuffer nullInput = { NULL, 0, 0 }; + const int stableInput = (zcs->appliedParams.inBufferMode == ZSTD_bm_stable); + return stableInput ? zcs->expectedInBuffer : nullInput; +} + /*! ZSTD_flushStream() : * @return : amount of data remaining to flush */ size_t ZSTD_flushStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { - ZSTD_inBuffer input = { NULL, 0, 0 }; + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); + input.size = input.pos; /* do not ingest more input during flush */ return ZSTD_compressStream2(zcs, output, &input, ZSTD_e_flush); } size_t ZSTD_endStream(ZSTD_CStream* zcs, ZSTD_outBuffer* output) { - ZSTD_inBuffer input = { NULL, 0, 0 }; + ZSTD_inBuffer input = inBuffer_forEndFlush(zcs); size_t const remainingToFlush = ZSTD_compressStream2(zcs, output, &input, ZSTD_e_end); - FORWARD_IF_ERROR( remainingToFlush , "ZSTD_compressStream2 failed"); + FORWARD_IF_ERROR(remainingToFlush , "ZSTD_compressStream2(,,ZSTD_e_end) failed"); if (zcs->appliedParams.nbWorkers > 0) return remainingToFlush; /* minimal estimation */ /* single thread mode : attempt to calculate remaining to flush more precisely */ { size_t const lastBlockSize = zcs->frameEnded ? 0 : ZSTD_BLOCKHEADERSIZE; @@ -6291,7 +6978,7 @@ static ZSTD_compressionParameters ZSTD_getCParams_internal(int compressionLevel, cp.targetLength = (unsigned)(-clampedCompressionLevel); } /* refine parameters based on srcSize & dictSize */ - return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode); + return ZSTD_adjustCParams_internal(cp, srcSizeHint, dictSize, mode, ZSTD_ps_auto); } } @@ -6327,4 +7014,22 @@ ZSTD_parameters ZSTD_getParams(int compressionLevel, unsigned long long srcSizeH return ZSTD_getParams_internal(compressionLevel, srcSizeHint, dictSize, ZSTD_cpm_unknown); } +void ZSTD_registerSequenceProducer( + ZSTD_CCtx* zc, void* mState, + ZSTD_sequenceProducer_F* mFinder +) { + if (mFinder != NULL) { + ZSTD_externalMatchCtx emctx; + emctx.mState = mState; + emctx.mFinder = mFinder; + emctx.seqBuffer = NULL; + emctx.seqBufferCapacity = 0; + zc->externalMatchCtx = emctx; + zc->requestedParams.useSequenceProducer = 1; + } else { + ZSTD_memset(&zc->externalMatchCtx, 0, sizeof(zc->externalMatchCtx)); + zc->requestedParams.useSequenceProducer = 0; + } +} + #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/zstd_compress_internal.h b/vendor/github.com/DataDog/zstd/zstd_compress_internal.h index 9310c57..c374603 100644 --- a/vendor/github.com/DataDog/zstd/zstd_compress_internal.h +++ b/vendor/github.com/DataDog/zstd/zstd_compress_internal.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -24,6 +24,7 @@ #ifdef ZSTD_MULTITHREAD # include "zstdmt_compress.h" #endif +#include "bits.h" /* ZSTD_highbit32, ZSTD_NbCommonBytes */ #if defined (__cplusplus) extern "C" { @@ -118,12 +119,13 @@ typedef struct { /** ZSTD_buildBlockEntropyStats() : * Builds entropy for the block. * @return : 0 on success or error code */ -size_t ZSTD_buildBlockEntropyStats(seqStore_t* seqStorePtr, - const ZSTD_entropyCTables_t* prevEntropy, - ZSTD_entropyCTables_t* nextEntropy, - const ZSTD_CCtx_params* cctxParams, - ZSTD_entropyCTablesMetadata_t* entropyMetadata, - void* workspace, size_t wkspSize); +size_t ZSTD_buildBlockEntropyStats( + const seqStore_t* seqStorePtr, + const ZSTD_entropyCTables_t* prevEntropy, + ZSTD_entropyCTables_t* nextEntropy, + const ZSTD_CCtx_params* cctxParams, + ZSTD_entropyCTablesMetadata_t* entropyMetadata, + void* workspace, size_t wkspSize); /********************************* * Compression internals structs * @@ -149,6 +151,12 @@ typedef struct { size_t capacity; /* The capacity starting from `seq` pointer */ } rawSeqStore_t; +typedef struct { + U32 idx; /* Index in array of ZSTD_Sequence */ + U32 posInSequence; /* Position within sequence at idx */ + size_t posInSrc; /* Number of bytes given by sequences provided so far */ +} ZSTD_sequencePosition; + UNUSED_ATTR static const rawSeqStore_t kNullRawSeqStore = {NULL, 0, 0, 0, 0}; typedef struct { @@ -219,8 +227,10 @@ struct ZSTD_matchState_t { U32 hashLog3; /* dispatch table for matches of len==3 : larger == faster, more memory */ U32 rowHashLog; /* For row-based matchfinder: Hashlog based on nb of rows in the hashTable.*/ - U16* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */ + BYTE* tagTable; /* For row-based matchFinder: A row-based table containing the hashes and head index. */ U32 hashCache[ZSTD_ROW_HASH_CACHE_SIZE]; /* For row-based matchFinder: a cache of hashes to improve speed */ + U64 hashSalt; /* For row-based matchFinder: salts the hash for re-use of tag table */ + U32 hashSaltEntropy; /* For row-based matchFinder: collects entropy for salt generation */ U32* hashTable; U32* hashTable3; @@ -235,6 +245,18 @@ struct ZSTD_matchState_t { const ZSTD_matchState_t* dictMatchState; ZSTD_compressionParameters cParams; const rawSeqStore_t* ldmSeqStore; + + /* Controls prefetching in some dictMatchState matchfinders. + * This behavior is controlled from the cctx ms. + * This parameter has no effect in the cdict ms. */ + int prefetchCDictTables; + + /* When == 0, lazy match finders insert every position. + * When != 0, lazy match finders only insert positions they search. + * This allows them to skip much faster over incompressible data, + * at a small cost to compression ratio. + */ + int lazySkipping; }; typedef struct { @@ -331,6 +353,24 @@ struct ZSTD_CCtx_params_s { /* Internal use, for createCCtxParams() and freeCCtxParams() only */ ZSTD_customMem customMem; + + /* Controls prefetching in some dictMatchState matchfinders */ + ZSTD_paramSwitch_e prefetchCDictTables; + + /* Controls whether zstd will fall back to an internal matchfinder + * if the external matchfinder returns an error code. */ + int enableMatchFinderFallback; + + /* Indicates whether an external matchfinder has been referenced. + * Users can't set this externally. + * It is set internally in ZSTD_registerSequenceProducer(). */ + int useSequenceProducer; + + /* Adjust the max block size*/ + size_t maxBlockSize; + + /* Controls repcode search in external sequence parsing */ + ZSTD_paramSwitch_e searchForExternalRepcodes; }; /* typedef'd to ZSTD_CCtx_params within "zstd.h" */ #define COMPRESS_SEQUENCES_WORKSPACE_SIZE (sizeof(unsigned) * (MaxSeq + 2)) @@ -362,6 +402,14 @@ typedef struct { ZSTD_entropyCTablesMetadata_t entropyMetadata; } ZSTD_blockSplitCtx; +/* Context for block-level external matchfinder API */ +typedef struct { + void* mState; + ZSTD_sequenceProducer_F* mFinder; + ZSTD_Sequence* seqBuffer; + size_t seqBufferCapacity; +} ZSTD_externalMatchCtx; + struct ZSTD_CCtx_s { ZSTD_compressionStage_e stage; int cParamsChanged; /* == 1 if cParams(except wlog) or compression level are changed in requestedParams. Triggers transmission of new params to ZSTDMT (if available) then reset to 0. */ @@ -411,6 +459,7 @@ struct ZSTD_CCtx_s { /* Stable in/out buffer verification */ ZSTD_inBuffer expectedInBuffer; + size_t stableIn_notConsumed; /* nb bytes within stable input buffer that are said to be consumed but are not */ size_t expectedOutBufferSize; /* Dictionary */ @@ -430,9 +479,13 @@ struct ZSTD_CCtx_s { /* Workspace for block splitter */ ZSTD_blockSplitCtx blockSplitCtx; + + /* Workspace for external matchfinder */ + ZSTD_externalMatchCtx externalMatchCtx; }; typedef enum { ZSTD_dtlm_fast, ZSTD_dtlm_full } ZSTD_dictTableLoadMethod_e; +typedef enum { ZSTD_tfp_forCCtx, ZSTD_tfp_forCDict } ZSTD_tableFillPurpose_e; typedef enum { ZSTD_noDict = 0, @@ -454,7 +507,7 @@ typedef enum { * In this mode we take both the source size and the dictionary size * into account when selecting and adjusting the parameters. */ - ZSTD_cpm_unknown = 3, /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams. + ZSTD_cpm_unknown = 3 /* ZSTD_getCParams, ZSTD_getParams, ZSTD_adjustParams. * We don't know what these parameters are for. We default to the legacy * behavior of taking both the source size and the dict size into account * when selecting and adjusting parameters. @@ -513,9 +566,11 @@ MEM_STATIC int ZSTD_cParam_withinBounds(ZSTD_cParameter cParam, int value) /* ZSTD_noCompressBlock() : * Writes uncompressed block to dst buffer from given src. * Returns the size of the block */ -MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) +MEM_STATIC size_t +ZSTD_noCompressBlock(void* dst, size_t dstCapacity, const void* src, size_t srcSize, U32 lastBlock) { U32 const cBlockHeader24 = lastBlock + (((U32)bt_raw)<<1) + (U32)(srcSize << 3); + DEBUGLOG(5, "ZSTD_noCompressBlock (srcSize=%zu, dstCapacity=%zu)", srcSize, dstCapacity); RETURN_ERROR_IF(srcSize + ZSTD_blockHeaderSize > dstCapacity, dstSize_tooSmall, "dst buf too small for uncompressed block"); MEM_writeLE24(dst, cBlockHeader24); @@ -523,7 +578,8 @@ MEM_STATIC size_t ZSTD_noCompressBlock (void* dst, size_t dstCapacity, const voi return ZSTD_blockHeaderSize + srcSize; } -MEM_STATIC size_t ZSTD_rleCompressBlock (void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) +MEM_STATIC size_t +ZSTD_rleCompressBlock(void* dst, size_t dstCapacity, BYTE src, size_t srcSize, U32 lastBlock) { BYTE* const op = (BYTE*)dst; U32 const cBlockHeader = lastBlock + (((U32)bt_rle)<<1) + (U32)(srcSize << 3); @@ -542,7 +598,7 @@ MEM_STATIC size_t ZSTD_minGain(size_t srcSize, ZSTD_strategy strat) { U32 const minlog = (strat>=ZSTD_btultra) ? (U32)(strat) - 1 : 6; ZSTD_STATIC_ASSERT(ZSTD_btultra == 8); - assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, strat)); + assert(ZSTD_cParam_withinBounds(ZSTD_c_strategy, (int)strat)); return (srcSize >> minlog) + 2; } @@ -578,29 +634,27 @@ ZSTD_safecopyLiterals(BYTE* op, BYTE const* ip, BYTE const* const iend, BYTE con while (ip < iend) *op++ = *ip++; } -#define ZSTD_REP_MOVE (ZSTD_REP_NUM-1) -#define STORE_REPCODE_1 STORE_REPCODE(1) -#define STORE_REPCODE_2 STORE_REPCODE(2) -#define STORE_REPCODE_3 STORE_REPCODE(3) -#define STORE_REPCODE(r) (assert((r)>=1), assert((r)<=3), (r)-1) -#define STORE_OFFSET(o) (assert((o)>0), o + ZSTD_REP_MOVE) -#define STORED_IS_OFFSET(o) ((o) > ZSTD_REP_MOVE) -#define STORED_IS_REPCODE(o) ((o) <= ZSTD_REP_MOVE) -#define STORED_OFFSET(o) (assert(STORED_IS_OFFSET(o)), (o)-ZSTD_REP_MOVE) -#define STORED_REPCODE(o) (assert(STORED_IS_REPCODE(o)), (o)+1) /* returns ID 1,2,3 */ -#define STORED_TO_OFFBASE(o) ((o)+1) -#define OFFBASE_TO_STORED(o) ((o)-1) + +#define REPCODE1_TO_OFFBASE REPCODE_TO_OFFBASE(1) +#define REPCODE2_TO_OFFBASE REPCODE_TO_OFFBASE(2) +#define REPCODE3_TO_OFFBASE REPCODE_TO_OFFBASE(3) +#define REPCODE_TO_OFFBASE(r) (assert((r)>=1), assert((r)<=ZSTD_REP_NUM), (r)) /* accepts IDs 1,2,3 */ +#define OFFSET_TO_OFFBASE(o) (assert((o)>0), o + ZSTD_REP_NUM) +#define OFFBASE_IS_OFFSET(o) ((o) > ZSTD_REP_NUM) +#define OFFBASE_IS_REPCODE(o) ( 1 <= (o) && (o) <= ZSTD_REP_NUM) +#define OFFBASE_TO_OFFSET(o) (assert(OFFBASE_IS_OFFSET(o)), (o) - ZSTD_REP_NUM) +#define OFFBASE_TO_REPCODE(o) (assert(OFFBASE_IS_REPCODE(o)), (o)) /* returns ID 1,2,3 */ /*! ZSTD_storeSeq() : - * Store a sequence (litlen, litPtr, offCode and matchLength) into seqStore_t. - * @offBase_minus1 : Users should use employ macros STORE_REPCODE_X and STORE_OFFSET(). + * Store a sequence (litlen, litPtr, offBase and matchLength) into seqStore_t. + * @offBase : Users should employ macros REPCODE_TO_OFFBASE() and OFFSET_TO_OFFBASE(). * @matchLength : must be >= MINMATCH - * Allowed to overread literals up to litLimit. + * Allowed to over-read literals up to litLimit. */ HINT_INLINE UNUSED_ATTR void ZSTD_storeSeq(seqStore_t* seqStorePtr, size_t litLength, const BYTE* literals, const BYTE* litLimit, - U32 offBase_minus1, + U32 offBase, size_t matchLength) { BYTE const* const litLimit_w = litLimit - WILDCOPY_OVERLENGTH; @@ -609,8 +663,8 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr, static const BYTE* g_start = NULL; if (g_start==NULL) g_start = (const BYTE*)literals; /* note : index only works for compression within a single segment */ { U32 const pos = (U32)((const BYTE*)literals - g_start); - DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offCode%7u", - pos, (U32)litLength, (U32)matchLength, (U32)offBase_minus1); + DEBUGLOG(6, "Cpos%7u :%3u literals, match%4u bytes at offBase%7u", + pos, (U32)litLength, (U32)matchLength, (U32)offBase); } #endif assert((size_t)(seqStorePtr->sequences - seqStorePtr->sequencesStart) < seqStorePtr->maxNbSeq); @@ -620,9 +674,9 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr, assert(literals + litLength <= litLimit); if (litEnd <= litLimit_w) { /* Common case we can use wildcopy. - * First copy 16 bytes, because literals are likely short. - */ - assert(WILDCOPY_OVERLENGTH >= 16); + * First copy 16 bytes, because literals are likely short. + */ + ZSTD_STATIC_ASSERT(WILDCOPY_OVERLENGTH >= 16); ZSTD_copy16(seqStorePtr->lit, literals); if (litLength > 16) { ZSTD_wildcopy(seqStorePtr->lit+16, literals+16, (ptrdiff_t)litLength-16, ZSTD_no_overlap); @@ -641,7 +695,7 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr, seqStorePtr->sequences[0].litLength = (U16)litLength; /* match offset */ - seqStorePtr->sequences[0].offBase = STORED_TO_OFFBASE(offBase_minus1); + seqStorePtr->sequences[0].offBase = offBase; /* match Length */ assert(matchLength >= MINMATCH); @@ -659,17 +713,17 @@ ZSTD_storeSeq(seqStore_t* seqStorePtr, /* ZSTD_updateRep() : * updates in-place @rep (array of repeat offsets) - * @offBase_minus1 : sum-type, with same numeric representation as ZSTD_storeSeq() + * @offBase : sum-type, using numeric representation of ZSTD_storeSeq() */ MEM_STATIC void -ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0) +ZSTD_updateRep(U32 rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) { - if (STORED_IS_OFFSET(offBase_minus1)) { /* full offset */ + if (OFFBASE_IS_OFFSET(offBase)) { /* full offset */ rep[2] = rep[1]; rep[1] = rep[0]; - rep[0] = STORED_OFFSET(offBase_minus1); + rep[0] = OFFBASE_TO_OFFSET(offBase); } else { /* repcode */ - U32 const repCode = STORED_REPCODE(offBase_minus1) - 1 + ll0; + U32 const repCode = OFFBASE_TO_REPCODE(offBase) - 1 + ll0; if (repCode > 0) { /* note : if repCode==0, no change */ U32 const currentOffset = (repCode==ZSTD_REP_NUM) ? (rep[0] - 1) : rep[repCode]; rep[2] = (repCode >= 2) ? rep[1] : rep[2]; @@ -686,11 +740,11 @@ typedef struct repcodes_s { } repcodes_t; MEM_STATIC repcodes_t -ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0) +ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase, U32 const ll0) { repcodes_t newReps; ZSTD_memcpy(&newReps, rep, sizeof(newReps)); - ZSTD_updateRep(newReps.rep, offBase_minus1, ll0); + ZSTD_updateRep(newReps.rep, offBase, ll0); return newReps; } @@ -698,103 +752,6 @@ ZSTD_newRep(U32 const rep[ZSTD_REP_NUM], U32 const offBase_minus1, U32 const ll0 /*-************************************* * Match length counter ***************************************/ -static unsigned ZSTD_NbCommonBytes (size_t val) -{ - if (MEM_isLittleEndian()) { - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) -# if STATIC_BMI2 - return _tzcnt_u64(val) >> 3; -# else - if (val != 0) { - unsigned long r; - _BitScanForward64(&r, (U64)val); - return (unsigned)(r >> 3); - } else { - /* Should not reach this code path */ - __assume(0); - } -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 4) - return (__builtin_ctzll((U64)val) >> 3); -# else - static const int DeBruijnBytePos[64] = { 0, 0, 0, 0, 0, 1, 1, 2, - 0, 3, 1, 3, 1, 4, 2, 7, - 0, 2, 3, 6, 1, 5, 3, 5, - 1, 3, 4, 4, 2, 5, 6, 7, - 7, 0, 1, 2, 3, 3, 4, 6, - 2, 6, 5, 5, 3, 4, 5, 6, - 7, 1, 2, 4, 6, 4, 4, 5, - 7, 2, 6, 5, 7, 6, 7, 7 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - if (val != 0) { - unsigned long r; - _BitScanForward(&r, (U32)val); - return (unsigned)(r >> 3); - } else { - /* Should not reach this code path */ - __assume(0); - } -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_ctz((U32)val) >> 3); -# else - static const int DeBruijnBytePos[32] = { 0, 0, 3, 0, 3, 1, 3, 0, - 3, 2, 2, 1, 3, 2, 0, 1, - 3, 3, 1, 2, 2, 2, 2, 0, - 3, 1, 2, 0, 1, 0, 1, 1 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } - } else { /* Big Endian CPU */ - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) -# if STATIC_BMI2 - return _lzcnt_u64(val) >> 3; -# else - if (val != 0) { - unsigned long r; - _BitScanReverse64(&r, (U64)val); - return (unsigned)(r >> 3); - } else { - /* Should not reach this code path */ - __assume(0); - } -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 4) - return (__builtin_clzll(val) >> 3); -# else - unsigned r; - const unsigned n32 = sizeof(size_t)*4; /* calculate this way due to compiler complaining in 32-bits mode */ - if (!(val>>n32)) { r=4; } else { r=0; val>>=n32; } - if (!(val>>16)) { r+=2; val>>=8; } else { val>>=24; } - r += (!val); - return r; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - if (val != 0) { - unsigned long r; - _BitScanReverse(&r, (unsigned long)val); - return (unsigned)(r >> 3); - } else { - /* Should not reach this code path */ - __assume(0); - } -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return (__builtin_clz((U32)val) >> 3); -# else - unsigned r; - if (!(val>>16)) { r=2; val>>=8; } else { r=0; val>>=24; } - r += (!val); - return r; -# endif - } } -} - - MEM_STATIC size_t ZSTD_count(const BYTE* pIn, const BYTE* pMatch, const BYTE* const pInLimit) { const BYTE* const pStart = pIn; @@ -840,32 +797,43 @@ ZSTD_count_2segments(const BYTE* ip, const BYTE* match, * Hashes ***************************************/ static const U32 prime3bytes = 506832829U; -static U32 ZSTD_hash3(U32 u, U32 h) { return ((u << (32-24)) * prime3bytes) >> (32-h) ; } -MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h); } /* only in zstd_opt.h */ +static U32 ZSTD_hash3(U32 u, U32 h, U32 s) { assert(h <= 32); return (((u << (32-24)) * prime3bytes) ^ s) >> (32-h) ; } +MEM_STATIC size_t ZSTD_hash3Ptr(const void* ptr, U32 h) { return ZSTD_hash3(MEM_readLE32(ptr), h, 0); } /* only in zstd_opt.h */ +MEM_STATIC size_t ZSTD_hash3PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash3(MEM_readLE32(ptr), h, s); } static const U32 prime4bytes = 2654435761U; -static U32 ZSTD_hash4(U32 u, U32 h) { return (u * prime4bytes) >> (32-h) ; } -static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_read32(ptr), h); } +static U32 ZSTD_hash4(U32 u, U32 h, U32 s) { assert(h <= 32); return ((u * prime4bytes) ^ s) >> (32-h) ; } +static size_t ZSTD_hash4Ptr(const void* ptr, U32 h) { return ZSTD_hash4(MEM_readLE32(ptr), h, 0); } +static size_t ZSTD_hash4PtrS(const void* ptr, U32 h, U32 s) { return ZSTD_hash4(MEM_readLE32(ptr), h, s); } static const U64 prime5bytes = 889523592379ULL; -static size_t ZSTD_hash5(U64 u, U32 h) { return (size_t)(((u << (64-40)) * prime5bytes) >> (64-h)) ; } -static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h); } +static size_t ZSTD_hash5(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-40)) * prime5bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash5Ptr(const void* p, U32 h) { return ZSTD_hash5(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash5PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash5(MEM_readLE64(p), h, s); } static const U64 prime6bytes = 227718039650203ULL; -static size_t ZSTD_hash6(U64 u, U32 h) { return (size_t)(((u << (64-48)) * prime6bytes) >> (64-h)) ; } -static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h); } +static size_t ZSTD_hash6(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-48)) * prime6bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash6Ptr(const void* p, U32 h) { return ZSTD_hash6(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash6PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash6(MEM_readLE64(p), h, s); } static const U64 prime7bytes = 58295818150454627ULL; -static size_t ZSTD_hash7(U64 u, U32 h) { return (size_t)(((u << (64-56)) * prime7bytes) >> (64-h)) ; } -static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h); } +static size_t ZSTD_hash7(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u << (64-56)) * prime7bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash7Ptr(const void* p, U32 h) { return ZSTD_hash7(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash7PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash7(MEM_readLE64(p), h, s); } static const U64 prime8bytes = 0xCF1BBCDCB7A56463ULL; -static size_t ZSTD_hash8(U64 u, U32 h) { return (size_t)(((u) * prime8bytes) >> (64-h)) ; } -static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h); } +static size_t ZSTD_hash8(U64 u, U32 h, U64 s) { assert(h <= 64); return (size_t)((((u) * prime8bytes) ^ s) >> (64-h)) ; } +static size_t ZSTD_hash8Ptr(const void* p, U32 h) { return ZSTD_hash8(MEM_readLE64(p), h, 0); } +static size_t ZSTD_hash8PtrS(const void* p, U32 h, U64 s) { return ZSTD_hash8(MEM_readLE64(p), h, s); } + MEM_STATIC FORCE_INLINE_ATTR size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) { + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + switch(mls) { default: @@ -877,6 +845,24 @@ size_t ZSTD_hashPtr(const void* p, U32 hBits, U32 mls) } } +MEM_STATIC FORCE_INLINE_ATTR +size_t ZSTD_hashPtrSalted(const void* p, U32 hBits, U32 mls, const U64 hashSalt) { + /* Although some of these hashes do support hBits up to 64, some do not. + * To be on the safe side, always avoid hBits > 32. */ + assert(hBits <= 32); + + switch(mls) + { + default: + case 4: return ZSTD_hash4PtrS(p, hBits, (U32)hashSalt); + case 5: return ZSTD_hash5PtrS(p, hBits, hashSalt); + case 6: return ZSTD_hash6PtrS(p, hBits, hashSalt); + case 7: return ZSTD_hash7PtrS(p, hBits, hashSalt); + case 8: return ZSTD_hash8PtrS(p, hBits, hashSalt); + } +} + + /** ZSTD_ipow() : * Return base^exponent. */ @@ -1224,10 +1210,15 @@ ZSTD_checkDictValidity(const ZSTD_window_t* window, (unsigned)blockEndIdx, (unsigned)maxDist, (unsigned)loadedDictEnd); assert(blockEndIdx >= loadedDictEnd); - if (blockEndIdx > loadedDictEnd + maxDist) { + if (blockEndIdx > loadedDictEnd + maxDist || loadedDictEnd != window->dictLimit) { /* On reaching window size, dictionaries are invalidated. * For simplification, if window size is reached anywhere within next block, * the dictionary is invalidated for the full block. + * + * We also have to invalidate the dictionary if ZSTD_window_update() has detected + * non-contiguous segments, which means that loadedDictEnd != window->dictLimit. + * loadedDictEnd may be 0, if forceWindow is true, but in that case we never use + * dictMatchState, so setting it to NULL is not a problem. */ DEBUGLOG(6, "invalidating dictionary for current block (distance > windowSize)"); *loadedDictEndPtr = 0; @@ -1359,6 +1350,42 @@ MEM_STATIC void ZSTD_debugTable(const U32* table, U32 max) #endif +/* Short Cache */ + +/* Normally, zstd matchfinders follow this flow: + * 1. Compute hash at ip + * 2. Load index from hashTable[hash] + * 3. Check if *ip == *(base + index) + * In dictionary compression, loading *(base + index) is often an L2 or even L3 miss. + * + * Short cache is an optimization which allows us to avoid step 3 most of the time + * when the data doesn't actually match. With short cache, the flow becomes: + * 1. Compute (hash, currentTag) at ip. currentTag is an 8-bit independent hash at ip. + * 2. Load (index, matchTag) from hashTable[hash]. See ZSTD_writeTaggedIndex to understand how this works. + * 3. Only if currentTag == matchTag, check *ip == *(base + index). Otherwise, continue. + * + * Currently, short cache is only implemented in CDict hashtables. Thus, its use is limited to + * dictMatchState matchfinders. + */ +#define ZSTD_SHORT_CACHE_TAG_BITS 8 +#define ZSTD_SHORT_CACHE_TAG_MASK ((1u << ZSTD_SHORT_CACHE_TAG_BITS) - 1) + +/* Helper function for ZSTD_fillHashTable and ZSTD_fillDoubleHashTable. + * Unpacks hashAndTag into (hash, tag), then packs (index, tag) into hashTable[hash]. */ +MEM_STATIC void ZSTD_writeTaggedIndex(U32* const hashTable, size_t hashAndTag, U32 index) { + size_t const hash = hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + U32 const tag = (U32)(hashAndTag & ZSTD_SHORT_CACHE_TAG_MASK); + assert(index >> (32 - ZSTD_SHORT_CACHE_TAG_BITS) == 0); + hashTable[hash] = (index << ZSTD_SHORT_CACHE_TAG_BITS) | tag; +} + +/* Helper function for short cache matchfinders. + * Unpacks tag1 and tag2 from lower bits of packedTag1 and packedTag2, then checks if the tags match. */ +MEM_STATIC int ZSTD_comparePackedTags(size_t packedTag1, size_t packedTag2) { + U32 const tag1 = packedTag1 & ZSTD_SHORT_CACHE_TAG_MASK; + U32 const tag2 = packedTag2 & ZSTD_SHORT_CACHE_TAG_MASK; + return tag1 == tag2; +} #if defined (__cplusplus) } @@ -1456,6 +1483,53 @@ U32 ZSTD_cycleLog(U32 hashLog, ZSTD_strategy strat); */ void ZSTD_CCtx_trace(ZSTD_CCtx* cctx, size_t extraCSize); +/* Returns 0 on success, and a ZSTD_error otherwise. This function scans through an array of + * ZSTD_Sequence, storing the sequences it finds, until it reaches a block delimiter. + * Note that the block delimiter must include the last literals of the block. + */ +size_t +ZSTD_copySequencesToSeqStoreExplicitBlockDelim(ZSTD_CCtx* cctx, + ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); + +/* Returns the number of bytes to move the current read position back by. + * Only non-zero if we ended up splitting a sequence. + * Otherwise, it may return a ZSTD error if something went wrong. + * + * This function will attempt to scan through blockSize bytes + * represented by the sequences in @inSeqs, + * storing any (partial) sequences. + * + * Occasionally, we may want to change the actual number of bytes we consumed from inSeqs to + * avoid splitting a match, or to avoid splitting a match such that it would produce a match + * smaller than MINMATCH. In this case, we return the number of bytes that we didn't read from this block. + */ +size_t +ZSTD_copySequencesToSeqStoreNoBlockDelim(ZSTD_CCtx* cctx, ZSTD_sequencePosition* seqPos, + const ZSTD_Sequence* const inSeqs, size_t inSeqsSize, + const void* src, size_t blockSize, ZSTD_paramSwitch_e externalRepSearch); + + +/* =============================================================== + * Deprecated definitions that are still used internally to avoid + * deprecation warnings. These functions are exactly equivalent to + * their public variants, but avoid the deprecation warnings. + * =============================================================== */ + +size_t ZSTD_compressBegin_usingCDict_deprecated(ZSTD_CCtx* cctx, const ZSTD_CDict* cdict); + +size_t ZSTD_compressContinue_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressEnd_public(ZSTD_CCtx* cctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + +size_t ZSTD_compressBlock_deprecated(ZSTD_CCtx* cctx, void* dst, size_t dstCapacity, const void* src, size_t srcSize); + + #endif /* ZSTD_COMPRESS_H */ #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/zstd_compress_literals.c b/vendor/github.com/DataDog/zstd/zstd_compress_literals.c index c8bf022..7b4e075 100644 --- a/vendor/github.com/DataDog/zstd/zstd_compress_literals.c +++ b/vendor/github.com/DataDog/zstd/zstd_compress_literals.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -14,11 +14,36 @@ ***************************************/ #include "zstd_compress_literals.h" + +/* ************************************************************** +* Debug Traces +****************************************************************/ +#if DEBUGLEVEL >= 2 + +static size_t showHexa(const void* src, size_t srcSize) +{ + const BYTE* const ip = (const BYTE*)src; + size_t u; + for (u=0; u31) + (srcSize>4095); + DEBUGLOG(5, "ZSTD_noCompressLiterals: srcSize=%zu, dstCapacity=%zu", srcSize, dstCapacity); + RETURN_ERROR_IF(srcSize + flSize > dstCapacity, dstSize_tooSmall, ""); switch(flSize) @@ -37,16 +62,30 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, } ZSTD_memcpy(ostart + flSize, src, srcSize); - DEBUGLOG(5, "Raw literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); + DEBUGLOG(5, "Raw (uncompressed) literals: %u -> %u", (U32)srcSize, (U32)(srcSize + flSize)); return srcSize + flSize; } +static int allBytesIdentical(const void* src, size_t srcSize) +{ + assert(srcSize >= 1); + assert(src != NULL); + { const BYTE b = ((const BYTE*)src)[0]; + size_t p; + for (p=1; p31) + (srcSize>4095); - (void)dstCapacity; /* dstCapacity already guaranteed to be >=4, hence large enough */ + assert(dstCapacity >= 4); (void)dstCapacity; + assert(allBytesIdentical(src, srcSize)); switch(flSize) { @@ -64,28 +103,51 @@ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* } ostart[flSize] = *(const BYTE*)src; - DEBUGLOG(5, "RLE literals: %u -> %u", (U32)srcSize, (U32)flSize + 1); + DEBUGLOG(5, "RLE : Repeated Literal (%02X: %u times) -> %u bytes encoded", ((const BYTE*)src)[0], (U32)srcSize, (U32)flSize + 1); return flSize+1; } -size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_strategy strategy, int disableLiteralCompression, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize, - void* entropyWorkspace, size_t entropyWorkspaceSize, - const int bmi2, - unsigned suspectUncompressible) +/* ZSTD_minLiteralsToCompress() : + * returns minimal amount of literals + * for literal compression to even be attempted. + * Minimum is made tighter as compression strategy increases. + */ +static size_t +ZSTD_minLiteralsToCompress(ZSTD_strategy strategy, HUF_repeat huf_repeat) +{ + assert((int)strategy >= 0); + assert((int)strategy <= 9); + /* btultra2 : min 8 bytes; + * then 2x larger for each successive compression strategy + * max threshold 64 bytes */ + { int const shift = MIN(9-(int)strategy, 3); + size_t const mintc = (huf_repeat == HUF_repeat_valid) ? 6 : (size_t)8 << shift; + DEBUGLOG(7, "minLiteralsToCompress = %zu", mintc); + return mintc; + } +} + +size_t ZSTD_compressLiterals ( + void* dst, size_t dstCapacity, + const void* src, size_t srcSize, + void* entropyWorkspace, size_t entropyWorkspaceSize, + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, + int disableLiteralCompression, + int suspectUncompressible, + int bmi2) { - size_t const minGain = ZSTD_minGain(srcSize, strategy); size_t const lhSize = 3 + (srcSize >= 1 KB) + (srcSize >= 16 KB); BYTE* const ostart = (BYTE*)dst; U32 singleStream = srcSize < 256; symbolEncodingType_e hType = set_compressed; size_t cLitSize; - DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i srcSize=%u)", - disableLiteralCompression, (U32)srcSize); + DEBUGLOG(5,"ZSTD_compressLiterals (disableLiteralCompression=%i, srcSize=%u, dstCapacity=%zu)", + disableLiteralCompression, (U32)srcSize, dstCapacity); + + DEBUGLOG(6, "Completed literals listing (%zu bytes)", showHexa(src, srcSize)); /* Prepare nextEntropy assuming reusing the existing table */ ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); @@ -93,40 +155,51 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, if (disableLiteralCompression) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - /* small ? don't even attempt compression (speed opt) */ -# define COMPRESS_LITERALS_SIZE_MIN 63 - { size_t const minLitSize = (prevHuf->repeatMode == HUF_repeat_valid) ? 6 : COMPRESS_LITERALS_SIZE_MIN; - if (srcSize <= minLitSize) return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } + /* if too small, don't even attempt compression (speed opt) */ + if (srcSize < ZSTD_minLiteralsToCompress(strategy, prevHuf->repeatMode)) + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); RETURN_ERROR_IF(dstCapacity < lhSize+1, dstSize_tooSmall, "not enough space for compression"); { HUF_repeat repeat = prevHuf->repeatMode; - int const preferRepeat = strategy < ZSTD_lazy ? srcSize <= 1024 : 0; + int const flags = 0 + | (bmi2 ? HUF_flags_bmi2 : 0) + | (strategy < ZSTD_lazy && srcSize <= 1024 ? HUF_flags_preferRepeat : 0) + | (strategy >= HUF_OPTIMAL_DEPTH_THRESHOLD ? HUF_flags_optimalDepth : 0) + | (suspectUncompressible ? HUF_flags_suspectUncompressible : 0); + + typedef size_t (*huf_compress_f)(void*, size_t, const void*, size_t, unsigned, unsigned, void*, size_t, HUF_CElt*, HUF_repeat*, int); + huf_compress_f huf_compress; if (repeat == HUF_repeat_valid && lhSize == 3) singleStream = 1; - cLitSize = singleStream ? - HUF_compress1X_repeat( - ostart+lhSize, dstCapacity-lhSize, src, srcSize, - HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, - (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible) : - HUF_compress4X_repeat( - ostart+lhSize, dstCapacity-lhSize, src, srcSize, - HUF_SYMBOLVALUE_MAX, HUF_TABLELOG_DEFAULT, entropyWorkspace, entropyWorkspaceSize, - (HUF_CElt*)nextHuf->CTable, &repeat, preferRepeat, bmi2, suspectUncompressible); + huf_compress = singleStream ? HUF_compress1X_repeat : HUF_compress4X_repeat; + cLitSize = huf_compress(ostart+lhSize, dstCapacity-lhSize, + src, srcSize, + HUF_SYMBOLVALUE_MAX, LitHufLog, + entropyWorkspace, entropyWorkspaceSize, + (HUF_CElt*)nextHuf->CTable, + &repeat, flags); + DEBUGLOG(5, "%zu literals compressed into %zu bytes (before header)", srcSize, cLitSize); if (repeat != HUF_repeat_none) { /* reused the existing table */ - DEBUGLOG(5, "Reusing previous huffman table"); + DEBUGLOG(5, "reusing statistics from previous huffman block"); hType = set_repeat; } } - if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) { - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); - } + { size_t const minGain = ZSTD_minGain(srcSize, strategy); + if ((cLitSize==0) || (cLitSize >= srcSize - minGain) || ERR_isError(cLitSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_noCompressLiterals(dst, dstCapacity, src, srcSize); + } } if (cLitSize==1) { - ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); - return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); - } + /* A return value of 1 signals that the alphabet consists of a single symbol. + * However, in some rare circumstances, it could be the compressed size (a single byte). + * For that outcome to have a chance to happen, it's necessary that `srcSize < 8`. + * (it's also necessary to not generate statistics). + * Therefore, in such a case, actively check that all bytes are identical. */ + if ((srcSize >= 8) || allBytesIdentical(src, srcSize)) { + ZSTD_memcpy(nextHuf, prevHuf, sizeof(*prevHuf)); + return ZSTD_compressRleLiteralsBlock(dst, dstCapacity, src, srcSize); + } } if (hType == set_compressed) { /* using a newly constructed table */ @@ -137,16 +210,19 @@ size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, switch(lhSize) { case 3: /* 2 - 2 - 10 - 10 */ - { U32 const lhc = hType + ((!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); + if (!singleStream) assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); + { U32 const lhc = hType + ((U32)(!singleStream) << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<14); MEM_writeLE24(ostart, lhc); break; } case 4: /* 2 - 2 - 14 - 14 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); { U32 const lhc = hType + (2 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<18); MEM_writeLE32(ostart, lhc); break; } case 5: /* 2 - 2 - 18 - 18 */ + assert(srcSize >= MIN_LITERALS_FOR_4_STREAMS); { U32 const lhc = hType + (3 << 2) + ((U32)srcSize<<4) + ((U32)cLitSize<<22); MEM_writeLE32(ostart, lhc); ostart[4] = (BYTE)(cLitSize >> 10); diff --git a/vendor/github.com/DataDog/zstd/zstd_compress_literals.h b/vendor/github.com/DataDog/zstd/zstd_compress_literals.h index f710412..80cb1db 100644 --- a/vendor/github.com/DataDog/zstd/zstd_compress_literals.h +++ b/vendor/github.com/DataDog/zstd/zstd_compress_literals.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -17,17 +17,25 @@ size_t ZSTD_noCompressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize); +/* ZSTD_compressRleLiteralsBlock() : + * Conditions : + * - All bytes in @src are identical + * - dstCapacity >= 4 */ size_t ZSTD_compressRleLiteralsBlock (void* dst, size_t dstCapacity, const void* src, size_t srcSize); -/* If suspectUncompressible then some sampling checks will be run to potentially skip huffman coding */ -size_t ZSTD_compressLiterals (ZSTD_hufCTables_t const* prevHuf, - ZSTD_hufCTables_t* nextHuf, - ZSTD_strategy strategy, int disableLiteralCompression, - void* dst, size_t dstCapacity, +/* ZSTD_compressLiterals(): + * @entropyWorkspace: must be aligned on 4-bytes boundaries + * @entropyWorkspaceSize : must be >= HUF_WORKSPACE_SIZE + * @suspectUncompressible: sampling checks, to potentially skip huffman coding + */ +size_t ZSTD_compressLiterals (void* dst, size_t dstCapacity, const void* src, size_t srcSize, void* entropyWorkspace, size_t entropyWorkspaceSize, - const int bmi2, - unsigned suspectUncompressible); + const ZSTD_hufCTables_t* prevHuf, + ZSTD_hufCTables_t* nextHuf, + ZSTD_strategy strategy, int disableLiteralCompression, + int suspectUncompressible, + int bmi2); #endif /* ZSTD_COMPRESS_LITERALS_H */ diff --git a/vendor/github.com/DataDog/zstd/zstd_compress_sequences.c b/vendor/github.com/DataDog/zstd/zstd_compress_sequences.c index 15f1691..465f378 100644 --- a/vendor/github.com/DataDog/zstd/zstd_compress_sequences.c +++ b/vendor/github.com/DataDog/zstd/zstd_compress_sequences.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -59,7 +59,7 @@ static unsigned ZSTD_useLowProbCount(size_t const nbSeq) { /* Heuristic: This should cover most blocks <= 16K and * start to fade out after 16K to about 32K depending on - * comprssibility. + * compressibility. */ return nbSeq >= 2048; } @@ -167,7 +167,7 @@ ZSTD_selectEncodingType( if (mostFrequent == nbSeq) { *repeatMode = FSE_repeat_none; if (isDefaultAllowed && nbSeq <= 2) { - /* Prefer set_basic over set_rle when there are 2 or less symbols, + /* Prefer set_basic over set_rle when there are 2 or fewer symbols, * since RLE uses 1 byte, but set_basic uses 5-6 bits per symbol. * If basic encoding isn't possible, always choose RLE. */ diff --git a/vendor/github.com/DataDog/zstd/zstd_compress_sequences.h b/vendor/github.com/DataDog/zstd/zstd_compress_sequences.h index a294ffe..d3d2a4d 100644 --- a/vendor/github.com/DataDog/zstd/zstd_compress_sequences.h +++ b/vendor/github.com/DataDog/zstd/zstd_compress_sequences.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_compress_superblock.c b/vendor/github.com/DataDog/zstd/zstd_compress_superblock.c index 2e61e21..61302a4 100644 --- a/vendor/github.com/DataDog/zstd/zstd_compress_superblock.c +++ b/vendor/github.com/DataDog/zstd/zstd_compress_superblock.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -37,13 +37,14 @@ * If it is set_compressed, first sub-block's literals section will be Treeless_Literals_Block * and the following sub-blocks' literals sections will be Treeless_Literals_Block. * @return : compressed size of literals section of a sub-block - * Or 0 if it unable to compress. + * Or 0 if unable to compress. * Or error code */ -static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, - const ZSTD_hufCTablesMetadata_t* hufMetadata, - const BYTE* literals, size_t litSize, - void* dst, size_t dstSize, - const int bmi2, int writeEntropy, int* entropyWritten) +static size_t +ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, + const ZSTD_hufCTablesMetadata_t* hufMetadata, + const BYTE* literals, size_t litSize, + void* dst, size_t dstSize, + const int bmi2, int writeEntropy, int* entropyWritten) { size_t const header = writeEntropy ? 200 : 0; size_t const lhSize = 3 + (litSize >= (1 KB - header)) + (litSize >= (16 KB - header)); @@ -54,8 +55,6 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, symbolEncodingType_e hType = writeEntropy ? hufMetadata->hType : set_repeat; size_t cLitSize = 0; - (void)bmi2; /* TODO bmi2... */ - DEBUGLOG(5, "ZSTD_compressSubBlock_literal (litSize=%zu, lhSize=%zu, writeEntropy=%d)", litSize, lhSize, writeEntropy); *entropyWritten = 0; @@ -77,9 +76,9 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, DEBUGLOG(5, "ZSTD_compressSubBlock_literal (hSize=%zu)", hufMetadata->hufDesSize); } - /* TODO bmi2 */ - { const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable) - : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable); + { int const flags = bmi2 ? HUF_flags_bmi2 : 0; + const size_t cSize = singleStream ? HUF_compress1X_usingCTable(op, oend-op, literals, litSize, hufTable, flags) + : HUF_compress4X_usingCTable(op, oend-op, literals, litSize, hufTable, flags); op += cSize; cLitSize += cSize; if (cSize == 0 || ERR_isError(cSize)) { @@ -127,7 +126,11 @@ static size_t ZSTD_compressSubBlock_literal(const HUF_CElt* hufTable, return op-ostart; } -static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* sequences, size_t nbSeq, size_t litSize, int lastSequence) { +static size_t +ZSTD_seqDecompressedSize(seqStore_t const* seqStore, + const seqDef* sequences, size_t nbSeq, + size_t litSize, int lastSequence) +{ const seqDef* const sstart = sequences; const seqDef* const send = sequences + nbSeq; const seqDef* sp = sstart; @@ -157,13 +160,14 @@ static size_t ZSTD_seqDecompressedSize(seqStore_t const* seqStore, const seqDef* * @return : compressed size of sequences section of a sub-block * Or 0 if it is unable to compress * Or error code. */ -static size_t ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables, - const ZSTD_fseCTablesMetadata_t* fseMetadata, - const seqDef* sequences, size_t nbSeq, - const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, - const ZSTD_CCtx_params* cctxParams, - void* dst, size_t dstCapacity, - const int bmi2, int writeEntropy, int* entropyWritten) +static size_t +ZSTD_compressSubBlock_sequences(const ZSTD_fseCTables_t* fseTables, + const ZSTD_fseCTablesMetadata_t* fseMetadata, + const seqDef* sequences, size_t nbSeq, + const BYTE* llCode, const BYTE* mlCode, const BYTE* ofCode, + const ZSTD_CCtx_params* cctxParams, + void* dst, size_t dstCapacity, + const int bmi2, int writeEntropy, int* entropyWritten) { const int longOffsets = cctxParams->cParams.windowLog > STREAM_ACCUMULATOR_MIN; BYTE* const ostart = (BYTE*)dst; @@ -540,7 +544,7 @@ static size_t ZSTD_compressSubBlock_multi(const seqStore_t* seqStorePtr, repcodes_t rep; ZSTD_memcpy(&rep, prevCBlock->rep, sizeof(rep)); for (seq = sstart; seq < sp; ++seq) { - ZSTD_updateRep(rep.rep, seq->offBase - 1, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); + ZSTD_updateRep(rep.rep, seq->offBase, ZSTD_getSequenceLength(seqStorePtr, seq).litLength == 0); } ZSTD_memcpy(nextCBlock->rep, &rep, sizeof(rep)); } diff --git a/vendor/github.com/DataDog/zstd/zstd_compress_superblock.h b/vendor/github.com/DataDog/zstd/zstd_compress_superblock.h index 2308218..fdc4b16 100644 --- a/vendor/github.com/DataDog/zstd/zstd_compress_superblock.h +++ b/vendor/github.com/DataDog/zstd/zstd_compress_superblock.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_cwksp.h b/vendor/github.com/DataDog/zstd/zstd_cwksp.h index d530c58..a2746c5 100644 --- a/vendor/github.com/DataDog/zstd/zstd_cwksp.h +++ b/vendor/github.com/DataDog/zstd/zstd_cwksp.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -15,7 +15,9 @@ /*-************************************* * Dependencies ***************************************/ +#include "allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ #include "zstd_internal.h" +#include "portability_macros.h" #if defined (__cplusplus) extern "C" { @@ -45,8 +47,9 @@ extern "C" { ***************************************/ typedef enum { ZSTD_cwksp_alloc_objects, - ZSTD_cwksp_alloc_buffers, - ZSTD_cwksp_alloc_aligned + ZSTD_cwksp_alloc_aligned_init_once, + ZSTD_cwksp_alloc_aligned, + ZSTD_cwksp_alloc_buffers } ZSTD_cwksp_alloc_phase_e; /** @@ -99,8 +102,8 @@ typedef enum { * * Workspace Layout: * - * [ ... workspace ... ] - * [objects][tables ... ->] free space [<- ... aligned][<- ... buffers] + * [ ... workspace ... ] + * [objects][tables ->] free space [<- buffers][<- aligned][<- init once] * * The various objects that live in the workspace are divided into the * following categories, and are allocated separately: @@ -124,9 +127,18 @@ typedef enum { * uint32_t arrays, all of whose values are between 0 and (nextSrc - base). * Their sizes depend on the cparams. These tables are 64-byte aligned. * - * - Aligned: these buffers are used for various purposes that require 4 byte - * alignment, but don't require any initialization before they're used. These - * buffers are each aligned to 64 bytes. + * - Init once: these buffers require to be initialized at least once before + * use. They should be used when we want to skip memory initialization + * while not triggering memory checkers (like Valgrind) when reading from + * from this memory without writing to it first. + * These buffers should be used carefully as they might contain data + * from previous compressions. + * Buffers are aligned to 64 bytes. + * + * - Aligned: these buffers don't require any initialization before they're + * used. The user of the buffer should make sure they write into a buffer + * location before reading from it. + * Buffers are aligned to 64 bytes. * * - Buffers: these buffers are used for various purposes that don't require * any alignment or initialization before they're used. This means they can @@ -138,8 +150,9 @@ typedef enum { * correctly packed into the workspace buffer. That order is: * * 1. Objects - * 2. Buffers - * 3. Aligned/Tables + * 2. Init once / Tables + * 3. Aligned / Tables + * 4. Buffers / Tables * * Attempts to reserve objects of different types out of order will fail. */ @@ -151,6 +164,7 @@ typedef struct { void* tableEnd; void* tableValidEnd; void* allocStart; + void* initOnceStart; BYTE allocFailed; int workspaceOversizedDuration; @@ -163,6 +177,7 @@ typedef struct { ***************************************/ MEM_STATIC size_t ZSTD_cwksp_available_space(ZSTD_cwksp* ws); +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws); MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { (void)ws; @@ -172,6 +187,20 @@ MEM_STATIC void ZSTD_cwksp_assert_internal_consistency(ZSTD_cwksp* ws) { assert(ws->tableEnd <= ws->allocStart); assert(ws->tableValidEnd <= ws->allocStart); assert(ws->allocStart <= ws->workspaceEnd); + assert(ws->initOnceStart <= ZSTD_cwksp_initialAllocStart(ws)); + assert(ws->workspace <= ws->initOnceStart); +#if ZSTD_MEMORY_SANITIZER + { + intptr_t const offset = __msan_test_shadow(ws->initOnceStart, + (U8*)ZSTD_cwksp_initialAllocStart(ws) - (U8*)ws->initOnceStart); +#if defined(ZSTD_MSAN_PRINT) + if(offset!=-1) { + __msan_print_shadow((U8*)ws->initOnceStart + offset - 8, 32); + } +#endif + assert(offset==-1); + }; +#endif } /** @@ -218,14 +247,10 @@ MEM_STATIC size_t ZSTD_cwksp_aligned_alloc_size(size_t size) { * for internal purposes (currently only alignment). */ MEM_STATIC size_t ZSTD_cwksp_slack_space_required(void) { - /* For alignment, the wksp will always allocate an additional n_1=[1, 64] bytes - * to align the beginning of tables section, as well as another n_2=[0, 63] bytes - * to align the beginning of the aligned section. - * - * n_1 + n_2 == 64 bytes if the cwksp is freshly allocated, due to tables and - * aligneds being sized in multiples of 64 bytes. + /* For alignment, the wksp will always allocate an additional 2*ZSTD_CWKSP_ALIGNMENT_BYTES + * bytes to align the beginning of tables section and end of buffers; */ - size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES; + size_t const slackSpace = ZSTD_CWKSP_ALIGNMENT_BYTES * 2; return slackSpace; } @@ -238,10 +263,18 @@ MEM_STATIC size_t ZSTD_cwksp_bytes_to_align_ptr(void* ptr, const size_t alignByt size_t const alignBytesMask = alignBytes - 1; size_t const bytes = (alignBytes - ((size_t)ptr & (alignBytesMask))) & alignBytesMask; assert((alignBytes & alignBytesMask) == 0); - assert(bytes != ZSTD_CWKSP_ALIGNMENT_BYTES); + assert(bytes < alignBytes); return bytes; } +/** + * Returns the initial value for allocStart which is used to determine the position from + * which we can allocate from the end of the workspace. + */ +MEM_STATIC void* ZSTD_cwksp_initialAllocStart(ZSTD_cwksp* ws) { + return (void*)((size_t)ws->workspaceEnd & ~(ZSTD_CWKSP_ALIGNMENT_BYTES-1)); +} + /** * Internal function. Do not use directly. * Reserves the given number of bytes within the aligned/buffer segment of the wksp, @@ -282,27 +315,16 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase { assert(phase >= ws->phase); if (phase > ws->phase) { - /* Going from allocating objects to allocating buffers */ - if (ws->phase < ZSTD_cwksp_alloc_buffers && - phase >= ZSTD_cwksp_alloc_buffers) { + /* Going from allocating objects to allocating initOnce / tables */ + if (ws->phase < ZSTD_cwksp_alloc_aligned_init_once && + phase >= ZSTD_cwksp_alloc_aligned_init_once) { ws->tableValidEnd = ws->objectEnd; - } + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); - /* Going from allocating buffers to allocating aligneds/tables */ - if (ws->phase < ZSTD_cwksp_alloc_aligned && - phase >= ZSTD_cwksp_alloc_aligned) { - { /* Align the start of the "aligned" to 64 bytes. Use [1, 64] bytes. */ - size_t const bytesToAlign = - ZSTD_CWKSP_ALIGNMENT_BYTES - ZSTD_cwksp_bytes_to_align_ptr(ws->allocStart, ZSTD_CWKSP_ALIGNMENT_BYTES); - DEBUGLOG(5, "reserving aligned alignment addtl space: %zu", bytesToAlign); - ZSTD_STATIC_ASSERT((ZSTD_CWKSP_ALIGNMENT_BYTES & (ZSTD_CWKSP_ALIGNMENT_BYTES - 1)) == 0); /* power of 2 */ - RETURN_ERROR_IF(!ZSTD_cwksp_reserve_internal_buffer_space(ws, bytesToAlign), - memory_allocation, "aligned phase - alignment initial allocation failed!"); - } { /* Align the start of the tables to 64 bytes. Use [0, 63] bytes */ - void* const alloc = ws->objectEnd; + void *const alloc = ws->objectEnd; size_t const bytesToAlign = ZSTD_cwksp_bytes_to_align_ptr(alloc, ZSTD_CWKSP_ALIGNMENT_BYTES); - void* const objectEnd = (BYTE*)alloc + bytesToAlign; + void *const objectEnd = (BYTE *) alloc + bytesToAlign; DEBUGLOG(5, "reserving table alignment addtl space: %zu", bytesToAlign); RETURN_ERROR_IF(objectEnd > ws->workspaceEnd, memory_allocation, "table phase - alignment initial allocation failed!"); @@ -310,7 +332,9 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase ws->tableEnd = objectEnd; /* table area starts being empty */ if (ws->tableValidEnd < ws->tableEnd) { ws->tableValidEnd = ws->tableEnd; - } } } + } + } + } ws->phase = phase; ZSTD_cwksp_assert_internal_consistency(ws); } @@ -322,7 +346,7 @@ ZSTD_cwksp_internal_advance_phase(ZSTD_cwksp* ws, ZSTD_cwksp_alloc_phase_e phase */ MEM_STATIC int ZSTD_cwksp_owns_buffer(const ZSTD_cwksp* ws, const void* ptr) { - return (ptr != NULL) && (ws->workspace <= ptr) && (ptr <= ws->workspaceEnd); + return (ptr != NULL) && (ws->workspace <= ptr) && (ptr < ws->workspaceEnd); } /** @@ -349,7 +373,9 @@ ZSTD_cwksp_reserve_internal(ZSTD_cwksp* ws, size_t bytes, ZSTD_cwksp_alloc_phase if (alloc) { alloc = (BYTE *)alloc + ZSTD_CWKSP_ASAN_REDZONE_SIZE; if (ws->isStatic == ZSTD_cwksp_dynamic_alloc) { - __asan_unpoison_memory_region(alloc, bytes); + /* We need to keep the redzone poisoned while unpoisoning the bytes that + * are actually allocated. */ + __asan_unpoison_memory_region(alloc, bytes - 2 * ZSTD_CWKSP_ASAN_REDZONE_SIZE); } } #endif @@ -365,6 +391,36 @@ MEM_STATIC BYTE* ZSTD_cwksp_reserve_buffer(ZSTD_cwksp* ws, size_t bytes) return (BYTE*)ZSTD_cwksp_reserve_internal(ws, bytes, ZSTD_cwksp_alloc_buffers); } +/** + * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). + * This memory has been initialized at least once in the past. + * This doesn't mean it has been initialized this time, and it might contain data from previous + * operations. + * The main usage is for algorithms that might need read access into uninitialized memory. + * The algorithm must maintain safety under these conditions and must make sure it doesn't + * leak any of the past data (directly or in side channels). + */ +MEM_STATIC void* ZSTD_cwksp_reserve_aligned_init_once(ZSTD_cwksp* ws, size_t bytes) +{ + size_t const alignedBytes = ZSTD_cwksp_align(bytes, ZSTD_CWKSP_ALIGNMENT_BYTES); + void* ptr = ZSTD_cwksp_reserve_internal(ws, alignedBytes, ZSTD_cwksp_alloc_aligned_init_once); + assert(((size_t)ptr & (ZSTD_CWKSP_ALIGNMENT_BYTES-1))== 0); + if(ptr && ptr < ws->initOnceStart) { + /* We assume the memory following the current allocation is either: + * 1. Not usable as initOnce memory (end of workspace) + * 2. Another initOnce buffer that has been allocated before (and so was previously memset) + * 3. An ASAN redzone, in which case we don't want to write on it + * For these reasons it should be fine to not explicitly zero every byte up to ws->initOnceStart. + * Note that we assume here that MSAN and ASAN cannot run in the same time. */ + ZSTD_memset(ptr, 0, MIN((size_t)((U8*)ws->initOnceStart - (U8*)ptr), alignedBytes)); + ws->initOnceStart = ptr; + } +#if ZSTD_MEMORY_SANITIZER + assert(__msan_test_shadow(ptr, bytes) == -1); +#endif + return ptr; +} + /** * Reserves and returns memory sized on and aligned on ZSTD_CWKSP_ALIGNMENT_BYTES (64 bytes). */ @@ -383,13 +439,17 @@ MEM_STATIC void* ZSTD_cwksp_reserve_aligned(ZSTD_cwksp* ws, size_t bytes) */ MEM_STATIC void* ZSTD_cwksp_reserve_table(ZSTD_cwksp* ws, size_t bytes) { - const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned; + const ZSTD_cwksp_alloc_phase_e phase = ZSTD_cwksp_alloc_aligned_init_once; void* alloc; void* end; void* top; - if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) { - return NULL; + /* We can only start allocating tables after we are done reserving space for objects at the + * start of the workspace */ + if(ws->phase < phase) { + if (ZSTD_isError(ZSTD_cwksp_internal_advance_phase(ws, phase))) { + return NULL; + } } alloc = ws->tableEnd; end = (BYTE *)alloc + bytes; @@ -468,11 +528,19 @@ MEM_STATIC void ZSTD_cwksp_mark_tables_dirty(ZSTD_cwksp* ws) #if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) /* To validate that the table re-use logic is sound, and that we don't * access table space that we haven't cleaned, we re-"poison" the table - * space every time we mark it dirty. */ + * space every time we mark it dirty. + * Since tableValidEnd space and initOnce space may overlap we don't poison + * the initOnce portion as it break its promise. This means that this poisoning + * check isn't always applied fully. */ { size_t size = (BYTE*)ws->tableValidEnd - (BYTE*)ws->objectEnd; assert(__msan_test_shadow(ws->objectEnd, size) == -1); - __msan_poison(ws->objectEnd, size); + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + __msan_poison(ws->objectEnd, size); + } else { + assert(ws->initOnceStart >= ws->objectEnd); + __msan_poison(ws->objectEnd, (BYTE*)ws->initOnceStart - (BYTE*)ws->objectEnd); + } } #endif @@ -500,7 +568,7 @@ MEM_STATIC void ZSTD_cwksp_clean_tables(ZSTD_cwksp* ws) { assert(ws->tableValidEnd >= ws->objectEnd); assert(ws->tableValidEnd <= ws->allocStart); if (ws->tableValidEnd < ws->tableEnd) { - ZSTD_memset(ws->tableValidEnd, 0, (BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd); + ZSTD_memset(ws->tableValidEnd, 0, (size_t)((BYTE*)ws->tableEnd - (BYTE*)ws->tableValidEnd)); } ZSTD_cwksp_mark_tables_clean(ws); } @@ -537,11 +605,14 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { #if ZSTD_MEMORY_SANITIZER && !defined (ZSTD_MSAN_DONT_POISON_WORKSPACE) /* To validate that the context re-use logic is sound, and that we don't * access stuff that this compression hasn't initialized, we re-"poison" - * the workspace (or at least the non-static, non-table parts of it) - * every time we start a new compression. */ + * the workspace except for the areas in which we expect memory re-use + * without initialization (objects, valid tables area and init once + * memory). */ { - size_t size = (BYTE*)ws->workspaceEnd - (BYTE*)ws->tableValidEnd; - __msan_poison(ws->tableValidEnd, size); + if((BYTE*)ws->tableValidEnd < (BYTE*)ws->initOnceStart) { + size_t size = (BYTE*)ws->initOnceStart - (BYTE*)ws->tableValidEnd; + __msan_poison(ws->tableValidEnd, size); + } } #endif @@ -557,10 +628,10 @@ MEM_STATIC void ZSTD_cwksp_clear(ZSTD_cwksp* ws) { #endif ws->tableEnd = ws->objectEnd; - ws->allocStart = ws->workspaceEnd; + ws->allocStart = ZSTD_cwksp_initialAllocStart(ws); ws->allocFailed = 0; - if (ws->phase > ZSTD_cwksp_alloc_buffers) { - ws->phase = ZSTD_cwksp_alloc_buffers; + if (ws->phase > ZSTD_cwksp_alloc_aligned_init_once) { + ws->phase = ZSTD_cwksp_alloc_aligned_init_once; } ZSTD_cwksp_assert_internal_consistency(ws); } @@ -577,6 +648,7 @@ MEM_STATIC void ZSTD_cwksp_init(ZSTD_cwksp* ws, void* start, size_t size, ZSTD_c ws->workspaceEnd = (BYTE*)start + size; ws->objectEnd = ws->workspace; ws->tableValidEnd = ws->objectEnd; + ws->initOnceStart = ZSTD_cwksp_initialAllocStart(ws); ws->phase = ZSTD_cwksp_alloc_objects; ws->isStatic = isStatic; ZSTD_cwksp_clear(ws); @@ -629,17 +701,11 @@ MEM_STATIC int ZSTD_cwksp_reserve_failed(const ZSTD_cwksp* ws) { * Returns if the estimated space needed for a wksp is within an acceptable limit of the * actual amount of space used. */ -MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp* const ws, - size_t const estimatedSpace, int resizedWorkspace) { - if (resizedWorkspace) { - /* Resized/newly allocated wksp should have exact bounds */ - return ZSTD_cwksp_used(ws) == estimatedSpace; - } else { - /* Due to alignment, when reusing a workspace, we can actually consume 63 fewer or more bytes - * than estimatedSpace. See the comments in zstd_cwksp.h for details. - */ - return (ZSTD_cwksp_used(ws) >= estimatedSpace - 63) && (ZSTD_cwksp_used(ws) <= estimatedSpace + 63); - } +MEM_STATIC int ZSTD_cwksp_estimated_space_within_bounds(const ZSTD_cwksp *const ws, size_t const estimatedSpace) { + /* We have an alignment space between objects and tables between tables and buffers, so we can have up to twice + * the alignment bytes difference between estimation and actual usage */ + return (estimatedSpace - ZSTD_cwksp_slack_space_required()) <= ZSTD_cwksp_used(ws) && + ZSTD_cwksp_used(ws) <= estimatedSpace; } diff --git a/vendor/github.com/DataDog/zstd/zstd_ddict.c b/vendor/github.com/DataDog/zstd/zstd_ddict.c index ddad94d..c77a9df 100644 --- a/vendor/github.com/DataDog/zstd/zstd_ddict.c +++ b/vendor/github.com/DataDog/zstd/zstd_ddict.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -15,12 +15,12 @@ /*-******************************************************* * Dependencies *********************************************************/ +#include "allocations.h" /* ZSTD_customMalloc, ZSTD_customFree */ #include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ #include "cpu.h" /* bmi2 */ #include "mem.h" /* low level memory routines */ #define FSE_STATIC_LINKING_ONLY #include "fse.h" -#define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_decompress_internal.h" #include "zstd_ddict.h" @@ -135,7 +135,7 @@ static size_t ZSTD_initDDict_internal(ZSTD_DDict* ddict, ZSTD_memcpy(internalBuffer, dict, dictSize); } ddict->dictSize = dictSize; - ddict->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + ddict->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ /* parse dictionary content */ FORWARD_IF_ERROR( ZSTD_loadEntropy_intoDDict(ddict, dictContentType) , ""); @@ -241,7 +241,7 @@ size_t ZSTD_sizeof_DDict(const ZSTD_DDict* ddict) unsigned ZSTD_getDictID_fromDDict(const ZSTD_DDict* ddict) { if (ddict==NULL) return 0; - return ZSTD_getDictID_fromDict(ddict->dictContent, ddict->dictSize); + return ddict->dictID; } #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/zstd_ddict.h b/vendor/github.com/DataDog/zstd/zstd_ddict.h index d0ff0af..97e432e 100644 --- a/vendor/github.com/DataDog/zstd/zstd_ddict.h +++ b/vendor/github.com/DataDog/zstd/zstd_ddict.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_decompress.c b/vendor/github.com/DataDog/zstd/zstd_decompress.c index 1f2606a..d48e87b 100644 --- a/vendor/github.com/DataDog/zstd/zstd_decompress.c +++ b/vendor/github.com/DataDog/zstd/zstd_decompress.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -56,17 +56,18 @@ /*-******************************************************* * Dependencies *********************************************************/ +#include "allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ #include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memmove, ZSTD_memset */ #include "mem.h" /* low level memory routines */ #define FSE_STATIC_LINKING_ONLY #include "fse.h" -#define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "xxhash.h" /* XXH64_reset, XXH64_update, XXH64_digest, XXH64 */ #include "zstd_internal.h" /* blockProperties_t */ #include "zstd_decompress_internal.h" /* ZSTD_DCtx */ #include "zstd_ddict.h" /* ZSTD_DDictDictContent */ #include "zstd_decompress_block.h" /* ZSTD_decompressBlock_internal */ +#include "bits.h" /* ZSTD_highbit32 */ #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) #include "zstd_legacy.h" @@ -79,11 +80,11 @@ *************************************/ #define DDICT_HASHSET_MAX_LOAD_FACTOR_COUNT_MULT 4 -#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. - * Currently, that means a 0.75 load factor. - * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded - * the load factor of the ddict hash set. - */ +#define DDICT_HASHSET_MAX_LOAD_FACTOR_SIZE_MULT 3 /* These two constants represent SIZE_MULT/COUNT_MULT load factor without using a float. + * Currently, that means a 0.75 load factor. + * So, if count * COUNT_MULT / size * SIZE_MULT != 0, then we've exceeded + * the load factor of the ddict hash set. + */ #define DDICT_HASHSET_TABLE_BASE_SIZE 64 #define DDICT_HASHSET_RESIZE_FACTOR 2 @@ -244,6 +245,7 @@ static void ZSTD_DCtx_resetParameters(ZSTD_DCtx* dctx) dctx->outBufferMode = ZSTD_bm_buffered; dctx->forceIgnoreChecksum = ZSTD_d_validateChecksum; dctx->refMultipleDDicts = ZSTD_rmd_refSingleDDict; + dctx->disableHufAsm = 0; } static void ZSTD_initDCtx_internal(ZSTD_DCtx* dctx) @@ -439,16 +441,40 @@ size_t ZSTD_frameHeaderSize(const void* src, size_t srcSize) * note : only works for formats ZSTD_f_zstd1 and ZSTD_f_zstd1_magicless * @return : 0, `zfhPtr` is correctly filled, * >0, `srcSize` is too small, value is wanted `srcSize` amount, - * or an error code, which can be tested using ZSTD_isError() */ +** or an error code, which can be tested using ZSTD_isError() */ size_t ZSTD_getFrameHeader_advanced(ZSTD_frameHeader* zfhPtr, const void* src, size_t srcSize, ZSTD_format_e format) { const BYTE* ip = (const BYTE*)src; size_t const minInputSize = ZSTD_startingInputLength(format); - ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzer do not understand that zfhPtr is only going to be read only if return value is zero, since they are 2 different signals */ - if (srcSize < minInputSize) return minInputSize; - RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter"); + DEBUGLOG(5, "ZSTD_getFrameHeader_advanced: minInputSize = %zu, srcSize = %zu", minInputSize, srcSize); + if (srcSize > 0) { + /* note : technically could be considered an assert(), since it's an invalid entry */ + RETURN_ERROR_IF(src==NULL, GENERIC, "invalid parameter : src==NULL, but srcSize>0"); + } + if (srcSize < minInputSize) { + if (srcSize > 0 && format != ZSTD_f_zstd1_magicless) { + /* when receiving less than @minInputSize bytes, + * control these bytes at least correspond to a supported magic number + * in order to error out early if they don't. + **/ + size_t const toCopy = MIN(4, srcSize); + unsigned char hbuf[4]; MEM_writeLE32(hbuf, ZSTD_MAGICNUMBER); + assert(src != NULL); + ZSTD_memcpy(hbuf, src, toCopy); + if ( MEM_readLE32(hbuf) != ZSTD_MAGICNUMBER ) { + /* not a zstd frame : let's check if it's a skippable frame */ + MEM_writeLE32(hbuf, ZSTD_MAGIC_SKIPPABLE_START); + ZSTD_memcpy(hbuf, src, toCopy); + if ((MEM_readLE32(hbuf) & ZSTD_MAGIC_SKIPPABLE_MASK) != ZSTD_MAGIC_SKIPPABLE_START) { + RETURN_ERROR(prefix_unknown, + "first bytes don't correspond to any supported magic number"); + } } } + return minInputSize; + } + + ZSTD_memset(zfhPtr, 0, sizeof(*zfhPtr)); /* not strictly necessary, but static analyzers may not understand that zfhPtr will be read only if return value is zero, since they are 2 different signals */ if ( (format != ZSTD_f_zstd1_magicless) && (MEM_readLE32(src) != ZSTD_MAGICNUMBER) ) { if ((MEM_readLE32(src) & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { @@ -564,49 +590,52 @@ static size_t readSkippableFrameSize(void const* src, size_t srcSize) sizeU32 = MEM_readLE32((BYTE const*)src + ZSTD_FRAMEIDSIZE); RETURN_ERROR_IF((U32)(sizeU32 + ZSTD_SKIPPABLEHEADERSIZE) < sizeU32, frameParameter_unsupported, ""); - { - size_t const skippableSize = skippableHeaderSize + sizeU32; + { size_t const skippableSize = skippableHeaderSize + sizeU32; RETURN_ERROR_IF(skippableSize > srcSize, srcSize_wrong, ""); return skippableSize; } } /*! ZSTD_readSkippableFrame() : - * Retrieves a zstd skippable frame containing data given by src, and writes it to dst buffer. + * Retrieves content of a skippable frame, and writes it to dst buffer. * * The parameter magicVariant will receive the magicVariant that was supplied when the frame was written, * i.e. magicNumber - ZSTD_MAGIC_SKIPPABLE_START. This can be NULL if the caller is not interested * in the magicVariant. * - * Returns an error if destination buffer is not large enough, or if the frame is not skippable. + * Returns an error if destination buffer is not large enough, or if this is not a valid skippable frame. * * @return : number of bytes written or a ZSTD error. */ -ZSTDLIB_API size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, unsigned* magicVariant, - const void* src, size_t srcSize) +size_t ZSTD_readSkippableFrame(void* dst, size_t dstCapacity, + unsigned* magicVariant, /* optional, can be NULL */ + const void* src, size_t srcSize) { - U32 const magicNumber = MEM_readLE32(src); - size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); - size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; + RETURN_ERROR_IF(srcSize < ZSTD_SKIPPABLEHEADERSIZE, srcSize_wrong, ""); - /* check input validity */ - RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); - RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); - RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + { U32 const magicNumber = MEM_readLE32(src); + size_t skippableFrameSize = readSkippableFrameSize(src, srcSize); + size_t skippableContentSize = skippableFrameSize - ZSTD_SKIPPABLEHEADERSIZE; - /* deliver payload */ - if (skippableContentSize > 0 && dst != NULL) - ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); - if (magicVariant != NULL) - *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; - return skippableContentSize; + /* check input validity */ + RETURN_ERROR_IF(!ZSTD_isSkippableFrame(src, srcSize), frameParameter_unsupported, ""); + RETURN_ERROR_IF(skippableFrameSize < ZSTD_SKIPPABLEHEADERSIZE || skippableFrameSize > srcSize, srcSize_wrong, ""); + RETURN_ERROR_IF(skippableContentSize > dstCapacity, dstSize_tooSmall, ""); + + /* deliver payload */ + if (skippableContentSize > 0 && dst != NULL) + ZSTD_memcpy(dst, (const BYTE *)src + ZSTD_SKIPPABLEHEADERSIZE, skippableContentSize); + if (magicVariant != NULL) + *magicVariant = magicNumber - ZSTD_MAGIC_SKIPPABLE_START; + return skippableContentSize; + } } /** ZSTD_findDecompressedSize() : - * compatible with legacy mode * `srcSize` must be the exact length of some number of ZSTD compressed and/or * skippable frames - * @return : decompressed size of the frames contained */ + * note: compatible with legacy mode + * @return : decompressed size of the frames contained */ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) { unsigned long long totalDstSize = 0; @@ -616,9 +645,7 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { size_t const skippableSize = readSkippableFrameSize(src, srcSize); - if (ZSTD_isError(skippableSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } + if (ZSTD_isError(skippableSize)) return ZSTD_CONTENTSIZE_ERROR; assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; @@ -626,17 +653,17 @@ unsigned long long ZSTD_findDecompressedSize(const void* src, size_t srcSize) continue; } - { unsigned long long const ret = ZSTD_getFrameContentSize(src, srcSize); - if (ret >= ZSTD_CONTENTSIZE_ERROR) return ret; + { unsigned long long const fcs = ZSTD_getFrameContentSize(src, srcSize); + if (fcs >= ZSTD_CONTENTSIZE_ERROR) return fcs; - /* check for overflow */ - if (totalDstSize + ret < totalDstSize) return ZSTD_CONTENTSIZE_ERROR; - totalDstSize += ret; + if (totalDstSize + fcs < totalDstSize) + return ZSTD_CONTENTSIZE_ERROR; /* check for overflow */ + totalDstSize += fcs; } + /* skip to next frame */ { size_t const frameSrcSize = ZSTD_findFrameCompressedSize(src, srcSize); - if (ZSTD_isError(frameSrcSize)) { - return ZSTD_CONTENTSIZE_ERROR; - } + if (ZSTD_isError(frameSrcSize)) return ZSTD_CONTENTSIZE_ERROR; + assert(frameSrcSize <= srcSize); src = (const BYTE *)src + frameSrcSize; srcSize -= frameSrcSize; @@ -758,10 +785,11 @@ static ZSTD_frameSizeInfo ZSTD_findFrameSizeInfo(const void* src, size_t srcSize ip += 4; } + frameSizeInfo.nbBlocks = nbBlocks; frameSizeInfo.compressedSize = (size_t)(ip - ipstart); frameSizeInfo.decompressedBound = (zfh.frameContentSize != ZSTD_CONTENTSIZE_UNKNOWN) ? zfh.frameContentSize - : nbBlocks * zfh.blockSizeMax; + : (unsigned long long)nbBlocks * zfh.blockSizeMax; return frameSizeInfo; } } @@ -801,6 +829,48 @@ unsigned long long ZSTD_decompressBound(const void* src, size_t srcSize) return bound; } +size_t ZSTD_decompressionMargin(void const* src, size_t srcSize) +{ + size_t margin = 0; + unsigned maxBlockSize = 0; + + /* Iterate over each frame */ + while (srcSize > 0) { + ZSTD_frameSizeInfo const frameSizeInfo = ZSTD_findFrameSizeInfo(src, srcSize); + size_t const compressedSize = frameSizeInfo.compressedSize; + unsigned long long const decompressedBound = frameSizeInfo.decompressedBound; + ZSTD_frameHeader zfh; + + FORWARD_IF_ERROR(ZSTD_getFrameHeader(&zfh, src, srcSize), ""); + if (ZSTD_isError(compressedSize) || decompressedBound == ZSTD_CONTENTSIZE_ERROR) + return ERROR(corruption_detected); + + if (zfh.frameType == ZSTD_frame) { + /* Add the frame header to our margin */ + margin += zfh.headerSize; + /* Add the checksum to our margin */ + margin += zfh.checksumFlag ? 4 : 0; + /* Add 3 bytes per block */ + margin += 3 * frameSizeInfo.nbBlocks; + + /* Compute the max block size */ + maxBlockSize = MAX(maxBlockSize, zfh.blockSizeMax); + } else { + assert(zfh.frameType == ZSTD_skippableFrame); + /* Add the entire skippable frame size to our margin. */ + margin += compressedSize; + } + + assert(srcSize >= compressedSize); + src = (const BYTE*)src + compressedSize; + srcSize -= compressedSize; + } + + /* Add the max block size back to the margin. */ + margin += maxBlockSize; + + return margin; +} /*-************************************************************* * Frame decoding @@ -826,7 +896,7 @@ static size_t ZSTD_copyRawBlock(void* dst, size_t dstCapacity, if (srcSize == 0) return 0; RETURN_ERROR(dstBuffer_null, ""); } - ZSTD_memcpy(dst, src, srcSize); + ZSTD_memmove(dst, src, srcSize); return srcSize; } @@ -904,6 +974,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, /* Loop on each block */ while (1) { + BYTE* oBlockEnd = oend; size_t decodedSize; blockProperties_t blockProperties; size_t const cBlockSize = ZSTD_getcBlockSize(ip, remainingSrcSize, &blockProperties); @@ -913,16 +984,34 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, remainingSrcSize -= ZSTD_blockHeaderSize; RETURN_ERROR_IF(cBlockSize > remainingSrcSize, srcSize_wrong, ""); + if (ip >= op && ip < oBlockEnd) { + /* We are decompressing in-place. Limit the output pointer so that we + * don't overwrite the block that we are currently reading. This will + * fail decompression if the input & output pointers aren't spaced + * far enough apart. + * + * This is important to set, even when the pointers are far enough + * apart, because ZSTD_decompressBlock_internal() can decide to store + * literals in the output buffer, after the block it is decompressing. + * Since we don't want anything to overwrite our input, we have to tell + * ZSTD_decompressBlock_internal to never write past ip. + * + * See ZSTD_allocateLiteralsBuffer() for reference. + */ + oBlockEnd = op + (ip - op); + } + switch(blockProperties.blockType) { case bt_compressed: - decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oend-op), ip, cBlockSize, /* frame */ 1, not_streaming); + decodedSize = ZSTD_decompressBlock_internal(dctx, op, (size_t)(oBlockEnd-op), ip, cBlockSize, /* frame */ 1, not_streaming); break; case bt_raw : + /* Use oend instead of oBlockEnd because this function is safe to overlap. It uses memmove. */ decodedSize = ZSTD_copyRawBlock(op, (size_t)(oend-op), ip, cBlockSize); break; case bt_rle : - decodedSize = ZSTD_setRleBlock(op, (size_t)(oend-op), *ip, blockProperties.origSize); + decodedSize = ZSTD_setRleBlock(op, (size_t)(oBlockEnd-op), *ip, blockProperties.origSize); break; case bt_reserved : default: @@ -957,6 +1046,7 @@ static size_t ZSTD_decompressFrame(ZSTD_DCtx* dctx, } ZSTD_DCtx_trace_end(dctx, (U64)(op-ostart), (U64)(ip-istart), /* streaming */ 0); /* Allow caller to get size read */ + DEBUGLOG(4, "ZSTD_decompressFrame: decompressed frame of size %zi, consuming %zi bytes of input", op-ostart, ip - (const BYTE*)*srcPtr); *srcPtr = ip; *srcSizePtr = remainingSrcSize; return (size_t)(op-ostart); @@ -1003,17 +1093,18 @@ static size_t ZSTD_decompressMultiFrame(ZSTD_DCtx* dctx, } #endif - { U32 const magicNumber = MEM_readLE32(src); - DEBUGLOG(4, "reading magic number %08X (expecting %08X)", - (unsigned)magicNumber, ZSTD_MAGICNUMBER); + if (srcSize >= 4) { + U32 const magicNumber = MEM_readLE32(src); + DEBUGLOG(5, "reading magic number %08X", (unsigned)magicNumber); if ((magicNumber & ZSTD_MAGIC_SKIPPABLE_MASK) == ZSTD_MAGIC_SKIPPABLE_START) { + /* skippable frame detected : skip it */ size_t const skippableSize = readSkippableFrameSize(src, srcSize); - FORWARD_IF_ERROR(skippableSize, "readSkippableFrameSize failed"); + FORWARD_IF_ERROR(skippableSize, "invalid skippable frame"); assert(skippableSize <= srcSize); src = (const BYTE *)src + skippableSize; srcSize -= skippableSize; - continue; + continue; /* check next frame */ } } if (ddict) { @@ -1109,8 +1200,8 @@ size_t ZSTD_decompress(void* dst, size_t dstCapacity, const void* src, size_t sr size_t ZSTD_nextSrcSizeToDecompress(ZSTD_DCtx* dctx) { return dctx->expected; } /** - * Similar to ZSTD_nextSrcSizeToDecompress(), but when when a block input can be streamed, - * we allow taking a partial block as the input. Currently only raw uncompressed blocks can + * Similar to ZSTD_nextSrcSizeToDecompress(), but when a block input can be streamed, we + * allow taking a partial block as the input. Currently only raw uncompressed blocks can * be streamed. * * For blocks that can be streamed, this allows us to reduce the latency until we produce @@ -1310,7 +1401,7 @@ size_t ZSTD_decompressContinue(ZSTD_DCtx* dctx, void* dst, size_t dstCapacity, c default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } @@ -1351,11 +1442,11 @@ ZSTD_loadDEntropy(ZSTD_entropyDTables_t* entropy, /* in minimal huffman, we always use X1 variants */ size_t const hSize = HUF_readDTableX1_wksp(entropy->hufTable, dictPtr, dictEnd - dictPtr, - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #else size_t const hSize = HUF_readDTableX2_wksp(entropy->hufTable, dictPtr, (size_t)(dictEnd - dictPtr), - workspace, workspaceSize); + workspace, workspaceSize, /* flags */ 0); #endif RETURN_ERROR_IF(HUF_isError(hSize), dictionary_corrupted, ""); dictPtr += hSize; @@ -1454,7 +1545,7 @@ size_t ZSTD_decompressBegin(ZSTD_DCtx* dctx) dctx->prefixStart = NULL; dctx->virtualStart = NULL; dctx->dictEnd = NULL; - dctx->entropy.hufTable[0] = (HUF_DTable)((HufLog)*0x1000001); /* cover both little and big endian */ + dctx->entropy.hufTable[0] = (HUF_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); /* cover both little and big endian */ dctx->litEntropy = dctx->fseEntropy = 0; dctx->dictID = 0; dctx->bType = bt_reserved; @@ -1516,7 +1607,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * This could for one of the following reasons : * - The frame does not require a dictionary (most common case). * - The frame was built with dictID intentionally removed. - * Needed dictionary is a hidden information. + * Needed dictionary is a hidden piece of information. * Note : this use case also happens when using a non-conformant dictionary. * - `srcSize` is too small, and as a result, frame header could not be decoded. * Note : possible if `srcSize < ZSTD_FRAMEHEADERSIZE_MAX`. @@ -1525,7 +1616,7 @@ unsigned ZSTD_getDictID_fromDict(const void* dict, size_t dictSize) * ZSTD_getFrameHeader(), which will provide a more precise error code. */ unsigned ZSTD_getDictID_fromFrame(const void* src, size_t srcSize) { - ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0 }; + ZSTD_frameHeader zfp = { 0, 0, 0, ZSTD_frame, 0, 0, 0, 0, 0 }; size_t const hError = ZSTD_getFrameHeader(&zfp, src, srcSize); if (ZSTD_isError(hError)) return 0; return zfp.dictID; @@ -1632,7 +1723,9 @@ size_t ZSTD_initDStream_usingDict(ZSTD_DStream* zds, const void* dict, size_t di size_t ZSTD_initDStream(ZSTD_DStream* zds) { DEBUGLOG(4, "ZSTD_initDStream"); - return ZSTD_initDStream_usingDDict(zds, NULL); + FORWARD_IF_ERROR(ZSTD_DCtx_reset(zds, ZSTD_reset_session_only), ""); + FORWARD_IF_ERROR(ZSTD_DCtx_refDDict(zds, NULL), ""); + return ZSTD_startingInputLength(zds->format); } /* ZSTD_initDStream_usingDDict() : @@ -1640,6 +1733,7 @@ size_t ZSTD_initDStream(ZSTD_DStream* zds) * this function cannot fail */ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) { + DEBUGLOG(4, "ZSTD_initDStream_usingDDict"); FORWARD_IF_ERROR( ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only) , ""); FORWARD_IF_ERROR( ZSTD_DCtx_refDDict(dctx, ddict) , ""); return ZSTD_startingInputLength(dctx->format); @@ -1650,6 +1744,7 @@ size_t ZSTD_initDStream_usingDDict(ZSTD_DStream* dctx, const ZSTD_DDict* ddict) * this function cannot fail */ size_t ZSTD_resetDStream(ZSTD_DStream* dctx) { + DEBUGLOG(4, "ZSTD_resetDStream"); FORWARD_IF_ERROR(ZSTD_DCtx_reset(dctx, ZSTD_reset_session_only), ""); return ZSTD_startingInputLength(dctx->format); } @@ -1721,6 +1816,11 @@ ZSTD_bounds ZSTD_dParam_getBounds(ZSTD_dParameter dParam) bounds.lowerBound = (int)ZSTD_rmd_refSingleDDict; bounds.upperBound = (int)ZSTD_rmd_refMultipleDDicts; return bounds; + case ZSTD_d_disableHuffmanAssembly: + bounds.lowerBound = 0; + bounds.upperBound = 1; + return bounds; + default:; } bounds.error = ERROR(parameter_unsupported); @@ -1761,6 +1861,9 @@ size_t ZSTD_DCtx_getParameter(ZSTD_DCtx* dctx, ZSTD_dParameter param, int* value case ZSTD_d_refMultipleDDicts: *value = (int)dctx->refMultipleDDicts; return 0; + case ZSTD_d_disableHuffmanAssembly: + *value = (int)dctx->disableHufAsm; + return 0; default:; } RETURN_ERROR(parameter_unsupported, ""); @@ -1794,6 +1897,10 @@ size_t ZSTD_DCtx_setParameter(ZSTD_DCtx* dctx, ZSTD_dParameter dParam, int value } dctx->refMultipleDDicts = (ZSTD_refMultipleDDicts_e)value; return 0; + case ZSTD_d_disableHuffmanAssembly: + CHECK_DBOUNDS(ZSTD_d_disableHuffmanAssembly, value); + dctx->disableHufAsm = value != 0; + return 0; default:; } RETURN_ERROR(parameter_unsupported, ""); @@ -1981,7 +2088,6 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if (zds->refMultipleDDicts && zds->ddictSet) { ZSTD_DCtx_selectFrameDDict(zds); } - DEBUGLOG(5, "header size : %u", (U32)hSize); if (ZSTD_isError(hSize)) { #if defined(ZSTD_LEGACY_SUPPORT) && (ZSTD_LEGACY_SUPPORT>=1) U32 const legacyVersion = ZSTD_isLegacy(istart, iend-istart); @@ -2013,6 +2119,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB zds->lhSize += remainingInput; } input->pos = input->size; + /* check first few bytes */ + FORWARD_IF_ERROR( + ZSTD_getFrameHeader_advanced(&zds->fParams, zds->headerBuffer, zds->lhSize, zds->format), + "First few bytes detected incorrect" ); + /* return hint input size */ return (MAX((size_t)ZSTD_FRAMEHEADERSIZE_MIN(zds->format), hSize) - zds->lhSize) + ZSTD_blockHeaderSize; /* remaining header bytes + next block header */ } assert(ip != NULL); @@ -2030,8 +2141,9 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB size_t const decompressedSize = ZSTD_decompress_usingDDict(zds, op, (size_t)(oend-op), istart, cSize, ZSTD_getDDict(zds)); if (ZSTD_isError(decompressedSize)) return decompressedSize; DEBUGLOG(4, "shortcut to single-pass ZSTD_decompress_usingDDict()") + assert(istart != NULL); ip = istart + cSize; - op += decompressedSize; + op = op ? op + decompressedSize : op; /* can occur if frameContentSize = 0 (empty frame) */ zds->expected = 0; zds->streamStage = zdss_init; someMoreWork = 0; @@ -2115,6 +2227,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB } if ((size_t)(iend-ip) >= neededInSize) { /* decode directly from src */ FORWARD_IF_ERROR(ZSTD_decompressContinueStream(zds, &op, oend, ip, neededInSize), ""); + assert(ip != NULL); ip += neededInSize; /* Function modifies the stage so we must break */ break; @@ -2129,7 +2242,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB int const isSkipFrame = ZSTD_isSkipFrame(zds); size_t loadedSize; /* At this point we shouldn't be decompressing a block that we can stream. */ - assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, iend - ip)); + assert(neededInSize == ZSTD_nextSrcSizeToDecompressWithInputSize(zds, (size_t)(iend - ip))); if (isSkipFrame) { loadedSize = MIN(toLoad, (size_t)(iend-ip)); } else { @@ -2138,8 +2251,11 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB "should never happen"); loadedSize = ZSTD_limitCopy(zds->inBuff + zds->inPos, toLoad, ip, (size_t)(iend-ip)); } - ip += loadedSize; - zds->inPos += loadedSize; + if (loadedSize != 0) { + /* ip may be NULL */ + ip += loadedSize; + zds->inPos += loadedSize; + } if (loadedSize < toLoad) { someMoreWork = 0; break; } /* not enough input, wait for more */ /* decode loaded input */ @@ -2149,14 +2265,17 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB break; } case zdss_flush: - { size_t const toFlushSize = zds->outEnd - zds->outStart; + { + size_t const toFlushSize = zds->outEnd - zds->outStart; size_t const flushedSize = ZSTD_limitCopy(op, (size_t)(oend-op), zds->outBuff + zds->outStart, toFlushSize); - op += flushedSize; + + op = op ? op + flushedSize : op; + zds->outStart += flushedSize; if (flushedSize == toFlushSize) { /* flush completed */ zds->streamStage = zdss_read; if ( (zds->outBuffSize < zds->fParams.frameContentSize) - && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { + && (zds->outStart + zds->fParams.blockSizeMax > zds->outBuffSize) ) { DEBUGLOG(5, "restart filling outBuff from beginning (left:%i, needed:%u)", (int)(zds->outBuffSize - zds->outStart), (U32)zds->fParams.blockSizeMax); @@ -2170,7 +2289,7 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB default: assert(0); /* impossible */ - RETURN_ERROR(GENERIC, "impossible to reach"); /* some compiler require default to do something */ + RETURN_ERROR(GENERIC, "impossible to reach"); /* some compilers require default to do something */ } } /* result */ @@ -2183,8 +2302,8 @@ size_t ZSTD_decompressStream(ZSTD_DStream* zds, ZSTD_outBuffer* output, ZSTD_inB if ((ip==istart) && (op==ostart)) { /* no forward progress */ zds->noForwardProgress ++; if (zds->noForwardProgress >= ZSTD_NO_FORWARD_PROGRESS_MAX) { - RETURN_ERROR_IF(op==oend, dstSize_tooSmall, ""); - RETURN_ERROR_IF(ip==iend, srcSize_wrong, ""); + RETURN_ERROR_IF(op==oend, noForwardProgress_destFull, ""); + RETURN_ERROR_IF(ip==iend, noForwardProgress_inputEmpty, ""); assert(0); } } else { @@ -2221,13 +2340,19 @@ size_t ZSTD_decompressStream_simpleArgs ( void* dst, size_t dstCapacity, size_t* dstPos, const void* src, size_t srcSize, size_t* srcPos) { - ZSTD_outBuffer output = { dst, dstCapacity, *dstPos }; - ZSTD_inBuffer input = { src, srcSize, *srcPos }; - /* ZSTD_compress_generic() will check validity of dstPos and srcPos */ - size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); - *dstPos = output.pos; - *srcPos = input.pos; - return cErr; + ZSTD_outBuffer output; + ZSTD_inBuffer input; + output.dst = dst; + output.size = dstCapacity; + output.pos = *dstPos; + input.src = src; + input.size = srcSize; + input.pos = *srcPos; + { size_t const cErr = ZSTD_decompressStream(dctx, &output, &input); + *dstPos = output.pos; + *srcPos = input.pos; + return cErr; + } } #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/zstd_decompress_block.c b/vendor/github.com/DataDog/zstd/zstd_decompress_block.c index 9d5c1c2..98767b1 100644 --- a/vendor/github.com/DataDog/zstd/zstd_decompress_block.c +++ b/vendor/github.com/DataDog/zstd/zstd_decompress_block.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -21,12 +21,12 @@ #include "mem.h" /* low level memory routines */ #define FSE_STATIC_LINKING_ONLY #include "fse.h" -#define HUF_STATIC_LINKING_ONLY #include "huf.h" #include "zstd_internal.h" #include "zstd_decompress_internal.h" /* ZSTD_DCtx */ #include "zstd_ddict.h" /* ZSTD_DDictDictContent */ #include "zstd_decompress_block.h" +#include "bits.h" /* ZSTD_highbit32 */ /*_******************************************************* * Macros @@ -90,7 +90,7 @@ static void ZSTD_allocateLiteralsBuffer(ZSTD_DCtx* dctx, void* const dst, const dctx->litBufferEnd = dctx->litBuffer + litSize - ZSTD_LITBUFFEREXTRASIZE; } else { - /* initially this will be stored entirely in dst during huffman decoding, it will partially shifted to litExtraBuffer after */ + /* initially this will be stored entirely in dst during huffman decoding, it will partially be shifted to litExtraBuffer after */ dctx->litBuffer = (BYTE*)dst + expectedWriteSize - litSize; dctx->litBufferEnd = (BYTE*)dst + expectedWriteSize; } @@ -135,13 +135,16 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, ZSTD_FALLTHROUGH; case set_compressed: - RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need up to 5 for case 3"); + RETURN_ERROR_IF(srcSize < 5, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need up to 5 for case 3"); { size_t lhSize, litSize, litCSize; U32 singleStream=0; U32 const lhlCode = (istart[0] >> 2) & 3; U32 const lhc = MEM_readLE32(istart); size_t hufSuccess; size_t expectedWriteSize = MIN(ZSTD_BLOCKSIZE_MAX, dstCapacity); + int const flags = 0 + | (ZSTD_DCtx_get_bmi2(dctx) ? HUF_flags_bmi2 : 0) + | (dctx->disableHufAsm ? HUF_flags_disableAsm : 0); switch(lhlCode) { case 0: case 1: default: /* note : default is impossible, since lhlCode into [0..3] */ @@ -166,6 +169,10 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, } RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); RETURN_ERROR_IF(litSize > ZSTD_BLOCKSIZE_MAX, corruption_detected, ""); + if (!singleStream) + RETURN_ERROR_IF(litSize < MIN_LITERALS_FOR_4_STREAMS, literals_headerWrong, + "Not enough literals (%zu) for the 4-streams mode (min %u)", + litSize, MIN_LITERALS_FOR_4_STREAMS); RETURN_ERROR_IF(litCSize + lhSize > srcSize, corruption_detected, ""); RETURN_ERROR_IF(expectedWriteSize < litSize , dstSize_tooSmall, ""); ZSTD_allocateLiteralsBuffer(dctx, dst, dstCapacity, litSize, streaming, expectedWriteSize, 0); @@ -177,13 +184,14 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, if (litEncType==set_repeat) { if (singleStream) { - hufSuccess = HUF_decompress1X_usingDTable_bmi2( + hufSuccess = HUF_decompress1X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx)); + dctx->HUFptr, flags); } else { - hufSuccess = HUF_decompress4X_usingDTable_bmi2( + assert(litSize >= MIN_LITERALS_FOR_4_STREAMS); + hufSuccess = HUF_decompress4X_usingDTable( dctx->litBuffer, litSize, istart+lhSize, litCSize, - dctx->HUFptr, ZSTD_DCtx_get_bmi2(dctx)); + dctx->HUFptr, flags); } } else { if (singleStream) { @@ -191,18 +199,18 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, hufSuccess = HUF_decompress1X_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace)); + sizeof(dctx->workspace), flags); #else - hufSuccess = HUF_decompress1X1_DCtx_wksp_bmi2( + hufSuccess = HUF_decompress1X1_DCtx_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); + sizeof(dctx->workspace), flags); #endif } else { - hufSuccess = HUF_decompress4X_hufOnly_wksp_bmi2( + hufSuccess = HUF_decompress4X_hufOnly_wksp( dctx->entropy.hufTable, dctx->litBuffer, litSize, istart+lhSize, litCSize, dctx->workspace, - sizeof(dctx->workspace), ZSTD_DCtx_get_bmi2(dctx)); + sizeof(dctx->workspace), flags); } } if (dctx->litBufferLocation == ZSTD_split) @@ -238,6 +246,7 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize = 3"); litSize = MEM_readLE24(istart) >> 4; break; } @@ -280,12 +289,13 @@ size_t ZSTD_decodeLiteralsBlock(ZSTD_DCtx* dctx, break; case 1: lhSize = 2; + RETURN_ERROR_IF(srcSize<3, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 3"); litSize = MEM_readLE16(istart) >> 4; break; case 3: lhSize = 3; + RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 2; here we need lhSize+1 = 4"); litSize = MEM_readLE24(istart) >> 4; - RETURN_ERROR_IF(srcSize<4, corruption_detected, "srcSize >= MIN_CBLOCK_SIZE == 3; here we need lhSize+1 = 4"); break; } RETURN_ERROR_IF(litSize > 0 && dst == NULL, dstSize_tooSmall, "NULL not handled"); @@ -507,14 +517,15 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i = 8; i < n; i += 8) { MEM_write64(spread + pos + i, sv); } - pos += n; + assert(n>=0); + pos += (size_t)n; } } /* Now we spread those positions across the table. - * The benefit of doing it in two stages is that we avoid the the + * The benefit of doing it in two stages is that we avoid the * variable size inner loop, which caused lots of branch misses. * Now we can run through all the positions without any branch misses. - * We unroll the loop twice, since that is what emperically worked best. + * We unroll the loop twice, since that is what empirically worked best. */ { size_t position = 0; @@ -541,7 +552,7 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (i=0; i highThreshold) position = (position + step) & tableMask; /* lowprob area */ + while (UNLIKELY(position > highThreshold)) position = (position + step) & tableMask; /* lowprob area */ } } assert(position == 0); /* position must reach all cells once, otherwise normalizedCounter is incorrect */ } @@ -552,7 +563,7 @@ void ZSTD_buildFSETable_body(ZSTD_seqSymbol* dt, for (u=0; ustateLL.table + seqState->stateLL.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(mlDInfo, seqState->stateML.table + seqState->stateML.state, sizeof(ZSTD_seqSymbol)); + ZSTD_memcpy(ofDInfo, seqState->stateOffb.table + seqState->stateOffb.state, sizeof(ZSTD_seqSymbol)); +#else const ZSTD_seqSymbol* const llDInfo = seqState->stateLL.table + seqState->stateLL.state; const ZSTD_seqSymbol* const mlDInfo = seqState->stateML.table + seqState->stateML.state; const ZSTD_seqSymbol* const ofDInfo = seqState->stateOffb.table + seqState->stateOffb.state; +#endif seq.matchLength = mlDInfo->baseValue; seq.litLength = llDInfo->baseValue; { U32 const ofBase = ofDInfo->baseValue; @@ -1187,28 +1221,31 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) U32 const llnbBits = llDInfo->nbBits; U32 const mlnbBits = mlDInfo->nbBits; U32 const ofnbBits = ofDInfo->nbBits; + + assert(llBits <= MaxLLBits); + assert(mlBits <= MaxMLBits); + assert(ofBits <= MaxOff); /* * As gcc has better branch and block analyzers, sometimes it is only - * valuable to mark likelyness for clang, it gives around 3-4% of + * valuable to mark likeliness for clang, it gives around 3-4% of * performance. */ /* sequence */ { size_t offset; - #if defined(__clang__) - if (LIKELY(ofBits > 1)) { - #else if (ofBits > 1) { - #endif ZSTD_STATIC_ASSERT(ZSTD_lo_isLongOffset == 1); ZSTD_STATIC_ASSERT(LONG_OFFSETS_MAX_EXTRA_BITS_32 == 5); - assert(ofBits <= MaxOff); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 > LONG_OFFSETS_MAX_EXTRA_BITS_32); + ZSTD_STATIC_ASSERT(STREAM_ACCUMULATOR_MIN_32 - LONG_OFFSETS_MAX_EXTRA_BITS_32 >= MaxMLBits); if (MEM_32bits() && longOffsets && (ofBits >= STREAM_ACCUMULATOR_MIN_32)) { - U32 const extraBits = ofBits - MIN(ofBits, 32 - seqState->DStream.bitsConsumed); + /* Always read extra bits, this keeps the logic simple, + * avoids branches, and avoids accidentally reading 0 bits. + */ + U32 const extraBits = LONG_OFFSETS_MAX_EXTRA_BITS_32; offset = ofBase + (BIT_readBitsFast(&seqState->DStream, ofBits - extraBits) << extraBits); BIT_reloadDStream(&seqState->DStream); - if (extraBits) offset += BIT_readBitsFast(&seqState->DStream, extraBits); - assert(extraBits <= LONG_OFFSETS_MAX_EXTRA_BITS_32); /* to avoid another reload */ + offset += BIT_readBitsFast(&seqState->DStream, extraBits); } else { offset = ofBase + BIT_readBitsFast(&seqState->DStream, ofBits/*>0*/); /* <= (ZSTD_WINDOWLOG_MAX-1) bits */ if (MEM_32bits()) BIT_reloadDStream(&seqState->DStream); @@ -1233,11 +1270,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) seq.offset = offset; } - #if defined(__clang__) - if (UNLIKELY(mlBits > 0)) - #else if (mlBits > 0) - #endif seq.matchLength += BIT_readBitsFast(&seqState->DStream, mlBits/*>0*/); if (MEM_32bits() && (mlBits+llBits >= STREAM_ACCUMULATOR_MIN_32-LONG_OFFSETS_MAX_EXTRA_BITS_32)) @@ -1247,11 +1280,7 @@ ZSTD_decodeSequence(seqState_t* seqState, const ZSTD_longOffset_e longOffsets) /* Ensure there are enough bits to read the rest of data in 64-bit mode. */ ZSTD_STATIC_ASSERT(16+LLFSELog+MLFSELog+OffFSELog < STREAM_ACCUMULATOR_MIN_64); - #if defined(__clang__) - if (UNLIKELY(llBits > 0)) - #else if (llBits > 0) - #endif seq.litLength += BIT_readBitsFast(&seqState->DStream, llBits/*>0*/); if (MEM_32bits()) @@ -1553,7 +1582,7 @@ ZSTD_decompressSequences_body(ZSTD_DCtx* dctx, const BYTE* const prefixStart = (const BYTE*)(dctx->prefixStart); const BYTE* const vBase = (const BYTE*)(dctx->virtualStart); const BYTE* const dictEnd = (const BYTE*)(dctx->dictEnd); - DEBUGLOG(5, "ZSTD_decompressSequences_body"); + DEBUGLOG(5, "ZSTD_decompressSequences_body: nbSeq = %d", nbSeq); (void)frame; /* Regen sequences */ @@ -1946,34 +1975,79 @@ ZSTD_decompressSequencesLong(ZSTD_DCtx* dctx, #endif /* ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT */ +/** + * @returns The total size of the history referenceable by zstd, including + * both the prefix and the extDict. At @p op any offset larger than this + * is invalid. + */ +static size_t ZSTD_totalHistorySize(BYTE* op, BYTE const* virtualStart) +{ + return (size_t)(op - virtualStart); +} -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) -/* ZSTD_getLongOffsetsShare() : +typedef struct { + unsigned longOffsetShare; + unsigned maxNbAdditionalBits; +} ZSTD_OffsetInfo; + +/* ZSTD_getOffsetInfo() : * condition : offTable must be valid * @return : "share" of long offsets (arbitrarily defined as > (1<<23)) - * compared to maximum possible of (1< 22) total += 1; + assert(max <= (1 << OffFSELog)); /* max not too large */ + for (u=0; u 22) info.longOffsetShare += 1; + } + + assert(tableLog <= OffFSELog); + info.longOffsetShare <<= (OffFSELog - tableLog); /* scale to OffFSELog */ } - assert(tableLog <= OffFSELog); - total <<= (OffFSELog - tableLog); /* scale to OffFSELog */ - - return total; + return info; +} + +/** + * @returns The maximum offset we can decode in one read of our bitstream, without + * reloading more bits in the middle of the offset bits read. Any offsets larger + * than this must use the long offset decoder. + */ +static size_t ZSTD_maxShortOffset(void) +{ + if (MEM_64bits()) { + /* We can decode any offset without reloading bits. + * This might change if the max window size grows. + */ + ZSTD_STATIC_ASSERT(ZSTD_WINDOWLOG_MAX <= 31); + return (size_t)-1; + } else { + /* The maximum offBase is (1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1. + * This offBase would require STREAM_ACCUMULATOR_MIN extra bits. + * Then we have to subtract ZSTD_REP_NUM to get the maximum possible offset. + */ + size_t const maxOffbase = ((size_t)1 << (STREAM_ACCUMULATOR_MIN + 1)) - 1; + size_t const maxOffset = maxOffbase - ZSTD_REP_NUM; + assert(ZSTD_highbit32((U32)maxOffbase) == STREAM_ACCUMULATOR_MIN); + return maxOffset; + } } -#endif size_t ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, @@ -1981,20 +2055,21 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, const void* src, size_t srcSize, const int frame, const streaming_operation streaming) { /* blockType == blockCompressed */ const BYTE* ip = (const BYTE*)src; - /* isLongOffset must be true if there are long offsets. - * Offsets are long if they are larger than 2^STREAM_ACCUMULATOR_MIN. - * We don't expect that to be the case in 64-bit mode. - * In block mode, window size is not known, so we have to be conservative. - * (note: but it could be evaluated from current-lowLimit) - */ - ZSTD_longOffset_e const isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (!frame || (dctx->fParams.windowSize > (1ULL << STREAM_ACCUMULATOR_MIN)))); DEBUGLOG(5, "ZSTD_decompressBlock_internal (size : %u)", (U32)srcSize); - RETURN_ERROR_IF(srcSize >= ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); + /* Note : the wording of the specification + * allows compressed block to be sized exactly ZSTD_BLOCKSIZE_MAX. + * This generally does not happen, as it makes little sense, + * since an uncompressed block would feature same size and have no decompression cost. + * Also, note that decoder from reference libzstd before < v1.5.4 + * would consider this edge case as an error. + * As a consequence, avoid generating compressed blocks of size ZSTD_BLOCKSIZE_MAX + * for broader compatibility with the deployed ecosystem of zstd decoders */ + RETURN_ERROR_IF(srcSize > ZSTD_BLOCKSIZE_MAX, srcSize_wrong, ""); /* Decode literals section */ { size_t const litCSize = ZSTD_decodeLiteralsBlock(dctx, src, srcSize, dst, dstCapacity, streaming); - DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : %u", (U32)litCSize); + DEBUGLOG(5, "ZSTD_decodeLiteralsBlock : cSize=%u, nbLiterals=%zu", (U32)litCSize, dctx->litSize); if (ZSTD_isError(litCSize)) return litCSize; ip += litCSize; srcSize -= litCSize; @@ -2002,6 +2077,23 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, /* Build Decoding Tables */ { + /* Compute the maximum block size, which must also work when !frame and fParams are unset. + * Additionally, take the min with dstCapacity to ensure that the totalHistorySize fits in a size_t. + */ + size_t const blockSizeMax = MIN(dstCapacity, (frame ? dctx->fParams.blockSizeMax : ZSTD_BLOCKSIZE_MAX)); + size_t const totalHistorySize = ZSTD_totalHistorySize((BYTE*)dst + blockSizeMax, (BYTE const*)dctx->virtualStart); + /* isLongOffset must be true if there are long offsets. + * Offsets are long if they are larger than ZSTD_maxShortOffset(). + * We don't expect that to be the case in 64-bit mode. + * + * We check here to see if our history is large enough to allow long offsets. + * If it isn't, then we can't possible have (valid) long offsets. If the offset + * is invalid, then it is okay to read it incorrectly. + * + * If isLongOffsets is true, then we will later check our decoding table to see + * if it is even possible to generate long offsets. + */ + ZSTD_longOffset_e isLongOffset = (ZSTD_longOffset_e)(MEM_32bits() && (totalHistorySize > ZSTD_maxShortOffset())); /* These macros control at build-time which decompressor implementation * we use. If neither is defined, we do some inspection and dispatch at * runtime. @@ -2009,6 +2101,11 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) int usePrefetchDecoder = dctx->ddictIsCold; +#else + /* Set to 1 to avoid computing offset info if we don't need to. + * Otherwise this value is ignored. + */ + int usePrefetchDecoder = 1; #endif int nbSeq; size_t const seqHSize = ZSTD_decodeSeqHeaders(dctx, &nbSeq, ip, srcSize); @@ -2016,28 +2113,42 @@ ZSTD_decompressBlock_internal(ZSTD_DCtx* dctx, ip += seqHSize; srcSize -= seqHSize; - RETURN_ERROR_IF(dst == NULL && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF((dst == NULL || dstCapacity == 0) && nbSeq > 0, dstSize_tooSmall, "NULL not handled"); + RETURN_ERROR_IF(MEM_64bits() && sizeof(size_t) == sizeof(void*) && (size_t)(-1) - (size_t)dst < (size_t)(1 << 20), dstSize_tooSmall, + "invalid dst"); -#if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ - !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if ( !usePrefetchDecoder - && (!frame || (dctx->fParams.windowSize > (1<<24))) - && (nbSeq>ADVANCED_SEQS) ) { /* could probably use a larger nbSeq limit */ - U32 const shareLongOffsets = ZSTD_getLongOffsetsShare(dctx->OFTptr); - U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ - usePrefetchDecoder = (shareLongOffsets >= minShare); + /* If we could potentially have long offsets, or we might want to use the prefetch decoder, + * compute information about the share of long offsets, and the maximum nbAdditionalBits. + * NOTE: could probably use a larger nbSeq limit + */ + if (isLongOffset || (!usePrefetchDecoder && (totalHistorySize > (1u << 24)) && (nbSeq > 8))) { + ZSTD_OffsetInfo const info = ZSTD_getOffsetInfo(dctx->OFTptr, nbSeq); + if (isLongOffset && info.maxNbAdditionalBits <= STREAM_ACCUMULATOR_MIN) { + /* If isLongOffset, but the maximum number of additional bits that we see in our table is small + * enough, then we know it is impossible to have too long an offset in this block, so we can + * use the regular offset decoder. + */ + isLongOffset = ZSTD_lo_isRegularOffset; + } + if (!usePrefetchDecoder) { + U32 const minShare = MEM_64bits() ? 7 : 20; /* heuristic values, correspond to 2.73% and 7.81% */ + usePrefetchDecoder = (info.longOffsetShare >= minShare); + } } -#endif dctx->ddictIsCold = 0; #if !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT) && \ !defined(ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG) - if (usePrefetchDecoder) + if (usePrefetchDecoder) { +#else + (void)usePrefetchDecoder; + { #endif #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_SHORT return ZSTD_decompressSequencesLong(dctx, dst, dstCapacity, ip, srcSize, nbSeq, isLongOffset, frame); #endif + } #ifndef ZSTD_FORCE_DECOMPRESS_SEQUENCES_LONG /* else */ @@ -2061,9 +2172,9 @@ void ZSTD_checkContinuity(ZSTD_DCtx* dctx, const void* dst, size_t dstSize) } -size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, - void* dst, size_t dstCapacity, - const void* src, size_t srcSize) +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) { size_t dSize; ZSTD_checkContinuity(dctx, dst, dstCapacity); @@ -2072,4 +2183,13 @@ size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, return dSize; } + +/* NOTE: Must just wrap ZSTD_decompressBlock_deprecated() */ +size_t ZSTD_decompressBlock(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize) +{ + return ZSTD_decompressBlock_deprecated(dctx, dst, dstCapacity, src, srcSize); +} + #endif /* USE_EXTERNAL_ZSTD */ diff --git a/vendor/github.com/DataDog/zstd/zstd_decompress_block.h b/vendor/github.com/DataDog/zstd/zstd_decompress_block.h index 0c1929b..a6df799 100644 --- a/vendor/github.com/DataDog/zstd/zstd_decompress_block.h +++ b/vendor/github.com/DataDog/zstd/zstd_decompress_block.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -65,6 +65,11 @@ void ZSTD_buildFSETable(ZSTD_seqSymbol* dt, unsigned tableLog, void* wksp, size_t wkspSize, int bmi2); +/* Internal definition of ZSTD_decompressBlock() to avoid deprecation warnings. */ +size_t ZSTD_decompressBlock_deprecated(ZSTD_DCtx* dctx, + void* dst, size_t dstCapacity, + const void* src, size_t srcSize); + #endif /* ZSTD_DEC_BLOCK_H */ diff --git a/vendor/github.com/DataDog/zstd/zstd_decompress_internal.h b/vendor/github.com/DataDog/zstd/zstd_decompress_internal.h index 7504889..7ce4632 100644 --- a/vendor/github.com/DataDog/zstd/zstd_decompress_internal.h +++ b/vendor/github.com/DataDog/zstd/zstd_decompress_internal.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -76,12 +76,13 @@ static UNUSED_ATTR const U32 ML_base[MaxML+1] = { #define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE (sizeof(S16) * (MaxSeq + 1) + (1u << MaxFSELog) + sizeof(U64)) #define ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32 ((ZSTD_BUILD_FSE_TABLE_WKSP_SIZE + sizeof(U32) - 1) / sizeof(U32)) +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 typedef struct { ZSTD_seqSymbol LLTable[SEQSYMBOL_TABLE_SIZE(LLFSELog)]; /* Note : Space reserved for FSE Tables */ ZSTD_seqSymbol OFTable[SEQSYMBOL_TABLE_SIZE(OffFSELog)]; /* is also used as temporary workspace while building hufTable during DDict creation */ ZSTD_seqSymbol MLTable[SEQSYMBOL_TABLE_SIZE(MLFSELog)]; /* and therefore must be at least HUF_DECOMPRESS_WORKSPACE_SIZE large */ - HUF_DTable hufTable[HUF_DTABLE_SIZE(HufLog)]; /* can accommodate HUF_decompress4X */ + HUF_DTable hufTable[HUF_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUF_decompress4X */ U32 rep[ZSTD_REP_NUM]; U32 workspace[ZSTD_BUILD_FSE_TABLE_WKSP_SIZE_U32]; } ZSTD_entropyDTables_t; @@ -165,6 +166,7 @@ struct ZSTD_DCtx_s ZSTD_dictUses_e dictUses; ZSTD_DDictHashSet* ddictSet; /* Hash set for multiple ddicts */ ZSTD_refMultipleDDicts_e refMultipleDDicts; /* User specified: if == 1, will allow references to multiple DDicts. Default == 0 (disabled) */ + int disableHufAsm; /* streaming */ ZSTD_dStreamStage streamStage; diff --git a/vendor/github.com/DataDog/zstd/zstd_deps.h b/vendor/github.com/DataDog/zstd/zstd_deps.h index ae5f022..949e023 100644 --- a/vendor/github.com/DataDog/zstd/zstd_deps.h +++ b/vendor/github.com/DataDog/zstd/zstd_deps.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_double_fast.c b/vendor/github.com/DataDog/zstd/zstd_double_fast.c index c12810b..458533f 100644 --- a/vendor/github.com/DataDog/zstd/zstd_double_fast.c +++ b/vendor/github.com/DataDog/zstd/zstd_double_fast.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -12,8 +12,43 @@ #include "zstd_compress_internal.h" #include "zstd_double_fast.h" +static void ZSTD_fillDoubleHashTableForCDict(ZSTD_matchState_t* ms, + void const* end, ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashLarge = ms->hashTable; + U32 const hBitsL = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + U32* const hashSmall = ms->chainTable; + U32 const hBitsS = cParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; -void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + /* Always insert every fastHashFillStep position into the hash tables. + * Insert the other positions into the large hash table if their entry + * is empty. + */ + for (; ip + fastHashFillStep - 1 <= iend; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + U32 i; + for (i = 0; i < fastHashFillStep; ++i) { + size_t const smHashAndTag = ZSTD_hashPtr(ip + i, hBitsS, mls); + size_t const lgHashAndTag = ZSTD_hashPtr(ip + i, hBitsL, 8); + if (i == 0) { + ZSTD_writeTaggedIndex(hashSmall, smHashAndTag, curr + i); + } + if (i == 0 || hashLarge[lgHashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { + ZSTD_writeTaggedIndex(hashLarge, lgHashAndTag, curr + i); + } + /* Only load extra positions for ZSTD_dtlm_full */ + if (dtlm == ZSTD_dtlm_fast) + break; + } } +} + +static void ZSTD_fillDoubleHashTableForCCtx(ZSTD_matchState_t* ms, void const* end, ZSTD_dictTableLoadMethod_e dtlm) { const ZSTD_compressionParameters* const cParams = &ms->cParams; @@ -44,7 +79,19 @@ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, /* Only load extra positions for ZSTD_dtlm_full */ if (dtlm == ZSTD_dtlm_fast) break; - } } + } } +} + +void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillDoubleHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillDoubleHashTableForCCtx(ms, end, dtlm); + } } @@ -68,7 +115,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; U32 offset_1=rep[0], offset_2=rep[1]; - U32 offsetSaved = 0; + U32 offsetSaved1 = 0, offsetSaved2 = 0; size_t mLength; U32 offset; @@ -101,8 +148,8 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( U32 const current = (U32)(ip - base); U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, current, cParams->windowLog); U32 const maxRep = current - windowLow; - if (offset_2 > maxRep) offsetSaved = offset_2, offset_2 = 0; - if (offset_1 > maxRep) offsetSaved = offset_1, offset_1 = 0; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; } /* Outer Loop: one iteration per match found and stored */ @@ -132,7 +179,7 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( if ((offset_1 > 0) & (MEM_read32(ip+1-offset_1) == MEM_read32(ip+1))) { mLength = ZSTD_count(ip+1+4, ip+1+4-offset_1, iend) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); goto _match_stored; } @@ -176,9 +223,13 @@ size_t ZSTD_compressBlock_doubleFast_noDict_generic( } while (ip1 <= ilimit); _cleanup: + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + /* save reps for next block */ - rep[0] = offset_1 ? offset_1 : offsetSaved; - rep[1] = offset_2 ? offset_2 : offsetSaved; + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; /* Return the last literals size */ return (size_t)(iend - anchor); @@ -218,7 +269,7 @@ _match_found: /* requires ip, offset, mLength */ hashLong[hl1] = (U32)(ip1 - base); } - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); _match_stored: /* match found */ @@ -244,7 +295,7 @@ _match_stored: U32 const tmpOff = offset_2; offset_2 = offset_1; offset_1 = tmpOff; /* swap offset_2 <=> offset_1 */ hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = (U32)(ip-base); hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = (U32)(ip-base); - ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, rLength); + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, rLength); ip += rLength; anchor = ip; continue; /* faster when present ... (?) */ @@ -276,7 +327,6 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; U32 offset_1=rep[0], offset_2=rep[1]; - U32 offsetSaved = 0; const ZSTD_matchState_t* const dms = ms->dictMatchState; const ZSTD_compressionParameters* const dictCParams = &dms->cParams; @@ -287,8 +337,8 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( const BYTE* const dictStart = dictBase + dictStartIndex; const BYTE* const dictEnd = dms->window.nextSrc; const U32 dictIndexDelta = prefixLowestIndex - (U32)(dictEnd - dictBase); - const U32 dictHBitsL = dictCParams->hashLog; - const U32 dictHBitsS = dictCParams->chainLog; + const U32 dictHBitsL = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + const U32 dictHBitsS = dictCParams->chainLog + ZSTD_SHORT_CACHE_TAG_BITS; const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictStart)); DEBUGLOG(5, "ZSTD_compressBlock_doubleFast_dictMatchState_generic"); @@ -296,6 +346,13 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( /* if a dictionary is attached, it must be within window range */ assert(ms->window.dictLimit + (1U << cParams->windowLog) >= endIndex); + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + size_t const chainTableBytes = (((size_t)1) << dictCParams->chainLog) * sizeof(U32); + PREFETCH_AREA(dictHashLong, hashTableBytes) + PREFETCH_AREA(dictHashSmall, chainTableBytes) + } + /* init */ ip += (dictAndPrefixLength == 0); @@ -310,8 +367,12 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( U32 offset; size_t const h2 = ZSTD_hashPtr(ip, hBitsL, 8); size_t const h = ZSTD_hashPtr(ip, hBitsS, mls); - size_t const dictHL = ZSTD_hashPtr(ip, dictHBitsL, 8); - size_t const dictHS = ZSTD_hashPtr(ip, dictHBitsS, mls); + size_t const dictHashAndTagL = ZSTD_hashPtr(ip, dictHBitsL, 8); + size_t const dictHashAndTagS = ZSTD_hashPtr(ip, dictHBitsS, mls); + U32 const dictMatchIndexAndTagL = dictHashLong[dictHashAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS]; + U32 const dictMatchIndexAndTagS = dictHashSmall[dictHashAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL = ZSTD_comparePackedTags(dictMatchIndexAndTagL, dictHashAndTagL); + int const dictTagsMatchS = ZSTD_comparePackedTags(dictMatchIndexAndTagS, dictHashAndTagS); U32 const curr = (U32)(ip-base); U32 const matchIndexL = hashLong[h2]; U32 matchIndexS = hashSmall[h]; @@ -329,7 +390,7 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); goto _match_stored; } @@ -341,9 +402,9 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( while (((ip>anchor) & (matchLong>prefixLowest)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ goto _match_found; } - } else { + } else if (dictTagsMatchL) { /* check dictMatchState long match */ - U32 const dictMatchIndexL = dictHashLong[dictHL]; + U32 const dictMatchIndexL = dictMatchIndexAndTagL >> ZSTD_SHORT_CACHE_TAG_BITS; const BYTE* dictMatchL = dictBase + dictMatchIndexL; assert(dictMatchL < dictEnd); @@ -359,9 +420,9 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( if (MEM_read32(match) == MEM_read32(ip)) { goto _search_next_long; } - } else { + } else if (dictTagsMatchS) { /* check dictMatchState short match */ - U32 const dictMatchIndexS = dictHashSmall[dictHS]; + U32 const dictMatchIndexS = dictMatchIndexAndTagS >> ZSTD_SHORT_CACHE_TAG_BITS; match = dictBase + dictMatchIndexS; matchIndexS = dictMatchIndexS + dictIndexDelta; @@ -376,10 +437,11 @@ size_t ZSTD_compressBlock_doubleFast_dictMatchState_generic( continue; _search_next_long: - { size_t const hl3 = ZSTD_hashPtr(ip+1, hBitsL, 8); - size_t const dictHLNext = ZSTD_hashPtr(ip+1, dictHBitsL, 8); + size_t const dictHashAndTagL3 = ZSTD_hashPtr(ip+1, dictHBitsL, 8); U32 const matchIndexL3 = hashLong[hl3]; + U32 const dictMatchIndexAndTagL3 = dictHashLong[dictHashAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int const dictTagsMatchL3 = ZSTD_comparePackedTags(dictMatchIndexAndTagL3, dictHashAndTagL3); const BYTE* matchL3 = base + matchIndexL3; hashLong[hl3] = curr + 1; @@ -392,9 +454,9 @@ _search_next_long: while (((ip>anchor) & (matchL3>prefixLowest)) && (ip[-1] == matchL3[-1])) { ip--; matchL3--; mLength++; } /* catch up */ goto _match_found; } - } else { + } else if (dictTagsMatchL3) { /* check dict long +1 match */ - U32 const dictMatchIndexL3 = dictHashLong[dictHLNext]; + U32 const dictMatchIndexL3 = dictMatchIndexAndTagL3 >> ZSTD_SHORT_CACHE_TAG_BITS; const BYTE* dictMatchL3 = dictBase + dictMatchIndexL3; assert(dictMatchL3 < dictEnd); if (dictMatchL3 > dictStart && MEM_read64(dictMatchL3) == MEM_read64(ip+1)) { @@ -420,7 +482,7 @@ _match_found: offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); _match_stored: /* match found */ @@ -449,7 +511,7 @@ _match_stored: const BYTE* const repEnd2 = repIndex2 < prefixLowestIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixLowest) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2); + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; ip += repLength2; @@ -462,8 +524,8 @@ _match_stored: } /* while (ip < ilimit) */ /* save reps for next block */ - rep[0] = offset_1 ? offset_1 : offsetSaved; - rep[1] = offset_2 ? offset_2 : offsetSaved; + rep[0] = offset_1; + rep[1] = offset_2; /* Return the last literals size */ return (size_t)(iend - anchor); @@ -586,7 +648,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); } else { if ((matchLongIndex > dictStartIndex) && (MEM_read64(matchLong) == MEM_read64(ip))) { const BYTE* const matchEnd = matchLongIndex < prefixStartIndex ? dictEnd : iend; @@ -597,7 +659,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( while (((ip>anchor) & (matchLong>lowMatchPtr)) && (ip[-1] == matchLong[-1])) { ip--; matchLong--; mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); } else if ((matchIndex > dictStartIndex) && (MEM_read32(match) == MEM_read32(ip))) { size_t const h3 = ZSTD_hashPtr(ip+1, hBitsL, 8); @@ -622,7 +684,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( } offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); } else { ip += ((ip-anchor) >> kSearchStrength) + 1; @@ -654,7 +716,7 @@ static size_t ZSTD_compressBlock_doubleFast_extDict_generic( const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2); + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); hashSmall[ZSTD_hashPtr(ip, hBitsS, mls)] = current2; hashLong[ZSTD_hashPtr(ip, hBitsL, 8)] = current2; ip += repLength2; diff --git a/vendor/github.com/DataDog/zstd/zstd_double_fast.h b/vendor/github.com/DataDog/zstd/zstd_double_fast.h index 5343b06..bdafd49 100644 --- a/vendor/github.com/DataDog/zstd/zstd_double_fast.h +++ b/vendor/github.com/DataDog/zstd/zstd_double_fast.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -20,7 +20,8 @@ extern "C" { #include "zstd_compress_internal.h" /* ZSTD_CCtx, size_t */ void ZSTD_fillDoubleHashTable(ZSTD_matchState_t* ms, - void const* end, ZSTD_dictTableLoadMethod_e dtlm); + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); size_t ZSTD_compressBlock_doubleFast( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); diff --git a/vendor/github.com/DataDog/zstd/zstd_errors.h b/vendor/github.com/DataDog/zstd/zstd_errors.h index 6e8f6d1..4064ecb 100644 --- a/vendor/github.com/DataDog/zstd/zstd_errors.h +++ b/vendor/github.com/DataDog/zstd/zstd_errors.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -21,19 +21,31 @@ extern "C" { /* ===== ZSTDERRORLIB_API : control library symbols visibility ===== */ -#ifndef ZSTDERRORLIB_VISIBILITY -# if defined(__GNUC__) && (__GNUC__ >= 4) -# define ZSTDERRORLIB_VISIBILITY __attribute__ ((visibility ("default"))) +#ifndef ZSTDERRORLIB_VISIBLE + /* Backwards compatibility with old macro name */ +# ifdef ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE ZSTDERRORLIB_VISIBILITY +# elif defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_VISIBLE __attribute__ ((visibility ("default"))) # else -# define ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_VISIBLE # endif #endif + +#ifndef ZSTDERRORLIB_HIDDEN +# if defined(__GNUC__) && (__GNUC__ >= 4) && !defined(__MINGW32__) +# define ZSTDERRORLIB_HIDDEN __attribute__ ((visibility ("hidden"))) +# else +# define ZSTDERRORLIB_HIDDEN +# endif +#endif + #if defined(ZSTD_DLL_EXPORT) && (ZSTD_DLL_EXPORT==1) -# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_API __declspec(dllexport) ZSTDERRORLIB_VISIBLE #elif defined(ZSTD_DLL_IMPORT) && (ZSTD_DLL_IMPORT==1) -# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBILITY /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ +# define ZSTDERRORLIB_API __declspec(dllimport) ZSTDERRORLIB_VISIBLE /* It isn't required but allows to generate better code, saving a function pointer load from the IAT and an indirect jump.*/ #else -# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBILITY +# define ZSTDERRORLIB_API ZSTDERRORLIB_VISIBLE #endif /*-********************************************* @@ -59,14 +71,17 @@ typedef enum { ZSTD_error_frameParameter_windowTooLarge = 16, ZSTD_error_corruption_detected = 20, ZSTD_error_checksum_wrong = 22, + ZSTD_error_literals_headerWrong = 24, ZSTD_error_dictionary_corrupted = 30, ZSTD_error_dictionary_wrong = 32, ZSTD_error_dictionaryCreation_failed = 34, ZSTD_error_parameter_unsupported = 40, + ZSTD_error_parameter_combination_unsupported = 41, ZSTD_error_parameter_outOfBound = 42, ZSTD_error_tableLog_tooLarge = 44, ZSTD_error_maxSymbolValue_tooLarge = 46, ZSTD_error_maxSymbolValue_tooSmall = 48, + ZSTD_error_stabilityCondition_notRespected = 50, ZSTD_error_stage_wrong = 60, ZSTD_error_init_missing = 62, ZSTD_error_memory_allocation = 64, @@ -74,11 +89,15 @@ typedef enum { ZSTD_error_dstSize_tooSmall = 70, ZSTD_error_srcSize_wrong = 72, ZSTD_error_dstBuffer_null = 74, + ZSTD_error_noForwardProgress_destFull = 80, + ZSTD_error_noForwardProgress_inputEmpty = 82, /* following error codes are __NOT STABLE__, they can be removed or changed in future versions */ ZSTD_error_frameIndex_tooLarge = 100, ZSTD_error_seekableIO = 102, ZSTD_error_dstBuffer_wrong = 104, ZSTD_error_srcBuffer_wrong = 105, + ZSTD_error_sequenceProducer_failed = 106, + ZSTD_error_externalSequences_invalid = 107, ZSTD_error_maxCode = 120 /* never EVER use this value directly, it can change in future versions! Use ZSTD_isError() instead */ } ZSTD_ErrorCode; diff --git a/vendor/github.com/DataDog/zstd/zstd_fast.c b/vendor/github.com/DataDog/zstd/zstd_fast.c index bd2dde1..1b60c21 100644 --- a/vendor/github.com/DataDog/zstd/zstd_fast.c +++ b/vendor/github.com/DataDog/zstd/zstd_fast.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -12,8 +12,42 @@ #include "zstd_compress_internal.h" /* ZSTD_hashPtr, ZSTD_count, ZSTD_storeSeq */ #include "zstd_fast.h" +static void ZSTD_fillHashTableForCDict(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm) +{ + const ZSTD_compressionParameters* const cParams = &ms->cParams; + U32* const hashTable = ms->hashTable; + U32 const hBits = cParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; + U32 const mls = cParams->minMatch; + const BYTE* const base = ms->window.base; + const BYTE* ip = base + ms->nextToUpdate; + const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; + const U32 fastHashFillStep = 3; -void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + /* Currently, we always use ZSTD_dtlm_full for filling CDict tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_full); + + /* Always insert every fastHashFillStep position into the hash table. + * Insert the other positions if their hash entry is empty. + */ + for ( ; ip + fastHashFillStep < iend + 2; ip += fastHashFillStep) { + U32 const curr = (U32)(ip - base); + { size_t const hashAndTag = ZSTD_hashPtr(ip, hBits, mls); + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr); } + + if (dtlm == ZSTD_dtlm_fast) continue; + /* Only load extra positions for ZSTD_dtlm_full */ + { U32 p; + for (p = 1; p < fastHashFillStep; ++p) { + size_t const hashAndTag = ZSTD_hashPtr(ip + p, hBits, mls); + if (hashTable[hashAndTag >> ZSTD_SHORT_CACHE_TAG_BITS] == 0) { /* not yet filled */ + ZSTD_writeTaggedIndex(hashTable, hashAndTag, curr + p); + } } } } +} + +static void ZSTD_fillHashTableForCCtx(ZSTD_matchState_t* ms, const void* const end, ZSTD_dictTableLoadMethod_e dtlm) { @@ -26,6 +60,10 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, const BYTE* const iend = ((const BYTE*)end) - HASH_READ_SIZE; const U32 fastHashFillStep = 3; + /* Currently, we always use ZSTD_dtlm_fast for filling CCtx tables. + * Feel free to remove this assert if there's a good reason! */ + assert(dtlm == ZSTD_dtlm_fast); + /* Always insert every fastHashFillStep position into the hash table. * Insert the other positions if their hash entry is empty. */ @@ -43,6 +81,18 @@ void ZSTD_fillHashTable(ZSTD_matchState_t* ms, } } } } } +void ZSTD_fillHashTable(ZSTD_matchState_t* ms, + const void* const end, + ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp) +{ + if (tfp == ZSTD_tfp_forCDict) { + ZSTD_fillHashTableForCDict(ms, end, dtlm); + } else { + ZSTD_fillHashTableForCCtx(ms, end, dtlm); + } +} + /** * If you squint hard enough (and ignore repcodes), the search operation at any @@ -118,7 +168,7 @@ ZSTD_compressBlock_fast_noDict_generic( U32 rep_offset1 = rep[0]; U32 rep_offset2 = rep[1]; - U32 offsetSaved = 0; + U32 offsetSaved1 = 0, offsetSaved2 = 0; size_t hash0; /* hash for ip0 */ size_t hash1; /* hash for ip1 */ @@ -142,8 +192,8 @@ ZSTD_compressBlock_fast_noDict_generic( { U32 const curr = (U32)(ip0 - base); U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, cParams->windowLog); U32 const maxRep = curr - windowLow; - if (rep_offset2 > maxRep) offsetSaved = rep_offset2, rep_offset2 = 0; - if (rep_offset1 > maxRep) offsetSaved = rep_offset1, rep_offset1 = 0; + if (rep_offset2 > maxRep) offsetSaved2 = rep_offset2, rep_offset2 = 0; + if (rep_offset1 > maxRep) offsetSaved1 = rep_offset1, rep_offset1 = 0; } /* start each op */ @@ -181,8 +231,14 @@ _start: /* Requires: ip0 */ mLength = ip0[-1] == match0[-1]; ip0 -= mLength; match0 -= mLength; - offcode = STORE_REPCODE_1; + offcode = REPCODE1_TO_OFFBASE; mLength += 4; + + /* First write next hash table entry; we've already calculated it. + * This write is known to be safe because the ip1 is before the + * repcode (ip2). */ + hashTable[hash1] = (U32)(ip1 - base); + goto _match; } @@ -196,6 +252,12 @@ _start: /* Requires: ip0 */ /* check match at ip[0] */ if (MEM_read32(ip0) == mval) { /* found a match! */ + + /* First write next hash table entry; we've already calculated it. + * This write is known to be safe because the ip1 == ip0 + 1, so + * we know we will resume searching after ip1 */ + hashTable[hash1] = (U32)(ip1 - base); + goto _offset; } @@ -225,6 +287,21 @@ _start: /* Requires: ip0 */ /* check match at ip[0] */ if (MEM_read32(ip0) == mval) { /* found a match! */ + + /* first write next hash table entry; we've already calculated it */ + if (step <= 4) { + /* We need to avoid writing an index into the hash table >= the + * position at which we will pick up our searching after we've + * taken this match. + * + * The minimum possible match has length 4, so the earliest ip0 + * can be after we take this match will be the current ip0 + 4. + * ip1 is ip0 + step - 1. If ip1 is >= ip0 + 4, we can't safely + * write this position. + */ + hashTable[hash1] = (U32)(ip1 - base); + } + goto _offset; } @@ -255,9 +332,24 @@ _cleanup: * However, it seems to be a meaningful performance hit to try to search * them. So let's not. */ + /* When the repcodes are outside of the prefix, we set them to zero before the loop. + * When the offsets are still zero, we need to restore them after the block to have a correct + * repcode history. If only one offset was invalid, it is easy. The tricky case is when both + * offsets were invalid. We need to figure out which offset to refill with. + * - If both offsets are zero they are in the same order. + * - If both offsets are non-zero, we won't restore the offsets from `offsetSaved[12]`. + * - If only one is zero, we need to decide which offset to restore. + * - If rep_offset1 is non-zero, then rep_offset2 must be offsetSaved1. + * - It is impossible for rep_offset2 to be non-zero. + * + * So if rep_offset1 started invalid (offsetSaved1 != 0) and became valid (rep_offset1 != 0), then + * set rep[0] = rep_offset1 and rep[1] = offsetSaved1. + */ + offsetSaved2 = ((offsetSaved1 != 0) && (rep_offset1 != 0)) ? offsetSaved1 : offsetSaved2; + /* save reps for next block */ - rep[0] = rep_offset1 ? rep_offset1 : offsetSaved; - rep[1] = rep_offset2 ? rep_offset2 : offsetSaved; + rep[0] = rep_offset1 ? rep_offset1 : offsetSaved1; + rep[1] = rep_offset2 ? rep_offset2 : offsetSaved2; /* Return the last literals size */ return (size_t)(iend - anchor); @@ -268,7 +360,7 @@ _offset: /* Requires: ip0, idx */ match0 = base + idx; rep_offset2 = rep_offset1; rep_offset1 = (U32)(ip0-match0); - offcode = STORE_OFFSET(rep_offset1); + offcode = OFFSET_TO_OFFBASE(rep_offset1); mLength = 4; /* Count the backwards match length. */ @@ -288,11 +380,6 @@ _match: /* Requires: ip0, match0, offcode */ ip0 += mLength; anchor = ip0; - /* write next hash table entry */ - if (ip1 < ip0) { - hashTable[hash1] = (U32)(ip1 - base); - } - /* Fill table and check for immediate repcode. */ if (ip0 <= ilimit) { /* Fill Table */ @@ -307,7 +394,7 @@ _match: /* Requires: ip0, match0, offcode */ { U32 const tmpOff = rep_offset2; rep_offset2 = rep_offset1; rep_offset1 = tmpOff; } /* swap rep_offset2 <=> rep_offset1 */ hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); ip0 += rLength; - ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, STORE_REPCODE_1, rLength); + ZSTD_storeSeq(seqStore, 0 /*litLen*/, anchor, iend, REPCODE1_TO_OFFBASE, rLength); anchor = ip0; continue; /* faster when present (confirmed on gcc-8) ... (?) */ } } } @@ -381,14 +468,14 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( U32 const stepSize = cParams->targetLength + !(cParams->targetLength); const BYTE* const base = ms->window.base; const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; + const BYTE* ip0 = istart; + const BYTE* ip1 = ip0 + stepSize; /* we assert below that stepSize >= 1 */ const BYTE* anchor = istart; const U32 prefixStartIndex = ms->window.dictLimit; const BYTE* const prefixStart = base + prefixStartIndex; const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - HASH_READ_SIZE; U32 offset_1=rep[0], offset_2=rep[1]; - U32 offsetSaved = 0; const ZSTD_matchState_t* const dms = ms->dictMatchState; const ZSTD_compressionParameters* const dictCParams = &dms->cParams ; @@ -398,13 +485,13 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( const BYTE* const dictStart = dictBase + dictStartIndex; const BYTE* const dictEnd = dms->window.nextSrc; const U32 dictIndexDelta = prefixStartIndex - (U32)(dictEnd - dictBase); - const U32 dictAndPrefixLength = (U32)(ip - prefixStart + dictEnd - dictStart); - const U32 dictHLog = dictCParams->hashLog; + const U32 dictAndPrefixLength = (U32)(istart - prefixStart + dictEnd - dictStart); + const U32 dictHBits = dictCParams->hashLog + ZSTD_SHORT_CACHE_TAG_BITS; /* if a dictionary is still attached, it necessarily means that * it is within window size. So we just check it. */ const U32 maxDistance = 1U << cParams->windowLog; - const U32 endIndex = (U32)((size_t)(ip - base) + srcSize); + const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); assert(endIndex - prefixStartIndex <= maxDistance); (void)maxDistance; (void)endIndex; /* these variables are not used when assert() is disabled */ @@ -414,106 +501,155 @@ size_t ZSTD_compressBlock_fast_dictMatchState_generic( * when translating a dict index into a local index */ assert(prefixStartIndex >= (U32)(dictEnd - dictBase)); + if (ms->prefetchCDictTables) { + size_t const hashTableBytes = (((size_t)1) << dictCParams->hashLog) * sizeof(U32); + PREFETCH_AREA(dictHashTable, hashTableBytes) + } + /* init */ DEBUGLOG(5, "ZSTD_compressBlock_fast_dictMatchState_generic"); - ip += (dictAndPrefixLength == 0); + ip0 += (dictAndPrefixLength == 0); /* dictMatchState repCode checks don't currently handle repCode == 0 * disabling. */ assert(offset_1 <= dictAndPrefixLength); assert(offset_2 <= dictAndPrefixLength); - /* Main Search Loop */ - while (ip < ilimit) { /* < instead of <=, because repcode check at (ip+1) */ + /* Outer search loop */ + assert(stepSize >= 1); + while (ip1 <= ilimit) { /* repcode check at (ip0 + 1) is safe because ip0 < ip1 */ size_t mLength; - size_t const h = ZSTD_hashPtr(ip, hlog, mls); - U32 const curr = (U32)(ip-base); - U32 const matchIndex = hashTable[h]; - const BYTE* match = base + matchIndex; - const U32 repIndex = curr + 1 - offset_1; - const BYTE* repMatch = (repIndex < prefixStartIndex) ? - dictBase + (repIndex - dictIndexDelta) : - base + repIndex; - hashTable[h] = curr; /* update hash table */ + size_t hash0 = ZSTD_hashPtr(ip0, hlog, mls); - if ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - mLength = ZSTD_count_2segments(ip+1+4, repMatch+4, iend, repMatchEnd, prefixStart) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, mLength); - } else if ( (matchIndex <= prefixStartIndex) ) { - size_t const dictHash = ZSTD_hashPtr(ip, dictHLog, mls); - U32 const dictMatchIndex = dictHashTable[dictHash]; - const BYTE* dictMatch = dictBase + dictMatchIndex; - if (dictMatchIndex <= dictStartIndex || - MEM_read32(dictMatch) != MEM_read32(ip)) { - assert(stepSize >= 1); - ip += ((ip-anchor) >> kSearchStrength) + stepSize; - continue; - } else { - /* found a dict match */ - U32 const offset = (U32)(curr-dictMatchIndex-dictIndexDelta); - mLength = ZSTD_count_2segments(ip+4, dictMatch+4, iend, dictEnd, prefixStart) + 4; - while (((ip>anchor) & (dictMatch>dictStart)) - && (ip[-1] == dictMatch[-1])) { - ip--; dictMatch--; mLength++; + size_t const dictHashAndTag0 = ZSTD_hashPtr(ip0, dictHBits, mls); + U32 dictMatchIndexAndTag = dictHashTable[dictHashAndTag0 >> ZSTD_SHORT_CACHE_TAG_BITS]; + int dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag0); + + U32 matchIndex = hashTable[hash0]; + U32 curr = (U32)(ip0 - base); + size_t step = stepSize; + const size_t kStepIncr = 1 << kSearchStrength; + const BYTE* nextStep = ip0 + kStepIncr; + + /* Inner search loop */ + while (1) { + const BYTE* match = base + matchIndex; + const U32 repIndex = curr + 1 - offset_1; + const BYTE* repMatch = (repIndex < prefixStartIndex) ? + dictBase + (repIndex - dictIndexDelta) : + base + repIndex; + const size_t hash1 = ZSTD_hashPtr(ip1, hlog, mls); + size_t const dictHashAndTag1 = ZSTD_hashPtr(ip1, dictHBits, mls); + hashTable[hash0] = curr; /* update hash table */ + + if (((U32) ((prefixStartIndex - 1) - repIndex) >= + 3) /* intentional underflow : ensure repIndex isn't overlapping dict + prefix */ + && (MEM_read32(repMatch) == MEM_read32(ip0 + 1))) { + const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + mLength = ZSTD_count_2segments(ip0 + 1 + 4, repMatch + 4, iend, repMatchEnd, prefixStart) + 4; + ip0++; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, REPCODE1_TO_OFFBASE, mLength); + break; + } + + if (dictTagsMatch) { + /* Found a possible dict match */ + const U32 dictMatchIndex = dictMatchIndexAndTag >> ZSTD_SHORT_CACHE_TAG_BITS; + const BYTE* dictMatch = dictBase + dictMatchIndex; + if (dictMatchIndex > dictStartIndex && + MEM_read32(dictMatch) == MEM_read32(ip0)) { + /* To replicate extDict parse behavior, we only use dict matches when the normal matchIndex is invalid */ + if (matchIndex <= prefixStartIndex) { + U32 const offset = (U32) (curr - dictMatchIndex - dictIndexDelta); + mLength = ZSTD_count_2segments(ip0 + 4, dictMatch + 4, iend, dictEnd, prefixStart) + 4; + while (((ip0 > anchor) & (dictMatch > dictStart)) + && (ip0[-1] == dictMatch[-1])) { + ip0--; + dictMatch--; + mLength++; + } /* catch up */ + offset_2 = offset_1; + offset_1 = offset; + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; + } + } + } + + if (matchIndex > prefixStartIndex && MEM_read32(match) == MEM_read32(ip0)) { + /* found a regular match */ + U32 const offset = (U32) (ip0 - match); + mLength = ZSTD_count(ip0 + 4, match + 4, iend) + 4; + while (((ip0 > anchor) & (match > prefixStart)) + && (ip0[-1] == match[-1])) { + ip0--; + match--; + mLength++; } /* catch up */ offset_2 = offset_1; offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); + ZSTD_storeSeq(seqStore, (size_t) (ip0 - anchor), anchor, iend, OFFSET_TO_OFFBASE(offset), mLength); + break; } - } else if (MEM_read32(match) != MEM_read32(ip)) { - /* it's not a match, and we're not going to check the dictionary */ - assert(stepSize >= 1); - ip += ((ip-anchor) >> kSearchStrength) + stepSize; - continue; - } else { - /* found a regular match */ - U32 const offset = (U32)(ip-match); - mLength = ZSTD_count(ip+4, match+4, iend) + 4; - while (((ip>anchor) & (match>prefixStart)) - && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - offset_2 = offset_1; - offset_1 = offset; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); - } + + /* Prepare for next iteration */ + dictMatchIndexAndTag = dictHashTable[dictHashAndTag1 >> ZSTD_SHORT_CACHE_TAG_BITS]; + dictTagsMatch = ZSTD_comparePackedTags(dictMatchIndexAndTag, dictHashAndTag1); + matchIndex = hashTable[hash1]; + + if (ip1 >= nextStep) { + step++; + nextStep += kStepIncr; + } + ip0 = ip1; + ip1 = ip1 + step; + if (ip1 > ilimit) goto _cleanup; + + curr = (U32)(ip0 - base); + hash0 = hash1; + } /* end inner search loop */ /* match found */ - ip += mLength; - anchor = ip; + assert(mLength); + ip0 += mLength; + anchor = ip0; - if (ip <= ilimit) { + if (ip0 <= ilimit) { /* Fill Table */ assert(base+curr+2 > istart); /* check base overflow */ hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; /* here because curr+2 could be > iend-8 */ - hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); /* check immediate repcode */ - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); + while (ip0 <= ilimit) { + U32 const current2 = (U32)(ip0-base); U32 const repIndex2 = current2 - offset_2; const BYTE* repMatch2 = repIndex2 < prefixStartIndex ? dictBase - dictIndexDelta + repIndex2 : base + repIndex2; if ( ((U32)((prefixStartIndex-1) - (U32)repIndex2) >= 3 /* intentional overflow */) - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { + && (MEM_read32(repMatch2) == MEM_read32(ip0))) { const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; U32 tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, repLength2); - hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; - ip += repLength2; - anchor = ip; + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = current2; + ip0 += repLength2; + anchor = ip0; continue; } break; } } + + /* Prepare for next iteration */ + assert(ip0 == anchor); + ip1 = ip0 + stepSize; } +_cleanup: /* save reps for next block */ - rep[0] = offset_1 ? offset_1 : offsetSaved; - rep[1] = offset_2 ? offset_2 : offsetSaved; + rep[0] = offset_1; + rep[1] = offset_2; /* Return the last literals size */ return (size_t)(iend - anchor); @@ -554,11 +690,10 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( U32* const hashTable = ms->hashTable; U32 const hlog = cParams->hashLog; /* support stepSize of 0 */ - U32 const stepSize = cParams->targetLength + !(cParams->targetLength); + size_t const stepSize = cParams->targetLength + !(cParams->targetLength) + 1; const BYTE* const base = ms->window.base; const BYTE* const dictBase = ms->window.dictBase; const BYTE* const istart = (const BYTE*)src; - const BYTE* ip = istart; const BYTE* anchor = istart; const U32 endIndex = (U32)((size_t)(istart - base) + srcSize); const U32 lowLimit = ZSTD_getLowestMatchIndex(ms, endIndex, cParams->windowLog); @@ -571,6 +706,28 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( const BYTE* const iend = istart + srcSize; const BYTE* const ilimit = iend - 8; U32 offset_1=rep[0], offset_2=rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; + + const BYTE* ip0 = istart; + const BYTE* ip1; + const BYTE* ip2; + const BYTE* ip3; + U32 current0; + + + size_t hash0; /* hash for ip0 */ + size_t hash1; /* hash for ip1 */ + U32 idx; /* match idx for ip0 */ + const BYTE* idxBase; /* base pointer for idx */ + + U32 offcode; + const BYTE* match0; + size_t mLength; + const BYTE* matchEnd = 0; /* initialize to avoid warning, assert != 0 later */ + + size_t step; + const BYTE* nextStep; + const size_t kStepIncr = (1 << (kSearchStrength - 1)); (void)hasStep; /* not currently specialized on whether it's accelerated */ @@ -580,75 +737,202 @@ static size_t ZSTD_compressBlock_fast_extDict_generic( if (prefixStartIndex == dictStartIndex) return ZSTD_compressBlock_fast(ms, seqStore, rep, src, srcSize); - /* Search Loop */ - while (ip < ilimit) { /* < instead of <=, because (ip+1) */ - const size_t h = ZSTD_hashPtr(ip, hlog, mls); - const U32 matchIndex = hashTable[h]; - const BYTE* const matchBase = matchIndex < prefixStartIndex ? dictBase : base; - const BYTE* match = matchBase + matchIndex; - const U32 curr = (U32)(ip-base); - const U32 repIndex = curr + 1 - offset_1; - const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; - const BYTE* const repMatch = repBase + repIndex; - hashTable[h] = curr; /* update hash table */ - DEBUGLOG(7, "offset_1 = %u , curr = %u", offset_1, curr); + { U32 const curr = (U32)(ip0 - base); + U32 const maxRep = curr - dictStartIndex; + if (offset_2 >= maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 >= maxRep) offsetSaved1 = offset_1, offset_1 = 0; + } - if ( ( ((U32)((prefixStartIndex-1) - repIndex) >= 3) /* intentional underflow */ - & (offset_1 <= curr+1 - dictStartIndex) ) /* note: we are searching at curr+1 */ - && (MEM_read32(repMatch) == MEM_read32(ip+1)) ) { - const BYTE* const repMatchEnd = repIndex < prefixStartIndex ? dictEnd : iend; - size_t const rLength = ZSTD_count_2segments(ip+1 +4, repMatch +4, iend, repMatchEnd, prefixStart) + 4; - ip++; - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_REPCODE_1, rLength); - ip += rLength; - anchor = ip; - } else { - if ( (matchIndex < dictStartIndex) || - (MEM_read32(match) != MEM_read32(ip)) ) { - assert(stepSize >= 1); - ip += ((ip-anchor) >> kSearchStrength) + stepSize; - continue; + /* start each op */ +_start: /* Requires: ip0 */ + + step = stepSize; + nextStep = ip0 + kStepIncr; + + /* calculate positions, ip0 - anchor == 0, so we skip step calc */ + ip1 = ip0 + 1; + ip2 = ip0 + step; + ip3 = ip2 + 1; + + if (ip3 >= ilimit) { + goto _cleanup; + } + + hash0 = ZSTD_hashPtr(ip0, hlog, mls); + hash1 = ZSTD_hashPtr(ip1, hlog, mls); + + idx = hashTable[hash0]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + do { + { /* load repcode match for ip[2] */ + U32 const current2 = (U32)(ip2 - base); + U32 const repIndex = current2 - offset_1; + const BYTE* const repBase = repIndex < prefixStartIndex ? dictBase : base; + U32 rval; + if ( ((U32)(prefixStartIndex - repIndex) >= 4) /* intentional underflow */ + & (offset_1 > 0) ) { + rval = MEM_read32(repBase + repIndex); + } else { + rval = MEM_read32(ip2) ^ 1; /* guaranteed to not match. */ } - { const BYTE* const matchEnd = matchIndex < prefixStartIndex ? dictEnd : iend; - const BYTE* const lowMatchPtr = matchIndex < prefixStartIndex ? dictStart : prefixStart; - U32 const offset = curr - matchIndex; - size_t mLength = ZSTD_count_2segments(ip+4, match+4, iend, matchEnd, prefixStart) + 4; - while (((ip>anchor) & (match>lowMatchPtr)) && (ip[-1] == match[-1])) { ip--; match--; mLength++; } /* catch up */ - offset_2 = offset_1; offset_1 = offset; /* update offset history */ - ZSTD_storeSeq(seqStore, (size_t)(ip-anchor), anchor, iend, STORE_OFFSET(offset), mLength); - ip += mLength; - anchor = ip; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + /* check repcode at ip[2] */ + if (MEM_read32(ip2) == rval) { + ip0 = ip2; + match0 = repBase + repIndex; + matchEnd = repIndex < prefixStartIndex ? dictEnd : iend; + assert((match0 != prefixStart) & (match0 != dictStart)); + mLength = ip0[-1] == match0[-1]; + ip0 -= mLength; + match0 -= mLength; + offcode = REPCODE1_TO_OFFBASE; + mLength += 4; + goto _match; } } - if (ip <= ilimit) { - /* Fill Table */ - hashTable[ZSTD_hashPtr(base+curr+2, hlog, mls)] = curr+2; - hashTable[ZSTD_hashPtr(ip-2, hlog, mls)] = (U32)(ip-2-base); - /* check immediate repcode */ - while (ip <= ilimit) { - U32 const current2 = (U32)(ip-base); - U32 const repIndex2 = current2 - offset_2; - const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; - if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 <= curr - dictStartIndex)) /* intentional overflow */ - && (MEM_read32(repMatch2) == MEM_read32(ip)) ) { - const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; - size_t const repLength2 = ZSTD_count_2segments(ip+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; - { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, STORE_REPCODE_1, repLength2); - hashTable[ZSTD_hashPtr(ip, hlog, mls)] = current2; - ip += repLength2; - anchor = ip; - continue; - } - break; - } } } + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip3; + + /* write back hash table entry */ + current0 = (U32)(ip0 - base); + hashTable[hash0] = current0; + + { /* load match for ip[0] */ + U32 const mval = idx >= dictStartIndex ? + MEM_read32(idxBase + idx) : + MEM_read32(ip0) ^ 1; /* guaranteed not to match */ + + /* check match at ip[0] */ + if (MEM_read32(ip0) == mval) { + /* found a match! */ + goto _offset; + } } + + /* lookup ip[1] */ + idx = hashTable[hash1]; + idxBase = idx < prefixStartIndex ? dictBase : base; + + /* hash ip[2] */ + hash0 = hash1; + hash1 = ZSTD_hashPtr(ip2, hlog, mls); + + /* advance to next positions */ + ip0 = ip1; + ip1 = ip2; + ip2 = ip0 + step; + ip3 = ip1 + step; + + /* calculate step */ + if (ip2 >= nextStep) { + step++; + PREFETCH_L1(ip1 + 64); + PREFETCH_L1(ip1 + 128); + nextStep += kStepIncr; + } + } while (ip3 < ilimit); + +_cleanup: + /* Note that there are probably still a couple positions we could search. + * However, it seems to be a meaningful performance hit to try to search + * them. So let's not. */ + + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; /* save reps for next block */ - rep[0] = offset_1; - rep[1] = offset_2; + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; /* Return the last literals size */ return (size_t)(iend - anchor); + +_offset: /* Requires: ip0, idx, idxBase */ + + /* Compute the offset code. */ + { U32 const offset = current0 - idx; + const BYTE* const lowMatchPtr = idx < prefixStartIndex ? dictStart : prefixStart; + matchEnd = idx < prefixStartIndex ? dictEnd : iend; + match0 = idxBase + idx; + offset_2 = offset_1; + offset_1 = offset; + offcode = OFFSET_TO_OFFBASE(offset); + mLength = 4; + + /* Count the backwards match length. */ + while (((ip0>anchor) & (match0>lowMatchPtr)) && (ip0[-1] == match0[-1])) { + ip0--; + match0--; + mLength++; + } } + +_match: /* Requires: ip0, match0, offcode, matchEnd */ + + /* Count the forward length. */ + assert(matchEnd != 0); + mLength += ZSTD_count_2segments(ip0 + mLength, match0 + mLength, iend, matchEnd, prefixStart); + + ZSTD_storeSeq(seqStore, (size_t)(ip0 - anchor), anchor, iend, offcode, mLength); + + ip0 += mLength; + anchor = ip0; + + /* write next hash table entry */ + if (ip1 < ip0) { + hashTable[hash1] = (U32)(ip1 - base); + } + + /* Fill table and check for immediate repcode. */ + if (ip0 <= ilimit) { + /* Fill Table */ + assert(base+current0+2 > istart); /* check base overflow */ + hashTable[ZSTD_hashPtr(base+current0+2, hlog, mls)] = current0+2; /* here because current+2 could be > iend-8 */ + hashTable[ZSTD_hashPtr(ip0-2, hlog, mls)] = (U32)(ip0-2-base); + + while (ip0 <= ilimit) { + U32 const repIndex2 = (U32)(ip0-base) - offset_2; + const BYTE* const repMatch2 = repIndex2 < prefixStartIndex ? dictBase + repIndex2 : base + repIndex2; + if ( (((U32)((prefixStartIndex-1) - repIndex2) >= 3) & (offset_2 > 0)) /* intentional underflow */ + && (MEM_read32(repMatch2) == MEM_read32(ip0)) ) { + const BYTE* const repEnd2 = repIndex2 < prefixStartIndex ? dictEnd : iend; + size_t const repLength2 = ZSTD_count_2segments(ip0+4, repMatch2+4, iend, repEnd2, prefixStart) + 4; + { U32 const tmpOffset = offset_2; offset_2 = offset_1; offset_1 = tmpOffset; } /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0 /*litlen*/, anchor, iend, REPCODE1_TO_OFFBASE, repLength2); + hashTable[ZSTD_hashPtr(ip0, hlog, mls)] = (U32)(ip0-base); + ip0 += repLength2; + anchor = ip0; + continue; + } + break; + } } + + goto _start; } ZSTD_GEN_FAST_FN(extDict, 4, 0) @@ -661,6 +945,7 @@ size_t ZSTD_compressBlock_fast_extDict( void const* src, size_t srcSize) { U32 const mls = ms->cParams.minMatch; + assert(ms->dictMatchState == NULL); switch(mls) { default: /* includes case 3 */ diff --git a/vendor/github.com/DataDog/zstd/zstd_fast.h b/vendor/github.com/DataDog/zstd/zstd_fast.h index 6217aaf..88f8f8e 100644 --- a/vendor/github.com/DataDog/zstd/zstd_fast.h +++ b/vendor/github.com/DataDog/zstd/zstd_fast.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -20,7 +20,8 @@ extern "C" { #include "zstd_compress_internal.h" void ZSTD_fillHashTable(ZSTD_matchState_t* ms, - void const* end, ZSTD_dictTableLoadMethod_e dtlm); + void const* end, ZSTD_dictTableLoadMethod_e dtlm, + ZSTD_tableFillPurpose_e tfp); size_t ZSTD_compressBlock_fast( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); diff --git a/vendor/github.com/DataDog/zstd/zstd_internal.h b/vendor/github.com/DataDog/zstd/zstd_internal.h index e256618..0970754 100644 --- a/vendor/github.com/DataDog/zstd/zstd_internal.h +++ b/vendor/github.com/DataDog/zstd/zstd_internal.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -29,7 +29,6 @@ #include "zstd.h" #define FSE_STATIC_LINKING_ONLY #include "fse.h" -#define HUF_STATIC_LINKING_ONLY #include "huf.h" #ifndef XXH_STATIC_LINKING_ONLY # define XXH_STATIC_LINKING_ONLY /* XXH64_state_t */ @@ -94,9 +93,9 @@ typedef enum { bt_raw, bt_rle, bt_compressed, bt_reserved } blockType_e; #define ZSTD_FRAMECHECKSUMSIZE 4 #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ -#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ +#define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */) /* for a non-null block */ +#define MIN_LITERALS_FOR_4_STREAMS 6 -#define HufLog 12 typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingType_e; #define LONGNBSEQ 0x7F00 @@ -104,6 +103,7 @@ typedef enum { set_basic, set_rle, set_compressed, set_repeat } symbolEncodingTy #define MINMATCH 3 #define Litbits 8 +#define LitHufLog 11 #define MaxLit ((1<= length) return; op += 16; @@ -251,7 +247,6 @@ void ZSTD_wildcopy(void* dst, const void* src, ptrdiff_t length, ZSTD_overlap_e COPY16(op, ip); } while (op < oend); -#endif } } @@ -300,11 +295,11 @@ typedef enum { typedef struct { seqDef* sequencesStart; seqDef* sequences; /* ptr to end of sequences */ - BYTE* litStart; - BYTE* lit; /* ptr to end of literals */ - BYTE* llCode; - BYTE* mlCode; - BYTE* ofCode; + BYTE* litStart; + BYTE* lit; /* ptr to end of literals */ + BYTE* llCode; + BYTE* mlCode; + BYTE* ofCode; size_t maxNbSeq; size_t maxNbLit; @@ -312,8 +307,8 @@ typedef struct { * in the seqStore that has a value larger than U16 (if it exists). To do so, we increment * the existing value of the litLength or matchLength by 0x10000. */ - ZSTD_longLengthType_e longLengthType; - U32 longLengthPos; /* Index of the sequence to apply long length modification to */ + ZSTD_longLengthType_e longLengthType; + U32 longLengthPos; /* Index of the sequence to apply long length modification to */ } seqStore_t; typedef struct { @@ -332,10 +327,10 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore seqLen.matchLength = seq->mlBase + MINMATCH; if (seqStore->longLengthPos == (U32)(seq - seqStore->sequencesStart)) { if (seqStore->longLengthType == ZSTD_llt_literalLength) { - seqLen.litLength += 0xFFFF; + seqLen.litLength += 0x10000; } if (seqStore->longLengthType == ZSTD_llt_matchLength) { - seqLen.matchLength += 0xFFFF; + seqLen.matchLength += 0x10000; } } return seqLen; @@ -348,109 +343,13 @@ MEM_STATIC ZSTD_sequenceLength ZSTD_getSequenceLength(seqStore_t const* seqStore * `decompressedBound != ZSTD_CONTENTSIZE_ERROR` */ typedef struct { + size_t nbBlocks; size_t compressedSize; unsigned long long decompressedBound; } ZSTD_frameSizeInfo; /* decompress & legacy */ const seqStore_t* ZSTD_getSeqStore(const ZSTD_CCtx* ctx); /* compress & dictBuilder */ -void ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ - -/* custom memory allocation functions */ -void* ZSTD_customMalloc(size_t size, ZSTD_customMem customMem); -void* ZSTD_customCalloc(size_t size, ZSTD_customMem customMem); -void ZSTD_customFree(void* ptr, ZSTD_customMem customMem); - - -MEM_STATIC U32 ZSTD_highbit32(U32 val) /* compress, dictBuilder, decodeCorpus */ -{ - assert(val != 0); - { -# if defined(_MSC_VER) /* Visual */ -# if STATIC_BMI2 == 1 - return _lzcnt_u32(val)^31; -# else - if (val != 0) { - unsigned long r; - _BitScanReverse(&r, val); - return (unsigned)r; - } else { - /* Should not reach this code path */ - __assume(0); - } -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 3) /* GCC Intrinsic */ - return __builtin_clz (val) ^ 31; -# elif defined(__ICCARM__) /* IAR Intrinsic */ - return 31 - __CLZ(val); -# else /* Software version */ - static const U32 DeBruijnClz[32] = { 0, 9, 1, 10, 13, 21, 2, 29, 11, 14, 16, 18, 22, 25, 3, 30, 8, 12, 20, 28, 15, 17, 24, 7, 19, 27, 23, 6, 26, 5, 4, 31 }; - U32 v = val; - v |= v >> 1; - v |= v >> 2; - v |= v >> 4; - v |= v >> 8; - v |= v >> 16; - return DeBruijnClz[(v * 0x07C4ACDDU) >> 27]; -# endif - } -} - -/** - * Counts the number of trailing zeros of a `size_t`. - * Most compilers should support CTZ as a builtin. A backup - * implementation is provided if the builtin isn't supported, but - * it may not be terribly efficient. - */ -MEM_STATIC unsigned ZSTD_countTrailingZeros(size_t val) -{ - if (MEM_64bits()) { -# if defined(_MSC_VER) && defined(_WIN64) -# if STATIC_BMI2 - return _tzcnt_u64(val); -# else - if (val != 0) { - unsigned long r; - _BitScanForward64(&r, (U64)val); - return (unsigned)r; - } else { - /* Should not reach this code path */ - __assume(0); - } -# endif -# elif defined(__GNUC__) && (__GNUC__ >= 4) - return __builtin_ctzll((U64)val); -# else - static const int DeBruijnBytePos[64] = { 0, 1, 2, 7, 3, 13, 8, 19, - 4, 25, 14, 28, 9, 34, 20, 56, - 5, 17, 26, 54, 15, 41, 29, 43, - 10, 31, 38, 35, 21, 45, 49, 57, - 63, 6, 12, 18, 24, 27, 33, 55, - 16, 53, 40, 42, 30, 37, 44, 48, - 62, 11, 23, 32, 52, 39, 36, 47, - 61, 22, 51, 46, 60, 50, 59, 58 }; - return DeBruijnBytePos[((U64)((val & -(long long)val) * 0x0218A392CDABBD3FULL)) >> 58]; -# endif - } else { /* 32 bits */ -# if defined(_MSC_VER) - if (val != 0) { - unsigned long r; - _BitScanForward(&r, (U32)val); - return (unsigned)r; - } else { - /* Should not reach this code path */ - __assume(0); - } -# elif defined(__GNUC__) && (__GNUC__ >= 3) - return __builtin_ctz((U32)val); -# else - static const int DeBruijnBytePos[32] = { 0, 1, 28, 2, 29, 14, 24, 3, - 30, 22, 20, 15, 25, 17, 4, 8, - 31, 27, 13, 23, 21, 19, 16, 7, - 26, 12, 18, 6, 11, 5, 10, 9 }; - return DeBruijnBytePos[((U32)((val & -(S32)val) * 0x077CB531U)) >> 27]; -# endif - } -} +int ZSTD_seqToCodes(const seqStore_t* seqStorePtr); /* compress, dictBuilder, decodeCorpus (shouldn't get its definition from here) */ /* ZSTD_invalidateRepCodes() : diff --git a/vendor/github.com/DataDog/zstd/zstd_lazy.c b/vendor/github.com/DataDog/zstd/zstd_lazy.c index f73f1e9..9463e66 100644 --- a/vendor/github.com/DataDog/zstd/zstd_lazy.c +++ b/vendor/github.com/DataDog/zstd/zstd_lazy.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -11,6 +11,9 @@ #include "zstd_compress_internal.h" #include "zstd_lazy.h" +#include "bits.h" /* ZSTD_countTrailingZeros64 */ + +#define kLazySkippingStep 8 /*-************************************* @@ -198,8 +201,8 @@ ZSTD_DUBT_findBetterDictMatch ( U32 matchIndex = dictMatchIndex + dictIndexDelta; if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) { DEBUGLOG(9, "ZSTD_DUBT_findBetterDictMatch(%u) : found better match length %u -> %u and offsetCode %u -> %u (dictMatchIndex %u, matchIndex %u)", - curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, STORE_OFFSET(curr - matchIndex), dictMatchIndex, matchIndex); - bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex); + curr, (U32)bestLength, (U32)matchLength, (U32)*offsetPtr, OFFSET_TO_OFFBASE(curr - matchIndex), dictMatchIndex, matchIndex); + bestLength = matchLength, *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); } if (ip+matchLength == iend) { /* reached end of input : ip[matchLength] is not valid, no way to know if it's larger or smaller than match */ break; /* drop, to guarantee consistency (miss a little bit of compression) */ @@ -219,7 +222,7 @@ ZSTD_DUBT_findBetterDictMatch ( } if (bestLength >= MINMATCH) { - U32 const mIndex = curr - (U32)STORED_OFFSET(*offsetPtr); (void)mIndex; + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offsetPtr); (void)mIndex; DEBUGLOG(8, "ZSTD_DUBT_findBetterDictMatch(%u) : found match of length %u and offsetCode %u (pos %u)", curr, (U32)bestLength, (U32)*offsetPtr, mIndex); } @@ -231,7 +234,7 @@ ZSTD_DUBT_findBetterDictMatch ( static size_t ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, const BYTE* const ip, const BYTE* const iend, - size_t* offsetPtr, + size_t* offBasePtr, U32 const mls, const ZSTD_dictMode_e dictMode) { @@ -328,8 +331,8 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, if (matchLength > bestLength) { if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; - if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr-matchIndex+1) - ZSTD_highbit32((U32)offsetPtr[0]+1)) ) - bestLength = matchLength, *offsetPtr = STORE_OFFSET(curr - matchIndex); + if ( (4*(int)(matchLength-bestLength)) > (int)(ZSTD_highbit32(curr - matchIndex + 1) - ZSTD_highbit32((U32)*offBasePtr)) ) + bestLength = matchLength, *offBasePtr = OFFSET_TO_OFFBASE(curr - matchIndex); if (ip+matchLength == iend) { /* equal : no way to know if inf or sup */ if (dictMode == ZSTD_dictMatchState) { nbCompares = 0; /* in addition to avoiding checking any @@ -362,16 +365,16 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, if (dictMode == ZSTD_dictMatchState && nbCompares) { bestLength = ZSTD_DUBT_findBetterDictMatch( ms, ip, iend, - offsetPtr, bestLength, nbCompares, + offBasePtr, bestLength, nbCompares, mls, dictMode); } assert(matchEndIdx > curr+8); /* ensure nextToUpdate is increased */ ms->nextToUpdate = matchEndIdx - 8; /* skip repetitive patterns */ if (bestLength >= MINMATCH) { - U32 const mIndex = curr - (U32)STORED_OFFSET(*offsetPtr); (void)mIndex; + U32 const mIndex = curr - (U32)OFFBASE_TO_OFFSET(*offBasePtr); (void)mIndex; DEBUGLOG(8, "ZSTD_DUBT_findBestMatch(%u) : found match of length %u and offsetCode %u (pos %u)", - curr, (U32)bestLength, (U32)*offsetPtr, mIndex); + curr, (U32)bestLength, (U32)*offBasePtr, mIndex); } return bestLength; } @@ -382,14 +385,14 @@ ZSTD_DUBT_findBestMatch(ZSTD_matchState_t* ms, FORCE_INLINE_TEMPLATE size_t ZSTD_BtFindBestMatch( ZSTD_matchState_t* ms, const BYTE* const ip, const BYTE* const iLimit, - size_t* offsetPtr, + size_t* offBasePtr, const U32 mls /* template */, const ZSTD_dictMode_e dictMode) { DEBUGLOG(7, "ZSTD_BtFindBestMatch"); if (ip < ms->window.base + ms->nextToUpdate) return 0; /* skipped area */ ZSTD_updateDUBT(ms, ip, iLimit, mls); - return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offsetPtr, mls, dictMode); + return ZSTD_DUBT_findBestMatch(ms, ip, iLimit, offBasePtr, mls, dictMode); } /*********************************** @@ -562,7 +565,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb /* save best solution */ if (currentMl > ml) { ml = currentMl; - *offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta)); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); if (ip+currentMl == iLimit) { /* best possible, avoids read overflow on next attempt */ return ml; @@ -599,7 +602,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb /* save best solution */ if (currentMl > ml) { ml = currentMl; - *offsetPtr = STORE_OFFSET(curr - (matchIndex + ddsIndexDelta)); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + ddsIndexDelta)); if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ } } @@ -618,7 +621,7 @@ size_t ZSTD_dedicatedDictSearch_lazy_search(size_t* offsetPtr, size_t ml, U32 nb FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal( ZSTD_matchState_t* ms, const ZSTD_compressionParameters* const cParams, - const BYTE* ip, U32 const mls) + const BYTE* ip, U32 const mls, U32 const lazySkipping) { U32* const hashTable = ms->hashTable; const U32 hashLog = cParams->hashLog; @@ -633,6 +636,9 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal( NEXT_IN_CHAIN(idx, chainMask) = hashTable[h]; hashTable[h] = idx; idx++; + /* Stop inserting every position when in the lazy skipping mode. */ + if (lazySkipping) + break; } ms->nextToUpdate = target; @@ -641,7 +647,7 @@ FORCE_INLINE_TEMPLATE U32 ZSTD_insertAndFindFirstIndex_internal( U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip) { const ZSTD_compressionParameters* const cParams = &ms->cParams; - return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch); + return ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, ms->cParams.minMatch, /* lazySkipping*/ 0); } /* inlining is important to hardwire a hot branch (template emulation) */ @@ -685,14 +691,15 @@ size_t ZSTD_HcFindBestMatch( } /* HC4 match finder */ - matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls); + matchIndex = ZSTD_insertAndFindFirstIndex_internal(ms, cParams, ip, mls, ms->lazySkipping); for ( ; (matchIndex>=lowLimit) & (nbAttempts>0) ; nbAttempts--) { size_t currentMl=0; if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { const BYTE* const match = base + matchIndex; assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ - if (match[ml] == ip[ml]) /* potentially better */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ currentMl = ZSTD_count(ip, match, iLimit); } else { const BYTE* const match = dictBase + matchIndex; @@ -704,7 +711,7 @@ size_t ZSTD_HcFindBestMatch( /* save best solution */ if (currentMl > ml) { ml = currentMl; - *offsetPtr = STORE_OFFSET(curr - matchIndex); + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ } @@ -740,7 +747,7 @@ size_t ZSTD_HcFindBestMatch( if (currentMl > ml) { ml = currentMl; assert(curr > matchIndex + dmsIndexDelta); - *offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta)); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ } @@ -757,8 +764,6 @@ size_t ZSTD_HcFindBestMatch( * (SIMD) Row-based matchfinder ***********************************/ /* Constants for row-based hash */ -#define ZSTD_ROW_HASH_TAG_OFFSET 16 /* byte offset of hashes in the match state's tagTable from the beginning of a row */ -#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */ #define ZSTD_ROW_HASH_TAG_MASK ((1u << ZSTD_ROW_HASH_TAG_BITS) - 1) #define ZSTD_ROW_HASH_MAX_ENTRIES 64 /* absolute maximum number of entries per row, for all configurations */ @@ -770,73 +775,19 @@ typedef U64 ZSTD_VecMask; /* Clarifies when we are interacting with a U64 repr * Starting from the LSB, returns the idx of the next non-zero bit. * Basically counting the nb of trailing zeroes. */ -static U32 ZSTD_VecMask_next(ZSTD_VecMask val) { - assert(val != 0); -# if defined(_MSC_VER) && defined(_WIN64) - if (val != 0) { - unsigned long r; - _BitScanForward64(&r, val); - return (U32)(r); - } else { - /* Should not reach this code path */ - __assume(0); - } -# elif (defined(__GNUC__) && ((__GNUC__ > 3) || ((__GNUC__ == 3) && (__GNUC_MINOR__ >= 4)))) - if (sizeof(size_t) == 4) { - U32 mostSignificantWord = (U32)(val >> 32); - U32 leastSignificantWord = (U32)val; - if (leastSignificantWord == 0) { - return 32 + (U32)__builtin_ctz(mostSignificantWord); - } else { - return (U32)__builtin_ctz(leastSignificantWord); - } - } else { - return (U32)__builtin_ctzll(val); - } -# else - /* Software ctz version: http://aggregate.org/MAGIC/#Trailing%20Zero%20Count - * and: https://stackoverflow.com/questions/2709430/count-number-of-bits-in-a-64-bit-long-big-integer - */ - val = ~val & (val - 1ULL); /* Lowest set bit mask */ - val = val - ((val >> 1) & 0x5555555555555555); - val = (val & 0x3333333333333333ULL) + ((val >> 2) & 0x3333333333333333ULL); - return (U32)((((val + (val >> 4)) & 0xF0F0F0F0F0F0F0FULL) * 0x101010101010101ULL) >> 56); -# endif -} - -/* ZSTD_rotateRight_*(): - * Rotates a bitfield to the right by "count" bits. - * https://en.wikipedia.org/w/index.php?title=Circular_shift&oldid=991635599#Implementing_circular_shifts - */ -FORCE_INLINE_TEMPLATE -U64 ZSTD_rotateRight_U64(U64 const value, U32 count) { - assert(count < 64); - count &= 0x3F; /* for fickle pattern recognition */ - return (value >> count) | (U64)(value << ((0U - count) & 0x3F)); -} - -FORCE_INLINE_TEMPLATE -U32 ZSTD_rotateRight_U32(U32 const value, U32 count) { - assert(count < 32); - count &= 0x1F; /* for fickle pattern recognition */ - return (value >> count) | (U32)(value << ((0U - count) & 0x1F)); -} - -FORCE_INLINE_TEMPLATE -U16 ZSTD_rotateRight_U16(U16 const value, U32 count) { - assert(count < 16); - count &= 0x0F; /* for fickle pattern recognition */ - return (value >> count) | (U16)(value << ((0U - count) & 0x0F)); +MEM_STATIC U32 ZSTD_VecMask_next(ZSTD_VecMask val) { + return ZSTD_countTrailingZeros64(val); } /* ZSTD_row_nextIndex(): * Returns the next index to insert at within a tagTable row, and updates the "head" - * value to reflect the update. Essentially cycles backwards from [0, {entries per row}) + * value to reflect the update. Essentially cycles backwards from [1, {entries per row}) */ FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextIndex(BYTE* const tagRow, U32 const rowMask) { - U32 const next = (*tagRow - 1) & rowMask; - *tagRow = (BYTE)next; - return next; + U32 next = (*tagRow-1) & rowMask; + next += (next == 0) ? rowMask : 0; /* skip first position */ + *tagRow = (BYTE)next; + return next; } /* ZSTD_isAligned(): @@ -850,7 +801,7 @@ MEM_STATIC int ZSTD_isAligned(void const* ptr, size_t align) { /* ZSTD_row_prefetch(): * Performs prefetching for the hashTable and tagTable at a given row. */ -FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, U16 const* tagTable, U32 const relRow, U32 const rowLog) { +FORCE_INLINE_TEMPLATE void ZSTD_row_prefetch(U32 const* hashTable, BYTE const* tagTable, U32 const relRow, U32 const rowLog) { PREFETCH_L1(hashTable + relRow); if (rowLog >= 5) { PREFETCH_L1(hashTable + relRow + 16); @@ -874,13 +825,13 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const B U32 idx, const BYTE* const iLimit) { U32 const* const hashTable = ms->hashTable; - U16 const* const tagTable = ms->tagTable; + BYTE const* const tagTable = ms->tagTable; U32 const hashLog = ms->rowHashLog; U32 const maxElemsToPrefetch = (base + idx) > iLimit ? 0 : (U32)(iLimit - (base + idx) + 1); U32 const lim = idx + MIN(ZSTD_ROW_HASH_CACHE_SIZE, maxElemsToPrefetch); for (; idx < lim; ++idx) { - U32 const hash = (U32)ZSTD_hashPtr(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const hash = (U32)ZSTD_hashPtrSalted(base + idx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); U32 const row = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); ms->hashCache[idx & ZSTD_ROW_HASH_CACHE_MASK] = hash; @@ -896,11 +847,12 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_fillHashCache(ZSTD_matchState_t* ms, const B * base + idx + ZSTD_ROW_HASH_CACHE_SIZE. Also prefetches the appropriate rows from hashTable and tagTable. */ FORCE_INLINE_TEMPLATE U32 ZSTD_row_nextCachedHash(U32* cache, U32 const* hashTable, - U16 const* tagTable, BYTE const* base, + BYTE const* tagTable, BYTE const* base, U32 idx, U32 const hashLog, - U32 const rowLog, U32 const mls) + U32 const rowLog, U32 const mls, + U64 const hashSalt) { - U32 const newHash = (U32)ZSTD_hashPtr(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const newHash = (U32)ZSTD_hashPtrSalted(base+idx+ZSTD_ROW_HASH_CACHE_SIZE, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); U32 const row = (newHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; ZSTD_row_prefetch(hashTable, tagTable, row, rowLog); { U32 const hash = cache[idx & ZSTD_ROW_HASH_CACHE_MASK]; @@ -918,22 +870,21 @@ FORCE_INLINE_TEMPLATE void ZSTD_row_update_internalImpl(ZSTD_matchState_t* ms, U32 const rowMask, U32 const useCache) { U32* const hashTable = ms->hashTable; - U16* const tagTable = ms->tagTable; + BYTE* const tagTable = ms->tagTable; U32 const hashLog = ms->rowHashLog; const BYTE* const base = ms->window.base; DEBUGLOG(6, "ZSTD_row_update_internalImpl(): updateStartIdx=%u, updateEndIdx=%u", updateStartIdx, updateEndIdx); for (; updateStartIdx < updateEndIdx; ++updateStartIdx) { - U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls) - : (U32)ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls); + U32 const hash = useCache ? ZSTD_row_nextCachedHash(ms->hashCache, hashTable, tagTable, base, updateStartIdx, hashLog, rowLog, mls, ms->hashSalt) + : (U32)ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt); U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; U32* const row = hashTable + relRow; - BYTE* tagRow = (BYTE*)(tagTable + relRow); /* Though tagTable is laid out as a table of U16, each tag is only 1 byte. - Explicit cast allows us to get exact desired position within each row */ + BYTE* tagRow = tagTable + relRow; U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); - assert(hash == ZSTD_hashPtr(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls)); - ((BYTE*)tagRow)[pos + ZSTD_ROW_HASH_TAG_OFFSET] = hash & ZSTD_ROW_HASH_TAG_MASK; + assert(hash == ZSTD_hashPtrSalted(base + updateStartIdx, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, ms->hashSalt)); + tagRow[pos] = hash & ZSTD_ROW_HASH_TAG_MASK; row[pos] = updateStartIdx; } } @@ -981,7 +932,35 @@ void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip) { const U32 mls = MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */); DEBUGLOG(5, "ZSTD_row_update(), rowLog=%u", rowLog); - ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* dont use cache */); + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 0 /* don't use cache */); +} + +/* Returns the mask width of bits group of which will be set to 1. Given not all + * architectures have easy movemask instruction, this helps to iterate over + * groups of bits easier and faster. + */ +FORCE_INLINE_TEMPLATE U32 +ZSTD_row_matchMaskGroupWidth(const U32 rowEntries) +{ + assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64); + assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES); + (void)rowEntries; +#if defined(ZSTD_ARCH_ARM_NEON) + /* NEON path only works for little endian */ + if (!MEM_isLittleEndian()) { + return 1; + } + if (rowEntries == 16) { + return 4; + } + if (rowEntries == 32) { + return 2; + } + if (rowEntries == 64) { + return 1; + } +#endif + return 1; } #if defined(ZSTD_ARCH_X86_SSE2) @@ -1004,71 +983,82 @@ ZSTD_row_getSSEMask(int nbChunks, const BYTE* const src, const BYTE tag, const U } #endif -/* Returns a ZSTD_VecMask (U32) that has the nth bit set to 1 if the newly-computed "tag" matches - * the hash at the nth position in a row of the tagTable. - * Each row is a circular buffer beginning at the value of "head". So we must rotate the "matches" bitfield - * to match up with the actual layout of the entries within the hashTable */ +#if defined(ZSTD_ARCH_ARM_NEON) FORCE_INLINE_TEMPLATE ZSTD_VecMask -ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 head, const U32 rowEntries) +ZSTD_row_getNEONMask(const U32 rowEntries, const BYTE* const src, const BYTE tag, const U32 headGrouped) { - const BYTE* const src = tagRow + ZSTD_ROW_HASH_TAG_OFFSET; + assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64); + if (rowEntries == 16) { + /* vshrn_n_u16 shifts by 4 every u16 and narrows to 8 lower bits. + * After that groups of 4 bits represent the equalMask. We lower + * all bits except the highest in these groups by doing AND with + * 0x88 = 0b10001000. + */ + const uint8x16_t chunk = vld1q_u8(src); + const uint16x8_t equalMask = vreinterpretq_u16_u8(vceqq_u8(chunk, vdupq_n_u8(tag))); + const uint8x8_t res = vshrn_n_u16(equalMask, 4); + const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0); + return ZSTD_rotateRight_U64(matches, headGrouped) & 0x8888888888888888ull; + } else if (rowEntries == 32) { + /* Same idea as with rowEntries == 16 but doing AND with + * 0x55 = 0b01010101. + */ + const uint16x8x2_t chunk = vld2q_u16((const uint16_t*)(const void*)src); + const uint8x16_t chunk0 = vreinterpretq_u8_u16(chunk.val[0]); + const uint8x16_t chunk1 = vreinterpretq_u8_u16(chunk.val[1]); + const uint8x16_t dup = vdupq_n_u8(tag); + const uint8x8_t t0 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk0, dup)), 6); + const uint8x8_t t1 = vshrn_n_u16(vreinterpretq_u16_u8(vceqq_u8(chunk1, dup)), 6); + const uint8x8_t res = vsli_n_u8(t0, t1, 4); + const U64 matches = vget_lane_u64(vreinterpret_u64_u8(res), 0) ; + return ZSTD_rotateRight_U64(matches, headGrouped) & 0x5555555555555555ull; + } else { /* rowEntries == 64 */ + const uint8x16x4_t chunk = vld4q_u8(src); + const uint8x16_t dup = vdupq_n_u8(tag); + const uint8x16_t cmp0 = vceqq_u8(chunk.val[0], dup); + const uint8x16_t cmp1 = vceqq_u8(chunk.val[1], dup); + const uint8x16_t cmp2 = vceqq_u8(chunk.val[2], dup); + const uint8x16_t cmp3 = vceqq_u8(chunk.val[3], dup); + + const uint8x16_t t0 = vsriq_n_u8(cmp1, cmp0, 1); + const uint8x16_t t1 = vsriq_n_u8(cmp3, cmp2, 1); + const uint8x16_t t2 = vsriq_n_u8(t1, t0, 2); + const uint8x16_t t3 = vsriq_n_u8(t2, t2, 4); + const uint8x8_t t4 = vshrn_n_u16(vreinterpretq_u16_u8(t3), 4); + const U64 matches = vget_lane_u64(vreinterpret_u64_u8(t4), 0); + return ZSTD_rotateRight_U64(matches, headGrouped); + } +} +#endif + +/* Returns a ZSTD_VecMask (U64) that has the nth group (determined by + * ZSTD_row_matchMaskGroupWidth) of bits set to 1 if the newly-computed "tag" + * matches the hash at the nth position in a row of the tagTable. + * Each row is a circular buffer beginning at the value of "headGrouped". So we + * must rotate the "matches" bitfield to match up with the actual layout of the + * entries within the hashTable */ +FORCE_INLINE_TEMPLATE ZSTD_VecMask +ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 headGrouped, const U32 rowEntries) +{ + const BYTE* const src = tagRow; assert((rowEntries == 16) || (rowEntries == 32) || rowEntries == 64); assert(rowEntries <= ZSTD_ROW_HASH_MAX_ENTRIES); + assert(ZSTD_row_matchMaskGroupWidth(rowEntries) * rowEntries <= sizeof(ZSTD_VecMask) * 8); #if defined(ZSTD_ARCH_X86_SSE2) - return ZSTD_row_getSSEMask(rowEntries / 16, src, tag, head); + return ZSTD_row_getSSEMask(rowEntries / 16, src, tag, headGrouped); #else /* SW or NEON-LE */ # if defined(ZSTD_ARCH_ARM_NEON) /* This NEON path only works for little endian - otherwise use SWAR below */ if (MEM_isLittleEndian()) { - if (rowEntries == 16) { - const uint8x16_t chunk = vld1q_u8(src); - const uint16x8_t equalMask = vreinterpretq_u16_u8(vceqq_u8(chunk, vdupq_n_u8(tag))); - const uint16x8_t t0 = vshlq_n_u16(equalMask, 7); - const uint32x4_t t1 = vreinterpretq_u32_u16(vsriq_n_u16(t0, t0, 14)); - const uint64x2_t t2 = vreinterpretq_u64_u32(vshrq_n_u32(t1, 14)); - const uint8x16_t t3 = vreinterpretq_u8_u64(vsraq_n_u64(t2, t2, 28)); - const U16 hi = (U16)vgetq_lane_u8(t3, 8); - const U16 lo = (U16)vgetq_lane_u8(t3, 0); - return ZSTD_rotateRight_U16((hi << 8) | lo, head); - } else if (rowEntries == 32) { - const uint16x8x2_t chunk = vld2q_u16((const U16*)(const void*)src); - const uint8x16_t chunk0 = vreinterpretq_u8_u16(chunk.val[0]); - const uint8x16_t chunk1 = vreinterpretq_u8_u16(chunk.val[1]); - const uint8x16_t equalMask0 = vceqq_u8(chunk0, vdupq_n_u8(tag)); - const uint8x16_t equalMask1 = vceqq_u8(chunk1, vdupq_n_u8(tag)); - const int8x8_t pack0 = vqmovn_s16(vreinterpretq_s16_u8(equalMask0)); - const int8x8_t pack1 = vqmovn_s16(vreinterpretq_s16_u8(equalMask1)); - const uint8x8_t t0 = vreinterpret_u8_s8(pack0); - const uint8x8_t t1 = vreinterpret_u8_s8(pack1); - const uint8x8_t t2 = vsri_n_u8(t1, t0, 2); - const uint8x8x2_t t3 = vuzp_u8(t2, t0); - const uint8x8_t t4 = vsri_n_u8(t3.val[1], t3.val[0], 4); - const U32 matches = vget_lane_u32(vreinterpret_u32_u8(t4), 0); - return ZSTD_rotateRight_U32(matches, head); - } else { /* rowEntries == 64 */ - const uint8x16x4_t chunk = vld4q_u8(src); - const uint8x16_t dup = vdupq_n_u8(tag); - const uint8x16_t cmp0 = vceqq_u8(chunk.val[0], dup); - const uint8x16_t cmp1 = vceqq_u8(chunk.val[1], dup); - const uint8x16_t cmp2 = vceqq_u8(chunk.val[2], dup); - const uint8x16_t cmp3 = vceqq_u8(chunk.val[3], dup); - - const uint8x16_t t0 = vsriq_n_u8(cmp1, cmp0, 1); - const uint8x16_t t1 = vsriq_n_u8(cmp3, cmp2, 1); - const uint8x16_t t2 = vsriq_n_u8(t1, t0, 2); - const uint8x16_t t3 = vsriq_n_u8(t2, t2, 4); - const uint8x8_t t4 = vshrn_n_u16(vreinterpretq_u16_u8(t3), 4); - const U64 matches = vget_lane_u64(vreinterpret_u64_u8(t4), 0); - return ZSTD_rotateRight_U64(matches, head); - } + return ZSTD_row_getNEONMask(rowEntries, src, tag, headGrouped); } # endif /* ZSTD_ARCH_ARM_NEON */ /* SWAR */ - { const size_t chunkSize = sizeof(size_t); + { const int chunkSize = sizeof(size_t); const size_t shiftAmount = ((chunkSize * 8) - chunkSize); const size_t xFF = ~((size_t)0); const size_t x01 = xFF / 0xFF; @@ -1101,11 +1091,11 @@ ZSTD_row_getMatchMask(const BYTE* const tagRow, const BYTE tag, const U32 head, } matches = ~matches; if (rowEntries == 16) { - return ZSTD_rotateRight_U16((U16)matches, head); + return ZSTD_rotateRight_U16((U16)matches, headGrouped); } else if (rowEntries == 32) { - return ZSTD_rotateRight_U32((U32)matches, head); + return ZSTD_rotateRight_U32((U32)matches, headGrouped); } else { - return ZSTD_rotateRight_U64((U64)matches, head); + return ZSTD_rotateRight_U64((U64)matches, headGrouped); } } #endif @@ -1135,7 +1125,7 @@ size_t ZSTD_RowFindBestMatch( const U32 rowLog) { U32* const hashTable = ms->hashTable; - U16* const tagTable = ms->tagTable; + BYTE* const tagTable = ms->tagTable; U32* const hashCache = ms->hashCache; const U32 hashLog = ms->rowHashLog; const ZSTD_compressionParameters* const cParams = &ms->cParams; @@ -1153,8 +1143,11 @@ size_t ZSTD_RowFindBestMatch( const U32 rowEntries = (1U << rowLog); const U32 rowMask = rowEntries - 1; const U32 cappedSearchLog = MIN(cParams->searchLog, rowLog); /* nb of searches is capped at nb entries per row */ + const U32 groupWidth = ZSTD_row_matchMaskGroupWidth(rowEntries); + const U64 hashSalt = ms->hashSalt; U32 nbAttempts = 1U << cappedSearchLog; size_t ml=4-1; + U32 hash; /* DMS/DDS variables that may be referenced laster */ const ZSTD_matchState_t* const dms = ms->dictMatchState; @@ -1178,7 +1171,7 @@ size_t ZSTD_RowFindBestMatch( if (dictMode == ZSTD_dictMatchState) { /* Prefetch DMS rows */ U32* const dmsHashTable = dms->hashTable; - U16* const dmsTagTable = dms->tagTable; + BYTE* const dmsTagTable = dms->tagTable; U32 const dmsHash = (U32)ZSTD_hashPtr(ip, dms->rowHashLog + ZSTD_ROW_HASH_TAG_BITS, mls); U32 const dmsRelRow = (dmsHash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; dmsTag = dmsHash & ZSTD_ROW_HASH_TAG_MASK; @@ -1188,23 +1181,34 @@ size_t ZSTD_RowFindBestMatch( } /* Update the hashTable and tagTable up to (but not including) ip */ - ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */); + if (!ms->lazySkipping) { + ZSTD_row_update_internal(ms, ip, mls, rowLog, rowMask, 1 /* useCache */); + hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls, hashSalt); + } else { + /* Stop inserting every position when in the lazy skipping mode. + * The hash cache is also not kept up to date in this mode. + */ + hash = (U32)ZSTD_hashPtrSalted(ip, hashLog + ZSTD_ROW_HASH_TAG_BITS, mls, hashSalt); + ms->nextToUpdate = curr; + } + ms->hashSaltEntropy += hash; /* collect salt entropy */ + { /* Get the hash for ip, compute the appropriate row */ - U32 const hash = ZSTD_row_nextCachedHash(hashCache, hashTable, tagTable, base, curr, hashLog, rowLog, mls); U32 const relRow = (hash >> ZSTD_ROW_HASH_TAG_BITS) << rowLog; U32 const tag = hash & ZSTD_ROW_HASH_TAG_MASK; U32* const row = hashTable + relRow; BYTE* tagRow = (BYTE*)(tagTable + relRow); - U32 const head = *tagRow & rowMask; + U32 const headGrouped = (*tagRow & rowMask) * groupWidth; U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; size_t numMatches = 0; size_t currMatch = 0; - ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, head, rowEntries); + ZSTD_VecMask matches = ZSTD_row_getMatchMask(tagRow, (BYTE)tag, headGrouped, rowEntries); /* Cycle through the matches and prefetch */ - for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) { - U32 const matchPos = (head + ZSTD_VecMask_next(matches)) & rowMask; + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; U32 const matchIndex = row[matchPos]; + if(matchPos == 0) continue; assert(numMatches < rowEntries); if (matchIndex < lowLimit) break; @@ -1214,13 +1218,14 @@ size_t ZSTD_RowFindBestMatch( PREFETCH_L1(dictBase + matchIndex); } matchBuffer[numMatches++] = matchIndex; + --nbAttempts; } /* Speed opt: insert current byte into hashtable too. This allows us to avoid one iteration of the loop in ZSTD_row_update_internal() at the next search. */ { U32 const pos = ZSTD_row_nextIndex(tagRow, rowMask); - tagRow[pos + ZSTD_ROW_HASH_TAG_OFFSET] = (BYTE)tag; + tagRow[pos] = (BYTE)tag; row[pos] = ms->nextToUpdate++; } @@ -1234,7 +1239,8 @@ size_t ZSTD_RowFindBestMatch( if ((dictMode != ZSTD_extDict) || matchIndex >= dictLimit) { const BYTE* const match = base + matchIndex; assert(matchIndex >= dictLimit); /* ensures this is true if dictMode != ZSTD_extDict */ - if (match[ml] == ip[ml]) /* potentially better */ + /* read 4B starting from (match + ml + 1 - sizeof(U32)) */ + if (MEM_read32(match + ml - 3) == MEM_read32(ip + ml - 3)) /* potentially better */ currentMl = ZSTD_count(ip, match, iLimit); } else { const BYTE* const match = dictBase + matchIndex; @@ -1246,7 +1252,7 @@ size_t ZSTD_RowFindBestMatch( /* Save best solution */ if (currentMl > ml) { ml = currentMl; - *offsetPtr = STORE_OFFSET(curr - matchIndex); + *offsetPtr = OFFSET_TO_OFFBASE(curr - matchIndex); if (ip+currentMl == iLimit) break; /* best possible, avoids read overflow on next attempt */ } } @@ -1264,19 +1270,21 @@ size_t ZSTD_RowFindBestMatch( const U32 dmsSize = (U32)(dmsEnd - dmsBase); const U32 dmsIndexDelta = dictLimit - dmsSize; - { U32 const head = *dmsTagRow & rowMask; + { U32 const headGrouped = (*dmsTagRow & rowMask) * groupWidth; U32 matchBuffer[ZSTD_ROW_HASH_MAX_ENTRIES]; size_t numMatches = 0; size_t currMatch = 0; - ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, head, rowEntries); + ZSTD_VecMask matches = ZSTD_row_getMatchMask(dmsTagRow, (BYTE)dmsTag, headGrouped, rowEntries); - for (; (matches > 0) && (nbAttempts > 0); --nbAttempts, matches &= (matches - 1)) { - U32 const matchPos = (head + ZSTD_VecMask_next(matches)) & rowMask; + for (; (matches > 0) && (nbAttempts > 0); matches &= (matches - 1)) { + U32 const matchPos = ((headGrouped + ZSTD_VecMask_next(matches)) / groupWidth) & rowMask; U32 const matchIndex = dmsRow[matchPos]; + if(matchPos == 0) continue; if (matchIndex < dmsLowestIndex) break; PREFETCH_L1(dmsBase + matchIndex); matchBuffer[numMatches++] = matchIndex; + --nbAttempts; } /* Return the longest match */ @@ -1295,7 +1303,7 @@ size_t ZSTD_RowFindBestMatch( if (currentMl > ml) { ml = currentMl; assert(curr > matchIndex + dmsIndexDelta); - *offsetPtr = STORE_OFFSET(curr - (matchIndex + dmsIndexDelta)); + *offsetPtr = OFFSET_TO_OFFBASE(curr - (matchIndex + dmsIndexDelta)); if (ip+currentMl == iLimit) break; } } @@ -1305,14 +1313,10 @@ size_t ZSTD_RowFindBestMatch( } -typedef size_t (*searchMax_f)( - ZSTD_matchState_t* ms, - const BYTE* ip, const BYTE* iLimit, size_t* offsetPtr); - /** - * This struct contains the functions necessary for lazy to search. - * Currently, that is only searchMax. However, it is still valuable to have the - * VTable because this makes it easier to add more functions to the VTable later. + * Generate search functions templated on (dictMode, mls, rowLog). + * These functions are outlined for code size & compilation time. + * ZSTD_searchMax() dispatches to the correct implementation function. * * TODO: The start of the search function involves loading and calculating a * bunch of constants from the ZSTD_matchState_t. These computations could be @@ -1330,25 +1334,25 @@ typedef size_t (*searchMax_f)( * the single segment loop. It should go in searchMax instead of its own * function to avoid having multiple virtual function calls per search. */ -typedef struct { - searchMax_f searchMax; -} ZSTD_LazyVTable; -#define GEN_ZSTD_BT_VTABLE(dictMode, mls) \ - static size_t ZSTD_BtFindBestMatch_##dictMode##_##mls( \ - ZSTD_matchState_t* ms, \ - const BYTE* ip, const BYTE* const iLimit, \ - size_t* offsetPtr) \ - { \ - assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ - return ZSTD_BtFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ - } \ - static const ZSTD_LazyVTable ZSTD_BtVTable_##dictMode##_##mls = { \ - ZSTD_BtFindBestMatch_##dictMode##_##mls \ - }; +#define ZSTD_BT_SEARCH_FN(dictMode, mls) ZSTD_BtFindBestMatch_##dictMode##_##mls +#define ZSTD_HC_SEARCH_FN(dictMode, mls) ZSTD_HcFindBestMatch_##dictMode##_##mls +#define ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog -#define GEN_ZSTD_HC_VTABLE(dictMode, mls) \ - static size_t ZSTD_HcFindBestMatch_##dictMode##_##mls( \ +#define ZSTD_SEARCH_FN_ATTRS FORCE_NOINLINE + +#define GEN_ZSTD_BT_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_BT_SEARCH_FN(dictMode, mls)( \ + ZSTD_matchState_t* ms, \ + const BYTE* ip, const BYTE* const iLimit, \ + size_t* offBasePtr) \ + { \ + assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ + return ZSTD_BtFindBestMatch(ms, ip, iLimit, offBasePtr, mls, ZSTD_##dictMode); \ + } \ + +#define GEN_ZSTD_HC_SEARCH_FN(dictMode, mls) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_HC_SEARCH_FN(dictMode, mls)( \ ZSTD_matchState_t* ms, \ const BYTE* ip, const BYTE* const iLimit, \ size_t* offsetPtr) \ @@ -1356,12 +1360,9 @@ typedef struct { assert(MAX(4, MIN(6, ms->cParams.minMatch)) == mls); \ return ZSTD_HcFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode); \ } \ - static const ZSTD_LazyVTable ZSTD_HcVTable_##dictMode##_##mls = { \ - ZSTD_HcFindBestMatch_##dictMode##_##mls \ - }; -#define GEN_ZSTD_ROW_VTABLE(dictMode, mls, rowLog) \ - static size_t ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog( \ +#define GEN_ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + ZSTD_SEARCH_FN_ATTRS size_t ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)( \ ZSTD_matchState_t* ms, \ const BYTE* ip, const BYTE* const iLimit, \ size_t* offsetPtr) \ @@ -1370,9 +1371,6 @@ typedef struct { assert(MAX(4, MIN(6, ms->cParams.searchLog)) == rowLog); \ return ZSTD_RowFindBestMatch(ms, ip, iLimit, offsetPtr, mls, ZSTD_##dictMode, rowLog); \ } \ - static const ZSTD_LazyVTable ZSTD_RowVTable_##dictMode##_##mls##_##rowLog = { \ - ZSTD_RowFindBestMatch_##dictMode##_##mls##_##rowLog \ - }; #define ZSTD_FOR_EACH_ROWLOG(X, dictMode, mls) \ X(dictMode, mls, 4) \ @@ -1395,83 +1393,102 @@ typedef struct { X(__VA_ARGS__, dictMatchState) \ X(__VA_ARGS__, dedicatedDictSearch) -/* Generate Row VTables for each combination of (dictMode, mls, rowLog) */ -ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_VTABLE) -/* Generate Binary Tree VTables for each combination of (dictMode, mls) */ -ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_VTABLE) -/* Generate Hash Chain VTables for each combination of (dictMode, mls) */ -ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_VTABLE) +/* Generate row search fns for each combination of (dictMode, mls, rowLog) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS_ROWLOG, GEN_ZSTD_ROW_SEARCH_FN) +/* Generate binary Tree search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_BT_SEARCH_FN) +/* Generate hash chain search fns for each combination of (dictMode, mls) */ +ZSTD_FOR_EACH_DICT_MODE(ZSTD_FOR_EACH_MLS, GEN_ZSTD_HC_SEARCH_FN) -#define GEN_ZSTD_BT_VTABLE_ARRAY(dictMode) \ - { \ - &ZSTD_BtVTable_##dictMode##_4, \ - &ZSTD_BtVTable_##dictMode##_5, \ - &ZSTD_BtVTable_##dictMode##_6 \ +typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e; + +#define GEN_ZSTD_CALL_BT_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_BT_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_HC_SEARCH_FN(dictMode, mls) \ + case mls: \ + return ZSTD_HC_SEARCH_FN(dictMode, mls)(ms, ip, iend, offsetPtr); +#define GEN_ZSTD_CALL_ROW_SEARCH_FN(dictMode, mls, rowLog) \ + case rowLog: \ + return ZSTD_ROW_SEARCH_FN(dictMode, mls, rowLog)(ms, ip, iend, offsetPtr); + +#define ZSTD_SWITCH_MLS(X, dictMode) \ + switch (mls) { \ + ZSTD_FOR_EACH_MLS(X, dictMode) \ } -#define GEN_ZSTD_HC_VTABLE_ARRAY(dictMode) \ - { \ - &ZSTD_HcVTable_##dictMode##_4, \ - &ZSTD_HcVTable_##dictMode##_5, \ - &ZSTD_HcVTable_##dictMode##_6 \ - } +#define ZSTD_SWITCH_ROWLOG(dictMode, mls) \ + case mls: \ + switch (rowLog) { \ + ZSTD_FOR_EACH_ROWLOG(GEN_ZSTD_CALL_ROW_SEARCH_FN, dictMode, mls) \ + } \ + ZSTD_UNREACHABLE; \ + break; -#define GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, mls) \ - { \ - &ZSTD_RowVTable_##dictMode##_##mls##_4, \ - &ZSTD_RowVTable_##dictMode##_##mls##_5, \ - &ZSTD_RowVTable_##dictMode##_##mls##_6 \ - } +#define ZSTD_SWITCH_SEARCH_METHOD(dictMode) \ + switch (searchMethod) { \ + case search_hashChain: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_HC_SEARCH_FN, dictMode) \ + break; \ + case search_binaryTree: \ + ZSTD_SWITCH_MLS(GEN_ZSTD_CALL_BT_SEARCH_FN, dictMode) \ + break; \ + case search_rowHash: \ + ZSTD_SWITCH_MLS(ZSTD_SWITCH_ROWLOG, dictMode) \ + break; \ + } \ + ZSTD_UNREACHABLE; -#define GEN_ZSTD_ROW_VTABLE_ARRAY(dictMode) \ - { \ - GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, 4), \ - GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, 5), \ - GEN_ZSTD_ROW_VTABLE_ARRAY_(dictMode, 6) \ - } - -#define GEN_ZSTD_VTABLE_ARRAY(X) \ - { \ - X(noDict), \ - X(extDict), \ - X(dictMatchState), \ - X(dedicatedDictSearch) \ +/** + * Searches for the longest match at @p ip. + * Dispatches to the correct implementation function based on the + * (searchMethod, dictMode, mls, rowLog). We use switch statements + * here instead of using an indirect function call through a function + * pointer because after Spectre and Meltdown mitigations, indirect + * function calls can be very costly, especially in the kernel. + * + * NOTE: dictMode and searchMethod should be templated, so those switch + * statements should be optimized out. Only the mls & rowLog switches + * should be left. + * + * @param ms The match state. + * @param ip The position to search at. + * @param iend The end of the input data. + * @param[out] offsetPtr Stores the match offset into this pointer. + * @param mls The minimum search length, in the range [4, 6]. + * @param rowLog The row log (if applicable), in the range [4, 6]. + * @param searchMethod The search method to use (templated). + * @param dictMode The dictMode (templated). + * + * @returns The length of the longest match found, or < mls if no match is found. + * If a match is found its offset is stored in @p offsetPtr. + */ +FORCE_INLINE_TEMPLATE size_t ZSTD_searchMax( + ZSTD_matchState_t* ms, + const BYTE* ip, + const BYTE* iend, + size_t* offsetPtr, + U32 const mls, + U32 const rowLog, + searchMethod_e const searchMethod, + ZSTD_dictMode_e const dictMode) +{ + if (dictMode == ZSTD_noDict) { + ZSTD_SWITCH_SEARCH_METHOD(noDict) + } else if (dictMode == ZSTD_extDict) { + ZSTD_SWITCH_SEARCH_METHOD(extDict) + } else if (dictMode == ZSTD_dictMatchState) { + ZSTD_SWITCH_SEARCH_METHOD(dictMatchState) + } else if (dictMode == ZSTD_dedicatedDictSearch) { + ZSTD_SWITCH_SEARCH_METHOD(dedicatedDictSearch) } + ZSTD_UNREACHABLE; + return 0; +} /* ******************************* * Common parser - lazy strategy *********************************/ -typedef enum { search_hashChain=0, search_binaryTree=1, search_rowHash=2 } searchMethod_e; - -/** - * This table is indexed first by the four ZSTD_dictMode_e values, and then - * by the two searchMethod_e values. NULLs are placed for configurations - * that should never occur (extDict modes go to the other implementation - * below and there is no DDSS for binary tree search yet). - */ - -static ZSTD_LazyVTable const* -ZSTD_selectLazyVTable(ZSTD_matchState_t const* ms, searchMethod_e searchMethod, ZSTD_dictMode_e dictMode) -{ - /* Fill the Hc/Bt VTable arrays with the right functions for the (dictMode, mls) combination. */ - ZSTD_LazyVTable const* const hcVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_HC_VTABLE_ARRAY); - ZSTD_LazyVTable const* const btVTables[4][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_BT_VTABLE_ARRAY); - /* Fill the Row VTable array with the right functions for the (dictMode, mls, rowLog) combination. */ - ZSTD_LazyVTable const* const rowVTables[4][3][3] = GEN_ZSTD_VTABLE_ARRAY(GEN_ZSTD_ROW_VTABLE_ARRAY); - - U32 const mls = MAX(4, MIN(6, ms->cParams.minMatch)); - U32 const rowLog = MAX(4, MIN(6, ms->cParams.searchLog)); - switch (searchMethod) { - case search_hashChain: - return hcVTables[dictMode][mls - 4]; - case search_binaryTree: - return btVTables[dictMode][mls - 4]; - case search_rowHash: - return rowVTables[dictMode][mls - 4][rowLog - 4]; - default: - return NULL; - } -} FORCE_INLINE_TEMPLATE size_t ZSTD_compressBlock_lazy_generic( @@ -1489,9 +1506,11 @@ ZSTD_compressBlock_lazy_generic( const BYTE* const base = ms->window.base; const U32 prefixLowestIndex = ms->window.dictLimit; const BYTE* const prefixLowest = base + prefixLowestIndex; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); - searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, dictMode)->searchMax; - U32 offset_1 = rep[0], offset_2 = rep[1], savedOffset=0; + U32 offset_1 = rep[0], offset_2 = rep[1]; + U32 offsetSaved1 = 0, offsetSaved2 = 0; const int isDMS = dictMode == ZSTD_dictMatchState; const int isDDS = dictMode == ZSTD_dedicatedDictSearch; @@ -1506,16 +1525,14 @@ ZSTD_compressBlock_lazy_generic( 0; const U32 dictAndPrefixLength = (U32)((ip - prefixLowest) + (dictEnd - dictLowest)); - assert(searchMax != NULL); - DEBUGLOG(5, "ZSTD_compressBlock_lazy_generic (dictMode=%u) (searchFunc=%u)", (U32)dictMode, (U32)searchMethod); ip += (dictAndPrefixLength == 0); if (dictMode == ZSTD_noDict) { U32 const curr = (U32)(ip - base); U32 const windowLow = ZSTD_getLowestPrefixIndex(ms, curr, ms->cParams.windowLog); U32 const maxRep = curr - windowLow; - if (offset_2 > maxRep) savedOffset = offset_2, offset_2 = 0; - if (offset_1 > maxRep) savedOffset = offset_1, offset_1 = 0; + if (offset_2 > maxRep) offsetSaved2 = offset_2, offset_2 = 0; + if (offset_1 > maxRep) offsetSaved1 = offset_1, offset_1 = 0; } if (isDxS) { /* dictMatchState repCode checks don't currently handle repCode == 0 @@ -1524,11 +1541,11 @@ ZSTD_compressBlock_lazy_generic( assert(offset_2 <= dictAndPrefixLength); } + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; + if (searchMethod == search_rowHash) { - const U32 rowLog = MAX(4, MIN(6, ms->cParams.searchLog)); - ZSTD_row_fillHashCache(ms, base, rowLog, - MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */), - ms->nextToUpdate, ilimit); + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); } /* Match Loop */ @@ -1540,7 +1557,7 @@ ZSTD_compressBlock_lazy_generic( #endif while (ip < ilimit) { size_t matchLength=0; - size_t offcode=STORE_REPCODE_1; + size_t offBase = REPCODE1_TO_OFFBASE; const BYTE* start=ip+1; DEBUGLOG(7, "search baseline (depth 0)"); @@ -1565,14 +1582,23 @@ ZSTD_compressBlock_lazy_generic( } /* first search (depth 0) */ - { size_t offsetFound = 999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); + { size_t offbaseFound = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &offbaseFound, mls, rowLog, searchMethod, dictMode); if (ml2 > matchLength) - matchLength = ml2, start = ip, offcode=offsetFound; + matchLength = ml2, start = ip, offBase = offbaseFound; } if (matchLength < 4) { - ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */; + ip += step; + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; continue; } @@ -1582,12 +1608,12 @@ ZSTD_compressBlock_lazy_generic( DEBUGLOG(7, "search depth 1"); ip ++; if ( (dictMode == ZSTD_noDict) - && (offcode) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + && (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; int const gain2 = (int)(mlRep * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip; + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; } if (isDxS) { const U32 repIndex = (U32)(ip - base) - offset_1; @@ -1599,17 +1625,17 @@ ZSTD_compressBlock_lazy_generic( const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; int const gain2 = (int)(mlRep * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip; + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; } } - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 4); + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offcode = offset2, start = ip; + matchLength = ml2, offBase = ofbCandidate, start = ip; continue; /* search a better one */ } } @@ -1618,12 +1644,12 @@ ZSTD_compressBlock_lazy_generic( DEBUGLOG(7, "search depth 2"); ip ++; if ( (dictMode == ZSTD_noDict) - && (offcode) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { + && (offBase) && ((offset_1>0) & (MEM_read32(ip) == MEM_read32(ip - offset_1)))) { size_t const mlRep = ZSTD_count(ip+4, ip+4-offset_1, iend) + 4; int const gain2 = (int)(mlRep * 4); - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip; + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; } if (isDxS) { const U32 repIndex = (U32)(ip - base) - offset_1; @@ -1635,17 +1661,17 @@ ZSTD_compressBlock_lazy_generic( const BYTE* repMatchEnd = repIndex < prefixLowestIndex ? dictEnd : iend; size_t const mlRep = ZSTD_count_2segments(ip+4, repMatch+4, iend, repMatchEnd, prefixLowest) + 4; int const gain2 = (int)(mlRep * 4); - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); if ((mlRep >= 4) && (gain2 > gain1)) - matchLength = mlRep, offcode = STORE_REPCODE_1, start = ip; + matchLength = mlRep, offBase = REPCODE1_TO_OFFBASE, start = ip; } } - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 7); + { size_t ofbCandidate=999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, dictMode); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offcode = offset2, start = ip; + matchLength = ml2, offBase = ofbCandidate, start = ip; continue; } } } break; /* nothing found : store previous solution */ @@ -1656,26 +1682,33 @@ ZSTD_compressBlock_lazy_generic( * notably if `value` is unsigned, resulting in a large positive `-value`. */ /* catch up */ - if (STORED_IS_OFFSET(offcode)) { + if (OFFBASE_IS_OFFSET(offBase)) { if (dictMode == ZSTD_noDict) { - while ( ((start > anchor) & (start - STORED_OFFSET(offcode) > prefixLowest)) - && (start[-1] == (start-STORED_OFFSET(offcode))[-1]) ) /* only search for offset within prefix */ + while ( ((start > anchor) & (start - OFFBASE_TO_OFFSET(offBase) > prefixLowest)) + && (start[-1] == (start-OFFBASE_TO_OFFSET(offBase))[-1]) ) /* only search for offset within prefix */ { start--; matchLength++; } } if (isDxS) { - U32 const matchIndex = (U32)((size_t)(start-base) - STORED_OFFSET(offcode)); + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); const BYTE* match = (matchIndex < prefixLowestIndex) ? dictBase + matchIndex - dictIndexDelta : base + matchIndex; const BYTE* const mStart = (matchIndex < prefixLowestIndex) ? dictLowest : prefixLowest; while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ } - offset_2 = offset_1; offset_1 = (U32)STORED_OFFSET(offcode); + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); } /* store sequence */ _storeSequence: { size_t const litLength = (size_t)(start - anchor); - ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offcode, matchLength); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); anchor = ip = start + matchLength; } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + ms->lazySkipping = 0; + } /* check immediate repcode */ if (isDxS) { @@ -1689,8 +1722,8 @@ _storeSequence: && (MEM_read32(repMatch) == MEM_read32(ip)) ) { const BYTE* const repEnd2 = repIndex < prefixLowestIndex ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd2, prefixLowest) + 4; - offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap offset_2 <=> offset_1 */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength); + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset_2 <=> offset_1 */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); ip += matchLength; anchor = ip; continue; @@ -1704,16 +1737,20 @@ _storeSequence: && (MEM_read32(ip) == MEM_read32(ip - offset_2)) ) { /* store sequence */ matchLength = ZSTD_count(ip+4, ip+4-offset_2, iend) + 4; - offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap repcodes */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength); + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap repcodes */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ } } } - /* Save reps for next block */ - rep[0] = offset_1 ? offset_1 : savedOffset; - rep[1] = offset_2 ? offset_2 : savedOffset; + /* If offset_1 started invalid (offsetSaved1 != 0) and became valid (offset_1 != 0), + * rotate saved offsets. See comment in ZSTD_compressBlock_fast_noDict for more context. */ + offsetSaved2 = ((offsetSaved1 != 0) && (offset_1 != 0)) ? offsetSaved1 : offsetSaved2; + + /* save reps for next block */ + rep[0] = offset_1 ? offset_1 : offsetSaved1; + rep[1] = offset_2 ? offset_2 : offsetSaved2; /* Return the last literals size */ return (size_t)(iend - anchor); @@ -1882,19 +1919,20 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( const BYTE* const dictEnd = dictBase + dictLimit; const BYTE* const dictStart = dictBase + ms->window.lowLimit; const U32 windowLog = ms->cParams.windowLog; - const U32 rowLog = ms->cParams.searchLog < 5 ? 4 : 5; + const U32 mls = BOUNDED(4, ms->cParams.minMatch, 6); + const U32 rowLog = BOUNDED(4, ms->cParams.searchLog, 6); - searchMax_f const searchMax = ZSTD_selectLazyVTable(ms, searchMethod, ZSTD_extDict)->searchMax; U32 offset_1 = rep[0], offset_2 = rep[1]; DEBUGLOG(5, "ZSTD_compressBlock_lazy_extDict_generic (searchFunc=%u)", (U32)searchMethod); + /* Reset the lazy skipping state */ + ms->lazySkipping = 0; + /* init */ ip += (ip == prefixStart); if (searchMethod == search_rowHash) { - ZSTD_row_fillHashCache(ms, base, rowLog, - MIN(ms->cParams.minMatch, 6 /* mls caps out at 6 */), - ms->nextToUpdate, ilimit); + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); } /* Match Loop */ @@ -1906,7 +1944,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( #endif while (ip < ilimit) { size_t matchLength=0; - size_t offcode=STORE_REPCODE_1; + size_t offBase = REPCODE1_TO_OFFBASE; const BYTE* start=ip+1; U32 curr = (U32)(ip-base); @@ -1925,14 +1963,23 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( } } /* first search (depth 0) */ - { size_t offsetFound = 999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offsetFound); + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); if (ml2 > matchLength) - matchLength = ml2, start = ip, offcode=offsetFound; + matchLength = ml2, start = ip, offBase = ofbCandidate; } if (matchLength < 4) { - ip += ((ip-anchor) >> kSearchStrength) + 1; /* jump faster over incompressible sections */ + size_t const step = ((size_t)(ip-anchor) >> kSearchStrength); + ip += step + 1; /* jump faster over incompressible sections */ + /* Enter the lazy skipping mode once we are skipping more than 8 bytes at a time. + * In this mode we stop inserting every position into our tables, and only insert + * positions that we search, which is one in step positions. + * The exact cutoff is flexible, I've just chosen a number that is reasonably high, + * so we minimize the compression ratio loss in "normal" scenarios. This mode gets + * triggered once we've gone 2KB without finding any matches. + */ + ms->lazySkipping = step > kLazySkippingStep; continue; } @@ -1942,7 +1989,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( ip ++; curr++; /* check repCode */ - if (offcode) { + if (offBase) { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog); const U32 repIndex = (U32)(curr - offset_1); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; @@ -1954,18 +2001,18 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; int const gain2 = (int)(repLength * 3); - int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + int const gain1 = (int)(matchLength*3 - ZSTD_highbit32((U32)offBase) + 1); if ((repLength >= 4) && (gain2 > gain1)) - matchLength = repLength, offcode = STORE_REPCODE_1, start = ip; + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; } } /* search match, depth 1 */ - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 4); + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 4); if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offcode = offset2, start = ip; + matchLength = ml2, offBase = ofbCandidate, start = ip; continue; /* search a better one */ } } @@ -1974,7 +2021,7 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( ip ++; curr++; /* check repCode */ - if (offcode) { + if (offBase) { const U32 windowLow = ZSTD_getLowestMatchIndex(ms, curr, windowLog); const U32 repIndex = (U32)(curr - offset_1); const BYTE* const repBase = repIndex < dictLimit ? dictBase : base; @@ -1986,38 +2033,45 @@ size_t ZSTD_compressBlock_lazy_extDict_generic( const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; size_t const repLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; int const gain2 = (int)(repLength * 4); - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 1); + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 1); if ((repLength >= 4) && (gain2 > gain1)) - matchLength = repLength, offcode = STORE_REPCODE_1, start = ip; + matchLength = repLength, offBase = REPCODE1_TO_OFFBASE, start = ip; } } /* search match, depth 2 */ - { size_t offset2=999999999; - size_t const ml2 = searchMax(ms, ip, iend, &offset2); - int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offset2))); /* raw approx */ - int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)STORED_TO_OFFBASE(offcode)) + 7); + { size_t ofbCandidate = 999999999; + size_t const ml2 = ZSTD_searchMax(ms, ip, iend, &ofbCandidate, mls, rowLog, searchMethod, ZSTD_extDict); + int const gain2 = (int)(ml2*4 - ZSTD_highbit32((U32)ofbCandidate)); /* raw approx */ + int const gain1 = (int)(matchLength*4 - ZSTD_highbit32((U32)offBase) + 7); if ((ml2 >= 4) && (gain2 > gain1)) { - matchLength = ml2, offcode = offset2, start = ip; + matchLength = ml2, offBase = ofbCandidate, start = ip; continue; } } } break; /* nothing found : store previous solution */ } /* catch up */ - if (STORED_IS_OFFSET(offcode)) { - U32 const matchIndex = (U32)((size_t)(start-base) - STORED_OFFSET(offcode)); + if (OFFBASE_IS_OFFSET(offBase)) { + U32 const matchIndex = (U32)((size_t)(start-base) - OFFBASE_TO_OFFSET(offBase)); const BYTE* match = (matchIndex < dictLimit) ? dictBase + matchIndex : base + matchIndex; const BYTE* const mStart = (matchIndex < dictLimit) ? dictStart : prefixStart; while ((start>anchor) && (match>mStart) && (start[-1] == match[-1])) { start--; match--; matchLength++; } /* catch up */ - offset_2 = offset_1; offset_1 = (U32)STORED_OFFSET(offcode); + offset_2 = offset_1; offset_1 = (U32)OFFBASE_TO_OFFSET(offBase); } /* store sequence */ _storeSequence: { size_t const litLength = (size_t)(start - anchor); - ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offcode, matchLength); + ZSTD_storeSeq(seqStore, litLength, anchor, iend, (U32)offBase, matchLength); anchor = ip = start + matchLength; } + if (ms->lazySkipping) { + /* We've found a match, disable lazy skipping mode, and refill the hash cache. */ + if (searchMethod == search_rowHash) { + ZSTD_row_fillHashCache(ms, base, rowLog, mls, ms->nextToUpdate, ilimit); + } + ms->lazySkipping = 0; + } /* check immediate repcode */ while (ip <= ilimit) { @@ -2032,8 +2086,8 @@ _storeSequence: /* repcode detected we should take it */ const BYTE* const repEnd = repIndex < dictLimit ? dictEnd : iend; matchLength = ZSTD_count_2segments(ip+4, repMatch+4, iend, repEnd, prefixStart) + 4; - offcode = offset_2; offset_2 = offset_1; offset_1 = (U32)offcode; /* swap offset history */ - ZSTD_storeSeq(seqStore, 0, anchor, iend, STORE_REPCODE_1, matchLength); + offBase = offset_2; offset_2 = offset_1; offset_1 = (U32)offBase; /* swap offset history */ + ZSTD_storeSeq(seqStore, 0, anchor, iend, REPCODE1_TO_OFFBASE, matchLength); ip += matchLength; anchor = ip; continue; /* faster when present ... (?) */ @@ -2099,7 +2153,6 @@ size_t ZSTD_compressBlock_lazy_extDict_row( size_t ZSTD_compressBlock_lazy2_extDict_row( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize) - { return ZSTD_compressBlock_lazy_extDict_generic(ms, seqStore, rep, src, srcSize, search_rowHash, 2); } diff --git a/vendor/github.com/DataDog/zstd/zstd_lazy.h b/vendor/github.com/DataDog/zstd/zstd_lazy.h index dc5046b..abadc7c 100644 --- a/vendor/github.com/DataDog/zstd/zstd_lazy.h +++ b/vendor/github.com/DataDog/zstd/zstd_lazy.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -26,6 +26,8 @@ extern "C" { */ #define ZSTD_LAZY_DDSS_BUCKET_LOG 2 +#define ZSTD_ROW_HASH_TAG_BITS 8 /* nb bits to use for the tag */ + U32 ZSTD_insertAndFindFirstIndex(ZSTD_matchState_t* ms, const BYTE* ip); void ZSTD_row_update(ZSTD_matchState_t* const ms, const BYTE* ip); @@ -117,7 +119,7 @@ size_t ZSTD_compressBlock_lazy2_extDict_row( size_t ZSTD_compressBlock_btlazy2_extDict( ZSTD_matchState_t* ms, seqStore_t* seqStore, U32 rep[ZSTD_REP_NUM], void const* src, size_t srcSize); - + #if defined (__cplusplus) } diff --git a/vendor/github.com/DataDog/zstd/zstd_ldm.c b/vendor/github.com/DataDog/zstd/zstd_ldm.c index ce4cece..b6c4d96 100644 --- a/vendor/github.com/DataDog/zstd/zstd_ldm.c +++ b/vendor/github.com/DataDog/zstd/zstd_ldm.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -243,11 +243,11 @@ static size_t ZSTD_ldm_fillFastTables(ZSTD_matchState_t* ms, switch(ms->cParams.strategy) { case ZSTD_fast: - ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast); + ZSTD_fillHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); break; case ZSTD_dfast: - ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast); + ZSTD_fillDoubleHashTable(ms, iend, ZSTD_dtlm_fast, ZSTD_tfp_forCCtx); break; case ZSTD_greedy: @@ -550,7 +550,7 @@ size_t ZSTD_ldm_generateSequences( * the window through early invalidation. * TODO: * Test the chunk size. * * Try invalidation after the sequence generation and test the - * the offset against maxDist directly. + * offset against maxDist directly. * * NOTE: Because of dictionaries + sequence splitting we MUST make sure * that any offset used is valid at the END of the sequence, since it may @@ -712,7 +712,7 @@ size_t ZSTD_ldm_blockCompress(rawSeqStore_t* rawSeqStore, rep[0] = sequence.offset; /* Store the sequence */ ZSTD_storeSeq(seqStore, newLitLength, ip - newLitLength, iend, - STORE_OFFSET(sequence.offset), + OFFSET_TO_OFFBASE(sequence.offset), sequence.matchLength); ip += sequence.matchLength; } diff --git a/vendor/github.com/DataDog/zstd/zstd_ldm.h b/vendor/github.com/DataDog/zstd/zstd_ldm.h index 41aea58..ceabc23 100644 --- a/vendor/github.com/DataDog/zstd/zstd_ldm.h +++ b/vendor/github.com/DataDog/zstd/zstd_ldm.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_ldm_geartab.h b/vendor/github.com/DataDog/zstd/zstd_ldm_geartab.h index bd5fa4a..5d799e7 100644 --- a/vendor/github.com/DataDog/zstd/zstd_ldm_geartab.h +++ b/vendor/github.com/DataDog/zstd/zstd_ldm_geartab.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_legacy.h b/vendor/github.com/DataDog/zstd/zstd_legacy.h index b68c53f..a43c232 100644 --- a/vendor/github.com/DataDog/zstd/zstd_legacy.h +++ b/vendor/github.com/DataDog/zstd/zstd_legacy.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -243,6 +243,13 @@ MEM_STATIC ZSTD_frameSizeInfo ZSTD_findFrameSizeInfoLegacy(const void *src, size frameSizeInfo.compressedSize = ERROR(srcSize_wrong); frameSizeInfo.decompressedBound = ZSTD_CONTENTSIZE_ERROR; } + /* In all cases, decompressedBound == nbBlocks * ZSTD_BLOCKSIZE_MAX. + * So we can compute nbBlocks without having to change every function. + */ + if (frameSizeInfo.decompressedBound != ZSTD_CONTENTSIZE_ERROR) { + assert((frameSizeInfo.decompressedBound & (ZSTD_BLOCKSIZE_MAX - 1)) == 0); + frameSizeInfo.nbBlocks = (size_t)(frameSizeInfo.decompressedBound / ZSTD_BLOCKSIZE_MAX); + } return frameSizeInfo; } diff --git a/vendor/github.com/DataDog/zstd/zstd_opt.c b/vendor/github.com/DataDog/zstd/zstd_opt.c index e673a9e..ca5fdb7 100644 --- a/vendor/github.com/DataDog/zstd/zstd_opt.c +++ b/vendor/github.com/DataDog/zstd/zstd_opt.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -17,7 +17,7 @@ #define ZSTD_LITFREQ_ADD 2 /* scaling factor for litFreq, so that frequencies adapt faster to new stats */ #define ZSTD_MAX_PRICE (1<<30) -#define ZSTD_PREDEF_THRESHOLD 1024 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ +#define ZSTD_PREDEF_THRESHOLD 8 /* if srcSize < ZSTD_PREDEF_THRESHOLD, symbols' cost is assumed static, directly determined by pre-defined distributions */ /*-************************************* @@ -27,27 +27,35 @@ #if 0 /* approximation at bit level (for tests) */ # define BITCOST_ACCURACY 0 # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat, opt) ((void)opt, ZSTD_bitWeight(stat)) +# define WEIGHT(stat, opt) ((void)(opt), ZSTD_bitWeight(stat)) #elif 0 /* fractional bit accuracy (for tests) */ # define BITCOST_ACCURACY 8 # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat,opt) ((void)opt, ZSTD_fracWeight(stat)) +# define WEIGHT(stat,opt) ((void)(opt), ZSTD_fracWeight(stat)) #else /* opt==approx, ultra==accurate */ # define BITCOST_ACCURACY 8 # define BITCOST_MULTIPLIER (1 << BITCOST_ACCURACY) -# define WEIGHT(stat,opt) (opt ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) +# define WEIGHT(stat,opt) ((opt) ? ZSTD_fracWeight(stat) : ZSTD_bitWeight(stat)) #endif +/* ZSTD_bitWeight() : + * provide estimated "cost" of a stat in full bits only */ MEM_STATIC U32 ZSTD_bitWeight(U32 stat) { return (ZSTD_highbit32(stat+1) * BITCOST_MULTIPLIER); } +/* ZSTD_fracWeight() : + * provide fractional-bit "cost" of a stat, + * using linear interpolation approximation */ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) { U32 const stat = rawStat + 1; U32 const hb = ZSTD_highbit32(stat); U32 const BWeight = hb * BITCOST_MULTIPLIER; + /* Fweight was meant for "Fractional weight" + * but it's effectively a value between 1 and 2 + * using fixed point arithmetic */ U32 const FWeight = (stat << BITCOST_ACCURACY) >> hb; U32 const weight = BWeight + FWeight; assert(hb + BITCOST_ACCURACY < 31); @@ -58,7 +66,7 @@ MEM_STATIC U32 ZSTD_fracWeight(U32 rawStat) /* debugging function, * @return price in bytes as fractional value * for debug messages only */ -MEM_STATIC double ZSTD_fCost(U32 price) +MEM_STATIC double ZSTD_fCost(int price) { return (double)price / (BITCOST_MULTIPLIER*8); } @@ -89,20 +97,26 @@ static U32 sum_u32(const unsigned table[], size_t nbElts) return total; } -static U32 ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift) +typedef enum { base_0possible=0, base_1guaranteed=1 } base_directive_e; + +static U32 +ZSTD_downscaleStats(unsigned* table, U32 lastEltIndex, U32 shift, base_directive_e base1) { U32 s, sum=0; - DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", (unsigned)lastEltIndex+1, (unsigned)shift); + DEBUGLOG(5, "ZSTD_downscaleStats (nbElts=%u, shift=%u)", + (unsigned)lastEltIndex+1, (unsigned)shift ); assert(shift < 30); for (s=0; s> shift); - sum += table[s]; + unsigned const base = base1 ? 1 : (table[s]>0); + unsigned const newStat = base + (table[s] >> shift); + sum += newStat; + table[s] = newStat; } return sum; } /* ZSTD_scaleStats() : - * reduce all elements in table is sum too large + * reduce all elt frequencies in table if sum too large * return the resulting sum of elements */ static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) { @@ -111,7 +125,7 @@ static U32 ZSTD_scaleStats(unsigned* table, U32 lastEltIndex, U32 logTarget) DEBUGLOG(5, "ZSTD_scaleStats (nbElts=%u, target=%u)", (unsigned)lastEltIndex+1, (unsigned)logTarget); assert(logTarget < 30); if (factor <= 1) return prevsum; - return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor)); + return ZSTD_downscaleStats(table, lastEltIndex, ZSTD_highbit32(factor), base_1guaranteed); } /* ZSTD_rescaleFreqs() : @@ -130,18 +144,22 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, DEBUGLOG(5, "ZSTD_rescaleFreqs (srcSize=%u)", (unsigned)srcSize); optPtr->priceType = zop_dynamic; - if (optPtr->litLengthSum == 0) { /* first block : init */ - if (srcSize <= ZSTD_PREDEF_THRESHOLD) { /* heuristic */ - DEBUGLOG(5, "(srcSize <= ZSTD_PREDEF_THRESHOLD) => zop_predef"); + if (optPtr->litLengthSum == 0) { /* no literals stats collected -> first block assumed -> init */ + + /* heuristic: use pre-defined stats for too small inputs */ + if (srcSize <= ZSTD_PREDEF_THRESHOLD) { + DEBUGLOG(5, "srcSize <= %i : use predefined stats", ZSTD_PREDEF_THRESHOLD); optPtr->priceType = zop_predef; } assert(optPtr->symbolCosts != NULL); if (optPtr->symbolCosts->huf.repeatMode == HUF_repeat_valid) { - /* huffman table presumed generated by dictionary */ + + /* huffman stats covering the full value set : table presumed generated by dictionary */ optPtr->priceType = zop_dynamic; if (compressedLiterals) { + /* generate literals statistics from huffman table */ unsigned lit; assert(optPtr->litFreq != NULL); optPtr->litSum = 0; @@ -189,13 +207,14 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, optPtr->offCodeSum += optPtr->offCodeFreq[of]; } } - } else { /* not a dictionary */ + } else { /* first block, no dictionary */ assert(optPtr->litFreq != NULL); if (compressedLiterals) { + /* base initial cost of literals on direct frequency within src */ unsigned lit = MaxLit; HIST_count_simple(optPtr->litFreq, &lit, src, srcSize); /* use raw first block to init statistics */ - optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8); + optPtr->litSum = ZSTD_downscaleStats(optPtr->litFreq, MaxLit, 8, base_0possible); } { unsigned const baseLLfreqs[MaxLL+1] = { @@ -225,10 +244,9 @@ ZSTD_rescaleFreqs(optState_t* const optPtr, optPtr->offCodeSum = sum_u32(baseOFCfreqs, MaxOff+1); } - } - } else { /* new block : re-use previous statistics, scaled down */ + } else { /* new block : scale down accumulated statistics */ if (compressedLiterals) optPtr->litSum = ZSTD_scaleStats(optPtr->litFreq, MaxLit, 12); @@ -256,11 +274,14 @@ static U32 ZSTD_rawLiteralsCost(const BYTE* const literals, U32 const litLength, return (litLength*6) * BITCOST_MULTIPLIER; /* 6 bit per literal - no statistic used */ /* dynamic statistics */ - { U32 price = litLength * optPtr->litSumBasePrice; + { U32 price = optPtr->litSumBasePrice * litLength; + U32 const litPriceMax = optPtr->litSumBasePrice - BITCOST_MULTIPLIER; U32 u; + assert(optPtr->litSumBasePrice >= BITCOST_MULTIPLIER); for (u=0; u < litLength; u++) { - assert(WEIGHT(optPtr->litFreq[literals[u]], optLevel) <= optPtr->litSumBasePrice); /* literal cost should never be negative */ - price -= WEIGHT(optPtr->litFreq[literals[u]], optLevel); + U32 litPrice = WEIGHT(optPtr->litFreq[literals[u]], optLevel); + if (UNLIKELY(litPrice > litPriceMax)) litPrice = litPriceMax; + price -= litPrice; } return price; } @@ -273,10 +294,11 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP assert(litLength <= ZSTD_BLOCKSIZE_MAX); if (optPtr->priceType == zop_predef) return WEIGHT(litLength, optLevel); - /* We can't compute the litLength price for sizes >= ZSTD_BLOCKSIZE_MAX - * because it isn't representable in the zstd format. So instead just - * call it 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. In this case the block - * would be all literals. + + /* ZSTD_LLcode() can't compute litLength price for sizes >= ZSTD_BLOCKSIZE_MAX + * because it isn't representable in the zstd format. + * So instead just pretend it would cost 1 bit more than ZSTD_BLOCKSIZE_MAX - 1. + * In such a case, the block would be all literals. */ if (litLength == ZSTD_BLOCKSIZE_MAX) return BITCOST_MULTIPLIER + ZSTD_litLengthPrice(ZSTD_BLOCKSIZE_MAX - 1, optPtr, optLevel); @@ -290,24 +312,25 @@ static U32 ZSTD_litLengthPrice(U32 const litLength, const optState_t* const optP } /* ZSTD_getMatchPrice() : - * Provides the cost of the match part (offset + matchLength) of a sequence + * Provides the cost of the match part (offset + matchLength) of a sequence. * Must be combined with ZSTD_fullLiteralsCost() to get the full cost of a sequence. - * @offcode : expects a scale where 0,1,2 are repcodes 1-3, and 3+ are real_offsets+2 + * @offBase : sumtype, representing an offset or a repcode, and using numeric representation of ZSTD_storeSeq() * @optLevel: when <2, favors small offset for decompression speed (improved cache efficiency) */ FORCE_INLINE_TEMPLATE U32 -ZSTD_getMatchPrice(U32 const offcode, +ZSTD_getMatchPrice(U32 const offBase, U32 const matchLength, const optState_t* const optPtr, int const optLevel) { U32 price; - U32 const offCode = ZSTD_highbit32(STORED_TO_OFFBASE(offcode)); + U32 const offCode = ZSTD_highbit32(offBase); U32 const mlBase = matchLength - MINMATCH; assert(matchLength >= MINMATCH); - if (optPtr->priceType == zop_predef) /* fixed scheme, do not use statistics */ - return WEIGHT(mlBase, optLevel) + ((16 + offCode) * BITCOST_MULTIPLIER); + if (optPtr->priceType == zop_predef) /* fixed scheme, does not use statistics */ + return WEIGHT(mlBase, optLevel) + + ((16 + offCode) * BITCOST_MULTIPLIER); /* emulated offset cost */ /* dynamic statistics */ price = (offCode * BITCOST_MULTIPLIER) + (optPtr->offCodeSumBasePrice - WEIGHT(optPtr->offCodeFreq[offCode], optLevel)); @@ -326,10 +349,10 @@ ZSTD_getMatchPrice(U32 const offcode, } /* ZSTD_updateStats() : - * assumption : literals + litLengtn <= iend */ + * assumption : literals + litLength <= iend */ static void ZSTD_updateStats(optState_t* const optPtr, U32 litLength, const BYTE* literals, - U32 offsetCode, U32 matchLength) + U32 offBase, U32 matchLength) { /* literals */ if (ZSTD_compressedLiterals(optPtr)) { @@ -345,8 +368,8 @@ static void ZSTD_updateStats(optState_t* const optPtr, optPtr->litLengthSum++; } - /* offset code : expected to follow storeSeq() numeric representation */ - { U32 const offCode = ZSTD_highbit32(STORED_TO_OFFBASE(offsetCode)); + /* offset code : follows storeSeq() numeric representation */ + { U32 const offCode = ZSTD_highbit32(offBase); assert(offCode <= MaxOff); optPtr->offCodeFreq[offCode]++; optPtr->offCodeSum++; @@ -553,16 +576,17 @@ void ZSTD_updateTree(ZSTD_matchState_t* ms, const BYTE* ip, const BYTE* iend) { ZSTD_updateTree_internal(ms, ip, iend, ms->cParams.minMatch, ZSTD_noDict); } -FORCE_INLINE_TEMPLATE -U32 ZSTD_insertBtAndGetAllMatches ( - ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ - ZSTD_matchState_t* ms, - U32* nextToUpdate3, - const BYTE* const ip, const BYTE* const iLimit, const ZSTD_dictMode_e dictMode, - const U32 rep[ZSTD_REP_NUM], - U32 const ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ - const U32 lengthToBeat, - U32 const mls /* template */) +FORCE_INLINE_TEMPLATE U32 +ZSTD_insertBtAndGetAllMatches ( + ZSTD_match_t* matches, /* store result (found matches) in this table (presumed large enough) */ + ZSTD_matchState_t* ms, + U32* nextToUpdate3, + const BYTE* const ip, const BYTE* const iLimit, + const ZSTD_dictMode_e dictMode, + const U32 rep[ZSTD_REP_NUM], + const U32 ll0, /* tells if associated literal length is 0 or not. This value must be 0 or 1 */ + const U32 lengthToBeat, + const U32 mls /* template */) { const ZSTD_compressionParameters* const cParams = &ms->cParams; U32 const sufficient_len = MIN(cParams->targetLength, ZSTD_OPT_NUM -1); @@ -645,7 +669,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( DEBUGLOG(8, "found repCode %u (ll0:%u, offset:%u) of length %u", repCode, ll0, repOffset, repLen); bestLength = repLen; - matches[mnum].off = STORE_REPCODE(repCode - ll0 + 1); /* expect value between 1 and 3 */ + matches[mnum].off = REPCODE_TO_OFFBASE(repCode - ll0 + 1); /* expect value between 1 and 3 */ matches[mnum].len = (U32)repLen; mnum++; if ( (repLen > sufficient_len) @@ -674,7 +698,7 @@ U32 ZSTD_insertBtAndGetAllMatches ( bestLength = mlen; assert(curr > matchIndex3); assert(mnum==0); /* no prior solution */ - matches[0].off = STORE_OFFSET(curr - matchIndex3); + matches[0].off = OFFSET_TO_OFFBASE(curr - matchIndex3); matches[0].len = (U32)mlen; mnum = 1; if ( (mlen > sufficient_len) | @@ -707,13 +731,13 @@ U32 ZSTD_insertBtAndGetAllMatches ( } if (matchLength > bestLength) { - DEBUGLOG(8, "found match of length %u at distance %u (offCode=%u)", - (U32)matchLength, curr - matchIndex, STORE_OFFSET(curr - matchIndex)); + DEBUGLOG(8, "found match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); assert(matchEndIdx > matchIndex); if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; bestLength = matchLength; - matches[mnum].off = STORE_OFFSET(curr - matchIndex); + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); matches[mnum].len = (U32)matchLength; mnum++; if ( (matchLength > ZSTD_OPT_NUM) @@ -755,12 +779,12 @@ U32 ZSTD_insertBtAndGetAllMatches ( if (matchLength > bestLength) { matchIndex = dictMatchIndex + dmsIndexDelta; - DEBUGLOG(8, "found dms match of length %u at distance %u (offCode=%u)", - (U32)matchLength, curr - matchIndex, STORE_OFFSET(curr - matchIndex)); + DEBUGLOG(8, "found dms match of length %u at distance %u (offBase=%u)", + (U32)matchLength, curr - matchIndex, OFFSET_TO_OFFBASE(curr - matchIndex)); if (matchLength > matchEndIdx - matchIndex) matchEndIdx = matchIndex + (U32)matchLength; bestLength = matchLength; - matches[mnum].off = STORE_OFFSET(curr - matchIndex); + matches[mnum].off = OFFSET_TO_OFFBASE(curr - matchIndex); matches[mnum].len = (U32)matchLength; mnum++; if ( (matchLength > ZSTD_OPT_NUM) @@ -961,7 +985,7 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, const ZSTD_optLdm_t* optLdm, U32 currPosInBlock) { U32 const posDiff = currPosInBlock - optLdm->startPosInBlock; - /* Note: ZSTD_match_t actually contains offCode and matchLength (before subtracting MINMATCH) */ + /* Note: ZSTD_match_t actually contains offBase and matchLength (before subtracting MINMATCH) */ U32 const candidateMatchLength = optLdm->endPosInBlock - optLdm->startPosInBlock - posDiff; /* Ensure that current block position is not outside of the match */ @@ -972,11 +996,11 @@ static void ZSTD_optLdm_maybeAddMatch(ZSTD_match_t* matches, U32* nbMatches, } if (*nbMatches == 0 || ((candidateMatchLength > matches[*nbMatches-1].len) && *nbMatches < ZSTD_OPT_NUM)) { - U32 const candidateOffCode = STORE_OFFSET(optLdm->offset); - DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offCode: %u matchLength %u) at block position=%u", - candidateOffCode, candidateMatchLength, currPosInBlock); + U32 const candidateOffBase = OFFSET_TO_OFFBASE(optLdm->offset); + DEBUGLOG(6, "ZSTD_optLdm_maybeAddMatch(): Adding ldm candidate match (offBase: %u matchLength %u) at block position=%u", + candidateOffBase, candidateMatchLength, currPosInBlock); matches[*nbMatches].len = candidateMatchLength; - matches[*nbMatches].off = candidateOffCode; + matches[*nbMatches].off = candidateOffBase; (*nbMatches)++; } } @@ -1063,6 +1087,8 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, ZSTD_optimal_t lastSequence; ZSTD_optLdm_t optLdm; + ZSTD_memset(&lastSequence, 0, sizeof(ZSTD_optimal_t)); + optLdm.seqStore = ms->ldmSeqStore ? *ms->ldmSeqStore : kNullRawSeqStore; optLdm.endPosInBlock = optLdm.startPosInBlock = optLdm.offset = 0; ZSTD_opt_getNextMatchAndUpdateSeqStore(&optLdm, (U32)(ip-istart), (U32)(iend-ip)); @@ -1099,14 +1125,14 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, /* large match -> immediate encoding */ { U32 const maxML = matches[nbMatches-1].len; - U32 const maxOffcode = matches[nbMatches-1].off; - DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffCode=%u at cPos=%u => start new series", - nbMatches, maxML, maxOffcode, (U32)(ip-prefixStart)); + U32 const maxOffBase = matches[nbMatches-1].off; + DEBUGLOG(6, "found %u matches of maxLength=%u and maxOffBase=%u at cPos=%u => start new series", + nbMatches, maxML, maxOffBase, (U32)(ip-prefixStart)); if (maxML > sufficient_len) { lastSequence.litlen = litlen; lastSequence.mlen = maxML; - lastSequence.off = maxOffcode; + lastSequence.off = maxOffBase; DEBUGLOG(6, "large match (%u>%u), immediate encoding", maxML, sufficient_len); cur = 0; @@ -1123,15 +1149,15 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, opt[pos].price = ZSTD_MAX_PRICE; /* mlen, litlen and price will be fixed during forward scanning */ } for (matchNb = 0; matchNb < nbMatches; matchNb++) { - U32 const offcode = matches[matchNb].off; + U32 const offBase = matches[matchNb].off; U32 const end = matches[matchNb].len; for ( ; pos <= end ; pos++ ) { - U32 const matchPrice = ZSTD_getMatchPrice(offcode, pos, optStatePtr, optLevel); + U32 const matchPrice = ZSTD_getMatchPrice(offBase, pos, optStatePtr, optLevel); U32 const sequencePrice = literalsPrice + matchPrice; DEBUGLOG(7, "rPos:%u => set initial price : %.2f", - pos, ZSTD_fCost(sequencePrice)); + pos, ZSTD_fCost((int)sequencePrice)); opt[pos].mlen = pos; - opt[pos].off = offcode; + opt[pos].off = offBase; opt[pos].litlen = litlen; opt[pos].price = (int)sequencePrice; } } @@ -1231,7 +1257,7 @@ ZSTD_compressBlock_opt_generic(ZSTD_matchState_t* ms, U32 const startML = (matchNb>0) ? matches[matchNb-1].len+1 : minMatch; U32 mlen; - DEBUGLOG(7, "testing match %u => offCode=%4u, mlen=%2u, llen=%2u", + DEBUGLOG(7, "testing match %u => offBase=%4u, mlen=%2u, llen=%2u", matchNb, matches[matchNb].off, lastML, litlen); for (mlen = lastML; mlen >= startML; mlen--) { /* scan downward */ @@ -1297,7 +1323,7 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ for (storePos=storeStart; storePos <= storeEnd; storePos++) { U32 const llen = opt[storePos].litlen; U32 const mlen = opt[storePos].mlen; - U32 const offCode = opt[storePos].off; + U32 const offBase = opt[storePos].off; U32 const advance = llen + mlen; DEBUGLOG(6, "considering seq starting at %zi, llen=%u, mlen=%u", anchor - istart, (unsigned)llen, (unsigned)mlen); @@ -1309,8 +1335,8 @@ _shortestPath: /* cur, last_pos, best_mlen, best_off have to be set */ } assert(anchor + llen <= iend); - ZSTD_updateStats(optStatePtr, llen, anchor, offCode, mlen); - ZSTD_storeSeq(seqStore, llen, anchor, iend, offCode, mlen); + ZSTD_updateStats(optStatePtr, llen, anchor, offBase, mlen); + ZSTD_storeSeq(seqStore, llen, anchor, iend, offBase, mlen); anchor += advance; ip = anchor; } } @@ -1350,7 +1376,7 @@ size_t ZSTD_compressBlock_btopt( /* ZSTD_initStats_ultra(): * make a first compression pass, just to seed stats with more accurate starting values. * only works on first block, with no dictionary and no ldm. - * this function cannot error, hence its contract must be respected. + * this function cannot error out, its narrow contract must be respected. */ static void ZSTD_initStats_ultra(ZSTD_matchState_t* ms, @@ -1369,7 +1395,7 @@ ZSTD_initStats_ultra(ZSTD_matchState_t* ms, ZSTD_compressBlock_opt2(ms, seqStore, tmpRep, src, srcSize, ZSTD_noDict); /* generate stats into ms->opt*/ - /* invalidate first scan from history */ + /* invalidate first scan from history, only keep entropy stats */ ZSTD_resetSeqStore(seqStore); ms->window.base -= srcSize; ms->window.dictLimit += (U32)srcSize; @@ -1393,20 +1419,20 @@ size_t ZSTD_compressBlock_btultra2( U32 const curr = (U32)((const BYTE*)src - ms->window.base); DEBUGLOG(5, "ZSTD_compressBlock_btultra2 (srcSize=%zu)", srcSize); - /* 2-pass strategy: + /* 2-passes strategy: * this strategy makes a first pass over first block to collect statistics - * and seed next round's statistics with it. - * After 1st pass, function forgets everything, and starts a new block. + * in order to seed next round's statistics with it. + * After 1st pass, function forgets history, and starts a new block. * Consequently, this can only work if no data has been previously loaded in tables, * aka, no dictionary, no prefix, no ldm preprocessing. * The compression ratio gain is generally small (~0.5% on first block), - * the cost is 2x cpu time on first block. */ + ** the cost is 2x cpu time on first block. */ assert(srcSize <= ZSTD_BLOCKSIZE_MAX); if ( (ms->opt.litLengthSum==0) /* first block */ && (seqStore->sequences == seqStore->sequencesStart) /* no ldm */ && (ms->window.dictLimit == ms->window.lowLimit) /* no dictionary */ - && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ - && (srcSize > ZSTD_PREDEF_THRESHOLD) + && (curr == ms->window.dictLimit) /* start of frame, nothing already loaded nor skipped */ + && (srcSize > ZSTD_PREDEF_THRESHOLD) /* input large enough to not employ default stats */ ) { ZSTD_initStats_ultra(ms, seqStore, rep, src, srcSize); } diff --git a/vendor/github.com/DataDog/zstd/zstd_opt.h b/vendor/github.com/DataDog/zstd/zstd_opt.h index 0de190a..3eb02f5 100644 --- a/vendor/github.com/DataDog/zstd/zstd_opt.h +++ b/vendor/github.com/DataDog/zstd/zstd_opt.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_stream.go b/vendor/github.com/DataDog/zstd/zstd_stream.go index 5e5718f..e3b6c09 100644 --- a/vendor/github.com/DataDog/zstd/zstd_stream.go +++ b/vendor/github.com/DataDog/zstd/zstd_stream.go @@ -72,6 +72,7 @@ import ( var errShortRead = errors.New("short read") var errReaderClosed = errors.New("Reader is closed") +var ErrNoParallelSupport = errors.New("No parallel support") // Writer is an io.WriteCloser that zstd-compresses its input. type Writer struct { @@ -302,6 +303,28 @@ func (w *Writer) Close() error { return getError(int(C.ZSTD_freeCStream(w.ctx))) } +// Set the number of workers to run the compression in parallel using multiple threads +// If > 1, the Write() call will become asynchronous. This means data will be buffered until processed. +// If you call Write() too fast, you might incur a memory buffer up to as large as your input. +// Consider calling Flush() periodically if you need to compress a very large file that would not fit all in memory. +// By default only one worker is used. +func (w *Writer) SetNbWorkers(n int) error { + if w.firstError != nil { + return w.firstError + } + if err := getError(int(C.ZSTD_CCtx_setParameter(w.ctx, C.ZSTD_c_nbWorkers, C.int(n)))); err != nil { + w.firstError = err + // First error case, a shared libary is used, and the library was compiled without parallel support + if err.Error() == "Unsupported parameter" { + return ErrNoParallelSupport + } else { + // This could happen if a very large number is passed in, and possibly zstd refuse to create as many threads, or the OS fails to do so + return err + } + } + return nil +} + // cSize is the recommended size of reader.compressionBuffer. This func and // invocation allow for a one-time check for validity. var cSize = func() int { diff --git a/vendor/github.com/DataDog/zstd/zstd_trace.h b/vendor/github.com/DataDog/zstd/zstd_trace.h index ee92fd4..168ace4 100644 --- a/vendor/github.com/DataDog/zstd/zstd_trace.h +++ b/vendor/github.com/DataDog/zstd/zstd_trace.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -22,13 +22,13 @@ extern "C" { * For now, enable conservatively: * - Only GNUC * - Only ELF - * - Only x86-64 and i386 + * - Only x86-64, i386 and aarch64 * Also, explicitly disable on platforms known not to work so they aren't * forgotten in the future. */ #if !defined(ZSTD_HAVE_WEAK_SYMBOLS) && \ defined(__GNUC__) && defined(__ELF__) && \ - (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86)) && \ + (defined(__x86_64__) || defined(_M_X64) || defined(__i386__) || defined(_M_IX86) || defined(__aarch64__)) && \ !defined(__APPLE__) && !defined(_WIN32) && !defined(__MINGW32__) && \ !defined(__CYGWIN__) && !defined(_AIX) # define ZSTD_HAVE_WEAK_SYMBOLS 1 diff --git a/vendor/github.com/DataDog/zstd/zstd_v01.c b/vendor/github.com/DataDog/zstd/zstd_v01.c index d16cc94..cee369d 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v01.c +++ b/vendor/github.com/DataDog/zstd/zstd_v01.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -191,25 +191,6 @@ typedef signed long long S64; /**************************************************************** * Memory I/O *****************************************************************/ -/* FSE_FORCE_MEMORY_ACCESS - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets generating assembly depending on alignment. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef FSE_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) -# define FSE_FORCE_MEMORY_ACCESS 1 -# endif -#endif - static unsigned FSE_32bits(void) { @@ -222,24 +203,6 @@ static unsigned FSE_isLittleEndian(void) return one.c[0]; } -#if defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==2) - -static U16 FSE_read16(const void* memPtr) { return *(const U16*) memPtr; } -static U32 FSE_read32(const void* memPtr) { return *(const U32*) memPtr; } -static U64 FSE_read64(const void* memPtr) { return *(const U64*) memPtr; } - -#elif defined(FSE_FORCE_MEMORY_ACCESS) && (FSE_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign; - -static U16 FSE_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -static U32 FSE_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -static U64 FSE_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } - -#else - static U16 FSE_read16(const void* memPtr) { U16 val; memcpy(&val, memPtr, sizeof(val)); return val; @@ -255,8 +218,6 @@ static U64 FSE_read64(const void* memPtr) U64 val; memcpy(&val, memPtr, sizeof(val)); return val; } -#endif /* FSE_FORCE_MEMORY_ACCESS */ - static U16 FSE_readLE16(const void* memPtr) { if (FSE_isLittleEndian()) @@ -1191,7 +1152,7 @@ static size_t HUF_decompress (void* dst, size_t maxDstSize, const void* cSrc, si zstd - standard compression library Copyright (C) 2014-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1760,20 +1721,26 @@ static size_t ZSTD_execSequence(BYTE* op, static const int dec32table[] = {0, 1, 2, 1, 4, 4, 4, 4}; /* added */ static const int dec64table[] = {8, 8, 8, 7, 8, 9,10,11}; /* subtracted */ const BYTE* const ostart = op; + BYTE* const oLitEnd = op + sequence.litLength; const size_t litLength = sequence.litLength; BYTE* const endMatch = op + litLength + sequence.matchLength; /* risk : address space overflow (32-bits) */ const BYTE* const litEnd = *litPtr + litLength; - /* check */ + /* checks */ + size_t const seqLength = sequence.litLength + sequence.matchLength; + + if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall); + if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected); + /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */ + if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected); + if (endMatch > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ - if (litEnd > litLimit) return ERROR(corruption_detected); - if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall); /* overwrite literal segment */ + if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */ + if (sequence.matchLength > (size_t)(*litPtr-op)) return ERROR(dstSize_tooSmall); /* overwrite literal segment */ /* copy Literals */ - if (((size_t)(*litPtr - op) < 8) || ((size_t)(oend-litEnd) < 8) || (op+litLength > oend-8)) - memmove(op, *litPtr, litLength); /* overwrite risk */ - else - ZSTD_wildcopy(op, *litPtr, litLength); + ZSTD_memmove(op, *litPtr, sequence.litLength); /* note : v0.1 seems to allow scenarios where output or input are close to end of buffer */ + op += litLength; *litPtr = litEnd; /* update for next sequence */ diff --git a/vendor/github.com/DataDog/zstd/zstd_v01.h b/vendor/github.com/DataDog/zstd/zstd_v01.h index 1f4cdad..29c7143 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v01.h +++ b/vendor/github.com/DataDog/zstd/zstd_v01.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_v02.c b/vendor/github.com/DataDog/zstd/zstd_v02.c index 38fc5df..cc33658 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v02.c +++ b/vendor/github.com/DataDog/zstd/zstd_v02.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -29,7 +29,7 @@ low-level memory access routines Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -116,24 +116,6 @@ extern "C" { /**************************************************************** * Memory I/O *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets generating assembly depending on alignment. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) -# define MEM_FORCE_MEMORY_ACCESS 1 -# endif -#endif MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; } MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; } @@ -144,33 +126,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void) return one.c[0]; } -#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) - -/* violates C standard on structure alignment. -Only use if no other choice to achieve best performance on target platform */ -MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } -MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } -MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } - -#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign; - -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } - -#else - -/* default method, safe and standard. - can sometimes prove slower */ - MEM_STATIC U16 MEM_read16(const void* memPtr) { U16 val; memcpy(&val, memPtr, sizeof(val)); return val; @@ -191,9 +146,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value) memcpy(memPtr, &value, sizeof(value)); } -#endif /* MEM_FORCE_MEMORY_ACCESS */ - - MEM_STATIC U16 MEM_readLE16(const void* memPtr) { if (MEM_isLittleEndian()) @@ -270,7 +222,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr) header file (to include) Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -434,7 +386,7 @@ MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits) } /*! BIT_lookBitsFast : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits) { const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; @@ -454,7 +406,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) } /*!BIT_readBitsFast : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) { size_t value = BIT_lookBitsFast(bitD, nbBits); @@ -511,7 +463,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) Error codes and messages Copyright (C) 2013-2015, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -610,7 +562,7 @@ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be mor header file for static linking (only) Copyright (C) 2013-2015, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -754,7 +706,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) header file for static linking (only) Copyright (C) 2013-2015, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -823,7 +775,7 @@ static size_t HUF_decompress4X6 (void* dst, size_t dstSize, const void* cSrc, si Header File Copyright (C) 2014-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -883,7 +835,7 @@ typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */ Header File for static linking only Copyright (C) 2014-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -947,7 +899,7 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx; FSE : Finite State Entropy coder Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1451,7 +1403,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz Huff0 : Huffman coder, part of New Generation Entropy library Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2610,7 +2562,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_ zstd - standard compression library Copyright (C) 2014-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -3115,12 +3067,19 @@ static size_t ZSTD_execSequence(BYTE* op, const BYTE* const litEnd = *litPtr + sequence.litLength; /* checks */ - if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ + size_t const seqLength = sequence.litLength + sequence.matchLength; + + if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall); + if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected); + /* Now we know there are no overflow in literal nor match lengths, can use the pointer check */ + if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); + if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected); + if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */ /* copy Literals */ - ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ + ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ op = oLitEnd; *litPtr = litEnd; /* update for next sequence */ diff --git a/vendor/github.com/DataDog/zstd/zstd_v02.h b/vendor/github.com/DataDog/zstd/zstd_v02.h index 7c6a84a..71d79ca 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v02.h +++ b/vendor/github.com/DataDog/zstd/zstd_v02.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_v03.c b/vendor/github.com/DataDog/zstd/zstd_v03.c index 552e403..af98cac 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v03.c +++ b/vendor/github.com/DataDog/zstd/zstd_v03.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -30,7 +30,7 @@ low-level memory access routines Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -117,24 +117,6 @@ extern "C" { /**************************************************************** * Memory I/O *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets generating assembly depending on alignment. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) -# define MEM_FORCE_MEMORY_ACCESS 1 -# endif -#endif MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; } MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; } @@ -145,33 +127,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void) return one.c[0]; } -#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) - -/* violates C standard on structure alignment. -Only use if no other choice to achieve best performance on target platform */ -MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } -MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } -MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } - -#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign; - -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } - -#else - -/* default method, safe and standard. - can sometimes prove slower */ - MEM_STATIC U16 MEM_read16(const void* memPtr) { U16 val; memcpy(&val, memPtr, sizeof(val)); return val; @@ -192,10 +147,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value) memcpy(memPtr, &value, sizeof(value)); } - -#endif /* MEM_FORCE_MEMORY_ACCESS */ - - MEM_STATIC U16 MEM_readLE16(const void* memPtr) { if (MEM_isLittleEndian()) @@ -272,7 +223,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr) header file (to include) Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -436,7 +387,7 @@ MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits) } /*! BIT_lookBitsFast : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits) { const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; @@ -456,7 +407,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) } /*!BIT_readBitsFast : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) { size_t value = BIT_lookBitsFast(bitD, nbBits); @@ -513,7 +464,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) Error codes and messages Copyright (C) 2013-2015, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -612,7 +563,7 @@ typedef unsigned FSE_DTable; /* don't allocate that. It's just a way to be mor header file for static linking (only) Copyright (C) 2013-2015, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -756,7 +707,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) header file for static linking (only) Copyright (C) 2013-2015, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -824,7 +775,7 @@ static size_t HUF_decompress4X4 (void* dst, size_t dstSize, const void* cSrc, si Header File Copyright (C) 2014-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -884,7 +835,7 @@ typedef struct ZSTD_CCtx_s ZSTD_CCtx; /* incomplete type */ Header File for static linking only Copyright (C) 2014-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -948,7 +899,7 @@ typedef struct ZSTD_DCtx_s ZSTD_DCtx; FSE : Finite State Entropy coder Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1452,7 +1403,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz Huff0 : Huffman coder, part of New Generation Entropy library Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2249,7 +2200,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_ zstd - standard compression library Copyright (C) 2014-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2756,18 +2707,24 @@ static size_t ZSTD_execSequence(BYTE* op, const BYTE* const litEnd = *litPtr + sequence.litLength; /* checks */ - if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ + size_t const seqLength = sequence.litLength + sequence.matchLength; + + if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall); + if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected); + /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */ + if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); + if (sequence.offset > (U32)(oLitEnd - base)) return ERROR(corruption_detected); + if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */ /* copy Literals */ - ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ + ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ op = oLitEnd; *litPtr = litEnd; /* update for next sequence */ /* copy Match */ - { - const BYTE* match = op - sequence.offset; + { const BYTE* match = op - sequence.offset; /* check */ if (sequence.offset > (size_t)op) return ERROR(corruption_detected); /* address space overflow test (this test seems kept by clang optimizer) */ diff --git a/vendor/github.com/DataDog/zstd/zstd_v03.h b/vendor/github.com/DataDog/zstd/zstd_v03.h index f6f3776..ec3d897 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v03.h +++ b/vendor/github.com/DataDog/zstd/zstd_v03.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_v04.c b/vendor/github.com/DataDog/zstd/zstd_v04.c index eaf41a6..b353ddc 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v04.c +++ b/vendor/github.com/DataDog/zstd/zstd_v04.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -88,24 +88,6 @@ extern "C" { /**************************************************************** * Memory I/O *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets generating assembly depending on alignment. - * But in some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) -# define MEM_FORCE_MEMORY_ACCESS 1 -# endif -#endif MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; } MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; } @@ -116,33 +98,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void) return one.c[0]; } -#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) - -/* violates C standard on structure alignment. -Only use if no other choice to achieve best performance on target platform */ -MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } -MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } -MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } - -#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; U64 u64; } __attribute__((packed)) unalign; - -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } - -#else - -/* default method, safe and standard. - can sometimes prove slower */ - MEM_STATIC U16 MEM_read16(const void* memPtr) { U16 val; memcpy(&val, memPtr, sizeof(val)); return val; @@ -163,9 +118,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value) memcpy(memPtr, &value, sizeof(value)); } -#endif /* MEM_FORCE_MEMORY_ACCESS */ - - MEM_STATIC U16 MEM_readLE16(const void* memPtr) { if (MEM_isLittleEndian()) @@ -543,7 +495,7 @@ If there is an error, the function will return an error code, which can be teste header file (to include) Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -701,7 +653,7 @@ MEM_STATIC size_t BIT_lookBits(BIT_DStream_t* bitD, U32 nbBits) } /*! BIT_lookBitsFast : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BIT_lookBitsFast(BIT_DStream_t* bitD, U32 nbBits) { const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; @@ -721,7 +673,7 @@ MEM_STATIC size_t BIT_readBits(BIT_DStream_t* bitD, U32 nbBits) } /*!BIT_readBitsFast : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BIT_readBitsFast(BIT_DStream_t* bitD, U32 nbBits) { size_t value = BIT_lookBitsFast(bitD, nbBits); @@ -782,7 +734,7 @@ MEM_STATIC unsigned BIT_endOfDStream(const BIT_DStream_t* DStream) header file for static linking (only) Copyright (C) 2013-2015, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -931,7 +883,7 @@ MEM_STATIC unsigned FSE_endOfDState(const FSE_DState_t* DStatePtr) FSE : Finite State Entropy coder Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1437,7 +1389,7 @@ static size_t FSE_decompress(void* dst, size_t maxDstSize, const void* cSrc, siz header file Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1515,7 +1467,7 @@ static unsigned HUF_isError(size_t code); /* tells if a return value i header file for static linking (only) Copyright (C) 2013-2015, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1602,7 +1554,7 @@ static size_t HUF_decompress4X4_usingDTable(void* dst, size_t maxDstSize, const Huff0 : Huffman coder, part of New Generation Entropy library Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2402,7 +2354,7 @@ static size_t HUF_decompress (void* dst, size_t dstSize, const void* cSrc, size_ zstd - decompression module fo v0.4 legacy format Copyright (C) 2015-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2877,13 +2829,19 @@ static size_t ZSTD_execSequence(BYTE* op, const BYTE* const litEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; - /* check */ - if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ + /* checks */ + size_t const seqLength = sequence.litLength + sequence.matchLength; + + if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall); + if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected); + /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */ + if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); + if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ - if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */ + if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */ /* copy Literals */ - ZSTD_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ + ZSTD_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ op = oLitEnd; *litPtr = litEnd; /* update for next sequence */ @@ -3284,7 +3242,7 @@ static void ZSTD_decompress_insertDictionary(ZSTD_DCtx* ctx, const void* dict, s Buffered version of Zstd compression library Copyright (C) 2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/vendor/github.com/DataDog/zstd/zstd_v04.h b/vendor/github.com/DataDog/zstd/zstd_v04.h index 4c2d8f6..b904be0 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v04.h +++ b/vendor/github.com/DataDog/zstd/zstd_v04.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_v05.c b/vendor/github.com/DataDog/zstd/zstd_v05.c index 6746a62..a91166a 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v05.c +++ b/vendor/github.com/DataDog/zstd/zstd_v05.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -20,7 +20,7 @@ low-level memory access routines Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -107,24 +107,6 @@ extern "C" { /*-************************************************************** * Memory I/O *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) -# define MEM_FORCE_MEMORY_ACCESS 1 -# endif -#endif MEM_STATIC unsigned MEM_32bits(void) { return sizeof(void*)==4; } MEM_STATIC unsigned MEM_64bits(void) { return sizeof(void*)==8; } @@ -135,37 +117,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void) return one.c[0]; } -#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) - -/* violates C standard, by lying on structure alignment. -Only use if no other choice to achieve best performance on target platform */ -MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } -MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } -MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { *(U32*)memPtr = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { *(U64*)memPtr = value; } - -#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; - -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } -MEM_STATIC void MEM_write32(void* memPtr, U32 value) { ((unalign*)memPtr)->u32 = value; } -MEM_STATIC void MEM_write64(void* memPtr, U64 value) { ((unalign*)memPtr)->u64 = value; } - -#else - -/* default method, safe and standard. - can sometimes prove slower */ - MEM_STATIC U16 MEM_read16(const void* memPtr) { U16 val; memcpy(&val, memPtr, sizeof(val)); return val; @@ -196,9 +147,6 @@ MEM_STATIC void MEM_write64(void* memPtr, U64 value) memcpy(memPtr, &value, sizeof(value)); } -#endif /* MEM_FORCE_MEMORY_ACCESS */ - - MEM_STATIC U16 MEM_readLE16(const void* memPtr) { if (MEM_isLittleEndian()) @@ -263,7 +211,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr) Header File for static linking only Copyright (C) 2014-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -287,7 +235,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr) OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd homepage : http://www.zstd.net + - zstd homepage : https://facebook.github.io/zstd */ #ifndef ZSTD_STATIC_H #define ZSTD_STATIC_H @@ -399,7 +347,7 @@ size_t ZSTDv05_decompressBlock(ZSTDv05_DCtx* dctx, void* dst, size_t dstCapacity Header File for include Copyright (C) 2014-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -486,7 +434,7 @@ static const size_t ZSTDv05_frameHeaderSize_min = 5; #define FSEv05_ENCODING_DYNAMIC 3 -#define HufLog 12 +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ @@ -554,7 +502,7 @@ typedef struct { header file Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -673,7 +621,7 @@ size_t FSEv05_decompress_usingDTable(void* dst, size_t dstCapacity, const void* header file (to include) Copyright (C) 2013-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -827,7 +775,7 @@ MEM_STATIC size_t BITv05_lookBits(BITv05_DStream_t* bitD, U32 nbBits) } /*! BITv05_lookBitsFast : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BITv05_lookBitsFast(BITv05_DStream_t* bitD, U32 nbBits) { const U32 bitMask = sizeof(bitD->bitContainer)*8 - 1; @@ -847,7 +795,7 @@ MEM_STATIC size_t BITv05_readBits(BITv05_DStream_t* bitD, unsigned nbBits) } /*!BITv05_readBitsFast : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BITv05_readBitsFast(BITv05_DStream_t* bitD, unsigned nbBits) { size_t value = BITv05_lookBitsFast(bitD, nbBits); @@ -902,7 +850,7 @@ MEM_STATIC unsigned BITv05_endOfDStream(const BITv05_DStream_t* DStream) header file for static linking (only) Copyright (C) 2013-2015, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1052,7 +1000,7 @@ MEM_STATIC unsigned FSEv05_endOfDState(const FSEv05_DState_t* DStatePtr) FSEv05 : Finite State Entropy coder Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1538,7 +1486,7 @@ size_t FSEv05_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t header file Copyright (C) 2013-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1611,7 +1559,7 @@ const char* HUFv05_getErrorName(size_t code); /* provides error code string (u header file, for static linking only Copyright (C) 2013-2016, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1703,7 +1651,7 @@ size_t HUFv05_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void Huff0 : Huffman coder, part of New Generation Entropy library Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2548,7 +2496,7 @@ size_t HUFv05_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS zstd - standard compression library Copyright (C) 2014-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2646,7 +2594,7 @@ struct ZSTDv05_DCtx_s FSEv05_DTable LLTable[FSEv05_DTABLE_SIZE_U32(LLFSEv05Log)]; FSEv05_DTable OffTable[FSEv05_DTABLE_SIZE_U32(OffFSEv05Log)]; FSEv05_DTable MLTable[FSEv05_DTABLE_SIZE_U32(MLFSEv05Log)]; - unsigned hufTableX4[HUFv05_DTABLE_SIZE(HufLog)]; + unsigned hufTableX4[HUFv05_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; const void* previousDstEnd; const void* base; const void* vBase; @@ -2674,7 +2622,7 @@ size_t ZSTDv05_decompressBegin(ZSTDv05_DCtx* dctx) dctx->base = NULL; dctx->vBase = NULL; dctx->dictEnd = NULL; - dctx->hufTableX4[0] = HufLog; + dctx->hufTableX4[0] = ZSTD_HUFFDTABLE_CAPACITY_LOG; dctx->flagStaticTables = 0; return 0; } @@ -3235,13 +3183,19 @@ static size_t ZSTDv05_execSequence(BYTE* op, const BYTE* const litEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; - /* check */ - if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ + /* checks */ + size_t const seqLength = sequence.litLength + sequence.matchLength; + + if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall); + if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected); + /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */ + if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); + if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ - if (litEnd > litLimit) return ERROR(corruption_detected); /* risk read beyond lit buffer */ + if (litEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */ /* copy Literals */ - ZSTDv05_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ + ZSTDv05_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ op = oLitEnd; *litPtr = litEnd; /* update for next sequence */ @@ -3747,7 +3701,7 @@ size_t ZSTDv05_decompressBegin_usingDict(ZSTDv05_DCtx* dctx, const void* dict, s Buffered version of Zstd compression library Copyright (C) 2015-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are diff --git a/vendor/github.com/DataDog/zstd/zstd_v05.h b/vendor/github.com/DataDog/zstd/zstd_v05.h index b7c8bed..ebb7831 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v05.h +++ b/vendor/github.com/DataDog/zstd/zstd_v05.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_v06.c b/vendor/github.com/DataDog/zstd/zstd_v06.c index a7e85d9..0fd4225 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v06.c +++ b/vendor/github.com/DataDog/zstd/zstd_v06.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -24,7 +24,7 @@ low-level memory access routines Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -109,24 +109,6 @@ extern "C" { /*-************************************************************** * Memory I/O *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) -# define MEM_FORCE_MEMORY_ACCESS 1 -# endif -#endif MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } @@ -137,33 +119,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void) return one.c[0]; } -#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) - -/* violates C standard, by lying on structure alignment. -Only use if no other choice to achieve best performance on target platform */ -MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } -MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } -MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } - -#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; - -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } - -#else - -/* default method, safe and standard. - can sometimes prove slower */ - MEM_STATIC U16 MEM_read16(const void* memPtr) { U16 val; memcpy(&val, memPtr, sizeof(val)); return val; @@ -184,9 +139,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value) memcpy(memPtr, &value, sizeof(value)); } - -#endif /* MEM_FORCE_MEMORY_ACCESS */ - MEM_STATIC U32 MEM_swap32(U32 in) { #if defined(_MSC_VER) /* Visual Studio */ @@ -282,7 +234,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr) Header File for static linking only Copyright (C) 2014-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -306,7 +258,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr) OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd homepage : http://www.zstd.net + - zstd homepage : https://facebook.github.io/zstd */ #ifndef ZSTDv06_STATIC_H #define ZSTDv06_STATIC_H @@ -413,7 +365,7 @@ ZSTDLIBv06_API size_t ZSTDv06_decompressBlock(ZSTDv06_DCtx* dctx, void* dst, siz Header File for include Copyright (C) 2014-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -480,7 +432,7 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ -#define HufLog 12 +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 #define IS_HUF 0 #define IS_PCH 1 @@ -620,7 +572,7 @@ void ZSTDv06_seqToCodes(const seqStore_t* seqStorePtr, size_t const nbSeq); Public Prototypes declaration Copyright (C) 2013-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -768,7 +720,7 @@ If there is an error, the function will return an error code, which can be teste header file (to include) Copyright (C) 2013-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -929,7 +881,7 @@ MEM_STATIC size_t BITv06_initDStream(BITv06_DStream_t* bitD, const void* srcBuff } /*! BITv06_lookBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BITv06_lookBitsFast(const BITv06_DStream_t* bitD, U32 nbBits) { U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; @@ -949,7 +901,7 @@ MEM_STATIC size_t BITv06_readBits(BITv06_DStream_t* bitD, U32 nbBits) } /*! BITv06_readBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BITv06_readBitsFast(BITv06_DStream_t* bitD, U32 nbBits) { size_t const value = BITv06_lookBitsFast(bitD, nbBits); @@ -1003,7 +955,7 @@ MEM_STATIC unsigned BITv06_endOfDStream(const BITv06_DStream_t* DStream) header file for static linking (only) Copyright (C) 2013-2015, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1211,7 +1163,7 @@ MEM_STATIC BYTE FSEv06_decodeSymbolFast(FSEv06_DState_t* DStatePtr, BITv06_DStre Common functions of New Generation Entropy library Copyright (C) 2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1356,7 +1308,7 @@ size_t FSEv06_readNCount (short* normalizedCounter, unsigned* maxSVPtr, unsigned FSE : Finite State Entropy decoder Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1680,7 +1632,7 @@ size_t FSEv06_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t header file Copyright (C) 2013-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1750,7 +1702,7 @@ size_t HUFv06_compressBound(size_t size); /**< maximum compressed size */ header file, for static linking only Copyright (C) 2013-2016, Yann Collet - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1932,7 +1884,7 @@ MEM_STATIC size_t HUFv06_readStats(BYTE* huffWeight, size_t hwSize, U32* rankSta Huffman decoder, part of New Generation Entropy library Copyright (C) 2013-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2677,7 +2629,7 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS Common functions of Zstd compression library Copyright (C) 2015-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2701,7 +2653,7 @@ size_t HUFv06_decompress (void* dst, size_t dstSize, const void* cSrc, size_t cS OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd homepage : http://www.zstd.net/ + - zstd homepage : https://facebook.github.io/zstd/ */ @@ -2731,7 +2683,7 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er zstd - standard compression library Copyright (C) 2014-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2755,7 +2707,7 @@ const char* ZBUFFv06_getErrorName(size_t errorCode) { return ERR_getErrorName(er OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd homepage : http://www.zstd.net + - zstd homepage : https://facebook.github.io/zstd */ /* *************************************************************** @@ -2807,7 +2759,7 @@ struct ZSTDv06_DCtx_s FSEv06_DTable LLTable[FSEv06_DTABLE_SIZE_U32(LLFSELog)]; FSEv06_DTable OffTable[FSEv06_DTABLE_SIZE_U32(OffFSELog)]; FSEv06_DTable MLTable[FSEv06_DTABLE_SIZE_U32(MLFSELog)]; - unsigned hufTableX4[HUFv06_DTABLE_SIZE(HufLog)]; + unsigned hufTableX4[HUFv06_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; const void* previousDstEnd; const void* base; const void* vBase; @@ -2835,7 +2787,7 @@ size_t ZSTDv06_decompressBegin(ZSTDv06_DCtx* dctx) dctx->base = NULL; dctx->vBase = NULL; dctx->dictEnd = NULL; - dctx->hufTableX4[0] = HufLog; + dctx->hufTableX4[0] = ZSTD_HUFFDTABLE_CAPACITY_LOG; dctx->flagRepeatTable = 0; return 0; } @@ -3371,13 +3323,19 @@ static size_t ZSTDv06_execSequence(BYTE* op, const BYTE* const iLitEnd = *litPtr + sequence.litLength; const BYTE* match = oLitEnd - sequence.offset; - /* check */ - if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of 8 from oend */ + /* checks */ + size_t const seqLength = sequence.litLength + sequence.matchLength; + + if (seqLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall); + if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected); + /* Now we know there are no overflow in literal nor match lengths, can use pointer checks */ + if (oLitEnd > oend_8) return ERROR(dstSize_tooSmall); + if (oMatchEnd > oend) return ERROR(dstSize_tooSmall); /* overwrite beyond dst buffer */ - if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + if (iLitEnd > litLimit) return ERROR(corruption_detected); /* overRead beyond lit buffer */ /* copy Literals */ - ZSTDv06_wildcopy(op, *litPtr, sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ + ZSTDv06_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : oLitEnd <= oend-8 : no risk of overwrite beyond oend */ op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ @@ -3890,7 +3848,7 @@ size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, s Buffered version of Zstd compression library Copyright (C) 2015-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -3914,7 +3872,7 @@ size_t ZSTDv06_decompressBegin_usingDict(ZSTDv06_DCtx* dctx, const void* dict, s OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd homepage : http://www.zstd.net/ + - zstd homepage : https://facebook.github.io/zstd/ */ @@ -4036,7 +3994,8 @@ size_t ZBUFFv06_decompressContinue(ZBUFFv06_DCtx* zbd, size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ if (ZSTDv06_isError(hSize)) return hSize; if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ - memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); + if (ip != NULL) + memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); zbd->lhSize += iend-ip; *dstCapacityPtr = 0; return (hSize - zbd->lhSize) + ZSTDv06_blockHeaderSize; /* remaining header bytes + next block header */ diff --git a/vendor/github.com/DataDog/zstd/zstd_v06.h b/vendor/github.com/DataDog/zstd/zstd_v06.h index 2f81817..f511b10 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v06.h +++ b/vendor/github.com/DataDog/zstd/zstd_v06.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstd_v07.c b/vendor/github.com/DataDog/zstd/zstd_v07.c index c7bb95a..60d77e0 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v07.c +++ b/vendor/github.com/DataDog/zstd/zstd_v07.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -185,7 +185,7 @@ ZSTDLIBv07_API size_t ZSTDv07_insertBlock(ZSTDv07_DCtx* dctx, const void* blockS low-level memory access routines Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -269,24 +269,6 @@ extern "C" { /*-************************************************************** * Memory I/O *****************************************************************/ -/* MEM_FORCE_MEMORY_ACCESS : - * By default, access to unaligned memory is controlled by `memcpy()`, which is safe and portable. - * Unfortunately, on some target/compiler combinations, the generated assembly is sub-optimal. - * The below switch allow to select different access method for improved performance. - * Method 0 (default) : use `memcpy()`. Safe and portable. - * Method 1 : `__packed` statement. It depends on compiler extension (ie, not portable). - * This method is safe if your compiler supports it, and *generally* as fast or faster than `memcpy`. - * Method 2 : direct access. This method is portable but violate C standard. - * It can generate buggy code on targets depending on alignment. - * In some circumstances, it's the only known way to get the most performance (ie GCC + ARMv6) - * See http://fastcompression.blogspot.fr/2015/08/accessing-unaligned-memory.html for details. - * Prefer these methods in priority order (0 > 1 > 2) - */ -#ifndef MEM_FORCE_MEMORY_ACCESS /* can be defined externally, on command line for example */ -# if defined(__INTEL_COMPILER) || defined(__GNUC__) || defined(__ICCARM__) -# define MEM_FORCE_MEMORY_ACCESS 1 -# endif -#endif MEM_STATIC unsigned MEM_32bits(void) { return sizeof(size_t)==4; } MEM_STATIC unsigned MEM_64bits(void) { return sizeof(size_t)==8; } @@ -297,33 +279,6 @@ MEM_STATIC unsigned MEM_isLittleEndian(void) return one.c[0]; } -#if defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==2) - -/* violates C standard, by lying on structure alignment. -Only use if no other choice to achieve best performance on target platform */ -MEM_STATIC U16 MEM_read16(const void* memPtr) { return *(const U16*) memPtr; } -MEM_STATIC U32 MEM_read32(const void* memPtr) { return *(const U32*) memPtr; } -MEM_STATIC U64 MEM_read64(const void* memPtr) { return *(const U64*) memPtr; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { *(U16*)memPtr = value; } - -#elif defined(MEM_FORCE_MEMORY_ACCESS) && (MEM_FORCE_MEMORY_ACCESS==1) - -/* __pack instructions are safer, but compiler specific, hence potentially problematic for some compilers */ -/* currently only defined for gcc and icc */ -typedef union { U16 u16; U32 u32; U64 u64; size_t st; } __attribute__((packed)) unalign; - -MEM_STATIC U16 MEM_read16(const void* ptr) { return ((const unalign*)ptr)->u16; } -MEM_STATIC U32 MEM_read32(const void* ptr) { return ((const unalign*)ptr)->u32; } -MEM_STATIC U64 MEM_read64(const void* ptr) { return ((const unalign*)ptr)->u64; } - -MEM_STATIC void MEM_write16(void* memPtr, U16 value) { ((unalign*)memPtr)->u16 = value; } - -#else - -/* default method, safe and standard. - can sometimes prove slower */ - MEM_STATIC U16 MEM_read16(const void* memPtr) { U16 val; memcpy(&val, memPtr, sizeof(val)); return val; @@ -344,8 +299,6 @@ MEM_STATIC void MEM_write16(void* memPtr, U16 value) memcpy(memPtr, &value, sizeof(value)); } -#endif /* MEM_FORCE_MEMORY_ACCESS */ - MEM_STATIC U32 MEM_swap32(U32 in) { #if defined(_MSC_VER) /* Visual Studio */ @@ -440,7 +393,7 @@ MEM_STATIC size_t MEM_readLEST(const void* memPtr) header file (to include) Copyright (C) 2013-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -597,7 +550,7 @@ MEM_STATIC size_t BITv07_initDStream(BITv07_DStream_t* bitD, const void* srcBuff } /*! BITv07_lookBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BITv07_lookBitsFast(const BITv07_DStream_t* bitD, U32 nbBits) { U32 const bitMask = sizeof(bitD->bitContainer)*8 - 1; @@ -617,7 +570,7 @@ MEM_STATIC size_t BITv07_readBits(BITv07_DStream_t* bitD, U32 nbBits) } /*! BITv07_readBitsFast() : -* unsafe version; only works only if nbBits >= 1 */ +* unsafe version; only works if nbBits >= 1 */ MEM_STATIC size_t BITv07_readBitsFast(BITv07_DStream_t* bitD, U32 nbBits) { size_t const value = BITv07_lookBitsFast(bitD, nbBits); @@ -671,7 +624,7 @@ MEM_STATIC unsigned BITv07_endOfDStream(const BITv07_DStream_t* DStream) Public Prototypes declaration Copyright (C) 2013-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -979,7 +932,7 @@ MEM_STATIC BYTE FSEv07_decodeSymbolFast(FSEv07_DState_t* DStatePtr, BITv07_DStre header file Copyright (C) 2013-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1152,7 +1105,7 @@ size_t HUFv07_decompress1X4_usingDTable(void* dst, size_t maxDstSize, const void Common functions of New Generation Entropy library Copyright (C) 2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1376,7 +1329,7 @@ size_t HUFv07_readStats(BYTE* huffWeight, size_t hwSize, U32* rankStats, FSE : Finite State Entropy decoder Copyright (C) 2013-2015, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -1700,7 +1653,7 @@ size_t FSEv07_decompress(void* dst, size_t maxDstSize, const void* cSrc, size_t Huffman decoder, part of New Generation Entropy library Copyright (C) 2013-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2578,7 +2531,7 @@ size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, Common functions of Zstd compression library Copyright (C) 2015-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2602,7 +2555,7 @@ size_t HUFv07_decompress1X_DCtx (HUFv07_DTable* dctx, void* dst, size_t dstSize, OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd homepage : http://www.zstd.net/ + - zstd homepage : https://facebook.github.io/zstd/ */ @@ -2648,7 +2601,7 @@ static void ZSTDv07_defaultFreeFunction(void* opaque, void* address) Header File for include Copyright (C) 2014-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2718,7 +2671,7 @@ typedef enum { bt_compressed, bt_raw, bt_rle, bt_end } blockType_t; #define MIN_SEQUENCES_SIZE 1 /* nbSeq==0 */ #define MIN_CBLOCK_SIZE (1 /*litCSize*/ + 1 /* RLE or RAW */ + MIN_SEQUENCES_SIZE /* nbSeq==0 */) /* for a non-null block */ -#define HufLog 12 +#define ZSTD_HUFFDTABLE_CAPACITY_LOG 12 typedef enum { lbt_huffman, lbt_repeat, lbt_raw, lbt_rle } litBlockType_t; #define LONGNBSEQ 0x7F00 @@ -2855,7 +2808,7 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction zstd - standard compression library Copyright (C) 2014-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -2879,7 +2832,7 @@ static const ZSTDv07_customMem defaultCustomMem = { ZSTDv07_defaultAllocFunction OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd homepage : http://www.zstd.net + - zstd homepage : https://facebook.github.io/zstd */ /* *************************************************************** @@ -2932,7 +2885,7 @@ struct ZSTDv07_DCtx_s FSEv07_DTable LLTable[FSEv07_DTABLE_SIZE_U32(LLFSELog)]; FSEv07_DTable OffTable[FSEv07_DTABLE_SIZE_U32(OffFSELog)]; FSEv07_DTable MLTable[FSEv07_DTABLE_SIZE_U32(MLFSELog)]; - HUFv07_DTable hufTable[HUFv07_DTABLE_SIZE(HufLog)]; /* can accommodate HUFv07_decompress4X */ + HUFv07_DTable hufTable[HUFv07_DTABLE_SIZE(ZSTD_HUFFDTABLE_CAPACITY_LOG)]; /* can accommodate HUFv07_decompress4X */ const void* previousDstEnd; const void* base; const void* vBase; @@ -2968,7 +2921,7 @@ size_t ZSTDv07_decompressBegin(ZSTDv07_DCtx* dctx) dctx->base = NULL; dctx->vBase = NULL; dctx->dictEnd = NULL; - dctx->hufTable[0] = (HUFv07_DTable)((HufLog)*0x1000001); + dctx->hufTable[0] = (HUFv07_DTable)((ZSTD_HUFFDTABLE_CAPACITY_LOG)*0x1000001); dctx->litEntropy = dctx->fseEntropy = 0; dctx->dictID = 0; { int i; for (i=0; irep[i] = repStartValue[i]; } @@ -3600,11 +3553,14 @@ size_t ZSTDv07_execSequence(BYTE* op, const BYTE* match = oLitEnd - sequence.offset; /* check */ - if ((oLitEnd>oend_w) | (oMatchEnd>oend)) return ERROR(dstSize_tooSmall); /* last match must start at a minimum distance of WILDCOPY_OVERLENGTH from oend */ - if (iLitEnd > litLimit) return ERROR(corruption_detected); /* over-read beyond lit buffer */ + assert(oend >= op); + if (sequence.litLength + WILDCOPY_OVERLENGTH > (size_t)(oend - op)) return ERROR(dstSize_tooSmall); + if (sequenceLength > (size_t)(oend - op)) return ERROR(dstSize_tooSmall); + assert(litLimit >= *litPtr); + if (sequence.litLength > (size_t)(litLimit - *litPtr)) return ERROR(corruption_detected);; /* copy Literals */ - ZSTDv07_wildcopy(op, *litPtr, sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ + ZSTDv07_wildcopy(op, *litPtr, (ptrdiff_t)sequence.litLength); /* note : since oLitEnd <= oend-WILDCOPY_OVERLENGTH, no risk of overwrite beyond oend */ op = oLitEnd; *litPtr = iLitEnd; /* update for next sequence */ @@ -3618,7 +3574,7 @@ size_t ZSTDv07_execSequence(BYTE* op, return sequenceLength; } /* span extDict & currentPrefixSegment */ - { size_t const length1 = dictEnd - match; + { size_t const length1 = (size_t)(dictEnd - match); memmove(oLitEnd, match, length1); op = oLitEnd + length1; sequence.matchLength -= length1; @@ -4254,7 +4210,7 @@ ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, Buffered version of Zstd compression library Copyright (C) 2015-2016, Yann Collet. - BSD 2-Clause License (http://www.opensource.org/licenses/bsd-license.php) + BSD 2-Clause License (https://opensource.org/licenses/bsd-license.php) Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are @@ -4278,7 +4234,7 @@ ZSTDLIBv07_API size_t ZSTDv07_decompress_usingDDict(ZSTDv07_DCtx* dctx, OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. You can contact the author at : - - zstd homepage : http://www.zstd.net/ + - zstd homepage : https://facebook.github.io/zstd/ */ @@ -4418,7 +4374,8 @@ size_t ZBUFFv07_decompressContinue(ZBUFFv07_DCtx* zbd, if (hSize != 0) { size_t const toLoad = hSize - zbd->lhSize; /* if hSize!=0, hSize > zbd->lhSize */ if (toLoad > (size_t)(iend-ip)) { /* not enough input to load full header */ - memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); + if (ip != NULL) + memcpy(zbd->headerBuffer + zbd->lhSize, ip, iend-ip); zbd->lhSize += iend-ip; *dstCapacityPtr = 0; return (hSize - zbd->lhSize) + ZSTDv07_blockHeaderSize; /* remaining header bytes + next block header */ diff --git a/vendor/github.com/DataDog/zstd/zstd_v07.h b/vendor/github.com/DataDog/zstd/zstd_v07.h index 986ab65..385b10c 100644 --- a/vendor/github.com/DataDog/zstd/zstd_v07.h +++ b/vendor/github.com/DataDog/zstd/zstd_v07.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Yann Collet, Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/DataDog/zstd/zstdmt_compress.c b/vendor/github.com/DataDog/zstd/zstdmt_compress.c index 290892c..215cf8b 100644 --- a/vendor/github.com/DataDog/zstd/zstdmt_compress.c +++ b/vendor/github.com/DataDog/zstd/zstdmt_compress.c @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the @@ -21,6 +21,7 @@ /* ====== Dependencies ====== */ +#include "allocations.h" /* ZSTD_customMalloc, ZSTD_customCalloc, ZSTD_customFree */ #include "zstd_deps.h" /* ZSTD_memcpy, ZSTD_memset, INT_MAX, UINT_MAX */ #include "mem.h" /* MEM_STATIC */ #include "pool.h" /* threadpool */ @@ -267,11 +268,11 @@ static void ZSTDMT_releaseBuffer(ZSTDMT_bufferPool* bufPool, buffer_t buf) * 1 buffer for input loading * 1 buffer for "next input" when submitting current one * 1 buffer stuck in queue */ -#define BUF_POOL_MAX_NB_BUFFERS(nbWorkers) 2*nbWorkers + 3 +#define BUF_POOL_MAX_NB_BUFFERS(nbWorkers) (2*(nbWorkers) + 3) /* After a worker releases its rawSeqStore, it is immediately ready for reuse. * So we only need one seq buffer per worker. */ -#define SEQ_POOL_MAX_NB_BUFFERS(nbWorkers) nbWorkers +#define SEQ_POOL_MAX_NB_BUFFERS(nbWorkers) (nbWorkers) /* ===== Seq Pool Wrapper ====== */ @@ -720,7 +721,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) ZSTDMT_serialState_update(job->serial, cctx, rawSeqStore, job->src, job->jobID); if (!job->firstJob) { /* flush and overwrite frame header when it's not first job */ - size_t const hSize = ZSTD_compressContinue(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); + size_t const hSize = ZSTD_compressContinue_public(cctx, dstBuff.start, dstBuff.capacity, job->src.start, 0); if (ZSTD_isError(hSize)) JOB_ERROR(hSize); DEBUGLOG(5, "ZSTDMT_compressionJob: flush and overwrite %u bytes of frame header (not first job)", (U32)hSize); ZSTD_invalidateRepCodes(cctx); @@ -738,7 +739,7 @@ static void ZSTDMT_compressionJob(void* jobDescription) DEBUGLOG(5, "ZSTDMT_compressionJob: compress %u bytes in %i blocks", (U32)job->src.size, nbChunks); assert(job->cSize == 0); for (chunkNb = 1; chunkNb < nbChunks; chunkNb++) { - size_t const cSize = ZSTD_compressContinue(cctx, op, oend-op, ip, chunkSize); + size_t const cSize = ZSTD_compressContinue_public(cctx, op, oend-op, ip, chunkSize); if (ZSTD_isError(cSize)) JOB_ERROR(cSize); ip += chunkSize; op += cSize; assert(op < oend); @@ -758,8 +759,8 @@ static void ZSTDMT_compressionJob(void* jobDescription) size_t const lastBlockSize1 = job->src.size & (chunkSize-1); size_t const lastBlockSize = ((lastBlockSize1==0) & (job->src.size>=chunkSize)) ? chunkSize : lastBlockSize1; size_t const cSize = (job->lastJob) ? - ZSTD_compressEnd (cctx, op, oend-op, ip, lastBlockSize) : - ZSTD_compressContinue(cctx, op, oend-op, ip, lastBlockSize); + ZSTD_compressEnd_public(cctx, op, oend-op, ip, lastBlockSize) : + ZSTD_compressContinue_public(cctx, op, oend-op, ip, lastBlockSize); if (ZSTD_isError(cSize)) JOB_ERROR(cSize); lastCBlockSize = cSize; } } @@ -1735,7 +1736,7 @@ findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) } } else { /* We have enough bytes buffered to initialize the hash, - * and are have processed enough bytes to find a sync point. + * and have processed enough bytes to find a sync point. * Start scanning at the beginning of the input. */ assert(mtctx->inBuff.filled >= RSYNC_MIN_BLOCK_SIZE); @@ -1762,17 +1763,24 @@ findSynchronizationPoint(ZSTDMT_CCtx const* mtctx, ZSTD_inBuffer const input) * then a block will be emitted anyways, but this is okay, since if we * are already synchronized we will remain synchronized. */ + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); for (; pos < syncPoint.toLoad; ++pos) { BYTE const toRemove = pos < RSYNC_LENGTH ? prev[pos] : istart[pos - RSYNC_LENGTH]; - assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + /* This assert is very expensive, and Debian compiles with asserts enabled. + * So disable it for now. We can get similar coverage by checking it at the + * beginning & end of the loop. + * assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); + */ hash = ZSTD_rollingHash_rotate(hash, toRemove, istart[pos], primePower); assert(mtctx->inBuff.filled + pos >= RSYNC_MIN_BLOCK_SIZE); if ((hash & hitMask) == hitMask) { syncPoint.toLoad = pos + 1; syncPoint.flush = 1; + ++pos; /* for assert */ break; } } + assert(pos < RSYNC_LENGTH || ZSTD_rollingHash_compute(istart + pos - RSYNC_LENGTH, RSYNC_LENGTH) == hash); return syncPoint; } diff --git a/vendor/github.com/DataDog/zstd/zstdmt_compress.h b/vendor/github.com/DataDog/zstd/zstdmt_compress.h index 325e2af..3485e0d 100644 --- a/vendor/github.com/DataDog/zstd/zstdmt_compress.h +++ b/vendor/github.com/DataDog/zstd/zstdmt_compress.h @@ -1,6 +1,6 @@ #ifndef USE_EXTERNAL_ZSTD /* - * Copyright (c) Yann Collet, Facebook, Inc. + * Copyright (c) Meta Platforms, Inc. and affiliates. * All rights reserved. * * This source code is licensed under both the BSD-style license (found in the diff --git a/vendor/github.com/cespare/xxhash/v2/README.md b/vendor/github.com/cespare/xxhash/v2/README.md index 792b4a6..8bf0e5b 100644 --- a/vendor/github.com/cespare/xxhash/v2/README.md +++ b/vendor/github.com/cespare/xxhash/v2/README.md @@ -3,8 +3,7 @@ [![Go Reference](https://pkg.go.dev/badge/github.com/cespare/xxhash/v2.svg)](https://pkg.go.dev/github.com/cespare/xxhash/v2) [![Test](https://github.com/cespare/xxhash/actions/workflows/test.yml/badge.svg)](https://github.com/cespare/xxhash/actions/workflows/test.yml) -xxhash is a Go implementation of the 64-bit -[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a +xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a high-quality hashing algorithm that is much faster than anything in the Go standard library. @@ -25,8 +24,11 @@ func (*Digest) WriteString(string) (int, error) func (*Digest) Sum64() uint64 ``` -This implementation provides a fast pure-Go implementation and an even faster -assembly implementation for amd64. +The package is written with optimized pure Go and also contains even faster +assembly implementations for amd64 and arm64. If desired, the `purego` build tag +opts into using the Go code even on those architectures. + +[xxHash]: http://cyan4973.github.io/xxHash/ ## Compatibility @@ -45,19 +47,20 @@ I recommend using the latest release of Go. Here are some quick benchmarks comparing the pure-Go and assembly implementations of Sum64. -| input size | purego | asm | -| --- | --- | --- | -| 5 B | 979.66 MB/s | 1291.17 MB/s | -| 100 B | 7475.26 MB/s | 7973.40 MB/s | -| 4 KB | 17573.46 MB/s | 17602.65 MB/s | -| 10 MB | 17131.46 MB/s | 17142.16 MB/s | +| input size | purego | asm | +| ---------- | --------- | --------- | +| 4 B | 1.3 GB/s | 1.2 GB/s | +| 16 B | 2.9 GB/s | 3.5 GB/s | +| 100 B | 6.9 GB/s | 8.1 GB/s | +| 4 KB | 11.7 GB/s | 16.7 GB/s | +| 10 MB | 12.0 GB/s | 17.3 GB/s | -These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using -the following commands under Go 1.11.2: +These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C +CPU using the following commands under Go 1.19.2: ``` -$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes' -$ go test -benchtime 10s -bench '/xxhash,direct,bytes' +benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$') +benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$') ``` ## Projects using this package diff --git a/vendor/github.com/cespare/xxhash/v2/testall.sh b/vendor/github.com/cespare/xxhash/v2/testall.sh new file mode 100644 index 0000000..94b9c44 --- /dev/null +++ b/vendor/github.com/cespare/xxhash/v2/testall.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -eu -o pipefail + +# Small convenience script for running the tests with various combinations of +# arch/tags. This assumes we're running on amd64 and have qemu available. + +go test ./... +go test -tags purego ./... +GOARCH=arm64 go test +GOARCH=arm64 go test -tags purego diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash.go b/vendor/github.com/cespare/xxhash/v2/xxhash.go index 15c835d..a9e0d45 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash.go +++ b/vendor/github.com/cespare/xxhash/v2/xxhash.go @@ -16,19 +16,11 @@ const ( prime5 uint64 = 2870177450012600261 ) -// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where -// possible in the Go code is worth a small (but measurable) performance boost -// by avoiding some MOVQs. Vars are needed for the asm and also are useful for -// convenience in the Go code in a few places where we need to intentionally -// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the -// result overflows a uint64). -var ( - prime1v = prime1 - prime2v = prime2 - prime3v = prime3 - prime4v = prime4 - prime5v = prime5 -) +// Store the primes in an array as well. +// +// The consts are used when possible in Go code to avoid MOVs but we need a +// contiguous array of the assembly code. +var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5} // Digest implements hash.Hash64. type Digest struct { @@ -50,10 +42,10 @@ func New() *Digest { // Reset clears the Digest's state so that it can be reused. func (d *Digest) Reset() { - d.v1 = prime1v + prime2 + d.v1 = primes[0] + prime2 d.v2 = prime2 d.v3 = 0 - d.v4 = -prime1v + d.v4 = -primes[0] d.total = 0 d.n = 0 } @@ -69,21 +61,23 @@ func (d *Digest) Write(b []byte) (n int, err error) { n = len(b) d.total += uint64(n) + memleft := d.mem[d.n&(len(d.mem)-1):] + if d.n+n < 32 { // This new data doesn't even fill the current block. - copy(d.mem[d.n:], b) + copy(memleft, b) d.n += n return } if d.n > 0 { // Finish off the partial block. - copy(d.mem[d.n:], b) + c := copy(memleft, b) d.v1 = round(d.v1, u64(d.mem[0:8])) d.v2 = round(d.v2, u64(d.mem[8:16])) d.v3 = round(d.v3, u64(d.mem[16:24])) d.v4 = round(d.v4, u64(d.mem[24:32])) - b = b[32-d.n:] + b = b[c:] d.n = 0 } @@ -133,21 +127,20 @@ func (d *Digest) Sum64() uint64 { h += d.total - i, end := 0, d.n - for ; i+8 <= end; i += 8 { - k1 := round(0, u64(d.mem[i:i+8])) + b := d.mem[:d.n&(len(d.mem)-1)] + for ; len(b) >= 8; b = b[8:] { + k1 := round(0, u64(b[:8])) h ^= k1 h = rol27(h)*prime1 + prime4 } - if i+4 <= end { - h ^= uint64(u32(d.mem[i:i+4])) * prime1 + if len(b) >= 4 { + h ^= uint64(u32(b[:4])) * prime1 h = rol23(h)*prime2 + prime3 - i += 4 + b = b[4:] } - for i < end { - h ^= uint64(d.mem[i]) * prime5 + for ; len(b) > 0; b = b[1:] { + h ^= uint64(b[0]) * prime5 h = rol11(h) * prime1 - i++ } h ^= h >> 33 diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.s b/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.s index be8db5b..3e8b132 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.s +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.s @@ -1,215 +1,209 @@ +//go:build !appengine && gc && !purego // +build !appengine // +build gc // +build !purego #include "textflag.h" -// Register allocation: -// AX h -// SI pointer to advance through b -// DX n -// BX loop end -// R8 v1, k1 -// R9 v2 -// R10 v3 -// R11 v4 -// R12 tmp -// R13 prime1v -// R14 prime2v -// DI prime4v +// Registers: +#define h AX +#define d AX +#define p SI // pointer to advance through b +#define n DX +#define end BX // loop end +#define v1 R8 +#define v2 R9 +#define v3 R10 +#define v4 R11 +#define x R12 +#define prime1 R13 +#define prime2 R14 +#define prime4 DI -// round reads from and advances the buffer pointer in SI. -// It assumes that R13 has prime1v and R14 has prime2v. -#define round(r) \ - MOVQ (SI), R12 \ - ADDQ $8, SI \ - IMULQ R14, R12 \ - ADDQ R12, r \ - ROLQ $31, r \ - IMULQ R13, r +#define round(acc, x) \ + IMULQ prime2, x \ + ADDQ x, acc \ + ROLQ $31, acc \ + IMULQ prime1, acc -// mergeRound applies a merge round on the two registers acc and val. -// It assumes that R13 has prime1v, R14 has prime2v, and DI has prime4v. -#define mergeRound(acc, val) \ - IMULQ R14, val \ - ROLQ $31, val \ - IMULQ R13, val \ - XORQ val, acc \ - IMULQ R13, acc \ - ADDQ DI, acc +// round0 performs the operation x = round(0, x). +#define round0(x) \ + IMULQ prime2, x \ + ROLQ $31, x \ + IMULQ prime1, x + +// mergeRound applies a merge round on the two registers acc and x. +// It assumes that prime1, prime2, and prime4 have been loaded. +#define mergeRound(acc, x) \ + round0(x) \ + XORQ x, acc \ + IMULQ prime1, acc \ + ADDQ prime4, acc + +// blockLoop processes as many 32-byte blocks as possible, +// updating v1, v2, v3, and v4. It assumes that there is at least one block +// to process. +#define blockLoop() \ +loop: \ + MOVQ +0(p), x \ + round(v1, x) \ + MOVQ +8(p), x \ + round(v2, x) \ + MOVQ +16(p), x \ + round(v3, x) \ + MOVQ +24(p), x \ + round(v4, x) \ + ADDQ $32, p \ + CMPQ p, end \ + JLE loop // func Sum64(b []byte) uint64 -TEXT ·Sum64(SB), NOSPLIT, $0-32 +TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32 // Load fixed primes. - MOVQ ·prime1v(SB), R13 - MOVQ ·prime2v(SB), R14 - MOVQ ·prime4v(SB), DI + MOVQ ·primes+0(SB), prime1 + MOVQ ·primes+8(SB), prime2 + MOVQ ·primes+24(SB), prime4 // Load slice. - MOVQ b_base+0(FP), SI - MOVQ b_len+8(FP), DX - LEAQ (SI)(DX*1), BX + MOVQ b_base+0(FP), p + MOVQ b_len+8(FP), n + LEAQ (p)(n*1), end // The first loop limit will be len(b)-32. - SUBQ $32, BX + SUBQ $32, end // Check whether we have at least one block. - CMPQ DX, $32 + CMPQ n, $32 JLT noBlocks // Set up initial state (v1, v2, v3, v4). - MOVQ R13, R8 - ADDQ R14, R8 - MOVQ R14, R9 - XORQ R10, R10 - XORQ R11, R11 - SUBQ R13, R11 + MOVQ prime1, v1 + ADDQ prime2, v1 + MOVQ prime2, v2 + XORQ v3, v3 + XORQ v4, v4 + SUBQ prime1, v4 - // Loop until SI > BX. -blockLoop: - round(R8) - round(R9) - round(R10) - round(R11) + blockLoop() - CMPQ SI, BX - JLE blockLoop + MOVQ v1, h + ROLQ $1, h + MOVQ v2, x + ROLQ $7, x + ADDQ x, h + MOVQ v3, x + ROLQ $12, x + ADDQ x, h + MOVQ v4, x + ROLQ $18, x + ADDQ x, h - MOVQ R8, AX - ROLQ $1, AX - MOVQ R9, R12 - ROLQ $7, R12 - ADDQ R12, AX - MOVQ R10, R12 - ROLQ $12, R12 - ADDQ R12, AX - MOVQ R11, R12 - ROLQ $18, R12 - ADDQ R12, AX - - mergeRound(AX, R8) - mergeRound(AX, R9) - mergeRound(AX, R10) - mergeRound(AX, R11) + mergeRound(h, v1) + mergeRound(h, v2) + mergeRound(h, v3) + mergeRound(h, v4) JMP afterBlocks noBlocks: - MOVQ ·prime5v(SB), AX + MOVQ ·primes+32(SB), h afterBlocks: - ADDQ DX, AX + ADDQ n, h - // Right now BX has len(b)-32, and we want to loop until SI > len(b)-8. - ADDQ $24, BX + ADDQ $24, end + CMPQ p, end + JG try4 - CMPQ SI, BX - JG fourByte +loop8: + MOVQ (p), x + ADDQ $8, p + round0(x) + XORQ x, h + ROLQ $27, h + IMULQ prime1, h + ADDQ prime4, h -wordLoop: - // Calculate k1. - MOVQ (SI), R8 - ADDQ $8, SI - IMULQ R14, R8 - ROLQ $31, R8 - IMULQ R13, R8 + CMPQ p, end + JLE loop8 - XORQ R8, AX - ROLQ $27, AX - IMULQ R13, AX - ADDQ DI, AX +try4: + ADDQ $4, end + CMPQ p, end + JG try1 - CMPQ SI, BX - JLE wordLoop + MOVL (p), x + ADDQ $4, p + IMULQ prime1, x + XORQ x, h -fourByte: - ADDQ $4, BX - CMPQ SI, BX - JG singles + ROLQ $23, h + IMULQ prime2, h + ADDQ ·primes+16(SB), h - MOVL (SI), R8 - ADDQ $4, SI - IMULQ R13, R8 - XORQ R8, AX - - ROLQ $23, AX - IMULQ R14, AX - ADDQ ·prime3v(SB), AX - -singles: - ADDQ $4, BX - CMPQ SI, BX +try1: + ADDQ $4, end + CMPQ p, end JGE finalize -singlesLoop: - MOVBQZX (SI), R12 - ADDQ $1, SI - IMULQ ·prime5v(SB), R12 - XORQ R12, AX +loop1: + MOVBQZX (p), x + ADDQ $1, p + IMULQ ·primes+32(SB), x + XORQ x, h + ROLQ $11, h + IMULQ prime1, h - ROLQ $11, AX - IMULQ R13, AX - - CMPQ SI, BX - JL singlesLoop + CMPQ p, end + JL loop1 finalize: - MOVQ AX, R12 - SHRQ $33, R12 - XORQ R12, AX - IMULQ R14, AX - MOVQ AX, R12 - SHRQ $29, R12 - XORQ R12, AX - IMULQ ·prime3v(SB), AX - MOVQ AX, R12 - SHRQ $32, R12 - XORQ R12, AX + MOVQ h, x + SHRQ $33, x + XORQ x, h + IMULQ prime2, h + MOVQ h, x + SHRQ $29, x + XORQ x, h + IMULQ ·primes+16(SB), h + MOVQ h, x + SHRQ $32, x + XORQ x, h - MOVQ AX, ret+24(FP) + MOVQ h, ret+24(FP) RET -// writeBlocks uses the same registers as above except that it uses AX to store -// the d pointer. - // func writeBlocks(d *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT, $0-40 +TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 // Load fixed primes needed for round. - MOVQ ·prime1v(SB), R13 - MOVQ ·prime2v(SB), R14 + MOVQ ·primes+0(SB), prime1 + MOVQ ·primes+8(SB), prime2 // Load slice. - MOVQ b_base+8(FP), SI - MOVQ b_len+16(FP), DX - LEAQ (SI)(DX*1), BX - SUBQ $32, BX + MOVQ b_base+8(FP), p + MOVQ b_len+16(FP), n + LEAQ (p)(n*1), end + SUBQ $32, end // Load vN from d. - MOVQ d+0(FP), AX - MOVQ 0(AX), R8 // v1 - MOVQ 8(AX), R9 // v2 - MOVQ 16(AX), R10 // v3 - MOVQ 24(AX), R11 // v4 + MOVQ s+0(FP), d + MOVQ 0(d), v1 + MOVQ 8(d), v2 + MOVQ 16(d), v3 + MOVQ 24(d), v4 // We don't need to check the loop condition here; this function is // always called with at least one block of data to process. -blockLoop: - round(R8) - round(R9) - round(R10) - round(R11) - - CMPQ SI, BX - JLE blockLoop + blockLoop() // Copy vN back to d. - MOVQ R8, 0(AX) - MOVQ R9, 8(AX) - MOVQ R10, 16(AX) - MOVQ R11, 24(AX) + MOVQ v1, 0(d) + MOVQ v2, 8(d) + MOVQ v3, 16(d) + MOVQ v4, 24(d) - // The number of bytes written is SI minus the old base pointer. - SUBQ b_base+8(FP), SI - MOVQ SI, ret+32(FP) + // The number of bytes written is p minus the old base pointer. + SUBQ b_base+8(FP), p + MOVQ p, ret+32(FP) RET diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_arm64.s b/vendor/github.com/cespare/xxhash/v2/xxhash_arm64.s new file mode 100644 index 0000000..7e3145a --- /dev/null +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_arm64.s @@ -0,0 +1,183 @@ +//go:build !appengine && gc && !purego +// +build !appengine +// +build gc +// +build !purego + +#include "textflag.h" + +// Registers: +#define digest R1 +#define h R2 // return value +#define p R3 // input pointer +#define n R4 // input length +#define nblocks R5 // n / 32 +#define prime1 R7 +#define prime2 R8 +#define prime3 R9 +#define prime4 R10 +#define prime5 R11 +#define v1 R12 +#define v2 R13 +#define v3 R14 +#define v4 R15 +#define x1 R20 +#define x2 R21 +#define x3 R22 +#define x4 R23 + +#define round(acc, x) \ + MADD prime2, acc, x, acc \ + ROR $64-31, acc \ + MUL prime1, acc + +// round0 performs the operation x = round(0, x). +#define round0(x) \ + MUL prime2, x \ + ROR $64-31, x \ + MUL prime1, x + +#define mergeRound(acc, x) \ + round0(x) \ + EOR x, acc \ + MADD acc, prime4, prime1, acc + +// blockLoop processes as many 32-byte blocks as possible, +// updating v1, v2, v3, and v4. It assumes that n >= 32. +#define blockLoop() \ + LSR $5, n, nblocks \ + PCALIGN $16 \ + loop: \ + LDP.P 16(p), (x1, x2) \ + LDP.P 16(p), (x3, x4) \ + round(v1, x1) \ + round(v2, x2) \ + round(v3, x3) \ + round(v4, x4) \ + SUB $1, nblocks \ + CBNZ nblocks, loop + +// func Sum64(b []byte) uint64 +TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32 + LDP b_base+0(FP), (p, n) + + LDP ·primes+0(SB), (prime1, prime2) + LDP ·primes+16(SB), (prime3, prime4) + MOVD ·primes+32(SB), prime5 + + CMP $32, n + CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 } + BLT afterLoop + + ADD prime1, prime2, v1 + MOVD prime2, v2 + MOVD $0, v3 + NEG prime1, v4 + + blockLoop() + + ROR $64-1, v1, x1 + ROR $64-7, v2, x2 + ADD x1, x2 + ROR $64-12, v3, x3 + ROR $64-18, v4, x4 + ADD x3, x4 + ADD x2, x4, h + + mergeRound(h, v1) + mergeRound(h, v2) + mergeRound(h, v3) + mergeRound(h, v4) + +afterLoop: + ADD n, h + + TBZ $4, n, try8 + LDP.P 16(p), (x1, x2) + + round0(x1) + + // NOTE: here and below, sequencing the EOR after the ROR (using a + // rotated register) is worth a small but measurable speedup for small + // inputs. + ROR $64-27, h + EOR x1 @> 64-27, h, h + MADD h, prime4, prime1, h + + round0(x2) + ROR $64-27, h + EOR x2 @> 64-27, h, h + MADD h, prime4, prime1, h + +try8: + TBZ $3, n, try4 + MOVD.P 8(p), x1 + + round0(x1) + ROR $64-27, h + EOR x1 @> 64-27, h, h + MADD h, prime4, prime1, h + +try4: + TBZ $2, n, try2 + MOVWU.P 4(p), x2 + + MUL prime1, x2 + ROR $64-23, h + EOR x2 @> 64-23, h, h + MADD h, prime3, prime2, h + +try2: + TBZ $1, n, try1 + MOVHU.P 2(p), x3 + AND $255, x3, x1 + LSR $8, x3, x2 + + MUL prime5, x1 + ROR $64-11, h + EOR x1 @> 64-11, h, h + MUL prime1, h + + MUL prime5, x2 + ROR $64-11, h + EOR x2 @> 64-11, h, h + MUL prime1, h + +try1: + TBZ $0, n, finalize + MOVBU (p), x4 + + MUL prime5, x4 + ROR $64-11, h + EOR x4 @> 64-11, h, h + MUL prime1, h + +finalize: + EOR h >> 33, h + MUL prime2, h + EOR h >> 29, h + MUL prime3, h + EOR h >> 32, h + + MOVD h, ret+24(FP) + RET + +// func writeBlocks(d *Digest, b []byte) int +TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 + LDP ·primes+0(SB), (prime1, prime2) + + // Load state. Assume v[1-4] are stored contiguously. + MOVD d+0(FP), digest + LDP 0(digest), (v1, v2) + LDP 16(digest), (v3, v4) + + LDP b_base+8(FP), (p, n) + + blockLoop() + + // Store updated state. + STP (v1, v2), 0(digest) + STP (v3, v4), 16(digest) + + BIC $31, n + MOVD n, ret+32(FP) + RET diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.go b/vendor/github.com/cespare/xxhash/v2/xxhash_asm.go similarity index 73% rename from vendor/github.com/cespare/xxhash/v2/xxhash_amd64.go rename to vendor/github.com/cespare/xxhash/v2/xxhash_asm.go index ad14b80..9216e0a 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash_amd64.go +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_asm.go @@ -1,3 +1,5 @@ +//go:build (amd64 || arm64) && !appengine && gc && !purego +// +build amd64 arm64 // +build !appengine // +build gc // +build !purego diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_other.go b/vendor/github.com/cespare/xxhash/v2/xxhash_other.go index 4a5a821..26df13b 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash_other.go +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_other.go @@ -1,4 +1,5 @@ -// +build !amd64 appengine !gc purego +//go:build (!amd64 && !arm64) || appengine || !gc || purego +// +build !amd64,!arm64 appengine !gc purego package xxhash @@ -14,10 +15,10 @@ func Sum64(b []byte) uint64 { var h uint64 if n >= 32 { - v1 := prime1v + prime2 + v1 := primes[0] + prime2 v2 := prime2 v3 := uint64(0) - v4 := -prime1v + v4 := -primes[0] for len(b) >= 32 { v1 = round(v1, u64(b[0:8:len(b)])) v2 = round(v2, u64(b[8:16:len(b)])) @@ -36,19 +37,18 @@ func Sum64(b []byte) uint64 { h += uint64(n) - i, end := 0, len(b) - for ; i+8 <= end; i += 8 { - k1 := round(0, u64(b[i:i+8:len(b)])) + for ; len(b) >= 8; b = b[8:] { + k1 := round(0, u64(b[:8])) h ^= k1 h = rol27(h)*prime1 + prime4 } - if i+4 <= end { - h ^= uint64(u32(b[i:i+4:len(b)])) * prime1 + if len(b) >= 4 { + h ^= uint64(u32(b[:4])) * prime1 h = rol23(h)*prime2 + prime3 - i += 4 + b = b[4:] } - for ; i < end; i++ { - h ^= uint64(b[i]) * prime5 + for ; len(b) > 0; b = b[1:] { + h ^= uint64(b[0]) * prime5 h = rol11(h) * prime1 } diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_safe.go b/vendor/github.com/cespare/xxhash/v2/xxhash_safe.go index fc9bea7..e86f1b5 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash_safe.go +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_safe.go @@ -1,3 +1,4 @@ +//go:build appengine // +build appengine // This file contains the safe implementations of otherwise unsafe-using code. diff --git a/vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go b/vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go index 376e0ca..1c1638f 100644 --- a/vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go +++ b/vendor/github.com/cespare/xxhash/v2/xxhash_unsafe.go @@ -1,3 +1,4 @@ +//go:build !appengine // +build !appengine // This file encapsulates usage of unsafe. @@ -11,7 +12,7 @@ import ( // In the future it's possible that compiler optimizations will make these // XxxString functions unnecessary by realizing that calls such as -// Sum64([]byte(s)) don't need to copy s. See https://golang.org/issue/2205. +// Sum64([]byte(s)) don't need to copy s. See https://go.dev/issue/2205. // If that happens, even if we keep these functions they can be replaced with // the trivial safe code. diff --git a/vendor/github.com/cockroachdb/errors/.travis.yml b/vendor/github.com/cockroachdb/errors/.travis.yml deleted file mode 100644 index 0900075..0000000 --- a/vendor/github.com/cockroachdb/errors/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -language: go - -go: -- 1.13.x -- 1.14.x -- 1.15.x -- 1.16.x diff --git a/vendor/github.com/cockroachdb/errors/markers/markers.go b/vendor/github.com/cockroachdb/errors/markers/markers.go index f718d10..d270125 100644 --- a/vendor/github.com/cockroachdb/errors/markers/markers.go +++ b/vendor/github.com/cockroachdb/errors/markers/markers.go @@ -46,10 +46,12 @@ func Is(err, reference error) bool { return err == nil } + isComparable := reflect.TypeOf(reference).Comparable() + // Direct reference comparison is the fastest, and most // likely to be true, so do this first. for c := err; c != nil; c = errbase.UnwrapOnce(c) { - if c == reference { + if isComparable && c == reference { return true } // Compatibility with std go errors: if the error object itself @@ -141,10 +143,27 @@ func If(err error, pred func(err error) (interface{}, bool)) (interface{}, bool) // package location or a different type, ensure that // RegisterTypeMigration() was called prior to IsAny(). func IsAny(err error, references ...error) bool { - // First try using direct reference comparison. - for c := err; ; c = errbase.UnwrapOnce(c) { + if err == nil { for _, refErr := range references { - if c == refErr { + if refErr == nil { + return true + } + } + // The mark-based comparison below will never match anything if + // the error is nil, so don't bother with computing the marks in + // that case. This avoids the computational expense of computing + // the reference marks upfront. + return false + } + + // First try using direct reference comparison. + for c := err; c != nil; c = errbase.UnwrapOnce(c) { + for _, refErr := range references { + if refErr == nil { + continue + } + isComparable := reflect.TypeOf(refErr).Comparable() + if isComparable && c == refErr { return true } // Compatibility with std go errors: if the error object itself @@ -153,19 +172,6 @@ func IsAny(err error, references ...error) bool { return true } } - if c == nil { - // This special case is to support a comparison to a nil - // reference. - break - } - } - - if err == nil { - // The mark-based comparison below will never match anything if - // the error is nil, so don't bother with computing the marks in - // that case. This avoids the computational expense of computing - // the reference marks upfront. - return false } // Try harder with marks. diff --git a/vendor/github.com/cockroachdb/logtags/buffer.go b/vendor/github.com/cockroachdb/logtags/buffer.go index e984151..1365a48 100644 --- a/vendor/github.com/cockroachdb/logtags/buffer.go +++ b/vendor/github.com/cockroachdb/logtags/buffer.go @@ -38,6 +38,20 @@ func (b *Buffer) Get() []Tag { return b.tags } +// GetTag returns the tag corresponding to the given key. If the tag doesn't +// exist, the bool return value will be false. +func (b *Buffer) GetTag(key string) (Tag, bool) { + if b == nil { + return Tag{}, false + } + for _, t := range b.tags { + if t.Key() == key { + return t, true + } + } + return Tag{}, false +} + // Add returns a new buffer with one more tag. If the tag has the same key as an // earlier tag, that tag is overwritten. // The receiver can be nil. @@ -52,6 +66,25 @@ func (b *Buffer) Add(key string, value interface{}) *Buffer { return res } +// Remove returns a new Buffer with the tag with key `key` removed. If the tag +// does not exist, the receiver is returned (unchanged). The bool return value +// is true if the tag existed. +func (b *Buffer) Remove(key string) (*Buffer, bool) { + if b == nil { + return nil, false + } + for i, t := range b.tags { + if t.Key() == key { + res := &Buffer{} + res.init(len(b.tags)-1, 0 /* capacityHint */) + copy(res.tags, b.tags[:i]) + copy(res.tags[i:], b.tags[i+1:]) + return res, true + } + } + return b, false +} + // Merge returns a new buffer which contains tags from the receiver, followed by // the tags from . // @@ -121,12 +154,12 @@ func (b *Buffer) String() string { } // FormatToString emits the k/v pairs to a strings.Builder. -// - the k/v pairs are separated by commas (& no spaces). -// - if there is no value, only the key is printed. -// - if there is a value, and the key is just 1 character long, -// the key and the value are concatenated. -// This supports e.g. printing k="n", v=123 as "n123". -// - otherwise, it prints "k=v". +// - the k/v pairs are separated by commas (& no spaces). +// - if there is no value, only the key is printed. +// - if there is a value, and the key is just 1 character long, +// the key and the value are concatenated. +// This supports e.g. printing k="n", v=123 as "n123". +// - otherwise, it prints "k=v". func (b *Buffer) FormatToString(buf *strings.Builder) { comma := "" for _, t := range b.Get() { @@ -142,13 +175,19 @@ func (b *Buffer) FormatToString(buf *strings.Builder) { } } -func (b *Buffer) init(length, maxLenHint int) { +// init initializes b.tags to a slice of len `length` (filled with zero values). +// The slice may have capacity capacityHint. If capacityHint is less than +// length, it is ignored. +func (b *Buffer) init(length, capacityHint int) { if length <= staticSlots { - // Even if maxLenHint is larger that staticSlots, we still want to try to + // Even if capacityHint is larger that staticSlots, we still want to try to // avoid the allocation (especially since tags frequently get deduplicated). b.tags = b.prealloc[:length] } else { - b.tags = make([]Tag, length, maxLenHint) + if capacityHint < length { + capacityHint = length + } + b.tags = make([]Tag, length, capacityHint) } } diff --git a/vendor/github.com/cockroachdb/logtags/context.go b/vendor/github.com/cockroachdb/logtags/context.go index 240be91..61e2027 100644 --- a/vendor/github.com/cockroachdb/logtags/context.go +++ b/vendor/github.com/cockroachdb/logtags/context.go @@ -41,6 +41,18 @@ func AddTag(ctx context.Context, key string, value interface{}) context.Context return WithTags(ctx, b.Add(key, value)) } +// RemoveTag returns a context that has the tags in the given context except the +// tag with key `key`. If such a tag does not exist, the given context is +// returned. +func RemoveTag(ctx context.Context, key string) context.Context { + b := FromContext(ctx) + newB, ok := b.Remove(key) + if !ok { + return ctx + } + return WithTags(ctx, newB) +} + // AddTags returns a context that has the tags in the given context plus another // set of tags. Tags are deduplicated (see Buffer.AddTags). func AddTags(ctx context.Context, tags *Buffer) context.Context { diff --git a/vendor/github.com/cockroachdb/pebble/Makefile b/vendor/github.com/cockroachdb/pebble/Makefile index e6af673..2377f86 100644 --- a/vendor/github.com/cockroachdb/pebble/Makefile +++ b/vendor/github.com/cockroachdb/pebble/Makefile @@ -27,6 +27,21 @@ test: testrace: testflags += -race -timeout 20m testrace: test +testasan: testflags += -asan -timeout 20m +testasan: test + +testmsan: export CC=clang +testmsan: testflags += -msan -timeout 20m +testmsan: test + +.PHONY: testobjiotracing +testobjiotracing: + ${GO} test -tags '$(TAGS) pebble_obj_io_tracing' ${testflags} -run ${TESTS} ./objstorage/objstorageprovider/objiotracing + +.PHONY: lint +lint: + ${GO} test -tags '$(TAGS)' ${testflags} -run ${TESTS} ./internal/lint + .PHONY: stress stressrace stressrace: testflags += -race stress stressrace: testflags += -exec 'stress ${STRESSFLAGS}' -timeout 0 -test.v @@ -48,7 +63,7 @@ crossversion-meta: .PHONY: stress-crossversion stress-crossversion: - STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 master + STRESS=1 ./scripts/run-crossversion-meta.sh crl-release-21.2 crl-release-22.1 crl-release-22.2 crl-release-23.1 master .PHONY: generate generate: @@ -81,11 +96,11 @@ endif @${GO} mod tidy $(MAKE) git-clean-check +# TODO(radu): switch back to @latest once bogus doc changes are +# addressed; see https://github.com/cockroachdb/crlfmt/pull/44 .PHONY: format format: - for _file in $$(gofmt -s -l .); do \ - gofmt -s -w $$_file ; \ - done + go install github.com/cockroachdb/crlfmt@44a36ec7 && crlfmt -w -tab 2 . .PHONY: format-check format-check: diff --git a/vendor/github.com/cockroachdb/pebble/batch.go b/vendor/github.com/cockroachdb/pebble/batch.go index 6544dbe..ff84fe3 100644 --- a/vendor/github.com/cockroachdb/pebble/batch.go +++ b/vendor/github.com/cockroachdb/pebble/batch.go @@ -5,6 +5,7 @@ package pebble import ( + "context" "encoding/binary" "fmt" "io" @@ -12,6 +13,7 @@ import ( "sort" "sync" "sync/atomic" + "time" "unsafe" "github.com/cockroachdb/errors" @@ -82,7 +84,7 @@ func (d DeferredBatchOp) Finish() error { // consumers should use a batch per goroutine or provide their own // synchronization. // -// Indexing +// # Indexing // // Batches can be optionally indexed (see DB.NewIndexedBatch). An indexed batch // allows iteration via an Iterator (see Batch.NewIter). The iterator provides @@ -104,7 +106,7 @@ func (d DeferredBatchOp) Finish() error { // significantly slower than inserting into a non-indexed batch. Only use an // indexed batch if you require reading from it. // -// Atomic commit +// # Atomic commit // // The operations in a batch are persisted by calling Batch.Commit which is // equivalent to calling DB.Apply(batch). A batch is committed atomically by @@ -115,7 +117,7 @@ func (d DeferredBatchOp) Finish() error { // Batch.Commit will guarantee that the batch is persisted to disk before // returning. See commitPipeline for more on the implementation details. // -// Large batches +// # Large batches // // The size of a batch is limited only by available memory (be aware that // indexed batches require considerably additional memory for the skiplist @@ -140,34 +142,35 @@ func (d DeferredBatchOp) Finish() error { // memtable have the same big-O time, but the constant factor dominates // here. Sorting is significantly faster and uses significantly less memory. // -// Internal representation +// # Internal representation // // The internal batch representation is a contiguous byte buffer with a fixed // 12-byte header, followed by a series of records. // -// +-------------+------------+--- ... ---+ -// | SeqNum (8B) | Count (4B) | Entries | -// +-------------+------------+--- ... ---+ +// +-------------+------------+--- ... ---+ +// | SeqNum (8B) | Count (4B) | Entries | +// +-------------+------------+--- ... ---+ // // Each record has a 1-byte kind tag prefix, followed by 1 or 2 length prefixed // strings (varstring): // -// +-----------+-----------------+-------------------+ -// | Kind (1B) | Key (varstring) | Value (varstring) | -// +-----------+-----------------+-------------------+ +// +-----------+-----------------+-------------------+ +// | Kind (1B) | Key (varstring) | Value (varstring) | +// +-----------+-----------------+-------------------+ // // A varstring is a varint32 followed by N bytes of data. The Kind tags are // exactly those specified by InternalKeyKind. The following table shows the // format for records of each kind: // -// InternalKeyKindDelete varstring -// InternalKeyKindLogData varstring -// InternalKeyKindSet varstring varstring -// InternalKeyKindMerge varstring varstring -// InternalKeyKindRangeDelete varstring varstring -// InternalKeyKindRangeKeySet varstring varstring -// InternalKeyKindRangeKeyUnset varstring varstring -// InternalKeyKindRangeKeyDelete varstring varstring +// InternalKeyKindDelete varstring +// InternalKeyKindLogData varstring +// InternalKeyKindIngestSST varstring +// InternalKeyKindSet varstring varstring +// InternalKeyKindMerge varstring varstring +// InternalKeyKindRangeDelete varstring varstring +// InternalKeyKindRangeKeySet varstring varstring +// InternalKeyKindRangeKeyUnset varstring varstring +// InternalKeyKindRangeKeyDelete varstring varstring // // The intuitive understanding here are that the arguments to Delete, Set, // Merge, DeleteRange and RangeKeyDelete are encoded into the batch. The @@ -210,6 +213,8 @@ type Batch struct { // The db to which the batch will be committed. Do not change this field // after the batch has been created as it might invalidate internal state. + // Batch.memTableSize is only refreshed if Batch.db is set. Setting db to + // nil once it has been set implies that the Batch has encountered an error. db *DB // The count of records in the batch. This count will be stored in the batch @@ -255,9 +260,74 @@ type Batch struct { // memtable. flushable *flushableBatch + // ingestedSSTBatch indicates that the batch contains one or more key kinds + // of InternalKeyKindIngestSST. If the batch contains key kinds of IngestSST + // then it will only contain key kinds of IngestSST. + ingestedSSTBatch bool + + // Synchronous Apply uses the commit WaitGroup for both publishing the + // seqnum and waiting for the WAL fsync (if needed). Asynchronous + // ApplyNoSyncWait, which implies WriteOptions.Sync is true, uses the commit + // WaitGroup for publishing the seqnum and the fsyncWait WaitGroup for + // waiting for the WAL fsync. + // + // TODO(sumeer): if we find that ApplyNoSyncWait in conjunction with + // SyncWait is causing higher memory usage because of the time duration + // between when the sync is already done, and a goroutine calls SyncWait + // (followed by Batch.Close), we could separate out {fsyncWait, commitErr} + // into a separate struct that is allocated separately (using another + // sync.Pool), and only that struct needs to outlive Batch.Close (which + // could then be called immediately after ApplyNoSyncWait). commitStats + // will also need to be in this separate struct. commit sync.WaitGroup + fsyncWait sync.WaitGroup + + commitStats BatchCommitStats + commitErr error - applied uint32 // updated atomically + applied atomic.Bool +} + +// BatchCommitStats exposes stats related to committing a batch. +// +// NB: there is no Pebble internal tracing (using LoggerAndTracer) of slow +// batch commits. The caller can use these stats to do their own tracing as +// needed. +type BatchCommitStats struct { + // TotalDuration is the time spent in DB.{Apply,ApplyNoSyncWait} or + // Batch.Commit, plus the time waiting in Batch.SyncWait. If there is a gap + // between calling ApplyNoSyncWait and calling SyncWait, that gap could + // include some duration in which real work was being done for the commit + // and will not be included here. This missing time is considered acceptable + // since the goal of these stats is to understand user-facing latency. + // + // TotalDuration includes time spent in various queues both inside Pebble + // and outside Pebble (I/O queues, goroutine scheduler queue, mutex wait + // etc.). For some of these queues (which we consider important) the wait + // times are included below -- these expose low-level implementation detail + // and are meant for expert diagnosis and subject to change. There may be + // unaccounted time after subtracting those values from TotalDuration. + TotalDuration time.Duration + // SemaphoreWaitDuration is the wait time for semaphores in + // commitPipeline.Commit. + SemaphoreWaitDuration time.Duration + // WALQueueWaitDuration is the wait time for allocating memory blocks in the + // LogWriter (due to the LogWriter not writing fast enough). + WALQueueWaitDuration time.Duration + // MemTableWriteStallDuration is the wait caused by a write stall due to too + // many memtables (due to not flushing fast enough). + MemTableWriteStallDuration time.Duration + // L0ReadAmpWriteStallDuration is the wait caused by a write stall due to + // high read amplification in L0 (due to not compacting fast enough out of + // L0). + L0ReadAmpWriteStallDuration time.Duration + // WALRotationDuration is the wait time for WAL rotation, which includes + // syncing and closing the old WAL and creating (or reusing) a new one. + WALRotationDuration time.Duration + // CommitWaitDuration is the wait for publishing the seqnum plus the + // duration for the WAL sync (if requested). The former should be tiny and + // one can assume that this is all due to the WAL sync. + CommitWaitDuration time.Duration } var _ Reader = (*Batch)(nil) @@ -348,13 +418,16 @@ func (b *Batch) refreshMemTableSize() { if !ok { break } - b.memTableSize += memTableEntrySize(len(key), len(value)) switch kind { case InternalKeyKindRangeDelete: b.countRangeDels++ case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: b.countRangeKeys++ + case InternalKeyKindIngestSST: + // This key kind doesn't contribute to the memtable size. + continue } + b.memTableSize += memTableEntrySize(len(key), len(value)) } } @@ -362,6 +435,9 @@ func (b *Batch) refreshMemTableSize() { // // It is safe to modify the contents of the arguments after Apply returns. func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error { + if b.ingestedSSTBatch { + panic("pebble: invalid batch application") + } if len(batch.data) == 0 { return nil } @@ -392,6 +468,8 @@ func (b *Batch) Apply(batch *Batch, _ *WriteOptions) error { b.countRangeDels++ case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: b.countRangeKeys++ + case InternalKeyKindIngestSST: + panic("pebble: invalid key kind for batch") } if b.index != nil { var err error @@ -797,6 +875,28 @@ func (b *Batch) LogData(data []byte, _ *WriteOptions) error { return nil } +// IngestSST adds the FileNum for an sstable to the batch. The data will only be +// written to the WAL (not added to memtables or sstables). +func (b *Batch) ingestSST(fileNum base.FileNum) { + if b.Empty() { + b.ingestedSSTBatch = true + } else if !b.ingestedSSTBatch { + // Batch contains other key kinds. + panic("pebble: invalid call to ingestSST") + } + + origMemTableSize := b.memTableSize + var buf [binary.MaxVarintLen64]byte + length := binary.PutUvarint(buf[:], uint64(fileNum)) + b.prepareDeferredKeyRecord(length, InternalKeyKindIngestSST) + copy(b.deferredOp.Key, buf[:length]) + // Since IngestSST writes only to the WAL and does not affect the memtable, + // we restore b.memTableSize to its original value. Note that Batch.count + // is not reset because for the InternalKeyKindIngestSST the count is the + // number of sstable paths which have been added to the batch. + b.memTableSize = origMemTableSize +} + // Empty returns true if the batch is empty, and false otherwise. func (b *Batch) Empty() bool { return len(b.data) <= batchHeaderLen @@ -845,10 +945,16 @@ func (b *Batch) SetRepr(data []byte) error { // later mutations. Its view can be refreshed via RefreshBatchSnapshot or // SetOptions(). func (b *Batch) NewIter(o *IterOptions) *Iterator { + return b.NewIterWithContext(context.Background(), o) +} + +// NewIterWithContext is like NewIter, and additionally accepts a context for +// tracing. +func (b *Batch) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterator { if b.index == nil { return &Iterator{err: ErrNotIndexed} } - return b.db.newIterInternal(b, nil /* snapshot */, o) + return b.db.newIter(ctx, b, nil /* snapshot */, o) } // newInternalIter creates a new internalIterator that iterates over the @@ -1089,8 +1195,10 @@ func (b *Batch) Reset() { b.rangeKeysSeqNum = 0 b.flushable = nil b.commit = sync.WaitGroup{} + b.fsyncWait = sync.WaitGroup{} + b.commitStats = BatchCommitStats{} b.commitErr = nil - atomic.StoreUint32(&b.applied, 0) + b.applied.Store(false) if b.data != nil { if cap(b.data) > batchMaxRetainedSize { // If the capacity of the buffer is larger than our maximum @@ -1159,7 +1267,9 @@ func (b *Batch) setCount(v uint32) { } // Count returns the count of memtable-modifying operations in this batch. All -// operations with the except of LogData increment this count. +// operations with the except of LogData increment this count. For IngestSSTs, +// count is only used to indicate the number of SSTs ingested in the record, the +// batch isn't applied to the memtable. func (b *Batch) Count() uint32 { if b.count > math.MaxUint32 { panic(ErrInvalidBatch) @@ -1205,6 +1315,26 @@ func batchDecodeStr(data []byte) (odata []byte, s []byte, ok bool) { return data[v:], data[:v], true } +// SyncWait is to be used in conjunction with DB.ApplyNoSyncWait. +func (b *Batch) SyncWait() error { + now := time.Now() + b.fsyncWait.Wait() + if b.commitErr != nil { + b.db = nil // prevent batch reuse on error + } + waitDuration := time.Since(now) + b.commitStats.CommitWaitDuration += waitDuration + b.commitStats.TotalDuration += waitDuration + return b.commitErr +} + +// CommitStats returns stats related to committing the batch. Should be called +// after Batch.Commit, DB.Apply. If DB.ApplyNoSyncWait is used, should be +// called after Batch.SyncWait. +func (b *Batch) CommitStats() BatchCommitStats { + return b.commitStats +} + // BatchReader iterates over the entries contained in a batch. type BatchReader []byte @@ -1336,6 +1466,19 @@ func (i *batchIter) Next() (*InternalKey, base.LazyValue) { return ikey, base.MakeInPlaceValue(i.value()) } +func (i *batchIter) NextPrefix(succKey []byte) (*InternalKey, LazyValue) { + // Because NextPrefix was invoked `succKey` must be ≥ the key at i's current + // position. Seek the arena iterator using TrySeekUsingNext. + ikey := i.iter.SeekGE(succKey, base.SeekGEFlagsNone.EnableTrySeekUsingNext()) + for ikey != nil && ikey.SeqNum() >= i.snapshot { + ikey = i.iter.Next() + } + if ikey == nil { + return nil, base.LazyValue{} + } + return ikey, base.MakeInPlaceValue(i.value()) +} + func (i *batchIter) Prev() (*InternalKey, base.LazyValue) { ikey := i.iter.Prev() for ikey != nil && ikey.SeqNum() >= i.snapshot { @@ -1773,6 +1916,12 @@ func (i *flushableBatchIter) Prev() (*InternalKey, base.LazyValue) { return &i.key, i.value() } +// Note: flushFlushableBatchIter.NextPrefix mirrors the implementation of +// flushableBatchIter.NextPrefix due to performance. Keep the two in sync. +func (i *flushableBatchIter) NextPrefix(succKey []byte) (*InternalKey, LazyValue) { + return i.SeekGE(succKey, base.SeekGEFlagsNone.EnableTrySeekUsingNext()) +} + func (i *flushableBatchIter) getKey(index int) InternalKey { e := &i.offsets[index] kind := InternalKeyKind(i.data[e.offset]) @@ -1866,6 +2015,10 @@ func (i *flushFlushableBatchIter) First() (*InternalKey, base.LazyValue) { return key, val } +func (i *flushFlushableBatchIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + panic("pebble: Prev unimplemented") +} + // Note: flushFlushableBatchIter.Next mirrors the implementation of // flushableBatchIter.Next due to performance. Keep the two in sync. func (i *flushFlushableBatchIter) Next() (*InternalKey, base.LazyValue) { diff --git a/vendor/github.com/cockroachdb/pebble/cache.go b/vendor/github.com/cockroachdb/pebble/cache.go index a1856e8..91f5532 100644 --- a/vendor/github.com/cockroachdb/pebble/cache.go +++ b/vendor/github.com/cockroachdb/pebble/cache.go @@ -15,9 +15,9 @@ type Cache = cache.Cache // creator of the cache should usually release their reference after the DB is // created. // -// c := pebble.NewCache(...) -// defer c.Unref() -// d, err := pebble.Open(pebble.Options{Cache: c}) +// c := pebble.NewCache(...) +// defer c.Unref() +// d, err := pebble.Open(pebble.Options{Cache: c}) func NewCache(size int64) *cache.Cache { return cache.New(size) } diff --git a/vendor/github.com/cockroachdb/pebble/checkpoint.go b/vendor/github.com/cockroachdb/pebble/checkpoint.go index 07d08a1..a181246 100644 --- a/vendor/github.com/cockroachdb/pebble/checkpoint.go +++ b/vendor/github.com/cockroachdb/pebble/checkpoint.go @@ -5,10 +5,12 @@ package pebble import ( + "io" "os" "github.com/cockroachdb/errors/oserror" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/record" "github.com/cockroachdb/pebble/vfs" "github.com/cockroachdb/pebble/vfs/atomicfs" ) @@ -19,6 +21,9 @@ type checkpointOptions struct { // flushWAL set to true will force a flush and sync of the WAL prior to // checkpointing. flushWAL bool + + // If set, any SSTs that don't overlap with these spans are excluded from a checkpoint. + restrictToSpans []CheckpointSpan } // CheckpointOption set optional parameters used by `DB.Checkpoint`. @@ -39,6 +44,43 @@ func WithFlushedWAL() CheckpointOption { } } +// WithRestrictToSpans specifies spans of interest for the checkpoint. Any SSTs +// that don't overlap with any of these spans are excluded from the checkpoint. +// +// Note that the checkpoint can still surface keys outside of these spans (from +// the WAL and from SSTs that partially overlap with these spans). Moreover, +// these surface keys aren't necessarily "valid" in that they could have been +// modified but the SST containing the modification is excluded. +func WithRestrictToSpans(spans []CheckpointSpan) CheckpointOption { + return func(opt *checkpointOptions) { + opt.restrictToSpans = spans + } +} + +// CheckpointSpan is a key range [Start, End) (inclusive on Start, exclusive on +// End) of interest for a checkpoint. +type CheckpointSpan struct { + Start []byte + End []byte +} + +// excludeFromCheckpoint returns true if an SST file should be excluded from the +// checkpoint because it does not overlap with the spans of interest +// (opt.restrictToSpans). +func excludeFromCheckpoint(f *fileMetadata, opt *checkpointOptions, cmp Compare) bool { + if len(opt.restrictToSpans) == 0 { + // Option not set; don't exclude anything. + return false + } + for _, s := range opt.restrictToSpans { + if f.Overlaps(cmp, s.Start, s.End, true /* exclusiveEnd */) { + return false + } + } + // None of the restrictToSpans overlapped; we can exclude this file. + return true +} + // mkdirAllAndSyncParents creates destDir and any of its missing parents. // Those missing parents, as well as the closest existing ancestor, are synced. // Returns a handle to the directory created at destDir. @@ -93,6 +135,9 @@ func mkdirAllAndSyncParents(fs vfs.FS, destDir string) (vfs.File, error) { // space overhead for a checkpoint if hard links are disabled. Also beware that // even if hard links are used, the space overhead for the checkpoint will // increase over time as the DB performs compactions. +// +// TODO(bananabrick): Test checkpointing of virtual sstables once virtual +// sstables is running e2e. func (d *DB) Checkpoint( destDir string, opts ...CheckpointOption, ) ( @@ -146,7 +191,10 @@ func (d *DB) Checkpoint( manifestFileNum := d.mu.versions.manifestFileNum manifestSize := d.mu.versions.manifest.Size() optionsFileNum := d.optionsFileNum - + virtualBackingFiles := make(map[base.DiskFileNum]struct{}) + for diskFileNum := range d.mu.versions.fileBackingMap { + virtualBackingFiles[diskFileNum] = struct{}{} + } // Release the manifest and DB.mu so we don't block other operations on // the database. d.mu.versions.logUnlock() @@ -154,13 +202,10 @@ func (d *DB) Checkpoint( // Wrap the normal filesystem with one which wraps newly created files with // vfs.NewSyncingFile. - fs := syncingFS{ - FS: d.opts.FS, - syncOpts: vfs.SyncingFileOptions{ - NoSyncOnClose: d.opts.NoSyncOnClose, - BytesPerSync: d.opts.BytesPerSync, - }, - } + fs := vfs.NewSyncingFS(d.opts.FS, vfs.SyncingFileOptions{ + NoSyncOnClose: d.opts.NoSyncOnClose, + BytesPerSync: d.opts.BytesPerSync, + }) // Create the dir and its parents (if necessary), and sync them. var dir vfs.File @@ -214,45 +259,34 @@ func (d *DB) Checkpoint( } } - { - // Copy the MANIFEST, and create a pointer to it. We copy rather - // than link because additional version edits added to the - // MANIFEST after we took our snapshot of the sstables will - // reference sstables that aren't in our checkpoint. For a - // similar reason, we need to limit how much of the MANIFEST we - // copy. - srcPath := base.MakeFilepath(fs, d.dirname, fileTypeManifest, manifestFileNum) - destPath := fs.PathJoin(destDir, fs.PathBase(srcPath)) - ckErr = vfs.LimitedCopy(fs, srcPath, destPath, manifestSize) - if ckErr != nil { - return ckErr - } - - // Recent format versions use an atomic marker for setting the - // active manifest. Older versions use the CURRENT file. The - // setCurrentFunc function will return a closure that will - // take the appropriate action for the database's format - // version. - var manifestMarker *atomicfs.Marker - manifestMarker, _, ckErr = atomicfs.LocateMarker(fs, destDir, manifestMarkerName) - if ckErr != nil { - return ckErr - } - ckErr = setCurrentFunc(formatVers, manifestMarker, fs, destDir, dir)(manifestFileNum) - if ckErr != nil { - return ckErr - } - ckErr = manifestMarker.Close() - if ckErr != nil { - return ckErr - } - } - + var excludedFiles map[deletedFileEntry]*fileMetadata + // Set of FileBacking.DiskFileNum which will be required by virtual sstables + // in the checkpoint. + requiredVirtualBackingFiles := make(map[base.DiskFileNum]struct{}) // Link or copy the sstables. for l := range current.Levels { iter := current.Levels[l].Iter() for f := iter.First(); f != nil; f = iter.Next() { - srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, f.FileNum) + if excludeFromCheckpoint(f, opt, d.cmp) { + if excludedFiles == nil { + excludedFiles = make(map[deletedFileEntry]*fileMetadata) + } + excludedFiles[deletedFileEntry{ + Level: l, + FileNum: f.FileNum, + }] = f + continue + } + + fileBacking := f.FileBacking + if f.Virtual { + if _, ok := requiredVirtualBackingFiles[fileBacking.DiskFileNum]; ok { + continue + } + requiredVirtualBackingFiles[fileBacking.DiskFileNum] = struct{}{} + } + + srcPath := base.MakeFilepath(fs, d.dirname, fileTypeTable, fileBacking.DiskFileNum) destPath := fs.PathJoin(destDir, fs.PathBase(srcPath)) ckErr = vfs.LinkOrCopy(fs, srcPath, destPath) if ckErr != nil { @@ -261,6 +295,23 @@ func (d *DB) Checkpoint( } } + var removeBackingTables []base.DiskFileNum + for diskFileNum := range virtualBackingFiles { + if _, ok := requiredVirtualBackingFiles[diskFileNum]; !ok { + // The backing sstable associated with fileNum is no longer + // required. + removeBackingTables = append(removeBackingTables, diskFileNum) + } + } + + ckErr = d.writeCheckpointManifest( + fs, formatVers, destDir, dir, manifestFileNum.DiskFileNum(), manifestSize, + excludedFiles, removeBackingTables, + ) + if ckErr != nil { + return ckErr + } + // Copy the WAL files. We copy rather than link because WAL file recycling // will cause the WAL files to be reused which would invalidate the // checkpoint. @@ -269,7 +320,7 @@ func (d *DB) Checkpoint( if logNum == 0 { continue } - srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum) + srcPath := base.MakeFilepath(fs, d.walDirname, fileTypeLog, logNum.DiskFileNum()) destPath := fs.PathJoin(destDir, fs.PathBase(srcPath)) ckErr = vfs.Copy(fs, srcPath, destPath) if ckErr != nil { @@ -286,3 +337,99 @@ func (d *DB) Checkpoint( dir = nil return ckErr } + +func (d *DB) writeCheckpointManifest( + fs vfs.FS, + formatVers FormatMajorVersion, + destDirPath string, + destDir vfs.File, + manifestFileNum base.DiskFileNum, + manifestSize int64, + excludedFiles map[deletedFileEntry]*fileMetadata, + removeBackingTables []base.DiskFileNum, +) error { + // Copy the MANIFEST, and create a pointer to it. We copy rather + // than link because additional version edits added to the + // MANIFEST after we took our snapshot of the sstables will + // reference sstables that aren't in our checkpoint. For a + // similar reason, we need to limit how much of the MANIFEST we + // copy. + // If some files are excluded from the checkpoint, also append a block that + // records those files as deleted. + if err := func() error { + srcPath := base.MakeFilepath(fs, d.dirname, fileTypeManifest, manifestFileNum) + destPath := fs.PathJoin(destDirPath, fs.PathBase(srcPath)) + src, err := fs.Open(srcPath, vfs.SequentialReadsOption) + if err != nil { + return err + } + defer src.Close() + + dst, err := fs.Create(destPath) + if err != nil { + return err + } + defer dst.Close() + + // Copy all existing records. We need to copy at the record level in case we + // need to append another record with the excluded files (we cannot simply + // append a record after a raw data copy; see + // https://github.com/cockroachdb/cockroach/issues/100935). + r := record.NewReader(&io.LimitedReader{R: src, N: manifestSize}, manifestFileNum.FileNum()) + w := record.NewWriter(dst) + for { + rr, err := r.Next() + if err != nil { + if err == io.EOF { + break + } + return err + } + + rw, err := w.Next() + if err != nil { + return err + } + if _, err := io.Copy(rw, rr); err != nil { + return err + } + } + + if len(excludedFiles) > 0 { + // Write out an additional VersionEdit that deletes the excluded SST files. + ve := versionEdit{ + DeletedFiles: excludedFiles, + RemovedBackingTables: removeBackingTables, + } + + rw, err := w.Next() + if err != nil { + return err + } + if err := ve.Encode(rw); err != nil { + return err + } + } + if err := w.Close(); err != nil { + return err + } + return dst.Sync() + }(); err != nil { + return err + } + + // Recent format versions use an atomic marker for setting the + // active manifest. Older versions use the CURRENT file. The + // setCurrentFunc function will return a closure that will + // take the appropriate action for the database's format + // version. + var manifestMarker *atomicfs.Marker + manifestMarker, _, err := atomicfs.LocateMarker(fs, destDirPath, manifestMarkerName) + if err != nil { + return err + } + if err := setCurrentFunc(formatVers, manifestMarker, fs, destDirPath, destDir)(manifestFileNum.FileNum()); err != nil { + return err + } + return manifestMarker.Close() +} diff --git a/vendor/github.com/cockroachdb/pebble/commit.go b/vendor/github.com/cockroachdb/pebble/commit.go index 83d1a50..26919da 100644 --- a/vendor/github.com/cockroachdb/pebble/commit.go +++ b/vendor/github.com/cockroachdb/pebble/commit.go @@ -8,6 +8,7 @@ import ( "runtime" "sync" "sync/atomic" + "time" "unsafe" "github.com/cockroachdb/pebble/record" @@ -32,7 +33,7 @@ type commitQueue struct { // // The head index is stored in the most-significant bits so that we can // atomically add to it and the overflow is harmless. - headTail uint64 + headTail atomic.Uint64 // slots is a ring buffer of values stored in this queue. The size must be a // power of 2. A slot is in use until *both* the tail index has moved beyond @@ -57,10 +58,10 @@ func (q *commitQueue) pack(head, tail uint32) uint64 { } func (q *commitQueue) enqueue(b *Batch) { - ptrs := atomic.LoadUint64(&q.headTail) + ptrs := q.headTail.Load() head, tail := q.unpack(ptrs) if (tail+uint32(len(q.slots)))&(1< syncWAL +func (p *commitPipeline) Commit(b *Batch, syncWAL bool, noSyncWait bool) error { if b.Empty() { return nil } - p.sem <- struct{}{} + commitStartTime := time.Now() + // Acquire semaphores. + p.commitQueueSem <- struct{}{} + if syncWAL { + p.logSyncQSem <- struct{}{} + } + b.commitStats.SemaphoreWaitDuration = time.Since(commitStartTime) // Prepare the batch for committing: enqueuing the batch in the pending // queue, determining the batch sequence number and writing the data to the @@ -250,27 +307,43 @@ func (p *commitPipeline) Commit(b *Batch, syncWAL bool) error { // // NB: We set Batch.commitErr on error so that the batch won't be a candidate // for reuse. See Batch.release(). - mem, err := p.prepare(b, syncWAL) + mem, err := p.prepare(b, syncWAL, noSyncWait) if err != nil { b.db = nil // prevent batch reuse on error + // NB: we are not doing <-p.commitQueueSem since the batch is still + // sitting in the pending queue. We should consider fixing this by also + // removing the batch from the pending queue. return err } // Apply the batch to the memtable. if err := p.env.apply(b, mem); err != nil { b.db = nil // prevent batch reuse on error + // NB: we are not doing <-p.commitQueueSem since the batch is still + // sitting in the pending queue. We should consider fixing this by also + // removing the batch from the pending queue. return err } // Publish the batch sequence number. p.publish(b) - <-p.sem + <-p.commitQueueSem - if b.commitErr != nil { - b.db = nil // prevent batch reuse on error + if !noSyncWait { + // Already waited for commit, so look at the error. + if b.commitErr != nil { + b.db = nil // prevent batch reuse on error + err = b.commitErr + } } - return b.commitErr + // Else noSyncWait. The LogWriter can be concurrently writing to + // b.commitErr. We will read b.commitErr in Batch.SyncWait after the + // LogWriter is done writing. + + b.commitStats.TotalDuration = time.Since(commitStartTime) + + return err } // AllocateSeqNum allocates count sequence numbers, invokes the prepare @@ -280,7 +353,9 @@ func (p *commitPipeline) Commit(b *Batch, syncWAL bool) error { // sstable ingestion within the commit pipeline. The prepare callback is // invoked with commitPipeline.mu held, but note that DB.mu is not held and // must be locked if necessary. -func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(seqNum uint64)) { +func (p *commitPipeline) AllocateSeqNum( + count int, prepare func(seqNum uint64), apply func(seqNum uint64), +) { // This method is similar to Commit and prepare. Be careful about trying to // share additional code with those methods because Commit and prepare are // performance critical code paths. @@ -294,7 +369,7 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se b.setCount(uint32(count)) b.commit.Add(1) - p.sem <- struct{}{} + p.commitQueueSem <- struct{}{} p.mu.Lock() @@ -306,12 +381,12 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se // Assign the batch a sequence number. Note that we use atomic operations // here to handle concurrent reads of logSeqNum. commitPipeline.mu provides // mutual exclusion for other goroutines writing to logSeqNum. - logSeqNum := atomic.AddUint64(p.env.logSeqNum, uint64(count)) - uint64(count) + logSeqNum := p.env.logSeqNum.Add(uint64(count)) - uint64(count) seqNum := logSeqNum if seqNum == 0 { // We can't use the value 0 for the global seqnum during ingestion, because // 0 indicates no global seqnum. So allocate one more seqnum. - atomic.AddUint64(p.env.logSeqNum, 1) + p.env.logSeqNum.Add(1) seqNum++ } b.setSeqNum(seqNum) @@ -321,7 +396,7 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se // writes that were sequenced before the ingestion. The spin loop is // unfortunate, but obviates the need for additional synchronization. for { - visibleSeqNum := atomic.LoadUint64(p.env.visibleSeqNum) + visibleSeqNum := p.env.visibleSeqNum.Load() if visibleSeqNum == logSeqNum { break } @@ -331,7 +406,7 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se // Invoke the prepare callback. Note the lack of error reporting. Even if the // callback internally fails, the sequence number needs to be published in // order to allow the commit pipeline to proceed. - prepare() + prepare(b.SeqNum()) p.mu.Unlock() @@ -341,26 +416,33 @@ func (p *commitPipeline) AllocateSeqNum(count int, prepare func(), apply func(se // Publish the sequence number. p.publish(b) - <-p.sem + <-p.commitQueueSem } -func (p *commitPipeline) prepare(b *Batch, syncWAL bool) (*memTable, error) { +func (p *commitPipeline) prepare(b *Batch, syncWAL bool, noSyncWait bool) (*memTable, error) { n := uint64(b.Count()) if n == invalidBatchCount { return nil, ErrInvalidBatch } - count := 1 - if syncWAL { - count++ - } - // count represents the waiting needed for publish, and optionally the - // waiting needed for the WAL sync. - b.commit.Add(count) - var syncWG *sync.WaitGroup var syncErr *error - if syncWAL { - syncWG, syncErr = &b.commit, &b.commitErr + switch { + case !syncWAL: + // Only need to wait for the publish. + b.commit.Add(1) + // Remaining cases represent syncWAL=true. + case noSyncWait: + syncErr = &b.commitErr + syncWG = &b.fsyncWait + // Only need to wait synchronously for the publish. The user will + // (asynchronously) wait on the batch's fsyncWait. + b.commit.Add(1) + b.fsyncWait.Add(1) + case !noSyncWait: + syncErr = &b.commitErr + syncWG = &b.commit + // Must wait for both the publish and the WAL fsync. + b.commit.Add(2) } p.mu.Lock() @@ -373,7 +455,7 @@ func (p *commitPipeline) prepare(b *Batch, syncWAL bool) (*memTable, error) { // Assign the batch a sequence number. Note that we use atomic operations // here to handle concurrent reads of logSeqNum. commitPipeline.mu provides // mutual exclusion for other goroutines writing to logSeqNum. - b.setSeqNum(atomic.AddUint64(p.env.logSeqNum, n) - n) + b.setSeqNum(p.env.logSeqNum.Add(n) - n) // Write the data to the WAL. mem, err := p.env.write(b, syncWG, syncErr) @@ -385,7 +467,7 @@ func (p *commitPipeline) prepare(b *Batch, syncWAL bool) (*memTable, error) { func (p *commitPipeline) publish(b *Batch) { // Mark the batch as applied. - atomic.StoreUint32(&b.applied, 1) + b.applied.Store(true) // Loop dequeuing applied batches from the pending queue. If our batch was // the head of the pending queue we are guaranteed that either we'll publish @@ -399,10 +481,12 @@ func (p *commitPipeline) publish(b *Batch) { if t == nil { // Wait for another goroutine to publish us. We might also be waiting for // the WAL sync to finish. + now := time.Now() b.commit.Wait() + b.commitStats.CommitWaitDuration += time.Since(now) break } - if atomic.LoadUint32(&t.applied) != 1 { + if !t.applied.Load() { panic("not reached") } @@ -411,13 +495,13 @@ func (p *commitPipeline) publish(b *Batch) { // number for a subsequent batch. That's ok as all we're guaranteeing is // that the sequence number ratchets up. for { - curSeqNum := atomic.LoadUint64(p.env.visibleSeqNum) + curSeqNum := p.env.visibleSeqNum.Load() newSeqNum := t.SeqNum() + uint64(t.Count()) if newSeqNum <= curSeqNum { // t's sequence number has already been published. break } - if atomic.CompareAndSwapUint64(p.env.visibleSeqNum, curSeqNum, newSeqNum) { + if p.env.visibleSeqNum.CompareAndSwap(curSeqNum, newSeqNum) { // We successfully published t's sequence number. break } @@ -426,18 +510,3 @@ func (p *commitPipeline) publish(b *Batch) { t.commit.Done() } } - -// ratchetSeqNum allocates and marks visible all sequence numbers less than -// but excluding `nextSeqNum`. -func (p *commitPipeline) ratchetSeqNum(nextSeqNum uint64) { - p.mu.Lock() - defer p.mu.Unlock() - - logSeqNum := atomic.LoadUint64(p.env.logSeqNum) - if logSeqNum >= nextSeqNum { - return - } - count := nextSeqNum - logSeqNum - _ = atomic.AddUint64(p.env.logSeqNum, uint64(count)) - uint64(count) - atomic.StoreUint64(p.env.visibleSeqNum, nextSeqNum) -} diff --git a/vendor/github.com/cockroachdb/pebble/compaction.go b/vendor/github.com/cockroachdb/pebble/compaction.go index 302342b..fcc840d 100644 --- a/vendor/github.com/cockroachdb/pebble/compaction.go +++ b/vendor/github.com/cockroachdb/pebble/compaction.go @@ -13,7 +13,6 @@ import ( "runtime/pprof" "sort" "strings" - "sync/atomic" "time" "github.com/cockroachdb/errors" @@ -24,6 +23,8 @@ import ( "github.com/cockroachdb/pebble/internal/private" "github.com/cockroachdb/pebble/internal/rangedel" "github.com/cockroachdb/pebble/internal/rangekey" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/pebble/vfs" ) @@ -35,6 +36,12 @@ var compactLabels = pprof.Labels("pebble", "compact") var flushLabels = pprof.Labels("pebble", "flush") var gcLabels = pprof.Labels("pebble", "gc") +// getInternalWriterProperties accesses a private variable (in the +// internal/private package) initialized by the sstable Writer. This indirection +// is necessary to ensure non-Pebble users constructing sstables for ingestion +// are unable to set internal-only properties. +var getInternalWriterProperties = private.SSTableInternalProperties.(func(*sstable.Writer) *sstable.Properties) + // expandedCompactionByteSizeLimit is the maximum number of bytes in all // compacted files. We avoid expanding the lower level file set of a compaction // if it would make the total compaction cover more than this many bytes. @@ -82,17 +89,28 @@ type compactionLevel struct { files manifest.LevelSlice } +func (cl compactionLevel) Clone() compactionLevel { + newCL := compactionLevel{ + level: cl.level, + files: cl.files.Reslice(func(start, end *manifest.LevelIterator) {}), + } + return newCL +} +func (cl compactionLevel) String() string { + return fmt.Sprintf(`Level %d, Files %s`, cl.level, cl.files) +} + // Return output from compactionOutputSplitters. See comment on // compactionOutputSplitter.shouldSplitBefore() on how this value is used. -type compactionSplitSuggestion int +type maybeSplit int const ( - noSplit compactionSplitSuggestion = iota + noSplit maybeSplit = iota splitNow ) // String implements the Stringer interface. -func (c compactionSplitSuggestion) String() string { +func (c maybeSplit) String() string { if c == noSplit { return "no-split" } @@ -111,7 +129,7 @@ type compactionOutputSplitter interface { // means no split is advised. If shouldSplitBefore(a) advises a split then // shouldSplitBefore(b) should also advise a split given b >= a, until // onNewOutput is called. - shouldSplitBefore(key *InternalKey, tw *sstable.Writer) compactionSplitSuggestion + shouldSplitBefore(key *InternalKey, tw *sstable.Writer) maybeSplit // onNewOutput updates internal splitter state when the compaction switches // to a new sstable, and returns the next limit for the new output which // would get used to truncate range tombstones if the compaction iterator @@ -119,79 +137,218 @@ type compactionOutputSplitter interface { // compaction's comparator. The specified key is the first key in the new // output, or nil if this sstable will only contain range tombstones already // in the fragmenter. - onNewOutput(key *InternalKey) []byte + onNewOutput(key []byte) []byte } -// fileSizeSplitter is a compactionOutputSplitter that makes a determination -// to split outputs based on the estimated file size of the current output. -// Note that, unlike most other splitters, this splitter does not guarantee -// that it will advise splits only at user key change boundaries. +// fileSizeSplitter is a compactionOutputSplitter that enforces target file +// sizes. This splitter splits to a new output file when the estimated file size +// is 0.5x-2x the target file size. If there are overlapping grandparent files, +// this splitter will attempt to split at a grandparent boundary. For example, +// consider the example where a compaction wrote 'd' to the current output file, +// and the next key has a user key 'g': +// +// previous key next key +// | | +// | | +// +---------------|----+ +--|----------+ +// grandparents: | 000006 | | | | 000007 | +// +---------------|----+ +--|----------+ +// a b d e f g i +// +// Splitting the output file F before 'g' will ensure that the current output +// file F does not overlap the grandparent file 000007. Aligning sstable +// boundaries like this can significantly reduce write amplification, since a +// subsequent compaction of F into the grandparent level will avoid needlessly +// rewriting any keys within 000007 that do not overlap F's bounds. Consider the +// following compaction: +// +// +----------------------+ +// input | | +// level +----------------------+ +// \/ +// +---------------+ +---------------+ +// output |XXXXXXX| | | |XXXXXXXX| +// level +---------------+ +---------------+ +// +// The input-level file overlaps two files in the output level, but only +// partially. The beginning of the first output-level file and the end of the +// second output-level file will be rewritten verbatim. This write I/O is +// "wasted" in the sense that no merging is being performed. +// +// To prevent the above waste, this splitter attempts to split output files +// before the start key of grandparent files. It still strives to write output +// files of approximately the target file size, by constraining this splitting +// at grandparent points to apply only if the current output's file size is +// about the right order of magnitude. +// +// Note that, unlike most other splitters, this splitter does not guarantee that +// it will advise splits only at user key change boundaries. type fileSizeSplitter struct { - maxFileSize uint64 + frontier frontier + targetFileSize uint64 + atGrandparentBoundary bool + boundariesObserved uint64 + nextGrandparent *fileMetadata + grandparents manifest.LevelIterator } -func (f *fileSizeSplitter) shouldSplitBefore( - key *InternalKey, tw *sstable.Writer, -) compactionSplitSuggestion { - // The Kind != RangeDelete part exists because EstimatedSize doesn't grow - // rightaway when a range tombstone is added to the fragmenter. It's always - // better to make a sequence of range tombstones visible to the fragmenter. - if key.Kind() != InternalKeyKindRangeDelete && tw != nil && - tw.EstimatedSize() >= f.maxFileSize { +func newFileSizeSplitter( + f *frontiers, targetFileSize uint64, grandparents manifest.LevelIterator, +) *fileSizeSplitter { + s := &fileSizeSplitter{targetFileSize: targetFileSize} + s.nextGrandparent = grandparents.First() + s.grandparents = grandparents + if s.nextGrandparent != nil { + s.frontier.Init(f, s.nextGrandparent.Smallest.UserKey, s.reached) + } + return s +} + +func (f *fileSizeSplitter) reached(nextKey []byte) []byte { + f.atGrandparentBoundary = true + f.boundariesObserved++ + // NB: f.grandparents is a bounded iterator, constrained to the compaction + // key range. + f.nextGrandparent = f.grandparents.Next() + if f.nextGrandparent == nil { + return nil + } + // TODO(jackson): Should we also split before or immediately after + // grandparents' largest keys? Splitting before the start boundary prevents + // overlap with the grandparent. Also splitting after the end boundary may + // increase the probability of move compactions. + return f.nextGrandparent.Smallest.UserKey +} + +func (f *fileSizeSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) maybeSplit { + atGrandparentBoundary := f.atGrandparentBoundary + + // Clear f.atGrandparentBoundary unconditionally. + // + // This is a bit subtle. Even if do decide to split, it's possible that a + // higher-level splitter will ignore our request (eg, because we're between + // two internal keys with the same user key). In this case, the next call to + // shouldSplitBefore will find atGrandparentBoundary=false. This is + // desirable, because in this case we would've already written the earlier + // key with the same user key to the output file. The current output file is + // already doomed to overlap the grandparent whose bound triggered + // atGrandparentBoundary=true. We should continue on, waiting for the next + // grandparent boundary. + f.atGrandparentBoundary = false + + // If the key is a range tombstone, the EstimatedSize may not grow right + // away when a range tombstone is added to the fragmenter: It's dependent on + // whether or not the this new range deletion will start a new fragment. + // Range deletions are rare, so we choose to simply not split yet. + // TODO(jackson): Reconsider this, and consider range keys too as a part of + // #2321. + if key.Kind() == InternalKeyKindRangeDelete || tw == nil { + return noSplit + } + + estSize := tw.EstimatedSize() + switch { + case estSize < f.targetFileSize/2: + // The estimated file size is less than half the target file size. Don't + // split it, even if currently aligned with a grandparent file because + // it's too small. + return noSplit + case estSize >= 2*f.targetFileSize: + // The estimated file size is double the target file size. Split it even + // if we were not aligned with a grandparent file boundary to avoid + // excessively exceeding the target file size. + return splitNow + case !atGrandparentBoundary: + // Don't split if we're not at a grandparent, except if we've exhausted + // all the grandparents overlapping this compaction's key range. Then we + // may want to split purely based on file size. + if f.nextGrandparent == nil { + // There are no more grandparents. Optimize for the target file size + // and split as soon as we hit the target file size. + if estSize >= f.targetFileSize { + return splitNow + } + } + return noSplit + default: + // INVARIANT: atGrandparentBoundary + // INVARIANT: targetSize/2 < estSize < 2*targetSize + // + // The estimated file size is close enough to the target file size that + // we should consider splitting. + // + // Determine whether to split now based on how many grandparent + // boundaries we have already observed while building this output file. + // The intuition here is that if the grandparent level is dense in this + // part of the keyspace, we're likely to continue to have more + // opportunities to split this file aligned with a grandparent. If this + // is the first grandparent boundary observed, we split immediately + // (we're already at ≥50% the target file size). Otherwise, each + // overlapping grandparent we've observed increases the minimum file + // size by 5% of the target file size, up to at most 90% of the target + // file size. + // + // TODO(jackson): The particular thresholds are somewhat unprincipled. + // This is the same heuristic as RocksDB implements. Is there are more + // principled formulation that can, further reduce w-amp, produce files + // closer to the target file size, or is more understandable? + + // NB: Subtract 1 from `boundariesObserved` to account for the current + // boundary we're considering splitting at. `reached` will have + // incremented it at the same time it set `atGrandparentBoundary`. + minimumPctOfTargetSize := 50 + 5*minUint64(f.boundariesObserved-1, 8) + if estSize < (minimumPctOfTargetSize*f.targetFileSize)/100 { + return noSplit + } return splitNow } - return noSplit } -func (f *fileSizeSplitter) onNewOutput(key *InternalKey) []byte { +func minUint64(a, b uint64) uint64 { + if b < a { + a = b + } + return a +} + +func (f *fileSizeSplitter) onNewOutput(key []byte) []byte { + f.boundariesObserved = 0 return nil } +func newLimitFuncSplitter(f *frontiers, limitFunc func(userKey []byte) []byte) *limitFuncSplitter { + s := &limitFuncSplitter{limitFunc: limitFunc} + s.frontier.Init(f, nil, s.reached) + return s +} + type limitFuncSplitter struct { - c *compaction + frontier frontier limitFunc func(userKey []byte) []byte - limit []byte + split maybeSplit } -func (lf *limitFuncSplitter) shouldSplitBefore( - key *InternalKey, tw *sstable.Writer, -) compactionSplitSuggestion { - // NB: The limit must be applied using >= since lf.limit may be used as the - // `splitterSuggestion` ultimately passed to `compactionIter.Tombstones` to - // serve as an *exclusive* end boundary truncation point. If we used > then, - // we may have already added a key with the user key `lf.limit` to the - // previous sstable. - if lf.limit != nil && lf.c.cmp(key.UserKey, lf.limit) >= 0 { - return splitNow - } - return noSplit +func (lf *limitFuncSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) maybeSplit { + return lf.split } -func (lf *limitFuncSplitter) onNewOutput(key *InternalKey) []byte { - lf.limit = nil +func (lf *limitFuncSplitter) reached(nextKey []byte) []byte { + lf.split = splitNow + return nil +} + +func (lf *limitFuncSplitter) onNewOutput(key []byte) []byte { + lf.split = noSplit if key != nil { - lf.limit = lf.limitFunc(key.UserKey) - } else { - // Use the start key of the first pending tombstone to find the - // next limit. All pending tombstones have the same start key. - // We use this as opposed to the end key of the - // last written sstable to effectively handle cases like these: - // - // a.SET.3 - // (lf.limit at b) - // d.RANGEDEL.4:f - // - // In this case, the partition after b has only range deletions, - // so if we were to find the limit after the last written key at - // the split point (key a), we'd get the limit b again, and - // finishOutput() would not advance any further because the next - // range tombstone to write does not start until after the L0 - // split point. - if startKey := lf.c.rangeDelFrag.Start(); startKey != nil { - lf.limit = lf.limitFunc(startKey) - } + // TODO(jackson): For some users, like L0 flush splits, there's no need + // to binary search over all the flush splits every time. The next split + // point must be ahead of the previous flush split point. + limit := lf.limitFunc(key) + lf.frontier.Update(limit) + return limit } - return lf.limit + lf.frontier.Update(nil) + return nil } // splitterGroup is a compactionOutputSplitter that splits whenever one of its @@ -203,7 +360,7 @@ type splitterGroup struct { func (a *splitterGroup) shouldSplitBefore( key *InternalKey, tw *sstable.Writer, -) (suggestion compactionSplitSuggestion) { +) (suggestion maybeSplit) { for _, splitter := range a.splitters { if splitter.shouldSplitBefore(key, tw) == splitNow { return splitNow @@ -212,7 +369,7 @@ func (a *splitterGroup) shouldSplitBefore( return noSplit } -func (a *splitterGroup) onNewOutput(key *InternalKey) []byte { +func (a *splitterGroup) onNewOutput(key []byte) []byte { var earliestLimit []byte for _, splitter := range a.splitters { limit := splitter.onNewOutput(key) @@ -240,9 +397,21 @@ type userKeyChangeSplitter struct { unsafePrevUserKey func() []byte } -func (u *userKeyChangeSplitter) shouldSplitBefore( - key *InternalKey, tw *sstable.Writer, -) compactionSplitSuggestion { +func (u *userKeyChangeSplitter) shouldSplitBefore(key *InternalKey, tw *sstable.Writer) maybeSplit { + // NB: The userKeyChangeSplitter only needs to suffer a key comparison if + // the wrapped splitter requests a split. + // + // We could implement this splitter using frontiers: When the inner splitter + // requests a split before key `k`, we'd update a frontier to be + // ImmediateSuccessor(k). Then on the next key greater than >k, the + // frontier's `reached` func would be called and we'd return splitNow. + // This doesn't really save work since duplicate user keys are rare, and it + // requires us to materialize the ImmediateSuccessor key. It also prevents + // us from splitting on the same key that the inner splitter requested a + // split for—instead we need to wait until the next key. The current + // implementation uses `unsafePrevUserKey` to gain access to the previous + // key which allows it to immediately respect the inner splitter if + // possible. if split := u.splitter.shouldSplitBefore(key, tw); split != splitNow { return split } @@ -252,30 +421,29 @@ func (u *userKeyChangeSplitter) shouldSplitBefore( return noSplit } -func (u *userKeyChangeSplitter) onNewOutput(key *InternalKey) []byte { +func (u *userKeyChangeSplitter) onNewOutput(key []byte) []byte { return u.splitter.onNewOutput(key) } -// compactionFile is a vfs.File wrapper that, on every write, updates a metric -// in `versions` on bytes written by in-progress compactions so far. It also -// increments a per-compaction `written` int. -type compactionFile struct { - vfs.File +// compactionWritable is a objstorage.Writable wrapper that, on every write, +// updates a metric in `versions` on bytes written by in-progress compactions so +// far. It also increments a per-compaction `written` int. +type compactionWritable struct { + objstorage.Writable versions *versionSet written *int64 } -// Write implements the io.Writer interface. -func (c *compactionFile) Write(p []byte) (n int, err error) { - n, err = c.File.Write(p) - if err != nil { - return n, err +// Write is part of the objstorage.Writable interface. +func (c *compactionWritable) Write(p []byte) error { + if err := c.Writable.Write(p); err != nil { + return err } - *c.written += int64(n) - c.versions.incrementCompactionBytes(int64(n)) - return n, err + *c.written += int64(len(p)) + c.versions.incrementCompactionBytes(int64(len(p))) + return nil } type compactionKind int @@ -288,6 +456,7 @@ const ( compactionKindElisionOnly compactionKindRead compactionKindRewrite + compactionKindIngestedFlushable ) func (k compactionKind) String() string { @@ -306,6 +475,8 @@ func (k compactionKind) String() string { return "read" case compactionKindRewrite: return "rewrite" + case compactionKindIngestedFlushable: + return "ingested-flushable" } return "?" } @@ -315,7 +486,7 @@ func (k compactionKind) String() string { // keys in the last snapshot stripe, as well as coalesce range keys within // snapshot stripes. func rangeKeyCompactionTransform( - snapshots []uint64, elideRangeKey func(start, end []byte) bool, + eq base.Equal, snapshots []uint64, elideRangeKey func(start, end []byte) bool, ) keyspan.Transformer { return keyspan.TransformerFunc(func(cmp base.Compare, s keyspan.Span, dst *keyspan.Span) error { elideInLastStripe := func(keys []keyspan.Key) []keyspan.Key { @@ -348,7 +519,7 @@ func rangeKeyCompactionTransform( } if j > start { keysDst := dst.Keys[usedLen:cap(dst.Keys)] - if err := rangekey.Coalesce(cmp, s.Keys[start:j], &keysDst); err != nil { + if err := rangekey.Coalesce(cmp, eq, s.Keys[start:j], &keysDst); err != nil { return err } if j == len(s.Keys) { @@ -362,7 +533,7 @@ func rangeKeyCompactionTransform( } if j < len(s.Keys) { keysDst := dst.Keys[usedLen:cap(dst.Keys)] - if err := rangekey.Coalesce(cmp, s.Keys[j:], &keysDst); err != nil { + if err := rangekey.Coalesce(cmp, eq, s.Keys[j:], &keysDst); err != nil { return err } keysDst = elideInLastStripe(keysDst) @@ -384,6 +555,11 @@ type compaction struct { logger Logger version *version stats base.InternalIteratorStats + beganAt time.Time + // versionEditApplied is set to true when a compaction has completed and the + // resulting version has been installed (if successful), but the compaction + // goroutine is still cleaning up (eg, deleting obsolete files). + versionEditApplied bool score float64 @@ -512,7 +688,7 @@ func (c *compaction) makeInfo(jobID int) CompactionInfo { return info } -func newCompaction(pc *pickedCompaction, opts *Options) *compaction { +func newCompaction(pc *pickedCompaction, opts *Options, beganAt time.Time) *compaction { c := &compaction{ kind: compactionKindDefault, cmp: pc.cmp, @@ -525,6 +701,7 @@ func newCompaction(pc *pickedCompaction, opts *Options) *compaction { largest: pc.largest, logger: opts.Logger, version: pc.version, + beganAt: beganAt, maxOutputFileSize: pc.maxOutputFileSize, maxOverlapBytes: pc.maxOverlapBytes, l0SublevelInfo: pc.l0SublevelInfo, @@ -556,7 +733,9 @@ func newCompaction(pc *pickedCompaction, opts *Options) *compaction { return c } -func newDeleteOnlyCompaction(opts *Options, cur *version, inputs []compactionLevel) *compaction { +func newDeleteOnlyCompaction( + opts *Options, cur *version, inputs []compactionLevel, beganAt time.Time, +) *compaction { c := &compaction{ kind: compactionKindDeleteOnly, cmp: opts.Comparer.Compare, @@ -565,6 +744,7 @@ func newDeleteOnlyCompaction(opts *Options, cur *version, inputs []compactionLev formatKey: opts.Comparer.FormatKey, logger: opts.Logger, version: cur, + beganAt: beganAt, inputs: inputs, } @@ -644,7 +824,9 @@ func adjustGrandparentOverlapBytesForFlush(c *compaction, flushingBytes uint64) } } -func newFlush(opts *Options, cur *version, baseLevel int, flushing flushableList) *compaction { +func newFlush( + opts *Options, cur *version, baseLevel int, flushing flushableList, beganAt time.Time, +) *compaction { c := &compaction{ kind: compactionKindFlush, cmp: opts.Comparer.Compare, @@ -653,6 +835,7 @@ func newFlush(opts *Options, cur *version, baseLevel int, flushing flushableList formatKey: opts.Comparer.FormatKey, logger: opts.Logger, version: cur, + beganAt: beganAt, inputs: []compactionLevel{{level: -1}, {level: 0}}, maxOutputFileSize: math.MaxUint64, maxOverlapBytes: math.MaxUint64, @@ -660,6 +843,25 @@ func newFlush(opts *Options, cur *version, baseLevel int, flushing flushableList } c.startLevel = &c.inputs[0] c.outputLevel = &c.inputs[1] + + if len(flushing) > 0 { + if _, ok := flushing[0].flushable.(*ingestedFlushable); ok { + if len(flushing) != 1 { + panic("pebble: ingestedFlushable must be flushed one at a time.") + } + c.kind = compactionKindIngestedFlushable + return c + } + } + + // Make sure there's no ingestedFlushable after the first flushable in the + // list. + for _, f := range flushing { + if _, ok := f.flushable.(*ingestedFlushable); ok { + panic("pebble: flushing shouldn't contain ingestedFlushable flushable") + } + } + if cur.L0Sublevels != nil { c.l0Limits = cur.L0Sublevels.FlushSplitKeys() } @@ -1033,7 +1235,7 @@ func (c *compaction) newInputIter( } if rangeKeyIter := f.newRangeKeyIter(nil); rangeKeyIter != nil { mi := &keyspan.MergingIter{} - mi.Init(c.cmp, rangeKeyCompactionTransform(snapshots, c.elideRangeKey), rangeKeyIter) + mi.Init(c.cmp, rangeKeyCompactionTransform(c.equal, snapshots, c.elideRangeKey), new(keyspan.MergingBuffers), rangeKeyIter) c.rangeKeyInterleaving.Init(c.comparer, iter, mi, nil /* hooks */, nil /* lowerBound */, nil /* upperBound */) iter = &c.rangeKeyInterleaving } @@ -1060,7 +1262,7 @@ func (c *compaction) newInputIter( var iter internalIterator = newMergingIter(c.logger, &c.stats, c.cmp, nil, iters...) if len(rangeKeyIters) > 0 { mi := &keyspan.MergingIter{} - mi.Init(c.cmp, rangeKeyCompactionTransform(snapshots, c.elideRangeKey), rangeKeyIters...) + mi.Init(c.cmp, rangeKeyCompactionTransform(c.equal, snapshots, c.elideRangeKey), new(keyspan.MergingBuffers), rangeKeyIters...) c.rangeKeyInterleaving.Init(c.comparer, iter, mi, nil /* hooks */, nil /* lowerBound */, nil /* upperBound */) iter = &c.rangeKeyInterleaving } @@ -1133,8 +1335,8 @@ func (c *compaction) newInputIter( newRangeDelIter := func( f manifest.LevelFile, _ *IterOptions, bytesIterated *uint64, ) (keyspan.FragmentIterator, error) { - iter, rangeDelIter, err := newIters(f.FileMetadata, nil, /* iter options */ - internalIterOpts{bytesIterated: &c.bytesIterated}) + iter, rangeDelIter, err := newIters(context.Background(), f.FileMetadata, + nil /* iter options */, internalIterOpts{bytesIterated: &c.bytesIterated}) if err == nil { // TODO(peter): It is mildly wasteful to open the point iterator only to // immediately close it. One way to solve this would be to add new @@ -1200,7 +1402,9 @@ func (c *compaction) newInputIter( // of range tombstones outside the file's internal key bounds. Skip // any range tombstones completely outside file bounds. rangeDelIter = keyspan.Truncate( - c.cmp, rangeDelIter, lowerBound.UserKey, upperBound.UserKey, &f.Smallest, &f.Largest) + c.cmp, rangeDelIter, lowerBound.UserKey, upperBound.UserKey, + &f.Smallest, &f.Largest, false, /* panicOnPartialOverlap */ + ) } if rangeDelIter == nil { rangeDelIter = emptyKeyspanIter @@ -1291,7 +1495,7 @@ func (c *compaction) newInputIter( } return iter, err } - li.Init(keyspan.SpanIterOptions{}, c.cmp, newRangeKeyIterWrapper, level.files.Iter(), l, c.logger, manifest.KeyTypeRange) + li.Init(keyspan.SpanIterOptions{}, c.cmp, newRangeKeyIterWrapper, level.files.Iter(), l, manifest.KeyTypeRange) rangeKeyIters = append(rangeKeyIters, li) } return nil @@ -1330,9 +1534,9 @@ func (c *compaction) newInputIter( pointKeyIter := newMergingIter(c.logger, &c.stats, c.cmp, nil, iters...) if len(rangeKeyIters) > 0 { mi := &keyspan.MergingIter{} - mi.Init(c.cmp, rangeKeyCompactionTransform(snapshots, c.elideRangeKey), rangeKeyIters...) + mi.Init(c.cmp, rangeKeyCompactionTransform(c.equal, snapshots, c.elideRangeKey), new(keyspan.MergingBuffers), rangeKeyIters...) di := &keyspan.DefragmentingIter{} - di.Init(c.comparer, mi, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer) + di.Init(c.comparer, mi, keyspan.DefragmentInternal, keyspan.StaticDefragmentReducer, new(keyspan.DefragmentingBuffers)) c.rangeKeyInterleaving.Init(c.comparer, pointKeyIter, di, nil /* hooks */, nil /* lowerBound */, nil /* upperBound */) return &c.rangeKeyInterleaving, nil } @@ -1445,7 +1649,8 @@ func (d *DB) addInProgressCompaction(c *compaction) { // // DB.mu must be held when calling this method. All writes to the manifest for // this compaction should have completed by this point. -func (d *DB) removeInProgressCompaction(c *compaction, rollback bool) { +func (d *DB) clearCompactingState(c *compaction, rollback bool) { + c.versionEditApplied = true for _, cl := range c.inputs { iter := cl.files.Iter() for f := iter.First(); f != nil; f = iter.Next() { @@ -1469,24 +1674,22 @@ func (d *DB) removeInProgressCompaction(c *compaction, rollback bool) { f.IsIntraL0Compacting = false } } - delete(d.mu.compact.inProgress, c) - l0InProgress := inProgressL0Compactions(d.getInProgressCompactionInfoLocked(c)) d.mu.versions.currentVersion().L0Sublevels.InitCompactingFileInfo(l0InProgress) } func (d *DB) calculateDiskAvailableBytes() uint64 { if space, err := d.opts.FS.GetDiskUsage(d.dirname); err == nil { - atomic.StoreUint64(&d.atomic.diskAvailBytes, space.AvailBytes) + d.diskAvailBytes.Store(space.AvailBytes) return space.AvailBytes } else if !errors.Is(err, vfs.ErrUnsupported) { d.opts.EventListener.BackgroundError(err) } - return atomic.LoadUint64(&d.atomic.diskAvailBytes) + return d.diskAvailBytes.Load() } func (d *DB) getDiskAvailableBytesCached() uint64 { - return atomic.LoadUint64(&d.atomic.diskAvailBytes) + return d.diskAvailBytes.Load() } func (d *DB) getDeletionPacerInfo() deletionPacerInfo { @@ -1630,14 +1833,110 @@ func (d *DB) flush() { }) } +// runIngestFlush is used to generate a flush version edit for sstables which +// were ingested as flushables. Both DB.mu and the manifest lock must be held +// while runIngestFlush is called. +func (d *DB) runIngestFlush(c *compaction) (*manifest.VersionEdit, error) { + if len(c.flushing) != 1 { + panic("pebble: ingestedFlushable must be flushed one at a time.") + } + + // Construct the VersionEdit, levelMetrics etc. + c.metrics = make(map[int]*LevelMetrics, numLevels) + // Finding the target level for ingestion must use the latest version + // after the logLock has been acquired. + c.version = d.mu.versions.currentVersion() + + baseLevel := d.mu.versions.picker.getBaseLevel() + iterOpts := IterOptions{logger: d.opts.Logger} + ve := &versionEdit{} + var level int + var err error + for _, file := range c.flushing[0].flushable.(*ingestedFlushable).files { + level, err = ingestTargetLevel( + d.newIters, d.tableNewRangeKeyIter, iterOpts, d.cmp, + c.version, baseLevel, d.mu.compact.inProgress, file.FileMetadata, + ) + if err != nil { + return nil, err + } + ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: level, Meta: file.FileMetadata}) + levelMetrics := c.metrics[level] + if levelMetrics == nil { + levelMetrics = &LevelMetrics{} + c.metrics[level] = levelMetrics + } + levelMetrics.NumFiles++ + levelMetrics.Size += int64(file.Size) + levelMetrics.BytesIngested += file.Size + levelMetrics.TablesIngested++ + } + + return ve, nil +} + // flush runs a compaction that copies the immutable memtables from memory to // disk. // // d.mu must be held when calling this, but the mutex may be dropped and // re-acquired during the course of this method. func (d *DB) flush1() (bytesFlushed uint64, err error) { - var n int + // NB: The flushable queue can contain flushables of type ingestedFlushable. + // The sstables in ingestedFlushable.files must be placed into the appropriate + // level in the lsm. Let's say the flushable queue contains a prefix of + // regular immutable memtables, then an ingestedFlushable, and then the + // mutable memtable. When the flush of the ingestedFlushable is performed, + // it needs an updated view of the lsm. That is, the prefix of immutable + // memtables must have already been flushed. Similarly, if there are two + // contiguous ingestedFlushables in the queue, then the first flushable must + // be flushed, so that the second flushable can see an updated view of the + // lsm. + // + // Given the above, we restrict flushes to either some prefix of regular + // memtables, or a single flushable of type ingestedFlushable. The DB.flush + // function will call DB.maybeScheduleFlush again, so a new flush to finish + // the remaining flush work should be scheduled right away. + // + // NB: Large batches placed in the flushable queue share the WAL with the + // previous memtable in the queue. We must ensure the property that both the + // large batch and the memtable with which it shares a WAL are flushed + // together. The property ensures that the minimum unflushed log number + // isn't incremented incorrectly. Since a flushableBatch.readyToFlush always + // returns true, and since the large batch will always be placed right after + // the memtable with which it shares a WAL, the property is naturally + // ensured. The large batch will always be placed after the memtable with + // which it shares a WAL because we ensure it in DB.commitWrite by holding + // the commitPipeline.mu and then holding DB.mu. As an extra defensive + // measure, if we try to flush the memtable without also flushing the + // flushable batch in the same flush, since the memtable and flushableBatch + // have the same logNum, the errFlushInvariant check below will trigger and + // prevent the flush from continuing. + var n, inputs int + var ingest bool for ; n < len(d.mu.mem.queue)-1; n++ { + if f, ok := d.mu.mem.queue[n].flushable.(*ingestedFlushable); ok { + if n == 0 { + // The first flushable is of type ingestedFlushable. Since these + // must be flushed individually, we perform a flush for just + // this. + if !f.readyForFlush() { + // This check is almost unnecessary, but we guard against it + // just in case this invariant changes in the future. + panic("pebble: ingestedFlushable should always be ready to flush.") + } + // By setting n = 1, we ensure that the first flushable(n == 0) + // is scheduled for a flush. The number of tables added is equal to the + // number of files in the ingest operation. + n = 1 + inputs = len(f.files) + ingest = true + break + } else { + // There was some prefix of flushables which weren't of type + // ingestedFlushable. So, perform a flush for those. + break + } + } if !d.mu.mem.queue[n].readyForFlush() { break } @@ -1646,6 +1945,11 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { // None of the immutable memtables are ready for flushing. return 0, nil } + if !ingest { + // Flushes of memtables add the prefix of n memtables from the flushable + // queue. + inputs = n + } // Require that every memtable being flushed has a log number less than the // new minimum unflushed log number. @@ -1660,30 +1964,56 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { } c := newFlush(d.opts, d.mu.versions.currentVersion(), - d.mu.versions.picker.getBaseLevel(), d.mu.mem.queue[:n]) + d.mu.versions.picker.getBaseLevel(), d.mu.mem.queue[:n], d.timeNow()) d.addInProgressCompaction(c) jobID := d.mu.nextJobID d.mu.nextJobID++ d.opts.EventListener.FlushBegin(FlushInfo{ - JobID: jobID, - Input: n, + JobID: jobID, + Input: inputs, + Ingest: ingest, }) startTime := d.timeNow() - ve, pendingOutputs, err := d.runCompaction(jobID, c) + var ve *manifest.VersionEdit + var pendingOutputs []physicalMeta + var stats compactStats + // To determine the target level of the files in the ingestedFlushable, we + // need to acquire the logLock, and not release it for that duration. Since, + // we need to acquire the logLock below to perform the logAndApply step + // anyway, we create the VersionEdit for ingestedFlushable outside of + // runCompaction. For all other flush cases, we construct the VersionEdit + // inside runCompaction. + if c.kind != compactionKindIngestedFlushable { + ve, pendingOutputs, stats, err = d.runCompaction(jobID, c) + } + + // Acquire logLock. This will be released either on an error, by way of + // logUnlock, or through a call to logAndApply if there is no error. + d.mu.versions.logLock() + + if c.kind == compactionKindIngestedFlushable { + ve, err = d.runIngestFlush(c) + } info := FlushInfo{ JobID: jobID, - Input: n, + Input: inputs, Duration: d.timeNow().Sub(startTime), Done: true, + Ingest: ingest, Err: err, } if err == nil { for i := range ve.NewFiles { e := &ve.NewFiles[i] info.Output = append(info.Output, e.Meta.TableInfo()) + // Ingested tables are not necessarily flushed to L0. Record the level of + // each ingested file explicitly. + if ingest { + info.IngestLevels = append(info.IngestLevels, e.Level) + } } if len(ve.NewFiles) == 0 { info.Err = errEmptyTable @@ -1693,25 +2023,53 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { // want to bump the minimum unflushed log number to the log number of the // oldest unflushed memtable. ve.MinUnflushedLogNum = minUnflushedLogNum - metrics := c.metrics[0] - for i := 0; i < n; i++ { - metrics.BytesIn += d.mu.mem.queue[i].logSize + if c.kind != compactionKindIngestedFlushable { + metrics := c.metrics[0] + if d.opts.DisableWAL { + // If the WAL is disabled, every flushable has a zero [logSize], + // resulting in zero bytes in. Instead, use the number of bytes we + // flushed as the BytesIn. This ensures we get a reasonable w-amp + // calculation even when the WAL is disabled. + metrics.BytesIn = metrics.BytesFlushed + } else { + metrics := c.metrics[0] + for i := 0; i < n; i++ { + metrics.BytesIn += d.mu.mem.queue[i].logSize + } + } } - - d.mu.versions.logLock() err = d.mu.versions.logAndApply(jobID, ve, c.metrics, false, /* forceRotation */ func() []compactionInfo { return d.getInProgressCompactionInfoLocked(c) }) if err != nil { info.Err = err // TODO(peter): untested. - d.mu.versions.obsoleteTables = append(d.mu.versions.obsoleteTables, pendingOutputs...) - d.mu.versions.incrementObsoleteTablesLocked(pendingOutputs) + for _, f := range pendingOutputs { + // Note that the FileBacking for the file metadata might not have + // been set yet. So, we directly use the FileNum. Since these + // files were generated as compaction outputs, these must be + // physical files on disk. This property might not hold once + // https://github.com/cockroachdb/pebble/issues/389 is + // implemented if #389 creates virtual sstables as output files. + d.mu.versions.obsoleteTables = append( + d.mu.versions.obsoleteTables, + fileInfo{f.FileNum.DiskFileNum(), f.Size}, + ) + } + d.mu.versions.updateObsoleteTableMetricsLocked() } + } else { + // We won't be performing the logAndApply step because of the error, + // so logUnlock. + d.mu.versions.logUnlock() } bytesFlushed = c.bytesIterated + d.mu.snapshots.cumulativePinnedCount += stats.cumulativePinnedKeys + d.mu.snapshots.cumulativePinnedSize += stats.cumulativePinnedSize + d.maybeUpdateDeleteCompactionHints(c) - d.removeInProgressCompaction(c, err != nil) + d.clearCompactingState(c, err != nil) + delete(d.mu.compact.inProgress, c) d.mu.versions.incrementCompactions(c.kind, c.extraLevels) d.mu.versions.incrementCompactionBytes(-c.bytesWritten) @@ -1721,6 +2079,13 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { d.mu.mem.queue = d.mu.mem.queue[n:] d.updateReadStateLocked(d.opts.DebugCheck) d.updateTableStatsLocked(ve.NewFiles) + if ingest { + d.mu.versions.metrics.Flush.AsIngestCount++ + for _, l := range c.metrics { + d.mu.versions.metrics.Flush.AsIngestBytes += l.BytesIngested + d.mu.versions.metrics.Flush.AsIngestTableCount += l.TablesIngested + } + } } // Signal FlushEnd after installing the new readState. This helps for unit // tests that use the callback to trigger a read using an iterator with @@ -1728,6 +2093,15 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { info.TotalDuration = d.timeNow().Sub(startTime) d.opts.EventListener.FlushEnd(info) + // The order of these operations matters here for ease of testing. + // Removing the reader reference first allows tests to be guaranteed that + // the memtable reservation has been released by the time a synchronous + // flush returns. readerUnrefLocked may also produce obsolete files so the + // call to deleteObsoleteFiles must happen after it. + for i := range flushed { + flushed[i].readerUnrefLocked(true) + } + d.deleteObsoleteFiles(jobID, false /* waitForOngoing */) // Mark all the memtables we flushed as flushed. Note that we do this last so @@ -1737,13 +2111,9 @@ func (d *DB) flush1() (bytesFlushed uint64, err error) { // is not marked as completed until the deletion of obsolete files job has // completed. for i := range flushed { - // The order of these operations matters here for ease of testing. Removing - // the reader reference first allows tests to be guaranteed that the - // memtable reservation has been released by the time a synchronous flush - // returns. - flushed[i].readerUnref() close(flushed[i].flushed) } + return bytesFlushed, err } @@ -1814,7 +2184,8 @@ func (d *DB) maybeScheduleCompactionPicker( // Check for delete-only compactions first, because they're expected to be // cheap and reduce future compaction work. - if len(d.mu.compact.deletionHints) > 0 && + if !d.opts.private.disableDeleteOnlyCompactions && + len(d.mu.compact.deletionHints) > 0 && d.mu.compact.compactingCount < maxConcurrentCompactions && !d.opts.DisableAutomaticCompactions { v := d.mu.versions.currentVersion() @@ -1823,7 +2194,7 @@ func (d *DB) maybeScheduleCompactionPicker( d.mu.compact.deletionHints = unresolvedHints if len(inputs) > 0 { - c := newDeleteOnlyCompaction(d.opts, v, inputs) + c := newDeleteOnlyCompaction(d.opts, v, inputs, d.timeNow()) d.mu.compact.compactingCount++ d.addInProgressCompaction(c) go d.compact(c, nil) @@ -1835,7 +2206,7 @@ func (d *DB) maybeScheduleCompactionPicker( env.inProgressCompactions = d.getInProgressCompactionInfoLocked(nil) pc, retryLater := d.mu.versions.picker.pickManual(env, manual) if pc != nil { - c := newCompaction(pc, d.opts) + c := newCompaction(pc, d.opts, d.timeNow()) d.mu.compact.manual = d.mu.compact.manual[1:] d.mu.compact.compactingCount++ d.addInProgressCompaction(c) @@ -1862,7 +2233,7 @@ func (d *DB) maybeScheduleCompactionPicker( if pc == nil { break } - c := newCompaction(pc, d.opts) + c := newCompaction(pc, d.opts, d.timeNow()) d.mu.compact.compactingCount++ d.addInProgressCompaction(c) go d.compact(c, nil) @@ -2152,6 +2523,13 @@ func (d *DB) compact(c *compaction, errChannel chan error) { d.opts.EventListener.BackgroundError(err) } d.mu.compact.compactingCount-- + delete(d.mu.compact.inProgress, c) + // Add this compaction's duration to the cumulative duration. NB: This + // must be atomic with the above removal of c from + // d.mu.compact.InProgress to ensure Metrics.Compact.Duration does not + // miss or double count a completing compaction's duration. + d.mu.compact.duration += d.timeNow().Sub(c.beganAt) + // The previous compaction may have produced too many files in a // level, so reschedule another compaction if needed. d.maybeScheduleCompaction() @@ -2176,7 +2554,7 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { d.opts.EventListener.CompactionBegin(info) startTime := d.timeNow() - ve, pendingOutputs, err := d.runCompaction(jobID, c) + ve, pendingOutputs, stats, err := d.runCompaction(jobID, c) info.Duration = d.timeNow().Sub(startTime) if err == nil { @@ -2186,8 +2564,19 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { }) if err != nil { // TODO(peter): untested. - d.mu.versions.obsoleteTables = append(d.mu.versions.obsoleteTables, pendingOutputs...) - d.mu.versions.incrementObsoleteTablesLocked(pendingOutputs) + for _, f := range pendingOutputs { + // Note that the FileBacking for the file metadata might not have + // been set yet. So, we directly use the FileNum. Since these + // files were generated as compaction outputs, these must be + // physical files on disk. This property might not hold once + // https://github.com/cockroachdb/pebble/issues/389 is + // implemented if #389 creates virtual sstables as output files. + d.mu.versions.obsoleteTables = append( + d.mu.versions.obsoleteTables, + fileInfo{f.FileNum.DiskFileNum(), f.Size}, + ) + } + d.mu.versions.updateObsoleteTableMetricsLocked() } } @@ -2200,12 +2589,16 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { } } + d.mu.snapshots.cumulativePinnedCount += stats.cumulativePinnedKeys + d.mu.snapshots.cumulativePinnedSize += stats.cumulativePinnedSize d.maybeUpdateDeleteCompactionHints(c) - d.removeInProgressCompaction(c, err != nil) + // NB: clearing compacting state must occur before updating the read state; + // L0Sublevels initialization depends on it. + d.clearCompactingState(c, err != nil) d.mu.versions.incrementCompactions(c.kind, c.extraLevels) d.mu.versions.incrementCompactionBytes(-c.bytesWritten) - info.TotalDuration = d.timeNow().Sub(startTime) + info.TotalDuration = d.timeNow().Sub(c.beganAt) d.opts.EventListener.CompactionEnd(info) // Update the read state before deleting obsolete files because the @@ -2221,6 +2614,11 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { return err } +type compactStats struct { + cumulativePinnedKeys uint64 + cumulativePinnedSize uint64 +} + // runCompactions runs a compaction that produces new on-disk tables from // memtables or old on-disk tables. // @@ -2228,12 +2626,15 @@ func (d *DB) compact1(c *compaction, errChannel chan error) (err error) { // re-acquired during the course of this method. func (d *DB) runCompaction( jobID int, c *compaction, -) (ve *versionEdit, pendingOutputs []*fileMetadata, retErr error) { +) (ve *versionEdit, pendingOutputs []physicalMeta, stats compactStats, retErr error) { // As a sanity check, confirm that the smallest / largest keys for new and // deleted files in the new versionEdit pass a validation function before // returning the edit. defer func() { - if ve != nil { + // If we're handling a panic, don't expect the version edit to validate. + if r := recover(); r != nil { + panic(r) + } else if ve != nil { err := validateVersionEdit(ve, d.opts.Experimental.KeyValidationFunc, d.opts.Comparer.FormatKey) if err != nil { d.opts.Logger.Fatalf("pebble: version edit validation failed: %s", err) @@ -2261,7 +2662,11 @@ func (d *DB) runCompaction( } c.metrics[cl.level] = levelMetrics } - return ve, nil, nil + return ve, nil, stats, nil + } + + if c.kind == compactionKindIngestedFlushable { + panic("pebble: runCompaction cannot handle compactionKindIngestedFlushable.") } // Check for a trivial move of one table from one level to the next. We avoid @@ -2291,7 +2696,7 @@ func (d *DB) runCompaction( {Level: c.outputLevel.level, Meta: meta}, }, } - return ve, nil, nil + return ve, nil, stats, nil } defer func() { @@ -2302,9 +2707,6 @@ func (d *DB) runCompaction( snapshots := d.mu.snapshots.toSlice() formatVers := d.mu.formatVers.vers - // The table is written at the maximum allowable format implied by the current - // format major version of the DB. - tableFormat := formatVers.MaxTableFormat() // Release the d.mu lock while doing I/O. // Note the unusual order: Unlock and then Lock. @@ -2313,7 +2715,7 @@ func (d *DB) runCompaction( iiter, err := c.newInputIter(d.newIters, d.tableNewRangeKeyIter, snapshots) if err != nil { - return nil, pendingOutputs, err + return nil, pendingOutputs, stats, err } c.allowedZeroSeqNum = c.allowZeroSeqNum() iter := newCompactionIter(c.cmp, c.equal, c.formatKey, d.merge, iiter, snapshots, @@ -2321,8 +2723,11 @@ func (d *DB) runCompaction( c.elideRangeTombstone, d.FormatMajorVersion()) var ( - filenames []string - tw *sstable.Writer + createdFiles []base.DiskFileNum + tw *sstable.Writer + pinnedKeySize uint64 + pinnedValueSize uint64 + pinnedCount uint64 ) defer func() { if iter != nil { @@ -2332,8 +2737,8 @@ func (d *DB) runCompaction( retErr = firstError(retErr, tw.Close()) } if retErr != nil { - for _, filename := range filenames { - d.opts.FS.Remove(filename) + for _, fileNum := range createdFiles { + _ = d.objProvider.Remove(fileTypeTable, fileNum) } } for _, closer := range c.closers { @@ -2364,6 +2769,19 @@ func (d *DB) runCompaction( c.metrics[c.extraLevels[0].level] = &LevelMetrics{} } + // The table is typically written at the maximum allowable format implied by + // the current format major version of the DB. + tableFormat := formatVers.MaxTableFormat() + + // In format major versions with maximum table formats of Pebblev3, value + // blocks were conditional on an experimental setting. In format major + // versions with maximum table formats of Pebblev4 and higher, value blocks + // are always enabled. + if tableFormat == sstable.TableFormatPebblev3 && + (d.opts.Experimental.EnableValueBlocks == nil || !d.opts.Experimental.EnableValueBlocks()) { + tableFormat = sstable.TableFormatPebblev2 + } + writerOpts := d.opts.MakeWriterOptions(c.outputLevel.level, tableFormat) if formatVers < FormatBlockPropertyCollector { // Cannot yet write block properties. @@ -2391,14 +2809,37 @@ func (d *DB) runCompaction( d.mu.Lock() fileNum := d.mu.versions.getNextFileNum() fileMeta.FileNum = fileNum - pendingOutputs = append(pendingOutputs, fileMeta) + pendingOutputs = append(pendingOutputs, fileMeta.PhysicalMeta()) d.mu.Unlock() - filename := base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, fileNum) - file, err := d.opts.FS.Create(filename) + ctx := context.TODO() + if objiotracing.Enabled { + ctx = objiotracing.WithLevel(ctx, c.outputLevel.level) + switch c.kind { + case compactionKindFlush: + ctx = objiotracing.WithReason(ctx, objiotracing.ForFlush) + case compactionKindIngestedFlushable: + ctx = objiotracing.WithReason(ctx, objiotracing.ForIngestion) + default: + ctx = objiotracing.WithReason(ctx, objiotracing.ForCompaction) + } + } + // Prefer shared storage if present. + // + // TODO(bilal): This might be inefficient for short-lived files in higher + // levels if we're only writing to shared storage and not double-writing + // to local storage. Either implement double-writing functionality, or + // set PreferSharedStorage to c.outputLevel.level >= 5. The latter needs + // some careful handling around move compactions to ensure all files in + // lower levels are in shared storage. + createOpts := objstorage.CreateOptions{ + PreferSharedStorage: true, + } + writable, objMeta, err := d.objProvider.Create(ctx, fileTypeTable, fileNum.DiskFileNum(), createOpts) if err != nil { return err } + reason := "flushing" if c.flushing == nil { reason = "compacting" @@ -2406,21 +2847,16 @@ func (d *DB) runCompaction( d.opts.EventListener.TableCreated(TableCreateInfo{ JobID: jobID, Reason: reason, - Path: filename, + Path: d.objProvider.Path(objMeta), FileNum: fileNum, }) - file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{ - NoSyncOnClose: d.opts.NoSyncOnClose, - BytesPerSync: d.opts.BytesPerSync, - }) - file = &compactionFile{ - File: file, + writable = &compactionWritable{ + Writable: writable, versions: d.mu.versions, written: &c.bytesWritten, } - filenames = append(filenames, filename) - cacheOpts := private.SSTableCacheOpts(d.cacheID, fileNum).(sstable.WriterOption) - internalTableOpt := private.SSTableInternalTableOpt.(sstable.WriterOption) + createdFiles = append(createdFiles, fileNum.DiskFileNum()) + cacheOpts := private.SSTableCacheOpts(d.cacheID, fileNum.DiskFileNum()).(sstable.WriterOption) const MaxFileWriteAdditionalCPUTime = time.Millisecond * 100 cpuWorkHandle = d.opts.Experimental.CPUWorkPermissionGranter.GetPermission( @@ -2430,7 +2866,7 @@ func (d *DB) runCompaction( d.opts.Experimental.MaxWriterConcurrency > 0 && (cpuWorkHandle.Permitted() || d.opts.Experimental.ForceWriterParallelism) - tw = sstable.NewWriter(file, writerOpts, cacheOpts, internalTableOpt, &prevPointKey) + tw = sstable.NewWriter(writable, writerOpts, cacheOpts, &prevPointKey) fileMeta.CreationTime = time.Now().Unix() ve.NewFiles = append(ve.NewFiles, newFileEntry{ @@ -2529,7 +2965,22 @@ func (d *DB) runCompaction( if tw == nil { return nil } - + { + // Set internal sstable properties. + p := getInternalWriterProperties(tw) + // Set the external sst version to 0. This is what RocksDB expects for + // db-internal sstables; otherwise, it could apply a global sequence number. + p.ExternalFormatVersion = 0 + // Set the snapshot pinned totals. + p.SnapshotPinnedKeys = pinnedCount + p.SnapshotPinnedKeySize = pinnedKeySize + p.SnapshotPinnedValueSize = pinnedValueSize + stats.cumulativePinnedKeys += pinnedCount + stats.cumulativePinnedSize += pinnedKeySize + pinnedValueSize + pinnedCount = 0 + pinnedKeySize = 0 + pinnedValueSize = 0 + } if err := tw.Close(); err != nil { tw = nil return err @@ -2546,9 +2997,13 @@ func (d *DB) runCompaction( meta.Size = writerMeta.Size meta.SmallestSeqNum = writerMeta.SmallestSeqNum meta.LargestSeqNum = writerMeta.LargestSeqNum + meta.InitPhysicalBacking() + // If the file didn't contain any range deletions, we can fill its // table stats now, avoiding unnecessarily loading the table later. - maybeSetStatsFromProperties(meta, &writerMeta.Properties) + maybeSetStatsFromProperties( + meta.PhysicalMeta(), &writerMeta.Properties, + ) if c.flushing == nil { outputMetrics.TablesCompacted++ @@ -2559,6 +3014,8 @@ func (d *DB) runCompaction( } outputMetrics.Size += int64(meta.Size) outputMetrics.NumFiles++ + outputMetrics.Additional.BytesWrittenDataBlocks += writerMeta.Properties.DataSize + outputMetrics.Additional.BytesWrittenValueBlocks += writerMeta.Properties.ValueBlocksSize if n := len(ve.NewFiles); n > 1 { // This is not the first output file. Ensure the sstable boundaries @@ -2643,35 +3100,37 @@ func (d *DB) runCompaction( return nil } - // compactionOutputSplitters contain all logic to determine whether the - // compaction loop should stop writing to one output sstable and switch to - // a new one. Some splitters can wrap other splitters, and - // the splitterGroup can be composed of multiple splitters. In this case, - // we start off with splitters for file sizes, grandparent limits, and (for - // L0 splits) L0 limits, before wrapping them in an splitterGroup. + // Build a compactionOutputSplitter that contains all logic to determine + // whether the compaction loop should stop writing to one output sstable and + // switch to a new one. Some splitters can wrap other splitters, and the + // splitterGroup can be composed of multiple splitters. In this case, we + // start off with splitters for file sizes, grandparent limits, and (for L0 + // splits) L0 limits, before wrapping them in an splitterGroup. + sizeSplitter := newFileSizeSplitter(&iter.frontiers, c.maxOutputFileSize, c.grandparents.Iter()) + unsafePrevUserKey := func() []byte { + // Return the largest point key written to tw or the start of + // the current range deletion in the fragmenter, whichever is + // greater. + prevPoint := prevPointKey.UnsafeKey() + if c.cmp(prevPoint.UserKey, c.rangeDelFrag.Start()) > 0 { + return prevPoint.UserKey + } + return c.rangeDelFrag.Start() + } outputSplitters := []compactionOutputSplitter{ // We do not split the same user key across different sstables within // one flush or compaction. The fileSizeSplitter may request a split in // the middle of a user key, so the userKeyChangeSplitter ensures we are // at a user key change boundary when doing a split. &userKeyChangeSplitter{ - cmp: c.cmp, - splitter: &fileSizeSplitter{maxFileSize: c.maxOutputFileSize}, - unsafePrevUserKey: func() []byte { - // Return the largest point key written to tw or the start of - // the current range deletion in the fragmenter, whichever is - // greater. - prevPoint := prevPointKey.UnsafeKey() - if c.cmp(prevPoint.UserKey, c.rangeDelFrag.Start()) > 0 { - return prevPoint.UserKey - } - return c.rangeDelFrag.Start() - }, + cmp: c.cmp, + splitter: sizeSplitter, + unsafePrevUserKey: unsafePrevUserKey, }, - &limitFuncSplitter{c: c, limitFunc: c.findGrandparentLimit}, + newLimitFuncSplitter(&iter.frontiers, c.findGrandparentLimit), } if splitL0Outputs { - outputSplitters = append(outputSplitters, &limitFuncSplitter{c: c, limitFunc: c.findL0Limit}) + outputSplitters = append(outputSplitters, newLimitFuncSplitter(&iter.frontiers, c.findL0Limit)) } splitter := &splitterGroup{cmp: c.cmp, splitters: outputSplitters} @@ -2684,7 +3143,28 @@ func (d *DB) runCompaction( // progress guarantees ensure that eventually the input iterator will be // exhausted and the range tombstone fragments will all be flushed. for key, val := iter.First(); key != nil || !c.rangeDelFrag.Empty() || !c.rangeKeyFrag.Empty(); { - splitterSuggestion := splitter.onNewOutput(key) + var firstKey []byte + if key != nil { + firstKey = key.UserKey + } else if startKey := c.rangeDelFrag.Start(); startKey != nil { + // Pass the start key of the first pending tombstone to find the + // next limit. All pending tombstones have the same start key. We + // use this as opposed to the end key of the last written sstable to + // effectively handle cases like these: + // + // a.SET.3 + // (lf.limit at b) + // d.RANGEDEL.4:f + // + // In this case, the partition after b has only range deletions, so + // if we were to find the limit after the last written key at the + // split point (key a), we'd get the limit b again, and + // finishOutput() would not advance any further because the next + // range tombstone to write does not start until after the L0 split + // point. + firstKey = startKey + } + splitterSuggestion := splitter.onNewOutput(firstKey) // Each inner loop iteration processes one key from the input iterator. for ; key != nil; key, val = iter.Next() { @@ -2749,11 +3229,19 @@ func (d *DB) runCompaction( } if tw == nil { if err := newOutput(); err != nil { - return nil, pendingOutputs, err + return nil, pendingOutputs, stats, err } } if err := tw.Add(*key, val); err != nil { - return nil, pendingOutputs, err + return nil, pendingOutputs, stats, err + } + if iter.snapshotPinned { + // The kv pair we just added to the sstable was only surfaced by + // the compaction iterator because an open snapshot prevented + // its elision. Increment the stats. + pinnedCount++ + pinnedKeySize += uint64(len(key.UserKey)) + base.InternalTrailerLen + pinnedValueSize += uint64(len(val)) } } @@ -2781,7 +3269,7 @@ func (d *DB) runCompaction( splitKey = key.UserKey } if err := finishOutput(splitKey); err != nil { - return nil, pendingOutputs, err + return nil, pendingOutputs, stats, err } } @@ -2797,15 +3285,15 @@ func (d *DB) runCompaction( } } - if err := d.dataDir.Sync(); err != nil { - return nil, pendingOutputs, err + if err := d.objProvider.Sync(); err != nil { + return nil, pendingOutputs, stats, err } // Refresh the disk available statistic whenever a compaction/flush // completes, before re-acquiring the mutex. _ = d.calculateDiskAvailableBytes() - return ve, pendingOutputs, nil + return ve, pendingOutputs, stats, nil } // validateVersionEdit validates that start and end keys across new and deleted @@ -2852,74 +3340,97 @@ func (d *DB) scanObsoleteFiles(list []string) { d.opts.DisableAutomaticCompactions = true // Wait for any ongoing compaction to complete before continuing. - if d.mu.compact.compactingCount > 0 || d.mu.compact.flushing { + for d.mu.compact.compactingCount > 0 || d.mu.compact.flushing { d.mu.compact.cond.Wait() } - liveFileNums := make(map[FileNum]struct{}) + liveFileNums := make(map[base.DiskFileNum]struct{}) d.mu.versions.addLiveFileNums(liveFileNums) + // Protect against files which are only referred to by the ingestedFlushable + // from being deleted. These are added to the flushable queue on WAL replay + // during read only mode and aren't part of the Version. Note that if + // !d.opts.ReadOnly, then all flushables of type ingestedFlushable have + // already been flushed. + for _, fEntry := range d.mu.mem.queue { + if f, ok := fEntry.flushable.(*ingestedFlushable); ok { + for _, file := range f.files { + liveFileNums[file.FileBacking.DiskFileNum] = struct{}{} + } + } + } + minUnflushedLogNum := d.mu.versions.minUnflushedLogNum manifestFileNum := d.mu.versions.manifestFileNum var obsoleteLogs []fileInfo - var obsoleteTables []*fileMetadata + var obsoleteTables []fileInfo var obsoleteManifests []fileInfo var obsoleteOptions []fileInfo for _, filename := range list { - fileType, fileNum, ok := base.ParseFilename(d.opts.FS, filename) + fileType, diskFileNum, ok := base.ParseFilename(d.opts.FS, filename) if !ok { continue } switch fileType { case fileTypeLog: - if fileNum >= minUnflushedLogNum { + if diskFileNum.FileNum() >= minUnflushedLogNum { continue } - fi := fileInfo{fileNum: fileNum} + fi := fileInfo{fileNum: diskFileNum} if stat, err := d.opts.FS.Stat(filename); err == nil { fi.fileSize = uint64(stat.Size()) } obsoleteLogs = append(obsoleteLogs, fi) case fileTypeManifest: - if fileNum >= manifestFileNum { + if diskFileNum.FileNum() >= manifestFileNum { continue } - fi := fileInfo{fileNum: fileNum} + fi := fileInfo{fileNum: diskFileNum} if stat, err := d.opts.FS.Stat(filename); err == nil { fi.fileSize = uint64(stat.Size()) } obsoleteManifests = append(obsoleteManifests, fi) case fileTypeOptions: - if fileNum >= d.optionsFileNum { + if diskFileNum.FileNum() >= d.optionsFileNum.FileNum() { continue } - fi := fileInfo{fileNum: fileNum} + fi := fileInfo{fileNum: diskFileNum} if stat, err := d.opts.FS.Stat(filename); err == nil { fi.fileSize = uint64(stat.Size()) } obsoleteOptions = append(obsoleteOptions, fi) case fileTypeTable: - if _, ok := liveFileNums[fileNum]; ok { - continue - } - fileMeta := &fileMetadata{ - FileNum: fileNum, - } - if stat, err := d.opts.FS.Stat(filename); err == nil { - fileMeta.Size = uint64(stat.Size()) - } - obsoleteTables = append(obsoleteTables, fileMeta) + // Objects are handled through the objstorage provider below. default: // Don't delete files we don't know about. - continue + } + } + + objects := d.objProvider.List() + for _, obj := range objects { + switch obj.FileType { + case fileTypeTable: + if _, ok := liveFileNums[obj.DiskFileNum]; ok { + continue + } + fileInfo := fileInfo{ + fileNum: obj.DiskFileNum, + } + if size, err := d.objProvider.Size(obj); err == nil { + fileInfo.fileSize = uint64(size) + } + obsoleteTables = append(obsoleteTables, fileInfo) + + default: + // Ignore object types we don't know about. } } d.mu.log.queue = merge(d.mu.log.queue, obsoleteLogs) - d.mu.versions.metrics.WAL.Files += int64(len(obsoleteLogs)) - d.mu.versions.obsoleteTables = mergeFileMetas(d.mu.versions.obsoleteTables, obsoleteTables) - d.mu.versions.incrementObsoleteTablesLocked(obsoleteTables) + d.mu.versions.metrics.WAL.Files = int64(len(d.mu.log.queue)) + d.mu.versions.obsoleteTables = mergeFileInfo(d.mu.versions.obsoleteTables, obsoleteTables) + d.mu.versions.updateObsoleteTableMetricsLocked() d.mu.versions.obsoleteManifests = merge(d.mu.versions.obsoleteManifests, obsoleteManifests) d.mu.versions.obsoleteOptions = merge(d.mu.versions.obsoleteOptions, obsoleteOptions) } @@ -2982,9 +3493,9 @@ func (d *DB) releaseCleaningTurn() { d.mu.cleaner.cond.Broadcast() } -// deleteObsoleteFiles deletes those files that are no longer needed. If -// waitForOngoing is true, it waits for any ongoing cleaning turns to complete, -// and if false, it returns rightaway if a cleaning turn is ongoing. +// deleteObsoleteFiles deletes those files and objects that are no longer +// needed. If waitForOngoing is true, it waits for any ongoing cleaning turns to +// complete, and if false, it returns rightaway if a cleaning turn is ongoing. // // d.mu must be held when calling this, but the mutex may be dropped and // re-acquired during the course of this method. @@ -2999,13 +3510,13 @@ func (d *DB) deleteObsoleteFiles(jobID int, waitForOngoing bool) { // obsoleteFile holds information about a file that needs to be deleted soon. type obsoleteFile struct { dir string - fileNum base.FileNum + fileNum base.DiskFileNum fileType fileType fileSize uint64 } type fileInfo struct { - fileNum FileNum + fileNum base.DiskFileNum fileSize uint64 } @@ -3026,7 +3537,7 @@ func (d *DB) doDeleteObsoleteFiles(jobID int) { // log that has not had its contents flushed to an sstable. We can recycle // the prefix of d.mu.log.queue with log numbers less than // minUnflushedLogNum. - if d.mu.log.queue[i].fileNum >= d.mu.versions.minUnflushedLogNum { + if d.mu.log.queue[i].fileNum.FileNum() >= d.mu.versions.minUnflushedLogNum { obsoleteLogs = d.mu.log.queue[:i] d.mu.log.queue = d.mu.log.queue[i:] d.mu.versions.metrics.WAL.Files -= int64(len(obsoleteLogs)) @@ -3034,19 +3545,14 @@ func (d *DB) doDeleteObsoleteFiles(jobID int) { } } - for _, table := range d.mu.versions.obsoleteTables { - obsoleteTables = append(obsoleteTables, fileInfo{ - fileNum: table.FileNum, - fileSize: table.Size, - }) - } + obsoleteTables = append(obsoleteTables, d.mu.versions.obsoleteTables...) d.mu.versions.obsoleteTables = nil // Sort the manifests cause we want to delete some contiguous prefix // of the older manifests. sort.Slice(d.mu.versions.obsoleteManifests, func(i, j int) bool { - return d.mu.versions.obsoleteManifests[i].fileNum < - d.mu.versions.obsoleteManifests[j].fileNum + return d.mu.versions.obsoleteManifests[i].fileNum.FileNum() < + d.mu.versions.obsoleteManifests[j].fileNum.FileNum() }) var obsoleteManifests []fileInfo @@ -3082,7 +3588,7 @@ func (d *DB) doDeleteObsoleteFiles(jobID int) { // We sort to make the order of deletions deterministic, which is nice for // tests. sort.Slice(f.obsolete, func(i, j int) bool { - return f.obsolete[i].fileNum < f.obsolete[j].fileNum + return f.obsolete[i].fileNum.FileNum() < f.obsolete[j].fileNum.FileNum() }) for _, fi := range f.obsolete { dir := d.dirname @@ -3132,8 +3638,10 @@ func (d *DB) paceAndDeleteObsoleteFiles(jobID int, files []obsoleteFile) { d.mu.versions.metrics.Table.ObsoleteCount-- d.mu.versions.metrics.Table.ObsoleteSize -= of.fileSize d.mu.Unlock() + d.deleteObsoleteObject(fileTypeTable, jobID, of.fileNum) + } else { + d.deleteObsoleteFile(of.fileType, jobID, path, of.fileNum) } - d.deleteObsoleteFile(of.fileType, jobID, path, of.fileNum) } } @@ -3161,8 +3669,38 @@ func (d *DB) maybeScheduleObsoleteTableDeletion() { }() } -// deleteObsoleteFile deletes file that is no longer needed. -func (d *DB) deleteObsoleteFile(fileType fileType, jobID int, path string, fileNum FileNum) { +func (d *DB) deleteObsoleteObject(fileType fileType, jobID int, fileNum base.DiskFileNum) { + if fileType != fileTypeTable { + panic("not an object") + } + + var path string + meta, err := d.objProvider.Lookup(fileType, fileNum) + if err != nil { + path = "" + } else { + path = d.objProvider.Path(meta) + err = d.objProvider.Remove(fileType, fileNum) + } + if d.objProvider.IsNotExistError(err) { + return + } + + switch fileType { + case fileTypeTable: + d.opts.EventListener.TableDeleted(TableDeleteInfo{ + JobID: jobID, + Path: path, + FileNum: fileNum.FileNum(), + Err: err, + }) + } +} + +// deleteObsoleteFile deletes a (non-object) file that is no longer needed. +func (d *DB) deleteObsoleteFile( + fileType fileType, jobID int, path string, fileNum base.DiskFileNum, +) { // TODO(peter): need to handle this error, probably by re-adding the // file that couldn't be deleted to one of the obsolete slices map. err := d.opts.Cleaner.Clean(d.opts.FS, fileType, path) @@ -3175,23 +3713,18 @@ func (d *DB) deleteObsoleteFile(fileType fileType, jobID int, path string, fileN d.opts.EventListener.WALDeleted(WALDeleteInfo{ JobID: jobID, Path: path, - FileNum: fileNum, + FileNum: fileNum.FileNum(), Err: err, }) case fileTypeManifest: d.opts.EventListener.ManifestDeleted(ManifestDeleteInfo{ JobID: jobID, Path: path, - FileNum: fileNum, + FileNum: fileNum.FileNum(), Err: err, }) case fileTypeTable: - d.opts.EventListener.TableDeleted(TableDeleteInfo{ - JobID: jobID, - Path: path, - FileNum: fileNum, - Err: err, - }) + panic("invalid deletion of object file") } } @@ -3202,7 +3735,7 @@ func merge(a, b []fileInfo) []fileInfo { a = append(a, b...) sort.Slice(a, func(i, j int) bool { - return a[i].fileNum < a[j].fileNum + return a[i].fileNum.FileNum() < a[j].fileNum.FileNum() }) n := 0 @@ -3215,19 +3748,19 @@ func merge(a, b []fileInfo) []fileInfo { return a[:n] } -func mergeFileMetas(a, b []*fileMetadata) []*fileMetadata { +func mergeFileInfo(a, b []fileInfo) []fileInfo { if len(b) == 0 { return a } a = append(a, b...) sort.Slice(a, func(i, j int) bool { - return a[i].FileNum < a[j].FileNum + return a[i].fileNum.FileNum() < a[j].fileNum.FileNum() }) n := 0 for i := 0; i < len(a); i++ { - if n == 0 || a[i].FileNum != a[n-1].FileNum { + if n == 0 || a[i].fileNum != a[n-1].fileNum { a[n] = a[i] n++ } diff --git a/vendor/github.com/cockroachdb/pebble/compaction_iter.go b/vendor/github.com/cockroachdb/pebble/compaction_iter.go index cd8f972..e09e5b0 100644 --- a/vendor/github.com/cockroachdb/pebble/compaction_iter.go +++ b/vendor/github.com/cockroachdb/pebble/compaction_iter.go @@ -5,6 +5,7 @@ package pebble import ( + "bytes" "fmt" "io" "sort" @@ -77,22 +78,22 @@ import ( // is that snapshots define stripes and entries are collapsed within stripes, // but not across stripes. Consider the following scenario: // -// a.PUT.9 -// a.DEL.8 -// a.PUT.7 -// a.DEL.6 -// a.PUT.5 +// a.PUT.9 +// a.DEL.8 +// a.PUT.7 +// a.DEL.6 +// a.PUT.5 // // In the absence of snapshots these entries would be collapsed to // a.PUT.9. What if there is a snapshot at sequence number 7? The entries can // be divided into two stripes and collapsed within the stripes: // -// a.PUT.9 a.PUT.9 -// a.DEL.8 ---> -// a.PUT.7 -// -- -- -// a.DEL.6 ---> a.DEL.6 -// a.PUT.5 +// a.PUT.9 a.PUT.9 +// a.DEL.8 ---> +// a.PUT.7 +// -- -- +// a.DEL.6 ---> a.DEL.6 +// a.PUT.5 // // All of the rules described earlier still apply, but they are confined to // operate within a snapshot stripe. Snapshots only affect compaction when the @@ -111,13 +112,13 @@ import ( // subject to the rules for snapshots. For example, consider the two range // tombstones [a,e)#1 and [c,g)#2: // -// 2: c-------g -// 1: a-------e +// 2: c-------g +// 1: a-------e // // These tombstones will be fragmented into: // -// 2: c---e---g -// 1: a---c---e +// 2: c---e---g +// 1: a---c---e // // Do we output the fragment [c,e)#1? Since it is covered by [c-e]#2 the answer // depends on whether it is in a new snapshot stripe. @@ -174,14 +175,20 @@ type compactionIter struct { // advanced. valueBuf []byte // Is the current entry valid? - valid bool - iterKey *InternalKey - iterValue []byte + valid bool + iterKey *InternalKey + iterValue []byte + iterStripeChange stripeChangeType // `skip` indicates whether the remaining skippable entries in the current // snapshot stripe should be skipped or processed. An example of a non- // skippable entry is a range tombstone as we need to return it from the // `compactionIter`, even if a key covering its start key has already been // seen in the same stripe. `skip` has no effect when `pos == iterPosNext`. + // + // TODO(jackson): If we use keyspan.InterleavingIter for range deletions, + // like we do for range keys, the only remaining 'non-skippable' key is + // the invalid key. We should be able to simplify this logic and remove this + // field. skip bool // `pos` indicates the iterator position at the top of `Next()`. Its type's // (`iterPos`) values take on the following meanings in the context of @@ -194,6 +201,11 @@ type compactionIter struct { // user key to ensure we've seen all mergeable operands. // - `iterPosPrev`: this is invalid as compactionIter is forward-only. pos iterPos + // `snapshotPinned` indicates whether the last point key returned by the + // compaction iterator was only returned because an open snapshot prevents + // its elision. This field only applies to point keys, and not to range + // deletions or range keys. + snapshotPinned bool // The index of the snapshot for the current key within the snapshots slice. curSnapshotIdx int curSnapshotSeqNum uint64 @@ -201,6 +213,17 @@ type compactionIter struct { // numbers define the snapshot stripes (see the Snapshots description // above). The sequence numbers are in ascending order. snapshots []uint64 + // frontiers holds a heap of user keys that affect compaction behavior when + // they're exceeded. Before a new key is returned, the compaction iterator + // advances the frontier, notifying any code that subscribed to be notified + // when a key was reached. The primary use today is within the + // implementation of compactionOutputSplitters in compaction.go. Many of + // these splitters wait for the compaction iterator to call Advance(k) when + // it's returning a new key. If the key that they're waiting for is + // surpassed, these splitters update internal state recording that they + // should request a compaction split next time they're asked in + // [shouldSplitBefore]. + frontiers frontiers // Reference to the range deletion tombstone fragmenter (e.g., // `compaction.rangeDelFrag`). rangeDelFrag *keyspan.Fragmenter @@ -238,6 +261,7 @@ func newCompactionIter( merge: merge, iter: iter, snapshots: snapshots, + frontiers: frontiers{cmp: cmp}, rangeDelFrag: rangeDelFrag, rangeKeyFrag: rangeKeyFrag, allowZeroSeqNum: allowZeroSeqNum, @@ -268,6 +292,7 @@ func (i *compactionIter) First() (*InternalKey, []byte) { i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(i.iterKey.SeqNum(), i.snapshots) } i.pos = iterPosNext + i.iterStripeChange = newStripeNewKey return i.Next() } @@ -300,7 +325,19 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { i.pos = iterPosCurForward i.valid = false + for i.iterKey != nil { + // If we entered a new snapshot stripe with the same key, any key we + // return on this iteration is only returned because the open snapshot + // prevented it from being elided or merged with the key returned for + // the previous stripe. Mark it as pinned so that the compaction loop + // can correctly populate output tables' pinned statistics. We might + // also set snapshotPinned=true down below if we observe that the key is + // deleted by a range deletion in a higher stripe or that this key is a + // tombstone that could be elided if only it were in the last snapshot + // stripe. + i.snapshotPinned = i.iterStripeChange == newStripeSameKey + if i.iterKey.Kind() == InternalKeyKindRangeDelete || rangekey.IsRangeKey(i.iterKey.Kind()) { // Return the span so the compaction can use it for file truncation and add // it to the relevant fragmenter. We do not set `skip` to true before @@ -327,25 +364,42 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { // This goes against the comment on i.key in the struct, and // therefore warrants some investigation. i.saveKey() + // TODO(jackson): Handle tracking pinned statistics for range keys + // and range deletions. This would require updating + // emitRangeDelChunk and rangeKeyCompactionTransform to update + // statistics when they apply their own snapshot striping logic. + i.snapshotPinned = false i.value = i.iterValue i.valid = true return &i.key, i.value } - if i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum) { + if cover := i.rangeDelFrag.Covers(*i.iterKey, i.curSnapshotSeqNum); cover == keyspan.CoversVisibly { + // A pending range deletion deletes this key. Skip it. i.saveKey() i.skipInStripe() continue + } else if cover == keyspan.CoversInvisibly { + // i.iterKey would be deleted by a range deletion if there weren't + // any open snapshots. Mark it as pinned. + i.snapshotPinned = true } switch i.iterKey.Kind() { case InternalKeyKindDelete, InternalKeyKindSingleDelete: - // If we're at the last snapshot stripe and the tombstone can be elided - // skip skippable keys in the same stripe. - if i.curSnapshotIdx == 0 && i.elideTombstone(i.iterKey.UserKey) { - i.saveKey() - i.skipInStripe() - continue + if i.elideTombstone(i.iterKey.UserKey) { + if i.curSnapshotIdx == 0 { + // If we're at the last snapshot stripe and the tombstone + // can be elided skip skippable keys in the same stripe. + i.saveKey() + i.skipInStripe() + continue + } else { + // We're not at the last snapshot stripe, so the tombstone + // can NOT yet be elided. Mark it as pinned, so that it's + // included in table statistics appropriately. + i.snapshotPinned = true + } } switch i.iterKey.Kind() { @@ -360,7 +414,6 @@ func (i *compactionIter) Next() (*InternalKey, []byte) { if i.singleDeleteNext() { return &i.key, i.value } - continue } @@ -458,18 +511,13 @@ func snapshotIndex(seq uint64, snapshots []uint64) (int, uint64) { // skipInStripe skips over skippable keys in the same stripe and user key. func (i *compactionIter) skipInStripe() { i.skip = true - var change stripeChangeType - for { - change = i.nextInStripe() - if change == sameStripeNonSkippable || change == newStripe { - break - } + for i.nextInStripe() == sameStripeSkippable { } // Reset skip if we landed outside the original stripe. Otherwise, we landed // in the same stripe on a non-skippable key. In that case we should preserve // `i.skip == true` such that later keys in the stripe will continue to be // skipped. - if change == newStripe { + if i.iterStripeChange == newStripeNewKey || i.iterStripeChange == newStripeSameKey { i.skip = false } } @@ -484,12 +532,16 @@ func (i *compactionIter) iterNext() bool { return i.iterKey != nil } -// stripeChangeType indicates how the snapshot stripe changed relative to the previous -// key. If no change, it also indicates whether the current entry is skippable. +// stripeChangeType indicates how the snapshot stripe changed relative to the +// previous key. If no change, it also indicates whether the current entry is +// skippable. If the snapshot stripe changed, it also indicates whether the new +// stripe was entered because the iterator progressed onto an entirely new key +// or entered a new stripe within the same key. type stripeChangeType int const ( - newStripe stripeChangeType = iota + newStripeNewKey stripeChangeType = iota + newStripeSameKey sameStripeSkippable sameStripeNonSkippable ) @@ -503,8 +555,15 @@ const ( // overwriting or mutating the saved key or value before they have been returned // to the caller of the exported function (i.e. the caller of Next, First, etc.) func (i *compactionIter) nextInStripe() stripeChangeType { + i.iterStripeChange = i.nextInStripeHelper() + return i.iterStripeChange +} + +// nextInStripeHelper is an internal helper for nextInStripe; callers should use +// nextInStripe and not call nextInStripeHelper. +func (i *compactionIter) nextInStripeHelper() stripeChangeType { if !i.iterNext() { - return newStripe + return newStripeNewKey } key := i.iterKey @@ -525,10 +584,10 @@ func (i *compactionIter) nextInStripe() stripeChangeType { panic(fmt.Sprintf("pebble: invariant violation: %s and %s out of order", key, prevKey)) } i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots) - return newStripe + return newStripeNewKey } else if !i.equal(i.key.UserKey, key.UserKey) { i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots) - return newStripe + return newStripeNewKey } origSnapshotIdx := i.curSnapshotIdx i.curSnapshotIdx, i.curSnapshotSeqNum = snapshotIndex(key.SeqNum(), i.snapshots) @@ -540,7 +599,7 @@ func (i *compactionIter) nextInStripe() stripeChangeType { if i.curSnapshotIdx == origSnapshotIdx { return sameStripeNonSkippable } - return newStripe + return newStripeSameKey case InternalKeyKindRangeKeySet, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeyDelete: // Range keys are interleaved at the max sequence number for a given user // key, so we should not see any more range keys in this stripe. @@ -549,12 +608,12 @@ func (i *compactionIter) nextInStripe() stripeChangeType { if i.curSnapshotIdx == origSnapshotIdx { return sameStripeNonSkippable } - return newStripe + return newStripeSameKey } if i.curSnapshotIdx == origSnapshotIdx { return sameStripeSkippable } - return newStripe + return newStripeSameKey } func (i *compactionIter) setNext() { @@ -581,49 +640,50 @@ func (i *compactionIter) setNext() { // Else, we continue to loop through entries in the stripe looking for a // DEL. Note that we may stop *before* encountering a DEL, if one exists. for { - switch t := i.nextInStripe(); t { - case newStripe, sameStripeNonSkippable: + switch i.nextInStripe() { + case newStripeNewKey, newStripeSameKey: i.pos = iterPosNext - if t == sameStripeNonSkippable { - // We iterated onto a key that we cannot skip. We can - // conservatively transform the original SET into a SETWITHDEL - // as an indication that there *may* still be a DEL/SINGLEDEL - // under this SET, even if we did not actually encounter one. - // - // This is safe to do, as: - // - // - in the case that there *is not* actually a DEL/SINGLEDEL - // under this entry, any SINGLEDEL above this now-transformed - // SETWITHDEL will become a DEL when the two encounter in a - // compaction. The DEL will eventually be elided in a - // subsequent compaction. The cost for ensuring correctness is - // that this entry is kept around for an additional compaction - // cycle(s). - // - // - in the case there *is* indeed a DEL/SINGLEDEL under us - // (but in a different stripe or sstable), then we will have - // already done the work to transform the SET into a - // SETWITHDEL, and we will skip any additional iteration when - // this entry is encountered again in a subsequent compaction. - // - // Ideally, this codepath would be smart enough to handle the - // case of SET <- RANGEDEL <- ... <- DEL/SINGLEDEL <- .... - // This requires preserving any RANGEDEL entries we encounter - // along the way, then emitting the original (possibly - // transformed) key, followed by the RANGEDELs. This requires - // a sizable refactoring of the existing code, as nextInStripe - // currently returns a sameStripeNonSkippable when it - // encounters a RANGEDEL. - // TODO(travers): optimize to handle the RANGEDEL case if it - // turns out to be a performance problem. - i.key.SetKind(InternalKeyKindSetWithDelete) + return + case sameStripeNonSkippable: + i.pos = iterPosNext + // We iterated onto a key that we cannot skip. We can + // conservatively transform the original SET into a SETWITHDEL + // as an indication that there *may* still be a DEL/SINGLEDEL + // under this SET, even if we did not actually encounter one. + // + // This is safe to do, as: + // + // - in the case that there *is not* actually a DEL/SINGLEDEL + // under this entry, any SINGLEDEL above this now-transformed + // SETWITHDEL will become a DEL when the two encounter in a + // compaction. The DEL will eventually be elided in a + // subsequent compaction. The cost for ensuring correctness is + // that this entry is kept around for an additional compaction + // cycle(s). + // + // - in the case there *is* indeed a DEL/SINGLEDEL under us + // (but in a different stripe or sstable), then we will have + // already done the work to transform the SET into a + // SETWITHDEL, and we will skip any additional iteration when + // this entry is encountered again in a subsequent compaction. + // + // Ideally, this codepath would be smart enough to handle the + // case of SET <- RANGEDEL <- ... <- DEL/SINGLEDEL <- .... + // This requires preserving any RANGEDEL entries we encounter + // along the way, then emitting the original (possibly + // transformed) key, followed by the RANGEDELs. This requires + // a sizable refactoring of the existing code, as nextInStripe + // currently returns a sameStripeNonSkippable when it + // encounters a RANGEDEL. + // TODO(travers): optimize to handle the RANGEDEL case if it + // turns out to be a performance problem. + i.key.SetKind(InternalKeyKindSetWithDelete) - // By setting i.skip=true, we are saying that after the - // non-skippable key is emitted (which is likely a RANGEDEL), - // the remaining point keys that share the same user key as this - // saved key should be skipped. - i.skip = true - } + // By setting i.skip=true, we are saying that after the + // non-skippable key is emitted (which is likely a RANGEDEL), + // the remaining point keys that share the same user key as this + // saved key should be skipped. + i.skip = true return case sameStripeSkippable: // We're still in the same stripe. If this is a DEL/SINGLEDEL, we @@ -636,7 +696,7 @@ func (i *compactionIter) setNext() { return } default: - panic("pebble: unexpected stripeChangeType: " + strconv.Itoa(int(t))) + panic("pebble: unexpected stripeChangeType: " + strconv.Itoa(int(i.iterStripeChange))) } } } @@ -649,9 +709,9 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType { // Loop looking for older values in the current snapshot stripe and merge // them. for { - if change := i.nextInStripe(); change == sameStripeNonSkippable || change == newStripe { + if i.nextInStripe() != sameStripeSkippable { i.pos = iterPosNext - return change + return i.iterStripeChange } key := i.iterKey switch key.Kind() { @@ -677,7 +737,7 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType { return sameStripeSkippable case InternalKeyKindSet, InternalKeyKindSetWithDelete: - if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) { + if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly { // We change the kind of the result key to a Set so that it shadows // keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't // strictly necessary, but provides consistency with the behavior of @@ -701,7 +761,7 @@ func (i *compactionIter) mergeNext(valueMerger ValueMerger) stripeChangeType { return sameStripeSkippable case InternalKeyKindMerge: - if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) { + if i.rangeDelFrag.Covers(*key, i.curSnapshotSeqNum) == keyspan.CoversVisibly { // We change the kind of the result key to a Set so that it shadows // keys in lower levels. That is, MERGE+RANGEDEL -> SET. This isn't // strictly necessary, but provides consistency with the behavior of @@ -735,7 +795,7 @@ func (i *compactionIter) singleDeleteNext() bool { // Loop until finds a key to be passed to the next level. for { - if change := i.nextInStripe(); change == sameStripeNonSkippable || change == newStripe { + if i.nextInStripe() != sameStripeSkippable { i.pos = iterPosNext return true } @@ -770,6 +830,7 @@ func (i *compactionIter) saveKey() { i.key.UserKey = i.keyBuf i.key.Trailer = i.iterKey.Trailer i.keyTrailer = i.iterKey.Trailer + i.frontiers.Advance(i.key.UserKey) } func (i *compactionIter) cloneKey(key []byte) []byte { @@ -896,5 +957,206 @@ func (i *compactionIter) maybeZeroSeqnum(snapshotIdx int) { // This is not the last snapshot return } - i.key.SetSeqNum(0) + i.key.SetSeqNum(base.SeqNumZero) +} + +// A frontier is used to monitor a compaction's progression across the user +// keyspace. +// +// A frontier hold a user key boundary that it's concerned with in its `key` +// field. If/when the compaction iterator returns an InternalKey with a user key +// _k_ such that k ≥ frontier.key, the compaction iterator invokes the +// frontier's `reached` function, passing _k_ as its argument. +// +// The `reached` function returns a new value to use as the key. If `reached` +// returns nil, the frontier is forgotten and its `reached` method will not be +// invoked again, unless the user calls [Update] to set a new key. +// +// A frontier's key may be updated outside the context of a `reached` +// invocation at any time, through its Update method. +type frontier struct { + // container points to the containing *frontiers that was passed to Init + // when the frontier was initialized. + container *frontiers + + // key holds the frontier's current key. If nil, this frontier is inactive + // and its reached func will not be invoked. The value of this key may only + // be updated by the `frontiers` type, or the Update method. + key []byte + + // reached is invoked to inform a frontier that its key has been reached. + // It's invoked with the user key that reached the limit. The `key` argument + // is guaranteed to be ≥ the frontier's key. + // + // After reached is invoked, the frontier's key is updated to the return + // value of `reached`. Note bene, the frontier is permitted to update its + // key to a user key ≤ the argument `key`. + // + // If a frontier is set to key k1, and reached(k2) is invoked (k2 ≥ k1), the + // frontier will receive reached(k2) calls until it returns nil or a key + // `k3` such that k2 < k3. This property is useful for frontiers that use + // `reached` invocations to drive iteration through collections of keys that + // may contain multiple keys that are both < k2 and ≥ k1. + reached func(key []byte) (next []byte) +} + +// Init initializes the frontier with the provided key and reached callback. +// The frontier is attached to the provided *frontiers and the provided reached +// func will be invoked when the *frontiers is advanced to a key ≥ this +// frontier's key. +func (f *frontier) Init( + frontiers *frontiers, initialKey []byte, reached func(key []byte) (next []byte), +) { + *f = frontier{ + container: frontiers, + key: initialKey, + reached: reached, + } + if initialKey != nil { + f.container.push(f) + } +} + +// String implements fmt.Stringer. +func (f *frontier) String() string { + return string(f.key) +} + +// Update replaces the existing frontier's key with the provided key. The +// frontier's reached func will be invoked when the new key is reached. +func (f *frontier) Update(key []byte) { + c := f.container + prevKeyIsNil := f.key == nil + f.key = key + if prevKeyIsNil { + if key != nil { + c.push(f) + } + return + } + + // Find the frontier within the heap (it must exist within the heap because + // f.key was != nil). If the frontier key is now nil, remove it from the + // heap. Otherwise, fix up its position. + for i := 0; i < len(c.items); i++ { + if c.items[i] == f { + if key != nil { + c.fix(i) + } else { + n := c.len() - 1 + c.swap(i, n) + c.down(i, n) + c.items = c.items[:n] + } + return + } + } + panic("unreachable") +} + +// frontiers is used to track progression of a task (eg, compaction) across the +// keyspace. Clients that want to be informed when the task advances to a key ≥ +// some frontier may register a frontier, providing a callback. The task calls +// `Advance(k)` with each user key encountered, which invokes the `reached` func +// on all tracked frontiers with `key`s ≤ k. +// +// Internally, frontiers is implemented as a simple heap. +type frontiers struct { + cmp Compare + items []*frontier +} + +// String implements fmt.Stringer. +func (f *frontiers) String() string { + var buf bytes.Buffer + for i := 0; i < len(f.items); i++ { + if i > 0 { + fmt.Fprint(&buf, ", ") + } + fmt.Fprintf(&buf, "%s: %q", f.items[i], f.items[i].key) + } + return buf.String() +} + +// Advance notifies all member frontiers with keys ≤ k. +func (f *frontiers) Advance(k []byte) { + for len(f.items) > 0 && f.cmp(k, f.items[0].key) >= 0 { + // This frontier has been reached. Invoke the closure and update with + // the next frontier. + f.items[0].key = f.items[0].reached(k) + if f.items[0].key == nil { + // This was the final frontier that this user was concerned with. + // Remove it from the heap. + f.pop() + } else { + // Fix up the heap root. + f.fix(0) + } + } +} + +func (f *frontiers) len() int { + return len(f.items) +} + +func (f *frontiers) less(i, j int) bool { + return f.cmp(f.items[i].key, f.items[j].key) < 0 +} + +func (f *frontiers) swap(i, j int) { + f.items[i], f.items[j] = f.items[j], f.items[i] +} + +// fix, up and down are copied from the go stdlib. + +func (f *frontiers) fix(i int) { + if !f.down(i, f.len()) { + f.up(i) + } +} + +func (f *frontiers) push(ff *frontier) { + n := len(f.items) + f.items = append(f.items, ff) + f.up(n) +} + +func (f *frontiers) pop() *frontier { + n := f.len() - 1 + f.swap(0, n) + f.down(0, n) + item := f.items[n] + f.items = f.items[:n] + return item +} + +func (f *frontiers) up(j int) { + for { + i := (j - 1) / 2 // parent + if i == j || !f.less(j, i) { + break + } + f.swap(i, j) + j = i + } +} + +func (f *frontiers) down(i0, n int) bool { + i := i0 + for { + j1 := 2*i + 1 + if j1 >= n || j1 < 0 { // j1 < 0 after int overflow + break + } + j := j1 // left child + if j2 := j1 + 1; j2 < n && f.less(j2, j1) { + j = j2 // = 2*i + 2 // right child + } + if !f.less(j, i) { + break + } + f.swap(i, j) + i = j + } + return i > i0 } diff --git a/vendor/github.com/cockroachdb/pebble/compaction_picker.go b/vendor/github.com/cockroachdb/pebble/compaction_picker.go index ca2950e..a7af8cf 100644 --- a/vendor/github.com/cockroachdb/pebble/compaction_picker.go +++ b/vendor/github.com/cockroachdb/pebble/compaction_picker.go @@ -9,6 +9,7 @@ import ( "fmt" "math" "sort" + "strings" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/humanize" @@ -46,13 +47,17 @@ type readCompactionEnv struct { flushing bool } -// Information about in-progress compactions provided to the compaction picker. These are used to -// constrain the new compactions that will be picked. +// Information about in-progress compactions provided to the compaction picker. +// These are used to constrain the new compactions that will be picked. type compactionInfo struct { - inputs []compactionLevel - outputLevel int - smallest InternalKey - largest InternalKey + // versionEditApplied is true if this compaction's version edit has already + // been committed. The compaction may still be in-progress deleting newly + // obsolete files. + versionEditApplied bool + inputs []compactionLevel + outputLevel int + smallest InternalKey + largest InternalKey } func (info compactionInfo) String() string { @@ -98,6 +103,16 @@ type sublevelInfo struct { sublevel manifest.Level } +func (cl sublevelInfo) Clone() sublevelInfo { + return sublevelInfo{ + sublevel: cl.sublevel, + LevelSlice: cl.LevelSlice.Reslice(func(start, end *manifest.LevelIterator) {}), + } +} +func (cl sublevelInfo) String() string { + return fmt.Sprintf(`Sublevel %s; Levels %s`, cl.sublevel, cl.LevelSlice) +} + // generateSublevelInfo will generate the level slices for each of the sublevels // from the level slice for all of L0. func generateSublevelInfo(cmp base.Compare, levelFiles manifest.LevelSlice) []sublevelInfo { @@ -257,6 +272,69 @@ func newPickedCompactionFromL0( return pc } +func (pc *pickedCompaction) String() string { + var builder strings.Builder + builder.WriteString(fmt.Sprintf(`Score=%f, `, pc.score)) + builder.WriteString(fmt.Sprintf(`Kind=%s, `, pc.kind)) + builder.WriteString(fmt.Sprintf(`AdjustedOutputLevel=%d, `, pc.adjustedOutputLevel)) + builder.WriteString(fmt.Sprintf(`maxOutputFileSize=%d, `, pc.maxOutputFileSize)) + builder.WriteString(fmt.Sprintf(`maxReadCompactionBytes=%d, `, pc.maxReadCompactionBytes)) + builder.WriteString(fmt.Sprintf(`smallest=%s, `, pc.smallest)) + builder.WriteString(fmt.Sprintf(`largest=%s, `, pc.largest)) + builder.WriteString(fmt.Sprintf(`version=%s, `, pc.version)) + builder.WriteString(fmt.Sprintf(`inputs=%s, `, pc.inputs)) + builder.WriteString(fmt.Sprintf(`startlevel=%s, `, pc.startLevel)) + builder.WriteString(fmt.Sprintf(`outputLevel=%s, `, pc.outputLevel)) + builder.WriteString(fmt.Sprintf(`extraLevels=%s, `, pc.extraLevels)) + builder.WriteString(fmt.Sprintf(`l0SublevelInfo=%s, `, pc.l0SublevelInfo)) + builder.WriteString(fmt.Sprintf(`lcf=%s`, pc.lcf)) + return builder.String() +} + +// Clone creates a deep copy of the pickedCompaction +func (pc *pickedCompaction) clone() *pickedCompaction { + + // Quickly copy over fields that do not require special deep copy care, and + // set all fields that will require a deep copy to nil. + newPC := &pickedCompaction{ + cmp: pc.cmp, + score: pc.score, + kind: pc.kind, + adjustedOutputLevel: pc.adjustedOutputLevel, + maxOutputFileSize: pc.maxOutputFileSize, + maxOverlapBytes: pc.maxOverlapBytes, + maxReadCompactionBytes: pc.maxReadCompactionBytes, + smallest: pc.smallest.Clone(), + largest: pc.largest.Clone(), + + // Both copies see the same manifest, therefore, it's ok for them to se + // share the same pc. version. + version: pc.version, + } + + newPC.inputs = make([]compactionLevel, len(pc.inputs)) + newPC.extraLevels = make([]*compactionLevel, 0, len(pc.extraLevels)) + for i := range pc.inputs { + newPC.inputs[i] = pc.inputs[i].Clone() + if i == 0 { + newPC.startLevel = &newPC.inputs[i] + } else if i == len(pc.inputs)-1 { + newPC.outputLevel = &newPC.inputs[i] + } else { + newPC.extraLevels = append(newPC.extraLevels, &newPC.inputs[i]) + } + } + + newPC.l0SublevelInfo = make([]sublevelInfo, len(pc.l0SublevelInfo)) + for i := range pc.l0SublevelInfo { + newPC.l0SublevelInfo[i] = pc.l0SublevelInfo[i].Clone() + } + if pc.lcf != nil { + newPC.lcf = pc.lcf.Clone() + } + return newPC +} + // maybeExpandedBounds is a helper function for setupInputs which ensures the // pickedCompaction's smallest and largest internal keys are updated iff // the candidate keys expand the key span. This avoids a bug for multi-level @@ -286,6 +364,8 @@ func (pc *pickedCompaction) maybeExpandBounds(smallest InternalKey, largest Inte } } +// setupInputs returns true if a compaction has been set up. It returns false if +// a concurrent compaction is occurring on the start or output level files. func (pc *pickedCompaction) setupInputs( opts *Options, diskAvailBytes uint64, startLevel *compactionLevel, ) bool { @@ -362,8 +442,7 @@ func (pc *pickedCompaction) setupInputs( } }) } - - oldLcf := *pc.lcf + oldLcf := pc.lcf.Clone() if pc.version.L0Sublevels.ExtendL0ForBaseCompactionTo(smallestBaseKey, largestBaseKey, pc.lcf) { var newStartLevelFiles []*fileMetadata iter := pc.version.Levels[0].Iter() @@ -379,7 +458,7 @@ func (pc *pickedCompaction) setupInputs( pc.smallest, pc.largest = manifest.KeyRange(pc.cmp, startLevel.files.Iter(), pc.outputLevel.files.Iter()) } else { - *pc.lcf = oldLcf + *pc.lcf = *oldLcf } } } else if pc.grow(pc.smallest, pc.largest, maxExpandedBytes, startLevel) { @@ -433,12 +512,28 @@ func (pc *pickedCompaction) grow( return true } -// initMultiLevelCompaction returns true if it initiated a multilevel input -// compaction. This currently never inits a multiLevel compaction. -func (pc *pickedCompaction) initMultiLevelCompaction( - opts *Options, vers *version, levelMaxBytes [7]int64, diskAvailBytes uint64, -) bool { - return false +func (pc *pickedCompaction) compactionSize() uint64 { + var bytesToCompact uint64 + for i := range pc.inputs { + bytesToCompact += pc.inputs[i].files.SizeSum() + } + return bytesToCompact +} + +// setupMultiLevelCandidated returns true if it successfully added another level +// to the compaction. +func (pc *pickedCompaction) setupMultiLevelCandidate(opts *Options, diskAvailBytes uint64) bool { + pc.inputs = append(pc.inputs, compactionLevel{level: pc.outputLevel.level + 1}) + + // Recalibrate startLevel and outputLevel: + // - startLevel and outputLevel pointers may be obsolete after appending to pc.inputs. + // - push outputLevel to extraLevels and move the new level to outputLevel + pc.startLevel = &pc.inputs[0] + pc.extraLevels = []*compactionLevel{&pc.inputs[1]} + pc.outputLevel = &pc.inputs[2] + + pc.adjustedOutputLevel++ + return pc.setupInputs(opts, diskAvailBytes, pc.extraLevels[len(pc.extraLevels)-1]) } // expandToAtomicUnit expands the provided level slice within its level both @@ -461,18 +556,18 @@ func (pc *pickedCompaction) initMultiLevelCompaction( // truncation of range tombstones to atomic compaction unit boundaries. // Consider the scenario: // -// L3: -// 12:[a#2,15-b#1,1] -// 13:[b#0,15-d#72057594037927935,15] +// L3: +// 12:[a#2,15-b#1,1] +// 13:[b#0,15-d#72057594037927935,15] // // These sstables contain a range tombstone [a-d)#2 which spans the two // sstables. The two sstables need to always be kept together. Compacting // sstable 13 independently of sstable 12 would result in: // -// L3: -// 12:[a#2,15-b#1,1] -// L4: -// 14:[b#0,15-d#72057594037927935,15] +// L3: +// 12:[a#2,15-b#1,1] +// L4: +// 14:[b#0,15-d#72057594037927935,15] // // This state is still ok, but when sstable 12 is next compacted, its range // tombstones will be truncated at "b" (the largest key in its atomic @@ -500,6 +595,10 @@ func expandToAtomicUnit( return inputs, false } + // TODO(jackson): Update to avoid use of LevelIterator.Current(). The + // Reslice interface will require some tweaking, because we currently rely + // on Reslice having already positioned the LevelIterator appropriately. + inputs = inputs.Reslice(func(start, end *manifest.LevelIterator) { iter := start.Clone() iter.Prev() @@ -611,7 +710,7 @@ func (a compensatedSizeAnnotator) Accumulate( ) (v interface{}, cacheOK bool) { vptr := dst.(*uint64) *vptr = *vptr + compensatedSize(f, a.pointTombstoneWeight) - return vptr, f.StatsValidLocked() + return vptr, f.StatsValid() } func (a compensatedSizeAnnotator) Merge(src interface{}, dst interface{}) interface{} { @@ -1237,7 +1336,7 @@ func (a elisionOnlyAnnotator) Accumulate(f *fileMetadata, dst interface{}) (inte if f.IsCompacting() { return dst, true } - if !f.StatsValidLocked() { + if !f.StatsValid() { return dst, false } // Bottommost files are large and not worthwhile to compact just @@ -1328,6 +1427,9 @@ func markedMergeHelper(f *fileMetadata, dst interface{}) (interface{}, bool) { func (p *compactionPickerByScore) pickElisionOnlyCompaction( env compactionEnv, ) (pc *pickedCompaction) { + if p.opts.private.disableElisionOnlyCompactions { + return nil + } v := p.vers.Levels[numLevels-1].Annotation(elisionOnlyAnnotator{}) if v == nil { return nil @@ -1456,12 +1558,35 @@ func pickAutoLPositive( if !pc.setupInputs(opts, diskAvailBytes(), pc.startLevel) { return nil } - if opts.Experimental.MultiLevelCompaction && - pc.initMultiLevelCompaction(opts, vers, levelMaxBytes, diskAvailBytes()) { - if !pc.setupInputs(opts, diskAvailBytes(), pc.extraLevels[len(pc.extraLevels)-1]) { - return nil - } + return pc.maybeAddLevel(opts, diskAvailBytes()) +} + +// maybeAddLevel maybe adds a level to the picked compaction. +func (pc *pickedCompaction) maybeAddLevel(opts *Options, diskAvailBytes uint64) *pickedCompaction { + if pc.outputLevel.level == numLevels-1 { + // Don't add a level if the current output level is in L6 + return pc } + if pc.compactionSize() > expandedCompactionByteSizeLimit( + opts, pc.adjustedOutputLevel, diskAvailBytes) { + // Don't add a level if the current compaction exceeds the compaction size limit + return pc + } + return opts.Experimental.MultiLevelCompactionHueristic.pick(pc, opts, diskAvailBytes) +} + +// MultiLevelHeuristic evaluates whether to add files from the next level into the compaction. +type MultiLevelHeuristic interface { + // evaluate returns the preferred compaction. + pick(pc *pickedCompaction, opts *Options, diskAvailBytes uint64) *pickedCompaction +} + +// NoMultiLevel will never add an additional level to the compaction. +type NoMultiLevel struct{} + +func (nml NoMultiLevel) pick( + pc *pickedCompaction, opts *Options, diskAvailBytes uint64, +) *pickedCompaction { return pc } @@ -1585,13 +1710,7 @@ func pickManualHelper( if !pc.setupInputs(opts, diskAvailBytes(), pc.startLevel) { return nil } - if opts.Experimental.MultiLevelCompaction && pc.startLevel.level > 0 && - pc.initMultiLevelCompaction(opts, vers, levelMaxBytes, diskAvailBytes()) { - if !pc.setupInputs(opts, diskAvailBytes(), pc.extraLevels[len(pc.extraLevels)-1]) { - return nil - } - } - return pc + return pc.maybeAddLevel(opts, diskAvailBytes()) } func (p *compactionPickerByScore) pickReadTriggeredCompaction( diff --git a/vendor/github.com/cockroachdb/pebble/db.go b/vendor/github.com/cockroachdb/pebble/db.go index 2e2e019..a12ee56 100644 --- a/vendor/github.com/cockroachdb/pebble/db.go +++ b/vendor/github.com/cockroachdb/pebble/db.go @@ -6,6 +6,7 @@ package pebble // import "github.com/cockroachdb/pebble" import ( + "context" "fmt" "io" "os" @@ -20,6 +21,7 @@ import ( "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/manual" + "github.com/cockroachdb/pebble/objstorage" "github.com/cockroachdb/pebble/record" "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/pebble/vfs" @@ -230,25 +232,17 @@ func (d defaultCPUWorkGranter) CPUWorkDone(_ CPUWorkHandle) {} // Comparer: myComparer, // }) type DB struct { - // WARNING: The following struct `atomic` contains fields which are accessed - // atomically. - // - // Go allocations are guaranteed to be 64-bit aligned which we take advantage - // of by placing the 64-bit fields which we access atomically at the beginning - // of the DB struct. For more information, see https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - atomic struct { - // The count and size of referenced memtables. This includes memtables - // present in DB.mu.mem.queue, as well as memtables that have been flushed - // but are still referenced by an inuse readState. - memTableCount int64 - memTableReserved int64 // number of bytes reserved in the cache for memtables + // The count and size of referenced memtables. This includes memtables + // present in DB.mu.mem.queue, as well as memtables that have been flushed + // but are still referenced by an inuse readState. + memTableCount atomic.Int64 + memTableReserved atomic.Int64 // number of bytes reserved in the cache for memtables - // The size of the current log file (i.e. db.mu.log.queue[len(queue)-1]. - logSize uint64 + // The size of the current log file (i.e. db.mu.log.queue[len(queue)-1]. + logSize atomic.Uint64 - // The number of bytes available on disk. - diskAvailBytes uint64 - } + // The number of bytes available on disk. + diskAvailBytes atomic.Uint64 cacheID uint64 dirname string @@ -263,11 +257,14 @@ type DB struct { // inserted into a memtable. largeBatchThreshold int // The current OPTIONS file number. - optionsFileNum FileNum + optionsFileNum base.DiskFileNum // The on-disk size of the current OPTIONS file. optionsFileSize uint64 - fileLock io.Closer + // objProvider is used to access and manage SSTs. + objProvider objstorage.Provider + + fileLock *Lock dataDir vfs.File walDir vfs.File @@ -368,12 +365,10 @@ type DB struct { fsyncLatency prometheus.Histogram record.LogWriterMetrics } + registerLogWriterForTesting func(w *record.LogWriter) } mem struct { - // Condition variable used to serialize memtable switching. See - // DB.makeRoomForWrite(). - cond sync.Cond // The current mutable memTable. mutable *memTable // Queue of flushables (the mutable memtable is at end). Elements are @@ -381,9 +376,6 @@ type DB struct { // index is set it is never modified making a fixed slice immutable and // safe for concurrent reads. queue flushableList - // True when the memtable is actively being switched. Both mem.mutable and - // log.LogWriter are invalid while switching is true. - switching bool // nextSize is the size of the next memtable. The memtable size starts at // min(256KB,Options.MemTableSize) and doubles each time a new memtable // is allocated up to Options.MemTableSize. This reduces the memory @@ -408,6 +400,10 @@ type DB struct { // is at the start of the list. New entries are added to the end. manual []*manualCompaction // inProgress is the set of in-progress flushes and compactions. + // It's used in the calculation of some metrics and to initialize L0 + // sublevels' state. Some of the compactions contained within this + // map may have already committed an edit to the version but are + // lingering performing cleanup, like deleting obsolete files. inProgress map[*compaction]struct{} // rescheduleReadCompaction indicates to an iterator that a read compaction @@ -418,6 +414,9 @@ type DB struct { // compactions which we might have to perform. readCompactions readCompactionQueue + // The cumulative duration of all completed compactions since Open. + // Does not include flushes. + duration time.Duration // Flush throughput metric. flushWriteThroughput ThroughputMetric // The idle start time for the flush "loop", i.e., when the flushing @@ -440,8 +439,15 @@ type DB struct { disabled int } - // The list of active snapshots. - snapshots snapshotList + snapshots struct { + // The list of active snapshots. + snapshotList + + // The cumulative count and size of snapshot-pinned keys written to + // sstables. + cumulativePinnedCount uint64 + cumulativePinnedSize uint64 + } tableStats struct { // Condition variable used to signal the completion of a @@ -450,7 +456,7 @@ type DB struct { // True when a stat collection operation is in progress. loading bool // True if stat collection has loaded statistics for all tables - // other than those listed explcitly in pending. This flag starts + // other than those listed explicitly in pending. This flag starts // as false when a database is opened and flips to true once stat // collection has caught up. loadedInitial bool @@ -466,7 +472,8 @@ type DB struct { // job to validate one or more sstables. cond sync.Cond // pending is a slice of metadata for sstables waiting to be - // validated. + // validated. Only physical sstables should be added to the pending + // queue. pending []newFileEntry // validating is set to true when validation is running. validating bool @@ -475,11 +482,23 @@ type DB struct { // Normally equal to time.Now() but may be overridden in tests. timeNow func() time.Time + // the time at database Open; may be used to compute metrics like effective + // compaction concurrency + openedAt time.Time } var _ Reader = (*DB)(nil) var _ Writer = (*DB)(nil) +// TestOnlyWaitForCleaning MUST only be used in tests. +func (d *DB) TestOnlyWaitForCleaning() { + d.mu.Lock() + defer d.mu.Unlock() + for d.mu.cleaner.cleaning { + d.mu.cleaner.cond.Wait() + } +} + // Get gets the value for the given key. It returns ErrNotFound if the DB does // not contain the key. // @@ -519,7 +538,7 @@ func (d *DB) getInternal(key []byte, b *Batch, s *Snapshot) ([]byte, io.Closer, if s != nil { seqNum = s.seqNum } else { - seqNum = atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum) + seqNum = d.mu.versions.visibleSeqNum.Load() } buf := getIterAllocPool.Get().(*getIterAlloc) @@ -551,6 +570,7 @@ func (d *DB) getInternal(key []byte, b *Batch, s *Snapshot) ([]byte, io.Closer, i := &buf.dbi pointIter := get *i = Iterator{ + ctx: context.Background(), getIterAlloc: buf, iter: pointIter, pointIter: pointIter, @@ -724,10 +744,34 @@ func (d *DB) RangeKeyDelete(start, end []byte, opts *WriteOptions) error { // // It is safe to modify the contents of the arguments after Apply returns. func (d *DB) Apply(batch *Batch, opts *WriteOptions) error { + return d.applyInternal(batch, opts, false) +} + +// ApplyNoSyncWait must only be used when opts.Sync is true and the caller +// does not want to wait for the WAL fsync to happen. The method will return +// once the mutation is applied to the memtable and is visible (note that a +// mutation is visible before the WAL sync even in the wait case, so we have +// not weakened the durability semantics). The caller must call Batch.SyncWait +// to wait for the WAL fsync. The caller must not Close the batch without +// first calling Batch.SyncWait. +// +// RECOMMENDATION: Prefer using Apply unless you really understand why you +// need ApplyNoSyncWait. +// EXPERIMENTAL: API/feature subject to change. Do not yet use outside +// CockroachDB. +func (d *DB) ApplyNoSyncWait(batch *Batch, opts *WriteOptions) error { + if !opts.Sync { + return errors.Errorf("cannot request asynchonous apply when WriteOptions.Sync is false") + } + return d.applyInternal(batch, opts, true) +} + +// REQUIRES: noSyncWait => opts.Sync +func (d *DB) applyInternal(batch *Batch, opts *WriteOptions, noSyncWait bool) error { if err := d.closed.Load(); err != nil { panic(err) } - if atomic.LoadUint32(&batch.applied) != 0 { + if batch.applied.Load() { panic("pebble: batch already applied") } if d.opts.ReadOnly { @@ -762,10 +806,10 @@ func (d *DB) Apply(batch *Batch, opts *WriteOptions) error { if int(batch.memTableSize) >= d.largeBatchThreshold { batch.flushable = newFlushableBatch(batch, d.opts.Comparer) } - if err := d.commit.Commit(batch, sync); err != nil { + if err := d.commit.Commit(batch, sync, noSyncWait); err != nil { // There isn't much we can do on an error here. The commit pipeline will be // horked at this point. - d.opts.Logger.Fatalf("%v", err) + d.opts.Logger.Fatalf("pebble: fatal commit error: %v", err) } // If this is a large batch, we need to clear the batch contents as the // flushable batch may still be present in the flushables queue. @@ -833,7 +877,7 @@ func (d *DB) commitWrite(b *Batch, syncWG *sync.WaitGroup, syncErr *error) (*mem b.flushable.setSeqNum(b.SeqNum()) if !d.opts.DisableWAL { var err error - size, err = d.mu.log.SyncRecord(repr, syncWG, syncErr) + size, b.commitStats.WALQueueWaitDuration, err = d.mu.log.SyncRecord(repr, syncWG, syncErr) if err != nil { panic(err) } @@ -842,8 +886,14 @@ func (d *DB) commitWrite(b *Batch, syncWG *sync.WaitGroup, syncErr *error) (*mem d.mu.Lock() - // Switch out the memtable if there was not enough room to store the batch. - err := d.makeRoomForWrite(b) + var err error + if !b.ingestedSSTBatch { + // Batches which contain keys of kind InternalKeyKindIngestSST will + // never be applied to the memtable, so we don't need to make room for + // write. For the other cases, switch out the memtable if there was not + // enough room to store the batch. + err = d.makeRoomForWrite(b) + } if err == nil && !d.opts.DisableWAL { d.mu.log.bytesIn += uint64(len(repr)) @@ -865,13 +915,13 @@ func (d *DB) commitWrite(b *Batch, syncWG *sync.WaitGroup, syncErr *error) (*mem } if b.flushable == nil { - size, err = d.mu.log.SyncRecord(repr, syncWG, syncErr) + size, b.commitStats.WALQueueWaitDuration, err = d.mu.log.SyncRecord(repr, syncWG, syncErr) if err != nil { panic(err) } } - atomic.StoreUint64(&d.atomic.logSize, uint64(size)) + d.logSize.Store(uint64(size)) return mem, err } @@ -883,6 +933,7 @@ type iterAlloc struct { merging mergingIter mlevels [3 + numLevels]mergingIterLevel levels [3 + numLevels]levelIter + levelsPositioned [3 + numLevels]bool } var iterAllocPool = sync.Pool{ @@ -891,9 +942,9 @@ var iterAllocPool = sync.Pool{ }, } -// newIterInternal constructs a new iterator, merging in batch iterators as an extra +// newIter constructs a new iterator, merging in batch iterators as an extra // level. -func (d *DB) newIterInternal(batch *Batch, s *Snapshot, o *IterOptions) *Iterator { +func (d *DB) newIter(ctx context.Context, batch *Batch, s *Snapshot, o *IterOptions) *Iterator { if err := d.closed.Load(); err != nil { panic(err) } @@ -926,7 +977,7 @@ func (d *DB) newIterInternal(batch *Batch, s *Snapshot, o *IterOptions) *Iterato if s != nil { seqNum = s.seqNum } else { - seqNum = atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum) + seqNum = d.mu.versions.visibleSeqNum.Load() } // Bundle various structures under a single umbrella in order to allocate @@ -934,6 +985,7 @@ func (d *DB) newIterInternal(batch *Batch, s *Snapshot, o *IterOptions) *Iterato buf := iterAllocPool.Get().(*iterAlloc) dbi := &buf.dbi *dbi = Iterator{ + ctx: ctx, alloc: buf, merge: d.merge, comparer: *d.opts.Comparer, @@ -948,7 +1000,7 @@ func (d *DB) newIterInternal(batch *Batch, s *Snapshot, o *IterOptions) *Iterato } if o != nil { dbi.opts = *o - dbi.saveBounds(o.LowerBound, o.UpperBound) + dbi.processBounds(o.LowerBound, o.UpperBound) } dbi.opts.logger = d.opts.Logger if d.opts.private.disableLazyCombinedIteration { @@ -957,14 +1009,14 @@ func (d *DB) newIterInternal(batch *Batch, s *Snapshot, o *IterOptions) *Iterato if batch != nil { dbi.batchSeqNum = dbi.batch.nextSeqNum() } - return finishInitializingIter(buf) + return finishInitializingIter(ctx, buf) } // finishInitializingIter is a helper for doing the non-trivial initialization // of an Iterator. It's invoked to perform the initial initialization of an // Iterator during NewIter or Clone, and to perform reinitialization due to a // change in IterOptions by a call to Iterator.SetOptions. -func finishInitializingIter(buf *iterAlloc) *Iterator { +func finishInitializingIter(ctx context.Context, buf *iterAlloc) *Iterator { // Short-hand. dbi := &buf.dbi memtables := dbi.readState.memtables @@ -986,7 +1038,7 @@ func finishInitializingIter(buf *iterAlloc) *Iterator { // dbi.merging. If this is called during a SetOptions call and this // Iterator has already initialized dbi.merging, constructPointIter is a // noop and an initialized pointIter already exists in dbi.pointIter. - dbi.constructPointIter(memtables, buf) + dbi.constructPointIter(ctx, memtables, buf) dbi.iter = dbi.pointIter } else { dbi.iter = emptyIter @@ -1062,7 +1114,129 @@ func finishInitializingIter(buf *iterAlloc) *Iterator { return dbi } -func (i *Iterator) constructPointIter(memtables flushableList, buf *iterAlloc) { +// ScanInternal scans all internal keys within the specified bounds, truncating +// any rangedels and rangekeys to those bounds if they span past them. For use +// when an external user needs to be aware of all internal keys that make up a +// key range. +// +// Keys deleted by range deletions must not be returned or exposed by this +// method, while the range deletion deleting that key must be exposed using +// visitRangeDel. Keys that would be masked by range key masking (if an +// appropriate prefix were set) should be exposed, alongside the range key +// that would have masked it. This method also collapses all point keys into +// one InternalKey; so only one internal key at most per user key is returned +// to visitPointKey. +// +// If visitSharedFile is not nil, ScanInternal iterates in skip-shared iteration +// mode. In this iteration mode, sstables in levels L5 and L6 are skipped, and +// their metadatas truncated to [lower, upper) and passed into visitSharedFile. +// ErrInvalidSkipSharedIteration is returned if visitSharedFile is not nil and an +// sstable in L5 or L6 is found that is not in shared storage according to +// provider.IsShared. Examples of when this could happen could be if Pebble +// started writing sstables before a creator ID was set (as creator IDs are +// necessary to enable shared storage) resulting in some lower level SSTs being +// on non-shared storage. Skip-shared iteration is invalid in those cases. +func (d *DB) ScanInternal( + ctx context.Context, + lower, upper []byte, + visitPointKey func(key *InternalKey, value LazyValue) error, + visitRangeDel func(start, end []byte, seqNum uint64) error, + visitRangeKey func(start, end []byte, keys []keyspan.Key) error, + visitSharedFile func(sst *SharedSSTMeta) error, +) error { + iter := d.newInternalIter(nil /* snapshot */, &scanInternalOptions{ + IterOptions: IterOptions{ + KeyTypes: IterKeyTypePointsAndRanges, + LowerBound: lower, + UpperBound: upper, + }, + skipSharedLevels: visitSharedFile != nil, + }) + defer iter.close() + return scanInternalImpl(ctx, lower, upper, iter, visitPointKey, visitRangeDel, visitRangeKey, visitSharedFile) +} + +// newInternalIter constructs and returns a new scanInternalIterator on this db. +// If o.skipSharedLevels is true, levels below sharedLevelsStart are *not* added +// to the internal iterator. +// +// TODO(bilal): This method has a lot of similarities with db.newIter as well as +// finishInitializingIter. Both pairs of methods should be refactored to reduce +// this duplication. +func (d *DB) newInternalIter(s *Snapshot, o *scanInternalOptions) *scanInternalIterator { + if err := d.closed.Load(); err != nil { + panic(err) + } + // Grab and reference the current readState. This prevents the underlying + // files in the associated version from being deleted if there is a current + // compaction. The readState is unref'd by Iterator.Close(). + readState := d.loadReadState() + + // Determine the seqnum to read at after grabbing the read state (current and + // memtables) above. + var seqNum uint64 + if s == nil { + seqNum = d.mu.versions.visibleSeqNum.Load() + } else { + seqNum = s.seqNum + } + + // Bundle various structures under a single umbrella in order to allocate + // them together. + buf := iterAllocPool.Get().(*iterAlloc) + dbi := &scanInternalIterator{ + comparer: d.opts.Comparer, + merge: d.opts.Merger.Merge, + readState: readState, + alloc: buf, + newIters: d.newIters, + newIterRangeKey: d.tableNewRangeKeyIter, + seqNum: seqNum, + } + if o != nil { + dbi.opts = *o + } + dbi.opts.logger = d.opts.Logger + if d.opts.private.disableLazyCombinedIteration { + dbi.opts.disableLazyCombinedIteration = true + } + return finishInitializingInternalIter(buf, dbi) +} + +func finishInitializingInternalIter(buf *iterAlloc, i *scanInternalIterator) *scanInternalIterator { + // Short-hand. + memtables := i.readState.memtables + // We only need to read from memtables which contain sequence numbers older + // than seqNum. Trim off newer memtables. + for j := len(memtables) - 1; j >= 0; j-- { + if logSeqNum := memtables[j].logSeqNum; logSeqNum < i.seqNum { + break + } + memtables = memtables[:j] + } + i.initializeBoundBufs(i.opts.LowerBound, i.opts.UpperBound) + + i.constructPointIter(memtables, buf) + + // For internal iterators, we skip the lazy combined iteration optimization + // entirely, and create the range key iterator stack directly. + i.rangeKey = iterRangeKeyStateAllocPool.Get().(*iteratorRangeKeyState) + i.rangeKey.init(i.comparer.Compare, i.comparer.Split, &i.opts.IterOptions) + i.constructRangeKeyIter() + + // Wrap the point iterator (currently i.iter) with an interleaving + // iterator that interleaves range keys pulled from + // i.rangeKey.rangeKeyIter. + i.rangeKey.iiter.Init(i.comparer, i.iter, i.rangeKey.rangeKeyIter, + nil /* mask */, i.opts.LowerBound, i.opts.UpperBound) + i.iter = &i.rangeKey.iiter + + return i +} + +func (i *Iterator) constructPointIter( + ctx context.Context, memtables flushableList, buf *iterAlloc, +) { if i.pointIter != nil { // Already have one. return @@ -1149,7 +1323,8 @@ func (i *Iterator) constructPointIter(memtables flushableList, buf *iterAlloc) { addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) { li := &levels[levelsIndex] - li.init(i.opts, i.comparer.Compare, i.comparer.Split, i.newIters, files, level, internalOpts) + li.init( + ctx, i.opts, i.comparer.Compare, i.comparer.Split, i.newIters, files, level, internalOpts) li.initRangeDel(&mlevels[mlevelsIndex].rangeDelIter) li.initBoundaryContext(&mlevels[mlevelsIndex].levelIterBoundaryContext) li.initCombinedIterState(&i.lazyCombinedIter.combinedIterState) @@ -1173,9 +1348,11 @@ func (i *Iterator) constructPointIter(memtables flushableList, buf *iterAlloc) { addLevelIterForFiles(current.Levels[level].Iter(), manifest.Level(level)) } buf.merging.init(&i.opts, &i.stats.InternalStats, i.comparer.Compare, i.comparer.Split, mlevels...) + if len(mlevels) <= cap(buf.levelsPositioned) { + buf.merging.levelsPositioned = buf.levelsPositioned[:len(mlevels)] + } buf.merging.snapshot = i.seqNum buf.merging.batchSnapshot = i.batchSeqNum - buf.merging.elideRangeTombstones = true buf.merging.combinedIterState = &i.lazyCombinedIter.combinedIterState i.pointIter = &buf.merging i.merging = &buf.merging @@ -1205,7 +1382,13 @@ func (d *DB) NewIndexedBatch() *Batch { // apparent memory and disk usage leak. Use snapshots (see NewSnapshot) for // point-in-time snapshots which avoids these problems. func (d *DB) NewIter(o *IterOptions) *Iterator { - return d.newIterInternal(nil /* batch */, nil /* snapshot */, o) + return d.NewIterWithContext(context.Background(), o) +} + +// NewIterWithContext is like NewIter, and additionally accepts a context for +// tracing. +func (d *DB) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterator { + return d.newIter(ctx, nil /* batch */, nil /* snapshot */, o) } // NewSnapshot returns a point-in-time view of the current DB state. Iterators @@ -1224,7 +1407,7 @@ func (d *DB) NewSnapshot() *Snapshot { d.mu.Lock() s := &Snapshot{ db: d, - seqNum: atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum), + seqNum: d.mu.versions.visibleSeqNum.Load(), } d.mu.snapshots.pushBack(s) d.mu.Unlock() @@ -1316,9 +1499,13 @@ func (d *DB) Close() error { } for _, mem := range d.mu.mem.queue { - mem.readerUnref() + // Usually, we'd want to delete the files returned by readerUnref. But + // in this case, even if we're unreferencing the flushables, the + // flushables aren't obsolete. They will be reconstructed during WAL + // replay. + mem.readerUnrefLocked(false) } - if reserved := atomic.LoadInt64(&d.atomic.memTableReserved); reserved != 0 { + if reserved := d.memTableReserved.Load(); reserved != 0 { err = firstError(err, errors.Errorf("leaked memtable reservation: %d", errors.Safe(reserved))) } @@ -1344,6 +1531,8 @@ func (d *DB) Close() error { err = firstError(err, errors.Errorf("non-zero zombie file count: %d", ztbls)) } + err = firstError(err, d.objProvider.Close()) + // If the options include a closer to 'close' the filesystem, close it. if d.opts.private.fsCloser != nil { d.opts.private.fsCloser.Close() @@ -1546,9 +1735,16 @@ func (d *DB) Metrics() *Metrics { vers := d.mu.versions.currentVersion() *metrics = d.mu.versions.metrics metrics.Compact.EstimatedDebt = d.mu.versions.picker.estimatedCompactionDebt(0) - metrics.Compact.InProgressBytes = atomic.LoadInt64(&d.mu.versions.atomic.atomicInProgressBytes) + metrics.Compact.InProgressBytes = d.mu.versions.atomicInProgressBytes.Load() metrics.Compact.NumInProgress = int64(d.mu.compact.compactingCount) metrics.Compact.MarkedFiles = vers.Stats.MarkedForCompaction + metrics.Compact.Duration = d.mu.compact.duration + for c := range d.mu.compact.inProgress { + if c.kind != compactionKindFlush { + metrics.Compact.Duration += d.timeNow().Sub(c.beganAt) + } + } + for _, m := range d.mu.mem.queue { metrics.MemTable.Size += m.totalBytes() } @@ -1556,12 +1752,14 @@ func (d *DB) Metrics() *Metrics { if metrics.Snapshots.Count > 0 { metrics.Snapshots.EarliestSeqNum = d.mu.snapshots.earliest() } + metrics.Snapshots.PinnedKeys = d.mu.snapshots.cumulativePinnedCount + metrics.Snapshots.PinnedSize = d.mu.snapshots.cumulativePinnedSize metrics.MemTable.Count = int64(len(d.mu.mem.queue)) - metrics.MemTable.ZombieCount = atomic.LoadInt64(&d.atomic.memTableCount) - metrics.MemTable.Count - metrics.MemTable.ZombieSize = uint64(atomic.LoadInt64(&d.atomic.memTableReserved)) - metrics.MemTable.Size + metrics.MemTable.ZombieCount = d.memTableCount.Load() - metrics.MemTable.Count + metrics.MemTable.ZombieSize = uint64(d.memTableReserved.Load()) - metrics.MemTable.Size metrics.WAL.ObsoleteFiles = int64(recycledLogsCount) metrics.WAL.ObsoletePhysicalSize = recycledLogSize - metrics.WAL.Size = atomic.LoadUint64(&d.atomic.logSize) + metrics.WAL.Size = d.logSize.Load() // The current WAL size (d.atomic.logSize) is the current logical size, // which may be less than the WAL's physical size if it was recycled. // The file sizes in d.mu.log.queue are updated to the physical size @@ -1592,7 +1790,9 @@ func (d *DB) Metrics() *Metrics { } metrics.private.optionsFileSize = d.optionsFileSize + // TODO(jackson): Consider making these metrics optional. metrics.Keys.RangeKeySetsCount = countRangeKeySetFragments(vers) + metrics.Keys.TombstoneCount = countTombstones(vers) d.mu.versions.logLock() metrics.private.manifestFileSize = uint64(d.mu.versions.manifest.Size()) @@ -1603,12 +1803,19 @@ func (d *DB) Metrics() *Metrics { d.opts.Logger.Infof("metrics error: %s", err) } metrics.Flush.WriteThroughput = d.mu.compact.flushWriteThroughput + if d.mu.compact.flushing { + metrics.Flush.NumInProgress = 1 + } + for i := 0; i < numLevels; i++ { + metrics.Levels[i].Additional.ValueBlocksSize = valueBlocksSizeForLevel(vers, i) + } d.mu.Unlock() metrics.BlockCache = d.opts.Cache.Metrics() metrics.TableCache, metrics.Filter = d.tableCache.metrics() metrics.TableIters = int64(d.tableCache.iterCount()) + metrics.Uptime = d.timeNow().Sub(d.openedAt) return metrics } @@ -1634,8 +1841,15 @@ func WithProperties() SSTablesOption { // SSTableInfo export manifest.TableInfo with sstable.Properties type SSTableInfo struct { manifest.TableInfo + // Virtual indicates whether the sstable is virtual. + Virtual bool + // BackingSSTNum is the file number associated with backing sstable which + // backs the sstable associated with this SSTableInfo. If Virtual is false, + // then BackingSSTNum == FileNum. + BackingSSTNum base.FileNum - // Properties is the sstable properties of this table. + // Properties is the sstable properties of this table. If Virtual is true, + // then the Properties are associated with the backing sst. Properties *sstable.Properties } @@ -1671,12 +1885,16 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) { for m := iter.First(); m != nil; m = iter.Next() { destTables[j] = SSTableInfo{TableInfo: m.TableInfo()} if opt.withProperties { - p, err := d.tableCache.getTableProperties(m) + p, err := d.tableCache.getTableProperties( + m, + ) if err != nil { return nil, err } destTables[j].Properties = p } + destTables[j].Virtual = m.Virtual + destTables[j].BackingSSTNum = m.FileBacking.DiskFileNum.FileNum() j++ } destLevels[i] = destTables[:j] @@ -1688,14 +1906,16 @@ func (d *DB) SSTables(opts ...SSTablesOption) ([][]SSTableInfo, error) { // EstimateDiskUsage returns the estimated filesystem space used in bytes for // storing the range `[start, end]`. The estimation is computed as follows: // -// - For sstables fully contained in the range the whole file size is included. -// - For sstables partially contained in the range the overlapping data block sizes -// are included. Even if a data block partially overlaps, or we cannot determine -// overlap due to abbreviated index keys, the full data block size is included in -// the estimation. Note that unlike fully contained sstables, none of the -// meta-block space is counted for partially overlapped files. -// - There may also exist WAL entries for unflushed keys in this range. This -// estimation currently excludes space used for the range in the WAL. +// - For sstables fully contained in the range the whole file size is included. +// - For sstables partially contained in the range the overlapping data block sizes +// are included. Even if a data block partially overlaps, or we cannot determine +// overlap due to abbreviated index keys, the full data block size is included in +// the estimation. Note that unlike fully contained sstables, none of the +// meta-block space is counted for partially overlapped files. +// - For virtual sstables, we use the overlap between start, end and the virtual +// sstable bounds to determine disk usage. +// - There may also exist WAL entries for unflushed keys in this range. This +// estimation currently excludes space used for the range in the WAL. func (d *DB) EstimateDiskUsage(start, end []byte) (uint64, error) { if err := d.closed.Load(); err != nil { panic(err) @@ -1729,10 +1949,24 @@ func (d *DB) EstimateDiskUsage(start, end []byte) (uint64, error) { } else if d.opts.Comparer.Compare(file.Smallest.UserKey, end) <= 0 && d.opts.Comparer.Compare(start, file.Largest.UserKey) <= 0 { var size uint64 - err := d.tableCache.withReader(file, func(r *sstable.Reader) (err error) { - size, err = r.EstimateDiskUsage(start, end) - return err - }) + var err error + if file.Virtual { + err = d.tableCache.withVirtualReader( + file.VirtualMeta(), + func(r sstable.VirtualReader) (err error) { + size, err = r.EstimateDiskUsage(start, end) + return err + }, + ) + } else { + err = d.tableCache.withReader( + file.PhysicalMeta(), + func(r *sstable.Reader) (err error) { + size, err = r.EstimateDiskUsage(start, end) + return err + }, + ) + } if err != nil { return 0, err } @@ -1766,8 +2000,8 @@ func (d *DB) newMemTable(logNum FileNum, logSeqNum uint64) (*memTable, *flushabl } } - atomic.AddInt64(&d.atomic.memTableCount, 1) - atomic.AddInt64(&d.atomic.memTableReserved, int64(size)) + d.memTableCount.Add(1) + d.memTableReserved.Add(int64(size)) releaseAccountingReservation := d.opts.Cache.Reserve(size) mem := newMemTable(memTableOptions{ @@ -1783,8 +2017,8 @@ func (d *DB) newMemTable(logNum FileNum, logSeqNum uint64) (*memTable, *flushabl entry.releaseMemAccounting = func() { manual.Free(mem.arenaBuf) mem.arenaBuf = nil - atomic.AddInt64(&d.atomic.memTableCount, -1) - atomic.AddInt64(&d.atomic.memTableReserved, -int64(size)) + d.memTableCount.Add(-1) + d.memTableReserved.Add(-int64(size)) releaseAccountingReservation() } return mem, entry @@ -1792,11 +2026,13 @@ func (d *DB) newMemTable(logNum FileNum, logSeqNum uint64) (*memTable, *flushabl func (d *DB) newFlushableEntry(f flushable, logNum FileNum, logSeqNum uint64) *flushableEntry { return &flushableEntry{ - flushable: f, - flushed: make(chan struct{}), - logNum: logNum, - logSeqNum: logSeqNum, - readerRefs: 1, + flushable: f, + flushed: make(chan struct{}), + logNum: logNum, + logSeqNum: logSeqNum, + readerRefs: 1, + deleteFn: d.mu.versions.addObsolete, + deleteFnLocked: d.mu.versions.addObsoleteLocked, } } @@ -1810,13 +2046,13 @@ func (d *DB) newFlushableEntry(f flushable, logNum FileNum, logSeqNum uint64) *f // Both DB.mu and commitPipeline.mu must be held by the caller. Note that DB.mu // may be released and reacquired. func (d *DB) makeRoomForWrite(b *Batch) error { + if b != nil && b.ingestedSSTBatch { + panic("pebble: invalid function call") + } + force := b == nil || b.flushable != nil stalled := false for { - if d.mu.mem.switching { - d.mu.mem.cond.Wait() - continue - } if b != nil && b.flushable == nil { err := d.mu.mem.mutable.prepare(b) if err != arenaskl.ErrArenaFull { @@ -1846,7 +2082,11 @@ func (d *DB) makeRoomForWrite(b *Batch) error { Reason: "memtable count limit reached", }) } + now := time.Now() d.mu.compact.cond.Wait() + if b != nil { + b.commitStats.MemTableWriteStallDuration += time.Since(now) + } continue } } @@ -1859,133 +2099,22 @@ func (d *DB) makeRoomForWrite(b *Batch) error { Reason: "L0 file count limit exceeded", }) } + now := time.Now() d.mu.compact.cond.Wait() + if b != nil { + b.commitStats.L0ReadAmpWriteStallDuration += time.Since(now) + } continue } - var newLogNum FileNum - var newLogFile vfs.File - var newLogSize uint64 + var newLogNum base.FileNum var prevLogSize uint64 - var err error - if !d.opts.DisableWAL { - jobID := d.mu.nextJobID - d.mu.nextJobID++ - newLogNum = d.mu.versions.getNextFileNum() - d.mu.mem.switching = true - - prevLogSize = uint64(d.mu.log.Size()) - - // The previous log may have grown past its original physical - // size. Update its file size in the queue so we have a proper - // accounting of its file size. - if d.mu.log.queue[len(d.mu.log.queue)-1].fileSize < prevLogSize { - d.mu.log.queue[len(d.mu.log.queue)-1].fileSize = prevLogSize + now := time.Now() + newLogNum, prevLogSize = d.recycleWAL() + if b != nil { + b.commitStats.WALRotationDuration += time.Since(now) } - d.mu.Unlock() - - // Close the previous log first. This writes an EOF trailer - // signifying the end of the file and syncs it to disk. We must - // close the previous log before linking the new log file, - // otherwise a crash could leave both logs with unclean tails, and - // Open will treat the previous log as corrupt. - err = d.mu.log.LogWriter.Close() - metrics := d.mu.log.LogWriter.Metrics() - d.mu.Lock() - if err := d.mu.log.metrics.Merge(metrics); err != nil { - d.opts.Logger.Infof("metrics error: %s", err) - } - d.mu.Unlock() - - newLogName := base.MakeFilepath(d.opts.FS, d.walDirname, fileTypeLog, newLogNum) - - // Try to use a recycled log file. Recycling log files is an important - // performance optimization as it is faster to sync a file that has - // already been written, than one which is being written for the first - // time. This is due to the need to sync file metadata when a file is - // being written for the first time. Note this is true even if file - // preallocation is performed (e.g. fallocate). - var recycleLog fileInfo - var recycleOK bool - if err == nil { - recycleLog, recycleOK = d.logRecycler.peek() - if recycleOK { - recycleLogName := base.MakeFilepath(d.opts.FS, d.walDirname, fileTypeLog, recycleLog.fileNum) - newLogFile, err = d.opts.FS.ReuseForWrite(recycleLogName, newLogName) - base.MustExist(d.opts.FS, newLogName, d.opts.Logger, err) - } else { - newLogFile, err = d.opts.FS.Create(newLogName) - base.MustExist(d.opts.FS, newLogName, d.opts.Logger, err) - } - } - - if err == nil && recycleOK { - // Figure out the recycled WAL size. This Stat is necessary - // because ReuseForWrite's contract allows for removing the - // old file and creating a new one. We don't know whether the - // WAL was actually recycled. - // TODO(jackson): Adding a boolean to the ReuseForWrite return - // value indicating whether or not the file was actually - // reused would allow us to skip the stat and use - // recycleLog.fileSize. - var finfo os.FileInfo - finfo, err = newLogFile.Stat() - if err == nil { - newLogSize = uint64(finfo.Size()) - } - } - - if err == nil { - // TODO(peter): RocksDB delays sync of the parent directory until the - // first time the log is synced. Is that worthwhile? - err = d.walDir.Sync() - } - - if err != nil && newLogFile != nil { - newLogFile.Close() - } else if err == nil { - newLogFile = vfs.NewSyncingFile(newLogFile, vfs.SyncingFileOptions{ - NoSyncOnClose: d.opts.NoSyncOnClose, - BytesPerSync: d.opts.WALBytesPerSync, - PreallocateSize: d.walPreallocateSize(), - }) - } - - if recycleOK { - err = firstError(err, d.logRecycler.pop(recycleLog.fileNum)) - } - - d.opts.EventListener.WALCreated(WALCreateInfo{ - JobID: jobID, - Path: newLogName, - FileNum: newLogNum, - RecycledFileNum: recycleLog.fileNum, - Err: err, - }) - - d.mu.Lock() - d.mu.mem.switching = false - d.mu.mem.cond.Broadcast() - - d.mu.versions.metrics.WAL.Files++ - } - - if err != nil { - // TODO(peter): avoid chewing through file numbers in a tight loop if there - // is an error here. - // - // What to do here? Stumbling on doesn't seem worthwhile. If we failed to - // close the previous log it is possible we lost a write. - panic(err) - } - - if !d.opts.DisableWAL { - d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum, fileSize: newLogSize}) - d.mu.log.LogWriter = record.NewLogWriter(newLogFile, newLogNum, record.LogWriterConfig{ - WALFsyncLatency: d.mu.log.metrics.fsyncLatency, - WALMinSyncInterval: d.opts.WALMinSyncInterval, - }) } immMem := d.mu.mem.mutable @@ -2005,10 +2134,10 @@ func (d *DB) makeRoomForWrite(b *Batch) error { // The batch is too large to fit in the memtable so add it directly to // the immutable queue. The flushable batch is associated with the same // log as the immutable memtable, but logically occurs after it in - // seqnum space. So give the flushable batch the logNum and clear it from - // the immutable log. This is done as a defensive measure to prevent the - // WAL containing the large batch from being deleted prematurely if the - // corresponding memtable is flushed without flushing the large batch. + // seqnum space. We ensure while flushing that the flushable batch + // is flushed along with the previous memtable in the flushable + // queue. See the top level comment in DB.flush1 to learn how this + // is ensured. // // See DB.commitWrite for the special handling of log writes for large // batches. In particular, the large batch has already written to @@ -2018,7 +2147,6 @@ func (d *DB) makeRoomForWrite(b *Batch) error { // for it until it is flushed. entry.releaseMemAccounting = d.opts.Cache.Reserve(int(b.flushable.totalBytes())) d.mu.mem.queue = append(d.mu.mem.queue, entry) - imm.logNum = 0 } var logSeqNum uint64 @@ -2028,33 +2156,168 @@ func (d *DB) makeRoomForWrite(b *Batch) error { logSeqNum += uint64(b.Count()) } } else { - logSeqNum = atomic.LoadUint64(&d.mu.versions.atomic.logSeqNum) - } - - // Create a new memtable, scheduling the previous one for flushing. We do - // this even if the previous memtable was empty because the DB.Flush - // mechanism is dependent on being able to wait for the empty memtable to - // flush. We can't just mark the empty memtable as flushed here because we - // also have to wait for all previous immutable tables to - // flush. Additionally, the memtable is tied to particular WAL file and we - // want to go through the flush path in order to recycle that WAL file. - // - // NB: newLogNum corresponds to the WAL that contains mutations that are - // present in the new memtable. When immutable memtables are flushed to - // disk, a VersionEdit will be created telling the manifest the minimum - // unflushed log number (which will be the next one in d.mu.mem.mutable - // that was not flushed). - var entry *flushableEntry - d.mu.mem.mutable, entry = d.newMemTable(newLogNum, logSeqNum) - d.mu.mem.queue = append(d.mu.mem.queue, entry) - d.updateReadStateLocked(nil) - if immMem.writerUnref() { - d.maybeScheduleFlush() + logSeqNum = d.mu.versions.logSeqNum.Load() } + d.rotateMemtable(newLogNum, logSeqNum, immMem) force = false } } +// Both DB.mu and commitPipeline.mu must be held by the caller. +func (d *DB) rotateMemtable(newLogNum FileNum, logSeqNum uint64, prev *memTable) { + // Create a new memtable, scheduling the previous one for flushing. We do + // this even if the previous memtable was empty because the DB.Flush + // mechanism is dependent on being able to wait for the empty memtable to + // flush. We can't just mark the empty memtable as flushed here because we + // also have to wait for all previous immutable tables to + // flush. Additionally, the memtable is tied to particular WAL file and we + // want to go through the flush path in order to recycle that WAL file. + // + // NB: newLogNum corresponds to the WAL that contains mutations that are + // present in the new memtable. When immutable memtables are flushed to + // disk, a VersionEdit will be created telling the manifest the minimum + // unflushed log number (which will be the next one in d.mu.mem.mutable + // that was not flushed). + // + // NB: prev should be the current mutable memtable. + var entry *flushableEntry + d.mu.mem.mutable, entry = d.newMemTable(newLogNum, logSeqNum) + d.mu.mem.queue = append(d.mu.mem.queue, entry) + d.updateReadStateLocked(nil) + if prev.writerUnref() { + d.maybeScheduleFlush() + } +} + +// Both DB.mu and commitPipeline.mu must be held by the caller. Note that DB.mu +// may be released and reacquired. +func (d *DB) recycleWAL() (newLogNum FileNum, prevLogSize uint64) { + if d.opts.DisableWAL { + panic("pebble: invalid function call") + } + + jobID := d.mu.nextJobID + d.mu.nextJobID++ + newLogNum = d.mu.versions.getNextFileNum() + + prevLogSize = uint64(d.mu.log.Size()) + + // The previous log may have grown past its original physical + // size. Update its file size in the queue so we have a proper + // accounting of its file size. + if d.mu.log.queue[len(d.mu.log.queue)-1].fileSize < prevLogSize { + d.mu.log.queue[len(d.mu.log.queue)-1].fileSize = prevLogSize + } + d.mu.Unlock() + + var err error + // Close the previous log first. This writes an EOF trailer + // signifying the end of the file and syncs it to disk. We must + // close the previous log before linking the new log file, + // otherwise a crash could leave both logs with unclean tails, and + // Open will treat the previous log as corrupt. + err = d.mu.log.LogWriter.Close() + metrics := d.mu.log.LogWriter.Metrics() + d.mu.Lock() + if err := d.mu.log.metrics.Merge(metrics); err != nil { + d.opts.Logger.Infof("metrics error: %s", err) + } + d.mu.Unlock() + + newLogName := base.MakeFilepath(d.opts.FS, d.walDirname, fileTypeLog, newLogNum.DiskFileNum()) + + // Try to use a recycled log file. Recycling log files is an important + // performance optimization as it is faster to sync a file that has + // already been written, than one which is being written for the first + // time. This is due to the need to sync file metadata when a file is + // being written for the first time. Note this is true even if file + // preallocation is performed (e.g. fallocate). + var recycleLog fileInfo + var recycleOK bool + var newLogFile vfs.File + if err == nil { + recycleLog, recycleOK = d.logRecycler.peek() + if recycleOK { + recycleLogName := base.MakeFilepath(d.opts.FS, d.walDirname, fileTypeLog, recycleLog.fileNum) + newLogFile, err = d.opts.FS.ReuseForWrite(recycleLogName, newLogName) + base.MustExist(d.opts.FS, newLogName, d.opts.Logger, err) + } else { + newLogFile, err = d.opts.FS.Create(newLogName) + base.MustExist(d.opts.FS, newLogName, d.opts.Logger, err) + } + } + + var newLogSize uint64 + if err == nil && recycleOK { + // Figure out the recycled WAL size. This Stat is necessary + // because ReuseForWrite's contract allows for removing the + // old file and creating a new one. We don't know whether the + // WAL was actually recycled. + // TODO(jackson): Adding a boolean to the ReuseForWrite return + // value indicating whether or not the file was actually + // reused would allow us to skip the stat and use + // recycleLog.fileSize. + var finfo os.FileInfo + finfo, err = newLogFile.Stat() + if err == nil { + newLogSize = uint64(finfo.Size()) + } + } + + if err == nil { + // TODO(peter): RocksDB delays sync of the parent directory until the + // first time the log is synced. Is that worthwhile? + err = d.walDir.Sync() + } + + if err != nil && newLogFile != nil { + newLogFile.Close() + } else if err == nil { + newLogFile = vfs.NewSyncingFile(newLogFile, vfs.SyncingFileOptions{ + NoSyncOnClose: d.opts.NoSyncOnClose, + BytesPerSync: d.opts.WALBytesPerSync, + PreallocateSize: d.walPreallocateSize(), + }) + } + + if recycleOK { + err = firstError(err, d.logRecycler.pop(recycleLog.fileNum.FileNum())) + } + + d.opts.EventListener.WALCreated(WALCreateInfo{ + JobID: jobID, + Path: newLogName, + FileNum: newLogNum, + RecycledFileNum: recycleLog.fileNum.FileNum(), + Err: err, + }) + + d.mu.Lock() + + d.mu.versions.metrics.WAL.Files++ + + if err != nil { + // TODO(peter): avoid chewing through file numbers in a tight loop if there + // is an error here. + // + // What to do here? Stumbling on doesn't seem worthwhile. If we failed to + // close the previous log it is possible we lost a write. + panic(err) + } + + d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum.DiskFileNum(), fileSize: newLogSize}) + d.mu.log.LogWriter = record.NewLogWriter(newLogFile, newLogNum, record.LogWriterConfig{ + WALFsyncLatency: d.mu.log.metrics.fsyncLatency, + WALMinSyncInterval: d.opts.WALMinSyncInterval, + QueueSemChan: d.commit.logSyncQSem, + }) + if d.mu.log.registerLogWriterForTesting != nil { + d.mu.log.registerLogWriterForTesting(d.mu.log.LogWriter) + } + + return +} + func (d *DB) getEarliestUnflushedSeqNumLocked() uint64 { seqNum := InternalKeySeqNumMax for i := range d.mu.mem.queue { @@ -2070,10 +2333,11 @@ func (d *DB) getInProgressCompactionInfoLocked(finishing *compaction) (rv []comp for c := range d.mu.compact.inProgress { if len(c.flushing) == 0 && (finishing == nil || c != finishing) { info := compactionInfo{ - inputs: c.inputs, - smallest: c.smallest, - largest: c.largest, - outputLevel: -1, + versionEditApplied: c.versionEditApplied, + inputs: c.inputs, + smallest: c.smallest, + largest: c.largest, + outputLevel: -1, } if c.outputLevel != nil { info.outputLevel = c.outputLevel.level @@ -2087,6 +2351,14 @@ func (d *DB) getInProgressCompactionInfoLocked(finishing *compaction) (rv []comp func inProgressL0Compactions(inProgress []compactionInfo) []manifest.L0Compaction { var compactions []manifest.L0Compaction for _, info := range inProgress { + // Skip in-progress compactions that have already committed; the L0 + // sublevels initialization code requires the set of in-progress + // compactions to be consistent with the current version. Compactions + // with versionEditApplied=true are already applied to the current + // version and but are performing cleanup without the database mutex. + if info.versionEditApplied { + continue + } l0 := false for _, cl := range info.inputs { l0 = l0 || cl.level == 0 @@ -2111,3 +2383,22 @@ func firstError(err0, err1 error) error { } return err1 } + +// SetCreatorID sets the CreatorID which is needed in order to use shared objects. +// Shared object usage is disabled until this method is called the first time. +// Once set, the Creator ID is persisted and cannot change. +// +// Does nothing if SharedStorage was not set in the options when the DB was +// opened or if the DB is in read-only mode. +func (d *DB) SetCreatorID(creatorID uint64) error { + if d.opts.Experimental.SharedStorage == nil || d.opts.ReadOnly { + return nil + } + return d.objProvider.SetCreatorID(objstorage.CreatorID(creatorID)) +} + +// ObjProvider returns the objstorage.Provider for this database. Meant to be +// used for internal purposes only. +func (d *DB) ObjProvider() objstorage.Provider { + return d.objProvider +} diff --git a/vendor/github.com/cockroachdb/pebble/error_iter.go b/vendor/github.com/cockroachdb/pebble/error_iter.go index b57679c..de0ed35 100644 --- a/vendor/github.com/cockroachdb/pebble/error_iter.go +++ b/vendor/github.com/cockroachdb/pebble/error_iter.go @@ -50,6 +50,10 @@ func (c *errorIter) Prev() (*InternalKey, base.LazyValue) { return nil, base.LazyValue{} } +func (c *errorIter) NextPrefix([]byte) (*InternalKey, base.LazyValue) { + return nil, base.LazyValue{} +} + func (c *errorIter) Error() error { return c.err } diff --git a/vendor/github.com/cockroachdb/pebble/event.go b/vendor/github.com/cockroachdb/pebble/event.go index f85e820..f243e24 100644 --- a/vendor/github.com/cockroachdb/pebble/event.go +++ b/vendor/github.com/cockroachdb/pebble/event.go @@ -5,11 +5,15 @@ package pebble import ( + "fmt" "strings" "time" + "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/humanize" + "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/manifest" + "github.com/cockroachdb/pebble/vfs" "github.com/cockroachdb/redact" ) @@ -115,22 +119,7 @@ func (i levelInfos) SafeFormat(w redact.SafePrinter, _ rune) { // DiskSlowInfo contains the info for a disk slowness event when writing to a // file. -type DiskSlowInfo struct { - // Path of file being written to. - Path string - // Duration that has elapsed since this disk operation started. - Duration time.Duration -} - -func (i DiskSlowInfo) String() string { - return redact.StringWithoutMarkers(i) -} - -// SafeFormat implements redact.SafeFormatter. -func (i DiskSlowInfo) SafeFormat(w redact.SafePrinter, _ rune) { - w.Printf("disk slowness detected: write to file %s has been ongoing for %0.1fs", - i.Path, redact.Safe(i.Duration.Seconds())) -} +type DiskSlowInfo = vfs.DiskSlowInfo // FlushInfo contains the info for a flush event. type FlushInfo struct { @@ -149,8 +138,14 @@ type FlushInfo struct { // TotalDuration is the total wall-time duration of the flush, including // applying the flush to the database. TotalDuration is always ≥ Duration. TotalDuration time.Duration - Done bool - Err error + // Ingest is set to true if the flush is handling tables that were added to + // the flushable queue via an ingestion operation. + Ingest bool + // IngestLevels are the output levels for each ingested table in the flush. + // This field is only populated when Ingest is true. + IngestLevels []int + Done bool + Err error } func (i FlushInfo) String() string { @@ -169,20 +164,47 @@ func (i FlushInfo) SafeFormat(w redact.SafePrinter, _ rune) { plural = "" } if !i.Done { - w.Printf("[JOB %d] flushing %d memtable", redact.Safe(i.JobID), redact.Safe(i.Input)) - w.SafeString(plural) - w.Printf(" to L0") + w.Printf("[JOB %d] ", redact.Safe(i.JobID)) + if !i.Ingest { + w.Printf("flushing %d memtable", redact.Safe(i.Input)) + w.SafeString(plural) + w.Printf(" to L0") + } else { + w.Printf("flushing %d ingested table%s", redact.Safe(i.Input), plural) + } return } outputSize := tablesTotalSize(i.Output) - w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s", - redact.Safe(i.JobID), redact.Safe(i.Input), plural, - redact.Safe(formatFileNums(i.Output)), - redact.Safe(humanize.Uint64(outputSize)), - redact.Safe(i.Duration.Seconds()), - redact.Safe(i.TotalDuration.Seconds()), - redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) + if !i.Ingest { + if invariants.Enabled && len(i.IngestLevels) > 0 { + panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) == 0")) + } + w.Printf("[JOB %d] flushed %d memtable%s to L0 [%s] (%s), in %.1fs (%.1fs total), output rate %s/s", + redact.Safe(i.JobID), redact.Safe(i.Input), plural, + redact.Safe(formatFileNums(i.Output)), + redact.Safe(humanize.Uint64(outputSize)), + redact.Safe(i.Duration.Seconds()), + redact.Safe(i.TotalDuration.Seconds()), + redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) + } else { + if invariants.Enabled && len(i.IngestLevels) == 0 { + panic(errors.AssertionFailedf("pebble: expected len(IngestedLevels) > 0")) + } + w.Printf("[JOB %d] flushed %d ingested flushable%s", + redact.Safe(i.JobID), redact.Safe(len(i.Output)), plural) + for j, level := range i.IngestLevels { + file := i.Output[j] + if j > 0 { + w.Printf(" +") + } + w.Printf(" L%d:%s (%s)", level, redact.Safe(file.FileNum), humanize.IEC.Uint64(file.Size)) + } + w.Printf(" in %.1fs (%.1fs total), output rate %s/s", + redact.Safe(i.Duration.Seconds()), + redact.Safe(i.TotalDuration.Seconds()), + redact.Safe(humanize.Uint64(uint64(float64(outputSize)/i.Duration.Seconds())))) + } } // ManifestCreateInfo contains info about a manifest creation event. @@ -283,7 +305,10 @@ type TableIngestInfo struct { // GlobalSeqNum is the sequence number that was assigned to all entries in // the ingested table. GlobalSeqNum uint64 - Err error + // flushable indicates whether the ingested sstable was treated as a + // flushable. + flushable bool + Err error } func (i TableIngestInfo) String() string { @@ -297,13 +322,22 @@ func (i TableIngestInfo) SafeFormat(w redact.SafePrinter, _ rune) { return } - w.Printf("[JOB %d] ingested", redact.Safe(i.JobID)) + if i.flushable { + w.Printf("[JOB %d] ingested as flushable", redact.Safe(i.JobID)) + } else { + w.Printf("[JOB %d] ingested", redact.Safe(i.JobID)) + } + for j := range i.Tables { t := &i.Tables[j] if j > 0 { w.Printf(",") } - w.Printf(" L%d:%s (%s)", redact.Safe(t.Level), redact.Safe(t.FileNum), + levelStr := "" + if !i.flushable { + levelStr = fmt.Sprintf("L%d:", t.Level) + } + w.Printf(" %s%s (%s)", redact.Safe(levelStr), redact.Safe(t.FileNum), redact.Safe(humanize.Uint64(t.Size))) } } @@ -427,9 +461,14 @@ type EventListener struct { // has been installed. CompactionEnd func(CompactionInfo) - // DiskSlow is invoked after a disk write operation on a file created - // with a disk health checking vfs.FS (see vfs.DefaultWithDiskHealthChecks) - // is observed to exceed the specified disk slowness threshold duration. + // DiskSlow is invoked after a disk write operation on a file created with a + // disk health checking vfs.FS (see vfs.DefaultWithDiskHealthChecks) is + // observed to exceed the specified disk slowness threshold duration. DiskSlow + // is called on a goroutine that is monitoring slowness/stuckness. The callee + // MUST return without doing any IO, or blocking on anything (like a mutex) + // that is waiting on IO. This is imperative in order to reliably monitor for + // slowness, since if this goroutine gets stuck, the monitoring will stop + // working. DiskSlow func(DiskSlowInfo) // FlushBegin is invoked after the inputs to a flush have been determined, diff --git a/vendor/github.com/cockroachdb/pebble/external_iterator.go b/vendor/github.com/cockroachdb/pebble/external_iterator.go index b5f6fae..3c45c2c 100644 --- a/vendor/github.com/cockroachdb/pebble/external_iterator.go +++ b/vendor/github.com/cockroachdb/pebble/external_iterator.go @@ -5,6 +5,7 @@ package pebble import ( + "context" "fmt" "sort" @@ -79,6 +80,18 @@ func NewExternalIter( iterOpts *IterOptions, files [][]sstable.ReadableFile, extraOpts ...ExternalIterOption, +) (it *Iterator, err error) { + return NewExternalIterWithContext(context.Background(), o, iterOpts, files, extraOpts...) +} + +// NewExternalIterWithContext is like NewExternalIter, and additionally +// accepts a context for tracing. +func NewExternalIterWithContext( + ctx context.Context, + o *Options, + iterOpts *IterOptions, + files [][]sstable.ReadableFile, + extraOpts ...ExternalIterOption, ) (it *Iterator, err error) { if iterOpts != nil { if err := validateExternalIterOpts(iterOpts); err != nil { @@ -119,6 +132,7 @@ func NewExternalIter( buf := iterAllocPool.Get().(*iterAlloc) dbi := &buf.dbi *dbi = Iterator{ + ctx: ctx, alloc: buf, merge: o.Merger.Merge, comparer: *o.Comparer, @@ -130,7 +144,9 @@ func NewExternalIter( // Add the readers to the Iterator so that Close closes them, and // SetOptions can re-construct iterators from them. externalReaders: readers, - newIters: func(f *manifest.FileMetadata, opts *IterOptions, internalOpts internalIterOpts) (internalIterator, keyspan.FragmentIterator, error) { + newIters: func( + ctx context.Context, f *manifest.FileMetadata, opts *IterOptions, + internalOpts internalIterOpts) (internalIterator, keyspan.FragmentIterator, error) { // NB: External iterators are currently constructed without any // `levelIters`. newIters should never be called. When we support // organizing multiple non-overlapping files into a single level @@ -143,12 +159,12 @@ func NewExternalIter( } if iterOpts != nil { dbi.opts = *iterOpts - dbi.saveBounds(iterOpts.LowerBound, iterOpts.UpperBound) + dbi.processBounds(iterOpts.LowerBound, iterOpts.UpperBound) } for i := range extraOpts { extraOpts[i].iterApply(dbi) } - finishInitializingExternal(dbi) + finishInitializingExternal(ctx, dbi) return dbi, nil } @@ -168,7 +184,7 @@ func validateExternalIterOpts(iterOpts *IterOptions) error { return nil } -func createExternalPointIter(it *Iterator) (internalIterator, error) { +func createExternalPointIter(ctx context.Context, it *Iterator) (internalIterator, error) { // TODO(jackson): In some instances we could generate fewer levels by using // L0Sublevels code to organize nonoverlapping files into the same level. // This would allow us to use levelIters and keep a smaller set of data and @@ -193,12 +209,14 @@ func createExternalPointIter(it *Iterator) (internalIterator, error) { pointIter internalIterator err error ) - pointIter, err = r.NewIterWithBlockPropertyFilters( + pointIter, err = r.NewIterWithBlockPropertyFiltersAndContext( + ctx, it.opts.LowerBound, it.opts.UpperBound, nil, /* BlockPropertiesFilterer */ false, /* useFilterBlock */ &it.stats.InternalStats, + sstable.TrivialReaderProvider{Reader: r}, ) if err != nil { return nil, err @@ -248,12 +266,14 @@ func createExternalPointIter(it *Iterator) (internalIterator, error) { it.alloc.merging.init(&it.opts, &it.stats.InternalStats, it.comparer.Compare, it.comparer.Split, mlevels...) it.alloc.merging.snapshot = base.InternalKeySeqNumMax - it.alloc.merging.elideRangeTombstones = true + if len(mlevels) <= cap(it.alloc.levelsPositioned) { + it.alloc.merging.levelsPositioned = it.alloc.levelsPositioned[:len(mlevels)] + } return &it.alloc.merging, nil } -func finishInitializingExternal(it *Iterator) { - pointIter, err := createExternalPointIter(it) +func finishInitializingExternal(ctx context.Context, it *Iterator) { + pointIter, err := createExternalPointIter(ctx, it) if err != nil { it.pointIter = &errorIter{err: err} } else { @@ -291,6 +311,7 @@ func finishInitializingExternal(it *Iterator) { base.InternalKeySeqNumMax, it.opts.LowerBound, it.opts.UpperBound, &it.hasPrefix, &it.prefixOrFullSeekKey, + true /* onlySets */, &it.rangeKey.internal, ) for i := range rangeKeyIters { it.rangeKey.iterConfig.AddLevel(rangeKeyIters[i]) @@ -314,7 +335,11 @@ func openExternalTables( ) (readers []*sstable.Reader, err error) { readers = make([]*sstable.Reader, 0, len(files)) for i := range files { - r, err := sstable.NewReader(files[i], readerOpts, extraReaderOpts...) + readable, err := sstable.NewSimpleReadable(files[i]) + if err != nil { + return readers, err + } + r, err := sstable.NewReader(readable, readerOpts, extraReaderOpts...) if err != nil { return readers, err } @@ -474,6 +499,20 @@ func (s *simpleLevelIter) Next() (*base.InternalKey, base.LazyValue) { return s.skipEmptyFileForward(nil /* seekKey */, base.SeekGEFlagsNone) } +func (s *simpleLevelIter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) { + if s.err != nil { + return nil, base.LazyValue{} + } + if s.currentIdx < 0 || s.currentIdx >= len(s.filtered) { + return nil, base.LazyValue{} + } + if iterKey, val := s.filtered[s.currentIdx].NextPrefix(succKey); iterKey != nil { + return iterKey, val + } + s.currentIdx++ + return s.skipEmptyFileForward(succKey /* seekKey */, base.SeekGEFlagsNone) +} + func (s *simpleLevelIter) Prev() (*base.InternalKey, base.LazyValue) { panic("unimplemented") } diff --git a/vendor/github.com/cockroachdb/pebble/filenames.go b/vendor/github.com/cockroachdb/pebble/filenames.go index bc33b38..07d74c8 100644 --- a/vendor/github.com/cockroachdb/pebble/filenames.go +++ b/vendor/github.com/cockroachdb/pebble/filenames.go @@ -33,7 +33,7 @@ const ( // NB: This is a low-level routine and typically not what you want to // use. Newer versions of Pebble running newer format major versions do // not use the CURRENT file. See setCurrentFunc in version_set.go. -func setCurrentFile(dirname string, fs vfs.FS, fileNum FileNum) error { +func setCurrentFile(dirname string, fs vfs.FS, fileNum base.DiskFileNum) error { newFilename := base.MakeFilepath(fs, dirname, fileTypeCurrent, fileNum) oldFilename := base.MakeFilepath(fs, dirname, fileTypeTemp, fileNum) fs.Remove(oldFilename) diff --git a/vendor/github.com/cockroachdb/pebble/flush_external.go b/vendor/github.com/cockroachdb/pebble/flush_external.go deleted file mode 100644 index 179beb2..0000000 --- a/vendor/github.com/cockroachdb/pebble/flush_external.go +++ /dev/null @@ -1,111 +0,0 @@ -// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package pebble - -import ( - "time" - - "github.com/cockroachdb/pebble/internal/private" -) - -// flushExternalTable is installed as the private.FlushExternalTable hook. -// It's used by the internal/replay package to flush external tables into L0 -// for supporting the pebble bench compact commands. Clients should use the -// replay package rather than calling this private hook directly. -func flushExternalTable(untypedDB interface{}, path string, originalMeta *fileMetadata) error { - d := untypedDB.(*DB) - if err := d.closed.Load(); err != nil { - panic(err) - } - if d.opts.ReadOnly { - return ErrReadOnly - } - - d.mu.Lock() - fileNum := d.mu.versions.getNextFileNum() - jobID := d.mu.nextJobID - d.mu.nextJobID++ - d.mu.Unlock() - - m := &fileMetadata{ - FileNum: fileNum, - Size: originalMeta.Size, - CreationTime: time.Now().Unix(), - SmallestSeqNum: originalMeta.SmallestSeqNum, - LargestSeqNum: originalMeta.LargestSeqNum, - } - if originalMeta.HasPointKeys { - m.ExtendPointKeyBounds(d.cmp, originalMeta.SmallestPointKey, originalMeta.LargestPointKey) - } - if originalMeta.HasRangeKeys { - m.ExtendRangeKeyBounds(d.cmp, originalMeta.SmallestRangeKey, originalMeta.LargestRangeKey) - } - - // Hard link the sstable into the DB directory. - if err := ingestLink(jobID, d.opts, d.dirname, []string{path}, []*fileMetadata{m}); err != nil { - return err - } - if err := d.dataDir.Sync(); err != nil { - return err - } - - // Assign sequence numbers from its current value through to the - // external file's largest sequence number. It's possible that the current - // sequence number has already exceeded m.LargestSeqNum if there were many - // nonoverlapping ingestions, so skip if this doesn't bump the sequence - // number. - d.commit.ratchetSeqNum(m.LargestSeqNum + 1) - d.commit.mu.Lock() - defer d.commit.mu.Unlock() - - // Apply the version edit. - d.mu.Lock() - d.mu.versions.logLock() - ve := &versionEdit{ - NewFiles: []newFileEntry{{Level: 0, Meta: m}}, - } - metrics := map[int]*LevelMetrics{ - 0: { - NumFiles: 1, - Size: int64(m.Size), - BytesIngested: m.Size, - TablesIngested: 1, - }, - } - err := d.mu.versions.logAndApply(jobID, ve, metrics, false /* forceRotation */, func() []compactionInfo { - return d.getInProgressCompactionInfoLocked(nil) - }) - if err != nil { - // NB: logAndApply will release d.mu.versions.logLock unconditionally. - d.mu.Unlock() - if err2 := ingestCleanup(d.opts.FS, d.dirname, []*fileMetadata{m}); err2 != nil { - d.opts.Logger.Infof("flush external cleanup failed: %v", err2) - } - return err - } - d.updateReadStateLocked(d.opts.DebugCheck) - d.updateTableStatsLocked(ve.NewFiles) - d.deleteObsoleteFiles(jobID, true /* waitForOngoing */) - d.maybeScheduleCompaction() - d.mu.Unlock() - return nil -} - -// ratchetSeqNum is a hook for allocating and publishing sequence numbers up -// to a specific absolute value. Its first parameter is a *pebble.DB and its -// second is the new next sequence number. RatchetSeqNum does nothing if the -// next sequence is already greater than or equal to nextSeqNum. -// -// This function is used by the internal/replay package to ensure replayed -// operations receive the same absolute sequence number. -func ratchetSeqNum(untypedDB interface{}, nextSeqNum uint64) { - d := untypedDB.(*DB) - d.commit.ratchetSeqNum(nextSeqNum) -} - -func init() { - private.FlushExternalTable = flushExternalTable - private.RatchetSeqNum = ratchetSeqNum -} diff --git a/vendor/github.com/cockroachdb/pebble/flushable.go b/vendor/github.com/cockroachdb/pebble/flushable.go index c96c91a..0a5b2ae 100644 --- a/vendor/github.com/cockroachdb/pebble/flushable.go +++ b/vendor/github.com/cockroachdb/pebble/flushable.go @@ -5,11 +5,13 @@ package pebble import ( + "context" "fmt" "sync/atomic" "time" "github.com/cockroachdb/pebble/internal/keyspan" + "github.com/cockroachdb/pebble/internal/manifest" ) // flushable defines the interface for immutable memtables. @@ -53,12 +55,21 @@ type flushableEntry struct { // readerRefs tracks the read references on the flushable. The two sources of // reader references are DB.mu.mem.queue and readState.memtables. The memory // reserved by the flushable in the cache is released when the reader refs - // drop to zero. If the flushable is a memTable, when the reader refs drops - // to zero, the writer refs will already be zero because the memtable will - // have been flushed and that only occurs once the writer refs drops to zero. + // drop to zero. If the flushable is referencing sstables, then the file + // refount is also decreased once the reader refs drops to 0. If the + // flushable is a memTable, when the reader refs drops to zero, the writer + // refs will already be zero because the memtable will have been flushed and + // that only occurs once the writer refs drops to zero. readerRefs int32 // Closure to invoke to release memory accounting. releaseMemAccounting func() + // unrefFiles, if not nil, should be invoked to decrease the ref count of + // files which are backing the flushable. + unrefFiles func() []*fileBacking + // deleteFnLocked should be called if the caller is holding DB.mu. + deleteFnLocked func(obsolete []*fileBacking) + // deleteFn should be called if the caller is not holding DB.mu. + deleteFn func(obsolete []*fileBacking) } func (e *flushableEntry) readerRef() { @@ -68,7 +79,19 @@ func (e *flushableEntry) readerRef() { } } -func (e *flushableEntry) readerUnref() { +// db.mu must not be held when this is called. +func (e *flushableEntry) readerUnref(deleteFiles bool) { + e.readerUnrefHelper(deleteFiles, e.deleteFn) +} + +// db.mu must be held when this is called. +func (e *flushableEntry) readerUnrefLocked(deleteFiles bool) { + e.readerUnrefHelper(deleteFiles, e.deleteFnLocked) +} + +func (e *flushableEntry) readerUnrefHelper( + deleteFiles bool, deleteFn func(obsolete []*fileBacking), +) { switch v := atomic.AddInt32(&e.readerRefs, -1); { case v < 0: panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) @@ -78,7 +101,144 @@ func (e *flushableEntry) readerUnref() { } e.releaseMemAccounting() e.releaseMemAccounting = nil + if e.unrefFiles != nil { + obsolete := e.unrefFiles() + e.unrefFiles = nil + if deleteFiles { + deleteFn(obsolete) + } + } } } type flushableList []*flushableEntry + +// ingestedFlushable is the implementation of the flushable interface for the +// ingesting sstables which are added to the flushable list. +type ingestedFlushable struct { + files []physicalMeta + cmp Compare + split Split + newIters tableNewIters + newRangeKeyIters keyspan.TableNewSpanIter + + // Since the level slice is immutable, we construct and set it once. It + // should be safe to read from slice in future reads. + slice manifest.LevelSlice + // hasRangeKeys is set on ingestedFlushable construction. + hasRangeKeys bool +} + +func newIngestedFlushable( + files []*fileMetadata, + cmp Compare, + split Split, + newIters tableNewIters, + newRangeKeyIters keyspan.TableNewSpanIter, +) *ingestedFlushable { + var physicalFiles []physicalMeta + var hasRangeKeys bool + for _, f := range files { + if f.HasRangeKeys { + hasRangeKeys = true + } + physicalFiles = append(physicalFiles, f.PhysicalMeta()) + } + + ret := &ingestedFlushable{ + files: physicalFiles, + cmp: cmp, + split: split, + newIters: newIters, + newRangeKeyIters: newRangeKeyIters, + // slice is immutable and can be set once and used many times. + slice: manifest.NewLevelSliceKeySorted(cmp, files), + hasRangeKeys: hasRangeKeys, + } + + return ret +} + +// TODO(sumeer): ingestedFlushable iters also need to plumb context for +// tracing. + +func (s *ingestedFlushable) newIter(o *IterOptions) internalIterator { + var opts IterOptions + if o != nil { + opts = *o + } + // TODO(bananabrick): The manifest.Level in newLevelIter is only used for + // logging. Update the manifest.Level encoding to account for levels which + // aren't truly levels in the lsm. Right now, the encoding only supports + // L0 sublevels, and the rest of the levels in the lsm. + return newLevelIter( + opts, s.cmp, s.split, s.newIters, s.slice.Iter(), manifest.Level(0), nil, + ) +} + +func (s *ingestedFlushable) newFlushIter(o *IterOptions, bytesFlushed *uint64) internalIterator { + // newFlushIter is only used for writing memtables to disk as sstables. + // Since ingested sstables are already present on disk, they don't need to + // make use of a flush iter. + panic("pebble: not implemented") +} + +func (s *ingestedFlushable) constructRangeDelIter( + file *manifest.FileMetadata, _ *keyspan.SpanIterOptions, +) (keyspan.FragmentIterator, error) { + // Note that the keyspan level iter expects a non-nil iterator to be + // returned even if there is an error. So, we return the emptyKeyspanIter. + iter, rangeDelIter, err := s.newIters(context.Background(), file, nil, internalIterOpts{}) + if err != nil { + return emptyKeyspanIter, err + } + iter.Close() + if rangeDelIter == nil { + return emptyKeyspanIter, nil + } + return rangeDelIter, nil +} + +// TODO(bananabrick): Using a level iter instead of a keyspan level iter to +// surface range deletes is more efficient. +func (s *ingestedFlushable) newRangeDelIter(_ *IterOptions) keyspan.FragmentIterator { + return keyspan.NewLevelIter( + keyspan.SpanIterOptions{}, s.cmp, + s.constructRangeDelIter, s.slice.Iter(), manifest.Level(0), + manifest.KeyTypePoint, + ) +} + +func (s *ingestedFlushable) newRangeKeyIter(o *IterOptions) keyspan.FragmentIterator { + if !s.containsRangeKeys() { + return nil + } + + return keyspan.NewLevelIter( + keyspan.SpanIterOptions{}, s.cmp, s.newRangeKeyIters, + s.slice.Iter(), manifest.Level(0), manifest.KeyTypeRange, + ) +} + +func (s *ingestedFlushable) containsRangeKeys() bool { + return s.hasRangeKeys +} + +func (s *ingestedFlushable) inuseBytes() uint64 { + // inuseBytes is only used when memtables are flushed to disk as sstables. + panic("pebble: not implemented") +} + +func (s *ingestedFlushable) totalBytes() uint64 { + // We don't allocate additional bytes for the ingestedFlushable. + return 0 +} + +func (s *ingestedFlushable) readyForFlush() bool { + // ingestedFlushable should always be ready to flush. However, note that + // memtables before the ingested sstables in the memtable queue must be + // flushed before an ingestedFlushable can be flushed. This is because the + // ingested sstables need an updated view of the Version to + // determine where to place the files in the lsm. + return true +} diff --git a/vendor/github.com/cockroachdb/pebble/format_major_version.go b/vendor/github.com/cockroachdb/pebble/format_major_version.go index 9b6546e..6d96297 100644 --- a/vendor/github.com/cockroachdb/pebble/format_major_version.go +++ b/vendor/github.com/cockroachdb/pebble/format_major_version.go @@ -110,18 +110,49 @@ const ( // 23.1 versions. - // FormatPrePebblev1MarkedCompacted is a format major version that - // guarantees that all sstables explicitly marked for compaction in the - // manifest have been compacted. Ratcheting to this format version will block - // (without holding mutexes) until all necessary compactions for files marked - // for compaction are complete. + // FormatUnusedPrePebblev1MarkedCompacted is an unused format major version. + // This format major version was originally intended to ship in the 23.1 + // release. It was later decided that this should be deferred until a + // subsequent release. The original ordering is preserved so as not to + // introduce breaking changes in Cockroach. + FormatUnusedPrePebblev1MarkedCompacted + + // FormatSSTableValueBlocks is a format major version that adds support for + // storing values in value blocks in the sstable. Value block support is not + // necessarily enabled when writing sstables, when running with this format + // major version. + // + // WARNING: In development, so no production code should upgrade to this + // format, since a DB with this format major version will not actually + // interoperate correctly with another DB with the same format major + // version. This format major version is introduced so that tests can start + // being executed up to this version. Note that these tests succeed despite + // the incomplete support since they do not enable value blocks and use + // TableFormatPebblev2. + FormatSSTableValueBlocks + + // FormatFlushableIngest is a format major version that enables lazy + // addition of ingested sstables into the LSM structure. When an ingest + // overlaps with a memtable, a record of the ingest is written to the WAL + // without waiting for a flush. Subsequent reads treat the ingested files as + // a level above the overlapping memtable. Once the memtable is flushed, the + // ingested files are moved into the lowest possible levels. + // + // This feature is behind a format major version because it required + // breaking changes to the WAL format. + FormatFlushableIngest + + // 23.2 versions. + + // FormatPrePebblev1MarkedCompacted is a format major version that guarantees + // that all sstables explicitly marked for compaction in the manifest (see + // FormatPrePebblev1Marked) have been compacted. Ratcheting to this format + // version will block (without holding mutexes) until all necessary + // compactions for files marked for compaction are complete. FormatPrePebblev1MarkedCompacted // FormatNewest always contains the most recent format major version. - // NB: When adding new versions, the MaxTableFormat method should also be - // updated to return the maximum allowable version for the new - // FormatMajorVersion. - FormatNewest FormatMajorVersion = FormatPrePebblev1MarkedCompacted + FormatNewest FormatMajorVersion = iota - 1 ) // MaxTableFormat returns the maximum sstable.TableFormat that can be used at @@ -135,8 +166,10 @@ func (v FormatMajorVersion) MaxTableFormat() sstable.TableFormat { FormatSplitUserKeysMarkedCompacted: return sstable.TableFormatPebblev1 case FormatRangeKeys, FormatMinTableFormatPebblev1, FormatPrePebblev1Marked, - FormatPrePebblev1MarkedCompacted: + FormatUnusedPrePebblev1MarkedCompacted: return sstable.TableFormatPebblev2 + case FormatSSTableValueBlocks, FormatFlushableIngest, FormatPrePebblev1MarkedCompacted: + return sstable.TableFormatPebblev3 default: panic(fmt.Sprintf("pebble: unsupported format major version: %s", v)) } @@ -152,7 +185,8 @@ func (v FormatMajorVersion) MinTableFormat() sstable.TableFormat { FormatRangeKeys: return sstable.TableFormatLevelDB case FormatMinTableFormatPebblev1, FormatPrePebblev1Marked, - FormatPrePebblev1MarkedCompacted: + FormatUnusedPrePebblev1MarkedCompacted, FormatSSTableValueBlocks, + FormatFlushableIngest, FormatPrePebblev1MarkedCompacted: return sstable.TableFormatPebblev1 default: panic(fmt.Sprintf("pebble: unsupported format major version: %s", v)) @@ -184,7 +218,7 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{ // guaranteed to exist, because we unconditionally locate it // during Open. manifestFileNum := d.mu.versions.manifestFileNum - filename := base.MakeFilename(fileTypeManifest, manifestFileNum) + filename := base.MakeFilename(fileTypeManifest, manifestFileNum.DiskFileNum()) if err := d.mu.versions.manifestMarker.Move(filename); err != nil { return errors.Wrap(err, "moving manifest marker") } @@ -222,7 +256,7 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{ // version that does not know about format major versions // attempts to open the database, it will error avoiding // accidental corruption. - if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, 0); err != nil { + if err := setCurrentFile(d.mu.versions.dirname, d.mu.versions.fs, base.FileNum(0).DiskFileNum()); err != nil { return err } return d.finalizeFormatVersUpgrade(FormatVersioned) @@ -266,6 +300,16 @@ var formatMajorVersionMigrations = map[FormatMajorVersion]func(*DB) error{ } return d.finalizeFormatVersUpgrade(FormatPrePebblev1Marked) }, + FormatUnusedPrePebblev1MarkedCompacted: func(d *DB) error { + // Intentional no-op. + return d.finalizeFormatVersUpgrade(FormatUnusedPrePebblev1MarkedCompacted) + }, + FormatSSTableValueBlocks: func(d *DB) error { + return d.finalizeFormatVersUpgrade(FormatSSTableValueBlocks) + }, + FormatFlushableIngest: func(d *DB) error { + return d.finalizeFormatVersUpgrade(FormatFlushableIngest) + }, FormatPrePebblev1MarkedCompacted: func(d *DB) error { // Before finalizing the format major version, rewrite any sstables // still marked for compaction. Note all format major versions @@ -476,17 +520,24 @@ var markFilesPrePebblev1 = func(tc *tableCacheContainer) findFilesFunc { for l := numLevels - 1; l > 0; l-- { iter := v.Levels[l].Iter() for f := iter.First(); f != nil; f = iter.Next() { - err = tc.withReader(f, func(r *sstable.Reader) error { - tf, err := r.TableFormat() - if err != nil { - return err - } - if tf < sstable.TableFormatPebblev1 { - found = true - files[l] = append(files[l], f) - } - return nil - }) + if f.Virtual { + // Any physical sstable which has been virtualized must + // have already undergone this migration, and we don't + // need to worry about the virtual sstable themselves. + panic("pebble: unexpected virtual sstable during migration") + } + err = tc.withReader( + f.PhysicalMeta(), func(r *sstable.Reader) error { + tf, err := r.TableFormat() + if err != nil { + return err + } + if tf < sstable.TableFormatPebblev1 { + found = true + files[l] = append(files[l], f) + } + return nil + }) if err != nil { return } @@ -502,31 +553,24 @@ func (d *DB) markFilesLocked(findFn findFilesFunc) error { jobID := d.mu.nextJobID d.mu.nextJobID++ - vers := d.mu.versions.currentVersion() + // Acquire a read state to have a view of the LSM and a guarantee that none + // of the referenced files will be deleted until we've unreferenced the read + // state. Some findFilesFuncs may read the files, requiring they not be + // deleted. + rs := d.loadReadState() var ( found bool files [numLevels][]*fileMetadata err error ) func() { - // Disable file deletions. Some migrations that mark files may need to - // read the files themselves. The LSM may be changing while the - // migration is determining which files to mark. That's ok, because - // after re-acquiring the mutex and acquiring the manifest lock, we'll - // discard any sstables that have been removed from the LSM. However, we - // also need to make sure that the files we try to open during the scan - // are not removed before we read them. This is preferable over - // gracefully handling nonexistent files because some environments (eg, - // Windows) error if you attempt to remove an open file. - d.disableFileDeletions() - defer d.enableFileDeletions() - + defer rs.unrefLocked() // Note the unusual locking: unlock, defer Lock(). The scan of the files in // the version does not need to block other operations that require the // DB.mu. Drop it for the scan, before re-acquiring it. d.mu.Unlock() defer d.mu.Lock() - found, files, err = findFn(vers) + found, files, err = findFn(rs.current) }() if err != nil { return err @@ -547,7 +591,7 @@ func (d *DB) markFilesLocked(findFn findFilesFunc) error { // Lock the manifest for a coherent view of the LSM. The database lock has // been re-acquired by the defer within the above anonymous function. d.mu.versions.logLock() - vers = d.mu.versions.currentVersion() + vers := d.mu.versions.currentVersion() for l, filesToMark := range files { if len(filesToMark) == 0 { continue diff --git a/vendor/github.com/cockroachdb/pebble/get_iter.go b/vendor/github.com/cockroachdb/pebble/get_iter.go index e7243fa..f1fc25c 100644 --- a/vendor/github.com/cockroachdb/pebble/get_iter.go +++ b/vendor/github.com/cockroachdb/pebble/get_iter.go @@ -5,6 +5,7 @@ package pebble import ( + "context" "fmt" "github.com/cockroachdb/pebble/internal/base" @@ -158,7 +159,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) { files := g.l0[n-1].Iter() g.l0 = g.l0[:n-1] iterOpts := IterOptions{logger: g.logger} - g.levelIter.init(iterOpts, g.cmp, nil /* split */, g.newIters, + g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters, files, manifest.L0Sublevel(n), internalIterOpts{}) g.levelIter.initRangeDel(&g.rangeDelIter) g.iter = &g.levelIter @@ -177,7 +178,7 @@ func (g *getIter) Next() (*InternalKey, base.LazyValue) { } iterOpts := IterOptions{logger: g.logger} - g.levelIter.init(iterOpts, g.cmp, nil /* split */, g.newIters, + g.levelIter.init(context.Background(), iterOpts, g.cmp, nil /* split */, g.newIters, g.version.Levels[g.level].Iter(), manifest.Level(g.level), internalIterOpts{}) g.levelIter.initRangeDel(&g.rangeDelIter) g.level++ @@ -190,6 +191,10 @@ func (g *getIter) Prev() (*InternalKey, base.LazyValue) { panic("pebble: Prev unimplemented") } +func (g *getIter) NextPrefix([]byte) (*InternalKey, base.LazyValue) { + panic("pebble: NextPrefix unimplemented") +} + func (g *getIter) Valid() bool { return g.iterKey != nil && g.err == nil } diff --git a/vendor/github.com/cockroachdb/pebble/ingest.go b/vendor/github.com/cockroachdb/pebble/ingest.go index 20ca53b..617c8b2 100644 --- a/vendor/github.com/cockroachdb/pebble/ingest.go +++ b/vendor/github.com/cockroachdb/pebble/ingest.go @@ -5,6 +5,7 @@ package pebble import ( + "context" "sort" "time" @@ -13,8 +14,8 @@ import ( "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/private" + "github.com/cockroachdb/pebble/objstorage" "github.com/cockroachdb/pebble/sstable" - "github.com/cockroachdb/pebble/vfs" ) func sstableKeyCompare(userCmp Compare, a, b InternalKey) int { @@ -22,12 +23,12 @@ func sstableKeyCompare(userCmp Compare, a, b InternalKey) int { if c != 0 { return c } - if a.Trailer == InternalKeyRangeDeleteSentinel { - if b.Trailer != InternalKeyRangeDeleteSentinel { + if a.IsExclusiveSentinel() { + if !b.IsExclusiveSentinel() { return -1 } - } else if b.Trailer == InternalKeyRangeDeleteSentinel { - return 1 + } else if b.IsExclusiveSentinel() { + return +1 } return 0 } @@ -45,20 +46,20 @@ func ingestValidateKey(opts *Options, key *InternalKey) error { } func ingestLoad1( - opts *Options, fmv FormatMajorVersion, path string, cacheID uint64, fileNum FileNum, + opts *Options, fmv FormatMajorVersion, path string, cacheID uint64, fileNum base.DiskFileNum, ) (*fileMetadata, error) { - stat, err := opts.FS.Stat(path) - if err != nil { - return nil, err - } - f, err := opts.FS.Open(path) if err != nil { return nil, err } + readable, err := sstable.NewSimpleReadable(f) + if err != nil { + return nil, err + } + cacheOpts := private.SSTableCacheOpts(cacheID, fileNum).(sstable.ReaderOption) - r, err := sstable.NewReader(f, opts.MakeReaderOptions(), cacheOpts) + r, err := sstable.NewReader(readable, opts.MakeReaderOptions(), cacheOpts) if err != nil { return nil, err } @@ -77,9 +78,10 @@ func ingestLoad1( } meta := &fileMetadata{} - meta.FileNum = fileNum - meta.Size = uint64(stat.Size()) + meta.FileNum = fileNum.FileNum() + meta.Size = uint64(readable.Size()) meta.CreationTime = time.Now().Unix() + meta.InitPhysicalBacking() // Avoid loading into the table cache for collecting stats if we // don't need to. If there are no range deletions, we have all the @@ -90,7 +92,7 @@ func ingestLoad1( // disallowing removal of an open file. Under MemFS, if we don't populate // meta.Stats here, the file will be loaded into the table cache for // calculating stats before we can remove the original link. - maybeSetStatsFromProperties(meta, &r.Properties) + maybeSetStatsFromProperties(meta.PhysicalMeta(), &r.Properties) { iter, err := r.NewIter(nil /* lower */, nil /* upper */) @@ -194,7 +196,7 @@ func ingestLoad1( } func ingestLoad( - opts *Options, fmv FormatMajorVersion, paths []string, cacheID uint64, pending []FileNum, + opts *Options, fmv FormatMajorVersion, paths []string, cacheID uint64, pending []base.DiskFileNum, ) ([]*fileMetadata, []string, error) { meta := make([]*fileMetadata, 0, len(paths)) newPaths := make([]string, 0, len(paths)) @@ -252,54 +254,28 @@ func ingestSortAndVerify(cmp Compare, meta []*fileMetadata, paths []string) erro return nil } -func ingestCleanup(fs vfs.FS, dirname string, meta []*fileMetadata) error { +func ingestCleanup(objProvider objstorage.Provider, meta []*fileMetadata) error { var firstErr error for i := range meta { - target := base.MakeFilepath(fs, dirname, fileTypeTable, meta[i].FileNum) - if err := fs.Remove(target); err != nil { + if err := objProvider.Remove(fileTypeTable, meta[i].FileBacking.DiskFileNum); err != nil { firstErr = firstError(firstErr, err) } } return firstErr } +// ingestLink creates new objects which are backed by either hardlinks to or +// copies of the ingested files. func ingestLink( - jobID int, opts *Options, dirname string, paths []string, meta []*fileMetadata, + jobID int, opts *Options, objProvider objstorage.Provider, paths []string, meta []*fileMetadata, ) error { - // Wrap the normal filesystem with one which wraps newly created files with - // vfs.NewSyncingFile. - fs := syncingFS{ - FS: opts.FS, - syncOpts: vfs.SyncingFileOptions{ - NoSyncOnClose: opts.NoSyncOnClose, - BytesPerSync: opts.BytesPerSync, - }, - } - for i := range paths { - target := base.MakeFilepath(fs, dirname, fileTypeTable, meta[i].FileNum) - var err error - if _, ok := opts.FS.(*vfs.MemFS); ok && opts.DebugCheck != nil { - // The combination of MemFS+Ingest+DebugCheck produces awkwardness around - // the subsequent deletion of files. The problem is that MemFS implements - // the Windows semantics of disallowing removal of an open file. This is - // desirable because it helps catch bugs where we violate the - // requirements of the Windows semantics. The normal practice for Ingest - // is for the caller to remove the source files after the ingest - // completes successfully. Unfortunately, Options.DebugCheck causes - // ingest to run DB.CheckLevels() before the ingest finishes, and - // DB.CheckLevels() populates the table cache with the newly ingested - // files. - // - // The combination of MemFS+Ingest+DebugCheck is primarily used in - // tests. As a workaround, disable hard linking this combination - // occurs. See https://github.com/cockroachdb/pebble/issues/495. - err = vfs.Copy(fs, paths[i], target) - } else { - err = vfs.LinkOrCopy(fs, paths[i], target) - } + objMeta, err := objProvider.LinkOrCopyFromLocal( + context.TODO(), opts.FS, paths[i], fileTypeTable, meta[i].FileBacking.DiskFileNum, + objstorage.CreateOptions{PreferSharedStorage: true}, + ) if err != nil { - if err2 := ingestCleanup(fs, dirname, meta[:i]); err2 != nil { + if err2 := ingestCleanup(objProvider, meta[:i]); err2 != nil { opts.Logger.Infof("ingest cleanup failed: %v", err2) } return err @@ -308,7 +284,7 @@ func ingestLink( opts.EventListener.TableCreated(TableCreateInfo{ JobID: jobID, Reason: "ingesting", - Path: target, + Path: objProvider.Path(objMeta), FileNum: meta[i].FileNum, }) } @@ -320,18 +296,28 @@ func ingestLink( func ingestMemtableOverlaps(cmp Compare, mem flushable, meta []*fileMetadata) bool { iter := mem.newIter(nil) rangeDelIter := mem.newRangeDelIter(nil) - defer iter.Close() + rkeyIter := mem.newRangeKeyIter(nil) - if rangeDelIter != nil { - defer rangeDelIter.Close() + closeIters := func() error { + err := iter.Close() + if rangeDelIter != nil { + err = firstError(err, rangeDelIter.Close()) + } + if rkeyIter != nil { + err = firstError(err, rkeyIter.Close()) + } + return err } for _, m := range meta { - if overlapWithIterator(iter, &rangeDelIter, m, cmp) { + if overlapWithIterator(iter, &rangeDelIter, rkeyIter, m, cmp) { + closeIters() return true } } - return false + + // Assume overlap if any iterator errored out. + return closeIters() != nil } func ingestUpdateSeqNum( @@ -377,7 +363,11 @@ func ingestUpdateSeqNum( } func overlapWithIterator( - iter internalIterator, rangeDelIter *keyspan.FragmentIterator, meta *fileMetadata, cmp Compare, + iter internalIterator, + rangeDelIter *keyspan.FragmentIterator, + rkeyIter keyspan.FragmentIterator, + meta *fileMetadata, + cmp Compare, ) bool { // Check overlap with point operations. // @@ -405,36 +395,60 @@ func overlapWithIterator( return true } } + // Assume overlap if iterator errored. + if err := iter.Error(); err != nil { + return true + } + + computeOverlapWithSpans := func(rIter keyspan.FragmentIterator) bool { + // NB: The spans surfaced by the fragment iterator are non-overlapping. + span := rIter.SeekLT(meta.Smallest.UserKey) + if span == nil { + span = rIter.Next() + } + for ; span != nil; span = rIter.Next() { + if span.Empty() { + continue + } + key := span.SmallestKey() + c := sstableKeyCompare(cmp, key, meta.Largest) + if c > 0 { + // The start of the span is after the largest key in the + // ingested table. + return false + } + if cmp(span.End, meta.Smallest.UserKey) > 0 { + // The end of the span is greater than the smallest in the + // table. Note that the span end key is exclusive, thus ">0" + // instead of ">=0". + return true + } + } + // Assume overlap if iterator errored. + if err := rIter.Error(); err != nil { + return true + } + return false + } + + // rkeyIter is either a range key level iter, or a range key iterator + // over a single file. + if rkeyIter != nil { + if computeOverlapWithSpans(rkeyIter) { + return true + } + } // Check overlap with range deletions. if rangeDelIter == nil || *rangeDelIter == nil { return false } - rangeDelItr := *rangeDelIter - rangeDel := rangeDelItr.SeekLT(meta.Smallest.UserKey) - if rangeDel == nil { - rangeDel = rangeDelItr.Next() - } - for ; rangeDel != nil; rangeDel = rangeDelItr.Next() { - key := rangeDel.SmallestKey() - c := sstableKeyCompare(cmp, key, meta.Largest) - if c > 0 { - // The start of the tombstone is after the largest key in the - // ingested table. - return false - } - if cmp(rangeDel.End, meta.Smallest.UserKey) > 0 { - // The end of the tombstone is greater than the smallest in the - // table. Note that the tombstone end key is exclusive, thus ">0" - // instead of ">=0". - return true - } - } - return false + return computeOverlapWithSpans(*rangeDelIter) } func ingestTargetLevel( newIters tableNewIters, + newRangeKeyIter keyspan.TableNewSpanIter, iterOps IterOptions, cmp Compare, v *version, @@ -502,24 +516,33 @@ func ingestTargetLevel( targetLevel := 0 - // Do we overlap with keys in L0? - iter := v.Levels[0].Iter() - for meta0 := iter.First(); meta0 != nil; meta0 = iter.Next() { - c1 := sstableKeyCompare(cmp, meta.Smallest, meta0.Largest) - c2 := sstableKeyCompare(cmp, meta.Largest, meta0.Smallest) - if c1 > 0 || c2 < 0 { - continue - } + // This assertion implicitly checks that we have the current version of + // the metadata. + if v.L0Sublevels == nil { + return 0, errors.AssertionFailedf("could not read L0 sublevels") + } + // Check for overlap over the keys of L0 by iterating over the sublevels. + for subLevel := 0; subLevel < len(v.L0SublevelFiles); subLevel++ { + iter := newLevelIter(iterOps, cmp, nil /* split */, newIters, + v.L0Sublevels.Levels[subLevel].Iter(), manifest.Level(0), nil) - iter, rangeDelIter, err := newIters(iter.Current(), nil, internalIterOpts{}) + var rangeDelIter keyspan.FragmentIterator + // Pass in a non-nil pointer to rangeDelIter so that levelIter.findFileGE + // sets it up for the target file. + iter.initRangeDel(&rangeDelIter) + + levelIter := keyspan.LevelIter{} + levelIter.Init( + keyspan.SpanIterOptions{}, cmp, newRangeKeyIter, + v.L0Sublevels.Levels[subLevel].Iter(), manifest.Level(0), manifest.KeyTypeRange, + ) + + overlap := overlapWithIterator(iter, &rangeDelIter, &levelIter, meta, cmp) + err := iter.Close() // Closes range del iter as well. + err = firstError(err, levelIter.Close()) if err != nil { return 0, err } - overlap := overlapWithIterator(iter, &rangeDelIter, meta, cmp) - iter.Close() - if rangeDelIter != nil { - rangeDelIter.Close() - } if overlap { return targetLevel, nil } @@ -533,8 +556,19 @@ func ingestTargetLevel( // Pass in a non-nil pointer to rangeDelIter so that levelIter.findFileGE // sets it up for the target file. levelIter.initRangeDel(&rangeDelIter) - overlap := overlapWithIterator(levelIter, &rangeDelIter, meta, cmp) - levelIter.Close() // Closes range del iter as well. + + rkeyLevelIter := &keyspan.LevelIter{} + rkeyLevelIter.Init( + keyspan.SpanIterOptions{}, cmp, newRangeKeyIter, + v.Levels[level].Iter(), manifest.Level(level), manifest.KeyTypeRange, + ) + + overlap := overlapWithIterator(levelIter, &rangeDelIter, rkeyLevelIter, meta, cmp) + err := levelIter.Close() // Closes range del iter as well. + err = firstError(err, rkeyLevelIter.Close()) + if err != nil { + return 0, err + } if overlap { return targetLevel, nil } @@ -593,18 +627,18 @@ func ingestTargetLevel( // // The steps for ingestion are: // -// 1. Allocate file numbers for every sstable being ingested. -// 2. Load the metadata for all sstables being ingest. -// 3. Sort the sstables by smallest key, verifying non overlap. -// 4. Hard link (or copy) the sstables into the DB directory. -// 5. Allocate a sequence number to use for all of the entries in the -// sstables. This is the step where overlap with memtables is -// determined. If there is overlap, we remember the most recent memtable -// that overlaps. -// 6. Update the sequence number in the ingested sstables. -// 7. Wait for the most recent memtable that overlaps to flush (if any). -// 8. Add the ingested sstables to the version (DB.ingestApply). -// 9. Publish the ingestion sequence number. +// 1. Allocate file numbers for every sstable being ingested. +// 2. Load the metadata for all sstables being ingest. +// 3. Sort the sstables by smallest key, verifying non overlap. +// 4. Hard link (or copy) the sstables into the DB directory. +// 5. Allocate a sequence number to use for all of the entries in the +// sstables. This is the step where overlap with memtables is +// determined. If there is overlap, we remember the most recent memtable +// that overlaps. +// 6. Update the sequence number in the ingested sstables. +// 7. Wait for the most recent memtable that overlaps to flush (if any). +// 8. Add the ingested sstables to the version (DB.ingestApply). +// 9. Publish the ingestion sequence number. // // Note that if the mutable memtable overlaps with ingestion, a flush of the // memtable is forced equivalent to DB.Flush. Additionally, subsequent @@ -630,11 +664,14 @@ type IngestOperationStats struct { // Bytes is the total bytes in the ingested sstables. Bytes uint64 // ApproxIngestedIntoL0Bytes is the approximate number of bytes ingested - // into L0. - // Currently, this value is completely accurate, but we are allowing this to - // be approximate once https://github.com/cockroachdb/pebble/issues/25 is - // implemented. + // into L0. This value is approximate when flushable ingests are active and + // an ingest overlaps an entry in the flushable queue. Currently, this + // approximation is very rough, only including tables that overlapped the + // memtable. This estimate may be improved with #2112. ApproxIngestedIntoL0Bytes uint64 + // MemtableOverlappingFiles is the count of ingested sstables + // that overlapped keys in the memtables. + MemtableOverlappingFiles int } // IngestWithStats does the same as Ingest, and additionally returns @@ -649,6 +686,113 @@ func (d *DB) IngestWithStats(paths []string) (IngestOperationStats, error) { return d.ingest(paths, ingestTargetLevel) } +// Both DB.mu and commitPipeline.mu must be held while this is called. +func (d *DB) newIngestedFlushableEntry( + meta []*fileMetadata, seqNum uint64, logNum FileNum, +) (*flushableEntry, error) { + // Update the sequence number for all of the sstables in the + // metadata. Writing the metadata to the manifest when the + // version edit is applied is the mechanism that persists the + // sequence number. The sstables themselves are left unmodified. + // In this case, a version edit will only be written to the manifest + // when the flushable is eventually flushed. If Pebble restarts in that + // time, then we'll lose the ingest sequence number information. But this + // information will also be reconstructed on node restart. + if err := ingestUpdateSeqNum( + d.cmp, d.opts.Comparer.FormatKey, seqNum, meta, + ); err != nil { + return nil, err + } + + f := newIngestedFlushable(meta, d.cmp, d.split, d.newIters, d.tableNewRangeKeyIter) + + // NB: The logNum/seqNum are the WAL number which we're writing this entry + // to and the sequence number within the WAL which we'll write this entry + // to. + entry := d.newFlushableEntry(f, logNum, seqNum) + // The flushable entry starts off with a single reader ref, so increment + // the FileMetadata.Refs. + for _, file := range f.files { + file.Ref() + } + entry.unrefFiles = func() []*fileBacking { + var obsolete []*fileBacking + for _, file := range f.files { + if file.Unref() == 0 { + obsolete = append(obsolete, file.FileMetadata.FileBacking) + } + } + return obsolete + } + + entry.flushForced = true + entry.releaseMemAccounting = func() {} + return entry, nil +} + +// Both DB.mu and commitPipeline.mu must be held while this is called. Since +// we're holding both locks, the order in which we rotate the memtable or +// recycle the WAL in this function is irrelevant as long as the correct log +// numbers are assigned to the appropriate flushable. +func (d *DB) handleIngestAsFlushable(meta []*fileMetadata, seqNum uint64) error { + b := d.NewBatch() + for _, m := range meta { + b.ingestSST(m.FileNum) + } + b.setSeqNum(seqNum) + + // If the WAL is disabled, then the logNum used to create the flushable + // entry doesn't matter. We just use the logNum assigned to the current + // mutable memtable. If the WAL is enabled, then this logNum will be + // overwritten by the logNum of the log which will contain the log entry + // for the ingestedFlushable. + logNum := d.mu.mem.queue[len(d.mu.mem.queue)-1].logNum + if !d.opts.DisableWAL { + // We create a new WAL for the flushable instead of reusing the end of + // the previous WAL. This simplifies the increment of the minimum + // unflushed log number, and also simplifies WAL replay. + logNum, _ = d.recycleWAL() + d.mu.Unlock() + err := d.commit.directWrite(b) + if err != nil { + d.opts.Logger.Fatalf("%v", err) + } + d.mu.Lock() + } + + entry, err := d.newIngestedFlushableEntry(meta, seqNum, logNum) + if err != nil { + return err + } + nextSeqNum := seqNum + uint64(b.Count()) + + // Set newLogNum to the logNum of the previous flushable. This value is + // irrelevant if the WAL is disabled. If the WAL is enabled, then we set + // the appropriate value below. + newLogNum := d.mu.mem.queue[len(d.mu.mem.queue)-1].logNum + if !d.opts.DisableWAL { + // This is WAL num of the next mutable memtable which comes after the + // ingestedFlushable in the flushable queue. The mutable memtable + // will be created below. + newLogNum, _ = d.recycleWAL() + if err != nil { + return err + } + } + + currMem := d.mu.mem.mutable + // NB: Placing ingested sstables above the current memtables + // requires rotating of the existing memtables/WAL. There is + // some concern of churning through tiny memtables due to + // ingested sstables being placed on top of them, but those + // memtables would have to be flushed anyways. + d.mu.mem.queue = append(d.mu.mem.queue, entry) + d.rotateMemtable(newLogNum, nextSeqNum, currMem) + d.updateReadStateLocked(d.opts.DebugCheck) + d.maybeScheduleFlush() + return nil +} + func (d *DB) ingest( paths []string, targetLevelFunc ingestTargetLevelFunc, ) (IngestOperationStats, error) { @@ -658,9 +802,9 @@ func (d *DB) ingest( // ordering. The sorting of L0 tables by sequence number avoids relying on // that (busted) invariant. d.mu.Lock() - pendingOutputs := make([]FileNum, len(paths)) + pendingOutputs := make([]base.DiskFileNum, len(paths)) for i := range paths { - pendingOutputs[i] = d.mu.versions.getNextFileNum() + pendingOutputs[i] = d.mu.versions.getNextFileNum().DiskFileNum() } jobID := d.mu.nextJobID d.mu.nextJobID++ @@ -687,19 +831,25 @@ func (d *DB) ingest( // (e.g. because the files reside on a different filesystem), ingestLink will // fall back to copying, and if that fails we undo our work and return an // error. - if err := ingestLink(jobID, d.opts, d.dirname, paths, meta); err != nil { + if err := ingestLink(jobID, d.opts, d.objProvider, paths, meta); err != nil { return IngestOperationStats{}, err } - // Fsync the directory we added the tables to. We need to do this at some - // point before we update the MANIFEST (via logAndApply), otherwise a crash - // can have the tables referenced in the MANIFEST, but not present in the - // directory. - if err := d.dataDir.Sync(); err != nil { + // Make the new tables durable. We need to do this at some point before we + // update the MANIFEST (via logAndApply), otherwise a crash can have the + // tables referenced in the MANIFEST, but not present in the provider. + if err := d.objProvider.Sync(); err != nil { return IngestOperationStats{}, err } + // metaFlushableOverlaps is a slice parallel to meta indicating which of the + // ingested sstables overlap some table in the flushable queue. It's used to + // approximate ingest-into-L0 stats when using flushable ingests. + metaFlushableOverlaps := make([]bool, len(meta)) var mem *flushableEntry - prepare := func() { + var mut *memTable + // asFlushable indicates whether the sstable was ingested as a flushable. + var asFlushable bool + prepare := func(seqNum uint64) { // Note that d.commit.mu is held by commitPipeline when calling prepare. d.mu.Lock() @@ -709,24 +859,89 @@ func (d *DB) ingest( // is ordered from oldest to newest with the mutable memtable being the // last element in the slice. We want to wait for the newest table that // overlaps. + for i := len(d.mu.mem.queue) - 1; i >= 0; i-- { m := d.mu.mem.queue[i] - if ingestMemtableOverlaps(d.cmp, m, meta) { - mem = m - if mem.flushable == d.mu.mem.mutable { - err = d.makeRoomForWrite(nil) + iter := m.newIter(nil) + rangeDelIter := m.newRangeDelIter(nil) + rkeyIter := m.newRangeKeyIter(nil) + for i := range meta { + if metaFlushableOverlaps[i] { + // This table already overlapped a more recent flushable. + continue } - mem.flushForced = true - d.maybeScheduleFlush() - return + if overlapWithIterator(iter, &rangeDelIter, rkeyIter, meta[i], d.cmp) { + // If this is the first table to overlap a flushable, save + // the flushable. This ingest must be ingested or flushed + // after it. + if mem == nil { + mem = m + } + metaFlushableOverlaps[i] = true + } + } + err := iter.Close() + if rangeDelIter != nil { + err = firstError(err, rangeDelIter.Close()) + } + if rkeyIter != nil { + err = firstError(err, rkeyIter.Close()) + } + if err != nil { + d.opts.Logger.Infof("ingest error reading flushable for log %s: %s", m.logNum, err) } } + + if mem == nil { + // No overlap with any of the queued flushables, so no need to queue + // after them. + + // New writes with higher sequence numbers may be concurrently + // committed. We must ensure they don't flush before this ingest + // completes. To do that, we ref the mutable memtable as a writer, + // preventing its flushing (and the flushing of all subsequent + // flushables in the queue). Once we've acquired the manifest lock + // to add the ingested sstables to the LSM, we can unref as we're + // guaranteed that the flush won't edit the LSM before this ingest. + mut = d.mu.mem.mutable + mut.writerRef() + return + } + // The ingestion overlaps with some entry in the flushable queue. + if d.mu.formatVers.vers < FormatFlushableIngest || + d.opts.Experimental.DisableIngestAsFlushable() || + (len(d.mu.mem.queue) > d.opts.MemTableStopWritesThreshold-1) { + // We're not able to ingest as a flushable, + // so we must synchronously flush. + if mem.flushable == d.mu.mem.mutable { + err = d.makeRoomForWrite(nil) + } + // New writes with higher sequence numbers may be concurrently + // committed. We must ensure they don't flush before this ingest + // completes. To do that, we ref the mutable memtable as a writer, + // preventing its flushing (and the flushing of all subsequent + // flushables in the queue). Once we've acquired the manifest lock + // to add the ingested sstables to the LSM, we can unref as we're + // guaranteed that the flush won't edit the LSM before this ingest. + mut = d.mu.mem.mutable + mut.writerRef() + mem.flushForced = true + d.maybeScheduleFlush() + return + } + // Since there aren't too many memtables already queued up, we can + // slide the ingested sstables on top of the existing memtables. + asFlushable = true + err = d.handleIngestAsFlushable(meta, seqNum) } var ve *versionEdit apply := func(seqNum uint64) { - if err != nil { + if err != nil || asFlushable { // An error occurred during prepare. + if mut != nil { + mut.writerUnref() + } return } @@ -737,6 +952,9 @@ func (d *DB) ingest( if err = ingestUpdateSeqNum( d.cmp, d.opts.Comparer.FormatKey, seqNum, meta, ); err != nil { + if mut != nil { + mut.writerUnref() + } return } @@ -748,16 +966,18 @@ func (d *DB) ingest( // Assign the sstables to the correct level in the LSM and apply the // version edit. - ve, err = d.ingestApply(jobID, meta, targetLevelFunc) + ve, err = d.ingestApply(jobID, meta, targetLevelFunc, mut) } d.commit.AllocateSeqNum(len(meta), prepare, apply) if err != nil { - if err2 := ingestCleanup(d.opts.FS, d.dirname, meta); err2 != nil { + if err2 := ingestCleanup(d.objProvider, meta); err2 != nil { d.opts.Logger.Infof("ingest cleanup failed: %v", err2) } } else { + // Since we either created a hard link to the ingesting files, or copied + // them over, it is safe to remove the originals paths. for _, path := range paths { if err2 := d.opts.FS.Remove(path); err2 != nil { d.opts.Logger.Infof("ingest failed to remove original file: %s", err2) @@ -769,6 +989,7 @@ func (d *DB) ingest( JobID: jobID, GlobalSeqNum: meta[0].SmallestSeqNum, Err: err, + flushable: asFlushable, } var stats IngestOperationStats if ve != nil { @@ -784,6 +1005,31 @@ func (d *DB) ingest( if e.Level == 0 { stats.ApproxIngestedIntoL0Bytes += e.Meta.Size } + if metaFlushableOverlaps[i] { + stats.MemtableOverlappingFiles++ + } + } + } else if asFlushable { + info.Tables = make([]struct { + TableInfo + Level int + }, len(meta)) + for i, f := range meta { + info.Tables[i].Level = -1 + info.Tables[i].TableInfo = f.TableInfo() + stats.Bytes += f.Size + // We don't have exact stats on which files will be ingested into + // L0, because actual ingestion into the LSM has been deferred until + // flush time. Instead, we infer based on memtable overlap. + // + // TODO(jackson): If we optimistically compute data overlap (#2112) + // before entering the commit pipeline, we can use that overlap to + // improve our approximation by incorporating overlap with L0, not + // just memtables. + if metaFlushableOverlaps[i] { + stats.ApproxIngestedIntoL0Bytes += f.Size + stats.MemtableOverlappingFiles++ + } } } d.opts.EventListener.TableIngested(info) @@ -793,6 +1039,7 @@ func (d *DB) ingest( type ingestTargetLevelFunc func( newIters tableNewIters, + newRangeKeyIter keyspan.TableNewSpanIter, iterOps IterOptions, cmp Compare, v *version, @@ -802,7 +1049,7 @@ type ingestTargetLevelFunc func( ) (int, error) func (d *DB) ingestApply( - jobID int, meta []*fileMetadata, findTargetLevel ingestTargetLevelFunc, + jobID int, meta []*fileMetadata, findTargetLevel ingestTargetLevelFunc, mut *memTable, ) (*versionEdit, error) { d.mu.Lock() defer d.mu.Unlock() @@ -819,6 +1066,16 @@ func (d *DB) ingestApply( // logAndApply unconditionally releases the manifest lock, but any earlier // returns must unlock the manifest. d.mu.versions.logLock() + + if mut != nil { + // Unref the mutable memtable to allows its flush to proceed. Now that we've + // acquired the manifest lock, we can be certain that if the mutable + // memtable has received more recent conflicting writes, the flush won't + // beat us to applying to the manifest resulting in sequence number + // inversion. + mut.writerUnref() + } + current := d.mu.versions.currentVersion() baseLevel := d.mu.versions.picker.getBaseLevel() iterOps := IterOptions{logger: d.opts.Logger} @@ -828,7 +1085,7 @@ func (d *DB) ingestApply( m := meta[i] f := &ve.NewFiles[i] var err error - f.Level, err = findTargetLevel(d.newIters, iterOps, d.cmp, current, baseLevel, d.mu.compact.inProgress, m) + f.Level, err = findTargetLevel(d.newIters, d.tableNewRangeKeyIter, iterOps, d.cmp, current, baseLevel, d.mu.compact.inProgress, m) if err != nil { d.mu.versions.logUnlock() return nil, err @@ -851,7 +1108,6 @@ func (d *DB) ingestApply( } d.updateReadStateLocked(d.opts.DebugCheck) d.updateTableStatsLocked(ve.NewFiles) - d.deleteObsoleteFiles(jobID, false /* waitForOngoing */) // The ingestion may have pushed a level over the threshold for compaction, // so check to see if one is necessary and schedule it. d.maybeScheduleCompaction() @@ -862,12 +1118,21 @@ func (d *DB) ingestApply( // maybeValidateSSTablesLocked adds the slice of newFileEntrys to the pending // queue of files to be validated, when the feature is enabled. // DB.mu must be locked when calling. +// +// TODO(bananabrick): Make sure that the ingestion step only passes in the +// physical sstables for validation here. func (d *DB) maybeValidateSSTablesLocked(newFiles []newFileEntry) { // Only add to the validation queue when the feature is enabled. if !d.opts.Experimental.ValidateOnIngest { return } + for _, f := range newFiles { + if f.Meta.Virtual { + panic("pebble: invalid call to maybeValidateSSTablesLocked") + } + } + d.mu.tableValidation.pending = append(d.mu.tableValidation.pending, newFiles...) if d.shouldValidateSSTablesLocked() { go d.validateSSTables() @@ -927,9 +1192,10 @@ func (d *DB) validateSSTables() { } } - err := d.tableCache.withReader(f.Meta, func(r *sstable.Reader) error { - return r.ValidateBlockChecksums() - }) + err := d.tableCache.withReader( + f.Meta.PhysicalMeta(), func(r *sstable.Reader) error { + return r.ValidateBlockChecksums() + }) if err != nil { // TODO(travers): Hook into the corruption reporting pipeline, once // available. See pebble#1192. diff --git a/vendor/github.com/cockroachdb/pebble/internal.go b/vendor/github.com/cockroachdb/pebble/internal.go index a856e99..b505207 100644 --- a/vendor/github.com/cockroachdb/pebble/internal.go +++ b/vendor/github.com/cockroachdb/pebble/internal.go @@ -22,6 +22,7 @@ const ( InternalKeyKindRangeKeySet = base.InternalKeyKindRangeKeySet InternalKeyKindRangeKeyUnset = base.InternalKeyKindRangeKeyUnset InternalKeyKindRangeKeyDelete = base.InternalKeyKindRangeKeyDelete + InternalKeyKindIngestSST = base.InternalKeyKindIngestSST InternalKeyKindInvalid = base.InternalKeyKindInvalid InternalKeySeqNumBatch = base.InternalKeySeqNumBatch InternalKeySeqNumMax = base.InternalKeySeqNumMax @@ -36,3 +37,14 @@ type internalIterator = base.InternalIterator // ErrCorruption is a marker to indicate that data in a file (WAL, MANIFEST, // sstable) isn't in the expected format. var ErrCorruption = base.ErrCorruption + +// AttributeAndLen exports the base.AttributeAndLen type. +type AttributeAndLen = base.AttributeAndLen + +// ShortAttribute exports the base.ShortAttribute type. +type ShortAttribute = base.ShortAttribute + +// LazyFetcher exports the base.LazyFetcher type. This export is needed since +// LazyValue.Clone requires a pointer to a LazyFetcher struct to avoid +// allocations. No code outside Pebble needs to peer into a LazyFetcher. +type LazyFetcher = base.LazyFetcher diff --git a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/flush_iterator.go b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/flush_iterator.go index 179dcdb..2a7ea03 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/flush_iterator.go +++ b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/flush_iterator.go @@ -79,6 +79,10 @@ func (it *flushIterator) Next() (*base.InternalKey, base.LazyValue) { return &it.key, base.MakeInPlaceValue(it.value()) } +func (it *flushIterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) { + panic("pebble: NextPrefix unimplemented") +} + func (it *flushIterator) Prev() (*base.InternalKey, base.LazyValue) { panic("pebble: Prev unimplemented") } diff --git a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go index b6c637d..3a40a20 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go +++ b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/iterator.go @@ -194,6 +194,13 @@ func (it *Iterator) Next() (*base.InternalKey, base.LazyValue) { return &it.key, base.MakeInPlaceValue(it.value()) } +// NextPrefix advances to the next position with a new prefix. Returns the key +// and value if the iterator is pointing at a valid entry, and (nil, nil) +// otherwise. +func (it *Iterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) { + return it.SeekGE(succKey, base.SeekGEFlagsNone.EnableTrySeekUsingNext()) +} + // Prev moves to the previous position. Returns the key and value if the // iterator is pointing at a valid entry, and (nil, nil) otherwise. func (it *Iterator) Prev() (*base.InternalKey, base.LazyValue) { diff --git a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go index 3f29447..b86d0af 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go +++ b/vendor/github.com/cockroachdb/pebble/internal/arenaskl/skl.go @@ -68,7 +68,7 @@ const ( // keys. var ErrRecordExists = errors.New("record with this key already exists") -// Skiplist is a fast, cocnurrent skiplist implementation that supports forward +// Skiplist is a fast, concurrent skiplist implementation that supports forward // and backward iteration. See batchskl.Skiplist for a non-concurrent // skiplist. Keys and values are immutable once added to the skiplist and // deletion is not supported. Instead, higher-level code is expected to add new diff --git a/vendor/github.com/cockroachdb/pebble/internal/base/comparer.go b/vendor/github.com/cockroachdb/pebble/internal/base/comparer.go index 6e5384e..a630962 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/base/comparer.go +++ b/vendor/github.com/cockroachdb/pebble/internal/base/comparer.go @@ -70,11 +70,11 @@ type Successor func(dst, a []byte) []byte // returns the smallest key that is larger than the given prefix a. // ImmediateSuccessor must return a prefix key k such that: // -// Split(k) == len(k) and Compare(k, a) > 0 +// Split(k) == len(k) and Compare(k, a) > 0 // // and there exists no representable k2 such that: // -// Split(k2) == len(k2) and Compare(k2, a) > 0 and Compare(k2, k) < 0 +// Split(k2) == len(k2) and Compare(k2, a) > 0 and Compare(k2, k) < 0 // // As an example, an implementation built on the natural byte ordering using // bytes.Compare could append a `\0` to `a`. @@ -98,24 +98,23 @@ type ImmediateSuccessor func(dst, a []byte) []byte // // The returned prefix must have the following properties: // -// 1) The prefix must be a byte prefix: +// 1. The prefix must be a byte prefix: // -// bytes.HasPrefix(a, prefix(a)) +// bytes.HasPrefix(a, prefix(a)) // -// 2) A key consisting of just a prefix must sort before all other keys with -// that prefix: +// 2. A key consisting of just a prefix must sort before all other keys with +// that prefix: // -// Compare(prefix(a), a) < 0 if len(suffix(a)) > 0 +// Compare(prefix(a), a) < 0 if len(suffix(a)) > 0 // -// 3) Prefixes must be used to order keys before suffixes: +// 3. Prefixes must be used to order keys before suffixes: // -// If Compare(a, b) <= 0, then Compare(prefix(a), prefix(b)) <= 0 +// If Compare(a, b) <= 0, then Compare(prefix(a), prefix(b)) <= 0 // -// 4) Suffixes themselves must be valid keys and comparable, respecting the same -// ordering as within a key. -// -// If Compare(prefix(a), prefix(b)) == 0, then Compare(suffix(a), suffix(b)) == Compare(a, b) +// 4. Suffixes themselves must be valid keys and comparable, respecting the same +// ordering as within a key. // +// If Compare(prefix(a), prefix(b)) == 0, then Compare(suffix(a), suffix(b)) == Compare(a, b) type Split func(a []byte) int // Comparer defines a total ordering over the space of []byte keys: a 'less diff --git a/vendor/github.com/cockroachdb/pebble/internal/base/filenames.go b/vendor/github.com/cockroachdb/pebble/internal/base/filenames.go index 269d14b..b080698 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/base/filenames.go +++ b/vendor/github.com/cockroachdb/pebble/internal/base/filenames.go @@ -20,6 +20,31 @@ type FileNum uint64 // String returns a string representation of the file number. func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) } +// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be +// called if the caller can ensure that the FileNum belongs to a physical file +// on disk. These could be manifests, log files, physical sstables on disk, the +// options file, but not virtual sstables. +func (fn FileNum) DiskFileNum() DiskFileNum { + return DiskFileNum{fn} +} + +// A DiskFileNum is just a FileNum belonging to a file which exists on disk. +// Note that a FileNum is an internal DB identifier and it could belong to files +// which don't exist on disk. An example would be virtual sstable FileNums. +// Converting a DiskFileNum to a FileNum is always valid, whereas converting a +// FileNum to DiskFileNum may not be valid and care should be taken to prove +// that the FileNum actually exists on disk. +type DiskFileNum struct { + fn FileNum +} + +func (dfn DiskFileNum) String() string { return dfn.fn.String() } + +// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid. +func (dfn DiskFileNum) FileNum() FileNum { + return dfn.fn +} + // FileType enumerates the types of files found in a DB. type FileType int @@ -36,92 +61,92 @@ const ( ) // MakeFilename builds a filename from components. -func MakeFilename(fileType FileType, fileNum FileNum) string { +func MakeFilename(fileType FileType, dfn DiskFileNum) string { switch fileType { case FileTypeLog: - return fmt.Sprintf("%s.log", fileNum) + return fmt.Sprintf("%s.log", dfn) case FileTypeLock: return "LOCK" case FileTypeTable: - return fmt.Sprintf("%s.sst", fileNum) + return fmt.Sprintf("%s.sst", dfn) case FileTypeManifest: - return fmt.Sprintf("MANIFEST-%s", fileNum) + return fmt.Sprintf("MANIFEST-%s", dfn) case FileTypeCurrent: return "CURRENT" case FileTypeOptions: - return fmt.Sprintf("OPTIONS-%s", fileNum) + return fmt.Sprintf("OPTIONS-%s", dfn) case FileTypeOldTemp: - return fmt.Sprintf("CURRENT.%s.dbtmp", fileNum) + return fmt.Sprintf("CURRENT.%s.dbtmp", dfn) case FileTypeTemp: - return fmt.Sprintf("temporary.%s.dbtmp", fileNum) + return fmt.Sprintf("temporary.%s.dbtmp", dfn) } panic("unreachable") } // MakeFilepath builds a filepath from components. -func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, fileNum FileNum) string { - return fs.PathJoin(dirname, MakeFilename(fileType, fileNum)) +func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string { + return fs.PathJoin(dirname, MakeFilename(fileType, dfn)) } // ParseFilename parses the components from a filename. -func ParseFilename(fs vfs.FS, filename string) (fileType FileType, fileNum FileNum, ok bool) { +func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) { filename = fs.PathBase(filename) switch { case filename == "CURRENT": - return FileTypeCurrent, 0, true + return FileTypeCurrent, DiskFileNum{0}, true case filename == "LOCK": - return FileTypeLock, 0, true + return FileTypeLock, DiskFileNum{0}, true case strings.HasPrefix(filename, "MANIFEST-"): - fileNum, ok = parseFileNum(filename[len("MANIFEST-"):]) + dfn, ok = parseDiskFileNum(filename[len("MANIFEST-"):]) if !ok { break } - return FileTypeManifest, fileNum, true + return FileTypeManifest, dfn, true case strings.HasPrefix(filename, "OPTIONS-"): - fileNum, ok = parseFileNum(filename[len("OPTIONS-"):]) + dfn, ok = parseDiskFileNum(filename[len("OPTIONS-"):]) if !ok { break } - return FileTypeOptions, fileNum, ok + return FileTypeOptions, dfn, ok case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"): s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp") - fileNum, ok = parseFileNum(s) + dfn, ok = parseDiskFileNum(s) if !ok { break } - return FileTypeOldTemp, fileNum, ok + return FileTypeOldTemp, dfn, ok case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"): s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp") - fileNum, ok = parseFileNum(s) + dfn, ok = parseDiskFileNum(s) if !ok { break } - return FileTypeTemp, fileNum, ok + return FileTypeTemp, dfn, ok default: i := strings.IndexByte(filename, '.') if i < 0 { break } - fileNum, ok = parseFileNum(filename[:i]) + dfn, ok = parseDiskFileNum(filename[:i]) if !ok { break } switch filename[i+1:] { case "sst": - return FileTypeTable, fileNum, true + return FileTypeTable, dfn, true case "log": - return FileTypeLog, fileNum, true + return FileTypeLog, dfn, true } } - return 0, fileNum, false + return 0, dfn, false } -func parseFileNum(s string) (fileNum FileNum, ok bool) { +func parseDiskFileNum(s string) (dfn DiskFileNum, ok bool) { u, err := strconv.ParseUint(s, 10, 64) if err != nil { - return fileNum, false + return dfn, false } - return FileNum(u), true + return DiskFileNum{FileNum(u)}, true } // A Fataler fatals a process with a message when called. diff --git a/vendor/github.com/cockroachdb/pebble/internal/base/internal.go b/vendor/github.com/cockroachdb/pebble/internal/base/internal.go index 2144596..5be485d 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/base/internal.go +++ b/vendor/github.com/cockroachdb/pebble/internal/base/internal.go @@ -60,6 +60,12 @@ const ( InternalKeyKindRangeKeyUnset InternalKeyKind = 20 InternalKeyKindRangeKeySet InternalKeyKind = 21 + // InternalKeyKindIngestSST is used to distinguish a batch that corresponds to + // the WAL entry for ingested sstables that are added to the flushable + // queue. This InternalKeyKind cannot appear, amongst other key kinds in a + // batch, or in an sstable. + InternalKeyKindIngestSST InternalKeyKind = 22 + // This maximum value isn't part of the file format. It's unlikely, // but future extensions may increase this value. // @@ -69,7 +75,7 @@ const ( // which sorts 'less than or equal to' any other valid internalKeyKind, when // searching for any kind of internal key formed by a certain user key and // seqNum. - InternalKeyKindMax InternalKeyKind = 21 + InternalKeyKindMax InternalKeyKind = 22 // InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a // zero sequence number. @@ -111,6 +117,7 @@ var internalKeyKindNames = []string{ InternalKeyKindRangeKeySet: "RANGEKEYSET", InternalKeyKindRangeKeyUnset: "RANGEKEYUNSET", InternalKeyKindRangeKeyDelete: "RANGEKEYDEL", + InternalKeyKindIngestSST: "INGESTSST", InternalKeyKindInvalid: "INVALID", } @@ -187,6 +194,7 @@ var kindsMap = map[string]InternalKeyKind{ "DEL": InternalKeyKindDelete, "SINGLEDEL": InternalKeyKindSingleDelete, "RANGEDEL": InternalKeyKindRangeDelete, + "LOGDATA": InternalKeyKindLogData, "SET": InternalKeyKindSet, "MERGE": InternalKeyKindMerge, "INVALID": InternalKeyKindInvalid, @@ -195,6 +203,7 @@ var kindsMap = map[string]InternalKeyKind{ "RANGEKEYSET": InternalKeyKindRangeKeySet, "RANGEKEYUNSET": InternalKeyKindRangeKeyUnset, "RANGEKEYDEL": InternalKeyKindRangeKeyDelete, + "INGESTSST": InternalKeyKindIngestSST, } // ParseInternalKey parses the string representation of an internal key. The @@ -362,9 +371,14 @@ func (k *InternalKey) SetKind(kind InternalKeyKind) { k.Trailer = (k.Trailer &^ 0xff) | uint64(kind) } -// Kind returns the kind compoment of the key. +// Kind returns the kind component of the key. func (k InternalKey) Kind() InternalKeyKind { - return InternalKeyKind(k.Trailer & 0xff) + return TrailerKind(k.Trailer) +} + +// TrailerKind returns the key kind of the key trailer. +func TrailerKind(trailer uint64) InternalKeyKind { + return InternalKeyKind(trailer & 0xff) } // Valid returns true if the key has a valid kind. @@ -383,6 +397,13 @@ func (k InternalKey) Clone() InternalKey { } } +// CopyFrom converts this InternalKey into a clone of the passed-in InternalKey, +// reusing any space already used for the current UserKey. +func (k *InternalKey) CopyFrom(k2 InternalKey) { + k.UserKey = append(k.UserKey[:0], k2.UserKey...) + k.Trailer = k2.Trailer +} + // String returns a string representation of the key. func (k InternalKey) String() string { return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind()) @@ -413,7 +434,11 @@ type prettyInternalKey struct { } func (k prettyInternalKey) Format(s fmt.State, c rune) { - fmt.Fprintf(s, "%s#%d,%s", k.formatKey(k.UserKey), k.SeqNum(), k.Kind()) + if seqNum := k.SeqNum(); seqNum == InternalKeySeqNumMax { + fmt.Fprintf(s, "%s#inf,%s", k.formatKey(k.UserKey), k.Kind()) + } else { + fmt.Fprintf(s, "%s#%d,%s", k.formatKey(k.UserKey), k.SeqNum(), k.Kind()) + } } // ParsePrettyInternalKey parses the pretty string representation of an @@ -425,6 +450,11 @@ func ParsePrettyInternalKey(s string) InternalKey { if !ok { panic(fmt.Sprintf("unknown kind: %q", x[2])) } - seqNum, _ := strconv.ParseUint(x[1], 10, 64) + var seqNum uint64 + if x[1] == "max" || x[1] == "inf" { + seqNum = InternalKeySeqNumMax + } else { + seqNum, _ = strconv.ParseUint(x[1], 10, 64) + } return MakeInternalKey([]byte(ukey), seqNum, kind) } diff --git a/vendor/github.com/cockroachdb/pebble/internal/base/iterator.go b/vendor/github.com/cockroachdb/pebble/internal/base/iterator.go index 338e085..8ab10a6 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/base/iterator.go +++ b/vendor/github.com/cockroachdb/pebble/internal/base/iterator.go @@ -4,7 +4,10 @@ package base -import "fmt" +import ( + "fmt" + "time" +) // InternalIterator iterates over a DB's key/value pairs in key order. Unlike // the Iterator interface, the returned keys are InternalKeys composed of the @@ -40,7 +43,9 @@ import "fmt" // context is not a strict byte prefix, but defined by byte equality for the // result of the Comparer.Split method. An InternalIterator is not required to // support prefix iteration mode, and can implement SeekPrefixGE by forwarding -// to SeekGE. +// to SeekGE. When the iteration prefix is exhausted, it is not valid to call +// Next on an internal iterator that's already returned (nil,nilv) or a key +// beyond the prefix. // // Bounds, [lower, upper), can be set on iterators, either using the SetBounds() // function in the interface, or in implementation specific ways during iterator @@ -152,6 +157,22 @@ type InternalIterator interface { // SeekPrefixGE or Next returned (nil, nilv). Next() (*InternalKey, LazyValue) + // NextPrefix moves the iterator to the next key/value pair with a different + // prefix than the key at the current iterator position. Returns the key and + // value if the iterator is pointing at a valid entry, and (nil, nil) + // otherwise. Note that NextPrefix only checks the upper bound. It is up to + // the caller to ensure that key is greater than or equal to the lower + // bound. + // + // NextPrefix is passed the immediate successor to the current prefix key. A + // valid implementation of NextPrefix is to call SeekGE with succKey. + // + // It is not allowed to call NextPrefix when the previous call was a reverse + // positioning operation or a call to a forward positioning method that + // returned (nil, nilv). It is also not allowed to call NextPrefix when the + // iterator is in prefix iteration mode. + NextPrefix(succKey []byte) (*InternalKey, LazyValue) + // Prev moves the iterator to the previous key/value pair. Returns the key // and value if the iterator is pointing at a valid entry, and (nil, nilv) // otherwise. Note that Prev only checks the lower bound. It is up to the @@ -330,7 +351,11 @@ type InternalIteratorStats struct { BlockBytes uint64 // Subset of BlockBytes that were in the block cache. BlockBytesInCache uint64 - + // BlockReadDuration accumulates the duration spent fetching blocks + // due to block cache misses. + // TODO(sumeer): this currently excludes the time spent in Reader creation, + // and in reading the rangedel and rangekey blocks. Fix that. + BlockReadDuration time.Duration // The following can repeatedly count the same points if they are iterated // over multiple times. Additionally, they may count a point twice when // switching directions. The latter could be improved if needed. @@ -339,7 +364,7 @@ type InternalIteratorStats struct { // included. KeyBytes uint64 // Bytes in values that were iterated over. Currently, only point values are - // included. + // included. For separated values, this is the size of the handle. ValueBytes uint64 // The count of points iterated over. PointCount uint64 @@ -347,14 +372,38 @@ type InternalIteratorStats struct { // can be useful for discovering instances of // https://github.com/cockroachdb/pebble/issues/1070. PointsCoveredByRangeTombstones uint64 + + // Stats related to points in value blocks encountered during iteration. + // These are useful to understand outliers, since typical user facing + // iteration should tend to only look at the latest point, and hence have + // the following stats close to 0. + SeparatedPointValue struct { + // Count is a count of points that were in value blocks. This is not a + // subset of PointCount: PointCount is produced by mergingIter and if + // positioned once, and successful in returning a point, will have a + // PointCount of 1, regardless of how many sstables (and memtables etc.) + // in the heap got positioned. The count here includes every sstable + // iterator that got positioned in the heap. + Count uint64 + // ValueBytes represent the total byte length of the values (in value + // blocks) of the points corresponding to Count. + ValueBytes uint64 + // ValueBytesFetched is the total byte length of the values (in value + // blocks) that were retrieved. + ValueBytesFetched uint64 + } } // Merge merges the stats in from into the given stats. func (s *InternalIteratorStats) Merge(from InternalIteratorStats) { s.BlockBytes += from.BlockBytes s.BlockBytesInCache += from.BlockBytesInCache + s.BlockReadDuration += from.BlockReadDuration s.KeyBytes += from.KeyBytes s.ValueBytes += from.ValueBytes s.PointCount += from.PointCount s.PointsCoveredByRangeTombstones += from.PointsCoveredByRangeTombstones + s.SeparatedPointValue.Count += from.SeparatedPointValue.Count + s.SeparatedPointValue.ValueBytes += from.SeparatedPointValue.ValueBytes + s.SeparatedPointValue.ValueBytesFetched += from.SeparatedPointValue.ValueBytesFetched } diff --git a/vendor/github.com/cockroachdb/pebble/internal/base/lazy_value.go b/vendor/github.com/cockroachdb/pebble/internal/base/lazy_value.go index bb0daa1..cc6d56d 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/base/lazy_value.go +++ b/vendor/github.com/cockroachdb/pebble/internal/base/lazy_value.go @@ -16,12 +16,8 @@ import "github.com/cockroachdb/pebble/internal/invariants" // it can call the ShortAttributeExtractor to extract the attribute and store // it together with the key. This allows for cheap retrieval of // AttributeAndLen on the read-path, without doing a more expensive retrieval -// of the value. -// -// In general, the extraction code may want to also look at the key to decide -// how to treat the value. Our current needs can be satisfied by configuring a -// keyspan for which the ShortAttributeExtractor should be used, so we do not -// expose the key to the extractor. +// of the value. In general, the extraction code may want to also look at the +// key to decide how to treat the value, hence the key* parameters. // // Write path performance: The ShortAttributeExtractor func cannot be inlined, // so we will pay the cost of this function call. However, we will only pay @@ -38,7 +34,8 @@ const MaxShortAttribute = 7 // ShortAttributeExtractor is an extractor that given the value, will return // the ShortAttribute. -type ShortAttributeExtractor func(value []byte) (ShortAttribute, error) +type ShortAttributeExtractor func( + key []byte, keyPrefixLen int, value []byte) (ShortAttribute, error) // AttributeAndLen represents the pair of value length and the short // attribute. @@ -70,24 +67,26 @@ type AttributeAndLen struct { // Memory management: // This is subtle, but important for performance. // -// - A LazyValue returned by an InternalIterator or Iterator is unstable in -// that repositioning the iterator will invalidate the memory inside it. A -// caller wishing to maintain that LazyValue needs to call -// LazyValue.Clone(). Note that this does not fetch the value if it is not -// in-place. Additionally, Clone() *must* only be called if -// LazyValue.Value() has *not* been called, since (a) it only makes a -// promise about stability of LazyValue and not the underlying value, (b) if -// LazyValue.Value() has been called already, the caller has the value -// []byte slice so it is unnecessary to fiddle around with LazyValue. +// A LazyValue returned by an InternalIterator or Iterator is unstable in that +// repositioning the iterator will invalidate the memory inside it. A caller +// wishing to maintain that LazyValue needs to call LazyValue.Clone(). Note +// that this does not fetch the value if it is not in-place. Clone() should +// ideally not be called if LazyValue.Value() has been called, since the +// cloned LazyValue will forget the extracted/fetched value, and calling +// Value() on this clone will cause the value to be extracted again. That is, +// Clone() does not make any promise about the memory stability of the +// underlying value. // -// - A user of an iterator that calls LazyValue.Value() wants as much as -// possible for the returned value []byte to point to iterator owned memory. -// - [P1] The underlying iterator that owns that memory also needs a promise +// A user of an iterator that calls LazyValue.Value() wants as much as +// possible for the returned value []byte to point to iterator owned memory. +// +// 1. [P1] The underlying iterator that owns that memory also needs a promise // from that user that at any time there is at most one value []byte slice // that the caller is expecting it to maintain. Otherwise, the underlying // iterator has to maintain multiple such []byte slices which results in // more complicated and inefficient code. -// - [P2] The underlying iterator, in order to make the promise that it is +// +// 2. [P2] The underlying iterator, in order to make the promise that it is // maintaining the one value []byte slice, also needs a way to know when // it is relieved of that promise. One way it is relieved of that promise // is by being told that it is being repositioned. Typically, the owner of @@ -105,28 +104,31 @@ type AttributeAndLen struct { // the caller. This will be true if the callee attempted to use buf and // either successfully used it or allocated a new []byte slice. // -// To ground the above in reality, we consider three examples of callers of -// LazyValue.Value(): -// - Iterator: it calls LazyValue.Value for its own use when merging values. -// When merging during reverse iteration, it may have cloned the LazyValue. -// In this case it calls LazyValue.Value() on the cloned value, merges it, -// and then calls LazyValue.Value() on the current iterator position and -// merges it. So it is honoring P1. -// - Iterator on behalf of Iterator clients: The Iterator.Value() method -// needs to call LazyValue.Value(). The client of Iterator is satisfying P1 -// because of the inherent Iterator interface constraint, i.e., it is calling -// Iterator.Value() on the current Iterator position. It is possible that -// the Iterator has cloned this LazyValue (for the reverse iteration case), -// which the client is unaware of, so the underlying sstable iterator may -// not be able to satisfy P2. This is ok because Iterator will call -// LazyValue.Value with its (reusable) owned buffer. -// - CockroachDB's pebbleMVCCScanner: This will use LazyValues from Iterator -// since during reverse iteration in order to find the highest version that -// satisfies a read it needs to clone the LazyValue, step back the iterator -// and then decide whether it needs the value from the previously cloned -// LazyValue. The pebbleMVCCScanner will satisfy P1. The P2 story is -// similar to the previous case in that it will call LazyValue.Value with -// its (reusable) owned buffer. +// To ground the above in reality, we consider three examples of callers of +// LazyValue.Value(): +// +// - Iterator: it calls LazyValue.Value for its own use when merging values. +// When merging during reverse iteration, it may have cloned the LazyValue. +// In this case it calls LazyValue.Value() on the cloned value, merges it, +// and then calls LazyValue.Value() on the current iterator position and +// merges it. So it is honoring P1. +// +// - Iterator on behalf of Iterator clients: The Iterator.Value() method +// needs to call LazyValue.Value(). The client of Iterator is satisfying P1 +// because of the inherent Iterator interface constraint, i.e., it is calling +// Iterator.Value() on the current Iterator position. It is possible that +// the Iterator has cloned this LazyValue (for the reverse iteration case), +// which the client is unaware of, so the underlying sstable iterator may +// not be able to satisfy P2. This is ok because Iterator will call +// LazyValue.Value with its (reusable) owned buffer. +// +// - CockroachDB's pebbleMVCCScanner: This will use LazyValues from Iterator +// since during reverse iteration in order to find the highest version that +// satisfies a read it needs to clone the LazyValue, step back the iterator +// and then decide whether it needs the value from the previously cloned +// LazyValue. The pebbleMVCCScanner will satisfy P1. The P2 story is +// similar to the previous case in that it will call LazyValue.Value with +// its (reusable) owned buffer. // // Corollary: callers that directly use InternalIterator can know that they // have done nothing to interfere with promise P2 can pass in a nil buf and be @@ -154,9 +156,24 @@ type LazyValue struct { } // LazyFetcher supports fetching a lazy value. +// +// Fetcher and Attribute are to be initialized at creation time. The fields +// are arranged to reduce the sizeof this struct. type LazyFetcher struct { - // ValueFetcher, given a handle, returns the value. It is acceptable to call - // the ValueFetcher as long as the DB is open. However, one should assume + // Fetcher, given a handle, returns the value. + Fetcher ValueFetcher + err error + value []byte + // Attribute includes the short attribute and value length. + Attribute AttributeAndLen + fetched bool + callerOwned bool +} + +// ValueFetcher is an interface for fetching a value. +type ValueFetcher interface { + // Fetch returns the value, given the handle. It is acceptable to call the + // ValueFetcher.Fetch as long as the DB is open. However, one should assume // there is a fast-path when the iterator tree has not moved off the sstable // iterator that initially provided this LazyValue. Hence, to utilize this // fast-path the caller should try to decide whether it needs the value or @@ -166,13 +183,8 @@ type LazyFetcher struct { // If the fetcher attempted to use buf *and* len(buf) was insufficient, it // will allocate a new slice for the value. In either case it will set // callerOwned to true. - ValueFetcher func(handle []byte, buf []byte) (val []byte, callerOwned bool, err error) - // Attribute includes the short attribute and value length. - Attribute AttributeAndLen - fetched bool - value []byte - callerOwned bool - err error + Fetch( + handle []byte, valLen int32, buf []byte) (val []byte, callerOwned bool, err error) } // Value returns the underlying value. @@ -197,7 +209,8 @@ func (lv *LazyValue) fetchValue(buf []byte) (val []byte, callerOwned bool, err e f := lv.Fetcher if !f.fetched { f.fetched = true - f.value, f.callerOwned, f.err = lv.Fetcher.ValueFetcher(lv.ValueOrHandle, buf) + f.value, f.callerOwned, f.err = f.Fetcher.Fetch( + lv.ValueOrHandle, lv.Fetcher.Attribute.ValueLen, buf) } return f.value, f.callerOwned, f.err } @@ -219,18 +232,46 @@ func (lv *LazyValue) Len() int { return int(lv.Fetcher.Attribute.ValueLen) } -// Clone creates a stable copy of the LazyValue, by appending bytes to buf. -// REQUIRES: LazyValue.Value() has not been called. -// Ensure that this is only called before LazyValue.Value is called. We don't -// actually care if it was in-place but we don't want confusing things about whether -// we need to clone the value inside the fetcher. -func (lv *LazyValue) Clone(buf []byte) (LazyValue, []byte) { - if invariants.Enabled && lv.Fetcher != nil && lv.Fetcher.fetched { - panic("value has already been fetched") +// TryGetShortAttribute returns the ShortAttribute and a bool indicating +// whether the ShortAttribute was populated. +func (lv *LazyValue) TryGetShortAttribute() (ShortAttribute, bool) { + if lv.Fetcher == nil { + return 0, false + } + return lv.Fetcher.Attribute.ShortAttribute, true +} + +// Clone creates a stable copy of the LazyValue, by appending bytes to buf. +// The fetcher parameter must be non-nil and may be over-written and used +// inside the returned LazyValue -- this is needed to avoid an allocation. +// Most callers have at most K cloned LazyValues, where K is hard-coded, so +// they can have a pool of exactly K LazyFetcher structs they can reuse in +// these calls. The alternative of allocating LazyFetchers from a sync.Pool is +// not viable since we have no code trigger for returning to the pool +// (LazyValues are simply GC'd). +// +// NB: It is highly preferable that LazyValue.Value() has not been called, +// since the Clone will forget any previously extracted value, and a future +// call to Value will cause it to be fetched again. We do this since we don't +// want to reason about whether or not to clone an already extracted value +// inside the Fetcher (we don't). Property P1 applies here too: if lv1.Value() +// has been called, and then lv2 is created as a clone of lv1, then calling +// lv2.Value() can invalidate any backing memory maintained inside the fetcher +// for lv1 (even though these are the same values). We initially prohibited +// calling LazyValue.Clone() if LazyValue.Value() has been called, but there +// is at least one complex caller (pebbleMVCCScanner inside CockroachDB) where +// it is not easy to prove this invariant. +func (lv *LazyValue) Clone(buf []byte, fetcher *LazyFetcher) (LazyValue, []byte) { + var lvCopy LazyValue + if lv.Fetcher != nil { + *fetcher = LazyFetcher{ + Fetcher: lv.Fetcher.Fetcher, + Attribute: lv.Fetcher.Attribute, + // Not copying anything that has been extracted. + } + lvCopy.Fetcher = fetcher } - // INVARIANT: LazyFetcher.value is nil, so there is nothing to copy there. vLen := len(lv.ValueOrHandle) - lvCopy := LazyValue{Fetcher: lv.Fetcher} if vLen == 0 { return lvCopy, buf } @@ -244,13 +285,3 @@ func (lv *LazyValue) Clone(buf []byte) (LazyValue, []byte) { func MakeInPlaceValue(val []byte) LazyValue { return LazyValue{ValueOrHandle: val} } - -// TODO(sumeer): test that callers are adhering to promise P1. In the sstable -// iterators, when invariants are enabled, create a testingFetcher struct that -// contains the sstable name and the []byte backing for the last fetched -// value. The sstable iterator, instead of returning an in-place value, will -// return a copy of the key in ValueOrHandle, and a ValueFetcher implemented -// by testingFetcher.valueFetcher. Each time this method is called, it will -// mangle the last fetched value []byte slice, allocate a new one, open the -// sstable, seek to the key, and copy the value into this new byte slice and -// return. diff --git a/vendor/github.com/cockroachdb/pebble/internal/base/logger.go b/vendor/github.com/cockroachdb/pebble/internal/base/logger.go new file mode 100644 index 0000000..e4a2f44 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/internal/base/logger.go @@ -0,0 +1,144 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package base + +import ( + "bytes" + "context" + "fmt" + "log" + "os" + "runtime" + "sync" + + "github.com/cockroachdb/pebble/internal/invariants" +) + +// Logger defines an interface for writing log messages. +type Logger interface { + Infof(format string, args ...interface{}) + Fatalf(format string, args ...interface{}) +} +type defaultLogger struct{} + +// DefaultLogger logs to the Go stdlib logs. +var DefaultLogger defaultLogger + +var _ Logger = DefaultLogger + +// Infof implements the Logger.Infof interface. +func (defaultLogger) Infof(format string, args ...interface{}) { + _ = log.Output(2, fmt.Sprintf(format, args...)) +} + +// Fatalf implements the Logger.Fatalf interface. +func (defaultLogger) Fatalf(format string, args ...interface{}) { + _ = log.Output(2, fmt.Sprintf(format, args...)) + os.Exit(1) +} + +// InMemLogger implements Logger using an in-memory buffer (used for testing). +// The buffer can be read via String() and cleared via Reset(). +type InMemLogger struct { + mu struct { + sync.Mutex + buf bytes.Buffer + } +} + +var _ Logger = (*InMemLogger)(nil) + +// Reset clears the internal buffer. +func (b *InMemLogger) Reset() { + b.mu.Lock() + defer b.mu.Unlock() + b.mu.buf.Reset() +} + +// String returns the current internal buffer. +func (b *InMemLogger) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.mu.buf.String() +} + +// Infof is part of the Logger interface. +func (b *InMemLogger) Infof(format string, args ...interface{}) { + s := fmt.Sprintf(format, args...) + b.mu.Lock() + defer b.mu.Unlock() + b.mu.buf.Write([]byte(s)) + if n := len(s); n == 0 || s[n-1] != '\n' { + b.mu.buf.Write([]byte("\n")) + } +} + +// Fatalf is part of the Logger interface. +func (b *InMemLogger) Fatalf(format string, args ...interface{}) { + b.Infof(format, args...) + runtime.Goexit() +} + +// LoggerAndTracer defines an interface for logging and tracing. +type LoggerAndTracer interface { + Logger + // Eventf formats and emits a tracing log, if tracing is enabled in the + // current context. + Eventf(ctx context.Context, format string, args ...interface{}) + // IsTracingEnabled returns true if tracing is enabled. It can be used as an + // optimization to avoid calling Eventf (which will be a noop when tracing + // is not enabled) to avoid the overhead of boxing the args. + IsTracingEnabled(ctx context.Context) bool +} + +// LoggerWithNoopTracer wraps a logger and does no tracing. +type LoggerWithNoopTracer struct { + Logger +} + +var _ LoggerAndTracer = &LoggerWithNoopTracer{} + +// Eventf implements LoggerAndTracer. +func (*LoggerWithNoopTracer) Eventf(ctx context.Context, format string, args ...interface{}) { + if invariants.Enabled && ctx == nil { + panic("Eventf context is nil") + } +} + +// IsTracingEnabled implements LoggerAndTracer. +func (*LoggerWithNoopTracer) IsTracingEnabled(ctx context.Context) bool { + if invariants.Enabled && ctx == nil { + panic("IsTracingEnabled ctx is nil") + } + return false +} + +// NoopLoggerAndTracer does no logging and tracing. Remember that struct{} is +// special cased in Go and does not incur an allocation when it backs the +// interface LoggerAndTracer. +type NoopLoggerAndTracer struct{} + +var _ LoggerAndTracer = NoopLoggerAndTracer{} + +// Infof implements LoggerAndTracer. +func (l NoopLoggerAndTracer) Infof(format string, args ...interface{}) {} + +// Fatalf implements LoggerAndTracer. +func (l NoopLoggerAndTracer) Fatalf(format string, args ...interface{}) {} + +// Eventf implements LoggerAndTracer. +func (l NoopLoggerAndTracer) Eventf(ctx context.Context, format string, args ...interface{}) { + if invariants.Enabled && ctx == nil { + panic("Eventf context is nil") + } +} + +// IsTracingEnabled implements LoggerAndTracer. +func (l NoopLoggerAndTracer) IsTracingEnabled(ctx context.Context) bool { + if invariants.Enabled && ctx == nil { + panic("IsTracingEnabled ctx is nil") + } + return false +} diff --git a/vendor/github.com/cockroachdb/pebble/internal/base/merger.go b/vendor/github.com/cockroachdb/pebble/internal/base/merger.go index 8a8450d..757d150 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/base/merger.go +++ b/vendor/github.com/cockroachdb/pebble/internal/base/merger.go @@ -23,7 +23,7 @@ type Merge func(key, value []byte) (ValueMerger, error) // // The merge operation must be associative. That is, for the values A, B, C: // -// Merge(A).MergeOlder(B).MergeOlder(C) == Merge(C).MergeNewer(B).MergeNewer(A) +// Merge(A).MergeOlder(B).MergeOlder(C) == Merge(C).MergeNewer(B).MergeNewer(A) // // Examples of merge operators are integer addition, list append, and string // concatenation. diff --git a/vendor/github.com/cockroachdb/pebble/internal/base/seqnums.go b/vendor/github.com/cockroachdb/pebble/internal/base/seqnums.go new file mode 100644 index 0000000..f563c1f --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/internal/base/seqnums.go @@ -0,0 +1,54 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package base + +// This file defines sequence numbers that are reserved for foreign keys i.e. +// internal keys coming from other Pebble instances and existing in shared +// storage, as those will "slot below" any internal keys added by our own Pebble +// instance. Any keys created by this Pebble instance need to be greater than +// all reserved sequence numbers (i.e. >= SeqNumStart). +const ( + // SeqNumZero is the zero sequence number, set by compactions if they can + // guarantee there are no keys underneath an internal key. + SeqNumZero = uint64(0) + + // SeqNumL6Point is the sequence number reserved for foreign point keys in L6. + // This sequence number must be lower than SeqNumL6RangeKey for range key + // masking to work correctly. + SeqNumL6Point = uint64(1) + + // SeqNumL6RangeKey is the sequence number reserved for foreign range keys in + // L6. Only RangeKeySets are expected at this level. + SeqNumL6RangeKey = uint64(2) + + // SeqNumL5RangeDel is the sequence number reserved for foreign range deletes + // in L5. These keys could delete L6 points. + SeqNumL5RangeDel = uint64(3) + + // SeqNumL5Point is the sequence number reserved for foreign point keys in L5. + // Any sst-local range deletions would have already been applied to these keys, + // so they can safely get a sequence number higher than SeqNumL5RangeDel. + // However they must have a sequence number lower than SeqNumL5RangeKey* for + // range key masking to work correctly. + SeqNumL5Point = uint64(4) + + // SeqNumL5RangeKeyUnsetDel is the sequence number reserved for foreign + // range key unsets/deletes in L5. These operations could apply to L6 + // RangeKeySets, so this sequence number must be > SeqNumL6RangeKey. + SeqNumL5RangeKeyUnsetDel = uint64(5) + + // SeqNumL5RangeKeySet is the sequence number reserved for foreign range key + // Sets in L5. These operations could apply to L6 RangeKeySets, so this + // sequence number must be > SeqNumL6RangeKey. Any SST-local rangekey + // unsets/dels have already been applied to them, so their sequence number must + // be > SeqNumL5RangeKeyUnsetDel. + SeqNumL5RangeKeySet = uint64(6) + + // Sequence numbers 7-9 are reserved for future use. + + // SeqNumStart is the first sequence number assigned to a key written by + // ourselves. + SeqNumStart = uint64(10) +) diff --git a/vendor/github.com/cockroachdb/pebble/internal/bytealloc/bytealloc.go b/vendor/github.com/cockroachdb/pebble/internal/bytealloc/bytealloc.go index b17f647..b905270 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/bytealloc/bytealloc.go +++ b/vendor/github.com/cockroachdb/pebble/internal/bytealloc/bytealloc.go @@ -26,7 +26,7 @@ import "github.com/cockroachdb/pebble/internal/rawalloc" type A []byte const chunkAllocMinSize = 512 -const chunkAllocMaxSize = 16384 +const chunkAllocMaxSize = 512 << 10 // 512 KB func (a A) reserve(n int) A { allocSize := cap(a) * 2 @@ -59,3 +59,11 @@ func (a A) Copy(src []byte) (A, []byte) { copy(alloc, src) return a, alloc } + +// Reset returns the current chunk, resetting allocated memory back to none. +// Future allocations will use memory previously allocated by previous calls to +// Alloc or Copy, so the caller must know know that none of the previously +// allocated byte slices are still in use. +func (a A) Reset() A { + return a[:0] +} diff --git a/vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go b/vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go index a5f8fd0..4e9492c 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go +++ b/vendor/github.com/cockroachdb/pebble/internal/cache/clockpro.go @@ -33,7 +33,7 @@ import ( type fileKey struct { // id is the namespace for fileNums. id uint64 - fileNum base.FileNum + fileNum base.DiskFileNum } type key struct { @@ -113,7 +113,7 @@ type shard struct { countTest int64 } -func (c *shard) Get(id uint64, fileNum base.FileNum, offset uint64) Handle { +func (c *shard) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle { c.mu.RLock() var value *Value if e := c.blocks.Get(key{fileKey{id, fileNum}, offset}); e != nil { @@ -131,7 +131,7 @@ func (c *shard) Get(id uint64, fileNum base.FileNum, offset uint64) Handle { return Handle{value: value} } -func (c *shard) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle { +func (c *shard) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle { if n := value.refs(); n != 1 { panic(fmt.Sprintf("pebble: Value has already been added to the cache: refs=%d", n)) } @@ -222,7 +222,7 @@ func (c *shard) checkConsistency() { } // Delete deletes the cached value for the specified file and offset. -func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) { +func (c *shard) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) { // The common case is there is nothing to delete, so do a quick check with // shared lock. k := key{fileKey{id, fileNum}, offset} @@ -246,7 +246,7 @@ func (c *shard) Delete(id uint64, fileNum base.FileNum, offset uint64) { } // EvictFile evicts all of the cache values for the specified file. -func (c *shard) EvictFile(id uint64, fileNum base.FileNum) { +func (c *shard) EvictFile(id uint64, fileNum base.DiskFileNum) { c.mu.Lock() defer c.mu.Unlock() @@ -603,7 +603,7 @@ type Metrics struct { // efficient eviction of all of the blocks for a file which is used when an // sstable is deleted from disk. // -// Memory Management +// # Memory Management // // In order to reduce pressure on the Go GC, manual memory management is // performed for the data stored in the cache. Manual memory management is @@ -639,9 +639,9 @@ type Cache struct { // creator of the cache should usually release their reference after the DB is // created. // -// c := cache.New(...) -// defer c.Unref() -// d, err := pebble.Open(pebble.Options{Cache: c}) +// c := cache.New(...) +// defer c.Unref() +// d, err := pebble.Open(pebble.Options{Cache: c}) func New(size int64) *Cache { return newShards(size, 2*runtime.GOMAXPROCS(0)) } @@ -683,7 +683,7 @@ func newShards(size int64, shards int) *Cache { return c } -func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard { +func (c *Cache) getShard(id uint64, fileNum base.DiskFileNum, offset uint64) *shard { if id == 0 { panic("pebble: 0 cache ID is invalid") } @@ -698,10 +698,11 @@ func (c *Cache) getShard(id uint64, fileNum base.FileNum, offset uint64) *shard h ^= uint64(id & 0xff) id >>= 8 } + fileNumVal := uint64(fileNum.FileNum()) for i := 0; i < 8; i++ { h *= prime64 - h ^= uint64(fileNum & 0xff) - fileNum >>= 8 + h ^= uint64(fileNumVal) & 0xff + fileNumVal >>= 8 } for i := 0; i < 8; i++ { h *= prime64 @@ -738,7 +739,7 @@ func (c *Cache) Unref() { // Get retrieves the cache value for the specified file and offset, returning // nil if no value is present. -func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle { +func (c *Cache) Get(id uint64, fileNum base.DiskFileNum, offset uint64) Handle { return c.getShard(id, fileNum, offset).Get(id, fileNum, offset) } @@ -746,17 +747,17 @@ func (c *Cache) Get(id uint64, fileNum base.FileNum, offset uint64) Handle { // existing value if present. A Handle is returned which provides faster // retrieval of the cached value than Get (lock-free and avoidance of the map // lookup). The value must have been allocated by Cache.Alloc. -func (c *Cache) Set(id uint64, fileNum base.FileNum, offset uint64, value *Value) Handle { +func (c *Cache) Set(id uint64, fileNum base.DiskFileNum, offset uint64, value *Value) Handle { return c.getShard(id, fileNum, offset).Set(id, fileNum, offset, value) } // Delete deletes the cached value for the specified file and offset. -func (c *Cache) Delete(id uint64, fileNum base.FileNum, offset uint64) { +func (c *Cache) Delete(id uint64, fileNum base.DiskFileNum, offset uint64) { c.getShard(id, fileNum, offset).Delete(id, fileNum, offset) } // EvictFile evicts all of the cache values for the specified file. -func (c *Cache) EvictFile(id uint64, fileNum base.FileNum) { +func (c *Cache) EvictFile(id uint64, fileNum base.DiskFileNum) { if id == 0 { panic("pebble: 0 cache ID is invalid") } diff --git a/vendor/github.com/cockroachdb/pebble/internal/cache/entry.go b/vendor/github.com/cockroachdb/pebble/internal/cache/entry.go index 1b96e67..beb4891 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/cache/entry.go +++ b/vendor/github.com/cockroachdb/pebble/internal/cache/entry.go @@ -30,7 +30,7 @@ func (p entryType) String() string { // Using manual memory management for entries is technically a volation of the // Cgo pointer rules: // -// https://golang.org/cmd/cgo/#hdr-Passing_pointers +// https://golang.org/cmd/cgo/#hdr-Passing_pointers // // Specifically, Go pointers should not be stored in C allocated memory. The // reason for this rule is that the Go GC will not look at C allocated memory diff --git a/vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go b/vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go index 8972810..6e093fd 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go +++ b/vendor/github.com/cockroachdb/pebble/internal/cache/robin_hood.go @@ -24,7 +24,7 @@ func robinHoodHash(k key, shift uint32) uint32 { const m = 11400714819323198485 h := hashSeed h ^= k.id * m - h ^= uint64(k.fileNum) * m + h ^= uint64(k.fileNum.FileNum()) * m h ^= k.offset * m return uint32(h >> shift) } @@ -95,11 +95,11 @@ func (e robinHoodEntries) free() { // table is 2, maxDist is computed as 4 and the actual size of the entry slice // is 6. // -// +---+---+---+---+---+---+ -// | 0 | 1 | 2 | 3 | 4 | 5 | -// +---+---+---+---+---+---+ -// ^ -// size +// +---+---+---+---+---+---+ +// | 0 | 1 | 2 | 3 | 4 | 5 | +// +---+---+---+---+---+---+ +// ^ +// size // // In this scenario, the target entry for a key will always be in the range // [0,1]. Valid entries may reside in the range [0,4] due to the linear probing diff --git a/vendor/github.com/cockroachdb/pebble/internal/crc/crc.go b/vendor/github.com/cockroachdb/pebble/internal/crc/crc.go index 990952e..4021a2e 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/crc/crc.go +++ b/vendor/github.com/cockroachdb/pebble/internal/crc/crc.go @@ -10,7 +10,9 @@ // look like a checksum. // // To calculate the uint32 checksum of some data: +// // var u uint32 = crc.New(data).Value() +// // In pebble, the uint32 value is then stored in little-endian format. package crc // import "github.com/cockroachdb/pebble/internal/crc" diff --git a/vendor/github.com/cockroachdb/pebble/internal/fastrand/fastrand.go b/vendor/github.com/cockroachdb/pebble/internal/fastrand/fastrand.go index 75529bb..dd3ec9c 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/fastrand/fastrand.go +++ b/vendor/github.com/cockroachdb/pebble/internal/fastrand/fastrand.go @@ -7,9 +7,11 @@ package fastrand import _ "unsafe" // required by go:linkname // Uint32 returns a lock free uint32 value. +// //go:linkname Uint32 runtime.fastrand func Uint32() uint32 // Uint32n returns a lock free uint32 value in the interval [0, n). +// //go:linkname Uint32n runtime.fastrandn func Uint32n(n uint32) uint32 diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/bounded.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/bounded.go index 98aa507..70dd395 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/bounded.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/bounded.go @@ -66,6 +66,9 @@ type BoundedIter struct { // indicating whether the iterator is in prefix iteration mode and the prefix // key if it is. This is used to exclude spans that are outside the iteration // prefix. +// +// hasPrefix and prefix are allowed to be nil, however if hasPrefix != nil, +// prefix must also not be nil. func (i *BoundedIter) Init( cmp base.Compare, split base.Split, @@ -146,7 +149,7 @@ func (i *BoundedIter) Next() *Span { } // Similarly, if the span extends to the next prefix and we're in prefix // iteration mode, we can avoid advancing. - if i.iterSpan != nil && *i.hasPrefix { + if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix { ei := i.split(i.iterSpan.End) if i.cmp(i.iterSpan.End[:ei], *i.prefix) > 0 { i.pos = posAtUpperLimit @@ -178,7 +181,7 @@ func (i *BoundedIter) Prev() *Span { } // Similarly, if the span extends to or beyond the current prefix and // we're in prefix iteration mode, we can avoid advancing. - if i.iterSpan != nil && *i.hasPrefix { + if i.iterSpan != nil && i.hasPrefix != nil && *i.hasPrefix { si := i.split(i.iterSpan.Start) if i.cmp(i.iterSpan.Start[:si], *i.prefix) < 0 { i.pos = posAtLowerLimit @@ -215,7 +218,7 @@ func (i *BoundedIter) SetBounds(lower, upper []byte) { func (i *BoundedIter) checkPrefixSpanStart(span *Span) *Span { // Compare to the prefix's bounds, if in prefix iteration mode. - if span != nil && *i.hasPrefix { + if span != nil && i.hasPrefix != nil && *i.hasPrefix { si := i.split(span.Start) if i.cmp(span.Start[:si], *i.prefix) > 0 { // This span starts at a prefix that sorts after our current prefix. @@ -242,7 +245,7 @@ func (i *BoundedIter) checkForwardBound(span *Span) *Span { func (i *BoundedIter) checkPrefixSpanEnd(span *Span) *Span { // Compare to the prefix's bounds, if in prefix iteration mode. - if span != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 { + if span != nil && i.hasPrefix != nil && *i.hasPrefix && i.cmp(span.End, *i.prefix) <= 0 { // This span ends before the current prefix. span = nil } diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go index 258f17a..ed0247e 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/defragment.go @@ -8,14 +8,20 @@ import ( "bytes" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/bytealloc" "github.com/cockroachdb/pebble/internal/invariants" ) // bufferReuseMaxCapacity is the maximum capacity of a DefragmentingIter buffer -// that DefragmentingIter will reuse. Buffers greater than this will be +// that DefragmentingIter will reuse. Buffers larger than this will be // discarded and reallocated as necessary. const bufferReuseMaxCapacity = 10 << 10 // 10 KB +// keysReuseMaxCapacity is the maximum capacity of a []keyspan.Key buffer that +// DefragmentingIter will reuse. Buffers larger than this will be discarded and +// reallocated as necessary. +const keysReuseMaxCapacity = 100 + // DefragmentMethod configures the defragmentation performed by the // DefragmentingIter. type DefragmentMethod interface { @@ -113,30 +119,21 @@ const ( // // Seeking (SeekGE, SeekLT) poses an obstacle to defragmentation. A seek may // land on a physical fragment in the middle of several fragments that must be -// defragmented. A seek first degfragments in the opposite direction of -// iteration to find the beginning of the defragmented span, and then -// defragments in the iteration direction, ensuring it's found a whole -// defragmented span. +// defragmented. A seek that lands in a fragment straddling the seek key must +// first degfragment in the opposite direction of iteration to find the +// beginning of the defragmented span, and then defragments in the iteration +// direction, ensuring it's found a whole defragmented span. type DefragmentingIter struct { + // DefragmentingBuffers holds buffers used for copying iterator state. + *DefragmentingBuffers comparer *base.Comparer equal base.Equal iter FragmentIterator iterSpan *Span iterPos iterPos - // curr holds the span at the current iterator position. currBuf is a buffer - // for use when copying user keys for curr. keysBuf is a buffer for use when - // copying Keys for curr. currBuf is cleared between positioning methods. - // - // keyBuf is a buffer specifically for the defragmented start key when - // defragmenting backwards or the defragmented end key when defragmenting - // forwards. These bounds are overwritten repeatedly during defragmentation, - // and the defragmentation routines overwrite keyBuf repeatedly to store - // these extended bounds. - curr Span - currBuf []byte - keysBuf []Key - keyBuf []byte + // curr holds the span at the current iterator position. + curr Span // method is a comparison function for two spans. method is called when two // spans are abutting to determine whether they may be defragmented. @@ -148,20 +145,53 @@ type DefragmentingIter struct { reduce DefragmentReducer } +// DefragmentingBuffers holds buffers used for copying iterator state. +type DefragmentingBuffers struct { + // currBuf is a buffer for use when copying user keys for curr. currBuf is + // cleared between positioning methods. + currBuf bytealloc.A + // keysBuf is a buffer for use when copying Keys for DefragmentingIter.curr. + keysBuf []Key + // keyBuf is a buffer specifically for the defragmented start key when + // defragmenting backwards or the defragmented end key when defragmenting + // forwards. These bounds are overwritten repeatedly during defragmentation, + // and the defragmentation routines overwrite keyBuf repeatedly to store + // these extended bounds. + keyBuf []byte +} + +// PrepareForReuse discards any excessively large buffers. +func (bufs *DefragmentingBuffers) PrepareForReuse() { + if cap(bufs.currBuf) > bufferReuseMaxCapacity { + bufs.currBuf = nil + } + if cap(bufs.keyBuf) > bufferReuseMaxCapacity { + bufs.keyBuf = nil + } + if cap(bufs.keysBuf) > keysReuseMaxCapacity { + bufs.keysBuf = nil + } +} + // Assert that *DefragmentingIter implements the FragmentIterator interface. var _ FragmentIterator = (*DefragmentingIter)(nil) // Init initializes the defragmenting iter using the provided defragment // method. func (i *DefragmentingIter) Init( - comparer *base.Comparer, iter FragmentIterator, equal DefragmentMethod, reducer DefragmentReducer, + comparer *base.Comparer, + iter FragmentIterator, + equal DefragmentMethod, + reducer DefragmentReducer, + bufs *DefragmentingBuffers, ) { *i = DefragmentingIter{ - comparer: comparer, - equal: comparer.Equal, - iter: iter, - method: equal, - reduce: reducer, + DefragmentingBuffers: bufs, + comparer: comparer, + equal: comparer.Equal, + iter: iter, + method: equal, + reduce: reducer, } } @@ -175,8 +205,9 @@ func (i *DefragmentingIter) Close() error { return i.iter.Close() } -// SeekGE seeks the iterator to the first span with a start key greater than or -// equal to key and returns it. +// SeekGE moves the iterator to the first span covering a key greater than or +// equal to the given key. This is equivalent to seeking to the first span with +// an end key greater than the given key. func (i *DefragmentingIter) SeekGE(key []byte) *Span { i.iterSpan = i.iter.SeekGE(key) if i.iterSpan == nil { @@ -186,30 +217,28 @@ func (i *DefragmentingIter) SeekGE(key []byte) *Span { i.iterPos = iterPosCurr return i.iterSpan } - // Save the current span and peek backwards. - i.saveCurrent() - i.iterSpan = i.iter.Prev() - if i.iterSpan != nil && i.equal(i.curr.Start, i.iterSpan.End) && i.checkEqual(i.iterSpan, &i.curr) { - // A continuation. The span we originally landed on and defragmented - // backwards has a true Start key < key. To obey the FragmentIterator - // contract, we must not return this defragmented span. Defragment - // forward to finish defragmenting the span in the forward direction. - i.defragmentForward() - - // Now we must be on a span that truly has a defragmented Start key > - // key. + // If the span starts strictly after key, we know there mustn't be an + // earlier span that ends at i.iterSpan.Start, otherwise i.iter would've + // returned that span instead. + if i.comparer.Compare(i.iterSpan.Start, key) > 0 { return i.defragmentForward() } - // The span previous to i.curr does not defragment, so we should return it. - // Next the underlying iterator back onto the span we previously saved to - // i.curr and then defragment forward. - i.iterSpan = i.iter.Next() + // The span we landed on has a Start bound ≤ key. There may be additional + // fragments before this span. Defragment backward to find the start of the + // defragmented span. + i.defragmentBackward() + if i.iterPos == iterPosPrev { + // Next once back onto the span. + i.iterSpan = i.iter.Next() + } + // Defragment the full span from its start. return i.defragmentForward() } -// SeekLT seeks the iterator to the last span with a start key less than -// key and returns it. +// SeekLT moves the iterator to the last span covering a key less than the +// given key. This is equivalent to seeking to the last span with a start +// key less than the given key. func (i *DefragmentingIter) SeekLT(key []byte) *Span { i.iterSpan = i.iter.SeekLT(key) if i.iterSpan == nil { @@ -219,7 +248,16 @@ func (i *DefragmentingIter) SeekLT(key []byte) *Span { i.iterPos = iterPosCurr return i.iterSpan } - // Defragment forward to find the end of the defragmented span. + // If the span ends strictly before key, we know there mustn't be a later + // span that starts at i.iterSpan.End, otherwise i.iter would've returned + // that span instead. + if i.comparer.Compare(i.iterSpan.End, key) < 0 { + return i.defragmentBackward() + } + + // The span we landed on has a End bound ≥ key. There may be additional + // fragments after this span. Defragment forward to find the end of the + // defragmented span. i.defragmentForward() if i.iterPos == iterPosNext { // Prev once back onto the span. @@ -437,15 +475,9 @@ func (i *DefragmentingIter) defragmentBackward() *Span { } func (i *DefragmentingIter) saveCurrent() { - i.currBuf = i.currBuf[:0] + i.currBuf.Reset() i.keysBuf = i.keysBuf[:0] i.keyBuf = i.keyBuf[:0] - if cap(i.currBuf) > bufferReuseMaxCapacity { - i.currBuf = nil - } - if cap(i.keyBuf) > bufferReuseMaxCapacity { - i.keyBuf = nil - } if i.iterSpan == nil { return } @@ -468,7 +500,6 @@ func (i *DefragmentingIter) saveBytes(b []byte) []byte { if b == nil { return nil } - ret := append(i.currBuf, b...) - i.currBuf = ret[len(ret):] - return ret + i.currBuf, b = i.currBuf.Copy(b) + return b } diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go index aad9ecb..d4a410d 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/fragmenter.go @@ -108,20 +108,20 @@ func (f *Fragmenter) checkInvariants(buf []Span) { // increasing start key order. That is, Add must be called with a series // of spans like: // -// a---e -// c---g -// c-----i -// j---n -// j-l +// a---e +// c---g +// c-----i +// j---n +// j-l // // We need to fragment the spans at overlap points. In the above // example, we'd create: // -// a-c-e -// c-e-g -// c-e-g-i -// j-l-n -// j-l +// a-c-e +// c-e-g +// c-e-g-i +// j-l-n +// j-l // // The fragments need to be output sorted by start key, and for equal start // keys, sorted by descending sequence number. This last part requires a mild @@ -134,42 +134,42 @@ func (f *Fragmenter) checkInvariants(buf []Span) { // // Walking through the example above, we start with: // -// a---e +// a---e // // Next we add [c,g) resulting in: // -// a-c-e -// c---g +// a-c-e +// c---g // // The fragment [a,c) is flushed leaving the pending spans as: // -// c-e -// c---g +// c-e +// c---g // // The next span is [c,i): // -// c-e -// c---g -// c-----i +// c-e +// c---g +// c-----i // // No fragments are flushed. The next span is [j,n): // -// c-e -// c---g -// c-----i -// j---n +// c-e +// c---g +// c-----i +// j---n // // The fragments [c,e), [c,g) and [c,i) are flushed. We sort these fragments // by their end key, then split the fragments on the end keys: // -// c-e -// c-e-g -// c-e---i +// c-e +// c-e-g +// c-e---i // // The [c,e) fragments all get flushed leaving: // -// e-g -// e---i +// e-g +// e---i // // This process continues until there are no more fragments to flush. // @@ -229,16 +229,35 @@ func (f *Fragmenter) Add(s Span) { f.pending = append(f.pending, s) } -// Covers returns true if the specified key is covered by one of the pending -// spans. The key must be consistent with the ordering of the -// spans. That is, it is invalid to specify a key here that is out of -// order with the span start keys passed to Add. -func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) bool { +// Cover is returned by Framenter.Covers and describes a span's relationship to +// a key at a particular snapshot. +type Cover int8 + +const ( + // NoCover indicates the tested key does not fall within the span's bounds, + // or the span contains no keys with sequence numbers higher than the key's. + NoCover Cover = iota + // CoversInvisibly indicates the tested key does fall within the span's + // bounds and the span contains at least one key with a higher sequence + // number, but none visible at the provided snapshot. + CoversInvisibly + // CoversVisibly indicates the tested key does fall within the span's + // bounds, and the span constains at least one key with a sequence number + // higher than the key's sequence number that is visible at the provided + // snapshot. + CoversVisibly +) + +// Covers returns an enum indicating whether the specified key is covered by one +// of the pending keys. The provided key must be consistent with the ordering of +// the spans. That is, it is invalid to specify a key here that is out of order +// with the span start keys passed to Add. +func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) Cover { if f.finished { panic("pebble: span fragmenter already finished") } if len(f.pending) == 0 { - return false + return NoCover } if f.Cmp(f.pending[0].Start, key.UserKey) > 0 { @@ -246,18 +265,31 @@ func (f *Fragmenter) Covers(key base.InternalKey, snapshot uint64) bool { f.Format(f.pending[0].Start), key.Pretty(f.Format))) } + cover := NoCover seqNum := key.SeqNum() for _, s := range f.pending { if f.Cmp(key.UserKey, s.End) < 0 { // NB: A range deletion tombstone does not delete a point operation // at the same sequence number, and broadly a span is not considered // to cover a point operation at the same sequence number. - if s.CoversAt(snapshot, seqNum) { - return true + + for i := range s.Keys { + if kseq := s.Keys[i].SeqNum(); kseq > seqNum { + // This key from the span has a higher sequence number than + // `key`. It covers `key`, although the span's key might not + // be visible if its snapshot is too high. + // + // Batch keys are always be visible. + if kseq < snapshot || kseq&base.InternalKeySeqNumBatch != 0 { + return CoversVisibly + } + // s.Keys[i] is not visible. + cover = CoversInvisibly + } } } } - return false + return cover } // Empty returns true if all fragments added so far have finished flushing. @@ -270,20 +302,20 @@ func (f *Fragmenter) Empty() bool { // emitting of spans which straddle an sstable boundary. Consider // the scenario: // -// a---------k#10 -// f#8 -// f#7 -/// +// a---------k#10 +// f#8 +// f#7 +// // Let's say the next user key after f is g. Calling TruncateAndFlushTo(g) will // flush this span: // -// a-------g#10 -// f#8 -// f#7 +// a-------g#10 +// f#8 +// f#7 // // And leave this one in f.pending: // -// g----k#10 +// g----k#10 // // WARNING: The fragmenter could hold on to the specified end key. Ensure it's // a safe byte slice that could outlast the current sstable output, and one @@ -374,8 +406,8 @@ func (f *Fragmenter) truncateAndFlush(key []byte) { // we want to flush (but not truncate) all range tombstones that start at or // before the first key in the next sstable. Consider: // -// a---e#10 -// a------h#9 +// a---e#10 +// a------h#9 // // If a compaction splits the sstables at key c we want the first sstable to // contain the tombstones [a,e)#10 and [a,e)#9. Fragmentation would naturally diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go index 6699de2..a507b75 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/interleaving_iter.go @@ -29,9 +29,9 @@ type SpanMask interface { SpanChanged(*Span) // SkipPoint is invoked by the interleaving iterator whenever the iterator // encounters a point key covered by a Span. If SkipPoint returns true, the - // interleaving iterator skips the point key without returning it. This is - // used during range key iteration to skip over point keys 'masked' by range - // keys. + // interleaving iterator skips the point key and all larger keys with the + // same prefix. This is used during range key iteration to skip over point + // keys 'masked' by range keys. SkipPoint(userKey []byte) bool } @@ -79,13 +79,13 @@ type SpanMask interface { // SeekGE or SeekPrefixGE search key. Consider, for example SetBounds('c', 'e'), // with an iterator containing the Span [a,z): // -// First() = `c#72057594037927935,21` Span() = [c,e) -// SeekGE('d') = `d#72057594037927935,21` Span() = [c,e) +// First() = `c#72057594037927935,21` Span() = [c,e) +// SeekGE('d') = `d#72057594037927935,21` Span() = [c,e) // // InterleavedIter does not interleave synthetic markers for spans that do not // contain any keys. // -// SpanMask +// # SpanMask // // InterelavingIter takes a SpanMask parameter that may be used to configure the // behavior of the iterator. See the documentation on the SpanMask type. @@ -479,6 +479,30 @@ func (i *InterleavingIter) Next() (*base.InternalKey, base.LazyValue) { return i.interleaveForward(i.lower, nil /* prefix */) } +// NextPrefix implements (base.InternalIterator).NextPrefix. +func (i *InterleavingIter) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) { + if i.dir == -1 { + panic("pebble: cannot switch directions with NextPrefix") + } + + // Refresh the point key if the current point key has already been + // interleaved, OR if the next uninterleaved point key has the old suffix + // (it's less than succKey). + if i.pointKeyInterleaved || (i.pointKey != nil && i.cmp(i.pointKey.UserKey, succKey) < 0) { + i.pointKey, i.pointVal = i.pointIter.NextPrefix(succKey) + i.pointKeyInterleaved = false + } + // If we already interleaved the current span start key, and the point key + // is ≥ the span's end key, move to the next span. + if i.keyspanInterleaved && i.pointKey != nil && i.span != nil && + i.cmp(i.pointKey.UserKey, i.span.End) >= 0 { + i.span = i.keyspanIter.Next() + i.checkForwardBound(nil) + i.savedKeyspan() + } + return i.interleaveForward(i.lower, nil) +} + // Prev implements (base.InternalIterator).Prev. func (i *InterleavingIter) Prev() (*base.InternalKey, base.LazyValue) { if i.dir == +1 { @@ -677,9 +701,29 @@ func (i *InterleavingIter) interleaveForward( // if we have stepped outside of the span last saved as a mask, // so that the decision to skip is made with the correct // knowledge of the covering span. - i.maybeUpdateMask(true /*covered */) + i.maybeUpdateMask(true /* covered */) if i.mask != nil && i.mask.SkipPoint(i.pointKey.UserKey) { + if i.prefix { + // During prefix-iteration node, once a point is masked, + // all subsequent keys with the same prefix must also be + // masked according to the key ordering. We can stop and + // return nil. + // + // NB: The above is not just an optimization. During + // prefix-iteration mode, the internal iterator contract + // prohibits us from Next-ing beyond the first key + // beyond the iteration prefix. If we didn't already + // stop early, we would need to check if this masked + // point is already beyond the prefix. + return i.yieldNil() + } + // TODO(jackson): If we thread a base.Comparer through to + // InterleavingIter so that we have access to + // ImmediateSuccessor, we could use NextPrefix. We'd need to + // tweak the SpanMask interface slightly, but it's probably + // worthwhile. + i.pointKey, i.pointVal = i.pointIter.Next() // We may have just invalidated the invariant that // ensures the span's End is > the point key, so @@ -780,27 +824,16 @@ func (i *InterleavingIter) interleaveBackward() (*base.InternalKey, base.LazyVal } } -// keyspanSeekGE seeks the keyspan iterator to the first span covering k ≥ key. -// Note that this differs from the FragmentIterator.SeekGE semantics, which -// seek to the first span with a start key ≥ key. -func (i *InterleavingIter) keyspanSeekGE(key []byte, prefix []byte) { - // Seek using SeekLT to look for a span that starts before key, with an end - // boundary extending beyond key. - i.span = i.keyspanIter.SeekLT(key) - if i.span == nil || i.cmp(i.span.End, key) <= 0 { - // The iterator is exhausted in the reverse direction, or the span we - // found ends before key. Next to the first key with a start ≥ key. - i.span = i.keyspanIter.Next() - } +// keyspanSeekGE seeks the keyspan iterator to the first span covering a key ≥ k. +func (i *InterleavingIter) keyspanSeekGE(k []byte, prefix []byte) { + i.span = i.keyspanIter.SeekGE(k) i.checkForwardBound(prefix) i.savedKeyspan() } -// keyspanSeekLT seeks the keyspan iterator to the last span covering k < key. -// Note that this differs from the FragmentIterator.SeekLT semantics, which -// seek to the last span with a start key < key. -func (i *InterleavingIter) keyspanSeekLT(key []byte) { - i.span = i.keyspanIter.SeekLT(key) +// keyspanSeekLT seeks the keyspan iterator to the last span covering a key < k. +func (i *InterleavingIter) keyspanSeekLT(k []byte) { + i.span = i.keyspanIter.SeekLT(k) i.checkBackwardBound() // The current span's start key is not guaranteed to be less than key, // because of the bounds enforcement. Consider the following example: @@ -813,7 +846,7 @@ func (i *InterleavingIter) keyspanSeekLT(key []byte) { // // This problem is a consequence of the SeekLT's exclusive search key and // the fact that we don't perform bounds truncation at every leaf iterator. - if i.span != nil && i.truncated && i.cmp(i.truncatedSpan.Start, key) >= 0 { + if i.span != nil && i.truncated && i.cmp(i.truncatedSpan.Start, k) >= 0 { i.span = nil } i.savedKeyspan() diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go index 373246e..cf2adc3 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/internal_iter_shim.go @@ -16,6 +16,7 @@ import "github.com/cockroachdb/pebble/internal/base" // keyspan.FragmentIterator with point keys. type InternalIteratorShim struct { miter MergingIter + mbufs MergingBuffers span *Span iterKey base.InternalKey } @@ -26,7 +27,7 @@ var _ base.InternalIterator = &InternalIteratorShim{} // Init initializes the internal iterator shim to merge the provided fragment // iterators. func (i *InternalIteratorShim) Init(cmp base.Compare, iters ...FragmentIterator) { - i.miter.Init(cmp, noopTransform, iters...) + i.miter.Init(cmp, noopTransform, &i.mbufs, iters...) } // Span returns the span containing the full set of keys over the key span at @@ -87,6 +88,11 @@ func (i *InternalIteratorShim) Next() (*base.InternalKey, base.LazyValue) { return &i.iterKey, base.MakeInPlaceValue(i.span.End) } +// NextPrefix implements (base.InternalIterator).NextPrefix. +func (i *InternalIteratorShim) NextPrefix([]byte) (*base.InternalKey, base.LazyValue) { + panic("unimplemented") +} + // Prev implements (base.InternalIterator).Prev. func (i *InternalIteratorShim) Prev() (*base.InternalKey, base.LazyValue) { panic("unimplemented") diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go index d7f5068..4798561 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/iter.go @@ -18,12 +18,14 @@ import ( // longer lifetimes but implementations need only guarantee stability until the // next positioning method. type FragmentIterator interface { - // SeekGE moves the iterator to the first span whose start key is greater - // than or equal to the given key. + // SeekGE moves the iterator to the first span covering a key greater than + // or equal to the given key. This is equivalent to seeking to the first + // span with an end key greater than the given key. SeekGE(key []byte) *Span - // SeekLT moves the iterator to the last span whose start key is less than - // the given key. + // SeekLT moves the iterator to the last span covering a key less than the + // given key. This is equivalent to seeking to the last span with a start + // key less than the given key. SeekLT(key []byte) *Span // First moves the iterator to the first span. @@ -58,7 +60,8 @@ type FragmentIterator interface { Close() error } -// TableNewSpanIter creates a new iterator for spans for the given file. +// TableNewSpanIter creates a new iterator for range key spans for the given +// file. type TableNewSpanIter func(file *manifest.FileMetadata, iterOptions *SpanIterOptions) (FragmentIterator, error) // SpanIterOptions is a subset of IterOptions that are necessary to instantiate @@ -104,6 +107,9 @@ func (i *Iter) Init(cmp base.Compare, spans []Span) { func (i *Iter) SeekGE(key []byte) *Span { // NB: manually inlined sort.Search is ~5% faster. // + // Define f(j) = false iff the span i.spans[j] is strictly before `key` + // (equivalently, i.spans[j].End ≤ key.) + // // Define f(-1) == false and f(n) == true. // Invariant: f(index-1) == false, f(upper) == true. i.index = 0 @@ -111,12 +117,13 @@ func (i *Iter) SeekGE(key []byte) *Span { for i.index < upper { h := int(uint(i.index+upper) >> 1) // avoid overflow when computing h // i.index ≤ h < upper - if i.cmp(key, i.spans[h].Start) > 0 { + if i.cmp(key, i.spans[h].End) >= 0 { i.index = h + 1 // preserves f(i-1) == false } else { upper = h // preserves f(j) == true } } + // i.index == upper, f(i.index-1) == false, and f(upper) (= f(i.index)) == // true => answer is i.index. if i.index >= len(i.spans) { diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go index c65c022..047c6a2 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/level_iter.go @@ -8,25 +8,23 @@ import ( "fmt" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/manifest" ) -// Logger defines an interface for writing log messages. -type Logger interface { - Infof(format string, args ...interface{}) - Fatalf(format string, args ...interface{}) -} - // LevelIter provides a merged view of spans from sstables in a level. // It takes advantage of level invariants to only have one sstable span block // open at one time, opened using the newIter function passed in. type LevelIter struct { - logger Logger - cmp base.Compare - // Denotes if this level iter should read point key spans (i.e. rangedels, - // or range keys. If key type is Point, no straddle spans are emitted between - // files, and point key bounds are used to find files instead of range key - // bounds. + cmp base.Compare + // Denotes the kind of key the level iterator should read. If the key type + // is KeyTypePoint, the level iterator will read range tombstones (which + // only affect point keys). If the key type is KeyTypeRange, the level + // iterator will read range keys. It is invalid to configure an iterator + // with the KeyTypePointAndRange key type. + // + // If key type is KeyTypePoint, no straddle spans are emitted between files, + // and point key bounds are used to find files instead of range key bounds. // // TODO(bilal): Straddle spans can safely be produced in rangedel mode once // we can guarantee that we will never read sstables in a level that split @@ -50,14 +48,17 @@ type LevelIter struct { dir int straddle Span straddleDir int - // The iter for the current file. It is nil under any of the following conditions: + // The iter for the current file (iterFile). It is nil under any of the + // following conditions: // - files.Current() == nil // - err != nil // - straddleDir != 0, in which case iterFile is not nil and points to the // next file (in the straddleDir direction). // - some other constraint, like the bounds in opts, caused the file at index to not // be relevant to the iteration. - iter FragmentIterator + iter FragmentIterator + // iterFile holds the current file. + // INVARIANT: iterFile = files.Current() iterFile *manifest.FileMetadata newIter TableNewSpanIter files manifest.LevelIterator @@ -72,18 +73,17 @@ type LevelIter struct { // LevelIter implements the keyspan.FragmentIterator interface. var _ FragmentIterator = (*LevelIter)(nil) -// newLevelIter returns a LevelIter. -func newLevelIter( +// NewLevelIter returns a LevelIter. +func NewLevelIter( opts SpanIterOptions, cmp base.Compare, newIter TableNewSpanIter, files manifest.LevelIterator, level manifest.Level, - logger Logger, keyType manifest.KeyType, ) *LevelIter { l := &LevelIter{} - l.Init(opts, cmp, newIter, files, level, logger, keyType) + l.Init(opts, cmp, newIter, files, level, keyType) return l } @@ -94,18 +94,19 @@ func (l *LevelIter) Init( newIter TableNewSpanIter, files manifest.LevelIterator, level manifest.Level, - logger Logger, keyType manifest.KeyType, ) { l.err = nil l.level = level - l.logger = logger l.tableOpts.RangeKeyFilters = opts.RangeKeyFilters l.cmp = cmp l.iterFile = nil l.newIter = newIter switch keyType { - case manifest.KeyTypePoint, manifest.KeyTypeRange: + case manifest.KeyTypePoint: + l.keyType = keyType + l.files = files.Filter(keyType) + case manifest.KeyTypeRange: l.keyType = keyType l.files = files.Filter(keyType) default: @@ -196,7 +197,9 @@ func (l *LevelIter) SeekGE(key []byte) *Span { f := l.findFileGE(key) if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(key, f.SmallestRangeKey.UserKey) < 0 { + // Peek at the previous file. prevFile := l.files.Prev() + l.files.Next() if prevFile != nil { // We could unconditionally return an empty span between the seek key and // f.SmallestRangeKey, however if this span is to the left of all range @@ -212,11 +215,10 @@ func (l *LevelIter) SeekGE(key []byte) *Span { // // TODO(bilal): Investigate ways to be able to return straddle spans in // cases similar to the above, while still retaining correctness. - l.files.Next() // Return a straddling key instead of loading the file. l.iterFile = f if err := l.Close(); err != nil { - return nil + return l.verify(nil) } l.straddleDir = +1 l.straddle = Span{ @@ -224,15 +226,15 @@ func (l *LevelIter) SeekGE(key []byte) *Span { End: f.SmallestRangeKey.UserKey, Keys: nil, } - return &l.straddle + return l.verify(&l.straddle) } } loadFileIndicator := l.loadFile(f, +1) if loadFileIndicator == noFileLoaded { - return nil + return l.verify(nil) } if span := l.iter.SeekGE(key); span != nil { - return span + return l.verify(span) } return l.skipEmptyFileForward() } @@ -246,7 +248,9 @@ func (l *LevelIter) SeekLT(key []byte) *Span { f := l.findFileLT(key) if f != nil && l.keyType == manifest.KeyTypeRange && l.cmp(f.LargestRangeKey.UserKey, key) < 0 { + // Peek at the next file. nextFile := l.files.Next() + l.files.Prev() if nextFile != nil { // We could unconditionally return an empty span between f.LargestRangeKey // and the seek key, however if this span is to the right of all range keys @@ -262,11 +266,10 @@ func (l *LevelIter) SeekLT(key []byte) *Span { // // TODO(bilal): Investigate ways to be able to return straddle spans in // cases similar to the above, while still retaining correctness. - l.files.Prev() // Return a straddling key instead of loading the file. l.iterFile = f if err := l.Close(); err != nil { - return nil + return l.verify(nil) } l.straddleDir = -1 l.straddle = Span{ @@ -274,14 +277,14 @@ func (l *LevelIter) SeekLT(key []byte) *Span { End: nextFile.SmallestRangeKey.UserKey, Keys: nil, } - return &l.straddle + return l.verify(&l.straddle) } } - if l.loadFile(l.findFileLT(key), -1) == noFileLoaded { - return nil + if l.loadFile(f, -1) == noFileLoaded { + return l.verify(nil) } if span := l.iter.SeekLT(key); span != nil { - return span + return l.verify(span) } return l.skipEmptyFileBackward() } @@ -294,10 +297,10 @@ func (l *LevelIter) First() *Span { l.err = nil // clear cached iteration error if l.loadFile(l.files.First(), +1) == noFileLoaded { - return nil + return l.verify(nil) } if span := l.iter.First(); span != nil { - return span + return l.verify(span) } return l.skipEmptyFileForward() } @@ -310,10 +313,10 @@ func (l *LevelIter) Last() *Span { l.err = nil // clear cached iteration error if l.loadFile(l.files.Last(), -1) == noFileLoaded { - return nil + return l.verify(nil) } if span := l.iter.Last(); span != nil { - return span + return l.verify(span) } return l.skipEmptyFileBackward() } @@ -321,7 +324,7 @@ func (l *LevelIter) Last() *Span { // Next implements keyspan.FragmentIterator. func (l *LevelIter) Next() *Span { if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir > 0) { - return nil + return l.verify(nil) } if l.iter == nil && l.iterFile == nil { // l.dir <= 0 @@ -331,7 +334,7 @@ func (l *LevelIter) Next() *Span { if l.iter != nil { if span := l.iter.Next(); span != nil { - return span + return l.verify(span) } } return l.skipEmptyFileForward() @@ -340,7 +343,7 @@ func (l *LevelIter) Next() *Span { // Prev implements keyspan.FragmentIterator. func (l *LevelIter) Prev() *Span { if l.err != nil || (l.iter == nil && l.iterFile == nil && l.dir < 0) { - return nil + return l.verify(nil) } if l.iter == nil && l.iterFile == nil { // l.dir >= 0 @@ -350,7 +353,7 @@ func (l *LevelIter) Prev() *Span { if l.iter != nil { if span := l.iter.Prev(); span != nil { - return span + return l.verify(span) } } return l.skipEmptyFileBackward() @@ -367,7 +370,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span { // Straddle spans are not created in rangedel mode. if err := l.Close(); err != nil { l.err = err - return nil + return l.verify(nil) } startKey := l.iterFile.LargestRangeKey.UserKey // Resetting l.iterFile without loading the file into l.iter is okay and @@ -375,7 +378,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span { // which it should be due to the Close() call above. l.iterFile = l.files.Next() if l.iterFile == nil { - return nil + return l.verify(nil) } endKey := l.iterFile.SmallestRangeKey.UserKey if l.cmp(startKey, endKey) < 0 { @@ -386,7 +389,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span { End: endKey, } l.straddleDir = +1 - return &l.straddle + return l.verify(&l.straddle) } } else if l.straddleDir < 0 { // We were at a straddle key, but are now changing directions. l.iterFile @@ -405,7 +408,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span { fileToLoad = l.files.Next() } if l.loadFile(fileToLoad, +1) == noFileLoaded { - return nil + return l.verify(nil) } span = l.iter.First() // In rangedel mode, we can expect to get empty files that we'd need to @@ -414,7 +417,7 @@ func (l *LevelIter) skipEmptyFileForward() *Span { break } } - return span + return l.verify(span) } func (l *LevelIter) skipEmptyFileBackward() *Span { @@ -428,7 +431,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span { l.iterFile != nil && l.iter != nil { if err := l.Close(); err != nil { l.err = err - return nil + return l.verify(nil) } endKey := l.iterFile.SmallestRangeKey.UserKey // Resetting l.iterFile without loading the file into l.iter is okay and @@ -436,7 +439,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span { // which it should be due to the Close() call above. l.iterFile = l.files.Prev() if l.iterFile == nil { - return nil + return l.verify(nil) } startKey := l.iterFile.LargestRangeKey.UserKey if l.cmp(startKey, endKey) < 0 { @@ -447,7 +450,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span { End: endKey, } l.straddleDir = -1 - return &l.straddle + return l.verify(&l.straddle) } } else if l.straddleDir > 0 { // We were at a straddle key, but are now changing directions. l.iterFile @@ -464,7 +467,7 @@ func (l *LevelIter) skipEmptyFileBackward() *Span { fileToLoad = l.files.Prev() } if l.loadFile(fileToLoad, -1) == noFileLoaded { - return nil + return l.verify(nil) } span = l.iter.Last() // In rangedel mode, we can expect to get empty files that we'd need to @@ -474,7 +477,22 @@ func (l *LevelIter) skipEmptyFileBackward() *Span { break } } - return span + return l.verify(span) +} + +// verify is invoked whenever a span is returned from an iterator positioning +// method to a caller. During invariant builds, it asserts invariants to the +// caller. +func (l *LevelIter) verify(s *Span) *Span { + // NB: Do not add any logic outside the invariants.Enabled conditional to + // ensure that verify is always compiled away in production builds. + if invariants.Enabled { + if f := l.files.Current(); f != l.iterFile { + panic(fmt.Sprintf("LevelIter.files.Current (%s) and l.iterFile (%s) diverged", + f, l.iterFile)) + } + } + return s } // Error implements keyspan.FragmentIterator. diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go index 996de03..2476024 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/merging_iter.go @@ -21,6 +21,10 @@ import ( // seeks would require introducing key comparisons to switchTo{Min,Max}Heap // where there currently are none. +// TODO(jackson): There are several opportunities to use base.Equal in the +// MergingIter implementation, but will require a bit of plumbing to thread the +// Equal function. + // Transformer defines a transformation to be applied to a Span. type Transformer interface { // Transform takes a Span as input and writes the transformed Span to the @@ -45,9 +49,9 @@ var noopTransform Transformer = TransformerFunc(func(_ base.Compare, s Span, dst return nil }) -// visibleTransform filters keys that are invisible at the provided snapshot +// VisibleTransform filters keys that are invisible at the provided snapshot // sequence number. -func visibleTransform(snapshot uint64) Transformer { +func VisibleTransform(snapshot uint64) Transformer { return TransformerFunc(func(_ base.Compare, s Span, dst *Span) error { dst.Start, dst.End = s.Start, s.End dst.Keys = dst.Keys[:0] @@ -76,34 +80,34 @@ func visibleTransform(snapshot uint64) Transformer { // trailer descending. If the MergingIter is configured with a Transformer, it's // permitted to modify the ordering of the spans' keys returned by MergingIter. // -// Algorithm +// # Algorithm // // The merging iterator wraps child iterators, merging and fragmenting spans // across levels. The high-level algorithm is: // -// 1. Initialize the heap with bound keys from child iterators' spans. -// 2. Find the next [or previous] two unique user keys' from bounds. -// 3. Consider the span formed between the two unique user keys a candidate -// span. -// 4. Determine if any of the child iterators' spans overlap the candidate -// span. -// 4a. If any of the child iterator's current bounds are end keys -// (during forward iteration) or start keys (during reverse -// iteration), then all the spans with that bound overlap the -// candidate span. -// 4b. Apply the configured transform, which may remove keys. -// 4b. If no spans overlap, forget the smallest (forward iteration) -// or largest (reverse iteration) unique user key and advance -// the iterators to the next unique user key. Start again from 3. +// 1. Initialize the heap with bound keys from child iterators' spans. +// 2. Find the next [or previous] two unique user keys' from bounds. +// 3. Consider the span formed between the two unique user keys a candidate +// span. +// 4. Determine if any of the child iterators' spans overlap the candidate +// span. +// 4a. If any of the child iterator's current bounds are end keys +// (during forward iteration) or start keys (during reverse +// iteration), then all the spans with that bound overlap the +// candidate span. +// 4b. Apply the configured transform, which may remove keys. +// 4c. If no spans overlap, forget the smallest (forward iteration) +// or largest (reverse iteration) unique user key and advance +// the iterators to the next unique user key. Start again from 3. // -// Detailed algorithm +// # Detailed algorithm // // Each level (i0, i1, ...) has a user-provided input FragmentIterator. The // merging iterator steps through individual boundaries of the underlying // spans separately. If the underlying FragmentIterator has fragments // [a,b){#2,#1} [b,c){#1} the mergingIterLevel.{next,prev} step through: // -// (a, start), (b, end), (b, start), (c, end) +// (a, start), (b, end), (b, start), (c, end) // // Note that (a, start) and (b, end) are observed ONCE each, despite two keys // sharing those bounds. Also note that (b, end) and (b, start) are two distinct @@ -119,18 +123,18 @@ func visibleTransform(snapshot uint64) Transformer { // determine which span is next, but it's also responsible for fragmenting // overlapping spans. Consider the example: // -// i0: b---d e-----h -// i1: a---c h-----k -// i2: a------------------------------p +// i0: b---d e-----h +// i1: a---c h-----k +// i2: a------------------------------p // -// fragments: a-b-c-d-e-----h-----k----------p +// fragments: a-b-c-d-e-----h-----k----------p // // None of the individual child iterators contain a span with the exact bounds // [c,d), but the merging iterator must produce a span [c,d). To accomplish // this, the merging iterator visits every span between unique boundary user // keys. In the above example, this is: // -// [a,b), [b,c), [c,d), [d,e), [e, h), [h, k), [k, p) +// [a,b), [b,c), [c,d), [d,e), [e, h), [h, k), [k, p) // // The merging iterator first initializes the heap to prepare for iteration. // The description below discusses the mechanics of forward iteration after a @@ -141,9 +145,9 @@ func visibleTransform(snapshot uint64) Transformer { // mergingIterLevel to the first bound of the first fragment. In the above // example, this seeks the child iterators to: // -// i0: (b, boundKindFragmentStart, [ [b,d) ]) -// i1: (a, boundKindFragmentStart, [ [a,c) ]) -// i2: (a, boundKindFragmentStart, [ [a,p) ]) +// i0: (b, boundKindFragmentStart, [ [b,d) ]) +// i1: (a, boundKindFragmentStart, [ [a,c) ]) +// i2: (a, boundKindFragmentStart, [ [a,p) ]) // // After fixing up the heap, the root of the heap is a boundKey with the // smallest user key ('a' in the example). Once the heap is setup for iteration @@ -160,9 +164,9 @@ func visibleTransform(snapshot uint64) Transformer { // In the above example, this results in m.start = 'a', m.end = 'b' and child // iterators in the following positions: // -// i0: (b, boundKindFragmentStart, [ [b,d) ]) -// i1: (c, boundKindFragmentEnd, [ [a,c) ]) -// i2: (p, boundKindFragmentEnd, [ [a,p) ]) +// i0: (b, boundKindFragmentStart, [ [b,d) ]) +// i1: (c, boundKindFragmentEnd, [ [a,c) ]) +// i2: (p, boundKindFragmentEnd, [ [a,p) ]) // // With the user key bounds of the next merged span established, // findNextFragmentSet must determine which, if any, fragments overlap the span. @@ -191,7 +195,7 @@ func visibleTransform(snapshot uint64) Transformer { // findNextFragmentSet loops, repeating the above process again until it finds a // span that does contain keys. // -// Memory safety +// # Memory safety // // The FragmentIterator interface only guarantees stability of a Span and its // associated slices until the next positioning method is called. Adjacent Spans @@ -207,24 +211,14 @@ func visibleTransform(snapshot uint64) Transformer { // accommodate this, find{Next,Prev}FragmentSet copy the initial boundary if the // subsequent Next/Prev would move to the next span. type MergingIter struct { - levels []mergingIterLevel - heap mergingIterHeap + *MergingBuffers // start and end hold the bounds for the span currently under the // iterator position. // // Invariant: None of the levels' iterators contain spans with a bound // between start and end. For all bounds b, b ≤ start || b ≥ end. start, end []byte - // buf is a buffer used to save [start, end) boundary keys. - buf []byte - // keys holds all of the keys across all levels that overlap the key span - // [start, end), sorted by Trailer descending. This slice is reconstituted - // in synthesizeKeys from each mergingIterLevel's keys every time the - // [start, end) bounds change. - // - // Each element points into a child iterator's memory, so the keys may not - // be directly modified. - keys keysBySeqNumKind + // transformer defines a transformation to be applied to a span before it's // yielded to the user. Transforming may filter individual keys contained // within the span. @@ -233,9 +227,8 @@ type MergingIter struct { // destination for transforms. Every tranformed span overwrites the // previous. span Span - - err error - dir int8 + err error + dir int8 // alloc preallocates mergingIterLevel and mergingIterItems for use by the // merging iterator. As long as the merging iterator is used with @@ -249,6 +242,35 @@ type MergingIter struct { } } +// MergingBuffers holds buffers used while merging keyspans. +type MergingBuffers struct { + // keys holds all of the keys across all levels that overlap the key span + // [start, end), sorted by Trailer descending. This slice is reconstituted + // in synthesizeKeys from each mergingIterLevel's keys every time the + // [start, end) bounds change. + // + // Each element points into a child iterator's memory, so the keys may not + // be directly modified. + keys keysBySeqNumKind + // levels holds levels allocated by MergingIter.init. The MergingIter will + // prefer use of its `manifest.NumLevels+3` array, so this slice will be + // longer if set. + levels []mergingIterLevel + // heap holds a slice for the merging iterator heap allocated by + // MergingIter.init. The MergingIter will prefer use of its + // `manifest.NumLevels+3` items array, so this slice will be longer if set. + heap mergingIterHeap + // buf is a buffer used to save [start, end) boundary keys. + buf []byte +} + +// PrepareForReuse discards any excessively large buffers. +func (bufs *MergingBuffers) PrepareForReuse() { + if cap(bufs.buf) > bufferReuseMaxCapacity { + bufs.buf = nil + } +} + // MergingIter implements the FragmentIterator interface. var _ FragmentIterator = (*MergingIter)(nil) @@ -300,13 +322,15 @@ func (l *mergingIterLevel) prev() { } // Init initializes the merging iterator with the provided fragment iterators. -func (m *MergingIter) Init(cmp base.Compare, transformer Transformer, iters ...FragmentIterator) { - levels, items := m.levels, m.heap.items - +func (m *MergingIter) Init( + cmp base.Compare, transformer Transformer, bufs *MergingBuffers, iters ...FragmentIterator, +) { *m = MergingIter{ - heap: mergingIterHeap{cmp: cmp}, - transformer: transformer, + MergingBuffers: bufs, + transformer: transformer, } + m.heap.cmp = cmp + levels, items := m.levels, m.heap.items // Invariant: cap(levels) >= cap(items) // Invariant: cap(alloc.levels) == cap(alloc.items) @@ -334,38 +358,70 @@ func (m *MergingIter) AddLevel(iter FragmentIterator) { m.levels = append(m.levels, mergingIterLevel{iter: iter}) } -// SeekGE moves the iterator to the first span with a start key greater than or -// equal to key. +// SeekGE moves the iterator to the first span covering a key greater than +// or equal to the given key. This is equivalent to seeking to the first +// span with an end key greater than the given key. func (m *MergingIter) SeekGE(key []byte) *Span { m.invalidate() // clear state about current position + + // SeekGE(k) seeks to the first span with an end key greater than the given + // key. The merged span M that we're searching for might straddle the seek + // `key`. In this case, the M.Start may be a key ≤ the seek key. + // + // Consider a SeekGE(dog) in the following example. + // + // i0: b---d e-----h + // i1: a---c h-----k + // i2: a------------------------------p + // merged: a-b-c-d-e-----h-----k----------p + // + // The merged span M containing 'dog' is [d,e). The 'd' of the merged span + // comes from i0's [b,d)'s end boundary. The [b,d) span does not cover any + // key >= dog, so we cannot find the span by positioning the child iterators + // using a SeekGE(dog). + // + // Instead, if we take all the child iterators' spans bounds: + // a b c d e h k p + // We want to partition them into keys ≤ `key` and keys > `key`. + // dog + // │ + // a b c d│e h k p + // │ + // The largest key on the left of the partition forms the merged span's + // start key, and the smallest key on the right of the partition forms the + // merged span's end key. Recharacterized: + // + // M.Start: the largest boundary ≤ k of any child span + // M.End: the smallest boundary > k of any child span + // + // The FragmentIterator interface doesn't implement seeking by all bounds, + // it implements seeking by containment. A SeekGE(k) will ensure we observe + // all start boundaries ≥ k and all end boundaries > k but does not ensure + // we observe end boundaries = k or any boundaries < k. A SeekLT(k) will + // ensure we observe all start boundaries < k and all end boundaries ≤ k but + // does not ensure we observe any start boundaries = k or any boundaries > + // k. This forces us to seek in one direction and step in the other. + // + // In a SeekGE, we want to end up oriented in the forward direction when + // complete, so we begin with searching for M.Start by SeekLT-ing every + // child iterator to `k`. For every child span found, we determine the + // largest bound ≤ `k` and use it to initialize our max heap. The resulting + // root of the max heap is a preliminary value for `M.Start`. for i := range m.levels { l := &m.levels[i] - - // A SeekGE requires we position each level at the smallest bound ≥ key. - // We must search through both inclusive start and exclusive end bounds. - // Note that this search requirement differs from FragmentIterator's - // .SeekGE'semantics, which returns the span with the smallest start key - // ≥ key. To remedy this difference, we find the last span less than - // key. If its end boundary is greater than or equal to key, we use it. - // Otherwise we use the start boundary of the next span which - // necessarily has a start ≥ key. s := l.iter.SeekLT(key) - if s != nil && m.cmp(s.End, key) >= 0 { - // s.End ≥ key - // We need to use this span's end bound. + if s == nil { + l.heapKey = boundKey{kind: boundKindInvalid} + } else if m.cmp(s.End, key) <= 0 { l.heapKey = boundKey{ kind: boundKindFragmentEnd, key: s.End, span: s, } - continue - } - // s.End < key - // The span `s` ends before key. Next to the first span with a Start ≥ - // key, and use that. - if s = l.iter.Next(); s == nil { - l.heapKey = boundKey{kind: boundKindInvalid} } else { + // s.End > key && s.Start < key + // We need to use this span's start bound, since that's the largest + // bound ≤ key. l.heapKey = boundKey{ kind: boundKindFragmentStart, key: s.Start, @@ -373,36 +429,201 @@ func (m *MergingIter) SeekGE(key []byte) *Span { } } } - m.initMinHeap() - return m.findNextFragmentSet() -} + m.initMaxHeap() + if m.err != nil { + return nil + } else if len(m.heap.items) == 0 { + // There are no spans covering any key < `key`. There is no span that + // straddles the seek key. Reorient the heap into a min heap and return + // the first span we find in the forward direction. + m.switchToMinHeap() + return m.findNextFragmentSet() + } -// SeekLT moves the iterator to the last span with a start key less than key. -func (m *MergingIter) SeekLT(key []byte) *Span { - // TODO(jackson): Evaluate whether there's an implementation of SeekLT - // independent of SeekGE that is more efficient. It's tricky, because the - // span we should return might straddle `key` itself. + // The heap root is now the largest boundary key b such that: + // 1. b < k + // 2. b = k, and b is an end boundary + // There's a third case that we will need to consider later, after we've + // switched to a min heap: + // 3. there exists a start boundary key b such that b = k. + // A start boundary key equal to k would not be surfaced when we seeked all + // the levels using SeekLT(k), since no key `key`, which will serve as our candidate end + // bound. + m.switchToMinHeap() + if m.err != nil { + return nil + } else if len(m.heap.items) == 0 { return nil } - // Prev to the previous span. - return m.Prev() + + // Check for the case 3 described above. It's possible that when we switch + // heap directions, we discover a start boundary of some child span that is + // equal to the seek key `key`. In this case, we want this key to be our + // start boundary. + if m.heap.items[0].boundKey.kind == boundKindFragmentStart && + m.cmp(m.heap.items[0].boundKey.key, key) == 0 { + // Call findNextFragmentSet, which will set m.start to the heap root and + // proceed forward. + return m.findNextFragmentSet() + } + + m.end = m.heap.items[0].boundKey.key + if found, s := m.synthesizeKeys(+1); found && s != nil { + return s + } + return m.findNextFragmentSet() + +} + +// SeekLT moves the iterator to the last span covering a key less than the +// given key. This is equivalent to seeking to the last span with a start +// key less than the given key. +func (m *MergingIter) SeekLT(key []byte) *Span { + m.invalidate() // clear state about current position + + // SeekLT(k) seeks to the last span with a start key less than the given + // key. The merged span M that we're searching for might straddle the seek + // `key`. In this case, the M.End may be a key ≥ the seek key. + // + // Consider a SeekLT(dog) in the following example. + // + // i0: b---d e-----h + // i1: a---c h-----k + // i2: a------------------------------p + // merged: a-b-c-d-e-----h-----k----------p + // + // The merged span M containing the largest key <'dog' is [d,e). The 'e' of + // the merged span comes from i0's [e,h)'s start boundary. The [e,h) span + // does not cover any key < dog, so we cannot find the span by positioning + // the child iterators using a SeekLT(dog). + // + // Instead, if we take all the child iterators' spans bounds: + // a b c d e h k p + // We want to partition them into keys < `key` and keys ≥ `key`. + // dog + // │ + // a b c d│e h k p + // │ + // The largest key on the left of the partition forms the merged span's + // start key, and the smallest key on the right of the partition forms the + // merged span's end key. Recharacterized: + // + // M.Start: the largest boundary < k of any child span + // M.End: the smallest boundary ≥ k of any child span + // + // The FragmentIterator interface doesn't implement seeking by all bounds, + // it implements seeking by containment. A SeekGE(k) will ensure we observe + // all start boundaries ≥ k and all end boundaries > k but does not ensure + // we observe end boundaries = k or any boundaries < k. A SeekLT(k) will + // ensure we observe all start boundaries < k and all end boundaries ≤ k but + // does not ensure we observe any start boundaries = k or any boundaries > + // k. This forces us to seek in one direction and step in the other. + // + // In a SeekLT, we want to end up oriented in the backward direction when + // complete, so we begin with searching for M.End by SeekGE-ing every + // child iterator to `k`. For every child span found, we determine the + // smallest bound ≥ `k` and use it to initialize our min heap. The resulting + // root of the min heap is a preliminary value for `M.End`. + for i := range m.levels { + l := &m.levels[i] + s := l.iter.SeekGE(key) + if s == nil { + l.heapKey = boundKey{kind: boundKindInvalid} + } else if m.cmp(s.Start, key) >= 0 { + l.heapKey = boundKey{ + kind: boundKindFragmentStart, + key: s.Start, + span: s, + } + } else { + // s.Start < key + // We need to use this span's end bound, since that's the smallest + // bound > key. + l.heapKey = boundKey{ + kind: boundKindFragmentEnd, + key: s.End, + span: s, + } + } + } + m.initMinHeap() + if m.err != nil { + return nil + } else if len(m.heap.items) == 0 { + // There are no spans covering any key ≥ `key`. There is no span that + // straddles the seek key. Reorient the heap into a max heap and return + // the first span we find in the reverse direction. + m.switchToMaxHeap() + return m.findPrevFragmentSet() + } + + // The heap root is now the smallest boundary key b such that: + // 1. b > k + // 2. b = k, and b is a start boundary + // There's a third case that we will need to consider later, after we've + // switched to a max heap: + // 3. there exists an end boundary key b such that b = k. + // An end boundary key equal to k would not be surfaced when we seeked all + // the levels using SeekGE(k), since k would not be contained within the + // exclusive end boundary. + // + // Assume that the tightest boundary ≥ k is the current heap root (cases 1 & + // 2). After we switch to a max heap, we'll check for the third case and + // adjust the end boundary if necessary. + m.end = m.heap.items[0].boundKey.key + + // Before switching the direction of the heap, save a copy of the end + // boundary if it's the start boundary of some child span. Prev-ing the + // child iterator might switch files and invalidate the memory of the bound. + if m.heap.items[0].boundKey.kind == boundKindFragmentStart { + m.buf = append(m.buf[:0], m.end...) + m.end = m.buf + } + + // Switch to a max heap. This will move each level to the previous bound in + // every level, and then establish a max heap. This allows us to obtain the + // largest boundary key < `key`, which will serve as our candidate start + // bound. + m.switchToMaxHeap() + if m.err != nil { + return nil + } else if len(m.heap.items) == 0 { + return nil + } + // Check for the case 3 described above. It's possible that when we switch + // heap directions, we discover an end boundary of some child span that is + // equal to the seek key `key`. In this case, we want this key to be our end + // boundary. + if m.heap.items[0].boundKey.kind == boundKindFragmentEnd && + m.cmp(m.heap.items[0].boundKey.key, key) == 0 { + // Call findPrevFragmentSet, which will set m.end to the heap root and + // proceed backwards. + return m.findPrevFragmentSet() + } + + m.start = m.heap.items[0].boundKey.key + if found, s := m.synthesizeKeys(-1); found && s != nil { + return s + } + return m.findPrevFragmentSet() } // First seeks the iterator to the first span. @@ -823,8 +1044,8 @@ func (m *MergingIter) synthesizeKeys(dir int8) (bool, *Span) { // sorted by trailer descending for the range key iteration use case. sort.Sort(&m.keys) - // Apply the configured transform. See visibleTransform. - s := Span{ + // Apply the configured transform. See VisibleTransform. + m.span = Span{ Start: m.start, End: m.end, Keys: m.keys, @@ -832,7 +1053,7 @@ func (m *MergingIter) synthesizeKeys(dir int8) (bool, *Span) { } // NB: m.heap.cmp is a base.Compare, whereas m.cmp is a method on // MergingIter. - if err := m.transformer.Transform(m.heap.cmp, s, &m.span); err != nil { + if err := m.transformer.Transform(m.heap.cmp, m.span, &m.span); err != nil { m.err = err return false, nil } diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go index 59a4884..efcf682 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/seek.go @@ -6,34 +6,6 @@ package keyspan import "github.com/cockroachdb/pebble/internal/base" -// SeekGE seeks to the span that contains the target key or the first span past -// the target key. -func SeekGE(cmp base.Compare, iter FragmentIterator, key []byte) *Span { - // NB: We use SeekLT in order to land on the proper span for a search - // key that resides in the middle of a span. Consider the scenario: - // - // a---e - // e---i - // - // The spans are indexed by their start keys `a` and `e`. If the - // search key is `c` we want to land on the span [a,e). If we were to - // use SeekGE then the search key `c` would land on the span [e,i) and - // we'd have to backtrack. The one complexity here is what happens for the - // search key `e`. In that case SeekLT will land us on the span [a,e) - // and we'll have to move forward. - iterSpan := iter.SeekLT(key) - - // Invariant: key > iterSpan.Start - - if iterSpan == nil || cmp(key, iterSpan.End) >= 0 { - // The current span lies entirely before the search key, or the iterator - // is exhausted. Advance the iterator to the next span which is - // guaranteed to lie at or past the search key. - iterSpan = iter.Next() - } - return iterSpan -} - // SeekLE seeks to the span that contains or is before the target key. func SeekLE(cmp base.Compare, iter FragmentIterator, key []byte) *Span { // NB: We use SeekLT in order to land on the proper span for a search diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go index 755cbfc..92694d5 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/span.go @@ -24,7 +24,8 @@ import ( // Note that the start user key is inclusive and the end user key is exclusive. // // Currently the only supported key kinds are: -// RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL. +// +// RANGEDEL, RANGEKEYSET, RANGEKEYUNSET, RANGEKEYDEL. type Span struct { // Start and End encode the user key range of all the contained items, with // an inclusive start key and exclusive end key. Both Start and End must be @@ -75,6 +76,15 @@ func (k Key) SeqNum() uint64 { return k.Trailer >> 8 } +// VisibleAt returns true if the provided key is visible at the provided +// snapshot sequence number. It interprets batch sequence numbers as always +// visible, because non-visible batch span keys are filtered when they're +// fragmented. +func (k Key) VisibleAt(snapshot uint64) bool { + seq := k.SeqNum() + return seq < snapshot || seq&base.InternalKeySeqNumBatch != 0 +} + // Kind returns the kind component of the key. func (k Key) Kind() base.InternalKeyKind { return base.InternalKeyKind(k.Trailer & 0xff) @@ -396,6 +406,17 @@ func SortKeysByTrailer(keys *[]Key) { sort.Sort(sorted) } +// KeysBySuffix implements sort.Interface, sorting its member Keys slice to by +// Suffix in the order dictated by Cmp. +type KeysBySuffix struct { + Cmp base.Compare + Keys []Key +} + +func (s *KeysBySuffix) Len() int { return len(s.Keys) } +func (s *KeysBySuffix) Less(i, j int) bool { return s.Cmp(s.Keys[i].Suffix, s.Keys[j].Suffix) < 0 } +func (s *KeysBySuffix) Swap(i, j int) { s.Keys[i], s.Keys[j] = s.Keys[j], s.Keys[i] } + // ParseSpan parses the string representation of a Span. It's intended for // tests. ParseSpan panics if passed a malformed span representation. func ParseSpan(input string) Span { diff --git a/vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go b/vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go index af3dddd..79d5cf6 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go +++ b/vendor/github.com/cockroachdb/pebble/internal/keyspan/truncate.go @@ -10,13 +10,18 @@ import "github.com/cockroachdb/pebble/internal/base" // truncated to be contained within the range [lower, upper). If start and end // are specified, filter out any spans that are completely outside those bounds. func Truncate( - cmp base.Compare, iter FragmentIterator, lower, upper []byte, start, end *base.InternalKey, + cmp base.Compare, + iter FragmentIterator, + lower, upper []byte, + start, end *base.InternalKey, + panicOnPartialOverlap bool, ) FragmentIterator { return Filter(iter, func(in *Span, out *Span) (keep bool) { out.Start, out.End = in.Start, in.End out.Keys = append(out.Keys[:0], in.Keys...) - // Ignore this span if it lies completely outside [start, end]. + // Ignore this span if it lies completely outside start, end. Note that + // end endInclusive indicated whether end is inclusive. // // The comparison between s.End and start is by user key only, as // the span is exclusive at s.End, so comparing by user keys @@ -48,13 +53,22 @@ func Truncate( // Wholly within the end bound. Keep it. } } + + var truncated bool // Truncate the bounds to lower and upper. if cmp(in.Start, lower) < 0 { + truncated = true out.Start = lower } if cmp(in.End, upper) > 0 { + truncated = true out.End = upper } + + if panicOnPartialOverlap && truncated { + panic("pebble: bounds should not be truncated") + } + return !out.Empty() && cmp(out.Start, out.End) < 0 }) } diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go index 13ccaf5..1a99de7 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/btree.go @@ -12,6 +12,7 @@ import ( "unsafe" "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/invariants" ) // The Annotator type defined below is used by other packages to lazily @@ -110,7 +111,16 @@ type leafNode struct { ref int32 count int16 leaf bool - items [maxItems]*FileMetadata + // subtreeCount holds the count of files in the entire subtree formed by + // this node. For leaf nodes, subtreeCount is always equal to count. For + // non-leaf nodes, it's the sum of count plus all the children's + // subtreeCounts. + // + // NB: We could move this field to the end of the node struct, since leaf => + // count=subtreeCount, however the unsafe casting [leafToNode] performs make + // it risky and cumbersome. + subtreeCount int + items [maxItems]*FileMetadata // annot contains one annotation per annotator, merged over the entire // node's files (and all descendants for non-leaf nodes). annot []annotation @@ -167,7 +177,7 @@ func mut(n **node) *node { // reference count to be greater than 1, we might be racing // with another call to decRef on this node. c := (*n).clone() - (*n).decRef(true /* recursive */, nil) + (*n).decRef(true /* contentsToo */, nil) *n = c // NB: We don't need to clear annotations, because (*node).clone does not // copy them. @@ -179,19 +189,25 @@ func (n *node) incRef() { atomic.AddInt32(&n.ref, 1) } -// decRef releases a reference to the node. If requested, the method -// will recurse into child nodes and decrease their refcounts as well. +// decRef releases a reference to the node. If requested, the method will unref +// its items and recurse into child nodes and decrease their refcounts as well. +// Some internal codepaths that manually copy the node's items or children to +// new nodes pass contentsToo=false to preserve existing reference counts during +// operations that should yield a net-zero change to descendant refcounts. // When a node is released, its contained files are dereferenced. -func (n *node) decRef(recursive bool, obsolete *[]*FileMetadata) { +func (n *node) decRef(contentsToo bool, obsolete *[]*FileMetadata) { if atomic.AddInt32(&n.ref, -1) > 0 { // Other references remain. Can't free. return } - // Dereference the node's metadata and release child references. - if recursive { + // Dereference the node's metadata and release child references if + // requested. Some internal callers may not want to propagate the deref + // because they're manually copying the filemetadata and children to other + // nodes, and they want to preserve the existing reference count. + if contentsToo { for _, f := range n.items[:n.count] { - if atomic.AddInt32(&f.refs, -1) == 0 { + if f.Unref() == 0 { // There are two sources of node dereferences: tree mutations // and Version dereferences. Files should only be made obsolete // during Version dereferences, during which `obsolete` will be @@ -204,7 +220,7 @@ func (n *node) decRef(recursive bool, obsolete *[]*FileMetadata) { } if !n.leaf { for i := int16(0); i <= n.count; i++ { - n.children[i].decRef(true /* recursive */, obsolete) + n.children[i].decRef(true /* contentsToo */, obsolete) } } } @@ -222,9 +238,10 @@ func (n *node) clone() *node { // triggering the race detector and looking like a data race. c.count = n.count c.items = n.items + c.subtreeCount = n.subtreeCount // Increase the refcount of each contained item. for _, f := range n.items[:n.count] { - atomic.AddInt32(&f.refs, 1) + f.Ref() } if !c.leaf { // Copy children and increase each refcount. @@ -236,6 +253,9 @@ func (n *node) clone() *node { return c } +// insertAt inserts the provided file and node at the provided index. This +// function is for use only as a helper function for internal B-Tree code. +// Clients should not invoke it directly. func (n *node) insertAt(index int, item *FileMetadata, nd *node) { if index < int(n.count) { copy(n.items[index+1:n.count+1], n.items[index:n.count]) @@ -250,6 +270,9 @@ func (n *node) insertAt(index int, item *FileMetadata, nd *node) { n.count++ } +// pushBack inserts the provided file and node at the tail of the node's items. +// This function is for use only as a helper function for internal B-Tree code. +// Clients should not invoke it directly. func (n *node) pushBack(item *FileMetadata, nd *node) { n.items[n.count] = item if !n.leaf { @@ -258,6 +281,9 @@ func (n *node) pushBack(item *FileMetadata, nd *node) { n.count++ } +// pushFront inserts the provided file and node at the head of the +// node's items. This function is for use only as a helper function for internal B-Tree +// code. Clients should not invoke it directly. func (n *node) pushFront(item *FileMetadata, nd *node) { if !n.leaf { copy(n.children[1:n.count+2], n.children[:n.count+1]) @@ -269,7 +295,8 @@ func (n *node) pushFront(item *FileMetadata, nd *node) { } // removeAt removes a value at a given index, pulling all subsequent values -// back. +// back. This function is for use only as a helper function for internal B-Tree +// code. Clients should not invoke it directly. func (n *node) removeAt(index int) (*FileMetadata, *node) { var child *node if !n.leaf { @@ -284,7 +311,9 @@ func (n *node) removeAt(index int) (*FileMetadata, *node) { return out, child } -// popBack removes and returns the last element in the list. +// popBack removes and returns the last element in the list. This function is +// for use only as a helper function for internal B-Tree code. Clients should +// not invoke it directly. func (n *node) popBack() (*FileMetadata, *node) { n.count-- out := n.items[n.count] @@ -297,7 +326,9 @@ func (n *node) popBack() (*FileMetadata, *node) { return out, child } -// popFront removes and returns the first element in the list. +// popFront removes and returns the first element in the list. This function is +// for use only as a helper function for internal B-Tree code. Clients should +// not invoke it directly. func (n *node) popFront() (*FileMetadata, *node) { n.count-- var child *node @@ -315,6 +346,9 @@ func (n *node) popFront() (*FileMetadata, *node) { // find returns the index where the given item should be inserted into this // list. 'found' is true if the item already exists in the list at the given // index. +// +// This function is for use only as a helper function for internal B-Tree code. +// Clients should not invoke it directly. func (n *node) find(cmp btreeCmp, item *FileMetadata) (index int, found bool) { // Logic copied from sort.Search. Inlining this gave // an 11% speedup on BenchmarkBTreeDeleteInsert. @@ -338,23 +372,34 @@ func (n *node) find(cmp btreeCmp, item *FileMetadata) (index int, found bool) { // and this function returns the item that existed at that index and a new // node containing all items/children after it. // -// Before: +// split is called when we want to perform a transformation like the one +// depicted in the following diagram. // -// +-----------+ -// | x y z | -// +--/-/-\-\--+ +// Before: +// +-----------+ +// n *node | x y z | +// +--/-/-\-\--+ // -// After: +// After: +// +-----------+ +// | y | n's parent +// +----/-\----+ +// / \ +// v v +// +-----------+ +-----------+ +// n *node | x | | z | next *node +// +-----------+ +-----------+ // -// +-----------+ -// | y | -// +----/-\----+ -// / \ -// v v -// +-----------+ +-----------+ -// | x | | z | -// +-----------+ +-----------+ +// split does not perform the complete transformation; the caller is responsible +// for updating the parent appropriately. split splits `n` into two nodes, `n` +// and `next`, returning `next` and the file that separates them. In the diagram +// above, `n.split` removes y and z from `n`, returning y in the first return +// value and `next` in the second return value. The caller is responsible for +// updating n's parent to now contain `y` as the separator between nodes `n` and +// `next`. // +// This function is for use only as a helper function for internal B-Tree code. +// Clients should not invoke it directly. func (n *node) split(i int) (*FileMetadata, *node) { out := n.items[i] var next *node @@ -370,17 +415,27 @@ func (n *node) split(i int) (*FileMetadata, *node) { } if !n.leaf { copy(next.children[:], n.children[i+1:n.count+1]) + descendantsMoved := 0 for j := int16(i + 1); j <= n.count; j++ { + descendantsMoved += n.children[j].subtreeCount n.children[j] = nil } + n.subtreeCount -= descendantsMoved + next.subtreeCount += descendantsMoved } n.count = int16(i) + // NB: We subtract one more than `next.count` from n's subtreeCount because + // the item at index `i` was removed from `n.items`. We'll return the item + // at index `i`, and the caller is responsible for updating the subtree + // count of whichever node adopts it. + n.subtreeCount -= int(next.count) + 1 + next.subtreeCount += int(next.count) return out, next } -// insert inserts a item into the subtree rooted at this node, making sure no +// Insert inserts a item into the subtree rooted at this node, making sure no // nodes in the subtree exceed maxItems items. -func (n *node) insert(cmp btreeCmp, item *FileMetadata) error { +func (n *node) Insert(cmp btreeCmp, item *FileMetadata) error { i, found := n.find(cmp, item) if found { // cmp provides a total ordering of the files within a level. @@ -391,6 +446,7 @@ func (n *node) insert(cmp btreeCmp, item *FileMetadata) error { } if n.leaf { n.insertAt(i, item, nil) + n.subtreeCount++ return nil } if n.children[i].count >= maxItems { @@ -410,14 +466,21 @@ func (n *node) insert(cmp btreeCmp, item *FileMetadata) error { errors.Safe(item.FileNum), errors.Safe(n.items[i].FileNum)) } } - return mut(&n.children[i]).insert(cmp, item) + + err := mut(&n.children[i]).Insert(cmp, item) + if err == nil { + n.subtreeCount++ + } + return err } -// removeMax removes and returns the maximum item from the subtree rooted -// at this node. +// removeMax removes and returns the maximum item from the subtree rooted at +// this node. This function is for use only as a helper function for internal +// B-Tree code. Clients should not invoke it directly. func (n *node) removeMax() *FileMetadata { if n.leaf { n.count-- + n.subtreeCount-- out := n.items[n.count] n.items[n.count] = nil return out @@ -427,16 +490,18 @@ func (n *node) removeMax() *FileMetadata { n.rebalanceOrMerge(int(n.count)) return n.removeMax() } + n.subtreeCount-- return child.removeMax() } -// remove removes a item from the subtree rooted at this node. Returns +// Remove removes a item from the subtree rooted at this node. Returns // the item that was removed or nil if no matching item was found. -func (n *node) remove(cmp btreeCmp, item *FileMetadata) (out *FileMetadata) { +func (n *node) Remove(cmp btreeCmp, item *FileMetadata) (out *FileMetadata) { i, found := n.find(cmp, item) if n.leaf { if found { out, _ = n.removeAt(i) + n.subtreeCount-- return out } return nil @@ -444,22 +509,28 @@ func (n *node) remove(cmp btreeCmp, item *FileMetadata) (out *FileMetadata) { if n.children[i].count <= minItems { // Child not large enough to remove from. n.rebalanceOrMerge(i) - return n.remove(cmp, item) + return n.Remove(cmp, item) } child := mut(&n.children[i]) if found { // Replace the item being removed with the max item in our left child. out = n.items[i] n.items[i] = child.removeMax() + n.subtreeCount-- return out } - // Latch is not in this node and child is large enough to remove from. - out = child.remove(cmp, item) + // File is not in this node and child is large enough to remove from. + out = child.Remove(cmp, item) + if out != nil { + n.subtreeCount-- + } return out } -// rebalanceOrMerge grows child 'i' to ensure it has sufficient room to remove -// a item from it while keeping it at or above minItems. +// rebalanceOrMerge grows child 'i' to ensure it has sufficient room to remove a +// item from it while keeping it at or above minItems. This function is for use +// only as a helper function for internal B-Tree code. Clients should not invoke +// it directly. func (n *node) rebalanceOrMerge(i int) { switch { case i > 0 && n.children[i-1].count > minItems: @@ -497,6 +568,12 @@ func (n *node) rebalanceOrMerge(i int) { yLa := n.items[i-1] child.pushFront(yLa, grandChild) n.items[i-1] = xLa + child.subtreeCount++ + left.subtreeCount-- + if grandChild != nil { + child.subtreeCount += grandChild.subtreeCount + left.subtreeCount -= grandChild.subtreeCount + } case i < int(n.count) && n.children[i+1].count > minItems: // Rebalance from right sibling. @@ -532,6 +609,12 @@ func (n *node) rebalanceOrMerge(i int) { xLa, grandChild := right.popFront() yLa := n.items[i] child.pushBack(yLa, grandChild) + child.subtreeCount++ + right.subtreeCount-- + if grandChild != nil { + child.subtreeCount += grandChild.subtreeCount + right.subtreeCount -= grandChild.subtreeCount + } n.items[i] = xLa default: @@ -570,12 +653,15 @@ func (n *node) rebalanceOrMerge(i int) { copy(child.children[child.count+1:], mergeChild.children[:mergeChild.count+1]) } child.count += mergeChild.count + 1 + child.subtreeCount += mergeChild.subtreeCount + 1 - mergeChild.decRef(false /* recursive */, nil) + mergeChild.decRef(false /* contentsToo */, nil) } } -func (n *node) invalidateAnnotation(a Annotator) { +// InvalidateAnnotation removes any existing cached annotations for the provided +// annotator from this node's subtree. +func (n *node) InvalidateAnnotation(a Annotator) { // Find this annotator's annotation on this node. var annot *annotation for i := range n.annot { @@ -590,12 +676,17 @@ func (n *node) invalidateAnnotation(a Annotator) { } if !n.leaf { for i := int16(0); i <= n.count; i++ { - n.children[i].invalidateAnnotation(a) + n.children[i].InvalidateAnnotation(a) } } } -func (n *node) annotation(a Annotator) (interface{}, bool) { +// Annotation retrieves, computing if not already computed, the provided +// annotator's annotation of this node. The second return value indicates +// whether the future reads of this annotation may use the first return value +// as-is. If false, the annotation is not stable and may change on a subsequent +// computation. +func (n *node) Annotation(a Annotator) (interface{}, bool) { // Find this annotator's annotation on this node. var annot *annotation for i := range n.annot { @@ -627,7 +718,7 @@ func (n *node) annotation(a Annotator) (interface{}, bool) { annot.valid = true for i := int16(0); i <= n.count; i++ { if !n.leaf { - v, ok := n.children[i].annotation(a) + v, ok := n.children[i].Annotation(a) annot.v = a.Merge(v, annot.v) annot.valid = annot.valid && ok } @@ -640,6 +731,20 @@ func (n *node) annotation(a Annotator) (interface{}, bool) { return annot.v, annot.valid } +func (n *node) verifyInvariants() { + recomputedSubtreeCount := int(n.count) + if !n.leaf { + for i := int16(0); i <= n.count; i++ { + n.children[i].verifyInvariants() + recomputedSubtreeCount += n.children[i].subtreeCount + } + } + if recomputedSubtreeCount != n.subtreeCount { + panic(fmt.Sprintf("recomputed subtree count (%d) ≠ n.subtreeCount (%d)", + recomputedSubtreeCount, n.subtreeCount)) + } +} + // btree is an implementation of a B-Tree. // // btree stores FileMetadata in an ordered structure, allowing easy insertion, @@ -650,25 +755,23 @@ func (n *node) annotation(a Annotator) (interface{}, bool) { // Write operations are not safe for concurrent mutation by multiple // goroutines, but Read operations are. type btree struct { - root *node - length int - cmp btreeCmp + root *node + cmp btreeCmp } -// release dereferences and clears the root node of the btree, removing all +// Release dereferences and clears the root node of the btree, removing all // items from the btree. In doing so, it decrements contained file counts. // It returns a slice of newly obsolete files, if any. -func (t *btree) release() (obsolete []*FileMetadata) { +func (t *btree) Release() (obsolete []*FileMetadata) { if t.root != nil { - t.root.decRef(true /* recursive */, &obsolete) + t.root.decRef(true /* contentsToo */, &obsolete) t.root = nil } - t.length = 0 return obsolete } -// clone clones the btree, lazily. It does so in constant time. -func (t *btree) clone() btree { +// Clone clones the btree, lazily. It does so in constant time. +func (t *btree) Clone() btree { c := *t if c.root != nil { // Incrementing the reference count on the root node is sufficient to @@ -690,15 +793,17 @@ func (t *btree) clone() btree { return c } -// delete removes the provided file from the tree. +// Delete removes the provided file from the tree. // It returns true if the file now has a zero reference count. -func (t *btree) delete(item *FileMetadata) (obsolete bool) { +func (t *btree) Delete(item *FileMetadata) (obsolete bool) { if t.root == nil || t.root.count == 0 { return false } - if out := mut(&t.root).remove(t.cmp, item); out != nil { - t.length-- - obsolete = atomic.AddInt32(&out.refs, -1) == 0 + if out := mut(&t.root).Remove(t.cmp, item); out != nil { + obsolete = out.Unref() == 0 + } + if invariants.Enabled { + t.root.verifyInvariants() } if t.root.count == 0 { old := t.root @@ -707,14 +812,14 @@ func (t *btree) delete(item *FileMetadata) (obsolete bool) { } else { t.root = t.root.children[0] } - old.decRef(false /* recursive */, nil) + old.decRef(false /* contentsToo */, nil) } return obsolete } -// insert adds the given item to the tree. If a item in the tree already -// equals the given one, insert panics. -func (t *btree) insert(item *FileMetadata) error { +// Insert adds the given item to the tree. If a item in the tree already +// equals the given one, Insert panics. +func (t *btree) Insert(item *FileMetadata) error { if t.root == nil { t.root = newLeafNode() } else if t.root.count >= maxItems { @@ -724,39 +829,36 @@ func (t *btree) insert(item *FileMetadata) error { newRoot.items[0] = splitLa newRoot.children[0] = t.root newRoot.children[1] = splitNode + newRoot.subtreeCount = t.root.subtreeCount + splitNode.subtreeCount + 1 t.root = newRoot } - atomic.AddInt32(&item.refs, 1) - err := mut(&t.root).insert(t.cmp, item) - t.length++ + item.Ref() + err := mut(&t.root).Insert(t.cmp, item) + if invariants.Enabled { + t.root.verifyInvariants() + } return err } -// iter returns a new iterator object. It is not safe to continue using an +// Iter returns a new iterator object. It is not safe to continue using an // iterator after modifications are made to the tree. If modifications are made, // create a new iterator. -func (t *btree) iter() iterator { +func (t *btree) Iter() iterator { return iterator{r: t.root, pos: -1, cmp: t.cmp} } -// height returns the height of the tree. -func (t *btree) height() int { +// Count returns the number of files contained within the B-Tree. +func (t *btree) Count() int { if t.root == nil { return 0 } - h := 1 - n := t.root - for !n.leaf { - n = n.children[0] - h++ - } - return h + return t.root.subtreeCount } // String returns a string description of the tree. The format is // similar to the https://en.wikipedia.org/wiki/Newick_format. func (t *btree) String() string { - if t.length == 0 { + if t.Count() == 0 { return ";" } var b strings.Builder @@ -873,6 +975,8 @@ type iterator struct { r *node // n and pos make up the current position of the iterator. // If valid, n.items[pos] is the current value of the iterator. + // + // n may be nil iff i.r is nil. n *node pos int16 // cmp dictates the ordering of the FileMetadata. @@ -883,6 +987,59 @@ type iterator struct { s iterStack } +// countLeft returns the count of files that are to the left of the current +// iterator position. +func (i *iterator) countLeft() int { + if i.r == nil { + return 0 + } + + // Each iterator has a stack of frames marking the path from the root node + // to the current iterator position. All files (n.items) and all subtrees + // (n.children) with indexes less than [pos] are to the left of the current + // iterator position. + // + // +------------------------+ - + // | Root pos:5 | | + // +------------------------+ | stack + // | Root/5 pos:3 | | frames + // +------------------------+ | [i.s] + // | Root/5/3 pos:9 | | + // +========================+ - + // | | + // | i.n: Root/5/3/9 i.pos:2| + // +------------------------+ + // + var count int + // Walk all the ancestors in the iterator stack [i.s], tallying up all the + // files and subtrees to the left of the stack frame's position. + f, ok := i.s.nth(0) + for fi := 0; ok; fi++ { + // There are [f.pos] files contained within [f.n.items] that sort to the + // left of the subtree the iterator has descended. + count += int(f.pos) + // Any subtrees that fall before the stack frame's position are entirely + // to the left of the iterator's current position. + for j := int16(0); j < f.pos; j++ { + count += f.n.children[j].subtreeCount + } + f, ok = i.s.nth(fi + 1) + } + + // The bottommost stack frame is inlined within the iterator struct. Again, + // [i.pos] files fall to the left of the current iterator position. + count += int(i.pos) + if !i.n.leaf { + // NB: Unlike above, we use a `<= i.pos` comparison. The iterator is + // positioned at item `i.n.items[i.pos]`, which sorts after everything + // in the subtree at `i.n.children[i.pos]`. + for j := int16(0); j <= i.pos; j++ { + count += i.n.children[j].subtreeCount + } + } + return count +} + func (i *iterator) clone() iterator { c := *i c.s = i.s.clone() @@ -904,7 +1061,7 @@ func (i iterator) String() string { } fmt.Fprintf(&buf, "%p: %02d/%02d\n", f.n, f.pos, f.n.count) } - if i.n == nil { + if i.r == nil { fmt.Fprintf(&buf, ": %02d", i.pos) } else { fmt.Fprintf(&buf, "%p: %02d/%02d", i.n, i.pos, i.n.count) @@ -1034,7 +1191,7 @@ func (i *iterator) ascend() { // tree's files, and then true for the (possibly empty) remainder. func (i *iterator) seek(fn func(*FileMetadata) bool) { i.reset() - if i.n == nil { + if i.r == nil { return } @@ -1066,7 +1223,7 @@ func (i *iterator) seek(fn func(*FileMetadata) bool) { // first seeks to the first item in the btree. func (i *iterator) first() { i.reset() - if i.n == nil { + if i.r == nil { return } for !i.n.leaf { @@ -1078,7 +1235,7 @@ func (i *iterator) first() { // last seeks to the last item in the btree. func (i *iterator) last() { i.reset() - if i.n == nil { + if i.r == nil { return } for !i.n.leaf { @@ -1090,7 +1247,7 @@ func (i *iterator) last() { // next positions the iterator to the item immediately following // its current position. func (i *iterator) next() { - if i.n == nil { + if i.r == nil { return } @@ -1117,7 +1274,7 @@ func (i *iterator) next() { // prev positions the iterator to the item immediately preceding // its current position. func (i *iterator) prev() { - if i.n == nil { + if i.r == nil { return } @@ -1148,5 +1305,8 @@ func (i *iterator) valid() bool { // cur returns the item at the iterator's current position. It is illegal // to call cur if the iterator is not valid. func (i *iterator) cur() *FileMetadata { + if invariants.Enabled && !i.valid() { + panic("btree iterator.cur invoked on invalid iterator") + } return i.n.items[i.pos] } diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go index a7b1e01..3646a03 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/l0_sublevels.go @@ -112,13 +112,15 @@ func sortAndSweep(keys []intervalKeyTemp, cmp Compare) []intervalKeyTemp { sorter := intervalKeySorter{keys: keys, cmp: cmp} sort.Sort(sorter) - // intervalKeys are generated using the file bounds. Specifically, there are 2 intervalKeys - // for each file, and len(keys) = 2 * number of files. Each intervalKeyTemp stores information - // about which file it was generated from, and whether the key represents the end key of the file. - // So, as we're deduplicating the `keys` slice, we're guaranteed to iterate over the interval - // keys belonging to each of the files. Since the file.{min,max}IntervalIndex points to the position - // of the files bounds in the deduplicated `keys` slice, we can determine file.{min,max}IntervalIndex - // during the iteration. + // intervalKeys are generated using the file bounds. Specifically, there are + // 2 intervalKeys for each file, and len(keys) = 2 * number of files. Each + // `intervalKeyTemp` stores information about which file it was generated + // from, and whether the key represents the end key of the file. So, as + // we're deduplicating the `keys` slice, we're guaranteed to iterate over + // the interval keys belonging to each of the files. Since the + // file.{min,max}IntervalIndex points to the position of the files bounds in + // the deduplicated `keys` slice, we can determine + // file.{min,max}IntervalIndex during the iteration. i := 0 j := 0 for i < len(keys) { @@ -151,24 +153,24 @@ type fileInterval struct { // cannot have any files participate in L0 -> Lbase compactions. isBaseCompacting bool - // The min and max intervals index across all the files that overlap with this - // interval. Inclusive on both sides. + // The min and max intervals index across all the files that overlap with + // this interval. Inclusive on both sides. filesMinIntervalIndex int filesMaxIntervalIndex int // True if another interval that has a file extending into this interval is - // undergoing a compaction into Lbase. In other words, this bool is true - // if any interval in [filesMinIntervalIndex, - // filesMaxIntervalIndex] has isBaseCompacting set to true. This - // lets the compaction picker de-prioritize this interval for picking - // compactions, since there's a high chance that a base compaction with a - // sufficient height of sublevels rooted at this interval could not be - // chosen due to the ongoing base compaction in the - // other interval. If the file straddling the two intervals is at a - // sufficiently high sublevel (with enough compactible files below it to - // satisfy minCompactionDepth), this is not an issue, but to optimize for - // quickly picking base compactions far away from other base compactions, - // this bool is used as a heuristic (but not as a complete disqualifier). + // undergoing a compaction into Lbase. In other words, this bool is true if + // any interval in [filesMinIntervalIndex, filesMaxIntervalIndex] has + // isBaseCompacting set to true. This lets the compaction picker + // de-prioritize this interval for picking compactions, since there's a high + // chance that a base compaction with a sufficient height of sublevels + // rooted at this interval could not be chosen due to the ongoing base + // compaction in the other interval. If the file straddling the two + // intervals is at a sufficiently high sublevel (with enough compactible + // files below it to satisfy minCompactionDepth), this is not an issue, but + // to optimize for quickly picking base compactions far away from other base + // compactions, this bool is used as a heuristic (but not as a complete + // disqualifier). intervalRangeIsBaseCompacting bool // All files in this interval, in increasing sublevel order. @@ -178,9 +180,8 @@ type fileInterval struct { // starting new compactions. This metric is not precise since the // compactingFileCount can include files that are part of N (where N > 1) // intra-L0 compactions, so the stack depth after those complete will be - // len(files) - compactingFileCount + N. We ignore this imprecision since - // we don't want to track which files are part of which intra-L0 - // compaction. + // len(files) - compactingFileCount + N. We ignore this imprecision since we + // don't want to track which files are part of which intra-L0 compaction. compactingFileCount int // Interpolated from files in this interval. For files spanning multiple @@ -332,17 +333,17 @@ func NewL0Sublevels( for _, sublevelFiles := range s.levelFiles { tr, ls := makeBTree(btreeCmpSmallestKey(cmp), sublevelFiles) s.Levels = append(s.Levels, ls) - tr.release() + tr.Release() } s.calculateFlushSplitKeys(flushSplitMaxBytes) return s, nil } -// Helper function to merge new intervalKeys into an existing slice -// of old fileIntervals, into result. Returns the new result and a slice of ints -// mapping old interval indices to new ones. The added intervalKeys do not -// need to be sorted; they get sorted and deduped in this function. +// Helper function to merge new intervalKeys into an existing slice of old +// fileIntervals, into result. Returns the new result and a slice of ints +// mapping old interval indices to new ones. The added intervalKeys do not need +// to be sorted; they get sorted and deduped in this function. func mergeIntervals( old, result []fileInterval, added []intervalKeyTemp, compare Compare, ) ([]fileInterval, []int) { @@ -413,35 +414,36 @@ func mergeIntervals( return result, oldToNewMap } -// AddL0Files incrementally builds a new L0Sublevels for when the only -// change since the receiver L0Sublevels was an addition of the specified files, -// with no L0 deletions. The common case of this is an ingestion or a flush. -// These files can "sit on top" of existing sublevels, creating at most one -// new sublevel for a flush (and possibly multiple for an ingestion), and at -// most 2*len(files) additions to s.orderedIntervals. No files must have been -// deleted from L0, and the added files must all be newer in sequence numbers -// than existing files in L0Sublevels. The files parameter must be sorted in -// seqnum order. The levelMetadata parameter corresponds to the new L0 post -// addition of files. This method is meant to be significantly more performant -// than NewL0Sublevels. +// AddL0Files incrementally builds a new L0Sublevels for when the only change +// since the receiver L0Sublevels was an addition of the specified files, with +// no L0 deletions. The common case of this is an ingestion or a flush. These +// files can "sit on top" of existing sublevels, creating at most one new +// sublevel for a flush (and possibly multiple for an ingestion), and at most +// 2*len(files) additions to s.orderedIntervals. No files must have been deleted +// from L0, and the added files must all be newer in sequence numbers than +// existing files in L0Sublevels. The files parameter must be sorted in seqnum +// order. The levelMetadata parameter corresponds to the new L0 post addition of +// files. This method is meant to be significantly more performant than +// NewL0Sublevels. // // Note that this function can only be called once on a given receiver; it // appends to some slices in s which is only safe when done once. This is okay, // as the common case (generating a new L0Sublevels after a flush/ingestion) is // only going to necessitate one call of this method on a given receiver. The -// returned value, if non-nil, can then have AddL0Files called on it again, and -// so on. If errInvalidL0SublevelsOpt is returned as an error, it likely means -// the optimization could not be applied (i.e. files added were older than files -// already in the sublevels, which is possible around ingestions and in tests). -// Eg. it can happen when an ingested file was ingested without queueing a flush -// since it did not actually overlap with any keys in the memtable. Later on the -// memtable was flushed, and the memtable had keys spanning around the ingested -// file, producing a flushed file that overlapped with the ingested file in file -// bounds but not in keys. It's possible for that flushed file to have a lower -// LargestSeqNum than the ingested file if all the additions after the ingestion -// were to another flushed file that was split into a separate sstable during -// flush. Any other non-nil error means L0Sublevels generation failed in the same -// way as NewL0Sublevels would likely fail. +// returned value, if non-nil, can then have [*L0Sublevels.AddL0Files] called on +// it again, and so on. If [errInvalidL0SublevelsOpt] is returned as an error, +// it likely means the optimization could not be applied (i.e. files added were +// older than files already in the sublevels, which is possible around +// ingestions and in tests). Eg. it can happen when an ingested file was +// ingested without queueing a flush since it did not actually overlap with any +// keys in the memtable. Later on the memtable was flushed, and the memtable had +// keys spanning around the ingested file, producing a flushed file that +// overlapped with the ingested file in file bounds but not in keys. It's +// possible for that flushed file to have a lower LargestSeqNum than the +// ingested file if all the additions after the ingestion were to another +// flushed file that was split into a separate sstable during flush. Any other +// non-nil error means [L0Sublevels] generation failed in the same way as +// [NewL0Sublevels] would likely fail. func (s *L0Sublevels) AddL0Files( files []*FileMetadata, flushSplitMaxBytes int64, levelMetadata *LevelMetadata, ) (*L0Sublevels, error) { @@ -485,8 +487,8 @@ func (s *L0Sublevels) AddL0Files( keys := make([]fileInterval, 0, 2*levelMetadata.Len()) var oldToNewMap []int // We can avoid the sortAndSweep step on the combined length of - // s.orderedIntervals and fileKeys by treating this as a merge of two - // sorted runs, fileKeys and s.orderedIntervals, into `keys` which will form + // s.orderedIntervals and fileKeys by treating this as a merge of two sorted + // runs, fileKeys and s.orderedIntervals, into `keys` which will form // newVal.orderedIntervals. keys, oldToNewMap = mergeIntervals(s.orderedIntervals, keys, fileKeys, s.cmp) if invariants.Enabled { @@ -502,20 +504,21 @@ func (s *L0Sublevels) AddL0Files( newInterval := &keys[newIdx] newInterval.index = newIdx // This code, and related code in the for loop below, adjusts - // files{Min,Max}IntervalIndex just for interval indices shifting due to new - // intervals, and not for any of the new files being added to the same - // intervals. The goal is to produce a state of the system that's accurate - // for all existing files, and has all the new intervals to support new - // files. Once that's done, we can just call addFileToSublevel to adjust - // all relevant intervals for new files. + // files{Min,Max}IntervalIndex just for interval indices shifting due to + // new intervals, and not for any of the new files being added to the + // same intervals. The goal is to produce a state of the system that's + // accurate for all existing files, and has all the new intervals to + // support new files. Once that's done, we can just call + // addFileToSublevel to adjust all relevant intervals for new files. newInterval.filesMinIntervalIndex = oldToNewMap[newInterval.filesMinIntervalIndex] // maxIntervalIndexes are special. Since it's an inclusive end bound, we // actually have to map it to the _next_ old interval's new previous // interval. This logic is easier to understand if you see // [f.minIntervalIndex, f.maxIntervalIndex] as [f.minIntervalIndex, - // f.maxIntervalIndex+1). The other case to remember is when the interval is - // completely empty (i.e. len(newInterval.files) == 0); in that case we want - // to refer back to ourselves regardless of additions to the right of us. + // f.maxIntervalIndex+1). The other case to remember is when the + // interval is completely empty (i.e. len(newInterval.files) == 0); in + // that case we want to refer back to ourselves regardless of additions + // to the right of us. if newInterval.filesMaxIntervalIndex < len(oldToNewMap)-1 && len(newInterval.files) > 0 { newInterval.filesMaxIntervalIndex = oldToNewMap[newInterval.filesMaxIntervalIndex+1] - 1 } else { @@ -523,16 +526,16 @@ func (s *L0Sublevels) AddL0Files( newInterval.filesMaxIntervalIndex = oldToNewMap[newInterval.filesMaxIntervalIndex] } } - // Loop through all instances of new intervals added between two old intervals - // and expand [filesMinIntervalIndex, filesMaxIntervalIndex] of new intervals - // to reflect that of adjacent old intervals. + // Loop through all instances of new intervals added between two old + // intervals and expand [filesMinIntervalIndex, filesMaxIntervalIndex] of + // new intervals to reflect that of adjacent old intervals. { // We can skip cases where new intervals were added to the left of all // existing intervals (eg. if the first entry in oldToNewMap is - // oldToNewMap[0] >= 1). Those intervals will only contain newly added files - // and will have their parameters adjusted down in addFileToSublevels. The - // same can also be said about new intervals that are to the right of all - // existing intervals. + // oldToNewMap[0] >= 1). Those intervals will only contain newly added + // files and will have their parameters adjusted down in + // addFileToSublevels. The same can also be said about new intervals + // that are to the right of all existing intervals. lastIdx := 0 for _, newIdx := range oldToNewMap { for i := lastIdx + 1; i < newIdx; i++ { @@ -555,28 +558,29 @@ func (s *L0Sublevels) AddL0Files( // TODO(bilal): This is the only place in this method where we loop through // all existing files, which could be much more in number than newly added // files. See if we can avoid the need for this, either by getting rid of - // f.minIntervalIndex and f.maxIntervalIndex and calculating them on the - // fly with a binary search, or by only looping through files to the right - // of the first interval touched by this method. + // f.minIntervalIndex and f.maxIntervalIndex and calculating them on the fly + // with a binary search, or by only looping through files to the right of + // the first interval touched by this method. for sublevel := range s.Levels { s.Levels[sublevel].Each(func(f *FileMetadata) { oldIntervalDelta := f.maxIntervalIndex - f.minIntervalIndex + 1 oldMinIntervalIndex := f.minIntervalIndex f.minIntervalIndex = oldToNewMap[f.minIntervalIndex] - // maxIntervalIndex is special. Since it's an inclusive end bound, we - // actually have to map it to the _next_ old interval's new previous - // interval. This logic is easier to understand if you see - // [f.minIntervalIndex, f.maxIntervalIndex] as - // [f.minIntervalIndex, f.maxIntervalIndex+1). + // maxIntervalIndex is special. Since it's an inclusive end bound, + // we actually have to map it to the _next_ old interval's new + // previous interval. This logic is easier to understand if you see + // [f.minIntervalIndex, f.maxIntervalIndex] as [f.minIntervalIndex, + // f.maxIntervalIndex+1). f.maxIntervalIndex = oldToNewMap[f.maxIntervalIndex+1] - 1 newIntervalDelta := f.maxIntervalIndex - f.minIntervalIndex + 1 - // Recalculate estimatedBytes for all old files across new intervals, but - // only if new intervals were added in between. + // Recalculate estimatedBytes for all old files across new + // intervals, but only if new intervals were added in between. if oldIntervalDelta != newIntervalDelta { - // j is incremented so that oldToNewMap[j] points to the next old - // interval. This is used to distinguish between old intervals (i.e. - // ones where we need to subtract f.Size/oldIntervalDelta) from new - // ones (where we don't need to subtract). In both cases we need to add + // j is incremented so that oldToNewMap[j] points to the next + // old interval. This is used to distinguish between old + // intervals (i.e. ones where we need to subtract + // f.Size/oldIntervalDelta) from new ones (where we don't need + // to subtract). In both cases we need to add // f.Size/newIntervalDelta. j := oldMinIntervalIndex for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ { @@ -628,7 +632,7 @@ func (s *L0Sublevels) AddL0Files( // populated correctly. newVal.Levels[sublevel] = ls } - tr.release() + tr.Release() } newVal.flushSplitUserKeys = nil @@ -636,12 +640,12 @@ func (s *L0Sublevels) AddL0Files( return newVal, nil } -// addFileToSublevels is called during L0Sublevels generation, and adds f to -// the correct sublevel's levelFiles, the relevant intervals' files slices, and -// sets interval indices on f. This method, if called successively on multiple -// files, _must_ be called on successively newer files (by seqnum). If -// checkInvariant is true, it could check for this in some cases and return -// errInvalidL0SublevelsOpt if that invariant isn't held. +// addFileToSublevels is called during L0Sublevels generation, and adds f to the +// correct sublevel's levelFiles, the relevant intervals' files slices, and sets +// interval indices on f. This method, if called successively on multiple files, +// _must_ be called on successively newer files (by seqnum). If checkInvariant +// is true, it could check for this in some cases and return +// [errInvalidL0SublevelsOpt] if that invariant isn't held. func (s *L0Sublevels) addFileToSublevels(f *FileMetadata, checkInvariant bool) error { // This is a simple and not very accurate estimate of the number of // bytes this SSTable contributes to the intervals it is a part of. @@ -654,14 +658,18 @@ func (s *L0Sublevels) addFileToSublevels(f *FileMetadata, checkInvariant bool) e // Update state in every fileInterval for this file. for i := f.minIntervalIndex; i <= f.maxIntervalIndex; i++ { interval := &s.orderedIntervals[i] - if len(interval.files) > 0 && - subLevel <= interval.files[len(interval.files)-1].SubLevel { + if len(interval.files) > 0 { if checkInvariant && interval.files[len(interval.files)-1].LargestSeqNum > f.LargestSeqNum { // We are sliding this file "underneath" an existing file. Throw away // and start over in NewL0Sublevels. return errInvalidL0SublevelsOpt } - subLevel = interval.files[len(interval.files)-1].SubLevel + 1 + // interval.files is sorted by sublevels, from lowest to highest. + // AddL0Files can only add files at sublevels higher than existing files + // in the same key intervals. + if maxSublevel := interval.files[len(interval.files)-1].SubLevel; subLevel <= maxSublevel { + subLevel = maxSublevel + 1 + } } interval.estimatedBytes += interpolatedBytes if f.minIntervalIndex < interval.filesMinIntervalIndex { @@ -738,11 +746,10 @@ func (s *L0Sublevels) InitCompactingFileInfo(inProgress []L0Compaction) { } } - // Some intervals may be base compacting without the files contained - // within those intervals being marked as compacting. This is possible if - // the files were added after the compaction initiated, and the active - // compaction files straddle the input file. Mark these intervals as base - // compacting. + // Some intervals may be base compacting without the files contained within + // those intervals being marked as compacting. This is possible if the files + // were added after the compaction initiated, and the active compaction + // files straddle the input file. Mark these intervals as base compacting. for _, c := range inProgress { startIK := intervalKey{key: c.Smallest.UserKey, isLargest: false} endIK := intervalKey{key: c.Largest.UserKey, isLargest: !c.Largest.IsExclusiveSentinel()} @@ -924,35 +931,35 @@ func (s *L0Sublevels) InUseKeyRanges(smallest, largest []byte) []UserKeyRange { curr = &keyRanges[len(keyRanges)-1] } - // If the filesMaxIntervalIndex is not the current index, we can jump - // to the max index, knowing that all intermediary intervals are - // overlapped by some file. + // If the filesMaxIntervalIndex is not the current index, we can jump to + // the max index, knowing that all intermediary intervals are overlapped + // by some file. if maxIdx := s.orderedIntervals[i].filesMaxIntervalIndex; maxIdx != i { // Note that end may be less than or equal to maxIdx if we're // concerned with a key range that ends before the interval at - // maxIdx starts. We must set curr.End now, before making that - // leap, because this iteration may be the last. + // maxIdx starts. We must set curr.End now, before making that leap, + // because this iteration may be the last. i = maxIdx curr.End = s.orderedIntervals[i+1].startKey.key continue } // No files overlapping with this interval overlap with the next - // interval. Update the current end to be the next interval's start - // key. Note that curr is not necessarily finished, because there may - // be an abutting non-empty interval. + // interval. Update the current end to be the next interval's start key. + // Note that curr is not necessarily finished, because there may be an + // abutting non-empty interval. curr.End = s.orderedIntervals[i+1].startKey.key i++ } return keyRanges } -// FlushSplitKeys returns a slice of user keys to split flushes at. -// Used by flushes to avoid writing sstables that straddle these split keys. -// These should be interpreted as the keys to start the next sstable (not the -// last key to include in the prev sstable). These are user keys so that -// range tombstones can be properly truncated (untruncated range tombstones -// are not permitted for L0 files). +// FlushSplitKeys returns a slice of user keys to split flushes at. Used by +// flushes to avoid writing sstables that straddle these split keys. These +// should be interpreted as the keys to start the next sstable (not the last key +// to include in the prev sstable). These are user keys so that range tombstones +// can be properly truncated (untruncated range tombstones are not permitted for +// L0 files). func (s *L0Sublevels) FlushSplitKeys() [][]byte { return s.flushSplitUserKeys } @@ -978,6 +985,7 @@ func (s *L0Sublevels) MaxDepthAfterOngoingCompactions() int { // // TODO(bilal): Simplify away the debugging statements in this method, and make // this a pure sanity checker. +// //lint:ignore U1000 - useful for debugging func (s *L0Sublevels) checkCompaction(c *L0CompactionFiles) error { includedFiles := newBitSet(s.levelMetadata.Len()) @@ -1074,7 +1082,7 @@ func (s *L0Sublevels) checkCompaction(c *L0CompactionFiles) error { // recently started compaction. isBase specifies if this is a base compaction; // if false, this is assumed to be an intra-L0 compaction. The specified // compaction must be involving L0 SSTables. It's assumed that the Compacting -// and IsIntraL0Compacting fields are already set on all FileMetadatas passed +// and IsIntraL0Compacting fields are already set on all [FileMetadata]s passed // in. func (s *L0Sublevels) UpdateStateForStartedCompaction(inputs []LevelSlice, isBase bool) error { minIntervalIndex := -1 @@ -1106,11 +1114,10 @@ func (s *L0Sublevels) UpdateStateForStartedCompaction(inputs []LevelSlice, isBas return nil } -// L0CompactionFiles represents a candidate set of L0 files for compaction. -// Also referred to as "lcf". Contains state information useful -// for generating the compaction (such as Files), as well as for picking -// between candidate compactions (eg. fileBytes and -// seedIntervalStackDepthReduction). +// L0CompactionFiles represents a candidate set of L0 files for compaction. Also +// referred to as "lcf". Contains state information useful for generating the +// compaction (such as Files), as well as for picking between candidate +// compactions (eg. fileBytes and seedIntervalStackDepthReduction). type L0CompactionFiles struct { Files []*FileMetadata @@ -1146,6 +1153,23 @@ type L0CompactionFiles struct { filesAdded []*FileMetadata } +// Clone allocates a new L0CompactionFiles, with the same underlying data. Note +// that the two fileMetadata slices contain values that point to the same +// underlying fileMetadata object. This is safe because these objects are read +// only. +func (l *L0CompactionFiles) Clone() *L0CompactionFiles { + oldLcf := *l + return &oldLcf +} + +// String merely prints the starting address of the first file, if it exists. +func (l *L0CompactionFiles) String() string { + if len(l.Files) > 0 { + return fmt.Sprintf("First File Address: %p", &l.Files[0]) + } + return "" +} + // addFile adds the specified file to the LCF. func (l *L0CompactionFiles) addFile(f *FileMetadata) { if l.FilesIncluded[f.L0Index] { @@ -1180,12 +1204,11 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) { // Compactions: // -// The sub-levels and intervals can be visualized in 2 dimensions as the X -// axis containing intervals in increasing order and the Y axis containing -// sub-levels (older to younger). The intervals can be sparse wrt sub-levels. -// We observe that the system is typically under severe pressure in L0 during -// large numbers of ingestions where most files added to L0 are narrow and -// non-overlapping. +// The sub-levels and intervals can be visualized in 2 dimensions as the X axis +// containing intervals in increasing order and the Y axis containing sub-levels +// (older to younger). The intervals can be sparse wrt sub-levels. We observe +// that the system is typically under severe pressure in L0 during large numbers +// of ingestions where most files added to L0 are narrow and non-overlapping. // // L0.1 d---g // L0.0 c--e g--j o--s u--x @@ -1203,23 +1226,23 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) { // it amenable to concurrent L0 -> Lbase compactions. // // L0 -> Lbase: The high-level goal of a L0 -> Lbase compaction is to reduce -// stack depth, by compacting files in the intervals with the highest -// (fileCount - compactingCount). Additionally, we would like compactions to -// not involve a huge number of files, so that they finish quickly, and to -// allow for concurrent L0 -> Lbase compactions when needed. In order to -// achieve these goals we would like compactions to visualize as capturing -// thin and tall rectangles. The approach below is to consider intervals in -// some order and then try to construct a compaction using the interval. The -// first interval we can construct a compaction for is the compaction that is -// started. There can be multiple heuristics in choosing the ordering of the -// intervals -- the code uses one heuristic that worked well for a large -// ingestion stemming from a cockroachdb import, but additional experimentation -// is necessary to pick a general heuristic. Additionally, the compaction that -// gets picked may be not as desirable as one that could be constructed later -// in terms of reducing stack depth (since adding more files to the compaction -// can get blocked by needing to encompass files that are already being -// compacted). So an alternative would be to try to construct more than one -// compaction and pick the best one. +// stack depth, by compacting files in the intervals with the highest (fileCount +// - compactingCount). Additionally, we would like compactions to not involve a +// huge number of files, so that they finish quickly, and to allow for +// concurrent L0 -> Lbase compactions when needed. In order to achieve these +// goals we would like compactions to visualize as capturing thin and tall +// rectangles. The approach below is to consider intervals in some order and +// then try to construct a compaction using the interval. The first interval we +// can construct a compaction for is the compaction that is started. There can +// be multiple heuristics in choosing the ordering of the intervals -- the code +// uses one heuristic that worked well for a large ingestion stemming from a +// cockroachdb import, but additional experimentation is necessary to pick a +// general heuristic. Additionally, the compaction that gets picked may be not +// as desirable as one that could be constructed later in terms of reducing +// stack depth (since adding more files to the compaction can get blocked by +// needing to encompass files that are already being compacted). So an +// alternative would be to try to construct more than one compaction and pick +// the best one. // // Here's a visualization of an ideal L0->LBase compaction selection: // @@ -1243,9 +1266,9 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) { // Lbase a--------i m---------w // // Note that running this compaction will mark the a--i file in Lbase as -// compacting, and when ExtendL0ForBaseCompactionTo is called with the bounds -// of that base file, it'll expand the compaction to also include all L0 files -// in the a-d interval. The resultant compaction would then be: +// compacting, and when ExtendL0ForBaseCompactionTo is called with the bounds of +// that base file, it'll expand the compaction to also include all L0 files in +// the a-d interval. The resultant compaction would then be: // // _____________ // L0.3 |a--d g-j| @@ -1255,11 +1278,10 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) { // // Lbase a--------i m---------w // -// The next best interval for base compaction would therefore -// be the one including r--t in L0.2 and p--x in L0.0, and both this compaction -// and the one picked earlier can run in parallel. This is assuming -// minCompactionDepth >= 2, otherwise the second compaction has too little -// depth to pick. +// The next best interval for base compaction would therefore be the one +// including r--t in L0.2 and p--x in L0.0, and both this compaction and the one +// picked earlier can run in parallel. This is assuming minCompactionDepth >= 2, +// otherwise the second compaction has too little depth to pick. // // _____________ // L0.3 |a--d g-j| _________ @@ -1292,24 +1314,23 @@ func (is intervalSorterByDecreasingScore) Swap(i, j int) { // // Lbase a--------ij--lm---------w // -// Intra-L0: If the L0 score is high, but PickBaseCompaction() is unable to -// pick a compaction, PickIntraL0Compaction will be used to pick an intra-L0 -// compaction. Similar to L0 -> Lbase compactions, we want to allow for -// multiple intra-L0 compactions and not generate wide output files that -// hinder later concurrency of L0 -> Lbase compactions. Also compactions -// that produce wide files don't reduce stack depth -- they represent wide -// rectangles in our visualization, which means many intervals have their -// depth reduced by a small amount. Typically, L0 files have non-overlapping -// sequence numbers, and sticking to that invariant would require us to -// consider intra-L0 compactions that proceed from youngest to oldest files, -// which could result in the aforementioned undesirable wide rectangle -// shape. But this non-overlapping sequence number is already relaxed in -// RocksDB -- sstables are primarily ordered by their largest sequence -// number. So we can arrange for intra-L0 compactions to capture thin and -// tall rectangles starting with the top of the stack (youngest files). -// Like the L0 -> Lbase case we order the intervals using a heuristic and -// consider each in turn. The same comment about better L0 -> Lbase heuristics -// and not being greedy applies here. +// Intra-L0: If the L0 score is high, but PickBaseCompaction() is unable to pick +// a compaction, PickIntraL0Compaction will be used to pick an intra-L0 +// compaction. Similar to L0 -> Lbase compactions, we want to allow for multiple +// intra-L0 compactions and not generate wide output files that hinder later +// concurrency of L0 -> Lbase compactions. Also compactions that produce wide +// files don't reduce stack depth -- they represent wide rectangles in our +// visualization, which means many intervals have their depth reduced by a small +// amount. Typically, L0 files have non-overlapping sequence numbers, and +// sticking to that invariant would require us to consider intra-L0 compactions +// that proceed from youngest to oldest files, which could result in the +// aforementioned undesirable wide rectangle shape. But this non-overlapping +// sequence number is already relaxed in RocksDB -- sstables are primarily +// ordered by their largest sequence number. So we can arrange for intra-L0 +// compactions to capture thin and tall rectangles starting with the top of the +// stack (youngest files). Like the L0 -> Lbase case we order the intervals +// using a heuristic and consider each in turn. The same comment about better L0 +// -> Lbase heuristics and not being greedy applies here. // // Going back to a modified version of our example from earlier, let's say these // are the base compactions in progress: @@ -1387,10 +1408,9 @@ func (s *L0Sublevels) PickBaseCompaction( // Pick the seed file for the interval as the file // in the lowest sub-level. f := interval.files[0] - // Don't bother considering the intervals that are - // covered by the seed file since they are likely - // nearby. Note that it is possible that those intervals - // have seed files at lower sub-levels so could be + // Don't bother considering the intervals that are covered by the seed + // file since they are likely nearby. Note that it is possible that + // those intervals have seed files at lower sub-levels so could be // viable for compaction. if f == nil { return nil, errors.New("no seed file found in sublevel intervals") @@ -1398,23 +1418,22 @@ func (s *L0Sublevels) PickBaseCompaction( consideredIntervals.markBits(f.minIntervalIndex, f.maxIntervalIndex+1) if f.IsCompacting() { if f.IsIntraL0Compacting { - // If we're picking a base compaction and we came across a - // seed file candidate that's being intra-L0 compacted, skip - // the interval instead of erroring out. + // If we're picking a base compaction and we came across a seed + // file candidate that's being intra-L0 compacted, skip the + // interval instead of erroring out. continue } - // We chose a compaction seed file that should not be - // compacting. Usually means the score is not accurately - // accounting for files already compacting, or internal state is - // inconsistent. + // We chose a compaction seed file that should not be compacting. + // Usually means the score is not accurately accounting for files + // already compacting, or internal state is inconsistent. return nil, errors.Errorf("file %s chosen as seed file for compaction should not be compacting", f.FileNum) } c := s.baseCompactionUsingSeed(f, interval.index, minCompactionDepth) if c != nil { - // Check if the chosen compaction overlaps with any files - // in Lbase that have Compacting = true. If that's the case, - // this compaction cannot be chosen. + // Check if the chosen compaction overlaps with any files in Lbase + // that have Compacting = true. If that's the case, this compaction + // cannot be chosen. baseIter := baseFiles.Iter() // An interval starting at ImmediateSuccessor(key) can never be the // first interval of a compaction since no file can start at that @@ -1454,10 +1473,9 @@ func (s *L0Sublevels) baseCompactionUsingSeed( c.addFile(f) // The first iteration of this loop builds the compaction at the seed file's - // sublevel. Future iterations expand on this compaction by stacking - // more files from intervalIndex and repeating. This is an - // optional activity so when it fails we can fallback to the last - // successful candidate. + // sublevel. Future iterations expand on this compaction by stacking more + // files from intervalIndex and repeating. This is an optional activity so + // when it fails we can fallback to the last successful candidate. var lastCandidate *L0CompactionFiles interval := &s.orderedIntervals[intervalIndex] @@ -1467,29 +1485,28 @@ func (s *L0Sublevels) baseCompactionUsingSeed( c.seedIntervalStackDepthReduction++ c.seedIntervalMaxLevel = sl c.addFile(f2) - // The seed file is in the lowest sublevel in the seed interval, but it may - // overlap with other files in even lower sublevels. For - // correctness we need to grow our interval to include those files, and - // capture all files in the next level that fall in this extended interval - // and so on. This can result in a triangular shape like the following - // where again the X axis is the key intervals and the Y axis - // is oldest to youngest. Note that it is not necessary for - // correctness to fill out the shape at the higher sub-levels - // to make it more rectangular since the invariant only requires - // that younger versions of a key not be moved to Lbase while - // leaving behind older versions. + // The seed file is in the lowest sublevel in the seed interval, but it + // may overlap with other files in even lower sublevels. For correctness + // we need to grow our interval to include those files, and capture all + // files in the next level that fall in this extended interval and so + // on. This can result in a triangular shape like the following where + // again the X axis is the key intervals and the Y axis is oldest to + // youngest. Note that it is not necessary for correctness to fill out + // the shape at the higher sub-levels to make it more rectangular since + // the invariant only requires that younger versions of a key not be + // moved to Lbase while leaving behind older versions. // - // --- // ----- - // It may be better for performance to have a more rectangular - // shape since the files being left behind will overlap with the - // same Lbase key range as that of this compaction. But there is - // also the danger that in trying to construct a more rectangular - // shape we will be forced to pull in a file that is already - // compacting. We expect extendCandidateToRectangle to eventually be called - // on this compaction if it's chosen, at which point we would iterate - // backward and choose those files. This logic is similar to compaction.grow - // for non-L0 compactions. + // It may be better for performance to have a more rectangular shape + // since the files being left behind will overlap with the same Lbase + // key range as that of this compaction. But there is also the danger + // that in trying to construct a more rectangular shape we will be + // forced to pull in a file that is already compacting. We expect + // extendCandidateToRectangle to eventually be called on this compaction + // if it's chosen, at which point we would iterate backward and choose + // those files. This logic is similar to compaction.grow for non-L0 + // compactions. done := false for currLevel := sl - 1; currLevel >= 0; currLevel-- { if !s.extendFiles(currLevel, math.MaxUint64, c) { @@ -1509,8 +1526,8 @@ func (s *L0Sublevels) baseCompactionUsingSeed( // depth, but long running compactions reduce flexibility in what can // run concurrently in L0 and even Lbase -> Lbase+1. An increase more // than 150% in bytes since the last candidate compaction (along with a - // total compaction size in excess of 100mb), or a total compaction - // size beyond a hard limit of 500mb, is criteria for rejecting this + // total compaction size in excess of 100mb), or a total compaction size + // beyond a hard limit of 500mb, is criteria for rejecting this // candidate. This lets us prefer slow growths as we add files, while // still having a hard limit. Note that if this is the first compaction // candidate to reach a stack depth reduction of minCompactionDepth or @@ -1568,7 +1585,7 @@ func (s *L0Sublevels) extendFiles( // PickIntraL0Compaction picks an intra-L0 compaction for files in this // sublevel. This method is only called when a base compaction cannot be chosen. -// See comment above PickBaseCompaction for heuristics involved in this +// See comment above [PickBaseCompaction] for heuristics involved in this // selection. func (s *L0Sublevels) PickIntraL0Compaction( earliestUnflushedSeqNum uint64, minCompactionDepth int, @@ -1584,9 +1601,8 @@ func (s *L0Sublevels) PickIntraL0Compaction( } sort.Sort(intervalSorterByDecreasingScore(scoredIntervals)) - // Optimization to avoid considering different intervals that - // are likely to choose the same seed file. Again this is just - // to reduce wasted work. + // Optimization to avoid considering different intervals that are likely to + // choose the same seed file. Again this is just to reduce wasted work. consideredIntervals := newBitSet(len(s.orderedIntervals)) for _, scoredInterval := range scoredIntervals { interval := &s.orderedIntervals[scoredInterval.interval] @@ -1595,8 +1611,8 @@ func (s *L0Sublevels) PickIntraL0Compaction( } var f *FileMetadata - // Pick the seed file for the interval as the file - // in the highest sub-level. + // Pick the seed file for the interval as the file in the highest + // sub-level. stackDepthReduction := scoredInterval.score for i := len(interval.files) - 1; i >= 0; i-- { f = interval.files[i] @@ -1604,9 +1620,8 @@ func (s *L0Sublevels) PickIntraL0Compaction( break } consideredIntervals.markBits(f.minIntervalIndex, f.maxIntervalIndex+1) - // Can this be the seed file? Files with newer sequence - // numbers than earliestUnflushedSeqNum cannot be in - // the compaction. + // Can this be the seed file? Files with newer sequence numbers than + // earliestUnflushedSeqNum cannot be in the compaction. if f.LargestSeqNum >= earliestUnflushedSeqNum { stackDepthReduction-- if stackDepthReduction == 0 { @@ -1669,10 +1684,10 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed( } // The first iteration of this loop produces an intra-L0 compaction at the // seed level. Iterations after that optionally add to the compaction by - // stacking more files from intervalIndex and repeating. This is an - // optional activity so when it fails we can fallback to the last - // successful candidate. The code stops adding when it can't add more, or - // when fileBytes grows too large. + // stacking more files from intervalIndex and repeating. This is an optional + // activity so when it fails we can fallback to the last successful + // candidate. The code stops adding when it can't add more, or when + // fileBytes grows too large. for ; slIndex >= 0; slIndex-- { f2 := interval.files[slIndex] sl := f2.SubLevel @@ -1684,23 +1699,24 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed( c.addFile(f2) // The seed file captures all files in the higher level that fall in the // range of intervals. That may extend the range of intervals so for - // correctness we need to capture all files in the next higher level that - // fall in this extended interval and so on. This can result in an - // inverted triangular shape like the following where again the X axis is the - // key intervals and the Y axis is oldest to youngest. Note that it is not - // necessary for correctness to fill out the shape at lower sub-levels to - // make it more rectangular since the invariant only requires that if we - // move an older seqnum for key k into a file that has a higher seqnum, we - // also move all younger seqnums for that key k into that file. + // correctness we need to capture all files in the next higher level + // that fall in this extended interval and so on. This can result in an + // inverted triangular shape like the following where again the X axis + // is the key intervals and the Y axis is oldest to youngest. Note that + // it is not necessary for correctness to fill out the shape at lower + // sub-levels to make it more rectangular since the invariant only + // requires that if we move an older seqnum for key k into a file that + // has a higher seqnum, we also move all younger seqnums for that key k + // into that file. // ----- // --- // - - // - // It may be better for performance to have a more rectangular shape since - // it will reduce the stack depth for more intervals. But there is also - // the danger that in explicitly trying to construct a more rectangular - // shape we will be forced to pull in a file that is already compacting. - // We assume that the performance concern is not a practical issue. + // It may be better for performance to have a more rectangular shape + // since it will reduce the stack depth for more intervals. But there is + // also the danger that in explicitly trying to construct a more + // rectangular shape we will be forced to pull in a file that is already + // compacting. We assume that the performance concern is not a practical + // issue. done := false for currLevel := sl + 1; currLevel < len(s.levelFiles); currLevel++ { if !s.extendFiles(currLevel, earliestUnflushedSeqNum, c) { @@ -1734,11 +1750,11 @@ func (s *L0Sublevels) intraL0CompactionUsingSeed( } // ExtendL0ForBaseCompactionTo extends the specified base compaction candidate -// L0CompactionFiles to optionally cover more files in L0 without "touching" -// any of the passed-in keys (i.e. the smallest/largest bounds are exclusive), -// as including any user keys for those internal keys -// could require choosing more files in LBase which is undesirable. Unbounded -// start/end keys are indicated by passing in the InvalidInternalKey. +// L0CompactionFiles to optionally cover more files in L0 without "touching" any +// of the passed-in keys (i.e. the smallest/largest bounds are exclusive), as +// including any user keys for those internal keys could require choosing more +// files in LBase which is undesirable. Unbounded start/end keys are indicated +// by passing in the InvalidInternalKey. func (s *L0Sublevels) ExtendL0ForBaseCompactionTo( smallest, largest InternalKey, candidate *L0CompactionFiles, ) bool { @@ -1794,65 +1810,66 @@ func (s *L0Sublevels) ExtendL0ForBaseCompactionTo( // Consider this scenario (original candidate is inside the rectangle), with // isBase = true and interval bounds a-j (from the union of base file bounds and // that of compaction candidate): -// _______ -// L0.3 a--d | g-j| -// L0.2 | f--j| r-t -// L0.1 b-d |e---j| -// L0.0 a--d | f--j| l--o p-----x // -// Lbase a--------i m---------w +// _______ +// L0.3 a--d | g-j| +// L0.2 | f--j| r-t +// L0.1 b-d |e---j| +// L0.0 a--d | f--j| l--o p-----x +// +// Lbase a--------i m---------w // // This method will iterate from the bottom up. At L0.0, it will add a--d since // it's in the bounds, then add b-d, then a--d, and so on, to produce this: // -// _____________ -// L0.3 |a--d g-j| -// L0.2 | f--j| r-t -// L0.1 | b-d e---j| -// L0.0 |a--d f--j| l--o p-----x +// _____________ +// L0.3 |a--d g-j| +// L0.2 | f--j| r-t +// L0.1 | b-d e---j| +// L0.0 |a--d f--j| l--o p-----x // -// Lbase a-------i m---------w +// Lbase a-------i m---------w // // Let's assume that, instead of a--d in the top sublevel, we had 3 files, a-b, // bb-c, and cc-d, of which bb-c is compacting. Let's also add another sublevel // L0.4 with some files, all of which aren't compacting: // -// L0.4 a------c ca--d _______ -// L0.3 a-b bb-c cc-d | g-j| -// L0.2 | f--j| r-t -// L0.1 b----------d |e---j| -// L0.0 a------------d | f--j| l--o p-----x +// L0.4 a------c ca--d _______ +// L0.3 a-b bb-c cc-d | g-j| +// L0.2 | f--j| r-t +// L0.1 b----------d |e---j| +// L0.0 a------------d | f--j| l--o p-----x // -// Lbase a------------------i m---------w +// Lbase a------------------i m---------w // -// This method then needs to choose between the left side of L0.3 bb-c -// (i.e. a-b), or the right side (i.e. cc-d and g-j) for inclusion in this -// compaction. Since the right side has more files as well as one file that has -// already been picked, it gets chosen at that sublevel, resulting in this -// intermediate compaction: +// This method then needs to choose between the left side of L0.3 bb-c (i.e. +// a-b), or the right side (i.e. cc-d and g-j) for inclusion in this compaction. +// Since the right side has more files as well as one file that has already been +// picked, it gets chosen at that sublevel, resulting in this intermediate +// compaction: // -// L0.4 a------c ca--d -// ______________ -// L0.3 a-b bb-c| cc-d g-j| -// L0.2 _________| f--j| r-t -// L0.1 | b----------d e---j| -// L0.0 |a------------d f--j| l--o p-----x +// L0.4 a------c ca--d +// ______________ +// L0.3 a-b bb-c| cc-d g-j| +// L0.2 _________| f--j| r-t +// L0.1 | b----------d e---j| +// L0.0 |a------------d f--j| l--o p-----x // -// Lbase a------------------i m---------w +// Lbase a------------------i m---------w // // Since bb-c had to be excluded at L0.3, the interval bounds for L0.4 are // actually ca-j, since ca is the next interval start key after the end interval // of bb-c. This would result in only ca-d being chosen at that sublevel, even // though a--c is also not compacting. This is the final result: // -// ______________ -// L0.4 a------c|ca--d | -// L0.3 a-b bb-c| cc-d g-j| -// L0.2 _________| f--j| r-t -// L0.1 | b----------d e---j| -// L0.0 |a------------d f--j| l--o p-----x +// ______________ +// L0.4 a------c|ca--d | +// L0.3 a-b bb-c| cc-d g-j| +// L0.2 _________| f--j| r-t +// L0.1 | b----------d e---j| +// L0.0 |a------------d f--j| l--o p-----x // -// Lbase a------------------i m---------w +// Lbase a------------------i m---------w // // TODO(bilal): Add more targeted tests for this method, through // ExtendL0ForBaseCompactionTo and intraL0CompactionUsingSeed. @@ -1883,19 +1900,19 @@ func (s *L0Sublevels) extendCandidateToRectangle( } // Stats for files. addedCount := 0 - // Iterate from the oldest sub-level for L0 -> Lbase and youngest - // sub-level for intra-L0. The idea here is that anything that can't - // be included from that level constrains what can be included from - // the next level. This change in constraint is directly incorporated - // into minIntervalIndex, maxIntervalIndex. + // Iterate from the oldest sub-level for L0 -> Lbase and youngest sub-level + // for intra-L0. The idea here is that anything that can't be included from + // that level constrains what can be included from the next level. This + // change in constraint is directly incorporated into minIntervalIndex, + // maxIntervalIndex. for sl := startLevel; sl != endLevel; sl += increment { files := s.levelFiles[sl] // Find the first file that overlaps with minIntervalIndex. index := sort.Search(len(files), func(i int) bool { return minIntervalIndex <= files[i].maxIntervalIndex }) - // Track the files that are fully within the current constraint - // of [minIntervalIndex, maxIntervalIndex]. + // Track the files that are fully within the current constraint of + // [minIntervalIndex, maxIntervalIndex]. firstIndex := -1 lastIndex := -1 for ; index < len(files); index++ { @@ -1904,8 +1921,8 @@ func (s *L0Sublevels) extendCandidateToRectangle( break } include := true - // Extends out on the left so can't be included. This narrows - // what we can included in the next level. + // Extends out on the left so can't be included. This narrows what + // we can included in the next level. if f.minIntervalIndex < minIntervalIndex { include = false minIntervalIndex = f.maxIntervalIndex + 1 @@ -1932,13 +1949,13 @@ func (s *L0Sublevels) extendCandidateToRectangle( continue } // We have the files in [firstIndex, lastIndex] as potential for - // inclusion. Some of these may already have been picked. Some - // of them may be already compacting. The latter is tricky since - // we have to decide whether to contract minIntervalIndex or - // maxIntervalIndex when we encounter an already compacting file. - // We pick the longest sequence between firstIndex - // and lastIndex of non-compacting files -- this is represented by - // [candidateNonCompactingFirst, candidateNonCompactingLast]. + // inclusion. Some of these may already have been picked. Some of them + // may be already compacting. The latter is tricky since we have to + // decide whether to contract minIntervalIndex or maxIntervalIndex when + // we encounter an already compacting file. We pick the longest sequence + // between firstIndex and lastIndex of non-compacting files -- this is + // represented by [candidateNonCompactingFirst, + // candidateNonCompactingLast]. nonCompactingFirst := -1 currentRunHasAlreadyPickedFiles := false candidateNonCompactingFirst := -1 @@ -1985,8 +2002,8 @@ func (s *L0Sublevels) extendCandidateToRectangle( } } if candidateNonCompactingFirst == -1 { - // All files are compacting. There will be gaps that we could exploit - // to continue, but don't bother. + // All files are compacting. There will be gaps that we could + // exploit to continue, but don't bother. break } // May need to shrink [minIntervalIndex, maxIntervalIndex] for the next level. diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go index b3a5038..3e1d26d 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/level_metadata.go @@ -9,6 +9,7 @@ import ( "fmt" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" ) // LevelMetadata contains metadata for all of the files within @@ -23,12 +24,12 @@ type LevelMetadata struct { func (lm *LevelMetadata) clone() LevelMetadata { return LevelMetadata{ level: lm.level, - tree: lm.tree.clone(), + tree: lm.tree.Clone(), } } func (lm *LevelMetadata) release() (obsolete []*FileMetadata) { - return lm.tree.release() + return lm.tree.Release() } func makeLevelMetadata(cmp Compare, level int, files []*FileMetadata) LevelMetadata { @@ -46,29 +47,29 @@ func makeBTree(cmp btreeCmp, files []*FileMetadata) (btree, LevelSlice) { var t btree t.cmp = cmp for _, f := range files { - t.insert(f) + t.Insert(f) } - return t, LevelSlice{iter: t.iter(), length: t.length} + return t, newLevelSlice(t.Iter()) } // Empty indicates whether there are any files in the level. func (lm *LevelMetadata) Empty() bool { - return lm.tree.length == 0 + return lm.tree.Count() == 0 } // Len returns the number of files within the level. func (lm *LevelMetadata) Len() int { - return lm.tree.length + return lm.tree.Count() } // Iter constructs a LevelIterator over the entire level. func (lm *LevelMetadata) Iter() LevelIterator { - return LevelIterator{iter: lm.tree.iter()} + return LevelIterator{iter: lm.tree.Iter()} } // Slice constructs a slice containing the entire level. func (lm *LevelMetadata) Slice() LevelSlice { - return LevelSlice{iter: lm.tree.iter(), length: lm.tree.length} + return newLevelSlice(lm.tree.Iter()) } // Find finds the provided file in the level if it exists. @@ -99,7 +100,7 @@ func (lm *LevelMetadata) Annotation(annotator Annotator) interface{} { if lm.Empty() { return annotator.Zero(nil) } - v, _ := lm.tree.root.annotation(annotator) + v, _ := lm.tree.root.Annotation(annotator) return v } @@ -112,7 +113,7 @@ func (lm *LevelMetadata) InvalidateAnnotation(annotator Annotator) { if lm.Empty() { return } - lm.tree.root.invalidateAnnotation(annotator) + lm.tree.root.InvalidateAnnotation(annotator) } // LevelFile holds a file's metadata along with its position @@ -133,7 +134,8 @@ func (lf LevelFile) Slice() LevelSlice { // a slice constructor like this? func NewLevelSliceSeqSorted(files []*FileMetadata) LevelSlice { tr, slice := makeBTree(btreeCmpSeqNum, files) - tr.release() + tr.Release() + slice.verifyInvariants() return slice } @@ -143,7 +145,8 @@ func NewLevelSliceSeqSorted(files []*FileMetadata) LevelSlice { // a slice constructor like this? func NewLevelSliceKeySorted(cmp base.Compare, files []*FileMetadata) LevelSlice { tr, slice := makeBTree(btreeCmpSmallestKey(cmp), files) - tr.release() + tr.Release() + slice.verifyInvariants() return slice } @@ -153,13 +156,56 @@ func NewLevelSliceKeySorted(cmp base.Compare, files []*FileMetadata) LevelSlice // TODO(jackson): Update tests to avoid requiring this and remove it. func NewLevelSliceSpecificOrder(files []*FileMetadata) LevelSlice { tr, slice := makeBTree(btreeCmpSpecificOrder(files), files) - tr.release() + tr.Release() + slice.verifyInvariants() return slice } +// newLevelSlice constructs a new LevelSlice backed by iter. +func newLevelSlice(iter iterator) LevelSlice { + s := LevelSlice{iter: iter} + if iter.r != nil { + s.length = iter.r.subtreeCount + } + s.verifyInvariants() + return s +} + +// newBoundedLevelSlice constructs a new LevelSlice backed by iter and bounded +// by the provided start and end bounds. The provided startBound and endBound +// iterators must be iterators over the same B-Tree. Both start and end bounds +// are inclusive. +func newBoundedLevelSlice(iter iterator, startBound, endBound *iterator) LevelSlice { + s := LevelSlice{ + iter: iter, + start: startBound, + end: endBound, + } + if iter.valid() { + s.length = endBound.countLeft() - startBound.countLeft() + // NB: The +1 is a consequence of the end bound being inclusive. + if endBound.valid() { + s.length++ + } + // NB: A slice that's empty due to its bounds may have an endBound + // positioned before the startBound due to the inclusive bounds. + // TODO(jackson): Consider refactoring the end boundary to be exclusive; + // it would simplify some areas (eg, here) and complicate others (eg, + // Reslice-ing to grow compactions). + if s.length < 0 { + s.length = 0 + } + } + s.verifyInvariants() + return s +} + // LevelSlice contains a slice of the files within a level of the LSM. // A LevelSlice is immutable once created, but may be used to construct a // mutable LevelIterator over the slice's files. +// +// LevelSlices should be constructed through one of the existing constructors, +// not manually initialized. type LevelSlice struct { iter iterator length int @@ -170,6 +216,19 @@ type LevelSlice struct { end *iterator } +func (ls LevelSlice) verifyInvariants() { + if invariants.Enabled { + i := ls.Iter() + var length int + for f := i.First(); f != nil; f = i.Next() { + length++ + } + if ls.length != length { + panic(fmt.Sprintf("LevelSlice %s has length %d value; actual length is %d", ls, ls.length, length)) + } + } +} + // Each invokes fn for each element in the slice. func (ls LevelSlice) Each(fn func(*FileMetadata)) { iter := ls.Iter() @@ -181,6 +240,7 @@ func (ls LevelSlice) Each(fn func(*FileMetadata)) { // String implements fmt.Stringer. func (ls LevelSlice) String() string { var buf bytes.Buffer + fmt.Fprintf(&buf, "%d files: ", ls.length) ls.Each(func(f *FileMetadata) { if buf.Len() > 0 { fmt.Fprintf(&buf, " ") @@ -246,18 +306,7 @@ func (ls LevelSlice) Reslice(resliceFunc func(start, end *LevelIterator)) LevelS end.iter = ls.end.clone() } resliceFunc(&start, &end) - - s := LevelSlice{ - iter: start.iter.clone(), - start: &start.iter, - end: &end.iter, - } - // Calculate the new slice's length. - iter := s.Iter() - for f := iter.First(); f != nil; f = iter.Next() { - s.length++ - } - return s + return newBoundedLevelSlice(start.iter.clone(), &start.iter, &end.iter) } // KeyType is used to specify the type of keys we're looking for in @@ -387,8 +436,13 @@ func (i *LevelIterator) Clone() LevelIterator { } // Current returns the item at the current iterator position. +// +// Current is deprecated. Callers should instead use the return value of a +// positioning operation. func (i *LevelIterator) Current() *FileMetadata { - if !i.iter.valid() { + if !i.iter.valid() || + (i.end != nil && cmpIter(i.iter, *i.end) > 0) || + (i.start != nil && cmpIter(i.iter, *i.start) < 0) { return nil } return i.iter.cur() @@ -426,7 +480,7 @@ func (i *LevelIterator) First() *FileMetadata { if !i.iter.valid() { return nil } - return i.filteredNextFile(i.iter.cur()) + return i.skipFilteredForward(i.iter.cur()) } // Last seeks to the last file in the iterator and returns it. @@ -442,31 +496,37 @@ func (i *LevelIterator) Last() *FileMetadata { if !i.iter.valid() { return nil } - return i.filteredPrevFile(i.iter.cur()) + return i.skipFilteredBackward(i.iter.cur()) } // Next advances the iterator to the next file and returns it. func (i *LevelIterator) Next() *FileMetadata { + if i.iter.r == nil { + return nil + } + if invariants.Enabled && (i.iter.pos >= i.iter.n.count || (i.end != nil && cmpIter(i.iter, *i.end) > 0)) { + panic("pebble: cannot next forward-exhausted iterator") + } i.iter.next() if !i.iter.valid() { return nil } - if i.end != nil && cmpIter(i.iter, *i.end) > 0 { - return nil - } - return i.filteredNextFile(i.iter.cur()) + return i.skipFilteredForward(i.iter.cur()) } // Prev moves the iterator the previous file and returns it. func (i *LevelIterator) Prev() *FileMetadata { + if i.iter.r == nil { + return nil + } + if invariants.Enabled && (i.iter.pos < 0 || (i.start != nil && cmpIter(i.iter, *i.start) < 0)) { + panic("pebble: cannot prev backward-exhausted iterator") + } i.iter.prev() if !i.iter.valid() { return nil } - if i.start != nil && cmpIter(i.iter, *i.start) < 0 { - return nil - } - return i.filteredPrevFile(i.iter.cur()) + return i.skipFilteredBackward(i.iter.cur()) } // SeekGE seeks to the first file in the iterator's file set with a largest @@ -475,28 +535,23 @@ func (i *LevelIterator) Prev() *FileMetadata { // be sorted by user keys and non-overlapping. func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata { // TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey. - if i.empty() { + if i.iter.r == nil { return nil } - meta := i.seek(func(m *FileMetadata) bool { + m := i.seek(func(m *FileMetadata) bool { return cmp(m.Largest.UserKey, userKey) >= 0 }) - for meta != nil { - switch i.filter { - case KeyTypePointAndRange: - return meta - case KeyTypePoint: - if meta.HasPointKeys && cmp(meta.LargestPointKey.UserKey, userKey) >= 0 { - return meta - } - case KeyTypeRange: - if meta.HasRangeKeys && cmp(meta.LargestRangeKey.UserKey, userKey) >= 0 { - return meta - } + if i.filter != KeyTypePointAndRange && m != nil { + b, ok := m.LargestBound(i.filter) + if !ok { + m = i.Next() + } else if c := cmp(b.UserKey, userKey); c < 0 || c == 0 && b.IsExclusiveSentinel() { + // This file does not contain any keys of the type ≥ lower. It + // should be filtered, even though it does contain point keys. + m = i.Next() } - meta = i.Next() } - return i.filteredNextFile(meta) + return i.skipFilteredForward(m) } // SeekLT seeks to the last file in the iterator's file set with a smallest @@ -505,71 +560,76 @@ func (i *LevelIterator) SeekGE(cmp Compare, userKey []byte) *FileMetadata { // by user keys and non-overlapping. func (i *LevelIterator) SeekLT(cmp Compare, userKey []byte) *FileMetadata { // TODO(jackson): Assert that i.iter.cmp == btreeCmpSmallestKey. - if i.empty() { + if i.iter.r == nil { return nil } i.seek(func(m *FileMetadata) bool { return cmp(m.Smallest.UserKey, userKey) >= 0 }) - meta := i.Prev() - for meta != nil { - switch i.filter { - case KeyTypePointAndRange: - return meta - case KeyTypePoint: - if meta.HasPointKeys && cmp(meta.SmallestPointKey.UserKey, userKey) < 0 { - return meta - } - case KeyTypeRange: - if meta.HasRangeKeys && cmp(meta.SmallestRangeKey.UserKey, userKey) < 0 { - return meta - } + m := i.Prev() + // Although i.Prev() guarantees that the current file contains keys of the + // relevant type, it doesn't guarantee that the keys of the relevant type + // are < userKey. + if i.filter != KeyTypePointAndRange && m != nil { + b, ok := m.SmallestBound(i.filter) + if !ok { + panic("unreachable") + } + if c := cmp(b.UserKey, userKey); c >= 0 { + // This file does not contain any keys of the type ≥ lower. It + // should be filtered, even though it does contain point keys. + m = i.Prev() } - meta = i.Prev() } - return i.filteredPrevFile(meta) + return i.skipFilteredBackward(m) } -func (i *LevelIterator) filteredNextFile(meta *FileMetadata) *FileMetadata { - switch i.filter { - case KeyTypePoint: - for meta != nil && !meta.HasPointKeys { - meta = i.Next() +// skipFilteredForward takes the file metadata at the iterator's current +// position, and skips forward if the current key-type filter (i.filter) +// excludes the file. It skips until it finds an unfiltered file or exhausts the +// level. If lower is != nil, skipFilteredForward skips any files that do not +// contain keys with the provided key-type ≥ lower. +// +// skipFilteredForward also enforces the upper bound, returning nil if at any +// point the upper bound is exceeded. +func (i *LevelIterator) skipFilteredForward(meta *FileMetadata) *FileMetadata { + for meta != nil && !meta.ContainsKeyType(i.filter) { + i.iter.next() + if !i.iter.valid() { + meta = nil + } else { + meta = i.iter.cur() } - return meta - case KeyTypeRange: - // TODO(bilal): Range keys are expected to be rare and sparse. Add an - // optimization to annotate the tree and efficiently skip over files that - // do not contain range keys right at the seek step, to reduce iterations - // here. - for meta != nil && !meta.HasRangeKeys { - meta = i.Next() - } - return meta - default: - return meta } + if meta != nil && i.end != nil && cmpIter(i.iter, *i.end) > 0 { + // Exceeded upper bound. + meta = nil + } + return meta } -func (i *LevelIterator) filteredPrevFile(meta *FileMetadata) *FileMetadata { - switch i.filter { - case KeyTypePoint: - for meta != nil && !meta.HasPointKeys { - meta = i.Prev() +// skipFilteredBackward takes the file metadata at the iterator's current +// position, and skips backward if the current key-type filter (i.filter) +// excludes the file. It skips until it finds an unfiltered file or exhausts the +// level. If upper is != nil, skipFilteredBackward skips any files that do not +// contain keys with the provided key-type < upper. +// +// skipFilteredBackward also enforces the lower bound, returning nil if at any +// point the lower bound is exceeded. +func (i *LevelIterator) skipFilteredBackward(meta *FileMetadata) *FileMetadata { + for meta != nil && !meta.ContainsKeyType(i.filter) { + i.iter.prev() + if !i.iter.valid() { + meta = nil + } else { + meta = i.iter.cur() } - return meta - case KeyTypeRange: - // TODO(bilal): Range keys are expected to be rare and sparse. Add an - // optimization to annotate the tree and efficiently skip over files that - // do not contain range keys right at the seek step, to reduce iterations - // here. - for meta != nil && !meta.HasRangeKeys { - meta = i.Prev() - } - return meta - default: - return meta } + if meta != nil && i.start != nil && cmpIter(i.iter, *i.start) < 0 { + // Exceeded lower bound. + meta = nil + } + return meta } func (i *LevelIterator) seek(fn func(*FileMetadata) bool) *FileMetadata { @@ -594,7 +654,6 @@ func (i *LevelIterator) seek(fn func(*FileMetadata) bool) *FileMetadata { return nil } else if i.start != nil && cmpIter(i.iter, *i.start) < 0 { i.iter = i.start.clone() - return i.iter.cur() } if !i.iter.valid() { return nil @@ -614,13 +673,9 @@ func (i *LevelIterator) Take() LevelFile { // the same position for a LevelFile because they're inclusive, so we can // share one iterator stack between the two bounds. boundsIter := i.iter.clone() + s := newBoundedLevelSlice(i.iter.clone(), &boundsIter, &boundsIter) return LevelFile{ FileMetadata: m, - slice: LevelSlice{ - iter: i.iter.clone(), - start: &boundsIter, - end: &boundsIter, - length: 1, - }, + slice: s, } } diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go index 6c3bca0..ca16ba1 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/version.go @@ -17,7 +17,6 @@ import ( "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/invariants" - "github.com/cockroachdb/pebble/vfs" ) // Compare exports the base.Compare type. @@ -42,7 +41,8 @@ type TableInfo struct { LargestSeqNum uint64 } -// TableStats contains statistics on a table used for compaction heuristics. +// TableStats contains statistics on a table used for compaction heuristics, +// and export via Metrics. type TableStats struct { // The total number of entries in the table. NumEntries uint64 @@ -60,10 +60,14 @@ type TableStats struct { // overlapping data in L0 and ignores L0 sublevels, but the error that // introduces is expected to be small. // - // Tables in the bottommost level of the LSM may have a nonzero estimate - // if snapshots or move compactions prevented the elision of their range - // tombstones. + // Tables in the bottommost level of the LSM may have a nonzero estimate if + // snapshots or move compactions prevented the elision of their range + // tombstones. A table in the bottommost level that was ingested into L6 + // will have a zero estimate, because the file's sequence numbers indicate + // that the tombstone cannot drop any data contained within the file itself. RangeDeletionsBytesEstimate uint64 + // Total size of value blocks and value index block. + ValueBlocksSize uint64 } // boundType represents the type of key (point or range) present as the smallest @@ -79,10 +83,10 @@ const ( // // The following shows the valid state transitions: // -// NotCompacting --> Compacting --> Compacted -// ^ | -// | | -// +-------<-------+ +// NotCompacting --> Compacting --> Compacted +// ^ | +// | | +// +-------<-------+ // // Input files to a compaction transition to Compacting when a compaction is // picked. A file that has finished compacting typically transitions into the @@ -117,43 +121,81 @@ func (s CompactionState) String() string { } } -// FileMetadata holds the metadata for an on-disk table. +// FileMetadata is maintained for leveled-ssts, i.e., they belong to a level of +// some version. FileMetadata does not contain the actual level of the sst, +// since such leveled-ssts can move across levels in different versions, while +// sharing the same FileMetadata. There are two kinds of leveled-ssts, physical +// and virtual. Underlying both leveled-ssts is a backing-sst, for which the +// only state is FileBacking. A backing-sst is level-less. It is possible for a +// backing-sst to be referred to by a physical sst in one version and by one or +// more virtual ssts in one or more versions. A backing-sst becomes obsolete +// and can be deleted once it is no longer required by any physical or virtual +// sst in any version. +// +// We maintain some invariants: +// +// 1. Each physical and virtual sst will have a unique FileMetadata.FileNum, +// and there will be exactly one FileMetadata associated with the FileNum. +// +// 2. Within a version, a backing-sst is either only referred to by one +// physical sst or one or more virtual ssts. +// +// 3. Once a backing-sst is referred to by a virtual sst in the latest version, +// it cannot go back to being referred to by a physical sst in any future +// version. +// +// Once a physical sst is no longer needed by any version, we will no longer +// maintain the file metadata associated with it. We will still maintain the +// FileBacking associated with the physical sst if the backing sst is required +// by any virtual ssts in any version. type FileMetadata struct { - // Atomic contains fields which are accessed atomically. Go allocations - // are guaranteed to be 64-bit aligned which we take advantage of by - // placing the 64-bit fields which we access atomically at the beginning - // of the FileMetadata struct. For more information, see - // https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - Atomic struct { - // AllowedSeeks is used to determine if a file should be picked for - // a read triggered compaction. It is decremented when read sampling - // in pebble.Iterator after every after every positioning operation - // that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc). - AllowedSeeks int64 + // AllowedSeeks is used to determine if a file should be picked for + // a read triggered compaction. It is decremented when read sampling + // in pebble.Iterator after every after every positioning operation + // that returns a user key (eg. Next, Prev, SeekGE, SeekLT, etc). + AllowedSeeks atomic.Int64 - // statsValid is 1 if stats have been loaded for the table. The - // TableStats structure is populated only if valid is 1. - statsValid uint32 - } + // statsValid indicates if stats have been loaded for the table. The + // TableStats structure is populated only if valid is true. + statsValid atomic.Bool + + // FileBacking is the state which backs either a physical or virtual + // sstables. + FileBacking *FileBacking // InitAllowedSeeks is the inital value of allowed seeks. This is used // to re-set allowed seeks on a file once it hits 0. InitAllowedSeeks int64 - - // Reference count for the file: incremented when a file is added to a - // version and decremented when the version is unreferenced. The file is - // obsolete when the reference count falls to zero. - refs int32 // FileNum is the file number. + // + // INVARIANT: when !FileMetadata.Virtual, FileNum == FileBacking.DiskFileNum. + // + // TODO(bananabrick): Consider creating separate types for + // FileMetadata.FileNum and FileBacking.FileNum. FileNum is used both as + // an indentifier for the FileMetadata in Pebble, and also as a handle to + // perform reads and writes. We should ensure through types that + // FileMetadata.FileNum isn't used to perform reads, and that + // FileBacking.FileNum isn't used as an identifier for the FileMetadata. FileNum base.FileNum - // Size is the size of the file, in bytes. + // Size is the size of the file, in bytes. Size is an approximate value for + // virtual sstables. + // + // INVARIANT: when !FileMetadata.Virtual, Size == FileBacking.Size. + // + // TODO(bananabrick): Size is currently used in metrics, and for many key + // Pebble level heuristics. Make sure that the heuristics will still work + // appropriately with an approximate value of size. Size uint64 // File creation time in seconds since the epoch (1970-01-01 00:00:00 // UTC). For ingested sstables, this corresponds to the time the file was - // ingested. + // ingested. For virtual sstables, this corresponds to the wall clock time + // when the FileMetadata for the virtual sstable was first created. CreationTime int64 - // Smallest and largest sequence numbers in the table, across both point and - // range keys. + // Lower and upper bounds for the smallest and largest sequence numbers in + // the table, across both point and range keys. For physical sstables, these + // values are tight bounds. For virtual sstables, there is no guarantee that + // there will be keys with SmallestSeqNum or LargestSeqNum within virtual + // sstable bounds. SmallestSeqNum uint64 LargestSeqNum uint64 // SmallestPointKey and LargestPointKey are the inclusive bounds for the @@ -176,6 +218,13 @@ type FileMetadata struct { Smallest InternalKey Largest InternalKey // Stats describe table statistics. Protected by DB.mu. + // + // For virtual sstables, set stats upon virtual sstable creation as + // asynchronous computation of stats is not currently supported. + // + // TODO(bananabrick): To support manifest replay for virtual sstables, we + // probably need to compute virtual sstable stats asynchronously. Otherwise, + // we'd have to write virtual sstable stats to the version edit. Stats TableStats SubLevel int @@ -224,6 +273,173 @@ type FileMetadata struct { // key type (point or range) corresponds to the smallest and largest overall // table bounds. boundTypeSmallest, boundTypeLargest boundType + // Virtual is true if the FileMetadata belongs to a virtual sstable. + Virtual bool +} + +// PhysicalFileMeta is used by functions which want a guarantee that their input +// belongs to a physical sst and not a virtual sst. +// +// NB: This type should only be constructed by calling +// FileMetadata.PhysicalMeta. +type PhysicalFileMeta struct { + *FileMetadata +} + +// VirtualFileMeta is used by functions which want a guarantee that their input +// belongs to a virtual sst and not a physical sst. +// +// NB: This type should only be constructed by calling FileMetadata.VirtualMeta. +type VirtualFileMeta struct { + *FileMetadata +} + +// PhysicalMeta should be the only source of creating the PhysicalFileMeta +// wrapper type. +func (m *FileMetadata) PhysicalMeta() PhysicalFileMeta { + if m.Virtual { + panic("pebble: file metadata does not belong to a physical sstable") + } + return PhysicalFileMeta{ + m, + } +} + +// VirtualMeta should be the only source of creating the VirtualFileMeta wrapper +// type. +func (m *FileMetadata) VirtualMeta() VirtualFileMeta { + if !m.Virtual { + panic("pebble: file metadata does not belong to a virtual sstable") + } + return VirtualFileMeta{ + m, + } +} + +// FileBacking either backs a single physical sstable, or one or more virtual +// sstables. +// +// See the comment above the FileMetadata type for sstable terminology. +type FileBacking struct { + // Reference count for the backing file on disk: incremented when a + // physical or virtual sstable which is backed by the FileBacking is + // added to a version and decremented when the version is unreferenced. + // We ref count in order to determine when it is safe to delete a + // backing sst file from disk. The backing file is obsolete when the + // reference count falls to zero. + refs atomic.Int32 + // latestVersionRefs are the references to the FileBacking in the + // latest version. This reference can be through a single physical + // sstable in the latest version, or one or more virtual sstables in the + // latest version. + // + // INVARIANT: latestVersionRefs <= refs. + latestVersionRefs atomic.Int32 + // VirtualizedSize is set iff the backing sst is only referred to by + // virtual ssts in the latest version. VirtualizedSize is the sum of the + // virtual sstable sizes of all of the virtual sstables in the latest + // version which are backed by the physical sstable. When a virtual + // sstable is removed from the latest version, we will decrement the + // VirtualizedSize. During compaction picking, we'll compensate a + // virtual sstable file size by + // (FileBacking.Size - FileBacking.VirtualizedSize) / latestVersionRefs. + // The intuition is that if FileBacking.Size - FileBacking.VirtualizedSize + // is high, then the space amplification due to virtual sstables is + // high, and we should pick the virtual sstable with a higher priority. + // + // TODO(bananabrick): Compensate the virtual sstable file size using + // the VirtualizedSize during compaction picking and test. + VirtualizedSize atomic.Uint64 + DiskFileNum base.DiskFileNum + Size uint64 +} + +// InitPhysicalBacking allocates and sets the FileBacking which is required by a +// physical sstable FileMetadata. +// +// Ensure that the state required by FileBacking, such as the FileNum, is +// already set on the FileMetadata before InitPhysicalBacking is called. +// Calling InitPhysicalBacking only after the relevant state has been set in the +// FileMetadata is not necessary in tests which don't rely on FileBacking. +func (m *FileMetadata) InitPhysicalBacking() { + if m.Virtual { + panic("pebble: virtual sstables should use a pre-existing FileBacking") + } + if m.FileBacking == nil { + m.FileBacking = &FileBacking{Size: m.Size, DiskFileNum: m.FileNum.DiskFileNum()} + } +} + +// ValidateVirtual should be called once the FileMetadata for a virtual sstable +// is created to verify that the fields of the virtual sstable are sound. +func (m *FileMetadata) ValidateVirtual(createdFrom *FileMetadata) { + if !m.Virtual { + panic("pebble: invalid virtual sstable") + } + + if createdFrom.SmallestSeqNum != m.SmallestSeqNum { + panic("pebble: invalid smallest sequence number for virtual sstable") + } + + if createdFrom.LargestSeqNum != m.LargestSeqNum { + panic("pebble: invalid largest sequence number for virtual sstable") + } + + if createdFrom.FileBacking != nil && createdFrom.FileBacking != m.FileBacking { + panic("pebble: invalid physical sstable state for virtual sstable") + } + + if m.Size == 0 { + panic("pebble: virtual sstable size must be set upon creation") + } +} + +// Refs returns the refcount of backing sstable. +func (m *FileMetadata) Refs() int32 { + return m.FileBacking.refs.Load() +} + +// Ref increments the ref count associated with the backing sstable. +func (m *FileMetadata) Ref() { + m.FileBacking.refs.Add(1) +} + +// Unref decrements the ref count associated with the backing sstable. +func (m *FileMetadata) Unref() int32 { + v := m.FileBacking.refs.Add(-1) + if invariants.Enabled && v < 0 { + panic("pebble: invalid FileMetadata refcounting") + } + return v +} + +// LatestRef increments the latest ref count associated with the backing +// sstable. +func (m *FileMetadata) LatestRef() { + m.FileBacking.latestVersionRefs.Add(1) + + if m.Virtual { + m.FileBacking.VirtualizedSize.Add(m.Size) + } +} + +// LatestUnref decrements the latest ref count associated with the backing +// sstable. +func (m *FileMetadata) LatestUnref() int32 { + if m.Virtual { + m.FileBacking.VirtualizedSize.Add(-m.Size) + } + + v := m.FileBacking.latestVersionRefs.Add(-1) + if invariants.Enabled && v < 0 { + panic("pebble: invalid FileMetadata latest refcounting") + } + return v +} + +// LatestRefs returns the latest ref count associated with the backing sstable. +func (m *FileMetadata) LatestRefs() int32 { + return m.FileBacking.latestVersionRefs.Load() } // SetCompactionState transitions this file's compaction state to the given @@ -261,22 +477,14 @@ func (m *FileMetadata) IsCompacting() bool { // returns true, the Stats field may be read (with or without holding the // database mutex). func (m *FileMetadata) StatsValid() bool { - return atomic.LoadUint32(&m.Atomic.statsValid) == 1 -} - -// StatsValidLocked returns true if the table stats have been populated. -// StatsValidLocked requires DB.mu is held when it's invoked, and it avoids the -// overhead of an atomic load. This is possible because table stats validity is -// only set while DB.mu is held. -func (m *FileMetadata) StatsValidLocked() bool { - return m.Atomic.statsValid == 1 + return m.statsValid.Load() } // StatsMarkValid marks the TableStats as valid. The caller must hold DB.mu // while populating TableStats and calling StatsMarkValud. Once stats are // populated, they must not be mutated. func (m *FileMetadata) StatsMarkValid() { - atomic.StoreUint32(&m.Atomic.statsValid, 1) + m.statsValid.Store(true) } // ExtendPointKeyBounds attempts to extend the lower and upper point key bounds @@ -354,6 +562,66 @@ func (m *FileMetadata) extendOverallBounds( } } +// Overlaps returns true if the file key range overlaps with the given range. +func (m *FileMetadata) Overlaps(cmp Compare, start []byte, end []byte, exclusiveEnd bool) bool { + if c := cmp(m.Largest.UserKey, start); c < 0 || (c == 0 && m.Largest.IsExclusiveSentinel()) { + // f is completely before the specified range; no overlap. + return false + } + if c := cmp(m.Smallest.UserKey, end); c > 0 || (c == 0 && exclusiveEnd) { + // f is completely after the specified range; no overlap. + return false + } + return true +} + +// ContainsKeyType returns whether or not the file contains keys of the provided +// type. +func (m *FileMetadata) ContainsKeyType(kt KeyType) bool { + switch kt { + case KeyTypePointAndRange: + return true + case KeyTypePoint: + return m.HasPointKeys + case KeyTypeRange: + return m.HasRangeKeys + default: + panic("unrecognized key type") + } +} + +// SmallestBound returns the file's smallest bound of the key type. It returns a +// false second return value if the file does not contain any keys of the key +// type. +func (m *FileMetadata) SmallestBound(kt KeyType) (*InternalKey, bool) { + switch kt { + case KeyTypePointAndRange: + return &m.Smallest, true + case KeyTypePoint: + return &m.SmallestPointKey, m.HasPointKeys + case KeyTypeRange: + return &m.SmallestRangeKey, m.HasRangeKeys + default: + panic("unrecognized key type") + } +} + +// LargestBound returns the file's largest bound of the key type. It returns a +// false second return value if the file does not contain any keys of the key +// type. +func (m *FileMetadata) LargestBound(kt KeyType) (*InternalKey, bool) { + switch kt { + case KeyTypePointAndRange: + return &m.Largest, true + case KeyTypePoint: + return &m.LargestPointKey, m.HasPointKeys + case KeyTypeRange: + return &m.LargestRangeKey, m.HasRangeKeys + default: + panic("unrecognized key type") + } +} + const ( maskContainsPointKeys = 1 << 0 maskSmallest = 1 << 1 @@ -417,7 +685,7 @@ func (m *FileMetadata) DebugString(format base.FormatKey, verbose bool) string { // ParseFileMetadataDebug parses a FileMetadata from its DebugString // representation. -func ParseFileMetadataDebug(s string) (m FileMetadata, err error) { +func ParseFileMetadataDebug(s string) (*FileMetadata, error) { // Split lines of the form: // 000000:[a#0,SET-z#0,SET] points:[...] ranges:[...] fields := strings.FieldsFunc(s, func(c rune) bool { @@ -429,8 +697,9 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) { } }) if len(fields)%3 != 0 { - return m, errors.Newf("malformed input: %s", s) + return nil, errors.Newf("malformed input: %s", s) } + m := &FileMetadata{} for len(fields) > 0 { prefix := fields[0] smallest := base.ParsePrettyInternalKey(fields[1]) @@ -460,7 +729,8 @@ func ParseFileMetadataDebug(s string) (m FileMetadata, err error) { m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest m.HasPointKeys = true } - return + m.InitPhysicalBacking() + return m, nil } // Validate validates the metadata for consistency with itself, returning an @@ -522,6 +792,11 @@ func (m *FileMetadata) Validate(cmp Compare, formatKey base.FormatKey) error { } } + // Ensure that FileMetadata.Init was called. + if m.FileBacking == nil { + return base.CorruptionErrorf("file metadata FileBacking not set") + } + return nil } @@ -626,51 +901,50 @@ func SortBySmallest(files []*FileMetadata, cmp Compare) { func overlaps(iter LevelIterator, cmp Compare, start, end []byte, exclusiveEnd bool) LevelSlice { startIter := iter.Clone() - startIter.SeekGE(cmp, start) - - // SeekGE compares user keys. The user key `start` may be equal to the - // f.Largest because f.Largest is a range deletion sentinel, indicating that - // the user key `start` is NOT contained within the file f. If that's the - // case, we can narrow the overlapping bounds to exclude the file with the - // sentinel. - if f := startIter.Current(); f != nil && f.Largest.IsExclusiveSentinel() && - cmp(f.Largest.UserKey, start) == 0 { - startIter.Next() + { + startIterFile := startIter.SeekGE(cmp, start) + // SeekGE compares user keys. The user key `start` may be equal to the + // f.Largest because f.Largest is a range deletion sentinel, indicating + // that the user key `start` is NOT contained within the file f. If + // that's the case, we can narrow the overlapping bounds to exclude the + // file with the sentinel. + if startIterFile != nil && startIterFile.Largest.IsExclusiveSentinel() && + cmp(startIterFile.Largest.UserKey, start) == 0 { + startIterFile = startIter.Next() + } + _ = startIterFile // Ignore unused assignment. } endIter := iter.Clone() - endIter.SeekGE(cmp, end) + { + endIterFile := endIter.SeekGE(cmp, end) - if !exclusiveEnd { - // endIter is now pointing at the *first* file with a largest key >= end. - // If there are multiple files including the user key `end`, we want all - // of them, so move forward. - for f := endIter.Current(); f != nil && cmp(f.Largest.UserKey, end) == 0; { - f = endIter.Next() + if !exclusiveEnd { + // endIter is now pointing at the *first* file with a largest key >= end. + // If there are multiple files including the user key `end`, we want all + // of them, so move forward. + for endIterFile != nil && cmp(endIterFile.Largest.UserKey, end) == 0 { + endIterFile = endIter.Next() + } } - } - // LevelSlice uses inclusive bounds, so if we seeked to the end sentinel - // or nexted too far because Largest.UserKey equaled `end`, go back. - // - // Consider !exclusiveEnd and end = 'f', with the following file bounds: - // - // [b,d] [e, f] [f, f] [g, h] - // - // the above for loop will Next until it arrives at [g, h]. We need to - // observe that g > f, and Prev to the file with bounds [f, f]. - if !endIter.iter.valid() { - endIter.Prev() - } else if c := cmp(endIter.Current().Smallest.UserKey, end); c > 0 || c == 0 && exclusiveEnd { - endIter.Prev() - } - - iter = startIter.Clone() - return LevelSlice{ - iter: iter.iter, - start: &startIter.iter, - end: &endIter.iter, + // LevelSlice uses inclusive bounds, so if we seeked to the end sentinel + // or nexted too far because Largest.UserKey equaled `end`, go back. + // + // Consider !exclusiveEnd and end = 'f', with the following file bounds: + // + // [b,d] [e, f] [f, f] [g, h] + // + // the above for loop will Next until it arrives at [g, h]. We need to + // observe that g > f, and Prev to the file with bounds [f, f]. + if endIterFile == nil { + endIterFile = endIter.Prev() + } else if c := cmp(endIterFile.Smallest.UserKey, end); c > 0 || c == 0 && exclusiveEnd { + endIterFile = endIter.Prev() + } + _ = endIterFile // Ignore unused assignment. } + return newBoundedLevelSlice(startIter.Clone().iter, &startIter.iter, &endIter.iter) } // NumLevels is the number of levels a Version contains. @@ -744,7 +1018,7 @@ type Version struct { // usage. // // L0Sublevels.Levels contains L0 files ordered by sublevels. All the files - // in Files[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to + // in Levels[0] are in L0Sublevels.Levels. L0SublevelFiles is also set to // a reference to that slice, as that slice is necessary for iterator // creation and needs to outlast L0Sublevels. L0Sublevels *L0Sublevels @@ -759,7 +1033,7 @@ type Version struct { // The callback to invoke when the last reference to a version is // removed. Will be called with list.mu held. - Deleted func(obsolete []*FileMetadata) + Deleted func(obsolete []*FileBacking) // Stats holds aggregated stats about the version maintained from // version to version. @@ -788,15 +1062,21 @@ func (v *Version) DebugString(format base.FormatKey) string { return v.string(format, true) } +func describeSublevels(format base.FormatKey, verbose bool, sublevels []LevelSlice) string { + var buf bytes.Buffer + for sublevel := len(sublevels) - 1; sublevel >= 0; sublevel-- { + fmt.Fprintf(&buf, "0.%d:\n", sublevel) + sublevels[sublevel].Each(func(f *FileMetadata) { + fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose)) + }) + } + return buf.String() +} + func (v *Version) string(format base.FormatKey, verbose bool) string { var buf bytes.Buffer if len(v.L0SublevelFiles) > 0 { - for sublevel := len(v.L0SublevelFiles) - 1; sublevel >= 0; sublevel-- { - fmt.Fprintf(&buf, "0.%d:\n", sublevel) - v.L0SublevelFiles[sublevel].Each(func(f *FileMetadata) { - fmt.Fprintf(&buf, " %s\n", f.DebugString(format, verbose)) - }) - } + fmt.Fprintf(&buf, "%s", describeSublevels(format, verbose, v.L0SublevelFiles)) } for level := 1; level < NumLevels; level++ { if v.Levels[level].Empty() { @@ -837,7 +1117,7 @@ func ParseVersionDebug( m.SmallestPointKey, m.LargestPointKey = m.Smallest, m.Largest m.HasPointKeys = true } - files[level] = append(files[level], &m) + files[level] = append(files[level], m) } } // Reverse the order of L0 files. This ensures we construct the same @@ -865,11 +1145,15 @@ func (v *Version) Ref() { // locked. func (v *Version) Unref() { if atomic.AddInt32(&v.refs, -1) == 0 { - obsolete := v.unrefFiles() l := v.list l.mu.Lock() l.Remove(v) - v.Deleted(obsolete) + obsolete := v.unrefFiles() + fileBacking := make([]*FileBacking, len(obsolete)) + for i, f := range obsolete { + fileBacking[i] = f.FileBacking + } + v.Deleted(fileBacking) l.mu.Unlock() } } @@ -881,7 +1165,12 @@ func (v *Version) Unref() { func (v *Version) UnrefLocked() { if atomic.AddInt32(&v.refs, -1) == 0 { v.list.Remove(v) - v.Deleted(v.unrefFiles()) + obsolete := v.unrefFiles() + fileBacking := make([]*FileBacking, len(obsolete)) + for i, f := range obsolete { + fileBacking[i] = f.FileBacking + } + v.Deleted(fileBacking) } } @@ -933,11 +1222,11 @@ func (v *Version) Contains(level int, cmp Compare, m *FileMetadata) bool { } // Overlaps returns all elements of v.files[level] whose user key range -// intersects the inclusive range [start, end]. If level is non-zero then the -// user key ranges of v.files[level] are assumed to not overlap (although they -// may touch). If level is zero then that assumption cannot be made, and the -// [start, end] range is expanded to the union of those matching ranges so far -// and the computation is repeated until [start, end] stabilizes. +// intersects the given range. If level is non-zero then the user key ranges of +// v.files[level] are assumed to not overlap (although they may touch). If level +// is zero then that assumption cannot be made, and the [start, end] range is +// expanded to the union of those matching ranges so far and the computation is +// repeated until [start, end] stabilizes. // The returned files are a subsequence of the input files, i.e., the ordering // is not changed. func (v *Version) Overlaps( @@ -957,20 +1246,16 @@ func (v *Version) Overlaps( if selected { continue } - smallest := meta.Smallest.UserKey - largest := meta.Largest.UserKey - if c := cmp(largest, start); c < 0 || c == 0 && meta.Largest.IsExclusiveSentinel() { - // meta is completely before the specified range; skip it. - continue - } - if c := cmp(smallest, end); c > 0 || c == 0 && exclusiveEnd { - // meta is completely after the specified range; skip it. + if !meta.Overlaps(cmp, start, end, exclusiveEnd) { + // meta is completely outside the specified range; skip it. continue } // Overlaps. selectedIndices[i] = true numSelected++ + smallest := meta.Smallest.UserKey + largest := meta.Largest.UserKey // Since level == 0, check if the newly added fileMetadata has // expanded the range. We expand the range immediately for files // we have remaining to check in this loop. All already checked @@ -998,17 +1283,17 @@ func (v *Version) Overlaps( tr.cmp = v.Levels[level].tree.cmp for i, meta := 0, l0Iter.First(); meta != nil; i, meta = i+1, l0Iter.Next() { if selectedIndices[i] { - err := tr.insert(meta) + err := tr.Insert(meta) if err != nil { panic(err) } } } - slice = LevelSlice{iter: tr.iter(), length: tr.length} + slice = newLevelSlice(tr.Iter()) // TODO(jackson): Avoid the oddity of constructing and // immediately releasing a B-Tree. Make LevelSlice an // interface? - tr.release() + tr.Release() break } // Continue looping to retry the files that were not selected. @@ -1038,37 +1323,6 @@ func (v *Version) CheckOrdering(cmp Compare, format base.FormatKey) error { return nil } -// CheckConsistency checks that all of the files listed in the version exist -// and their on-disk sizes match the sizes listed in the version. -func (v *Version) CheckConsistency(dirname string, fs vfs.FS) error { - var buf bytes.Buffer - var args []interface{} - - for level, files := range v.Levels { - iter := files.Iter() - for f := iter.First(); f != nil; f = iter.Next() { - path := base.MakeFilepath(fs, dirname, base.FileTypeTable, f.FileNum) - info, err := fs.Stat(path) - if err != nil { - buf.WriteString("L%d: %s: %v\n") - args = append(args, errors.Safe(level), errors.Safe(f.FileNum), err) - continue - } - if info.Size() != int64(f.Size) { - buf.WriteString("L%d: %s: file size mismatch (%s): %d (disk) != %d (MANIFEST)\n") - args = append(args, errors.Safe(level), errors.Safe(f.FileNum), path, - errors.Safe(info.Size()), errors.Safe(f.Size)) - continue - } - } - } - - if buf.Len() == 0 { - return nil - } - return errors.Errorf(buf.String(), args...) -} - // VersionList holds a list of versions. The versions are ordered from oldest // to newest. type VersionList struct { diff --git a/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go b/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go index 75509ab..33f70d6 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manifest/version_edit.go @@ -8,11 +8,14 @@ import ( "bufio" "bytes" "encoding/binary" + "fmt" "io" - "sync/atomic" + "sort" + "time" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" ) // TODO(peter): describe the MANIFEST file format, independently of the C++ @@ -107,9 +110,40 @@ type VersionEdit struct { // found that there was no overlapping file at the higher level). DeletedFiles map[DeletedFileEntry]*FileMetadata NewFiles []NewFileEntry + // CreatedBackingTables can be used to preserve the FileBacking associated + // with a physical sstable. This is useful when virtual sstables in the + // latest version are reconstructed during manifest replay, and we also need + // to reconstruct the FileBacking which is required by these virtual + // sstables. + // + // INVARIANT: The FileBacking associated with a physical sstable must only + // be added as a backing file in the same version edit where the physical + // sstable is first virtualized. This means that the physical sstable must + // be present in DeletedFiles and that there must be at least one virtual + // sstable with the same FileBacking as the physical sstable in NewFiles. A + // file must be present in CreatedBackingTables in exactly one version edit. + // The physical sstable associated with the FileBacking must also not be + // present in NewFiles. + CreatedBackingTables []*FileBacking + // RemovedBackingTables is used to remove the FileBacking associated with a + // virtual sstable. Note that a backing sstable can be removed as soon as + // there are no virtual sstables in the latest version which are using the + // backing sstable, but the backing sstable doesn't necessarily have to be + // removed atomically with the version edit which removes the last virtual + // sstable associated with the backing sstable. The removal can happen in a + // future version edit. + // + // INVARIANT: A file must only be added to RemovedBackingTables if it was + // added to CreateBackingTables in a prior version edit. The same version + // edit also cannot have the same file present in both CreateBackingTables + // and RemovedBackingTables. A file must be present in RemovedBackingTables + // in exactly one version edit. + RemovedBackingTables []base.DiskFileNum } // Decode decodes an edit from the specified reader. +// +// TODO(bananabrick): Support decoding of virtual sstable state. func (v *VersionEdit) Decode(r io.Reader) error { br, ok := r.(byteReader) if !ok { @@ -342,6 +376,7 @@ func (v *VersionEdit) Decode(r io.Reader) error { } } m.boundsSet = true + m.InitPhysicalBacking() v.NewFiles = append(v.NewFiles, NewFileEntry{ Level: level, Meta: m, @@ -364,7 +399,51 @@ func (v *VersionEdit) Decode(r io.Reader) error { return nil } +// String implements fmt.Stringer for a VersionEdit. +func (v *VersionEdit) String() string { + var buf bytes.Buffer + if v.ComparerName != "" { + fmt.Fprintf(&buf, " comparer: %s", v.ComparerName) + } + if v.MinUnflushedLogNum != 0 { + fmt.Fprintf(&buf, " log-num: %d\n", v.MinUnflushedLogNum) + } + if v.ObsoletePrevLogNum != 0 { + fmt.Fprintf(&buf, " prev-log-num: %d\n", v.ObsoletePrevLogNum) + } + if v.NextFileNum != 0 { + fmt.Fprintf(&buf, " next-file-num: %d\n", v.NextFileNum) + } + if v.LastSeqNum != 0 { + fmt.Fprintf(&buf, " last-seq-num: %d\n", v.LastSeqNum) + } + entries := make([]DeletedFileEntry, 0, len(v.DeletedFiles)) + for df := range v.DeletedFiles { + entries = append(entries, df) + } + sort.Slice(entries, func(i, j int) bool { + if entries[i].Level != entries[j].Level { + return entries[i].Level < entries[j].Level + } + return entries[i].FileNum < entries[j].FileNum + }) + for _, df := range entries { + fmt.Fprintf(&buf, " deleted: L%d %s\n", df.Level, df.FileNum) + } + for _, nf := range v.NewFiles { + fmt.Fprintf(&buf, " added: L%d %s", nf.Level, nf.Meta.String()) + if nf.Meta.CreationTime != 0 { + fmt.Fprintf(&buf, " (%s)", + time.Unix(nf.Meta.CreationTime, 0).UTC().Format(time.RFC3339)) + } + fmt.Fprintln(&buf) + } + return buf.String() +} + // Encode encodes an edit to the specified writer. +// +// TODO(bananabrick): Support encoding of virtual sstable state. func (v *VersionEdit) Encode(w io.Writer) error { e := versionEditEncoder{new(bytes.Buffer)} @@ -536,10 +615,28 @@ func (e versionEditEncoder) writeUvarint(u uint64) { // BulkVersionEdit summarizes the files added and deleted from a set of version // edits. +// +// INVARIANTS: +// No file can be added to a level more than once. This is true globally, and +// also true for all of the calls to Accumulate for a single bulk version edit. +// +// No file can be removed from a level more than once. This is true globally, +// and also true for all of the calls to Accumulate for a single bulk version +// edit. +// +// A file must not be added and removed from a given level in the same version +// edit. +// +// A file that is being removed from a level must have been added to that level +// before (in a prior version edit). Note that a given file can be deleted from +// a level and added to another level in a single version edit type BulkVersionEdit struct { - Added [NumLevels][]*FileMetadata + Added [NumLevels]map[base.FileNum]*FileMetadata Deleted [NumLevels]map[base.FileNum]*FileMetadata + AddedFileBacking []*FileBacking + RemovedFileBacking []base.DiskFileNum + // AddedByFileNum maps file number to file metadata for all added files // from accumulated version edits. AddedByFileNum is only populated if set // to non-nil by a caller. It must be set to non-nil when replaying @@ -558,6 +655,19 @@ type BulkVersionEdit struct { // Accumulate adds the file addition and deletions in the specified version // edit to the bulk edit's internal state. +// +// INVARIANTS: +// If a file is added to a given level in a call to Accumulate and then removed +// from that level in a subsequent call, the file will not be present in the +// resulting BulkVersionEdit.Deleted for that level. +// +// After accumulation of version edits, the bulk version edit may have +// information about a file which has been deleted from a level, but it may +// not have information about the same file added to the same level. The add +// could've occurred as part of a previous bulk version edit. In this case, +// the deleted file must be present in BulkVersionEdit.Deleted, at the end +// of the accumulation, because we need to decrease the refcount of the +// deleted file in Apply. func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error { for df, m := range ve.DeletedFiles { dmap := b.Deleted[df.Level] @@ -579,7 +689,12 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error { if m.MarkedForCompaction { b.MarkedForCompactionCountDiff-- } - dmap[df.FileNum] = m + if _, ok := b.Added[df.Level][df.FileNum]; !ok { + dmap[df.FileNum] = m + } else { + // Present in b.Added for the same level. + delete(b.Added[df.Level], df.FileNum) + } } for _, nf := range ve.NewFiles { @@ -590,7 +705,10 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error { return base.CorruptionErrorf("pebble: file deleted L%d.%s before it was inserted", nf.Level, nf.Meta.FileNum) } } - b.Added[nf.Level] = append(b.Added[nf.Level], nf.Meta) + if b.Added[nf.Level] == nil { + b.Added[nf.Level] = make(map[base.FileNum]*FileMetadata) + } + b.Added[nf.Level][nf.Meta.FileNum] = nf.Meta if b.AddedByFileNum != nil { b.AddedByFileNum[nf.Meta.FileNum] = nf.Meta } @@ -598,36 +716,101 @@ func (b *BulkVersionEdit) Accumulate(ve *VersionEdit) error { b.MarkedForCompactionCountDiff++ } } + + // Generate state for the backing files. + b.AddedFileBacking = append(b.AddedFileBacking, ve.CreatedBackingTables...) + + // Since a file can be removed from backing files in exactly one version + // edit it is safe to just append without any de-duplication. + b.RemovedFileBacking = append(b.RemovedFileBacking, ve.RemovedBackingTables...) + return nil } +// AccumulateIncompleteAndApplySingleVE should be called if a single version edit +// is to be applied to the provided curr Version and if the caller needs to +// update the versionSet.zombieTables map. This function exists separately from +// BulkVersionEdit.Apply because it is easier to reason about properties +// regarding BulkVersionedit.Accumulate/Apply and zombie table generation, if we +// know that exactly one version edit is being accumulated. +// +// Note that the version edit passed into this function may be incomplete +// because compactions don't have the ref counting information necessary to +// populate VersionEdit.RemovedBackingTables. This function will complete such a +// version edit by populating RemovedBackingTables. +// +// Invariant: Any file being deleted through ve must belong to the curr Version. +// We can't have a delete for some arbitrary file which does not exist in curr. +func AccumulateIncompleteAndApplySingleVE( + ve *VersionEdit, + curr *Version, + cmp Compare, + formatKey base.FormatKey, + flushSplitBytes int64, + readCompactionRate int64, + backingStateMap map[base.DiskFileNum]*FileBacking, +) (_ *Version, zombies map[base.DiskFileNum]uint64, _ error) { + if len(ve.RemovedBackingTables) != 0 { + panic("pebble: invalid incomplete version edit") + } + var b BulkVersionEdit + err := b.Accumulate(ve) + if err != nil { + return nil, nil, err + } + zombies = make(map[base.DiskFileNum]uint64) + v, err := b.Apply( + curr, cmp, formatKey, flushSplitBytes, readCompactionRate, zombies, + ) + if err != nil { + return nil, nil, err + } + + for _, s := range b.AddedFileBacking { + backingStateMap[s.DiskFileNum] = s + } + + for fileNum := range zombies { + if _, ok := backingStateMap[fileNum]; ok { + // This table was backing some virtual sstable in the latest version, + // but is now a zombie. We add RemovedBackingTables entries for + // these, before the version edit is written to disk. + ve.RemovedBackingTables = append( + ve.RemovedBackingTables, fileNum, + ) + delete(backingStateMap, fileNum) + } + } + + return v, zombies, nil +} + // Apply applies the delta b to the current version to produce a new // version. The new version is consistent with respect to the comparer cmp. // // curr may be nil, which is equivalent to a pointer to a zero version. // -// On success, a map of zombie files containing the file numbers and sizes of -// deleted files is returned. These files are considered zombies because they -// are no longer referenced by the returned Version, but cannot be deleted from -// disk as they are still in use by the incoming Version. +// On success, if a non-nil zombies map is provided to Apply, the map is updated +// with file numbers and files sizes of deleted files. These files are +// considered zombies because they are no longer referenced by the returned +// Version, but cannot be deleted from disk as they are still in use by the +// incoming Version. func (b *BulkVersionEdit) Apply( curr *Version, cmp Compare, formatKey base.FormatKey, flushSplitBytes int64, readCompactionRate int64, -) (_ *Version, zombies map[base.FileNum]uint64, _ error) { - addZombie := func(fileNum base.FileNum, size uint64) { - if zombies == nil { - zombies = make(map[base.FileNum]uint64) - } - zombies[fileNum] = size - } - // The remove zombie function is used to handle tables that are moved from - // one level to another during a version edit (i.e. a "move" compaction). - removeZombie := func(fileNum base.FileNum) { + zombies map[base.DiskFileNum]uint64, +) (*Version, error) { + addZombie := func(state *FileBacking) { if zombies != nil { - delete(zombies, fileNum) + zombies[state.DiskFileNum] = state.Size + } + } + removeZombie := func(state *FileBacking) { + if zombies != nil { + delete(zombies, state.DiskFileNum) } } @@ -639,7 +822,7 @@ func (b *BulkVersionEdit) Apply( } v.Stats.MarkedForCompaction += b.MarkedForCompactionCountDiff if v.Stats.MarkedForCompaction < 0 { - return nil, nil, base.CorruptionErrorf("pebble: version marked for compaction count negative") + return nil, base.CorruptionErrorf("pebble: version marked for compaction count negative") } for level := range v.Levels { @@ -660,7 +843,7 @@ func (b *BulkVersionEdit) Apply( // Initialize L0Sublevels. if curr == nil || curr.L0Sublevels == nil { if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil { - return nil, nil, errors.Wrap(err, "pebble: internal error") + return nil, errors.Wrap(err, "pebble: internal error") } } else { v.L0Sublevels = curr.L0Sublevels @@ -673,47 +856,70 @@ func (b *BulkVersionEdit) Apply( // Some edits on this level. lm := &v.Levels[level] lmRange := &v.RangeKeyLevels[level] - addedFiles := b.Added[level] - deletedMap := b.Deleted[level] - if n := v.Levels[level].Len() + len(addedFiles); n == 0 { - return nil, nil, base.CorruptionErrorf( + + addedFilesMap := b.Added[level] + deletedFilesMap := b.Deleted[level] + if n := v.Levels[level].Len() + len(addedFilesMap); n == 0 { + return nil, base.CorruptionErrorf( "pebble: internal error: No current or added files but have deleted files: %d", - errors.Safe(len(deletedMap))) + errors.Safe(len(deletedFilesMap))) } - // NB: addedFiles may be empty and it also is not necessarily - // internally consistent: it does not reflect deletions in deletedMap. + // NB: addedFilesMap may be empty. If a file is present in addedFilesMap + // for a level, it won't be present in deletedFilesMap for the same + // level. - for _, f := range deletedMap { - addZombie(f.FileNum, f.Size) - if obsolete := v.Levels[level].tree.delete(f); obsolete { + for _, f := range deletedFilesMap { + if obsolete := v.Levels[level].tree.Delete(f); obsolete { // Deleting a file from the B-Tree may decrement its // reference count. However, because we cloned the // previous level's B-Tree, this should never result in a // file's reference count dropping to zero. err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during B-Tree removal", level, f.FileNum) - return nil, nil, err + return nil, err } if f.HasRangeKeys { - if obsolete := v.RangeKeyLevels[level].tree.delete(f); obsolete { + if obsolete := v.RangeKeyLevels[level].tree.Delete(f); obsolete { // Deleting a file from the B-Tree may decrement its // reference count. However, because we cloned the // previous level's B-Tree, this should never result in a // file's reference count dropping to zero. err := errors.Errorf("pebble: internal error: file L%d.%s obsolete during range-key B-Tree removal", level, f.FileNum) - return nil, nil, err + return nil, err } } + + // Note that a backing sst will only become a zombie if the + // references to it in the latest version is 0. We will remove the + // backing sst from the zombie list in the next loop if one of the + // addedFiles in any of the levels is referencing the backing sst. + // This is possible if a physical sstable is virtualized, or if it + // is moved. + latestRefCount := f.LatestRefs() + if latestRefCount <= 0 { + // If a file is present in deletedFilesMap for a level, then it + // must have already been added to the level previously, which + // means that its latest ref count cannot be 0. + err := errors.Errorf("pebble: internal error: incorrect latestRefs reference counting for file", f.FileNum) + return nil, err + } else if f.LatestUnref() == 0 { + addZombie(f.FileBacking) + } } + addedFiles := make([]*FileMetadata, 0, len(addedFilesMap)) + for _, f := range addedFilesMap { + addedFiles = append(addedFiles, f) + } + // Sort addedFiles by file number. This isn't necessary, but tests which + // replay invalid manifests check the error output, and the error output + // depends on the order in which files are added to the btree. + sort.Slice(addedFiles, func(i, j int) bool { + return addedFiles[i].FileNum < addedFiles[j].FileNum + }) + var sm, la *FileMetadata for _, f := range addedFiles { - if _, ok := deletedMap[f.FileNum]; ok { - // Already called addZombie on this file in the preceding - // loop, so we don't need to do it here. - continue - } - // NB: allowedSeeks is used for read triggered compactions. It is set using // Options.Experimental.ReadCompactionRate which defaults to 32KB. var allowedSeeks int64 @@ -723,20 +929,23 @@ func (b *BulkVersionEdit) Apply( if allowedSeeks < 100 { allowedSeeks = 100 } - atomic.StoreInt64(&f.Atomic.AllowedSeeks, allowedSeeks) + f.AllowedSeeks.Store(allowedSeeks) f.InitAllowedSeeks = allowedSeeks - err := lm.tree.insert(f) + err := lm.tree.Insert(f) + // We're adding this file to the new version, so increment the + // latest refs count. + f.LatestRef() if err != nil { - return nil, nil, errors.Wrap(err, "pebble") + return nil, errors.Wrap(err, "pebble") } if f.HasRangeKeys { - err = lmRange.tree.insert(f) + err = lmRange.tree.Insert(f) if err != nil { - return nil, nil, errors.Wrap(err, "pebble") + return nil, errors.Wrap(err, "pebble") } } - removeZombie(f.FileNum) + removeZombie(f.FileBacking) // Track the keys with the smallest and largest keys, so that we can // check consistency of the modified span. if sm == nil || base.InternalCompare(cmp, sm.Smallest, f.Smallest) > 0 { @@ -748,27 +957,36 @@ func (b *BulkVersionEdit) Apply( } if level == 0 { - if curr != nil && curr.L0Sublevels != nil && len(deletedMap) == 0 { + if curr != nil && curr.L0Sublevels != nil && len(deletedFilesMap) == 0 { // Flushes and ingestions that do not delete any L0 files do not require // a regeneration of L0Sublevels from scratch. We can instead generate // it incrementally. var err error // AddL0Files requires addedFiles to be sorted in seqnum order. - addedFiles = append([]*FileMetadata(nil), addedFiles...) SortBySeqNum(addedFiles) v.L0Sublevels, err = curr.L0Sublevels.AddL0Files(addedFiles, flushSplitBytes, &v.Levels[0]) if errors.Is(err, errInvalidL0SublevelsOpt) { err = v.InitL0Sublevels(cmp, formatKey, flushSplitBytes) + } else if invariants.Enabled && err == nil { + copyOfSublevels, err := NewL0Sublevels(&v.Levels[0], cmp, formatKey, flushSplitBytes) + if err != nil { + panic(fmt.Sprintf("error when regenerating sublevels: %s", err)) + } + s1 := describeSublevels(base.DefaultFormatter, false /* verbose */, copyOfSublevels.Levels) + s2 := describeSublevels(base.DefaultFormatter, false /* verbose */, v.L0Sublevels.Levels) + if s1 != s2 { + panic(fmt.Sprintf("incremental L0 sublevel generation produced different output than regeneration: %s != %s", s1, s2)) + } } if err != nil { - return nil, nil, errors.Wrap(err, "pebble: internal error") + return nil, errors.Wrap(err, "pebble: internal error") } v.L0SublevelFiles = v.L0Sublevels.Levels } else if err := v.InitL0Sublevels(cmp, formatKey, flushSplitBytes); err != nil { - return nil, nil, errors.Wrap(err, "pebble: internal error") + return nil, errors.Wrap(err, "pebble: internal error") } if err := CheckOrdering(cmp, formatKey, Level(0), v.Levels[level].Iter()); err != nil { - return nil, nil, errors.Wrap(err, "pebble: internal error") + return nil, errors.Wrap(err, "pebble: internal error") } continue } @@ -789,9 +1007,9 @@ func (b *BulkVersionEdit) Apply( } }) if err := CheckOrdering(cmp, formatKey, Level(level), check.Iter()); err != nil { - return nil, nil, errors.Wrap(err, "pebble: internal error") + return nil, errors.Wrap(err, "pebble: internal error") } } } - return v, zombies, nil + return v, nil } diff --git a/vendor/github.com/cockroachdb/pebble/internal/manual/manual_64bit.go b/vendor/github.com/cockroachdb/pebble/internal/manual/manual_64bit.go index 5f3793c..8c08232 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/manual/manual_64bit.go +++ b/vendor/github.com/cockroachdb/pebble/internal/manual/manual_64bit.go @@ -2,8 +2,8 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 -// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 +//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64 +// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 riscv64 package manual diff --git a/vendor/github.com/cockroachdb/pebble/internal/private/flush_external.go b/vendor/github.com/cockroachdb/pebble/internal/private/flush_external.go deleted file mode 100644 index 398b4d3..0000000 --- a/vendor/github.com/cockroachdb/pebble/internal/private/flush_external.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package private - -import "github.com/cockroachdb/pebble/internal/manifest" - -// FlushExternalTable is a hook for linking files into L0 without assigning a -// global sequence number, mimicking a flush. FlushExternalTable takes a -// *pebble.DB, the path and the metadata of a sstable to flush directly into -// L0. -// -// Calls to flush a sstable may fail if the file's sequence numbers are not -// greater than the current commit pipeline's sequence number. On success the -// commit pipeline's published sequence number will be moved to the file's -// highest sequence number. -// -// This function is wrapped in a safer, more ergonomic API in the -// internal/replay package. Clients should use the replay package rather than -// calling this private hook directly. -var FlushExternalTable func(interface{}, string, *manifest.FileMetadata) error - -// RatchetSeqNum is a hook for allocating and publishing sequence numbers up -// to a specific absolute value. Its first parameter is a *pebble.DB and its -// second is the new next sequence number. RatchetSeqNum does nothing if the -// next sequence is already greater than or equal to nextSeqNum. -// -// This function is used by the internal/replay package to ensure replayed -// operations receive the same absolute sequence number. -var RatchetSeqNum func(interface{}, uint64) diff --git a/vendor/github.com/cockroachdb/pebble/internal/private/sstable.go b/vendor/github.com/cockroachdb/pebble/internal/private/sstable.go index f4a823d..780dd56 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/private/sstable.go +++ b/vendor/github.com/cockroachdb/pebble/internal/private/sstable.go @@ -8,7 +8,7 @@ import "github.com/cockroachdb/pebble/internal/base" // SSTableCacheOpts is a hook for specifying cache options to // sstable.NewReader. -var SSTableCacheOpts func(cacheID uint64, fileNum base.FileNum) interface{} +var SSTableCacheOpts func(cacheID uint64, fileNum base.DiskFileNum) interface{} // SSTableRawTombstonesOpt is a sstable.Reader option for disabling // fragmentation of the range tombstones returned by @@ -22,7 +22,8 @@ var SSTableRawTombstonesOpt interface{} // tool/make_test_sstables.go. var SSTableWriterDisableKeyOrderChecks func(interface{}) -// SSTableInternalTableOpt is an sstable.Writer option that sets properties for -// sstables being created by the db itself (i.e. through flushes and -// compactions), as opposed to those meant for ingestion. -var SSTableInternalTableOpt interface{} +// SSTableInternalProperties is a func(*sstable.Writer) *sstable.Properties +// function that allows Pebble-internal code to mutate properties that external +// sstable writers are not permitted to edit. It's an untyped interface{} to +// avoid a cyclic dependency. +var SSTableInternalProperties interface{} diff --git a/vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go b/vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go index 3e3cf11..a25b64d 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go +++ b/vendor/github.com/cockroachdb/pebble/internal/rangekey/coalesce.go @@ -6,6 +6,7 @@ package rangekey import ( "bytes" + "math" "sort" "github.com/cockroachdb/pebble/internal/base" @@ -15,7 +16,41 @@ import ( ) // UserIteratorConfig holds state for constructing the range key iterator stack -// for user iteration. +// for user iteration. The range key iterator must merge range key spans across +// the levels of the LSM. This merging is performed by a keyspan.MergingIter +// on-the-fly. The UserIteratorConfig implements keyspan.Transformer, evaluating +// range-key semantics and shadowing, so the spans returned by a MergingIter are +// fully resolved. +// +// The MergingIter is wrapped by a BoundedIter, which elides spans that are +// outside the iterator bounds (or the current prefix's bounds, during prefix +// iteration mode). +// +// To provide determinisim during iteration, the BoundedIter is wrapped by a +// DefragmentingIter that defragments abutting spans with identical +// user-observable state. +// +// At the top-level an InterleavingIter interleaves range keys with point keys +// and performs truncation to iterator bounds. +// +// Below is an abbreviated diagram illustrating the mechanics of a SeekGE. +// +// InterleavingIter.SeekGE +// │ +// DefragmentingIter.SeekGE +// │ +// BoundedIter.SeekGE +// │ +// ╭────────────────┴───────────────╮ +// │ ├── defragmentBwd* +// MergingIter.SeekGE │ +// │ ╰── defragmentFwd +// ╰─╶╶ per level╶╶ ─╮ +// │ +// │ +// ├── .SeekLT +// │ +// ╰── .Next type UserIteratorConfig struct { snapshot uint64 comparer *base.Comparer @@ -24,14 +59,29 @@ type UserIteratorConfig struct { diter keyspan.DefragmentingIter liters [manifest.NumLevels]keyspan.LevelIter litersUsed int - sortBuf keysBySuffix + onlySets bool + bufs *Buffers +} + +// Buffers holds various buffers used for range key iteration. They're exposed +// so that they may be pooled and reused between iterators. +type Buffers struct { + merging keyspan.MergingBuffers + defragmenting keyspan.DefragmentingBuffers + sortBuf keyspan.KeysBySuffix +} + +// PrepareForReuse discards any excessively large buffers. +func (bufs *Buffers) PrepareForReuse() { + bufs.merging.PrepareForReuse() + bufs.defragmenting.PrepareForReuse() } // Init initializes the range key iterator stack for user iteration. The // resulting fragment iterator applies range key semantics, defragments spans -// according to their user-observable state and removes all Keys other than -// RangeKeySets describing the current state of range keys. The resulting spans -// contain Keys sorted by Suffix. +// according to their user-observable state and, if onlySets = true, removes all +// Keys other than RangeKeySets describing the current state of range keys. The +// resulting spans contain Keys sorted by Suffix. // // The snapshot sequence number parameter determines which keys are visible. Any // keys not visible at the provided snapshot are ignored. @@ -41,14 +91,18 @@ func (ui *UserIteratorConfig) Init( lower, upper []byte, hasPrefix *bool, prefix *[]byte, + onlySets bool, + bufs *Buffers, iters ...keyspan.FragmentIterator, ) keyspan.FragmentIterator { ui.snapshot = snapshot ui.comparer = comparer - ui.miter.Init(comparer.Compare, ui, iters...) + ui.onlySets = onlySets + ui.miter.Init(comparer.Compare, ui, &bufs.merging, iters...) ui.biter.Init(comparer.Compare, comparer.Split, &ui.miter, lower, upper, hasPrefix, prefix) - ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer) + ui.diter.Init(comparer, &ui.biter, ui, keyspan.StaticDefragmentReducer, &bufs.defragmenting) ui.litersUsed = 0 + ui.bufs = bufs return &ui.diter } @@ -87,17 +141,17 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k // Apply shadowing of keys. dst.Start = s.Start dst.End = s.End - ui.sortBuf = keysBySuffix{ - cmp: cmp, - keys: dst.Keys[:0], + ui.bufs.sortBuf = keyspan.KeysBySuffix{ + Cmp: cmp, + Keys: ui.bufs.sortBuf.Keys[:0], } - if err := coalesce(&ui.sortBuf, s.Visible(ui.snapshot).Keys, &dst.Keys); err != nil { + if err := coalesce(ui.comparer.Equal, &ui.bufs.sortBuf, ui.snapshot, s.Keys); err != nil { return err } - // During user iteration over range keys, unsets and deletes don't - // matter. Remove them. This step helps logical defragmentation during - // iteration. - keys := dst.Keys + // During user iteration over range keys, unsets and deletes don't matter. + // Remove them if onlySets = true. This step helps logical defragmentation + // during iteration. + keys := ui.bufs.sortBuf.Keys dst.Keys = dst.Keys[:0] for i := range keys { switch keys[i].Kind() { @@ -110,11 +164,17 @@ func (ui *UserIteratorConfig) Transform(cmp base.Compare, s keyspan.Span, dst *k if invariants.Enabled && len(dst.Keys) > 0 && cmp(dst.Keys[len(dst.Keys)-1].Suffix, keys[i].Suffix) > 0 { panic("pebble: keys unexpectedly not in ascending suffix order") } - // Skip. - continue + if ui.onlySets { + // Skip. + continue + } + dst.Keys = append(dst.Keys, keys[i]) case base.InternalKeyKindRangeKeyDelete: - // Skip. - continue + if ui.onlySets { + // Skip. + continue + } + dst.Keys = append(dst.Keys, keys[i]) default: return base.CorruptionErrorf("pebble: unrecognized range key kind %s", keys[i].Kind()) } @@ -148,8 +208,8 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S ret := true for i := range a.Keys { if invariants.Enabled { - if a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet || - b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet { + if ui.onlySets && (a.Keys[i].Kind() != base.InternalKeyKindRangeKeySet || + b.Keys[i].Kind() != base.InternalKeyKindRangeKeySet) { panic("pebble: unexpected non-RangeKeySet during defragmentation") } if i > 0 && (ui.comparer.Compare(a.Keys[i].Suffix, a.Keys[i-1].Suffix) < 0 || @@ -193,8 +253,9 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S // // Additionally, internal range keys at the same sequence number have subtle // mechanics: -// * RANGEKEYSETs shadow RANGEKEYUNSETs of the same suffix. -// * RANGEKEYDELs only apply to keys at lower sequence numbers. +// - RANGEKEYSETs shadow RANGEKEYUNSETs of the same suffix. +// - RANGEKEYDELs only apply to keys at lower sequence numbers. +// // This is required for ingestion. Ingested sstables are assigned a single // sequence number for the file, at which all of the file's keys are visible. // The RANGEKEYSET, RANGEKEYUNSET and RANGEKEYDEL key kinds are ordered such @@ -204,98 +265,105 @@ func (ui *UserIteratorConfig) ShouldDefragment(equal base.Equal, a, b *keyspan.S // set or unset but not both. // // The resulting dst Keys slice is sorted by Trailer. -func Coalesce(cmp base.Compare, keys []keyspan.Key, dst *[]keyspan.Key) error { +func Coalesce(cmp base.Compare, eq base.Equal, keys []keyspan.Key, dst *[]keyspan.Key) error { // TODO(jackson): Currently, Coalesce doesn't actually perform the sequence // number promotion described in the comment above. - keysBySuffix := keysBySuffix{ - cmp: cmp, - keys: (*dst)[:0], + keysBySuffix := keyspan.KeysBySuffix{ + Cmp: cmp, + Keys: (*dst)[:0], } - if err := coalesce(&keysBySuffix, keys, dst); err != nil { + if err := coalesce(eq, &keysBySuffix, math.MaxUint64, keys); err != nil { return err } - // coalesce left the keys in *dst sorted by suffix. Re-sort them by trailer. + // Update the span with the (potentially reduced) keys slice. coalesce left + // the keys in *dst sorted by suffix. Re-sort them by trailer. + *dst = keysBySuffix.Keys keyspan.SortKeysByTrailer(dst) return nil } -func coalesce(keysBySuffix *keysBySuffix, keys []keyspan.Key, dst *[]keyspan.Key) error { - var deleted bool - for i := 0; i < len(keys) && !deleted; i++ { - k := keys[i] - if invariants.Enabled && i > 0 && k.Trailer > keys[i-1].Trailer { +func coalesce( + equal base.Equal, keysBySuffix *keyspan.KeysBySuffix, snapshot uint64, keys []keyspan.Key, +) error { + // First, enforce visibility and RangeKeyDelete mechanics. We only need to + // consider the prefix of keys before and including the first + // RangeKeyDelete. We also must skip any keys that aren't visible at the + // provided snapshot sequence number. + // + // NB: Within a given sequence number, keys are ordered as: + // RangeKeySet > RangeKeyUnset > RangeKeyDelete + // This is significant, because this ensures that a Set or Unset sharing a + // sequence number with a Delete do not shadow each other. + deleteIdx := -1 + for i := range keys { + if invariants.Enabled && i > 0 && keys[i].Trailer > keys[i-1].Trailer { panic("pebble: invariant violation: span keys unordered") } + if !keys[i].VisibleAt(snapshot) { + continue + } + // Once a RangeKeyDelete is observed, we know it shadows all subsequent + // keys and we can break early. We don't add the RangeKeyDelete key to + // keysBySuffix.keys yet, because we don't want a suffix-less key + // that appeared earlier in the slice to elide it. It'll be added back + // in at the end. + if keys[i].Kind() == base.InternalKeyKindRangeKeyDelete { + deleteIdx = i + break + } + keysBySuffix.Keys = append(keysBySuffix.Keys, keys[i]) + } - // NB: Within a given sequence number, keys are ordered as: - // RangeKeySet > RangeKeyUnset > RangeKeyDelete - // This is significant, because this ensures that none of the range keys - // sharing a sequence number shadow each other. - switch k.Kind() { - case base.InternalKeyKindRangeKeySet: - n := len(keysBySuffix.keys) + // Sort the accumulated keys by suffix. There may be duplicates within a + // suffix, in which case the one with a larger trailer survives. + // + // We use a stable sort so that the first key with a given suffix is the one + // that with the highest Trailer (because the input `keys` was sorted by + // trailer descending). + sort.Stable(keysBySuffix) - if keysBySuffix.get(n, k.Suffix) < n { - // This suffix is already set or unset at a higher sequence - // number. Skip. - continue + // Grab a handle of the full sorted slice, before reslicing + // keysBySuffix.keys to accumulate the final coalesced keys. + sorted := keysBySuffix.Keys + keysBySuffix.Keys = keysBySuffix.Keys[:0] + + var ( + // prevSuffix is updated on each iteration of the below loop, and + // compared by the subsequent iteration to determine whether adjacent + // keys are defined at the same suffix. + prevSuffix []byte + // shadowing is set to true once any Key is shadowed by another key. + // When it's set to true—or after the loop if no keys are shadowed—the + // keysBySuffix.keys slice is resliced to contain the prefix of + // unshadowed keys. This avoids copying them incrementally in the common + // case of no shadowing. + shadowing bool + ) + for i := range sorted { + if i > 0 && equal(prevSuffix, sorted[i].Suffix) { + // Skip; this key is shadowed by the predecessor that had a larger + // Trailer. If this is the first shadowed key, set shadowing=true + // and reslice keysBySuffix.keys to hold the entire unshadowed + // prefix. + if !shadowing { + keysBySuffix.Keys = keysBySuffix.Keys[:i] + shadowing = true } - keysBySuffix.keys = append(keysBySuffix.keys, k) - sort.Sort(keysBySuffix) - case base.InternalKeyKindRangeKeyUnset: - n := len(keysBySuffix.keys) - - if keysBySuffix.get(n, k.Suffix) < n { - // This suffix is already set or unset at a higher sequence - // number. Skip. - continue - } - keysBySuffix.keys = append(keysBySuffix.keys, k) - sort.Sort(keysBySuffix) - case base.InternalKeyKindRangeKeyDelete: - // All remaining range keys in this span have been deleted by this - // RangeKeyDelete. There's no need to continue looping, because all - // the remaining keys are shadowed by this one. The for loop - // condition will terminate when it sees the last key is a - // range key deletion. - keysBySuffix.keys = append(keysBySuffix.keys, k) - deleted = true - default: - return base.CorruptionErrorf("pebble: unexpected range key kind %s", k.Kind()) + continue + } + prevSuffix = sorted[i].Suffix + if shadowing { + keysBySuffix.Keys = append(keysBySuffix.Keys, sorted[i]) } } - - // Update the span with the (potentially reduced) keys slice. - // NB: We don't re-sort by Trailer. The exported Coalesce function however - // will. - *dst = keysBySuffix.keys + // If there was no shadowing, keysBySuffix.keys is untouched. We can simply + // set it to the existing `sorted` slice (also backed by keysBySuffix.keys). + if !shadowing { + keysBySuffix.Keys = sorted + } + // If the original input `keys` slice contained a RangeKeyDelete, add it. + if deleteIdx >= 0 { + keysBySuffix.Keys = append(keysBySuffix.Keys, keys[deleteIdx]) + } return nil } - -type keysBySuffix struct { - cmp base.Compare - keys []keyspan.Key -} - -// get searches for suffix among the first n keys in keys. If the suffix is -// found, it returns the index of the item with the suffix. If the suffix is not -// found, it returns n. -func (s *keysBySuffix) get(n int, suffix []byte) (i int) { - // Binary search for the suffix to see if there's an existing key with the - // suffix. Only binary search among the first n items. get is called while - // appending new keys with suffixes that may sort before existing keys. - // The n parameter indicates what portion of the keys slice is sorted and - // may contain relevant keys. - - i = sort.Search(n, func(i int) bool { - return s.cmp(s.keys[i].Suffix, suffix) >= 0 - }) - if i < n && s.cmp(s.keys[i].Suffix, suffix) == 0 { - return i - } - return n -} - -func (s *keysBySuffix) Len() int { return len(s.keys) } -func (s *keysBySuffix) Less(i, j int) bool { return s.cmp(s.keys[i].Suffix, s.keys[j].Suffix) < 0 } -func (s *keysBySuffix) Swap(i, j int) { s.keys[i], s.keys[j] = s.keys[j], s.keys[i] } diff --git a/vendor/github.com/cockroachdb/pebble/internal/rate/rate.go b/vendor/github.com/cockroachdb/pebble/internal/rate/rate.go index abd1dde..565626f 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/rate/rate.go +++ b/vendor/github.com/cockroachdb/pebble/internal/rate/rate.go @@ -196,13 +196,15 @@ func (lim *Limiter) Reserve() Reservation { // The Limiter takes this Reservation into account when allowing future events. // ReserveN returns false if n exceeds the Limiter's burst size. // Usage example: -// r := lim.ReserveN(time.Now(), 1) -// if !r.OK() { -// // Not allowed to act! Did you remember to set lim.burst to be > 0 ? -// return -// } -// time.Sleep(r.Delay()) -// Act() +// +// r := lim.ReserveN(time.Now(), 1) +// if !r.OK() { +// // Not allowed to act! Did you remember to set lim.burst to be > 0 ? +// return +// } +// time.Sleep(r.Delay()) +// Act() +// // Use this method if you wish to wait and slow down in accordance with the rate limit without dropping events. // If you need to respect a deadline or cancel the delay, use Wait instead. // To drop or skip events exceeding rate limit, use Allow instead. diff --git a/vendor/github.com/cockroachdb/pebble/internal/rawalloc/rawalloc_64bit.go b/vendor/github.com/cockroachdb/pebble/internal/rawalloc/rawalloc_64bit.go index ea1d952..c97ba33 100644 --- a/vendor/github.com/cockroachdb/pebble/internal/rawalloc/rawalloc_64bit.go +++ b/vendor/github.com/cockroachdb/pebble/internal/rawalloc/rawalloc_64bit.go @@ -12,8 +12,8 @@ // implied. See the License for the specific language governing // permissions and limitations under the License. -//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 -// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 +//go:build amd64 || arm64 || arm64be || ppc64 || ppc64le || mips64 || mips64le || s390x || sparc64 || riscv64 +// +build amd64 arm64 arm64be ppc64 ppc64le mips64 mips64le s390x sparc64 riscv64 package rawalloc diff --git a/vendor/github.com/cockroachdb/pebble/iterator.go b/vendor/github.com/cockroachdb/pebble/iterator.go index 23288d0..3348274 100644 --- a/vendor/github.com/cockroachdb/pebble/iterator.go +++ b/vendor/github.com/cockroachdb/pebble/iterator.go @@ -6,13 +6,14 @@ package pebble import ( "bytes" + "context" "io" "sync" - "sync/atomic" "unsafe" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/bytealloc" "github.com/cockroachdb/pebble/internal/fastrand" "github.com/cockroachdb/pebble/internal/humanize" "github.com/cockroachdb/pebble/internal/invariants" @@ -34,9 +35,9 @@ import ( // always a single internalIterator position corresponding to the position // returned to the user. Consider the example: // -// a.MERGE.9 a.MERGE.8 a.MERGE.7 a.SET.6 b.DELETE.9 b.DELETE.5 b.SET.4 -// \ / -// \ Iterator.Key() = 'a' / +// a.MERGE.9 a.MERGE.8 a.MERGE.7 a.SET.6 b.DELETE.9 b.DELETE.5 b.SET.4 +// \ / +// \ Iterator.Key() = 'a' / // // The Iterator exposes one valid position at user key 'a' and the two exhausted // positions at the beginning and end of iteration. The underlying @@ -118,6 +119,7 @@ type IteratorStats struct { // ReverseStepCount includes Prev. ReverseStepCount [NumStatsKind]int InternalStats InternalIteratorStats + RangeKeyStats RangeKeyIteratorStats } var _ redact.SafeFormatter = &IteratorStats{} @@ -126,6 +128,32 @@ var _ redact.SafeFormatter = &IteratorStats{} // iterators. type InternalIteratorStats = base.InternalIteratorStats +// RangeKeyIteratorStats contains miscellaneous stats about range keys +// encountered by the iterator. +type RangeKeyIteratorStats struct { + // Count records the number of range keys encountered during + // iteration. Range keys may be counted multiple times if the iterator + // leaves a range key's bounds and then returns. + Count int + // ContainedPoints records the number of point keys encountered within the + // bounds of a range key. Note that this includes point keys with suffixes + // that sort both above and below the covering range key's suffix. + ContainedPoints int + // SkippedPoints records the count of the subset of ContainedPoints point + // keys that were skipped during iteration due to range-key masking. It does + // not include point keys that were never loaded because a + // RangeKeyMasking.Filter excluded the entire containing block. + SkippedPoints int +} + +// Merge adds all of the argument's statistics to the receiver. It may be used +// to accumulate stats across multiple iterators. +func (s *RangeKeyIteratorStats) Merge(o RangeKeyIteratorStats) { + s.Count += o.Count + s.ContainedPoints += o.ContainedPoints + s.SkippedPoints += o.SkippedPoints +} + // LazyValue is a lazy value. See the long comment in base.LazyValue. type LazyValue = base.LazyValue @@ -149,6 +177,11 @@ type LazyValue = base.LazyValue // Next, Prev) return without advancing if the iterator has an accumulated // error. type Iterator struct { + // The context is stored here since (a) Iterators are expected to be + // short-lived (since they pin memtables and sstables), (b) plumbing a + // context into every method is very painful, (c) they do not (yet) respect + // context cancellation and are only used for tracing. + ctx context.Context opts IterOptions merge Merge comparer base.Comparer @@ -172,6 +205,7 @@ type Iterator struct { value LazyValue // For use in LazyValue.Clone. valueBuf []byte + fetcher base.LazyFetcher // For use in LazyValue.Value. lazyValueBuf []byte valueCloser io.Closer @@ -278,6 +312,9 @@ type Iterator struct { closePointIterOnce bool // Used in some tests to disable the random disabling of seek optimizations. forceEnableSeekOpt bool + // Set to true if NextPrefix is not currently permitted. Defaults to false + // in case an iterator never had any bounds. + nextPrefixNotPermittedByUpperBound bool } // cmp is a convenience shorthand for the i.comparer.Compare function. @@ -343,11 +380,8 @@ type iteratorRangeKeyState struct { // start and end are the [start, end) boundaries of the current range keys. start []byte end []byte - // keys is sorted by Suffix ascending. - keys []RangeKeyData - // buf is used to save range-key data before moving the range-key iterator. - // Start and end boundaries, suffixes and values are all copied into buf. - buf []byte + + rangeKeyBuffers // iterConfig holds fields that are used for the construction of the // iterator stack, but do not need to be directly accessed during iteration. @@ -356,6 +390,31 @@ type iteratorRangeKeyState struct { iterConfig rangekey.UserIteratorConfig } +type rangeKeyBuffers struct { + // keys is sorted by Suffix ascending. + keys []RangeKeyData + // buf is used to save range-key data before moving the range-key iterator. + // Start and end boundaries, suffixes and values are all copied into buf. + buf bytealloc.A + // internal holds buffers used by the range key internal iterators. + internal rangekey.Buffers +} + +func (b *rangeKeyBuffers) PrepareForReuse() { + const maxKeysReuse = 100 + if len(b.keys) > maxKeysReuse { + b.keys = nil + } + // Avoid caching the key buf if it is overly large. The constant is + // fairly arbitrary. + if cap(b.buf) >= maxKeyBufCacheSize { + b.buf = nil + } else { + b.buf = b.buf[:0] + } + b.internal.PrepareForReuse() +} + func (i *iteratorRangeKeyState) init(cmp base.Compare, split base.Split, opts *IterOptions) { i.cmp = cmp i.split = split @@ -717,16 +776,16 @@ func (i *Iterator) sampleRead() { if len(mi.levels) > 1 { mi.ForEachLevelIter(func(li *levelIter) bool { l := manifest.LevelToInt(li.level) - if file := li.files.Current(); file != nil { + if f := li.iterFile; f != nil { var containsKey bool if i.pos == iterPosNext || i.pos == iterPosCurForward || i.pos == iterPosCurForwardPaused { - containsKey = i.cmp(file.SmallestPointKey.UserKey, i.key) <= 0 + containsKey = i.cmp(f.SmallestPointKey.UserKey, i.key) <= 0 } else if i.pos == iterPosPrev || i.pos == iterPosCurReverse || i.pos == iterPosCurReversePaused { - containsKey = i.cmp(file.LargestPointKey.UserKey, i.key) >= 0 + containsKey = i.cmp(f.LargestPointKey.UserKey, i.key) >= 0 } - // Do nothing if the current key is not contained in file's + // Do nothing if the current key is not contained in f's // bounds. We could seek the LevelIterator at this level // to find the right file, but the performance impacts of // doing that are significant enough to negate the benefits @@ -740,7 +799,7 @@ func (i *Iterator) sampleRead() { return true } topLevel = l - topFile = file + topFile = f } } return false @@ -751,14 +810,14 @@ func (i *Iterator) sampleRead() { return } if numOverlappingLevels >= 2 { - allowedSeeks := atomic.AddInt64(&topFile.Atomic.AllowedSeeks, -1) + allowedSeeks := topFile.AllowedSeeks.Add(-1) if allowedSeeks == 0 { // Since the compaction queue can handle duplicates, we can keep // adding to the queue even once allowedSeeks hits 0. // In fact, we NEED to keep adding to the queue, because the queue // is small and evicts older and possibly useful compactions. - atomic.AddInt64(&topFile.Atomic.AllowedSeeks, topFile.InitAllowedSeeks) + topFile.AllowedSeeks.Add(topFile.InitAllowedSeeks) read := readCompaction{ start: topFile.SmallestPointKey.UserKey, @@ -899,7 +958,7 @@ func (i *Iterator) findPrevEntry(limit []byte) { // call, so use valueBuf instead. Note that valueBuf is only used // in this one instance; everywhere else (eg. in findNextEntry), // we just point i.value to the unsafe i.iter-owned value buffer. - i.value, i.valueBuf = i.iterValue.Clone(i.valueBuf[:0]) + i.value, i.valueBuf = i.iterValue.Clone(i.valueBuf[:0], &i.fetcher) i.saveRangeKey() i.iterValidityState = IterValid i.iterKey, i.iterValue = i.iter.Prev() @@ -1226,7 +1285,7 @@ func (i *Iterator) SeekGEWithLimit(key []byte, limit []byte) IterValidityState { // An example Split function may separate a timestamp suffix from the prefix of // the key. // -// Split(@) -> +// Split(@) -> // // Consider the keys "a@1", "a@2", "aa@3", "aa@4". The prefixes for these keys // are "a", and "aa". Note that despite "a" and "aa" sharing a prefix by the @@ -1234,20 +1293,20 @@ func (i *Iterator) SeekGEWithLimit(key []byte, limit []byte) IterValidityState { // function. To see how this works, consider the following set of calls on this // data set: // -// SeekPrefixGE("a@0") -> "a@1" -// Next() -> "a@2" -// Next() -> EOF +// SeekPrefixGE("a@0") -> "a@1" +// Next() -> "a@2" +// Next() -> EOF // // If you're just looking to iterate over keys with a shared prefix, as // defined by the configured comparer, set iterator bounds instead: // -// iter := db.NewIter(&pebble.IterOptions{ -// LowerBound: []byte("prefix"), -// UpperBound: []byte("prefiy"), -// }) -// for iter.First(); iter.Valid(); iter.Next() { -// // Only keys beginning with "prefix" will be visited. -// } +// iter := db.NewIter(&pebble.IterOptions{ +// LowerBound: []byte("prefix"), +// UpperBound: []byte("prefiy"), +// }) +// for iter.First(); iter.Valid(); iter.Next() { +// // Only keys beginning with "prefix" will be visited. +// } // // See ExampleIterator_SeekPrefixGE for a working example. // @@ -1487,13 +1546,7 @@ func (i *Iterator) First() bool { i.requiresReposition = false i.stats.ForwardSeekCount[InterfaceCall]++ - if lowerBound := i.opts.GetLowerBound(); lowerBound != nil { - i.iterKey, i.iterValue = i.iter.SeekGE(lowerBound, base.SeekGEFlagsNone) - i.stats.ForwardSeekCount[InternalIterCall]++ - } else { - i.iterKey, i.iterValue = i.iter.First() - i.stats.ForwardSeekCount[InternalIterCall]++ - } + i.iterFirstWithinBounds() i.findNextEntry(nil) i.maybeSampleRead() return i.iterValidityState == IterValid @@ -1525,13 +1578,7 @@ func (i *Iterator) Last() bool { i.requiresReposition = false i.stats.ReverseSeekCount[InterfaceCall]++ - if upperBound := i.opts.GetUpperBound(); upperBound != nil { - i.iterKey, i.iterValue = i.iter.SeekLT(upperBound, base.SeekLTFlagsNone) - i.stats.ReverseSeekCount[InternalIterCall]++ - } else { - i.iterKey, i.iterValue = i.iter.Last() - i.stats.ReverseSeekCount[InternalIterCall]++ - } + i.iterLastWithinBounds() i.findPrevEntry(nil) i.maybeSampleRead() return i.iterValidityState == IterValid @@ -1540,7 +1587,7 @@ func (i *Iterator) Last() bool { // Next moves the iterator to the next key/value pair. Returns true if the // iterator is pointing at a valid entry and false otherwise. func (i *Iterator) Next() bool { - return i.NextWithLimit(nil) == IterValid + return i.nextWithLimit(nil) == IterValid } // NextWithLimit moves the iterator to the next key/value pair. @@ -1554,6 +1601,161 @@ func (i *Iterator) Next() bool { // guarantees it will surface any range keys with bounds overlapping the // keyspace up to limit. func (i *Iterator) NextWithLimit(limit []byte) IterValidityState { + return i.nextWithLimit(limit) +} + +// NextPrefix moves the iterator to the next key/value pair with a key +// containing a different prefix than the current key. Prefixes are determined +// by Comparer.Split. Exhausts the iterator if invoked while in prefix-iteration +// mode. +// +// It is not permitted to invoke NextPrefix while at a IterAtLimit position. +// When called in this condition, NextPrefix has non-deterministic behavior. +// +// It is not permitted to invoke NextPrefix when the Iterator has an +// upper-bound that is a versioned MVCC key (see the comment for +// Comparer.Split). It returns an error in this case. +func (i *Iterator) NextPrefix() bool { + if i.nextPrefixNotPermittedByUpperBound { + i.lastPositioningOp = unknownLastPositionOp + i.requiresReposition = false + i.err = errors.Errorf("NextPrefix not permitted with upper bound %s", + i.comparer.FormatKey(i.opts.UpperBound)) + i.iterValidityState = IterExhausted + return false + } + if i.hasPrefix { + i.iterValidityState = IterExhausted + return false + } + return i.nextPrefix() == IterValid +} + +func (i *Iterator) nextPrefix() IterValidityState { + if i.rangeKey != nil { + // NB: Check Valid() before clearing requiresReposition. + i.rangeKey.prevPosHadRangeKey = i.rangeKey.hasRangeKey && i.Valid() + // If we have a range key but did not expose it at the previous iterator + // position (because the iterator was not at a valid position), updated + // must be true. This ensures that after an iterator op sequence like: + // - Next() → (IterValid, RangeBounds() = [a,b)) + // - NextWithLimit(...) → (IterAtLimit, RangeBounds() = -) + // - NextWithLimit(...) → (IterValid, RangeBounds() = [a,b)) + // the iterator returns RangeKeyChanged()=true. + // + // The remainder of this function will only update i.rangeKey.updated if + // the iterator moves into a new range key, or out of the current range + // key. + i.rangeKey.updated = i.rangeKey.hasRangeKey && !i.Valid() && i.opts.rangeKeys() + } + + // Although NextPrefix documents that behavior at IterAtLimit is undefined, + // this function handles these cases as a simple prefix-agnostic Next. This + // is done for deterministic behavior in the metamorphic tests. + // + // TODO(jackson): If the metamorphic test operation generator is adjusted to + // make generation of some operations conditional on the previous + // operations, then we can remove this behavior and explicitly error. + + i.lastPositioningOp = unknownLastPositionOp + i.requiresReposition = false + switch i.pos { + case iterPosCurForward: + // Positioned on the current key. Advance to the next prefix. + i.internalNextPrefix(i.split(i.key)) + case iterPosCurForwardPaused: + // Positioned at a limit. Implement as a prefix-agnostic Next. See TODO + // up above. The iterator is already positioned at the next key. + case iterPosCurReverse: + // Switching directions. + // Unless the iterator was exhausted, reverse iteration needs to + // position the iterator at iterPosPrev. + if i.iterKey != nil { + i.err = errors.New("switching from reverse to forward but iter is not at prev") + i.iterValidityState = IterExhausted + return i.iterValidityState + } + // The Iterator is exhausted and i.iter is positioned before the first + // key. Reposition to point to the first internal key. + i.iterFirstWithinBounds() + case iterPosCurReversePaused: + // Positioned at a limit. Implement as a prefix-agnostic Next. See TODO + // up above. + // + // Switching directions; The iterator must not be exhausted since it + // paused. + if i.iterKey == nil { + i.err = errors.New("switching paused from reverse to forward but iter is exhausted") + i.iterValidityState = IterExhausted + return i.iterValidityState + } + i.nextUserKey() + case iterPosPrev: + // The underlying iterator is pointed to the previous key (this can + // only happen when switching iteration directions). + if i.iterKey == nil { + // We're positioned before the first key. Need to reposition to point to + // the first key. + i.iterFirstWithinBounds() + } else { + // Move the internal iterator back onto the user key stored in + // i.key. iterPosPrev guarantees that it's positioned at the last + // key with the user key less than i.key, so we're guaranteed to + // land on the correct key with a single Next. + i.iterKey, i.iterValue = i.iter.Next() + if invariants.Enabled && !i.equal(i.iterKey.UserKey, i.key) { + i.opts.logger.Fatalf("pebble: invariant violation: Nexting internal iterator from iterPosPrev landed on %q, not %q", + i.iterKey.UserKey, i.key) + } + } + // The internal iterator is now positioned at i.key. Advance to the next + // prefix. + i.internalNextPrefix(i.split(i.key)) + case iterPosNext: + // Already positioned on the next key. Only call nextPrefixKey if the + // next key shares the same prefix. + if i.iterKey != nil { + currKeyPrefixLen := i.split(i.key) + iterKeyPrefixLen := i.split(i.iterKey.UserKey) + if bytes.Equal(i.iterKey.UserKey[:iterKeyPrefixLen], i.key[:currKeyPrefixLen]) { + i.internalNextPrefix(currKeyPrefixLen) + } + } + } + + i.stats.ForwardStepCount[InterfaceCall]++ + i.findNextEntry(nil /* limit */) + i.maybeSampleRead() + return i.iterValidityState +} + +func (i *Iterator) internalNextPrefix(currKeyPrefixLen int) { + if i.iterKey == nil { + return + } + // The Next "fast-path" is not really a fast-path when there is more than + // one version. However, even with TableFormatPebblev3, there is a small + // slowdown (~10%) for one version if we remove it and only call NextPrefix. + // When there are two versions, only calling NextPrefix is ~30% faster. + i.stats.ForwardStepCount[InternalIterCall]++ + if i.iterKey, i.iterValue = i.iter.Next(); i.iterKey == nil { + return + } + iterKeyPrefixLen := i.split(i.iterKey.UserKey) + if !bytes.Equal(i.iterKey.UserKey[:iterKeyPrefixLen], i.key[:currKeyPrefixLen]) { + return + } + i.stats.ForwardStepCount[InternalIterCall]++ + i.prefixOrFullSeekKey = i.comparer.ImmediateSuccessor(i.prefixOrFullSeekKey[:0], i.key[:currKeyPrefixLen]) + i.iterKey, i.iterValue = i.iter.NextPrefix(i.prefixOrFullSeekKey) + if invariants.Enabled && i.iterKey != nil { + if iterKeyPrefixLen := i.split(i.iterKey.UserKey); i.cmp(i.iterKey.UserKey[:iterKeyPrefixLen], i.prefixOrFullSeekKey) < 0 { + panic("pebble: iter.NextPrefix did not advance beyond the current prefix") + } + } +} + +func (i *Iterator) nextWithLimit(limit []byte) IterValidityState { i.stats.ForwardStepCount[InterfaceCall]++ if i.hasPrefix { if limit != nil { @@ -1607,13 +1809,7 @@ func (i *Iterator) NextWithLimit(limit []byte) IterValidityState { } // We're positioned before the first key. Need to reposition to point to // the first key. - if lowerBound := i.opts.GetLowerBound(); lowerBound != nil { - i.iterKey, i.iterValue = i.iter.SeekGE(lowerBound, base.SeekGEFlagsNone) - i.stats.ForwardSeekCount[InternalIterCall]++ - } else { - i.iterKey, i.iterValue = i.iter.First() - i.stats.ForwardSeekCount[InternalIterCall]++ - } + i.iterFirstWithinBounds() case iterPosCurReversePaused: // Switching directions. // The iterator must not be exhausted since it paused. @@ -1633,13 +1829,7 @@ func (i *Iterator) NextWithLimit(limit []byte) IterValidityState { if i.iterKey == nil { // We're positioned before the first key. Need to reposition to point to // the first key. - if lowerBound := i.opts.GetLowerBound(); lowerBound != nil { - i.iterKey, i.iterValue = i.iter.SeekGE(lowerBound, base.SeekGEFlagsNone) - i.stats.ForwardSeekCount[InternalIterCall]++ - } else { - i.iterKey, i.iterValue = i.iter.First() - i.stats.ForwardSeekCount[InternalIterCall]++ - } + i.iterFirstWithinBounds() } else { i.nextUserKey() } @@ -1737,13 +1927,7 @@ func (i *Iterator) PrevWithLimit(limit []byte) IterValidityState { if i.iterKey == nil { // We're positioned after the last key. Need to reposition to point to // the last key. - if upperBound := i.opts.GetUpperBound(); upperBound != nil { - i.iterKey, i.iterValue = i.iter.SeekLT(upperBound, base.SeekLTFlagsNone) - i.stats.ReverseSeekCount[InternalIterCall]++ - } else { - i.iterKey, i.iterValue = i.iter.Last() - i.stats.ReverseSeekCount[InternalIterCall]++ - } + i.iterLastWithinBounds() } else { i.prevUserKey() } @@ -1756,6 +1940,28 @@ func (i *Iterator) PrevWithLimit(limit []byte) IterValidityState { return i.iterValidityState } +// iterFirstWithinBounds moves the internal iterator to the first key, +// respecting bounds. +func (i *Iterator) iterFirstWithinBounds() { + i.stats.ForwardSeekCount[InternalIterCall]++ + if lowerBound := i.opts.GetLowerBound(); lowerBound != nil { + i.iterKey, i.iterValue = i.iter.SeekGE(lowerBound, base.SeekGEFlagsNone) + } else { + i.iterKey, i.iterValue = i.iter.First() + } +} + +// iterLastWithinBounds moves the internal iterator to the last key, respecting +// bounds. +func (i *Iterator) iterLastWithinBounds() { + i.stats.ReverseSeekCount[InternalIterCall]++ + if upperBound := i.opts.GetUpperBound(); upperBound != nil { + i.iterKey, i.iterValue = i.iter.SeekLT(upperBound, base.SeekLTFlagsNone) + } else { + i.iterKey, i.iterValue = i.iter.Last() + } +} + // RangeKeyData describes a range key's data, set through RangeKeySet. The key // boundaries of the range key is provided by Iterator.RangeBounds. type RangeKeyData struct { @@ -1827,14 +2033,13 @@ func (i *Iterator) saveRangeKey() { i.rangeKey.hasRangeKey = true return } - + i.stats.RangeKeyStats.Count += len(s.Keys) + i.rangeKey.buf.Reset() i.rangeKey.hasRangeKey = true i.rangeKey.updated = true i.rangeKey.stale = false - i.rangeKey.buf = append(i.rangeKey.buf[:0], s.Start...) - i.rangeKey.start = i.rangeKey.buf - i.rangeKey.buf = append(i.rangeKey.buf, s.End...) - i.rangeKey.end = i.rangeKey.buf[len(i.rangeKey.buf)-len(s.End):] + i.rangeKey.buf, i.rangeKey.start = i.rangeKey.buf.Copy(s.Start) + i.rangeKey.buf, i.rangeKey.end = i.rangeKey.buf.Copy(s.End) i.rangeKey.keys = i.rangeKey.keys[:0] for j := 0; j < len(s.Keys); j++ { if invariants.Enabled { @@ -1844,14 +2049,10 @@ func (i *Iterator) saveRangeKey() { panic("pebble: user iteration encountered range keys not in suffix order") } } - i.rangeKey.buf = append(i.rangeKey.buf, s.Keys[j].Suffix...) - suffix := i.rangeKey.buf[len(i.rangeKey.buf)-len(s.Keys[j].Suffix):] - i.rangeKey.buf = append(i.rangeKey.buf, s.Keys[j].Value...) - value := i.rangeKey.buf[len(i.rangeKey.buf)-len(s.Keys[j].Value):] - i.rangeKey.keys = append(i.rangeKey.keys, RangeKeyData{ - Suffix: suffix, - Value: value, - }) + var rkd RangeKeyData + i.rangeKey.buf, rkd.Suffix = i.rangeKey.buf.Copy(s.Keys[j].Suffix) + i.rangeKey.buf, rkd.Value = i.rangeKey.buf.Copy(s.Keys[j].Value) + i.rangeKey.keys = append(i.rangeKey.keys, rkd) } } @@ -1959,6 +2160,8 @@ func (i *Iterator) Error() error { return i.err } +const maxKeyBufCacheSize = 4 << 10 // 4 KB + // Close closes the iterator and returns any accumulated error. Exhausting // all the key/value pairs in a table is not considered to be an error. // It is not valid to call any method, including Close, after the iterator @@ -2023,15 +2226,12 @@ func (i *Iterator) Close() error { i.valueCloser = nil } - const maxKeyBufCacheSize = 4 << 10 // 4 KB - if i.rangeKey != nil { - // Avoid caching the key buf if it is overly large. The constant is - // fairly arbitrary. - if cap(i.rangeKey.buf) >= maxKeyBufCacheSize { - i.rangeKey.buf = nil + + i.rangeKey.rangeKeyBuffers.PrepareForReuse() + *i.rangeKey = iteratorRangeKeyState{ + rangeKeyBuffers: i.rangeKey.rangeKeyBuffers, } - *i.rangeKey = iteratorRangeKeyState{buf: i.rangeKey.buf} iterRangeKeyStateAllocPool.Put(i.rangeKey) i.rangeKey = nil } @@ -2098,7 +2298,7 @@ func (i *Iterator) SetBounds(lower, upper []byte) { // Copy the user-provided bounds into an Iterator-owned buffer, and set them // on i.opts.{Lower,Upper}Bound. - i.saveBounds(lower, upper) + i.processBounds(lower, upper) i.iter.SetBounds(i.opts.LowerBound, i.opts.UpperBound) // If the iterator has an open point iterator that's not currently being @@ -2120,7 +2320,10 @@ func (i *Iterator) SetBounds(lower, upper []byte) { i.invalidate() } -func (i *Iterator) saveBounds(lower, upper []byte) { +// Initialization and changing of the bounds must call processBounds. +// processBounds saves the bounds and computes derived state from those +// bounds. +func (i *Iterator) processBounds(lower, upper []byte) { // Copy the user-provided bounds into an Iterator-owned buffer. We can't // overwrite the current bounds, because some internal iterators compare old // and new bounds for optimizations. @@ -2132,9 +2335,19 @@ func (i *Iterator) saveBounds(lower, upper []byte) { } else { i.opts.LowerBound = nil } + i.nextPrefixNotPermittedByUpperBound = false if upper != nil { buf = append(buf, upper...) i.opts.UpperBound = buf[len(buf)-len(upper):] + if i.comparer.Split != nil { + if i.comparer.Split(i.opts.UpperBound) != len(i.opts.UpperBound) { + // Setting an upper bound that is a versioned MVCC key. This means + // that a key can have some MVCC versions before the upper bound and + // some after. This causes significant complications for NextPrefix, + // so we bar the user of NextPrefix. + i.nextPrefixNotPermittedByUpperBound = true + } + } } else { i.opts.UpperBound = nil } @@ -2319,7 +2532,7 @@ func (i *Iterator) SetOptions(o *IterOptions) { i.opts.LowerBound, i.opts.UpperBound = lower, upper } else { i.opts = *o - i.saveBounds(o.LowerBound, o.UpperBound) + i.processBounds(o.LowerBound, o.UpperBound) // Propagate the changed bounds to the existing point iterator. // NB: We propagate i.opts.{Lower,Upper}Bound, not o.{Lower,Upper}Bound // because i.opts now point to buffers owned by Pebble. @@ -2338,10 +2551,10 @@ func (i *Iterator) SetOptions(o *IterOptions) { // Iterators created through NewExternalIter have a different iterator // initialization process. if i.externalReaders != nil { - finishInitializingExternal(i) + finishInitializingExternal(i.ctx, i) return } - finishInitializingIter(i.alloc) + finishInitializingIter(i.ctx, i.alloc) } func (i *Iterator) invalidate() { @@ -2421,6 +2634,12 @@ type CloneOptions struct { // will cause an increase in memory and disk usage (use NewSnapshot for that // purpose). func (i *Iterator) Clone(opts CloneOptions) (*Iterator, error) { + return i.CloneWithContext(context.Background(), opts) +} + +// CloneWithContext is like Clone, and additionally accepts a context for +// tracing. +func (i *Iterator) CloneWithContext(ctx context.Context, opts CloneOptions) (*Iterator, error) { if opts.IterOptions == nil { opts.IterOptions = &i.opts } @@ -2436,6 +2655,7 @@ func (i *Iterator) Clone(opts CloneOptions) (*Iterator, error) { buf := iterAllocPool.Get().(*iterAlloc) dbi := &buf.dbi *dbi = Iterator{ + ctx: ctx, opts: *opts.IterOptions, alloc: buf, merge: i.merge, @@ -2450,7 +2670,7 @@ func (i *Iterator) Clone(opts CloneOptions) (*Iterator, error) { newIterRangeKey: i.newIterRangeKey, seqNum: i.seqNum, } - dbi.saveBounds(dbi.opts.LowerBound, dbi.opts.UpperBound) + dbi.processBounds(dbi.opts.LowerBound, dbi.opts.UpperBound) // If the caller requested the clone have a current view of the indexed // batch, set the clone's batch sequence number appropriately. @@ -2458,7 +2678,7 @@ func (i *Iterator) Clone(opts CloneOptions) (*Iterator, error) { dbi.batchSeqNum = (uint64(len(i.batch.data)) | base.InternalKeySeqNumBatch) } - return finishInitializingIter(buf), nil + return finishInitializingIter(ctx, buf), nil } // Merge adds all of the argument's statistics to the receiver. It may be used @@ -2471,6 +2691,7 @@ func (stats *IteratorStats) Merge(o IteratorStats) { stats.ReverseStepCount[i] += o.ReverseStepCount[i] } stats.InternalStats.Merge(o.InternalStats) + stats.RangeKeyStats.Merge(o.RangeKeyStats) } func (stats *IteratorStats) String() string { @@ -2492,14 +2713,30 @@ func (stats *IteratorStats) SafeFormat(s redact.SafePrinter, verb rune) { } if stats.InternalStats != (InternalIteratorStats{}) { s.SafeString(",\n(internal-stats: ") - s.Printf("(block-bytes: (total %s, cached %s)), "+ - "(points: (count %s, key-bytes %s, value-bytes %s, tombstoned: %s))", + s.Printf("(block-bytes: (total %s, cached %s, read-time %s)), "+ + "(points: (count %s, key-bytes %s, value-bytes %s, tombstoned %s))", humanize.IEC.Uint64(stats.InternalStats.BlockBytes), humanize.IEC.Uint64(stats.InternalStats.BlockBytesInCache), + humanize.FormattedString(stats.InternalStats.BlockReadDuration.String()), humanize.SI.Uint64(stats.InternalStats.PointCount), humanize.SI.Uint64(stats.InternalStats.KeyBytes), humanize.SI.Uint64(stats.InternalStats.ValueBytes), humanize.SI.Uint64(stats.InternalStats.PointsCoveredByRangeTombstones), ) + if stats.InternalStats.SeparatedPointValue.Count != 0 { + s.Printf(", (separated: (count %s, bytes %s, fetched %s)))", + humanize.SI.Uint64(stats.InternalStats.SeparatedPointValue.Count), + humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytes), + humanize.IEC.Uint64(stats.InternalStats.SeparatedPointValue.ValueBytesFetched)) + } else { + s.Printf(")") + } + } + if stats.RangeKeyStats != (RangeKeyIteratorStats{}) { + s.SafeString(",\n(range-key-stats: ") + s.Printf("(count %d), (contained points: (count %d, skipped %d)))", + stats.RangeKeyStats.Count, + stats.RangeKeyStats.ContainedPoints, + stats.RangeKeyStats.SkippedPoints) } } diff --git a/vendor/github.com/cockroachdb/pebble/level_checker.go b/vendor/github.com/cockroachdb/pebble/level_checker.go index 31fcaf0..4bbd9a4 100644 --- a/vendor/github.com/cockroachdb/pebble/level_checker.go +++ b/vendor/github.com/cockroachdb/pebble/level_checker.go @@ -5,10 +5,10 @@ package pebble import ( + "context" "fmt" "io" "sort" - "sync/atomic" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" @@ -59,7 +59,7 @@ type simpleMergingIterLevel struct { type simpleMergingIter struct { levels []simpleMergingIterLevel snapshot uint64 - heap mergingIterHeap + heap simpleMergingIterHeap // The last point's key and level. For validation. lastKey InternalKey lastLevel int @@ -86,12 +86,12 @@ func (m *simpleMergingIter) init( m.snapshot = snapshot m.lastLevel = -1 m.heap.cmp = cmp - m.heap.items = make([]mergingIterItem, 0, len(levels)) + m.heap.items = make([]simpleMergingIterItem, 0, len(levels)) for i := range m.levels { l := &m.levels[i] l.iterKey, l.iterValue = l.iter.First() if l.iterKey != nil { - item := mergingIterItem{ + item := simpleMergingIterItem{ index: i, value: l.iterValue, } @@ -117,7 +117,7 @@ func (m *simpleMergingIter) positionRangeDels() { if l.rangeDelIter == nil { continue } - l.tombstone = keyspan.SeekGE(m.heap.cmp, l.rangeDelIter, item.key.UserKey) + l.tombstone = l.rangeDelIter.SeekGE(item.key.UserKey) } } @@ -378,7 +378,8 @@ func checkRangeTombstones(c *checkConfig) error { lf := files.Take() atomicUnit, _ := expandToAtomicUnit(c.cmp, lf.Slice(), true /* disableIsCompacting */) lower, upper := manifest.KeyRange(c.cmp, atomicUnit.Iter()) - iterToClose, iter, err := c.newIters(lf.FileMetadata, nil, internalIterOpts{}) + iterToClose, iter, err := c.newIters( + context.Background(), lf.FileMetadata, nil, internalIterOpts{}) if err != nil { return err } @@ -554,11 +555,11 @@ type CheckLevelsStats struct { } // CheckLevels checks: -// - Every entry in the DB is consistent with the level invariant. See the -// comment at the top of the file. -// - Point keys in sstables are ordered. -// - Range delete tombstones in sstables are ordered and fragmented. -// - Successful processing of all MERGE records. +// - Every entry in the DB is consistent with the level invariant. See the +// comment at the top of the file. +// - Point keys in sstables are ordered. +// - Range delete tombstones in sstables are ordered and fragmented. +// - Successful processing of all MERGE records. func (d *DB) CheckLevels(stats *CheckLevelsStats) error { // Grab and reference the current readState. readState := d.loadReadState() @@ -566,7 +567,7 @@ func (d *DB) CheckLevels(stats *CheckLevelsStats) error { // Determine the seqnum to read at after grabbing the read state (current and // memtables) above. - seqNum := atomic.LoadUint64(&d.mu.versions.atomic.visibleSeqNum) + seqNum := d.mu.versions.visibleSeqNum.Load() checkConfig := &checkConfig{ logger: d.opts.Logger, @@ -638,7 +639,7 @@ func checkLevelsInternal(c *checkConfig) (err error) { manifestIter := current.L0SublevelFiles[sublevel].Iter() iterOpts := IterOptions{logger: c.logger} li := &levelIter{} - li.init(iterOpts, c.cmp, nil /* split */, c.newIters, manifestIter, + li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters, manifestIter, manifest.L0Sublevel(sublevel), internalIterOpts{}) li.initRangeDel(&mlevelAlloc[0].rangeDelIter) li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext) @@ -652,7 +653,7 @@ func checkLevelsInternal(c *checkConfig) (err error) { iterOpts := IterOptions{logger: c.logger} li := &levelIter{} - li.init(iterOpts, c.cmp, nil /* split */, c.newIters, + li.init(context.Background(), iterOpts, c.cmp, nil /* split */, c.newIters, current.Levels[level].Iter(), manifest.Level(level), internalIterOpts{}) li.initRangeDel(&mlevelAlloc[0].rangeDelIter) li.initBoundaryContext(&mlevelAlloc[0].levelIterBoundaryContext) @@ -674,3 +675,92 @@ func checkLevelsInternal(c *checkConfig) (err error) { // Phase 2: Check that the tombstones are mutually consistent. return checkRangeTombstones(c) } + +type simpleMergingIterItem struct { + index int + key InternalKey + value base.LazyValue +} + +type simpleMergingIterHeap struct { + cmp Compare + reverse bool + items []simpleMergingIterItem +} + +func (h *simpleMergingIterHeap) len() int { + return len(h.items) +} + +func (h *simpleMergingIterHeap) less(i, j int) bool { + ikey, jkey := h.items[i].key, h.items[j].key + if c := h.cmp(ikey.UserKey, jkey.UserKey); c != 0 { + if h.reverse { + return c > 0 + } + return c < 0 + } + if h.reverse { + return ikey.Trailer < jkey.Trailer + } + return ikey.Trailer > jkey.Trailer +} + +func (h *simpleMergingIterHeap) swap(i, j int) { + h.items[i], h.items[j] = h.items[j], h.items[i] +} + +// init, fix, up and down are copied from the go stdlib. +func (h *simpleMergingIterHeap) init() { + // heapify + n := h.len() + for i := n/2 - 1; i >= 0; i-- { + h.down(i, n) + } +} + +func (h *simpleMergingIterHeap) fix(i int) { + if !h.down(i, h.len()) { + h.up(i) + } +} + +func (h *simpleMergingIterHeap) pop() *simpleMergingIterItem { + n := h.len() - 1 + h.swap(0, n) + h.down(0, n) + item := &h.items[n] + h.items = h.items[:n] + return item +} + +func (h *simpleMergingIterHeap) up(j int) { + for { + i := (j - 1) / 2 // parent + if i == j || !h.less(j, i) { + break + } + h.swap(i, j) + j = i + } +} + +func (h *simpleMergingIterHeap) down(i0, n int) bool { + i := i0 + for { + j1 := 2*i + 1 + if j1 >= n || j1 < 0 { // j1 < 0 after int overflow + break + } + j := j1 // left child + if j2 := j1 + 1; j2 < n && h.less(j2, j1) { + j = j2 // = 2*i + 2 // right child + } + if !h.less(j, i) { + break + } + h.swap(i, j) + i = j + } + return i > i0 +} diff --git a/vendor/github.com/cockroachdb/pebble/level_iter.go b/vendor/github.com/cockroachdb/pebble/level_iter.go index a366685..feb809d 100644 --- a/vendor/github.com/cockroachdb/pebble/level_iter.go +++ b/vendor/github.com/cockroachdb/pebble/level_iter.go @@ -5,9 +5,9 @@ package pebble import ( + "context" "fmt" "runtime/debug" - "unsafe" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/invariants" @@ -17,14 +17,38 @@ import ( ) // tableNewIters creates a new point and range-del iterator for the given file -// number. If bytesIterated is specified, it is incremented as the given file is -// iterated through. +// number. +// +// On success, the internalIterator is not-nil and must be closed; the +// FragmentIterator can be nil. +// TODO(radu): always return a non-nil FragmentIterator. +// +// On error, the iterators are nil. +// +// The only (non-test) implementation of tableNewIters is tableCacheContainer.newIters(). type tableNewIters func( + ctx context.Context, file *manifest.FileMetadata, opts *IterOptions, internalOpts internalIterOpts, ) (internalIterator, keyspan.FragmentIterator, error) +// tableNewRangeDelIter takes a tableNewIters and returns a TableNewSpanIter +// for the rangedel iterator returned by tableNewIters. +func tableNewRangeDelIter(ctx context.Context, newIters tableNewIters) keyspan.TableNewSpanIter { + return func(file *manifest.FileMetadata, iterOptions *keyspan.SpanIterOptions) (keyspan.FragmentIterator, error) { + iter, rangeDelIter, err := newIters( + ctx, file, &IterOptions{RangeKeyFilters: iterOptions.RangeKeyFilters}, internalIterOpts{}) + if iter != nil { + _ = iter.Close() + } + if rangeDelIter == nil { + rangeDelIter = emptyKeyspanIter + } + return rangeDelIter, err + } +} + type internalIterOpts struct { bytesIterated *uint64 stats *base.InternalIteratorStats @@ -54,6 +78,11 @@ type internalIterOpts struct { // kind InternalKeyKindRangeDeletion which will be used to pause the levelIter // at the sstable until the mergingIter is ready to advance past it. type levelIter struct { + // The context is stored here since (a) iterators are expected to be + // short-lived (since they pin sstables), (b) plumbing a context into every + // method is very painful, (c) they do not (yet) respect context + // cancellation and are only used for tracing. + ctx context.Context logger Logger cmp Compare split Split @@ -91,7 +120,8 @@ type levelIter struct { // - err != nil // - some other constraint, like the bounds in opts, caused the file at index to not // be relevant to the iteration. - iter internalIterator + iter internalIterator + // iterFile holds the current file. It is always equal to l.files.Current(). iterFile *fileMetadata // filteredIter is an optional interface that may be implemented by internal // iterators that perform filtering of keys. When a new file's iterator is @@ -214,11 +244,13 @@ func newLevelIter( bytesIterated *uint64, ) *levelIter { l := &levelIter{} - l.init(opts, cmp, split, newIters, files, level, internalIterOpts{bytesIterated: bytesIterated}) + l.init(context.Background(), opts, cmp, split, newIters, files, level, + internalIterOpts{bytesIterated: bytesIterated}) return l } func (l *levelIter) init( + ctx context.Context, opts IterOptions, cmp Compare, split Split, @@ -227,6 +259,7 @@ func (l *levelIter) init( level manifest.Level, internalOpts internalIterOpts, ) { + l.ctx = ctx l.err = nil l.level = level l.logger = opts.getLogger() @@ -313,9 +346,10 @@ func (l *levelIter) maybeTriggerCombinedIteration(file *fileMetadata, dir int) { func (l *levelIter) findFileGE(key []byte, flags base.SeekGEFlags) *fileMetadata { // Find the earliest file whose largest key is >= key. - if invariants.Enabled && flags.TrySeekUsingNext() && disableSeekOpt(key, uintptr(unsafe.Pointer(l))) { - flags = flags.DisableTrySeekUsingNext() - } + // NB: if flags.TrySeekUsingNext()=true, the levelIter must respect it. If + // the levelIter is positioned at the key P, it must return a key ≥ P. If + // used within a merging iterator, the merging iterator will depend on the + // levelIter only moving forward to maintain heap invariants. // Ordinarily we seek the LevelIterator using SeekGE. In some instances, we // Next instead. In other instances, we try Next-ing first, falling back to @@ -632,7 +666,7 @@ func (l *levelIter) loadFile(file *fileMetadata, dir int) loadFileReturnIndicato var rangeDelIter keyspan.FragmentIterator var iter internalIterator - iter, rangeDelIter, l.err = l.newIters(l.files.Current(), &l.tableOpts, l.internalOpts) + iter, rangeDelIter, l.err = l.newIters(l.ctx, l.iterFile, &l.tableOpts, l.internalOpts) l.iter = iter if l.err != nil { return noFileLoaded @@ -655,7 +689,7 @@ func (l *levelIter) loadFile(file *fileMetadata, dir int) loadFileReturnIndicato if l.boundaryContext != nil { l.boundaryContext.smallestUserKey = file.Smallest.UserKey l.boundaryContext.largestUserKey = file.Largest.UserKey - l.boundaryContext.isLargestUserKeyRangeDelSentinel = file.Largest.IsExclusiveSentinel() + l.boundaryContext.isLargestUserKeyExclusive = file.Largest.IsExclusiveSentinel() } return newFileLoaded } @@ -687,7 +721,6 @@ func (l *levelIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba l.boundaryContext.isSyntheticIterBoundsKey = false l.boundaryContext.isIgnorableBoundaryKey = false } - // NB: the top-level Iterator has already adjusted key based on // IterOptions.LowerBound. loadFileIndicator := l.loadFile(l.findFileGE(key, flags), +1) @@ -864,6 +897,56 @@ func (l *levelIter) Next() (*InternalKey, base.LazyValue) { return l.verify(l.skipEmptyFileForward()) } +func (l *levelIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + if l.err != nil || l.iter == nil { + return nil, base.LazyValue{} + } + if l.boundaryContext != nil { + l.boundaryContext.isSyntheticIterBoundsKey = false + l.boundaryContext.isIgnorableBoundaryKey = false + } + + switch { + case l.largestBoundary != nil: + if l.tableOpts.UpperBound != nil { + // The UpperBound was within this file, so don't load the next + // file. We leave the largestBoundary unchanged so that subsequent + // calls to Next() stay at this file. If a Seek/First/Last call is + // made and this file continues to be relevant, loadFile() will + // set the largestBoundary to nil. + if l.rangeDelIterPtr != nil { + *l.rangeDelIterPtr = nil + } + return nil, base.LazyValue{} + } + // We're stepping past the boundary key, so we need to load a later + // file. + + default: + // Reset the smallest boundary since we're moving away from it. + l.smallestBoundary = nil + + if key, val := l.iter.NextPrefix(succKey); key != nil { + return l.verify(key, val) + } + // Fall through to seeking. + } + + // Seek the manifest level iterator using TrySeekUsingNext=true and + // RelativeSeek=true so that we take advantage of the knowledge that + // `succKey` can only be contained in later files. + metadataSeekFlags := base.SeekGEFlagsNone.EnableTrySeekUsingNext().EnableRelativeSeek() + if l.loadFile(l.findFileGE(succKey, metadataSeekFlags), +1) != noFileLoaded { + // NB: The SeekGE on the file's iterator must not set TrySeekUsingNext, + // because l.iter is unpositioned. + if key, val := l.iter.SeekGE(succKey, base.SeekGEFlagsNone); key != nil { + return l.verify(key, val) + } + return l.verify(l.skipEmptyFileForward()) + } + return nil, base.LazyValue{} +} + func (l *levelIter) Prev() (*InternalKey, base.LazyValue) { if l.err != nil || l.iter == nil { return nil, base.LazyValue{} @@ -955,6 +1038,9 @@ func (l *levelIter) skipEmptyFileForward() (*InternalKey, base.LazyValue) { // If the boundary is a range deletion tombstone, return that key. if l.iterFile.LargestPointKey.Kind() == InternalKeyKindRangeDelete { l.largestBoundary = &l.iterFile.LargestPointKey + if l.boundaryContext != nil { + l.boundaryContext.isIgnorableBoundaryKey = true + } return l.largestBoundary, base.LazyValue{} } // If the last point iterator positioning op might've skipped keys, @@ -1042,6 +1128,9 @@ func (l *levelIter) skipEmptyFileBackward() (*InternalKey, base.LazyValue) { // If the boundary is a range deletion tombstone, return that key. if l.iterFile.SmallestPointKey.Kind() == InternalKeyKindRangeDelete { l.smallestBoundary = &l.iterFile.SmallestPointKey + if l.boundaryContext != nil { + l.boundaryContext.isIgnorableBoundaryKey = true + } return l.smallestBoundary, base.LazyValue{} } // If the last point iterator positioning op skipped keys, it's diff --git a/vendor/github.com/cockroachdb/pebble/log_recycler.go b/vendor/github.com/cockroachdb/pebble/log_recycler.go index 85e64cc..c8c2ff4 100644 --- a/vendor/github.com/cockroachdb/pebble/log_recycler.go +++ b/vendor/github.com/cockroachdb/pebble/log_recycler.go @@ -32,14 +32,14 @@ type logRecycler struct { // the log file should not be deleted (i.e. the log is being recycled), and // false otherwise. func (r *logRecycler) add(logInfo fileInfo) bool { - if logInfo.fileNum < r.minRecycleLogNum { + if logInfo.fileNum.FileNum() < r.minRecycleLogNum { return false } r.mu.Lock() defer r.mu.Unlock() - if logInfo.fileNum <= r.mu.maxLogNum { + if logInfo.fileNum.FileNum() <= r.mu.maxLogNum { // The log file number was already considered for recycling. Don't consider // it again. This avoids a race between adding the same log file for // recycling multiple times, and removing the log file for actual @@ -49,7 +49,7 @@ func (r *logRecycler) add(logInfo fileInfo) bool { // shouldn't be deleted. return true } - r.mu.maxLogNum = logInfo.fileNum + r.mu.maxLogNum = logInfo.fileNum.FileNum() if len(r.mu.logs) >= r.limit { return false } @@ -89,7 +89,7 @@ func (r *logRecycler) pop(logNum FileNum) error { if len(r.mu.logs) == 0 { return errors.New("pebble: log recycler empty") } - if r.mu.logs[0].fileNum != logNum { + if r.mu.logs[0].fileNum.FileNum() != logNum { return errors.Errorf("pebble: log recycler invalid %d vs %d", errors.Safe(logNum), errors.Safe(fileInfoNums(r.mu.logs))) } r.mu.logs = r.mu.logs[1:] @@ -102,7 +102,7 @@ func fileInfoNums(finfos []fileInfo) []FileNum { } nums := make([]FileNum, len(finfos)) for i := range finfos { - nums[i] = finfos[i].fileNum + nums[i] = finfos[i].fileNum.FileNum() } return nums } diff --git a/vendor/github.com/cockroachdb/pebble/logger.go b/vendor/github.com/cockroachdb/pebble/logger.go index 20987c0..5e3e92e 100644 --- a/vendor/github.com/cockroachdb/pebble/logger.go +++ b/vendor/github.com/cockroachdb/pebble/logger.go @@ -4,30 +4,13 @@ package pebble -import ( - "fmt" - "log" - "os" -) +import "github.com/cockroachdb/pebble/internal/base" // Logger defines an interface for writing log messages. -type Logger interface { - Infof(format string, args ...interface{}) - Fatalf(format string, args ...interface{}) -} - -type defaultLogger struct{} +type Logger = base.Logger // DefaultLogger logs to the Go stdlib logs. -var DefaultLogger defaultLogger +var DefaultLogger = base.DefaultLogger -// Infof implements the Logger.Infof interface. -func (defaultLogger) Infof(format string, args ...interface{}) { - _ = log.Output(2, fmt.Sprintf(format, args...)) -} - -// Fatalf implements the Logger.Fatalf interface. -func (defaultLogger) Fatalf(format string, args ...interface{}) { - _ = log.Output(2, fmt.Sprintf(format, args...)) - os.Exit(1) -} +// LoggerAndTracer defines an interface for logging and tracing. +type LoggerAndTracer = base.LoggerAndTracer diff --git a/vendor/github.com/cockroachdb/pebble/mem_table.go b/vendor/github.com/cockroachdb/pebble/mem_table.go index 2108834..86ef7a3 100644 --- a/vendor/github.com/cockroachdb/pebble/mem_table.go +++ b/vendor/github.com/cockroachdb/pebble/mem_table.go @@ -80,7 +80,7 @@ type memTable struct { // inflight mutations that have reserved space in the memtable but not yet // applied. The memtable cannot be flushed to disk until the writer refs // drops to zero. - writerRefs int32 + writerRefs atomic.Int32 tombstones keySpanCache rangeKeys keySpanCache // The current logSeqNum at the time the memtable was created. This is @@ -115,13 +115,13 @@ func newMemTable(opts memTableOptions) *memTable { } m := &memTable{ - cmp: opts.Comparer.Compare, - formatKey: opts.Comparer.FormatKey, - equal: opts.Comparer.Equal, - arenaBuf: opts.arenaBuf, - writerRefs: 1, - logSeqNum: opts.logSeqNum, + cmp: opts.Comparer.Compare, + formatKey: opts.Comparer.FormatKey, + equal: opts.Comparer.Equal, + arenaBuf: opts.arenaBuf, + logSeqNum: opts.logSeqNum, } + m.writerRefs.Store(1) m.tombstones = keySpanCache{ cmp: m.cmp, formatKey: m.formatKey, @@ -143,18 +143,19 @@ func newMemTable(opts memTableOptions) *memTable { m.skl.Reset(arena, m.cmp) m.rangeDelSkl.Reset(arena, m.cmp) m.rangeKeySkl.Reset(arena, m.cmp) + m.reserved = arena.Size() return m } func (m *memTable) writerRef() { - switch v := atomic.AddInt32(&m.writerRefs, 1); { + switch v := m.writerRefs.Add(1); { case v <= 1: panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) } } func (m *memTable) writerUnref() bool { - switch v := atomic.AddInt32(&m.writerRefs, -1); { + switch v := m.writerRefs.Add(-1); { case v < 0: panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) case v == 0: @@ -165,7 +166,7 @@ func (m *memTable) writerUnref() bool { } func (m *memTable) readyForFlush() bool { - return atomic.LoadInt32(&m.writerRefs) == 0 + return m.writerRefs.Load() == 0 } // Prepare reserves space for the batch in the memtable and references the @@ -210,6 +211,8 @@ func (m *memTable) apply(batch *Batch, seqNum uint64) error { // Don't increment seqNum for LogData, since these are not applied // to the memtable. seqNum-- + case InternalKeyKindIngestSST: + panic("pebble: cannot apply ingested sstable key kind to memtable") default: err = ins.Add(&m.skl, ikey, value) } @@ -263,7 +266,7 @@ func (m *memTable) containsRangeKeys() bool { func (m *memTable) availBytes() uint32 { a := m.skl.Arena() - if atomic.LoadInt32(&m.writerRefs) == 1 { + if m.writerRefs.Load() == 1 { // If there are no other concurrent apply operations, we can update the // reserved bytes setting to accurately reflect how many bytes of been // allocated vs the over-estimation present in memTableEntrySize. diff --git a/vendor/github.com/cockroachdb/pebble/merging_iter.go b/vendor/github.com/cockroachdb/pebble/merging_iter.go index 59b279f..4e2426d 100644 --- a/vendor/github.com/cockroachdb/pebble/merging_iter.go +++ b/vendor/github.com/cockroachdb/pebble/merging_iter.go @@ -8,6 +8,7 @@ import ( "bytes" "fmt" "runtime/debug" + "unsafe" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" @@ -16,7 +17,8 @@ import ( ) type mergingIterLevel struct { - iter internalIterator + index int + iter internalIterator // rangeDelIter is set to the range-deletion iterator for the level. When // configured with a levelIter, this pointer changes as sstable boundaries // are crossed. See levelIter.initRangeDel and the Range Deletions comment @@ -44,20 +46,25 @@ type levelIterBoundaryContext struct { // smallestUserKey and largestUserKey are populated with the smallest and // largest boundaries of the current file. smallestUserKey, largestUserKey []byte - // isLargestUserKeyRangeDelSentinel is set to true when a file's largest - // boundary is an exclusive range deletion sentinel. If true, the file does - // not contain any keys with the provided user key, and the largestUserKey - // bound is exclusive. - isLargestUserKeyRangeDelSentinel bool + // isLargestUserKeyExclusive is set to true when a file's largest boundary + // is an exclusive key, (eg, a range deletion sentinel). If true, the file + // does not contain any keys with the provided user key, and the + // largestUserKey bound is exclusive. + isLargestUserKeyExclusive bool // isSyntheticIterBoundsKey is set to true iff the key returned by the level - // iterator is a synthetic key derived from the iterator bounds. This is - // used to prevent the mergingIter from being stuck at such a synthetic key - // if it becomes the top element of the heap. + // iterator is a synthetic key derived from the iterator bounds. This is used + // to prevent the mergingIter from being stuck at such a synthetic key if it + // becomes the top element of the heap. When used with a user-facing Iterator, + // the only range deletions exposed by this mergingIter should be those with + // `isSyntheticIterBoundsKey || isIgnorableBoundaryKey`. isSyntheticIterBoundsKey bool // isIgnorableBoundaryKey is set to true iff the key returned by the level - // iterator is a file boundary key that should be ignored. This is used to + // iterator is a file boundary key that should be ignored when returning to + // the parent iterator. File boundary keys are used by the level iter to // keep a levelIter file's range deletion iterator open as long as other - // levels within the merging iterator require it. + // levels within the merging iterator require it. When used with a user-facing + // Iterator, the only range deletions exposed by this mergingIter should be + // those with `isSyntheticIterBoundsKey || isIgnorableBoundaryKey`. isIgnorableBoundaryKey bool } @@ -73,7 +80,7 @@ type levelIterBoundaryContext struct { // property. When one of the child iterators is exhausted during Next/Prev // iteration, it is removed from the heap. // -// Range Deletions +// # Range Deletions // // A mergingIter can optionally be configured with a slice of range deletion // iterators. The range deletion iterator slice must exactly parallel the point @@ -143,11 +150,11 @@ type levelIterBoundaryContext struct { // to skip processing entries in that shadow. For example, consider the // scenario: // -// r0: a---e -// r1: d---h -// r2: g---k -// r3: j---n -// r4: m---q +// r0: a---e +// r1: d---h +// r2: g---k +// r3: j---n +// r4: m---q // // This is showing 5 levels of range deletions. Consider what happens upon // SeekGE("b"). We first seek the point iterator for level 0 (the point values @@ -189,17 +196,17 @@ type levelIterBoundaryContext struct { // // For a full example, consider the following setup: // -// p0: o -// r0: m---q +// p0: o +// r0: m---q // -// p1: n p -// r1: g---k +// p1: n p +// r1: g---k // -// p2: b d i -// r2: a---e q----v +// p2: b d i +// r2: a---e q----v // -// p3: e -// r3: +// p3: e +// r3: // // If we start iterating from the beginning, the first key we encounter is "b" // in p2. When the mergingIter is pointing at a valid entry, the range deletion @@ -243,12 +250,15 @@ type mergingIter struct { upper []byte stats *InternalIteratorStats + // levelsPositioned, if non-nil, is a slice of the same length as levels. + // It's used by NextPrefix to record which levels have already been + // repositioned. It's created lazily by the first call to NextPrefix. + levelsPositioned []bool + combinedIterState *combinedIterState - // Elide range tombstones from being returned during iteration. Set to true - // when mergingIter is a child of Iterator and the mergingIter is processing - // range tombstones. - elideRangeTombstones bool + // Used in some tests to disable the random disabling of seek optimizations. + forceEnableSeekOpt bool } // mergingIter implements the base.InternalIterator interface. @@ -300,21 +310,20 @@ func (m *mergingIter) init( m.split = split m.stats = stats if cap(m.heap.items) < len(levels) { - m.heap.items = make([]mergingIterItem, 0, len(levels)) + m.heap.items = make([]*mergingIterLevel, 0, len(levels)) } else { m.heap.items = m.heap.items[:0] } + for l := range m.levels { + m.levels[l].index = l + } } func (m *mergingIter) initHeap() { m.heap.items = m.heap.items[:0] for i := range m.levels { if l := &m.levels[i]; l.iterKey != nil { - m.heap.items = append(m.heap.items, mergingIterItem{ - index: i, - key: *l.iterKey, - value: l.iterValue, - }) + m.heap.items = append(m.heap.items, l) } else { m.err = firstError(m.err, l.iter.Error()) if m.err != nil { @@ -343,13 +352,13 @@ func (m *mergingIter) initMinRangeDelIters(oldTopLevel int) { } // Position the range-del iterators at levels <= m.heap.items[0].index. - item := &m.heap.items[0] + item := m.heap.items[0] for level := oldTopLevel + 1; level <= item.index; level++ { l := &m.levels[level] if l.rangeDelIter == nil { continue } - l.tombstone = keyspan.SeekGE(m.heap.cmp, l.rangeDelIter, item.key.UserKey) + l.tombstone = l.rangeDelIter.SeekGE(item.iterKey.UserKey) } } @@ -370,13 +379,13 @@ func (m *mergingIter) initMaxRangeDelIters(oldTopLevel int) { return } // Position the range-del iterators at levels <= m.heap.items[0].index. - item := &m.heap.items[0] + item := m.heap.items[0] for level := oldTopLevel + 1; level <= item.index; level++ { l := &m.levels[level] if l.rangeDelIter == nil { continue } - l.tombstone = keyspan.SeekLE(m.heap.cmp, l.rangeDelIter, item.key.UserKey) + l.tombstone = keyspan.SeekLE(m.heap.cmp, l.rangeDelIter, item.iterKey.UserKey) } } @@ -400,8 +409,8 @@ func (m *mergingIter) switchToMinHeap() { // The current key is a:2 and i2 is pointed at a:1. When we switch to forward // iteration, we want to return a key that is greater than a:2. - key := m.heap.items[0].key - cur := &m.levels[m.heap.items[0].index] + key := m.heap.items[0].iterKey + cur := m.heap.items[0] for i := range m.levels { l := &m.levels[i] @@ -442,7 +451,7 @@ func (m *mergingIter) switchToMinHeap() { } } for ; l.iterKey != nil; l.iterKey, l.iterValue = l.iter.Next() { - if base.InternalCompare(m.heap.cmp, key, *l.iterKey) < 0 { + if base.InternalCompare(m.heap.cmp, *key, *l.iterKey) < 0 { // key < iter-key break } @@ -483,8 +492,8 @@ func (m *mergingIter) switchToMaxHeap() { // // The current key is b:2 and i2 is pointing at b:1. When we switch to // reverse iteration, we want to return a key that is less than b:2. - key := m.heap.items[0].key - cur := &m.levels[m.heap.items[0].index] + key := m.heap.items[0].iterKey + cur := m.heap.items[0] for i := range m.levels { l := &m.levels[i] @@ -524,7 +533,7 @@ func (m *mergingIter) switchToMaxHeap() { } } for ; l.iterKey != nil; l.iterKey, l.iterValue = l.iter.Prev() { - if base.InternalCompare(m.heap.cmp, key, *l.iterKey) > 0 { + if base.InternalCompare(m.heap.cmp, *key, *l.iterKey) > 0 { // key > iter-key break } @@ -547,13 +556,69 @@ func (m *mergingIter) switchToMaxHeap() { m.initMaxHeap() } -// Steps to the next entry. item is the current top item in the heap. -func (m *mergingIter) nextEntry(item *mergingIterItem) { - l := &m.levels[item.index] - oldTopLevel := item.index +// maybeNextEntryWithinPrefix steps to the next entry, as long as the iteration +// prefix has not already been exceeded. If it has, it exhausts the iterator by +// resetting the heap to empty. +func (m *mergingIter) maybeNextEntryWithinPrefix(l *mergingIterLevel) { + if s := m.split(l.iterKey.UserKey); !bytes.Equal(m.prefix, l.iterKey.UserKey[:s]) { + // The item at the root of the heap already exceeds the iteration + // prefix. We should not advance any more. Clear the heap to reflect + // that the iterator is now exhausted (within this prefix, at + // least). + m.heap.items = m.heap.items[:0] + return + } + m.nextEntry(l, nil /* succKey */) +} + +// nextEntry unconditionally steps to the next entry. item is the current top +// item in the heap. +// +// nextEntry should be called directly when not in prefix-iteration mode, or by +// Next. During prefix iteration mode, all other callers should use +// maybeNextEntryWithinPrefix which will avoid advancing the iterator if the +// current iteration prefix has been exhausted. See the comment within +// nextEntry's body for an explanation of why other callers should call +// maybeNextEntryWithinPrefix, which will ensure the documented invariant is +// preserved. +func (m *mergingIter) nextEntry(l *mergingIterLevel, succKey []byte) { + // INVARIANT: If in prefix iteration mode, item.iterKey must have a prefix equal + // to m.prefix. This invariant is important for ensuring TrySeekUsingNext + // optimizations behave correctly. + // + // During prefix iteration, the iterator does not have a full view of the + // LSM. Some level iterators may omit keys that are known to fall outside + // the seek prefix (eg, due to sstable bloom filter exclusion). It's + // important that in such cases we don't position any iterators beyond + // m.prefix, because doing so may interfere with future seeks. + // + // Let prefixes P1 < P2 < P3. Imagine a SeekPrefixGE to prefix P1, followed + // by a SeekPrefixGE to prefix P2. Imagine there exist live keys at prefix + // P2, but they're not visible to the SeekPrefixGE(P1) (because of + // bloom-filter exclusion or a range tombstone that deletes prefix P1 but + // not P2). If the SeekPrefixGE(P1) is allowed to move any level iterators + // to P3, the SeekPrefixGE(P2, TrySeekUsingNext=true) may mistakenly think + // the level contains no point keys or range tombstones within the prefix + // P2. Care is taken to avoid ever advancing the iterator beyond the current + // prefix. If nextEntry is ever invoked while we're already beyond the + // current prefix, we're violating the invariant. + if invariants.Enabled && m.prefix != nil { + if s := m.split(l.iterKey.UserKey); !bytes.Equal(m.prefix, l.iterKey.UserKey[:s]) { + m.logger.Fatalf("mergingIter: prefix violation: nexting beyond prefix %q; existing heap root %q\n%s", + m.prefix, l.iterKey, debug.Stack()) + } + } + + oldTopLevel := l.index oldRangeDelIter := l.rangeDelIter - if l.iterKey, l.iterValue = l.iter.Next(); l.iterKey != nil { - item.key, item.value = *l.iterKey, l.iterValue + + if succKey == nil { + l.iterKey, l.iterValue = l.iter.Next() + } else { + l.iterKey, l.iterValue = l.iter.NextPrefix(succKey) + } + + if l.iterKey != nil { if m.heap.len() > 1 { m.heap.fix(0) } @@ -575,11 +640,15 @@ func (m *mergingIter) nextEntry(item *mergingIterItem) { m.initMinRangeDelIters(oldTopLevel) } -// isNextEntryDeleted() starts from the current entry (as the next entry) and if it is deleted, -// moves the iterators forward as needed and returns true, else it returns false. item is the top -// item in the heap. -func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool { - // Look for a range deletion tombstone containing item.key at higher +// isNextEntryDeleted starts from the current entry (as the next entry) and if +// it is deleted, moves the iterators forward as needed and returns true, else +// it returns false. item is the top item in the heap. +// +// During prefix iteration mode, isNextEntryDeleted will exhaust the iterator by +// clearing the heap if the deleted key(s) extend beyond the iteration prefix +// during prefix-iteration mode. +func (m *mergingIter) isNextEntryDeleted(item *mergingIterLevel) bool { + // Look for a range deletion tombstone containing item.iterKey at higher // levels (level < item.index). If we find such a range tombstone we know // it deletes the key in the current level. Also look for a range // deletion at the current level (level == item.index). If we find such a @@ -593,16 +662,16 @@ func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool { // direction. continue } - if m.heap.cmp(l.tombstone.End, item.key.UserKey) <= 0 { + if m.heap.cmp(l.tombstone.End, item.iterKey.UserKey) <= 0 { // The current key is at or past the tombstone end key. // // NB: for the case that this l.rangeDelIter is provided by a levelIter we know that - // the levelIter must be positioned at a key >= item.key. So it is sufficient to seek the + // the levelIter must be positioned at a key >= item.iterKey. So it is sufficient to seek the // current l.rangeDelIter (since any range del iterators that will be provided by the - // levelIter in the future cannot contain item.key). Also, it is possible that we + // levelIter in the future cannot contain item.iterKey). Also, it is possible that we // will encounter parts of the range delete that should be ignored -- we handle that // below. - l.tombstone = keyspan.SeekGE(m.heap.cmp, l.rangeDelIter, item.key.UserKey) + l.tombstone = l.rangeDelIter.SeekGE(item.iterKey.UserKey) } if l.tombstone == nil { continue @@ -611,9 +680,9 @@ func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool { // Reasoning for correctness of untruncated tombstone handling when the untruncated // tombstone is at a higher level: // The iterator corresponding to this tombstone is still in the heap so it must be - // positioned >= item.key. Which means the Largest key bound of the sstable containing this - // tombstone is >= item.key. So the upper limit of this tombstone cannot be file-bounds-constrained - // to < item.key. But it is possible that item.key < smallestUserKey, in which + // positioned >= item.iterKey. Which means the Largest key bound of the sstable containing this + // tombstone is >= item.iterKey. So the upper limit of this tombstone cannot be file-bounds-constrained + // to < item.iterKey. But it is possible that item.key < smallestUserKey, in which // case this tombstone should be ignored. // // Example 1: @@ -626,11 +695,11 @@ func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool { // does not cover this key. // // For a tombstone at the same level as the key, the file bounds are trivially satisfied. - if (l.smallestUserKey == nil || m.heap.cmp(l.smallestUserKey, item.key.UserKey) <= 0) && - l.tombstone.VisibleAt(m.snapshot) && l.tombstone.Contains(m.heap.cmp, item.key.UserKey) { + if (l.smallestUserKey == nil || m.heap.cmp(l.smallestUserKey, item.iterKey.UserKey) <= 0) && + l.tombstone.VisibleAt(m.snapshot) && l.tombstone.Contains(m.heap.cmp, item.iterKey.UserKey) { if level < item.index { // We could also do m.seekGE(..., level + 1). The levels from - // [level + 1, item.index) are already after item.key so seeking them may be + // [level + 1, item.index) are already after item.iterKey so seeking them may be // wasteful. // We can seek up to the min of largestUserKey and tombstone.End. @@ -642,33 +711,70 @@ func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool { // the exclusive bound of the tombstone. We do not need to look at // isLargestKeyRangeDelSentinel. // - // Progress argument: Since this file is at a higher level than item.key we know + // Progress argument: Since this file is at a higher level than item.iterKey we know // that the iterator in this file must be positioned within its bounds and at a key - // X > item.key (otherwise it would be the min of the heap). It is not - // possible for X.UserKey == item.key.UserKey, since it is incompatible with - // X > item.key (a lower version cannot be in a higher sstable), so it must be that - // X.UserKey > item.key.UserKey. Which means l.largestUserKey > item.key.UserKey. - // We also know that l.tombstone.End > item.key.UserKey. So the min of these, - // seekKey, computed below, is > item.key.UserKey, so the call to seekGE() will + // X > item.iterKey (otherwise it would be the min of the heap). It is not + // possible for X.UserKey == item.iterKey.UserKey, since it is incompatible with + // X > item.iterKey (a lower version cannot be in a higher sstable), so it must be that + // X.UserKey > item.iterKey.UserKey. Which means l.largestUserKey > item.key.UserKey. + // We also know that l.tombstone.End > item.iterKey.UserKey. So the min of these, + // seekKey, computed below, is > item.iterKey.UserKey, so the call to seekGE() will // make forward progress. seekKey := l.tombstone.End if l.largestUserKey != nil && m.heap.cmp(l.largestUserKey, seekKey) < 0 { seekKey = l.largestUserKey } - // This seek is not directly due to a SeekGE call, so we don't - // know enough about the underlying iterator positions, and so - // we keep the try-seek-using-next optimization disabled. + // This seek is not directly due to a SeekGE call, so we don't know + // enough about the underlying iterator positions, and so we keep the + // try-seek-using-next optimization disabled. Additionally, if we're in + // prefix-seek mode and a re-seek would have moved us past the original + // prefix, we can remove all merging iter levels below the rangedel + // tombstone's level and return immediately instead of re-seeking. This + // is correct since those levels cannot provide a key that matches the + // prefix, and is also visible. Additionally, this is important to make + // subsequent `TrySeekUsingNext` work correctly, as a re-seek on a + // different prefix could have resulted in this iterator skipping visible + // keys at prefixes in between m.prefix and seekKey, that are currently + // not in the heap due to a bloom filter mismatch. // // Additionally, we set the relative-seek flag. This is // important when iterating with lazy combined iteration. If // there's a range key between this level's current file and the // file the seek will land on, we need to detect it in order to // trigger construction of the combined iterator. + if m.prefix != nil { + if n := m.split(seekKey); !bytes.Equal(m.prefix, seekKey[:n]) { + for i := item.index; i < len(m.levels); i++ { + // Remove this level from the heap. Setting iterKey and iterValue + // to their zero values should be sufficient for initMinHeap to not + // re-initialize the heap with them in it. Other fields in + // mergingIterLevel can remain as-is; the iter/rangeDelIter needs + // to stay intact for future trySeekUsingNexts to work, the level + // iter boundary context is owned by the levelIter which is not + // being repositioned, and any tombstones in these levels will be + // irrelevant for us anyway. + m.levels[i].iterKey = nil + m.levels[i].iterValue = base.LazyValue{} + } + // TODO(bilal): Consider a more efficient way of removing levels from + // the heap without reinitializing all of it. This would likely + // necessitate tracking the heap positions of each mergingIterHeap + // item in the mergingIterLevel, and then swapping that item in the + // heap with the last-positioned heap item, and shrinking the heap by + // one. + m.initMinHeap() + return true + } + } m.seekGE(seekKey, item.index, base.SeekGEFlagsNone.EnableRelativeSeek()) return true } - if l.tombstone.CoversAt(m.snapshot, item.key.SeqNum()) { - m.nextEntry(item) + if l.tombstone.CoversAt(m.snapshot, item.iterKey.SeqNum()) { + if m.prefix == nil { + m.nextEntry(item, nil /* succKey */) + } else { + m.maybeNextEntryWithinPrefix(item) + } return true } } @@ -678,73 +784,56 @@ func (m *mergingIter) isNextEntryDeleted(item *mergingIterItem) bool { // Starting from the current entry, finds the first (next) entry that can be returned. func (m *mergingIter) findNextEntry() (*InternalKey, base.LazyValue) { - var reseeked bool for m.heap.len() > 0 && m.err == nil { - item := &m.heap.items[0] + item := m.heap.items[0] if m.levels[item.index].isSyntheticIterBoundsKey { break } - // For prefix iteration, stop if we've already exceeded the iterator's - // current prefix. There are two cases where we perform this check. - // - // 1. We already re-seeked the iterator due to a range tombstone. We - // could amortize the cost of this comparison, by doing it only after - // we have iterated in this for loop a few times. But unless we find - // a performance benefit to that, we do the simple thing and compare - // each time. Note that isNextEntryDeleted already did at least 4 key - // comparisons in order to return true, and additionally at least one - // heap comparison to step to the next entry. - // - // Note that we cannot move this comparison into the - // isNextEntryDeleted branch. Once isNextEntryDeleted determines a - // key is deleted and seeks the level's iterator, item.key's memory - // is potentially invalid. If the iterator is now exhausted, item.key - // may be garbage. - // - // 2. If the heap root is an ignorable boundary key, Next-ing the - // iterator will close the associated file and open the next file in - // the level. We don't want to do this unnecessarily because the - // 'exhausted' file may only appear exhausted because it failed a - // bloom filter check. Avoiding Next-ing the file has a performance - // benefit of unnecessarily advancing to the next file, but it's also - // necessary for correctness of some TrySeekUsingNext optimizations - // deeper in the iterator stack: - // - // There may exist valid keys between our current prefix (m.prefix) - // and the ignorable boundary key (item.key) that were not returned - // due to the bloom filter test. A subsequent SeekPrefixGE call with - // a seek key in the range (m.prefix - item.key) will have the - // TrySeekUsingNext flag enabled. The levelIter relies on having not - // skipped these keys in order to avoid more expensive full seeks. - if m.prefix != nil && (reseeked || m.levels[item.index].isIgnorableBoundaryKey) { - if n := m.split(item.key.UserKey); !bytes.Equal(m.prefix, item.key.UserKey[:n]) { - return nil, base.LazyValue{} - } - } m.addItemStats(item) - if m.isNextEntryDeleted(item) { - m.stats.PointsCoveredByRangeTombstones++ - reseeked = true + + // Skip ignorable boundary keys. These are not real keys and exist to + // keep sstables open until we've surpassed their end boundaries so that + // their range deletions are visible. + if m.levels[item.index].isIgnorableBoundaryKey { + if m.prefix == nil { + m.nextEntry(item, nil /* succKey */) + } else { + m.maybeNextEntryWithinPrefix(item) + } continue } - if item.key.Visible(m.snapshot, m.batchSnapshot) && - (!m.levels[item.index].isIgnorableBoundaryKey) && - (item.key.Kind() != InternalKeyKindRangeDelete || !m.elideRangeTombstones) { - return &item.key, item.value + + // Check if the heap root key is deleted by a range tombstone in a + // higher level. If it is, isNextEntryDeleted will advance the iterator + // to a later key (through seeking or nexting). + if m.isNextEntryDeleted(item) { + m.stats.PointsCoveredByRangeTombstones++ + continue } - m.nextEntry(item) + + // Check if the key is visible at the iterator sequence numbers. + if !item.iterKey.Visible(m.snapshot, m.batchSnapshot) { + if m.prefix == nil { + m.nextEntry(item, nil /* succKey */) + } else { + m.maybeNextEntryWithinPrefix(item) + } + continue + } + + // The heap root is visible and not deleted by any range tombstones. + // Return it. + return item.iterKey, item.iterValue } return nil, base.LazyValue{} } // Steps to the prev entry. item is the current top item in the heap. -func (m *mergingIter) prevEntry(item *mergingIterItem) { - l := &m.levels[item.index] - oldTopLevel := item.index +func (m *mergingIter) prevEntry(l *mergingIterLevel) { + oldTopLevel := l.index oldRangeDelIter := l.rangeDelIter if l.iterKey, l.iterValue = l.iter.Prev(); l.iterKey != nil { - item.key, item.value = *l.iterKey, l.iterValue if m.heap.len() > 1 { m.heap.fix(0) } @@ -770,8 +859,8 @@ func (m *mergingIter) prevEntry(item *mergingIterItem) { // isPrevEntryDeleted() starts from the current entry (as the prev entry) and if it is deleted, // moves the iterators backward as needed and returns true, else it returns false. item is the top // item in the heap. -func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool { - // Look for a range deletion tombstone containing item.key at higher +func (m *mergingIter) isPrevEntryDeleted(item *mergingIterLevel) bool { + // Look for a range deletion tombstone containing item.iterKey at higher // levels (level < item.index). If we find such a range tombstone we know // it deletes the key in the current level. Also look for a range // deletion at the current level (level == item.index). If we find such a @@ -785,16 +874,16 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool { // direction. continue } - if m.heap.cmp(item.key.UserKey, l.tombstone.Start) < 0 { + if m.heap.cmp(item.iterKey.UserKey, l.tombstone.Start) < 0 { // The current key is before the tombstone start key. // // NB: for the case that this l.rangeDelIter is provided by a levelIter we know that - // the levelIter must be positioned at a key < item.key. So it is sufficient to seek the + // the levelIter must be positioned at a key < item.iterKey. So it is sufficient to seek the // current l.rangeDelIter (since any range del iterators that will be provided by the - // levelIter in the future cannot contain item.key). Also, it is it is possible that we + // levelIter in the future cannot contain item.iterKey). Also, it is it is possible that we // will encounter parts of the range delete that should be ignored -- we handle that // below. - l.tombstone = keyspan.SeekLE(m.heap.cmp, l.rangeDelIter, item.key.UserKey) + l.tombstone = keyspan.SeekLE(m.heap.cmp, l.rangeDelIter, item.iterKey.UserKey) } if l.tombstone == nil { continue @@ -804,9 +893,9 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool { // tombstone is at a higher level: // // The iterator corresponding to this tombstone is still in the heap so it must be - // positioned <= item.key. Which means the Smallest key bound of the sstable containing this - // tombstone is <= item.key. So the lower limit of this tombstone cannot have been - // file-bounds-constrained to > item.key. But it is possible that item.key >= Largest + // positioned <= item.iterKey. Which means the Smallest key bound of the sstable containing this + // tombstone is <= item.iterKey. So the lower limit of this tombstone cannot have been + // file-bounds-constrained to > item.iterKey. But it is possible that item.key >= Largest // key bound of this sstable, in which case this tombstone should be ignored. // // Example 1: @@ -823,20 +912,20 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool { // Example 3: // sstable bounds [c#8, g#RangeDelSentinel] containing [b, i)#7 and the key is g#10. // This key is not deleted by this tombstone. We need to look at - // isLargestUserKeyRangeDelSentinel. + // isLargestUserKeyExclusive. // // For a tombstone at the same level as the key, the file bounds are trivially satisfied. // Default to within bounds. withinLargestSSTableBound := true if l.largestUserKey != nil { - cmpResult := m.heap.cmp(l.largestUserKey, item.key.UserKey) - withinLargestSSTableBound = cmpResult > 0 || (cmpResult == 0 && !l.isLargestUserKeyRangeDelSentinel) + cmpResult := m.heap.cmp(l.largestUserKey, item.iterKey.UserKey) + withinLargestSSTableBound = cmpResult > 0 || (cmpResult == 0 && !l.isLargestUserKeyExclusive) } - if withinLargestSSTableBound && l.tombstone.Contains(m.heap.cmp, item.key.UserKey) && l.tombstone.VisibleAt(m.snapshot) { + if withinLargestSSTableBound && l.tombstone.Contains(m.heap.cmp, item.iterKey.UserKey) && l.tombstone.VisibleAt(m.snapshot) { if level < item.index { // We could also do m.seekLT(..., level + 1). The levels from - // [level + 1, item.index) are already before item.key so seeking them may be + // [level + 1, item.index) are already before item.iterKey so seeking them may be // wasteful. // We can seek up to the max of smallestUserKey and tombstone.Start.UserKey. @@ -844,10 +933,10 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool { // Using example 1 above, we can seek to the larger of c and b, which is c. // // Progress argument: We know that the iterator in this file is positioned within - // its bounds and at a key X < item.key (otherwise it would be the max of the heap). - // So smallestUserKey <= item.key.UserKey and we already know that - // l.tombstone.Start.UserKey <= item.key.UserKey. So the seekKey computed below - // is <= item.key.UserKey, and since we do a seekLT() we will make backwards + // its bounds and at a key X < item.iterKey (otherwise it would be the max of the heap). + // So smallestUserKey <= item.iterKey.UserKey and we already know that + // l.tombstone.Start.UserKey <= item.iterKey.UserKey. So the seekKey computed below + // is <= item.iterKey.UserKey, and since we do a seekLT() we will make backwards // progress. seekKey := l.tombstone.Start if l.smallestUserKey != nil && m.heap.cmp(l.smallestUserKey, seekKey) > 0 { @@ -861,7 +950,7 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool { m.seekLT(seekKey, item.index, base.SeekLTFlagsNone.EnableRelativeSeek()) return true } - if l.tombstone.CoversAt(m.snapshot, item.key.SeqNum()) { + if l.tombstone.CoversAt(m.snapshot, item.iterKey.SeqNum()) { m.prevEntry(item) return true } @@ -873,7 +962,7 @@ func (m *mergingIter) isPrevEntryDeleted(item *mergingIterItem) bool { // Starting from the current entry, finds the first (prev) entry that can be returned. func (m *mergingIter) findPrevEntry() (*InternalKey, base.LazyValue) { for m.heap.len() > 0 && m.err == nil { - item := &m.heap.items[0] + item := m.heap.items[0] if m.levels[item.index].isSyntheticIterBoundsKey { break } @@ -882,10 +971,9 @@ func (m *mergingIter) findPrevEntry() (*InternalKey, base.LazyValue) { m.stats.PointsCoveredByRangeTombstones++ continue } - if item.key.Visible(m.snapshot, m.batchSnapshot) && - (!m.levels[item.index].isIgnorableBoundaryKey) && - (item.key.Kind() != InternalKeyKindRangeDelete || !m.elideRangeTombstones) { - return &item.key, item.value + if item.iterKey.Visible(m.snapshot, m.batchSnapshot) && + (!m.levels[item.index].isIgnorableBoundaryKey) { + return item.iterKey, item.iterValue } m.prevEntry(item) } @@ -915,6 +1003,37 @@ func (m *mergingIter) seekGE(key []byte, level int, flags base.SeekGEFlags) { // level (and any lower levels) when the current iterator position is // contained within a range tombstone at a higher level. + // Deterministically disable the TrySeekUsingNext optimizations sometimes in + // invariant builds to encourage the metamorphic tests to surface bugs. Note + // that we cannot disable the optimization within individual levels. It must + // be disabled for all levels or none. If one lower-level iterator performs + // a fresh seek whereas another takes advantage of its current iterator + // position, the heap can become inconsistent. Consider the following + // example: + // + // L5: [ [b-c) ] [ d ]* + // L6: [ b ] [e]* + // + // Imagine a SeekGE(a). The [b-c) range tombstone deletes the L6 point key + // 'b', resulting in the iterator positioned at d with the heap: + // + // {L5: d, L6: e} + // + // A subsequent SeekGE(b) is seeking to a larger key, so the caller may set + // TrySeekUsingNext()=true. If the L5 iterator used the TrySeekUsingNext + // optimization but the L6 iterator did not, the iterator would have the + // heap: + // + // {L6: b, L5: d} + // + // Because the L5 iterator has already advanced to the next sstable, the + // merging iterator cannot observe the [b-c) range tombstone and will + // mistakenly return L6's deleted point key 'b'. + if invariants.Enabled && flags.TrySeekUsingNext() && !m.forceEnableSeekOpt && + disableSeekOpt(key, uintptr(unsafe.Pointer(m))) { + flags = flags.DisableTrySeekUsingNext() + } + for ; level < len(m.levels); level++ { if invariants.Enabled && m.lower != nil && m.heap.cmp(key, m.lower) < 0 { m.logger.Fatalf("mergingIter: lower bound violation: %s < %s\n%s", key, m.lower, debug.Stack()) @@ -950,7 +1069,7 @@ func (m *mergingIter) seekGE(key []byte, level int, flags base.SeekGEFlags) { // so we can have a sstable with bounds [c#8, i#InternalRangeDelSentinel], and the // tombstone is [b, k)#8 and the seek key is i: levelIter.SeekGE(i) will move past // this sstable since it realizes the largest key is a InternalRangeDelSentinel. - l.tombstone = keyspan.SeekGE(m.heap.cmp, rangeDelIter, key) + l.tombstone = rangeDelIter.SeekGE(key) if l.tombstone != nil && l.tombstone.VisibleAt(m.snapshot) && l.tombstone.Contains(m.heap.cmp, key) && (l.smallestUserKey == nil || m.heap.cmp(l.smallestUserKey, key) <= 0) { // NB: Based on the comment above l.largestUserKey >= key, and based on the @@ -1040,7 +1159,7 @@ func (m *mergingIter) seekLT(key []byte, level int, flags base.SeekLTFlags) { withinLargestSSTableBound := true if l.largestUserKey != nil { cmpResult := m.heap.cmp(l.largestUserKey, key) - withinLargestSSTableBound = cmpResult > 0 || (cmpResult == 0 && !l.isLargestUserKeyRangeDelSentinel) + withinLargestSSTableBound = cmpResult > 0 || (cmpResult == 0 && !l.isLargestUserKeyExclusive) } l.tombstone = keyspan.SeekLE(m.heap.cmp, rangeDelIter, key) @@ -1123,7 +1242,59 @@ func (m *mergingIter) Next() (*InternalKey, base.LazyValue) { return nil, base.LazyValue{} } - m.nextEntry(&m.heap.items[0]) + // NB: It's okay to call nextEntry directly even during prefix iteration + // mode (as opposed to indirectly through maybeNextEntryWithinPrefix). + // During prefix iteration mode, we rely on the caller to not call Next if + // the iterator has already advanced beyond the iteration prefix. See the + // comment above the base.InternalIterator interface. + m.nextEntry(m.heap.items[0], nil /* succKey */) + return m.findNextEntry() +} + +func (m *mergingIter) NextPrefix(succKey []byte) (*InternalKey, LazyValue) { + if m.dir != 1 { + panic("pebble: cannot switch directions with NextPrefix") + } + if m.err != nil || m.heap.len() == 0 { + return nil, LazyValue{} + } + if m.levelsPositioned == nil { + m.levelsPositioned = make([]bool, len(m.levels)) + } else { + for i := range m.levelsPositioned { + m.levelsPositioned[i] = false + } + } + + // The heap root necessarily must be positioned at a key < succKey, because + // NextPrefix was invoked. + root := &m.heap.items[0] + m.levelsPositioned[(*root).index] = true + if invariants.Enabled && m.heap.cmp((*root).iterKey.UserKey, succKey) >= 0 { + m.logger.Fatalf("pebble: invariant violation: NextPrefix(%q) called on merging iterator already positioned at %q", + succKey, (*root).iterKey) + } + m.nextEntry(*root, succKey) + // NB: root is a pointer to the heap root. nextEntry may have changed + // the heap root, so we must not expect root to still point to the same + // level (or to even be valid, if the heap is now exhaused). + + for m.heap.len() > 0 { + if m.levelsPositioned[(*root).index] { + // A level we've previously positioned is at the top of the heap, so + // there are no other levels positioned at keys < succKey. We've + // advanced as far as we need to. + break + } + // Since this level was not the original heap root when NextPrefix was + // called, we don't know whether this level's current key has the + // previous prefix or a new one. + if m.heap.cmp((*root).iterKey.UserKey, succKey) >= 0 { + break + } + m.levelsPositioned[(*root).index] = true + m.nextEntry(*root, succKey) + } return m.findNextEntry() } @@ -1145,7 +1316,7 @@ func (m *mergingIter) Prev() (*InternalKey, base.LazyValue) { return nil, base.LazyValue{} } - m.prevEntry(&m.heap.items[0]) + m.prevEntry(m.heap.items[0]) return m.findPrevEntry() } @@ -1188,7 +1359,7 @@ func (m *mergingIter) DebugString() string { sep := "" for m.heap.len() > 0 { item := m.heap.pop() - fmt.Fprintf(&buf, "%s%s", sep, item.key) + fmt.Fprintf(&buf, "%s%s", sep, item.iterKey) sep = " " } if m.dir == 1 { @@ -1209,10 +1380,10 @@ func (m *mergingIter) ForEachLevelIter(fn func(li *levelIter) bool) { } } -func (m *mergingIter) addItemStats(item *mergingIterItem) { +func (m *mergingIter) addItemStats(l *mergingIterLevel) { m.stats.PointCount++ - m.stats.KeyBytes += uint64(len(item.key.UserKey)) - m.stats.ValueBytes += uint64(len(item.value.ValueOrHandle)) + m.stats.KeyBytes += uint64(len(l.iterKey.UserKey)) + m.stats.ValueBytes += uint64(len(l.iterValue.ValueOrHandle)) } var _ internalIterator = &mergingIter{} diff --git a/vendor/github.com/cockroachdb/pebble/merging_iter_heap.go b/vendor/github.com/cockroachdb/pebble/merging_iter_heap.go index d4f22a6..c8c336f 100644 --- a/vendor/github.com/cockroachdb/pebble/merging_iter_heap.go +++ b/vendor/github.com/cockroachdb/pebble/merging_iter_heap.go @@ -4,18 +4,10 @@ package pebble -import "github.com/cockroachdb/pebble/internal/base" - -type mergingIterItem struct { - index int - key InternalKey - value base.LazyValue -} - type mergingIterHeap struct { cmp Compare reverse bool - items []mergingIterItem + items []*mergingIterLevel } func (h *mergingIterHeap) len() int { @@ -27,7 +19,7 @@ func (h *mergingIterHeap) clear() { } func (h *mergingIterHeap) less(i, j int) bool { - ikey, jkey := h.items[i].key, h.items[j].key + ikey, jkey := h.items[i].iterKey, h.items[j].iterKey if c := h.cmp(ikey.UserKey, jkey.UserKey); c != 0 { if h.reverse { return c > 0 @@ -59,11 +51,11 @@ func (h *mergingIterHeap) fix(i int) { } } -func (h *mergingIterHeap) pop() *mergingIterItem { +func (h *mergingIterHeap) pop() *mergingIterLevel { n := h.len() - 1 h.swap(0, n) h.down(0, n) - item := &h.items[n] + item := h.items[n] h.items = h.items[:n] return item } diff --git a/vendor/github.com/cockroachdb/pebble/metrics.go b/vendor/github.com/cockroachdb/pebble/metrics.go index bcc5754..cd90d03 100644 --- a/vendor/github.com/cockroachdb/pebble/metrics.go +++ b/vendor/github.com/cockroachdb/pebble/metrics.go @@ -6,6 +6,7 @@ package pebble import ( "fmt" + "strings" "time" "github.com/cockroachdb/pebble/internal/base" @@ -78,6 +79,18 @@ type LevelMetrics struct { TablesIngested uint64 // The number of sstables moved to this level by a "move" compaction. TablesMoved uint64 + // Additional contains misc additional metrics that are not always printed. + Additional struct { + // The sum of Properties.ValueBlocksSize for all the sstables in this + // level. Printed by LevelMetrics.format iff there is at least one level + // with a non-zero value. + ValueBlocksSize uint64 + // Cumulative metrics about bytes written to data blocks and value blocks, + // via compactions (except move compactions) or flushes. Not printed by + // LevelMetrics.format, but are available to sophisticated clients. + BytesWrittenDataBlocks uint64 + BytesWrittenValueBlocks uint64 + } } // Add updates the counter metrics for the level. @@ -94,6 +107,9 @@ func (m *LevelMetrics) Add(u *LevelMetrics) { m.TablesFlushed += u.TablesFlushed m.TablesIngested += u.TablesIngested m.TablesMoved += u.TablesMoved + m.Additional.BytesWrittenDataBlocks += u.Additional.BytesWrittenDataBlocks + m.Additional.BytesWrittenValueBlocks += u.Additional.BytesWrittenValueBlocks + m.Additional.ValueBlocksSize += u.Additional.ValueBlocksSize } // WriteAmp computes the write amplification for compactions at this @@ -107,8 +123,10 @@ func (m *LevelMetrics) WriteAmp() float64 { // format generates a string of the receiver's metrics, formatting it into the // supplied buffer. -func (m *LevelMetrics) format(w redact.SafePrinter, score redact.SafeValue) { - w.Printf("%9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7d %7.1f\n", +func (m *LevelMetrics) format( + w redact.SafePrinter, score redact.SafeValue, includeValueBlocksSize bool, +) { + w.Printf("%9d %7s %7s %7s %7s %7s %7s %7s %7s %7s %7s %7d %7.1f", redact.Safe(m.NumFiles), humanize.IEC.Int64(m.Size), score, @@ -122,6 +140,11 @@ func (m *LevelMetrics) format(w redact.SafePrinter, score redact.SafeValue) { humanize.IEC.Uint64(m.BytesRead), redact.Safe(m.Sublevels), redact.Safe(m.WriteAmp())) + if includeValueBlocksSize { + w.Printf(" %7s\n", humanize.IEC.Uint64(m.Additional.ValueBlocksSize)) + } else { + w.SafeString("\n") + } } // Metrics holds metrics for various subsystems of the DB such as the Cache, @@ -156,12 +179,27 @@ type Metrics struct { // compaction. Such files are compacted in a rewrite compaction // when no other compactions are picked. MarkedFiles int + // Duration records the cumulative duration of all compactions since the + // database was opened. + Duration time.Duration } Flush struct { // The total number of flushes. Count int64 WriteThroughput ThroughputMetric + // Number of flushes that are in-progress. In the current implementation + // this will always be zero or one. + NumInProgress int64 + // AsIngestCount is a monotonically increasing counter of flush operations + // handling ingested tables. + AsIngestCount uint64 + // AsIngestCount is a monotonically increasing counter of tables ingested as + // flushables. + AsIngestTableCount uint64 + // AsIngestBytes is a monotonically increasing counter of the bytes flushed + // for flushables that originated as ingestion operations. + AsIngestBytes uint64 } Filter FilterMetrics @@ -184,6 +222,9 @@ type Metrics struct { Keys struct { // The approximate count of internal range key set keys in the database. RangeKeySetsCount uint64 + // The approximate count of internal tombstones (DEL, SINGLEDEL and + // RANGEDEL key kinds) within the database. + TombstoneCount uint64 } Snapshots struct { @@ -191,6 +232,14 @@ type Metrics struct { Count int // The sequence number of the earliest, currently open snapshot. EarliestSeqNum uint64 + // A running tally of keys written to sstables during flushes or + // compactions that would've been elided if it weren't for open + // snapshots. + PinnedKeys uint64 + // A running cumulative sum of the size of keys and values written to + // sstables during flushes or compactions that would've been elided if + // it weren't for open snapshots. + PinnedSize uint64 } Table struct { @@ -210,6 +259,8 @@ type Metrics struct { // Count of the number of open sstable iterators. TableIters int64 + // Uptime is the total time since this DB was opened. + Uptime time.Duration WAL struct { // Number of live WAL files. @@ -329,27 +380,27 @@ func (m *Metrics) formatWAL(w redact.SafePrinter) { // String pretty-prints the metrics, showing a line for the WAL, a line per-level, and // a total: // -// __level_____count____size___score______in__ingest(sz_cnt)____move(sz_cnt)___write(sz_cnt)____read___w-amp -// WAL 1 27 B - 48 B - - - - 108 B - - 2.2 -// 0 2 1.6 K 0.50 81 B 825 B 1 0 B 0 2.4 K 3 0 B 30.6 -// 1 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0.0 -// 2 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0.0 -// 3 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0.0 -// 4 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0.0 -// 5 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0.0 -// 6 1 825 B 0.00 1.6 K 0 B 0 0 B 0 825 B 1 1.6 K 0.5 -// total 3 2.4 K - 933 B 825 B 1 0 B 0 4.1 K 4 1.6 K 4.5 -// flush 3 -// compact 1 1.6 K 0 B 1 (size == estimated-debt, score = in-progress-bytes, in = num-in-progress) -// ctype 0 0 0 0 0 (default, delete, elision, move, read) -// memtbl 1 4.0 M -// zmemtbl 0 0 B -// ztbl 0 0 B -// bcache 4 752 B 7.7% (score == hit-rate) -// tcache 0 0 B 0.0% (score == hit-rate) -// snapshots 0 0 (score == earliest seq num) -// titers 0 -// filter - - 0.0% (score == utility) +// __level_____count____size___score______in__ingest(sz_cnt)____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp +// WAL 1 28 B - 17 B - - - - 56 B - - - 3.3 +// 0 1 770 B 0.25 28 B 0 B 0 0 B 0 770 B 1 0 B 1 27.5 +// 1 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 +// 2 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 +// 3 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 +// 4 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 +// 5 0 0 B 0.00 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 +// 6 0 0 B - 0 B 0 B 0 0 B 0 0 B 0 0 B 0 0.0 +// total 1 770 B - 56 B 0 B 0 0 B 0 826 B 1 0 B 1 14.8 +// flush 1 0 B 0 0 (ingest = ingested-as-flushable, move = tables-ingested) +// compact 0 0 B 0 B 0 (size == estimated-debt, score = in-progress-bytes, in = num-in-progress) +// ctype 0 0 0 0 0 0 0 (default, delete, elision, move, read, rewrite, multi-level) +// memtbl 1 256 K +// zmemtbl 1 256 K +// ztbl 0 0 B +// bcache 4 697 B 0.0% (score == hit-rate) +// tcache 1 696 B 0.0% (score == hit-rate) +// snaps 0 - 0 (score == earliest seq num) +// titers 1 +// filter - - 0.0% (score == utility) // // The WAL "in" metric is the size of the batches written to the WAL. The WAL // "write" metric is the size of the physical data written to the WAL which @@ -376,9 +427,18 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) { // width specifiers. When the issue is fixed, we can convert these to // RedactableStrings. https://github.com/cockroachdb/redact/issues/17 + haveValueBlocks := false + var valueBlocksHeading redact.SafeString + for level := 0; level < numLevels; level++ { + if m.Levels[level].Additional.ValueBlocksSize > 0 { + haveValueBlocks = true + valueBlocksHeading = "__val-bl" + break + } + } var total LevelMetrics - w.SafeString("__level_____count____size___score______in__ingest(sz_cnt)" + - "____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp\n") + w.Printf("__level_____count____size___score______in__ingest(sz_cnt)"+ + "____move(sz_cnt)___write(sz_cnt)____read___r-amp___w-amp%s\n", valueBlocksHeading) m.formatWAL(w) for level := 0; level < numLevels; level++ { l := &m.Levels[level] @@ -389,7 +449,7 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) { if level < numLevels-1 { score = redact.Safe(fmt.Sprintf("%0.2f", l.Score)) } - l.format(w, score) + l.format(w, score, haveValueBlocks) total.Add(l) total.Sublevels += l.Sublevels } @@ -400,23 +460,30 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) { // ingested. total.BytesFlushed += total.BytesIn w.SafeString(" total ") - total.format(w, notApplicable) + total.format(w, notApplicable, haveValueBlocks) - w.Printf(" flush %9d\n", redact.Safe(m.Flush.Count)) - w.Printf("compact %9d %7s %7s %7d %7s (size == estimated-debt, score = in-progress-bytes, in = num-in-progress)\n", + w.Printf(" flush %9d %31s %7d %7d %s\n", + redact.Safe(m.Flush.Count), + humanize.IEC.Uint64(m.Flush.AsIngestBytes), + redact.Safe(m.Flush.AsIngestTableCount), + redact.Safe(m.Flush.AsIngestCount), + redact.SafeString(`(ingest = tables-ingested, move = ingested-as-flushable)`)) + w.Printf("compact %9d %7s %7s %7d %s %s\n", redact.Safe(m.Compact.Count), humanize.IEC.Uint64(m.Compact.EstimatedDebt), humanize.IEC.Int64(m.Compact.InProgressBytes), redact.Safe(m.Compact.NumInProgress), - redact.SafeString("")) - w.Printf(" ctype %9d %7d %7d %7d %7d %7d %7d (default, delete, elision, move, read, rewrite, multi-level)\n", + redact.SafeString(strings.Repeat(" ", 24)), + redact.SafeString(`(size == estimated-debt, score = in-progress-bytes, in = num-in-progress)`)) + w.Printf(" ctype %9d %7d %7d %7d %7d %7d %7d %s\n", redact.Safe(m.Compact.DefaultCount), redact.Safe(m.Compact.DeleteOnlyCount), redact.Safe(m.Compact.ElisionOnlyCount), redact.Safe(m.Compact.MoveCount), redact.Safe(m.Compact.ReadCount), redact.Safe(m.Compact.RewriteCount), - redact.Safe(m.Compact.MultiLevelCount)) + redact.Safe(m.Compact.MultiLevelCount), + redact.SafeString(`(default, delete, elision, move, read, rewrite, multi-level)`)) w.Printf(" memtbl %9d %7s\n", redact.Safe(m.MemTable.Count), humanize.IEC.Uint64(m.MemTable.Size)) diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/noop_readahead.go b/vendor/github.com/cockroachdb/pebble/objstorage/noop_readahead.go new file mode 100644 index 0000000..5632be7 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/noop_readahead.go @@ -0,0 +1,34 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorage + +import "context" + +// NoopReadHandle can be used by Readable implementations that don't +// support read-ahead. +type NoopReadHandle struct { + readable Readable +} + +// MakeNoopReadHandle initializes a NoopReadHandle. +func MakeNoopReadHandle(r Readable) NoopReadHandle { + return NoopReadHandle{readable: r} +} + +var _ ReadHandle = (*NoopReadHandle)(nil) + +// ReadAt is part of the ReadHandle interface. +func (h *NoopReadHandle) ReadAt(ctx context.Context, p []byte, off int64) error { + return h.readable.ReadAt(ctx, p, off) +} + +// Close is part of the ReadHandle interface. +func (*NoopReadHandle) Close() error { return nil } + +// MaxReadahead is part of the ReadHandle interface. +func (*NoopReadHandle) MaxReadahead() {} + +// RecordCacheHit is part of the ReadHandle interface. +func (*NoopReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go new file mode 100644 index 0000000..5cc22a1 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorage.go @@ -0,0 +1,252 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorage + +import ( + "context" + "fmt" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/vfs" +) + +// Readable is the handle for an object that is open for reading. +type Readable interface { + // ReadAt reads len(p) bytes into p starting at offset off. + // + // Clients of ReadAt can execute parallel ReadAt calls on the + // same Readable. + ReadAt(ctx context.Context, p []byte, off int64) error + + Close() error + + // Size returns the size of the object. + Size() int64 + + // NewReadHandle creates a read handle for ReadAt requests that are related + // and can benefit from optimizations like read-ahead. + // + // The ReadHandle must be closed before the Readable is closed. + // + // Multiple separate ReadHandles can be used. + NewReadHandle(ctx context.Context) ReadHandle +} + +// ReadHandle is used to perform reads that are related and might benefit from +// optimizations like read-ahead. +type ReadHandle interface { + // ReadAt reads len(p) bytes into p starting at offset off. + // + // Parallel ReadAt calls on the same ReadHandle are not allowed. + ReadAt(ctx context.Context, p []byte, off int64) error + + Close() error + + // MaxReadahead configures the implementation to expect large sequential + // reads. Used to skip any initial read-ahead ramp-up. + MaxReadahead() + + // RecordCacheHit informs the implementation that we were able to retrieve a + // block from cache. This is useful for example when the implementation is + // trying to detect a sequential reading pattern. + RecordCacheHit(ctx context.Context, offset, size int64) +} + +// Writable is the handle for an object that is open for writing. +// Either Finish or Abort must be called. +type Writable interface { + // Write writes len(p) bytes from p to the underlying object. The data is not + // guaranteed to be durable until Finish is called. + // + // Note that Write *is* allowed to modify the slice passed in, whether + // temporarily or permanently. Callers of Write need to take this into + // account. + Write(p []byte) error + + // Finish completes the object and makes the data durable. + // No further calls are allowed after calling Finish. + Finish() error + + // Abort gives up on finishing the object. There is no guarantee about whether + // the object exists after calling Abort. + // No further calls are allowed after calling Abort. + Abort() +} + +// ObjectMetadata contains the metadata required to be able to access an object. +type ObjectMetadata struct { + DiskFileNum base.DiskFileNum + FileType base.FileType + + // The fields below are only set if the object is on shared storage. + Shared struct { + // CreatorID identifies the DB instance that originally created the object. + CreatorID CreatorID + // CreatorFileNum is the identifier for the object within the context of the + // DB instance that originally created the object. + CreatorFileNum base.DiskFileNum + + CleanupMethod SharedCleanupMethod + } +} + +// IsShared returns true if the object is on shared storage. +func (meta *ObjectMetadata) IsShared() bool { + return meta.Shared.CreatorID.IsSet() +} + +// CreatorID identifies the DB instance that originally created a shared object. +// This ID is incorporated in backing object names. +// Must be non-zero. +type CreatorID uint64 + +// IsSet returns true if the CreatorID is not zero. +func (c CreatorID) IsSet() bool { return c != 0 } + +func (c CreatorID) String() string { return fmt.Sprintf("%d", c) } + +// SharedCleanupMethod indicates the method for cleaning up unused shared objects. +type SharedCleanupMethod uint8 + +const ( + // SharedRefTracking is used for shared objects for which objstorage providers + // keep track of references via reference marker objects. + SharedRefTracking SharedCleanupMethod = iota + + // SharedNoCleanup is used for shared objects that are managed externally; the + // objstorage provider never deletes such objects. + SharedNoCleanup +) + +// OpenOptions contains optional arguments for OpenForReading. +type OpenOptions struct { + // MustExist triggers a fatal error if the file does not exist. The fatal + // error message contains extra information helpful for debugging. + MustExist bool +} + +// CreateOptions contains optional arguments for Create. +type CreateOptions struct { + // PreferSharedStorage causes the object to be created on shared storage if + // the provider has shared storage configured. + PreferSharedStorage bool + + // SharedCleanupMethod is used for the object when it is created on shared storage. + // The default (zero) value is SharedRefTracking. + SharedCleanupMethod SharedCleanupMethod +} + +// Provider is a singleton object used to access and manage objects. +// +// An object is conceptually like a large immutable file. The main use of +// objects is for storing sstables; in the future it could also be used for blob +// storage. +// +// The Provider can only manage objects that it knows about - either objects +// created by the provider, or existing objects the Provider was informed about +// via AddObjects. +// +// Objects are currently backed by a vfs.File or a shared.Storage object. +type Provider interface { + // OpenForReading opens an existing object. + OpenForReading( + ctx context.Context, fileType base.FileType, FileNum base.DiskFileNum, opts OpenOptions, + ) (Readable, error) + + // Create creates a new object and opens it for writing. + // + // The object is not guaranteed to be durable (accessible in case of crashes) + // until Sync is called. + Create( + ctx context.Context, fileType base.FileType, FileNum base.DiskFileNum, opts CreateOptions, + ) (w Writable, meta ObjectMetadata, err error) + + // Remove removes an object. + // + // The object is not guaranteed to be durably removed until Sync is called. + Remove(fileType base.FileType, FileNum base.DiskFileNum) error + + // Sync flushes the metadata from creation or removal of objects since the last Sync. + // This includes objects that have been Created but for which + // Writable.Finish() has not yet been called. + Sync() error + + // LinkOrCopyFromLocal creates a new object that is either a copy of a given + // local file or a hard link (if the new object is created on the same FS, and + // if the FS supports it). + // + // The object is not guaranteed to be durable (accessible in case of crashes) + // until Sync is called. + LinkOrCopyFromLocal( + ctx context.Context, + srcFS vfs.FS, + srcFilePath string, + dstFileType base.FileType, + dstFileNum base.DiskFileNum, + opts CreateOptions, + ) (ObjectMetadata, error) + + // Lookup returns the metadata of an object that is already known to the Provider. + // Does not perform any I/O. + Lookup(fileType base.FileType, FileNum base.DiskFileNum) (ObjectMetadata, error) + + // Path returns an internal, implementation-dependent path for the object. It is + // meant to be used for informational purposes (like logging). + Path(meta ObjectMetadata) string + + // Size returns the size of the object. + Size(meta ObjectMetadata) (int64, error) + + // List returns the objects currently known to the provider. Does not perform any I/O. + List() []ObjectMetadata + + // SetCreatorID sets the CreatorID which is needed in order to use shared + // objects. Shared object usage is disabled until this method is called the + // first time. Once set, the Creator ID is persisted and cannot change. + // + // Cannot be called if shared storage is not configured for the provider. + SetCreatorID(creatorID CreatorID) error + + // SharedObjectBacking encodes the shared object metadata. + SharedObjectBacking(meta *ObjectMetadata) (SharedObjectBackingHandle, error) + + // AttachSharedObjects registers existing shared objects with this provider. + AttachSharedObjects(objs []SharedObjectToAttach) ([]ObjectMetadata, error) + + Close() error + + // IsNotExistError indicates whether the error is known to report that a file or + // directory does not exist. + IsNotExistError(err error) bool +} + +// SharedObjectBacking encodes the metadata necessary to incorporate a shared +// object into a different Pebble instance. The encoding is specific to a given +// Provider implementation. +type SharedObjectBacking []byte + +// SharedObjectBackingHandle is a container for a SharedObjectBacking which +// ensures that the backing stays valid. A backing can otherwise become invalid +// if this provider unrefs the shared object. The SharedObjectBackingHandle +// delays any unref until Close. +type SharedObjectBackingHandle interface { + // Get returns the backing. The backing is only guaranteed to be valid until + // Close is called (or until the Provider is closed). If Close was already + // called, returns an error. + Get() (SharedObjectBacking, error) + Close() +} + +// SharedObjectToAttach contains the arguments needed to attach an existing shared object. +type SharedObjectToAttach struct { + // FileNum is the file number that will be used to refer to this object (in + // the context of this instance). + FileNum base.DiskFileNum + FileType base.FileType + // Backing contains the metadata for the shared object backing (normally + // generated from a different instance, but using the same Provider + // implementation). + Backing SharedObjectBacking +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go new file mode 100644 index 0000000..4956935 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing.go @@ -0,0 +1,68 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objiotracing + +import "github.com/cockroachdb/pebble/internal/base" + +// OpType indicates the type of operation. +type OpType uint8 + +// OpType values. +const ( + ReadOp OpType = iota + WriteOp + // RecordCacheHitOp happens when a read is satisfied from the block cache. See + // objstorage.ReadHandle.RecordCacheHit(). + RecordCacheHitOp + // MaxReadaheadOp is a "meta operation" that configures a read handle for + // large sequential reads. See objstorage.ReadHandle.MaxReadahead(). + MaxReadaheadOp +) + +// Reason indicates the higher-level context of the operation. +type Reason uint8 + +// Reason values. +const ( + UnknownReason Reason = iota + ForFlush + ForCompaction + ForIngestion + // TODO(radu): add ForUserFacing. +) + +// BlockType indicates the type of data block relevant to an operation. +type BlockType uint8 + +// BlockType values. +const ( + UnknownBlock BlockType = iota + DataBlock + ValueBlock + FilterBlock + MetadataBlock +) + +// Event is the on-disk format of a tracing event. It is exported here so that +// trace processing tools can use it by importing this package. +type Event struct { + // Event start time as a Unix time (see time.Time.StartUnixNano()). + // Note that recorded events are not necessarily ordered by time - this is + // because separate event "streams" use local buffers (for performance). + StartUnixNano int64 + Op OpType + Reason Reason + BlockType BlockType + // LSM level plus one (with 0 indicating unknown level). + LevelPlusOne uint8 + // Hardcoded padding so that struct layout doesn't depend on architecture. + _ uint32 + FileNum base.FileNum + // HandleID is a unique identifier corresponding to an objstorage.ReadHandle; + // only set for read operations performed through a ReadHandle. + HandleID uint64 + Offset int64 + Size int64 +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go new file mode 100644 index 0000000..a4923ab --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_off.go @@ -0,0 +1,59 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +//go:build !pebble_obj_io_tracing +// +build !pebble_obj_io_tracing + +package objiotracing + +import ( + "context" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/vfs" +) + +// Enabled is used to short circuit tracing-related code in regular builds. +const Enabled = false + +// Tracer manages the writing of object IO traces to files. +type Tracer struct{} + +// Open creates a Tracer which generates trace files in the given directory. +// Each trace file contains a series of Events (as they are in memory). +func Open(fs vfs.FS, fsDir string) *Tracer { + return nil +} + +// Close the tracer, flushing any remaining events. +func (*Tracer) Close() {} + +// WrapReadable wraps an objstorage.Readable with one that generates tracing +// events. +func (*Tracer) WrapReadable( + ctx context.Context, r objstorage.Readable, fileNum base.DiskFileNum, +) objstorage.Readable { + return r +} + +// WrapWritable wraps an objstorage.Writable with one that generates tracing +// events. +func (t *Tracer) WrapWritable( + ctx context.Context, w objstorage.Writable, fileNum base.DiskFileNum, +) objstorage.Writable { + return w +} + +// WithReason creates a context that has an associated Reason (which ends up in +// traces created under that context). +func WithReason(ctx context.Context, reason Reason) context.Context { return ctx } + +// WithBlockType creates a context that has an associated BlockType (which ends up in +// traces created under that context). +func WithBlockType(ctx context.Context, blockType BlockType) context.Context { return ctx } + +// WithLevel creates a context that has an associated level (which ends up in +// traces created under that context). +func WithLevel(ctx context.Context, level int) context.Context { return ctx } diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go new file mode 100644 index 0000000..8030538 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go @@ -0,0 +1,410 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +//go:build pebble_obj_io_tracing +// +build pebble_obj_io_tracing + +package objiotracing + +import ( + "bufio" + "context" + "fmt" + "math/rand" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/vfs" +) + +// Enabled is used to short circuit tracing-related code in regular builds. +const Enabled = true + +// Tracer manages the writing of object IO traces to files. +// +// The tracer runs a background worker goroutine which receives trace event +// buffers over a channel and dumps them to IOTRACES- files. Wrapper +// implementations for Readable, ReadHandle, Writable are producers of traces; +// they maintain internal buffers of events which get flushed to the buffered +// channel when they get full. This allows for minimal synchronization per IO +// (as for most of these structures, an instance only allows a single IO at a +// time). +type Tracer struct { + fs vfs.FS + fsDir string + + handleID atomic.Uint64 + + workerStopCh chan struct{} + workerDataCh chan eventBuf + workerWait sync.WaitGroup +} + +// Open creates a Tracer which generates trace files in the given directory. +// Each trace file contains a series of Events (as they are in memory). +func Open(fs vfs.FS, fsDir string) *Tracer { + t := &Tracer{ + fs: fs, + fsDir: fsDir, + workerStopCh: make(chan struct{}), + workerDataCh: make(chan eventBuf, channelBufSize), + } + + t.handleID.Store(uint64(rand.NewSource(time.Now().UnixNano()).Int63())) + + t.workerWait.Add(1) + go t.workerLoop() + return t +} + +// Close the tracer, flushing any remaining events. +func (t *Tracer) Close() { + if t.workerStopCh == nil { + return + } + // Tell the worker to stop and wait for it to finish up. + close(t.workerStopCh) + t.workerWait.Wait() + t.workerStopCh = nil +} + +// WrapWritable wraps an objstorage.Writable with one that generates tracing +// events. +func (t *Tracer) WrapWritable( + ctx context.Context, w objstorage.Writable, fileNum base.FileNum, +) objstorage.Writable { + return &writable{ + w: w, + fileNum: fileNum, + g: makeEventGenerator(ctx, t), + } +} + +type writable struct { + w objstorage.Writable + fileNum base.FileNum + curOffset int64 + g eventGenerator +} + +var _ objstorage.Writable = (*writable)(nil) + +// Write is part of the objstorage.Writable interface. +func (w *writable) Write(p []byte) error { + w.g.add(context.Background(), Event{ + Op: WriteOp, + FileNum: w.fileNum, + Offset: w.curOffset, + Size: int64(len(p)), + }) + // If w.w.Write(p) returns an error, a new writable + // will be used, so even tho all of p may not have + // been written to the underlying "file", it is okay + // to add len(p) to curOffset. + w.curOffset += int64(len(p)) + return w.w.Write(p) +} + +// Finish is part of the objstorage.Writable interface. +func (w *writable) Finish() error { + w.g.flush() + return w.w.Finish() +} + +// Abort is part of the objstorage.Writable interface. +func (w *writable) Abort() { + w.g.flush() + w.w.Abort() +} + +// WrapReadable wraps an objstorage.Readable with one that generates tracing +// events. +func (t *Tracer) WrapReadable( + ctx context.Context, r objstorage.Readable, fileNum base.FileNum, +) objstorage.Readable { + res := &readable{ + r: r, + fileNum: fileNum, + } + res.mu.g = makeEventGenerator(ctx, t) + return res +} + +type readable struct { + r objstorage.Readable + fileNum base.FileNum + mu struct { + sync.Mutex + g eventGenerator + } +} + +var _ objstorage.Readable = (*readable)(nil) + +// ReadAt is part of the objstorage.Readable interface. +func (r *readable) ReadAt(ctx context.Context, v []byte, off int64) (n int, err error) { + r.mu.Lock() + r.mu.g.add(ctx, Event{ + Op: ReadOp, + FileNum: r.fileNum, + Offset: off, + Size: int64(len(v)), + }) + r.mu.Unlock() + return r.r.ReadAt(ctx, v, off) +} + +// Close is part of the objstorage.Readable interface. +func (r *readable) Close() error { + r.mu.g.flush() + return r.r.Close() +} + +// Size is part of the objstorage.Readable interface. +func (r *readable) Size() int64 { + return r.r.Size() +} + +// NewReadHandle is part of the objstorage.Readable interface. +func (r *readable) NewReadHandle(ctx context.Context) objstorage.ReadHandle { + // It's safe to get the tracer from the generator without the mutex since it never changes. + t := r.mu.g.t + return &readHandle{ + rh: r.r.NewReadHandle(ctx), + fileNum: r.fileNum, + handleID: t.handleID.Add(1), + g: makeEventGenerator(ctx, t), + } +} + +type readHandle struct { + rh objstorage.ReadHandle + fileNum base.FileNum + handleID uint64 + g eventGenerator +} + +var _ objstorage.ReadHandle = (*readHandle)(nil) + +// ReadAt is part of the objstorage.ReadHandle interface. +func (rh *readHandle) ReadAt(ctx context.Context, p []byte, off int64) (n int, err error) { + rh.g.add(ctx, Event{ + Op: ReadOp, + FileNum: rh.fileNum, + HandleID: rh.handleID, + Offset: off, + Size: int64(len(p)), + }) + return rh.rh.ReadAt(ctx, p, off) +} + +// Close is part of the objstorage.ReadHandle interface. +func (rh *readHandle) Close() error { + rh.g.flush() + return rh.rh.Close() +} + +// MaxReadahead is part of the objstorage.ReadHandle interface. +func (rh *readHandle) MaxReadahead() { + rh.g.add(context.Background(), Event{ + Op: MaxReadaheadOp, + FileNum: rh.fileNum, + HandleID: rh.handleID, + }) + rh.rh.MaxReadahead() +} + +// RecordCacheHit is part of the objstorage.ReadHandle interface. +func (rh *readHandle) RecordCacheHit(ctx context.Context, offset, size int64) { + rh.g.add(ctx, Event{ + Op: RecordCacheHitOp, + FileNum: rh.fileNum, + HandleID: rh.handleID, + Offset: offset, + Size: size, + }) + rh.rh.RecordCacheHit(ctx, offset, size) +} + +type ctxInfo struct { + reason Reason + blockType BlockType + levelPlusOne uint8 +} + +func mergeCtxInfo(base, other ctxInfo) ctxInfo { + res := other + if res.reason == 0 { + res.reason = base.reason + } + if res.blockType == 0 { + res.blockType = base.blockType + } + if res.levelPlusOne == 0 { + res.levelPlusOne = base.levelPlusOne + } + return res +} + +type ctxInfoKey struct{} + +func withInfo(ctx context.Context, info ctxInfo) context.Context { + return context.WithValue(ctx, ctxInfoKey{}, info) +} + +func infoFromCtx(ctx context.Context) ctxInfo { + res := ctx.Value(ctxInfoKey{}) + if res == nil { + return ctxInfo{} + } + return res.(ctxInfo) +} + +// WithReason creates a context that has an associated Reason (which ends up in +// traces created under that context). +func WithReason(ctx context.Context, reason Reason) context.Context { + info := infoFromCtx(ctx) + info.reason = reason + return withInfo(ctx, info) +} + +// WithBlockType creates a context that has an associated BlockType (which ends up in +// traces created under that context). +func WithBlockType(ctx context.Context, blockType BlockType) context.Context { + info := infoFromCtx(ctx) + info.blockType = blockType + return withInfo(ctx, info) +} + +// WithLevel creates a context that has an associated level (which ends up in +// traces created under that context). +func WithLevel(ctx context.Context, level int) context.Context { + info := infoFromCtx(ctx) + info.levelPlusOne = uint8(level) + 1 + return withInfo(ctx, info) +} + +const ( + eventSize = int(unsafe.Sizeof(Event{})) + targetEntriesPerFile = 256 * 1024 * 1024 / eventSize // 256MB files + eventsPerBuf = 16 + channelBufSize = 512 * 1024 / eventsPerBuf // 512K events. + bytesPerFileSync = 128 * 1024 +) + +type eventBuf struct { + events [eventsPerBuf]Event + num int +} + +type eventGenerator struct { + t *Tracer + baseCtxInfo ctxInfo + buf eventBuf +} + +func makeEventGenerator(ctx context.Context, t *Tracer) eventGenerator { + return eventGenerator{ + t: t, + baseCtxInfo: infoFromCtx(ctx), + } +} + +func (g *eventGenerator) flush() { + if g.buf.num > 0 { + g.t.workerDataCh <- g.buf + g.buf.num = 0 + } +} + +func (g *eventGenerator) add(ctx context.Context, e Event) { + e.StartUnixNano = time.Now().UnixNano() + info := infoFromCtx(ctx) + info = mergeCtxInfo(g.baseCtxInfo, info) + e.Reason = info.reason + e.BlockType = info.blockType + e.LevelPlusOne = info.levelPlusOne + if g.buf.num == eventsPerBuf { + g.flush() + } + g.buf.events[g.buf.num] = e + g.buf.num++ +} + +type workerState struct { + curFile vfs.File + curBW *bufio.Writer + numEntriesInFile int +} + +func (t *Tracer) workerLoop() { + defer t.workerWait.Done() + stopCh := t.workerStopCh + dataCh := t.workerDataCh + var state workerState + t.workerNewFile(&state) + for { + select { + case <-stopCh: + close(dataCh) + // Flush any remaining traces. + for data := range dataCh { + t.workerWriteTraces(&state, data) + } + t.workerCloseFile(&state) + return + + case data := <-dataCh: + t.workerWriteTraces(&state, data) + } + } +} + +func (t *Tracer) workerWriteTraces(state *workerState, data eventBuf) { + if state.numEntriesInFile >= targetEntriesPerFile { + t.workerCloseFile(state) + t.workerNewFile(state) + } + state.numEntriesInFile += data.num + p := unsafe.Pointer(&data.events[0]) + b := unsafe.Slice((*byte)(p), eventSize*data.num) + if _, err := state.curBW.Write(b); err != nil { + panic(err) + } +} + +func (t *Tracer) workerNewFile(state *workerState) { + filename := fmt.Sprintf("IOTRACES-%s", time.Now().UTC().Format(time.RFC3339Nano)) + + file, err := t.fs.Create(t.fs.PathJoin(t.fsDir, filename)) + if err != nil { + panic(err) + } + file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{ + BytesPerSync: bytesPerFileSync, + }) + state.curFile = file + state.curBW = bufio.NewWriter(file) + state.numEntriesInFile = 0 +} + +func (t *Tracer) workerCloseFile(state *workerState) { + if state.curFile != nil { + if err := state.curBW.Flush(); err != nil { + panic(err) + } + if err := state.curFile.Sync(); err != nil { + panic(err) + } + if err := state.curFile.Close(); err != nil { + panic(err) + } + state.curFile = nil + state.curBW = nil + } +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go new file mode 100644 index 0000000..9e3d34f --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/provider.go @@ -0,0 +1,460 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "context" + "io" + "os" + "sort" + "sync" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/errors/oserror" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat" + "github.com/cockroachdb/pebble/objstorage/shared" + "github.com/cockroachdb/pebble/vfs" +) + +// provider is the implementation of objstorage.Provider. +type provider struct { + st Settings + + fsDir vfs.File + + tracer *objiotracing.Tracer + + shared sharedSubsystem + + mu struct { + sync.RWMutex + + shared struct { + // catalogBatch accumulates shared object creations and deletions until + // Sync is called. + catalogBatch sharedobjcat.Batch + } + + // localObjectsChanged is set if non-shared objects were created or deleted + // but Sync was not yet called. + localObjectsChanged bool + + // knownObjects maintains information about objects that are known to the provider. + // It is initialized with the list of files in the manifest when we open a DB. + knownObjects map[base.DiskFileNum]objstorage.ObjectMetadata + + // protectedObjects are objects that cannot be unreferenced because they + // have outstanding SharedObjectBackingHandles. The value is a count of outstanding handles + protectedObjects map[base.DiskFileNum]int + } +} + +var _ objstorage.Provider = (*provider)(nil) + +// Settings that must be specified when creating the provider. +type Settings struct { + Logger base.Logger + + // Local filesystem configuration. + FS vfs.FS + FSDirName string + + // FSDirInitialListing is a listing of FSDirName at the time of calling Open. + // + // This is an optional optimization to avoid double listing on Open when the + // higher layer already has a listing. When nil, we obtain the listing on + // Open. + FSDirInitialListing []string + + // Cleaner cleans obsolete files from the local filesystem. + // + // The default cleaner uses the DeleteCleaner. + FSCleaner base.Cleaner + + // NoSyncOnClose decides whether the implementation will enforce a + // close-time synchronization (e.g., fdatasync() or sync_file_range()) + // on files it writes to. Setting this to true removes the guarantee for a + // sync on close. Some implementations can still issue a non-blocking sync. + NoSyncOnClose bool + + // BytesPerSync enables periodic syncing of files in order to smooth out + // writes to disk. This option does not provide any persistence guarantee, but + // is used to avoid latency spikes if the OS automatically decides to write + // out a large chunk of dirty filesystem buffers. + BytesPerSync int + + // Fields here are set only if the provider is to support shared objects + // (experimental). + Shared struct { + Storage shared.Storage + } +} + +// DefaultSettings initializes default settings (with no shared storage), +// suitable for tests and tools. +func DefaultSettings(fs vfs.FS, dirName string) Settings { + return Settings{ + Logger: base.DefaultLogger, + FS: fs, + FSDirName: dirName, + FSCleaner: base.DeleteCleaner{}, + NoSyncOnClose: false, + BytesPerSync: 512 * 1024, // 512KB + } +} + +// Open creates the provider. +func Open(settings Settings) (objstorage.Provider, error) { + return open(settings) +} + +func open(settings Settings) (p *provider, _ error) { + fsDir, err := settings.FS.OpenDir(settings.FSDirName) + if err != nil { + return nil, err + } + + defer func() { + if p == nil { + fsDir.Close() + } + }() + + p = &provider{ + st: settings, + fsDir: fsDir, + } + p.mu.knownObjects = make(map[base.DiskFileNum]objstorage.ObjectMetadata) + p.mu.protectedObjects = make(map[base.DiskFileNum]int) + + if objiotracing.Enabled { + p.tracer = objiotracing.Open(settings.FS, settings.FSDirName) + } + + // Add local FS objects. + if err := p.vfsInit(); err != nil { + return nil, err + } + + // Initialize shared subsystem (if configured) and add shared objects. + if err := p.sharedInit(); err != nil { + return nil, err + } + + return p, nil +} + +// Close is part of the objstorage.Provider interface. +func (p *provider) Close() error { + var err error + if p.fsDir != nil { + err = p.fsDir.Close() + p.fsDir = nil + } + if objiotracing.Enabled { + if p.tracer != nil { + p.tracer.Close() + p.tracer = nil + } + } + return err +} + +// OpenForReading opens an existing object. +func (p *provider) OpenForReading( + ctx context.Context, + fileType base.FileType, + fileNum base.DiskFileNum, + opts objstorage.OpenOptions, +) (objstorage.Readable, error) { + meta, err := p.Lookup(fileType, fileNum) + if err != nil { + if opts.MustExist { + p.st.Logger.Fatalf("%v", err) + } + return nil, err + } + + var r objstorage.Readable + if !meta.IsShared() { + r, err = p.vfsOpenForReading(ctx, fileType, fileNum, opts) + } else { + r, err = p.sharedOpenForReading(ctx, meta, opts) + } + if err != nil { + return nil, err + } + if objiotracing.Enabled { + r = p.tracer.WrapReadable(ctx, r, fileNum) + } + return r, nil +} + +// Create creates a new object and opens it for writing. +// +// The object is not guaranteed to be durable (accessible in case of crashes) +// until Sync is called. +func (p *provider) Create( + ctx context.Context, + fileType base.FileType, + fileNum base.DiskFileNum, + opts objstorage.CreateOptions, +) (w objstorage.Writable, meta objstorage.ObjectMetadata, err error) { + if opts.PreferSharedStorage && p.st.Shared.Storage != nil { + w, meta, err = p.sharedCreate(ctx, fileType, fileNum, opts) + } else { + w, meta, err = p.vfsCreate(ctx, fileType, fileNum) + } + if err != nil { + err = errors.Wrapf(err, "creating object %s", errors.Safe(fileNum)) + return nil, objstorage.ObjectMetadata{}, err + } + p.addMetadata(meta) + if objiotracing.Enabled { + w = p.tracer.WrapWritable(ctx, w, fileNum) + } + return w, meta, nil +} + +// Remove removes an object. +// +// Note that if the object is shared, the object is only (conceptually) removed +// from this provider. If other providers have references on the shared object, +// it will not be removed. +// +// The object is not guaranteed to be durably removed until Sync is called. +func (p *provider) Remove(fileType base.FileType, fileNum base.DiskFileNum) error { + meta, err := p.Lookup(fileType, fileNum) + if err != nil { + return err + } + + if !meta.IsShared() { + err = p.vfsRemove(fileType, fileNum) + } else { + // TODO(radu): implement shared object removal (i.e. deref). + err = p.sharedUnref(meta) + } + if err != nil && !p.IsNotExistError(err) { + // We want to be able to retry a Remove, so we keep the object in our list. + // TODO(radu): we should mark the object as "zombie" and not allow any other + // operations. + return errors.Wrapf(err, "removing object %s", errors.Safe(fileNum)) + } + + p.removeMetadata(fileNum) + return err +} + +// IsNotExistError is part of the objstorage.Provider interface. +func (p *provider) IsNotExistError(err error) bool { + if oserror.IsNotExist(err) { + return true + } + if p.sharedStorage() != nil && p.sharedStorage().IsNotExistError(err) { + return true + } + return false +} + +// Sync flushes the metadata from creation or removal of objects since the last Sync. +func (p *provider) Sync() error { + if err := p.vfsSync(); err != nil { + return err + } + if err := p.sharedSync(); err != nil { + return err + } + return nil +} + +// LinkOrCopyFromLocal creates a new object that is either a copy of a given +// local file or a hard link (if the new object is created on the same FS, and +// if the FS supports it). +// +// The object is not guaranteed to be durable (accessible in case of crashes) +// until Sync is called. +func (p *provider) LinkOrCopyFromLocal( + ctx context.Context, + srcFS vfs.FS, + srcFilePath string, + dstFileType base.FileType, + dstFileNum base.DiskFileNum, + opts objstorage.CreateOptions, +) (objstorage.ObjectMetadata, error) { + shared := opts.PreferSharedStorage && p.st.Shared.Storage != nil + if !shared && srcFS == p.st.FS { + // Wrap the normal filesystem with one which wraps newly created files with + // vfs.NewSyncingFile. + fs := vfs.NewSyncingFS(p.st.FS, vfs.SyncingFileOptions{ + NoSyncOnClose: p.st.NoSyncOnClose, + BytesPerSync: p.st.BytesPerSync, + }) + dstPath := p.vfsPath(dstFileType, dstFileNum) + if err := vfs.LinkOrCopy(fs, srcFilePath, dstPath); err != nil { + return objstorage.ObjectMetadata{}, err + } + + meta := objstorage.ObjectMetadata{ + DiskFileNum: dstFileNum, + FileType: dstFileType, + } + p.addMetadata(meta) + return meta, nil + } + // Create the object and copy the data. + w, meta, err := p.Create(ctx, dstFileType, dstFileNum, opts) + if err != nil { + return objstorage.ObjectMetadata{}, err + } + f, err := srcFS.Open(srcFilePath, vfs.SequentialReadsOption) + if err != nil { + return objstorage.ObjectMetadata{}, err + } + defer f.Close() + buf := make([]byte, 64*1024) + for { + n, readErr := f.Read(buf) + if readErr != nil && readErr != io.EOF { + w.Abort() + return objstorage.ObjectMetadata{}, readErr + } + + if n > 0 { + if err := w.Write(buf[:n]); err != nil { + w.Abort() + return objstorage.ObjectMetadata{}, err + } + } + + if readErr == io.EOF { + break + } + } + if err := w.Finish(); err != nil { + return objstorage.ObjectMetadata{}, err + } + return meta, nil +} + +// Lookup is part of the objstorage.Provider interface. +func (p *provider) Lookup( + fileType base.FileType, fileNum base.DiskFileNum, +) (objstorage.ObjectMetadata, error) { + p.mu.RLock() + defer p.mu.RUnlock() + meta, ok := p.mu.knownObjects[fileNum] + if !ok { + return objstorage.ObjectMetadata{}, errors.Wrapf( + os.ErrNotExist, + "file %s (type %d) unknown to the objstorage provider", + errors.Safe(fileNum), errors.Safe(fileType), + ) + } + if meta.FileType != fileType { + return objstorage.ObjectMetadata{}, errors.AssertionFailedf( + "file %s type mismatch (known type %d, expected type %d)", + errors.Safe(fileNum), errors.Safe(meta.FileType), errors.Safe(fileType), + ) + } + return meta, nil +} + +// Path is part of the objstorage.Provider interface. +func (p *provider) Path(meta objstorage.ObjectMetadata) string { + if !meta.IsShared() { + return p.vfsPath(meta.FileType, meta.DiskFileNum) + } + return p.sharedPath(meta) +} + +// Size returns the size of the object. +func (p *provider) Size(meta objstorage.ObjectMetadata) (int64, error) { + if !meta.IsShared() { + return p.vfsSize(meta.FileType, meta.DiskFileNum) + } + return p.sharedSize(meta) +} + +// List is part of the objstorage.Provider interface. +func (p *provider) List() []objstorage.ObjectMetadata { + p.mu.RLock() + defer p.mu.RUnlock() + res := make([]objstorage.ObjectMetadata, 0, len(p.mu.knownObjects)) + for _, meta := range p.mu.knownObjects { + res = append(res, meta) + } + sort.Slice(res, func(i, j int) bool { + return res[i].DiskFileNum.FileNum() < res[j].DiskFileNum.FileNum() + }) + return res +} + +func (p *provider) addMetadata(meta objstorage.ObjectMetadata) { + p.mu.Lock() + defer p.mu.Unlock() + p.mu.knownObjects[meta.DiskFileNum] = meta + if meta.IsShared() { + p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{ + FileNum: meta.DiskFileNum, + FileType: meta.FileType, + CreatorID: meta.Shared.CreatorID, + CreatorFileNum: meta.Shared.CreatorFileNum, + CleanupMethod: meta.Shared.CleanupMethod, + }) + } else { + p.mu.localObjectsChanged = true + } +} + +func (p *provider) removeMetadata(fileNum base.DiskFileNum) { + p.mu.Lock() + defer p.mu.Unlock() + + meta, ok := p.mu.knownObjects[fileNum] + if !ok { + return + } + delete(p.mu.knownObjects, fileNum) + if meta.IsShared() { + p.mu.shared.catalogBatch.DeleteObject(fileNum) + } else { + p.mu.localObjectsChanged = true + } +} + +// protectObject prevents the unreferencing of a shared object until +// unprotectObject is called. +func (p *provider) protectObject(fileNum base.DiskFileNum) { + p.mu.Lock() + defer p.mu.Unlock() + p.mu.protectedObjects[fileNum] = p.mu.protectedObjects[fileNum] + 1 +} + +func (p *provider) unprotectObject(fileNum base.DiskFileNum) { + p.mu.Lock() + defer p.mu.Unlock() + v := p.mu.protectedObjects[fileNum] + if invariants.Enabled && v == 0 { + panic("invalid protection count") + } + if v > 1 { + p.mu.protectedObjects[fileNum] = v - 1 + } else { + delete(p.mu.protectedObjects, fileNum) + // TODO(radu): check if the object is still in knownObject; if not, unref it + // now. + } +} + +func (p *provider) isProtected(fileNum base.DiskFileNum) bool { + p.mu.Lock() + defer p.mu.Unlock() + return p.mu.protectedObjects[fileNum] > 0 +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go new file mode 100644 index 0000000..ec04efb --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/readahead.go @@ -0,0 +1,200 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +const ( + // Constants for dynamic readahead of data blocks. Note that the size values + // make sense as some multiple of the default block size; and they should + // both be larger than the default block size. + minFileReadsForReadahead = 2 + // TODO(bilal): Have the initial size value be a factor of the block size, + // as opposed to a hardcoded value. + initialReadaheadSize = 64 << 10 /* 64KB */ + maxReadaheadSize = 256 << 10 /* 256KB */ +) + +// readaheadState contains state variables related to readahead. Updated on +// file reads. +type readaheadState struct { + // Number of sequential reads. + numReads int64 + // Size issued to the next call to Prefetch. Starts at or above + // initialReadaheadSize and grows exponentially until maxReadaheadSize. + size int64 + // prevSize is the size used in the last Prefetch call. + prevSize int64 + // The byte offset up to which the OS has been asked to read ahead / cached. + // When reading ahead, reads up to this limit should not incur an IO + // operation. Reads after this limit can benefit from a new call to + // Prefetch. + limit int64 +} + +func makeReadaheadState() readaheadState { + return readaheadState{ + size: initialReadaheadSize, + } +} + +func (rs *readaheadState) recordCacheHit(offset, blockLength int64) { + currentReadEnd := offset + blockLength + if rs.numReads >= minFileReadsForReadahead { + if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { + // This is a read that would have resulted in a readahead, had it + // not been a cache hit. + rs.limit = currentReadEnd + return + } + if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize { + // We read too far away from rs.limit to benefit from readahead in + // any scenario. Reset all variables. + rs.numReads = 1 + rs.limit = currentReadEnd + rs.size = initialReadaheadSize + rs.prevSize = 0 + return + } + // Reads in the range [rs.limit - rs.prevSize, rs.limit] end up + // here. This is a read that is potentially benefitting from a past + // readahead. + return + } + if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { + // Blocks are being read sequentially and would benefit from readahead + // down the line. + rs.numReads++ + return + } + // We read too far ahead of the last read, or before it. This indicates + // a random read, where readahead is not desirable. Reset all variables. + rs.numReads = 1 + rs.limit = currentReadEnd + rs.size = initialReadaheadSize + rs.prevSize = 0 +} + +// maybeReadahead updates state and determines whether to issue a readahead / +// prefetch call for a block read at offset for blockLength bytes. +// Returns a size value (greater than 0) that should be prefetched if readahead +// would be beneficial. +func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { + currentReadEnd := offset + blockLength + if rs.numReads >= minFileReadsForReadahead { + // The minimum threshold of sequential reads to justify reading ahead + // has been reached. + // There are two intervals: the interval being read: + // [offset, currentReadEnd] + // as well as the interval where a read would benefit from read ahead: + // [rs.limit, rs.limit + rs.size] + // We increase the latter interval to + // [rs.limit, rs.limit + maxReadaheadSize] to account for cases where + // readahead may not be beneficial with a small readahead size, but over + // time the readahead size would increase exponentially to make it + // beneficial. + if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { + // We are doing a read in the interval ahead of + // the last readahead range. In the diagrams below, ++++ is the last + // readahead range, ==== is the range represented by + // [rs.limit, rs.limit + maxReadaheadSize], and ---- is the range + // being read. + // + // rs.limit rs.limit + maxReadaheadSize + // ++++++++++|===========================| + // + // |-------------| + // offset currentReadEnd + // + // This case is also possible, as are all cases with an overlap + // between [rs.limit, rs.limit + maxReadaheadSize] and [offset, + // currentReadEnd]: + // + // rs.limit rs.limit + maxReadaheadSize + // ++++++++++|===========================| + // + // |-------------| + // offset currentReadEnd + // + // + rs.numReads++ + rs.limit = offset + rs.size + rs.prevSize = rs.size + // Increase rs.size for the next read. + rs.size *= 2 + if rs.size > maxReadaheadSize { + rs.size = maxReadaheadSize + } + return rs.prevSize + } + if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize { + // The above conditional has rs.limit > rs.prevSize to confirm that + // rs.limit - rs.prevSize would not underflow. + // We read too far away from rs.limit to benefit from readahead in + // any scenario. Reset all variables. + // The case where we read too far ahead: + // + // (rs.limit - rs.prevSize) (rs.limit) (rs.limit + maxReadaheadSize) + // |+++++++++++++|=============| + // + // |-------------| + // offset currentReadEnd + // + // Or too far behind: + // + // (rs.limit - rs.prevSize) (rs.limit) (rs.limit + maxReadaheadSize) + // |+++++++++++++|=============| + // + // |-------------| + // offset currentReadEnd + // + rs.numReads = 1 + rs.limit = currentReadEnd + rs.size = initialReadaheadSize + rs.prevSize = 0 + + return 0 + } + // Reads in the range [rs.limit - rs.prevSize, rs.limit] end up + // here. This is a read that is potentially benefitting from a past + // readahead, but there's no reason to issue a readahead call at the + // moment. + // + // (rs.limit - rs.prevSize) (rs.limit + maxReadaheadSize) + // |+++++++++++++|===============| + // (rs.limit) + // + // |-------| + // offset currentReadEnd + // + rs.numReads++ + return 0 + } + if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { + // Blocks are being read sequentially and would benefit from readahead + // down the line. + // + // (rs.limit) (rs.limit + maxReadaheadSize) + // |=============| + // + // |-------| + // offset currentReadEnd + // + rs.numReads++ + return 0 + } + // We read too far ahead of the last read, or before it. This indicates + // a random read, where readahead is not desirable. Reset all variables. + // + // (rs.limit - maxReadaheadSize) (rs.limit) (rs.limit + maxReadaheadSize) + // |+++++++++++++|=============| + // + // |-------| + // offset currentReadEnd + // + rs.numReads = 1 + rs.limit = currentReadEnd + rs.size = initialReadaheadSize + rs.prevSize = 0 + return 0 +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared.go new file mode 100644 index 0000000..b6fd71d --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared.go @@ -0,0 +1,255 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "context" + "sync" + "sync/atomic" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat" + "github.com/cockroachdb/pebble/objstorage/shared" +) + +// sharedSubsystem contains the provider fields related to shared storage. +// All fields remain unset if shared storage is not configured. +type sharedSubsystem struct { + catalog *sharedobjcat.Catalog + + // checkRefsOnOpen controls whether we check the ref marker file when opening + // an object. Normally this is true when invariants are enabled (but the provider + // test tweaks this field). + checkRefsOnOpen bool + + // initialized guards access to the creatorID field. + initialized atomic.Bool + creatorID objstorage.CreatorID + initOnce sync.Once +} + +func (ss *sharedSubsystem) init(creatorID objstorage.CreatorID) { + ss.initOnce.Do(func() { + ss.creatorID = creatorID + ss.initialized.Store(true) + }) +} + +func (p *provider) sharedStorage() shared.Storage { + return p.st.Shared.Storage +} + +// sharedInit initializes the shared object subsystem (if configured) and finds +// any shared objects. +func (p *provider) sharedInit() error { + if p.sharedStorage() == nil { + return nil + } + catalog, contents, err := sharedobjcat.Open(p.st.FS, p.st.FSDirName) + if err != nil { + return errors.Wrapf(err, "pebble: could not open shared object catalog") + } + p.shared.catalog = catalog + p.shared.checkRefsOnOpen = invariants.Enabled + + // The creator ID may or may not be initialized yet. + if contents.CreatorID.IsSet() { + p.shared.init(contents.CreatorID) + p.st.Logger.Infof("shared storage configured; creatorID = %s", contents.CreatorID) + } else { + p.st.Logger.Infof("shared storage configured; no creatorID yet") + } + + for _, meta := range contents.Objects { + o := objstorage.ObjectMetadata{ + DiskFileNum: meta.FileNum, + FileType: meta.FileType, + } + o.Shared.CreatorID = meta.CreatorID + o.Shared.CreatorFileNum = meta.CreatorFileNum + o.Shared.CleanupMethod = meta.CleanupMethod + p.mu.knownObjects[o.DiskFileNum] = o + } + return nil +} + +// SetCreatorID is part of the objstorage.Provider interface. +func (p *provider) SetCreatorID(creatorID objstorage.CreatorID) error { + if p.sharedStorage() == nil { + return errors.AssertionFailedf("attempt to set CreatorID but shared storage not enabled") + } + // Note: this call is a cheap no-op if the creator ID was already set. This + // call also checks if we are trying to change the ID. + if err := p.shared.catalog.SetCreatorID(creatorID); err != nil { + return err + } + if !p.shared.initialized.Load() { + p.st.Logger.Infof("shared storage creatorID set to %s", creatorID) + p.shared.init(creatorID) + } + return nil +} + +func (p *provider) sharedCheckInitialized() error { + if p.sharedStorage() == nil { + return errors.Errorf("shared object support not configured") + } + if !p.shared.initialized.Load() { + return errors.Errorf("shared object support not available: shared creator ID not yet set") + } + return nil +} + +func (p *provider) sharedSync() error { + batch := func() sharedobjcat.Batch { + p.mu.Lock() + defer p.mu.Unlock() + res := p.mu.shared.catalogBatch.Copy() + p.mu.shared.catalogBatch.Reset() + return res + }() + + if batch.IsEmpty() { + return nil + } + + err := p.shared.catalog.ApplyBatch(batch) + if err != nil { + // We have to put back the batch (for the next Sync). + p.mu.Lock() + defer p.mu.Unlock() + batch.Append(p.mu.shared.catalogBatch) + p.mu.shared.catalogBatch = batch + return err + } + + return nil +} + +func (p *provider) sharedPath(meta objstorage.ObjectMetadata) string { + return "shared://" + sharedObjectName(meta) +} + +// sharedCreateRef creates a reference marker object. +func (p *provider) sharedCreateRef(meta objstorage.ObjectMetadata) error { + if meta.Shared.CleanupMethod != objstorage.SharedRefTracking { + return nil + } + refName := p.sharedObjectRefName(meta) + writer, err := p.sharedStorage().CreateObject(refName) + if err == nil { + // The object is empty, just close the writer. + err = writer.Close() + } + if err != nil { + return errors.Wrapf(err, "creating marker object %q", refName) + } + return nil +} + +func (p *provider) sharedCreate( + _ context.Context, + fileType base.FileType, + fileNum base.DiskFileNum, + opts objstorage.CreateOptions, +) (objstorage.Writable, objstorage.ObjectMetadata, error) { + if err := p.sharedCheckInitialized(); err != nil { + return nil, objstorage.ObjectMetadata{}, err + } + meta := objstorage.ObjectMetadata{ + DiskFileNum: fileNum, + FileType: fileType, + } + meta.Shared.CreatorID = p.shared.creatorID + meta.Shared.CreatorFileNum = fileNum + meta.Shared.CleanupMethod = opts.SharedCleanupMethod + + objName := sharedObjectName(meta) + writer, err := p.sharedStorage().CreateObject(objName) + if err != nil { + return nil, objstorage.ObjectMetadata{}, errors.Wrapf(err, "creating object %q", objName) + } + return &sharedWritable{ + p: p, + meta: meta, + storageWriter: writer, + }, meta, nil +} + +func (p *provider) sharedOpenForReading( + ctx context.Context, meta objstorage.ObjectMetadata, opts objstorage.OpenOptions, +) (objstorage.Readable, error) { + if err := p.sharedCheckInitialized(); err != nil { + return nil, err + } + // Verify we have a reference on this object; for performance reasons, we only + // do this in testing scenarios. + if p.shared.checkRefsOnOpen && meta.Shared.CleanupMethod == objstorage.SharedRefTracking { + refName := p.sharedObjectRefName(meta) + if _, err := p.sharedStorage().Size(refName); err != nil { + if p.sharedStorage().IsNotExistError(err) { + if opts.MustExist { + p.st.Logger.Fatalf("marker object %q does not exist", refName) + // TODO(radu): maybe list references for the object. + } + return nil, errors.Errorf("marker object %q does not exist", refName) + } + return nil, errors.Wrapf(err, "checking marker object %q", refName) + } + } + objName := sharedObjectName(meta) + reader, size, err := p.sharedStorage().ReadObject(ctx, objName) + if err != nil { + if opts.MustExist && p.sharedStorage().IsNotExistError(err) { + p.st.Logger.Fatalf("object %q does not exist", objName) + // TODO(radu): maybe list references for the object. + } + return nil, err + } + return newSharedReadable(reader, size), nil +} + +func (p *provider) sharedSize(meta objstorage.ObjectMetadata) (int64, error) { + if err := p.sharedCheckInitialized(); err != nil { + return 0, err + } + objName := sharedObjectName(meta) + return p.sharedStorage().Size(objName) +} + +// sharedUnref implements object "removal" with the shared backend. The ref +// marker object is removed and the backing object is removed only if there are +// no other ref markers. +func (p *provider) sharedUnref(meta objstorage.ObjectMetadata) error { + if meta.Shared.CleanupMethod == objstorage.SharedNoCleanup { + // Never delete objects in this mode. + return nil + } + if p.isProtected(meta.DiskFileNum) { + // TODO(radu): we need a mechanism to unref the object when it becomes + // unprotected. + return nil + } + + refName := p.sharedObjectRefName(meta) + // Tolerate a not-exists error. + if err := p.sharedStorage().Delete(refName); err != nil && !p.sharedStorage().IsNotExistError(err) { + return err + } + otherRefs, err := p.sharedStorage().List(sharedObjectRefPrefix(meta), "" /* delimiter */) + if err != nil { + return err + } + if len(otherRefs) == 0 { + objName := sharedObjectName(meta) + if err := p.sharedStorage().Delete(objName); err != nil && !p.sharedStorage().IsNotExistError(err) { + return err + } + } + return nil +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_backing.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_backing.go new file mode 100644 index 0000000..921bd54 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_backing.go @@ -0,0 +1,240 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "bytes" + "encoding/binary" + "io" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat" +) + +const ( + tagCreatorID = 1 + tagCreatorFileNum = 2 + tagCleanupMethod = 3 + // tagRefCheckID encodes the information for a ref marker that needs to be + // checked when attaching this object to another provider. This is set to the + // creator ID and FileNum for the provider that encodes the backing, and + // allows the "target" provider to check that the "source" provider kept its + // reference on the object alive. + tagRefCheckID = 4 + + // Any new tags that don't have the tagNotSafeToIgnoreMask bit set must be + // followed by the length of the data (so they can be skipped). + + // Any new tags that have the tagNotSafeToIgnoreMask bit set cause errors if + // they are encountered by earlier code that doesn't know the tag. + tagNotSafeToIgnoreMask = 64 +) + +func (p *provider) encodeSharedObjectBacking( + meta *objstorage.ObjectMetadata, +) (objstorage.SharedObjectBacking, error) { + if !meta.IsShared() { + return nil, errors.AssertionFailedf("object %s not on shared storage", meta.DiskFileNum) + } + + buf := make([]byte, 0, binary.MaxVarintLen64*4) + buf = binary.AppendUvarint(buf, tagCreatorID) + buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorID)) + // TODO(radu): encode file type as well? + buf = binary.AppendUvarint(buf, tagCreatorFileNum) + buf = binary.AppendUvarint(buf, uint64(meta.Shared.CreatorFileNum.FileNum())) + buf = binary.AppendUvarint(buf, tagCleanupMethod) + buf = binary.AppendUvarint(buf, uint64(meta.Shared.CleanupMethod)) + if meta.Shared.CleanupMethod == objstorage.SharedRefTracking { + buf = binary.AppendUvarint(buf, tagRefCheckID) + buf = binary.AppendUvarint(buf, uint64(p.shared.creatorID)) + buf = binary.AppendUvarint(buf, uint64(meta.DiskFileNum.FileNum())) + } + return buf, nil +} + +type sharedObjectBackingHandle struct { + backing objstorage.SharedObjectBacking + fileNum base.DiskFileNum + p *provider +} + +func (s *sharedObjectBackingHandle) Get() (objstorage.SharedObjectBacking, error) { + if s.backing == nil { + return nil, errors.Errorf("SharedObjectBackingHandle.Get() called after Close()") + } + return s.backing, nil +} + +func (s *sharedObjectBackingHandle) Close() { + if s.backing != nil { + s.backing = nil + s.p.unprotectObject(s.fileNum) + } +} + +var _ objstorage.SharedObjectBackingHandle = (*sharedObjectBackingHandle)(nil) + +// SharedObjectBacking is part of the objstorage.Provider interface. +func (p *provider) SharedObjectBacking( + meta *objstorage.ObjectMetadata, +) (objstorage.SharedObjectBackingHandle, error) { + backing, err := p.encodeSharedObjectBacking(meta) + if err != nil { + return nil, err + } + p.protectObject(meta.DiskFileNum) + return &sharedObjectBackingHandle{ + backing: backing, + fileNum: meta.DiskFileNum, + p: p, + }, nil +} + +type decodedBacking struct { + meta objstorage.ObjectMetadata + // refToCheck is set only when meta.Shared.CleanupMethod is RefTracking + refToCheck struct { + creatorID objstorage.CreatorID + fileNum base.DiskFileNum + } +} + +// decodeSharedObjectBacking decodes the shared object metadata. +// Returns the object metadata and (optionally) the creator ID of the provider +// that encoded the backing whose ref marker needs to be checked. +func decodeSharedObjectBacking( + fileType base.FileType, fileNum base.DiskFileNum, buf objstorage.SharedObjectBacking, +) (decodedBacking, error) { + var creatorID, creatorFileNum, cleanupMethod, refCheckCreatorID, refCheckFileNum uint64 + br := bytes.NewReader(buf) + for { + tag, err := binary.ReadUvarint(br) + if err == io.EOF { + break + } + if err != nil { + return decodedBacking{}, err + } + switch tag { + case tagCreatorID: + creatorID, err = binary.ReadUvarint(br) + + case tagCreatorFileNum: + creatorFileNum, err = binary.ReadUvarint(br) + + case tagCleanupMethod: + cleanupMethod, err = binary.ReadUvarint(br) + + case tagRefCheckID: + refCheckCreatorID, err = binary.ReadUvarint(br) + if err == nil { + refCheckFileNum, err = binary.ReadUvarint(br) + } + + default: + // Ignore unknown tags, unless they're not safe to ignore. + if tag&tagNotSafeToIgnoreMask != 0 { + return decodedBacking{}, errors.Newf("unknown tag %d", tag) + } + var dataLen uint64 + dataLen, err = binary.ReadUvarint(br) + if err == nil { + _, err = br.Seek(int64(dataLen), io.SeekCurrent) + } + } + if err != nil { + return decodedBacking{}, err + } + } + if creatorID == 0 { + return decodedBacking{}, errors.Newf("shared object backing missing creator ID") + } + if creatorFileNum == 0 { + return decodedBacking{}, errors.Newf("shared object backing missing creator file num") + } + var res decodedBacking + res.meta.DiskFileNum = fileNum + res.meta.FileType = fileType + res.meta.Shared.CreatorID = objstorage.CreatorID(creatorID) + res.meta.Shared.CreatorFileNum = base.FileNum(creatorFileNum).DiskFileNum() + res.meta.Shared.CleanupMethod = objstorage.SharedCleanupMethod(cleanupMethod) + + if res.meta.Shared.CleanupMethod == objstorage.SharedRefTracking { + if refCheckCreatorID == 0 || refCheckFileNum == 0 { + return decodedBacking{}, errors.Newf("shared object backing missing ref to check") + } + res.refToCheck.creatorID = objstorage.CreatorID(refCheckCreatorID) + res.refToCheck.fileNum = base.FileNum(refCheckFileNum).DiskFileNum() + } + return res, nil +} + +// AttachSharedObjects is part of the objstorage.Provider interface. +func (p *provider) AttachSharedObjects( + objs []objstorage.SharedObjectToAttach, +) ([]objstorage.ObjectMetadata, error) { + decoded := make([]decodedBacking, len(objs)) + for i, o := range objs { + var err error + decoded[i], err = decodeSharedObjectBacking(o.FileType, o.FileNum, o.Backing) + if err != nil { + return nil, err + } + } + + // Create the reference marker objects. + // TODO(radu): parallelize this. + for _, d := range decoded { + if d.meta.Shared.CleanupMethod != objstorage.SharedRefTracking { + continue + } + if err := p.sharedCreateRef(d.meta); err != nil { + // TODO(radu): clean up references previously created in this loop. + return nil, err + } + // Check the "origin"'s reference. + refName := sharedObjectRefName(d.meta, d.refToCheck.creatorID, d.refToCheck.fileNum) + if _, err := p.sharedStorage().Size(refName); err != nil { + _ = p.sharedUnref(d.meta) + // TODO(radu): clean up references previously created in this loop. + if p.sharedStorage().IsNotExistError(err) { + return nil, errors.Errorf("origin marker object %q does not exist;"+ + " object probably removed from the provider which created the backing", refName) + } + return nil, errors.Wrapf(err, "checking origin's marker object %s", refName) + } + } + + func() { + p.mu.Lock() + defer p.mu.Unlock() + for _, d := range decoded { + p.mu.shared.catalogBatch.AddObject(sharedobjcat.SharedObjectMetadata{ + FileNum: d.meta.DiskFileNum, + FileType: d.meta.FileType, + CreatorID: d.meta.Shared.CreatorID, + CreatorFileNum: d.meta.Shared.CreatorFileNum, + }) + } + }() + if err := p.sharedSync(); err != nil { + return nil, err + } + + metas := make([]objstorage.ObjectMetadata, len(decoded)) + for i, d := range decoded { + metas[i] = d.meta + } + + p.mu.Lock() + defer p.mu.Unlock() + for _, meta := range metas { + p.mu.knownObjects[meta.DiskFileNum] = meta + } + return metas, nil +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_obj_name.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_obj_name.go new file mode 100644 index 0000000..49695f8 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_obj_name.go @@ -0,0 +1,80 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "fmt" + + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" +) + +// sharedObjectName returns the name of an object on shared storage. +// +// For sstables, the format is: --.sst +// For example: 1a3f-2-000001.sst +func sharedObjectName(meta objstorage.ObjectMetadata) string { + switch meta.FileType { + case base.FileTypeTable: + return fmt.Sprintf( + "%04x-%d-%06d.sst", + objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(), + ) + } + panic("unknown FileType") +} + +// sharedObjectRefName returns the name of the object's ref marker associated +// with a given referencing provider. This name is the object's name concatenated with +// ".ref..". +// +// For example: 1a3f-2-000001.sst.ref.5.000008 +func sharedObjectRefName( + meta objstorage.ObjectMetadata, refCreatorID objstorage.CreatorID, refFileNum base.DiskFileNum, +) string { + if meta.Shared.CleanupMethod != objstorage.SharedRefTracking { + panic("ref object used when ref tracking disabled") + } + switch meta.FileType { + case base.FileTypeTable: + return fmt.Sprintf( + "%04x-%d-%06d.sst.ref.%d.%06d", + objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(), refCreatorID, refFileNum.FileNum(), + ) + } + panic("unknown FileType") +} + +func sharedObjectRefPrefix(meta objstorage.ObjectMetadata) string { + switch meta.FileType { + case base.FileTypeTable: + return fmt.Sprintf( + "%04x-%d-%06d.sst.ref.", + objHash(meta), meta.Shared.CreatorID, meta.Shared.CreatorFileNum.FileNum(), + ) + } + panic("unknown FileType") +} + +// sharedObjectRefName returns the name of the object's ref marker associated +// with this provider. This name is the object's name concatenated with +// ".ref..". +// +// For example: 1a3f-2-000001.sst.ref.5.000008 +func (p *provider) sharedObjectRefName(meta objstorage.ObjectMetadata) string { + if meta.Shared.CleanupMethod != objstorage.SharedRefTracking { + panic("ref object used when ref tracking disabled") + } + return sharedObjectRefName(meta, p.shared.creatorID, meta.DiskFileNum) +} + +// objHash returns a 16-bit hash value derived from the creator ID and creator +// file num. We prepend this value to object names to ensure balanced +// partitioning with AWS (and likely other blob storage providers). +func objHash(meta objstorage.ObjectMetadata) uint16 { + const prime1 = 7459 + const prime2 = 17539 + return uint16(uint64(meta.Shared.CreatorID)*prime1 + uint64(meta.Shared.CreatorFileNum.FileNum())*prime2) +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_readable.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_readable.go new file mode 100644 index 0000000..ee203bf --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_readable.go @@ -0,0 +1,65 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "context" + + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/shared" +) + +// sharedReadable is a very simple implementation of Readable on top of the +// ReadCloser returned by shared.Storage.CreateObject. +type sharedReadable struct { + objReader shared.ObjectReader + size int64 +} + +var _ objstorage.Readable = (*sharedReadable)(nil) + +func newSharedReadable(objReader shared.ObjectReader, size int64) *sharedReadable { + return &sharedReadable{ + objReader: objReader, + size: size, + } +} + +func (r *sharedReadable) ReadAt(ctx context.Context, p []byte, offset int64) error { + return r.objReader.ReadAt(ctx, p, offset) +} + +func (r *sharedReadable) Close() error { + defer func() { r.objReader = nil }() + return r.objReader.Close() +} + +func (r *sharedReadable) Size() int64 { + return r.size +} + +func (r *sharedReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle { + // TODO(radu): use a pool. + return &sharedReadHandle{readable: r} +} + +type sharedReadHandle struct { + readable *sharedReadable +} + +var _ objstorage.ReadHandle = (*sharedReadHandle)(nil) + +func (r *sharedReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error { + return r.readable.ReadAt(ctx, p, offset) +} + +func (r *sharedReadHandle) Close() error { + r.readable = nil + return nil +} + +func (r *sharedReadHandle) MaxReadahead() {} + +func (r *sharedReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go new file mode 100644 index 0000000..c4f9dc5 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/shared_writable.go @@ -0,0 +1,54 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "io" + + "github.com/cockroachdb/pebble/objstorage" +) + +// sharedWritable is a very simple implementation of Writable on top of the +// WriteCloser returned by shared.Storage.CreateObject. +type sharedWritable struct { + p *provider + meta objstorage.ObjectMetadata + storageWriter io.WriteCloser +} + +var _ objstorage.Writable = (*sharedWritable)(nil) + +// Write is part of the Writable interface. +func (w *sharedWritable) Write(p []byte) error { + _, err := w.storageWriter.Write(p) + return err +} + +// Finish is part of the Writable interface. +func (w *sharedWritable) Finish() error { + err := w.storageWriter.Close() + w.storageWriter = nil + if err != nil { + w.Abort() + return err + } + + // Create the marker object. + if err := w.p.sharedCreateRef(w.meta); err != nil { + w.Abort() + return err + } + return nil +} + +// Abort is part of the Writable interface. +func (w *sharedWritable) Abort() { + if w.storageWriter != nil { + _ = w.storageWriter.Close() + w.storageWriter = nil + } + w.p.removeMetadata(w.meta.DiskFileNum) + // TODO(radu): delete the object if it was created. +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/catalog.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/catalog.go new file mode 100644 index 0000000..25f205e --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/catalog.go @@ -0,0 +1,376 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package sharedobjcat + +import ( + "fmt" + "io" + "sort" + "sync" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/record" + "github.com/cockroachdb/pebble/vfs" + "github.com/cockroachdb/pebble/vfs/atomicfs" +) + +// Catalog is used to manage the on-disk shared object catalog. +// +// The catalog file is a log of records, where each record is an encoded +// versionEdit. +type Catalog struct { + fs vfs.FS + dirname string + mu struct { + sync.Mutex + + creatorID objstorage.CreatorID + objects map[base.DiskFileNum]SharedObjectMetadata + + marker *atomicfs.Marker + + catalogFile vfs.File + catalogRecWriter *record.Writer + + rotationHelper record.RotationHelper + + // catalogFilename is the filename of catalogFile when catalogFile != nil, otherwise + // it is the filename of the last catalog file. + catalogFilename string + } +} + +// SharedObjectMetadata encapsulates the data stored in the catalog file for each object. +type SharedObjectMetadata struct { + // FileNum is the identifier for the object within the context of a single DB + // instance. + FileNum base.DiskFileNum + // FileType is the type of the object. Only certain FileTypes are possible. + FileType base.FileType + // CreatorID identifies the DB instance that originally created the object. + CreatorID objstorage.CreatorID + // CreatorFileNum is the identifier for the object within the context of the + // DB instance that originally created the object. + CreatorFileNum base.DiskFileNum + + CleanupMethod objstorage.SharedCleanupMethod +} + +const ( + catalogFilenameBase = "SHARED-CATALOG" + catalogMarkerName = "shared-catalog" + + // We create a new file when the size exceeds 1MB (and some other conditions + // hold; see record.RotationHelper). + rotateFileSize = 1024 * 1024 // 1MB +) + +// CatalogContents contains the shared objects in the catalog. +type CatalogContents struct { + // CreatorID, if it is set. + CreatorID objstorage.CreatorID + Objects []SharedObjectMetadata +} + +// Open creates a Catalog and loads any existing catalog file, returning the +// creator ID (if it is set) and the contents. +func Open(fs vfs.FS, dirname string) (*Catalog, CatalogContents, error) { + c := &Catalog{ + fs: fs, + dirname: dirname, + } + c.mu.objects = make(map[base.DiskFileNum]SharedObjectMetadata) + + var err error + c.mu.marker, c.mu.catalogFilename, err = atomicfs.LocateMarker(fs, dirname, catalogMarkerName) + if err != nil { + return nil, CatalogContents{}, err + } + // If the filename is empty, there is no existing catalog. + if c.mu.catalogFilename != "" { + if err := c.loadFromCatalogFile(c.mu.catalogFilename); err != nil { + return nil, CatalogContents{}, err + } + if err := c.mu.marker.RemoveObsolete(); err != nil { + return nil, CatalogContents{}, err + } + // TODO(radu): remove obsolete catalog files. + } + res := CatalogContents{ + CreatorID: c.mu.creatorID, + Objects: make([]SharedObjectMetadata, 0, len(c.mu.objects)), + } + for _, meta := range c.mu.objects { + res.Objects = append(res.Objects, meta) + } + // Sort the objects so the function is deterministic. + sort.Slice(res.Objects, func(i, j int) bool { + return res.Objects[i].FileNum.FileNum() < res.Objects[j].FileNum.FileNum() + }) + return c, res, nil +} + +// SetCreatorID sets the creator ID. If it is already set, it must match. +func (c *Catalog) SetCreatorID(id objstorage.CreatorID) error { + if !id.IsSet() { + return errors.AssertionFailedf("attempt to unset CreatorID") + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.mu.creatorID.IsSet() { + if c.mu.creatorID != id { + return errors.AssertionFailedf("attempt to change CreatorID from %s to %s", c.mu.creatorID, id) + } + return nil + } + + ve := versionEdit{CreatorID: id} + if err := c.writeToCatalogFileLocked(&ve); err != nil { + return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err) + } + c.mu.creatorID = id + return nil +} + +// Close any open files. +func (c *Catalog) Close() error { + return c.closeCatalogFile() +} + +func (c *Catalog) closeCatalogFile() error { + if c.mu.catalogFile == nil { + return nil + } + err1 := c.mu.catalogRecWriter.Close() + err2 := c.mu.catalogFile.Close() + c.mu.catalogRecWriter = nil + c.mu.catalogFile = nil + if err1 != nil { + return err1 + } + return err2 +} + +// Batch is used to perform multiple object additions/deletions at once. +type Batch struct { + ve versionEdit +} + +// AddObject adds a new object to the batch. +// +// The given FileNum must be new - it must not match that of any object that was +// ever in the catalog. +func (b *Batch) AddObject(meta SharedObjectMetadata) { + b.ve.NewObjects = append(b.ve.NewObjects, meta) +} + +// DeleteObject adds an object removal to the batch. +func (b *Batch) DeleteObject(fileNum base.DiskFileNum) { + b.ve.DeletedObjects = append(b.ve.DeletedObjects, fileNum) +} + +// Reset clears the batch. +func (b *Batch) Reset() { + b.ve.NewObjects = b.ve.NewObjects[:0] + b.ve.DeletedObjects = b.ve.DeletedObjects[:0] +} + +// IsEmpty returns true if the batch is empty. +func (b *Batch) IsEmpty() bool { + return len(b.ve.NewObjects) == 0 && len(b.ve.DeletedObjects) == 0 +} + +// Copy returns a copy of the Batch. +func (b *Batch) Copy() Batch { + var res Batch + if len(b.ve.NewObjects) > 0 { + res.ve.NewObjects = make([]SharedObjectMetadata, len(b.ve.NewObjects)) + copy(res.ve.NewObjects, b.ve.NewObjects) + } + if len(b.ve.DeletedObjects) > 0 { + res.ve.DeletedObjects = make([]base.DiskFileNum, len(b.ve.DeletedObjects)) + copy(res.ve.DeletedObjects, b.ve.DeletedObjects) + } + return res +} + +// Append merges two batches. +func (b *Batch) Append(other Batch) { + b.ve.NewObjects = append(b.ve.NewObjects, other.ve.NewObjects...) + b.ve.DeletedObjects = append(b.ve.DeletedObjects, other.ve.DeletedObjects...) +} + +// ApplyBatch applies a batch of updates; returns after the change is stably +// recorded on storage. +func (c *Catalog) ApplyBatch(b Batch) error { + c.mu.Lock() + defer c.mu.Unlock() + + for _, n := range b.ve.DeletedObjects { + if _, exists := c.mu.objects[n]; !exists { + return errors.AssertionFailedf("deleting non-existent object %s", n) + } + } + for _, meta := range b.ve.NewObjects { + if _, exists := c.mu.objects[meta.FileNum]; exists { + return errors.AssertionFailedf("adding existing object %s", meta.FileNum) + } + } + + if err := c.writeToCatalogFileLocked(&b.ve); err != nil { + return errors.Wrapf(err, "pebble: could not write to shared object catalog: %v", err) + } + + // Apply the batch to our current state. + for _, n := range b.ve.DeletedObjects { + delete(c.mu.objects, n) + } + for _, meta := range b.ve.NewObjects { + c.mu.objects[meta.FileNum] = meta + } + b.Reset() + return nil +} + +func (c *Catalog) loadFromCatalogFile(filename string) error { + catalogPath := c.fs.PathJoin(c.dirname, filename) + f, err := c.fs.Open(catalogPath) + if err != nil { + return errors.Wrapf( + err, "pebble: could not open shared object catalog file %q for DB %q", + errors.Safe(filename), c.dirname, + ) + } + defer f.Close() + rr := record.NewReader(f, 0 /* logNum */) + for { + r, err := rr.Next() + if err == io.EOF || record.IsInvalidRecord(err) { + break + } + if err != nil { + return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q", + errors.Safe(filename)) + } + var ve versionEdit + err = ve.Decode(r) + if err != nil { + return errors.Wrapf(err, "pebble: error when loading shared object catalog file %q", + errors.Safe(filename)) + } + // Apply the version edit to the current state. + if ve.CreatorID.IsSet() { + c.mu.creatorID = ve.CreatorID + } + for _, fileNum := range ve.DeletedObjects { + delete(c.mu.objects, fileNum) + } + for _, meta := range ve.NewObjects { + c.mu.objects[meta.FileNum] = meta + } + } + return nil +} + +// writeToCatalogFileLocked writes a versionEdit to the catalog file. +// Creates a new file if this is the first write. +func (c *Catalog) writeToCatalogFileLocked(ve *versionEdit) error { + c.mu.rotationHelper.AddRecord(int64(len(ve.NewObjects) + len(ve.DeletedObjects))) + snapshotSize := int64(len(c.mu.objects)) + + var shouldRotate bool + if c.mu.catalogFile == nil { + shouldRotate = true + } else if c.mu.catalogRecWriter.Size() >= rotateFileSize { + shouldRotate = c.mu.rotationHelper.ShouldRotate(snapshotSize) + } + + if shouldRotate { + if c.mu.catalogFile != nil { + if err := c.closeCatalogFile(); err != nil { + return err + } + } + if err := c.createNewCatalogFileLocked(); err != nil { + return err + } + c.mu.rotationHelper.Rotate(snapshotSize) + } + return writeRecord(ve, c.mu.catalogFile, c.mu.catalogRecWriter) +} + +func makeCatalogFilename(iter uint64) string { + return fmt.Sprintf("%s-%06d", catalogFilenameBase, iter) +} + +// createNewCatalogFileLocked creates a new catalog file, populates it with the +// current catalog and sets c.mu.catalogFile and c.mu.catalogRecWriter. +func (c *Catalog) createNewCatalogFileLocked() (outErr error) { + if c.mu.catalogFile != nil { + return errors.AssertionFailedf("catalogFile already open") + } + filename := makeCatalogFilename(c.mu.marker.NextIter()) + filepath := c.fs.PathJoin(c.dirname, filename) + file, err := c.fs.Create(filepath) + if err != nil { + return err + } + recWriter := record.NewWriter(file) + err = func() error { + // Create a versionEdit that gets us from an empty catalog to the current state. + var ve versionEdit + ve.CreatorID = c.mu.creatorID + ve.NewObjects = make([]SharedObjectMetadata, 0, len(c.mu.objects)) + for _, meta := range c.mu.objects { + ve.NewObjects = append(ve.NewObjects, meta) + } + if err := writeRecord(&ve, file, recWriter); err != nil { + return err + } + + // Move the marker to the new filename. Move handles syncing the data + // directory as well. + if err := c.mu.marker.Move(filename); err != nil { + return errors.Wrap(err, "moving marker") + } + + return nil + }() + + if err != nil { + _ = recWriter.Close() + _ = file.Close() + _ = c.fs.Remove(filepath) + return err + } + + // Remove any previous file (ignoring any error). + if c.mu.catalogFilename != "" { + _ = c.fs.Remove(c.fs.PathJoin(c.dirname, c.mu.catalogFilename)) + } + + c.mu.catalogFile = file + c.mu.catalogRecWriter = recWriter + c.mu.catalogFilename = filename + return nil +} + +func writeRecord(ve *versionEdit, file vfs.File, recWriter *record.Writer) error { + w, err := recWriter.Next() + if err != nil { + return err + } + if err := ve.Encode(w); err != nil { + return err + } + if err := recWriter.Flush(); err != nil { + return err + } + return file.Sync() +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/version_edit.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/version_edit.go new file mode 100644 index 0000000..c71e2a8 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat/version_edit.go @@ -0,0 +1,165 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package sharedobjcat + +import ( + "bufio" + "encoding/binary" + "io" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" +) + +// versionEdit is a modification to the shared object state which can be encoded +// into a record. +// +// TODO(radu): consider adding creation and deletion time for debugging purposes. +type versionEdit struct { + NewObjects []SharedObjectMetadata + DeletedObjects []base.DiskFileNum + CreatorID objstorage.CreatorID +} + +const ( + // tagNewObject is followed by the FileNum, creator ID, creator FileNum, and + // cleanup method. + tagNewObject = 1 + // tagDeletedObject is followed by the FileNum. + tagDeletedObject = 2 + // tagCreatorID is followed by the Creator ID for this store. This ID can + // never change. + tagCreatorID = 3 +) + +// Object type values. We don't want to encode FileType directly because it is +// more general (and we want freedom to change it in the future). +const ( + objTypeTable = 1 +) + +func objTypeToFileType(objType uint64) (base.FileType, error) { + switch objType { + case objTypeTable: + return base.FileTypeTable, nil + default: + return 0, errors.Newf("unknown object type %d", objType) + } +} + +func fileTypeToObjType(fileType base.FileType) (uint64, error) { + switch fileType { + case base.FileTypeTable: + return objTypeTable, nil + + default: + return 0, errors.Newf("unknown object type for file type %d", fileType) + } +} + +// Encode encodes an edit to the specified writer. +func (v *versionEdit) Encode(w io.Writer) error { + buf := make([]byte, 0, binary.MaxVarintLen64*(len(v.NewObjects)*4+len(v.DeletedObjects)*2+2)) + for _, meta := range v.NewObjects { + objType, err := fileTypeToObjType(meta.FileType) + if err != nil { + return err + } + buf = binary.AppendUvarint(buf, uint64(tagNewObject)) + buf = binary.AppendUvarint(buf, uint64(meta.FileNum.FileNum())) + buf = binary.AppendUvarint(buf, objType) + buf = binary.AppendUvarint(buf, uint64(meta.CreatorID)) + buf = binary.AppendUvarint(buf, uint64(meta.CreatorFileNum.FileNum())) + buf = binary.AppendUvarint(buf, uint64(meta.CleanupMethod)) + } + + for _, dfn := range v.DeletedObjects { + buf = binary.AppendUvarint(buf, uint64(tagDeletedObject)) + buf = binary.AppendUvarint(buf, uint64(dfn.FileNum())) + } + if v.CreatorID.IsSet() { + buf = binary.AppendUvarint(buf, uint64(tagCreatorID)) + buf = binary.AppendUvarint(buf, uint64(v.CreatorID)) + } + _, err := w.Write(buf) + return err +} + +// Decode decodes an edit from the specified reader. +func (v *versionEdit) Decode(r io.Reader) error { + br, ok := r.(io.ByteReader) + if !ok { + br = bufio.NewReader(r) + } + for { + tag, err := binary.ReadUvarint(br) + if err == io.EOF { + break + } + if err != nil { + return err + } + + err = nil + switch tag { + case tagNewObject: + var fileNum, creatorID, creatorFileNum, cleanupMethod uint64 + var fileType base.FileType + fileNum, err = binary.ReadUvarint(br) + if err == nil { + var objType uint64 + objType, err = binary.ReadUvarint(br) + if err == nil { + fileType, err = objTypeToFileType(objType) + } + } + if err == nil { + creatorID, err = binary.ReadUvarint(br) + } + if err == nil { + creatorFileNum, err = binary.ReadUvarint(br) + } + if err == nil { + cleanupMethod, err = binary.ReadUvarint(br) + } + if err == nil { + v.NewObjects = append(v.NewObjects, SharedObjectMetadata{ + FileNum: base.FileNum(fileNum).DiskFileNum(), + FileType: fileType, + CreatorID: objstorage.CreatorID(creatorID), + CreatorFileNum: base.FileNum(creatorFileNum).DiskFileNum(), + CleanupMethod: objstorage.SharedCleanupMethod(cleanupMethod), + }) + } + + case tagDeletedObject: + var fileNum uint64 + fileNum, err = binary.ReadUvarint(br) + if err == nil { + v.DeletedObjects = append(v.DeletedObjects, base.FileNum(fileNum).DiskFileNum()) + } + + case tagCreatorID: + var id uint64 + id, err = binary.ReadUvarint(br) + if err == nil { + v.CreatorID = objstorage.CreatorID(id) + } + + default: + // Ignore unknown tags. + } + if err != nil { + if err == io.EOF { + return errCorruptCatalog + } + return err + } + } + return nil +} + +var errCorruptCatalog = base.CorruptionErrorf("pebble: corrupt shared object catalog") diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go new file mode 100644 index 0000000..5b20251 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs.go @@ -0,0 +1,109 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "context" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/vfs" +) + +func (p *provider) vfsPath(fileType base.FileType, fileNum base.DiskFileNum) string { + return base.MakeFilepath(p.st.FS, p.st.FSDirName, fileType, fileNum) +} + +func (p *provider) vfsOpenForReading( + ctx context.Context, + fileType base.FileType, + fileNum base.DiskFileNum, + opts objstorage.OpenOptions, +) (objstorage.Readable, error) { + filename := p.vfsPath(fileType, fileNum) + file, err := p.st.FS.Open(filename, vfs.RandomReadsOption) + if err != nil { + if opts.MustExist { + base.MustExist(p.st.FS, filename, p.st.Logger, err) + } + return nil, err + } + return newFileReadable(file, p.st.FS, filename) +} + +func (p *provider) vfsCreate( + _ context.Context, fileType base.FileType, fileNum base.DiskFileNum, +) (objstorage.Writable, objstorage.ObjectMetadata, error) { + filename := p.vfsPath(fileType, fileNum) + file, err := p.st.FS.Create(filename) + if err != nil { + return nil, objstorage.ObjectMetadata{}, err + } + file = vfs.NewSyncingFile(file, vfs.SyncingFileOptions{ + NoSyncOnClose: p.st.NoSyncOnClose, + BytesPerSync: p.st.BytesPerSync, + }) + meta := objstorage.ObjectMetadata{ + DiskFileNum: fileNum, + FileType: fileType, + } + return newFileBufferedWritable(file), meta, nil +} + +func (p *provider) vfsRemove(fileType base.FileType, fileNum base.DiskFileNum) error { + return p.st.FSCleaner.Clean(p.st.FS, fileType, p.vfsPath(fileType, fileNum)) +} + +// vfsInit finds any local FS objects. +func (p *provider) vfsInit() error { + listing := p.st.FSDirInitialListing + if listing == nil { + var err error + listing, err = p.st.FS.List(p.st.FSDirName) + if err != nil { + return errors.Wrapf(err, "pebble: could not list store directory") + } + } + + for _, filename := range listing { + fileType, fileNum, ok := base.ParseFilename(p.st.FS, filename) + if ok && fileType == base.FileTypeTable { + o := objstorage.ObjectMetadata{ + FileType: fileType, + DiskFileNum: fileNum, + } + p.mu.knownObjects[o.DiskFileNum] = o + } + } + return nil +} + +func (p *provider) vfsSync() error { + p.mu.Lock() + shouldSync := p.mu.localObjectsChanged + p.mu.localObjectsChanged = false + p.mu.Unlock() + + if !shouldSync { + return nil + } + if err := p.fsDir.Sync(); err != nil { + p.mu.Lock() + defer p.mu.Unlock() + p.mu.localObjectsChanged = true + return err + } + return nil +} + +func (p *provider) vfsSize(fileType base.FileType, fileNum base.DiskFileNum) (int64, error) { + filename := p.vfsPath(fileType, fileNum) + stat, err := p.st.FS.Stat(filename) + if err != nil { + return 0, err + } + return stat.Size(), nil +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go new file mode 100644 index 0000000..25cea57 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_readable.go @@ -0,0 +1,210 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "context" + "fmt" + "os" + "sync" + + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/vfs" +) + +// fileReadable implements objstorage.Readable on top of a vfs.File. +// +// The implementation might use Prealloc and might reopen the file with +// SequentialReadsOption. +type fileReadable struct { + file vfs.File + size int64 + + // The following fields are used to possibly open the file again using the + // sequential reads option (see vfsReadHandle). + filename string + fs vfs.FS +} + +var _ objstorage.Readable = (*fileReadable)(nil) + +func newFileReadable(file vfs.File, fs vfs.FS, filename string) (*fileReadable, error) { + info, err := file.Stat() + if err != nil { + return nil, err + } + r := &fileReadable{ + file: file, + size: info.Size(), + filename: filename, + fs: fs, + } + invariants.SetFinalizer(r, func(obj interface{}) { + if obj.(*fileReadable).file != nil { + fmt.Fprintf(os.Stderr, "Readable was not closed") + os.Exit(1) + } + }) + return r, nil +} + +// ReadAt is part of the objstorage.Readable interface. +func (r *fileReadable) ReadAt(_ context.Context, p []byte, off int64) error { + n, err := r.file.ReadAt(p, off) + if invariants.Enabled && err == nil && n != len(p) { + panic("short read") + } + return err +} + +// Close is part of the objstorage.Readable interface. +func (r *fileReadable) Close() error { + defer func() { r.file = nil }() + return r.file.Close() +} + +// Size is part of the objstorage.Readable interface. +func (r *fileReadable) Size() int64 { + return r.size +} + +// NewReadHandle is part of the objstorage.Readable interface. +func (r *fileReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle { + rh := readHandlePool.Get().(*vfsReadHandle) + rh.r = r + rh.rs = makeReadaheadState() + return rh +} + +type vfsReadHandle struct { + r *fileReadable + rs readaheadState + + // sequentialFile holds a file descriptor to the same underlying File, + // except with fadvise(FADV_SEQUENTIAL) called on it to take advantage of + // OS-level readahead. Once this is non-nil, the other variables in + // readaheadState don't matter much as we defer to OS-level readahead. + sequentialFile vfs.File +} + +var _ objstorage.ReadHandle = (*vfsReadHandle)(nil) + +var readHandlePool = sync.Pool{ + New: func() interface{} { + i := &vfsReadHandle{} + // Note: this is a no-op if invariants are disabled or race is enabled. + invariants.SetFinalizer(i, func(obj interface{}) { + if obj.(*vfsReadHandle).r != nil { + fmt.Fprintf(os.Stderr, "ReadHandle was not closed") + os.Exit(1) + } + }) + return i + }, +} + +// Close is part of the objstorage.ReadHandle interface. +func (rh *vfsReadHandle) Close() error { + var err error + if rh.sequentialFile != nil { + err = rh.sequentialFile.Close() + } + *rh = vfsReadHandle{} + readHandlePool.Put(rh) + return err +} + +// ReadAt is part of the objstorage.ReadHandle interface. +func (rh *vfsReadHandle) ReadAt(_ context.Context, p []byte, offset int64) error { + var n int + var err error + if rh.sequentialFile != nil { + // Use OS-level read-ahead. + n, err = rh.sequentialFile.ReadAt(p, offset) + } else { + if readaheadSize := rh.rs.maybeReadahead(offset, int64(len(p))); readaheadSize > 0 { + if readaheadSize >= maxReadaheadSize { + // We've reached the maximum readahead size. Beyond this point, rely on + // OS-level readahead. + rh.MaxReadahead() + } else { + _ = rh.r.file.Prefetch(offset, readaheadSize) + } + } + n, err = rh.r.file.ReadAt(p, offset) + } + if invariants.Enabled && err == nil && n != len(p) { + panic("short read") + } + return err +} + +// MaxReadahead is part of the objstorage.ReadHandle interface. +func (rh *vfsReadHandle) MaxReadahead() { + if rh.sequentialFile != nil { + return + } + + // TODO(radu): we could share the reopened file descriptor across multiple + // handles. + f, err := rh.r.fs.Open(rh.r.filename, vfs.SequentialReadsOption) + if err == nil { + rh.sequentialFile = f + } +} + +// RecordCacheHit is part of the objstorage.ReadHandle interface. +func (rh *vfsReadHandle) RecordCacheHit(_ context.Context, offset, size int64) { + if rh.sequentialFile != nil { + // Using OS-level readahead, so do nothing. + return + } + rh.rs.recordCacheHit(offset, size) +} + +// TestingCheckMaxReadahead returns true if the ReadHandle has switched to +// OS-level read-ahead. +func TestingCheckMaxReadahead(rh objstorage.ReadHandle) bool { + switch rh := rh.(type) { + case *vfsReadHandle: + return rh.sequentialFile != nil + case *PreallocatedReadHandle: + return rh.sequentialFile != nil + default: + panic("unknown ReadHandle type") + } +} + +// PreallocatedReadHandle is used to avoid an allocation in NewReadHandle; see +// UsePreallocatedReadHandle. +type PreallocatedReadHandle struct { + vfsReadHandle +} + +// Close is part of the objstorage.ReadHandle interface. +func (rh *PreallocatedReadHandle) Close() error { + var err error + if rh.sequentialFile != nil { + err = rh.sequentialFile.Close() + } + rh.vfsReadHandle = vfsReadHandle{} + return err +} + +// UsePreallocatedReadHandle is equivalent to calling readable.NewReadHandle() +// but uses the existing storage of a PreallocatedReadHandle when possible +// (currently this happens if we are reading from a local file). +// The returned handle still needs to be closed. +func UsePreallocatedReadHandle( + ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle, +) objstorage.ReadHandle { + if r, ok := readable.(*fileReadable); ok { + // See fileReadable.NewReadHandle. + rh.vfsReadHandle = vfsReadHandle{r: r} + return rh + } + return readable.NewReadHandle(ctx) +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_writable.go b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_writable.go new file mode 100644 index 0000000..c9b207c --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/objstorageprovider/vfs_writable.go @@ -0,0 +1,65 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package objstorageprovider + +import ( + "bufio" + + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/vfs" +) + +// NewFileWritable returns a Writable that uses a file as underlying storage. +func NewFileWritable(file vfs.File) objstorage.Writable { + return newFileBufferedWritable(file) +} + +type fileBufferedWritable struct { + file vfs.File + bw *bufio.Writer +} + +var _ objstorage.Writable = (*fileBufferedWritable)(nil) + +func newFileBufferedWritable(file vfs.File) *fileBufferedWritable { + return &fileBufferedWritable{ + file: file, + bw: bufio.NewWriter(file), + } +} + +// Write is part of the objstorage.Writable interface. +func (w *fileBufferedWritable) Write(p []byte) error { + // Ignoring the length written since bufio.Writer.Write is guaranteed to + // return an error if the length written is < len(p). + _, err := w.bw.Write(p) + return err +} + +// Finish is part of the objstorage.Writable interface. +func (w *fileBufferedWritable) Finish() error { + err := w.bw.Flush() + if err == nil { + err = w.file.Sync() + } + err = firstError(err, w.file.Close()) + w.bw = nil + w.file = nil + return err +} + +// Abort is part of the objstorage.Writable interface. +func (w *fileBufferedWritable) Abort() { + _ = w.file.Close() + w.bw = nil + w.file = nil +} + +func firstError(err0, err1 error) error { + if err0 != nil { + return err0 + } + return err1 +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/shared/logging.go b/vendor/github.com/cockroachdb/pebble/objstorage/shared/logging.go new file mode 100644 index 0000000..da3853a --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/shared/logging.go @@ -0,0 +1,139 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package shared + +import ( + "context" + "fmt" + "io" + "sort" +) + +// WithLogging wraps the given Storage implementation and emits logs for various +// operations. +func WithLogging(wrapped Storage, logf func(fmt string, args ...interface{})) Storage { + return &loggingStore{ + logf: logf, + wrapped: wrapped, + } +} + +// loggingStore wraps a shared.Storage implementation and emits logs of the +// operations. +type loggingStore struct { + logf func(fmt string, args ...interface{}) + wrapped Storage +} + +var _ Storage = (*loggingStore)(nil) + +func (l *loggingStore) Close() error { + l.logf("close") + return l.wrapped.Close() +} + +func (l *loggingStore) ReadObject( + ctx context.Context, objName string, +) (_ ObjectReader, objSize int64, _ error) { + r, size, err := l.wrapped.ReadObject(ctx, objName) + l.logf("create reader for object %q: %s", objName, errOrPrintf(err, "%d bytes", size)) + if err != nil { + return nil, 0, err + } + return &loggingReader{ + l: l, + name: objName, + wrapped: r, + }, size, nil +} + +type loggingReader struct { + l *loggingStore + name string + wrapped ObjectReader +} + +var _ ObjectReader = (*loggingReader)(nil) + +func (l *loggingReader) ReadAt(ctx context.Context, p []byte, offset int64) error { + if err := l.wrapped.ReadAt(ctx, p, offset); err != nil { + l.l.logf("read object %q at %d (length %d): error %v", l.name, offset, len(p), err) + return err + } + l.l.logf("read object %q at %d (length %d)", l.name, offset, len(p)) + return nil +} + +func (l *loggingReader) Close() error { + l.l.logf("close reader for %q", l.name) + return l.wrapped.Close() +} + +func (l *loggingStore) CreateObject(objName string) (io.WriteCloser, error) { + l.logf("create object %q", objName) + writer, err := l.wrapped.CreateObject(objName) + if err != nil { + return nil, err + } + return &loggingWriter{ + l: l, + name: objName, + WriteCloser: writer, + }, nil +} + +type loggingWriter struct { + l *loggingStore + name string + bytesWritten int64 + io.WriteCloser +} + +func (l *loggingWriter) Write(p []byte) (n int, err error) { + n, err = l.WriteCloser.Write(p) + l.bytesWritten += int64(n) + return n, err +} + +func (l *loggingWriter) Close() error { + l.l.logf("close writer for %q after %d bytes", l.name, l.bytesWritten) + return l.WriteCloser.Close() +} + +func (l *loggingStore) List(prefix, delimiter string) ([]string, error) { + l.logf("list (prefix=%q, delimiter=%q)", prefix, delimiter) + res, err := l.wrapped.List(prefix, delimiter) + if err != nil { + return nil, err + } + sorted := append([]string(nil), res...) + sort.Strings(sorted) + for _, s := range sorted { + l.logf(" - %s", s) + } + return res, nil +} + +func (l *loggingStore) Delete(objName string) error { + l.logf("delete object %q", objName) + return l.wrapped.Delete(objName) +} + +func (l *loggingStore) Size(objName string) (int64, error) { + size, err := l.wrapped.Size(objName) + l.logf("size of object %q: %s", objName, errOrPrintf(err, "%d", size)) + return size, err +} + +func errOrPrintf(err error, format string, args ...interface{}) string { + if err != nil { + return fmt.Sprintf("error: %v", err) + } + return fmt.Sprintf(format, args...) +} + +func (l *loggingStore) IsNotExistError(err error) bool { + return l.wrapped.IsNotExistError(err) +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/shared/mem.go b/vendor/github.com/cockroachdb/pebble/objstorage/shared/mem.go new file mode 100644 index 0000000..2b6df70 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/shared/mem.go @@ -0,0 +1,161 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package shared + +import ( + "bytes" + "context" + "io" + "os" + "strings" + "sync" +) + +// NewInMem returns an in-memory implementation of the shared.Storage +// interface (for testing). +func NewInMem() Storage { + store := &inMemStore{} + store.mu.objects = make(map[string]*inMemObj) + return store +} + +// inMemStore is an in-memory implementation of the shared.Storage interface +// (for testing). +type inMemStore struct { + mu struct { + sync.Mutex + objects map[string]*inMemObj + } +} + +var _ Storage = (*inMemStore)(nil) + +type inMemObj struct { + name string + data []byte +} + +func (s *inMemStore) Close() error { + *s = inMemStore{} + return nil +} + +func (s *inMemStore) ReadObject( + ctx context.Context, objName string, +) (_ ObjectReader, objSize int64, _ error) { + obj, err := s.getObj(objName) + if err != nil { + return nil, 0, err + } + return &inMemReader{data: obj.data}, int64(len(obj.data)), nil +} + +type inMemReader struct { + data []byte +} + +var _ ObjectReader = (*inMemReader)(nil) + +func (r *inMemReader) ReadAt(ctx context.Context, p []byte, offset int64) error { + if offset+int64(len(p)) > int64(len(r.data)) { + return io.EOF + } + copy(p, r.data[offset:]) + return nil +} + +func (r *inMemReader) Close() error { + r.data = nil + return nil +} + +func (s *inMemStore) CreateObject(objName string) (io.WriteCloser, error) { + return &inMemWriter{ + store: s, + name: objName, + }, nil +} + +type inMemWriter struct { + store *inMemStore + name string + buf bytes.Buffer +} + +var _ io.WriteCloser = (*inMemWriter)(nil) + +func (o *inMemWriter) Write(p []byte) (n int, err error) { + if o.store == nil { + panic("Write after Close") + } + return o.buf.Write(p) +} + +func (o *inMemWriter) Close() error { + if o.store != nil { + o.store.addObj(&inMemObj{ + name: o.name, + data: o.buf.Bytes(), + }) + o.store = nil + } + return nil +} + +func (s *inMemStore) List(prefix, delimiter string) ([]string, error) { + if delimiter != "" { + panic("delimiter unimplemented") + } + + s.mu.Lock() + defer s.mu.Unlock() + res := make([]string, 0, len(s.mu.objects)) + for name := range s.mu.objects { + if strings.HasPrefix(name, prefix) { + res = append(res, name) + } + } + return res, nil +} + +func (s *inMemStore) Delete(objName string) error { + s.rmObj(objName) + return nil +} + +// Size returns the length of the named object in bytesWritten. +func (s *inMemStore) Size(objName string) (int64, error) { + obj, err := s.getObj(objName) + if err != nil { + return 0, err + } + return int64(len(obj.data)), nil +} + +func (s *inMemStore) IsNotExistError(err error) bool { + return err == os.ErrNotExist +} + +func (s *inMemStore) getObj(name string) (*inMemObj, error) { + s.mu.Lock() + defer s.mu.Unlock() + obj, ok := s.mu.objects[name] + if !ok { + return nil, os.ErrNotExist + } + return obj, nil +} + +func (s *inMemStore) addObj(o *inMemObj) { + s.mu.Lock() + defer s.mu.Unlock() + s.mu.objects[o.name] = o +} + +func (s *inMemStore) rmObj(name string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.mu.objects, name) +} diff --git a/vendor/github.com/cockroachdb/pebble/objstorage/shared/storage.go b/vendor/github.com/cockroachdb/pebble/objstorage/shared/storage.go new file mode 100644 index 0000000..c910c70 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/objstorage/shared/storage.go @@ -0,0 +1,74 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package shared + +import ( + "context" + "io" +) + +// Storage is an interface for a blob storage driver. This is lower-level +// than an FS-like interface, however FS/File-like abstractions can be built on +// top of these methods. +// +// TODO(bilal): Consider pushing shared file obsoletion as well as path +// generation behind this interface. +type Storage interface { + io.Closer + + // ReadObject returns an ObjectReader that can be used to perform reads on an + // object, along with the total size of the object. + ReadObject(ctx context.Context, objName string) (_ ObjectReader, objSize int64, _ error) + + // CreateObject returns a writer for the object at the request name. A new + // empty object is created if CreateObject is called on an existing object. + // + // A Writer *must* be closed via either Close, and if closing returns a + // non-nil error, that error should be handled or reported to the user -- an + // implementation may buffer written data until Close and only then return + // an error, or Write may return an opaque io.EOF with the underlying cause + // returned by the subsequent Close(). + // + // TODO(radu): if we encounter some unrelated error while writing to the + // WriteCloser, we'd want to abort the whole thing rather than letting Close + // finalize the upload. + CreateObject(objName string) (io.WriteCloser, error) + + // List enumerates files within the supplied prefix, returning a list of + // objects within that prefix. If delimiter is non-empty, names which have the + // same prefix, prior to the delimiter but after the prefix, are grouped into a + // single result which is that prefix. The order that results are returned is + // undefined. If a prefix is specified, the prefix is trimmed from the result + // list. + // + // An example would be, if the storage contains objects a, b/4, b/5 and b/6, + // these would be the return values: + // List("", "") -> ["a", "b/4", "b/5", "b/6"] + // List("", "/") -> ["a", "b"] + // List("b", "/") -> ["4", "5", "6"] + // List("b", "") -> ["/4", "/5", "/6"] + List(prefix, delimiter string) ([]string, error) + + // Delete removes the named object from the store. + Delete(objName string) error + + // Size returns the length of the named object in bytesWritten. + Size(objName string) (int64, error) + + // IsNotExistError returns true if the given error (returned by a method in + // this interface) indicates that the object does not exist. + IsNotExistError(err error) bool +} + +// ObjectReader is used to perform reads on an object. +type ObjectReader interface { + // ReadAt reads len(p) bytes into p starting at offset off. + // + // Clients of ReadAt can execute parallel ReadAt calls on the + // same ObjectReader. + ReadAt(ctx context.Context, p []byte, offset int64) error + + Close() error +} diff --git a/vendor/github.com/cockroachdb/pebble/open.go b/vendor/github.com/cockroachdb/pebble/open.go index 4d0cf96..ac45175 100644 --- a/vendor/github.com/cockroachdb/pebble/open.go +++ b/vendor/github.com/cockroachdb/pebble/open.go @@ -6,6 +6,7 @@ package pebble import ( "bytes" + "encoding/binary" "fmt" "io" "math" @@ -15,12 +16,16 @@ import ( "time" "github.com/cockroachdb/errors" + "github.com/cockroachdb/errors/oserror" "github.com/cockroachdb/pebble/internal/arenaskl" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/cache" "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/manual" "github.com/cockroachdb/pebble/internal/rate" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider" "github.com/cockroachdb/pebble/record" "github.com/cockroachdb/pebble/vfs" "github.com/prometheus/client_golang/prometheus" @@ -58,6 +63,85 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { if err := opts.Validate(); err != nil { return nil, err } + if opts.LoggerAndTracer == nil { + opts.LoggerAndTracer = &base.LoggerWithNoopTracer{Logger: opts.Logger} + } else { + opts.Logger = opts.LoggerAndTracer + } + + // In all error cases, we return db = nil; this is used by various + // deferred cleanups. + + // Open the database and WAL directories first. + walDirname, dataDir, walDir, err := prepareAndOpenDirs(dirname, opts) + if err != nil { + return nil, err + } + defer func() { + if db == nil { + if walDir != dataDir { + walDir.Close() + } + dataDir.Close() + } + }() + + // Lock the database directory. + var fileLock *Lock + if opts.Lock != nil { + // The caller already acquired the database lock. Ensure that the + // directory matches. + if dirname != opts.Lock.dirname { + return nil, errors.Newf("pebble: opts.Lock acquired in %q not %q", opts.Lock.dirname, dirname) + } + if err := opts.Lock.refForOpen(); err != nil { + return nil, err + } + fileLock = opts.Lock + } else { + fileLock, err = LockDirectory(dirname, opts.FS) + if err != nil { + return nil, err + } + } + defer func() { + if db == nil { + fileLock.Close() + } + }() + + // Establish the format major version. + formatVersion, formatVersionMarker, err := lookupFormatMajorVersion(opts.FS, dirname) + if err != nil { + return nil, err + } + defer func() { + if db == nil { + formatVersionMarker.Close() + } + }() + + // Find the currently active manifest, if there is one. + manifestMarker, manifestFileNum, manifestExists, err := findCurrentManifest(formatVersion, opts.FS, dirname) + if err != nil { + return nil, errors.Wrapf(err, "pebble: database %q", dirname) + } + defer func() { + if db == nil { + manifestMarker.Close() + } + }() + + // Atomic markers may leave behind obsolete files if there's a crash + // mid-update. Clean these up if we're not in read-only mode. + if !opts.ReadOnly { + if err := formatVersionMarker.RemoveObsolete(); err != nil { + return nil, err + } + if err := manifestMarker.RemoveObsolete(); err != nil { + return nil, err + } + } if opts.Cache == nil { opts.Cache = cache.New(cacheDefaultSize) @@ -68,7 +152,7 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { d := &DB{ cacheID: opts.Cache.NewID(), dirname: dirname, - walDirname: opts.WALDir, + walDirname: walDirname, opts: opts, cmp: opts.Comparer.Compare, equal: opts.equal(), @@ -76,12 +160,15 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { split: opts.Comparer.Split, abbreviatedKey: opts.Comparer.AbbreviatedKey, largeBatchThreshold: (opts.MemTableSize - int(memTableEmptySize)) / 2, + fileLock: fileLock, + dataDir: dataDir, + walDir: walDir, logRecycler: logRecycler{limit: opts.MemTableStopWritesThreshold + 1}, closed: new(atomic.Value), closedCh: make(chan struct{}), } d.mu.versions = &versionSet{} - d.atomic.diskAvailBytes = math.MaxUint64 + d.diskAvailBytes.Store(math.MaxUint64) d.mu.versions.diskAvailBytes = d.getDiskAvailableBytesCached defer func() { @@ -107,20 +194,18 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { t.arenaBuf = nil } } + if d.objProvider != nil { + d.objProvider.Close() + } if r != nil { panic(r) } } }() - tableCacheSize := TableCacheSize(opts.MaxOpenFiles) - d.tableCache = newTableCacheContainer(opts.TableCache, d.cacheID, dirname, opts.FS, d.opts, tableCacheSize) - d.newIters = d.tableCache.newIters - d.tableNewRangeKeyIter = d.tableCache.newRangeKeyIter - d.commit = newCommitPipeline(commitEnv{ - logSeqNum: &d.mu.versions.atomic.logSeqNum, - visibleSeqNum: &d.mu.versions.atomic.visibleSeqNum, + logSeqNum: &d.mu.versions.logSeqNum, + visibleSeqNum: &d.mu.versions.visibleSeqNum, apply: d.commitApply, write: d.commitWrite, }) @@ -132,150 +217,53 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { if d.mu.mem.nextSize > initialMemTableSize { d.mu.mem.nextSize = initialMemTableSize } - d.mu.mem.cond.L = &d.mu.Mutex d.mu.cleaner.cond.L = &d.mu.Mutex d.mu.compact.cond.L = &d.mu.Mutex d.mu.compact.inProgress = make(map[*compaction]struct{}) d.mu.compact.noOngoingFlushStartTime = time.Now() d.mu.snapshots.init() // logSeqNum is the next sequence number that will be assigned. Start - // assigning sequence numbers from 1 to match rocksdb. - d.mu.versions.atomic.logSeqNum = 1 + // assigning sequence numbers from base.SeqNumStart to leave room for reserved + // sequence numbers (see comments around SeqNumStart). + d.mu.versions.logSeqNum.Store(base.SeqNumStart) + d.mu.formatVers.vers = formatVersion + d.mu.formatVers.marker = formatVersionMarker d.timeNow = time.Now + d.openedAt = d.timeNow() d.mu.Lock() defer d.mu.Unlock() - if !d.opts.ReadOnly { - err := opts.FS.MkdirAll(dirname, 0755) - if err != nil { - return nil, err - } - } - - // Ensure we close resources if we error out early. If the database is - // successfully opened, the named return value `db` will be set to `d`. - defer func() { - if db != nil { - // The database was successfully opened. - return - } - if d.dataDir != nil { - d.dataDir.Close() - } - if d.walDirname != d.dirname && d.walDir != nil { - d.walDir.Close() - } - if d.mu.formatVers.marker != nil { - d.mu.formatVers.marker.Close() - } - }() - - // Open the database and WAL directories first in order to check for their - // existence. - var err error - d.dataDir, err = opts.FS.OpenDir(dirname) - if err != nil { - return nil, err - } - if d.walDirname == "" { - d.walDirname = d.dirname - } - if d.walDirname == d.dirname { - d.walDir = d.dataDir - } else { - if !d.opts.ReadOnly { - err := opts.FS.MkdirAll(d.walDirname, 0755) - if err != nil { - return nil, err - } - } - d.walDir, err = opts.FS.OpenDir(d.walDirname) - if err != nil { - return nil, err - } - } - - // Lock the database directory. - fileLock, err := opts.FS.Lock(base.MakeFilepath(opts.FS, dirname, fileTypeLock, 0)) - if err != nil { - d.dataDir.Close() - if d.dataDir != d.walDir { - d.walDir.Close() - } - return nil, err - } - defer func() { - if fileLock != nil { - fileLock.Close() - } - }() - - // Establish the format major version. - { - d.mu.formatVers.vers, d.mu.formatVers.marker, err = lookupFormatMajorVersion(opts.FS, dirname) - if err != nil { - return nil, err - } - if !d.opts.ReadOnly { - if err := d.mu.formatVers.marker.RemoveObsolete(); err != nil { - return nil, err - } - } - } - jobID := d.mu.nextJobID d.mu.nextJobID++ - // Find the currently active manifest, if there is one. - manifestMarker, manifestFileNum, exists, err := findCurrentManifest(d.mu.formatVers.vers, opts.FS, dirname) setCurrent := setCurrentFunc(d.mu.formatVers.vers, manifestMarker, opts.FS, dirname, d.dataDir) - defer func() { - // Ensure we close the manifest marker if we error out for any reason. - // If the database is successfully opened, the *versionSet will take - // ownership over the manifest marker, ensuring it's closed when the DB - // is closed. - if db == nil { - manifestMarker.Close() - } - }() - if err != nil { - return nil, errors.Wrapf(err, "pebble: database %q", dirname) - } else if !exists && !d.opts.ReadOnly && !d.opts.ErrorIfNotExists { - // Create the DB if it did not already exist. + if !manifestExists { + // DB does not exist. + if d.opts.ErrorIfNotExists || d.opts.ReadOnly { + return nil, errors.Wrapf(ErrDBDoesNotExist, "dirname=%q", dirname) + } + + // Create the DB. if err := d.mu.versions.create(jobID, dirname, opts, manifestMarker, setCurrent, &d.mu.Mutex); err != nil { return nil, err } - } else if opts.ErrorIfExists { - return nil, errors.Errorf("pebble: database %q already exists", dirname) } else { + if opts.ErrorIfExists { + return nil, errors.Wrapf(ErrDBAlreadyExists, "dirname=%q", dirname) + } // Load the version set. - if err := d.mu.versions.load(dirname, opts, manifestFileNum, manifestMarker, setCurrent, &d.mu.Mutex); err != nil { + if err := d.mu.versions.load(dirname, opts, manifestFileNum.FileNum(), manifestMarker, setCurrent, &d.mu.Mutex); err != nil { return nil, err } - if err := d.mu.versions.currentVersion().CheckConsistency(dirname, opts.FS); err != nil { - return nil, err - } - } - - // If the Options specify a format major version higher than the - // loaded database's, upgrade it. If this is a new database, this - // code path also performs an initial upgrade from the starting - // implicit MostCompatible version. - if !d.opts.ReadOnly && opts.FormatMajorVersion > d.mu.formatVers.vers { - if err := d.ratchetFormatMajorVersionLocked(opts.FormatMajorVersion); err != nil { - return nil, err - } - } - - // Atomic markers like the one used for the MANIFEST may leave - // behind obsolete files if there's a crash mid-update. Clean these - // up if we're not in read-only mode. - if !d.opts.ReadOnly { - if err := manifestMarker.RemoveObsolete(); err != nil { - return nil, err + if opts.ErrorIfNotPristine { + liveFileNums := make(map[base.DiskFileNum]struct{}) + d.mu.versions.addLiveFileNums(liveFileNums) + if len(liveFileNums) != 0 { + return nil, errors.Wrapf(ErrDBNotPristine, "dirname=%q", dirname) + } } } @@ -284,10 +272,11 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { // sequence number of the first batch that will be inserted. if !d.opts.ReadOnly { var entry *flushableEntry - d.mu.mem.mutable, entry = d.newMemTable(0 /* logNum */, d.mu.versions.atomic.logSeqNum) + d.mu.mem.mutable, entry = d.newMemTable(0 /* logNum */, d.mu.versions.logSeqNum.Load()) d.mu.mem.queue = append(d.mu.mem.queue, entry) } + // List the objects ls, err := opts.FS.List(d.walDirname) if err != nil { return nil, err @@ -299,6 +288,33 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { } ls = append(ls, ls2...) } + providerSettings := objstorageprovider.Settings{ + Logger: opts.Logger, + FS: opts.FS, + FSDirName: dirname, + FSDirInitialListing: ls, + FSCleaner: opts.Cleaner, + NoSyncOnClose: opts.NoSyncOnClose, + BytesPerSync: opts.BytesPerSync, + } + providerSettings.Shared.Storage = opts.Experimental.SharedStorage + + d.objProvider, err = objstorageprovider.Open(providerSettings) + if err != nil { + return nil, err + } + + if manifestExists { + curVersion := d.mu.versions.currentVersion() + if err := checkConsistency(curVersion, dirname, d.objProvider); err != nil { + return nil, err + } + } + + tableCacheSize := TableCacheSize(opts.MaxOpenFiles) + d.tableCache = newTableCacheContainer(opts.TableCache, d.cacheID, d.objProvider, d.opts, tableCacheSize) + d.newIters = d.tableCache.newIters + d.tableNewRangeKeyIter = d.tableCache.newRangeKeyIter // Replay any newer log files than the ones named in the manifest. type fileNumAndName struct { @@ -316,21 +332,21 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { // Don't reuse any obsolete file numbers to avoid modifying an // ingested sstable's original external file. - if d.mu.versions.nextFileNum <= fn { - d.mu.versions.nextFileNum = fn + 1 + if d.mu.versions.nextFileNum <= fn.FileNum() { + d.mu.versions.nextFileNum = fn.FileNum() + 1 } switch ft { case fileTypeLog: - if fn >= d.mu.versions.minUnflushedLogNum { - logFiles = append(logFiles, fileNumAndName{fn, filename}) + if fn.FileNum() >= d.mu.versions.minUnflushedLogNum { + logFiles = append(logFiles, fileNumAndName{fn.FileNum(), filename}) } - if d.logRecycler.minRecycleLogNum <= fn { - d.logRecycler.minRecycleLogNum = fn + 1 + if d.logRecycler.minRecycleLogNum <= fn.FileNum() { + d.logRecycler.minRecycleLogNum = fn.FileNum() + 1 } case fileTypeOptions: - if previousOptionsFileNum < fn { - previousOptionsFileNum = fn + if previousOptionsFileNum < fn.FileNum() { + previousOptionsFileNum = fn.FileNum() previousOptionsFilename = filename } case fileTypeTemp, fileTypeOldTemp: @@ -362,19 +378,21 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { }) var ve versionEdit + var toFlush flushableList for i, lf := range logFiles { lastWAL := i == len(logFiles)-1 - maxSeqNum, err := d.replayWAL(jobID, &ve, opts.FS, + flush, maxSeqNum, err := d.replayWAL(jobID, &ve, opts.FS, opts.FS.PathJoin(d.walDirname, lf.name), lf.num, strictWALTail && !lastWAL) if err != nil { return nil, err } + toFlush = append(toFlush, flush...) d.mu.versions.markFileNumUsed(lf.num) - if d.mu.versions.atomic.logSeqNum < maxSeqNum { - d.mu.versions.atomic.logSeqNum = maxSeqNum + if d.mu.versions.logSeqNum.Load() < maxSeqNum { + d.mu.versions.logSeqNum.Store(maxSeqNum) } } - d.mu.versions.atomic.visibleSeqNum = d.mu.versions.atomic.logSeqNum + d.mu.versions.visibleSeqNum.Store(d.mu.versions.logSeqNum.Load()) if !d.opts.ReadOnly { // Create an empty .log file. @@ -396,8 +414,12 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { return nil, err } - newLogName := base.MakeFilepath(opts.FS, d.walDirname, fileTypeLog, newLogNum) - d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum, fileSize: 0}) + for _, entry := range toFlush { + entry.readerUnrefLocked(true) + } + + newLogName := base.MakeFilepath(opts.FS, d.walDirname, fileTypeLog, newLogNum.DiskFileNum()) + d.mu.log.queue = append(d.mu.log.queue, fileInfo{fileNum: newLogNum.DiskFileNum(), fileSize: 0}) logFile, err := opts.FS.Create(newLogName) if err != nil { return nil, err @@ -426,15 +448,29 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { logWriterConfig := record.LogWriterConfig{ WALMinSyncInterval: d.opts.WALMinSyncInterval, WALFsyncLatency: d.mu.log.metrics.fsyncLatency, + QueueSemChan: d.commit.logSyncQSem, } d.mu.log.LogWriter = record.NewLogWriter(logFile, newLogNum, logWriterConfig) d.mu.versions.metrics.WAL.Files++ } d.updateReadStateLocked(d.opts.DebugCheck) + // If the Options specify a format major version higher than the + // loaded database's, upgrade it. If this is a new database, this + // code path also performs an initial upgrade from the starting + // implicit MostCompatible version. + // + // We ratchet the version this far into Open so that migrations have a read + // state available. + if !d.opts.ReadOnly && opts.FormatMajorVersion > d.mu.formatVers.vers { + if err := d.ratchetFormatMajorVersionLocked(opts.FormatMajorVersion); err != nil { + return nil, err + } + } + if !d.opts.ReadOnly { // Write the current options to disk. - d.optionsFileNum = d.mu.versions.getNextFileNum() + d.optionsFileNum = d.mu.versions.getNextFileNum().DiskFileNum() tmpPath := base.MakeFilepath(opts.FS, dirname, fileTypeTemp, d.optionsFileNum) optionsPath := base.MakeFilepath(opts.FS, dirname, fileTypeOptions, d.optionsFileNum) @@ -509,10 +545,53 @@ func Open(dirname string, opts *Options) (db *DB, _ error) { } }) - d.fileLock, fileLock = fileLock, nil return d, nil } +// prepareAndOpenDirs opens the directories for the store (and creates them if +// necessary). +// +// Returns an error if ReadOnly is set and the directories don't exist. +func prepareAndOpenDirs( + dirname string, opts *Options, +) (walDirname string, dataDir vfs.File, walDir vfs.File, err error) { + walDirname = opts.WALDir + if opts.WALDir == "" { + walDirname = dirname + } + + // Create directories if needed. + if !opts.ReadOnly { + if err := opts.FS.MkdirAll(dirname, 0755); err != nil { + return "", nil, nil, err + } + if walDirname != dirname { + if err := opts.FS.MkdirAll(walDirname, 0755); err != nil { + return "", nil, nil, err + } + } + } + + dataDir, err = opts.FS.OpenDir(dirname) + if err != nil { + if opts.ReadOnly && oserror.IsNotExist(err) { + return "", nil, nil, errors.Errorf("pebble: database %q does not exist", dirname) + } + return "", nil, nil, err + } + + if walDirname == dirname { + walDir = dataDir + } else { + walDir, err = opts.FS.OpenDir(walDirname) + if err != nil { + dataDir.Close() + return "", nil, nil, err + } + } + return walDirname, dataDir, walDir, nil +} + // GetVersion returns the engine version string from the latest options // file present in dir. Used to check what Pebble or RocksDB version was last // used to write to the database stored in this directory. An empty string is @@ -535,9 +614,9 @@ func GetVersion(dir string, fs vfs.FS) (string, error) { // processed, reset version. This is because rocksdb often // writes multiple options files without deleting previous ones. // Otherwise, skip parsing this options file. - if fn > lastOptionsSeen { + if fn.FileNum() > lastOptionsSeen { version = "" - lastOptionsSeen = fn + lastOptionsSeen = fn.FileNum() } else { continue } @@ -571,25 +650,31 @@ func GetVersion(dir string, fs vfs.FS) (string, error) { return version, nil } -// replayWAL replays the edits in the specified log file. +// replayWAL replays the edits in the specified log file. If the DB is in +// read only mode, then the WALs are replayed into memtables and not flushed. If +// the DB is not in read only mode, then the contents of the WAL are guaranteed +// to be flushed. +// +// The toFlush return value is a list of flushables associated with the WAL +// being replayed which will be flushed. Once the version edit has been applied +// to the manifest, it is up to the caller of replayWAL to unreference the +// toFlush flushables returned by replayWAL. // // d.mu must be held when calling this, but the mutex may be dropped and // re-acquired during the course of this method. func (d *DB) replayWAL( jobID int, ve *versionEdit, fs vfs.FS, filename string, logNum FileNum, strictWALTail bool, -) (maxSeqNum uint64, err error) { +) (toFlush flushableList, maxSeqNum uint64, err error) { file, err := fs.Open(filename) if err != nil { - return 0, err + return nil, 0, err } defer file.Close() - var ( b Batch buf bytes.Buffer mem *memTable entry *flushableEntry - toFlush flushableList rr = record.NewReader(file, logNum) offset int64 // byte offset in rr lastFlushOffset int64 @@ -633,6 +718,23 @@ func (d *DB) replayWAL( d.mu.mem.queue = append(d.mu.mem.queue, entry) } } + + // updateVE is used to update ve with information about new files created + // during the flush of any flushable not of type ingestedFlushable. For the + // flushable of type ingestedFlushable we use custom handling below. + updateVE := func() error { + // TODO(bananabrick): See if we can use the actual base level here, + // instead of using 1. + c := newFlush(d.opts, d.mu.versions.currentVersion(), + 1 /* base level */, toFlush, d.timeNow()) + newVE, _, _, err := d.runCompaction(jobID, c) + if err != nil { + return errors.Wrapf(err, "running compaction during WAL replay") + } + ve.NewFiles = append(ve.NewFiles, newVE.NewFiles...) + return nil + } + for { offset = rr.Offset() r, err := rr.Next() @@ -650,14 +752,18 @@ func (d *DB) replayWAL( } else if record.IsInvalidRecord(err) && !strictWALTail { break } - return 0, errors.Wrap(err, "pebble: error when replaying WAL") + return nil, 0, errors.Wrap(err, "pebble: error when replaying WAL") } if buf.Len() < batchHeaderLen { - return 0, base.CorruptionErrorf("pebble: corrupt log file %q (num %s)", + return nil, 0, base.CorruptionErrorf("pebble: corrupt log file %q (num %s)", filename, errors.Safe(logNum)) } + if d.opts.ErrorIfNotPristine { + return nil, 0, errors.WithDetailf(ErrDBNotPristine, "location: %q", d.dirname) + } + // Specify Batch.db so that Batch.SetRepr will compute Batch.memTableSize // which is used below. b = Batch{db: d} @@ -665,6 +771,100 @@ func (d *DB) replayWAL( seqNum := b.SeqNum() maxSeqNum = seqNum + uint64(b.Count()) + { + br := b.Reader() + if kind, encodedFileNum, _, _ := br.Next(); kind == InternalKeyKindIngestSST { + fileNums := make([]base.DiskFileNum, 0, b.Count()) + addFileNum := func(encodedFileNum []byte) { + fileNum, n := binary.Uvarint(encodedFileNum) + if n <= 0 { + panic("pebble: ingest sstable file num is invalid.") + } + fileNums = append(fileNums, base.FileNum(fileNum).DiskFileNum()) + } + addFileNum(encodedFileNum) + + for i := 1; i < int(b.Count()); i++ { + kind, encodedFileNum, _, ok := br.Next() + if kind != InternalKeyKindIngestSST { + panic("pebble: invalid batch key kind.") + } + if !ok { + panic("pebble: invalid batch count.") + } + addFileNum(encodedFileNum) + } + + if _, _, _, ok := br.Next(); ok { + panic("pebble: invalid number of entries in batch.") + } + + paths := make([]string, len(fileNums)) + for i, n := range fileNums { + paths[i] = base.MakeFilepath(d.opts.FS, d.dirname, fileTypeTable, n) + } + + var meta []*manifest.FileMetadata + meta, _, err = ingestLoad( + d.opts, d.mu.formatVers.vers, paths, d.cacheID, fileNums, + ) + if err != nil { + return nil, 0, err + } + + if uint32(len(meta)) != b.Count() { + panic("pebble: couldn't load all files in WAL entry.") + } + + entry, err = d.newIngestedFlushableEntry( + meta, seqNum, logNum, + ) + if err != nil { + return nil, 0, err + } + + if d.opts.ReadOnly { + d.mu.mem.queue = append(d.mu.mem.queue, entry) + // We added the IngestSST flushable to the queue. But there + // must be at least one WAL entry waiting to be replayed. We + // have to ensure this newer WAL entry isn't replayed into + // the current value of d.mu.mem.mutable because the current + // mutable memtable exists before this flushable entry in + // the memtable queue. To ensure this, we just need to unset + // d.mu.mem.mutable. When a newer WAL is replayed, we will + // set d.mu.mem.mutable to a newer value. + d.mu.mem.mutable = nil + } else { + toFlush = append(toFlush, entry) + // During WAL replay, the lsm only has L0, hence, the + // baseLevel is 1. For the sake of simplicity, we place the + // ingested files in L0 here, instead of finding their + // target levels. This is a simplification for the sake of + // simpler code. It is expected that WAL replay should be + // rare, and that flushables of type ingestedFlushable + // should also be rare. So, placing the ingested files in L0 + // is alright. + // + // TODO(bananabrick): Maybe refactor this function to allow + // us to easily place ingested files in levels as low as + // possible during WAL replay. It would require breaking up + // the application of ve to the manifest into chunks and is + // not pretty w/o a refactor to this function and how it's + // used. + c := newFlush( + d.opts, d.mu.versions.currentVersion(), + 1, /* base level */ + []*flushableEntry{entry}, + d.timeNow(), + ) + for _, file := range c.flushing[0].flushable.(*ingestedFlushable).files { + ve.NewFiles = append(ve.NewFiles, newFileEntry{Level: 0, Meta: file.FileMetadata}) + } + } + return toFlush, maxSeqNum, nil + } + } + if b.memTableSize >= uint64(d.largeBatchThreshold) { flushMem() // Make a copy of the data slice since it is currently owned by buf and will @@ -677,13 +877,22 @@ func (d *DB) replayWAL( entry.readerRefs++ if d.opts.ReadOnly { d.mu.mem.queue = append(d.mu.mem.queue, entry) + // We added the flushable batch to the flushable to the queue. + // But there must be at least one WAL entry waiting to be + // replayed. We have to ensure this newer WAL entry isn't + // replayed into the current value of d.mu.mem.mutable because + // the current mutable memtable exists before this flushable + // entry in the memtable queue. To ensure this, we just need to + // unset d.mu.mem.mutable. When a newer WAL is replayed, we will + // set d.mu.mem.mutable to a newer value. + d.mu.mem.mutable = nil } else { toFlush = append(toFlush, entry) } } else { ensureMem(seqNum) if err = mem.prepare(&b); err != nil && err != arenaskl.ErrArenaFull { - return 0, err + return nil, 0, err } // We loop since DB.newMemTable() slowly grows the size of allocated memtables, so the // batch may not initially fit, but will eventually fit (since it is smaller than @@ -693,11 +902,11 @@ func (d *DB) replayWAL( ensureMem(seqNum) err = mem.prepare(&b) if err != nil && err != arenaskl.ErrArenaFull { - return 0, err + return nil, 0, err } } if err = mem.apply(&b, seqNum); err != nil { - return 0, err + return nil, 0, err } mem.writerUnref() } @@ -706,18 +915,12 @@ func (d *DB) replayWAL( flushMem() // mem is nil here. if !d.opts.ReadOnly { - c := newFlush(d.opts, d.mu.versions.currentVersion(), - 1 /* base level */, toFlush) - newVE, _, err := d.runCompaction(jobID, c) + err = updateVE() if err != nil { - return 0, err - } - ve.NewFiles = append(ve.NewFiles, newVE.NewFiles...) - for i := range toFlush { - toFlush[i].readerUnref() + return nil, 0, err } } - return maxSeqNum, err + return toFlush, maxSeqNum, err } func checkOptions(opts *Options, path string) (strictWALTail bool, err error) { @@ -748,7 +951,7 @@ type DBDesc struct { // Peek looks for an existing database in dirname on the provided FS. It // returns a brief description of the database. Peek is read-only and -// does not open the database. +// does not open the database func Peek(dirname string, fs vfs.FS) (*DBDesc, error) { vers, versMarker, err := lookupFormatMajorVersion(fs, dirname) if err != nil { @@ -780,3 +983,124 @@ func Peek(dirname string, fs vfs.FS) (*DBDesc, error) { } return desc, nil } + +// LockDirectory acquires the database directory lock in the named directory, +// preventing another process from opening the database. LockDirectory returns a +// handle to the held lock that may be passed to Open through Options.Lock to +// subsequently open the database, skipping lock acquistion during Open. +// +// LockDirectory may be used to expand the critical section protected by the +// database lock to include setup before the call to Open. +func LockDirectory(dirname string, fs vfs.FS) (*Lock, error) { + fileLock, err := fs.Lock(base.MakeFilepath(fs, dirname, fileTypeLock, base.FileNum(0).DiskFileNum())) + if err != nil { + return nil, err + } + l := &Lock{dirname: dirname, fileLock: fileLock} + l.refs.Store(1) + invariants.SetFinalizer(l, func(obj interface{}) { + if refs := obj.(*Lock).refs.Load(); refs > 0 { + panic(errors.AssertionFailedf("lock for %q finalized with %d refs", dirname, refs)) + } + }) + return l, nil +} + +// Lock represents a file lock on a directory. It may be passed to Open through +// Options.Lock to elide lock aquisition during Open. +type Lock struct { + dirname string + fileLock io.Closer + // refs is a count of the number of handles on the lock. refs must be 0, 1 + // or 2. + // + // When acquired by the client and passed to Open, refs = 1 and the Open + // call increments it to 2. When the database is closed, it's decremented to + // 1. Finally when the original caller, calls Close on the Lock, it's + // drecemented to zero and the underlying file lock is released. + // + // When Open acquires the file lock, refs remains at 1 until the database is + // closed. + refs atomic.Int32 +} + +func (l *Lock) refForOpen() error { + // During Open, when a user passed in a lock, the reference count must be + // exactly 1. If it's zero, the lock is no longer held and is invalid. If + // it's 2, the lock is already in use by another database within the + // process. + if !l.refs.CompareAndSwap(1, 2) { + return errors.Errorf("pebble: unexpected Lock reference count; is the lock already in use?") + } + return nil +} + +// Close releases the lock, permitting another process to lock and open the +// database. Close must not be called until after a database using the Lock has +// been closed. +func (l *Lock) Close() error { + if l.refs.Add(-1) > 0 { + return nil + } + defer func() { l.fileLock = nil }() + return l.fileLock.Close() +} + +// ErrDBDoesNotExist is generated when ErrorIfNotExists is set and the database +// does not exist. +// +// Note that errors can be wrapped with more details; use errors.Is(). +var ErrDBDoesNotExist = errors.New("pebble: database does not exist") + +// ErrDBAlreadyExists is generated when ErrorIfExists is set and the database +// already exists. +// +// Note that errors can be wrapped with more details; use errors.Is(). +var ErrDBAlreadyExists = errors.New("pebble: database already exists") + +// ErrDBNotPristine is generated when ErrorIfNotPristine is set and the database +// already exists and is not pristine. +// +// Note that errors can be wrapped with more details; use errors.Is(). +var ErrDBNotPristine = errors.New("pebble: database already exists and is not pristine") + +func checkConsistency(v *manifest.Version, dirname string, objProvider objstorage.Provider) error { + var buf bytes.Buffer + var args []interface{} + + dedup := make(map[base.DiskFileNum]struct{}) + for level, files := range v.Levels { + iter := files.Iter() + for f := iter.First(); f != nil; f = iter.Next() { + backingState := f.FileBacking + if _, ok := dedup[backingState.DiskFileNum]; ok { + continue + } + dedup[backingState.DiskFileNum] = struct{}{} + fileNum := backingState.DiskFileNum + fileSize := backingState.Size + meta, err := objProvider.Lookup(base.FileTypeTable, fileNum) + var size int64 + if err == nil { + size, err = objProvider.Size(meta) + } + if err != nil { + buf.WriteString("L%d: %s: %v\n") + args = append(args, errors.Safe(level), errors.Safe(fileNum), err) + continue + } + + if size != int64(fileSize) { + buf.WriteString("L%d: %s: object size mismatch (%s): %d (disk) != %d (MANIFEST)\n") + args = append(args, errors.Safe(level), errors.Safe(fileNum), objProvider.Path(meta), + errors.Safe(size), errors.Safe(fileSize)) + continue + } + } + } + + if buf.Len() == 0 { + return nil + } + return errors.Errorf(buf.String(), args...) +} diff --git a/vendor/github.com/cockroachdb/pebble/options.go b/vendor/github.com/cockroachdb/pebble/options.go index db5b86c..2bdc9d5 100644 --- a/vendor/github.com/cockroachdb/pebble/options.go +++ b/vendor/github.com/cockroachdb/pebble/options.go @@ -18,6 +18,7 @@ import ( "github.com/cockroachdb/pebble/internal/cache" "github.com/cockroachdb/pebble/internal/humanize" "github.com/cockroachdb/pebble/internal/manifest" + "github.com/cockroachdb/pebble/objstorage/shared" "github.com/cockroachdb/pebble/sstable" "github.com/cockroachdb/pebble/vfs" ) @@ -61,6 +62,12 @@ type BlockPropertyCollector = sstable.BlockPropertyCollector // BlockPropertyFilter exports the sstable.BlockPropertyFilter type. type BlockPropertyFilter = base.BlockPropertyFilter +// ShortAttributeExtractor exports the base.ShortAttributeExtractor type. +type ShortAttributeExtractor = base.ShortAttributeExtractor + +// UserKeyPrefixBound exports the sstable.UserKeyPrefixBound type. +type UserKeyPrefixBound = sstable.UserKeyPrefixBound + // IterKeyType configures which types of keys an iterator should surface. type IterKeyType int8 @@ -215,6 +222,16 @@ func (o *IterOptions) getLogger() Logger { return o.logger } +// scanInternalOptions is similar to IterOptions, meant for use with +// scanInternalIterator. +type scanInternalOptions struct { + IterOptions + + // skipSharedLevels skips levels that are shareable (level >= + // sharedLevelStart). + skipSharedLevels bool +} + // RangeKeyMasking configures automatic hiding of point keys by range keys. A // non-nil Suffix enables range-key masking. When enabled, range keys with // suffixes ≥ Suffix behave as masks. All point keys that are contained within a @@ -224,7 +241,7 @@ func (o *IterOptions) getLogger() Logger { // Specifically, when configured with a RangeKeyMasking.Suffix _s_, and there // exists a range key with suffix _r_ covering a point key with suffix _p_, and // -// _s_ ≤ _r_ < _p_ +// _s_ ≤ _r_ < _p_ // // then the point key is elided. // @@ -387,6 +404,8 @@ func (o *LevelOptions) EnsureDefaults() *LevelOptions { } if o.BlockSize <= 0 { o.BlockSize = base.DefaultBlockSize + } else if o.BlockSize > sstable.MaximumBlockSize { + panic(errors.Errorf("BlockSize %d exceeds MaximumBlockSize", o.BlockSize)) } if o.BlockSizeThreshold <= 0 { o.BlockSizeThreshold = base.DefaultBlockSizeThreshold @@ -445,21 +464,31 @@ type Options struct { // TODO(peter): untested DisableWAL bool - // ErrorIfExists is whether it is an error if the database already exists. + // ErrorIfExists causes an error on Open if the database already exists. + // The error can be checked with errors.Is(err, ErrDBAlreadyExists). // // The default value is false. ErrorIfExists bool - // ErrorIfNotExists is whether it is an error if the database does not - // already exist. + // ErrorIfNotExists causes an error on Open if the database does not already + // exist. The error can be checked with errors.Is(err, ErrDBDoesNotExist). // // The default value is false which will cause a database to be created if it // does not already exist. ErrorIfNotExists bool + // ErrorIfNotPristine causes an error on Open if the database already exists + // and any operations have been performed on the database. The error can be + // checked with errors.Is(err, ErrDBNotPristine). + // + // Note that a database that contained keys that were all subsequently deleted + // may or may not trigger the error. Currently, we check if there are any live + // SSTs or log records to replay. + ErrorIfNotPristine bool + // EventListener provides hooks to listening to significant DB events such as // flushes, compactions, and table deletion. - EventListener EventListener + EventListener *EventListener // Experimental contains experimental options which are off by default. // These options are temporary and will eventually either be deleted, moved @@ -554,10 +583,10 @@ type Options struct { // desired size of each level of the LSM. Defaults to 10. LevelMultiplier int - // MultiLevelCompaction allows the compaction of SSTs from more than two - // levels iff a conventional two level compaction will quickly trigger a - // compaction in the output level. - MultiLevelCompaction bool + // MultiLevelCompactionHueristic determines whether to add an additional + // level to a conventional two level compaction. If nil, a multilevel + // compaction will never get triggered. + MultiLevelCompactionHueristic MultiLevelHeuristic // MaxWriterConcurrency is used to indicate the maximum number of // compression workers the compression queue is allowed to use. If @@ -582,6 +611,49 @@ type Options struct { // // The default value is 1, which results in no scaling of point tombstones. PointTombstoneWeight float64 + + // EnableValueBlocks is used to decide whether to enable writing + // TableFormatPebblev3 sstables. This setting is only respected by a + // specific subset of format major versions: FormatSSTableValueBlocks, + // FormatFlushableIngest and FormatPrePebblev1MarkedCompacted. In lower + // format major versions, value blocks are never enabled. In higher + // format major versions, value blocks are always enabled. + EnableValueBlocks func() bool + + // ShortAttributeExtractor is used iff EnableValueBlocks() returns true + // (else ignored). If non-nil, a ShortAttribute can be extracted from the + // value and stored with the key, when the value is stored elsewhere. + ShortAttributeExtractor ShortAttributeExtractor + + // RequiredInPlaceValueBound specifies an optional span of user key + // prefixes for which the values must be stored with the key. This is + // useful for statically known exclusions to value separation. In + // CockroachDB, this will be used for the lock table key space that has + // non-empty suffixes, but those locks don't represent actual MVCC + // versions (the suffix ordering is arbitrary). We will also need to add + // support for dynamically configured exclusions (we want the default to + // be to allow Pebble to decide whether to separate the value or not, + // hence this is structured as exclusions), for example, for users of + // CockroachDB to dynamically exclude certain tables. + // + // Any change in exclusion behavior takes effect only on future written + // sstables, and does not start rewriting existing sstables. + RequiredInPlaceValueBound UserKeyPrefixBound + + // DisableIngestAsFlushable disables lazy ingestion of sstables through + // a WAL write and memtable rotation. Only effectual if the the format + // major version is at least `FormatFlushableIngest`. + DisableIngestAsFlushable func() bool + + // SharedStorage is a second FS-like storage medium that can be shared + // between multiple Pebble instances. It is used to store sstables only, and + // is managed by objstorage.Provider. Each sstable might only be written to + // by one Pebble instance, but other Pebble instances can possibly read the + // same files if they have the path to get to them. The pebble instance that + // wrote a file should not delete it if other Pebble instances are known to + // be reading this file. This FS is expected to have slower read/write + // performance than the default FS above. + SharedStorage shared.Storage } // Filters is a map from filter policy name to filter policy. It is used for @@ -635,6 +707,17 @@ type Options struct { // The default value uses the underlying operating system's file system. FS vfs.FS + // Lock, if set, must be a database lock acquired through LockDirectory for + // the same directory passed to Open. If provided, Open will skip locking + // the directory. Closing the database will not release the lock, and it's + // the responsibility of the caller to release the lock after closing the + // database. + // + // Open will enforce that the Lock passed locks the same directory passed to + // Open. Concurrent calls to Open using the same Lock are detected and + // prohibited. + Lock *Lock + // The count of L0 files necessary to trigger an L0 compaction. L0CompactionFileThreshold int @@ -656,10 +739,15 @@ type Options struct { // options for the last level are used for all subsequent levels. Levels []LevelOptions + // LoggerAndTracer will be used, if non-nil, else Logger will be used and + // tracing will be a noop. + // Logger used to write log messages. // // The default logger uses the Go standard library log package. Logger Logger + // LoggerAndTracer is used for writing log messages and traces. + LoggerAndTracer LoggerAndTracer // MaxManifestFileSize is the maximum size the MANIFEST file is allowed to // become. When the MANIFEST exceeds this size it is rolled over and a new @@ -783,6 +871,23 @@ type Options struct { // RocksDB 6.2.1. strictWALTail bool + // disableDeleteOnlyCompactions prevents the scheduling of delete-only + // compactions that drop sstables wholy covered by range tombstones or + // range key tombstones. + disableDeleteOnlyCompactions bool + + // disableElisionOnlyCompactions prevents the scheduling of elision-only + // compactions that rewrite sstables in place in order to elide obsolete + // keys. + disableElisionOnlyCompactions bool + + // disableLazyCombinedIteration is a private option used by the + // metamorphic tests to test equivalence between lazy-combined iteration + // and constructing the range-key iterator upfront. It's a private + // option to avoid littering the public interface with options that we + // do not want to allow users to actually configure. + disableLazyCombinedIteration bool + // A private option to disable stats collection. disableTableStats bool @@ -795,13 +900,6 @@ type Options struct { // against the FS are made after the DB is closed, the FS may leak a // goroutine indefinitely. fsCloser io.Closer - - // disableLazyCombinedIteration is a private option used by the - // metamorphic tests to test equivalence between lazy-combined iteration - // and constructing the range-key iterator upfront. It's a private - // option to avoid littering the public interface with options that we - // do not want to allow users to actually configure. - disableLazyCombinedIteration bool } } @@ -827,6 +925,9 @@ func (o *Options) EnsureDefaults() *Options { if o.Comparer == nil { o.Comparer = DefaultComparer } + if o.Experimental.DisableIngestAsFlushable == nil { + o.Experimental.DisableIngestAsFlushable = func() bool { return false } + } if o.Experimental.L0CompactionConcurrency <= 0 { o.Experimental.L0CompactionConcurrency = 10 } @@ -890,6 +991,9 @@ func (o *Options) EnsureDefaults() *Options { if o.Logger == nil { o.Logger = DefaultLogger } + if o.EventListener == nil { + o.EventListener = &EventListener{} + } o.EventListener.EnsureDefaults(o.Logger) if o.MaxManifestFileSize == 0 { o.MaxManifestFileSize = 128 << 20 // 128 MB @@ -919,13 +1023,7 @@ func (o *Options) EnsureDefaults() *Options { } if o.FS == nil { - o.FS, o.private.fsCloser = vfs.WithDiskHealthChecks(vfs.Default, 5*time.Second, - func(name string, duration time.Duration) { - o.EventListener.DiskSlow(DiskSlowInfo{ - Path: name, - Duration: duration, - }) - }) + o.WithFSDefaults() } if o.FlushSplitBytes <= 0 { o.FlushSplitBytes = 2 * o.Levels[0].TargetFileSize @@ -949,10 +1047,36 @@ func (o *Options) EnsureDefaults() *Options { o.Experimental.PointTombstoneWeight = 1 } + if o.Experimental.MultiLevelCompactionHueristic == nil { + o.Experimental.MultiLevelCompactionHueristic = NoMultiLevel{} + } + o.initMaps() return o } +// WithFSDefaults configures the Options to wrap the configured filesystem with +// the default virtual file system middleware, like disk-health checking. +func (o *Options) WithFSDefaults() *Options { + if o.FS == nil { + o.FS = vfs.Default + } + o.FS, o.private.fsCloser = vfs.WithDiskHealthChecks(o.FS, 5*time.Second, + func(info vfs.DiskSlowInfo) { + o.EventListener.DiskSlow(info) + }) + return o +} + +// AddEventListener adds the provided event listener to the Options, in addition +// to any existing event listener. +func (o *Options) AddEventListener(l EventListener) { + if o.EventListener != nil { + l = TeeEventListener(l, *o.EventListener) + } + o.EventListener = &l +} + func (o *Options) equal() Equal { if o.Comparer.Equal == nil { return bytes.Equal @@ -1023,6 +1147,9 @@ func (o *Options) String() string { fmt.Fprintf(&buf, " compaction_debt_concurrency=%d\n", o.Experimental.CompactionDebtConcurrency) fmt.Fprintf(&buf, " comparer=%s\n", o.Comparer.Name) fmt.Fprintf(&buf, " disable_wal=%t\n", o.DisableWAL) + if o.Experimental.DisableIngestAsFlushable != nil && o.Experimental.DisableIngestAsFlushable() { + fmt.Fprintf(&buf, " disable_ingest_as_flushable=%t\n", true) + } fmt.Fprintf(&buf, " flush_delay_delete_range=%s\n", o.FlushDelayDeleteRange) fmt.Fprintf(&buf, " flush_delay_range_key=%s\n", o.FlushDelayRangeKey) fmt.Fprintf(&buf, " flush_split_bytes=%d\n", o.FlushSplitBytes) @@ -1064,11 +1191,18 @@ func (o *Options) String() string { fmt.Fprintf(&buf, " force_writer_parallelism=%t\n", o.Experimental.ForceWriterParallelism) // Private options. + // + // These options are only encoded if true, because we do not want them to + // appear in production serialized Options files, since they're testing-only + // options. They're only serialized when true, which still ensures that the + // metamorphic tests may propagate them to subprocesses. + if o.private.disableDeleteOnlyCompactions { + fmt.Fprintln(&buf, " disable_delete_only_compactions=true") + } + if o.private.disableElisionOnlyCompactions { + fmt.Fprintln(&buf, " disable_elision_only_compactions=true") + } if o.private.disableLazyCombinedIteration { - // This option is only encoded if true, because we do not want it to - // appear in production serialized Options files, since it's a - // testing-only option. It's only serialized when true, which still - // ensures that the metamorphic tests may propagate it to subprocesses. fmt.Fprintln(&buf, " disable_lazy_combined_iteration=true") } @@ -1078,6 +1212,7 @@ func (o *Options) String() string { fmt.Fprintf(&buf, "[Level \"%d\"]\n", i) fmt.Fprintf(&buf, " block_restart_interval=%d\n", l.BlockRestartInterval) fmt.Fprintf(&buf, " block_size=%d\n", l.BlockSize) + fmt.Fprintf(&buf, " block_size_threshold=%d\n", l.BlockSizeThreshold) fmt.Fprintf(&buf, " compression=%s\n", l.Compression) fmt.Fprintf(&buf, " filter_policy=%s\n", filterPolicyName(l.FilterPolicy)) fmt.Fprintf(&buf, " filter_type=%s\n", l.FilterType) @@ -1212,10 +1347,20 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error { // NB: This is a deprecated serialization of the // `flush_delay_delete_range`. o.FlushDelayDeleteRange, err = time.ParseDuration(value) - case "disable_wal": - o.DisableWAL, err = strconv.ParseBool(value) + case "disable_delete_only_compactions": + o.private.disableDeleteOnlyCompactions, err = strconv.ParseBool(value) + case "disable_elision_only_compactions": + o.private.disableElisionOnlyCompactions, err = strconv.ParseBool(value) + case "disable_ingest_as_flushable": + var v bool + v, err = strconv.ParseBool(value) + if err == nil { + o.Experimental.DisableIngestAsFlushable = func() bool { return v } + } case "disable_lazy_combined_iteration": o.private.disableLazyCombinedIteration, err = strconv.ParseBool(value) + case "disable_wal": + o.DisableWAL, err = strconv.ParseBool(value) case "flush_delay_delete_range": o.FlushDelayDeleteRange, err = time.ParseDuration(value) case "flush_delay_range_key": @@ -1346,6 +1491,8 @@ func (o *Options) Parse(s string, hooks *ParseHooks) error { l.BlockRestartInterval, err = strconv.Atoi(value) case "block_size": l.BlockSize, err = strconv.Atoi(value) + case "block_size_threshold": + l.BlockSizeThreshold, err = strconv.Atoi(value) case "compression": switch value { case "Default": @@ -1471,6 +1618,7 @@ func (o *Options) MakeReaderOptions() sstable.ReaderOptions { if o.Merger != nil { readerOpts.MergerName = o.Merger.Name } + readerOpts.LoggerAndTracer = o.LoggerAndTracer } return readerOpts } @@ -1489,6 +1637,10 @@ func (o *Options) MakeWriterOptions(level int, format sstable.TableFormat) sstab writerOpts.TablePropertyCollectors = o.TablePropertyCollectors writerOpts.BlockPropertyCollectors = o.BlockPropertyCollectors } + if format >= sstable.TableFormatPebblev3 { + writerOpts.ShortAttributeExtractor = o.Experimental.ShortAttributeExtractor + writerOpts.RequiredInPlaceValueBound = o.Experimental.RequiredInPlaceValueBound + } levelOpts := o.Level(level) writerOpts.BlockRestartInterval = levelOpts.BlockRestartInterval writerOpts.BlockSize = levelOpts.BlockSize diff --git a/vendor/github.com/cockroachdb/pebble/range_keys.go b/vendor/github.com/cockroachdb/pebble/range_keys.go index 096cb28..30549d5 100644 --- a/vendor/github.com/cockroachdb/pebble/range_keys.go +++ b/vendor/github.com/cockroachdb/pebble/range_keys.go @@ -17,7 +17,7 @@ import ( func (i *Iterator) constructRangeKeyIter() { i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init( &i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound, - &i.hasPrefix, &i.prefixOrFullSeekKey) + &i.hasPrefix, &i.prefixOrFullSeekKey, true /* onlySets */, &i.rangeKey.rangeKeyBuffers.internal) // If there's an indexed batch with range keys, include it. if i.batch != nil { @@ -80,7 +80,7 @@ func (i *Iterator) constructRangeKeyIter() { li := i.rangeKey.iterConfig.NewLevelIter() spanIterOpts := keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters} li.Init(spanIterOpts, i.cmp, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(), - manifest.Level(level), i.opts.logger, manifest.KeyTypeRange) + manifest.Level(level), manifest.KeyTypeRange) i.rangeKey.iterConfig.AddLevel(li) } } @@ -266,24 +266,24 @@ func (m *rangeKeyMasking) SpanChanged(s *keyspan.Span) { // _t_, and there exists a span with suffix _r_ covering a point key with suffix // _p_, and // -// _t_ ≤ _r_ < _p_ +// _t_ ≤ _r_ < _p_ // // then the point key is elided. Consider the following rendering, where using // integer suffixes with higher integers sort before suffixes with lower // integers, (for example @7 ≤ @6 < @5): // -// ^ -// @9 | •―――――――――――――――○ [e,m)@9 -// s 8 | • l@8 -// u 7 |------------------------------------ @7 RangeKeyMasking.Suffix -// f 6 | [h,q)@6 •―――――――――――――――――○ (threshold) -// f 5 | • h@5 -// f 4 | • n@4 -// i 3 | •―――――――――――○ [f,l)@3 -// x 2 | • b@2 -// 1 | -// 0 |___________________________________ -// a b c d e f g h i j k l m n o p q +// ^ +// @9 | •―――――――――――――――○ [e,m)@9 +// s 8 | • l@8 +// u 7 |------------------------------------ @7 RangeKeyMasking.Suffix +// f 6 | [h,q)@6 •―――――――――――――――――○ (threshold) +// f 5 | • h@5 +// f 4 | • n@4 +// i 3 | •―――――――――――○ [f,l)@3 +// x 2 | • b@2 +// 1 | +// 0 |___________________________________ +// a b c d e f g h i j k l m n o p q // // An iterator scanning the entire keyspace with the masking threshold set to @7 // will observe point keys b@2 and l@8. The span keys [h,q)@6 and [f,l)@3 serve @@ -297,6 +297,7 @@ func (m *rangeKeyMasking) SpanChanged(s *keyspan.Span) { // Invariant: The userKey is within the user key bounds of the span most // recently provided to `SpanChanged`. func (m *rangeKeyMasking) SkipPoint(userKey []byte) bool { + m.parent.stats.RangeKeyStats.ContainedPoints++ if m.maskSpan == nil { // No range key is currently acting as a mask, so don't skip. return false @@ -309,7 +310,11 @@ func (m *rangeKeyMasking) SkipPoint(userKey []byte) bool { // the InterleavingIter). Skip the point key if the range key's suffix is // greater than the point key's suffix. pointSuffix := userKey[m.split(userKey):] - return len(pointSuffix) > 0 && m.cmp(m.maskActiveSuffix, pointSuffix) < 0 + if len(pointSuffix) > 0 && m.cmp(m.maskActiveSuffix, pointSuffix) < 0 { + m.parent.stats.RangeKeyStats.SkippedPoints++ + return true + } + return false } // The iteratorRangeKeyState type implements the sstable package's @@ -322,9 +327,9 @@ func (m *rangeKeyMasking) SkipPoint(userKey []byte) bool { // Consider the range key [a,m)@10, and an iterator positioned just before the // below block, bounded by index separators `c` and `z`: // -// c z -// x | c@9 c@5 c@1 d@7 e@4 y@4 | ... -// iter pos +// c z +// x | c@9 c@5 c@1 d@7 e@4 y@4 | ... +// iter pos // // The next block cannot be skipped, despite the range key suffix @10 is greater // than all the block's keys' suffixes, because it contains a key (y@4) outside @@ -496,24 +501,6 @@ func (i *lazyCombinedIter) initCombinedIteration( } } - if i.parent.hasPrefix { - si := i.parent.comparer.Split(seekKey) - if i.parent.cmp(seekKey[:si], i.parent.prefixOrFullSeekKey) > 0 { - // The earliest possible range key has a start key with a prefix - // greater than the current iteration prefix. There's no need to - // switch to combined iteration, because there are not any range - // keys within the bounds of the prefix. Additionally, using a seek - // key that is outside the scope of the prefix can violate - // invariants within the range key iterator stack. Optimizations - // that exit early due to exhausting the prefix may result in - // `seekKey` being larger than the next range key's start key. - // - // See the testdata/rangekeys test case associated with #1893. - i.combinedIterState = combinedIterState{initialized: false} - return pointKey, pointValue - } - } - // An operation on the point iterator observed a file containing range keys, // so we must switch to combined interleaving iteration. First, construct // the range key iterator stack. It must not exist, otherwise we'd already @@ -628,6 +615,17 @@ func (i *lazyCombinedIter) Next() (*InternalKey, base.LazyValue) { return k, v } +func (i *lazyCombinedIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + if i.combinedIterState.initialized { + return i.parent.rangeKey.iiter.NextPrefix(succKey) + } + k, v := i.pointIter.NextPrefix(succKey) + if i.combinedIterState.triggered { + return i.initCombinedIteration(+1, k, v, nil) + } + return k, v +} + func (i *lazyCombinedIter) Prev() (*InternalKey, base.LazyValue) { if i.combinedIterState.initialized { return i.parent.rangeKey.iiter.Prev() diff --git a/vendor/github.com/cockroachdb/pebble/read_state.go b/vendor/github.com/cockroachdb/pebble/read_state.go index 67d5a7e..a389817 100644 --- a/vendor/github.com/cockroachdb/pebble/read_state.go +++ b/vendor/github.com/cockroachdb/pebble/read_state.go @@ -40,7 +40,7 @@ func (s *readState) unref() { } s.current.Unref() for _, mem := range s.memtables { - mem.readerUnref() + mem.readerUnref(true) } // The last reference to the readState was released. Check to see if there @@ -58,7 +58,7 @@ func (s *readState) unrefLocked() { } s.current.UnrefLocked() for _, mem := range s.memtables { - mem.readerUnref() + mem.readerUnrefLocked(true) } // NB: Unlike readState.unref(), we don't attempt to cleanup newly obsolete @@ -78,8 +78,7 @@ func (d *DB) loadReadState() *readState { // updateReadStateLocked creates a new readState from the current version and // list of memtables. Requires DB.mu is held. If checker is not nil, it is -// called after installing the new readState. If atomicFunc is not nil, it is -// executed atomically with the transition to the new read state. +// called after installing the new readState. func (d *DB) updateReadStateLocked(checker func(*DB) error) { s := &readState{ db: d, diff --git a/vendor/github.com/cockroachdb/pebble/record/log_writer.go b/vendor/github.com/cockroachdb/pebble/record/log_writer.go index 6098d19..9ef1655 100644 --- a/vendor/github.com/cockroachdb/pebble/record/log_writer.go +++ b/vendor/github.com/cockroachdb/pebble/record/log_writer.go @@ -133,7 +133,8 @@ func (q *syncQueue) load() (head, tail, realLength uint32) { return head, tail, realLength } -func (q *syncQueue) pop(head, tail uint32, err error) error { +// REQUIRES: queueSemChan is non-nil. +func (q *syncQueue) pop(head, tail uint32, err error, queueSemChan chan struct{}) error { if tail == head { // Queue is empty. return nil @@ -153,6 +154,10 @@ func (q *syncQueue) pop(head, tail uint32, err error) error { // will try to enqueue before we've "freed" space in the queue. atomic.AddUint64(&q.headTail, 1) wg.Done() + // Is always non-nil in production. + if queueSemChan != nil { + <-queueSemChan + } } return nil @@ -291,12 +296,20 @@ type LogWriter struct { // used for min-sync-interval. In normal operation this points to // time.AfterFunc. afterFunc func(d time.Duration, f func()) syncTimer + + // See the comment for LogWriterConfig.QueueSemChan. + queueSemChan chan struct{} } // LogWriterConfig is a struct used for configuring new LogWriters type LogWriterConfig struct { WALMinSyncInterval durationFunc WALFsyncLatency prometheus.Histogram + // QueueSemChan is an optional channel to pop from when popping from + // LogWriter.flusher.syncQueue. It functions as a semaphore that prevents + // the syncQueue from overflowing (which will cause a panic). All production + // code ensures this is non-nil. + QueueSemChan chan struct{} } // CapAllocatedBlocks is the maximum number of blocks allocated by the @@ -319,6 +332,7 @@ func NewLogWriter(w io.Writer, logNum base.FileNum, logWriterConfig LogWriterCon afterFunc: func(d time.Duration, f func()) syncTimer { return time.AfterFunc(d, f) }, + queueSemChan: logWriterConfig.QueueSemChan, } r.free.cond.L = &r.free.Mutex r.free.blocks = make([]*block, 0, CapAllocatedBlocks) @@ -441,7 +455,7 @@ func (w *LogWriter) flushLoop(context.Context) { // If flusher has an error, we propagate it to waiters. Note in spite of // error we consume the pending list above to free blocks for writers. if f.err != nil { - f.syncQ.pop(head, tail, f.err) + f.syncQ.pop(head, tail, f.err, w.queueSemChan) // Update the idleStartTime if work could not be done, so that we don't // include the duration we tried to do work as idle. We don't bother // with the rest of the accounting, which means we will undercount. @@ -518,7 +532,7 @@ func (w *LogWriter) flushPending( syncLatency, err = w.syncWithLatency() } f := &w.flusher - if popErr := f.syncQ.pop(head, tail, err); popErr != nil { + if popErr := f.syncQ.pop(head, tail, err, w.queueSemChan); popErr != nil { return synced, syncLatency, bytesWritten, popErr } } @@ -548,7 +562,7 @@ func (w *LogWriter) flushBlock(b *block) error { // queueBlock queues the current block for writing to the underlying writer, // allocates a new block and reserves space for the next header. -func (w *LogWriter) queueBlock() { +func (w *LogWriter) queueBlock() (waitDuration time.Duration) { // Allocate a new block, blocking until one is available. We do this first // because w.block is protected by w.flusher.Mutex. w.free.Lock() @@ -557,9 +571,11 @@ func (w *LogWriter) queueBlock() { w.free.allocated++ w.free.blocks = append(w.free.blocks, &block{}) } else { + now := time.Now() for len(w.free.blocks) == 0 { w.free.cond.Wait() } + waitDuration = time.Since(now) } } nextBlock := w.free.blocks[len(w.free.blocks)-1] @@ -575,6 +591,28 @@ func (w *LogWriter) queueBlock() { f.Unlock() w.blockNum++ + return waitDuration +} + +// ReserveAllFreeBlocksForTesting is used to only for testing. +func (w *LogWriter) ReserveAllFreeBlocksForTesting() (releaseFunc func()) { + w.free.Lock() + defer w.free.Unlock() + free := w.free.blocks + w.free.blocks = nil + return func() { + w.free.Lock() + defer w.free.Unlock() + // It is possible that someone has pushed a free block and w.free.blocks + // is no longer nil. That is harmless. Also, the waiter loops on the + // condition len(w.free.blocks) == 0, so to actually unblock it we need to + // give it a free block. + if len(free) == 0 { + free = append(free, &block{}) + } + w.free.blocks = free + w.free.cond.Broadcast() + } } // Close flushes and syncs any unwritten data and closes the writer. @@ -626,7 +664,8 @@ func (w *LogWriter) Close() error { // of the record. // External synchronisation provided by commitPipeline.mu. func (w *LogWriter) WriteRecord(p []byte) (int64, error) { - return w.SyncRecord(p, nil, nil) + logSize, _, err := w.SyncRecord(p, nil, nil) + return logSize, err } // SyncRecord writes a complete record. If wg!= nil the record will be @@ -634,9 +673,11 @@ func (w *LogWriter) WriteRecord(p []byte) (int64, error) { // the wait group upon completion. Returns the offset just past the end of the // record. // External synchronisation provided by commitPipeline.mu. -func (w *LogWriter) SyncRecord(p []byte, wg *sync.WaitGroup, err *error) (int64, error) { +func (w *LogWriter) SyncRecord( + p []byte, wg *sync.WaitGroup, err *error, +) (logSize int64, waitDuration time.Duration, err2 error) { if w.err != nil { - return -1, w.err + return -1, 0, w.err } // The `i == 0` condition ensures we handle empty records. Such records can @@ -644,7 +685,9 @@ func (w *LogWriter) SyncRecord(p []byte, wg *sync.WaitGroup, err *error) (int64, // MANIFEST is currently written using Writer, it is good to support the same // semantics with LogWriter. for i := 0; i == 0 || len(p) > 0; i++ { - p = w.emitFragment(i, p) + var wd time.Duration + p, wd = w.emitFragment(i, p) + waitDuration += wd } if wg != nil { @@ -663,7 +706,7 @@ func (w *LogWriter) SyncRecord(p []byte, wg *sync.WaitGroup, err *error) (int64, // race with our read. That's ok because the only error we could be seeing is // one to syncing for which the caller can receive notification of by passing // in a non-nil err argument. - return offset, nil + return offset, waitDuration, nil } // Size returns the current size of the file. @@ -684,7 +727,7 @@ func (w *LogWriter) emitEOFTrailer() { atomic.StoreInt32(&b.written, i+int32(recyclableHeaderSize)) } -func (w *LogWriter) emitFragment(n int, p []byte) []byte { +func (w *LogWriter) emitFragment(n int, p []byte) (remainingP []byte, waitDuration time.Duration) { b := w.block i := b.written first := n == 0 @@ -718,9 +761,9 @@ func (w *LogWriter) emitFragment(n int, p []byte) []byte { for i := b.written; i < blockSize; i++ { b.buf[i] = 0 } - w.queueBlock() + waitDuration = w.queueBlock() } - return p[r:] + return p[r:], waitDuration } // Metrics must be called after Close. The callee will no longer modify the diff --git a/vendor/github.com/cockroachdb/pebble/record/record.go b/vendor/github.com/cockroachdb/pebble/record/record.go index f1583b3..9b42a4c 100644 --- a/vendor/github.com/cockroachdb/pebble/record/record.go +++ b/vendor/github.com/cockroachdb/pebble/record/record.go @@ -19,6 +19,7 @@ // Neither Readers or Writers are safe to use concurrently. // // Example code: +// // func read(r io.Reader) ([]string, error) { // var ss []string // records := record.NewReader(r) @@ -65,9 +66,9 @@ // A record maps to one or more chunks. There are two chunk formats: legacy and // recyclable. The legacy chunk format: // -// +----------+-----------+-----------+--- ... ---+ -// | CRC (4B) | Size (2B) | Type (1B) | Payload | -// +----------+-----------+-----------+--- ... ---+ +// +----------+-----------+-----------+--- ... ---+ +// | CRC (4B) | Size (2B) | Type (1B) | Payload | +// +----------+-----------+-----------+--- ... ---+ // // CRC is computed over the type and payload // Size is the length of the payload in bytes @@ -84,9 +85,9 @@ // metadata. Additionally, recycling log files is a prequisite for using direct // IO with log writing. The recyclyable format is: // -// +----------+-----------+-----------+----------------+--- ... ---+ -// | CRC (4B) | Size (2B) | Type (1B) | Log number (4B)| Payload | -// +----------+-----------+-----------+----------------+--- ... ---+ +// +----------+-----------+-----------+----------------+--- ... ---+ +// | CRC (4B) | Size (2B) | Type (1B) | Log number (4B)| Payload | +// +----------+-----------+-----------+----------------+--- ... ---+ // // Recyclable chunks are distinguished from legacy chunks by the addition of 4 // extra "recyclable" chunk types that map directly to the legacy chunk types @@ -247,6 +248,7 @@ func (r *Reader) nextChunk(wantFirst bool) error { r.begin = r.end + headerSize r.end = r.begin + int(length) if r.end > r.n { + // The chunk straddles a 32KB boundary (or the end of file). if r.recovering { r.recover() continue diff --git a/vendor/github.com/cockroachdb/pebble/record/rotation.go b/vendor/github.com/cockroachdb/pebble/record/rotation.go new file mode 100644 index 0000000..a2dd322 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/record/rotation.go @@ -0,0 +1,82 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package record + +// RotationHelper is a type used to inform the decision of rotating a record log +// file. +// +// The assumption is that multiple records can be coalesced into a single record +// (called a snapshot). Starting a new file, where the first record is a +// snapshot of the current state is referred to as "rotating" the log. +// +// Normally we rotate files when a certain file size is reached. But in certain +// cases (e.g. contents become very large), this can result in too frequent +// rotation. This helper contains logic to impose extra conditions on the +// rotation. +// +// The rotation helper uses "size" as a unit-less estimation that is correlated +// with the on-disk size of a record or snapshot. +type RotationHelper struct { + // lastSnapshotSize is the size of the last snapshot. + lastSnapshotSize int64 + // sizeSinceLastSnapshot is the sum of sizes of records applied since the last + // snapshot. + sizeSinceLastSnapshot int64 + lastRecordSize int64 +} + +// AddRecord makes the rotation helper aware of a new record. +func (rh *RotationHelper) AddRecord(recordSize int64) { + rh.sizeSinceLastSnapshot += recordSize + rh.lastRecordSize = recordSize +} + +// ShouldRotate returns whether we should start a new log file (with a snapshot). +// Does not need to be called if other rotation factors (log file size) are not +// satisfied. +func (rh *RotationHelper) ShouldRotate(nextSnapshotSize int64) bool { + // The primary goal is to ensure that when reopening a log file, the number of + // edits that need to be replayed on top of the snapshot is "sane" while + // keeping the rotation frequency as low as possible. + // + // For the purposes of this description, we assume that the log is mainly + // storing a collection of "entries", with edits adding or removing entries. + // Consider the following cases: + // + // - The number of live entries is roughly stable: after writing the snapshot + // (with S entries), we require that there be enough edits such that the + // cumulative number of entries in those edits, E, be greater than S. This + // will ensure that at most 50% of data written out is due to rotation. + // + // - The number of live entries K in the DB is shrinking drastically, say from + // S to S/10: After this shrinking, E = 0.9S, and so if we used the previous + // snapshot entry count, S, as the threshold that needs to be exceeded, we + // will further delay the snapshot writing. Which means on reopen we will + // need to replay 0.9S edits to get to a version with 0.1S entries. It would + // be better to create a new snapshot when E exceeds the number of entries in + // the current version. + // + // - The number of live entries L in the DB is growing; say the last snapshot + // had S entries, and now we have 10S entries, so E = 9S. If we required + // that E is at least the current number of entries, we would further delay + // writing a new snapshot (which is not desirable). + // + // The logic below uses the min of the last snapshot size count and the size + // count in the current version. + return rh.sizeSinceLastSnapshot > rh.lastSnapshotSize || rh.sizeSinceLastSnapshot > nextSnapshotSize +} + +// Rotate makes the rotation helper aware that we are rotating to a new snapshot +// (to which we will apply the latest edit). +func (rh *RotationHelper) Rotate(snapshotSize int64) { + rh.lastSnapshotSize = snapshotSize + rh.sizeSinceLastSnapshot = rh.lastRecordSize +} + +// DebugInfo returns the last snapshot size and size of the edits since the last +// snapshot; used for testing and debugging. +func (rh *RotationHelper) DebugInfo() (lastSnapshotSize int64, sizeSinceLastSnapshot int64) { + return rh.lastSnapshotSize, rh.sizeSinceLastSnapshot +} diff --git a/vendor/github.com/cockroachdb/pebble/scan_internal.go b/vendor/github.com/cockroachdb/pebble/scan_internal.go new file mode 100644 index 0000000..4984b0f --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/scan_internal.go @@ -0,0 +1,925 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package pebble + +import ( + "context" + "fmt" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/keyspan" + "github.com/cockroachdb/pebble/internal/manifest" + "github.com/cockroachdb/pebble/objstorage" +) + +const ( + // In skip-shared iteration mode, keys in levels sharedLevelsStart and greater + // (i.e. lower in the LSM) are skipped. + sharedLevelsStart = 5 +) + +// ErrInvalidSkipSharedIteration is returned by ScanInternal if it was called +// with a shared file visitor function, and a file in a shareable level (i.e. +// level >= sharedLevelsStart) was found to not be in shared storage according +// to objstorage.Provider. +var ErrInvalidSkipSharedIteration = errors.New("pebble: cannot use skip-shared iteration due to non-shared files in lower levels") + +// SharedSSTMeta represents an sstable on shared storage that can be ingested +// by another pebble instance. This struct must contain all fields that are +// required for a Pebble instance to ingest a foreign sstable on shared storage, +// including constructing any relevant objstorage.Provider / sharedobjcat.Catalog +// data structures, as well as creating virtual FileMetadatas. +// +// Note that the Pebble instance creating and returning a SharedSSTMeta might +// not be the one that created the underlying sstable on shared storage to begin +// with; it's possible for a Pebble instance to reshare an sstable that was +// shared to it. +type SharedSSTMeta struct { + // Backing is the shared object underlying this SST. Can be attached to an + // objstorage.Provider. + Backing objstorage.SharedObjectBackingHandle + + // Smallest and Largest internal keys for the overall bounds. The kind and + // SeqNum of these will reflect what is physically present on the source Pebble + // instance's view of the sstable; it's up to the ingesting instance to set the + // sequence number in the trailer to match the read-time sequence numbers + // reserved for the level this SST is being ingested into. The Kind is expected + // to remain unchanged by the ingesting instance. + // + // Note that these bounds could be narrower than the bounds of the underlying + // sstable; ScanInternal is expected to truncate sstable bounds to the user key + // bounds passed into that method. + Smallest, Largest InternalKey + + // SmallestRangeKey and LargestRangeKey are internal keys that denote the + // range key bounds of this sstable. Must lie within [Smallest, Largest]. + SmallestRangeKey, LargestRangeKey InternalKey + + // SmallestPointKey and LargestPointKey are internal keys that denote the + // point key bounds of this sstable. Must lie within [Smallest, Largest]. + SmallestPointKey, LargestPointKey InternalKey + + // Level denotes the level at which this file was present at read time. + // For files visited by ScanInternal, this value will only be 5 or 6. + Level uint8 + + // Size contains an estimate of the size of this sstable. + Size uint64 + + // fileNum at time of creation in the creator instance. Only used for + // debugging/tests. + fileNum base.FileNum +} + +func (s *SharedSSTMeta) cloneFromFileMeta(f *fileMetadata) { + *s = SharedSSTMeta{ + Smallest: f.Smallest.Clone(), + Largest: f.Largest.Clone(), + SmallestRangeKey: f.SmallestRangeKey.Clone(), + LargestRangeKey: f.LargestRangeKey.Clone(), + SmallestPointKey: f.SmallestPointKey.Clone(), + LargestPointKey: f.LargestPointKey.Clone(), + Size: f.Size, + fileNum: f.FileNum, + } +} + +type pcIterPos int + +const ( + pcIterPosCur pcIterPos = iota + pcIterPosNext +) + +// pointCollapsingIterator is an internalIterator that collapses point keys and +// returns at most one point internal key for each user key. Merges are merged, +// sets are emitted as-is, and SingleDeletes are collapsed with the next point +// key, however we don't guarantee that we generate and return a SetWithDel if +// a set shadows a del. Point keys deleted by rangedels are also considered shadowed and not +// exposed. +// +// TODO(bilal): Implement the unimplemented internalIterator methods below. +// Currently this iterator only supports the forward iteration case necessary +// for scanInternal, however foreign sstable iterators will also need to use this +// or a simplified version of this. +type pointCollapsingIterator struct { + iter keyspan.InterleavingIter + pos pcIterPos + comparer *base.Comparer + merge base.Merge + err error + seqNum uint64 + // The current position of `iter`. Could be backed by savedKey (i.e. iterKey == + // &savedKey) or could be owned by `iter`. findNextEntry and similar methods + // are expected to save the current value of this to savedKey if they're + // iterating away from the current key but still need to retain it. See + // comments in findNextEntry on how this field is used. + // + // At the end of a positioning call: + // - if pos == pcIterPosNext, iterKey is pointing to the next user key owned + // by `iter` while savedKey is holding a copy to our current key. + // - If pos == pcIterPosCur, iterKey is pointing to either a savedKey-backed + // copy of the current key, or an `iter`-owned current key in which case + // the value of savedKey is undefined. + iterKey *InternalKey + savedKey InternalKey + // Value at the current iterator position, at iterKey. + value base.LazyValue + // Used for Merge keys only. + valueMerger ValueMerger + valueBuf []byte +} + +// SeekPrefixGE implements the InternalIterator interface. +func (p *pointCollapsingIterator) SeekPrefixGE( + prefix, key []byte, flags base.SeekGEFlags, +) (*base.InternalKey, base.LazyValue) { + panic("unimplemented") +} + +// SeekGE implements the InternalIterator interface. +func (p *pointCollapsingIterator) SeekGE( + key []byte, flags base.SeekGEFlags, +) (*base.InternalKey, base.LazyValue) { + p.resetKey() + p.iterKey, p.value = p.iter.SeekGE(key, flags) + p.pos = pcIterPosCur + if p.iterKey == nil { + return nil, base.LazyValue{} + } + return p.findNextEntry() +} + +// SeekLT implements the InternalIterator interface. +func (p *pointCollapsingIterator) SeekLT( + key []byte, flags base.SeekLTFlags, +) (*base.InternalKey, base.LazyValue) { + panic("unimplemented") +} + +func (p *pointCollapsingIterator) resetKey() { + p.savedKey.UserKey = p.savedKey.UserKey[:0] + p.savedKey.Trailer = 0 + p.valueMerger = nil + p.valueBuf = p.valueBuf[:0] + p.iterKey = nil + p.pos = pcIterPosCur +} + +// findNextEntry is called to return the next key. p.iter must be positioned at +// the start of the first user key we are interested in. +func (p *pointCollapsingIterator) findNextEntry() (*base.InternalKey, base.LazyValue) { + finishAndReturnMerge := func() (*base.InternalKey, base.LazyValue) { + value, closer, err := p.valueMerger.Finish(true /* includesBase */) + if err != nil { + p.err = err + return nil, base.LazyValue{} + } + p.valueBuf = append(p.valueBuf[:0], value...) + if closer != nil { + _ = closer.Close() + } + p.valueMerger = nil + newValue := base.MakeInPlaceValue(value) + return &p.savedKey, newValue + } + + // saveKey sets p.iterKey (if not-nil) to &p.savedKey. We can use this equality + // as a proxy to determine if we're at the first internal key for a user key. + p.saveKey() + for p.iterKey != nil { + // NB: p.savedKey is either the current key (iff p.iterKey == &p.savedKey), + // or the previous key. + if p.iterKey != &p.savedKey && !p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) { + if p.valueMerger != nil { + if p.savedKey.Kind() != InternalKeyKindMerge { + panic(fmt.Sprintf("expected key %s to have MERGE kind", p.iterKey)) + } + p.pos = pcIterPosNext + return finishAndReturnMerge() + } + p.saveKey() + continue + } + if s := p.iter.Span(); s != nil && s.CoversAt(p.seqNum, p.iterKey.SeqNum()) { + // All future keys for this user key must be deleted. + if p.valueMerger != nil { + return finishAndReturnMerge() + } else if p.savedKey.Kind() == InternalKeyKindSingleDelete { + panic("cannot process singledel key in point collapsing iterator") + } + // Fast forward to the next user key. + p.saveKey() + p.iterKey, p.value = p.iter.Next() + for p.iterKey != nil && p.savedKey.SeqNum() >= p.iterKey.SeqNum() && p.comparer.Equal(p.iterKey.UserKey, p.savedKey.UserKey) { + p.iterKey, p.value = p.iter.Next() + } + continue + } + switch p.savedKey.Kind() { + case InternalKeyKindSet, InternalKeyKindDelete, InternalKeyKindSetWithDelete: + p.saveKey() + // Note that we return SETs directly, even if they would otherwise get + // compacted into a Del to turn into a SetWithDelete. This is a fast + // path optimization that can break SINGLEDEL determinism. To lead to + // consistent SINGLEDEL behaviour, this iterator should *not* be used for + // a keyspace where SINGLEDELs could be in use. If this iterator observes + // a SINGLEDEL as the first internal key for a user key, it will panic. + // + // As p.value is a lazy value owned by the child iterator, we can thread + // it through without loading it into p.valueBuf. + // + // TODO(bilal): We can even avoid saving the key in this fast path if + // we are in a block where setHasSamePrefix = false in a v3 sstable, + // guaranteeing that there's only one internal key for each user key. + // Thread this logic through the sstable iterators and/or consider + // collapsing (ha) this logic into the sstable iterators that are aware + // of blocks and can determine user key changes without doing key saves + // or comparisons. + p.pos = pcIterPosCur + return p.iterKey, p.value + case InternalKeyKindSingleDelete: + // Panic, as this iterator is not expected to observe single deletes. + panic("cannot process singledel key in point collapsing iterator") + case InternalKeyKindMerge: + if p.valueMerger == nil { + // Set up merger. This is the first Merge key encountered. + value, callerOwned, err := p.value.Value(p.valueBuf[:0]) + if err != nil { + p.err = err + return nil, base.LazyValue{} + } + if !callerOwned { + p.valueBuf = append(p.valueBuf[:0], value...) + } else { + p.valueBuf = value + } + p.valueMerger, err = p.merge(p.iterKey.UserKey, p.valueBuf) + if err != nil { + p.err = err + return nil, base.LazyValue{} + } + p.saveKey() + p.iterKey, p.value = p.iter.Next() + continue + } + switch p.iterKey.Kind() { + case InternalKeyKindSet, InternalKeyKindMerge, InternalKeyKindSetWithDelete: + // Merge into key. + value, callerOwned, err := p.value.Value(p.valueBuf[:0]) + if err != nil { + p.err = err + return nil, base.LazyValue{} + } + if !callerOwned { + p.valueBuf = append(p.valueBuf[:0], value...) + } else { + p.valueBuf = value + } + if err := p.valueMerger.MergeOlder(value); err != nil { + p.err = err + return nil, base.LazyValue{} + } + } + if p.iterKey.Kind() != InternalKeyKindMerge { + p.savedKey.SetKind(p.iterKey.Kind()) + p.pos = pcIterPosCur + return finishAndReturnMerge() + } + p.iterKey, p.value = p.iter.Next() + case InternalKeyKindRangeDelete: + // These are interleaved by the interleaving iterator ahead of all points. + // We should pass them as-is, but also account for any points ahead of + // them. + p.pos = pcIterPosCur + return p.iterKey, p.value + default: + panic(fmt.Sprintf("unexpected kind: %d", p.iterKey.Kind())) + } + } + if p.valueMerger != nil { + p.pos = pcIterPosNext + return finishAndReturnMerge() + } + p.resetKey() + return nil, base.LazyValue{} +} + +// First implements the InternalIterator interface. +func (p *pointCollapsingIterator) First() (*base.InternalKey, base.LazyValue) { + p.resetKey() + p.iterKey, p.value = p.iter.First() + p.pos = pcIterPosCur + if p.iterKey == nil { + return nil, base.LazyValue{} + } + return p.findNextEntry() +} + +// Last implements the InternalIterator interface. +func (p *pointCollapsingIterator) Last() (*base.InternalKey, base.LazyValue) { + panic("unimplemented") +} + +func (p *pointCollapsingIterator) saveKey() { + if p.iterKey == nil { + p.savedKey = InternalKey{UserKey: p.savedKey.UserKey[:0]} + return + } else if p.iterKey == &p.savedKey { + // Do nothing. + return + } + p.savedKey.CopyFrom(*p.iterKey) + p.iterKey = &p.savedKey +} + +// Next implements the InternalIterator interface. +func (p *pointCollapsingIterator) Next() (*base.InternalKey, base.LazyValue) { + switch p.pos { + case pcIterPosCur: + p.saveKey() + if p.iterKey != nil && p.iterKey.Kind() == InternalKeyKindRangeDelete { + // Step over the interleaved range delete and process the very next + // internal key, even if it's at the same user key. This is because a + // point for that user key has not been returned yet. + p.iterKey, p.value = p.iter.Next() + break + } + // Fast forward to the next user key. p.iterKey stayed at p.savedKey for + // this loop. + key, val := p.iter.Next() + // p.iterKey.SeqNum() >= key.SeqNum() is an optimization that allows us to + // use p.iterKey.SeqNum() < key.SeqNum() as a sign that the user key has + // changed, without needing to do the full key comparison. + for p.iterKey != nil && key != nil && p.iterKey.SeqNum() >= key.SeqNum() && + p.comparer.Equal(p.iterKey.UserKey, key.UserKey) { + key, val = p.iter.Next() + } + if key == nil { + // There are no keys to return. + p.resetKey() + return nil, base.LazyValue{} + } + p.iterKey, p.value = key, val + case pcIterPosNext: + p.pos = pcIterPosCur + } + if p.iterKey == nil { + p.resetKey() + return nil, base.LazyValue{} + } + return p.findNextEntry() +} + +// NextPrefix implements the InternalIterator interface. +func (p *pointCollapsingIterator) NextPrefix(succKey []byte) (*base.InternalKey, base.LazyValue) { + panic("unimplemented") +} + +// Prev implements the InternalIterator interface. +func (p *pointCollapsingIterator) Prev() (*base.InternalKey, base.LazyValue) { + panic("unimplemented") +} + +// Error implements the InternalIterator interface. +func (p *pointCollapsingIterator) Error() error { + if p.err != nil { + return p.err + } + return p.iter.Error() +} + +// Close implements the InternalIterator interface. +func (p *pointCollapsingIterator) Close() error { + return p.iter.Close() +} + +// SetBounds implements the InternalIterator interface. +func (p *pointCollapsingIterator) SetBounds(lower, upper []byte) { + p.resetKey() + p.iter.SetBounds(lower, upper) +} + +// String implements the InternalIterator interface. +func (p *pointCollapsingIterator) String() string { + return p.iter.String() +} + +var _ internalIterator = &pointCollapsingIterator{} + +// scanInternalIterator is an iterator that returns all internal keys, including +// tombstones. For instance, an InternalKeyKindDelete would be returned as an +// InternalKeyKindDelete instead of the iterator skipping over to the next key. +// Internal keys within a user key are collapsed, eg. if there are two SETs, the +// one with the higher sequence is returned. Useful if an external user of Pebble +// needs to observe and rebuild Pebble's history of internal keys, such as in +// node-to-node replication. For use with {db,snapshot}.ScanInternal(). +// +// scanInternalIterator is expected to ignore point keys deleted by range +// deletions, and range keys shadowed by a range key unset or delete, however it +// *must* return the range delete as well as the range key unset/delete that did +// the shadowing. +type scanInternalIterator struct { + opts scanInternalOptions + comparer *base.Comparer + merge Merge + iter internalIterator + readState *readState + rangeKey *iteratorRangeKeyState + pointKeyIter pointCollapsingIterator + iterKey *InternalKey + iterValue LazyValue + alloc *iterAlloc + newIters tableNewIters + newIterRangeKey keyspan.TableNewSpanIter + seqNum uint64 + + // boundsBuf holds two buffers used to store the lower and upper bounds. + // Whenever the InternalIterator's bounds change, the new bounds are copied + // into boundsBuf[boundsBufIdx]. The two bounds share a slice to reduce + // allocations. opts.LowerBound and opts.UpperBound point into this slice. + boundsBuf [2][]byte + boundsBufIdx int +} + +// truncateSharedFile truncates a shared file's [Smallest, Largest] fields to +// [lower, upper), potentially opening iterators on the file to find keys within +// the requested bounds. A SharedSSTMeta is produced that is suitable for +// external consumption by other Pebble instances. If shouldSkip is true, this +// file does not contain any keys in [lower, upper) and can be skipped. +// +// TODO(bilal): If opening iterators and doing reads in this method is too +// inefficient, consider producing non-tight file bounds instead. +func (d *DB) truncateSharedFile( + ctx context.Context, + lower, upper []byte, + level int, + file *fileMetadata, + objMeta objstorage.ObjectMetadata, +) (sst *SharedSSTMeta, shouldSkip bool, err error) { + cmp := d.cmp + sst = &SharedSSTMeta{} + sst.cloneFromFileMeta(file) + sst.Level = uint8(level) + sst.Backing, err = d.objProvider.SharedObjectBacking(&objMeta) + if err != nil { + return nil, false, err + } + needsLowerTruncate := cmp(lower, file.Smallest.UserKey) > 0 + needsUpperTruncate := cmp(upper, file.Largest.UserKey) < 0 || (cmp(upper, file.Largest.UserKey) == 0 && !file.Largest.IsExclusiveSentinel()) + // Fast path: file is entirely within [lower, upper). + if !needsLowerTruncate && !needsUpperTruncate { + return sst, false, nil + } + + // We will need to truncate file bounds in at least one direction. Open all + // relevant iterators. + // + // TODO(bilal): Once virtual sstables go in, verify that the constraining of + // bounds to virtual sstable bounds happens below this method, so we aren't + // unintentionally exposing keys we shouldn't be exposing. + iter, rangeDelIter, err := d.newIters(ctx, file, &IterOptions{ + LowerBound: lower, + UpperBound: upper, + level: manifest.Level(level), + }, internalIterOpts{}) + if err != nil { + return nil, false, err + } + defer iter.Close() + if rangeDelIter != nil { + rangeDelIter = keyspan.Truncate( + cmp, rangeDelIter, lower, upper, nil, nil, + false, /* panicOnPartialOverlap */ + ) + defer rangeDelIter.Close() + } + rangeKeyIter, err := d.tableNewRangeKeyIter(file, nil /* spanIterOptions */) + if err != nil { + return nil, false, err + } + if rangeKeyIter != nil { + rangeKeyIter = keyspan.Truncate( + cmp, rangeKeyIter, lower, upper, nil, nil, + false, /* panicOnPartialOverlap */ + ) + defer rangeKeyIter.Close() + } + // Check if we need to truncate on the left side. This means finding a new + // LargestPointKey and LargestRangeKey that is >= lower. + if needsLowerTruncate { + sst.SmallestPointKey.UserKey = sst.SmallestPointKey.UserKey[:0] + sst.SmallestPointKey.Trailer = 0 + key, _ := iter.SeekGE(lower, base.SeekGEFlagsNone) + if key != nil { + sst.SmallestPointKey.CopyFrom(*key) + } + if rangeDelIter != nil { + span := rangeDelIter.SeekGE(lower) + if span != nil && (len(sst.SmallestPointKey.UserKey) == 0 || base.InternalCompare(cmp, span.SmallestKey(), sst.SmallestPointKey) < 0) { + sst.SmallestPointKey.CopyFrom(span.SmallestKey()) + } + } + sst.SmallestRangeKey.UserKey = sst.SmallestRangeKey.UserKey[:0] + sst.SmallestRangeKey.Trailer = 0 + if rangeKeyIter != nil { + span := rangeKeyIter.SeekGE(lower) + if span != nil { + sst.SmallestRangeKey.CopyFrom(span.SmallestKey()) + } + } + } + // Check if we need to truncate on the right side. This means finding a new + // LargestPointKey and LargestRangeKey that is < upper. + if needsUpperTruncate { + sst.LargestPointKey.UserKey = sst.LargestPointKey.UserKey[:0] + sst.LargestPointKey.Trailer = 0 + key, _ := iter.SeekLT(upper, base.SeekLTFlagsNone) + if key != nil { + sst.LargestPointKey.CopyFrom(*key) + } + if rangeDelIter != nil { + span := rangeDelIter.SeekLT(upper) + if span != nil && (len(sst.LargestPointKey.UserKey) == 0 || base.InternalCompare(cmp, span.LargestKey(), sst.LargestPointKey) > 0) { + sst.LargestPointKey.CopyFrom(span.LargestKey()) + } + } + sst.LargestRangeKey.UserKey = sst.LargestRangeKey.UserKey[:0] + sst.LargestRangeKey.Trailer = 0 + if rangeKeyIter != nil { + span := rangeKeyIter.SeekLT(upper) + if span != nil { + sst.LargestRangeKey.CopyFrom(span.LargestKey()) + } + } + } + // Set overall bounds based on {Smallest,Largest}{Point,Range}Key. + switch { + case len(sst.SmallestRangeKey.UserKey) == 0: + sst.Smallest = sst.SmallestPointKey + case len(sst.SmallestPointKey.UserKey) == 0: + sst.Smallest = sst.SmallestRangeKey + default: + sst.Smallest = sst.SmallestPointKey + if base.InternalCompare(cmp, sst.SmallestRangeKey, sst.SmallestPointKey) < 0 { + sst.Smallest = sst.SmallestRangeKey + } + } + switch { + case len(sst.LargestRangeKey.UserKey) == 0: + sst.Largest = sst.LargestPointKey + case len(sst.LargestPointKey.UserKey) == 0: + sst.Largest = sst.LargestRangeKey + default: + sst.Largest = sst.LargestPointKey + if base.InternalCompare(cmp, sst.LargestRangeKey, sst.LargestPointKey) > 0 { + sst.Largest = sst.LargestRangeKey + } + } + // On rare occasion, a file might overlap with [lower, upper) but not actually + // have any keys within those bounds. Skip such files. + if len(sst.Smallest.UserKey) == 0 { + return nil, true, nil + } + return sst, false, nil +} + +func scanInternalImpl( + ctx context.Context, + lower, upper []byte, + iter *scanInternalIterator, + visitPointKey func(key *InternalKey, value LazyValue) error, + visitRangeDel func(start, end []byte, seqNum uint64) error, + visitRangeKey func(start, end []byte, keys []keyspan.Key) error, + visitSharedFile func(sst *SharedSSTMeta) error, +) error { + if visitSharedFile != nil && (lower == nil || upper == nil) { + panic("lower and upper bounds must be specified in skip-shared iteration mode") + } + // Before starting iteration, check if any files in levels sharedLevelsStart + // and below are *not* shared. Error out if that is the case, as skip-shared + // iteration will not produce a consistent point-in-time view of this range + // of keys. For files that are shared, call visitSharedFile with a truncated + // version of that file. + cmp := iter.comparer.Compare + db := iter.readState.db + provider := db.objProvider + if visitSharedFile != nil { + if provider == nil { + panic("expected non-nil Provider in skip-shared iteration mode") + } + for level := sharedLevelsStart; level < numLevels; level++ { + files := iter.readState.current.Levels[level].Iter() + for f := files.SeekGE(cmp, lower); f != nil && cmp(f.Smallest.UserKey, upper) < 0; f = files.Next() { + var objMeta objstorage.ObjectMetadata + var err error + objMeta, err = provider.Lookup(fileTypeTable, f.FileNum.DiskFileNum()) + if err != nil { + return err + } + if !objMeta.IsShared() { + return errors.Wrapf(ErrInvalidSkipSharedIteration, "when processing file %s", objMeta.DiskFileNum) + } + var sst *SharedSSTMeta + var skip bool + sst, skip, err = iter.readState.db.truncateSharedFile(ctx, lower, upper, level, f, objMeta) + if err != nil { + return err + } + if skip { + continue + } + if err = visitSharedFile(sst); err != nil { + return err + } + } + } + } + + for valid := iter.seekGE(lower); valid && iter.error() == nil; valid = iter.next() { + key := iter.unsafeKey() + + switch key.Kind() { + case InternalKeyKindRangeKeyDelete, InternalKeyKindRangeKeyUnset, InternalKeyKindRangeKeySet: + span := iter.unsafeSpan() + if err := visitRangeKey(span.Start, span.End, span.Keys); err != nil { + return err + } + case InternalKeyKindRangeDelete: + rangeDel := iter.unsafeRangeDel() + if err := visitRangeDel(rangeDel.Start, rangeDel.End, rangeDel.LargestSeqNum()); err != nil { + return err + } + default: + val := iter.lazyValue() + if err := visitPointKey(key, val); err != nil { + return err + } + } + } + + return nil +} + +// constructPointIter constructs a merging iterator and sets i.iter to it. +func (i *scanInternalIterator) constructPointIter(memtables flushableList, buf *iterAlloc) { + // Merging levels and levels from iterAlloc. + mlevels := buf.mlevels[:0] + levels := buf.levels[:0] + + // We compute the number of levels needed ahead of time and reallocate a slice if + // the array from the iterAlloc isn't large enough. Doing this allocation once + // should improve the performance. + numMergingLevels := len(memtables) + numLevelIters := 0 + + current := i.readState.current + numMergingLevels += len(current.L0SublevelFiles) + numLevelIters += len(current.L0SublevelFiles) + for level := 1; level < len(current.Levels); level++ { + if current.Levels[level].Empty() { + continue + } + if i.opts.skipSharedLevels && level >= sharedLevelsStart { + continue + } + numMergingLevels++ + numLevelIters++ + } + + if numMergingLevels > cap(mlevels) { + mlevels = make([]mergingIterLevel, 0, numMergingLevels) + } + if numLevelIters > cap(levels) { + levels = make([]levelIter, 0, numLevelIters) + } + // TODO(bilal): Push these into the iterAlloc buf. + var rangeDelMiter keyspan.MergingIter + rangeDelIters := make([]keyspan.FragmentIterator, 0, numMergingLevels) + rangeDelLevels := make([]keyspan.LevelIter, 0, numLevelIters) + + // Next are the memtables. + for j := len(memtables) - 1; j >= 0; j-- { + mem := memtables[j] + mlevels = append(mlevels, mergingIterLevel{ + iter: mem.newIter(&i.opts.IterOptions), + }) + if rdi := mem.newRangeDelIter(&i.opts.IterOptions); rdi != nil { + rangeDelIters = append(rangeDelIters, rdi) + } + } + + // Next are the file levels: L0 sub-levels followed by lower levels. + mlevelsIndex := len(mlevels) + levelsIndex := len(levels) + mlevels = mlevels[:numMergingLevels] + levels = levels[:numLevelIters] + rangeDelLevels = rangeDelLevels[:numLevelIters] + addLevelIterForFiles := func(files manifest.LevelIterator, level manifest.Level) { + li := &levels[levelsIndex] + rli := &rangeDelLevels[levelsIndex] + + li.init( + context.Background(), i.opts.IterOptions, i.comparer.Compare, i.comparer.Split, i.newIters, files, level, + internalIterOpts{}) + li.initBoundaryContext(&mlevels[mlevelsIndex].levelIterBoundaryContext) + mlevels[mlevelsIndex].iter = li + rli.Init(keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters}, + i.comparer.Compare, tableNewRangeDelIter(context.Background(), i.newIters), files, level, + manifest.KeyTypePoint) + rangeDelIters = append(rangeDelIters, rli) + + levelsIndex++ + mlevelsIndex++ + } + + // Add level iterators for the L0 sublevels, iterating from newest to + // oldest. + for i := len(current.L0SublevelFiles) - 1; i >= 0; i-- { + addLevelIterForFiles(current.L0SublevelFiles[i].Iter(), manifest.L0Sublevel(i)) + } + + // Add level iterators for the non-empty non-L0 levels. + for level := 1; level < numLevels; level++ { + if current.Levels[level].Empty() { + continue + } + if i.opts.skipSharedLevels && level >= sharedLevelsStart { + continue + } + addLevelIterForFiles(current.Levels[level].Iter(), manifest.Level(level)) + } + buf.merging.init(&i.opts.IterOptions, &InternalIteratorStats{}, i.comparer.Compare, i.comparer.Split, mlevels...) + buf.merging.snapshot = i.seqNum + rangeDelMiter.Init(i.comparer.Compare, keyspan.VisibleTransform(i.seqNum), new(keyspan.MergingBuffers), rangeDelIters...) + i.pointKeyIter = pointCollapsingIterator{ + comparer: i.comparer, + merge: i.merge, + seqNum: i.seqNum, + } + i.pointKeyIter.iter.Init(i.comparer, &buf.merging, &rangeDelMiter, nil /* mask */, i.opts.LowerBound, i.opts.UpperBound) + i.iter = &i.pointKeyIter +} + +// constructRangeKeyIter constructs the range-key iterator stack, populating +// i.rangeKey.rangeKeyIter with the resulting iterator. This is similar to +// Iterator.constructRangeKeyIter, except it doesn't handle batches and ensures +// iterConfig does *not* elide unsets/deletes. +func (i *scanInternalIterator) constructRangeKeyIter() { + // We want the bounded iter from iterConfig, but not the collapsing of + // RangeKeyUnsets and RangeKeyDels. + i.rangeKey.rangeKeyIter = i.rangeKey.iterConfig.Init( + i.comparer, i.seqNum, i.opts.LowerBound, i.opts.UpperBound, + nil /* hasPrefix */, nil /* prefix */, false, /* onlySets */ + &i.rangeKey.rangeKeyBuffers.internal) + + // Next are the flushables: memtables and large batches. + for j := len(i.readState.memtables) - 1; j >= 0; j-- { + mem := i.readState.memtables[j] + // We only need to read from memtables which contain sequence numbers older + // than seqNum. + if logSeqNum := mem.logSeqNum; logSeqNum >= i.seqNum { + continue + } + if rki := mem.newRangeKeyIter(&i.opts.IterOptions); rki != nil { + i.rangeKey.iterConfig.AddLevel(rki) + } + } + + current := i.readState.current + // Next are the file levels: L0 sub-levels followed by lower levels. + // + // Add file-specific iterators for L0 files containing range keys. This is less + // efficient than using levelIters for sublevels of L0 files containing + // range keys, but range keys are expected to be sparse anyway, reducing the + // cost benefit of maintaining a separate L0Sublevels instance for range key + // files and then using it here. + // + // NB: We iterate L0's files in reverse order. They're sorted by + // LargestSeqNum ascending, and we need to add them to the merging iterator + // in LargestSeqNum descending to preserve the merging iterator's invariants + // around Key Trailer order. + iter := current.RangeKeyLevels[0].Iter() + for f := iter.Last(); f != nil; f = iter.Prev() { + spanIterOpts := &keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters} + spanIter, err := i.newIterRangeKey(f, spanIterOpts) + if err != nil { + i.rangeKey.iterConfig.AddLevel(&errorKeyspanIter{err: err}) + continue + } + i.rangeKey.iterConfig.AddLevel(spanIter) + } + + // Add level iterators for the non-empty non-L0 levels. + for level := 1; level < len(current.RangeKeyLevels); level++ { + if current.RangeKeyLevels[level].Empty() { + continue + } + if i.opts.skipSharedLevels && level >= sharedLevelsStart { + continue + } + li := i.rangeKey.iterConfig.NewLevelIter() + spanIterOpts := keyspan.SpanIterOptions{RangeKeyFilters: i.opts.RangeKeyFilters} + li.Init(spanIterOpts, i.comparer.Compare, i.newIterRangeKey, current.RangeKeyLevels[level].Iter(), + manifest.Level(level), manifest.KeyTypeRange) + i.rangeKey.iterConfig.AddLevel(li) + } +} + +// seekGE seeks this iterator to the first key that's greater than or equal +// to the specified user key. +func (i *scanInternalIterator) seekGE(key []byte) bool { + i.iterKey, i.iterValue = i.iter.SeekGE(key, base.SeekGEFlagsNone) + return i.iterKey != nil +} + +// unsafeKey returns the unsafe InternalKey at the current position. The value +// is nil if the iterator is invalid or exhausted. +func (i *scanInternalIterator) unsafeKey() *InternalKey { + return i.iterKey +} + +// lazyValue returns a value pointer to the value at the current iterator +// position. Behaviour undefined if unsafeKey() returns a Range key or Rangedel +// kind key. +func (i *scanInternalIterator) lazyValue() LazyValue { + return i.iterValue +} + +// unsafeRangeDel returns a range key span. Behaviour undefined if UnsafeKey returns +// a non-rangedel kind. +func (i *scanInternalIterator) unsafeRangeDel() *keyspan.Span { + return i.pointKeyIter.iter.Span() +} + +// unsafeSpan returns a range key span. Behaviour undefined if UnsafeKey returns +// a non-rangekey type. +func (i *scanInternalIterator) unsafeSpan() *keyspan.Span { + return i.rangeKey.iiter.Span() +} + +// next advances the iterator in the forward direction, and returns the +// iterator's new validity state. +func (i *scanInternalIterator) next() bool { + i.iterKey, i.iterValue = i.iter.Next() + return i.iterKey != nil +} + +// error returns an error from the internal iterator, if there's any. +func (i *scanInternalIterator) error() error { + return i.iter.Error() +} + +// close closes this iterator, and releases any pooled objects. +func (i *scanInternalIterator) close() error { + if err := i.iter.Close(); err != nil { + return err + } + i.readState.unref() + if i.rangeKey != nil { + i.rangeKey.PrepareForReuse() + *i.rangeKey = iteratorRangeKeyState{ + rangeKeyBuffers: i.rangeKey.rangeKeyBuffers, + } + iterRangeKeyStateAllocPool.Put(i.rangeKey) + i.rangeKey = nil + } + if alloc := i.alloc; alloc != nil { + for j := range i.boundsBuf { + if cap(i.boundsBuf[j]) >= maxKeyBufCacheSize { + alloc.boundsBuf[j] = nil + } else { + alloc.boundsBuf[j] = i.boundsBuf[j] + } + } + *alloc = iterAlloc{ + keyBuf: alloc.keyBuf[:0], + boundsBuf: alloc.boundsBuf, + prefixOrFullSeekKey: alloc.prefixOrFullSeekKey[:0], + } + iterAllocPool.Put(alloc) + i.alloc = nil + } + return nil +} + +func (i *scanInternalIterator) initializeBoundBufs(lower, upper []byte) { + buf := i.boundsBuf[i.boundsBufIdx][:0] + if lower != nil { + buf = append(buf, lower...) + i.opts.LowerBound = buf + } else { + i.opts.LowerBound = nil + } + if upper != nil { + buf = append(buf, upper...) + i.opts.UpperBound = buf[len(buf)-len(upper):] + } else { + i.opts.UpperBound = nil + } + i.boundsBuf[i.boundsBufIdx] = buf + i.boundsBufIdx = 1 - i.boundsBufIdx +} diff --git a/vendor/github.com/cockroachdb/pebble/snapshot.go b/vendor/github.com/cockroachdb/pebble/snapshot.go index 5d53d7a..d2b8564 100644 --- a/vendor/github.com/cockroachdb/pebble/snapshot.go +++ b/vendor/github.com/cockroachdb/pebble/snapshot.go @@ -5,8 +5,11 @@ package pebble import ( + "context" "io" "math" + + "github.com/cockroachdb/pebble/internal/keyspan" ) // Snapshot provides a read-only point-in-time view of the DB state. @@ -42,10 +45,46 @@ func (s *Snapshot) Get(key []byte) ([]byte, io.Closer, error) { // return false). The iterator can be positioned via a call to SeekGE, // SeekLT, First or Last. func (s *Snapshot) NewIter(o *IterOptions) *Iterator { + return s.NewIterWithContext(context.Background(), o) +} + +// NewIterWithContext is like NewIter, and additionally accepts a context for +// tracing. +func (s *Snapshot) NewIterWithContext(ctx context.Context, o *IterOptions) *Iterator { if s.db == nil { panic(ErrClosed) } - return s.db.newIterInternal(nil /* batch */, s, o) + return s.db.newIter(ctx, nil /* batch */, s, o) +} + +// ScanInternal scans all internal keys within the specified bounds, truncating +// any rangedels and rangekeys to those bounds. For use when an external user +// needs to be aware of all internal keys that make up a key range. +// +// See comment on db.ScanInternal for the behaviour that can be expected of +// point keys deleted by range dels and keys masked by range keys. +func (s *Snapshot) ScanInternal( + ctx context.Context, + lower, upper []byte, + visitPointKey func(key *InternalKey, value LazyValue) error, + visitRangeDel func(start, end []byte, seqNum uint64) error, + visitRangeKey func(start, end []byte, keys []keyspan.Key) error, + visitSharedFile func(sst *SharedSSTMeta) error, +) error { + if s.db == nil { + panic(ErrClosed) + } + iter := s.db.newInternalIter(s, &scanInternalOptions{ + IterOptions: IterOptions{ + KeyTypes: IterKeyTypePointsAndRanges, + LowerBound: lower, + UpperBound: upper, + }, + skipSharedLevels: visitSharedFile != nil, + }) + defer iter.close() + + return scanInternalImpl(ctx, lower, upper, iter, visitPointKey, visitRangeDel, visitRangeKey, visitSharedFile) } // Close closes the snapshot, releasing its resources. Close must be called. diff --git a/vendor/github.com/cockroachdb/pebble/sstable/block.go b/vendor/github.com/cockroachdb/pebble/sstable/block.go index da951bc..27cf77f 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/block.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/block.go @@ -8,10 +8,12 @@ import ( "encoding/binary" "unsafe" + "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" "github.com/cockroachdb/pebble/internal/cache" "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/keyspan" + "github.com/cockroachdb/pebble/internal/manual" "github.com/cockroachdb/pebble/internal/rangedel" "github.com/cockroachdb/pebble/internal/rangekey" ) @@ -30,11 +32,33 @@ type blockWriter struct { nEntries int nextRestart int buf []byte - restarts []uint32 - curKey []byte - curValue []byte - prevKey []byte - tmp [4]byte + // For datablocks in TableFormatPebblev3, we steal the most significant bit + // in restarts for encoding setHasSameKeyPrefixSinceLastRestart. This leaves + // us with 31 bits, which is more than enough (no one needs > 2GB blocks). + // Typically, restarts occur every 16 keys, and by storing this bit with the + // restart, we can optimize for the case where a user wants to skip to the + // next prefix which happens to be in the same data block, but is > 16 keys + // away. We have seen production situations with 100+ versions per MVCC key + // (which share the same prefix). Additionally, for such writers, the prefix + // compression of the key, that shares the key with the preceding key, is + // limited to the prefix part of the preceding key -- this ensures that when + // doing NPrefix (see blockIter) we don't need to assemble the full key + // for each step since by limiting the length of the shared key we are + // ensuring that any of the keys with the same prefix can be used to + // assemble the full key when the prefix does change. + restarts []uint32 + curKey []byte + // curValue excludes the optional prefix provided to + // storeWithOptionalValuePrefix. + curValue []byte + prevKey []byte + tmp [4]byte + // We don't know the state of the sets that were at the end of the previous + // block, so this is initially 0. It may be true for the second and later + // restarts in a block. Not having inter-block information is fine since we + // will optimize by stepping through restarts only within the same block. + // Note that the first restart is the first key in the block. + setHasSameKeyPrefixSinceLastRestart bool } func (w *blockWriter) clear() { @@ -47,17 +71,41 @@ func (w *blockWriter) clear() { } } -func (w *blockWriter) store(keySize int, value []byte) { +// MaximumBlockSize is an extremely generous maximum block size of 256MiB. We +// explicitly place this limit to reserve a few bits in the restart for +// internal use. +const MaximumBlockSize = 1 << 28 +const setHasSameKeyPrefixRestartMask uint32 = 1 << 31 +const restartMaskLittleEndianHighByteWithoutSetHasSamePrefix byte = 0b0111_1111 +const restartMaskLittleEndianHighByteOnlySetHasSamePrefix byte = 0b1000_0000 + +// If !addValuePrefix, the valuePrefix is ignored. +func (w *blockWriter) storeWithOptionalValuePrefix( + keySize int, + value []byte, + maxSharedKeyLen int, + addValuePrefix bool, + valuePrefix valuePrefix, + setHasSameKeyPrefix bool, +) { shared := 0 + if !setHasSameKeyPrefix { + w.setHasSameKeyPrefixSinceLastRestart = false + } if w.nEntries == w.nextRestart { w.nextRestart = w.nEntries + w.restartInterval - w.restarts = append(w.restarts, uint32(len(w.buf))) + restart := uint32(len(w.buf)) + if w.setHasSameKeyPrefixSinceLastRestart { + restart = restart | setHasSameKeyPrefixRestartMask + } + w.setHasSameKeyPrefixSinceLastRestart = true + w.restarts = append(w.restarts, restart) } else { // TODO(peter): Manually inlined version of base.SharedPrefixLen(). This // is 3% faster on BenchmarkWriter on go1.16. Remove if future versions // show this to not be a performance win. For now, functions that use of // unsafe cannot be inlined. - n := len(w.curKey) + n := maxSharedKeyLen if n > len(w.prevKey) { n = len(w.prevKey) } @@ -72,7 +120,11 @@ func (w *blockWriter) store(keySize int, value []byte) { } } - needed := 3*binary.MaxVarintLen32 + len(w.curKey[shared:]) + len(value) + lenValuePlusOptionalPrefix := len(value) + if addValuePrefix { + lenValuePlusOptionalPrefix++ + } + needed := 3*binary.MaxVarintLen32 + len(w.curKey[shared:]) + lenValuePlusOptionalPrefix n := len(w.buf) if cap(w.buf) < n+needed { newCap := 2 * cap(w.buf) @@ -114,7 +166,7 @@ func (w *blockWriter) store(keySize int, value []byte) { } { - x := uint32(len(value)) + x := uint32(lenValuePlusOptionalPrefix) for x >= 0x80 { w.buf[n] = byte(x) | 0x80 x >>= 7 @@ -125,6 +177,10 @@ func (w *blockWriter) store(keySize int, value []byte) { } n += copy(w.buf[n:], w.curKey[shared:]) + if addValuePrefix { + w.buf[n : n+1][0] = byte(valuePrefix) + n++ + } n += copy(w.buf[n:], value) w.buf = w.buf[:n] @@ -134,6 +190,19 @@ func (w *blockWriter) store(keySize int, value []byte) { } func (w *blockWriter) add(key InternalKey, value []byte) { + w.addWithOptionalValuePrefix( + key, value, len(key.UserKey), false, 0, false) +} + +// Callers that always set addValuePrefix to false should use add() instead. +func (w *blockWriter) addWithOptionalValuePrefix( + key InternalKey, + value []byte, + maxSharedKeyLen int, + addValuePrefix bool, + valuePrefix valuePrefix, + setHasSameKeyPrefix bool, +) { w.curKey, w.prevKey = w.prevKey, w.curKey size := key.Size() @@ -143,7 +212,8 @@ func (w *blockWriter) add(key InternalKey, value []byte) { w.curKey = w.curKey[:size] key.Encode(w.curKey) - w.store(size, value) + w.storeWithOptionalValuePrefix( + size, value, maxSharedKeyLen, addValuePrefix, valuePrefix, setHasSameKeyPrefix) } func (w *blockWriter) finish() []byte { @@ -211,6 +281,36 @@ type blockEntry struct { // range keys since there is only a single range deletion and range key block // per sstable and the blockIter will not release the bytes for the block until // it is closed. +// +// Note on why blockIter knows about lazyValueHandling: +// +// blockIter's positioning functions (that return a LazyValue), are too +// complex to inline even prior to lazyValueHandling. blockIter.Next and +// blockIter.First were by far the cheapest and had costs 195 and 180 +// respectively, which exceeds the budget of 80. We initially tried to keep +// the lazyValueHandling logic out of blockIter by wrapping it with a +// lazyValueDataBlockIter. singleLevelIter and twoLevelIter would use this +// wrapped iter. The functions in lazyValueDataBlockIter were simple, in that +// they called the corresponding blockIter func and then decided whether the +// value was in fact in-place (so return immediately) or needed further +// handling. But these also turned out too costly for mid-stack inlining since +// simple calls like the following have a high cost that is barely under the +// budget of 80 +// +// k, v := i.data.SeekGE(key, flags) // cost 74 +// k, v := i.data.Next() // cost 72 +// +// We have 2 options for minimizing performance regressions: +// - Include the lazyValueHandling logic in the already non-inlineable +// blockIter functions: Since most of the time is spent in data block iters, +// it is acceptable to take the small hit of unnecessary branching (which +// hopefully branch prediction will predict correctly) for other kinds of +// blocks. +// - Duplicate the logic of singleLevelIterator and twoLevelIterator for the +// v3 sstable and only use the aforementioned lazyValueDataBlockIter for a +// v3 sstable. We would want to manage these copies via code generation. +// +// We have picked the first option here. type blockIter struct { cmp Compare // offset is the byte index that marks where the current key/value is @@ -248,6 +348,9 @@ type blockIter struct { // val contains the value the iterator is currently pointed at. If non-nil, // this points to a slice of the block data. val []byte + // lazyValue is val turned into a LazyValue, whenever a positioning method + // returns a non-nil key-value pair. + lazyValue base.LazyValue // ikey contains the decoded InternalKey the iterator is currently pointed // at. Note that the memory backing ikey.UserKey is either data stored // directly in the block, fullKey, or cachedBuf. The key stability guarantee @@ -273,7 +376,11 @@ type blockIter struct { cacheHandle cache.Handle // The first key in the block. This is used by the caller to set bounds // for block iteration for already loaded blocks. - firstKey InternalKey + firstKey InternalKey + lazyValueHandling struct { + vbr *valueBlockReader + hasValuePrefix bool + } } // blockIter implements the base.InternalIterator interface. @@ -413,6 +520,7 @@ func (i *blockIter) readEntry() { } unsharedKey := getBytes(ptr, int(unshared)) + // TODO(sumeer): move this into the else block below. i.fullKey = append(i.fullKey[:shared], unsharedKey...) if shared == 0 { // Provide stability for the key across positioning calls if the key @@ -553,7 +661,7 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba for index < upper { h := int32(uint(index+upper) >> 1) // avoid overflow when computing h // index ≤ h < upper - offset := int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*h:])) + offset := decodeRestart(i.data[i.restarts+4*h:]) // For a restart point, there are 0 bytes shared with the previous key. // The varint encoding of 0 occupies 1 byte. ptr := unsafe.Pointer(uintptr(i.ptr) + uintptr(offset+1)) @@ -621,18 +729,33 @@ func (i *blockIter) SeekGE(key []byte, flags base.SeekGEFlags) (*InternalKey, ba // 0, then all keys in this block are larger than the key sought, and offset // remains at zero. if index > 0 { - i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index-1):])) + i.offset = decodeRestart(i.data[i.restarts+4*(index-1):]) } i.readEntry() i.decodeInternalKey(i.key) // Iterate from that restart point to somewhere >= the key sought. - for ; i.valid(); i.Next() { + if !i.valid() { + return nil, base.LazyValue{} + } + if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 { + // Initialize i.lazyValue + if !i.lazyValueHandling.hasValuePrefix || + base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { + i.lazyValue = base.MakeInPlaceValue(i.val) + } else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) { + i.lazyValue = base.MakeInPlaceValue(i.val[1:]) + } else { + i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val) + } + return &i.ikey, i.lazyValue + } + for i.Next(); i.valid(); i.Next() { if base.InternalCompare(i.cmp, i.ikey, ikey) >= 0 { - return &i.ikey, base.MakeInPlaceValue(i.val) + // i.Next() has already initialized i.lazyValue. + return &i.ikey, i.lazyValue } } - return nil, base.LazyValue{} } @@ -666,7 +789,7 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba for index < upper { h := int32(uint(index+upper) >> 1) // avoid overflow when computing h // index ≤ h < upper - offset := int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*h:])) + offset := decodeRestart(i.data[i.restarts+4*h:]) // For a restart point, there are 0 bytes shared with the previous key. // The varint encoding of 0 occupies 1 byte. ptr := unsafe.Pointer(uintptr(i.ptr) + uintptr(offset+1)) @@ -733,9 +856,9 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba // index-1 will be the largest whose key is < the key sought. targetOffset := i.restarts if index > 0 { - i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index-1):])) + i.offset = decodeRestart(i.data[i.restarts+4*(index-1):]) if index < i.numRestarts { - targetOffset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index):])) + targetOffset = decodeRestart(i.data[i.restarts+4*(index):]) } } else if index == 0 { // If index == 0 then all keys in this block are larger than the key @@ -757,11 +880,12 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba if i.cmp(i.ikey.UserKey, ikey.UserKey) >= 0 { // The current key is greater than or equal to our search key. Back up to - // the previous key which was less than our search key. Note that his for + // the previous key which was less than our search key. Note that this for // loop will execute at least once with this if-block not being true, so // the key we are backing up to is the last one this loop cached. i.Prev() - return &i.ikey, base.MakeInPlaceValue(i.val) + // i.Prev() has already initialized i.lazyValue. + return &i.ikey, i.lazyValue } if i.nextOffset >= targetOffset { @@ -779,7 +903,15 @@ func (i *blockIter) SeekLT(key []byte, flags base.SeekLTFlags) (*InternalKey, ba if !i.valid() { return nil, base.LazyValue{} } - return &i.ikey, base.MakeInPlaceValue(i.val) + if !i.lazyValueHandling.hasValuePrefix || + base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { + i.lazyValue = base.MakeInPlaceValue(i.val) + } else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) { + i.lazyValue = base.MakeInPlaceValue(i.val[1:]) + } else { + i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val) + } + return &i.ikey, i.lazyValue } // First implements internalIterator.First, as documented in the pebble @@ -792,13 +924,27 @@ func (i *blockIter) First() (*InternalKey, base.LazyValue) { i.clearCache() i.readEntry() i.decodeInternalKey(i.key) - return &i.ikey, base.MakeInPlaceValue(i.val) + if !i.lazyValueHandling.hasValuePrefix || + base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { + i.lazyValue = base.MakeInPlaceValue(i.val) + } else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) { + i.lazyValue = base.MakeInPlaceValue(i.val[1:]) + } else { + i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val) + } + return &i.ikey, i.lazyValue +} + +func decodeRestart(b []byte) int32 { + _ = b[3] // bounds check hint to compiler; see golang.org/issue/14808 + return int32(uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | + uint32(b[3]&restartMaskLittleEndianHighByteWithoutSetHasSamePrefix)<<24) } // Last implements internalIterator.Last, as documented in the pebble package. func (i *blockIter) Last() (*InternalKey, base.LazyValue) { // Seek forward from the last restart point. - i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(i.numRestarts-1):])) + i.offset = decodeRestart(i.data[i.restarts+4*(i.numRestarts-1):]) if !i.valid() { return nil, base.LazyValue{} } @@ -813,7 +959,15 @@ func (i *blockIter) Last() (*InternalKey, base.LazyValue) { } i.decodeInternalKey(i.key) - return &i.ikey, base.MakeInPlaceValue(i.val) + if !i.lazyValueHandling.hasValuePrefix || + base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { + i.lazyValue = base.MakeInPlaceValue(i.val) + } else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) { + i.lazyValue = base.MakeInPlaceValue(i.val[1:]) + } else { + i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val) + } + return &i.ikey, i.lazyValue } // Next implements internalIterator.Next, as documented in the pebble @@ -850,7 +1004,307 @@ func (i *blockIter) Next() (*InternalKey, base.LazyValue) { i.ikey.Trailer = uint64(InternalKeyKindInvalid) i.ikey.UserKey = nil } - return &i.ikey, base.MakeInPlaceValue(i.val) + if !i.lazyValueHandling.hasValuePrefix || + base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { + i.lazyValue = base.MakeInPlaceValue(i.val) + } else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) { + i.lazyValue = base.MakeInPlaceValue(i.val[1:]) + } else { + i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val) + } + return &i.ikey, i.lazyValue +} + +// nextPrefix is used for implementing NPrefix. +func (i *blockIter) nextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + if i.lazyValueHandling.hasValuePrefix { + return i.nextPrefixV3(succKey) + } + const nextsBeforeSeek = 3 + k, v := i.Next() + for j := 1; k != nil && i.cmp(k.UserKey, succKey) < 0; j++ { + if j >= nextsBeforeSeek { + return i.SeekGE(succKey, base.SeekGEFlagsNone) + } + k, v = i.Next() + } + return k, v +} + +func (i *blockIter) nextPrefixV3(succKey []byte) (*InternalKey, base.LazyValue) { + // Doing nexts that involve a key comparison can be expensive (and the cost + // depends on the key length), so we use the same threshold of 3 that we use + // for TableFormatPebblev2 in blockIter.nextPrefix above. The next fast path + // that looks at setHasSamePrefix takes ~5ns per key, which is ~150x faster + // than doing a SeekGE within the block, so we do this 16 times + // (~5ns*16=80ns), and then switch to looking at restarts. Doing the binary + // search for the restart consumes > 100ns. If the number of versions is > + // 17, we will increment nextFastCount to 17, then do a binary search, and + // on average need to find a key between two restarts, so another 8 steps + // corresponding to nextFastCount, for a mean total of 17 + 8 = 25 such + // steps. + // + // TODO(sumeer): use the configured restartInterval for the sstable when it + // was written (which we don't currently store) instead of the default value + // of 16. + const nextCmpThresholdBeforeSeek = 3 + const nextFastThresholdBeforeRestarts = 16 + nextCmpCount := 0 + nextFastCount := 0 + usedRestarts := false + // INVARIANT: blockIter is valid. + if invariants.Enabled && !i.valid() { + panic(errors.AssertionFailedf("nextPrefixV3 called on invalid blockIter")) + } + prevKeyIsSet := i.ikey.Kind() == InternalKeyKindSet + for { + i.offset = i.nextOffset + if !i.valid() { + return nil, base.LazyValue{} + } + // Need to decode the length integers, so we can compute nextOffset. + ptr := unsafe.Pointer(uintptr(i.ptr) + uintptr(i.offset)) + // This is an ugly performance hack. Reading entries from blocks is one of + // the inner-most routines and decoding the 3 varints per-entry takes + // significant time. Neither go1.11 or go1.12 will inline decodeVarint for + // us, so we do it manually. This provides a 10-15% performance improvement + // on blockIter benchmarks on both go1.11 and go1.12. + // + // TODO(peter): remove this hack if go:inline is ever supported. + + // Decode the shared key length integer. + var shared uint32 + if a := *((*uint8)(ptr)); a < 128 { + shared = uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 1) + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { + shared = uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 2) + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { + shared = uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 3) + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { + shared = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 4) + } else { + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) + shared = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 5) + } + // Decode the unshared key length integer. + var unshared uint32 + if a := *((*uint8)(ptr)); a < 128 { + unshared = uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 1) + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { + unshared = uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 2) + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { + unshared = uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 3) + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { + unshared = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 4) + } else { + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) + unshared = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 5) + } + // Decode the value length integer. + var value uint32 + if a := *((*uint8)(ptr)); a < 128 { + value = uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 1) + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { + value = uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 2) + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { + value = uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 3) + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { + value = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 4) + } else { + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) + value = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 5) + } + // The starting position of the value. + valuePtr := unsafe.Pointer(uintptr(ptr) + uintptr(unshared)) + i.nextOffset = int32(uintptr(valuePtr)-uintptr(i.ptr)) + int32(value) + if invariants.Enabled && unshared < 8 { + // This should not happen since only the key prefix is shared, so even + // if the prefix length is the same as the user key length, the unshared + // will include the trailer. + panic(errors.AssertionFailedf("unshared %d is too small", unshared)) + } + // The trailer is written in little endian, so the key kind is the first + // byte in the trailer that is encoded in the slice [unshared-8:unshared]. + keyKind := InternalKeyKind((*[manual.MaxArrayLen]byte)(ptr)[unshared-8]) + prefixChanged := false + if keyKind == InternalKeyKindSet { + if invariants.Enabled && value == 0 { + panic(errors.AssertionFailedf("value is of length 0, but we expect a valuePrefix")) + } + valPrefix := *((*valuePrefix)(valuePtr)) + if setHasSamePrefix(valPrefix) { + // Fast-path. No need to assemble i.fullKey, or update i.key. We know + // that subsequent keys will not have a shared length that is greater + // than the prefix of the current key, which is also the prefix of + // i.key. Since we are continuing to iterate, we don't need to + // initialize i.ikey and i.lazyValue (these are initialized before + // returning). + nextFastCount++ + if nextFastCount > nextFastThresholdBeforeRestarts { + if usedRestarts { + // Exhausted iteration budget. This will never happen unless + // someone is using a restart interval > 16. It is just to guard + // against long restart intervals causing too much iteration. + break + } + // Haven't used restarts yet, so find the first restart at or beyond + // the current offset. + targetOffset := i.offset + var index int32 + { + // NB: manually inlined sort.Sort is ~5% faster. + // + // f defined for a restart point is true iff the offset >= + // targetOffset. + // Define f(-1) == false and f(i.numRestarts) == true. + // Invariant: f(index-1) == false, f(upper) == true. + upper := i.numRestarts + for index < upper { + h := int32(uint(index+upper) >> 1) // avoid overflow when computing h + // index ≤ h < upper + offset := decodeRestart(i.data[i.restarts+4*h:]) + if offset < targetOffset { + index = h + 1 // preserves f(index-1) == false + } else { + upper = h // preserves f(upper) == true + } + } + // index == upper, f(index-1) == false, and f(upper) (= f(index)) == true + // => answer is index. + } + usedRestarts = true + nextFastCount = 0 + if index == i.numRestarts { + // Already past the last real restart, so iterate a bit more until + // we are done with the block. + continue + } + // Have some real restarts after index. NB: index is the first + // restart at or beyond the current offset. + startingIndex := index + for index != i.numRestarts && + // The restart at index is 4 bytes written in little endian format + // starting at i.restart+4*index. The 0th byte is the least + // significant and the 3rd byte is the most significant. Since the + // most significant bit of the 3rd byte is what we use for + // encoding the set-has-same-prefix information, the indexing + // below has +3. + i.data[i.restarts+4*index+3]&restartMaskLittleEndianHighByteOnlySetHasSamePrefix != 0 { + // We still have the same prefix, so move to the next restart. + index++ + } + // index is the first restart that did not have the same prefix. + if index != startingIndex { + // Managed to skip past at least one restart. Resume iteration + // from index-1. Since nextFastCount has been reset to 0, we + // should be able to iterate to the next prefix. + i.offset = decodeRestart(i.data[i.restarts+4*(index-1):]) + i.readEntry() + } + // Else, unable to skip past any restart. Resume iteration. Since + // nextFastCount has been reset to 0, we should be able to iterate + // to the next prefix. + continue + } + continue + } else if prevKeyIsSet { + prefixChanged = true + } + } else { + prevKeyIsSet = false + } + // Slow-path cases: + // - (Likely) The prefix has changed. + // - (Unlikely) The prefix has not changed. + // We assemble the key etc. under the assumption that it is the likely + // case. + unsharedKey := getBytes(ptr, int(unshared)) + // TODO(sumeer): move this into the else block below. This is a bit tricky + // since the current logic assumes we have always copied the latest key + // into fullKey, which is why when we get to the next key we can (a) + // access i.fullKey[:shared], (b) append only the unsharedKey to + // i.fullKey. For (a), we can access i.key[:shared] since that memory is + // valid (even if unshared). For (b), we will need to remember whether + // i.key refers to i.fullKey or not, and can append the unsharedKey only + // in the former case and for the latter case need to copy the shared part + // too. This same comment applies to the other place where we can do this + // optimization, in readEntry(). + i.fullKey = append(i.fullKey[:shared], unsharedKey...) + i.val = getBytes(valuePtr, int(value)) + if shared == 0 { + // Provide stability for the key across positioning calls if the key + // doesn't share a prefix with the previous key. This removes requiring the + // key to be copied if the caller knows the block has a restart interval of + // 1. An important example of this is range-del blocks. + i.key = unsharedKey + } else { + i.key = i.fullKey + } + // Manually inlined version of i.decodeInternalKey(i.key). + if n := len(i.key) - 8; n >= 0 { + i.ikey.Trailer = binary.LittleEndian.Uint64(i.key[n:]) + i.ikey.UserKey = i.key[:n:n] + if i.globalSeqNum != 0 { + i.ikey.SetSeqNum(i.globalSeqNum) + } + } else { + i.ikey.Trailer = uint64(InternalKeyKindInvalid) + i.ikey.UserKey = nil + } + nextCmpCount++ + if invariants.Enabled && prefixChanged && i.cmp(i.ikey.UserKey, succKey) < 0 { + panic(errors.AssertionFailedf("prefix should have changed but %x < %x", + i.ikey.UserKey, succKey)) + } + if prefixChanged || i.cmp(i.ikey.UserKey, succKey) >= 0 { + // Prefix has changed. + if invariants.Enabled && !i.lazyValueHandling.hasValuePrefix { + panic(errors.AssertionFailedf("nextPrefixV3 being run for non-v3 sstable")) + } + if base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { + i.lazyValue = base.MakeInPlaceValue(i.val) + } else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) { + i.lazyValue = base.MakeInPlaceValue(i.val[1:]) + } else { + i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val) + } + return &i.ikey, i.lazyValue + } + // Else prefix has not changed. + + if nextCmpCount >= nextCmpThresholdBeforeSeek { + break + } + } + return i.SeekGE(succKey, base.SeekGEFlagsNone) +} + +// NextPrefix implements (base.InternalIterator).NextPrefix +func (i *blockIter) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + const nextsBeforeSeek = 3 + k, v := i.Next() + for j := 1; k != nil && i.cmp(k.UserKey, succKey) < 0; j++ { + if j >= nextsBeforeSeek { + return i.SeekGE(succKey, base.SeekGEFlagsNone) + } + k, v = i.Next() + } + return k, v } // Prev implements internalIterator.Prev, as documented in the pebble @@ -874,7 +1328,15 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) { i.ikey.UserKey = nil } i.cached = i.cached[:n] - return &i.ikey, base.MakeInPlaceValue(i.val) + if !i.lazyValueHandling.hasValuePrefix || + base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { + i.lazyValue = base.MakeInPlaceValue(i.val) + } else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) { + i.lazyValue = base.MakeInPlaceValue(i.val[1:]) + } else { + i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val) + } + return &i.ikey, i.lazyValue } i.clearCache() @@ -896,7 +1358,7 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) { for index < upper { h := int32(uint(index+upper) >> 1) // avoid overflow when computing h // index ≤ h < upper - offset := int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*h:])) + offset := decodeRestart(i.data[i.restarts+4*h:]) if offset < targetOffset { index = h + 1 // preserves f(i-1) == false } else { @@ -909,7 +1371,7 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) { i.offset = 0 if index > 0 { - i.offset = int32(binary.LittleEndian.Uint32(i.data[i.restarts+4*(index-1):])) + i.offset = decodeRestart(i.data[i.restarts+4*(index-1):]) } i.readEntry() @@ -921,7 +1383,15 @@ func (i *blockIter) Prev() (*InternalKey, base.LazyValue) { } i.decodeInternalKey(i.key) - return &i.ikey, base.MakeInPlaceValue(i.val) + if !i.lazyValueHandling.hasValuePrefix || + base.TrailerKind(i.ikey.Trailer) != InternalKeyKindSet { + i.lazyValue = base.MakeInPlaceValue(i.val) + } else if i.lazyValueHandling.vbr == nil || !isValueHandle(valuePrefix(i.val[0])) { + i.lazyValue = base.MakeInPlaceValue(i.val[1:]) + } else { + i.lazyValue = i.lazyValueHandling.vbr.getLazyValueForPrefixAndValueHandle(i.val) + } + return &i.ikey, i.lazyValue } // Key implements internalIterator.Key, as documented in the pebble package. @@ -930,7 +1400,7 @@ func (i *blockIter) Key() *InternalKey { } func (i *blockIter) value() base.LazyValue { - return base.MakeInPlaceValue(i.val) + return i.lazyValue } // Error implements internalIterator.Error, as documented in the pebble @@ -945,6 +1415,8 @@ func (i *blockIter) Close() error { i.cacheHandle.Release() i.cacheHandle = cache.Handle{} i.val = nil + i.lazyValue = base.LazyValue{} + i.lazyValueHandling.vbr = nil return nil } @@ -966,7 +1438,7 @@ func (i *blockIter) valid() bool { // gathers all the fragments with identical bounds within a block and returns a // single keyspan.Span describing all the keys defined over the span. // -// Memory lifetime +// # Memory lifetime // // A Span returned by fragmentBlockIter is only guaranteed to be stable until // the next fragmentBlockIter iteration positioning method. A Span's Keys slice @@ -1219,8 +1691,13 @@ func (i *fragmentBlockIter) Prev() *keyspan.Span { // SeekGE implements (keyspan.FragmentIterator).SeekGE. func (i *fragmentBlockIter) SeekGE(k []byte) *keyspan.Span { - i.dir = +1 - return i.gatherForward(i.blockIter.SeekGE(k, base.SeekGEFlags(0))) + if s := i.SeekLT(k); s != nil && i.blockIter.cmp(k, s.End) < 0 { + return s + } + // TODO(jackson): If the above i.SeekLT(k) discovers a span but the span + // doesn't meet the k < s.End comparison, then there's no need for the + // SeekLT to gatherBackward. + return i.Next() } // SeekLT implements (keyspan.FragmentIterator).SeekLT. diff --git a/vendor/github.com/cockroachdb/pebble/sstable/block_property.go b/vendor/github.com/cockroachdb/pebble/sstable/block_property.go index ff851e5..1e986fb 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/block_property.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/block_property.go @@ -82,17 +82,18 @@ import ( // properties must only apply to the key, and will be provided a nil value. // BlockPropertyCollector is used when writing a sstable. -// - All calls to Add are included in the next FinishDataBlock, after which -// the next data block is expected to start. // -// - The index entry generated for the data block, which contains the return -// value from FinishDataBlock, is not immediately included in the current -// index block. It is included when AddPrevDataBlockToIndexBlock is called. -// An alternative would be to return an opaque handle from FinishDataBlock -// and pass it to a new AddToIndexBlock method, which requires more -// plumbing, and passing of an interface{} results in a undesirable heap -// allocation. AddPrevDataBlockToIndexBlock must be called before keys are -// added to the new data block. +// - All calls to Add are included in the next FinishDataBlock, after which +// the next data block is expected to start. +// +// - The index entry generated for the data block, which contains the return +// value from FinishDataBlock, is not immediately included in the current +// index block. It is included when AddPrevDataBlockToIndexBlock is called. +// An alternative would be to return an opaque handle from FinishDataBlock +// and pass it to a new AddToIndexBlock method, which requires more +// plumbing, and passing of an interface{} results in a undesirable heap +// allocation. AddPrevDataBlockToIndexBlock must be called before keys are +// added to the new data block. type BlockPropertyCollector interface { // Name returns the name of the block property collector. Name() string @@ -586,9 +587,9 @@ var blockPropertiesFiltererPool = sync.Pool{ }, } -// NewBlockPropertiesFilterer returns a partially initialized filterer. To complete +// newBlockPropertiesFilterer returns a partially initialized filterer. To complete // initialization, call IntersectsUserPropsAndFinishInit. -func NewBlockPropertiesFilterer( +func newBlockPropertiesFilterer( filters []BlockPropertyFilter, limited BoundLimitedBlockPropertyFilter, ) *BlockPropertiesFilterer { filterer := blockPropertiesFiltererPool.Get().(*BlockPropertiesFilterer) @@ -608,11 +609,31 @@ func releaseBlockPropertiesFilterer(filterer *BlockPropertiesFilterer) { blockPropertiesFiltererPool.Put(filterer) } -// IntersectsUserPropsAndFinishInit is called with the user properties map for +// IntersectsTable evaluates the provided block-property filter against the +// provided set of table-level properties. If there is no intersection between +// the filters and the table or an error is encountered, IntersectsTable returns +// a nil filterer (and possibly an error). If there is an intersection, +// IntersectsTable returns a non-nil filterer that may be used by an iterator +// reading the table. +func IntersectsTable( + filters []BlockPropertyFilter, + limited BoundLimitedBlockPropertyFilter, + userProperties map[string]string, +) (*BlockPropertiesFilterer, error) { + f := newBlockPropertiesFilterer(filters, limited) + ok, err := f.intersectsUserPropsAndFinishInit(userProperties) + if !ok || err != nil { + releaseBlockPropertiesFilterer(f) + return nil, err + } + return f, nil +} + +// intersectsUserPropsAndFinishInit is called with the user properties map for // the sstable and returns whether the sstable intersects the filters. It // additionally initializes the shortIDToFiltersIndex for the filters that are // relevant to this sstable. -func (f *BlockPropertiesFilterer) IntersectsUserPropsAndFinishInit( +func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit( userProperties map[string]string, ) (bool, error) { for i := range f.filters { diff --git a/vendor/github.com/cockroachdb/pebble/sstable/filter.go b/vendor/github.com/cockroachdb/pebble/sstable/filter.go index 2d73e12..7b2e1ab 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/filter.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/filter.go @@ -18,14 +18,32 @@ type FilterMetrics struct { Misses int64 } -var dummyFilterMetrics FilterMetrics +// FilterMetricsTracker is used to keep track of filter metrics. It contains the +// same metrics as FilterMetrics, but they can be updated atomically. An +// instance of FilterMetricsTracker can be passed to a Reader as a ReaderOption. +type FilterMetricsTracker struct { + // See FilterMetrics.Hits. + hits atomic.Int64 + // See FilterMetrics.Misses. + misses atomic.Int64 +} -func (m *FilterMetrics) readerApply(r *Reader) { +var _ ReaderOption = (*FilterMetricsTracker)(nil) + +func (m *FilterMetricsTracker) readerApply(r *Reader) { if r.tableFilter != nil { r.tableFilter.metrics = m } } +// Load returns the current values as FilterMetrics. +func (m *FilterMetricsTracker) Load() FilterMetrics { + return FilterMetrics{ + Hits: m.hits.Load(), + Misses: m.misses.Load(), + } +} + // BlockHandle is the file offset and length of a block. type BlockHandle struct { Offset, Length uint64 @@ -47,22 +65,24 @@ type filterWriter interface { type tableFilterReader struct { policy FilterPolicy - metrics *FilterMetrics + metrics *FilterMetricsTracker } func newTableFilterReader(policy FilterPolicy) *tableFilterReader { return &tableFilterReader{ policy: policy, - metrics: &dummyFilterMetrics, + metrics: nil, } } func (f *tableFilterReader) mayContain(data, key []byte) bool { mayContain := f.policy.MayContain(TableFilter, data, key) - if mayContain { - atomic.AddInt64(&f.metrics.Misses, 1) - } else { - atomic.AddInt64(&f.metrics.Hits, 1) + if f.metrics != nil { + if mayContain { + f.metrics.misses.Add(1) + } else { + f.metrics.hits.Add(1) + } } return mayContain } diff --git a/vendor/github.com/cockroachdb/pebble/sstable/format.go b/vendor/github.com/cockroachdb/pebble/sstable/format.go index c6c484a..9190983 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/format.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/format.go @@ -23,8 +23,10 @@ const ( TableFormatRocksDBv2 TableFormatPebblev1 // Block properties. TableFormatPebblev2 // Range keys. + TableFormatPebblev3 // Value blocks. + TableFormatPebblev4 // Reserved. - TableFormatMax = TableFormatPebblev2 + TableFormatMax = TableFormatPebblev4 ) // ParseTableFormat parses the given magic bytes and version into its @@ -46,6 +48,10 @@ func ParseTableFormat(magic []byte, version uint32) (TableFormat, error) { return TableFormatPebblev1, nil case 2: return TableFormatPebblev2, nil + case 3: + return TableFormatPebblev3, nil + case 4: + return TableFormatPebblev4, nil default: return TableFormatUnspecified, base.CorruptionErrorf( "pebble/table: unsupported pebble format version %d", errors.Safe(version), @@ -53,7 +59,7 @@ func ParseTableFormat(magic []byte, version uint32) (TableFormat, error) { } default: return TableFormatUnspecified, base.CorruptionErrorf( - "pebble/table: invalid table (bad magic number)", + "pebble/table: invalid table (bad magic number: 0x%x)", magic, ) } } @@ -69,6 +75,10 @@ func (f TableFormat) AsTuple() (string, uint32) { return pebbleDBMagic, 1 case TableFormatPebblev2: return pebbleDBMagic, 2 + case TableFormatPebblev3: + return pebbleDBMagic, 3 + case TableFormatPebblev4: + return pebbleDBMagic, 4 default: panic("sstable: unknown table format version tuple") } @@ -85,6 +95,10 @@ func (f TableFormat) String() string { return "(Pebble,v1)" case TableFormatPebblev2: return "(Pebble,v2)" + case TableFormatPebblev3: + return "(Pebble,v3)" + case TableFormatPebblev4: + return "(Pebble,v4)" default: panic("sstable: unknown table format version tuple") } diff --git a/vendor/github.com/cockroachdb/pebble/sstable/internal.go b/vendor/github.com/cockroachdb/pebble/sstable/internal.go index 2dab9a7..9bab181 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/internal.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/internal.go @@ -21,6 +21,7 @@ const ( InternalKeyKindSet = base.InternalKeyKindSet InternalKeyKindMerge = base.InternalKeyKindMerge InternalKeyKindLogData = base.InternalKeyKindLogData + InternalKeyKindSingleDelete = base.InternalKeyKindSingleDelete InternalKeyKindRangeDelete = base.InternalKeyKindRangeDelete InternalKeyKindMax = base.InternalKeyKindMax InternalKeyKindInvalid = base.InternalKeyKindInvalid diff --git a/vendor/github.com/cockroachdb/pebble/sstable/options.go b/vendor/github.com/cockroachdb/pebble/sstable/options.go index d94a1e4..66dbdd4 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/options.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/options.go @@ -117,6 +117,9 @@ type ReaderOptions struct { // written with {Batch,DB}.Merge. The MergerName is checked for consistency // with the value stored in the sstable when it was written. MergerName string + + // Logger is an optional logger and tracer. + LoggerAndTracer base.LoggerAndTracer } func (o ReaderOptions) ensureDefaults() ReaderOptions { @@ -126,6 +129,9 @@ func (o ReaderOptions) ensureDefaults() ReaderOptions { if o.MergerName == "" { o.MergerName = base.DefaultMerger.Name } + if o.LoggerAndTracer == nil { + o.LoggerAndTracer = base.NoopLoggerAndTracer{} + } return o } @@ -221,6 +227,14 @@ type WriterOptions struct { // compress data blocks and write datablocks to disk in parallel with the // Writer client goroutine. Parallelism bool + + // ShortAttributeExtractor mirrors + // Options.Experimental.ShortAttributeExtractor. + ShortAttributeExtractor base.ShortAttributeExtractor + + // RequiredInPlaceValueBound mirrors + // Options.Experimental.RequiredInPlaceValueBound. + RequiredInPlaceValueBound UserKeyPrefixBound } func (o WriterOptions) ensureDefaults() WriterOptions { diff --git a/vendor/github.com/cockroachdb/pebble/sstable/properties.go b/vendor/github.com/cockroachdb/pebble/sstable/properties.go index 083480a..ca417ce 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/properties.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/properties.go @@ -120,6 +120,10 @@ type Properties struct { NumRangeKeySets uint64 `prop:"pebble.num.range-key-sets"` // The number of RANGEKEYUNSETs in this table. NumRangeKeyUnsets uint64 `prop:"pebble.num.range-key-unsets"` + // The number of value blocks in this table. Only serialized if > 0. + NumValueBlocks uint64 `prop:"pebble.num.value-blocks"` + // The number of values stored in value blocks. Only serialized if > 0. + NumValuesInValueBlocks uint64 `prop:"pebble.num.values.in.value-blocks"` // Timestamp of the earliest key. 0 if unknown. OldestKeyTime uint64 `prop:"rocksdb.oldest.key.time"` // The name of the prefix extractor used in this table. Empty if no prefix @@ -132,16 +136,29 @@ type Properties struct { PropertyCollectorNames string `prop:"rocksdb.property.collectors"` // Total raw key size. RawKeySize uint64 `prop:"rocksdb.raw.key.size"` + // Total raw key size of point deletion tombstones. This value is comparable + // to RawKeySize. + RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"` // Total raw rangekey key size. RawRangeKeyKeySize uint64 `prop:"pebble.raw.range-key.key.size"` // Total raw rangekey value size. RawRangeKeyValueSize uint64 `prop:"pebble.raw.range-key.value.size"` // Total raw value size. RawValueSize uint64 `prop:"rocksdb.raw.value.size"` + // The total number of keys in this table that were pinned by open snapshots. + SnapshotPinnedKeys uint64 `prop:"pebble.num.snapshot-pinned-keys"` + // The cumulative bytes of keys in this table that were pinned by + // open snapshots. This value is comparable to RawKeySize. + SnapshotPinnedKeySize uint64 `prop:"pebble.raw.snapshot-pinned-keys.size"` + // The cumulative bytes of values in this table that were pinned by + // open snapshots. This value is comparable to RawValueSize. + SnapshotPinnedValueSize uint64 `prop:"pebble.raw.snapshot-pinned-values.size"` // Size of the top-level index if kTwoLevelIndexSearch is used. TopLevelIndexSize uint64 `prop:"rocksdb.top-level.index.size"` // User collected properties. UserProperties map[string]string + // Total size of value blocks and value index block. Only serialized if > 0. + ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"` // If filtering is enabled, was the filter created on the whole key. WholeKeyFiltering bool `prop:"rocksdb.block.based.table.whole.key.filtering"` @@ -283,7 +300,7 @@ func (p *Properties) saveString(m map[string][]byte, offset uintptr, value strin m[propOffsetTagMap[offset]] = []byte(value) } -func (p *Properties) save(w *rawBlockWriter) { +func (p *Properties) save(tblFormat TableFormat, w *rawBlockWriter) { m := make(map[string][]byte) for k, v := range p.UserProperties { m[k] = []byte(v) @@ -333,6 +350,13 @@ func (p *Properties) save(w *rawBlockWriter) { p.saveUvarint(m, unsafe.Offsetof(p.NumDeletions), p.NumDeletions) p.saveUvarint(m, unsafe.Offsetof(p.NumMergeOperands), p.NumMergeOperands) p.saveUvarint(m, unsafe.Offsetof(p.NumRangeDeletions), p.NumRangeDeletions) + // NB: We only write out the RawTombstoneKeySize for Pebble formats. This + // isn't strictly necessary because unrecognized properties are interpreted + // as user-defined properties, however writing them prevents byte-for-byte + // equivalence with RocksDB files that some of our testing requires. + if p.RawPointTombstoneKeySize > 0 && tblFormat >= TableFormatPebblev1 { + p.saveUvarint(m, unsafe.Offsetof(p.RawPointTombstoneKeySize), p.RawPointTombstoneKeySize) + } if p.NumRangeKeys() > 0 { p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeyDels), p.NumRangeKeyDels) p.saveUvarint(m, unsafe.Offsetof(p.NumRangeKeySets), p.NumRangeKeySets) @@ -340,6 +364,12 @@ func (p *Properties) save(w *rawBlockWriter) { p.saveUvarint(m, unsafe.Offsetof(p.RawRangeKeyKeySize), p.RawRangeKeyKeySize) p.saveUvarint(m, unsafe.Offsetof(p.RawRangeKeyValueSize), p.RawRangeKeyValueSize) } + if p.NumValueBlocks > 0 { + p.saveUvarint(m, unsafe.Offsetof(p.NumValueBlocks), p.NumValueBlocks) + } + if p.NumValuesInValueBlocks > 0 { + p.saveUvarint(m, unsafe.Offsetof(p.NumValuesInValueBlocks), p.NumValuesInValueBlocks) + } p.saveUvarint(m, unsafe.Offsetof(p.OldestKeyTime), p.OldestKeyTime) if p.PrefixExtractorName != "" { p.saveString(m, unsafe.Offsetof(p.PrefixExtractorName), p.PrefixExtractorName) @@ -348,8 +378,16 @@ func (p *Properties) save(w *rawBlockWriter) { if p.PropertyCollectorNames != "" { p.saveString(m, unsafe.Offsetof(p.PropertyCollectorNames), p.PropertyCollectorNames) } + if p.SnapshotPinnedKeys > 0 { + p.saveUvarint(m, unsafe.Offsetof(p.SnapshotPinnedKeys), p.SnapshotPinnedKeys) + p.saveUvarint(m, unsafe.Offsetof(p.SnapshotPinnedKeySize), p.SnapshotPinnedKeySize) + p.saveUvarint(m, unsafe.Offsetof(p.SnapshotPinnedValueSize), p.SnapshotPinnedValueSize) + } p.saveUvarint(m, unsafe.Offsetof(p.RawKeySize), p.RawKeySize) p.saveUvarint(m, unsafe.Offsetof(p.RawValueSize), p.RawValueSize) + if p.ValueBlocksSize > 0 { + p.saveUvarint(m, unsafe.Offsetof(p.ValueBlocksSize), p.ValueBlocksSize) + } p.saveBool(m, unsafe.Offsetof(p.WholeKeyFiltering), p.WholeKeyFiltering) keys := make([]string, 0, len(m)) diff --git a/vendor/github.com/cockroachdb/pebble/sstable/raw_block.go b/vendor/github.com/cockroachdb/pebble/sstable/raw_block.go index c488266..d33b51a 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/raw_block.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/raw_block.go @@ -26,7 +26,8 @@ func (w *rawBlockWriter) add(key InternalKey, value []byte) { w.curKey = w.curKey[:size] copy(w.curKey, key.UserKey) - w.store(size, value) + w.storeWithOptionalValuePrefix( + size, value, len(key.UserKey), false, 0, false) } // rawBlockIter is an iterator over a single block of data. Unlike blockIter, diff --git a/vendor/github.com/cockroachdb/pebble/sstable/reader.go b/vendor/github.com/cockroachdb/pebble/sstable/reader.go index 84417d7..776e87e 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/reader.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/reader.go @@ -6,39 +6,34 @@ package sstable import ( "bytes" + "context" "encoding/binary" "fmt" "io" "os" "sort" "sync" + "time" "unsafe" "github.com/cespare/xxhash/v2" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/bytealloc" "github.com/cockroachdb/pebble/internal/cache" "github.com/cockroachdb/pebble/internal/crc" "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/keyspan" + "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/private" - "github.com/cockroachdb/pebble/vfs" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" ) var errCorruptIndexEntry = base.CorruptionErrorf("pebble/table: corrupt index entry") var errReaderClosed = errors.New("pebble/table: reader is closed") -const ( - // Constants for dynamic readahead of data blocks. Note that the size values - // make sense as some multiple of the default block size; and they should - // both be larger than the default block size. - minFileReadsForReadahead = 2 - // TODO(bilal): Have the initial size value be a factor of the block size, - // as opposed to a hardcoded value. - initialReadaheadSize = 64 << 10 /* 64KB */ - maxReadaheadSize = 256 << 10 /* 256KB */ -) - // decodeBlockHandle returns the block handle encoded at the start of src, as // well as the number of bytes it occupies. It returns zero if given invalid // input. A block handle for a data block or a first/lower level index block @@ -90,6 +85,9 @@ type block []byte type Iterator interface { base.InternalIterator + // NextPrefix implements (base.InternalIterator).NextPrefix. + NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) + // MaybeFilteredKeys may be called when an iterator is exhausted to indicate // whether or not the last positioning method may have skipped any keys due // to block-property filters. This is used by the Pebble levelIter to @@ -151,8 +149,10 @@ type Iterator interface { // bound is exhausted (due to block property filters filtering out index and // data blocks and going past the bound on the top level index block). Note // that if we tried to separate out the BPF case from others we could -// develop more knowledge here. - !PLDE or !PGDE of course imply that -// data-exhausted is not true. +// develop more knowledge here. +// - PGDE is true for twoLevelIterator. PLDE is true if it is a standalone +// singleLevelIterator. !PLDE or !PGDE of course imply that data-exhausted +// is not true. // // An implication of the above is that if we are going to somehow utilize // knowledge of data-exhausted in an optimization, we must not forget the @@ -171,11 +171,27 @@ type Iterator interface { // comments if trying to optimize this in the future. // - TrySeekUsingNext optimizations: these work regardless of exhaustion // state. +// +// Implementation detail: In the code PLDE only checks that +// i.data.isDataInvalidated(). This narrower check is safe, since this is a +// subset of the set expressed by the OR expression. Also, it is not a +// de-optimization since whenever we exhaust the iterator we explicitly call +// i.data.invalidate(). PGDE checks i.index.isDataInvalidated() && +// i.data.isDataInvalidated(). Again, this narrower check is safe, and not a +// de-optimization since whenever we exhaust the iterator we explicitly call +// i.index.invalidate() and i.data.invalidate(). The && is questionable -- for +// now this is a bit of defensive code. We should seriously consider removing +// it, since defensive code suggests we are not confident about our invariants +// (and if we are not confident, we need more invariant assertions, not +// defensive code). +// +// TODO(sumeer): remove the aforementioned defensive code. // singleLevelIterator iterates over an entire table of data. To seek for a given // key, it first looks in the index for the block that contains that key, and then // looks inside that block. type singleLevelIterator struct { + ctx context.Context cmp Compare // Global lower/upper bound for the iterator. lower []byte @@ -186,16 +202,29 @@ type singleLevelIterator struct { blockLower []byte blockUpper []byte reader *Reader - index blockIter - data blockIter - dataRS readaheadState + // vState will be set iff the iterator is constructed for virtual sstable + // iteration. + vState *virtualState + // endKeyInclusive is set to force the iterator to treat the upper field as + // inclusive while iterating instead of exclusive. This is used for virtual + // sstable iteration where the file bounds are inclusive. + endKeyInclusive bool + index blockIter + data blockIter + dataRH objstorage.ReadHandle + dataRHPrealloc objstorageprovider.PreallocatedReadHandle // dataBH refers to the last data block that the iterator considered // loading. It may not actually have loaded the block, due to an error or // because it was considered irrelevant. - dataBH BlockHandle - err error - closeHook func(i Iterator) error - stats *base.InternalIteratorStats + dataBH BlockHandle + vbReader *valueBlockReader + // vbRH is the read handle for value blocks, which are in a different + // part of the sstable than data blocks. + vbRH objstorage.ReadHandle + vbRHPrealloc objstorageprovider.PreallocatedReadHandle + err error + closeHook func(i Iterator) error + stats *base.InternalIteratorStats // boundsCmp and positionedUsingLatestBounds are for optimizing iteration // that uses multiple adjacent bounds. The seek after setting a new bound @@ -365,21 +394,33 @@ func checkRangeKeyFragmentBlockIterator(obj interface{}) { // init initializes a singleLevelIterator for reading from the table. It is // synonmous with Reader.NewIter, but allows for reusing of the iterator // between different Readers. +// +// Note that lower, upper passed into init has nothing to do with virtual sstable +// bounds. If the virtualState passed in is not nil, then virtual sstable bounds +// will be enforced. func (i *singleLevelIterator) init( + ctx context.Context, r *Reader, + v *virtualState, lower, upper []byte, filterer *BlockPropertiesFilterer, useFilter bool, stats *base.InternalIteratorStats, + rp ReaderProvider, ) error { if r.err != nil { return r.err } - indexH, err := r.readIndex(stats) + indexH, err := r.readIndex(ctx, stats) if err != nil { return err } + if v != nil { + i.vState = v + i.endKeyInclusive, lower, upper = v.constrainBounds(lower, upper, false /* endInclusive */) + } + i.ctx = ctx i.lower = lower i.upper = upper i.bpfs = filterer @@ -393,20 +434,39 @@ func (i *singleLevelIterator) init( _ = i.index.Close() return err } - i.dataRS.size = initialReadaheadSize + i.dataRH = objstorageprovider.UsePreallocatedReadHandle(ctx, r.readable, &i.dataRHPrealloc) + if r.tableFormat >= TableFormatPebblev3 { + if r.Properties.NumValueBlocks > 0 { + // NB: we cannot avoid this ~248 byte allocation, since valueBlockReader + // can outlive the singleLevelIterator due to be being embedded in a + // LazyValue. This consumes ~2% in microbenchmark CPU profiles, but we + // should only optimize this if it shows up as significant in end-to-end + // CockroachDB benchmarks, since it is tricky to do so. One possibility + // is that if many sstable iterators only get positioned at latest + // versions of keys, and therefore never expose a LazyValue that is + // separated to their callers, they can put this valueBlockReader into a + // sync.Pool. + i.vbReader = &valueBlockReader{ + ctx: ctx, + bpOpen: i, + rp: rp, + vbih: r.valueBIH, + stats: stats, + } + i.data.lazyValueHandling.vbr = i.vbReader + i.vbRH = objstorageprovider.UsePreallocatedReadHandle(ctx, r.readable, &i.vbRHPrealloc) + } + i.data.lazyValueHandling.hasValuePrefix = true + } return nil } // setupForCompaction sets up the singleLevelIterator for use with compactionIter. // Currently, it skips readahead ramp-up. It should be called after init is called. func (i *singleLevelIterator) setupForCompaction() { - if i.reader.fs != nil { - f, err := i.reader.fs.Open(i.reader.filename, vfs.SequentialReadsOption) - if err == nil { - // Given that this iterator is for a compaction, we can assume that it - // will be read sequentially and we can skip the readahead ramp-up. - i.dataRS.sequentialFile = f - } + i.dataRH.MaxReadahead() + if i.vbRH != nil { + i.vbRH.MaxReadahead() } } @@ -434,7 +494,12 @@ func (i *singleLevelIterator) initBounds() { if i.blockUpper != nil && i.cmp(i.blockUpper, i.index.Key().UserKey) > 0 { // The upper-bound is greater than the index key which itself is greater // than or equal to every key in the block. No need to check the - // upper-bound again for this block. + // upper-bound again for this block. Even if blockUpper is inclusive + // because of upper being inclusive, we can still safely set blockUpper + // to nil here. + // + // TODO(bananabrick): We could also set blockUpper to nil for the >= + // case, if blockUpper is inclusive. i.blockUpper = nil } } @@ -495,7 +560,8 @@ func (i *singleLevelIterator) loadBlock(dir int8) loadBlockResult { } // blockIntersects } - block, err := i.readBlockWithStats(i.dataBH, &i.dataRS) + ctx := objiotracing.WithBlockType(i.ctx, objiotracing.DataBlock) + block, err := i.reader.readBlock(ctx, i.dataBH, nil /* transform */, i.dataRH, i.stats) if err != nil { i.err = err return loadBlockFailed @@ -510,6 +576,15 @@ func (i *singleLevelIterator) loadBlock(dir int8) loadBlockResult { return loadBlockOK } +// readBlockForVBR implements the blockProviderWhenOpen interface for use by +// the valueBlockReader. +func (i *singleLevelIterator) readBlockForVBR( + ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats, +) (cache.Handle, error) { + ctx = objiotracing.WithBlockType(ctx, objiotracing.ValueBlock) + return i.reader.readBlock(ctx, h, nil, i.vbRH, stats) +} + // resolveMaybeExcluded is invoked when the block-property filterer has found // that a block is excluded according to its properties but only if its bounds // fall within the filter's current bounds. This function consults the @@ -577,12 +652,6 @@ func (i *singleLevelIterator) resolveMaybeExcluded(dir int8) intersectsResult { return blockIntersects } -func (i *singleLevelIterator) readBlockWithStats( - bh BlockHandle, raState *readaheadState, -) (cache.Handle, error) { - return i.reader.readBlock(bh, nil /* transform */, raState, i.stats) -} - func (i *singleLevelIterator) initBoundsForAlreadyLoadedBlock() { if i.data.firstKey.UserKey == nil { panic("initBoundsForAlreadyLoadedBlock must not be called on empty or corrupted block") @@ -617,9 +686,12 @@ func (i *singleLevelIterator) trySeekGEUsingNextWithinBlock( for j := 0; j < numStepsBeforeSeek; j++ { curKeyCmp := i.cmp(k.UserKey, key) if curKeyCmp >= 0 { - if i.blockUpper != nil && i.cmp(k.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{}, true + if i.blockUpper != nil { + cmp := i.cmp(k.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{}, true + } } return k, v, true } @@ -676,6 +748,17 @@ func (i *singleLevelIterator) recordOffset() uint64 { func (i *singleLevelIterator) SeekGE( key []byte, flags base.SeekGEFlags, ) (*InternalKey, base.LazyValue) { + if i.vState != nil { + // Callers of SeekGE don't know about virtual sstable bounds, so we may + // have to internally restrict the bounds. + // + // TODO(bananabrick): We can optimize this check away for the level iter + // if necessary. + if i.cmp(key, i.lower) < 0 { + key = i.lower + } + } + if flags.TrySeekUsingNext() { // The i.exhaustedBounds comparison indicates that the upper bound was // reached. The i.data.isDataInvalidated() indicates that the sstable was @@ -766,16 +849,18 @@ func (i *singleLevelIterator) seekGEHelper( less = i.cmp(currKey.UserKey, key) < 0 } if !less { - if i.blockUpper != nil && i.cmp(currKey.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} + if i.blockUpper != nil { + cmp := i.cmp(currKey.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } } return currKey, value } } // Slow-path. - // Since we're re-seeking the iterator, the previous value of // maybeFilteredKeysSingleLevel is irrelevant. If we filter out blocks // during seeking, loadBlock will set it to true. @@ -798,10 +883,14 @@ func (i *singleLevelIterator) seekGEHelper( // to the next block if upper bound is already exceeded. Note that // the next block starts with keys >= ikey.UserKey since even // though this is the block separator, the same user key can span - // multiple blocks. Since upper is exclusive we use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} + // multiple blocks. If upper is exclusive we use >= below, else + // we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } } // Want to skip to the next block. dontSeekWithinBlock = true @@ -809,9 +898,12 @@ func (i *singleLevelIterator) seekGEHelper( } if !dontSeekWithinBlock { if ikey, val := i.data.SeekGE(key, flags.DisableTrySeekUsingNext()); ikey != nil { - if i.blockUpper != nil && i.cmp(ikey.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} + if i.blockUpper != nil { + cmp := i.cmp(ikey.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } } return ikey, val } @@ -825,6 +917,16 @@ func (i *singleLevelIterator) seekGEHelper( func (i *singleLevelIterator) SeekPrefixGE( prefix, key []byte, flags base.SeekGEFlags, ) (*base.InternalKey, base.LazyValue) { + if i.vState != nil { + // Callers of SeekPrefixGE aren't aware of virtual sstable bounds, so + // we may have to internally restrict the bounds. + // + // TODO(bananabrick): We can optimize away this check for the level iter + // if necessary. + if i.cmp(key, i.lower) < 0 { + key = i.lower + } + } k, v := i.seekPrefixGE(prefix, key, flags, i.useFilter) return k, v } @@ -846,7 +948,7 @@ func (i *singleLevelIterator) seekPrefixGE( i.lastBloomFilterMatched = false // Check prefix bloom filter. var dataH cache.Handle - dataH, i.err = i.reader.readFilter(i.stats) + dataH, i.err = i.reader.readFilter(i.ctx, i.stats) if i.err != nil { i.data.invalidate() return nil, base.LazyValue{} @@ -893,12 +995,55 @@ func (i *singleLevelIterator) seekPrefixGE( return k, value } +// virtualLast should only be called if i.vReader != nil. +func (i *singleLevelIterator) virtualLast() (*InternalKey, base.LazyValue) { + if i.vState == nil { + panic("pebble: invalid call to virtualLast") + } + + // Seek to the first internal key. + ikey, _ := i.SeekGE(i.upper, base.SeekGEFlagsNone) + if i.endKeyInclusive { + // Let's say the virtual sstable upper bound is c#1, with the keys c#3, c#2, + // c#1, d, e, ... in the sstable. So, the last key in the virtual sstable is + // c#1. We can perform SeekGE(i.upper) and then keep nexting until we find + // the last key with userkey == i.upper. + // + // TODO(bananabrick): Think about how to improve this. If many internal keys + // with the same user key at the upper bound then this could be slow, but + // maybe the odds of having many internal keys with the same user key at the + // upper bound are low. + for ikey != nil && i.cmp(ikey.UserKey, i.upper) == 0 { + ikey, _ = i.Next() + } + return i.Prev() + } + + // We seeked to the first key >= i.upper. + return i.Prev() +} + // SeekLT implements internalIterator.SeekLT, as documented in the pebble // package. Note that SeekLT only checks the lower bound. It is up to the -// caller to ensure that key is less than the upper bound. +// caller to ensure that key is less than or equal to the upper bound. func (i *singleLevelIterator) SeekLT( key []byte, flags base.SeekLTFlags, ) (*InternalKey, base.LazyValue) { + if i.vState != nil { + // Might have to fix upper bound since virtual sstable bounds are not + // known to callers of SeekLT. + // + // TODO(bananabrick): We can optimize away this check for the level iter + // if necessary. + cmp := i.cmp(key, i.upper) + // key == i.upper is fine. We'll do the right thing and return the + // first internal key with user key < key. + if cmp > 0 { + // Return the last key in the virtual sstable. + return i.virtualLast() + } + } + i.exhaustedBounds = 0 i.err = nil // clear cached iteration error boundsCmp := i.boundsCmp @@ -1002,6 +1147,13 @@ func (i *singleLevelIterator) SeekLT( // to ensure that key is greater than or equal to the lower bound (e.g. via a // call to SeekGE(lower)). func (i *singleLevelIterator) First() (*InternalKey, base.LazyValue) { + // If the iterator was created on a virtual sstable, we will SeekGE to the + // lower bound instead of using First, because First does not respect + // bounds. + if i.vState != nil { + return i.SeekGE(i.lower, base.SeekGEFlagsNone) + } + if i.lower != nil { panic("singleLevelIterator.First() used despite lower bound") } @@ -1031,9 +1183,12 @@ func (i *singleLevelIterator) firstInternal() (*InternalKey, base.LazyValue) { } if result == loadBlockOK { if ikey, val := i.data.First(); ikey != nil { - if i.blockUpper != nil && i.cmp(ikey.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} + if i.blockUpper != nil { + cmp := i.cmp(ikey.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } } return ikey, val } @@ -1043,11 +1198,14 @@ func (i *singleLevelIterator) firstInternal() (*InternalKey, base.LazyValue) { // don't want to bother moving to the next block if upper bound is // already exceeded. Note that the next block starts with keys >= // ikey.UserKey since even though this is the block separator, the - // same user key can span multiple blocks. Since upper is exclusive we - // use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} + // same user key can span multiple blocks. If upper is exclusive we + // use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } } // Else fall through to skipForward. } @@ -1060,6 +1218,10 @@ func (i *singleLevelIterator) firstInternal() (*InternalKey, base.LazyValue) { // to ensure that key is less than the upper bound (e.g. via a call to // SeekLT(upper)) func (i *singleLevelIterator) Last() (*InternalKey, base.LazyValue) { + if i.vState != nil { + return i.virtualLast() + } + if i.upper != nil { panic("singleLevelIterator.Last() used despite upper bound") } @@ -1128,10 +1290,87 @@ func (i *singleLevelIterator) Next() (*InternalKey, base.LazyValue) { return nil, base.LazyValue{} } if key, val := i.data.Next(); key != nil { - if i.blockUpper != nil && i.cmp(key.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 + if i.blockUpper != nil { + cmp := i.cmp(key.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + return key, val + } + return i.skipForward() +} + +// NextPrefix implements (base.InternalIterator).NextPrefix. +func (i *singleLevelIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + if i.exhaustedBounds == +1 { + panic("NextPrefix called even though exhausted upper bound") + } + i.exhaustedBounds = 0 + i.maybeFilteredKeysSingleLevel = false + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + if i.err != nil { + return nil, base.LazyValue{} + } + if key, val := i.data.nextPrefix(succKey); key != nil { + if i.blockUpper != nil { + cmp := i.cmp(key.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + return key, val + } + // Did not find prefix in the existing data block. This is the slow-path + // where we effectively seek the iterator. + var ikey *InternalKey + // The key is likely to be in the next data block, so try one step. + if ikey, _ = i.index.Next(); ikey == nil { + // The target key is greater than any key in the index block. + // Invalidate the block iterator so that a subsequent call to Prev() + // will return the last key in the table. + i.data.invalidate() + return nil, base.LazyValue{} + } + if i.cmp(succKey, ikey.UserKey) > 0 { + // Not in the next data block, so seek the index. + if ikey, _ = i.index.SeekGE(succKey, base.SeekGEFlagsNone); ikey == nil { + // The target key is greater than any key in the index block. + // Invalidate the block iterator so that a subsequent call to Prev() + // will return the last key in the table. + i.data.invalidate() return nil, base.LazyValue{} } + } + result := i.loadBlock(+1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockIrrelevant { + // Enforce the upper bound here since don't want to bother moving + // to the next block if upper bound is already exceeded. Note that + // the next block starts with keys >= ikey.UserKey since even + // though this is the block separator, the same user key can span + // multiple blocks. If upper is exclusive we use >= below, else we use + // >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } + } else if key, val := i.data.SeekGE(succKey, base.SeekGEFlagsNone); key != nil { + if i.blockUpper != nil { + cmp := i.cmp(key.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } + } return key, val } return i.skipForward() @@ -1183,18 +1422,24 @@ func (i *singleLevelIterator) skipForward() (*InternalKey, base.LazyValue) { // since don't want to bother moving to the next block if upper // bound is already exceeded. Note that the next block starts with // keys >= key.UserKey since even though this is the block - // separator, the same user key can span multiple blocks. Since - // upper is exclusive we use >= below. - if i.upper != nil && i.cmp(key.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} + // separator, the same user key can span multiple blocks. If upper + // is exclusive we use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(key.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } } continue } if key, val := i.data.First(); key != nil { - if i.blockUpper != nil && i.cmp(key.UserKey, i.blockUpper) >= 0 { - i.exhaustedBounds = +1 - return nil, base.LazyValue{} + if i.blockUpper != nil { + cmp := i.cmp(key.UserKey, i.blockUpper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + return nil, base.LazyValue{} + } } return key, val } @@ -1282,20 +1527,30 @@ func (i *singleLevelIterator) Close() error { } err = firstError(err, i.data.Close()) err = firstError(err, i.index.Close()) - if i.dataRS.sequentialFile != nil { - err = firstError(err, i.dataRS.sequentialFile.Close()) - i.dataRS.sequentialFile = nil + if i.dataRH != nil { + err = firstError(err, i.dataRH.Close()) + i.dataRH = nil } err = firstError(err, i.err) if i.bpfs != nil { releaseBlockPropertiesFilterer(i.bpfs) } + if i.vbReader != nil { + i.vbReader.close() + } + if i.vbRH != nil { + err = firstError(err, i.vbRH.Close()) + i.vbRH = nil + } *i = i.resetForReuse() singleLevelIterPool.Put(i) return err } func (i *singleLevelIterator) String() string { + if i.vState != nil { + return i.vState.fileNum.String() + } return i.reader.fileNum.String() } @@ -1309,23 +1564,36 @@ func disableBoundsOpt(bound []byte, ptr uintptr) bool { } // SetBounds implements internalIterator.SetBounds, as documented in the pebble -// package. +// package. Note that the upper field is exclusive. func (i *singleLevelIterator) SetBounds(lower, upper []byte) { i.boundsCmp = 0 - if i.positionedUsingLatestBounds { - if i.upper != nil && lower != nil && i.cmp(i.upper, lower) <= 0 { - i.boundsCmp = +1 - if invariants.Enabled && disableBoundsOpt(lower, uintptr(unsafe.Pointer(i))) { - i.boundsCmp = 0 - } - } else if i.lower != nil && upper != nil && i.cmp(upper, i.lower) <= 0 { - i.boundsCmp = -1 - if invariants.Enabled && disableBoundsOpt(upper, uintptr(unsafe.Pointer(i))) { - i.boundsCmp = 0 + if i.vState != nil { + // If the reader is constructed for a virtual sstable, then we must + // constrain the bounds of the reader. For physical sstables, the bounds + // can be wider than the actual sstable's bounds because we won't + // accidentally expose additional keys as there are no additional keys. + i.endKeyInclusive, lower, upper = i.vState.constrainBounds( + lower, upper, false, + ) + } else { + // TODO(bananabrick): Figure out the logic here to enable the boundsCmp + // optimization for virtual sstables. + if i.positionedUsingLatestBounds { + if i.upper != nil && lower != nil && i.cmp(i.upper, lower) <= 0 { + i.boundsCmp = +1 + if invariants.Enabled && disableBoundsOpt(lower, uintptr(unsafe.Pointer(i))) { + i.boundsCmp = 0 + } + } else if i.lower != nil && upper != nil && i.cmp(upper, i.lower) <= 0 { + i.boundsCmp = -1 + if invariants.Enabled && disableBoundsOpt(upper, uintptr(unsafe.Pointer(i))) { + i.boundsCmp = 0 + } } } - i.positionedUsingLatestBounds = false } + + i.positionedUsingLatestBounds = false i.lower = lower i.upper = upper i.blockLower = nil @@ -1347,6 +1615,9 @@ type compactionIterator struct { var _ base.InternalIterator = (*compactionIterator)(nil) func (i *compactionIterator) String() string { + if i.vState != nil { + return i.vState.fileNum.String() + } return i.reader.fileNum.String() } @@ -1386,6 +1657,10 @@ func (i *compactionIterator) Next() (*InternalKey, base.LazyValue) { return i.skipForward(i.data.Next()) } +func (i *compactionIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + panic("pebble: NextPrefix unimplemented") +} + func (i *compactionIterator) Prev() (*InternalKey, base.LazyValue) { panic("pebble: Prev unimplemented") } @@ -1425,6 +1700,14 @@ func (i *compactionIterator) skipForward( curOffset := i.recordOffset() *i.bytesIterated += uint64(curOffset - i.prevOffset) i.prevOffset = curOffset + + if i.vState != nil && key != nil { + cmp := i.cmp(key.UserKey, i.vState.upper.UserKey) + if cmp > 0 || (i.vState.upper.IsExclusiveSentinel() && cmp == 0) { + return nil, base.LazyValue{} + } + } + return key, val } @@ -1474,7 +1757,8 @@ func (i *twoLevelIterator) loadIndex(dir int8) loadBlockResult { } // blockIntersects } - indexBlock, err := i.readBlockWithStats(bhp.BlockHandle, nil /* readaheadState */) + ctx := objiotracing.WithBlockType(i.ctx, objiotracing.MetadataBlock) + indexBlock, err := i.reader.readBlock(ctx, bhp.BlockHandle, nil /* transform */, nil /* readHandle */, i.stats) if err != nil { i.err = err return loadBlockFailed @@ -1549,21 +1833,33 @@ func (i *twoLevelIterator) resolveMaybeExcluded(dir int8) intersectsResult { return blockIntersects } +// Note that lower, upper passed into init has nothing to do with virtual sstable +// bounds. If the virtualState passed in is not nil, then virtual sstable bounds +// will be enforced. func (i *twoLevelIterator) init( + ctx context.Context, r *Reader, + v *virtualState, lower, upper []byte, filterer *BlockPropertiesFilterer, useFilter bool, stats *base.InternalIteratorStats, + rp ReaderProvider, ) error { if r.err != nil { return r.err } - topLevelIndexH, err := r.readIndex(stats) + topLevelIndexH, err := r.readIndex(ctx, stats) if err != nil { return err } + if v != nil { + i.vState = v + // Note that upper is exclusive here. + i.endKeyInclusive, lower, upper = v.constrainBounds(lower, upper, false /* endInclusive */) + } + i.ctx = ctx i.lower = lower i.upper = upper i.bpfs = filterer @@ -1577,10 +1873,28 @@ func (i *twoLevelIterator) init( _ = i.topLevelIndex.Close() return err } + i.dataRH = r.readable.NewReadHandle(ctx) + if r.tableFormat >= TableFormatPebblev3 { + if r.Properties.NumValueBlocks > 0 { + i.vbReader = &valueBlockReader{ + ctx: ctx, + bpOpen: i, + rp: rp, + vbih: r.valueBIH, + stats: stats, + } + i.data.lazyValueHandling.vbr = i.vbReader + i.vbRH = r.readable.NewReadHandle(ctx) + } + i.data.lazyValueHandling.hasValuePrefix = true + } return nil } func (i *twoLevelIterator) String() string { + if i.vState != nil { + return i.vState.fileNum.String() + } return i.reader.fileNum.String() } @@ -1605,17 +1919,29 @@ func (i *twoLevelIterator) MaybeFilteredKeys() bool { func (i *twoLevelIterator) SeekGE( key []byte, flags base.SeekGEFlags, ) (*InternalKey, base.LazyValue) { + if i.vState != nil { + // Callers of SeekGE don't know about virtual sstable bounds, so we may + // have to internally restrict the bounds. + // + // TODO(bananabrick): We can optimize away this check for the level iter + // if necessary. + if i.cmp(key, i.lower) < 0 { + key = i.lower + } + } + err := i.err i.err = nil // clear cached iteration error - // TODO(sumeer): we are not fully optimizing in the flags.TrySeekUsingNext() - // case when the twoLevelIterator is already exhausted. We will take the - // slow-path below, even though we could return now. We could do: - // if flags.TrySeekUsingNext() && (i.exhaustedBounds == +1 || (i.data.isDataInvalidated() && - // i.index.isDataInvalidated())) && i.err == nil { - // // Already exhausted, so return nil. - // return nil, nil - // } + // The twoLevelIterator could be already exhausted. Utilize that when + // trySeekUsingNext is true. See the comment about data-exhausted, PGDE, and + // bounds-exhausted near the top of the file. + if flags.TrySeekUsingNext() && + (i.exhaustedBounds == +1 || (i.data.isDataInvalidated() && i.index.isDataInvalidated())) && + err == nil { + // Already exhausted, so return nil. + return nil, base.LazyValue{} + } // SeekGE performs various step-instead-of-seeking optimizations: eg enabled // by trySeekUsingNext, or by monotonically increasing bounds (i.boundsCmp). @@ -1657,10 +1983,13 @@ func (i *twoLevelIterator) SeekGE( // to the next entry in the top level index if upper bound is // already exceeded. Note that the next entry starts with keys >= // ikey.UserKey since even though this is the block separator, the - // same user key can span multiple index blocks. Since upper is - // exclusive we use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 + // same user key can span multiple index blocks. If upper is + // exclusive we use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + } } // Fall through to skipForward. dontSeekWithinSingleLevelIter = true @@ -1742,6 +2071,17 @@ func (i *twoLevelIterator) SeekGE( func (i *twoLevelIterator) SeekPrefixGE( prefix, key []byte, flags base.SeekGEFlags, ) (*base.InternalKey, base.LazyValue) { + if i.vState != nil { + // Callers of SeekGE don't know about virtual sstable bounds, so we may + // have to internally restrict the bounds. + // + // TODO(bananabrick): We can optimize away this check for the level iter + // if necessary. + if i.cmp(key, i.lower) < 0 { + key = i.lower + } + } + // NOTE: prefix is only used for bloom filter checking and not later work in // this method. Hence, we can use the existing iterator position if the last // SeekPrefixGE did not fail bloom filter matching. @@ -1749,16 +2089,17 @@ func (i *twoLevelIterator) SeekPrefixGE( err := i.err i.err = nil // clear cached iteration error - // TODO(sumeer): we are not fully optimizing in the flags.TrySeekUsingNext() - // case when the twoLevelIterator is already exhausted. We will take the - // slow-path below, even though we could return now. We could do: - // filterUsedAndDidNotMatch := - // i.reader.tableFilter != nil && i.useFilter && !i.lastBloomFilterMatched - // if flags.TrySeekUsingNext() && (i.exhaustedBounds == +1 || (i.data.isDataInvalidated() && - // i.index.isDataInvalidated())) && i.err == nil { - // // Already exhausted, so return nil. - // return nil, nil - //} + // The twoLevelIterator could be already exhausted. Utilize that when + // trySeekUsingNext is true. See the comment about data-exhausted, PGDE, and + // bounds-exhausted near the top of the file. + filterUsedAndDidNotMatch := + i.reader.tableFilter != nil && i.useFilter && !i.lastBloomFilterMatched + if flags.TrySeekUsingNext() && !filterUsedAndDidNotMatch && + (i.exhaustedBounds == +1 || (i.data.isDataInvalidated() && i.index.isDataInvalidated())) && + err == nil { + // Already exhausted, so return nil. + return nil, base.LazyValue{} + } // Check prefix bloom filter. if i.reader.tableFilter != nil && i.useFilter { @@ -1768,7 +2109,7 @@ func (i *twoLevelIterator) SeekPrefixGE( } i.lastBloomFilterMatched = false var dataH cache.Handle - dataH, i.err = i.reader.readFilter(i.stats) + dataH, i.err = i.reader.readFilter(i.ctx, i.stats) if i.err != nil { i.data.invalidate() return nil, base.LazyValue{} @@ -1830,10 +2171,13 @@ func (i *twoLevelIterator) SeekPrefixGE( // to the next entry in the top level index if upper bound is // already exceeded. Note that the next entry starts with keys >= // ikey.UserKey since even though this is the block separator, the - // same user key can span multiple index blocks. Since upper is - // exclusive we use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 + // same user key can span multiple index blocks. If upper is + // exclusive we use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + } } // Fall through to skipForward. dontSeekWithinSingleLevelIter = true @@ -1900,12 +2244,54 @@ func (i *twoLevelIterator) SeekPrefixGE( return i.skipForward() } +// virtualLast should only be called if i.vReader != nil and i.endKeyInclusive +// is true. +func (i *twoLevelIterator) virtualLast() (*InternalKey, base.LazyValue) { + if i.vState == nil { + panic("pebble: invalid call to virtualLast") + } + + // Seek to the first internal key. + ikey, _ := i.SeekGE(i.upper, base.SeekGEFlagsNone) + if i.endKeyInclusive { + // Let's say the virtual sstable upper bound is c#1, with the keys c#3, c#2, + // c#1, d, e, ... in the sstable. So, the last key in the virtual sstable is + // c#1. We can perform SeekGE(i.upper) and then keep nexting until we find + // the last key with userkey == i.upper. + // + // TODO(bananabrick): Think about how to improve this. If many internal keys + // with the same user key at the upper bound then this could be slow, but + // maybe the odds of having many internal keys with the same user key at the + // upper bound are low. + for ikey != nil && i.cmp(ikey.UserKey, i.upper) == 0 { + ikey, _ = i.Next() + } + return i.Prev() + } + // We seeked to the first key >= i.upper. + return i.Prev() +} + // SeekLT implements internalIterator.SeekLT, as documented in the pebble // package. Note that SeekLT only checks the lower bound. It is up to the // caller to ensure that key is less than the upper bound. func (i *twoLevelIterator) SeekLT( key []byte, flags base.SeekLTFlags, ) (*InternalKey, base.LazyValue) { + if i.vState != nil { + // Might have to fix upper bound since virtual sstable bounds are not + // known to callers of SeekLT. + // + // TODO(bananabrick): We can optimize away this check for the level iter + // if necessary. + cmp := i.cmp(key, i.upper) + // key == i.upper is fine. We'll do the right thing and return the + // first internal key with user key < key. + if cmp > 0 { + return i.virtualLast() + } + } + i.exhaustedBounds = 0 i.err = nil // clear cached iteration error // Seek optimization only applies until iterator is first positioned after SetBounds. @@ -1975,6 +2361,13 @@ func (i *twoLevelIterator) SeekLT( // to ensure that key is greater than or equal to the lower bound (e.g. via a // call to SeekGE(lower)). func (i *twoLevelIterator) First() (*InternalKey, base.LazyValue) { + // If the iterator was created on a virtual sstable, we will SeekGE to the + // lower bound instead of using First, because First does not respect + // bounds. + if i.vState != nil { + return i.SeekGE(i.lower, base.SeekGEFlagsNone) + } + if i.lower != nil { panic("twoLevelIterator.First() used despite lower bound") } @@ -2004,9 +2397,12 @@ func (i *twoLevelIterator) First() (*InternalKey, base.LazyValue) { // index if upper bound is already exceeded. Note that the next entry // starts with keys >= ikey.UserKey since even though this is the // block separator, the same user key can span multiple index blocks. - // Since upper is exclusive we use >= below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 + // If upper is exclusive we use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + } } } // NB: skipForward checks whether exhaustedBounds is already +1. @@ -2018,6 +2414,13 @@ func (i *twoLevelIterator) First() (*InternalKey, base.LazyValue) { // to ensure that key is less than the upper bound (e.g. via a call to // SeekLT(upper)) func (i *twoLevelIterator) Last() (*InternalKey, base.LazyValue) { + if i.vState != nil { + if i.endKeyInclusive { + return i.virtualLast() + } + return i.SeekLT(i.upper, base.SeekLTFlagsNone) + } + if i.upper != nil { panic("twoLevelIterator.Last() used despite upper bound") } @@ -2073,6 +2476,50 @@ func (i *twoLevelIterator) Next() (*InternalKey, base.LazyValue) { return i.skipForward() } +// NextPrefix implements (base.InternalIterator).NextPrefix. +func (i *twoLevelIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + if i.exhaustedBounds == +1 { + panic("Next called even though exhausted upper bound") + } + // Seek optimization only applies until iterator is first positioned after SetBounds. + i.boundsCmp = 0 + i.maybeFilteredKeysTwoLevel = false + if i.err != nil { + return nil, base.LazyValue{} + } + if key, val := i.singleLevelIterator.NextPrefix(succKey); key != nil { + return key, val + } + // Did not find prefix in the existing second-level index block. This is the + // slow-path where we seek the iterator. + var ikey *InternalKey + if ikey, _ = i.topLevelIndex.SeekGE(succKey, base.SeekGEFlagsNone); ikey == nil { + i.data.invalidate() + i.index.invalidate() + return nil, base.LazyValue{} + } + result := i.loadIndex(+1) + if result == loadBlockFailed { + return nil, base.LazyValue{} + } + if result == loadBlockIrrelevant { + // Enforce the upper bound here since don't want to bother moving to the + // next entry in the top level index if upper bound is already exceeded. + // Note that the next entry starts with keys >= ikey.UserKey since even + // though this is the block separator, the same user key can span multiple + // index blocks. If upper is exclusive we use >= below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + } + } + } else if key, val := i.singleLevelIterator.SeekGE(succKey, base.SeekGEFlagsNone); key != nil { + return key, val + } + return i.skipForward() +} + // Prev implements internalIterator.Prev, as documented in the pebble // package. func (i *twoLevelIterator) Prev() (*InternalKey, base.LazyValue) { @@ -2116,11 +2563,14 @@ func (i *twoLevelIterator) skipForward() (*InternalKey, base.LazyValue) { // level index if upper bound is already exceeded. Note that the // next entry starts with keys >= ikey.UserKey since even though // this is the block separator, the same user key can span - // multiple index blocks. Since upper is exclusive we use >= - // below. - if i.upper != nil && i.cmp(ikey.UserKey, i.upper) >= 0 { - i.exhaustedBounds = +1 - // Next iteration will return. + // multiple index blocks. If upper is exclusive we use >= + // below, else we use >. + if i.upper != nil { + cmp := i.cmp(ikey.UserKey, i.upper) + if (!i.endKeyInclusive && cmp >= 0) || cmp > 0 { + i.exhaustedBounds = +1 + // Next iteration will return. + } } } } @@ -2173,14 +2623,21 @@ func (i *twoLevelIterator) Close() error { err = firstError(err, i.data.Close()) err = firstError(err, i.index.Close()) err = firstError(err, i.topLevelIndex.Close()) - if i.dataRS.sequentialFile != nil { - err = firstError(err, i.dataRS.sequentialFile.Close()) - i.dataRS.sequentialFile = nil + if i.dataRH != nil { + err = firstError(err, i.dataRH.Close()) + i.dataRH = nil } err = firstError(err, i.err) if i.bpfs != nil { releaseBlockPropertiesFilterer(i.bpfs) } + if i.vbReader != nil { + i.vbReader.close() + } + if i.vbRH != nil { + err = firstError(err, i.vbRH.Close()) + i.vbRH = nil + } *i = twoLevelIterator{ singleLevelIterator: i.singleLevelIterator.resetForReuse(), topLevelIndex: i.topLevelIndex.resetForReuse(), @@ -2240,11 +2697,18 @@ func (i *twoLevelCompactionIterator) Next() (*InternalKey, base.LazyValue) { return i.skipForward(i.singleLevelIterator.Next()) } +func (i *twoLevelCompactionIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyValue) { + panic("pebble: NextPrefix unimplemented") +} + func (i *twoLevelCompactionIterator) Prev() (*InternalKey, base.LazyValue) { panic("pebble: Prev unimplemented") } func (i *twoLevelCompactionIterator) String() string { + if i.vState != nil { + return i.vState.fileNum.String() + } return i.reader.fileNum.String() } @@ -2283,203 +2747,19 @@ func (i *twoLevelCompactionIterator) skipForward( curOffset := i.recordOffset() *i.bytesIterated += uint64(curOffset - i.prevOffset) i.prevOffset = curOffset + + if i.vState != nil && key != nil { + cmp := i.cmp(key.UserKey, i.vState.upper.UserKey) + if cmp > 0 || (i.vState.upper.IsExclusiveSentinel() && cmp == 0) { + return nil, base.LazyValue{} + } + } + return key, val } type blockTransform func([]byte) ([]byte, error) -// readaheadState contains state variables related to readahead. Updated on -// file reads. -type readaheadState struct { - // Number of sequential reads. - numReads int64 - // Size issued to the next call to Prefetch. Starts at or above - // initialReadaheadSize and grows exponentially until maxReadaheadSize. - size int64 - // prevSize is the size used in the last Prefetch call. - prevSize int64 - // The byte offset up to which the OS has been asked to read ahead / cached. - // When reading ahead, reads up to this limit should not incur an IO - // operation. Reads after this limit can benefit from a new call to - // Prefetch. - limit int64 - // sequentialFile holds a file descriptor to the same underlying File, - // except with fadvise(FADV_SEQUENTIAL) called on it to take advantage of - // OS-level readahead. Initialized when the iterator has been consistently - // reading blocks in a sequential access pattern. Once this is non-nil, - // the other variables in readaheadState don't matter much as we defer - // to OS-level readahead. - sequentialFile vfs.File -} - -func (rs *readaheadState) recordCacheHit(offset, blockLength int64) { - currentReadEnd := offset + blockLength - if rs.sequentialFile != nil { - // Using OS-level readahead instead, so do nothing. - return - } - if rs.numReads >= minFileReadsForReadahead { - if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { - // This is a read that would have resulted in a readahead, had it - // not been a cache hit. - rs.limit = currentReadEnd - return - } - if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize { - // We read too far away from rs.limit to benefit from readahead in - // any scenario. Reset all variables. - rs.numReads = 1 - rs.limit = currentReadEnd - rs.size = initialReadaheadSize - rs.prevSize = 0 - return - } - // Reads in the range [rs.limit - rs.prevSize, rs.limit] end up - // here. This is a read that is potentially benefitting from a past - // readahead. - return - } - if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { - // Blocks are being read sequentially and would benefit from readahead - // down the line. - rs.numReads++ - return - } - // We read too far ahead of the last read, or before it. This indicates - // a random read, where readahead is not desirable. Reset all variables. - rs.numReads = 1 - rs.limit = currentReadEnd - rs.size = initialReadaheadSize - rs.prevSize = 0 -} - -// maybeReadahead updates state and determines whether to issue a readahead / -// prefetch call for a block read at offset for blockLength bytes. -// Returns a size value (greater than 0) that should be prefetched if readahead -// would be beneficial. -func (rs *readaheadState) maybeReadahead(offset, blockLength int64) int64 { - currentReadEnd := offset + blockLength - if rs.sequentialFile != nil { - // Using OS-level readahead instead, so do nothing. - return 0 - } - if rs.numReads >= minFileReadsForReadahead { - // The minimum threshold of sequential reads to justify reading ahead - // has been reached. - // There are two intervals: the interval being read: - // [offset, currentReadEnd] - // as well as the interval where a read would benefit from read ahead: - // [rs.limit, rs.limit + rs.size] - // We increase the latter interval to - // [rs.limit, rs.limit + maxReadaheadSize] to account for cases where - // readahead may not be beneficial with a small readahead size, but over - // time the readahead size would increase exponentially to make it - // beneficial. - if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { - // We are doing a read in the interval ahead of - // the last readahead range. In the diagrams below, ++++ is the last - // readahead range, ==== is the range represented by - // [rs.limit, rs.limit + maxReadaheadSize], and ---- is the range - // being read. - // - // rs.limit rs.limit + maxReadaheadSize - // ++++++++++|===========================| - // - // |-------------| - // offset currentReadEnd - // - // This case is also possible, as are all cases with an overlap - // between [rs.limit, rs.limit + maxReadaheadSize] and [offset, - // currentReadEnd]: - // - // rs.limit rs.limit + maxReadaheadSize - // ++++++++++|===========================| - // - // |-------------| - // offset currentReadEnd - // - // - rs.numReads++ - rs.limit = offset + rs.size - rs.prevSize = rs.size - // Increase rs.size for the next read. - rs.size *= 2 - if rs.size > maxReadaheadSize { - rs.size = maxReadaheadSize - } - return rs.prevSize - } - if currentReadEnd < rs.limit-rs.prevSize || offset > rs.limit+maxReadaheadSize { - // The above conditional has rs.limit > rs.prevSize to confirm that - // rs.limit - rs.prevSize would not underflow. - // We read too far away from rs.limit to benefit from readahead in - // any scenario. Reset all variables. - // The case where we read too far ahead: - // - // (rs.limit - rs.prevSize) (rs.limit) (rs.limit + maxReadaheadSize) - // |+++++++++++++|=============| - // - // |-------------| - // offset currentReadEnd - // - // Or too far behind: - // - // (rs.limit - rs.prevSize) (rs.limit) (rs.limit + maxReadaheadSize) - // |+++++++++++++|=============| - // - // |-------------| - // offset currentReadEnd - // - rs.numReads = 1 - rs.limit = currentReadEnd - rs.size = initialReadaheadSize - rs.prevSize = 0 - return 0 - } - // Reads in the range [rs.limit - rs.prevSize, rs.limit] end up - // here. This is a read that is potentially benefitting from a past - // readahead, but there's no reason to issue a readahead call at the - // moment. - // - // (rs.limit - rs.prevSize) (rs.limit + maxReadaheadSize) - // |+++++++++++++|===============| - // (rs.limit) - // - // |-------| - // offset currentReadEnd - // - rs.numReads++ - return 0 - } - if currentReadEnd >= rs.limit && offset <= rs.limit+maxReadaheadSize { - // Blocks are being read sequentially and would benefit from readahead - // down the line. - // - // (rs.limit) (rs.limit + maxReadaheadSize) - // |=============| - // - // |-------| - // offset currentReadEnd - // - rs.numReads++ - return 0 - } - // We read too far ahead of the last read, or before it. This indicates - // a random read, where readahead is not desirable. Reset all variables. - // - // (rs.limit - maxReadaheadSize) (rs.limit) (rs.limit + maxReadaheadSize) - // |+++++++++++++|=============| - // - // |-------| - // offset currentReadEnd - // - rs.numReads = 1 - rs.limit = currentReadEnd - rs.size = initialReadaheadSize - rs.prevSize = 0 - return 0 -} - // ReaderOption provide an interface to do work on Reader while it is being // opened. type ReaderOption interface { @@ -2522,7 +2802,7 @@ func (m Mergers) readerApply(r *Reader) { // number. If not specified, a unique cache ID will be used. type cacheOpts struct { cacheID uint64 - fileNum base.FileNum + fileNum base.DiskFileNum } // Marker function to indicate the option should be applied before reading the @@ -2534,7 +2814,7 @@ func (c *cacheOpts) readerApply(r *Reader) { if r.cacheID == 0 { r.cacheID = c.cacheID } - if r.fileNum == 0 { + if r.fileNum.FileNum() == 0 { r.fileNum = c.fileNum } } @@ -2543,25 +2823,11 @@ func (c *cacheOpts) writerApply(w *Writer) { if w.cacheID == 0 { w.cacheID = c.cacheID } - if w.fileNum == 0 { + if w.fileNum.FileNum() == 0 { w.fileNum = c.fileNum } } -// FileReopenOpt is specified if this reader is allowed to reopen additional -// file descriptors for this file. Used to take advantage of OS-level readahead. -type FileReopenOpt struct { - FS vfs.FS - Filename string -} - -func (f FileReopenOpt) readerApply(r *Reader) { - if r.fs == nil { - r.fs = f.FS - r.filename = f.Filename - } -} - // rawTombstonesOpt is a Reader open option for specifying that range // tombstones returned by Reader.NewRangeDelIter() should not be // fragmented. Used by debug tools to get a raw view of the tombstones @@ -2575,26 +2841,179 @@ func (rawTombstonesOpt) readerApply(r *Reader) { } func init() { - private.SSTableCacheOpts = func(cacheID uint64, fileNum base.FileNum) interface{} { + private.SSTableCacheOpts = func(cacheID uint64, fileNum base.DiskFileNum) interface{} { return &cacheOpts{cacheID, fileNum} } private.SSTableRawTombstonesOpt = rawTombstonesOpt{} } +// VirtualReader wraps Reader. Its purpose is to restrict functionality of the +// Reader which should be inaccessible to virtual sstables, and enforce bounds +// invariants associated with virtual sstables. All reads on virtual sstables +// should go through a VirtualReader. +// +// INVARIANT: Any iterators created through a virtual reader will guarantee that +// they don't expose keys outside the virtual sstable bounds. +type VirtualReader struct { + vState virtualState + reader *Reader + Properties struct { + // RawKeySize, RawValueSize are set upon construction of a + // VirtualReader. The values of the fields is extrapolated. See + // MakeVirtualReader for implementation details. + RawKeySize uint64 + RawValueSize uint64 + } +} + +// Lightweight virtual sstable state which can be passed to sstable iterators. +type virtualState struct { + lower InternalKey + upper InternalKey + fileNum base.FileNum + Compare Compare +} + +// MakeVirtualReader is used to contruct a reader which can read from virtual +// sstables. +func MakeVirtualReader(reader *Reader, meta manifest.VirtualFileMeta) VirtualReader { + if reader.fileNum != meta.FileBacking.DiskFileNum { + panic("pebble: invalid call to MakeVirtualReader") + } + + vState := virtualState{ + lower: meta.Smallest, + upper: meta.Largest, + fileNum: meta.FileNum, + Compare: reader.Compare, + } + v := VirtualReader{ + vState: vState, + reader: reader, + } + + v.Properties.RawKeySize = + (reader.Properties.RawKeySize * meta.Size) / meta.FileBacking.Size + v.Properties.RawValueSize = + (reader.Properties.RawValueSize * meta.Size) / meta.FileBacking.Size + + return v +} + +// NewCompactionIter is the compaction iterator function for virtual readers. +func (v *VirtualReader) NewCompactionIter( + bytesIterated *uint64, rp ReaderProvider, +) (Iterator, error) { + return v.reader.newCompactionIter(bytesIterated, rp, &v.vState) +} + +// NewIterWithBlockPropertyFiltersAndContext wraps +// Reader.NewIterWithBlockPropertyFiltersAndContext. We assume that the passed +// in [lower, upper) bounds will have at least some overlap with the virtual +// sstable bounds. No overlap is not currently supported in the iterator. +func (v *VirtualReader) NewIterWithBlockPropertyFiltersAndContext( + ctx context.Context, + lower, upper []byte, + filterer *BlockPropertiesFilterer, + useFilterBlock bool, + stats *base.InternalIteratorStats, + rp ReaderProvider, +) (Iterator, error) { + return v.reader.newIterWithBlockPropertyFiltersAndContext( + ctx, + lower, upper, filterer, useFilterBlock, stats, rp, &v.vState, + ) +} + +// NewRawRangeDelIter wraps Reader.NewRawRangeDelIter. +func (v *VirtualReader) NewRawRangeDelIter() (keyspan.FragmentIterator, error) { + iter, err := v.reader.NewRawRangeDelIter() + if err != nil { + return nil, err + } + if iter == nil { + return nil, nil + } + + // There should be no spans which cross virtual sstable bounds. So, no + // truncation should occur. + return keyspan.Truncate( + v.reader.Compare, iter, v.vState.lower.UserKey, v.vState.upper.UserKey, + &v.vState.lower, &v.vState.upper, true, /* panicOnPartialOverlap */ + ), nil +} + +// NewRawRangeKeyIter wraps Reader.NewRawRangeKeyIter. +func (v *VirtualReader) NewRawRangeKeyIter() (keyspan.FragmentIterator, error) { + iter, err := v.reader.NewRawRangeKeyIter() + if err != nil { + return nil, err + } + if iter == nil { + return nil, nil + } + + // There should be no spans which cross virtual sstable bounds. So, no + // truncation should occur. + return keyspan.Truncate( + v.reader.Compare, iter, v.vState.lower.UserKey, v.vState.upper.UserKey, + &v.vState.lower, &v.vState.upper, true, /* panicOnPartialOverlap */ + ), nil +} + +// Constrain bounds will narrow the start, end bounds if they do not fit within +// the virtual sstable. The function will return if the new end key is +// inclusive. +func (v *virtualState) constrainBounds( + start, end []byte, endInclusive bool, +) (lastKeyInclusive bool, first []byte, last []byte) { + first = start + if start == nil || v.Compare(start, v.lower.UserKey) < 0 { + first = v.lower.UserKey + } + + // Note that we assume that start, end has some overlap with the virtual + // sstable bounds. + last = v.upper.UserKey + lastKeyInclusive = !v.upper.IsExclusiveSentinel() + if end != nil { + cmp := v.Compare(end, v.upper.UserKey) + switch { + case cmp == 0: + lastKeyInclusive = !v.upper.IsExclusiveSentinel() && endInclusive + last = v.upper.UserKey + case cmp > 0: + lastKeyInclusive = !v.upper.IsExclusiveSentinel() + last = v.upper.UserKey + default: + lastKeyInclusive = endInclusive + last = end + } + } + // TODO(bananabrick): What if someone passes in bounds completely outside of + // virtual sstable bounds? + return lastKeyInclusive, first, last +} + +// EstimateDiskUsage just calls VirtualReader.reader.EstimateDiskUsage after +// enforcing the virtual sstable bounds. +func (v *VirtualReader) EstimateDiskUsage(start, end []byte) (uint64, error) { + _, f, l := v.vState.constrainBounds(start, end, true /* endInclusive */) + return v.reader.EstimateDiskUsage(f, l) +} + // Reader is a table reader. type Reader struct { - file ReadableFile - fs vfs.FS - filename string + readable objstorage.Readable cacheID uint64 - fileNum base.FileNum - rawTombstones bool + fileNum base.DiskFileNum err error indexBH BlockHandle filterBH BlockHandle rangeDelBH BlockHandle rangeKeyBH BlockHandle rangeDelTransform blockTransform + valueBIH valueBlocksIndexHandle propertiesBH BlockHandle metaIndexBH BlockHandle footerBH BlockHandle @@ -2602,30 +3021,27 @@ type Reader struct { Compare Compare FormatKey base.FormatKey Split Split - mergerOK bool - checksumType ChecksumType tableFilter *tableFilterReader - tableFormat TableFormat - Properties Properties + // Keep types that are not multiples of 8 bytes at the end and with + // decreasing size. + Properties Properties + tableFormat TableFormat + rawTombstones bool + mergerOK bool + checksumType ChecksumType } // Close implements DB.Close, as documented in the pebble package. func (r *Reader) Close() error { r.opts.Cache.Unref() - if r.err != nil { - if r.file != nil { - r.file.Close() - r.file = nil - } - return r.err + if r.readable != nil { + r.err = firstError(r.err, r.readable.Close()) + r.readable = nil } - if r.file != nil { - r.err = r.file.Close() - r.file = nil - if r.err != nil { - return r.err - } + + if r.err != nil { + return r.err } // Make any future calls to Get, NewIter or Close return an error. r.err = errReaderClosed @@ -2640,13 +3056,46 @@ func (r *Reader) NewIterWithBlockPropertyFilters( filterer *BlockPropertiesFilterer, useFilterBlock bool, stats *base.InternalIteratorStats, + rp ReaderProvider, +) (Iterator, error) { + return r.newIterWithBlockPropertyFiltersAndContext( + context.Background(), + lower, upper, filterer, useFilterBlock, stats, rp, nil, + ) +} + +// NewIterWithBlockPropertyFiltersAndContext is similar to +// NewIterWithBlockPropertyFilters and additionally accepts a context for +// tracing. +func (r *Reader) NewIterWithBlockPropertyFiltersAndContext( + ctx context.Context, + lower, upper []byte, + filterer *BlockPropertiesFilterer, + useFilterBlock bool, + stats *base.InternalIteratorStats, + rp ReaderProvider, +) (Iterator, error) { + return r.newIterWithBlockPropertyFiltersAndContext( + ctx, + lower, upper, filterer, useFilterBlock, stats, rp, nil, + ) +} + +func (r *Reader) newIterWithBlockPropertyFiltersAndContext( + ctx context.Context, + lower, upper []byte, + filterer *BlockPropertiesFilterer, + useFilterBlock bool, + stats *base.InternalIteratorStats, + rp ReaderProvider, + v *virtualState, ) (Iterator, error) { // NB: pebble.tableCache wraps the returned iterator with one which performs // reference counting on the Reader, preventing the Reader from being closed // until the final iterator closes. if r.Properties.IndexType == twoLevelIndex { i := twoLevelIterPool.Get().(*twoLevelIterator) - err := i.init(r, lower, upper, filterer, useFilterBlock, stats) + err := i.init(ctx, r, v, lower, upper, filterer, useFilterBlock, stats, rp) if err != nil { return nil, err } @@ -2654,7 +3103,7 @@ func (r *Reader) NewIterWithBlockPropertyFilters( } i := singleLevelIterPool.Get().(*singleLevelIterator) - err := i.init(r, lower, upper, filterer, useFilterBlock, stats) + err := i.init(ctx, r, v, lower, upper, filterer, useFilterBlock, stats, rp) if err != nil { return nil, err } @@ -2662,18 +3111,32 @@ func (r *Reader) NewIterWithBlockPropertyFilters( } // NewIter returns an iterator for the contents of the table. If an error -// occurs, NewIter cleans up after itself and returns a nil iterator. +// occurs, NewIter cleans up after itself and returns a nil iterator. NewIter +// must only be used when the Reader is guaranteed to outlive any LazyValues +// returned from the iter. func (r *Reader) NewIter(lower, upper []byte) (Iterator, error) { - return r.NewIterWithBlockPropertyFilters(lower, upper, nil, true /* useFilterBlock */, nil /* stats */) + return r.NewIterWithBlockPropertyFilters( + lower, upper, nil, true /* useFilterBlock */, nil, /* stats */ + TrivialReaderProvider{Reader: r}) } // NewCompactionIter returns an iterator similar to NewIter but it also increments // the number of bytes iterated. If an error occurs, NewCompactionIter cleans up // after itself and returns a nil iterator. -func (r *Reader) NewCompactionIter(bytesIterated *uint64) (Iterator, error) { +func (r *Reader) NewCompactionIter(bytesIterated *uint64, rp ReaderProvider) (Iterator, error) { + return r.newCompactionIter(bytesIterated, rp, nil) +} + +func (r *Reader) newCompactionIter( + bytesIterated *uint64, rp ReaderProvider, v *virtualState, +) (Iterator, error) { if r.Properties.IndexType == twoLevelIndex { i := twoLevelIterPool.Get().(*twoLevelIterator) - err := i.init(r, nil /* lower */, nil /* upper */, nil, false /* useFilter */, nil /* stats */) + err := i.init( + context.Background(), + r, v, nil /* lower */, nil /* upper */, nil, + false /* useFilter */, nil /* stats */, rp, + ) if err != nil { return nil, err } @@ -2684,7 +3147,10 @@ func (r *Reader) NewCompactionIter(bytesIterated *uint64) (Iterator, error) { }, nil } i := singleLevelIterPool.Get().(*singleLevelIterator) - err := i.init(r, nil /* lower */, nil /* upper */, nil, false /* useFilter */, nil /* stats */) + err := i.init( + context.Background(), r, v, nil /* lower */, nil, /* upper */ + nil, false /* useFilter */, nil /* stats */, rp, + ) if err != nil { return nil, err } @@ -2698,6 +3164,9 @@ func (r *Reader) NewCompactionIter(bytesIterated *uint64) (Iterator, error) { // NewRawRangeDelIter returns an internal iterator for the contents of the // range-del block for the table. Returns nil if the table does not contain // any range deletions. +// +// TODO(sumeer): plumb context.Context since this path is relevant in the user-facing +// iterator. Add WithContext methods since the existing ones are public. func (r *Reader) NewRawRangeDelIter() (keyspan.FragmentIterator, error) { if r.rangeDelBH.Length == 0 { return nil, nil @@ -2716,6 +3185,9 @@ func (r *Reader) NewRawRangeDelIter() (keyspan.FragmentIterator, error) { // NewRawRangeKeyIter returns an internal iterator for the contents of the // range-key block for the table. Returns nil if the table does not contain any // range keys. +// +// TODO(sumeer): plumb context.Context since this path is relevant in the user-facing +// iterator. Add WithContext methods since the existing ones are public. func (r *Reader) NewRawRangeKeyIter() (keyspan.FragmentIterator, error) { if r.rangeKeyBH.Length == 0 { return nil, nil @@ -2742,20 +3214,28 @@ func (i *rangeKeyFragmentBlockIter) Close() error { return err } -func (r *Reader) readIndex(stats *base.InternalIteratorStats) (cache.Handle, error) { - return r.readBlock(r.indexBH, nil /* transform */, nil /* readaheadState */, stats) +func (r *Reader) readIndex( + ctx context.Context, stats *base.InternalIteratorStats, +) (cache.Handle, error) { + ctx = objiotracing.WithBlockType(ctx, objiotracing.MetadataBlock) + return r.readBlock(ctx, r.indexBH, nil, nil, stats) } -func (r *Reader) readFilter(stats *base.InternalIteratorStats) (cache.Handle, error) { - return r.readBlock(r.filterBH, nil /* transform */, nil /* readaheadState */, stats) +func (r *Reader) readFilter( + ctx context.Context, stats *base.InternalIteratorStats, +) (cache.Handle, error) { + ctx = objiotracing.WithBlockType(ctx, objiotracing.FilterBlock) + return r.readBlock(ctx, r.filterBH, nil /* transform */, nil /* readHandle */, stats) } func (r *Reader) readRangeDel(stats *base.InternalIteratorStats) (cache.Handle, error) { - return r.readBlock(r.rangeDelBH, r.rangeDelTransform, nil /* readaheadState */, stats) + ctx := objiotracing.WithBlockType(context.Background(), objiotracing.MetadataBlock) + return r.readBlock(ctx, r.rangeDelBH, r.rangeDelTransform, nil /* readHandle */, stats) } func (r *Reader) readRangeKey(stats *base.InternalIteratorStats) (cache.Handle, error) { - return r.readBlock(r.rangeKeyBH, nil /* transform */, nil /* readaheadState */, stats) + ctx := objiotracing.WithBlockType(context.Background(), objiotracing.MetadataBlock) + return r.readBlock(ctx, r.rangeKeyBH, nil /* transform */, nil /* readHandle */, stats) } func checkChecksum( @@ -2782,14 +3262,15 @@ func checkChecksum( // readBlock reads and decompresses a block from disk into memory. func (r *Reader) readBlock( + ctx context.Context, bh BlockHandle, transform blockTransform, - raState *readaheadState, + readHandle objstorage.ReadHandle, stats *base.InternalIteratorStats, -) (_ cache.Handle, _ error) { +) (handle cache.Handle, _ error) { if h := r.opts.Cache.Get(r.cacheID, r.fileNum, bh.Offset); h.Get() != nil { - if raState != nil { - raState.recordCacheHit(int64(bh.Offset), int64(bh.Length+blockTrailerLen)) + if readHandle != nil { + readHandle.RecordCacheHit(ctx, int64(bh.Offset), int64(bh.Length+blockTrailerLen)) } if stats != nil { stats.BlockBytes += bh.Length @@ -2797,53 +3278,38 @@ func (r *Reader) readBlock( } return h, nil } - file := r.file - - if raState != nil { - if raState.sequentialFile != nil { - file = raState.sequentialFile - } else if readaheadSize := raState.maybeReadahead(int64(bh.Offset), int64(bh.Length+blockTrailerLen)); readaheadSize > 0 { - if readaheadSize >= maxReadaheadSize { - // We've reached the maximum readahead size. Beyond this - // point, rely on OS-level readahead. Note that we can only - // reopen a new file handle with this optimization if - // r.fs != nil. This reader must have been created with the - // FileReopenOpt for this field to be set. - if r.fs != nil { - f, err := r.fs.Open(r.filename, vfs.SequentialReadsOption) - if err == nil { - // Use this new file handle for all sequential reads by - // this iterator going forward. - raState.sequentialFile = f - file = f - } - - // If we tried to load a table that doesn't exist, panic - // immediately. Something is seriously wrong if a table - // doesn't exist. - // See cockroachdb/cockroach#56490. - base.MustExist(r.fs, r.filename, panicFataler{}, err) - } - } - if raState.sequentialFile == nil { - type fd interface { - Fd() uintptr - } - if f, ok := r.file.(fd); ok { - _ = vfs.Prefetch(f.Fd(), bh.Offset, uint64(readaheadSize)) - } - } - } - } v := r.opts.Cache.Alloc(int(bh.Length + blockTrailerLen)) b := v.Buf() - if _, err := file.ReadAt(b, int64(bh.Offset)); err != nil { + readStartTime := time.Now() + var err error + if readHandle != nil { + err = readHandle.ReadAt(ctx, b, int64(bh.Offset)) + } else { + err = r.readable.ReadAt(ctx, b, int64(bh.Offset)) + } + readDuration := time.Since(readStartTime) + // TODO(sumeer): should the threshold be configurable. + const slowReadTracingThreshold = 5 * time.Millisecond + // The invariants.Enabled path is for deterministic testing. + if invariants.Enabled { + readDuration = slowReadTracingThreshold + } + // Call IsTracingEnabled to avoid the allocations of boxing integers into an + // interface{}, unless necessary. + if readDuration >= slowReadTracingThreshold && r.opts.LoggerAndTracer.IsTracingEnabled(ctx) { + r.opts.LoggerAndTracer.Eventf(ctx, "reading %d bytes took %s", + bh.Length+blockTrailerLen, readDuration.String()) + } + if stats != nil { + stats.BlockReadDuration += readDuration + } + if err != nil { r.opts.Cache.Free(v) return cache.Handle{}, err } - if err := checkChecksum(r.checksumType, b, bh, r.fileNum); err != nil { + if err := checkChecksum(r.checksumType, b, bh, r.fileNum.FileNum()); err != nil { r.opts.Cache.Free(v) return cache.Handle{}, err } @@ -2929,7 +3395,8 @@ func (r *Reader) transformRangeDelV1(b []byte) ([]byte, error) { } func (r *Reader) readMetaindex(metaindexBH BlockHandle) error { - b, err := r.readBlock(metaindexBH, nil /* transform */, nil /* readaheadState */, nil /* stats */) + b, err := r.readBlock( + context.Background(), metaindexBH, nil /* transform */, nil /* readHandle */, nil /* stats */) if err != nil { return err } @@ -2948,18 +3415,31 @@ func (r *Reader) readMetaindex(metaindexBH BlockHandle) error { meta := map[string]BlockHandle{} for valid := i.First(); valid; valid = i.Next() { - bh, n := decodeBlockHandle(i.Value()) - if n == 0 { - return base.CorruptionErrorf("pebble/table: invalid table (bad filter block handle)") + value := i.Value() + if bytes.Equal(i.Key().UserKey, []byte(metaValueIndexName)) { + vbih, n, err := decodeValueBlocksIndexHandle(i.Value()) + if err != nil { + return err + } + if n == 0 || n != len(value) { + return base.CorruptionErrorf("pebble/table: invalid table (bad value blocks index handle)") + } + r.valueBIH = vbih + } else { + bh, n := decodeBlockHandle(value) + if n == 0 || n != len(value) { + return base.CorruptionErrorf("pebble/table: invalid table (bad block handle)") + } + meta[string(i.Key().UserKey)] = bh } - meta[string(i.Key().UserKey)] = bh } if err := i.Close(); err != nil { return err } if bh, ok := meta[metaPropertiesName]; ok { - b, err = r.readBlock(bh, nil /* transform */, nil /* readaheadState */, nil /* stats */) + b, err = r.readBlock( + context.Background(), bh, nil /* transform */, nil /* readHandle */, nil /* stats */) if err != nil { return err } @@ -3025,18 +3505,20 @@ func (r *Reader) Layout() (*Layout, error) { Filter: r.filterBH, RangeDel: r.rangeDelBH, RangeKey: r.rangeKeyBH, + ValueIndex: r.valueBIH.h, Properties: r.propertiesBH, MetaIndex: r.metaIndexBH, Footer: r.footerBH, + Format: r.tableFormat, } - indexH, err := r.readIndex(nil /* stats */) + indexH, err := r.readIndex(context.Background(), nil) if err != nil { return nil, err } defer indexH.Release() - var alloc []byte + var alloc bytealloc.A if r.Properties.IndexPartitions == 0 { l.Index = append(l.Index, r.indexBH) @@ -3047,12 +3529,7 @@ func (r *Reader) Layout() (*Layout, error) { return nil, errCorruptIndexEntry } if len(dataBH.Props) > 0 { - if len(alloc) < len(dataBH.Props) { - alloc = make([]byte, 256<<10) - } - n := copy(alloc, dataBH.Props) - dataBH.Props = alloc[:n:n] - alloc = alloc[n:] + alloc, dataBH.Props = alloc.Copy(dataBH.Props) } l.Data = append(l.Data, dataBH) } @@ -3067,8 +3544,8 @@ func (r *Reader) Layout() (*Layout, error) { } l.Index = append(l.Index, indexBH.BlockHandle) - subIndex, err := r.readBlock( - indexBH.BlockHandle, nil /* transform */, nil /* readaheadState */, nil /* stats */) + subIndex, err := r.readBlock(context.Background(), + indexBH.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */) if err != nil { return nil, err } @@ -3078,12 +3555,7 @@ func (r *Reader) Layout() (*Layout, error) { for key, value := iter.First(); key != nil; key, value = iter.Next() { dataBH, err := decodeBlockHandleWithProperties(value.InPlaceValue()) if len(dataBH.Props) > 0 { - if len(alloc) < len(dataBH.Props) { - alloc = make([]byte, 256<<10) - } - n := copy(alloc, dataBH.Props) - dataBH.Props = alloc[:n:n] - alloc = alloc[n:] + alloc, dataBH.Props = alloc.Copy(dataBH.Props) } if err != nil { return nil, errCorruptIndexEntry @@ -3094,6 +3566,39 @@ func (r *Reader) Layout() (*Layout, error) { *iter = iter.resetForReuse() } } + if r.valueBIH.h.Length != 0 { + vbiH, err := r.readBlock(context.Background(), r.valueBIH.h, nil, nil, nil) + if err != nil { + return nil, err + } + defer vbiH.Release() + vbiBlock := vbiH.Get() + indexEntryLen := int(r.valueBIH.blockNumByteLength + r.valueBIH.blockOffsetByteLength + + r.valueBIH.blockLengthByteLength) + i := 0 + for len(vbiBlock) != 0 { + if len(vbiBlock) < indexEntryLen { + return nil, errors.Errorf( + "remaining value index block %d does not contain a full entry of length %d", + len(vbiBlock), indexEntryLen) + } + n := int(r.valueBIH.blockNumByteLength) + bn := int(littleEndianGet(vbiBlock, n)) + if bn != i { + return nil, errors.Errorf("unexpected block num %d, expected %d", + bn, i) + } + i++ + vbiBlock = vbiBlock[n:] + n = int(r.valueBIH.blockOffsetByteLength) + blockOffset := littleEndianGet(vbiBlock, n) + vbiBlock = vbiBlock[n:] + n = int(r.valueBIH.blockLengthByteLength) + blockLen := littleEndianGet(vbiBlock, n) + vbiBlock = vbiBlock[n:] + l.ValueBlock = append(l.ValueBlock, BlockHandle{Offset: blockOffset, Length: blockLen}) + } + } return l, nil } @@ -3123,9 +3628,9 @@ func (r *Reader) ValidateBlockChecksums() error { // Check all blocks sequentially. Make use of read-ahead, given we are // scanning the entire file from start to end. - blockRS := &readaheadState{ - size: initialReadaheadSize, - } + rh := r.readable.NewReadHandle(context.TODO()) + defer rh.Close() + for _, bh := range blocks { // Certain blocks may not be present, in which case we skip them. if bh.Length == 0 { @@ -3133,7 +3638,7 @@ func (r *Reader) ValidateBlockChecksums() error { } // Read the block, which validates the checksum. - h, err := r.readBlock(bh, nil /* transform */, blockRS, nil /* stats */) + h, err := r.readBlock(context.Background(), bh, nil, rh, nil) if err != nil { return err } @@ -3163,7 +3668,7 @@ func (r *Reader) EstimateDiskUsage(start, end []byte) (uint64, error) { return 0, r.err } - indexH, err := r.readIndex(nil /* stats */) + indexH, err := r.readIndex(context.Background(), nil) if err != nil { return 0, err } @@ -3195,8 +3700,8 @@ func (r *Reader) EstimateDiskUsage(start, end []byte) (uint64, error) { if err != nil { return 0, errCorruptIndexEntry } - startIdxBlock, err := r.readBlock( - startIdxBH.BlockHandle, nil /* transform */, nil /* readaheadState */, nil /* stats */) + startIdxBlock, err := r.readBlock(context.Background(), + startIdxBH.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */) if err != nil { return 0, err } @@ -3216,8 +3721,8 @@ func (r *Reader) EstimateDiskUsage(start, end []byte) (uint64, error) { if err != nil { return 0, errCorruptIndexEntry } - endIdxBlock, err := r.readBlock( - endIdxBH.BlockHandle, nil /* transform */, nil /* readaheadState */, nil /* stats */) + endIdxBlock, err := r.readBlock(context.Background(), + endIdxBH.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */) if err != nil { return 0, err } @@ -3241,9 +3746,23 @@ func (r *Reader) EstimateDiskUsage(start, end []byte) (uint64, error) { return 0, errCorruptIndexEntry } + includeInterpolatedValueBlocksSize := func(dataBlockSize uint64) uint64 { + // INVARIANT: r.Properties.DataSize > 0 since startIdxIter is not nil. + // Linearly interpolate what is stored in value blocks. + // + // TODO(sumeer): if we need more accuracy, without loading any data blocks + // (which contain the value handles, and which may also be insufficient if + // the values are in separate files), we will need to accumulate the + // logical size of the key-value pairs and store the cumulative value for + // each data block in the index block entry. This increases the size of + // the BlockHandle, so wait until this becomes necessary. + return dataBlockSize + + uint64((float64(dataBlockSize)/float64(r.Properties.DataSize))* + float64(r.Properties.ValueBlocksSize)) + } if endIdxIter == nil { // The range spans beyond this file. Include data blocks through the last. - return r.Properties.DataSize - startBH.Offset, nil + return includeInterpolatedValueBlocksSize(r.Properties.DataSize - startBH.Offset), nil } key, val = endIdxIter.SeekGE(end, base.SeekGEFlagsNone) if key == nil { @@ -3251,13 +3770,14 @@ func (r *Reader) EstimateDiskUsage(start, end []byte) (uint64, error) { return 0, err } // The range spans beyond this file. Include data blocks through the last. - return r.Properties.DataSize - startBH.Offset, nil + return includeInterpolatedValueBlocksSize(r.Properties.DataSize - startBH.Offset), nil } endBH, err := decodeBlockHandleWithProperties(val.InPlaceValue()) if err != nil { return 0, errCorruptIndexEntry } - return endBH.Offset + endBH.Length + blockTrailerLen - startBH.Offset, nil + return includeInterpolatedValueBlocksSize( + endBH.Offset + endBH.Length + blockTrailerLen - startBH.Offset), nil } // TableFormat returns the format version for the table. @@ -3268,20 +3788,13 @@ func (r *Reader) TableFormat() (TableFormat, error) { return r.tableFormat, nil } -// ReadableFile describes subset of vfs.File required for reading SSTs. -type ReadableFile interface { - io.ReaderAt - io.Closer - Stat() (os.FileInfo, error) -} - // NewReader returns a new table reader for the file. Closing the reader will // close the file. -func NewReader(f ReadableFile, o ReaderOptions, extraOpts ...ReaderOption) (*Reader, error) { +func NewReader(f objstorage.Readable, o ReaderOptions, extraOpts ...ReaderOption) (*Reader, error) { o = o.ensureDefaults() r := &Reader{ - file: f, - opts: o, + readable: f, + opts: o, } if r.opts.Cache == nil { r.opts.Cache = cache.New(0) @@ -3369,9 +3882,12 @@ type Layout struct { Filter BlockHandle RangeDel BlockHandle RangeKey BlockHandle + ValueBlock []BlockHandle + ValueIndex BlockHandle Properties BlockHandle MetaIndex BlockHandle Footer BlockHandle + Format TableFormat } // Describe returns a description of the layout. If the verbose parameter is @@ -3379,6 +3895,7 @@ type Layout struct { func (l *Layout) Describe( w io.Writer, verbose bool, r *Reader, fmtRecord func(key *base.InternalKey, value []byte), ) { + ctx := context.TODO() type block struct { BlockHandle name string @@ -3403,6 +3920,12 @@ func (l *Layout) Describe( if l.RangeKey.Length != 0 { blocks = append(blocks, block{l.RangeKey, "range-key"}) } + for i := range l.ValueBlock { + blocks = append(blocks, block{l.ValueBlock[i], "value-block"}) + } + if l.ValueIndex.Length != 0 { + blocks = append(blocks, block{l.ValueIndex, "value-index"}) + } if l.Properties.Length != 0 { blocks = append(blocks, block{l.Properties, "properties"}) } @@ -3434,7 +3957,7 @@ func (l *Layout) Describe( if b.name == "footer" || b.name == "leveldb-footer" { trailer, offset := make([]byte, b.Length), b.Offset - _, _ = r.file.ReadAt(trailer, int64(offset)) + _ = r.readable.ReadAt(ctx, trailer, int64(offset)) if b.name == "footer" { checksumType := ChecksumType(trailer[0]) @@ -3474,14 +3997,15 @@ func (l *Layout) Describe( continue } - h, err := r.readBlock(b.BlockHandle, nil /* transform */, nil /* readaheadState */, nil /* stats */) + h, err := r.readBlock( + context.Background(), b.BlockHandle, nil /* transform */, nil /* readHandle */, nil /* stats */) if err != nil { fmt.Fprintf(w, " [err: %s]\n", err) continue } getRestart := func(data []byte, restarts, i int32) int32 { - return int32(binary.LittleEndian.Uint32(data[restarts+4*i:])) + return decodeRestart(data[restarts+4*i:]) } formatIsRestart := func(data []byte, restarts, numRestarts, offset int32) { @@ -3506,7 +4030,7 @@ func (l *Layout) Describe( formatTrailer := func() { trailer := make([]byte, blockTrailerLen) offset := int64(b.Offset + b.Length) - _, _ = r.file.ReadAt(trailer, offset) + _ = r.readable.ReadAt(ctx, trailer, offset) bt := blockType(trailer[0]) checksum := binary.LittleEndian.Uint32(trailer[1:]) fmt.Fprintf(w, "%10d [trailer compression=%s checksum=0x%04x]\n", offset, bt, checksum) @@ -3539,12 +4063,23 @@ func (l *Layout) Describe( formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset) if fmtRecord != nil { fmt.Fprintf(w, " ") - // InPlaceValue() will succeed even for data blocks where the actual - // value is in a different location, since this value was fetched - // from blockIter, which only has local knowledge. - // TODO(sumeer): adjust the formatting to account for the difference - // between the real value and a handle to that value. - fmtRecord(key, value.InPlaceValue()) + if l.Format < TableFormatPebblev3 { + fmtRecord(key, value.InPlaceValue()) + } else { + // InPlaceValue() will succeed even for data blocks where the + // actual value is in a different location, since this value was + // fetched from a blockIter which does not know about value + // blocks. + v := value.InPlaceValue() + if base.TrailerKind(key.Trailer) != InternalKeyKindSet { + fmtRecord(key, v) + } else if !isValueHandle(valuePrefix(v[0])) { + fmtRecord(key, v[1:]) + } else { + vh := decodeValueHandle(v[1:]) + fmtRecord(key, []byte(fmt.Sprintf("value handle %+v", vh))) + } + } } if base.InternalCompare(r.Compare, lastKey, *key) >= 0 { @@ -3582,19 +4117,39 @@ func (l *Layout) Describe( iter, _ := newRawBlockIter(r.Compare, h.Get()) for valid := iter.First(); valid; valid = iter.Next() { value := iter.Value() - bh, n := decodeBlockHandle(value) + var bh BlockHandle + var n int + var vbih valueBlocksIndexHandle + isValueBlocksIndexHandle := false + if bytes.Equal(iter.Key().UserKey, []byte(metaValueIndexName)) { + vbih, n, err = decodeValueBlocksIndexHandle(value) + bh = vbih.h + isValueBlocksIndexHandle = true + } else { + bh, n = decodeBlockHandle(value) + } if n == 0 || n != len(value) { fmt.Fprintf(w, "%10d [err: %s]\n", b.Offset+uint64(iter.offset), err) continue } - - fmt.Fprintf(w, "%10d %s block:%d/%d", + var vbihStr string + if isValueBlocksIndexHandle { + vbihStr = fmt.Sprintf(" value-blocks-index-lengths: %d(num), %d(offset), %d(length)", + vbih.blockNumByteLength, vbih.blockOffsetByteLength, vbih.blockLengthByteLength) + } + fmt.Fprintf(w, "%10d %s block:%d/%d%s", b.Offset+uint64(iter.offset), iter.Key().UserKey, - bh.Offset, bh.Length) + bh.Offset, bh.Length, vbihStr) formatIsRestart(iter.data, iter.restarts, iter.numRestarts, iter.offset) } formatRestarts(iter.data, iter.restarts, iter.numRestarts) formatTrailer() + case "value-block": + // We don't peer into the value-block since it can't be interpreted + // without the valueHandles. + case "value-index": + // We have already read the value-index to construct the list of + // value-blocks, so no need to do it again. } h.Release() @@ -3604,8 +4159,58 @@ func (l *Layout) Describe( fmt.Fprintf(w, "%10d EOF\n", last.Offset+last.Length) } -type panicFataler struct{} - -func (panicFataler) Fatalf(format string, args ...interface{}) { - panic(errors.Errorf(format, args...)) +// ReadableFile describes the smallest subset of vfs.File that is required for +// reading SSTs. +type ReadableFile interface { + io.ReaderAt + io.Closer + Stat() (os.FileInfo, error) +} + +// NewSimpleReadable wraps a ReadableFile in a objstorage.Readable +// implementation (which does not support read-ahead) +func NewSimpleReadable(r ReadableFile) (objstorage.Readable, error) { + info, err := r.Stat() + if err != nil { + return nil, err + } + res := &simpleReadable{ + f: r, + size: info.Size(), + } + res.rh = objstorage.MakeNoopReadHandle(res) + return res, nil +} + +// simpleReadable wraps a ReadableFile to implement objstorage.Readable. +type simpleReadable struct { + f ReadableFile + size int64 + rh objstorage.NoopReadHandle +} + +var _ objstorage.Readable = (*simpleReadable)(nil) + +// ReadAt is part of the objstorage.Readable interface. +func (s *simpleReadable) ReadAt(_ context.Context, p []byte, off int64) error { + n, err := s.f.ReadAt(p, off) + if invariants.Enabled && err == nil && n != len(p) { + panic("short read") + } + return err +} + +// Close is part of the objstorage.Readable interface. +func (s *simpleReadable) Close() error { + return s.f.Close() +} + +// Size is part of the objstorage.Readable interface. +func (s *simpleReadable) Size() int64 { + return s.size +} + +// NewReaddHandle is part of the objstorage.Readable interface. +func (s *simpleReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle { + return &s.rh } diff --git a/vendor/github.com/cockroachdb/pebble/sstable/suffix_rewriter.go b/vendor/github.com/cockroachdb/pebble/sstable/suffix_rewriter.go index 18718a3..b1423fd 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/suffix_rewriter.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/suffix_rewriter.go @@ -2,23 +2,41 @@ package sstable import ( "bytes" + "context" "math" - "os" "sync" - "time" "github.com/cespare/xxhash/v2" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/bytealloc" + "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/rangekey" + "github.com/cockroachdb/pebble/objstorage" ) -// RewriteKeySuffixes copies the content of the passed SSTable bytes to a new -// sstable, written to `out`, in which the suffix `from` has is replaced with -// `to` in every key. The input sstable must consist of only Sets or RangeKeySets -// and every key must have `from` as its suffix as determined by the Split -// function of the Comparer in the passed WriterOptions. Range deletes must not -// exist in this sstable, as they will be ignored. +// RewriteKeySuffixes is deprecated. +// +// TODO(sumeer): remove after switching CockroachDB to RewriteKeySuffixesAndReturnFormat. +func RewriteKeySuffixes( + sst []byte, + rOpts ReaderOptions, + out objstorage.Writable, + o WriterOptions, + from, to []byte, + concurrency int, +) (*WriterMetadata, error) { + meta, _, err := RewriteKeySuffixesAndReturnFormat(sst, rOpts, out, o, from, to, concurrency) + return meta, err +} + +// RewriteKeySuffixesAndReturnFormat copies the content of the passed SSTable +// bytes to a new sstable, written to `out`, in which the suffix `from` has is +// replaced with `to` in every key. The input sstable must consist of only +// Sets or RangeKeySets and every key must have `from` as its suffix as +// determined by the Split function of the Comparer in the passed +// WriterOptions. Range deletes must not exist in this sstable, as they will +// be ignored. // // Data blocks are rewritten in parallel by `concurrency` workers and then // assembled into a final SST. Filters are copied from the original SST without @@ -27,58 +45,78 @@ import ( // // Any block and table property collectors configured in the WriterOptions must // implement SuffixReplaceableTableCollector/SuffixReplaceableBlockCollector. -func RewriteKeySuffixes( +// +// The WriterOptions.TableFormat is ignored, and the output sstable has the +// same TableFormat as the input, which is returned in case the caller wants +// to do some error checking. Suffix rewriting is meant to be efficient, and +// allowing changes in the TableFormat detracts from that efficiency. +func RewriteKeySuffixesAndReturnFormat( sst []byte, rOpts ReaderOptions, - out writeCloseSyncer, + out objstorage.Writable, o WriterOptions, from, to []byte, concurrency int, -) (*WriterMetadata, error) { +) (*WriterMetadata, TableFormat, error) { r, err := NewMemReader(sst, rOpts) if err != nil { - return nil, err + return nil, TableFormatUnspecified, err } defer r.Close() return rewriteKeySuffixesInBlocks(r, out, o, from, to, concurrency) } func rewriteKeySuffixesInBlocks( - r *Reader, out writeCloseSyncer, o WriterOptions, from, to []byte, concurrency int, -) (*WriterMetadata, error) { + r *Reader, out objstorage.Writable, o WriterOptions, from, to []byte, concurrency int, +) (*WriterMetadata, TableFormat, error) { if o.Comparer == nil || o.Comparer.Split == nil { - return nil, errors.New("a valid splitter is required to define suffix to replace replace suffix") + return nil, TableFormatUnspecified, + errors.New("a valid splitter is required to rewrite suffixes") } if concurrency < 1 { - return nil, errors.New("concurrency must be >= 1") + return nil, TableFormatUnspecified, errors.New("concurrency must be >= 1") + } + // Even though NumValueBlocks = 0 => NumValuesInValueBlocks = 0, check both + // as a defensive measure. + if r.Properties.NumValueBlocks > 0 || r.Properties.NumValuesInValueBlocks > 0 { + return nil, TableFormatUnspecified, + errors.New("sstable with a single suffix should not have value blocks") } + tableFormat := r.tableFormat + o.TableFormat = tableFormat w := NewWriter(out, o) - defer w.Close() + defer func() { + if w != nil { + w.Close() + } + }() for _, c := range w.propCollectors { if _, ok := c.(SuffixReplaceableTableCollector); !ok { - return nil, errors.Errorf("property collector %s does not support suffix replacement", c.Name()) + return nil, TableFormatUnspecified, + errors.Errorf("property collector %s does not support suffix replacement", c.Name()) } } for _, c := range w.blockPropCollectors { if _, ok := c.(SuffixReplaceableBlockCollector); !ok { - return nil, errors.Errorf("block property collector %s does not support suffix replacement", c.Name()) + return nil, TableFormatUnspecified, + errors.Errorf("block property collector %s does not support suffix replacement", c.Name()) } } l, err := r.Layout() if err != nil { - return nil, errors.Wrap(err, "reading layout") + return nil, TableFormatUnspecified, errors.Wrap(err, "reading layout") } if err := rewriteDataBlocksToWriter(r, w, l.Data, from, to, w.split, concurrency); err != nil { - return nil, errors.Wrap(err, "rewriting data blocks") + return nil, TableFormatUnspecified, errors.Wrap(err, "rewriting data blocks") } // Copy over the range key block and replace suffixes in it if it exists. if err := rewriteRangeKeyBlockToWriter(r, w, from, to); err != nil { - return nil, errors.Wrap(err, "rewriting range key blocks") + return nil, TableFormatUnspecified, errors.Wrap(err, "rewriting range key blocks") } // Copy over the filter block if it exists (rewriteDataBlocksToWriter will @@ -86,7 +124,7 @@ func rewriteKeySuffixesInBlocks( if w.filter != nil && l.Filter.Length > 0 { filterBlock, _, err := readBlockBuf(r, l.Filter, nil) if err != nil { - return nil, errors.Wrap(err, "reading filter") + return nil, TableFormatUnspecified, errors.Wrap(err, "reading filter") } w.filter = copyFilterWriter{ origPolicyName: w.filter.policyName(), origMetaName: w.filter.metaName(), data: filterBlock, @@ -94,10 +132,12 @@ func rewriteKeySuffixesInBlocks( } if err := w.Close(); err != nil { - return nil, err + w = nil + return nil, TableFormatUnspecified, err } - - return w.Metadata() + writerMeta, err := w.Metadata() + w = nil + return writerMeta, tableFormat, err } var errBadKind = errors.New("key does not have expected kind (set)") @@ -126,8 +166,8 @@ func rewriteBlocks( buf.checksummer.xxHasher = xxhash.New() } - var blockAlloc []byte - var keyAlloc []byte + var blockAlloc bytealloc.A + var keyAlloc bytealloc.A var scratch InternalKey var inputBlock, inputBlockBuf []byte @@ -178,7 +218,24 @@ func rewriteBlocks( copy(scratch.UserKey, key.UserKey[:si]) copy(scratch.UserKey[si:], to) + // NB: for TableFormatPebblev3 and higher, since + // !iter.lazyValueHandling.hasValuePrefix, it will return the raw value + // in the block, which includes the 1-byte prefix. This is fine since bw + // also does not know about the prefix and will preserve it in bw.add. v := val.InPlaceValue() + if invariants.Enabled && r.tableFormat >= TableFormatPebblev3 && + key.Kind() == InternalKeyKindSet { + if len(v) < 1 { + return errors.Errorf("value has no prefix") + } + prefix := valuePrefix(v[0]) + if isValueHandle(prefix) { + return errors.Errorf("value prefix is incorrect") + } + if setHasSamePrefix(prefix) { + return errors.Errorf("multiple keys with same key prefix") + } + } bw.add(scratch, v) if output[i].start.UserKey == nil { keyAlloc, output[i].start = cloneKeyWithBuf(scratch, keyAlloc) @@ -191,12 +248,7 @@ func rewriteBlocks( finished := compressAndChecksum(bw.finish(), compression, &buf) // copy our finished block into the output buffer. - sz := len(finished) + blockTrailerLen - if cap(blockAlloc) < sz { - blockAlloc = make([]byte, sz*128) - } - output[i].data = blockAlloc[:sz:sz] - blockAlloc = blockAlloc[sz:] + blockAlloc, output[i].data = blockAlloc.Alloc(len(finished) + blockTrailerLen) copy(output[i].data, finished) copy(output[i].data[len(finished):], buf.tmp[:blockTrailerLen]) } @@ -278,11 +330,11 @@ func rewriteDataBlocksToWriter( for i := range blocks { // Write the rewritten block to the file. - n, err := w.writer.Write(blocks[i].data) - if err != nil { + if err := w.writable.Write(blocks[i].data); err != nil { return err } + n := len(blocks[i].data) bh := BlockHandle{Offset: w.meta.Size, Length: uint64(n) - blockTrailerLen} // Update the overall size. w.meta.Size += uint64(n) @@ -306,8 +358,8 @@ func rewriteDataBlocksToWriter( } } - var bhp BlockHandleWithProperties - if bhp, err = w.maybeAddBlockPropertiesToBlockHandle(bh); err != nil { + bhp, err := w.maybeAddBlockPropertiesToBlockHandle(bh) + if err != nil { return err } var nextKey InternalKey @@ -384,13 +436,18 @@ func (c copyFilterWriter) policyName() string { return c.origPolicyName } // more work to rederive filters, props, etc, however re-doing that work makes // it less restrictive -- props no longer need to func RewriteKeySuffixesViaWriter( - r *Reader, out writeCloseSyncer, o WriterOptions, from, to []byte, + r *Reader, out objstorage.Writable, o WriterOptions, from, to []byte, ) (*WriterMetadata, error) { if o.Comparer == nil || o.Comparer.Split == nil { - return nil, errors.New("a valid splitter is required to define suffix to replace replace suffix") + return nil, errors.New("a valid splitter is required to rewrite suffixes") } w := NewWriter(out, o) + defer func() { + if w != nil { + w.Close() + } + }() i, err := r.NewIter(nil, nil) if err != nil { return nil, err @@ -424,18 +481,21 @@ func RewriteKeySuffixesViaWriter( return nil, err } if err := w.Close(); err != nil { + w = nil return nil, err } - return &w.meta, nil + writerMeta, err := w.Metadata() + w = nil + return writerMeta, err } // NewMemReader opens a reader over the SST stored in the passed []byte. func NewMemReader(sst []byte, o ReaderOptions) (*Reader, error) { - return NewReader(memReader{sst, bytes.NewReader(sst), sizeOnlyStat(int64(len(sst)))}, o) + return NewReader(newMemReader(sst), o) } func readBlockBuf(r *Reader, bh BlockHandle, buf []byte) ([]byte, []byte, error) { - raw := r.file.(memReader).b[bh.Offset : bh.Offset+bh.Length+blockTrailerLen] + raw := r.readable.(*memReader).b[bh.Offset : bh.Offset+bh.Length+blockTrailerLen] if err := checkChecksum(r.checksumType, raw, bh, 0); err != nil { return nil, buf, err } @@ -455,31 +515,46 @@ func readBlockBuf(r *Reader, bh BlockHandle, buf []byte) ([]byte, []byte, error) return res, buf, err } -// memReader is a thin wrapper around a []byte such that it can be passed to an +// memReader is a thin wrapper around a []byte such that it can be passed to // sstable.Reader. It supports concurrent use, and does so without locking in // contrast to the heavier read/write vfs.MemFile. type memReader struct { - b []byte - r *bytes.Reader - s sizeOnlyStat + b []byte + r *bytes.Reader + rh objstorage.NoopReadHandle } -var _ ReadableFile = memReader{} +var _ objstorage.Readable = (*memReader)(nil) -// ReadAt implements io.ReaderAt. -func (m memReader) ReadAt(p []byte, off int64) (n int, err error) { return m.r.ReadAt(p, off) } +func newMemReader(b []byte) *memReader { + r := &memReader{ + b: b, + r: bytes.NewReader(b), + } + r.rh = objstorage.MakeNoopReadHandle(r) + return r +} -// Close implements io.Closer. -func (memReader) Close() error { return nil } +// ReadAt is part of objstorage.Readable. +func (m *memReader) ReadAt(_ context.Context, p []byte, off int64) error { + n, err := m.r.ReadAt(p, off) + if invariants.Enabled && err == nil && n != len(p) { + panic("short read") + } + return err +} -// Stat implements ReadableFile. -func (m memReader) Stat() (os.FileInfo, error) { return m.s, nil } +// Close is part of objstorage.Readable. +func (*memReader) Close() error { + return nil +} -type sizeOnlyStat int64 +// Stat is part of objstorage.Readable. +func (m *memReader) Size() int64 { + return int64(len(m.b)) +} -func (s sizeOnlyStat) Size() int64 { return int64(s) } -func (sizeOnlyStat) IsDir() bool { panic(errors.AssertionFailedf("unimplemented")) } -func (sizeOnlyStat) ModTime() time.Time { panic(errors.AssertionFailedf("unimplemented")) } -func (sizeOnlyStat) Mode() os.FileMode { panic(errors.AssertionFailedf("unimplemented")) } -func (sizeOnlyStat) Name() string { panic(errors.AssertionFailedf("unimplemented")) } -func (sizeOnlyStat) Sys() interface{} { panic(errors.AssertionFailedf("unimplemented")) } +// NewReadHandle is part of objstorage.Readable. +func (m *memReader) NewReadHandle(_ context.Context) objstorage.ReadHandle { + return &m.rh +} diff --git a/vendor/github.com/cockroachdb/pebble/sstable/table.go b/vendor/github.com/cockroachdb/pebble/sstable/table.go index fd9a95f..5267123 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/table.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/table.go @@ -27,52 +27,53 @@ // // To return the value for a key: // -// r := table.NewReader(file, options) -// defer r.Close() -// i := r.NewIter(nil, nil) -// defer i.Close() -// ikey, value := r.SeekGE(key) -// if options.Comparer.Compare(ikey.UserKey, key) != 0 { -// // not found -// } else { -// // value is the first record containing key -// } +// r := table.NewReader(file, options) +// defer r.Close() +// i := r.NewIter(nil, nil) +// defer i.Close() +// ikey, value := r.SeekGE(key) +// if options.Comparer.Compare(ikey.UserKey, key) != 0 { +// // not found +// } else { +// // value is the first record containing key +// } // // To count the number of entries in a table: // -// i, n := r.NewIter(nil, nil), 0 -// for key, value := i.First(); key != nil; key, value = i.Next() { -// n++ -// } -// if err := i.Close(); err != nil { -// return 0, err -// } -// return n, nil +// i, n := r.NewIter(nil, nil), 0 +// for key, value := i.First(); key != nil; key, value = i.Next() { +// n++ +// } +// if err := i.Close(); err != nil { +// return 0, err +// } +// return n, nil // // To write a table with three entries: // -// w := table.NewWriter(file, options) -// if err := w.Set([]byte("apple"), []byte("red")); err != nil { -// w.Close() -// return err -// } -// if err := w.Set([]byte("banana"), []byte("yellow")); err != nil { -// w.Close() -// return err -// } -// if err := w.Set([]byte("cherry"), []byte("red")); err != nil { -// w.Close() -// return err -// } -// return w.Close() +// w := table.NewWriter(file, options) +// if err := w.Set([]byte("apple"), []byte("red")); err != nil { +// w.Close() +// return err +// } +// if err := w.Set([]byte("banana"), []byte("yellow")); err != nil { +// w.Close() +// return err +// } +// if err := w.Set([]byte("cherry"), []byte("red")); err != nil { +// w.Close() +// return err +// } +// return w.Close() package sstable // import "github.com/cockroachdb/pebble/sstable" import ( + "context" "encoding/binary" - "io" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/objstorage" ) /* @@ -87,6 +88,9 @@ The table file format looks like: [index block] (for single level index) [meta rangedel block] (optional) [meta range key block] (optional) +[value block 0] (optional) +[value block M-1] (optional) +[meta value index block] (optional) [meta properties block] [metaindex block] [footer] @@ -110,8 +114,9 @@ byte of the trailer (i.e. the block type), and is serialized as little-endian. The block type gives the per-block compression used; each block is compressed independently. The checksum algorithm is described in the pebble/crc package. -Most blocks, other than the meta filter block, contain key/value pairs. The -remainder of this comment refers to the decompressed block, which has its 5 byte +Most blocks, other than the meta filter block, value blocks and meta value +index block, contain key/value pairs. The remainder of this comment refers to +the decompressed block, containing key/value pairs, which has its 5 byte trailer stripped. The decompressed block data consists of a sequence of such key/value entries followed by a block suffix. Each key is encoded as a shared prefix length and a remainder string. For example, if two adjacent keys are @@ -154,6 +159,9 @@ lower-level index blocks. The metaindex block also contains block handles as values, with keys being the names of the meta blocks. +For a description of value blocks and the meta value index block, see +value_block.go + */ const ( @@ -183,6 +191,7 @@ const ( rocksDBFormatVersion2 = 2 metaRangeKeyName = "pebble.range_key" + metaValueIndexName = "pebble.value_index" metaPropertiesName = "rocksdb.properties" metaRangeDelName = "rocksdb.range_del" metaRangeDelV2Name = "rocksdb.range_del2" @@ -274,17 +283,20 @@ func (t blockType) String() string { } // legacy (LevelDB) footer format: -// metaindex handle (varint64 offset, varint64 size) -// index handle (varint64 offset, varint64 size) -// to make the total size 2 * BlockHandle::kMaxEncodedLength -// table_magic_number (8 bytes) +// +// metaindex handle (varint64 offset, varint64 size) +// index handle (varint64 offset, varint64 size) +// to make the total size 2 * BlockHandle::kMaxEncodedLength +// table_magic_number (8 bytes) +// // new (RocksDB) footer format: -// checksum type (char, 1 byte) -// metaindex handle (varint64 offset, varint64 size) -// index handle (varint64 offset, varint64 size) -// to make the total size 2 * BlockHandle::kMaxEncodedLength + 1 -// footer version (4 bytes) -// table_magic_number (8 bytes) +// +// checksum type (char, 1 byte) +// metaindex handle (varint64 offset, varint64 size) +// index handle (varint64 offset, varint64 size) +// to make the total size 2 * BlockHandle::kMaxEncodedLength + 1 +// footer version (4 bytes) +// table_magic_number (8 bytes) type footer struct { format TableFormat checksum ChecksumType @@ -293,26 +305,22 @@ type footer struct { footerBH BlockHandle } -func readFooter(f ReadableFile) (footer, error) { +func readFooter(f objstorage.Readable) (footer, error) { var footer footer - stat, err := f.Stat() - if err != nil { - return footer, errors.Wrap(err, "pebble/table: invalid table (could not stat file)") - } - if stat.Size() < minFooterLen { + size := f.Size() + if size < minFooterLen { return footer, base.CorruptionErrorf("pebble/table: invalid table (file size is too small)") } buf := make([]byte, maxFooterLen) - off := stat.Size() - maxFooterLen + off := size - maxFooterLen if off < 0 { off = 0 + buf = buf[:size] } - n, err := f.ReadAt(buf, off) - if err != nil && err != io.EOF { + if err := f.ReadAt(context.TODO(), buf, off); err != nil { return footer, errors.Wrap(err, "pebble/table: invalid table (could not read footer)") } - buf = buf[:n] switch magic := buf[len(buf)-len(rocksDBMagic):]; string(magic) { case levelDBMagic: @@ -354,11 +362,11 @@ func readFooter(f ReadableFile) (footer, error) { buf = buf[1:] default: - return footer, base.CorruptionErrorf("pebble/table: invalid table (bad magic number)") + return footer, base.CorruptionErrorf("pebble/table: invalid table (bad magic number: 0x%x)", magic) } { - end := uint64(stat.Size()) + end := uint64(size) var n int footer.metaindexBH, n = decodeBlockHandle(buf) if n == 0 || footer.metaindexBH.Offset+footer.metaindexBH.Length > end { @@ -420,7 +428,7 @@ func supportsTwoLevelIndex(format TableFormat) bool { switch format { case TableFormatLevelDB: return false - case TableFormatRocksDBv2, TableFormatPebblev1, TableFormatPebblev2: + case TableFormatRocksDBv2, TableFormatPebblev1, TableFormatPebblev2, TableFormatPebblev3, TableFormatPebblev4: return true default: panic("sstable: unspecified table format version") diff --git a/vendor/github.com/cockroachdb/pebble/sstable/value_block.go b/vendor/github.com/cockroachdb/pebble/sstable/value_block.go new file mode 100644 index 0000000..fc0e700 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/sstable/value_block.go @@ -0,0 +1,948 @@ +// Copyright 2022 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package sstable + +import ( + "context" + "encoding/binary" + "io" + "sync" + "unsafe" + + "github.com/cockroachdb/errors" + "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/cache" + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" + "golang.org/x/exp/rand" +) + +// Value blocks are supported in TableFormatPebblev3. +// +// 1. Motivation and overview +// +// Value blocks are a mechanism designed for sstables storing MVCC data, where +// there can be many versions of a key that need to be kept, but only the +// latest value is typically read (see the documentation for Comparer.Split +// regarding MVCC keys). The goal is faster reads. Unlike Pebble versions, +// which can be eagerly thrown away (except when there are snapshots), MVCC +// versions are long-lived (e.g. default CockroachDB garbage collection +// threshold for older versions is 24 hours) and can significantly slow down +// reads. We have seen CockroachDB production workloads with very slow reads +// due to: +// - 100s of versions for each key in a table. +// +// - Tables with mostly MVCC garbage consisting of 2 versions per key -- a +// real key-value pair, followed by a key-value pair whose value (usually +// with zero byte length) indicates it is an MVCC tombstone. +// +// The value blocks mechanism attempts to improve read throughput in these +// cases when the key size is smaller than the value sizes of older versions. +// This is done by moving the value of an older version to a value block in a +// different part of the sstable. This improves spatial locality of the data +// being read by the workload, which increases caching effectiveness. +// +// Additionally, even when the key size is not smaller than the value of older +// versions (e.g. secondary indexes in CockroachDB), TableFormatPebblev3 +// stores the result of key comparisons done at write time inside the sstable, +// which makes stepping from one key prefix to the next prefix (i.e., skipping +// over older versions of a MVCC key) more efficient by avoiding key +// comparisons and key decoding. See the results in +// https://github.com/cockroachdb/pebble/pull/2149 and more details in the +// comment inside BenchmarkIteratorScanNextPrefix. These improvements are also +// visible in end-to-end CockroachDB tests, as outlined in +// https://github.com/cockroachdb/cockroach/pull/96652. +// +// In TableFormatPebblev3, each SET has a one byte value prefix that tells us +// whether the value is in-place or in a value block. This 1 byte prefix +// encodes additional information: +// +// - ShortAttribute: This is an attribute of the value. Currently, CockroachDB +// uses it to represent whether the value is a tombstone or not. This avoids +// the need to fetch a value from the value block if the caller only wants +// to figure out whether it is an MVCC tombstone. The length of the value is +// another attribute that the caller can be interested in, and it is also +// accessible without reading the value in the value block (see the value +// handle in the details section). +// +// - SET-same-prefix: this enables the aforementioned optimization when +// stepping from one key prefix to the next key prefix. +// +// We further optimize this iteration over prefixes by using the restart +// points in a block to encode whether the SET at a restart point has the same +// prefix since the last restart point. This allows us to skip over restart +// points within the same block. See the comment in blockWriter, and how both +// SET-same-prefix and the restart point information is used in +// blockIter.nextPrefixV3. +// +// This flexibility of values that are in-place or in value blocks requires +// flexibility in the iterator interface. The InternalIterator interface +// returns a LazyValue instead of a byte slice. Additionally, pebble.Iterator +// allows the caller to ask for a LazyValue. See lazy_value.go for details, +// including the memory lifetime management. +// +// For historical discussions about this feature, see the issue +// https://github.com/cockroachdb/pebble/issues/1170 and the prototype in +// https://github.com/cockroachdb/pebble/pull/1443. +// +// The code in this file mainly covers value block and related encodings. We +// discuss these in the next section. +// +// 2. Details +// +// Note that the notion of the latest value is local to the sstable. It is +// possible that that latest value has been deleted by a sstable in a higher +// level, and what is the latest value from the perspective of the whole LSM +// is an older MVCC version. This only affects performance and not +// correctness. This local knowledge is also why we continue to store these +// older versions in the same sstable -- we need to be able to conveniently +// read them. The code in this file is agnostic to the policy regarding what +// should be stored in value blocks -- it allows even the latest MVCC version +// to be stored in a value block. The policy decision in made in the +// sstable.Writer. See Writer.makeAddPointDecisionV3. +// +// Data blocks contain two kinds of SET keys: those with in-place values and +// those with a value handle. To distinguish these two cases we use a single +// byte prefix (valuePrefix). This single byte prefix is split into multiple +// parts, where nb represents information that is encoded in n bits. +// +// +---------------+--------------------+-----------+--------------------+ +// | value-kind 2b | SET-same-prefix 1b | unused 2b | short-attribute 3b | +// +---------------+--------------------+-----------+--------------------+ +// +// The 2 bit value-kind specifies whether this is an in-place value or a value +// handle pointing to a value block. We use 2 bits here for future +// representation of values that are in separate files. The 1 bit +// SET-same-prefix is true if this key is a SET and is immediately preceded by +// a SET that shares the same prefix. The 3 bit short-attribute is described +// in base.ShortAttribute -- it stores user-defined attributes about the +// value. It is unused for in-place values. +// +// Value Handle and Value Blocks: +// valueHandles refer to values in value blocks. Value blocks are simpler than +// normal data blocks (that contain key-value pairs, and allow for binary +// search), which makes them cheap for value retrieval purposes. A valueHandle +// is a tuple (valueLen, blockNum, offsetInBlock), where blockNum is the 0 +// indexed value block number and offsetInBlock is the byte offset in that +// block containing the value. The valueHandle.valueLen is included since +// there are multiple use cases in CockroachDB that need the value length but +// not the value, for which we can avoid reading the value in the value block +// (see +// https://github.com/cockroachdb/pebble/issues/1170#issuecomment-958203245). +// +// A value block has a checksum like other blocks, and is optionally +// compressed. An uncompressed value block is a sequence of values with no +// separator or length (we rely on the valueHandle to demarcate). The +// valueHandle.offsetInBlock points to the value, of length +// valueHandle.valueLen. While writing a sstable, all the (possibly +// compressed) value blocks need to be held in-memory until they can be +// written. Value blocks are placed after the "meta rangedel" and "meta range +// key" blocks since value blocks are considered less likely to be read. +// +// Meta Value Index Block: +// Since the (key, valueHandle) pair are written before there is any knowledge +// of the byte offset of the value block in the file, or its compressed +// length, we need another lookup to map the valueHandle.blockNum to the +// information needed to read it from the file. This information is provided +// by the "value index block". The "value index block" is referred to by the +// metaindex block. The design intentionally avoids making the "value index +// block" a general purpose key-value block, since each caller wants to lookup +// the information for a particular blockNum (there is no need for SeekGE +// etc.). Instead, this index block stores a sequence of (blockNum, +// blockOffset, blockLength) tuples, where the blockNums are consecutive +// integers, and the tuples are encoded with a fixed width encoding. This +// allows a reader to find the tuple for block K by looking at the offset +// K*fixed-width. The fixed width for each field is decided by looking at the +// maximum value of each of these fields. As a concrete example of a large +// sstable with many value blocks, we constructed a 100MB sstable with many +// versions and had 2475 value blocks (~32KB each). This sstable had this +// tuple encoded using 2+4+2=8 bytes, which means the uncompressed value index +// block was 2475*8=~19KB, which is modest. Therefore, we don't support more +// than one value index block. Consider the example of 2 byte blockNum, 4 byte +// blockOffset and 2 byte blockLen. The value index block will look like: +// +// +---------------+------------------+---------------+ +// | blockNum (2B) | blockOffset (4B) | blockLen (2B) | +// +---------------+------------------+---------------+ +// | 0 | 7,123,456 | 30,000 | +// +---------------+------------------+---------------+ +// | 1 | 7,153,456 | 20,000 | +// +---------------+------------------+---------------+ +// | 2 | 7,173,456 | 25,567 | +// +---------------+------------------+---------------+ +// | .... | ... | ... | +// +// +// The metaindex block contains the valueBlocksIndexHandle which in addition +// to the BlockHandle also specifies the widths of these tuple fields. In the +// above example, the +// valueBlockIndexHandle.{blockNumByteLength,blockOffsetByteLength,blockLengthByteLength} +// will be (2,4,2). + +// valueHandle is stored with a key when the value is in a value block. This +// handle is the pointer to that value. +type valueHandle struct { + valueLen uint32 + blockNum uint32 + offsetInBlock uint32 +} + +// valuePrefix is the single byte prefix for either the in-place value or the +// encoded valueHandle. It encoded multiple kinds of information. +type valuePrefix byte + +const ( + // 2 most-significant bits of valuePrefix encodes the value-kind. + valueKindMask valuePrefix = '\xC0' + valueKindIsValueHandle valuePrefix = '\x80' + valueKindIsInPlaceValue valuePrefix = '\x00' + + // 1 bit indicates SET has same key prefix as immediately preceding key that + // is also a SET. If the immediately preceding key in the same block is a + // SET, AND this bit is 0, the prefix must have changed. + // + // Note that the current policy of only storing older MVCC versions in value + // blocks means that valueKindIsValueHandle => SET has same prefix. But no + // code should rely on this behavior. Also, SET has same prefix does *not* + // imply valueKindIsValueHandle. + setHasSameKeyPrefixMask valuePrefix = '\x20' + + // 3 least-significant bits for the user-defined base.ShortAttribute. + // Undefined for valueKindIsInPlaceValue. + userDefinedShortAttributeMask valuePrefix = '\x07' +) + +// valueHandle fields are varint encoded, so maximum 5 bytes each, plus 1 byte +// for the valuePrefix. This could alternatively be group varint encoded, but +// experiments were inconclusive +// (https://github.com/cockroachdb/pebble/pull/1443#issuecomment-1270298802). +const valueHandleMaxLen = 5*3 + 1 + +// Assert blockHandleLikelyMaxLen >= valueHandleMaxLen. +const _ = uint(blockHandleLikelyMaxLen - valueHandleMaxLen) + +func encodeValueHandle(dst []byte, v valueHandle) int { + n := 0 + n += binary.PutUvarint(dst[n:], uint64(v.valueLen)) + n += binary.PutUvarint(dst[n:], uint64(v.blockNum)) + n += binary.PutUvarint(dst[n:], uint64(v.offsetInBlock)) + return n +} + +func makePrefixForValueHandle(setHasSameKeyPrefix bool, attribute base.ShortAttribute) valuePrefix { + prefix := valueKindIsValueHandle | valuePrefix(attribute) + if setHasSameKeyPrefix { + prefix = prefix | setHasSameKeyPrefixMask + } + return prefix +} + +func makePrefixForInPlaceValue(setHasSameKeyPrefix bool) valuePrefix { + prefix := valueKindIsInPlaceValue + if setHasSameKeyPrefix { + prefix = prefix | setHasSameKeyPrefixMask + } + return prefix +} + +func isValueHandle(b valuePrefix) bool { + return b&valueKindMask == valueKindIsValueHandle +} + +// REQUIRES: isValueHandle(b) +func getShortAttribute(b valuePrefix) base.ShortAttribute { + return base.ShortAttribute(b & userDefinedShortAttributeMask) +} + +func setHasSamePrefix(b valuePrefix) bool { + return b&setHasSameKeyPrefixMask == setHasSameKeyPrefixMask +} + +func decodeLenFromValueHandle(src []byte) (uint32, []byte) { + ptr := unsafe.Pointer(&src[0]) + var v uint32 + if a := *((*uint8)(ptr)); a < 128 { + v = uint32(a) + src = src[1:] + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { + v = uint32(b)<<7 | uint32(a) + src = src[2:] + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { + v = uint32(c)<<14 | uint32(b)<<7 | uint32(a) + src = src[3:] + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { + v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + src = src[4:] + } else { + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) + v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + src = src[5:] + } + return v, src +} + +func decodeRemainingValueHandle(src []byte) valueHandle { + var vh valueHandle + ptr := unsafe.Pointer(&src[0]) + // Manually inlined uvarint decoding. Saves ~25% in benchmarks. Unrolling + // a loop for i:=0; i<2; i++, saves ~6%. + var v uint32 + if a := *((*uint8)(ptr)); a < 128 { + v = uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 1) + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { + v = uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 2) + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { + v = uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 3) + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { + v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 4) + } else { + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) + v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + ptr = unsafe.Pointer(uintptr(ptr) + 5) + } + vh.blockNum = v + + if a := *((*uint8)(ptr)); a < 128 { + v = uint32(a) + } else if a, b := a&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 1))); b < 128 { + v = uint32(b)<<7 | uint32(a) + } else if b, c := b&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 2))); c < 128 { + v = uint32(c)<<14 | uint32(b)<<7 | uint32(a) + } else if c, d := c&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 3))); d < 128 { + v = uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + } else { + d, e := d&0x7f, *((*uint8)(unsafe.Pointer(uintptr(ptr) + 4))) + v = uint32(e)<<28 | uint32(d)<<21 | uint32(c)<<14 | uint32(b)<<7 | uint32(a) + } + vh.offsetInBlock = v + + return vh +} + +func decodeValueHandle(src []byte) valueHandle { + valLen, src := decodeLenFromValueHandle(src) + vh := decodeRemainingValueHandle(src) + vh.valueLen = valLen + return vh +} + +// valueBlocksIndexHandle is placed in the metaindex if there are any value +// blocks. If there are no value blocks, there is no value blocks index, and +// no entry in the metaindex. Note that the lack of entry in the metaindex +// should not be used to ascertain whether the values are prefixed, since the +// former is an emergent property of the data that was written and not known +// until all the key-value pairs in the sstable are written. +type valueBlocksIndexHandle struct { + h BlockHandle + blockNumByteLength uint8 + blockOffsetByteLength uint8 + blockLengthByteLength uint8 +} + +const valueBlocksIndexHandleMaxLen = blockHandleMaxLenWithoutProperties + 3 + +// Assert blockHandleLikelyMaxLen >= valueBlocksIndexHandleMaxLen. +const _ = uint(blockHandleLikelyMaxLen - valueBlocksIndexHandleMaxLen) + +func encodeValueBlocksIndexHandle(dst []byte, v valueBlocksIndexHandle) int { + n := encodeBlockHandle(dst, v.h) + dst[n] = v.blockNumByteLength + n++ + dst[n] = v.blockOffsetByteLength + n++ + dst[n] = v.blockLengthByteLength + n++ + return n +} + +func decodeValueBlocksIndexHandle(src []byte) (valueBlocksIndexHandle, int, error) { + var vbih valueBlocksIndexHandle + var n int + vbih.h, n = decodeBlockHandle(src) + if n <= 0 { + return vbih, 0, errors.Errorf("bad BlockHandle %x", src) + } + if len(src) != n+3 { + return vbih, 0, errors.Errorf("bad BlockHandle %x", src) + } + vbih.blockNumByteLength = src[n] + vbih.blockOffsetByteLength = src[n+1] + vbih.blockLengthByteLength = src[n+2] + return vbih, n + 3, nil +} + +type valueBlocksAndIndexStats struct { + numValueBlocks uint64 + numValuesInValueBlocks uint64 + // Includes both value blocks and value index block. + valueBlocksAndIndexSize uint64 +} + +// valueBlockWriter writes a sequence of value blocks, and the value blocks +// index, for a sstable. +type valueBlockWriter struct { + // The configured uncompressed block size and size threshold + blockSize, blockSizeThreshold int + // Configured compression. + compression Compression + // checksummer with configured checksum type. + checksummer checksummer + // Block finished callback. + blockFinishedFunc func(compressedSize int) + + // buf is the current block being written to (uncompressed). + buf *blockBuffer + // compressedBuf is used for compressing the block. + compressedBuf *blockBuffer + // Sequence of blocks that are finished. + blocks []blockAndHandle + // Cumulative value block bytes written so far. + totalBlockBytes uint64 + numValues uint64 +} + +type blockAndHandle struct { + block *blockBuffer + handle BlockHandle + compressed bool +} + +type blockBuffer struct { + b []byte +} + +// Pool of block buffers that should be roughly the blockSize. +var uncompressedValueBlockBufPool = sync.Pool{ + New: func() interface{} { + return &blockBuffer{} + }, +} + +// Pool of block buffers for compressed value blocks. These may widely vary in +// size based on compression ratios. +var compressedValueBlockBufPool = sync.Pool{ + New: func() interface{} { + return &blockBuffer{} + }, +} + +func releaseToValueBlockBufPool(pool *sync.Pool, b *blockBuffer) { + // Don't pool buffers larger than 128KB, in case we had some rare large + // values. + if len(b.b) > 128*1024 { + return + } + if invariants.Enabled { + // Set the bytes to a random value. Cap the number of bytes being + // randomized to prevent test timeouts. + length := cap(b.b) + if length > 1000 { + length = 1000 + } + b.b = b.b[:length:length] + rand.Read(b.b) + } + pool.Put(b) +} + +var valueBlockWriterPool = sync.Pool{ + New: func() interface{} { + return &valueBlockWriter{} + }, +} + +func newValueBlockWriter( + blockSize int, + blockSizeThreshold int, + compression Compression, + checksumType ChecksumType, + // compressedSize should exclude the block trailer. + blockFinishedFunc func(compressedSize int), +) *valueBlockWriter { + w := valueBlockWriterPool.Get().(*valueBlockWriter) + *w = valueBlockWriter{ + blockSize: blockSize, + blockSizeThreshold: blockSizeThreshold, + compression: compression, + checksummer: checksummer{ + checksumType: checksumType, + }, + blockFinishedFunc: blockFinishedFunc, + buf: uncompressedValueBlockBufPool.Get().(*blockBuffer), + compressedBuf: compressedValueBlockBufPool.Get().(*blockBuffer), + blocks: w.blocks[:0], + } + w.buf.b = w.buf.b[:0] + w.compressedBuf.b = w.compressedBuf.b[:0] + return w +} + +func releaseValueBlockWriter(w *valueBlockWriter) { + for i := range w.blocks { + if w.blocks[i].compressed { + releaseToValueBlockBufPool(&compressedValueBlockBufPool, w.blocks[i].block) + } else { + releaseToValueBlockBufPool(&uncompressedValueBlockBufPool, w.blocks[i].block) + } + w.blocks[i].block = nil + } + if w.buf != nil { + releaseToValueBlockBufPool(&uncompressedValueBlockBufPool, w.buf) + } + if w.compressedBuf != nil { + releaseToValueBlockBufPool(&compressedValueBlockBufPool, w.compressedBuf) + } + *w = valueBlockWriter{ + blocks: w.blocks[:0], + } + valueBlockWriterPool.Put(w) +} + +func (w *valueBlockWriter) addValue(v []byte) (valueHandle, error) { + if invariants.Enabled && len(v) == 0 { + return valueHandle{}, errors.Errorf("cannot write empty value to value block") + } + w.numValues++ + blockLen := len(w.buf.b) + valueLen := len(v) + if blockLen >= w.blockSize || + (blockLen > w.blockSizeThreshold && blockLen+valueLen > w.blockSize) { + // Block is not currently empty and adding this value will become too big, + // so finish this block. + w.compressAndFlush() + blockLen = len(w.buf.b) + if invariants.Enabled && blockLen != 0 { + panic("blockLen of new block should be 0") + } + } + vh := valueHandle{ + valueLen: uint32(valueLen), + blockNum: uint32(len(w.blocks)), + offsetInBlock: uint32(blockLen), + } + blockLen = int(vh.offsetInBlock + vh.valueLen) + if cap(w.buf.b) < blockLen { + size := 2 * cap(w.buf.b) + if size < 1024 { + size = 1024 + } + for size < blockLen { + size *= 2 + } + buf := make([]byte, blockLen, size) + _ = copy(buf, w.buf.b) + w.buf.b = buf + } else { + w.buf.b = w.buf.b[:blockLen] + } + buf := w.buf.b[vh.offsetInBlock:] + n := copy(buf, v) + if n != len(buf) { + panic("incorrect length computation") + } + return vh, nil +} + +func (w *valueBlockWriter) compressAndFlush() { + // Compress the buffer, discarding the result if the improvement isn't at + // least 12.5%. + blockType := noCompressionBlockType + b := w.buf + if w.compression != NoCompression { + blockType, w.compressedBuf.b = + compressBlock(w.compression, w.buf.b, w.compressedBuf.b[:cap(w.compressedBuf.b)]) + if len(w.compressedBuf.b) < len(w.buf.b)-len(w.buf.b)/8 { + b = w.compressedBuf + } else { + blockType = noCompressionBlockType + } + } + n := len(b.b) + if n+blockTrailerLen > cap(b.b) { + block := make([]byte, n+blockTrailerLen) + copy(block, b.b) + b.b = block + } else { + b.b = b.b[:n+blockTrailerLen] + } + b.b[n] = byte(blockType) + w.computeChecksum(b.b) + bh := BlockHandle{Offset: w.totalBlockBytes, Length: uint64(n)} + w.totalBlockBytes += uint64(len(b.b)) + // blockFinishedFunc length excludes the block trailer. + w.blockFinishedFunc(n) + compressed := blockType != noCompressionBlockType + w.blocks = append(w.blocks, blockAndHandle{ + block: b, + handle: bh, + compressed: compressed, + }) + // Handed off a buffer to w.blocks, so need get a new one. + if compressed { + w.compressedBuf = compressedValueBlockBufPool.Get().(*blockBuffer) + } else { + w.buf = uncompressedValueBlockBufPool.Get().(*blockBuffer) + } + w.buf.b = w.buf.b[:0] +} + +func (w *valueBlockWriter) computeChecksum(block []byte) { + n := len(block) - blockTrailerLen + checksum := w.checksummer.checksum(block[:n], block[n:n+1]) + binary.LittleEndian.PutUint32(block[n+1:], checksum) +} + +func (w *valueBlockWriter) finish( + writer io.Writer, fileOffset uint64, +) (valueBlocksIndexHandle, valueBlocksAndIndexStats, error) { + if len(w.buf.b) > 0 { + w.compressAndFlush() + } + n := len(w.blocks) + if n == 0 { + return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, nil + } + largestOffset := uint64(0) + largestLength := uint64(0) + for i := range w.blocks { + _, err := writer.Write(w.blocks[i].block.b) + if err != nil { + return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, err + } + w.blocks[i].handle.Offset += fileOffset + largestOffset = w.blocks[i].handle.Offset + if largestLength < w.blocks[i].handle.Length { + largestLength = w.blocks[i].handle.Length + } + } + vbihOffset := fileOffset + w.totalBlockBytes + + vbih := valueBlocksIndexHandle{ + h: BlockHandle{ + Offset: vbihOffset, + }, + blockNumByteLength: uint8(lenLittleEndian(uint64(n - 1))), + blockOffsetByteLength: uint8(lenLittleEndian(largestOffset)), + blockLengthByteLength: uint8(lenLittleEndian(largestLength)), + } + var err error + if vbih, err = w.writeValueBlocksIndex(writer, vbih); err != nil { + return valueBlocksIndexHandle{}, valueBlocksAndIndexStats{}, err + } + stats := valueBlocksAndIndexStats{ + numValueBlocks: uint64(n), + numValuesInValueBlocks: w.numValues, + valueBlocksAndIndexSize: w.totalBlockBytes + vbih.h.Length + blockTrailerLen, + } + return vbih, stats, err +} + +func (w *valueBlockWriter) writeValueBlocksIndex( + writer io.Writer, h valueBlocksIndexHandle, +) (valueBlocksIndexHandle, error) { + blockLen := + int(h.blockNumByteLength+h.blockOffsetByteLength+h.blockLengthByteLength) * len(w.blocks) + h.h.Length = uint64(blockLen) + blockLen += blockTrailerLen + var buf []byte + if cap(w.buf.b) < blockLen { + buf = make([]byte, blockLen) + w.buf.b = buf + } else { + buf = w.buf.b[:blockLen] + } + b := buf + for i := range w.blocks { + littleEndianPut(uint64(i), b, int(h.blockNumByteLength)) + b = b[int(h.blockNumByteLength):] + littleEndianPut(w.blocks[i].handle.Offset, b, int(h.blockOffsetByteLength)) + b = b[int(h.blockOffsetByteLength):] + littleEndianPut(w.blocks[i].handle.Length, b, int(h.blockLengthByteLength)) + b = b[int(h.blockLengthByteLength):] + } + if len(b) != blockTrailerLen { + panic("incorrect length calculation") + } + b[0] = byte(noCompressionBlockType) + w.computeChecksum(buf) + if _, err := writer.Write(buf); err != nil { + return valueBlocksIndexHandle{}, err + } + return h, nil +} + +// littleEndianPut writes v to b using little endian encoding, under the +// assumption that v can be represented using n bytes. +func littleEndianPut(v uint64, b []byte, n int) { + _ = b[n-1] // bounds check + for i := 0; i < n; i++ { + b[i] = byte(v) + v = v >> 8 + } +} + +// lenLittleEndian returns the minimum number of bytes needed to encode v +// using little endian encoding. +func lenLittleEndian(v uint64) int { + n := 0 + for i := 0; i < 8; i++ { + n++ + v = v >> 8 + if v == 0 { + break + } + } + return n +} + +func littleEndianGet(b []byte, n int) uint64 { + _ = b[n-1] // bounds check + v := uint64(b[0]) + for i := 1; i < n; i++ { + v |= uint64(b[i]) << (8 * i) + } + return v +} + +// UserKeyPrefixBound represents a [Lower,Upper) bound of user key prefixes. +// If both are nil, there is no bound specified. Else, Compare(Lower,Upper) +// must be < 0. +type UserKeyPrefixBound struct { + // Lower is a lower bound user key prefix. + Lower []byte + // Upper is an upper bound user key prefix. + Upper []byte +} + +// IsEmpty returns true iff the bound is empty. +func (ukb *UserKeyPrefixBound) IsEmpty() bool { + return len(ukb.Lower) == 0 && len(ukb.Upper) == 0 +} + +type blockProviderWhenOpen interface { + readBlockForVBR( + ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats, + ) (cache.Handle, error) +} + +type blockProviderWhenClosed struct { + rp ReaderProvider + r *Reader +} + +func (bpwc *blockProviderWhenClosed) open() error { + var err error + bpwc.r, err = bpwc.rp.GetReader() + return err +} + +func (bpwc *blockProviderWhenClosed) close() { + bpwc.rp.Close() + bpwc.r = nil +} + +func (bpwc blockProviderWhenClosed) readBlockForVBR( + ctx context.Context, h BlockHandle, stats *base.InternalIteratorStats, +) (cache.Handle, error) { + ctx = objiotracing.WithBlockType(ctx, objiotracing.ValueBlock) + return bpwc.r.readBlock(ctx, h, nil, nil, stats) +} + +// ReaderProvider supports the implementation of blockProviderWhenClosed. +// GetReader and Close can be called multiple times in pairs. +type ReaderProvider interface { + GetReader() (r *Reader, err error) + Close() +} + +// TrivialReaderProvider implements ReaderProvider for a Reader that will +// outlive the top-level iterator in the iterator tree. +type TrivialReaderProvider struct { + *Reader +} + +var _ ReaderProvider = TrivialReaderProvider{} + +// GetReader implements ReaderProvider. +func (trp TrivialReaderProvider) GetReader() (*Reader, error) { + return trp.Reader, nil +} + +// Close implements ReaderProvider. +func (trp TrivialReaderProvider) Close() {} + +// valueBlockReader is used to retrieve values in value +// blocks. It is used when the sstable was written with +// Properties.ValueBlocksAreEnabled. +type valueBlockReader struct { + ctx context.Context + bpOpen blockProviderWhenOpen + rp ReaderProvider + vbih valueBlocksIndexHandle + stats *base.InternalIteratorStats + + // The value blocks index is lazily retrieved the first time the reader + // needs to read a value that resides in a value block. + vbiBlock []byte + vbiCache cache.Handle + // When sequentially iterating through all key-value pairs, the cost of + // repeatedly getting a block that is already in the cache and releasing the + // cache.Handle can be ~40% of the cpu overhead. So the reader remembers the + // last value block it retrieved, in case there is locality of access, and + // this value block can be used for the next value retrieval. + valueBlockNum uint32 + valueBlock []byte + valueBlockPtr unsafe.Pointer + valueCache cache.Handle + lazyFetcher base.LazyFetcher + closed bool + bufToMangle []byte +} + +func (r *valueBlockReader) getLazyValueForPrefixAndValueHandle(handle []byte) base.LazyValue { + fetcher := &r.lazyFetcher + valLen, h := decodeLenFromValueHandle(handle[1:]) + *fetcher = base.LazyFetcher{ + Fetcher: r, + Attribute: base.AttributeAndLen{ + ValueLen: int32(valLen), + ShortAttribute: getShortAttribute(valuePrefix(handle[0])), + }, + } + if r.stats != nil { + r.stats.SeparatedPointValue.Count++ + r.stats.SeparatedPointValue.ValueBytes += uint64(valLen) + } + return base.LazyValue{ + ValueOrHandle: h, + Fetcher: fetcher, + } +} + +func (r *valueBlockReader) close() { + r.bpOpen = nil + r.vbiBlock = nil + r.vbiCache.Release() + // Set the handle to empty since Release does not nil the Handle.value. If + // we were to reopen this valueBlockReader and retrieve the same + // Handle.value from the cache, we don't want to accidentally unref it when + // attempting to unref the old handle. + r.vbiCache = cache.Handle{} + r.valueBlock = nil + r.valueBlockPtr = nil + r.valueCache.Release() + // See comment above. + r.valueCache = cache.Handle{} + r.closed = true + // rp, vbih, stats remain valid, so that LazyFetcher.ValueFetcher can be + // implemented. +} + +// Fetch implements base.ValueFetcher. +func (r *valueBlockReader) Fetch( + handle []byte, valLen int32, buf []byte, +) (val []byte, callerOwned bool, err error) { + if !r.closed { + val, err := r.getValueInternal(handle, valLen) + if invariants.Enabled { + val = r.doValueMangling(val) + } + return val, false, err + } + + bp := blockProviderWhenClosed{rp: r.rp} + err = bp.open() + if err != nil { + return nil, false, err + } + defer bp.close() + defer r.close() + r.bpOpen = bp + var v []byte + v, err = r.getValueInternal(handle, valLen) + if err != nil { + return nil, false, err + } + buf = append(buf[:0], v...) + return buf, true, nil +} + +// doValueMangling attempts to uncover violations of the contract listed in +// the declaration comment of LazyValue. It is expensive, hence only called +// when invariants.Enabled. +func (r *valueBlockReader) doValueMangling(v []byte) []byte { + // Randomly set the bytes in the previous retrieved value to 0, since + // property P1 only requires the valueBlockReader to maintain the memory of + // one fetched value. + if rand.Intn(2) == 0 { + for i := range r.bufToMangle { + r.bufToMangle[i] = 0 + } + } + // Store the current value in a new buffer for future mangling. + r.bufToMangle = append([]byte(nil), v...) + return r.bufToMangle +} + +func (r *valueBlockReader) getValueInternal(handle []byte, valLen int32) (val []byte, err error) { + vh := decodeRemainingValueHandle(handle) + vh.valueLen = uint32(valLen) + if r.vbiBlock == nil { + ch, err := r.bpOpen.readBlockForVBR(r.ctx, r.vbih.h, r.stats) + if err != nil { + return nil, err + } + r.vbiCache = ch + r.vbiBlock = ch.Get() + } + if r.valueBlock == nil || r.valueBlockNum != vh.blockNum { + vbh, err := r.getBlockHandle(vh.blockNum) + if err != nil { + return nil, err + } + vbCacheHandle, err := r.bpOpen.readBlockForVBR(r.ctx, vbh, r.stats) + if err != nil { + return nil, err + } + r.valueBlockNum = vh.blockNum + r.valueCache.Release() + r.valueCache = vbCacheHandle + r.valueBlock = vbCacheHandle.Get() + r.valueBlockPtr = unsafe.Pointer(&r.valueBlock[0]) + } + if r.stats != nil { + r.stats.SeparatedPointValue.ValueBytesFetched += uint64(valLen) + } + return r.valueBlock[vh.offsetInBlock : vh.offsetInBlock+vh.valueLen], nil +} + +func (r *valueBlockReader) getBlockHandle(blockNum uint32) (BlockHandle, error) { + indexEntryLen := + int(r.vbih.blockNumByteLength + r.vbih.blockOffsetByteLength + r.vbih.blockLengthByteLength) + offsetInIndex := indexEntryLen * int(blockNum) + if len(r.vbiBlock) < offsetInIndex+indexEntryLen { + return BlockHandle{}, errors.Errorf( + "cannot read at offset %d and length %d from block of length %d", + offsetInIndex, indexEntryLen, len(r.vbiBlock)) + } + b := r.vbiBlock[offsetInIndex : offsetInIndex+indexEntryLen] + n := int(r.vbih.blockNumByteLength) + bn := littleEndianGet(b, n) + if uint32(bn) != blockNum { + return BlockHandle{}, + errors.Errorf("expected block num %d but found %d", blockNum, bn) + } + b = b[n:] + n = int(r.vbih.blockOffsetByteLength) + blockOffset := littleEndianGet(b, n) + b = b[n:] + n = int(r.vbih.blockLengthByteLength) + blockLen := littleEndianGet(b, n) + return BlockHandle{Offset: blockOffset, Length: blockLen}, nil +} diff --git a/vendor/github.com/cockroachdb/pebble/sstable/writer.go b/vendor/github.com/cockroachdb/pebble/sstable/writer.go index f94ff38..d818688 100644 --- a/vendor/github.com/cockroachdb/pebble/sstable/writer.go +++ b/vendor/github.com/cockroachdb/pebble/sstable/writer.go @@ -5,24 +5,25 @@ package sstable import ( - "bufio" "bytes" "encoding/binary" "fmt" - "io" "math" "runtime" + "sort" "sync" "github.com/cespare/xxhash/v2" "github.com/cockroachdb/errors" "github.com/cockroachdb/pebble/internal/base" + "github.com/cockroachdb/pebble/internal/bytealloc" "github.com/cockroachdb/pebble/internal/cache" "github.com/cockroachdb/pebble/internal/crc" "github.com/cockroachdb/pebble/internal/invariants" "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/private" "github.com/cockroachdb/pebble/internal/rangekey" + "github.com/cockroachdb/pebble/objstorage" ) // encodedBHPEstimatedSize estimates the size of the encoded BlockHandleWithProperties. @@ -109,27 +110,16 @@ func (m *WriterMetadata) updateSeqNum(seqNum uint64) { } } -type flusher interface { - Flush() error -} - -type writeCloseSyncer interface { - io.WriteCloser - Sync() error -} - // Writer is a table writer. type Writer struct { - writer io.Writer - bufWriter *bufio.Writer - syncer writeCloseSyncer - meta WriterMetadata - err error + writable objstorage.Writable + meta WriterMetadata + err error // cacheID and fileNum are used to remove blocks written to the sstable from // the cache, providing a defense in depth against bugs which cause cache // collisions. cacheID uint64 - fileNum base.FileNum + fileNum base.DiskFileNum // The following fields are copied from Options. blockSize int blockSizeThreshold int @@ -186,9 +176,9 @@ type Writer struct { // indexBlockAlloc is used to bulk-allocate byte slices used to store index // blocks in indexPartitions. These live until the index finishes. indexBlockAlloc []byte - // indexSepAlloc is used to bulk-allocate index block seperator slices stored + // indexSepAlloc is used to bulk-allocate index block separator slices stored // in indexPartitions. These live until the index finishes. - indexSepAlloc []byte + indexSepAlloc bytealloc.A // To allow potentially overlapping (i.e. un-fragmented) range keys spans to // be added to the Writer, a keyspan.Fragmenter is used to retain the keys @@ -196,7 +186,8 @@ type Writer struct { // keys must be added in order of their start user-key. fragmenter keyspan.Fragmenter rangeKeyEncoder rangekey.Encoder - rangeKeyCoalesced keyspan.Span + rangeKeysBySuffix keyspan.KeysBySuffix + rangeKeySpan keyspan.Span rkBuf []byte // dataBlockBuf consists of the state which is currently owned by and used by // the Writer client goroutine. This state can be handed off to other goroutines. @@ -206,6 +197,24 @@ type Writer struct { blockBuf blockBuf coordination coordinationState + + // Information (other than the byte slice) about the last point key, to + // avoid extracting it again. + lastPointKeyInfo pointKeyInfo + + // For value blocks. + shortAttributeExtractor base.ShortAttributeExtractor + requiredInPlaceValueBound UserKeyPrefixBound + valueBlockWriter *valueBlockWriter +} + +type pointKeyInfo struct { + trailer uint64 + // Only computed when w.valueBlockWriter is not nil. + userKeyLen int + // prefixLen uses w.split, if not nil. Only computed when w.valueBlockWriter + // is not nil. + prefixLen int } type coordinationState struct { @@ -238,35 +247,37 @@ func (c *coordinationState) init(parallelismEnabled bool, writer *Writer) { // sizeEstimate is a general purpose helper for estimating two kinds of sizes: // A. The compressed sstable size, which is useful for deciding when to start -// a new sstable during flushes or compactions. In practice, we use this in -// estimating the data size (excluding the index). +// +// a new sstable during flushes or compactions. In practice, we use this in +// estimating the data size (excluding the index). +// // B. The size of index blocks to decide when to start a new index block. // // There are some terminology peculiarities which are due to the origin of // sizeEstimate for use case A with parallel compression enabled (for which // the code has not been merged). Specifically this relates to the terms // "written" and "compressed". -// - The notion of "written" for case A is sufficiently defined by saying that -// the data block is compressed. Waiting for the actual data block write to -// happen can result in unnecessary estimation, when we already know how big -// it will be in compressed form. Additionally, with the forthcoming value -// blocks containing older MVCC values, these compressed block will be held -// in-memory until late in the sstable writing, and we do want to accurately -// account for them without waiting for the actual write. -// For case B, "written" means that the index entry has been fully -// generated, and has been added to the uncompressed block buffer for that -// index block. It does not include actually writing a potentially -// compressed index block. -// - The notion of "compressed" is to differentiate between a "inflight" size -// and the actual size, and is handled via computing a compression ratio -// observed so far (defaults to 1). -// For case A, this is actual data block compression, so the "inflight" size -// is uncompressed blocks (that are no longer being written to) and the -// "compressed" size is after they have been compressed. -// For case B the inflight size is for a key-value pair in the index for -// which the value size (the encoded size of the BlockHandleWithProperties) -// is not accurately known, while the compressed size is the size of that -// entry when it has been added to the (in-progress) index ssblock. +// - The notion of "written" for case A is sufficiently defined by saying that +// the data block is compressed. Waiting for the actual data block write to +// happen can result in unnecessary estimation, when we already know how big +// it will be in compressed form. Additionally, with the forthcoming value +// blocks containing older MVCC values, these compressed block will be held +// in-memory until late in the sstable writing, and we do want to accurately +// account for them without waiting for the actual write. +// For case B, "written" means that the index entry has been fully +// generated, and has been added to the uncompressed block buffer for that +// index block. It does not include actually writing a potentially +// compressed index block. +// - The notion of "compressed" is to differentiate between a "inflight" size +// and the actual size, and is handled via computing a compression ratio +// observed so far (defaults to 1). +// For case A, this is actual data block compression, so the "inflight" size +// is uncompressed blocks (that are no longer being written to) and the +// "compressed" size is after they have been compressed. +// For case B the inflight size is for a key-value pair in the index for +// which the value size (the encoded size of the BlockHandleWithProperties) +// is not accurately known, while the compressed size is the size of that +// entry when it has been added to the (in-progress) index ssblock. // // Usage: To update state, one can optionally provide an inflight write value // using addInflight (used for case B). When something is "written" the state @@ -727,40 +738,153 @@ func (w *Writer) Add(key InternalKey, value []byte) error { return w.addPoint(key, value) } -func (w *Writer) addPoint(key InternalKey, value []byte) error { - if !w.disableKeyOrderChecks && w.dataBlockBuf.dataBlock.nEntries >= 1 { - // curKey is guaranteed to be the last point key which was added to the Writer. - // Inlining base.DecodeInternalKey has a 2-3% improve in the BenchmarkWriter - // benchmark. - encodedKey := w.dataBlockBuf.dataBlock.curKey - n := len(encodedKey) - base.InternalTrailerLen - var trailer uint64 - if n >= 0 { - trailer = binary.LittleEndian.Uint64(encodedKey[n:]) - encodedKey = encodedKey[:n:n] - } else { - trailer = uint64(InternalKeyKindInvalid) - encodedKey = nil +func (w *Writer) makeAddPointDecisionV2(key InternalKey) error { + prevTrailer := w.lastPointKeyInfo.trailer + w.lastPointKeyInfo.trailer = key.Trailer + if w.dataBlockBuf.dataBlock.nEntries == 0 { + return nil + } + if !w.disableKeyOrderChecks { + prevPointUserKey := w.dataBlockBuf.dataBlock.curKey + n := len(prevPointUserKey) - base.InternalTrailerLen + if n < 0 { + panic("corrupt key in blockWriter buffer") } - largestPointKey := InternalKey{ - UserKey: encodedKey, - Trailer: trailer, - } - - if largestPointKey.UserKey != nil { - // TODO(peter): Manually inlined version of base.InternalCompare(). This is - // 3.5% faster on BenchmarkWriter on go1.13. Remove if go1.14 or future - // versions show this to not be a performance win. - x := w.compare(largestPointKey.UserKey, key.UserKey) - if x > 0 || (x == 0 && largestPointKey.Trailer <= key.Trailer) { - w.err = errors.Errorf("pebble: keys must be added in strictly increasing order: %s, %s", - largestPointKey.Pretty(w.formatKey), key.Pretty(w.formatKey)) - return w.err - } + prevPointUserKey = prevPointUserKey[:n:n] + cmpUser := w.compare(prevPointUserKey, key.UserKey) + if cmpUser > 0 || (cmpUser == 0 && prevTrailer <= key.Trailer) { + return errors.Errorf( + "pebble: keys must be added in strictly increasing order: %s, %s", + InternalKey{UserKey: prevPointUserKey, Trailer: prevTrailer}.Pretty(w.formatKey), + key.Pretty(w.formatKey)) } } + return nil +} - if err := w.maybeFlush(key, value); err != nil { +func (w *Writer) makeAddPointDecisionV3( + key InternalKey, valueLen int, +) (setHasSamePrefix bool, writeToValueBlock bool, err error) { + prevPointKeyInfo := w.lastPointKeyInfo + w.lastPointKeyInfo.userKeyLen = len(key.UserKey) + w.lastPointKeyInfo.prefixLen = w.lastPointKeyInfo.userKeyLen + if w.split != nil { + w.lastPointKeyInfo.prefixLen = w.split(key.UserKey) + } + w.lastPointKeyInfo.trailer = key.Trailer + if w.dataBlockBuf.dataBlock.nEntries == 0 { + return false, false, nil + } + prevPointUserKey := w.dataBlockBuf.dataBlock.curKey + n := len(prevPointUserKey) - base.InternalTrailerLen + if n < 0 { + panic("corrupt key in blockWriter buffer") + } + prevPointUserKey = prevPointUserKey[:n:n] + prevPointKey := InternalKey{UserKey: prevPointUserKey, Trailer: prevPointKeyInfo.trailer} + considerWriteToValueBlock := base.TrailerKind(prevPointKeyInfo.trailer) == InternalKeyKindSet && + base.TrailerKind(key.Trailer) == InternalKeyKindSet + if considerWriteToValueBlock && !w.requiredInPlaceValueBound.IsEmpty() { + keyPrefix := key.UserKey[:w.lastPointKeyInfo.prefixLen] + cmpUpper := w.compare( + w.requiredInPlaceValueBound.Upper, keyPrefix) + if cmpUpper <= 0 { + // Common case for CockroachDB. Make it empty since all future keys in + // this sstable will also have cmpUpper <= 0. + w.requiredInPlaceValueBound = UserKeyPrefixBound{} + } else if w.compare(keyPrefix, w.requiredInPlaceValueBound.Lower) >= 0 { + considerWriteToValueBlock = false + } + } + // cmpPrefix is initialized iff considerWriteToValueBlock. + var cmpPrefix int + var cmpUser int + if considerWriteToValueBlock { + // Compare the prefixes. + cmpPrefix = w.compare(prevPointUserKey[:prevPointKeyInfo.prefixLen], + key.UserKey[:w.lastPointKeyInfo.prefixLen]) + cmpUser = cmpPrefix + if cmpPrefix == 0 { + // Need to compare suffixes to compute cmpUser. + cmpUser = w.compare(prevPointUserKey[prevPointKeyInfo.prefixLen:], + key.UserKey[w.lastPointKeyInfo.prefixLen:]) + } + } else { + cmpUser = w.compare(prevPointUserKey, key.UserKey) + } + if !w.disableKeyOrderChecks && + (cmpUser > 0 || (cmpUser == 0 && prevPointKeyInfo.trailer <= key.Trailer)) { + return false, false, errors.Errorf( + "pebble: keys must be added in strictly increasing order: %s, %s", + prevPointKey.Pretty(w.formatKey), key.Pretty(w.formatKey)) + } + if !considerWriteToValueBlock { + return false, false, nil + } + // NB: it is possible that cmpUser == 0, i.e., these two SETs have identical + // user keys (because of an open snapshot). This should be the rare case. + setHasSamePrefix = cmpPrefix == 0 + considerWriteToValueBlock = setHasSamePrefix + // Use of 0 here is somewhat arbitrary. Given the minimum 3 byte encoding of + // valueHandle, this should be > 3. But tiny values are common in test and + // unlikely in production, so we use 0 here for better test coverage. + const tinyValueThreshold = 0 + if considerWriteToValueBlock && valueLen <= tinyValueThreshold { + considerWriteToValueBlock = false + } + return setHasSamePrefix, considerWriteToValueBlock, nil +} + +func (w *Writer) addPoint(key InternalKey, value []byte) error { + var err error + var setHasSameKeyPrefix, writeToValueBlock, addPrefixToValueStoredWithKey bool + maxSharedKeyLen := len(key.UserKey) + if w.valueBlockWriter != nil { + // maxSharedKeyLen is limited to the prefix of the preceding key. If the + // preceding key was in a different block, then the blockWriter will + // ignore this maxSharedKeyLen. + maxSharedKeyLen = w.lastPointKeyInfo.prefixLen + setHasSameKeyPrefix, writeToValueBlock, err = w.makeAddPointDecisionV3(key, len(value)) + addPrefixToValueStoredWithKey = base.TrailerKind(key.Trailer) == InternalKeyKindSet + } else { + err = w.makeAddPointDecisionV2(key) + } + if err != nil { + return err + } + var valueStoredWithKey []byte + var prefix valuePrefix + var valueStoredWithKeyLen int + if writeToValueBlock { + vh, err := w.valueBlockWriter.addValue(value) + if err != nil { + return err + } + n := encodeValueHandle(w.blockBuf.tmp[:], vh) + valueStoredWithKey = w.blockBuf.tmp[:n] + valueStoredWithKeyLen = len(valueStoredWithKey) + 1 + var attribute base.ShortAttribute + if w.shortAttributeExtractor != nil { + // TODO(sumeer): for compactions, it is possible that the input sstable + // already has this value in the value section and so we have already + // extracted the ShortAttribute. Avoid extracting it again. This will + // require changing the Writer.Add interface. + if attribute, err = w.shortAttributeExtractor( + key.UserKey, w.lastPointKeyInfo.prefixLen, value); err != nil { + return err + } + } + prefix = makePrefixForValueHandle(setHasSameKeyPrefix, attribute) + } else { + valueStoredWithKey = value + valueStoredWithKeyLen = len(value) + if addPrefixToValueStoredWithKey { + valueStoredWithKeyLen++ + } + prefix = makePrefixForInPlaceValue(setHasSameKeyPrefix) + } + + if err := w.maybeFlush(key, valueStoredWithKeyLen); err != nil { return err } @@ -771,14 +895,23 @@ func (w *Writer) addPoint(key InternalKey, value []byte) error { } } for i := range w.blockPropCollectors { - if err := w.blockPropCollectors[i].Add(key, value); err != nil { + v := value + if addPrefixToValueStoredWithKey { + // Values for SET are not required to be in-place, and in the future may + // not even be read by the compaction, so pass nil values. Block + // property collectors in such Pebble DB's must not look at the value. + v = nil + } + if err := w.blockPropCollectors[i].Add(key, v); err != nil { w.err = err return err } } w.maybeAddToFilter(key.UserKey) - w.dataBlockBuf.dataBlock.add(key, value) + w.dataBlockBuf.dataBlock.addWithOptionalValuePrefix( + key, valueStoredWithKey, maxSharedKeyLen, addPrefixToValueStoredWithKey, prefix, + setHasSameKeyPrefix) w.meta.updateSeqNum(key.SeqNum()) @@ -797,8 +930,9 @@ func (w *Writer) addPoint(key InternalKey, value []byte) error { w.props.NumEntries++ switch key.Kind() { - case InternalKeyKindDelete: + case InternalKeyKindDelete, InternalKeyKindSingleDelete: w.props.NumDeletions++ + w.props.RawPointTombstoneKeySize += uint64(key.Size()) case InternalKeyKindMerge: w.props.NumMergeOperands++ } @@ -903,10 +1037,15 @@ func (w *Writer) addTombstone(key InternalKey, value []byte) error { } // RangeKeySet sets a range between start (inclusive) and end (exclusive) with -// the given suffix to the given value. +// the given suffix to the given value. The resulting range key is given the +// sequence number zero, with the expectation that the resulting sstable will be +// ingested. // // Keys must be added to the table in increasing order of start key. Spans are -// not required to be fragmented. +// not required to be fragmented. The same suffix may not be set or unset twice +// over the same keyspan, because it would result in inconsistent state. Both +// the Set and Unset would share the zero sequence number, and a key cannot be +// both simultaneously set and unset. func (w *Writer) RangeKeySet(start, end, suffix, value []byte) error { return w.addRangeKeySpan(keyspan.Span{ Start: w.tempRangeKeyCopy(start), @@ -922,10 +1061,15 @@ func (w *Writer) RangeKeySet(start, end, suffix, value []byte) error { } // RangeKeyUnset un-sets a range between start (inclusive) and end (exclusive) -// with the given suffix. +// with the given suffix. The resulting range key is given the +// sequence number zero, with the expectation that the resulting sstable will be +// ingested. // // Keys must be added to the table in increasing order of start key. Spans are -// not required to be fragmented. +// not required to be fragmented. The same suffix may not be set or unset twice +// over the same keyspan, because it would result in inconsistent state. Both +// the Set and Unset would share the zero sequence number, and a key cannot be +// both simultaneously set and unset. func (w *Writer) RangeKeyUnset(start, end, suffix []byte) error { return w.addRangeKeySpan(keyspan.Span{ Start: w.tempRangeKeyCopy(start), @@ -970,6 +1114,11 @@ func (w *Writer) AddRangeKey(key InternalKey, value []byte) error { } func (w *Writer) addRangeKeySpan(span keyspan.Span) error { + if w.compare(span.Start, span.End) >= 0 { + return errors.Errorf( + "pebble: start key must be strictly less than end key", + ) + } if w.fragmenter.Start() != nil && w.compare(w.fragmenter.Start(), span.Start) > 0 { return errors.Errorf("pebble: spans must be added in order: %s > %s", w.formatKey(w.fragmenter.Start()), w.formatKey(span.Start)) @@ -979,20 +1128,23 @@ func (w *Writer) addRangeKeySpan(span keyspan.Span) error { return w.err } -func (w *Writer) coalesceSpans(span keyspan.Span) { - // This method is the emit function of the Fragmenter, so span.Keys is only - // owned by this span and it's safe to mutate. - w.rangeKeyCoalesced.Start = span.Start - w.rangeKeyCoalesced.End = span.End - err := rangekey.Coalesce(w.compare, span.Keys, &w.rangeKeyCoalesced.Keys) - if err != nil { - w.err = errors.Newf("sstable: could not coalesce span: %s", err) - return - } +func (w *Writer) encodeRangeKeySpan(span keyspan.Span) { + // This method is the emit function of the Fragmenter. + // + // NB: The span should only contain range keys and be internally consistent + // (eg, no duplicate suffixes, no additional keys after a RANGEKEYDEL). + // + // We use w.rangeKeysBySuffix and w.rangeKeySpan to avoid allocations. - // NB: The span only contains range keys and is internally consistent (eg, - // no duplicate suffixes, no additional keys after a RANGEKEYDEL). - w.err = firstError(w.err, w.rangeKeyEncoder.Encode(&w.rangeKeyCoalesced)) + // Sort the keys by suffix. Iteration doesn't *currently* depend on it, but + // we may want to in the future. + w.rangeKeysBySuffix.Cmp = w.compare + w.rangeKeysBySuffix.Keys = span.Keys + sort.Sort(&w.rangeKeysBySuffix) + + w.rangeKeySpan = span + w.rangeKeySpan.Keys = w.rangeKeysBySuffix.Keys + w.err = firstError(w.err, w.rangeKeyEncoder.Encode(&w.rangeKeySpan)) } func (w *Writer) addRangeKey(key InternalKey, value []byte) error { @@ -1201,8 +1353,8 @@ func (w *Writer) flush(key InternalKey) error { return err } -func (w *Writer) maybeFlush(key InternalKey, value []byte) error { - if !w.dataBlockBuf.shouldFlush(key, len(value), w.blockSize, w.blockSizeThreshold) { +func (w *Writer) maybeFlush(key InternalKey, valueLen int) error { + if !w.dataBlockBuf.shouldFlush(key, valueLen, w.blockSize, w.blockSizeThreshold) { return nil } @@ -1274,11 +1426,11 @@ func (w *Writer) indexEntrySep(prevKey, key InternalKey, dataBlockBuf *dataBlock // they're used when the index block is finished. // // Invariant: -// 1. addIndexEntry must not store references to the sep InternalKey, the tmp -// byte slice, bhp.Props. That is, these must be either deep copied or -// encoded. -// 2. addIndexEntry must not hold references to the flushIndexBuf, and the writeTo -// indexBlockBufs. +// 1. addIndexEntry must not store references to the sep InternalKey, the tmp +// byte slice, bhp.Props. That is, these must be either deep copied or +// encoded. +// 2. addIndexEntry must not hold references to the flushIndexBuf, and the writeTo +// indexBlockBufs. func (w *Writer) addIndexEntry( sep InternalKey, bhp BlockHandleWithProperties, @@ -1323,8 +1475,8 @@ func (w *Writer) addPrevDataBlockToIndexBlockProps() { // aren't being written asynchronously. // // Invariant: -// 1. addIndexEntrySync must not store references to the prevKey, key InternalKey's, -// the tmp byte slice. That is, these must be either deep copied or encoded. +// 1. addIndexEntrySync must not store references to the prevKey, key InternalKey's, +// the tmp byte slice. That is, these must be either deep copied or encoded. func (w *Writer) addIndexEntrySync( prevKey, key InternalKey, bhp BlockHandleWithProperties, tmp []byte, ) error { @@ -1388,25 +1540,22 @@ func shouldFlush( return newSize > targetBlockSize } -const keyAllocSize = 256 << 10 - -func cloneKeyWithBuf(k InternalKey, buf []byte) ([]byte, InternalKey) { +func cloneKeyWithBuf(k InternalKey, a bytealloc.A) (bytealloc.A, InternalKey) { if len(k.UserKey) == 0 { - return buf, k + return a, k } - if len(buf) < len(k.UserKey) { - buf = make([]byte, len(k.UserKey)+keyAllocSize) - } - n := copy(buf, k.UserKey) - return buf[n:], InternalKey{UserKey: buf[:n:n], Trailer: k.Trailer} + a, keyCopy := a.Copy(k.UserKey) + return a, InternalKey{UserKey: keyCopy, Trailer: k.Trailer} } // Invariants: The byte slice returned by finishIndexBlockProps is heap-allocated -// and has its own lifetime, independent of the Writer and the blockPropsEncoder, +// +// and has its own lifetime, independent of the Writer and the blockPropsEncoder, +// // and it is safe to: -// 1. Reuse w.blockPropsEncoder without first encoding the byte slice returned. -// 2. Store the byte slice in the Writer since it is a copy and not supported by -// an underlying buffer. +// 1. Reuse w.blockPropsEncoder without first encoding the byte slice returned. +// 2. Store the byte slice in the Writer since it is a copy and not supported by +// an underlying buffer. func (w *Writer) finishIndexBlockProps() ([]byte, error) { w.blockPropsEncoder.resetProps() for i := range w.blockPropCollectors { @@ -1426,11 +1575,11 @@ func (w *Writer) finishIndexBlockProps() ([]byte, error) { // level index block. This is only used when two level indexes are enabled. // // Invariants: -// 1. The props slice passed into finishedIndexBlock must not be a -// owned by any other struct, since it will be stored in the Writer.indexPartitions -// slice. -// 2. None of the buffers owned by indexBuf will be shallow copied and stored elsewhere. -// That is, it must be safe to reuse indexBuf after finishIndexBlock has been called. +// 1. The props slice passed into finishedIndexBlock must not be a +// owned by any other struct, since it will be stored in the Writer.indexPartitions +// slice. +// 2. None of the buffers owned by indexBuf will be shallow copied and stored elsewhere. +// That is, it must be safe to reuse indexBuf after finishIndexBlock has been called. func (w *Writer) finishIndexBlock(indexBuf *indexBlockBuf, props []byte) error { part := indexBlockAndBlockProperties{ nEntries: indexBuf.block.nEntries, properties: props, @@ -1512,7 +1661,7 @@ func compressAndChecksum(b []byte, compression Compression, blockBuf *blockBuf) func (w *Writer) writeCompressedBlock(block []byte, blockTrailerBuf []byte) (BlockHandle, error) { bh := BlockHandle{Offset: w.meta.Size, Length: uint64(len(block))} - if w.cacheID != 0 && w.fileNum != 0 { + if w.cacheID != 0 && w.fileNum.FileNum() != 0 { // Remove the block being written from the cache. This provides defense in // depth against bugs which cause cache collisions. // @@ -1522,20 +1671,38 @@ func (w *Writer) writeCompressedBlock(block []byte, blockTrailerBuf []byte) (Blo } // Write the bytes to the file. - n, err := w.writer.Write(block) - if err != nil { + if err := w.writable.Write(block); err != nil { return BlockHandle{}, err } - w.meta.Size += uint64(n) - n, err = w.writer.Write(blockTrailerBuf[:blockTrailerLen]) - if err != nil { + w.meta.Size += uint64(len(block)) + if err := w.writable.Write(blockTrailerBuf[:blockTrailerLen]); err != nil { return BlockHandle{}, err } - w.meta.Size += uint64(n) + w.meta.Size += blockTrailerLen return bh, nil } +// Write implements io.Writer. This is analogous to writeCompressedBlock for +// blocks that already incorporate the trailer, and don't need the callee to +// return a BlockHandle. +func (w *Writer) Write(blockWithTrailer []byte) (n int, err error) { + offset := w.meta.Size + if w.cacheID != 0 && w.fileNum.FileNum() != 0 { + // Remove the block being written from the cache. This provides defense in + // depth against bugs which cause cache collisions. + // + // TODO(peter): Alternatively, we could add the uncompressed value to the + // cache. + w.cache.Delete(w.cacheID, w.fileNum, offset) + } + w.meta.Size += uint64(len(blockWithTrailer)) + if err := w.writable.Write(blockWithTrailer); err != nil { + return 0, err + } + return len(blockWithTrailer), nil +} + func (w *Writer) writeBlock( b []byte, compression Compression, blockBuf *blockBuf, ) (BlockHandle, error) { @@ -1562,6 +1729,12 @@ func (w *Writer) assertFormatCompatibility() error { ) } + if (w.props.NumValueBlocks > 0 || w.props.NumValuesInValueBlocks > 0 || + w.props.ValueBlocksSize > 0) && w.tableFormat < TableFormatPebblev3 { + return errors.Newf( + "table format version %s is less than the minimum required version %s for value blocks", + w.tableFormat, TableFormatPebblev3) + } return nil } @@ -1569,21 +1742,28 @@ func (w *Writer) assertFormatCompatibility() error { // table was written to. func (w *Writer) Close() (err error) { defer func() { - if w.syncer == nil { - return + if w.valueBlockWriter != nil { + releaseValueBlockWriter(w.valueBlockWriter) + // Defensive code in case Close gets called again. We don't want to put + // the same object to a sync.Pool. + w.valueBlockWriter = nil } - err1 := w.syncer.Close() - if err == nil { - err = err1 + if w.writable != nil { + w.writable.Abort() + w.writable = nil + } + // Record any error in the writer (so we can exit early if Close is called + // again). + if err != nil { + w.err = err } - w.syncer = nil }() // finish must be called before we check for an error, because finish will // block until every single task added to the writeQueue has been processed, // and an error could be encountered while any of those tasks are processed. - if err = w.coordination.writeQueue.finish(); err != nil { - w.err = err + if err := w.coordination.writeQueue.finish(); err != nil { + return err } if w.err != nil { @@ -1609,17 +1789,14 @@ func (w *Writer) Close() (err error) { if w.dataBlockBuf.dataBlock.nEntries > 0 || w.indexBlock.block.nEntries == 0 { bh, err := w.writeBlock(w.dataBlockBuf.dataBlock.finish(), w.compression, &w.dataBlockBuf.blockBuf) if err != nil { - w.err = err - return w.err + return err } - var bhp BlockHandleWithProperties - if bhp, err = w.maybeAddBlockPropertiesToBlockHandle(bh); err != nil { - w.err = err + bhp, err := w.maybeAddBlockPropertiesToBlockHandle(bh) + if err != nil { return err } prevKey := base.DecodeInternalKey(w.dataBlockBuf.dataBlock.curKey) - if err = w.addIndexEntrySync(prevKey, InternalKey{}, bhp, w.dataBlockBuf.tmp[:]); err != nil { - w.err = err + if err := w.addIndexEntrySync(prevKey, InternalKey{}, bhp, w.dataBlockBuf.tmp[:]); err != nil { return err } } @@ -1631,13 +1808,11 @@ func (w *Writer) Close() (err error) { if w.filter != nil { b, err := w.filter.finish() if err != nil { - w.err = err - return w.err + return err } bh, err := w.writeBlock(b, NoCompression, &w.blockBuf) if err != nil { - w.err = err - return w.err + return err } n := encodeBlockHandle(w.blockBuf.tmp[:], bh) metaindex.add(InternalKey{UserKey: []byte(w.filter.metaName())}, w.blockBuf.tmp[:n]) @@ -1651,8 +1826,7 @@ func (w *Writer) Close() (err error) { // Write the two level index block. indexBH, err = w.writeTwoLevelIndex() if err != nil { - w.err = err - return w.err + return err } } else { w.props.IndexType = binarySearchIndex @@ -1665,8 +1839,7 @@ func (w *Writer) Close() (err error) { // Write the single level index block. indexBH, err = w.writeBlock(w.indexBlock.finish(), w.compression, &w.blockBuf) if err != nil { - w.err = err - return w.err + return err } } @@ -1691,8 +1864,7 @@ func (w *Writer) Close() (err error) { } rangeDelBH, err = w.writeBlock(w.rangeDelBlock.finish(), NoCompression, &w.blockBuf) if err != nil { - w.err = err - return w.err + return err } } @@ -1706,8 +1878,7 @@ func (w *Writer) Close() (err error) { kind := key.Kind() endKey, _, ok := rangekey.DecodeEndKey(kind, w.rangeKeyBlock.curValue) if !ok { - w.err = errors.Newf("invalid end key: %s", w.rangeKeyBlock.curValue) - return w.err + return errors.Newf("invalid end key: %s", w.rangeKeyBlock.curValue) } k := base.MakeExclusiveSentinelKey(kind, endKey).Clone() w.meta.SetLargestRangeKey(k) @@ -1716,8 +1887,21 @@ func (w *Writer) Close() (err error) { // enable compression on this block. rangeKeyBH, err = w.writeBlock(w.rangeKeyBlock.finish(), NoCompression, &w.blockBuf) if err != nil { - w.err = err - return w.err + return err + } + } + + if w.valueBlockWriter != nil { + vbiHandle, vbStats, err := w.valueBlockWriter.finish(w, w.meta.Size) + if err != nil { + return err + } + w.props.NumValueBlocks = vbStats.numValueBlocks + w.props.NumValuesInValueBlocks = vbStats.numValuesInValueBlocks + w.props.ValueBlocksSize = vbStats.valueBlocksAndIndexSize + if vbStats.numValueBlocks > 0 { + n := encodeValueBlocksIndexHandle(w.blockBuf.tmp[:], vbiHandle) + metaindex.add(InternalKey{UserKey: []byte(metaValueIndexName)}, w.blockBuf.tmp[:n]) } } @@ -1734,7 +1918,6 @@ func (w *Writer) Close() (err error) { userProps := make(map[string]string) for i := range w.propCollectors { if err := w.propCollectors[i].Finish(userProps); err != nil { - w.err = err return err } } @@ -1742,10 +1925,8 @@ func (w *Writer) Close() (err error) { scratch := w.blockPropsEncoder.getScratchForProp() // Place the shortID in the first byte. scratch = append(scratch, byte(i)) - buf, err := - w.blockPropCollectors[i].FinishTable(scratch) + buf, err := w.blockPropCollectors[i].FinishTable(scratch) if err != nil { - w.err = err return err } var prop string @@ -1768,11 +1949,10 @@ func (w *Writer) Close() (err error) { // reduces table size without a significant impact on performance. raw.restartInterval = propertiesBlockRestartInterval w.props.CompressionOptions = rocksDBCompressionOptions - w.props.save(&raw) + w.props.save(w.tableFormat, &raw) bh, err := w.writeBlock(raw.finish(), NoCompression, &w.blockBuf) if err != nil { - w.err = err - return w.err + return err } n := encodeBlockHandle(w.blockBuf.tmp[:], bh) metaindex.add(InternalKey{UserKey: []byte(metaPropertiesName)}, w.blockBuf.tmp[:n]) @@ -1798,8 +1978,7 @@ func (w *Writer) Close() (err error) { // expect the meta-index block to not be compressed. metaindexBH, err := w.writeBlock(metaindex.blockWriter.finish(), NoCompression, &w.blockBuf) if err != nil { - w.err = err - return w.err + return err } // Write the table footer. @@ -1809,34 +1988,25 @@ func (w *Writer) Close() (err error) { metaindexBH: metaindexBH, indexBH: indexBH, } - var n int - if n, err = w.writer.Write(footer.encode(w.blockBuf.tmp[:])); err != nil { - w.err = err - return w.err + encoded := footer.encode(w.blockBuf.tmp[:]) + if err := w.writable.Write(footer.encode(w.blockBuf.tmp[:])); err != nil { + return err } - w.meta.Size += uint64(n) + w.meta.Size += uint64(len(encoded)) w.meta.Properties = w.props - // Flush the buffer. - if w.bufWriter != nil { - if err := w.bufWriter.Flush(); err != nil { - w.err = err - return err - } - } - // Check that the features present in the table are compatible with the format // configured for the table. if err = w.assertFormatCompatibility(); err != nil { - w.err = err - return w.err - } - - if err := w.syncer.Sync(); err != nil { - w.err = err return err } + if err := w.writable.Finish(); err != nil { + w.writable = nil + return err + } + w.writable = nil + w.dataBlockBuf.clear() dataBlockBufPool.Put(w.dataBlockBuf) w.dataBlockBuf = nil @@ -1845,9 +2015,6 @@ func (w *Writer) Close() (err error) { w.indexBlock = nil // Make any future calls to Set or Close return an error. - if w.err != nil { - return w.err - } w.err = errWriterClosed return nil } @@ -1863,7 +2030,7 @@ func (w *Writer) EstimatedSize() uint64 { // Metadata returns the metadata for the finished sstable. Only valid to call // after the sstable has been finished. func (w *Writer) Metadata() (*WriterMetadata, error) { - if w.syncer != nil { + if w.writable != nil { return nil, errors.New("pebble: writer is not closed") } return &w.meta, nil @@ -1905,23 +2072,12 @@ func (o *PreviousPointKeyOpt) writerApply(w *Writer) { o.w = w } -// internalTableOpt is a WriterOption that sets properties for sstables being -// created by the db itself (i.e. through flushes and compactions), as opposed -// to those meant for ingestion. -type internalTableOpt struct{} - -func (i internalTableOpt) writerApply(w *Writer) { - // Set the external sst version to 0. This is what RocksDB expects for - // db-internal sstables; otherwise, it could apply a global sequence number. - w.props.ExternalFormatVersion = 0 -} - // NewWriter returns a new table writer for the file. Closing the writer will // close the file. -func NewWriter(f writeCloseSyncer, o WriterOptions, extraOpts ...WriterOption) *Writer { +func NewWriter(writable objstorage.Writable, o WriterOptions, extraOpts ...WriterOption) *Writer { o = o.ensureDefaults() w := &Writer{ - syncer: f, + writable: writable, meta: WriterMetadata{ SmallestSeqNum: math.MaxUint64, }, @@ -1954,6 +2110,14 @@ func NewWriter(f writeCloseSyncer, o WriterOptions, extraOpts ...WriterOption) * Format: o.Comparer.FormatKey, }, } + if w.tableFormat >= TableFormatPebblev3 { + w.shortAttributeExtractor = o.ShortAttributeExtractor + w.requiredInPlaceValueBound = o.RequiredInPlaceValueBound + w.valueBlockWriter = newValueBlockWriter( + w.blockSize, w.blockSizeThreshold, w.compression, w.checksumType, func(compressedSize int) { + w.coordination.sizeEstimate.dataBlockCompressed(compressedSize, 0) + }) + } w.dataBlockBuf = newDataBlockBuf(w.restartInterval, w.checksumType) @@ -1963,13 +2127,13 @@ func NewWriter(f writeCloseSyncer, o WriterOptions, extraOpts ...WriterOption) * w.coordination.init(o.Parallelism, w) - if f == nil { - w.err = errors.New("pebble: nil file") + if writable == nil { + w.err = errors.New("pebble: nil writable") return w } // Note that WriterOptions are applied in two places; the ones with a - // preApply() method are applied here, and the rest are applied after + // preApply() method are applied here. The rest are applied down below after // default properties are set. type preApply interface{ preApply() } for _, opt := range extraOpts { @@ -2038,29 +2202,30 @@ func NewWriter(f writeCloseSyncer, o WriterOptions, extraOpts ...WriterOption) * // Apply the remaining WriterOptions that do not have a preApply() method. for _, opt := range extraOpts { - if _, ok := opt.(preApply); !ok { - opt.writerApply(w) + if _, ok := opt.(preApply); ok { + continue } + opt.writerApply(w) } // Initialize the range key fragmenter and encoder. - w.fragmenter.Emit = w.coalesceSpans + w.fragmenter.Emit = w.encodeRangeKeySpan w.rangeKeyEncoder.Emit = w.addRangeKey - - // If f does not have a Flush method, do our own buffering. - if _, ok := f.(flusher); ok { - w.writer = f - } else { - w.bufWriter = bufio.NewWriter(f) - w.writer = w.bufWriter - } return w } +// internalGetProperties is a private, internal-use-only function that takes a +// Writer and returns a pointer to its Properties, allowing direct mutation. +// It's used by internal Pebble flushes and compactions to set internal +// properties. It gets installed in private. +func internalGetProperties(w *Writer) *Properties { + return &w.props +} + func init() { private.SSTableWriterDisableKeyOrderChecks = func(i interface{}) { w := i.(*Writer) w.disableKeyOrderChecks = true } - private.SSTableInternalTableOpt = internalTableOpt{} + private.SSTableInternalProperties = internalGetProperties } diff --git a/vendor/github.com/cockroachdb/pebble/syncing_fs.go b/vendor/github.com/cockroachdb/pebble/syncing_fs.go deleted file mode 100644 index 2229af4..0000000 --- a/vendor/github.com/cockroachdb/pebble/syncing_fs.go +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package pebble - -import "github.com/cockroachdb/pebble/vfs" - -// syncingFS wraps a vfs.FS with one that wraps newly created files with -// vfs.NewSyncingFile. -type syncingFS struct { - vfs.FS - syncOpts vfs.SyncingFileOptions -} - -func (fs syncingFS) Create(name string) (vfs.File, error) { - f, err := fs.FS.Create(name) - if err != nil { - return nil, err - } - return vfs.NewSyncingFile(f, fs.syncOpts), nil -} diff --git a/vendor/github.com/cockroachdb/pebble/table_cache.go b/vendor/github.com/cockroachdb/pebble/table_cache.go index 88c20c5..9a3c5b5 100644 --- a/vendor/github.com/cockroachdb/pebble/table_cache.go +++ b/vendor/github.com/cockroachdb/pebble/table_cache.go @@ -21,8 +21,9 @@ import ( "github.com/cockroachdb/pebble/internal/keyspan" "github.com/cockroachdb/pebble/internal/manifest" "github.com/cockroachdb/pebble/internal/private" + "github.com/cockroachdb/pebble/objstorage" + "github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing" "github.com/cockroachdb/pebble/sstable" - "github.com/cockroachdb/pebble/vfs" ) var emptyIter = &errorIter{err: nil} @@ -58,19 +59,15 @@ var tableCacheLabels = pprof.Labels("pebble", "table-cache") // are updated, we could have unnecessary evictions of those // fields, and the surrounding fields from the CPU caches. type tableCacheOpts struct { - atomic struct { - // iterCount in the tableCacheOpts keeps track of iterators - // opened or closed by a DB. It's used to keep track of - // leaked iterators on a per-db level. - iterCount *int32 - } + // iterCount keeps track of how many iterators are open. It is used to keep + // track of leaked iterators on a per-db level. + iterCount *atomic.Int32 - logger Logger - cacheID uint64 - dirname string - fs vfs.FS - opts sstable.ReaderOptions - filterMetrics *FilterMetrics + loggerAndTracer LoggerAndTracer + cacheID uint64 + objProvider objstorage.Provider + opts sstable.ReaderOptions + filterMetrics *sstable.FilterMetricsTracker } // tableCacheContainer contains the table cache and @@ -86,7 +83,7 @@ type tableCacheContainer struct { // newTableCacheContainer will panic if the underlying cache in the table cache // doesn't match Options.Cache. func newTableCacheContainer( - tc *TableCache, cacheID uint64, dirname string, fs vfs.FS, opts *Options, size int, + tc *TableCache, cacheID uint64, objProvider objstorage.Provider, opts *Options, size int, ) *tableCacheContainer { // We will release a ref to table cache acquired here when tableCacheContainer.close is called. if tc != nil { @@ -102,13 +99,12 @@ func newTableCacheContainer( t := &tableCacheContainer{} t.tableCache = tc - t.dbOpts.logger = opts.Logger + t.dbOpts.loggerAndTracer = opts.LoggerAndTracer t.dbOpts.cacheID = cacheID - t.dbOpts.dirname = dirname - t.dbOpts.fs = fs + t.dbOpts.objProvider = objProvider t.dbOpts.opts = opts.MakeReaderOptions() - t.dbOpts.filterMetrics = &FilterMetrics{} - t.dbOpts.atomic.iterCount = new(int32) + t.dbOpts.filterMetrics = &sstable.FilterMetricsTracker{} + t.dbOpts.iterCount = new(atomic.Int32) return t } @@ -119,7 +115,7 @@ func (c *tableCacheContainer) close() error { // by the DB using this container. Note that we'll still perform cleanup // below in the case that there are leaked iterators. var err error - if v := atomic.LoadInt32(c.dbOpts.atomic.iterCount); v > 0 { + if v := c.dbOpts.iterCount.Load(); v > 0 { err = errors.Errorf("leaked iterators: %d", errors.Safe(v)) } @@ -133,22 +129,27 @@ func (c *tableCacheContainer) close() error { } func (c *tableCacheContainer) newIters( - file *manifest.FileMetadata, opts *IterOptions, internalOpts internalIterOpts, + ctx context.Context, + file *manifest.FileMetadata, + opts *IterOptions, + internalOpts internalIterOpts, ) (internalIterator, keyspan.FragmentIterator, error) { - return c.tableCache.getShard(file.FileNum).newIters(file, opts, internalOpts, &c.dbOpts) + return c.tableCache.getShard(file.FileBacking.DiskFileNum).newIters(ctx, file, opts, internalOpts, &c.dbOpts) } func (c *tableCacheContainer) newRangeKeyIter( file *manifest.FileMetadata, opts *keyspan.SpanIterOptions, ) (keyspan.FragmentIterator, error) { - return c.tableCache.getShard(file.FileNum).newRangeKeyIter(file, opts, &c.dbOpts) + return c.tableCache.getShard(file.FileBacking.DiskFileNum).newRangeKeyIter(file, opts, &c.dbOpts) } +// getTableProperties returns the properties associated with the backing physical +// table if the input metadata belongs to a virtual sstable. func (c *tableCacheContainer) getTableProperties(file *fileMetadata) (*sstable.Properties, error) { - return c.tableCache.getShard(file.FileNum).getTableProperties(file, &c.dbOpts) + return c.tableCache.getShard(file.FileBacking.DiskFileNum).getTableProperties(file, &c.dbOpts) } -func (c *tableCacheContainer) evict(fileNum FileNum) { +func (c *tableCacheContainer) evict(fileNum base.DiskFileNum) { c.tableCache.getShard(fileNum).evict(fileNum, &c.dbOpts, false) } @@ -159,42 +160,44 @@ func (c *tableCacheContainer) metrics() (CacheMetrics, FilterMetrics) { s.mu.RLock() m.Count += int64(len(s.mu.nodes)) s.mu.RUnlock() - m.Hits += atomic.LoadInt64(&s.atomic.hits) - m.Misses += atomic.LoadInt64(&s.atomic.misses) + m.Hits += s.hits.Load() + m.Misses += s.misses.Load() } m.Size = m.Count * int64(unsafe.Sizeof(sstable.Reader{})) - f := FilterMetrics{ - Hits: atomic.LoadInt64(&c.dbOpts.filterMetrics.Hits), - Misses: atomic.LoadInt64(&c.dbOpts.filterMetrics.Misses), - } + f := c.dbOpts.filterMetrics.Load() return m, f } -func (c *tableCacheContainer) withReader(meta *fileMetadata, fn func(*sstable.Reader) error) error { - s := c.tableCache.getShard(meta.FileNum) - v := s.findNode(meta, &c.dbOpts) +func (c *tableCacheContainer) withReader(meta physicalMeta, fn func(*sstable.Reader) error) error { + s := c.tableCache.getShard(meta.FileBacking.DiskFileNum) + v := s.findNode(meta.FileMetadata, &c.dbOpts) defer s.unrefValue(v) if v.err != nil { - base.MustExist(c.dbOpts.fs, v.filename, c.dbOpts.logger, v.err) return v.err } return fn(v.reader) } +// withVirtualReader fetches a VirtualReader associated with a virtual sstable. +func (c *tableCacheContainer) withVirtualReader( + meta virtualMeta, fn func(sstable.VirtualReader) error, +) error { + s := c.tableCache.getShard(meta.FileBacking.DiskFileNum) + v := s.findNode(meta.FileMetadata, &c.dbOpts) + defer s.unrefValue(v) + if v.err != nil { + return v.err + } + return fn(sstable.MakeVirtualReader(v.reader, meta)) +} + func (c *tableCacheContainer) iterCount() int64 { - return int64(atomic.LoadInt32(c.dbOpts.atomic.iterCount)) + return int64(c.dbOpts.iterCount.Load()) } // TableCache is a shareable cache for open sstables. type TableCache struct { - // atomic contains fields which are accessed atomically. Go allocations - // are guaranteed to be 64-bit aligned which we take advantage of by - // placing the 64-bit fields which we access atomically at the beginning - // of the TableCache struct. For more information, see - // https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - atomic struct { - refs int64 - } + refs atomic.Int64 cache *Cache shards []*tableCacheShard @@ -204,7 +207,7 @@ type TableCache struct { // the table cache only remains valid if there is at least one reference // to it. func (c *TableCache) Ref() { - v := atomic.AddInt64(&c.atomic.refs, 1) + v := c.refs.Add(1) // We don't want the reference count to ever go from 0 -> 1, // cause a reference count of 0 implies that we've closed the cache. if v <= 1 { @@ -214,7 +217,7 @@ func (c *TableCache) Ref() { // Unref removes a reference to the table cache. func (c *TableCache) Unref() error { - v := atomic.AddInt64(&c.atomic.refs, -1) + v := c.refs.Add(-1) switch { case v < 0: panic(fmt.Sprintf("pebble: inconsistent reference count: %d", v)) @@ -256,31 +259,24 @@ func NewTableCache(cache *Cache, numShards int, size int) *TableCache { } // Hold a ref to the cache here. - c.atomic.refs = 1 + c.refs.Store(1) return c } -func (c *TableCache) getShard(fileNum FileNum) *tableCacheShard { - return c.shards[uint64(fileNum)%uint64(len(c.shards))] +func (c *TableCache) getShard(fileNum base.DiskFileNum) *tableCacheShard { + return c.shards[uint64(fileNum.FileNum())%uint64(len(c.shards))] } type tableCacheKey struct { cacheID uint64 - fileNum FileNum + fileNum base.DiskFileNum } type tableCacheShard struct { - // WARNING: The following struct `atomic` contains fields are accessed atomically. - // - // Go allocations are guaranteed to be 64-bit aligned which we take advantage - // of by placing the 64-bit fields which we access atomically at the beginning - // of the DB struct. For more information, see https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - atomic struct { - hits int64 - misses int64 - iterCount int32 - } + hits atomic.Int64 + misses atomic.Int64 + iterCount atomic.Int32 size int @@ -342,25 +338,31 @@ func (c *tableCacheShard) checkAndIntersectFilters( } if boundLimitedFilter != nil || len(blockPropertyFilters) > 0 { - filterer = sstable.NewBlockPropertiesFilterer(blockPropertyFilters, boundLimitedFilter) - intersects, err := - filterer.IntersectsUserPropsAndFinishInit(v.reader.Properties.UserProperties) - if err != nil { + filterer, err = sstable.IntersectsTable( + blockPropertyFilters, + boundLimitedFilter, + v.reader.Properties.UserProperties, + ) + // NB: IntersectsTable will return a nil filterer if the table-level + // properties indicate there's no intersection with the provided filters. + if filterer == nil || err != nil { return false, nil, err } - if !intersects { - return false, nil, nil - } } return true, filterer, nil } func (c *tableCacheShard) newIters( + ctx context.Context, file *manifest.FileMetadata, opts *IterOptions, internalOpts internalIterOpts, dbOpts *tableCacheOpts, ) (internalIterator, keyspan.FragmentIterator, error) { + // TODO(sumeer): constructing the Reader should also use a plumbed context, + // since parts of the sstable are read during the construction. The Reader + // should not remember that context since the Reader can be long-lived. + // Calling findNode gives us the responsibility of decrementing v's // refCount. If opening the underlying table resulted in error, then we // decrement this straight away. Otherwise, we pass that responsibility to @@ -368,7 +370,6 @@ func (c *tableCacheShard) newIters( v := c.findNode(file, dbOpts) if v.err != nil { defer c.unrefValue(v) - base.MustExist(dbOpts.fs, v.filename, dbOpts.logger, v.err) return nil, nil, v.err } @@ -384,9 +385,36 @@ func (c *tableCacheShard) newIters( return nil, nil, err } + type iterCreator interface { + NewRawRangeDelIter() (keyspan.FragmentIterator, error) + NewIterWithBlockPropertyFiltersAndContext( + ctx context.Context, + lower, upper []byte, + filterer *sstable.BlockPropertiesFilterer, + useFilterBlock bool, + stats *base.InternalIteratorStats, + rp sstable.ReaderProvider, + ) (sstable.Iterator, error) + NewCompactionIter( + bytesIterated *uint64, + rp sstable.ReaderProvider, + ) (sstable.Iterator, error) + } + + // TODO(bananabrick): We suffer an allocation if file is a virtual sstable. + var ic iterCreator = v.reader + if file.Virtual { + virtualReader := sstable.MakeVirtualReader( + v.reader, file.VirtualMeta(), + ) + ic = &virtualReader + } + // NB: range-del iterator does not maintain a reference to the table, nor // does it need to read from it after creation. - rangeDelIter, err := v.reader.NewRawRangeDelIter() + var rangeDelIter keyspan.FragmentIterator + rangeDelIter, err = ic.NewRawRangeDelIter() + if err != nil { c.unrefValue(v) return nil, nil, err @@ -412,12 +440,25 @@ func (c *tableCacheShard) newIters( useFilter := true if opts != nil { useFilter = manifest.LevelToInt(opts.level) != 6 || opts.UseL6Filters + ctx = objiotracing.WithLevel(ctx, manifest.LevelToInt(opts.level)) } + tableFormat, err := v.reader.TableFormat() + if err != nil { + return nil, nil, err + } + var rp sstable.ReaderProvider + if tableFormat >= sstable.TableFormatPebblev3 && v.reader.Properties.NumValueBlocks > 0 { + rp = &tableCacheShardReaderProvider{c: c, file: file, dbOpts: dbOpts} + } + if internalOpts.bytesIterated != nil { - iter, err = v.reader.NewCompactionIter(internalOpts.bytesIterated) + iter, err = ic.NewCompactionIter(internalOpts.bytesIterated, rp) } else { - iter, err = v.reader.NewIterWithBlockPropertyFilters( - opts.GetLowerBound(), opts.GetUpperBound(), filterer, useFilter, internalOpts.stats) + iter, err = ic.NewIterWithBlockPropertyFiltersAndContext( + ctx, + opts.GetLowerBound(), opts.GetUpperBound(), + filterer, useFilter, internalOpts.stats, rp, + ) } if err != nil { if rangeDelIter != nil { @@ -427,11 +468,11 @@ func (c *tableCacheShard) newIters( return nil, nil, err } // NB: v.closeHook takes responsibility for calling unrefValue(v) here. Take - // care to avoid introduceingan allocation here by adding a closure. + // care to avoid introducing an allocation here by adding a closure. iter.SetCloseHook(v.closeHook) - atomic.AddInt32(&c.atomic.iterCount, 1) - atomic.AddInt32(dbOpts.atomic.iterCount, 1) + c.iterCount.Add(1) + dbOpts.iterCount.Add(1) if invariants.RaceEnabled { c.mu.Lock() c.mu.iters[iter] = debug.Stack() @@ -450,7 +491,6 @@ func (c *tableCacheShard) newRangeKeyIter( v := c.findNode(file, dbOpts) if v.err != nil { defer c.unrefValue(v) - base.MustExist(dbOpts.fs, v.filename, dbOpts.logger, v.err) return nil, v.err } @@ -476,18 +516,73 @@ func (c *tableCacheShard) newRangeKeyIter( } var iter keyspan.FragmentIterator - iter, err = v.reader.NewRawRangeKeyIter() + if file.Virtual { + virtualReader := sstable.MakeVirtualReader( + v.reader, file.VirtualMeta(), + ) + iter, err = virtualReader.NewRawRangeKeyIter() + } else { + iter, err = v.reader.NewRawRangeKeyIter() + } + // iter is a block iter that holds the entire value of the block in memory. // No need to hold onto a ref of the cache value. c.unrefValue(v) - if err != nil || iter == nil { + if err != nil { return nil, err } + if iter == nil { + // NewRawRangeKeyIter can return nil even if there's no error. However, + // the keyspan.LevelIter expects a non-nil iterator if err is nil. + return emptyKeyspanIter, nil + } + return iter, nil } +type tableCacheShardReaderProvider struct { + c *tableCacheShard + file *manifest.FileMetadata + dbOpts *tableCacheOpts + v *tableCacheValue +} + +var _ sstable.ReaderProvider = &tableCacheShardReaderProvider{} + +// GetReader implements sstable.ReaderProvider. Note that it is not the +// responsibility of tableCacheShardReaderProvider to ensure that the file +// continues to exist. The ReaderProvider is used in iterators where the +// top-level iterator is pinning the read state and preventing the files from +// being deleted. +// +// The caller must call tableCacheShardReaderProvider.Close. +// +// Note that currently the Reader returned here is only used to read value +// blocks. This reader shouldn't be used for other purposes like reading keys +// outside of virtual sstable bounds. +// +// TODO(bananabrick): We could return a wrapper over the Reader to ensure +// that the reader isn't used for other purposes. +func (rp *tableCacheShardReaderProvider) GetReader() (*sstable.Reader, error) { + // Calling findNode gives us the responsibility of decrementing v's + // refCount. + v := rp.c.findNode(rp.file, rp.dbOpts) + if v.err != nil { + defer rp.c.unrefValue(v) + return nil, v.err + } + rp.v = v + return v.reader, nil +} + +// Close implements sstable.ReaderProvider. +func (rp *tableCacheShardReaderProvider) Close() { + rp.c.unrefValue(rp.v) + rp.v = nil +} + // getTableProperties return sst table properties for target file func (c *tableCacheShard) getTableProperties( file *fileMetadata, dbOpts *tableCacheOpts, @@ -515,7 +610,7 @@ func (c *tableCacheShard) releaseNode(n *tableCacheNode) { // // c.mu must be held when calling this. func (c *tableCacheShard) unlinkNode(n *tableCacheNode) { - key := tableCacheKey{n.cacheID, n.meta.FileNum} + key := tableCacheKey{n.cacheID, n.fileNum} delete(c.mu.nodes, key) switch n.ptype { @@ -560,7 +655,7 @@ func (c *tableCacheShard) clearNode(n *tableCacheNode) { // it is present in tableCacheShard.mu.nodes, so a reference count of 0 means // the node has already been removed from that map. func (c *tableCacheShard) unrefValue(v *tableCacheValue) { - if atomic.AddInt32(&v.refCount, -1) == 0 { + if v.refCount.Add(-1) == 0 { c.releasing.Add(1) c.releasingCh <- v } @@ -572,16 +667,16 @@ func (c *tableCacheShard) unrefValue(v *tableCacheValue) { func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) *tableCacheValue { // Fast-path for a hit in the cache. c.mu.RLock() - key := tableCacheKey{dbOpts.cacheID, meta.FileNum} + key := tableCacheKey{dbOpts.cacheID, meta.FileBacking.DiskFileNum} if n := c.mu.nodes[key]; n != nil && n.value != nil { // Fast-path hit. // // The caller is responsible for decrementing the refCount. v := n.value - atomic.AddInt32(&v.refCount, 1) + v.refCount.Add(1) c.mu.RUnlock() - atomic.StoreInt32(&n.referenced, 1) - atomic.AddInt64(&c.atomic.hits, 1) + n.referenced.Store(true) + c.hits.Add(1) <-v.loaded return v } @@ -594,8 +689,8 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * case n == nil: // Slow-path miss of a non-existent node. n = &tableCacheNode{ - meta: meta, - ptype: tableCacheNodeCold, + fileNum: meta.FileBacking.DiskFileNum, + ptype: tableCacheNodeCold, } c.addNode(n, dbOpts) c.mu.sizeCold++ @@ -605,9 +700,9 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * // // The caller is responsible for decrementing the refCount. v := n.value - atomic.AddInt32(&v.refCount, 1) - atomic.StoreInt32(&n.referenced, 1) - atomic.AddInt64(&c.atomic.hits, 1) + v.refCount.Add(1) + n.referenced.Store(true) + c.hits.Add(1) c.mu.Unlock() <-v.loaded return v @@ -620,18 +715,18 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * c.mu.coldTarget = c.size } - atomic.StoreInt32(&n.referenced, 0) + n.referenced.Store(false) n.ptype = tableCacheNodeHot c.addNode(n, dbOpts) c.mu.sizeHot++ } - atomic.AddInt64(&c.atomic.misses, 1) + c.misses.Add(1) v := &tableCacheValue{ - loaded: make(chan struct{}), - refCount: 2, + loaded: make(chan struct{}), } + v.refCount.Store(2) // Cache the closure invoked when an iterator is closed. This avoids an // allocation on every call to newIters. v.closeHook = func(i sstable.Iterator) error { @@ -641,8 +736,8 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * c.mu.Unlock() } c.unrefValue(v) - atomic.AddInt32(&c.atomic.iterCount, -1) - atomic.AddInt32(dbOpts.atomic.iterCount, -1) + c.iterCount.Add(-1) + dbOpts.iterCount.Add(-1) return nil } n.value = v @@ -652,7 +747,12 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * // Note adding to the cache lists must complete before we begin loading the // table as a failure during load will result in the node being unlinked. pprof.Do(context.Background(), tableCacheLabels, func(context.Context) { - v.load(meta, c, dbOpts) + v.load( + loadInfo{ + backingFileNum: meta.FileBacking.DiskFileNum, + smallestSeqNum: meta.SmallestSeqNum, + largestSeqNum: meta.LargestSeqNum, + }, c, dbOpts) }) return v } @@ -660,7 +760,7 @@ func (c *tableCacheShard) findNode(meta *fileMetadata, dbOpts *tableCacheOpts) * func (c *tableCacheShard) addNode(n *tableCacheNode, dbOpts *tableCacheOpts) { c.evictNodes() n.cacheID = dbOpts.cacheID - key := tableCacheKey{n.cacheID, n.meta.FileNum} + key := tableCacheKey{n.cacheID, n.fileNum} c.mu.nodes[key] = n n.links.next = n @@ -688,8 +788,8 @@ func (c *tableCacheShard) evictNodes() { func (c *tableCacheShard) runHandCold() { n := c.mu.handCold if n.ptype == tableCacheNodeCold { - if atomic.LoadInt32(&n.referenced) == 1 { - atomic.StoreInt32(&n.referenced, 0) + if n.referenced.Load() { + n.referenced.Store(false) n.ptype = tableCacheNodeHot c.mu.sizeCold-- c.mu.sizeHot++ @@ -721,8 +821,8 @@ func (c *tableCacheShard) runHandHot() { n := c.mu.handHot if n.ptype == tableCacheNodeHot { - if atomic.LoadInt32(&n.referenced) == 1 { - atomic.StoreInt32(&n.referenced, 0) + if n.referenced.Load() { + n.referenced.Store(false) } else { n.ptype = tableCacheNodeCold c.mu.sizeHot-- @@ -754,7 +854,7 @@ func (c *tableCacheShard) runHandTest() { c.mu.handTest = c.mu.handTest.next() } -func (c *tableCacheShard) evict(fileNum FileNum, dbOpts *tableCacheOpts, allowLeak bool) { +func (c *tableCacheShard) evict(fileNum base.DiskFileNum, dbOpts *tableCacheOpts, allowLeak bool) { c.mu.Lock() key := tableCacheKey{dbOpts.cacheID, fileNum} n := c.mu.nodes[key] @@ -769,8 +869,8 @@ func (c *tableCacheShard) evict(fileNum FileNum, dbOpts *tableCacheOpts, allowLe v = n.value if v != nil { if !allowLeak { - if t := atomic.AddInt32(&v.refCount, -1); t != 0 { - dbOpts.logger.Fatalf("sstable %s: refcount is not zero: %d\n%s", fileNum, t, debug.Stack()) + if t := v.refCount.Add(-1); t != 0 { + dbOpts.loggerAndTracer.Fatalf("sstable %s: refcount is not zero: %d\n%s", fileNum, t, debug.Stack()) } } c.releasing.Add(1) @@ -790,7 +890,7 @@ func (c *tableCacheShard) evict(fileNum FileNum, dbOpts *tableCacheOpts, allowLe // associated with dbOpts.cacheID. Make sure that there will // be no more accesses to the files associated with the DB. func (c *tableCacheShard) removeDB(dbOpts *tableCacheOpts) { - var fileNums []base.FileNum + var fileNums []base.DiskFileNum c.mu.RLock() // Collect the fileNums which need to be cleaned. @@ -802,7 +902,7 @@ func (c *tableCacheShard) removeDB(dbOpts *tableCacheOpts) { } if node.cacheID == dbOpts.cacheID { - fileNums = append(fileNums, node.meta.FileNum) + fileNums = append(fileNums, node.fileNum) } node = node.next() } @@ -811,8 +911,8 @@ func (c *tableCacheShard) removeDB(dbOpts *tableCacheOpts) { // Evict all the nodes associated with the DB. // This should synchronously close all the files // associated with the DB. - for _, fNum := range fileNums { - c.evict(fNum, dbOpts, true) + for _, fileNum := range fileNums { + c.evict(fileNum, dbOpts, true) } } @@ -823,7 +923,7 @@ func (c *tableCacheShard) Close() error { // Check for leaked iterators. Note that we'll still perform cleanup below in // the case that there are leaked iterators. var err error - if v := atomic.LoadInt32(&c.atomic.iterCount); v > 0 { + if v := c.iterCount.Load(); v > 0 { if !invariants.RaceEnabled { err = errors.Errorf("leaked iterators: %d", errors.Safe(v)) } else { @@ -838,7 +938,7 @@ func (c *tableCacheShard) Close() error { for c.mu.handHot != nil { n := c.mu.handHot if n.value != nil { - if atomic.AddInt32(&n.value.refCount, -1) == 0 { + if n.value.refCount.Add(-1) == 0 { c.releasing.Add(1) c.releasingCh <- n.value } @@ -870,27 +970,32 @@ func (c *tableCacheShard) Close() error { type tableCacheValue struct { closeHook func(i sstable.Iterator) error reader *sstable.Reader - filename string err error loaded chan struct{} // Reference count for the value. The reader is closed when the reference // count drops to zero. - refCount int32 + refCount atomic.Int32 } -func (v *tableCacheValue) load(meta *fileMetadata, c *tableCacheShard, dbOpts *tableCacheOpts) { - // Try opening the fileTypeTable first. - var f vfs.File - v.filename = base.MakeFilepath(dbOpts.fs, dbOpts.dirname, fileTypeTable, meta.FileNum) - f, v.err = dbOpts.fs.Open(v.filename, vfs.RandomReadsOption) +type loadInfo struct { + backingFileNum base.DiskFileNum + largestSeqNum uint64 + smallestSeqNum uint64 +} + +func (v *tableCacheValue) load(loadInfo loadInfo, c *tableCacheShard, dbOpts *tableCacheOpts) { + // Try opening the file first. + var f objstorage.Readable + f, v.err = dbOpts.objProvider.OpenForReading( + context.TODO(), fileTypeTable, loadInfo.backingFileNum, objstorage.OpenOptions{MustExist: true}, + ) if v.err == nil { - cacheOpts := private.SSTableCacheOpts(dbOpts.cacheID, meta.FileNum).(sstable.ReaderOption) - reopenOpt := sstable.FileReopenOpt{FS: dbOpts.fs, Filename: v.filename} - v.reader, v.err = sstable.NewReader(f, dbOpts.opts, cacheOpts, dbOpts.filterMetrics, reopenOpt) + cacheOpts := private.SSTableCacheOpts(dbOpts.cacheID, loadInfo.backingFileNum).(sstable.ReaderOption) + v.reader, v.err = sstable.NewReader(f, dbOpts.opts, cacheOpts, dbOpts.filterMetrics) } if v.err == nil { - if meta.SmallestSeqNum == meta.LargestSeqNum { - v.reader.Properties.GlobalSeqNum = meta.LargestSeqNum + if loadInfo.smallestSeqNum == loadInfo.largestSeqNum { + v.reader.Properties.GlobalSeqNum = loadInfo.largestSeqNum } } if v.err != nil { @@ -898,7 +1003,7 @@ func (v *tableCacheValue) load(meta *fileMetadata, c *tableCacheShard, dbOpts *t defer c.mu.Unlock() // Lookup the node in the cache again as it might have already been // removed. - key := tableCacheKey{dbOpts.cacheID, meta.FileNum} + key := tableCacheKey{dbOpts.cacheID, loadInfo.backingFileNum} n := c.mu.nodes[key] if n != nil && n.value == v { c.releaseNode(n) @@ -938,8 +1043,8 @@ func (p tableCacheNodeType) String() string { } type tableCacheNode struct { - meta *fileMetadata - value *tableCacheValue + fileNum base.DiskFileNum + value *tableCacheValue links struct { next *tableCacheNode @@ -948,7 +1053,7 @@ type tableCacheNode struct { ptype tableCacheNodeType // referenced is atomically set to indicate that this entry has been accessed // since the last time one of the clock hands swept it. - referenced int32 + referenced atomic.Bool // Storing the cache id associated with the DB instance here // avoids the need to thread the dbOpts struct through many functions. diff --git a/vendor/github.com/cockroachdb/pebble/table_stats.go b/vendor/github.com/cockroachdb/pebble/table_stats.go index 5a6571f..dadcabf 100644 --- a/vendor/github.com/cockroachdb/pebble/table_stats.go +++ b/vendor/github.com/cockroachdb/pebble/table_stats.go @@ -53,7 +53,7 @@ func (d *DB) maybeCollectTableStatsLocked() { func (d *DB) updateTableStatsLocked(newFiles []manifest.NewFileEntry) { var needStats bool for _, nf := range newFiles { - if !nf.Meta.StatsValidLocked() { + if !nf.Meta.StatsValid() { needStats = true break } @@ -131,7 +131,7 @@ func (d *DB) collectTableStats() bool { } d.mu.tableStats.cond.Broadcast() d.maybeCollectTableStatsLocked() - if len(hints) > 0 { + if len(hints) > 0 && !d.opts.private.disableDeleteOnlyCompactions { // Verify that all of the hint tombstones' files still exist in the // current version. Otherwise, the tombstone itself may have been // compacted into L6 and more recent keys may have had their sequence @@ -175,7 +175,7 @@ func (d *DB) loadNewFileStats( // collectTableStats updates f.Stats for active files, and we // ensure only one goroutine runs it at a time through // d.mu.tableStats.loading. - if nf.Meta.StatsValidLocked() { + if nf.Meta.StatsValid() { continue } @@ -186,7 +186,10 @@ func (d *DB) loadNewFileStats( continue } - stats, newHints, err := d.loadTableStats(rs.current, nf.Level, nf.Meta) + stats, newHints, err := d.loadTableStats( + rs.current, nf.Level, + nf.Meta.PhysicalMeta(), + ) if err != nil { d.opts.EventListener.BackgroundError(err) continue @@ -219,7 +222,7 @@ func (d *DB) scanReadStateTableStats( // and we ensure only one goroutine runs it at a time through // d.mu.tableStats.loading. This makes it safe to read validity // through f.Stats.ValidLocked despite not holding d.mu. - if f.StatsValidLocked() { + if f.StatsValid() { continue } @@ -233,7 +236,9 @@ func (d *DB) scanReadStateTableStats( return fill, hints, moreRemain } - stats, newHints, err := d.loadTableStats(rs.current, l, f) + stats, newHints, err := d.loadTableStats( + rs.current, l, f.PhysicalMeta(), + ) if err != nil { // Set `moreRemain` so we'll try again. moreRemain = true @@ -250,30 +255,38 @@ func (d *DB) scanReadStateTableStats( return fill, hints, moreRemain } +// loadTableStats currently only supports stats collection for physical +// sstables. +// +// TODO(bananabrick): Support stats collection for virtual sstables. func (d *DB) loadTableStats( - v *version, level int, meta *fileMetadata, + v *version, level int, meta physicalMeta, ) (manifest.TableStats, []deleteCompactionHint, error) { var stats manifest.TableStats var compactionHints []deleteCompactionHint - err := d.tableCache.withReader(meta, func(r *sstable.Reader) (err error) { - stats.NumEntries = r.Properties.NumEntries - stats.NumDeletions = r.Properties.NumDeletions - if r.Properties.NumPointDeletions() > 0 { - if err = d.loadTablePointKeyStats(r, v, level, meta, &stats); err != nil { - return + err := d.tableCache.withReader( + meta, func(r *sstable.Reader) (err error) { + stats.NumEntries = r.Properties.NumEntries + stats.NumDeletions = r.Properties.NumDeletions + if r.Properties.NumPointDeletions() > 0 { + if err = d.loadTablePointKeyStats(r, v, level, meta, &stats); err != nil { + return + } } - } - if r.Properties.NumRangeDeletions > 0 || r.Properties.NumRangeKeyDels > 0 { - if compactionHints, err = d.loadTableRangeDelStats(r, v, level, meta, &stats); err != nil { - return + if r.Properties.NumRangeDeletions > 0 || r.Properties.NumRangeKeyDels > 0 { + if compactionHints, err = d.loadTableRangeDelStats( + r, v, level, meta, &stats, + ); err != nil { + return + } } - } - // TODO(travers): Once we have real-world data, consider collecting - // additional stats that may provide improved heuristics for compaction - // picking. - stats.NumRangeKeySets = r.Properties.NumRangeKeySets - return - }) + // TODO(travers): Once we have real-world data, consider collecting + // additional stats that may provide improved heuristics for compaction + // picking. + stats.NumRangeKeySets = r.Properties.NumRangeKeySets + stats.ValueBlocksSize = r.Properties.ValueBlocksSize + return + }) if err != nil { return stats, nil, err } @@ -283,7 +296,7 @@ func (d *DB) loadTableStats( // loadTablePointKeyStats calculates the point key statistics for the given // table. The provided manifest.TableStats are updated. func (d *DB) loadTablePointKeyStats( - r *sstable.Reader, v *version, level int, meta *fileMetadata, stats *manifest.TableStats, + r *sstable.Reader, v *version, level int, meta physicalMeta, stats *manifest.TableStats, ) error { // TODO(jackson): If the file has a wide keyspace, the average // value size beneath the entire file might not be representative @@ -291,21 +304,21 @@ func (d *DB) loadTablePointKeyStats( // We could write the ranges of 'clusters' of point tombstones to // a sstable property and call averageValueSizeBeneath for each of // these narrower ranges to improve the estimate. - avgKeySize, avgValSize, err := d.averageEntrySizeBeneath(v, level, meta) + avgValSize, err := d.averagePhysicalValueSizeBeneath(v, level, meta) if err != nil { return err } stats.PointDeletionsBytesEstimate = - pointDeletionsBytesEstimate(&r.Properties, avgKeySize, avgValSize) + pointDeletionsBytesEstimate(meta.Size, &r.Properties, avgValSize) return nil } // loadTableRangeDelStats calculates the range deletion and range key deletion // statistics for the given table. func (d *DB) loadTableRangeDelStats( - r *sstable.Reader, v *version, level int, meta *fileMetadata, stats *manifest.TableStats, + r *sstable.Reader, v *version, level int, meta physicalMeta, stats *manifest.TableStats, ) ([]deleteCompactionHint, error) { - iter, err := newCombinedDeletionKeyspanIter(d.opts.Comparer, r, meta) + iter, err := newCombinedDeletionKeyspanIter(d.opts.Comparer, r, meta.FileMetadata) if err != nil { return nil, err } @@ -319,25 +332,50 @@ func (d *DB) loadTableRangeDelStats( for s := iter.First(); s != nil; s = iter.Next() { start, end := s.Start, s.End // We only need to consider deletion size estimates for tables that contain - // point keys. - var hasPoints bool + // RANGEDELs. + var maxRangeDeleteSeqNum uint64 for _, k := range s.Keys { - if k.Kind() == base.InternalKeyKindRangeDelete { - hasPoints = true + if k.Kind() == base.InternalKeyKindRangeDelete && maxRangeDeleteSeqNum < k.SeqNum() { + maxRangeDeleteSeqNum = k.SeqNum() break } } // If the file is in the last level of the LSM, there is no data beneath // it. The fact that there is still a range tombstone in a bottommost file - // suggests that an open snapshot kept the tombstone around. Estimate disk - // usage within the file itself. + // indicates two possibilites: + // 1. an open snapshot kept the tombstone around, and the data the + // tombstone deletes is contained within the file itself. + // 2. the file was ingested. + // In the first case, we'd like to estimate disk usage within the file + // itself since compacting the file will drop that covered data. In the + // second case, we expect that compacting the file will NOT drop any + // data and rewriting the file is a waste of write bandwidth. We can + // distinguish these cases by looking at the file metadata's sequence + // numbers. A file's range deletions can only delete data within the + // file at lower sequence numbers. All keys in an ingested sstable adopt + // the same sequence number, preventing tombstones from deleting keys + // within the same file. We check here if the largest RANGEDEL sequence + // number is greater than the file's smallest sequence number. If it is, + // the RANGEDEL could conceivably (although inconclusively) delete data + // within the same file. + // + // Note that this heuristic is imperfect. If a table containing a range + // deletion is ingested into L5 and subsequently compacted into L6 but + // an open snapshot prevents elision of covered keys in L6, the + // resulting RangeDeletionsBytesEstimate will incorrectly include all + // covered keys. + // + // TODO(jackson): We could prevent the above error in the heuristic by + // computing the file's RangeDeletionsBytesEstimate during the + // compaction itself. It's unclear how common this is. + // // NOTE: If the span `s` wholly contains a table containing range keys, // the returned size estimate will be slightly inflated by the range key // block. However, in practice, range keys are expected to be rare, and // the size of the range key block relative to the overall size of the // table is expected to be small. - if hasPoints && level == numLevels-1 { + if level == numLevels-1 && meta.SmallestSeqNum < maxRangeDeleteSeqNum { size, err := r.EstimateDiskUsage(start, end) if err != nil { return nil, err @@ -369,7 +407,7 @@ func (d *DB) loadTableRangeDelStats( hintType: hintType, start: make([]byte, len(start)), end: make([]byte, len(end)), - tombstoneFile: meta, + tombstoneFile: meta.FileMetadata, tombstoneLevel: level, tombstoneLargestSeqNum: s.LargestSeqNum(), tombstoneSmallestSeqNum: s.SmallestSeqNum(), @@ -382,9 +420,9 @@ func (d *DB) loadTableRangeDelStats( return compactionHints, err } -func (d *DB) averageEntrySizeBeneath( - v *version, level int, meta *fileMetadata, -) (avgKeySize, avgValueSize uint64, err error) { +func (d *DB) averagePhysicalValueSizeBeneath( + v *version, level int, meta physicalMeta, +) (avgValueSize uint64, err error) { // Find all files in lower levels that overlap with meta, // summing their value sizes and entry counts. var fileSum, keySum, valSum, entryCount uint64 @@ -393,39 +431,52 @@ func (d *DB) averageEntrySizeBeneath( meta.Largest.UserKey, meta.Largest.IsExclusiveSentinel()) iter := overlaps.Iter() for file := iter.First(); file != nil; file = iter.Next() { - err := d.tableCache.withReader(file, func(r *sstable.Reader) (err error) { - fileSum += file.Size - entryCount += r.Properties.NumEntries - keySum += r.Properties.RawKeySize - valSum += r.Properties.RawValueSize - return nil - }) + var err error + if file.Virtual { + err = d.tableCache.withVirtualReader( + file.VirtualMeta(), + func(v sstable.VirtualReader) (err error) { + fileSum += file.Size + entryCount += file.Stats.NumEntries + keySum += v.Properties.RawKeySize + valSum += v.Properties.RawValueSize + return nil + }) + } else { + err = d.tableCache.withReader( + file.PhysicalMeta(), + func(r *sstable.Reader) (err error) { + fileSum += file.Size + entryCount += r.Properties.NumEntries + keySum += r.Properties.RawKeySize + valSum += r.Properties.RawValueSize + return nil + }) + } + if err != nil { - return 0, 0, err + return 0, err } } } if entryCount == 0 { - return 0, 0, nil + return 0, nil } - // RawKeySize and RawValueSize are uncompressed totals. Scale them + // RawKeySize and RawValueSize are uncompressed totals. Scale the value sum // according to the data size to account for compression, index blocks and // metadata overhead. Eg: // - // Compression rate × Average uncompressed key size + // Compression rate × Average uncompressed value size // // ↓ // - // FileSize RawKeySize - // ----------------------- × ---------- + // FileSize RawValueSize + // ----------------------- × ------------ // RawKeySize+RawValueSize NumEntries - // - // We refactor the calculation to avoid error from rounding/truncation. - totalSizePerEntry := fileSum / entryCount - uncompressedSum := keySum + valSum - avgKeySize = keySum * totalSizePerEntry / uncompressedSum - avgValueSize = valSum * totalSizePerEntry / uncompressedSum - return avgKeySize, avgValueSize, err + uncompressedSum := float64(keySum + valSum) + compressionRatio := float64(fileSum) / uncompressedSum + avgCompressedValueSize := (float64(valSum) / float64(entryCount)) * compressionRatio + return uint64(avgCompressedValueSize), nil } func (d *DB) estimateReclaimedSizeBeneath( @@ -501,10 +552,21 @@ func (d *DB) estimateReclaimedSizeBeneath( continue } var size uint64 - err := d.tableCache.withReader(file, func(r *sstable.Reader) (err error) { - size, err = r.EstimateDiskUsage(start, end) - return err - }) + var err error + if file.Virtual { + err = d.tableCache.withVirtualReader( + file.VirtualMeta(), func(r sstable.VirtualReader) (err error) { + size, err = r.EstimateDiskUsage(start, end) + return err + }) + } else { + err = d.tableCache.withReader( + file.PhysicalMeta(), func(r *sstable.Reader) (err error) { + size, err = r.EstimateDiskUsage(start, end) + return err + }) + } + if err != nil { return 0, hintSeqNum, err } @@ -515,7 +577,7 @@ func (d *DB) estimateReclaimedSizeBeneath( return estimate, hintSeqNum, nil } -func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties) bool { +func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) bool { // If a table contains range deletions or range key deletions, we defer the // stats collection. There are two main reasons for this: // @@ -546,8 +608,8 @@ func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties) // doesn't require any additional IO and since the number of point // deletions in the file is low, the error introduced by this crude // estimate is expected to be small. - avgKeySize, avgValSize := estimateEntrySizes(meta.Size, props) - pointEstimate = pointDeletionsBytesEstimate(props, avgKeySize, avgValSize) + avgValSize := estimateValuePhysicalSize(meta.Size, props) + pointEstimate = pointDeletionsBytesEstimate(meta.Size, props, avgValSize) } meta.Stats.NumEntries = props.NumEntries @@ -555,51 +617,81 @@ func maybeSetStatsFromProperties(meta *fileMetadata, props *sstable.Properties) meta.Stats.NumRangeKeySets = props.NumRangeKeySets meta.Stats.PointDeletionsBytesEstimate = pointEstimate meta.Stats.RangeDeletionsBytesEstimate = 0 + meta.Stats.ValueBlocksSize = props.ValueBlocksSize meta.StatsMarkValid() return true } -func pointDeletionsBytesEstimate(props *sstable.Properties, avgKeySize, avgValSize uint64) uint64 { +func pointDeletionsBytesEstimate( + fileSize uint64, props *sstable.Properties, avgValPhysicalSize uint64, +) (estimate uint64) { if props.NumEntries == 0 { return 0 } - // Estimate the potential space to reclaim using the table's own - // properties. There may or may not be keys covered by any individual - // point tombstone. If not, compacting the point tombstone into L6 will at - // least allow us to drop the point deletion key and will reclaim the key - // bytes. If there are covered key(s), we also get to drop key and value - // bytes for each covered key. + numPointDels := props.NumPointDeletions() + if numPointDels == 0 { + return 0 + } + // Estimate the potential space to reclaim using the table's own properties. + // There may or may not be keys covered by any individual point tombstone. + // If not, compacting the point tombstone into L6 will at least allow us to + // drop the point deletion key and will reclaim the tombstone's key bytes. + // If there are covered key(s), we also get to drop key and value bytes for + // each covered key. // // We estimate assuming that each point tombstone on average covers 1 key. // This is almost certainly an overestimate, but that's probably okay // because point tombstones can slow range iterations even when they don't // cover a key. It may be beneficial in the future to more accurately // estimate which tombstones cover keys and which do not. - numPointDels := props.NumPointDeletions() - return numPointDels*avgKeySize + numPointDels*(avgKeySize+avgValSize) + + // Calculate the contribution of the key sizes: 1x for the point tombstone + // itself and 1x for the key that's expected to exist lower in the LSM. + var keysLogicalSize uint64 + if props.RawPointTombstoneKeySize > 0 { + // This table has a RawPointTombstoneKeySize property, so we know the + // exact size of the logical, uncompressed bytes of the point tombstone + // keys. + keysLogicalSize = 2 * props.RawPointTombstoneKeySize + } else { + // This sstable predates the existence of the RawPointTombstoneKeySize + // property. We can use the average key size within the file itself and + // the count of point deletions to estimate the size. + keysLogicalSize = 2 * numPointDels * props.RawKeySize / props.NumEntries + } + // Scale the logical key size by the logical:physical ratio of the file to + // account for compression, metadata overhead, etc. + // + // Physical FileSize + // ----------- = ----------------------- + // Logical RawKeySize+RawValueSize + // + estimate += (keysLogicalSize * fileSize) / (props.RawKeySize + props.RawValueSize) + + // Calculate the contribution of the deleted values. The caller has already + // computed an average physical size (possibly comptued across many + // sstables), so there's no need to scale it. + estimate += numPointDels * avgValPhysicalSize + return estimate } -func estimateEntrySizes( - fileSize uint64, props *sstable.Properties, -) (avgKeySize, avgValSize uint64) { - // RawKeySize and RawValueSize are uncompressed totals. Scale them - // according to the data size to account for compression, index blocks and - // metadata overhead. Eg: +func estimateValuePhysicalSize(fileSize uint64, props *sstable.Properties) (avgValSize uint64) { + // RawKeySize and RawValueSize are uncompressed totals. Scale according to + // the data size to account for compression, index blocks and metadata + // overhead. Eg: // - // Compression rate × Average uncompressed key size + // Compression rate × Average uncompressed value size // // ↓ // - // FileSize RawKeySize + // FileSize RawValSize // ----------------------- × ---------- // RawKeySize+RawValueSize NumEntries // - // We refactor the calculation to avoid error from rounding/truncation. - fileSizePerEntry := fileSize / props.NumEntries uncompressedSum := props.RawKeySize + props.RawValueSize - avgKeySize = props.RawKeySize * fileSizePerEntry / uncompressedSum - avgValSize = props.RawValueSize * fileSizePerEntry / uncompressedSum - return avgKeySize, avgValSize + compressionRatio := float64(fileSize) / float64(uncompressedSum) + avgCompressedValueSize := (float64(props.RawValueSize) / float64(props.NumEntries)) * compressionRatio + return uint64(avgCompressedValueSize) } // newCombinedDeletionKeyspanIter returns a keyspan.FragmentIterator that @@ -611,20 +703,20 @@ func estimateEntrySizes( // As an example, consider the following set of spans from the range deletion // and range key blocks of a table: // -// |---------| |---------| |-------| RANGEKEYDELs -// |-----------|-------------| |-----| RANGEDELs -// __________________________________________________________ -// a b c d e f g h i j k l m n o p q r s t u v w x y z +// |---------| |---------| |-------| RANGEKEYDELs +// |-----------|-------------| |-----| RANGEDELs +// __________________________________________________________ +// a b c d e f g h i j k l m n o p q r s t u v w x y z // // The tableRangedDeletionIter produces the following set of output spans, where // '1' indicates a span containing only range deletions, '2' is a span // containing only range key deletions, and '3' is a span containing a mixture // of both range deletions and range key deletions. // -// 1 3 1 3 2 1 3 2 -// |-----|---------|-----|---|-----| |---|-|-----| -// __________________________________________________________ -// a b c d e f g h i j k l m n o p q r s t u v w x y z +// 1 3 1 3 2 1 3 2 +// |-----|---------|-----|---|-----| |---|-|-----| +// __________________________________________________________ +// a b c d e f g h i j k l m n o p q r s t u v w x y z // // Algorithm. // @@ -639,11 +731,11 @@ func estimateEntrySizes( // have already been defragmented. To the left and right of any overlap, the // same reasoning applies. For example, // -// |--------| |-------| RANGEKEYDEL -// |---------------------------| RANGEDEL -// |----1---|----3---|----1----|---2---| Merged, fragmented spans. -// __________________________________________________________ -// a b c d e f g h i j k l m n o p q r s t u v w x y z +// |--------| |-------| RANGEKEYDEL +// |---------------------------| RANGEDEL +// |----1---|----3---|----1----|---2---| Merged, fragmented spans. +// __________________________________________________________ +// a b c d e f g h i j k l m n o p q r s t u v w x y z // // Any fragmented abutting spans produced by the merging iter will be of // differing types (i.e. a transition from a span with homogenous key kinds to a @@ -715,7 +807,7 @@ func newCombinedDeletionKeyspanIter( keyspan.SortKeysByTrailer(&out.Keys) return nil }) - mIter.Init(comparer.Compare, transform) + mIter.Init(comparer.Compare, transform, new(keyspan.MergingBuffers)) iter, err := r.NewRawRangeDelIter() if err != nil { @@ -723,12 +815,13 @@ func newCombinedDeletionKeyspanIter( } if iter != nil { dIter := &keyspan.DefragmentingIter{} - dIter.Init(comparer, iter, equal, reducer) + dIter.Init(comparer, iter, equal, reducer, new(keyspan.DefragmentingBuffers)) iter = dIter // Truncate tombstones to the containing file's bounds if necessary. // See docs/range_deletions.md for why this is necessary. iter = keyspan.Truncate( - comparer.Compare, iter, m.Smallest.UserKey, m.Largest.UserKey, nil, nil, + comparer.Compare, iter, m.Smallest.UserKey, m.Largest.UserKey, + nil, nil, false, /* panicOnPartialOverlap */ ) mIter.AddLevel(iter) } @@ -752,7 +845,7 @@ func newCombinedDeletionKeyspanIter( return len(out.Keys) > 0 }) dIter := &keyspan.DefragmentingIter{} - dIter.Init(comparer, iter, equal, reducer) + dIter.Init(comparer, iter, equal, reducer, new(keyspan.DefragmentingBuffers)) iter = dIter mIter.AddLevel(iter) } @@ -783,7 +876,7 @@ func (a rangeKeySetsAnnotator) Accumulate( ) (v interface{}, cacheOK bool) { vptr := dst.(*uint64) *vptr = *vptr + f.Stats.NumRangeKeySets - return vptr, f.StatsValidLocked() + return vptr, f.StatsValid() } func (a rangeKeySetsAnnotator) Merge(src interface{}, dst interface{}) interface{} { @@ -806,3 +899,97 @@ func countRangeKeySetFragments(v *version) (count uint64) { } return count } + +// tombstonesAnnotator implements manifest.Annotator, annotating B-Tree nodes +// with the sum of the files' counts of tombstones (DEL, SINGLEDEL and RANGEDELk +// eys). Its annotation type is a *uint64. The count of tombstones may change +// once a table's stats are loaded asynchronously, so its values are marked as +// cacheable only if a file's stats have been loaded. +type tombstonesAnnotator struct{} + +var _ manifest.Annotator = tombstonesAnnotator{} + +func (a tombstonesAnnotator) Zero(dst interface{}) interface{} { + if dst == nil { + return new(uint64) + } + v := dst.(*uint64) + *v = 0 + return v +} + +func (a tombstonesAnnotator) Accumulate( + f *fileMetadata, dst interface{}, +) (v interface{}, cacheOK bool) { + vptr := dst.(*uint64) + *vptr = *vptr + f.Stats.NumDeletions + return vptr, f.StatsValid() +} + +func (a tombstonesAnnotator) Merge(src interface{}, dst interface{}) interface{} { + srcV := src.(*uint64) + dstV := dst.(*uint64) + *dstV = *dstV + *srcV + return dstV +} + +// countTombstones counts the number of tombstone (DEL, SINGLEDEL and RANGEDEL) +// internal keys across all files of the LSM. It only counts keys in files for +// which table stats have been loaded. It uses a b-tree annotator to cache +// intermediate values between calculations when possible. +func countTombstones(v *version) (count uint64) { + for l := 0; l < numLevels; l++ { + if v.Levels[l].Empty() { + continue + } + count += *v.Levels[l].Annotation(tombstonesAnnotator{}).(*uint64) + } + return count +} + +// valueBlocksSizeAnnotator implements manifest.Annotator, annotating B-Tree +// nodes with the sum of the files' Properties.ValueBlocksSize. Its annotation +// type is a *uint64. The value block size may change once a table's stats are +// loaded asynchronously, so its values are marked as cacheable only if a +// file's stats have been loaded. +type valueBlocksSizeAnnotator struct{} + +var _ manifest.Annotator = valueBlocksSizeAnnotator{} + +func (a valueBlocksSizeAnnotator) Zero(dst interface{}) interface{} { + if dst == nil { + return new(uint64) + } + v := dst.(*uint64) + *v = 0 + return v +} + +func (a valueBlocksSizeAnnotator) Accumulate( + f *fileMetadata, dst interface{}, +) (v interface{}, cacheOK bool) { + vptr := dst.(*uint64) + *vptr = *vptr + f.Stats.ValueBlocksSize + return vptr, f.StatsValid() +} + +func (a valueBlocksSizeAnnotator) Merge(src interface{}, dst interface{}) interface{} { + srcV := src.(*uint64) + dstV := dst.(*uint64) + *dstV = *dstV + *srcV + return dstV +} + +// valueBlocksSizeForLevel returns the Properties.ValueBlocksSize across all +// files for a level of the LSM. It only includes the size for files for which +// table stats have been loaded. It uses a b-tree annotator to cache +// intermediate values between calculations when possible. It must not be +// called concurrently. +// +// REQUIRES: 0 <= level <= numLevels. +func valueBlocksSizeForLevel(v *version, level int) (count uint64) { + if v.Levels[level].Empty() { + return 0 + } + return *v.Levels[level].Annotation(valueBlocksSizeAnnotator{}).(*uint64) +} diff --git a/vendor/github.com/cockroachdb/pebble/version_set.go b/vendor/github.com/cockroachdb/pebble/version_set.go index 1b63e62..41246ab 100644 --- a/vendor/github.com/cockroachdb/pebble/version_set.go +++ b/vendor/github.com/cockroachdb/pebble/version_set.go @@ -30,6 +30,9 @@ const manifestMarkerName = `manifest` type bulkVersionEdit = manifest.BulkVersionEdit type deletedFileEntry = manifest.DeletedFileEntry type fileMetadata = manifest.FileMetadata +type physicalMeta = manifest.PhysicalFileMeta +type virtualMeta = manifest.VirtualFileMeta +type fileBacking = manifest.FileBacking type newFileEntry = manifest.NewFileEntry type version = manifest.Version type versionEdit = manifest.VersionEdit @@ -41,26 +44,19 @@ type versionList = manifest.VersionList // like it sounds: a delta from the previous version. Version edits are logged // to the MANIFEST file, which is replayed at startup. type versionSet struct { - // WARNING: The following struct `atomic` contains fields are accessed atomically. - // - // Go allocations are guaranteed to be 64-bit aligned which we take advantage - // of by placing the 64-bit fields which we access atomically at the beginning - // of the versionSet struct. - // For more information, see https://golang.org/pkg/sync/atomic/#pkg-note-BUG. - atomic struct { - logSeqNum uint64 // next seqNum to use for WAL writes + // Next seqNum to use for WAL writes. + logSeqNum atomic.Uint64 - // The upper bound on sequence numbers that have been assigned so far. - // A suffix of these sequence numbers may not have been written to a - // WAL. Both logSeqNum and visibleSeqNum are atomically updated by the - // commitPipeline. - visibleSeqNum uint64 // visible seqNum (<= logSeqNum) + // The upper bound on sequence numbers that have been assigned so far. A + // suffix of these sequence numbers may not have been written to a WAL. Both + // logSeqNum and visibleSeqNum are atomically updated by the commitPipeline. + // visibleSeqNum is <= logSeqNum. + visibleSeqNum atomic.Uint64 - // Number of bytes present in sstables being written by in-progress - // compactions. This value will be zero if there are no in-progress - // compactions. Updated and read atomically. - atomicInProgressBytes int64 - } + // Number of bytes present in sstables being written by in-progress + // compactions. This value will be zero if there are no in-progress + // compactions. Updated and read atomically. + atomicInProgressBytes atomic.Int64 // Immutable fields. dirname string @@ -83,14 +79,26 @@ type versionSet struct { // A pointer to versionSet.addObsoleteLocked. Avoids allocating a new closure // on the creation of every version. - obsoleteFn func(obsolete []*manifest.FileMetadata) - obsoleteTables []*manifest.FileMetadata + obsoleteFn func(obsolete []*fileBacking) + obsoleteTables []fileInfo obsoleteManifests []fileInfo obsoleteOptions []fileInfo // Zombie tables which have been removed from the current version but are // still referenced by an inuse iterator. - zombieTables map[FileNum]uint64 // filenum -> size + zombieTables map[base.DiskFileNum]uint64 // filenum -> size + + // fileBackingMap is a map for the FileBacking which is supporting virtual + // sstables in the latest version. Once the file backing is backing no + // virtual sstables in the latest version, it is removed from this map and + // the corresponding state is added to the zombieTables map. Note that we + // don't keep track of file backing which supports a virtual sstable + // which is not in the latest version. + // + // fileBackingMap is protected by the versionSet.logLock. It's populated + // during Open in versionSet.load, but it's not used concurrently during + // load. + fileBackingMap map[base.DiskFileNum]*fileBacking // minUnflushedLogNum is the smallest WAL log file number corresponding to // mutations that have not been flushed to an sstable. @@ -111,8 +119,7 @@ type versionSet struct { writing bool writerCond sync.Cond // State for deciding when to write a snapshot. Protected by mu. - lastSnapshotFileCount int64 - editsSinceLastSnapshotFileCount int64 + rotationHelper record.RotationHelper } func (vs *versionSet) init( @@ -132,7 +139,8 @@ func (vs *versionSet) init( vs.dynamicBaseLevel = true vs.versions.Init(mu) vs.obsoleteFn = vs.addObsoleteLocked - vs.zombieTables = make(map[FileNum]uint64) + vs.zombieTables = make(map[base.DiskFileNum]uint64) + vs.fileBackingMap = make(map[base.DiskFileNum]*fileBacking) vs.nextFileNum = 1 vs.manifestMarker = marker vs.setCurrent = setCurrent @@ -180,7 +188,7 @@ func (vs *versionSet) create( vs.opts.EventListener.ManifestCreated(ManifestCreateInfo{ JobID: jobID, - Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, vs.manifestFileNum), + Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, vs.manifestFileNum.DiskFileNum()), FileNum: vs.manifestFileNum, Err: err, }) @@ -202,7 +210,7 @@ func (vs *versionSet) load( vs.init(dirname, opts, marker, setCurrent, mu) vs.manifestFileNum = manifestFileNum - manifestPath := base.MakeFilepath(opts.FS, dirname, fileTypeManifest, vs.manifestFileNum) + manifestPath := base.MakeFilepath(opts.FS, dirname, fileTypeManifest, vs.manifestFileNum.DiskFileNum()) manifestFilename := opts.FS.PathBase(manifestPath) // Read the versionEdits in the manifest file. @@ -258,7 +266,14 @@ func (vs *versionSet) load( // (assuming no WALs contain higher sequence numbers than the // manifest's LastSeqNum). Increment LastSeqNum by 1 to get the // next sequence number that will be assigned. - vs.atomic.logSeqNum = ve.LastSeqNum + 1 + // + // If LastSeqNum is less than SeqNumStart, increase it to at least + // SeqNumStart to leave ample room for reserved sequence numbers. + if ve.LastSeqNum+1 < base.SeqNumStart { + vs.logSeqNum.Store(base.SeqNumStart) + } else { + vs.logSeqNum.Store(ve.LastSeqNum + 1) + } } } // We have already set vs.nextFileNum = 2 at the beginning of the @@ -278,7 +293,20 @@ func (vs *versionSet) load( } vs.markFileNumUsed(vs.minUnflushedLogNum) - newVersion, _, err := bve.Apply(nil, vs.cmp, opts.Comparer.FormatKey, opts.FlushSplitBytes, opts.Experimental.ReadCompactionRate) + // Populate the fileBackingMap since we have finished version + // edit accumulation. + for _, s := range bve.AddedFileBacking { + vs.fileBackingMap[s.DiskFileNum] = s + } + + for _, fileNum := range bve.RemovedFileBacking { + delete(vs.fileBackingMap, fileNum) + } + + newVersion, err := bve.Apply( + nil, vs.cmp, opts.Comparer.FormatKey, opts.FlushSplitBytes, + opts.Experimental.ReadCompactionRate, nil, /* zombies */ + ) if err != nil { return err } @@ -383,7 +411,7 @@ func (vs *versionSet) logAndApply( // in an unflushed memtable. logSeqNum is the _next_ sequence number that // will be assigned, so subtract that by 1 to get the upper bound on the // last assigned sequence number. - logSeqNum := atomic.LoadUint64(&vs.atomic.logSeqNum) + logSeqNum := vs.logSeqNum.Load() ve.LastSeqNum = logSeqNum - 1 if logSeqNum == 0 { // logSeqNum is initialized to 1 in Open() if there are no previous WAL @@ -433,37 +461,22 @@ func (vs *versionSet) logAndApply( // // The logic below uses the min of the last snapshot file count and the file // count in the current version. - editCount := int64(len(ve.DeletedFiles) + len(ve.NewFiles)) - vs.editsSinceLastSnapshotFileCount += editCount + vs.rotationHelper.AddRecord(int64(len(ve.DeletedFiles) + len(ve.NewFiles))) sizeExceeded := vs.manifest.Size() >= vs.opts.MaxManifestFileSize requireRotation := forceRotation || vs.manifest == nil - computeNextSnapshotFileCount := func() int64 { - var count int64 - for i := range vs.metrics.Levels { - count += vs.metrics.Levels[i].NumFiles - } - return count + + var nextSnapshotFilecount int64 + for i := range vs.metrics.Levels { + nextSnapshotFilecount += vs.metrics.Levels[i].NumFiles } - var nextSnapshotFileCount int64 if sizeExceeded && !requireRotation { - if vs.editsSinceLastSnapshotFileCount > vs.lastSnapshotFileCount { - requireRotation = true - } else { - nextSnapshotFileCount = computeNextSnapshotFileCount() - if vs.editsSinceLastSnapshotFileCount > nextSnapshotFileCount { - requireRotation = true - } - } + requireRotation = vs.rotationHelper.ShouldRotate(nextSnapshotFilecount) } var newManifestFileNum FileNum var prevManifestFileSize uint64 if requireRotation { newManifestFileNum = vs.getNextFileNum() prevManifestFileSize = uint64(vs.manifest.Size()) - if nextSnapshotFileCount == 0 { - // Haven't computed it, or happens to be 0. - nextSnapshotFileCount = computeNextSnapshotFileCount() - } } // Grab certain values before releasing vs.mu, in case createManifest() needs @@ -471,18 +484,17 @@ func (vs *versionSet) logAndApply( minUnflushedLogNum := vs.minUnflushedLogNum nextFileNum := vs.nextFileNum - var zombies map[FileNum]uint64 + var zombies map[base.DiskFileNum]uint64 if err := func() error { vs.mu.Unlock() defer vs.mu.Lock() - var bve bulkVersionEdit - if err := bve.Accumulate(ve); err != nil { - return err - } - var err error - newVersion, zombies, err = bve.Apply(currentVersion, vs.cmp, vs.opts.Comparer.FormatKey, vs.opts.FlushSplitBytes, vs.opts.Experimental.ReadCompactionRate) + newVersion, zombies, err = manifest.AccumulateIncompleteAndApplySingleVE( + ve, currentVersion, vs.cmp, vs.opts.Comparer.FormatKey, + vs.opts.FlushSplitBytes, vs.opts.Experimental.ReadCompactionRate, + vs.fileBackingMap, + ) if err != nil { return errors.Wrap(err, "MANIFEST apply failed") } @@ -491,7 +503,7 @@ func (vs *versionSet) logAndApply( if err := vs.createManifest(vs.dirname, newManifestFileNum, minUnflushedLogNum, nextFileNum); err != nil { vs.opts.EventListener.ManifestCreated(ManifestCreateInfo{ JobID: jobID, - Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, newManifestFileNum), + Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, newManifestFileNum.DiskFileNum()), FileNum: newManifestFileNum, Err: err, }) @@ -503,6 +515,7 @@ func (vs *versionSet) logAndApply( if err != nil { return errors.Wrap(err, "MANIFEST next record write failed") } + // NB: Any error from this point on is considered fatal as we don't now if // the MANIFEST write occurred or not. Trying to determine that is // fraught. Instead we rely on the standard recovery mechanism run when a @@ -524,7 +537,7 @@ func (vs *versionSet) logAndApply( } vs.opts.EventListener.ManifestCreated(ManifestCreateInfo{ JobID: jobID, - Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, newManifestFileNum), + Path: base.MakeFilepath(vs.fs, vs.dirname, fileTypeManifest, newManifestFileNum.DiskFileNum()), FileNum: newManifestFileNum, }) } @@ -542,8 +555,7 @@ func (vs *versionSet) logAndApply( if requireRotation { // Successfully rotated. - vs.lastSnapshotFileCount = nextSnapshotFileCount - vs.editsSinceLastSnapshotFileCount = editCount + vs.rotationHelper.Rotate(nextSnapshotFilecount) } // Now that DB.mu is held again, initialize compacting file info in // L0Sublevels. @@ -566,7 +578,7 @@ func (vs *versionSet) logAndApply( if newManifestFileNum != 0 { if vs.manifestFileNum != 0 { vs.obsoleteManifests = append(vs.obsoleteManifests, fileInfo{ - fileNum: vs.manifestFileNum, + fileNum: vs.manifestFileNum.DiskFileNum(), fileSize: prevManifestFileSize, }) } @@ -607,7 +619,7 @@ func (vs *versionSet) incrementCompactions(kind compactionKind, extraLevels []*c vs.metrics.Compact.Count++ vs.metrics.Compact.DefaultCount++ - case compactionKindFlush: + case compactionKindFlush, compactionKindIngestedFlushable: vs.metrics.Flush.Count++ case compactionKindMove: @@ -636,7 +648,7 @@ func (vs *versionSet) incrementCompactions(kind compactionKind, extraLevels []*c } func (vs *versionSet) incrementCompactionBytes(numBytes int64) { - atomic.AddInt64(&vs.atomic.atomicInProgressBytes, numBytes) + vs.atomicInProgressBytes.Add(numBytes) } // createManifest creates a manifest file that contains a snapshot of vs. @@ -644,7 +656,7 @@ func (vs *versionSet) createManifest( dirname string, fileNum, minUnflushedLogNum, nextFileNum FileNum, ) (err error) { var ( - filename = base.MakeFilepath(vs.fs, dirname, fileTypeManifest, fileNum) + filename = base.MakeFilepath(vs.fs, dirname, fileTypeManifest, fileNum.DiskFileNum()) manifestFile vfs.File manifest *record.Writer ) @@ -668,6 +680,7 @@ func (vs *versionSet) createManifest( snapshot := versionEdit{ ComparerName: vs.cmpName, } + dedup := make(map[base.DiskFileNum]struct{}) for level, levelMetadata := range vs.currentVersion().Levels { iter := levelMetadata.Iter() for meta := iter.First(); meta != nil; meta = iter.Next() { @@ -675,6 +688,14 @@ func (vs *versionSet) createManifest( Level: level, Meta: meta, }) + // TODO(bananabrick): Test snapshot changes. + if _, ok := dedup[meta.FileBacking.DiskFileNum]; meta.Virtual && !ok { + dedup[meta.FileBacking.DiskFileNum] = struct{}{} + snapshot.CreatedBackingTables = append( + snapshot.CreatedBackingTables, + meta.FileBacking, + ) + } } } @@ -739,13 +760,13 @@ func (vs *versionSet) currentVersion() *version { return vs.versions.Back() } -func (vs *versionSet) addLiveFileNums(m map[FileNum]struct{}) { +func (vs *versionSet) addLiveFileNums(m map[base.DiskFileNum]struct{}) { current := vs.currentVersion() for v := vs.versions.Front(); true; v = v.Next() { for _, lm := range v.Levels { iter := lm.Iter() for f := iter.First(); f != nil; f = iter.Next() { - m[f.FileNum] = struct{}{} + m[f.FileBacking.DiskFileNum] = struct{}{} } } if v == current { @@ -754,23 +775,59 @@ func (vs *versionSet) addLiveFileNums(m map[FileNum]struct{}) { } } -func (vs *versionSet) addObsoleteLocked(obsolete []*manifest.FileMetadata) { - for _, fileMeta := range obsolete { +// addObsoleteLocked will add the fileInfo associated with obsolete backing +// sstables to the obsolete tables list. +// +// The file backings in the obsolete list must not appear more than once. +// +// DB.mu must be held when addObsoleteLocked is called. +func (vs *versionSet) addObsoleteLocked(obsolete []*fileBacking) { + if len(obsolete) == 0 { + return + } + + obsoleteFileInfo := make([]fileInfo, len(obsolete)) + for i, bs := range obsolete { + obsoleteFileInfo[i].fileNum = bs.DiskFileNum + obsoleteFileInfo[i].fileSize = bs.Size + } + + if invariants.Enabled { + dedup := make(map[base.DiskFileNum]struct{}) + for _, fi := range obsoleteFileInfo { + dedup[fi.fileNum] = struct{}{} + } + if len(dedup) != len(obsoleteFileInfo) { + panic("pebble: duplicate FileBacking present in obsolete list") + } + } + + for _, fi := range obsoleteFileInfo { // Note that the obsolete tables are no longer zombie by the definition of // zombie, but we leave them in the zombie tables map until they are // deleted from disk. - if _, ok := vs.zombieTables[fileMeta.FileNum]; !ok { - vs.opts.Logger.Fatalf("MANIFEST obsolete table %s not marked as zombie", fileMeta.FileNum) + if _, ok := vs.zombieTables[fi.fileNum]; !ok { + vs.opts.Logger.Fatalf("MANIFEST obsolete table %s not marked as zombie", fi.fileNum) } } - vs.obsoleteTables = append(vs.obsoleteTables, obsolete...) - vs.incrementObsoleteTablesLocked(obsolete) + + vs.obsoleteTables = append(vs.obsoleteTables, obsoleteFileInfo...) + vs.updateObsoleteTableMetricsLocked() } -func (vs *versionSet) incrementObsoleteTablesLocked(obsolete []*manifest.FileMetadata) { - for _, fileMeta := range obsolete { - vs.metrics.Table.ObsoleteCount++ - vs.metrics.Table.ObsoleteSize += fileMeta.Size +// addObsolete will acquire DB.mu, so DB.mu must not be held when this is +// called. +func (vs *versionSet) addObsolete(obsolete []*fileBacking) { + vs.mu.Lock() + defer vs.mu.Unlock() + vs.addObsoleteLocked(obsolete) +} + +func (vs *versionSet) updateObsoleteTableMetricsLocked() { + vs.metrics.Table.ObsoleteCount = int64(len(vs.obsoleteTables)) + vs.metrics.Table.ObsoleteSize = 0 + for _, fi := range vs.obsoleteTables { + vs.metrics.Table.ObsoleteSize += fi.fileSize } } @@ -782,7 +839,7 @@ func setCurrentFunc( // the CURRENT file to signal which MANIFEST is current. Ignore // the filename read during LocateMarker. return func(manifestFileNum FileNum) error { - if err := setCurrentFile(dirname, fs, manifestFileNum); err != nil { + if err := setCurrentFile(dirname, fs, manifestFileNum.DiskFileNum()); err != nil { return err } if err := dir.Sync(); err != nil { @@ -800,13 +857,13 @@ func setCurrentFunc( func setCurrentFuncMarker(marker *atomicfs.Marker, fs vfs.FS, dirname string) func(FileNum) error { return func(manifestFileNum FileNum) error { - return marker.Move(base.MakeFilename(fileTypeManifest, manifestFileNum)) + return marker.Move(base.MakeFilename(fileTypeManifest, manifestFileNum.DiskFileNum())) } } func findCurrentManifest( vers FormatMajorVersion, fs vfs.FS, dirname string, -) (marker *atomicfs.Marker, manifestNum FileNum, exists bool, err error) { +) (marker *atomicfs.Marker, manifestNum base.DiskFileNum, exists bool, err error) { // NB: We always locate the manifest marker, even if we might not // actually use it (because we're opening the database at an earlier // format major version that uses the CURRENT file). Locating a @@ -814,7 +871,7 @@ func findCurrentManifest( var filename string marker, filename, err = atomicfs.LocateMarker(fs, dirname, manifestMarkerName) if err != nil { - return nil, 0, false, err + return nil, base.FileNum(0).DiskFileNum(), false, err } if vers < formatVersionedManifestMarker { @@ -824,9 +881,9 @@ func findCurrentManifest( manifestNum, err = readCurrentFile(fs, dirname) if oserror.IsNotExist(err) { - return marker, 0, false, nil + return marker, base.FileNum(0).DiskFileNum(), false, nil } else if err != nil { - return marker, 0, false, err + return marker, base.FileNum(0).DiskFileNum(), false, err } return marker, manifestNum, true, nil } @@ -837,48 +894,48 @@ func findCurrentManifest( if filename == "" { // The marker hasn't been set yet. This database doesn't exist. - return marker, 0, false, nil + return marker, base.FileNum(0).DiskFileNum(), false, nil } var ok bool _, manifestNum, ok = base.ParseFilename(fs, filename) if !ok { - return marker, 0, false, base.CorruptionErrorf("pebble: MANIFEST name %q is malformed", errors.Safe(filename)) + return marker, base.FileNum(0).DiskFileNum(), false, base.CorruptionErrorf("pebble: MANIFEST name %q is malformed", errors.Safe(filename)) } return marker, manifestNum, true, nil } -func readCurrentFile(fs vfs.FS, dirname string) (FileNum, error) { +func readCurrentFile(fs vfs.FS, dirname string) (base.DiskFileNum, error) { // Read the CURRENT file to find the current manifest file. - current, err := fs.Open(base.MakeFilepath(fs, dirname, fileTypeCurrent, 0)) + current, err := fs.Open(base.MakeFilepath(fs, dirname, fileTypeCurrent, base.FileNum(0).DiskFileNum())) if err != nil { - return 0, errors.Wrapf(err, "pebble: could not open CURRENT file for DB %q", dirname) + return base.FileNum(0).DiskFileNum(), errors.Wrapf(err, "pebble: could not open CURRENT file for DB %q", dirname) } defer current.Close() stat, err := current.Stat() if err != nil { - return 0, err + return base.FileNum(0).DiskFileNum(), err } n := stat.Size() if n == 0 { - return 0, errors.Errorf("pebble: CURRENT file for DB %q is empty", dirname) + return base.FileNum(0).DiskFileNum(), errors.Errorf("pebble: CURRENT file for DB %q is empty", dirname) } if n > 4096 { - return 0, errors.Errorf("pebble: CURRENT file for DB %q is too large", dirname) + return base.FileNum(0).DiskFileNum(), errors.Errorf("pebble: CURRENT file for DB %q is too large", dirname) } b := make([]byte, n) _, err = current.ReadAt(b, 0) if err != nil { - return 0, err + return base.FileNum(0).DiskFileNum(), err } if b[n-1] != '\n' { - return 0, base.CorruptionErrorf("pebble: CURRENT file for DB %q is malformed", dirname) + return base.FileNum(0).DiskFileNum(), base.CorruptionErrorf("pebble: CURRENT file for DB %q is malformed", dirname) } b = bytes.TrimSpace(b) _, manifestFileNum, ok := base.ParseFilename(fs, string(b)) if !ok { - return 0, base.CorruptionErrorf("pebble: MANIFEST name %q is malformed", errors.Safe(b)) + return base.FileNum(0).DiskFileNum(), base.CorruptionErrorf("pebble: MANIFEST name %q is malformed", errors.Safe(b)) } return manifestFileNum, nil } diff --git a/vendor/github.com/cockroachdb/pebble/vfs/atomicfs/marker.go b/vendor/github.com/cockroachdb/pebble/vfs/atomicfs/marker.go index f0ff558..6bdc506 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/atomicfs/marker.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/atomicfs/marker.go @@ -16,7 +16,7 @@ import ( // ReadMarker looks up the current state of a marker returning just the // current value of the marker. Callers that may need to move the marker -// should use LocateMarker. +// to a new value should use LocateMarker. func ReadMarker(fs vfs.FS, dir, markerName string) (string, error) { state, err := scanForMarker(fs, dir, markerName) if err != nil { @@ -49,9 +49,11 @@ func LocateMarker(fs vfs.FS, dir, markerName string) (*Marker, string, error) { } type scannedState struct { + // filename is the latest marker file found (the one with the highest iter value). filename string iter uint64 value string + // obsolete is a list of earlier markers that were found. obsolete []string } @@ -89,11 +91,15 @@ func scanForMarker(fs vfs.FS, dir, markerName string) (scannedState, error) { return state, nil } -// A Marker provides an interface for marking a single file on the -// filesystem. The marker may be atomically moved from name to name. -// Marker is not safe for concurrent use. Multiple processes may not -// read or move the same marker simultaneously. An Marker may only be -// constructed through LocateMarker. +// A Marker provides an interface for maintaining a single string value on the +// filesystem. The marker may be atomically moved from value to value. +// +// The implementation creates a new marker file for each new value, embedding +// the value in the marker filename. +// +// Marker is not safe for concurrent use. Multiple processes may not read or +// move the same marker simultaneously. A Marker may only be constructed through +// LocateMarker. // // Marker names must be unique within the directory. type Marker struct { @@ -112,11 +118,10 @@ type Marker struct { // monotonically increasing over the lifetime of a marker. Actual // marker files will always have a positive iter value. iter uint64 - // obsoleteFiles holds a list of files discovered by LocateMarker - // that are old values for this marker. These files may exist if the - // filesystem doesn't guarantee atomic renames (eg, if it's - // implemented as a link(newpath), remove(oldpath), and a crash in - // between may leave an entry at the old path). + // obsoleteFiles holds a list of marker files discovered by LocateMarker that + // are old values for this marker. These files may exist in certain error + // cases or crashes (e.g. if the deletion of the previous marker file failed + // during Move). obsoleteFiles []string } @@ -160,22 +165,21 @@ func (a *Marker) Close() error { return a.dirFD.Close() } -// Move atomically moves the marker to mark the provided filename. +// Move atomically moves the marker to a new value. +// // If Move returns a nil error, the new marker value is guaranteed to be // persisted to stable storage. If Move returns an error, the current // value of the marker may be the old value or the new value. Callers // may retry a Move error. // // If an error occurs while syncing the directory, Move panics. -// -// The provided filename does not need to exist on the filesystem. -func (a *Marker) Move(filename string) error { +func (a *Marker) Move(newValue string) error { a.iter++ - dstFilename := markerFilename(a.name, a.iter, filename) + dstFilename := markerFilename(a.name, a.iter, newValue) dstPath := a.fs.PathJoin(a.dir, dstFilename) oldFilename := a.filename - // The marker has never been placed. Create a new file. + // Create the new marker. f, err := a.fs.Create(dstPath) if err != nil { // On a distributed filesystem, an error doesn't guarantee that @@ -217,7 +221,7 @@ func (a *Marker) Move(filename string) error { } // NextIter returns the next iteration number that the marker will use. -// Clients may use this number for formulating filenames that are +// Clients may use this number for formulating new values that are // unused. func (a *Marker) NextIter() uint64 { return a.iter + 1 @@ -226,12 +230,11 @@ func (a *Marker) NextIter() uint64 { // RemoveObsolete removes any obsolete files discovered while locating // the marker or files unable to be removed during Move. func (a *Marker) RemoveObsolete() error { - obsolete := a.obsoleteFiles - for _, filename := range obsolete { + for i, filename := range a.obsoleteFiles { if err := a.fs.Remove(a.fs.PathJoin(a.dir, filename)); err != nil && !oserror.IsNotExist(err) { + a.obsoleteFiles = a.obsoleteFiles[i:] return err } - a.obsoleteFiles = obsolete[1:] } a.obsoleteFiles = nil return nil diff --git a/vendor/github.com/cockroachdb/pebble/vfs/clone.go b/vendor/github.com/cockroachdb/pebble/vfs/clone.go index 900d787..5d6edf2 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/clone.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/clone.go @@ -12,8 +12,9 @@ import ( ) type cloneOpts struct { - skip func(string) bool - sync bool + skip func(string) bool + sync bool + tryLink bool } // A CloneOption configures the behavior of Clone. @@ -28,6 +29,12 @@ func CloneSkip(fn func(string) bool) CloneOption { // CloneSync configures Clone to sync files and directories. var CloneSync CloneOption = func(o *cloneOpts) { o.sync = true } +// CloneTryLink configures Clone to link files to the destination if the source and +// destination filesystems are the same. If the source and destination +// filesystems are not the same or the filesystem does not support linking, then +// Clone falls back to copying. +var CloneTryLink CloneOption = func(o *cloneOpts) { o.tryLink = true } + // Clone recursively copies a directory structure from srcFS to dstFS. srcPath // specifies the path in srcFS to copy from and must be compatible with the // srcFS path format. dstDir is the target directory in dstFS and must be @@ -49,6 +56,7 @@ func Clone(srcFS, dstFS FS, srcPath, dstPath string, opts ...CloneOption) (bool, } return false, err } + defer srcFile.Close() stat, err := srcFile.Stat() if err != nil { @@ -56,9 +64,6 @@ func Clone(srcFS, dstFS FS, srcPath, dstPath string, opts ...CloneOption) (bool, } if stat.IsDir() { - if err := srcFile.Close(); err != nil { - return false, err - } if err := dstFS.MkdirAll(dstPath, 0755); err != nil { return false, err } @@ -94,13 +99,25 @@ func Clone(srcFS, dstFS FS, srcPath, dstPath string, opts ...CloneOption) (bool, return true, nil } + // If the source and destination filesystems are the same and the user + // specified they'd prefer to link if possible, try to use a hardlink, + // falling back to copying if it fails. + if srcFS == dstFS && o.tryLink { + if err := LinkOrCopy(srcFS, srcPath, dstPath); oserror.IsNotExist(err) { + // Clone's semantics are such that it returns (false,nil) if the + // source does not exist. + return false, nil + } else if err != nil { + return false, err + } else { + return true, nil + } + } + data, err := io.ReadAll(srcFile) if err != nil { return false, err } - if err := srcFile.Close(); err != nil { - return false, err - } dstFile, err := dstFS.Create(dstPath) if err != nil { return false, err @@ -113,7 +130,6 @@ func Clone(srcFS, dstFS FS, srcPath, dstPath string, opts ...CloneOption) (bool, return false, err } } - if err := dstFile.Close(); err != nil { return false, err } diff --git a/vendor/github.com/cockroachdb/pebble/vfs/default_linux.go b/vendor/github.com/cockroachdb/pebble/vfs/default_linux.go new file mode 100644 index 0000000..ec29074 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/vfs/default_linux.go @@ -0,0 +1,132 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +//go:build linux +// +build linux + +package vfs + +import ( + "os" + "syscall" + + "github.com/cockroachdb/errors" + "golang.org/x/sys/unix" +) + +func wrapOSFileImpl(f *os.File) File { + lf := &linuxFile{File: f, fd: f.Fd()} + if lf.fd != InvalidFd { + lf.useSyncRange = isSyncRangeSupported(lf.fd) + } + return lf +} + +func (defaultFS) OpenDir(name string) (File, error) { + f, err := os.OpenFile(name, syscall.O_CLOEXEC, 0) + if err != nil { + return nil, errors.WithStack(err) + } + return &linuxDir{f}, nil +} + +// Assert that linuxFile and linuxDir implement vfs.File. +var ( + _ File = (*linuxDir)(nil) + _ File = (*linuxFile)(nil) +) + +type linuxDir struct { + *os.File +} + +func (d *linuxDir) Prefetch(offset int64, length int64) error { return nil } +func (d *linuxDir) Preallocate(offset, length int64) error { return nil } +func (d *linuxDir) SyncData() error { return d.Sync() } +func (d *linuxDir) SyncTo(offset int64) (fullSync bool, err error) { return false, nil } + +type linuxFile struct { + *os.File + fd uintptr + useSyncRange bool +} + +func (f *linuxFile) Prefetch(offset int64, length int64) error { + _, _, err := unix.Syscall(unix.SYS_READAHEAD, uintptr(f.fd), uintptr(offset), uintptr(length)) + return err +} + +func (f *linuxFile) Preallocate(offset, length int64) error { + return unix.Fallocate(int(f.fd), unix.FALLOC_FL_KEEP_SIZE, offset, length) +} + +func (f *linuxFile) SyncData() error { + return unix.Fdatasync(int(f.fd)) +} + +func (f *linuxFile) SyncTo(offset int64) (fullSync bool, err error) { + if !f.useSyncRange { + // Use fdatasync, which does provide persistence guarantees but won't + // update all file metadata. From the `fdatasync` man page: + // + // fdatasync() is similar to fsync(), but does not flush modified + // metadata unless that metadata is needed in order to allow a + // subsequent data retrieval to be correctly handled. For example, + // changes to st_atime or st_mtime (respectively, time of last access + // and time of last modification; see stat(2)) do not require flushing + // because they are not necessary for a subsequent data read to be + // handled correctly. On the other hand, a change to the file size + // (st_size, as made by say ftruncate(2)), would require a metadata + // flush. + if err = unix.Fdatasync(int(f.fd)); err != nil { + return false, err + } + return true, nil + } + + const ( + waitBefore = 0x1 + write = 0x2 + // waitAfter = 0x4 + ) + + // By specifying write|waitBefore for the flags, we're instructing + // SyncFileRange to a) wait for any outstanding data being written to finish, + // and b) to queue any other dirty data blocks in the range [0,offset] for + // writing. The actual writing of this data will occur asynchronously. The + // use of `waitBefore` is to limit how much dirty data is allowed to + // accumulate. Linux sometimes behaves poorly when a large amount of dirty + // data accumulates, impacting other I/O operations. + return false, unix.SyncFileRange(int(f.fd), 0, offset, write|waitBefore) +} + +type syncFileRange func(fd int, off int64, n int64, flags int) (err error) + +// sync_file_range depends on both the filesystem, and the broader kernel +// support. In particular, Windows Subsystem for Linux does not support +// sync_file_range, even when used with ext{2,3,4}. syncRangeSmokeTest performs +// a test of of sync_file_range, returning false on ENOSYS, and true otherwise. +func syncRangeSmokeTest(fd uintptr, syncFn syncFileRange) bool { + err := syncFn(int(fd), 0 /* offset */, 0 /* nbytes */, 0 /* flags */) + return err != unix.ENOSYS +} + +func isSyncRangeSupported(fd uintptr) bool { + var stat unix.Statfs_t + if err := unix.Fstatfs(int(fd), &stat); err != nil { + return false + } + + // Allowlist which filesystems we allow using sync_file_range with as some + // filesystems treat that syscall as a noop (notably ZFS). A allowlist is + // used instead of a denylist in order to have a more graceful failure mode + // in case a filesystem we haven't tested is encountered. Currently only + // ext2/3/4 are known to work properly. + const extMagic = 0xef53 + switch stat.Type { + case extMagic: + return syncRangeSmokeTest(fd, unix.SyncFileRange) + } + return false +} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/default_unix.go b/vendor/github.com/cockroachdb/pebble/vfs/default_unix.go new file mode 100644 index 0000000..836185e --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/vfs/default_unix.go @@ -0,0 +1,49 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +//go:build darwin || dragonfly || freebsd || netbsd || openbsd || solaris +// +build darwin dragonfly freebsd netbsd openbsd solaris + +package vfs + +import ( + "os" + "syscall" + + "github.com/cockroachdb/errors" +) + +func wrapOSFileImpl(osFile *os.File) File { + return &unixFile{File: osFile, fd: osFile.Fd()} +} + +func (defaultFS) OpenDir(name string) (File, error) { + f, err := os.OpenFile(name, syscall.O_CLOEXEC, 0) + if err != nil { + return nil, errors.WithStack(err) + } + return &unixFile{f, InvalidFd}, nil +} + +// Assert that unixFile implements vfs.File. +var _ File = (*unixFile)(nil) + +type unixFile struct { + *os.File + fd uintptr +} + +func (*unixFile) Prefetch(offset int64, length int64) error { return nil } +func (*unixFile) Preallocate(offset, length int64) error { return nil } + +func (f *unixFile) SyncData() error { + return f.Sync() +} + +func (f *unixFile) SyncTo(int64) (fullSync bool, err error) { + if err = f.Sync(); err != nil { + return false, err + } + return true, nil +} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/default_windows.go b/vendor/github.com/cockroachdb/pebble/vfs/default_windows.go new file mode 100644 index 0000000..f314193 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/vfs/default_windows.go @@ -0,0 +1,61 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +//go:build windows +// +build windows + +package vfs + +import ( + "os" + "syscall" + + "github.com/cockroachdb/errors" +) + +func wrapOSFileImpl(f *os.File) File { + return &windowsFile{f} +} + +func (defaultFS) OpenDir(name string) (File, error) { + f, err := os.OpenFile(name, syscall.O_CLOEXEC, 0) + if err != nil { + return nil, errors.WithStack(err) + } + return &windowsDir{f}, nil +} + +// Assert that windowsFile and windowsDir implement vfs.File. +var ( + _ File = (*windowsFile)(nil) + _ File = (*windowsDir)(nil) +) + +type windowsDir struct { + *os.File +} + +func (*windowsDir) Prefetch(offset int64, length int64) error { return nil } +func (*windowsDir) Preallocate(off, length int64) error { return nil } + +// Silently ignore Sync() on Windows. This is the same behavior as +// RocksDB. See port/win/io_win.cc:WinDirectory::Fsync(). +func (*windowsDir) Sync() error { return nil } +func (*windowsDir) SyncData() error { return nil } +func (*windowsDir) SyncTo(length int64) (fullSync bool, err error) { return false, nil } + +type windowsFile struct { + *os.File +} + +func (*windowsFile) Prefetch(offset int64, length int64) error { return nil } +func (*windowsFile) Preallocate(offset, length int64) error { return nil } + +func (f *windowsFile) SyncData() error { return f.Sync() } +func (f *windowsFile) SyncTo(length int64) (fullSync bool, err error) { + if err = f.Sync(); err != nil { + return false, err + } + return true, nil +} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/dir_unix.go b/vendor/github.com/cockroachdb/pebble/vfs/dir_unix.go deleted file mode 100644 index c27aba9..0000000 --- a/vendor/github.com/cockroachdb/pebble/vfs/dir_unix.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2014 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -//go:build darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris -// +build darwin dragonfly freebsd linux netbsd openbsd solaris - -package vfs - -import ( - "os" - "syscall" - - "github.com/cockroachdb/errors" -) - -func (defaultFS) OpenDir(name string) (File, error) { - f, err := os.OpenFile(name, syscall.O_CLOEXEC, 0) - return f, errors.WithStack(err) -} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/dir_windows.go b/vendor/github.com/cockroachdb/pebble/vfs/dir_windows.go deleted file mode 100644 index d53690a..0000000 --- a/vendor/github.com/cockroachdb/pebble/vfs/dir_windows.go +++ /dev/null @@ -1,33 +0,0 @@ -// Copyright 2014 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -//go:build windows -// +build windows - -package vfs - -import ( - "os" - "syscall" - - "github.com/cockroachdb/errors" -) - -type windowsDir struct { - File -} - -func (windowsDir) Sync() error { - // Silently ignore Sync() on Windows. This is the same behavior as - // RocksDB. See port/win/io_win.cc:WinDirectory::Fsync(). - return nil -} - -func (defaultFS) OpenDir(name string) (File, error) { - f, err := os.OpenFile(name, syscall.O_CLOEXEC, 0) - if err != nil { - return nil, errors.WithStack(err) - } - return windowsDir{f}, nil -} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/disk_full.go b/vendor/github.com/cockroachdb/pebble/vfs/disk_full.go index c7d12ea..a5ef3e4 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/disk_full.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/disk_full.go @@ -39,68 +39,66 @@ func OnDiskFull(fs FS, fn func()) FS { } type enospcFS struct { - inner FS - atomic struct { - // generation is a monotonically increasing number that encodes the - // current state of ENOSPC error handling. Incoming writes are - // organized into generations to provide strong guarantees on when the - // disk full callback is invoked. The callback is invoked once per - // write generation. - // - // Special significance is given to the parity of this generation - // field to optimize incoming writes in the normal state, which only - // need to perform a single atomic load. If generation is odd, an - // ENOSPC error is being actively handled. The generations associated - // with writes are always even. - // - // The lifecycle of a write is: - // - // 1. Atomically load the current generation. - // a. If it's even, this is the write's generation number. - // b. If it's odd, an ENOSPC was recently encountered and the - // corresponding invocation of the disk full callback has not - // yet completed. The write must wait until the callback has - // completed and generation is updated to an even number, which - // becomes the write's generation number. - // 2. Perform the write. If it encounters no error or an error other - // than ENOSPC, the write returns and proceeds no further in this - // lifecycle. - // 3. Handle ENOSPC. If the write encounters ENOSPC, the callback must - // be invoked for the write's generation. The write's goroutine - // acquires the FS's mutex. - // a. If the FS's current generation is still equal to the write's - // generation, the write is the first write of its generation to - // encounter ENOSPC. It increments the FS's current generation - // to an odd number, signifying that an ENOSPC is being handled - // and invokes the callback. - // b. If the FS's current generation has changed, some other write - // from the same generation encountered an ENOSPC first. This - // write waits on the condition variable until the FS's current - // generation is updated indicating that the generation's - // callback invocation has completed. - // 3. Retry the write once. The callback for the write's generation - // has completed, either by this write's goroutine or another's. - // The write may proceed with the expectation that the callback - // remedied the full disk by freeing up disk space and an ENOSPC - // should not be encountered again for at least a few minutes. If - // we do encounter another ENOSPC on the retry, the callback was - // unable to remedy the full disk and another retry won't be - // useful. Any error, including ENOSPC, during the retry is - // returned without further handling. None of the retries invoke - // the callback. - // - // This scheme has a few nice properties: - // * Once the disk-full callback completes, it won't be invoked - // again unless a write that started strictly later encounters an - // ENOSPC. This is convenient if the callback strives to 'fix' the - // full disk, for example, by removing a ballast file. A new - // invocation of the callback guarantees a new problem. - // * Incoming writes block if there's an unhandled ENOSPC. Some - // writes, like WAL or MANIFEST fsyncs, are fatal if they encounter - // an ENOSPC. - generation uint32 - } - mu struct { + inner FS + // generation is a monotonically increasing number that encodes the + // current state of ENOSPC error handling. Incoming writes are + // organized into generations to provide strong guarantees on when the + // disk full callback is invoked. The callback is invoked once per + // write generation. + // + // Special significance is given to the parity of this generation + // field to optimize incoming writes in the normal state, which only + // need to perform a single atomic load. If generation is odd, an + // ENOSPC error is being actively handled. The generations associated + // with writes are always even. + // + // The lifecycle of a write is: + // + // 1. Atomically load the current generation. + // a. If it's even, this is the write's generation number. + // b. If it's odd, an ENOSPC was recently encountered and the + // corresponding invocation of the disk full callback has not + // yet completed. The write must wait until the callback has + // completed and generation is updated to an even number, which + // becomes the write's generation number. + // 2. Perform the write. If it encounters no error or an error other + // than ENOSPC, the write returns and proceeds no further in this + // lifecycle. + // 3. Handle ENOSPC. If the write encounters ENOSPC, the callback must + // be invoked for the write's generation. The write's goroutine + // acquires the FS's mutex. + // a. If the FS's current generation is still equal to the write's + // generation, the write is the first write of its generation to + // encounter ENOSPC. It increments the FS's current generation + // to an odd number, signifying that an ENOSPC is being handled + // and invokes the callback. + // b. If the FS's current generation has changed, some other write + // from the same generation encountered an ENOSPC first. This + // write waits on the condition variable until the FS's current + // generation is updated indicating that the generation's + // callback invocation has completed. + // 3. Retry the write once. The callback for the write's generation + // has completed, either by this write's goroutine or another's. + // The write may proceed with the expectation that the callback + // remedied the full disk by freeing up disk space and an ENOSPC + // should not be encountered again for at least a few minutes. If + // we do encounter another ENOSPC on the retry, the callback was + // unable to remedy the full disk and another retry won't be + // useful. Any error, including ENOSPC, during the retry is + // returned without further handling. None of the retries invoke + // the callback. + // + // This scheme has a few nice properties: + // * Once the disk-full callback completes, it won't be invoked + // again unless a write that started strictly later encounters an + // ENOSPC. This is convenient if the callback strives to 'fix' the + // full disk, for example, by removing a ballast file. A new + // invocation of the callback guarantees a new problem. + // * Incoming writes block if there's an unhandled ENOSPC. Some + // writes, like WAL or MANIFEST fsyncs, are fatal if they encounter + // an ENOSPC. + generation atomic.Uint32 + mu struct { sync.Mutex sync.Cond onDiskFull func() @@ -119,7 +117,7 @@ func (fs *enospcFS) Unwrap() FS { // waitUntilReady blocks until the callback returns. The returned generation // is always even. func (fs *enospcFS) waitUntilReady() uint32 { - gen := atomic.LoadUint32(&fs.atomic.generation) + gen := fs.generation.Load() if gen%2 == 0 { // An even generation indicates that we're not currently handling an // ENOSPC. Allow the write to proceed. @@ -132,10 +130,10 @@ func (fs *enospcFS) waitUntilReady() uint32 { defer fs.mu.Unlock() // Load the generation again with fs.mu locked. - gen = atomic.LoadUint32(&fs.atomic.generation) + gen = fs.generation.Load() for gen%2 == 1 { fs.mu.Wait() - gen = atomic.LoadUint32(&fs.atomic.generation) + gen = fs.generation.Load() } return gen } @@ -144,7 +142,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) { fs.mu.Lock() defer fs.mu.Unlock() - currentGeneration := atomic.LoadUint32(&fs.atomic.generation) + currentGeneration := fs.generation.Load() // If the current generation is still `gen`, this is the first goroutine // to hit an ENOSPC within this write generation, so this goroutine is @@ -153,7 +151,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) { // Increment the generation to an odd number, indicating that the FS // is out-of-disk space and incoming writes should pause and wait for // the next generation before continuing. - atomic.StoreUint32(&fs.atomic.generation, gen+1) + fs.generation.Store(gen + 1) func() { // Drop the mutex while we invoke the callback, re-acquiring @@ -165,7 +163,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) { // Update the current generation again to an even number, indicating // that the callback has completed for the write generation `gen`. - atomic.StoreUint32(&fs.atomic.generation, gen+2) + fs.generation.Store(gen + 2) fs.mu.Broadcast() return } @@ -185,7 +183,7 @@ func (fs *enospcFS) handleENOSPC(gen uint32) { // subsequent generations' callbacks. for currentGeneration == gen+1 { fs.mu.Wait() - currentGeneration = atomic.LoadUint32(&fs.atomic.generation) + currentGeneration = fs.generation.Load() } } @@ -199,10 +197,10 @@ func (fs *enospcFS) Create(name string) (File, error) { f, err = fs.inner.Create(name) } if f != nil { - f = WithFd(f, enospcFile{ + f = &enospcFile{ fs: fs, inner: f, - }) + } } return f, err } @@ -222,10 +220,10 @@ func (fs *enospcFS) Link(oldname, newname string) error { func (fs *enospcFS) Open(name string, opts ...OpenOption) (File, error) { f, err := fs.inner.Open(name, opts...) if f != nil { - f = WithFd(f, enospcFile{ + f = &enospcFile{ fs: fs, inner: f, - }) + } } return f, err } @@ -233,10 +231,10 @@ func (fs *enospcFS) Open(name string, opts ...OpenOption) (File, error) { func (fs *enospcFS) OpenDir(name string) (File, error) { f, err := fs.inner.OpenDir(name) if f != nil { - f = WithFd(f, enospcFile{ + f = &enospcFile{ fs: fs, inner: f, - }) + } } return f, err } @@ -288,10 +286,10 @@ func (fs *enospcFS) ReuseForWrite(oldname, newname string) (File, error) { } if f != nil { - f = WithFd(f, enospcFile{ + f = &enospcFile{ fs: fs, inner: f, - }) + } } return f, err } @@ -349,19 +347,21 @@ type enospcFile struct { inner File } -func (f enospcFile) Close() error { +var _ File = (*enospcFile)(nil) + +func (f *enospcFile) Close() error { return f.inner.Close() } -func (f enospcFile) Read(p []byte) (n int, err error) { +func (f *enospcFile) Read(p []byte) (n int, err error) { return f.inner.Read(p) } -func (f enospcFile) ReadAt(p []byte, off int64) (n int, err error) { +func (f *enospcFile) ReadAt(p []byte, off int64) (n int, err error) { return f.inner.ReadAt(p, off) } -func (f enospcFile) Write(p []byte) (n int, err error) { +func (f *enospcFile) Write(p []byte) (n int, err error) { gen := f.fs.waitUntilReady() n, err = f.inner.Write(p) @@ -375,11 +375,19 @@ func (f enospcFile) Write(p []byte) (n int, err error) { return n, err } -func (f enospcFile) Stat() (os.FileInfo, error) { +func (f *enospcFile) Prefetch(offset, length int64) error { + return f.inner.Prefetch(offset, length) +} + +func (f *enospcFile) Preallocate(offset, length int64) error { + return f.inner.Preallocate(offset, length) +} + +func (f *enospcFile) Stat() (os.FileInfo, error) { return f.inner.Stat() } -func (f enospcFile) Sync() error { +func (f *enospcFile) Sync() error { gen := f.fs.waitUntilReady() err := f.inner.Sync() @@ -399,7 +407,18 @@ func (f enospcFile) Sync() error { return err } -// Ensure that *enospcFS implements the FS interface. +func (f *enospcFile) SyncData() error { + return f.inner.SyncData() +} + +func (f *enospcFile) SyncTo(length int64) (fullSync bool, err error) { + return f.inner.SyncTo(length) +} + +func (f *enospcFile) Fd() uintptr { + return f.inner.Fd() +} + var _ FS = (*enospcFS)(nil) func isENOSPC(err error) bool { diff --git a/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go b/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go index e7b0b12..1c0b9c5 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/disk_health.go @@ -5,25 +5,102 @@ package vfs import ( + "fmt" "io" "os" + "path/filepath" "sync" "sync/atomic" "time" + + "github.com/cockroachdb/pebble/internal/invariants" + "github.com/cockroachdb/redact" ) const ( - // defaultTickInterval is the default interval between two ticks of each - // diskHealthCheckingFile loop iteration. - defaultTickInterval = 2 * time.Second // preallocatedSlotCount is the default number of slots available for // concurrent filesystem operations. The slot count may be exceeded, but // each additional slot will incur an additional allocation. We choose 16 // here with the expectation that it is significantly more than required in // practice. See the comment above the diskHealthCheckingFS type definition. preallocatedSlotCount = 16 + // deltaBits is the number of bits in the packed 64-bit integer used for + // identifying a delta from the file creation time in milliseconds. + deltaBits = 40 + // writeSizeBits is the number of bits in the packed 64-bit integer used for + // identifying the size of the write operation, if the operation is sized. See + // writeSizePrecision below for precision of size. + writeSizeBits = 20 + // Track size of writes at kilobyte precision. See comment above lastWritePacked for more. + writeSizePrecision = 1024 ) +// Variables to enable testing. +var ( + // defaultTickInterval is the default interval between two ticks of each + // diskHealthCheckingFile loop iteration. + defaultTickInterval = 2 * time.Second +) + +// OpType is the type of IO operation being monitored by a +// diskHealthCheckingFile. +type OpType uint8 + +// The following OpTypes is limited to the subset of file system operations that +// a diskHealthCheckingFile supports (namely writes and syncs). +const ( + OpTypeUnknown OpType = iota + OpTypeWrite + OpTypeSync + OpTypeSyncData + OpTypeSyncTo + OpTypeCreate + OpTypeLink + OpTypeMkdirAll + OpTypePreallocate + OpTypeRemove + OpTypeRemoveAll + OpTypeRename + OpTypeReuseForWrite + // Note: opTypeMax is just used in tests. It must appear last in the list + // of OpTypes. + opTypeMax +) + +// String implements fmt.Stringer. +func (o OpType) String() string { + switch o { + case OpTypeWrite: + return "write" + case OpTypeSync: + return "sync" + case OpTypeSyncData: + return "syncdata" + case OpTypeSyncTo: + return "syncto" + case OpTypeCreate: + return "create" + case OpTypeLink: + return "link" + case OpTypeMkdirAll: + return "mkdirall" + case OpTypePreallocate: + return "preallocate" + case OpTypeRemove: + return "remove" + case OpTypeRemoveAll: + return "removall" + case OpTypeRename: + return "rename" + case OpTypeReuseForWrite: + return "reuseforwrite" + case OpTypeUnknown: + return "unknown" + default: + panic(fmt.Sprintf("vfs: unknown op type: %d", o)) + } +} + // diskHealthCheckingFile is a File wrapper to detect slow disk operations, and // call onSlowDisk if a disk operation is seen to exceed diskSlowThreshold. // @@ -32,28 +109,51 @@ const ( // duration. This setup is preferable to creating a new timer at every disk // operation, as it reduces overhead per disk operation. type diskHealthCheckingFile struct { - File - - onSlowDisk func(time.Duration) + file File + onSlowDisk func(opType OpType, writeSizeInBytes int, duration time.Duration) diskSlowThreshold time.Duration tickInterval time.Duration - stopper chan struct{} - lastWriteNanos int64 + stopper chan struct{} + // lastWritePacked is a 64-bit unsigned int. The most significant + // 40 bits represent an delta (in milliseconds) from the creation + // time of the diskHealthCheckingFile. The next most significant 20 bits + // represent the size of the write in KBs, if the write has a size. (If + // it doesn't, the 20 bits are zeroed). The least significant four bits + // contains the OpType. + // + // The use of 40 bits for an delta provides ~34 years of effective + // monitoring time before the uint wraps around, at millisecond precision. + // ~34 years of process uptime "ought to be enough for anybody". Millisecond + // writeSizePrecision is sufficient, given that we are monitoring for writes that take + // longer than one millisecond. + // + // The use of 20 bits for the size in KBs allows representing sizes up + // to nearly one GB. If the write is larger than that, we round down to ~one GB. + // + // The use of four bits for OpType allows for 16 operation types. + // + // NB: this packing scheme is not persisted, and is therefore safe to adjust + // across process boundaries. + lastWritePacked uint64 + createTime time.Time } // newDiskHealthCheckingFile instantiates a new diskHealthCheckingFile, with the // specified time threshold and event listener. func newDiskHealthCheckingFile( - file File, diskSlowThreshold time.Duration, onSlowDisk func(time.Duration), + file File, + diskSlowThreshold time.Duration, + onSlowDisk func(OpType OpType, writeSizeInBytes int, duration time.Duration), ) *diskHealthCheckingFile { return &diskHealthCheckingFile{ - File: file, + file: file, onSlowDisk: onSlowDisk, diskSlowThreshold: diskSlowThreshold, tickInterval: defaultTickInterval, - stopper: make(chan struct{}), + stopper: make(chan struct{}), + createTime: time.Now(), } } @@ -74,16 +174,17 @@ func (d *diskHealthCheckingFile) startTicker() { return case <-ticker.C: - lastWriteNanos := atomic.LoadInt64(&d.lastWriteNanos) - if lastWriteNanos == 0 { + packed := atomic.LoadUint64(&d.lastWritePacked) + if packed == 0 { continue } - lastWrite := time.Unix(0, lastWriteNanos) + delta, writeSize, op := unpack(packed) + lastWrite := d.createTime.Add(delta) now := time.Now() if lastWrite.Add(d.diskSlowThreshold).Before(now) { // diskSlowThreshold was exceeded. Call the passed-in // listener. - d.onSlowDisk(now.Sub(lastWrite)) + d.onSlowDisk(op, writeSize, now.Sub(lastWrite)) } } } @@ -95,10 +196,25 @@ func (d *diskHealthCheckingFile) stopTicker() { close(d.stopper) } +// Fd implements (vfs.File).Fd. +func (d *diskHealthCheckingFile) Fd() uintptr { + return d.file.Fd() +} + +// Read implements (vfs.File).Read +func (d *diskHealthCheckingFile) Read(p []byte) (int, error) { + return d.file.Read(p) +} + +// ReadAt implements (vfs.File).ReadAt +func (d *diskHealthCheckingFile) ReadAt(p []byte, off int64) (int, error) { + return d.file.ReadAt(p, off) +} + // Write implements the io.Writer interface. func (d *diskHealthCheckingFile) Write(p []byte) (n int, err error) { - d.timeDiskOp(func() { - n, err = d.File.Write(p) + d.timeDiskOp(OpTypeWrite, int64(len(p)), func() { + n, err = d.file.Write(p) }) return n, err } @@ -106,32 +222,164 @@ func (d *diskHealthCheckingFile) Write(p []byte) (n int, err error) { // Close implements the io.Closer interface. func (d *diskHealthCheckingFile) Close() error { d.stopTicker() - return d.File.Close() + return d.file.Close() } -// Sync implements the io.Syncer interface. -func (d *diskHealthCheckingFile) Sync() (err error) { - d.timeDiskOp(func() { - err = d.File.Sync() +// Prefetch implements (vfs.File).Prefetch. +func (d *diskHealthCheckingFile) Prefetch(offset, length int64) error { + return d.file.Prefetch(offset, length) +} + +// Preallocate implements (vfs.File).Preallocate. +func (d *diskHealthCheckingFile) Preallocate(off, n int64) (err error) { + d.timeDiskOp(OpTypePreallocate, n, func() { + err = d.file.Preallocate(off, n) }) return err } +// Stat implements (vfs.File).Stat. +func (d *diskHealthCheckingFile) Stat() (os.FileInfo, error) { + return d.file.Stat() +} + +// Sync implements the io.Syncer interface. +func (d *diskHealthCheckingFile) Sync() (err error) { + d.timeDiskOp(OpTypeSync, 0, func() { + err = d.file.Sync() + }) + return err +} + +// SyncData implements (vfs.File).SyncData. +func (d *diskHealthCheckingFile) SyncData() (err error) { + d.timeDiskOp(OpTypeSyncData, 0, func() { + err = d.file.SyncData() + }) + return err +} + +// SyncTo implements (vfs.File).SyncTo. +func (d *diskHealthCheckingFile) SyncTo(length int64) (fullSync bool, err error) { + d.timeDiskOp(OpTypeSyncTo, length, func() { + fullSync, err = d.file.SyncTo(length) + }) + return fullSync, err +} + // timeDiskOp runs the specified closure and makes its timing visible to the // monitoring goroutine, in case it exceeds one of the slow disk durations. -func (d *diskHealthCheckingFile) timeDiskOp(op func()) { +// opType should always be set. writeSizeInBytes should be set if the write +// operation is sized. If not, it should be set to zero. +func (d *diskHealthCheckingFile) timeDiskOp(opType OpType, writeSizeInBytes int64, op func()) { if d == nil { op() return } - atomic.StoreInt64(&d.lastWriteNanos, time.Now().UnixNano()) + delta := time.Since(d.createTime) + packed := pack(delta, writeSizeInBytes, opType) + if invariants.Enabled { + if !atomic.CompareAndSwapUint64(&d.lastWritePacked, 0, packed) { + panic("concurrent write operations detected on file") + } + } else { + atomic.StoreUint64(&d.lastWritePacked, packed) + } defer func() { - atomic.StoreInt64(&d.lastWriteNanos, 0) + if invariants.Enabled { + if !atomic.CompareAndSwapUint64(&d.lastWritePacked, packed, 0) { + panic("concurrent write operations detected on file") + } + } else { + atomic.StoreUint64(&d.lastWritePacked, 0) + } }() op() } +// Note the slight lack of symmetry between pack & unpack. pack takes an int64 for writeSizeInBytes, since +// callers of pack use an int64. This is dictated by the vfs interface. unpack OTOH returns an int. This is +// safe because the packing scheme implies we only actually need 32 bits. +func pack(delta time.Duration, writeSizeInBytes int64, opType OpType) uint64 { + // We have no guarantee of clock monotonicity. If we have a small regression + // in the clock, we set deltaMillis to zero, so we can still catch the operation + // if happens to be slow. + deltaMillis := delta.Milliseconds() + if deltaMillis < 0 { + deltaMillis = 0 + } + // As of 3/7/2023, the use of 40 bits for an delta provides ~34 years + // of effective monitoring time before the uint wraps around, at millisecond + // precision. + if deltaMillis > 1< writeSizeCeiling { + writeSize = writeSizeCeiling + } + + return uint64(deltaMillis)<<(64-deltaBits) | uint64(writeSize)<<(64-deltaBits-writeSizeBits) | uint64(opType) +} + +func unpack(packed uint64) (delta time.Duration, writeSizeInBytes int, opType OpType) { + delta = time.Duration(packed>>(64-deltaBits)) * time.Millisecond + wz := int64(packed>>(64-deltaBits-writeSizeBits)) & ((1 << writeSizeBits) - 1) * writeSizePrecision + // Given the packing scheme, converting wz to an int will not truncate anything. + writeSizeInBytes = int(wz) + opType = OpType(packed & 0xf) + return delta, writeSizeInBytes, opType +} + +// diskHealthCheckingDir implements disk-health checking for directories. Unlike +// other files, we allow directories to receive concurrent write operations +// (Syncs are the only write operations supported by a directory.) Since the +// diskHealthCheckingFile's timeDiskOp can only track a single in-flight +// operation at a time, we time the operation using the filesystem-level +// timeFilesystemOp function instead. +type diskHealthCheckingDir struct { + File + name string + fs *diskHealthCheckingFS +} + +// Sync implements the io.Syncer interface. +func (d *diskHealthCheckingDir) Sync() (err error) { + d.fs.timeFilesystemOp(d.name, OpTypeSync, func() { + err = d.File.Sync() + }) + return err +} + +// DiskSlowInfo captures info about detected slow operations on the vfs. +type DiskSlowInfo struct { + // Path of file being written to. + Path string + // Operation being performed on the file. + OpType OpType + // Size of write in bytes, if the write is sized. + WriteSize int + // Duration that has elapsed since this disk operation started. + Duration time.Duration +} + +func (i DiskSlowInfo) String() string { + return redact.StringWithoutMarkers(i) +} + +// SafeFormat implements redact.SafeFormatter. +func (i DiskSlowInfo) SafeFormat(w redact.SafePrinter, _ rune) { + w.Printf("disk slowness detected: %s on file %s (%d bytes) has been ongoing for %0.1fs", + redact.Safe(i.OpType.String()), redact.Safe(filepath.Base(i.Path)), + redact.Safe(i.WriteSize), redact.Safe(i.Duration.Seconds())) +} + // diskHealthCheckingFS adds disk-health checking facilities to a VFS. // It times disk write operations in two ways: // @@ -156,22 +404,22 @@ func (d *diskHealthCheckingFile) timeDiskOp(op func()) { // maximum concurrent filesystem operations. This is expected to be very few // for these reasons: // 1. Pebble has limited write concurrency. Flushes, compactions and WAL -// rotations are the primary sources of filesystem metadata operations. With -// the default max-compaction concurrency, these operations require at most 5 -// concurrent slots if all 5 perform a filesystem metadata operation -// simultaneously. +// rotations are the primary sources of filesystem metadata operations. With +// the default max-compaction concurrency, these operations require at most 5 +// concurrent slots if all 5 perform a filesystem metadata operation +// simultaneously. // 2. Pebble's limited concurrent I/O writers spend most of their time -// performing file I/O, not performing the filesystem metadata operations that -// require recording a slot on the diskHealthCheckingFS. +// performing file I/O, not performing the filesystem metadata operations that +// require recording a slot on the diskHealthCheckingFS. // 3. In CockroachDB, each additional store/Pebble instance has its own vfs.FS -// which provides a separate goroutine and set of slots. +// which provides a separate goroutine and set of slots. // 4. In CockroachDB, many of the additional sources of filesystem metadata -// operations (like encryption-at-rest) are sequential with respect to Pebble's -// threads. +// operations (like encryption-at-rest) are sequential with respect to Pebble's +// threads. type diskHealthCheckingFS struct { tickInterval time.Duration diskSlowThreshold time.Duration - onSlowDisk func(string, time.Duration) + onSlowDisk func(DiskSlowInfo) fs FS mu struct { sync.Mutex @@ -194,6 +442,7 @@ type diskHealthCheckingFS struct { type slot struct { name string + opType OpType startNanos int64 } @@ -207,7 +456,7 @@ var _ FS = (*diskHealthCheckingFS)(nil) // // A threshold of zero disables disk-health checking. func WithDiskHealthChecks( - innerFS FS, diskSlowThreshold time.Duration, onSlowDisk func(string, time.Duration), + innerFS FS, diskSlowThreshold time.Duration, onSlowDisk func(info DiskSlowInfo), ) (FS, io.Closer) { if diskSlowThreshold == 0 { return innerFS, noopCloser{} @@ -230,7 +479,7 @@ func WithDiskHealthChecks( return fs, fs } -func (d *diskHealthCheckingFS) timeFilesystemOp(name string, op func()) { +func (d *diskHealthCheckingFS) timeFilesystemOp(name string, opType OpType, op func()) { if d == nil { op() return @@ -265,6 +514,7 @@ func (d *diskHealthCheckingFS) timeFilesystemOp(name string, op func()) { // This slot is not in use. Claim it. s = d.mu.inflight[i] s.name = name + s.opType = opType atomic.StoreInt64(&s.startNanos, startNanos) break } @@ -277,6 +527,7 @@ func (d *diskHealthCheckingFS) timeFilesystemOp(name string, op func()) { if s == nil { s = &slot{ name: name, + opType: opType, startNanos: startNanos, } d.mu.inflight = append(d.mu.inflight, s) @@ -297,23 +548,38 @@ func (d *diskHealthCheckingFS) startTickerLocked() { go func() { ticker := time.NewTicker(d.tickInterval) defer ticker.Stop() + var exceededSlots []slot for { select { case <-ticker.C: // Scan the inflight slots for any slots recording a start // time older than the diskSlowThreshold. + exceededSlots = exceededSlots[:0] d.mu.Lock() now := time.Now() for i := range d.mu.inflight { nanos := atomic.LoadInt64(&d.mu.inflight[i].startNanos) if nanos != 0 && time.Unix(0, nanos).Add(d.diskSlowThreshold).Before(now) { - // diskSlowThreshold was exceeded. Invoke the provided - // callback. - d.onSlowDisk(d.mu.inflight[i].name, now.Sub(time.Unix(0, nanos))) + // diskSlowThreshold was exceeded. Copy this inflightOp into + // exceededSlots and call d.onSlowDisk after dropping the mutex. + var inflightOp slot + inflightOp.name = d.mu.inflight[i].name + inflightOp.opType = d.mu.inflight[i].opType + inflightOp.startNanos = nanos + exceededSlots = append(exceededSlots, inflightOp) } } d.mu.Unlock() + for i := range exceededSlots { + d.onSlowDisk( + DiskSlowInfo{ + Path: exceededSlots[i].name, + OpType: exceededSlots[i].opType, + WriteSize: 0, // writes at the fs level are not sized + Duration: now.Sub(time.Unix(0, exceededSlots[i].startNanos)), + }) + } case <-stopper: return } @@ -354,7 +620,7 @@ func (d *diskHealthCheckingFS) Close() error { func (d *diskHealthCheckingFS) Create(name string) (File, error) { var f File var err error - d.timeFilesystemOp(name, func() { + d.timeFilesystemOp(name, OpTypeCreate, func() { f, err = d.fs.Create(name) }) if err != nil { @@ -363,11 +629,17 @@ func (d *diskHealthCheckingFS) Create(name string) (File, error) { if d.diskSlowThreshold == 0 { return f, nil } - checkingFile := newDiskHealthCheckingFile(f, d.diskSlowThreshold, func(duration time.Duration) { - d.onSlowDisk(name, duration) + checkingFile := newDiskHealthCheckingFile(f, d.diskSlowThreshold, func(opType OpType, writeSizeInBytes int, duration time.Duration) { + d.onSlowDisk( + DiskSlowInfo{ + Path: name, + OpType: opType, + WriteSize: writeSizeInBytes, + Duration: duration, + }) }) checkingFile.startTicker() - return WithFd(f, checkingFile), nil + return checkingFile, nil } // GetDiskUsage implements the FS interface. @@ -378,7 +650,7 @@ func (d *diskHealthCheckingFS) GetDiskUsage(path string) (DiskUsage, error) { // Link implements the FS interface. func (d *diskHealthCheckingFS) Link(oldname, newname string) error { var err error - d.timeFilesystemOp(newname, func() { + d.timeFilesystemOp(newname, OpTypeLink, func() { err = d.fs.Link(oldname, newname) }) return err @@ -397,7 +669,7 @@ func (d *diskHealthCheckingFS) Lock(name string) (io.Closer, error) { // MkdirAll implements the FS interface. func (d *diskHealthCheckingFS) MkdirAll(dir string, perm os.FileMode) error { var err error - d.timeFilesystemOp(dir, func() { + d.timeFilesystemOp(dir, OpTypeMkdirAll, func() { err = d.fs.MkdirAll(dir, perm) }) return err @@ -416,11 +688,11 @@ func (d *diskHealthCheckingFS) OpenDir(name string) (File, error) { } // Directories opened with OpenDir must be opened with health checking, // because they may be explicitly synced. - checkingFile := newDiskHealthCheckingFile(f, d.diskSlowThreshold, func(duration time.Duration) { - d.onSlowDisk(name, duration) - }) - checkingFile.startTicker() - return WithFd(f, checkingFile), nil + return &diskHealthCheckingDir{ + File: f, + name: name, + fs: d, + }, nil } // PathBase implements the FS interface. @@ -441,7 +713,7 @@ func (d *diskHealthCheckingFS) PathDir(path string) string { // Remove implements the FS interface. func (d *diskHealthCheckingFS) Remove(name string) error { var err error - d.timeFilesystemOp(name, func() { + d.timeFilesystemOp(name, OpTypeRemove, func() { err = d.fs.Remove(name) }) return err @@ -450,7 +722,7 @@ func (d *diskHealthCheckingFS) Remove(name string) error { // RemoveAll implements the FS interface. func (d *diskHealthCheckingFS) RemoveAll(name string) error { var err error - d.timeFilesystemOp(name, func() { + d.timeFilesystemOp(name, OpTypeRemoveAll, func() { err = d.fs.RemoveAll(name) }) return err @@ -459,7 +731,7 @@ func (d *diskHealthCheckingFS) RemoveAll(name string) error { // Rename implements the FS interface. func (d *diskHealthCheckingFS) Rename(oldname, newname string) error { var err error - d.timeFilesystemOp(newname, func() { + d.timeFilesystemOp(newname, OpTypeRename, func() { err = d.fs.Rename(oldname, newname) }) return err @@ -469,7 +741,7 @@ func (d *diskHealthCheckingFS) Rename(oldname, newname string) error { func (d *diskHealthCheckingFS) ReuseForWrite(oldname, newname string) (File, error) { var f File var err error - d.timeFilesystemOp(newname, func() { + d.timeFilesystemOp(newname, OpTypeReuseForWrite, func() { f, err = d.fs.ReuseForWrite(oldname, newname) }) if err != nil { @@ -478,11 +750,17 @@ func (d *diskHealthCheckingFS) ReuseForWrite(oldname, newname string) (File, err if d.diskSlowThreshold == 0 { return f, nil } - checkingFile := newDiskHealthCheckingFile(f, d.diskSlowThreshold, func(duration time.Duration) { - d.onSlowDisk(newname, duration) + checkingFile := newDiskHealthCheckingFile(f, d.diskSlowThreshold, func(opType OpType, writeSizeInBytes int, duration time.Duration) { + d.onSlowDisk( + DiskSlowInfo{ + Path: newname, + OpType: opType, + WriteSize: writeSizeInBytes, + Duration: duration, + }) }) checkingFile.startTicker() - return WithFd(f, checkingFile), nil + return checkingFile, nil } // Stat implements the FS interface. diff --git a/vendor/github.com/cockroachdb/pebble/vfs/disk_usage_netbsd.go b/vendor/github.com/cockroachdb/pebble/vfs/disk_usage_netbsd.go new file mode 100644 index 0000000..8b7e9dd --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/vfs/disk_usage_netbsd.go @@ -0,0 +1,26 @@ +// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +//go:build netbsd +// +build netbsd + +package vfs + +import "golang.org/x/sys/unix" + +func (defaultFS) GetDiskUsage(path string) (DiskUsage, error) { + stat := unix.Statvfs_t{} + if err := unix.Statvfs(path, &stat); err != nil { + return DiskUsage{}, err + } + + freeBytes := uint64(stat.Bsize) * uint64(stat.Bfree) + availBytes := uint64(stat.Bsize) * uint64(stat.Bavail) + totalBytes := uint64(stat.Bsize) * uint64(stat.Blocks) + return DiskUsage{ + AvailBytes: availBytes, + TotalBytes: totalBytes, + UsedBytes: totalBytes - freeBytes, + }, nil +} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/errors_unix.go b/vendor/github.com/cockroachdb/pebble/vfs/errors_unix.go index 19a671e..2d05e14 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/errors_unix.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/errors_unix.go @@ -2,19 +2,17 @@ // of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -//go:build darwin || dragonfly || freebsd || linux || openbsd -// +build darwin dragonfly freebsd linux openbsd +//go:build darwin || dragonfly || freebsd || linux || openbsd || netbsd +// +build darwin dragonfly freebsd linux openbsd netbsd package vfs import ( - "syscall" - "github.com/cockroachdb/errors" "golang.org/x/sys/unix" ) -var errNotEmpty = syscall.ENOTEMPTY +var errNotEmpty = unix.ENOTEMPTY // IsNoSpaceError returns true if the given error indicates that the disk is // out of space. diff --git a/vendor/github.com/cockroachdb/pebble/vfs/fd.go b/vendor/github.com/cockroachdb/pebble/vfs/fd.go deleted file mode 100644 index 199d304..0000000 --- a/vendor/github.com/cockroachdb/pebble/vfs/fd.go +++ /dev/null @@ -1,42 +0,0 @@ -// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -package vfs - -// fdGetter is an interface for a file with an Fd() method. A lot of -// File related optimizations (eg. Prefetch(), WAL recycling) rely on the -// existence of the Fd method to return a raw file descriptor. -type fdGetter interface { - Fd() uintptr -} - -// fdFileWrapper is a File wrapper that also exposes an Fd() method. Used to -// wrap outer (wrapped) Files that could unintentionally hide the Fd() method -// exposed by the inner (unwrapped) File. It effectively lets the Fd() method -// bypass the outer File and go to the inner File. -type fdFileWrapper struct { - File - - // All methods usually pass through to File above, except for Fd(), which - // bypasses it and gets called directly on the inner file. - inner fdGetter -} - -func (f *fdFileWrapper) Fd() uintptr { - return f.inner.Fd() -} - -// WithFd takes an inner (unwrapped) and an outer (wrapped) vfs.File, -// and returns an fdFileWrapper if the inner file has an Fd() method. Use this -// method to fix the hiding of the Fd() method and the subsequent unintentional -// disabling of Fd-related file optimizations. -func WithFd(inner, outer File) File { - if f, ok := inner.(fdGetter); ok { - return &fdFileWrapper{ - File: outer, - inner: f, - } - } - return outer -} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/file_lock_unix.go b/vendor/github.com/cockroachdb/pebble/vfs/file_lock_unix.go index 7c195c5..4d05e8c 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/file_lock_unix.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/file_lock_unix.go @@ -11,9 +11,9 @@ import ( "io" "os" "sync" - "syscall" "github.com/cockroachdb/errors" + "golang.org/x/sys/unix" ) var lockedFiles struct { @@ -54,14 +54,14 @@ func (defaultFS) Lock(name string) (io.Closer, error) { if err != nil { return nil, err } - spec := syscall.Flock_t{ - Type: syscall.F_WRLCK, + spec := unix.Flock_t{ + Type: unix.F_WRLCK, Whence: io.SeekStart, Start: 0, Len: 0, // 0 means to lock the entire file. Pid: int32(os.Getpid()), } - if err := syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &spec); err != nil { + if err := unix.FcntlFlock(f.Fd(), unix.F_SETLK, &spec); err != nil { f.Close() return nil, err } diff --git a/vendor/github.com/cockroachdb/pebble/vfs/file_lock_windows.go b/vendor/github.com/cockroachdb/pebble/vfs/file_lock_windows.go index 8c0feaa..31fd17a 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/file_lock_windows.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/file_lock_windows.go @@ -9,29 +9,30 @@ package vfs import ( "io" - "syscall" + + "golang.org/x/sys/windows" ) -// lockCloser hides all of an syscall.Handle's methods, except for Close. +// lockCloser hides all of an windows.Handle's methods, except for Close. type lockCloser struct { - fd syscall.Handle + fd windows.Handle } func (l lockCloser) Close() error { - return syscall.Close(l.fd) + return windows.Close(l.fd) } // Lock locks the given file. On Windows, Locking will fail if the file is // already open by the current process. func (defaultFS) Lock(name string) (io.Closer, error) { - p, err := syscall.UTF16PtrFromString(name) + p, err := windows.UTF16PtrFromString(name) if err != nil { return nil, err } - fd, err := syscall.CreateFile(p, - syscall.GENERIC_READ|syscall.GENERIC_WRITE, - 0, nil, syscall.CREATE_ALWAYS, - syscall.FILE_ATTRIBUTE_NORMAL, + fd, err := windows.CreateFile(p, + windows.GENERIC_READ|windows.GENERIC_WRITE, + 0, nil, windows.CREATE_ALWAYS, + windows.FILE_ATTRIBUTE_NORMAL, 0, ) if err != nil { diff --git a/vendor/github.com/cockroachdb/pebble/vfs/logging_fs.go b/vendor/github.com/cockroachdb/pebble/vfs/logging_fs.go new file mode 100644 index 0000000..3ba5584 --- /dev/null +++ b/vendor/github.com/cockroachdb/pebble/vfs/logging_fs.go @@ -0,0 +1,143 @@ +// Copyright 2021 The LevelDB-Go and Pebble Authors. All rights reserved. Use +// of this source code is governed by a BSD-style license that can be found in +// the LICENSE file. + +package vfs + +import ( + "io" + "os" +) + +// WithLogging wraps an FS and logs filesystem modification operations to the +// given logFn. +func WithLogging(fs FS, logFn LogFn) FS { + return &loggingFS{ + FS: fs, + logFn: logFn, + } +} + +// LogFn is a function that is used to capture a log when WithLogging is used. +type LogFn func(fmt string, args ...interface{}) + +type loggingFS struct { + FS + logFn LogFn +} + +var _ FS = (*loggingFS)(nil) + +func (fs *loggingFS) Create(name string) (File, error) { + fs.logFn("create: %s", name) + f, err := fs.FS.Create(name) + if err != nil { + return nil, err + } + return newLoggingFile(f, name, fs.logFn), nil +} + +func (fs *loggingFS) Open(name string, opts ...OpenOption) (File, error) { + fs.logFn("open: %s", name) + f, err := fs.FS.Open(name, opts...) + if err != nil { + return nil, err + } + return newLoggingFile(f, name, fs.logFn), nil +} + +func (fs *loggingFS) Link(oldname, newname string) error { + fs.logFn("link: %s -> %s", oldname, newname) + return fs.FS.Link(oldname, newname) +} + +func (fs *loggingFS) OpenDir(name string) (File, error) { + fs.logFn("open-dir: %s", name) + f, err := fs.FS.OpenDir(name) + if err != nil { + return nil, err + } + return newLoggingFile(f, name, fs.logFn), nil +} + +func (fs *loggingFS) Rename(oldname, newname string) error { + fs.logFn("rename: %s -> %s", oldname, newname) + return fs.FS.Rename(oldname, newname) +} + +func (fs *loggingFS) ReuseForWrite(oldname, newname string) (File, error) { + fs.logFn("reuseForWrite: %s -> %s", oldname, newname) + f, err := fs.FS.ReuseForWrite(oldname, newname) + if err != nil { + return nil, err + } + return newLoggingFile(f, newname, fs.logFn), nil +} + +func (fs *loggingFS) MkdirAll(dir string, perm os.FileMode) error { + fs.logFn("mkdir-all: %s %#o", dir, perm) + return fs.FS.MkdirAll(dir, perm) +} + +func (fs *loggingFS) Lock(name string) (io.Closer, error) { + fs.logFn("lock: %s", name) + return fs.FS.Lock(name) +} + +func (fs loggingFS) Remove(name string) error { + fs.logFn("remove: %s", name) + err := fs.FS.Remove(name) + return err +} + +func (fs loggingFS) RemoveAll(name string) error { + fs.logFn("remove-all: %s", name) + err := fs.FS.RemoveAll(name) + return err +} + +type loggingFile struct { + File + name string + logFn LogFn +} + +var _ File = (*loggingFile)(nil) + +func newLoggingFile(f File, name string, logFn LogFn) *loggingFile { + return &loggingFile{ + File: f, + name: name, + logFn: logFn, + } +} + +func (f *loggingFile) Close() error { + f.logFn("close: %s", f.name) + return f.File.Close() +} + +func (f *loggingFile) Sync() error { + f.logFn("sync: %s", f.name) + return f.File.Sync() +} + +func (f *loggingFile) SyncData() error { + f.logFn("sync-data: %s", f.name) + return f.File.SyncData() +} + +func (f *loggingFile) SyncTo(length int64) (fullSync bool, err error) { + f.logFn("sync-to(%d): %s", length, f.name) + return f.File.SyncTo(length) +} + +func (f *loggingFile) ReadAt(p []byte, offset int64) (int, error) { + f.logFn("read-at(%d, %d): %s", offset, len(p), f.name) + return f.File.ReadAt(p, offset) +} + +func (f *loggingFile) Prefetch(offset int64, length int64) error { + f.logFn("prefetch(%d, %d): %s", offset, length, f.name) + return f.File.Prefetch(offset, length) +} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/mem_fs.go b/vendor/github.com/cockroachdb/pebble/vfs/mem_fs.go index e343b3e..2b3a6d2 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/mem_fs.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/mem_fs.go @@ -36,22 +36,23 @@ func NewMem() *MemFS { // at which point they are discarded and no longer visible. // // Expected usage: -// strictFS := NewStrictMem() -// db := Open(..., &Options{FS: strictFS}) -// // Do and commit various operations. -// ... -// // Prevent any more changes to finalized state. -// strictFS.SetIgnoreSyncs(true) -// // This will finish any ongoing background flushes, compactions but none of these writes will -// // be finalized since syncs are being ignored. -// db.Close() -// // Discard unsynced state. -// strictFS.ResetToSyncedState() -// // Allow changes to finalized state. -// strictFS.SetIgnoreSyncs(false) -// // Open the DB. This DB should have the same state as if the earlier strictFS operations and -// // db.Close() were not called. -// db := Open(..., &Options{FS: strictFS}) +// +// strictFS := NewStrictMem() +// db := Open(..., &Options{FS: strictFS}) +// // Do and commit various operations. +// ... +// // Prevent any more changes to finalized state. +// strictFS.SetIgnoreSyncs(true) +// // This will finish any ongoing background flushes, compactions but none of these writes will +// // be finalized since syncs are being ignored. +// db.Close() +// // Discard unsynced state. +// strictFS.ResetToSyncedState() +// // Allow changes to finalized state. +// strictFS.SetIgnoreSyncs(false) +// // Open the DB. This DB should have the same state as if the earlier strictFS operations and +// // db.Close() were not called. +// db := Open(..., &Options{FS: strictFS}) func NewStrictMem() *MemFS { return &MemFS{ root: newRootMemNode(), @@ -141,6 +142,7 @@ func (y *MemFS) ResetToSyncedState() { // - "/", "foo", false // - "/foo/", "bar", false // - "/foo/bar/", "x", true +// // Similarly, walking "/y/z/", with a trailing slash, will result in 3 calls to f: // - "/", "y", false // - "/y/", "z", false @@ -624,6 +626,8 @@ type memFile struct { read, write bool } +var _ File = (*memFile)(nil) + func (f *memFile) Close() error { if n := atomic.AddInt32(&f.n.refs, -1); n < 0 { panic(fmt.Sprintf("pebble: close of unopened file: %d", n)) @@ -661,7 +665,11 @@ func (f *memFile) ReadAt(p []byte, off int64) (int, error) { if off >= int64(len(f.n.mu.data)) { return 0, io.EOF } - return copy(p, f.n.mu.data[off:]), nil + n := copy(p, f.n.mu.data[off:]) + if n < len(p) { + return n, io.EOF + } + return n, nil } func (f *memFile) Write(p []byte) (int, error) { @@ -694,6 +702,9 @@ func (f *memFile) Write(p []byte) (int, error) { return len(p), nil } +func (f *memFile) Prefetch(offset int64, length int64) error { return nil } +func (f *memFile) Preallocate(offset, length int64) error { return nil } + func (f *memFile) Stat() (os.FileInfo, error) { return f.n, nil } @@ -719,6 +730,22 @@ func (f *memFile) Sync() error { return nil } +func (f *memFile) SyncData() error { + return f.Sync() +} + +func (f *memFile) SyncTo(length int64) (fullSync bool, err error) { + // NB: This SyncTo implementation lies, with its return values claiming it + // synced the data up to `length`. When fullSync=false, SyncTo provides no + // durability guarantees, so this can help surface bugs where we improperly + // rely on SyncTo providing durability. + return false, nil +} + +func (f *memFile) Fd() uintptr { + return InvalidFd +} + // Flush is a no-op and present only to prevent buffering at higher levels // (e.g. it prevents sstable.Writer from using a bufio.Writer). func (f *memFile) Flush() error { diff --git a/vendor/github.com/cockroachdb/pebble/vfs/prefetch_generic.go b/vendor/github.com/cockroachdb/pebble/vfs/prefetch_generic.go deleted file mode 100644 index c18904a..0000000 --- a/vendor/github.com/cockroachdb/pebble/vfs/prefetch_generic.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -//go:build !linux -// +build !linux - -package vfs - -// Prefetch signals the OS (on supported platforms) to fetch the next size -// bytes in file (as returned by os.File.Fd()) after offset into cache. Any -// subsequent reads in that range will not issue disk IO. -func Prefetch(file uintptr, offset uint64, size uint64) error { - // No-op. - return nil -} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/prefetch_linux.go b/vendor/github.com/cockroachdb/pebble/vfs/prefetch_linux.go deleted file mode 100644 index 0c5b2be..0000000 --- a/vendor/github.com/cockroachdb/pebble/vfs/prefetch_linux.go +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright 2020 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -//go:build linux -// +build linux - -package vfs - -import "syscall" - -// Prefetch signals the OS (on supported platforms) to fetch the next size -// bytes in file (as returned by os.File.Fd()) after offset into cache. Any -// subsequent reads in that range will not issue disk IO. -func Prefetch(file uintptr, offset uint64, size uint64) error { - _, _, err := syscall.Syscall(syscall.SYS_READAHEAD, uintptr(file), uintptr(offset), uintptr(size)) - return err -} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/syncing_file.go b/vendor/github.com/cockroachdb/pebble/vfs/syncing_file.go index cbf7064..d1a4ae1 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/syncing_file.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/syncing_file.go @@ -12,6 +12,8 @@ import ( // SyncingFileOptions holds the options for a syncingFile. type SyncingFileOptions struct { + // NoSyncOnClose elides the automatic Sync during Close if it's not possible + // to sync the remainder of the file in a non-blocking way. NoSyncOnClose bool BytesPerSync int PreallocateSize int @@ -19,9 +21,8 @@ type SyncingFileOptions struct { type syncingFile struct { File + // fd can be InvalidFd if the underlying File does not support it. fd uintptr - useSyncRange bool - closing bool noSyncOnClose bool bytesPerSync int64 preallocateSize int64 @@ -36,9 +37,6 @@ type syncingFile struct { syncOffset int64 } preallocatedBlocks int64 - syncData func() error - syncTo func(offset int64) error - timeDiskOp func(op func()) } // NewSyncingFile wraps a writable file and ensures that data is synced @@ -49,6 +47,7 @@ type syncingFile struct { func NewSyncingFile(f File, opts SyncingFileOptions) File { s := &syncingFile{ File: f, + fd: f.Fd(), noSyncOnClose: bool(opts.NoSyncOnClose), bytesPerSync: int64(opts.BytesPerSync), preallocateSize: int64(opts.PreallocateSize), @@ -56,30 +55,7 @@ func NewSyncingFile(f File, opts SyncingFileOptions) File { // Ensure a file that is opened and then closed will be synced, even if no // data has been written to it. s.atomic.syncOffset = -1 - - type fd interface { - Fd() uintptr - } - if d, ok := f.(fd); ok { - s.fd = d.Fd() - } - type dhChecker interface { - timeDiskOp(op func()) - } - if d, ok := f.(dhChecker); ok { - s.timeDiskOp = d.timeDiskOp - } else { - s.timeDiskOp = func(op func()) { - op() - } - } - - s.init() - - if s.syncData == nil { - s.syncData = s.File.Sync - } - return WithFd(f, s) + return s } // NB: syncingFile.Write is unsafe for concurrent use! @@ -100,7 +76,7 @@ func (f *syncingFile) Write(p []byte) (n int, err error) { } func (f *syncingFile) preallocate(offset int64) error { - if f.fd == 0 || f.preallocateSize == 0 { + if f.fd == InvalidFd || f.preallocateSize == 0 { return nil } @@ -112,7 +88,7 @@ func (f *syncingFile) preallocate(offset int64) error { length := f.preallocateSize * (newPreallocatedBlocks - f.preallocatedBlocks) offset = f.preallocateSize * f.preallocatedBlocks f.preallocatedBlocks = newPreallocatedBlocks - return preallocExtend(f.fd, offset, length) + return f.Preallocate(offset, length) } func (f *syncingFile) ratchetSyncOffset(offset int64) { @@ -131,9 +107,10 @@ func (f *syncingFile) Sync() error { // We update syncOffset (atomically) in order to avoid spurious syncs in // maybeSync. Note that even if syncOffset is larger than the current file // offset, we still need to call the underlying file's sync for persistence - // guarantees (which are not provided by sync_file_range). + // guarantees which are not provided by SyncTo (or by sync_file_range on + // Linux). f.ratchetSyncOffset(atomic.LoadInt64(&f.atomic.offset)) - return f.syncData() + return f.SyncData() } func (f *syncingFile) maybeSync() error { @@ -163,30 +140,78 @@ func (f *syncingFile) maybeSync() error { return nil } - if f.fd == 0 { + if f.fd == InvalidFd { return errors.WithStack(f.Sync()) } - // Note that syncTo will always be called with an offset < atomic.offset. The - // syncTo implementation may choose to sync the entire file (i.e. on OSes - // which do not support syncing a portion of the file). The syncTo - // implementation must call ratchetSyncOffset with as much of the file as it - // has synced. - return errors.WithStack(f.syncTo(syncToOffset)) + // Note that SyncTo will always be called with an offset < atomic.offset. + // The SyncTo implementation may choose to sync the entire file (i.e. on + // OSes which do not support syncing a portion of the file). + fullSync, err := f.SyncTo(syncToOffset) + if err != nil { + return errors.WithStack(err) + } + if fullSync { + f.ratchetSyncOffset(offset) + } else { + f.ratchetSyncOffset(syncToOffset) + } + return nil } func (f *syncingFile) Close() error { // Sync any data that has been written but not yet synced unless the file // has noSyncOnClose option explicitly set. - // Note that if SyncFileRange was used, atomic.syncOffset will be less than - // atomic.offset. See syncingFile.syncToRange. - f.closing = true - if !f.noSyncOnClose || f.useSyncRange { - if atomic.LoadInt64(&f.atomic.offset) > atomic.LoadInt64(&f.atomic.syncOffset) { - if err := f.Sync(); err != nil { - return errors.WithStack(err) + // + // NB: If the file is capable of non-durability-guarantee SyncTos, and the + // caller has not called Sync since the last write, syncOffset is guaranteed + // to be less than atomic.offset. This ensures we fall into the below + // conditional and perform a full sync to durably persist the file. + if off := atomic.LoadInt64(&f.atomic.offset); off > atomic.LoadInt64(&f.atomic.syncOffset) { + // There's still remaining dirty data. + + if f.noSyncOnClose { + // If NoSyncOnClose is set, only perform a SyncTo. On linux, SyncTo + // translates to a non-blocking `sync_file_range` call which + // provides no persistence guarantee. Since it's non-blocking, + // there's no latency hit of a blocking sync call, but we still + // ensure we're not allowing significant dirty data to accumulate. + if _, err := f.File.SyncTo(off); err != nil { + return err } + f.ratchetSyncOffset(off) + } else if err := f.Sync(); err != nil { + return errors.WithStack(err) } } return errors.WithStack(f.File.Close()) } + +// NewSyncingFS wraps a vfs.FS with one that wraps newly created files with +// vfs.NewSyncingFile. +func NewSyncingFS(fs FS, syncOpts SyncingFileOptions) FS { + return &syncingFS{ + FS: fs, + syncOpts: syncOpts, + } +} + +type syncingFS struct { + FS + syncOpts SyncingFileOptions +} + +var _ FS = (*syncingFS)(nil) + +func (fs *syncingFS) Create(name string) (File, error) { + f, err := fs.FS.Create(name) + if err != nil { + return nil, err + } + return NewSyncingFile(f, fs.syncOpts), nil +} + +func (fs *syncingFS) ReuseForWrite(oldname, newname string) (File, error) { + // TODO(radu): implement this if needed. + panic("unimplemented") +} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/syncing_file_generic.go b/vendor/github.com/cockroachdb/pebble/vfs/syncing_file_generic.go deleted file mode 100644 index 7835b72..0000000 --- a/vendor/github.com/cockroachdb/pebble/vfs/syncing_file_generic.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -//go:build !linux || arm -// +build !linux arm - -package vfs - -func (f *syncingFile) init() { - f.syncTo = f.syncToGeneric -} - -func (f *syncingFile) syncToGeneric(_ int64) error { - return f.Sync() -} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/syncing_file_linux.go b/vendor/github.com/cockroachdb/pebble/vfs/syncing_file_linux.go deleted file mode 100644 index 10b23a1..0000000 --- a/vendor/github.com/cockroachdb/pebble/vfs/syncing_file_linux.go +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright 2019 The LevelDB-Go and Pebble Authors. All rights reserved. Use -// of this source code is governed by a BSD-style license that can be found in -// the LICENSE file. - -//go:build linux && !arm -// +build linux,!arm - -package vfs - -import "syscall" - -type syncFileRange func(fd int, off int64, n int64, flags int) (err error) - -// sync_file_range depends on both the filesystem, and the broader kernel -// support. In particular, Windows Subsystem for Linux does not support -// sync_file_range, even when used with ext{2,3,4}. syncRangeSmokeTest performs -// a test of of sync_file_range, returning false on ENOSYS, and true otherwise. -func syncRangeSmokeTest(fd uintptr, fn syncFileRange) bool { - err := fn(int(fd), 0 /* offset */, 0 /* nbytes */, 0 /* flags */) - return err != syscall.ENOSYS -} - -func isSyncRangeSupported(fd uintptr) bool { - var stat syscall.Statfs_t - if err := syscall.Fstatfs(int(fd), &stat); err != nil { - return false - } - - // Allowlist which filesystems we allow using sync_file_range with as some - // filesystems treat that syscall as a noop (notably ZFS). A allowlist is - // used instead of a denylist in order to have a more graceful failure mode - // in case a filesystem we haven't tested is encountered. Currently only - // ext2/3/4 are known to work properly. - const extMagic = 0xef53 - switch stat.Type { - case extMagic: - return syncRangeSmokeTest(fd, syscall.SyncFileRange) - } - return false -} - -func (f *syncingFile) init() { - if f.fd == 0 { - return - } - f.timeDiskOp(func() { - f.useSyncRange = isSyncRangeSupported(f.fd) - }) - if f.useSyncRange { - f.syncTo = f.syncToRange - } else { - f.syncTo = f.syncToFdatasync - } - f.syncData = f.syncFdatasync -} - -func (f *syncingFile) syncFdatasync() error { - if f.fd == 0 { - return f.File.Sync() - } - var err error - f.timeDiskOp(func() { - err = syscall.Fdatasync(int(f.fd)) - }) - return err -} - -func (f *syncingFile) syncToFdatasync(_ int64) error { - return f.Sync() -} - -func (f *syncingFile) syncToRange(offset int64) error { - const ( - waitBefore = 0x1 - write = 0x2 - // waitAfter = 0x4 - ) - - // The flags for the sync_file_range system call. Unless the file has - // noSyncOnClose explicitly set and it is being closed, the waitBefore - // flag will be set which may block the call. - flags := write - if !f.noSyncOnClose || !f.closing { - flags |= waitBefore - } - - // Note that syncToRange is only called with an offset that is guaranteed to - // be less than atomic.offset (i.e. the write offset). This implies the - // syncingFile.Close will Sync the rest of the data, as well as the file's - // metadata. - f.ratchetSyncOffset(offset) - - // By specifying write|waitBefore for the flags, we're instructing - // SyncFileRange to a) wait for any outstanding data being written to finish, - // and b) to queue any other dirty data blocks in the range [0,offset] for - // writing. The actual writing of this data will occur asynchronously. The - // use of `waitBefore` is to limit how much dirty data is allowed to - // accumulate. Linux sometimes behaves poorly when a large amount of dirty - // data accumulates, impacting other I/O operations. - var err error - f.timeDiskOp(func() { - err = syscall.SyncFileRange(int(f.fd), 0, offset, flags) - }) - return err -} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/vfs.go b/vendor/github.com/cockroachdb/pebble/vfs/vfs.go index c816a8a..7eafbf6 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/vfs.go +++ b/vendor/github.com/cockroachdb/pebble/vfs/vfs.go @@ -29,10 +29,53 @@ type File interface { // method *is* allowed to modify the slice passed in, whether temporarily // or permanently. Callers of Write() need to take this into account. io.Writer + + // Preallocate optionally preallocates storage for `length` at `offset` + // within the file. Implementations may choose to do nothing. + Preallocate(offset, length int64) error Stat() (os.FileInfo, error) Sync() error + + // SyncTo requests that a prefix of the file's data be synced to stable + // storage. The caller passes provides a `length`, indicating how many bytes + // to sync from the beginning of the file. SyncTo is a no-op for + // directories, and therefore always returns false. + // + // SyncTo returns a fullSync return value, indicating one of two possible + // outcomes. + // + // If fullSync is false, the first `length` bytes of the file was queued to + // be synced to stable storage. The syncing of the file prefix may happen + // asynchronously. No persistence guarantee is provided. + // + // If fullSync is true, the entirety of the file's contents were + // synchronously synced to stable storage, and a persistence guarantee is + // provided. In this outcome, any modified metadata for the file is not + // guaranteed to be synced unless that metadata is needed in order to allow + // a subsequent data retrieval to be correctly handled. + SyncTo(length int64) (fullSync bool, err error) + + // SyncData requires that all written data be persisted. File metadata is + // not required to be synced. Unsophisticated implementations may call Sync. + SyncData() error + + // Prefetch signals the OS (on supported platforms) to fetch the next length + // bytes in file (as returned by os.File.Fd()) after offset into cache. Any + // subsequent reads in that range will not issue disk IO. + Prefetch(offset int64, length int64) error + + // Fd returns the raw file descriptor when a File is backed by an *os.File. + // It can be used for specific functionality like Prefetch. + // Returns InvalidFd if not supported. + Fd() uintptr } +// InvalidFd is a special value returned by File.Fd() when the file is not +// backed by an OS descriptor. +// Note: the special value is consistent with what os.File implementation +// returns on a nil receiver. +const InvalidFd uintptr = ^(uintptr(0)) + // OpenOption provide an interface to do work on file handles in the Open() // call. type OpenOption interface { @@ -144,10 +187,18 @@ var Default FS = defaultFS{} type defaultFS struct{} +// wrapOSFile takes a standard library OS file and returns a vfs.File. f may be +// nil, in which case wrapOSFile must not panic. In such cases, it's okay if the +// returned vfs.File may panic if used. +func wrapOSFile(f *os.File) File { + // See the implementations in default_{linux,unix,windows}.go. + return wrapOSFileImpl(f) +} + func (defaultFS) Create(name string) (File, error) { const openFlags = os.O_RDWR | os.O_CREATE | os.O_EXCL | syscall.O_CLOEXEC - f, err := os.OpenFile(name, openFlags, 0666) + osFile, err := os.OpenFile(name, openFlags, 0666) // If the file already exists, remove it and try again. // // NB: We choose to remove the file instead of truncating it, despite the @@ -158,11 +209,11 @@ func (defaultFS) Create(name string) (File, error) { // attempting to create the a file at the same path. for oserror.IsExist(err) { if removeErr := os.Remove(name); removeErr != nil && !oserror.IsNotExist(removeErr) { - return f, errors.WithStack(removeErr) + return wrapOSFile(osFile), errors.WithStack(removeErr) } - f, err = os.OpenFile(name, openFlags, 0666) + osFile, err = os.OpenFile(name, openFlags, 0666) } - return f, errors.WithStack(err) + return wrapOSFile(osFile), errors.WithStack(err) } func (defaultFS) Link(oldname, newname string) error { @@ -170,10 +221,11 @@ func (defaultFS) Link(oldname, newname string) error { } func (defaultFS) Open(name string, opts ...OpenOption) (File, error) { - file, err := os.OpenFile(name, os.O_RDONLY|syscall.O_CLOEXEC, 0) + osFile, err := os.OpenFile(name, os.O_RDONLY|syscall.O_CLOEXEC, 0) if err != nil { return nil, errors.WithStack(err) } + file := wrapOSFile(osFile) for _, opt := range opts { opt.Apply(file) } @@ -197,7 +249,7 @@ func (fs defaultFS) ReuseForWrite(oldname, newname string) (File, error) { return nil, errors.WithStack(err) } f, err := os.OpenFile(newname, os.O_RDWR|os.O_CREATE|syscall.O_CLOEXEC, 0666) - return f, errors.WithStack(err) + return wrapOSFile(f), errors.WithStack(err) } func (defaultFS) MkdirAll(dir string, perm os.FileMode) error { @@ -240,11 +292,8 @@ var RandomReadsOption OpenOption = &randomReadsOption{} // Apply implements the OpenOption interface. func (randomReadsOption) Apply(f File) { - type fd interface { - Fd() uintptr - } - if fdFile, ok := f.(fd); ok { - _ = fadviseRandom(fdFile.Fd()) + if fd := f.Fd(); fd != InvalidFd { + _ = fadviseRandom(fd) } } @@ -257,24 +306,27 @@ var SequentialReadsOption OpenOption = &sequentialReadsOption{} // Apply implements the OpenOption interface. func (sequentialReadsOption) Apply(f File) { - type fd interface { - Fd() uintptr - } - if fdFile, ok := f.(fd); ok { - _ = fadviseSequential(fdFile.Fd()) + if fd := f.Fd(); fd != InvalidFd { + _ = fadviseSequential(fd) } } // Copy copies the contents of oldname to newname. If newname exists, it will // be overwritten. func Copy(fs FS, oldname, newname string) error { - src, err := fs.Open(oldname) + return CopyAcrossFS(fs, oldname, fs, newname) +} + +// CopyAcrossFS copies the contents of oldname on srcFS to newname dstFS. If +// newname exists, it will be overwritten. +func CopyAcrossFS(srcFS FS, oldname string, dstFS FS, newname string) error { + src, err := srcFS.Open(oldname, SequentialReadsOption) if err != nil { return err } defer src.Close() - dst, err := fs.Create(newname) + dst, err := dstFS.Create(newname) if err != nil { return err } @@ -289,7 +341,7 @@ func Copy(fs FS, oldname, newname string) error { // LimitedCopy copies up to maxBytes from oldname to newname. If newname // exists, it will be overwritten. func LimitedCopy(fs FS, oldname, newname string, maxBytes int64) error { - src, err := fs.Open(oldname) + src, err := fs.Open(oldname, SequentialReadsOption) if err != nil { return err } @@ -309,7 +361,7 @@ func LimitedCopy(fs FS, oldname, newname string, maxBytes int64) error { // LinkOrCopy creates newname as a hard link to the oldname file. If creating // the hard link fails, LinkOrCopy falls back to copying the file (which may -// also fail if newname doesn't exist or oldname already exists). +// also fail if oldname doesn't exist or newname already exists). func LinkOrCopy(fs FS, oldname, newname string) error { err := fs.Link(oldname, newname) if err == nil { diff --git a/vendor/github.com/cockroachdb/redact/builder/builder.go b/vendor/github.com/cockroachdb/redact/builder/builder.go index b4dfcc6..9c0ae29 100644 --- a/vendor/github.com/cockroachdb/redact/builder/builder.go +++ b/vendor/github.com/cockroachdb/redact/builder/builder.go @@ -67,6 +67,24 @@ func (b *StringBuilder) Write(s []byte) (int, error) { return b.Buffer.Write(s) } +// WriteString shadows the ib.Buffer.WriteString method. +func (b *StringBuilder) WriteString(s string) (int, error) { + b.SetMode(ib.UnsafeEscaped) + return b.Buffer.WriteString(s) +} + +// WriteByte shadows the ib.Buffer.WriteByte method. +func (b *StringBuilder) WriteByte(c byte) error { + b.SetMode(ib.UnsafeEscaped) + return b.Buffer.WriteByte(c) +} + +// WriteRune shadows the ib.Buffer.WriteRune method. +func (b *StringBuilder) WriteRune(r rune) error { + b.SetMode(ib.UnsafeEscaped) + return b.Buffer.WriteRune(r) +} + // StringBuilder implements SafeWriter. var _ i.SafeWriter = (*StringBuilder)(nil) diff --git a/vendor/github.com/cockroachdb/redact/internal/rfmt/fmtsort/sort.go b/vendor/github.com/cockroachdb/redact/internal/rfmt/fmtsort/sort.go index 9d2d580..a64d5cd 100644 --- a/vendor/github.com/cockroachdb/redact/internal/rfmt/fmtsort/sort.go +++ b/vendor/github.com/cockroachdb/redact/internal/rfmt/fmtsort/sort.go @@ -132,7 +132,7 @@ func compare(aVal, bVal reflect.Value) int { default: return -1 } - case reflect.Ptr: + case reflect.Ptr, reflect.UnsafePointer: a, b := aVal.Pointer(), bVal.Pointer() switch { case a < b: diff --git a/vendor/github.com/cockroachdb/redact/internal/rfmt/format.go b/vendor/github.com/cockroachdb/redact/internal/rfmt/format.go index 49296f4..56c81d3 100644 --- a/vendor/github.com/cockroachdb/redact/internal/rfmt/format.go +++ b/vendor/github.com/cockroachdb/redact/internal/rfmt/format.go @@ -315,7 +315,7 @@ func (f *fmt) fmtInteger(u uint64, base int, isSigned bool, verb rune, digits st f.zero = oldZero } -// truncate truncates the string s to the specified precision, if present. +// truncateString truncates the string s to the specified precision, if present. func (f *fmt) truncateString(s string) string { if f.precPresent { n := f.prec diff --git a/vendor/github.com/cockroachdb/redact/internal/rfmt/format.go.diff b/vendor/github.com/cockroachdb/redact/internal/rfmt/format.go.diff index 7bf918c..e2479b7 100644 --- a/vendor/github.com/cockroachdb/redact/internal/rfmt/format.go.diff +++ b/vendor/github.com/cockroachdb/redact/internal/rfmt/format.go.diff @@ -1,12 +1,18 @@ ---- format.go.orig 2021-06-18 16:45:51.065116000 +0200 -+++ format.go 2021-06-18 17:30:58.628179000 +0200 -@@ -1,3 +1,6 @@ +--- format.go.orig 2022-01-07 15:25:57.762807000 +0100 ++++ format.go 2022-01-07 15:26:16.154491000 +0100 +@@ -1,8 +1,11 @@ +// Code generated from the Go standard library. DO NOT EDIT +// GENERATED FILE DO NOT EDIT +// // Copyright 2009 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. + +-package fmt ++package rfmt + + import ( + "strconv" @@ -65,25 +68,17 @@ if n <= 0 { // No padding bytes needed. return diff --git a/vendor/github.com/getsentry/sentry-go/.codecov.yml b/vendor/github.com/getsentry/sentry-go/.codecov.yml new file mode 100644 index 0000000..df0b268 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/.codecov.yml @@ -0,0 +1,13 @@ +codecov: + # across + notify: + # Do not notify until at least this number of reports have been uploaded + # from the CI pipeline. We normally have more than that number, but 6 + # should be enough to get a first notification. + after_n_builds: 6 +coverage: + status: + project: + default: + # Do not fail the commit status if the coverage was reduced up to this value + threshold: 0.5% diff --git a/vendor/github.com/getsentry/sentry-go/.craft.yml b/vendor/github.com/getsentry/sentry-go/.craft.yml index 5f786f5..238288e 100644 --- a/vendor/github.com/getsentry/sentry-go/.craft.yml +++ b/vendor/github.com/getsentry/sentry-go/.craft.yml @@ -1,12 +1,13 @@ -minVersion: 0.23.1 -preReleaseCommand: bash scripts/craft-pre-release.sh -changelogPolicy: auto +minVersion: 0.35.0 +changelogPolicy: simple artifactProvider: name: none targets: - name: github - includeNames: /none/ tagPrefix: v + - name: github + tagPrefix: otel/v + tagOnly: true - name: registry sdks: github:getsentry/sentry-go: diff --git a/vendor/github.com/getsentry/sentry-go/.gitignore b/vendor/github.com/getsentry/sentry-go/.gitignore index 5d0e90c..b1249ba 100644 --- a/vendor/github.com/getsentry/sentry-go/.gitignore +++ b/vendor/github.com/getsentry/sentry-go/.gitignore @@ -1,4 +1,8 @@ +# Code coverage artifacts coverage.txt +coverage.out +coverage.html +.coverage/ # Just my personal way of tracking stuff — Kamil FIXME.md diff --git a/vendor/github.com/getsentry/sentry-go/.golangci.yml b/vendor/github.com/getsentry/sentry-go/.golangci.yml index 0deb9c0..0f35323 100644 --- a/vendor/github.com/getsentry/sentry-go/.golangci.yml +++ b/vendor/github.com/getsentry/sentry-go/.golangci.yml @@ -35,6 +35,7 @@ issues: exclude-rules: - path: _test\.go linters: + - goconst - prealloc - path: _test\.go text: "G306:" diff --git a/vendor/github.com/getsentry/sentry-go/CHANGELOG.md b/vendor/github.com/getsentry/sentry-go/CHANGELOG.md index 8c3ae0a..02c783d 100644 --- a/vendor/github.com/getsentry/sentry-go/CHANGELOG.md +++ b/vendor/github.com/getsentry/sentry-go/CHANGELOG.md @@ -1,17 +1,276 @@ # Changelog +## 0.21.0 + +The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.21.0. + +Note: this release includes one **breaking change** and some **deprecations**, which are listed below. + +### Breaking Changes + +**This change does not apply if you use [https://sentry.io](https://sentry.io)** + +- Remove support for the `/store` endpoint ([#631](https://github.com/getsentry/sentry-go/pull/631)) + - This change requires a self-hosted version of Sentry 20.6.0 or higher. If you are using a version of [self-hosted Sentry](https://develop.sentry.dev/self-hosted/) (aka *on-premise*) older than 20.6.0, then you will need to [upgrade](https://develop.sentry.dev/self-hosted/releases/) your instance. + +### Features + +- Rename four span option functions ([#611](https://github.com/getsentry/sentry-go/pull/611), [#624](https://github.com/getsentry/sentry-go/pull/624)) + - `TransctionSource` -> `WithTransactionSource` + - `SpanSampled` -> `WithSpanSampled` + - `OpName` -> `WithOpName` + - `TransactionName` -> `WithTransactionName` + - Old functions `TransctionSource`, `SpanSampled`, `OpName`, and `TransactionName` are still available but are now **deprecated** and will be removed in a future release. +- Make `client.EventFromMessage` and `client.EventFromException` methods public ([#607](https://github.com/getsentry/sentry-go/pull/607)) +- Add `client.SetException` method ([#607](https://github.com/getsentry/sentry-go/pull/607)) + - This allows to set or add errors to an existing `Event`. + +### Bug Fixes + +- Protect from panics while doing concurrent reads/writes to Span data fields ([#609](https://github.com/getsentry/sentry-go/pull/609)) +- [otel] Improve detection of Sentry-related spans ([#632](https://github.com/getsentry/sentry-go/pull/632), [#636](https://github.com/getsentry/sentry-go/pull/636)) + - Fixes cases when HTTP spans containing requests to Sentry were captured by Sentry ([#627](https://github.com/getsentry/sentry-go/issues/627)) + +### Misc + +- Drop testing in (legacy) GOPATH mode ([#618](https://github.com/getsentry/sentry-go/pull/618)) +- Remove outdated documentation from https://pkg.go.dev/github.com/getsentry/sentry-go ([#623](https://github.com/getsentry/sentry-go/pull/623)) + +## 0.20.0 + +The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.20.0. + +Note: this release has some **breaking changes**, which are listed below. + +### Breaking Changes + +- Remove the following methods: `Scope.SetTransaction()`, `Scope.Transaction()` ([#605](https://github.com/getsentry/sentry-go/pull/605)) + + Span.Name should be used instead to access the transaction's name. + + For example, the following [`TracesSampler`](https://docs.sentry.io/platforms/go/configuration/sampling/#setting-a-sampling-function) function should be now written as follows: + + **Before:** + ```go + TracesSampler: func(ctx sentry.SamplingContext) float64 { + hub := sentry.GetHubFromContext(ctx.Span.Context()) + if hub.Scope().Transaction() == "GET /health" { + return 0 + } + return 1 + }, + ``` + + **After:** + ```go + TracesSampler: func(ctx sentry.SamplingContext) float64 { + if ctx.Span.Name == "GET /health" { + return 0 + } + return 1 + }, + ``` + +### Features + +- Add `Span.SetContext()` method ([#599](https://github.com/getsentry/sentry-go/pull/599/)) + - It is recommended to use it instead of `hub.Scope().SetContext` when setting or updating context on transactions. +- Add `DebugMeta` interface to `Event` and extend `Frame` structure with more fields ([#606](https://github.com/getsentry/sentry-go/pull/606)) + - More about DebugMeta interface [here](https://develop.sentry.dev/sdk/event-payloads/debugmeta/). + +### Bug Fixes + +- [otel] Fix missing OpenTelemetry context on some events ([#599](https://github.com/getsentry/sentry-go/pull/599), [#605](https://github.com/getsentry/sentry-go/pull/605)) + - Fixes ([#596](https://github.com/getsentry/sentry-go/issues/596)). +- [otel] Better handling for HTTP span attributes ([#610](https://github.com/getsentry/sentry-go/pull/610)) + +### Misc + +- Bump minimum versions: `github.com/kataras/iris/v12` to 12.2.0, `github.com/labstack/echo/v4` to v4.10.0 ([#595](https://github.com/getsentry/sentry-go/pull/595)) + - Resolves [GO-2022-1144 / CVE-2022-41717](https://deps.dev/advisory/osv/GO-2022-1144), [GO-2023-1495 / CVE-2022-41721](https://deps.dev/advisory/osv/GO-2023-1495), [GO-2022-1059 / CVE-2022-32149](https://deps.dev/advisory/osv/GO-2022-1059). +- Bump `google.golang.org/protobuf` minimum required version to 1.29.1 ([#604](https://github.com/getsentry/sentry-go/pull/604)) + - This fixes a potential denial of service issue ([CVE-2023-24535](https://github.com/advisories/GHSA-hw7c-3rfg-p46j)). +- Exclude the `otel` module when building in GOPATH mode ([#615](https://github.com/getsentry/sentry-go/pull/615)) + +## 0.19.0 + +The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.19.0. + +### Features + +- Add support for exception mechanism metadata ([#564](https://github.com/getsentry/sentry-go/pull/564/)) + - More about exception mechanisms [here](https://develop.sentry.dev/sdk/event-payloads/exception/#exception-mechanism). + +### Bug Fixes +- [otel] Use the correct "trace" context when sending a Sentry error ([#580](https://github.com/getsentry/sentry-go/pull/580/)) + + +### Misc +- Drop support for Go 1.17, add support for Go 1.20 ([#563](https://github.com/getsentry/sentry-go/pull/563/)) + - According to our policy, we're officially supporting the last three minor releases of Go. +- Switch repository license to MIT ([#583](https://github.com/getsentry/sentry-go/pull/583/)) + - More about Sentry licensing [here](https://open.sentry.io/licensing/). +- Bump `golang.org/x/text` minimum required version to 0.3.8 ([#586](https://github.com/getsentry/sentry-go/pull/586)) + - This fixes [CVE-2022-32149](https://github.com/advisories/GHSA-69ch-w2m2-3vjp) vulnerability. + +## 0.18.0 + +The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.18.0. +This release contains initial support for [OpenTelemetry](https://opentelemetry.io/) and various other bug fixes and improvements. + +**Note**: This is the last release supporting Go 1.17. + +### Features + +- Initial support for [OpenTelemetry](https://opentelemetry.io/). + You can now send all your OpenTelemetry spans to Sentry. + + Install the `otel` module + + ```bash + go get github.com/getsentry/sentry-go \ + github.com/getsentry/sentry-go/otel + ``` + + Configure the Sentry and OpenTelemetry SDKs + + ```go + import ( + "go.opentelemetry.io/otel" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + "github.com/getsentry/sentry-go" + "github.com/getsentry/sentry-go/otel" + // ... + ) + + // Initlaize the Sentry SDK + sentry.Init(sentry.ClientOptions{ + Dsn: "__DSN__", + EnableTracing: true, + TracesSampleRate: 1.0, + }) + + // Set up the Sentry span processor + tp := sdktrace.NewTracerProvider( + sdktrace.WithSpanProcessor(sentryotel.NewSentrySpanProcessor()), + // ... + ) + otel.SetTracerProvider(tp) + + // Set up the Sentry propagator + otel.SetTextMapPropagator(sentryotel.NewSentryPropagator()) + ``` + + You can read more about using OpenTelemetry with Sentry in our [docs](https://docs.sentry.io/platforms/go/performance/instrumentation/opentelemetry/). + +### Bug Fixes + +- Do not freeze the Dynamic Sampling Context when no Sentry values are present in the baggage header ([#532](https://github.com/getsentry/sentry-go/pull/532)) +- Create a frozen Dynamic Sampling Context when calling `span.ToBaggage()` ([#566](https://github.com/getsentry/sentry-go/pull/566)) +- Fix baggage parsing and encoding in vendored otel package ([#568](https://github.com/getsentry/sentry-go/pull/568)) + +### Misc + +- Add `Span.SetDynamicSamplingContext()` ([#539](https://github.com/getsentry/sentry-go/pull/539/)) +- Add various getters for `Dsn` ([#540](https://github.com/getsentry/sentry-go/pull/540)) +- Add `SpanOption::SpanSampled` ([#546](https://github.com/getsentry/sentry-go/pull/546)) +- Add `Span.SetData()` ([#542](https://github.com/getsentry/sentry-go/pull/542)) +- Add `Span.IsTransaction()` ([#543](https://github.com/getsentry/sentry-go/pull/543)) +- Add `Span.GetTransaction()` method ([#558](https://github.com/getsentry/sentry-go/pull/558)) + +## 0.17.0 + +The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.17.0. +This release contains a new `BeforeSendTransaction` hook option and corrects two regressions introduced in `0.16.0`. + +### Features + +- Add `BeforeSendTransaction` hook to `ClientOptions` ([#517](https://github.com/getsentry/sentry-go/pull/517)) + - Here's [an example](https://github.com/getsentry/sentry-go/blob/master/_examples/http/main.go#L56-L66) of how BeforeSendTransaction can be used to modify or drop transaction events. + +### Bug Fixes + +- Do not crash in Span.Finish() when the Client is empty [#520](https://github.com/getsentry/sentry-go/pull/520) + - Fixes [#518](https://github.com/getsentry/sentry-go/issues/518) +- Attach non-PII/non-sensitive request headers to events when `ClientOptions.SendDefaultPii` is set to `false` ([#524](https://github.com/getsentry/sentry-go/pull/524)) + - Fixes [#523](https://github.com/getsentry/sentry-go/issues/523) + +### Misc + +- Clarify how to handle logrus.Fatalf events ([#501](https://github.com/getsentry/sentry-go/pull/501/)) +- Rename the `examples` directory to `_examples` ([#521](https://github.com/getsentry/sentry-go/pull/521)) + - This removes an indirect dependency to `github.com/golang-jwt/jwt` + +## 0.16.0 + +The Sentry SDK team is happy to announce the immediate availability of Sentry Go SDK v0.16.0. +Due to ongoing work towards a stable API for `v1.0.0`, we sadly had to include **two breaking changes** in this release. + +### Breaking Changes + +- Add `EnableTracing`, a boolean option flag to enable performance monitoring (`false` by default). + - If you're using `TracesSampleRate` or `TracesSampler`, this option is **required** to enable performance monitoring. + + ```go + sentry.Init(sentry.ClientOptions{ + EnableTracing: true, + TracesSampleRate: 1.0, + }) + ``` +- Unify TracesSampler [#498](https://github.com/getsentry/sentry-go/pull/498) + - `TracesSampler` was changed to a callback that must return a `float64` between `0.0` and `1.0`. + + For example, you can apply a sample rate of `1.0` (100%) to all `/api` transactions, and a sample rate of `0.5` (50%) to all other transactions. + You can read more about this in our [SDK docs](https://docs.sentry.io/platforms/go/configuration/filtering/#using-sampling-to-filter-transaction-events). + + ```go + sentry.Init(sentry.ClientOptions{ + TracesSampler: sentry.TracesSampler(func(ctx sentry.SamplingContext) float64 { + hub := sentry.GetHubFromContext(ctx.Span.Context()) + name := hub.Scope().Transaction() + + if strings.HasPrefix(name, "GET /api") { + return 1.0 + } + + return 0.5 + }), + } + ``` + +### Features + +- Send errors logged with [Logrus](https://github.com/sirupsen/logrus) to Sentry. + - Have a look at our [logrus examples](https://github.com/getsentry/sentry-go/blob/master/_examples/logrus/main.go) on how to use the integration. +- Add support for Dynamic Sampling [#491](https://github.com/getsentry/sentry-go/pull/491) + - You can read more about Dynamic Sampling in our [product docs](https://docs.sentry.io/product/data-management-settings/dynamic-sampling/). +- Add detailed logging about the reason transactions are being dropped. + - You can enable SDK logging via `sentry.ClientOptions.Debug: true`. + +### Bug Fixes + +- Do not clone the hub when calling `StartTransaction` [#505](https://github.com/getsentry/sentry-go/pull/505) + - Fixes [#502](https://github.com/getsentry/sentry-go/issues/502) + +## 0.15.0 + +- fix: Scope values should not override Event values (#446) +- feat: Make maximum amount of spans configurable (#460) +- feat: Add a method to start a transaction (#482) +- feat: Extend User interface by adding Data, Name and Segment (#483) +- feat: Add ClientOptions.SendDefaultPII (#485) + ## 0.14.0 - feat: Add function to continue from trace string (#434) - feat: Add `max-depth` options (#428) -- ref: Use a `Context` type mapping to a `map[string]interface{}` for all event contexts (#444) +- *[breaking]* ref: Use a `Context` type mapping to a `map[string]interface{}` for all event contexts (#444) +- *[breaking]* ref: Replace deprecated `ioutil` pkg with `os` & `io` (#454) - ref: Optimize `stacktrace.go` from size and speed (#467) -- ref: Replace deprecated `ioutil` pkg with `os` & `io` (#454) - ci: Test against `go1.19` and `go1.18`, drop `go1.16` and `go1.15` support (#432, #477) - deps: Dependency update to fix CVEs (#462, #464, #477) -_NOTE:_ -This version drops support for Go 1.16 and Go 1.15. The currently supported Go versions are the last 3 stable releases: 1.19, 1.18 and 1.17. +_NOTE:_ This version drops support for Go 1.16 and Go 1.15. The currently supported Go versions are the last 3 stable releases: 1.19, 1.18 and 1.17. ## v0.13.0 @@ -70,7 +329,7 @@ There are no breaking changes and upgrading should be a smooth experience for al _NOTE:_ This version introduces support for [Sentry's Performance Monitoring](https://docs.sentry.io/platforms/go/performance/). The new tracing capabilities are beta, and we plan to expand them on future versions. Feedback is welcome, please open new issues on GitHub. -The `sentryhttp` package got better API docs, an [updated usage example](https://github.com/getsentry/sentry-go/tree/master/example/http) and support for creating automatic transactions as part of Performance Monitoring. +The `sentryhttp` package got better API docs, an [updated usage example](https://github.com/getsentry/sentry-go/tree/master/_examples/http) and support for creating automatic transactions as part of Performance Monitoring. ## v0.8.0 diff --git a/vendor/github.com/getsentry/sentry-go/CONTRIBUTING.md b/vendor/github.com/getsentry/sentry-go/CONTRIBUTING.md index 094f521..9808f18 100644 --- a/vendor/github.com/getsentry/sentry-go/CONTRIBUTING.md +++ b/vendor/github.com/getsentry/sentry-go/CONTRIBUTING.md @@ -72,6 +72,8 @@ $ go test -race -coverprofile=coverage.txt -covermode=atomic && go tool cover -h ## Linting +Lint with [`golangci-lint`](https://github.com/golangci/golangci-lint): + ```console $ golangci-lint run ``` diff --git a/vendor/github.com/getsentry/sentry-go/LICENSE b/vendor/github.com/getsentry/sentry-go/LICENSE index 3e66f28..b1b358e 100644 --- a/vendor/github.com/getsentry/sentry-go/LICENSE +++ b/vendor/github.com/getsentry/sentry-go/LICENSE @@ -1,9 +1,21 @@ -Copyright (c) 2019 Sentry (https://sentry.io) and individual contributors. -All rights reserved. +MIT License -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: +Copyright (c) 2019 Functional Software, Inc. dba Sentry -* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. -* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/getsentry/sentry-go/Makefile b/vendor/github.com/getsentry/sentry-go/Makefile new file mode 100644 index 0000000..c1e27eb --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/Makefile @@ -0,0 +1,83 @@ +.DEFAULT_GOAL := help + +MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) +MKFILE_DIR := $(dir $(MKFILE_PATH)) +ALL_GO_MOD_DIRS := $(shell find . -type f -name 'go.mod' -exec dirname {} \; | sort) +GO = go +TIMEOUT = 300 + +# Parse Makefile and display the help +help: ## Show help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' +.PHONY: help + +build: ## Build everything + for dir in $(ALL_GO_MOD_DIRS); do \ + cd "$${dir}"; \ + echo ">>> Running 'go build' for module: $${dir}"; \ + go build ./...; \ + done; +.PHONY: build + +### Tests (inspired by https://github.com/open-telemetry/opentelemetry-go/blob/main/Makefile) +TEST_TARGETS := test-short test-verbose test-race +test-race: ARGS=-race +test-short: ARGS=-short +test-verbose: ARGS=-v -race +$(TEST_TARGETS): test +test: $(ALL_GO_MOD_DIRS:%=test/%) ## Run tests +test/%: DIR=$* +test/%: + @echo ">>> Running tests for module: $(DIR)" + @# We use '-count=1' to disable test caching. + (cd $(DIR) && $(GO) test -count=1 -timeout $(TIMEOUT)s $(ARGS) ./...) +.PHONY: $(TEST_TARGETS) test + +# Coverage +COVERAGE_MODE = atomic +COVERAGE_PROFILE = coverage.out +COVERAGE_REPORT_DIR = .coverage +COVERAGE_REPORT_DIR_ABS = "$(MKFILE_DIR)/$(COVERAGE_REPORT_DIR)" +$(COVERAGE_REPORT_DIR): + mkdir -p $(COVERAGE_REPORT_DIR) +clean-report-dir: $(COVERAGE_REPORT_DIR) + test $(COVERAGE_REPORT_DIR) && rm -f $(COVERAGE_REPORT_DIR)/* +test-coverage: $(COVERAGE_REPORT_DIR) clean-report-dir ## Test with coverage enabled + set -e ; \ + for dir in $(ALL_GO_MOD_DIRS); do \ + echo ">>> Running tests with coverage for module: $${dir}"; \ + DIR_ABS=$$(python -c 'import os, sys; print(os.path.realpath(sys.argv[1]))' $${dir}) ; \ + REPORT_NAME=$$(basename $${DIR_ABS}); \ + (cd "$${dir}" && \ + $(GO) test -count=1 -coverpkg=./... -covermode=$(COVERAGE_MODE) -coverprofile="$(COVERAGE_PROFILE)" ./... && \ + cp $(COVERAGE_PROFILE) "$(COVERAGE_REPORT_DIR_ABS)/$${REPORT_NAME}_$(COVERAGE_PROFILE)" && \ + $(GO) tool cover -html=$(COVERAGE_PROFILE) -o coverage.html); \ + done; +.PHONY: test-coverage clean-report-dir + +mod-tidy: ## Check go.mod tidiness + set -e ; \ + for dir in $(ALL_GO_MOD_DIRS); do \ + cd "$${dir}"; \ + echo ">>> Running 'go mod tidy' for module: $${dir}"; \ + go mod tidy -go=1.18 -compat=1.18; \ + done; \ + git diff --exit-code; +.PHONY: mod-tidy + +vet: ## Run "go vet" + set -e ; \ + for dir in $(ALL_GO_MOD_DIRS); do \ + cd "$${dir}"; \ + echo ">>> Running 'go vet' for module: $${dir}"; \ + go vet ./...; \ + done; +.PHONY: vet + +lint: ## Lint (using "golangci-lint") + golangci-lint run +.PHONY: lint + +fmt: ## Format all Go files + gofmt -l -w -s . +.PHONY: fmt diff --git a/vendor/github.com/getsentry/sentry-go/README.md b/vendor/github.com/getsentry/sentry-go/README.md index e5472d5..82f9293 100644 --- a/vendor/github.com/getsentry/sentry-go/README.md +++ b/vendor/github.com/getsentry/sentry-go/README.md @@ -29,7 +29,7 @@ The only requirement is a Go compiler. We verify this package against the 3 most recent releases of Go. Those are the supported versions. The exact versions are defined in -[`GitHub workflow`](.github/workflows/ci.yml). +[`GitHub workflow`](.github/workflows/test.yml). In addition, we run tests against the current master branch of the Go toolchain, though support for this configuration is best-effort. @@ -42,7 +42,7 @@ though support for this configuration is best-effort. $ go get github.com/getsentry/sentry-go@latest ``` -Check out the [list of released versions](https://pkg.go.dev/github.com/getsentry/sentry-go?tab=versions). +Check out the [list of released versions](https://github.com/getsentry/sentry-go/releases). ## Configuration @@ -62,9 +62,9 @@ More on this in the [Configuration section of the official Sentry Go SDK documen The SDK supports reporting errors and tracking application performance. -To get started, have a look at one of our [examples](example/): -- [Basic error instrumentation](example/basic/main.go) -- [Error and tracing for HTTP servers](example/http/main.go) +To get started, have a look at one of our [examples](_examples/): +- [Basic error instrumentation](_examples/basic/main.go) +- [Error and tracing for HTTP servers](_examples/http/main.go) We also provide a [complete API reference](https://pkg.go.dev/github.com/getsentry/sentry-go). @@ -96,7 +96,7 @@ checkout the official documentation: ## License Licensed under -[The 2-Clause BSD License](https://opensource.org/licenses/BSD-2-Clause), see +[The MIT License](https://opensource.org/licenses/mit/), see [`LICENSE`](LICENSE). ## Community diff --git a/vendor/github.com/getsentry/sentry-go/client.go b/vendor/github.com/getsentry/sentry-go/client.go index d889889..c201595 100644 --- a/vendor/github.com/getsentry/sentry-go/client.go +++ b/vendor/github.com/getsentry/sentry-go/client.go @@ -3,14 +3,12 @@ package sentry import ( "context" "crypto/x509" - "errors" "fmt" "io" "log" "math/rand" "net/http" "os" - "reflect" "sort" "strings" "sync" @@ -28,6 +26,11 @@ import ( // stack trace is often the most useful information. const maxErrorDepth = 10 +// defaultMaxSpans limits the default number of recorded spans per transaction. The limit is +// meant to bound memory usage and prevent too large transaction events that +// would be rejected by Sentry. +const defaultMaxSpans = 1000 + // hostname is the host name reported by the kernel. It is precomputed once to // avoid syscalls when capturing events. // @@ -121,6 +124,8 @@ type ClientOptions struct { // 0.0 is treated as if it was 1.0. To drop all events, set the DSN to the // empty string. SampleRate float64 + // Enable performance tracing. + EnableTracing bool // The sample rate for sampling traces in the range [0.0, 1.0]. TracesSampleRate float64 // Used to customize the sampling of traces, overrides TracesSampleRate. @@ -129,10 +134,15 @@ type ClientOptions struct { // and if applicable, caught errors type and value. // If the match is found, then a whole event will be dropped. IgnoreErrors []string + // If this flag is enabled, certain personally identifiable information (PII) is added by active integrations. + // By default, no such data is sent. + SendDefaultPII bool // BeforeSend is called before error events are sent to Sentry. // Use it to mutate the event or return nil to discard the event. - // See EventProcessor if you need to mutate transactions. BeforeSend func(event *Event, hint *EventHint) *Event + // BeforeSendTransaction is called before transaction events are sent to Sentry. + // Use it to mutate the transaction or return nil to discard the transaction. + BeforeSendTransaction func(event *Event, hint *EventHint) *Event // Before breadcrumb add callback. BeforeBreadcrumb func(breadcrumb *Breadcrumb, hint *BreadcrumbHint) *Breadcrumb // Integrations to be installed on the current Client, receives default @@ -175,6 +185,11 @@ type ClientOptions struct { // Maximum number of breadcrumbs // when MaxBreadcrumbs is negative then ignore breadcrumbs. MaxBreadcrumbs int + // Maximum number of spans. + // + // See https://develop.sentry.dev/sdk/envelopes/#size-limits for size limits + // applied during event ingestion. Events that exceed these limits might get dropped. + MaxSpans int // An optional pointer to http.Client that will be used with a default // HTTPTransport. Using your own client will make HTTPTransport, HTTPProxy, // HTTPSProxy and CaCerts options ignored. @@ -222,10 +237,6 @@ type Client struct { // single goroutine) or hub methods (for concurrent programs, for example web // servers). func NewClient(options ClientOptions) (*Client, error) { - if options.TracesSampleRate != 0.0 && options.TracesSampler != nil { - return nil, errors.New("TracesSampleRate and TracesSampler are mutually exclusive") - } - if options.Debug { debugWriter := options.DebugWriter if debugWriter == nil { @@ -250,6 +261,10 @@ func NewClient(options ClientOptions) (*Client, error) { options.MaxErrorDepth = maxErrorDepth } + if options.MaxSpans == 0 { + options.MaxSpans = defaultMaxSpans + } + // SENTRYGODEBUG is a comma-separated list of key=value pairs (similar // to GODEBUG). It is not a supported feature: recognized debug options // may change any time. @@ -306,7 +321,7 @@ func (client *Client) setupTransport() { // accommodate more concurrent events. // TODO(tracing): consider using separate buffers per // event type. - if opts.TracesSampleRate != 0 || opts.TracesSampler != nil { + if opts.EnableTracing { httpTransport.BufferSize = 1000 } transport = httpTransport @@ -362,13 +377,13 @@ func (client Client) Options() ClientOptions { // CaptureMessage captures an arbitrary message. func (client *Client) CaptureMessage(message string, hint *EventHint, scope EventModifier) *EventID { - event := client.eventFromMessage(message, LevelInfo) + event := client.EventFromMessage(message, LevelInfo) return client.CaptureEvent(event, hint, scope) } // CaptureException captures an error. func (client *Client) CaptureException(exception error, hint *EventHint, scope EventModifier) *EventID { - event := client.eventFromException(exception, LevelError) + event := client.EventFromException(exception, LevelError) return client.CaptureEvent(event, hint, scope) } @@ -423,11 +438,11 @@ func (client *Client) RecoverWithContext( var event *Event switch err := err.(type) { case error: - event = client.eventFromException(err, LevelFatal) + event = client.EventFromException(err, LevelFatal) case string: - event = client.eventFromMessage(err, LevelFatal) + event = client.EventFromMessage(err, LevelFatal) default: - event = client.eventFromMessage(fmt.Sprintf("%#v", err), LevelFatal) + event = client.EventFromMessage(fmt.Sprintf("%#v", err), LevelFatal) } return client.CaptureEvent(event, hint, scope) } @@ -447,10 +462,11 @@ func (client *Client) Flush(timeout time.Duration) bool { return client.Transport.Flush(timeout) } -func (client *Client) eventFromMessage(message string, level Level) *Event { +// EventFromMessage creates an event from the given message string. +func (client *Client) EventFromMessage(message string, level Level) *Event { if message == "" { err := usageError{fmt.Errorf("%s called with empty message", callerFunctionName())} - return client.eventFromException(err, level) + return client.EventFromException(err, level) } event := NewEvent() event.Level = level @@ -467,41 +483,17 @@ func (client *Client) eventFromMessage(message string, level Level) *Event { return event } -func (client *Client) eventFromException(exception error, level Level) *Event { +// EventFromException creates a new Sentry event from the given `error` instance. +func (client *Client) EventFromException(exception error, level Level) *Event { + event := NewEvent() + event.Level = level + err := exception if err == nil { err = usageError{fmt.Errorf("%s called with nil error", callerFunctionName())} } - event := NewEvent() - event.Level = level - - for i := 0; i < client.options.MaxErrorDepth && err != nil; i++ { - event.Exception = append(event.Exception, Exception{ - Value: err.Error(), - Type: reflect.TypeOf(err).String(), - Stacktrace: ExtractStacktrace(err), - }) - switch previous := err.(type) { - case interface{ Unwrap() error }: - err = previous.Unwrap() - case interface{ Cause() error }: - err = previous.Cause() - default: - err = nil - } - } - - // Add a trace of the current stack to the most recent error in a chain if - // it doesn't have a stack trace yet. - // We only add to the most recent error to avoid duplication and because the - // current stack is most likely unrelated to errors deeper in the chain. - if event.Exception[0].Stacktrace == nil { - event.Exception[0].Stacktrace = NewStacktrace() - } - - // event.Exception should be sorted such that the most recent error is last. - reverse(event.Exception) + event.SetException(err, client.options.MaxErrorDepth) return event } @@ -556,11 +548,18 @@ func (client *Client) processEvent(event *Event, hint *EventHint, scope EventMod return nil } - // As per spec, transactions do not go through BeforeSend. - if event.Type != transactionType && options.BeforeSend != nil { - if hint == nil { - hint = &EventHint{} + // Apply beforeSend* processors + if hint == nil { + hint = &EventHint{} + } + if event.Type == transactionType && options.BeforeSendTransaction != nil { + // Transaction events + if event = options.BeforeSendTransaction(event, hint); event == nil { + Logger.Println("Transaction dropped due to BeforeSendTransaction callback.") + return nil } + } else if event.Type != transactionType && options.BeforeSend != nil { + // All other events if event = options.BeforeSend(event, hint); event == nil { Logger.Println("Event dropped due to BeforeSend callback.") return nil @@ -607,12 +606,12 @@ func (client *Client) prepareEvent(event *Event, hint *EventHint, scope EventMod event.Platform = "go" event.Sdk = SdkInfo{ - Name: "sentry.go", - Version: Version, + Name: SDKIdentifier, + Version: SDKVersion, Integrations: client.listIntegrations(), Packages: []SdkPackage{{ Name: "sentry-go", - Version: Version, + Version: SDKVersion, }}, } diff --git a/vendor/github.com/getsentry/sentry-go/doc.go b/vendor/github.com/getsentry/sentry-go/doc.go index 1cc5e94..973020a 100644 --- a/vendor/github.com/getsentry/sentry-go/doc.go +++ b/vendor/github.com/getsentry/sentry-go/doc.go @@ -1,63 +1,6 @@ /* -Package sentry is the official Sentry SDK for Go. +Package repository: https://github.com/getsentry/sentry-go/ -Use it to report errors and track application performance through distributed -tracing. - -For more information about Sentry and SDK features please have a look at the -documentation site https://docs.sentry.io/platforms/go/. - -Basic Usage - -The first step is to initialize the SDK, providing at a minimum the DSN of your -Sentry project. This step is accomplished through a call to sentry.Init. - - func main() { - err := sentry.Init(...) - ... - } - -A more detailed yet simple example is available at -https://github.com/getsentry/sentry-go/blob/master/example/basic/main.go. - -Error Reporting - -The Capture* functions report messages and errors to Sentry. - - sentry.CaptureMessage(...) - sentry.CaptureException(...) - sentry.CaptureEvent(...) - -Use similarly named functions in the Hub for concurrent programs like web -servers. - -Performance Monitoring - -You can use Sentry to monitor your application's performance. More information -on the product page https://docs.sentry.io/product/performance/. - -The StartSpan function creates new spans. - - span := sentry.StartSpan(ctx, "operation") - ... - span.Finish() - -Integrations - -The SDK has support for several Go frameworks, available as subpackages. - -Getting Support - -For paid Sentry.io accounts, head out to https://sentry.io/support. - -For all users, support channels include: - Forum: https://forum.sentry.io - Discord: https://discord.gg/Ww9hbqr (#go channel) - -If you found an issue with the SDK, please report through -https://github.com/getsentry/sentry-go/issues/new/choose. - -For responsibly disclosing a security issue, please follow the steps in -https://sentry.io/security/#vulnerability-disclosure. +For more information about Sentry and SDK features, please have a look at the official documentation site: https://docs.sentry.io/platforms/go/ */ package sentry diff --git a/vendor/github.com/getsentry/sentry-go/dsn.go b/vendor/github.com/getsentry/sentry-go/dsn.go index 5f5a13c..e56f540 100644 --- a/vendor/github.com/getsentry/sentry-go/dsn.go +++ b/vendor/github.com/getsentry/sentry-go/dsn.go @@ -145,19 +145,44 @@ func (dsn Dsn) String() string { return url } -// StoreAPIURL returns the URL of the store endpoint of the project associated -// with the DSN. -func (dsn Dsn) StoreAPIURL() *url.URL { - return dsn.getAPIURL("store") +// Get the scheme of the DSN. +func (dsn Dsn) GetScheme() string { + return string(dsn.scheme) } -// EnvelopeAPIURL returns the URL of the envelope endpoint of the project +// Get the public key of the DSN. +func (dsn Dsn) GetPublicKey() string { + return dsn.publicKey +} + +// Get the secret key of the DSN. +func (dsn Dsn) GetSecretKey() string { + return dsn.secretKey +} + +// Get the host of the DSN. +func (dsn Dsn) GetHost() string { + return dsn.host +} + +// Get the port of the DSN. +func (dsn Dsn) GetPort() int { + return dsn.port +} + +// Get the path of the DSN. +func (dsn Dsn) GetPath() string { + return dsn.path +} + +// Get the project ID of the DSN. +func (dsn Dsn) GetProjectID() string { + return dsn.projectID +} + +// GetAPIURL returns the URL of the envelope endpoint of the project // associated with the DSN. -func (dsn Dsn) EnvelopeAPIURL() *url.URL { - return dsn.getAPIURL("envelope") -} - -func (dsn Dsn) getAPIURL(s string) *url.URL { +func (dsn Dsn) GetAPIURL() *url.URL { var rawURL string rawURL += fmt.Sprintf("%s://%s", dsn.scheme, dsn.host) if dsn.port != dsn.scheme.defaultPort() { @@ -166,7 +191,7 @@ func (dsn Dsn) getAPIURL(s string) *url.URL { if dsn.path != "" { rawURL += dsn.path } - rawURL += fmt.Sprintf("/api/%s/%s/", dsn.projectID, s) + rawURL += fmt.Sprintf("/api/%s/%s/", dsn.projectID, "envelope") parsedURL, _ := url.Parse(rawURL) return parsedURL } diff --git a/vendor/github.com/getsentry/sentry-go/dynamic_sampling_context.go b/vendor/github.com/getsentry/sentry-go/dynamic_sampling_context.go new file mode 100644 index 0000000..fda6fdf --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/dynamic_sampling_context.go @@ -0,0 +1,119 @@ +package sentry + +import ( + "strconv" + "strings" + + "github.com/getsentry/sentry-go/internal/otel/baggage" +) + +const ( + sentryPrefix = "sentry-" +) + +// DynamicSamplingContext holds information about the current event that can be used to make dynamic sampling decisions. +type DynamicSamplingContext struct { + Entries map[string]string + Frozen bool +} + +func DynamicSamplingContextFromHeader(header []byte) (DynamicSamplingContext, error) { + bag, err := baggage.Parse(string(header)) + if err != nil { + return DynamicSamplingContext{}, err + } + + entries := map[string]string{} + for _, member := range bag.Members() { + // We only store baggage members if their key starts with "sentry-". + if k, v := member.Key(), member.Value(); strings.HasPrefix(k, sentryPrefix) { + entries[strings.TrimPrefix(k, sentryPrefix)] = v + } + } + + return DynamicSamplingContext{ + Entries: entries, + // If there's at least one Sentry value, we consider the DSC frozen + Frozen: len(entries) > 0, + }, nil +} + +func DynamicSamplingContextFromTransaction(span *Span) DynamicSamplingContext { + entries := map[string]string{} + + hub := hubFromContext(span.Context()) + scope := hub.Scope() + client := hub.Client() + + if client == nil || scope == nil { + return DynamicSamplingContext{ + Entries: map[string]string{}, + Frozen: false, + } + } + + options := client.Options() + + if traceID := span.TraceID.String(); traceID != "" { + entries["trace_id"] = traceID + } + if sampleRate := span.sampleRate; sampleRate != 0 { + entries["sample_rate"] = strconv.FormatFloat(sampleRate, 'f', -1, 64) + } + + if dsn := client.dsn; dsn != nil { + if publicKey := dsn.publicKey; publicKey != "" { + entries["public_key"] = publicKey + } + } + if release := options.Release; release != "" { + entries["release"] = release + } + if environment := options.Environment; environment != "" { + entries["environment"] = environment + } + + // Only include the transaction name if it's of good quality (not empty and not SourceURL) + if span.Source != "" && span.Source != SourceURL { + if span.IsTransaction() { + entries["transaction"] = span.Name + } + } + + if userSegment := scope.user.Segment; userSegment != "" { + entries["user_segment"] = userSegment + } + + return DynamicSamplingContext{ + Entries: entries, + Frozen: true, + } +} + +func (d DynamicSamplingContext) HasEntries() bool { + return len(d.Entries) > 0 +} + +func (d DynamicSamplingContext) IsFrozen() bool { + return d.Frozen +} + +func (d DynamicSamplingContext) String() string { + members := []baggage.Member{} + for k, entry := range d.Entries { + member, err := baggage.NewMember(sentryPrefix+k, entry) + if err != nil { + continue + } + members = append(members, member) + } + if len(members) > 0 { + baggage, err := baggage.New(members...) + if err != nil { + return "" + } + return baggage.String() + } + + return "" +} diff --git a/vendor/github.com/getsentry/sentry-go/interfaces.go b/vendor/github.com/getsentry/sentry-go/interfaces.go index a825020..0be0d10 100644 --- a/vendor/github.com/getsentry/sentry-go/interfaces.go +++ b/vendor/github.com/getsentry/sentry-go/interfaces.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "net/http" + "reflect" "strings" "time" ) @@ -16,6 +17,9 @@ import ( // transactionType is the type of a transaction event. const transactionType = "transaction" +// eventType is the type of an error event. +const eventType = "event" + // Level marks the severity of the event. type Level string @@ -28,6 +32,15 @@ const ( LevelFatal Level = "fatal" ) +func getSensitiveHeaders() map[string]bool { + return map[string]bool{ + "Authorization": true, + "Cookie": true, + "X-Forwarded-For": true, + "X-Real-Ip": true, + } +} + // SdkInfo contains all metadata about about the SDK being used. type SdkInfo struct { Name string `json:"name,omitempty"` @@ -93,10 +106,45 @@ func (b *Breadcrumb) MarshalJSON() ([]byte, error) { // User describes the user associated with an Event. If this is used, at least // an ID or an IP address should be provided. type User struct { - Email string `json:"email,omitempty"` - ID string `json:"id,omitempty"` - IPAddress string `json:"ip_address,omitempty"` - Username string `json:"username,omitempty"` + ID string `json:"id,omitempty"` + Email string `json:"email,omitempty"` + IPAddress string `json:"ip_address,omitempty"` + Username string `json:"username,omitempty"` + Name string `json:"name,omitempty"` + Segment string `json:"segment,omitempty"` + Data map[string]string `json:"data,omitempty"` +} + +func (u User) IsEmpty() bool { + if len(u.ID) > 0 { + return false + } + + if len(u.Email) > 0 { + return false + } + + if len(u.IPAddress) > 0 { + return false + } + + if len(u.Username) > 0 { + return false + } + + if len(u.Name) > 0 { + return false + } + + if len(u.Segment) > 0 { + return false + } + + if len(u.Data) > 0 { + return false + } + + return true } // Request contains information on a HTTP request related to the event. @@ -121,23 +169,35 @@ func NewRequest(r *http.Request) *Request { } url := fmt.Sprintf("%s://%s%s", protocol, r.Host, r.URL.Path) - // We read only the first Cookie header because of the specification: - // https://tools.ietf.org/html/rfc6265#section-5.4 - // When the user agent generates an HTTP request, the user agent MUST NOT - // attach more than one Cookie header field. - cookies := r.Header.Get("Cookie") - - headers := make(map[string]string, len(r.Header)) - for k, v := range r.Header { - headers[k] = strings.Join(v, ",") - } - headers["Host"] = r.Host - + var cookies string var env map[string]string - if addr, port, err := net.SplitHostPort(r.RemoteAddr); err == nil { - env = map[string]string{"REMOTE_ADDR": addr, "REMOTE_PORT": port} + headers := map[string]string{} + + if client := CurrentHub().Client(); client != nil && client.Options().SendDefaultPII { + // We read only the first Cookie header because of the specification: + // https://tools.ietf.org/html/rfc6265#section-5.4 + // When the user agent generates an HTTP request, the user agent MUST NOT + // attach more than one Cookie header field. + cookies = r.Header.Get("Cookie") + + for k, v := range r.Header { + headers[k] = strings.Join(v, ",") + } + + if addr, port, err := net.SplitHostPort(r.RemoteAddr); err == nil { + env = map[string]string{"REMOTE_ADDR": addr, "REMOTE_PORT": port} + } + } else { + sensitiveHeaders := getSensitiveHeaders() + for k, v := range r.Header { + if _, ok := sensitiveHeaders[k]; !ok { + headers[k] = strings.Join(v, ",") + } + } } + headers["Host"] = r.Host + return &Request{ URL: url, Method: r.Method, @@ -148,6 +208,22 @@ func NewRequest(r *http.Request) *Request { } } +// Mechanism is the mechanism by which an exception was generated and handled. +type Mechanism struct { + Type string `json:"type,omitempty"` + Description string `json:"description,omitempty"` + HelpLink string `json:"help_link,omitempty"` + Handled *bool `json:"handled,omitempty"` + Data map[string]interface{} `json:"data,omitempty"` +} + +// SetUnhandled indicates that the exception is an unhandled exception, i.e. +// from a panic. +func (m *Mechanism) SetUnhandled() { + h := false + m.Handled = &h +} + // Exception specifies an error that occurred. type Exception struct { Type string `json:"type,omitempty"` // used as the main issue title @@ -155,6 +231,46 @@ type Exception struct { Module string `json:"module,omitempty"` ThreadID string `json:"thread_id,omitempty"` Stacktrace *Stacktrace `json:"stacktrace,omitempty"` + Mechanism *Mechanism `json:"mechanism,omitempty"` +} + +// SDKMetaData is a struct to stash data which is needed at some point in the SDK's event processing pipeline +// but which shouldn't get send to Sentry. +type SDKMetaData struct { + dsc DynamicSamplingContext +} + +// Contains information about how the name of the transaction was determined. +type TransactionInfo struct { + Source TransactionSource `json:"source,omitempty"` +} + +// The DebugMeta interface is not used in Golang apps, but may be populated +// when proxying Events from other platforms, like iOS, Android, and the +// Web. (See: https://develop.sentry.dev/sdk/event-payloads/debugmeta/ ). +type DebugMeta struct { + SdkInfo *DebugMetaSdkInfo `json:"sdk_info,omitempty"` + Images []DebugMetaImage `json:"images,omitempty"` +} + +type DebugMetaSdkInfo struct { + SdkName string `json:"sdk_name,omitempty"` + VersionMajor int `json:"version_major,omitempty"` + VersionMinor int `json:"version_minor,omitempty"` + VersionPatchlevel int `json:"version_patchlevel,omitempty"` +} + +type DebugMetaImage struct { + Type string `json:"type,omitempty"` // all + ImageAddr string `json:"image_addr,omitempty"` // macho,elf,pe + ImageSize int `json:"image_size,omitempty"` // macho,elf,pe + DebugID string `json:"debug_id,omitempty"` // macho,elf,pe,wasm,sourcemap + DebugFile string `json:"debug_file,omitempty"` // macho,elf,pe,wasm + CodeID string `json:"code_id,omitempty"` // macho,elf,pe,wasm + CodeFile string `json:"code_file,omitempty"` // macho,elf,pe,wasm,sourcemap + ImageVmaddr string `json:"image_vmaddr,omitempty"` // macho,elf,pe + Arch string `json:"arch,omitempty"` // macho,elf,pe + UUID string `json:"uuid,omitempty"` // proguard } // EventID is a hexadecimal string representing a unique uuid4 for an Event. @@ -187,12 +303,56 @@ type Event struct { Modules map[string]string `json:"modules,omitempty"` Request *Request `json:"request,omitempty"` Exception []Exception `json:"exception,omitempty"` + DebugMeta *DebugMeta `json:"debug_meta,omitempty"` // The fields below are only relevant for transactions. - Type string `json:"type,omitempty"` - StartTime time.Time `json:"start_timestamp"` - Spans []*Span `json:"spans,omitempty"` + Type string `json:"type,omitempty"` + StartTime time.Time `json:"start_timestamp"` + Spans []*Span `json:"spans,omitempty"` + TransactionInfo *TransactionInfo `json:"transaction_info,omitempty"` + + // The fields below are not part of the final JSON payload. + + sdkMetaData SDKMetaData +} + +// SetException appends the unwrapped errors to the event's exception list. +// +// maxErrorDepth is the maximum depth of the error chain we will look +// into while unwrapping the errors. +func (e *Event) SetException(exception error, maxErrorDepth int) { + err := exception + if err == nil { + return + } + + for i := 0; i < maxErrorDepth && err != nil; i++ { + e.Exception = append(e.Exception, Exception{ + Value: err.Error(), + Type: reflect.TypeOf(err).String(), + Stacktrace: ExtractStacktrace(err), + }) + switch previous := err.(type) { + case interface{ Unwrap() error }: + err = previous.Unwrap() + case interface{ Cause() error }: + err = previous.Cause() + default: + err = nil + } + } + + // Add a trace of the current stack to the most recent error in a chain if + // it doesn't have a stack trace yet. + // We only add to the most recent error to avoid duplication and because the + // current stack is most likely unrelated to errors deeper in the chain. + if e.Exception[0].Stacktrace == nil { + e.Exception[0].Stacktrace = NewStacktrace() + } + + // event.Exception should be sorted such that the most recent error is last. + reverse(e.Exception) } // TODO: Event.Contexts map[string]interface{} => map[string]EventContext, @@ -234,9 +394,10 @@ func (e *Event) defaultMarshalJSON() ([]byte, error) { // be sent for transactions. They shadow the respective fields in Event // and are meant to remain nil, triggering the omitempty behavior. - Type json.RawMessage `json:"type,omitempty"` - StartTime json.RawMessage `json:"start_timestamp,omitempty"` - Spans json.RawMessage `json:"spans,omitempty"` + Type json.RawMessage `json:"type,omitempty"` + StartTime json.RawMessage `json:"start_timestamp,omitempty"` + Spans json.RawMessage `json:"spans,omitempty"` + TransactionInfo json.RawMessage `json:"transaction_info,omitempty"` } x := errorEvent{event: (*event)(e)} diff --git a/vendor/github.com/getsentry/sentry-go/internal/crypto/randutil/randutil.go b/vendor/github.com/getsentry/sentry-go/internal/crypto/randutil/randutil.go deleted file mode 100644 index c37cebc..0000000 --- a/vendor/github.com/getsentry/sentry-go/internal/crypto/randutil/randutil.go +++ /dev/null @@ -1,23 +0,0 @@ -package randutil - -import ( - "crypto/rand" - "encoding/binary" -) - -const ( - floatMax = 1 << 53 - floatMask = floatMax - 1 -) - -// Float64 returns a cryptographically secure random number in [0.0, 1.0). -func Float64() float64 { - // The implementation is, in essence: - // return float64(rand.Int63n(1<<53)) / (1<<53) - b := make([]byte, 8) - _, err := rand.Read(b) - if err != nil { - panic(err) - } - return float64(binary.LittleEndian.Uint64(b)&floatMask) / floatMax -} diff --git a/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/README.md b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/README.md new file mode 100644 index 0000000..2718f31 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/README.md @@ -0,0 +1,12 @@ +## Why do we have this "otel/baggage" folder? + +The root sentry-go SDK (namely, the Dynamic Sampling functionality) needs an implementation of the [baggage spec](https://www.w3.org/TR/baggage/). +For that reason, we've taken the existing baggage implementation from the [opentelemetry-go](https://github.com/open-telemetry/opentelemetry-go/) repository, and fixed a few things that in our opinion were violating the specification. + +These issues are: +1. Baggage string value `one%20two` should be properly parsed as "one two" +1. Baggage string value `one+two` should be parsed as "one+two" +1. Go string value "one two" should be encoded as `one%20two` (percent encoding), and NOT as `one+two` (URL query encoding). +1. Go string value "1=1" might be encoded as `1=1`, because the spec says: "Note, value MAY contain any number of the equal sign (=) characters. Parsers MUST NOT assume that the equal sign is only used to separate key and value.". `1%3D1` is also valid, but to simplify the implementation we're not doing it. + +Changes were made in this PR: https://github.com/getsentry/sentry-go/pull/568 diff --git a/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/baggage.go b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/baggage.go new file mode 100644 index 0000000..1806555 --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/baggage.go @@ -0,0 +1,604 @@ +// Adapted from https://github.com/open-telemetry/opentelemetry-go/blob/c21b6b6bb31a2f74edd06e262f1690f3f6ea3d5c/baggage/baggage.go +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package baggage + +import ( + "errors" + "fmt" + "net/url" + "regexp" + "strings" + "unicode/utf8" + + "github.com/getsentry/sentry-go/internal/otel/baggage/internal/baggage" +) + +const ( + maxMembers = 180 + maxBytesPerMembers = 4096 + maxBytesPerBaggageString = 8192 + + listDelimiter = "," + keyValueDelimiter = "=" + propertyDelimiter = ";" + + keyDef = `([\x21\x23-\x27\x2A\x2B\x2D\x2E\x30-\x39\x41-\x5a\x5e-\x7a\x7c\x7e]+)` + valueDef = `([\x21\x23-\x2b\x2d-\x3a\x3c-\x5B\x5D-\x7e]*)` + keyValueDef = `\s*` + keyDef + `\s*` + keyValueDelimiter + `\s*` + valueDef + `\s*` +) + +var ( + keyRe = regexp.MustCompile(`^` + keyDef + `$`) + valueRe = regexp.MustCompile(`^` + valueDef + `$`) + propertyRe = regexp.MustCompile(`^(?:\s*` + keyDef + `\s*|` + keyValueDef + `)$`) +) + +var ( + errInvalidKey = errors.New("invalid key") + errInvalidValue = errors.New("invalid value") + errInvalidProperty = errors.New("invalid baggage list-member property") + errInvalidMember = errors.New("invalid baggage list-member") + errMemberNumber = errors.New("too many list-members in baggage-string") + errMemberBytes = errors.New("list-member too large") + errBaggageBytes = errors.New("baggage-string too large") +) + +// Property is an additional metadata entry for a baggage list-member. +type Property struct { + key, value string + + // hasValue indicates if a zero-value value means the property does not + // have a value or if it was the zero-value. + hasValue bool + + // hasData indicates whether the created property contains data or not. + // Properties that do not contain data are invalid with no other check + // required. + hasData bool +} + +// NewKeyProperty returns a new Property for key. +// +// If key is invalid, an error will be returned. +func NewKeyProperty(key string) (Property, error) { + if !keyRe.MatchString(key) { + return newInvalidProperty(), fmt.Errorf("%w: %q", errInvalidKey, key) + } + + p := Property{key: key, hasData: true} + return p, nil +} + +// NewKeyValueProperty returns a new Property for key with value. +// +// If key or value are invalid, an error will be returned. +func NewKeyValueProperty(key, value string) (Property, error) { + if !keyRe.MatchString(key) { + return newInvalidProperty(), fmt.Errorf("%w: %q", errInvalidKey, key) + } + if !valueRe.MatchString(value) { + return newInvalidProperty(), fmt.Errorf("%w: %q", errInvalidValue, value) + } + + p := Property{ + key: key, + value: value, + hasValue: true, + hasData: true, + } + return p, nil +} + +func newInvalidProperty() Property { + return Property{} +} + +// parseProperty attempts to decode a Property from the passed string. It +// returns an error if the input is invalid according to the W3C Baggage +// specification. +func parseProperty(property string) (Property, error) { + if property == "" { + return newInvalidProperty(), nil + } + + match := propertyRe.FindStringSubmatch(property) + if len(match) != 4 { + return newInvalidProperty(), fmt.Errorf("%w: %q", errInvalidProperty, property) + } + + p := Property{hasData: true} + if match[1] != "" { + p.key = match[1] + } else { + p.key = match[2] + p.value = match[3] + p.hasValue = true + } + + return p, nil +} + +// validate ensures p conforms to the W3C Baggage specification, returning an +// error otherwise. +func (p Property) validate() error { + errFunc := func(err error) error { + return fmt.Errorf("invalid property: %w", err) + } + + if !p.hasData { + return errFunc(fmt.Errorf("%w: %q", errInvalidProperty, p)) + } + + if !keyRe.MatchString(p.key) { + return errFunc(fmt.Errorf("%w: %q", errInvalidKey, p.key)) + } + if p.hasValue && !valueRe.MatchString(p.value) { + return errFunc(fmt.Errorf("%w: %q", errInvalidValue, p.value)) + } + if !p.hasValue && p.value != "" { + return errFunc(errors.New("inconsistent value")) + } + return nil +} + +// Key returns the Property key. +func (p Property) Key() string { + return p.key +} + +// Value returns the Property value. Additionally, a boolean value is returned +// indicating if the returned value is the empty if the Property has a value +// that is empty or if the value is not set. +func (p Property) Value() (string, bool) { + return p.value, p.hasValue +} + +// String encodes Property into a string compliant with the W3C Baggage +// specification. +func (p Property) String() string { + if p.hasValue { + return fmt.Sprintf("%s%s%v", p.key, keyValueDelimiter, p.value) + } + return p.key +} + +type properties []Property + +func fromInternalProperties(iProps []baggage.Property) properties { + if len(iProps) == 0 { + return nil + } + + props := make(properties, len(iProps)) + for i, p := range iProps { + props[i] = Property{ + key: p.Key, + value: p.Value, + hasValue: p.HasValue, + } + } + return props +} + +func (p properties) asInternal() []baggage.Property { + if len(p) == 0 { + return nil + } + + iProps := make([]baggage.Property, len(p)) + for i, prop := range p { + iProps[i] = baggage.Property{ + Key: prop.key, + Value: prop.value, + HasValue: prop.hasValue, + } + } + return iProps +} + +func (p properties) Copy() properties { + if len(p) == 0 { + return nil + } + + props := make(properties, len(p)) + copy(props, p) + return props +} + +// validate ensures each Property in p conforms to the W3C Baggage +// specification, returning an error otherwise. +func (p properties) validate() error { + for _, prop := range p { + if err := prop.validate(); err != nil { + return err + } + } + return nil +} + +// String encodes properties into a string compliant with the W3C Baggage +// specification. +func (p properties) String() string { + props := make([]string, len(p)) + for i, prop := range p { + props[i] = prop.String() + } + return strings.Join(props, propertyDelimiter) +} + +// Member is a list-member of a baggage-string as defined by the W3C Baggage +// specification. +type Member struct { + key, value string + properties properties + + // hasData indicates whether the created property contains data or not. + // Properties that do not contain data are invalid with no other check + // required. + hasData bool +} + +// NewMember returns a new Member from the passed arguments. The key will be +// used directly while the value will be url decoded after validation. An error +// is returned if the created Member would be invalid according to the W3C +// Baggage specification. +func NewMember(key, value string, props ...Property) (Member, error) { + m := Member{ + key: key, + value: value, + properties: properties(props).Copy(), + hasData: true, + } + if err := m.validate(); err != nil { + return newInvalidMember(), err + } + //// NOTE(anton): I don't think we need to unescape here + // decodedValue, err := url.PathUnescape(value) + // if err != nil { + // return newInvalidMember(), fmt.Errorf("%w: %q", errInvalidValue, value) + // } + // m.value = decodedValue + return m, nil +} + +func newInvalidMember() Member { + return Member{} +} + +// parseMember attempts to decode a Member from the passed string. It returns +// an error if the input is invalid according to the W3C Baggage +// specification. +func parseMember(member string) (Member, error) { + if n := len(member); n > maxBytesPerMembers { + return newInvalidMember(), fmt.Errorf("%w: %d", errMemberBytes, n) + } + + var ( + key, value string + props properties + ) + + parts := strings.SplitN(member, propertyDelimiter, 2) + switch len(parts) { + case 2: + // Parse the member properties. + for _, pStr := range strings.Split(parts[1], propertyDelimiter) { + p, err := parseProperty(pStr) + if err != nil { + return newInvalidMember(), err + } + props = append(props, p) + } + fallthrough + case 1: + // Parse the member key/value pair. + + // Take into account a value can contain equal signs (=). + kv := strings.SplitN(parts[0], keyValueDelimiter, 2) + if len(kv) != 2 { + return newInvalidMember(), fmt.Errorf("%w: %q", errInvalidMember, member) + } + // "Leading and trailing whitespaces are allowed but MUST be trimmed + // when converting the header into a data structure." + key = strings.TrimSpace(kv[0]) + value = strings.TrimSpace(kv[1]) + var err error + if !keyRe.MatchString(key) { + return newInvalidMember(), fmt.Errorf("%w: %q", errInvalidKey, key) + } + if !valueRe.MatchString(value) { + return newInvalidMember(), fmt.Errorf("%w: %q", errInvalidValue, value) + } + decodedValue, err := url.PathUnescape(value) + if err != nil { + return newInvalidMember(), fmt.Errorf("%w: %q", err, value) + } + value = decodedValue + default: + // This should never happen unless a developer has changed the string + // splitting somehow. Panic instead of failing silently and allowing + // the bug to slip past the CI checks. + panic("failed to parse baggage member") + } + + return Member{key: key, value: value, properties: props, hasData: true}, nil +} + +// validate ensures m conforms to the W3C Baggage specification. +// A key is just an ASCII string, but a value must be URL encoded UTF-8, +// returning an error otherwise. +func (m Member) validate() error { + if !m.hasData { + return fmt.Errorf("%w: %q", errInvalidMember, m) + } + + if !keyRe.MatchString(m.key) { + return fmt.Errorf("%w: %q", errInvalidKey, m.key) + } + //// NOTE(anton): IMO it's too early to validate the value here. + // if !valueRe.MatchString(m.value) { + // return fmt.Errorf("%w: %q", errInvalidValue, m.value) + // } + return m.properties.validate() +} + +// Key returns the Member key. +func (m Member) Key() string { return m.key } + +// Value returns the Member value. +func (m Member) Value() string { return m.value } + +// Properties returns a copy of the Member properties. +func (m Member) Properties() []Property { return m.properties.Copy() } + +// String encodes Member into a string compliant with the W3C Baggage +// specification. +func (m Member) String() string { + // A key is just an ASCII string, but a value is URL encoded UTF-8. + s := fmt.Sprintf("%s%s%s", m.key, keyValueDelimiter, percentEncodeValue(m.value)) + if len(m.properties) > 0 { + s = fmt.Sprintf("%s%s%s", s, propertyDelimiter, m.properties.String()) + } + return s +} + +// percentEncodeValue encodes the baggage value, using percent-encoding for +// disallowed octets. +func percentEncodeValue(s string) string { + const upperhex = "0123456789ABCDEF" + var sb strings.Builder + + for byteIndex, width := 0, 0; byteIndex < len(s); byteIndex += width { + runeValue, w := utf8.DecodeRuneInString(s[byteIndex:]) + width = w + char := string(runeValue) + if valueRe.MatchString(char) && char != "%" { + // The character is returned as is, no need to percent-encode + sb.WriteString(char) + } else { + // We need to percent-encode each byte of the multi-octet character + for j := 0; j < width; j++ { + b := s[byteIndex+j] + sb.WriteByte('%') + // Bitwise operations are inspired by "net/url" + sb.WriteByte(upperhex[b>>4]) + sb.WriteByte(upperhex[b&15]) + } + } + } + return sb.String() +} + +// Baggage is a list of baggage members representing the baggage-string as +// defined by the W3C Baggage specification. +type Baggage struct { //nolint:golint + list baggage.List +} + +// New returns a new valid Baggage. It returns an error if it results in a +// Baggage exceeding limits set in that specification. +// +// It expects all the provided members to have already been validated. +func New(members ...Member) (Baggage, error) { + if len(members) == 0 { + return Baggage{}, nil + } + + b := make(baggage.List) + for _, m := range members { + if !m.hasData { + return Baggage{}, errInvalidMember + } + + // OpenTelemetry resolves duplicates by last-one-wins. + b[m.key] = baggage.Item{ + Value: m.value, + Properties: m.properties.asInternal(), + } + } + + // Check member numbers after deduplication. + if len(b) > maxMembers { + return Baggage{}, errMemberNumber + } + + bag := Baggage{b} + if n := len(bag.String()); n > maxBytesPerBaggageString { + return Baggage{}, fmt.Errorf("%w: %d", errBaggageBytes, n) + } + + return bag, nil +} + +// Parse attempts to decode a baggage-string from the passed string. It +// returns an error if the input is invalid according to the W3C Baggage +// specification. +// +// If there are duplicate list-members contained in baggage, the last one +// defined (reading left-to-right) will be the only one kept. This diverges +// from the W3C Baggage specification which allows duplicate list-members, but +// conforms to the OpenTelemetry Baggage specification. +func Parse(bStr string) (Baggage, error) { + if bStr == "" { + return Baggage{}, nil + } + + if n := len(bStr); n > maxBytesPerBaggageString { + return Baggage{}, fmt.Errorf("%w: %d", errBaggageBytes, n) + } + + b := make(baggage.List) + for _, memberStr := range strings.Split(bStr, listDelimiter) { + m, err := parseMember(memberStr) + if err != nil { + return Baggage{}, err + } + // OpenTelemetry resolves duplicates by last-one-wins. + b[m.key] = baggage.Item{ + Value: m.value, + Properties: m.properties.asInternal(), + } + } + + // OpenTelemetry does not allow for duplicate list-members, but the W3C + // specification does. Now that we have deduplicated, ensure the baggage + // does not exceed list-member limits. + if len(b) > maxMembers { + return Baggage{}, errMemberNumber + } + + return Baggage{b}, nil +} + +// Member returns the baggage list-member identified by key. +// +// If there is no list-member matching the passed key the returned Member will +// be a zero-value Member. +// The returned member is not validated, as we assume the validation happened +// when it was added to the Baggage. +func (b Baggage) Member(key string) Member { + v, ok := b.list[key] + if !ok { + // We do not need to worry about distinguishing between the situation + // where a zero-valued Member is included in the Baggage because a + // zero-valued Member is invalid according to the W3C Baggage + // specification (it has an empty key). + return newInvalidMember() + } + + return Member{ + key: key, + value: v.Value, + properties: fromInternalProperties(v.Properties), + hasData: true, + } +} + +// Members returns all the baggage list-members. +// The order of the returned list-members does not have significance. +// +// The returned members are not validated, as we assume the validation happened +// when they were added to the Baggage. +func (b Baggage) Members() []Member { + if len(b.list) == 0 { + return nil + } + + members := make([]Member, 0, len(b.list)) + for k, v := range b.list { + members = append(members, Member{ + key: k, + value: v.Value, + properties: fromInternalProperties(v.Properties), + hasData: true, + }) + } + return members +} + +// SetMember returns a copy the Baggage with the member included. If the +// baggage contains a Member with the same key the existing Member is +// replaced. +// +// If member is invalid according to the W3C Baggage specification, an error +// is returned with the original Baggage. +func (b Baggage) SetMember(member Member) (Baggage, error) { + if !member.hasData { + return b, errInvalidMember + } + + n := len(b.list) + if _, ok := b.list[member.key]; !ok { + n++ + } + list := make(baggage.List, n) + + for k, v := range b.list { + // Do not copy if we are just going to overwrite. + if k == member.key { + continue + } + list[k] = v + } + + list[member.key] = baggage.Item{ + Value: member.value, + Properties: member.properties.asInternal(), + } + + return Baggage{list: list}, nil +} + +// DeleteMember returns a copy of the Baggage with the list-member identified +// by key removed. +func (b Baggage) DeleteMember(key string) Baggage { + n := len(b.list) + if _, ok := b.list[key]; ok { + n-- + } + list := make(baggage.List, n) + + for k, v := range b.list { + if k == key { + continue + } + list[k] = v + } + + return Baggage{list: list} +} + +// Len returns the number of list-members in the Baggage. +func (b Baggage) Len() int { + return len(b.list) +} + +// String encodes Baggage into a string compliant with the W3C Baggage +// specification. The returned string will be invalid if the Baggage contains +// any invalid list-members. +func (b Baggage) String() string { + members := make([]string, 0, len(b.list)) + for k, v := range b.list { + members = append(members, Member{ + key: k, + value: v.Value, + properties: fromInternalProperties(v.Properties), + }.String()) + } + return strings.Join(members, listDelimiter) +} diff --git a/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/internal/baggage/baggage.go b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/internal/baggage/baggage.go new file mode 100644 index 0000000..ea99ccb --- /dev/null +++ b/vendor/github.com/getsentry/sentry-go/internal/otel/baggage/internal/baggage/baggage.go @@ -0,0 +1,45 @@ +// Adapted from https://github.com/open-telemetry/opentelemetry-go/blob/c21b6b6bb31a2f74edd06e262f1690f3f6ea3d5c/internal/baggage/baggage.go +// +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/* +Package baggage provides base types and functionality to store and retrieve +baggage in Go context. This package exists because the OpenTracing bridge to +OpenTelemetry needs to synchronize state whenever baggage for a context is +modified and that context contains an OpenTracing span. If it were not for +this need this package would not need to exist and the +`go.opentelemetry.io/otel/baggage` package would be the singular place where +W3C baggage is handled. +*/ +package baggage + +// List is the collection of baggage members. The W3C allows for duplicates, +// but OpenTelemetry does not, therefore, this is represented as a map. +type List map[string]Item + +// Item is the value and metadata properties part of a list-member. +type Item struct { + Value string + Properties []Property +} + +// Property is a metadata entry for a list-member. +type Property struct { + Key, Value string + + // HasValue indicates if a zero-value value means the property does not + // have a value or if it was the zero-value. + HasValue bool +} diff --git a/vendor/github.com/getsentry/sentry-go/internal/ratelimit/rate_limits.go b/vendor/github.com/getsentry/sentry-go/internal/ratelimit/rate_limits.go index 2dcda27..579297e 100644 --- a/vendor/github.com/getsentry/sentry-go/internal/ratelimit/rate_limits.go +++ b/vendor/github.com/getsentry/sentry-go/internal/ratelimit/rate_limits.go @@ -15,7 +15,7 @@ var errInvalidXSRLRetryAfter = errors.New("invalid retry-after value") // // Example // -// X-Sentry-Rate-Limits: 60:transaction, 2700:default;error;security +// X-Sentry-Rate-Limits: 60:transaction, 2700:default;error;security // // This will rate limit transactions for the next 60 seconds and errors for the // next 2700 seconds. diff --git a/vendor/github.com/getsentry/sentry-go/scope.go b/vendor/github.com/getsentry/sentry-go/scope.go index 51dd7fd..03602bd 100644 --- a/vendor/github.com/getsentry/sentry-go/scope.go +++ b/vendor/github.com/getsentry/sentry-go/scope.go @@ -31,7 +31,6 @@ type Scope struct { extra map[string]interface{} fingerprint []string level Level - transaction string request *http.Request // requestBody holds a reference to the original request.Body. requestBody interface { @@ -275,22 +274,6 @@ func (scope *Scope) SetLevel(level Level) { scope.level = level } -// SetTransaction sets the transaction name for the current transaction. -func (scope *Scope) SetTransaction(name string) { - scope.mu.Lock() - defer scope.mu.Unlock() - - scope.transaction = name -} - -// Transaction returns the transaction name for the current transaction. -func (scope *Scope) Transaction() (name string) { - scope.mu.RLock() - defer scope.mu.RUnlock() - - return scope.transaction -} - // Clone returns a copy of the current scope with all data copied over. func (scope *Scope) Clone() *Scope { scope.mu.RLock() @@ -312,7 +295,6 @@ func (scope *Scope) Clone() *Scope { clone.fingerprint = make([]string, len(scope.fingerprint)) copy(clone.fingerprint, scope.fingerprint) clone.level = scope.level - clone.transaction = scope.transaction clone.request = scope.request clone.requestBody = scope.requestBody clone.eventProcessors = scope.eventProcessors @@ -365,7 +347,11 @@ func (scope *Scope) ApplyToEvent(event *Event, hint *EventHint) *Event { // to link errors and traces/spans in Sentry. continue } - event.Contexts[key] = value + + // Ensure we are not overwriting event fields + if _, ok := event.Contexts[key]; !ok { + event.Contexts[key] = value + } } } @@ -379,8 +365,7 @@ func (scope *Scope) ApplyToEvent(event *Event, hint *EventHint) *Event { } } - var emptyUser User - if event.User == emptyUser { + if event.User.IsEmpty() { event.User = scope.user } @@ -392,10 +377,6 @@ func (scope *Scope) ApplyToEvent(event *Event, hint *EventHint) *Event { event.Level = scope.level } - if scope.transaction != "" { - event.Transaction = scope.transaction - } - if event.Request == nil && scope.request != nil { event.Request = NewRequest(scope.request) // NOTE: The SDK does not attempt to send partial request body data. diff --git a/vendor/github.com/getsentry/sentry-go/sentry.go b/vendor/github.com/getsentry/sentry-go/sentry.go index 9564a7e..f222138 100644 --- a/vendor/github.com/getsentry/sentry-go/sentry.go +++ b/vendor/github.com/getsentry/sentry-go/sentry.go @@ -5,15 +5,21 @@ import ( "time" ) +// Deprecated: Use SDKVersion instead. +const Version = SDKVersion + // Version is the version of the SDK. -const Version = "0.14.0" +const SDKVersion = "0.21.0" + +// The identifier of the SDK. +const SDKIdentifier = "sentry.go" // apiVersion is the minimum version of the Sentry API compatible with the // sentry-go SDK. const apiVersion = "7" // userAgent is the User-Agent of outgoing HTTP requests. -const userAgent = "sentry-go/" + Version +const userAgent = "sentry-go/" + SDKVersion // Init initializes the SDK with options. The returned error is non-nil if // options is invalid, for instance if a malformed DSN is provided. diff --git a/vendor/github.com/getsentry/sentry-go/span_recorder.go b/vendor/github.com/getsentry/sentry-go/span_recorder.go index 9e611ca..137aa23 100644 --- a/vendor/github.com/getsentry/sentry-go/span_recorder.go +++ b/vendor/github.com/getsentry/sentry-go/span_recorder.go @@ -4,11 +4,6 @@ import ( "sync" ) -// maxSpans limits the number of recorded spans per transaction. The limit is -// meant to bound memory usage and prevent too large transaction events that -// would be rejected by Sentry. -const maxSpans = 1000 - // A spanRecorder stores a span tree that makes up a transaction. Safe for // concurrent use. It is okay to add child spans from multiple goroutines. type spanRecorder struct { @@ -20,6 +15,10 @@ type spanRecorder struct { // record stores a span. The first stored span is assumed to be the root of a // span tree. func (r *spanRecorder) record(s *Span) { + maxSpans := defaultMaxSpans + if client := CurrentHub().Client(); client != nil { + maxSpans = client.Options().MaxSpans + } r.mu.Lock() defer r.mu.Unlock() if len(r.spans) >= maxSpans { diff --git a/vendor/github.com/getsentry/sentry-go/stacktrace.go b/vendor/github.com/getsentry/sentry-go/stacktrace.go index 8c256ec..c3c5f5d 100644 --- a/vendor/github.com/getsentry/sentry-go/stacktrace.go +++ b/vendor/github.com/getsentry/sentry-go/stacktrace.go @@ -166,13 +166,7 @@ type Frame struct { Symbol string `json:"symbol,omitempty"` // Module is, despite the name, the Sentry protocol equivalent of a Go // package's import path. - Module string `json:"module,omitempty"` - // Package is not used for Go stack trace frames. In other platforms it - // refers to a container where the Module can be found. For example, a - // Java JAR, a .NET Assembly, or a native dynamic library. - // It exists for completeness, allowing the construction and reporting - // of custom event payloads. - Package string `json:"package,omitempty"` + Module string `json:"module,omitempty"` Filename string `json:"filename,omitempty"` AbsPath string `json:"abs_path,omitempty"` Lineno int `json:"lineno,omitempty"` @@ -182,6 +176,18 @@ type Frame struct { PostContext []string `json:"post_context,omitempty"` InApp bool `json:"in_app,omitempty"` Vars map[string]interface{} `json:"vars,omitempty"` + // Package and the below are not used for Go stack trace frames. In + // other platforms it refers to a container where the Module can be + // found. For example, a Java JAR, a .NET Assembly, or a native + // dynamic library. They exists for completeness, allowing the + // construction and reporting of custom event payloads. + Package string `json:"package,omitempty"` + InstructionAddr string `json:"instruction_addr,omitempty"` + AddrMode string `json:"addr_mode,omitempty"` + SymbolAddr string `json:"symbol_addr,omitempty"` + ImageAddr string `json:"image_addr,omitempty"` + Platform string `json:"platform,omitempty"` + StackStart bool `json:"stack_start,omitempty"` } // NewFrame assembles a stacktrace frame out of runtime.Frame. diff --git a/vendor/github.com/getsentry/sentry-go/traces_sampler.go b/vendor/github.com/getsentry/sentry-go/traces_sampler.go index 3a341aa..69e7cb7 100644 --- a/vendor/github.com/getsentry/sentry-go/traces_sampler.go +++ b/vendor/github.com/getsentry/sentry-go/traces_sampler.go @@ -1,171 +1,19 @@ package sentry -import ( - "fmt" - - "github.com/getsentry/sentry-go/internal/crypto/randutil" -) - -// A TracesSampler makes sampling decisions for spans. -// -// In addition to the sampling context passed to the Sample method, -// implementations may keep and use internal state to make decisions. -// -// Sampling is one of the last steps when starting a new span, such that the -// sampler can inspect most of the state of the span to make a decision. -// -// Implementations must be safe for concurrent use by multiple goroutines. -type TracesSampler interface { - Sample(ctx SamplingContext) Sampled -} - -// Implementation note: -// -// TracesSampler.Sample return type is Sampled (instead of bool or float64), so -// that we can compose samplers by letting a sampler return SampledUndefined to -// defer the decision to the next sampler. -// -// For example, a hypothetical InheritFromParentSampler would return -// SampledUndefined if there is no parent span in the SamplingContext, deferring -// the sampling decision to another sampler, like a UniformSampler. -// -// var _ TracesSampler = sentry.TracesSamplers{ -// sentry.InheritFromParentSampler, -// sentry.UniformTracesSampler(0.1), -// } -// -// Another example, we can provide a sampler that returns SampledFalse if the -// SamplingContext matches some condition, and SampledUndefined otherwise: -// -// var _ TracesSampler = sentry.TracesSamplers{ -// sentry.IgnoreTransaction(regexp.MustCompile(`^\w+ /(favicon.ico|healthz)`), -// sentry.InheritFromParentSampler, -// sentry.UniformTracesSampler(0.1), -// } -// -// If after running all samplers the decision is still undefined, the -// span/transaction is not sampled. - // A SamplingContext is passed to a TracesSampler to determine a sampling // decision. +// +// TODO(tracing): possibly expand SamplingContext to include custom / +// user-provided data. type SamplingContext struct { Span *Span // The current span, always non-nil. Parent *Span // The parent span, may be nil. } -// TODO(tracing): possibly expand SamplingContext to include custom / -// user-provided data. -// -// Unlike in other SDKs, the current http.Request is not part of the -// SamplingContext to avoid bloating it with possibly unnecessary values that -// could confuse people or have negative performance consequences. -// -// For the request to be provided in a SamplingContext, a request pointer would -// most likely need to be stored in the span context and it would open precedent -// for more arbitrary data like fasthttp.Request. -// -// Users wanting to influence the sampling decision based on the request can -// still do so, either by updating the transaction directly on their HTTP -// handler: -// -// func(w http.ResponseWriter, r *http.Request) { -// transaction := sentry.TransactionFromContext(r.Context()) -// if r.Header.Get("X-Custom-Sampling") == "yes" { -// transaction.Sampled = sentry.SampledTrue -// } else { -// transaction.Sampled = sentry.SampledFalse -// } -// } -// -// Or by having their own middleware that stores arbitrary data in the request -// context (a pointer to the request itself included): -// -// type myContextKey struct{} -// type myContextData struct { -// request *http.Request -// // ... -// } -// -// func middleware(h http.Handler) http.Handler { -// return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { -// data := &myContextData{ -// request: r, -// } -// ctx := context.WithValue(r.Context(), myContextKey{}, data) -// h.ServeHTTP(w, r.WithContext(ctx)) -// }) -// } -// -// func main() { -// err := sentry.Init(sentry.ClientOptions{ -// // A custom TracesSampler can access data from the span's context: -// TracesSampler: sentry.TracesSamplerFunc(func(ctx sentry.SamplingContext) bool { -// data, ok := ctx.Span.Context().Value(myContextKey{}).(*myContextData) -// if !ok { -// return false -// } -// return data.request.URL.Hostname() == "example.com" -// }), -// }) -// // ... -// } -// -// Note, however, that for the middleware to be effective, it would have to run -// before sentryhttp's own middleware, meaning the middleware itself is not -// instrumented to send panics to Sentry and it is not part of the timed -// transaction. -// -// If neither of those prove to be sufficient, we can consider including a -// (possibly nil) *http.Request field to SamplingContext. In that case, the SDK -// would need to track the request either in the Scope or the Span.Context. -// -// Alternatively, add a map-like type or simply a generic interface{} similar to -// the CustomSamplingContext type in the Java SDK: -// -// type SamplingContext struct { -// Span *Span // The current span, always non-nil. -// Parent *Span // The parent span, may be nil. -// CustomData interface{} -// } -// -// func CustomSamplingContext(data interface{}) SpanOption { -// return func(s *Span) { -// s.customSamplingContext = data -// } -// } -// -// func main() { -// // ... -// span := sentry.StartSpan(ctx, "op", CustomSamplingContext(data)) -// // ... -// } - -// The TracesSamplerFunc type is an adapter to allow the use of ordinary +// The TracesSample type is an adapter to allow the use of ordinary // functions as a TracesSampler. -type TracesSamplerFunc func(ctx SamplingContext) Sampled +type TracesSampler func(ctx SamplingContext) float64 -var _ TracesSampler = TracesSamplerFunc(nil) - -func (f TracesSamplerFunc) Sample(ctx SamplingContext) Sampled { +func (f TracesSampler) Sample(ctx SamplingContext) float64 { return f(ctx) } - -// UniformTracesSampler is a TracesSampler that samples root spans randomly at a -// uniform rate. -type UniformTracesSampler float64 - -var _ TracesSampler = UniformTracesSampler(0) - -func (s UniformTracesSampler) Sample(ctx SamplingContext) Sampled { - if s < 0.0 || s > 1.0 { - panic(fmt.Errorf("sampling rate out of range [0.0, 1.0]: %f", s)) - } - if randutil.Float64() < float64(s) { - return SampledTrue - } - return SampledFalse -} - -// TODO(tracing): implement and export basic TracesSampler implementations: -// parent-based, span ID / trace ID based, etc. It should be possible to compose -// parent-based with other samplers. diff --git a/vendor/github.com/getsentry/sentry-go/tracing.go b/vendor/github.com/getsentry/sentry-go/tracing.go index 6c992fa..519dc20 100644 --- a/vendor/github.com/getsentry/sentry-go/tracing.go +++ b/vendor/github.com/getsentry/sentry-go/tracing.go @@ -9,9 +9,15 @@ import ( "net/http" "regexp" "strings" + "sync" "time" ) +const ( + SentryTraceHeader = "sentry-trace" + SentryBaggageHeader = "baggage" +) + // A Span is the building block of a Sentry transaction. Spans build up a tree // structure of timed operations. The span tree makes up a transaction event // that is sent to Sentry when the root span is finished. @@ -21,6 +27,7 @@ type Span struct { //nolint: maligned // prefer readability over optimal memory TraceID TraceID `json:"trace_id"` SpanID SpanID `json:"span_id"` ParentSpanID SpanID `json:"parent_span_id"` + Name string `json:"name,omitempty"` Op string `json:"op,omitempty"` Description string `json:"description,omitempty"` Status SpanStatus `json:"status,omitempty"` @@ -28,24 +35,44 @@ type Span struct { //nolint: maligned // prefer readability over optimal memory StartTime time.Time `json:"start_timestamp"` EndTime time.Time `json:"timestamp"` Data map[string]interface{} `json:"data,omitempty"` + Sampled Sampled `json:"-"` + Source TransactionSource `json:"-"` - Sampled Sampled `json:"-"` - + // mu protects concurrent writes to map fields + mu sync.RWMutex + // sample rate the span was sampled with. + sampleRate float64 // ctx is the context where the span was started. Always non-nil. ctx context.Context - + // Dynamic Sampling context + dynamicSamplingContext DynamicSamplingContext // parent refers to the immediate local parent span. A remote parent span is // only referenced by setting ParentSpanID. parent *Span - // isTransaction is true only for the root span of a local span tree. The // root span is the first span started in a context. Note that a local root // span may have a remote parent belonging to the same trace, therefore // isTransaction depends on ctx and not on parent. isTransaction bool - // recorder stores all spans in a transaction. Guaranteed to be non-nil. recorder *spanRecorder + // span context, can only be set on transactions + contexts map[string]Context +} + +// TraceParentContext describes the context of a (remote) parent span. +// +// The context is normally extracted from a received "sentry-trace" header and +// used to initialize a new transaction. +// +// Note: the name might be not the best one. It was taken mostly to stay aligned +// with other SDKs, and it alludes to W3C "traceparent" header (https://www.w3.org/TR/trace-context/), +// which serves a similar purpose to "sentry-trace". We should eventually consider +// making this type internal-only and give it a better name. +type TraceParentContext struct { + TraceID TraceID + ParentSpanID SpanID + Sampled Sampled } // (*) Note on maligned: @@ -81,14 +108,19 @@ func StartSpan(ctx context.Context, operation string, options ...SpanOption) *Sp // defaults Op: operation, StartTime: time.Now(), + Sampled: SampledUndefined, ctx: context.WithValue(ctx, spanContextKey{}, &span), parent: parent, isTransaction: !hasParent, } + if hasParent { span.TraceID = parent.TraceID } else { + // Only set the Source if this is a transaction + span.Source = SourceCustom + // Implementation note: // // While math/rand is ~2x faster than crypto/rand (exact @@ -174,9 +206,6 @@ func (s *Span) Finish() { // (see https://github.com/getsentry/sentry-python/blob/f6f3525f8812f609/sentry_sdk/tracing.py#L372) hub := hubFromContext(s.ctx) - if hub.Scope().Transaction() == "" { - Logger.Printf("Missing transaction name for span with op = %q", s.Op) - } hub.CaptureEvent(event) } @@ -195,12 +224,66 @@ func (s *Span) StartChild(operation string, options ...SpanOption) *Span { // accessing the tags map directly as SetTag takes care of initializing the map // when necessary. func (s *Span) SetTag(name, value string) { + s.mu.Lock() + defer s.mu.Unlock() + if s.Tags == nil { s.Tags = make(map[string]string) } s.Tags[name] = value } +// SetData sets a data on the span. It is recommended to use SetData instead of +// accessing the data map directly as SetData takes care of initializing the map +// when necessary. +func (s *Span) SetData(name, value string) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.Data == nil { + s.Data = make(map[string]interface{}) + } + s.Data[name] = value +} + +// SetContext sets a context on the span. It is recommended to use SetContext instead of +// accessing the contexts map directly as SetContext takes care of initializing the map +// when necessary. +func (s *Span) SetContext(key string, value Context) { + s.mu.Lock() + defer s.mu.Unlock() + + if s.contexts == nil { + s.contexts = make(map[string]Context) + } + s.contexts[key] = value +} + +// IsTransaction checks if the given span is a transaction. +func (s *Span) IsTransaction() bool { + return s.isTransaction +} + +// GetTransaction returns the transaction that contains this span. +// +// For transaction spans it returns itself. For spans that were created manually +// the method returns "nil". +func (s *Span) GetTransaction() *Span { + spanRecorder := s.spanRecorder() + if spanRecorder == nil { + // This probably means that the Span was created manually (not via + // StartTransaction/StartSpan or StartChild). + // Return "nil" to indicate that it's not a normal situation. + return nil + } + recorderRoot := spanRecorder.root() + if recorderRoot == nil { + // Same as above: manually created Span. + return nil + } + return recorderRoot +} + // TODO(tracing): maybe add shortcuts to get/set transaction name. Right now the // transaction name is in the Scope, as it has existed there historically, prior // to tracing. @@ -210,8 +293,9 @@ func (s *Span) SetTag(name, value string) { // func (s *Span) TransactionName() string // func (s *Span) SetTransactionName(name string) -// ToSentryTrace returns the trace propagation value used with the sentry-trace -// HTTP header. +// ToSentryTrace returns the seralized TraceParentContext from a transaction/sapn. +// Use this function to propagate the TraceParentContext to a downstream SDK, +// either as the value of the "sentry-trace" HTTP header, or as an html "sentry-trace" meta tag. func (s *Span) ToSentryTrace() string { // TODO(tracing): add instrumentation for outgoing HTTP requests using // ToSentryTrace. @@ -226,25 +310,51 @@ func (s *Span) ToSentryTrace() string { return b.String() } +// ToBaggage returns the serialized DynamicSamplingContext from a transaction. +// Use this function to propagate the DynamicSamplingContext to a downstream SDK, +// either as the value of the "baggage" HTTP header, or as an html "baggage" meta tag. +func (s *Span) ToBaggage() string { + if containingTransaction := s.GetTransaction(); containingTransaction != nil { + // In case there is currently no frozen DynamicSamplingContext attached to the transaction, + // create one from the properties of the transaction. + if !s.dynamicSamplingContext.IsFrozen() { + // This will return a frozen DynamicSamplingContext. + s.dynamicSamplingContext = DynamicSamplingContextFromTransaction(containingTransaction) + } + + return containingTransaction.dynamicSamplingContext.String() + } + return "" +} + +// SetDynamicSamplingContext sets the given dynamic sampling context on the +// current transaction. +func (s *Span) SetDynamicSamplingContext(dsc DynamicSamplingContext) { + if s.isTransaction { + s.dynamicSamplingContext = dsc + } +} + // sentryTracePattern matches either // -// TRACE_ID - SPAN_ID -// [[:xdigit:]]{32}-[[:xdigit:]]{16} +// TRACE_ID - SPAN_ID +// [[:xdigit:]]{32}-[[:xdigit:]]{16} // // or // -// TRACE_ID - SPAN_ID - SAMPLED -// [[:xdigit:]]{32}-[[:xdigit:]]{16}-[01] +// TRACE_ID - SPAN_ID - SAMPLED +// [[:xdigit:]]{32}-[[:xdigit:]]{16}-[01] var sentryTracePattern = regexp.MustCompile(`^([[:xdigit:]]{32})-([[:xdigit:]]{16})(?:-([01]))?$`) // updateFromSentryTrace parses a sentry-trace HTTP header (as returned by // ToSentryTrace) and updates fields of the span. If the header cannot be -// recognized as valid, the span is left unchanged. -func (s *Span) updateFromSentryTrace(header []byte) { +// recognized as valid, the span is left unchanged. The returned value indicates +// whether the span was updated. +func (s *Span) updateFromSentryTrace(header []byte) (updated bool) { m := sentryTracePattern.FindSubmatch(header) if m == nil { // no match - return + return false } _, _ = hex.Decode(s.TraceID[:], m[1]) _, _ = hex.Decode(s.ParentSpanID[:], m[2]) @@ -256,6 +366,18 @@ func (s *Span) updateFromSentryTrace(header []byte) { s.Sampled = SampledTrue } } + return true +} + +func (s *Span) updateFromBaggage(header []byte) { + if s.isTransaction { + dsc, err := DynamicSamplingContextFromHeader(header) + if err != nil { + return + } + + s.dynamicSamplingContext = dsc + } } func (s *Span) MarshalJSON() ([]byte, error) { @@ -276,45 +398,104 @@ func (s *Span) MarshalJSON() ([]byte, error) { } func (s *Span) sample() Sampled { - // https://develop.sentry.dev/sdk/unified-api/tracing/#sampling - // #1 explicit sampling decision via StartSpan options. - if s.Sampled != SampledUndefined { - return s.Sampled - } hub := hubFromContext(s.ctx) var clientOptions ClientOptions client := hub.Client() if client != nil { clientOptions = hub.Client().Options() } - samplingContext := SamplingContext{Span: s, Parent: s.parent} + + // https://develop.sentry.dev/sdk/performance/#sampling + // #1 tracing is not enabled. + if !clientOptions.EnableTracing { + Logger.Printf("Dropping transaction: EnableTracing is set to %t", clientOptions.EnableTracing) + s.sampleRate = 0.0 + return SampledFalse + } + + // #2 explicit sampling decision via StartSpan/StartTransaction options. + if s.Sampled != SampledUndefined { + Logger.Printf("Using explicit sampling decision from StartSpan/StartTransaction: %v", s.Sampled) + switch s.Sampled { + case SampledTrue: + s.sampleRate = 1.0 + case SampledFalse: + s.sampleRate = 0.0 + } + return s.Sampled + } + // Variant for non-transaction spans: they inherit the parent decision. - // TracesSampler only runs for the root span. // Note: non-transaction should always have a parent, but we check both // conditions anyway -- the first for semantic meaning, the second to // avoid a nil pointer dereference. if !s.isTransaction && s.parent != nil { return s.parent.Sampled } - // #2 use TracesSampler from ClientOptions. + + // #3 use TracesSampler from ClientOptions. sampler := clientOptions.TracesSampler - if sampler != nil { - return sampler.Sample(samplingContext) + samplingContext := SamplingContext{ + Span: s, + Parent: s.parent, } - // #3 inherit parent decision. + + if sampler != nil { + tracesSamplerSampleRate := sampler.Sample(samplingContext) + s.sampleRate = tracesSamplerSampleRate + if tracesSamplerSampleRate < 0.0 || tracesSamplerSampleRate > 1.0 { + Logger.Printf("Dropping transaction: Returned TracesSampler rate is out of range [0.0, 1.0]: %f", tracesSamplerSampleRate) + return SampledFalse + } + if tracesSamplerSampleRate == 0 { + Logger.Printf("Dropping transaction: Returned TracesSampler rate is: %f", tracesSamplerSampleRate) + return SampledFalse + } + + if rng.Float64() < tracesSamplerSampleRate { + return SampledTrue + } + Logger.Printf("Dropping transaction: TracesSampler returned rate: %f", tracesSamplerSampleRate) + return SampledFalse + } + // #4 inherit parent decision. if s.parent != nil { + Logger.Printf("Using sampling decision from parent: %v", s.parent.Sampled) + switch s.parent.Sampled { + case SampledTrue: + s.sampleRate = 1.0 + case SampledFalse: + s.sampleRate = 0.0 + } return s.parent.Sampled } - // #4 uniform sampling using TracesSampleRate. - sampler = UniformTracesSampler(clientOptions.TracesSampleRate) - return sampler.Sample(samplingContext) + + // #5 use TracesSampleRate from ClientOptions. + sampleRate := clientOptions.TracesSampleRate + s.sampleRate = sampleRate + if sampleRate < 0.0 || sampleRate > 1.0 { + Logger.Printf("Dropping transaction: TracesSamplerRate out of range [0.0, 1.0]: %f", sampleRate) + return SampledFalse + } + if sampleRate == 0.0 { + Logger.Printf("Dropping transaction: TracesSampleRate rate is: %f", sampleRate) + return SampledFalse + } + + if rng.Float64() < sampleRate { + return SampledTrue + } + + return SampledFalse } func (s *Span) toEvent() *Event { + s.mu.Lock() + defer s.mu.Unlock() + if !s.isTransaction { return nil // only transactions can be transformed into events } - hub := hubFromContext(s.ctx) children := s.recorder.children() finished := make([]*Span, 0, len(children)) @@ -326,17 +507,33 @@ func (s *Span) toEvent() *Event { finished = append(finished, child) } + // Create and attach a DynamicSamplingContext to the transaction. + // If the DynamicSamplingContext is not frozen at this point, we can assume being head of trace. + if !s.dynamicSamplingContext.IsFrozen() { + s.dynamicSamplingContext = DynamicSamplingContextFromTransaction(s) + } + + contexts := map[string]Context{} + for k, v := range s.contexts { + contexts[k] = v + } + contexts["trace"] = s.traceContext().Map() + return &Event{ Type: transactionType, - Transaction: hub.Scope().Transaction(), - Contexts: map[string]Context{ - "trace": s.traceContext().Map(), + Transaction: s.Name, + Contexts: contexts, + Tags: s.Tags, + Extra: s.Data, + Timestamp: s.EndTime, + StartTime: s.StartTime, + Spans: finished, + TransactionInfo: &TransactionInfo{ + Source: s.Source, + }, + sdkMetaData: SDKMetaData{ + dsc: s.dynamicSamplingContext, }, - Tags: s.Tags, - Extra: s.Data, - Timestamp: s.EndTime, - StartTime: s.StartTime, - Spans: finished, } } @@ -354,6 +551,23 @@ func (s *Span) traceContext() *TraceContext { // spanRecorder stores the span tree. Guaranteed to be non-nil. func (s *Span) spanRecorder() *spanRecorder { return s.recorder } +// ParseTraceParentContext parses a sentry-trace header and builds a TraceParentContext from the +// parsed values. If the header was parsed correctly, the second returned argument +// ("valid") will be set to true, otherwise (e.g., empty or malformed header) it will +// be false. +func ParseTraceParentContext(header []byte) (traceParentContext TraceParentContext, valid bool) { + s := Span{} + updated := s.updateFromSentryTrace(header) + if !updated { + return TraceParentContext{}, false + } + return TraceParentContext{ + TraceID: s.TraceID, + ParentSpanID: s.ParentSpanID, + Sampled: s.Sampled, + }, true +} + // TraceID identifies a trace. type TraceID [16]byte @@ -394,6 +608,18 @@ var ( zeroSpanID SpanID ) +// Contains information about how the name of the transaction was determined. +type TransactionSource string + +const ( + SourceCustom TransactionSource = "custom" + SourceURL TransactionSource = "url" + SourceRoute TransactionSource = "route" + SourceView TransactionSource = "view" + SourceComponent TransactionSource = "component" + SourceTask TransactionSource = "task" +) + // SpanStatus is the status of a span. type SpanStatus uint8 @@ -530,9 +756,9 @@ type Sampled int8 // The possible trace sampling decisions are: SampledFalse, SampledUndefined // (default) and SampledTrue. const ( - SampledFalse Sampled = -1 + iota - SampledUndefined - SampledTrue + SampledFalse Sampled = -1 + SampledUndefined Sampled = 0 + SampledTrue Sampled = 1 ) func (s Sampled) String() string { @@ -561,9 +787,62 @@ type SpanOption func(s *Span) // A span tree has a single transaction name, therefore using this option when // starting a span affects the span tree as a whole, potentially overwriting a // name set previously. +// +// Deprecated: Use WithTransactionSource() instead. func TransactionName(name string) SpanOption { + return WithTransactionName(name) +} + +// WithTransactionName option sets the name of the current transaction. +// +// A span tree has a single transaction name, therefore using this option when +// starting a span affects the span tree as a whole, potentially overwriting a +// name set previously. +func WithTransactionName(name string) SpanOption { return func(s *Span) { - hubFromContext(s.Context()).Scope().SetTransaction(name) + s.Name = name + } +} + +// OpName sets the operation name for a given span. +// +// Deprecated: Use WithOpName() instead. +func OpName(name string) SpanOption { + return WithOpName(name) +} + +// WithOpName sets the operation name for a given span. +func WithOpName(name string) SpanOption { + return func(s *Span) { + s.Op = name + } +} + +// TransctionSource sets the source of the transaction name. +// +// Deprecated: Use WithTransactionSource() instead. +func TransctionSource(source TransactionSource) SpanOption { + return WithTransactionSource(source) +} + +// WithTransactionSource sets the source of the transaction name. +func WithTransactionSource(source TransactionSource) SpanOption { + return func(s *Span) { + s.Source = source + } +} + +// SpanSampled updates the sampling flag for a given span. +// +// Deprecated: Use WithSpanSampled() instead. +func SpanSampled(sampled Sampled) SpanOption { + return WithSpanSampled(sampled) +} + +// WithSpanSampled updates the sampling flag for a given span. +func WithSpanSampled(sampled Sampled) SpanOption { + return func(s *Span) { + s.Sampled = sampled } } @@ -572,9 +851,31 @@ func TransactionName(name string) SpanOption { // span will be left unchanged. // // ContinueFromRequest is an alias for: -// ContinueFromTrace(r.Header.Get("sentry-trace")) +// +// ContinueFromHeaders(r.Header.Get(SentryTraceHeader), r.Header.Get(SentryBaggageHeader)). func ContinueFromRequest(r *http.Request) SpanOption { - return ContinueFromTrace(r.Header.Get("sentry-trace")) + return ContinueFromHeaders(r.Header.Get(SentryTraceHeader), r.Header.Get(SentryBaggageHeader)) +} + +// ContinueFromHeaders returns a span option that updates the span to continue +// an existing TraceID and propagates the Dynamic Sampling context. +func ContinueFromHeaders(trace, baggage string) SpanOption { + return func(s *Span) { + if trace != "" { + s.updateFromSentryTrace([]byte(trace)) + } + if baggage != "" { + s.updateFromBaggage([]byte(baggage)) + } + + // In case a sentry-trace header is present but there are no sentry-related + // values in the baggage, create an empty, frozen DynamicSamplingContext. + if trace != "" && !s.dynamicSamplingContext.HasEntries() { + s.dynamicSamplingContext = DynamicSamplingContext{ + Frozen: true, + } + } + } } // ContinueFromTrace returns a span option that updates the span to continue @@ -612,7 +913,7 @@ func TransactionFromContext(ctx context.Context) *Span { // // Note the equivalence: // -// SpanFromContext(ctx).StartChild(...) === StartSpan(ctx, ...) +// SpanFromContext(ctx).StartChild(...) === StartSpan(ctx, ...) // // So we don't aim spanFromContext at creating spans, but mutating existing // spans that you'd have no access otherwise (because it was created in code you @@ -626,3 +927,19 @@ func spanFromContext(ctx context.Context) *Span { } return nil } + +// StartTransaction will create a transaction (root span) if there's no existing +// transaction in the context otherwise, it will return the existing transaction. +func StartTransaction(ctx context.Context, name string, options ...SpanOption) *Span { + currentTransaction, exists := ctx.Value(spanContextKey{}).(*Span) + if exists { + return currentTransaction + } + + options = append(options, WithTransactionName(name)) + return StartSpan( + ctx, + "", + options..., + ) +} diff --git a/vendor/github.com/getsentry/sentry-go/transport.go b/vendor/github.com/getsentry/sentry-go/transport.go index d8fa2be..a6abf1d 100644 --- a/vendor/github.com/getsentry/sentry-go/transport.go +++ b/vendor/github.com/getsentry/sentry-go/transport.go @@ -94,26 +94,53 @@ func getRequestBodyFromEvent(event *Event) []byte { return nil } -func transactionEnvelopeFromBody(eventID EventID, sentAt time.Time, body json.RawMessage) (*bytes.Buffer, error) { +func envelopeFromBody(event *Event, dsn *Dsn, sentAt time.Time, body json.RawMessage) (*bytes.Buffer, error) { var b bytes.Buffer enc := json.NewEncoder(&b) - // envelope header + + // Construct the trace envelope header + var trace = map[string]string{} + if dsc := event.sdkMetaData.dsc; dsc.HasEntries() { + for k, v := range dsc.Entries { + trace[k] = v + } + } + + // Envelope header err := enc.Encode(struct { - EventID EventID `json:"event_id"` - SentAt time.Time `json:"sent_at"` + EventID EventID `json:"event_id"` + SentAt time.Time `json:"sent_at"` + Dsn string `json:"dsn"` + Sdk map[string]string `json:"sdk"` + Trace map[string]string `json:"trace,omitempty"` }{ - EventID: eventID, + EventID: event.EventID, SentAt: sentAt, + Trace: trace, + Dsn: dsn.String(), + Sdk: map[string]string{ + "name": event.Sdk.Name, + "version": event.Sdk.Version, + }, }) if err != nil { return nil, err } - // item header + + var itemType string + switch event.Type { + case transactionType: + itemType = transactionType + default: + itemType = eventType + } + + // Item header err = enc.Encode(struct { Type string `json:"type"` Length int `json:"length"` }{ - Type: transactionType, + Type: itemType, Length: len(body), }) if err != nil { @@ -124,6 +151,7 @@ func transactionEnvelopeFromBody(eventID EventID, sentAt time.Time, body json.Ra if err != nil { return nil, err } + return &b, nil } @@ -137,21 +165,14 @@ func getRequestFromEvent(event *Event, dsn *Dsn) (r *http.Request, err error) { if body == nil { return nil, errors.New("event could not be marshaled") } - if event.Type == transactionType { - b, err := transactionEnvelopeFromBody(event.EventID, time.Now(), body) - if err != nil { - return nil, err - } - return http.NewRequest( - http.MethodPost, - dsn.EnvelopeAPIURL().String(), - b, - ) + envelope, err := envelopeFromBody(event, dsn, time.Now(), body) + if err != nil { + return nil, err } return http.NewRequest( http.MethodPost, - dsn.StoreAPIURL().String(), - bytes.NewReader(body), + dsn.GetAPIURL().String(), + envelope, ) } diff --git a/vendor/github.com/getsentry/sentry-go/util.go b/vendor/github.com/getsentry/sentry-go/util.go index b523f5d..e5717c6 100644 --- a/vendor/github.com/getsentry/sentry-go/util.go +++ b/vendor/github.com/getsentry/sentry-go/util.go @@ -35,7 +35,7 @@ func monotonicTimeSince(start time.Time) (end time.Time) { return start.Add(time.Since(start)) } -//nolint: deadcode, unused +// nolint: deadcode, unused func prettyPrint(data interface{}) { dbg, _ := json.MarshalIndent(data, "", " ") fmt.Println(string(dbg)) diff --git a/vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.pb.go b/vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.pb.go deleted file mode 100644 index a76f807..0000000 --- a/vendor/github.com/golang/protobuf/ptypes/timestamp/timestamp.pb.go +++ /dev/null @@ -1,64 +0,0 @@ -// Code generated by protoc-gen-go. DO NOT EDIT. -// source: github.com/golang/protobuf/ptypes/timestamp/timestamp.proto - -package timestamp - -import ( - protoreflect "google.golang.org/protobuf/reflect/protoreflect" - protoimpl "google.golang.org/protobuf/runtime/protoimpl" - timestamppb "google.golang.org/protobuf/types/known/timestamppb" - reflect "reflect" -) - -// Symbols defined in public import of google/protobuf/timestamp.proto. - -type Timestamp = timestamppb.Timestamp - -var File_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto protoreflect.FileDescriptor - -var file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_rawDesc = []byte{ - 0x0a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, - 0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x74, 0x79, - 0x70, 0x65, 0x73, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2f, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x1a, 0x1f, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, - 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x42, 0x37, - 0x5a, 0x35, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x67, 0x6f, 0x6c, - 0x61, 0x6e, 0x67, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x70, 0x74, 0x79, - 0x70, 0x65, 0x73, 0x2f, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x3b, 0x74, 0x69, - 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x50, 0x00, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, -} - -var file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_goTypes = []interface{}{} -var file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_depIdxs = []int32{ - 0, // [0:0] is the sub-list for method output_type - 0, // [0:0] is the sub-list for method input_type - 0, // [0:0] is the sub-list for extension type_name - 0, // [0:0] is the sub-list for extension extendee - 0, // [0:0] is the sub-list for field type_name -} - -func init() { file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_init() } -func file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_init() { - if File_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto != nil { - return - } - type x struct{} - out := protoimpl.TypeBuilder{ - File: protoimpl.DescBuilder{ - GoPackagePath: reflect.TypeOf(x{}).PkgPath(), - RawDescriptor: file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_rawDesc, - NumEnums: 0, - NumMessages: 0, - NumExtensions: 0, - NumServices: 0, - }, - GoTypes: file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_goTypes, - DependencyIndexes: file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_depIdxs, - }.Build() - File_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto = out.File - file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_rawDesc = nil - file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_goTypes = nil - file_github_com_golang_protobuf_ptypes_timestamp_timestamp_proto_depIdxs = nil -} diff --git a/vendor/github.com/klauspost/compress/.goreleaser.yml b/vendor/github.com/klauspost/compress/.goreleaser.yml index 0af08e6..7a008a4 100644 --- a/vendor/github.com/klauspost/compress/.goreleaser.yml +++ b/vendor/github.com/klauspost/compress/.goreleaser.yml @@ -3,7 +3,7 @@ before: hooks: - ./gen.sh - - go install mvdan.cc/garble@latest + - go install mvdan.cc/garble@v0.9.3 builds: - diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 9ec000f..efab55e 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -9,7 +9,6 @@ This package provides various compression algorithms. * [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding. * [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped requests efficiently. * [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation. -* [fuzz package](https://github.com/klauspost/compress-fuzz) for fuzz testing all compressors/decompressors here. [![Go Reference](https://pkg.go.dev/badge/klauspost/compress.svg)](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories) [![Go](https://github.com/klauspost/compress/actions/workflows/go.yml/badge.svg)](https://github.com/klauspost/compress/actions/workflows/go.yml) @@ -17,6 +16,52 @@ This package provides various compression algorithms. # changelog +* Apr 5, 2023 - [v1.16.4](https://github.com/klauspost/compress/releases/tag/v1.16.4) + * zstd: Improve zstd best efficiency by @greatroar and @klauspost in https://github.com/klauspost/compress/pull/784 + * zstd: Respect WithAllLitEntropyCompression https://github.com/klauspost/compress/pull/792 + * zstd: Fix amd64 not always detecting corrupt data https://github.com/klauspost/compress/pull/785 + * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 + * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 + * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 + * gzhttp: Suppport ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 + +* Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) + * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 + * gzhttp: Add optional [BREACH mitigation](https://github.com/klauspost/compress/tree/master/gzhttp#breach-mitigation). https://github.com/klauspost/compress/pull/762 https://github.com/klauspost/compress/pull/768 https://github.com/klauspost/compress/pull/769 https://github.com/klauspost/compress/pull/770 https://github.com/klauspost/compress/pull/767 + * s2: Add Intel LZ4s converter https://github.com/klauspost/compress/pull/766 + * zstd: Minor bug fixes https://github.com/klauspost/compress/pull/771 https://github.com/klauspost/compress/pull/772 https://github.com/klauspost/compress/pull/773 + * huff0: Speed up compress1xDo by @greatroar in https://github.com/klauspost/compress/pull/774 + +* Feb 26, 2023 - [v1.16.0](https://github.com/klauspost/compress/releases/tag/v1.16.0) + * s2: Add [Dictionary](https://github.com/klauspost/compress/tree/master/s2#dictionaries) support. https://github.com/klauspost/compress/pull/685 + * s2: Add Compression Size Estimate. https://github.com/klauspost/compress/pull/752 + * s2: Add support for custom stream encoder. https://github.com/klauspost/compress/pull/755 + * s2: Add LZ4 block converter. https://github.com/klauspost/compress/pull/748 + * s2: Support io.ReaderAt in ReadSeeker. https://github.com/klauspost/compress/pull/747 + * s2c/s2sx: Use concurrent decoding. https://github.com/klauspost/compress/pull/746 + +* Jan 21st, 2023 (v1.15.15) + * deflate: Improve level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/739 + * zstd: Add delta encoding support by @greatroar in https://github.com/klauspost/compress/pull/728 + * zstd: Various speed improvements by @greatroar https://github.com/klauspost/compress/pull/741 https://github.com/klauspost/compress/pull/734 https://github.com/klauspost/compress/pull/736 https://github.com/klauspost/compress/pull/744 https://github.com/klauspost/compress/pull/743 https://github.com/klauspost/compress/pull/745 + * gzhttp: Add SuffixETag() and DropETag() options to prevent ETag collisions on compressed responses by @willbicks in https://github.com/klauspost/compress/pull/740 + +* Jan 3rd, 2023 (v1.15.14) + + * flate: Improve speed in big stateless blocks https://github.com/klauspost/compress/pull/718 + * zstd: Minor speed tweaks by @greatroar in https://github.com/klauspost/compress/pull/716 https://github.com/klauspost/compress/pull/720 + * export NoGzipResponseWriter for custom ResponseWriter wrappers by @harshavardhana in https://github.com/klauspost/compress/pull/722 + * s2: Add example for indexing and existing stream https://github.com/klauspost/compress/pull/723 + +* Dec 11, 2022 (v1.15.13) + * zstd: Add [MaxEncodedSize](https://pkg.go.dev/github.com/klauspost/compress@v1.15.13/zstd#Encoder.MaxEncodedSize) to encoder https://github.com/klauspost/compress/pull/691 + * zstd: Various tweaks and improvements https://github.com/klauspost/compress/pull/693 https://github.com/klauspost/compress/pull/695 https://github.com/klauspost/compress/pull/696 https://github.com/klauspost/compress/pull/701 https://github.com/klauspost/compress/pull/702 https://github.com/klauspost/compress/pull/703 https://github.com/klauspost/compress/pull/704 https://github.com/klauspost/compress/pull/705 https://github.com/klauspost/compress/pull/706 https://github.com/klauspost/compress/pull/707 https://github.com/klauspost/compress/pull/708 + +* Oct 26, 2022 (v1.15.12) + + * zstd: Tweak decoder allocs. https://github.com/klauspost/compress/pull/680 + * gzhttp: Always delete `HeaderNoCompression` https://github.com/klauspost/compress/pull/683 + * Sept 26, 2022 (v1.15.11) * flate: Improve level 1-3 compression https://github.com/klauspost/compress/pull/678 @@ -579,6 +624,8 @@ Here are other packages of good quality and pure Go (no cgo wrappers or autoconv * [github.com/pierrec/lz4](https://github.com/pierrec/lz4) - strong multithreaded LZ4 compression. * [github.com/cosnicolaou/pbzip2](https://github.com/cosnicolaou/pbzip2) - multithreaded bzip2 decompression. * [github.com/dsnet/compress](https://github.com/dsnet/compress) - brotli decompression, bzip2 writer. +* [github.com/ronanh/intcomp](https://github.com/ronanh/intcomp) - Integer compression. +* [github.com/spenczar/fpc](https://github.com/spenczar/fpc) - Float compression. # license diff --git a/vendor/github.com/klauspost/compress/fse/compress.go b/vendor/github.com/klauspost/compress/fse/compress.go index 6f34191..dac97e5 100644 --- a/vendor/github.com/klauspost/compress/fse/compress.go +++ b/vendor/github.com/klauspost/compress/fse/compress.go @@ -146,54 +146,51 @@ func (s *Scratch) compress(src []byte) error { c1.encodeZero(tt[src[ip-2]]) ip -= 2 } + src = src[:ip] // Main compression loop. switch { case !s.zeroBits && s.actualTableLog <= 8: // We can encode 4 symbols without requiring a flush. // We do not need to check if any output is 0 bits. - for ip >= 4 { + for ; len(src) >= 4; src = src[:len(src)-4] { s.bw.flush32() - v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] c2.encode(tt[v0]) c1.encode(tt[v1]) c2.encode(tt[v2]) c1.encode(tt[v3]) - ip -= 4 } case !s.zeroBits: // We do not need to check if any output is 0 bits. - for ip >= 4 { + for ; len(src) >= 4; src = src[:len(src)-4] { s.bw.flush32() - v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] c2.encode(tt[v0]) c1.encode(tt[v1]) s.bw.flush32() c2.encode(tt[v2]) c1.encode(tt[v3]) - ip -= 4 } case s.actualTableLog <= 8: // We can encode 4 symbols without requiring a flush - for ip >= 4 { + for ; len(src) >= 4; src = src[:len(src)-4] { s.bw.flush32() - v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] c2.encodeZero(tt[v0]) c1.encodeZero(tt[v1]) c2.encodeZero(tt[v2]) c1.encodeZero(tt[v3]) - ip -= 4 } default: - for ip >= 4 { + for ; len(src) >= 4; src = src[:len(src)-4] { s.bw.flush32() - v3, v2, v1, v0 := src[ip-4], src[ip-3], src[ip-2], src[ip-1] + v3, v2, v1, v0 := src[len(src)-4], src[len(src)-3], src[len(src)-2], src[len(src)-1] c2.encodeZero(tt[v0]) c1.encodeZero(tt[v1]) s.bw.flush32() c2.encodeZero(tt[v2]) c1.encodeZero(tt[v3]) - ip -= 4 } } @@ -459,15 +456,17 @@ func (s *Scratch) countSimple(in []byte) (max int) { for _, v := range in { s.count[v]++ } - m := uint32(0) + m, symlen := uint32(0), s.symbolLen for i, v := range s.count[:] { + if v == 0 { + continue + } if v > m { m = v } - if v > 0 { - s.symbolLen = uint16(i) + 1 - } + symlen = uint16(i) + 1 } + s.symbolLen = symlen return int(m) } diff --git a/vendor/github.com/klauspost/compress/fse/decompress.go b/vendor/github.com/klauspost/compress/fse/decompress.go index 926f5f1..cc05d0f 100644 --- a/vendor/github.com/klauspost/compress/fse/decompress.go +++ b/vendor/github.com/klauspost/compress/fse/decompress.go @@ -260,7 +260,9 @@ func (s *Scratch) buildDtable() error { // If the buffer is over-read an error is returned. func (s *Scratch) decompress() error { br := &s.bits - br.init(s.br.unread()) + if err := br.init(s.br.unread()); err != nil { + return err + } var s1, s2 decoder // Initialize and decode first state and symbol. diff --git a/vendor/github.com/klauspost/compress/huff0/bitreader.go b/vendor/github.com/klauspost/compress/huff0/bitreader.go index 504a7be..e36d974 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitreader.go +++ b/vendor/github.com/klauspost/compress/huff0/bitreader.go @@ -67,7 +67,6 @@ func (b *bitReaderBytes) fillFast() { // 2 bounds checks. v := b.in[b.off-4 : b.off] - v = v[:4] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value |= uint64(low) << (b.bitsRead - 32) b.bitsRead -= 32 @@ -88,8 +87,7 @@ func (b *bitReaderBytes) fill() { return } if b.off > 4 { - v := b.in[b.off-4:] - v = v[:4] + v := b.in[b.off-4 : b.off] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value |= uint64(low) << (b.bitsRead - 32) b.bitsRead -= 32 @@ -179,7 +177,6 @@ func (b *bitReaderShifted) fillFast() { // 2 bounds checks. v := b.in[b.off-4 : b.off] - v = v[:4] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value |= uint64(low) << ((b.bitsRead - 32) & 63) b.bitsRead -= 32 @@ -200,8 +197,7 @@ func (b *bitReaderShifted) fill() { return } if b.off > 4 { - v := b.in[b.off-4:] - v = v[:4] + v := b.in[b.off-4 : b.off] low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) b.value |= uint64(low) << ((b.bitsRead - 32) & 63) b.bitsRead -= 32 diff --git a/vendor/github.com/klauspost/compress/huff0/bitwriter.go b/vendor/github.com/klauspost/compress/huff0/bitwriter.go index ec71f7a..aed2347 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitwriter.go +++ b/vendor/github.com/klauspost/compress/huff0/bitwriter.go @@ -60,6 +60,22 @@ func (b *bitWriter) encTwoSymbols(ct cTable, av, bv byte) { b.nBits += encA.nBits + encB.nBits } +// encFourSymbols adds up to 32 bits from four symbols. +// It will not check if there is space for them, +// so the caller must ensure that b has been flushed recently. +func (b *bitWriter) encFourSymbols(encA, encB, encC, encD cTableEntry) { + bitsA := encA.nBits + bitsB := bitsA + encB.nBits + bitsC := bitsB + encC.nBits + bitsD := bitsC + encD.nBits + combined := uint64(encA.val) | + (uint64(encB.val) << (bitsA & 63)) | + (uint64(encC.val) << (bitsB & 63)) | + (uint64(encD.val) << (bitsC & 63)) + b.bitContainer |= combined << (b.nBits & 63) + b.nBits += bitsD +} + // flush32 will flush out, so there are at least 32 bits available for writing. func (b *bitWriter) flush32() { if b.nBits < 32 { diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index 4d14542..4ee4fa1 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -248,8 +248,7 @@ func (s *Scratch) compress1xDo(dst, src []byte) ([]byte, error) { tmp := src[n : n+4] // tmp should be len 4 bw.flush32() - bw.encTwoSymbols(cTable, tmp[3], tmp[2]) - bw.encTwoSymbols(cTable, tmp[1], tmp[0]) + bw.encFourSymbols(cTable[tmp[3]], cTable[tmp[2]], cTable[tmp[1]], cTable[tmp[0]]) } } else { for ; n >= 0; n -= 4 { @@ -365,29 +364,29 @@ func (s *Scratch) countSimple(in []byte) (max int, reuse bool) { m := uint32(0) if len(s.prevTable) > 0 { for i, v := range s.count[:] { + if v == 0 { + continue + } if v > m { m = v } - if v > 0 { - s.symbolLen = uint16(i) + 1 - if i >= len(s.prevTable) { - reuse = false - } else { - if s.prevTable[i].nBits == 0 { - reuse = false - } - } + s.symbolLen = uint16(i) + 1 + if i >= len(s.prevTable) { + reuse = false + } else if s.prevTable[i].nBits == 0 { + reuse = false } } return int(m), reuse } for i, v := range s.count[:] { + if v == 0 { + continue + } if v > m { m = v } - if v > 0 { - s.symbolLen = uint16(i) + 1 - } + s.symbolLen = uint16(i) + 1 } return int(m), false } @@ -484,34 +483,35 @@ func (s *Scratch) buildCTable() error { // Different from reference implementation. huffNode0 := s.nodes[0 : huffNodesLen+1] - for huffNode[nonNullRank].count == 0 { + for huffNode[nonNullRank].count() == 0 { nonNullRank-- } lowS := int16(nonNullRank) nodeRoot := nodeNb + lowS - 1 lowN := nodeNb - huffNode[nodeNb].count = huffNode[lowS].count + huffNode[lowS-1].count - huffNode[lowS].parent, huffNode[lowS-1].parent = uint16(nodeNb), uint16(nodeNb) + huffNode[nodeNb].setCount(huffNode[lowS].count() + huffNode[lowS-1].count()) + huffNode[lowS].setParent(nodeNb) + huffNode[lowS-1].setParent(nodeNb) nodeNb++ lowS -= 2 for n := nodeNb; n <= nodeRoot; n++ { - huffNode[n].count = 1 << 30 + huffNode[n].setCount(1 << 30) } // fake entry, strong barrier - huffNode0[0].count = 1 << 31 + huffNode0[0].setCount(1 << 31) // create parents for nodeNb <= nodeRoot { var n1, n2 int16 - if huffNode0[lowS+1].count < huffNode0[lowN+1].count { + if huffNode0[lowS+1].count() < huffNode0[lowN+1].count() { n1 = lowS lowS-- } else { n1 = lowN lowN++ } - if huffNode0[lowS+1].count < huffNode0[lowN+1].count { + if huffNode0[lowS+1].count() < huffNode0[lowN+1].count() { n2 = lowS lowS-- } else { @@ -519,18 +519,19 @@ func (s *Scratch) buildCTable() error { lowN++ } - huffNode[nodeNb].count = huffNode0[n1+1].count + huffNode0[n2+1].count - huffNode0[n1+1].parent, huffNode0[n2+1].parent = uint16(nodeNb), uint16(nodeNb) + huffNode[nodeNb].setCount(huffNode0[n1+1].count() + huffNode0[n2+1].count()) + huffNode0[n1+1].setParent(nodeNb) + huffNode0[n2+1].setParent(nodeNb) nodeNb++ } // distribute weights (unlimited tree height) - huffNode[nodeRoot].nbBits = 0 + huffNode[nodeRoot].setNbBits(0) for n := nodeRoot - 1; n >= startNode; n-- { - huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1 + huffNode[n].setNbBits(huffNode[huffNode[n].parent()].nbBits() + 1) } for n := uint16(0); n <= nonNullRank; n++ { - huffNode[n].nbBits = huffNode[huffNode[n].parent].nbBits + 1 + huffNode[n].setNbBits(huffNode[huffNode[n].parent()].nbBits() + 1) } s.actualTableLog = s.setMaxHeight(int(nonNullRank)) maxNbBits := s.actualTableLog @@ -542,7 +543,7 @@ func (s *Scratch) buildCTable() error { var nbPerRank [tableLogMax + 1]uint16 var valPerRank [16]uint16 for _, v := range huffNode[:nonNullRank+1] { - nbPerRank[v.nbBits]++ + nbPerRank[v.nbBits()]++ } // determine stating value per rank { @@ -557,7 +558,7 @@ func (s *Scratch) buildCTable() error { // push nbBits per symbol, symbol order for _, v := range huffNode[:nonNullRank+1] { - s.cTable[v.symbol].nBits = v.nbBits + s.cTable[v.symbol()].nBits = v.nbBits() } // assign value within rank, symbol order @@ -603,12 +604,12 @@ func (s *Scratch) huffSort() { pos := rank[r].current rank[r].current++ prev := nodes[(pos-1)&huffNodesMask] - for pos > rank[r].base && c > prev.count { + for pos > rank[r].base && c > prev.count() { nodes[pos&huffNodesMask] = prev pos-- prev = nodes[(pos-1)&huffNodesMask] } - nodes[pos&huffNodesMask] = nodeElt{count: c, symbol: byte(n)} + nodes[pos&huffNodesMask] = makeNodeElt(c, byte(n)) } } @@ -617,7 +618,7 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { huffNode := s.nodes[1 : huffNodesLen+1] //huffNode = huffNode[: huffNodesLen] - largestBits := huffNode[lastNonNull].nbBits + largestBits := huffNode[lastNonNull].nbBits() // early exit : no elt > maxNbBits if largestBits <= maxNbBits { @@ -627,14 +628,14 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { baseCost := int(1) << (largestBits - maxNbBits) n := uint32(lastNonNull) - for huffNode[n].nbBits > maxNbBits { - totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits)) - huffNode[n].nbBits = maxNbBits + for huffNode[n].nbBits() > maxNbBits { + totalCost += baseCost - (1 << (largestBits - huffNode[n].nbBits())) + huffNode[n].setNbBits(maxNbBits) n-- } // n stops at huffNode[n].nbBits <= maxNbBits - for huffNode[n].nbBits == maxNbBits { + for huffNode[n].nbBits() == maxNbBits { n-- } // n end at index of smallest symbol using < maxNbBits @@ -655,10 +656,10 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { { currentNbBits := maxNbBits for pos := int(n); pos >= 0; pos-- { - if huffNode[pos].nbBits >= currentNbBits { + if huffNode[pos].nbBits() >= currentNbBits { continue } - currentNbBits = huffNode[pos].nbBits // < maxNbBits + currentNbBits = huffNode[pos].nbBits() // < maxNbBits rankLast[maxNbBits-currentNbBits] = uint32(pos) } } @@ -675,8 +676,8 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { if lowPos == noSymbol { break } - highTotal := huffNode[highPos].count - lowTotal := 2 * huffNode[lowPos].count + highTotal := huffNode[highPos].count() + lowTotal := 2 * huffNode[lowPos].count() if highTotal <= lowTotal { break } @@ -692,13 +693,14 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { // this rank is no longer empty rankLast[nBitsToDecrease-1] = rankLast[nBitsToDecrease] } - huffNode[rankLast[nBitsToDecrease]].nbBits++ + huffNode[rankLast[nBitsToDecrease]].setNbBits(1 + + huffNode[rankLast[nBitsToDecrease]].nbBits()) if rankLast[nBitsToDecrease] == 0 { /* special case, reached largest symbol */ rankLast[nBitsToDecrease] = noSymbol } else { rankLast[nBitsToDecrease]-- - if huffNode[rankLast[nBitsToDecrease]].nbBits != maxNbBits-nBitsToDecrease { + if huffNode[rankLast[nBitsToDecrease]].nbBits() != maxNbBits-nBitsToDecrease { rankLast[nBitsToDecrease] = noSymbol /* this rank is now empty */ } } @@ -706,15 +708,15 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { for totalCost < 0 { /* Sometimes, cost correction overshoot */ if rankLast[1] == noSymbol { /* special case : no rank 1 symbol (using maxNbBits-1); let's create one from largest rank 0 (using maxNbBits) */ - for huffNode[n].nbBits == maxNbBits { + for huffNode[n].nbBits() == maxNbBits { n-- } - huffNode[n+1].nbBits-- + huffNode[n+1].setNbBits(huffNode[n+1].nbBits() - 1) rankLast[1] = n + 1 totalCost++ continue } - huffNode[rankLast[1]+1].nbBits-- + huffNode[rankLast[1]+1].setNbBits(huffNode[rankLast[1]+1].nbBits() - 1) rankLast[1]++ totalCost++ } @@ -722,9 +724,26 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { return maxNbBits } -type nodeElt struct { - count uint32 - parent uint16 - symbol byte - nbBits uint8 +// A nodeElt is the fields +// +// count uint32 +// parent uint16 +// symbol byte +// nbBits uint8 +// +// in some order, all squashed into an integer so that the compiler +// always loads and stores entire nodeElts instead of separate fields. +type nodeElt uint64 + +func makeNodeElt(count uint32, symbol byte) nodeElt { + return nodeElt(count) | nodeElt(symbol)<<48 } + +func (e *nodeElt) count() uint32 { return uint32(*e) } +func (e *nodeElt) parent() uint16 { return uint16(*e >> 32) } +func (e *nodeElt) symbol() byte { return byte(*e >> 48) } +func (e *nodeElt) nbBits() uint8 { return uint8(*e >> 56) } + +func (e *nodeElt) setCount(c uint32) { *e = (*e)&0xffffffff00000000 | nodeElt(c) } +func (e *nodeElt) setParent(p int16) { *e = (*e)&0xffff0000ffffffff | nodeElt(uint16(p))<<32 } +func (e *nodeElt) setNbBits(n uint8) { *e = (*e)&0x00ffffffffffffff | nodeElt(n)<<56 } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index 42a237e..3c0b398 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -61,7 +61,7 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { b, err := fse.Decompress(in[:iSize], s.fse) s.fse.Out = nil if err != nil { - return s, nil, err + return s, nil, fmt.Errorf("fse decompress returned: %w", err) } if len(b) > 255 { return s, nil, errors.New("corrupt input: output table too large") diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s index 8d2187a..c4c7ab2 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -4,360 +4,349 @@ // func decompress4x_main_loop_amd64(ctx *decompress4xContext) TEXT ·decompress4x_main_loop_amd64(SB), $0-8 - XORQ DX, DX - // Preload values MOVQ ctx+0(FP), AX MOVBQZX 8(AX), DI - MOVQ 16(AX), SI - MOVQ 48(AX), BX - MOVQ 24(AX), R9 - MOVQ 32(AX), R10 - MOVQ (AX), R11 + MOVQ 16(AX), BX + MOVQ 48(AX), SI + MOVQ 24(AX), R8 + MOVQ 32(AX), R9 + MOVQ (AX), R10 // Main loop main_loop: - MOVQ SI, R8 - CMPQ R8, BX + XORL DX, DX + CMPQ BX, SI SETGE DL // br0.fillFast32() - MOVQ 32(R11), R12 - MOVBQZX 40(R11), R13 - CMPQ R13, $0x20 + MOVQ 32(R10), R11 + MOVBQZX 40(R10), R12 + CMPQ R12, $0x20 JBE skip_fill0 - MOVQ 24(R11), AX - SUBQ $0x20, R13 + MOVQ 24(R10), AX + SUBQ $0x20, R12 SUBQ $0x04, AX - MOVQ (R11), R14 + MOVQ (R10), R13 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R14*1), R14 - MOVQ R13, CX - SHLQ CL, R14 - MOVQ AX, 24(R11) - ORQ R14, R12 + MOVL (AX)(R13*1), R13 + MOVQ R12, CX + SHLQ CL, R13 + MOVQ AX, 24(R10) + ORQ R13, R11 - // exhausted = exhausted || (br0.off < 4) - CMPQ AX, $0x04 - SETLT AL - ORB AL, DL + // exhausted += (br0.off < 4) + CMPQ AX, $0x04 + ADCB $+0, DL skip_fill0: // val0 := br0.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br0.advance(uint8(v0.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val1 := br0.peekTopBits(peekBits) MOVQ DI, CX - MOVQ R12, R14 - SHRQ CL, R14 + MOVQ R11, R13 + SHRQ CL, R13 // v1 := table[val1&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br0.advance(uint8(v1.entry)) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // these two writes get coalesced // out[id * dstEvery + 0] = uint8(v0.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (R8) + MOVW AX, (BX) // update the bitreader structure - MOVQ R12, 32(R11) - MOVB R13, 40(R11) - ADDQ R9, R8 + MOVQ R11, 32(R10) + MOVB R12, 40(R10) // br1.fillFast32() - MOVQ 80(R11), R12 - MOVBQZX 88(R11), R13 - CMPQ R13, $0x20 + MOVQ 80(R10), R11 + MOVBQZX 88(R10), R12 + CMPQ R12, $0x20 JBE skip_fill1 - MOVQ 72(R11), AX - SUBQ $0x20, R13 + MOVQ 72(R10), AX + SUBQ $0x20, R12 SUBQ $0x04, AX - MOVQ 48(R11), R14 + MOVQ 48(R10), R13 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R14*1), R14 - MOVQ R13, CX - SHLQ CL, R14 - MOVQ AX, 72(R11) - ORQ R14, R12 + MOVL (AX)(R13*1), R13 + MOVQ R12, CX + SHLQ CL, R13 + MOVQ AX, 72(R10) + ORQ R13, R11 - // exhausted = exhausted || (br1.off < 4) - CMPQ AX, $0x04 - SETLT AL - ORB AL, DL + // exhausted += (br1.off < 4) + CMPQ AX, $0x04 + ADCB $+0, DL skip_fill1: // val0 := br1.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br1.advance(uint8(v0.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val1 := br1.peekTopBits(peekBits) MOVQ DI, CX - MOVQ R12, R14 - SHRQ CL, R14 + MOVQ R11, R13 + SHRQ CL, R13 // v1 := table[val1&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br1.advance(uint8(v1.entry)) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // these two writes get coalesced // out[id * dstEvery + 0] = uint8(v0.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (R8) + MOVW AX, (BX)(R8*1) // update the bitreader structure - MOVQ R12, 80(R11) - MOVB R13, 88(R11) - ADDQ R9, R8 + MOVQ R11, 80(R10) + MOVB R12, 88(R10) // br2.fillFast32() - MOVQ 128(R11), R12 - MOVBQZX 136(R11), R13 - CMPQ R13, $0x20 + MOVQ 128(R10), R11 + MOVBQZX 136(R10), R12 + CMPQ R12, $0x20 JBE skip_fill2 - MOVQ 120(R11), AX - SUBQ $0x20, R13 + MOVQ 120(R10), AX + SUBQ $0x20, R12 SUBQ $0x04, AX - MOVQ 96(R11), R14 + MOVQ 96(R10), R13 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R14*1), R14 - MOVQ R13, CX - SHLQ CL, R14 - MOVQ AX, 120(R11) - ORQ R14, R12 + MOVL (AX)(R13*1), R13 + MOVQ R12, CX + SHLQ CL, R13 + MOVQ AX, 120(R10) + ORQ R13, R11 - // exhausted = exhausted || (br2.off < 4) - CMPQ AX, $0x04 - SETLT AL - ORB AL, DL + // exhausted += (br2.off < 4) + CMPQ AX, $0x04 + ADCB $+0, DL skip_fill2: // val0 := br2.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br2.advance(uint8(v0.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val1 := br2.peekTopBits(peekBits) MOVQ DI, CX - MOVQ R12, R14 - SHRQ CL, R14 + MOVQ R11, R13 + SHRQ CL, R13 // v1 := table[val1&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br2.advance(uint8(v1.entry)) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // these two writes get coalesced // out[id * dstEvery + 0] = uint8(v0.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (R8) + MOVW AX, (BX)(R8*2) // update the bitreader structure - MOVQ R12, 128(R11) - MOVB R13, 136(R11) - ADDQ R9, R8 + MOVQ R11, 128(R10) + MOVB R12, 136(R10) // br3.fillFast32() - MOVQ 176(R11), R12 - MOVBQZX 184(R11), R13 - CMPQ R13, $0x20 + MOVQ 176(R10), R11 + MOVBQZX 184(R10), R12 + CMPQ R12, $0x20 JBE skip_fill3 - MOVQ 168(R11), AX - SUBQ $0x20, R13 + MOVQ 168(R10), AX + SUBQ $0x20, R12 SUBQ $0x04, AX - MOVQ 144(R11), R14 + MOVQ 144(R10), R13 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (AX)(R14*1), R14 - MOVQ R13, CX - SHLQ CL, R14 - MOVQ AX, 168(R11) - ORQ R14, R12 + MOVL (AX)(R13*1), R13 + MOVQ R12, CX + SHLQ CL, R13 + MOVQ AX, 168(R10) + ORQ R13, R11 - // exhausted = exhausted || (br3.off < 4) - CMPQ AX, $0x04 - SETLT AL - ORB AL, DL + // exhausted += (br3.off < 4) + CMPQ AX, $0x04 + ADCB $+0, DL skip_fill3: // val0 := br3.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br3.advance(uint8(v0.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val1 := br3.peekTopBits(peekBits) MOVQ DI, CX - MOVQ R12, R14 - SHRQ CL, R14 + MOVQ R11, R13 + SHRQ CL, R13 // v1 := table[val1&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br3.advance(uint8(v1.entry)) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // these two writes get coalesced // out[id * dstEvery + 0] = uint8(v0.entry >> 8) // out[id * dstEvery + 1] = uint8(v1.entry >> 8) - MOVW AX, (R8) + LEAQ (R8)(R8*2), CX + MOVW AX, (BX)(CX*1) // update the bitreader structure - MOVQ R12, 176(R11) - MOVB R13, 184(R11) - ADDQ $0x02, SI + MOVQ R11, 176(R10) + MOVB R12, 184(R10) + ADDQ $0x02, BX TESTB DL, DL JZ main_loop MOVQ ctx+0(FP), AX - SUBQ 16(AX), SI - SHLQ $0x02, SI - MOVQ SI, 40(AX) + SUBQ 16(AX), BX + SHLQ $0x02, BX + MOVQ BX, 40(AX) RET // func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) TEXT ·decompress4x_8b_main_loop_amd64(SB), $0-8 - XORQ DX, DX - // Preload values MOVQ ctx+0(FP), CX MOVBQZX 8(CX), DI MOVQ 16(CX), BX MOVQ 48(CX), SI - MOVQ 24(CX), R9 - MOVQ 32(CX), R10 - MOVQ (CX), R11 + MOVQ 24(CX), R8 + MOVQ 32(CX), R9 + MOVQ (CX), R10 // Main loop main_loop: - MOVQ BX, R8 - CMPQ R8, SI + XORL DX, DX + CMPQ BX, SI SETGE DL // br0.fillFast32() - MOVQ 32(R11), R12 - MOVBQZX 40(R11), R13 - CMPQ R13, $0x20 + MOVQ 32(R10), R11 + MOVBQZX 40(R10), R12 + CMPQ R12, $0x20 JBE skip_fill0 - MOVQ 24(R11), R14 - SUBQ $0x20, R13 - SUBQ $0x04, R14 - MOVQ (R11), R15 + MOVQ 24(R10), R13 + SUBQ $0x20, R12 + SUBQ $0x04, R13 + MOVQ (R10), R14 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R14)(R15*1), R15 - MOVQ R13, CX - SHLQ CL, R15 - MOVQ R14, 24(R11) - ORQ R15, R12 + MOVL (R13)(R14*1), R14 + MOVQ R12, CX + SHLQ CL, R14 + MOVQ R13, 24(R10) + ORQ R14, R11 - // exhausted = exhausted || (br0.off < 4) - CMPQ R14, $0x04 - SETLT AL - ORB AL, DL + // exhausted += (br0.off < 4) + CMPQ R13, $0x04 + ADCB $+0, DL skip_fill0: // val0 := br0.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br0.advance(uint8(v0.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val1 := br0.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v1 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br0.advance(uint8(v1.entry) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 BSWAPL AX // val2 := br0.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v2 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br0.advance(uint8(v2.entry) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val3 := br0.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v3 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br0.advance(uint8(v3.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 BSWAPL AX // these four writes get coalesced @@ -365,88 +354,86 @@ skip_fill0: // out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 3] = uint8(v2.entry >> 8) // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (R8) + MOVL AX, (BX) // update the bitreader structure - MOVQ R12, 32(R11) - MOVB R13, 40(R11) - ADDQ R9, R8 + MOVQ R11, 32(R10) + MOVB R12, 40(R10) // br1.fillFast32() - MOVQ 80(R11), R12 - MOVBQZX 88(R11), R13 - CMPQ R13, $0x20 + MOVQ 80(R10), R11 + MOVBQZX 88(R10), R12 + CMPQ R12, $0x20 JBE skip_fill1 - MOVQ 72(R11), R14 - SUBQ $0x20, R13 - SUBQ $0x04, R14 - MOVQ 48(R11), R15 + MOVQ 72(R10), R13 + SUBQ $0x20, R12 + SUBQ $0x04, R13 + MOVQ 48(R10), R14 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R14)(R15*1), R15 - MOVQ R13, CX - SHLQ CL, R15 - MOVQ R14, 72(R11) - ORQ R15, R12 + MOVL (R13)(R14*1), R14 + MOVQ R12, CX + SHLQ CL, R14 + MOVQ R13, 72(R10) + ORQ R14, R11 - // exhausted = exhausted || (br1.off < 4) - CMPQ R14, $0x04 - SETLT AL - ORB AL, DL + // exhausted += (br1.off < 4) + CMPQ R13, $0x04 + ADCB $+0, DL skip_fill1: // val0 := br1.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br1.advance(uint8(v0.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val1 := br1.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v1 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br1.advance(uint8(v1.entry) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 BSWAPL AX // val2 := br1.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v2 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br1.advance(uint8(v2.entry) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val3 := br1.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v3 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br1.advance(uint8(v3.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 BSWAPL AX // these four writes get coalesced @@ -454,88 +441,86 @@ skip_fill1: // out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 3] = uint8(v2.entry >> 8) // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (R8) + MOVL AX, (BX)(R8*1) // update the bitreader structure - MOVQ R12, 80(R11) - MOVB R13, 88(R11) - ADDQ R9, R8 + MOVQ R11, 80(R10) + MOVB R12, 88(R10) // br2.fillFast32() - MOVQ 128(R11), R12 - MOVBQZX 136(R11), R13 - CMPQ R13, $0x20 + MOVQ 128(R10), R11 + MOVBQZX 136(R10), R12 + CMPQ R12, $0x20 JBE skip_fill2 - MOVQ 120(R11), R14 - SUBQ $0x20, R13 - SUBQ $0x04, R14 - MOVQ 96(R11), R15 + MOVQ 120(R10), R13 + SUBQ $0x20, R12 + SUBQ $0x04, R13 + MOVQ 96(R10), R14 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R14)(R15*1), R15 - MOVQ R13, CX - SHLQ CL, R15 - MOVQ R14, 120(R11) - ORQ R15, R12 + MOVL (R13)(R14*1), R14 + MOVQ R12, CX + SHLQ CL, R14 + MOVQ R13, 120(R10) + ORQ R14, R11 - // exhausted = exhausted || (br2.off < 4) - CMPQ R14, $0x04 - SETLT AL - ORB AL, DL + // exhausted += (br2.off < 4) + CMPQ R13, $0x04 + ADCB $+0, DL skip_fill2: // val0 := br2.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br2.advance(uint8(v0.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val1 := br2.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v1 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br2.advance(uint8(v1.entry) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 BSWAPL AX // val2 := br2.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v2 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br2.advance(uint8(v2.entry) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val3 := br2.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v3 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br2.advance(uint8(v3.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 BSWAPL AX // these four writes get coalesced @@ -543,88 +528,86 @@ skip_fill2: // out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 3] = uint8(v2.entry >> 8) // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (R8) + MOVL AX, (BX)(R8*2) // update the bitreader structure - MOVQ R12, 128(R11) - MOVB R13, 136(R11) - ADDQ R9, R8 + MOVQ R11, 128(R10) + MOVB R12, 136(R10) // br3.fillFast32() - MOVQ 176(R11), R12 - MOVBQZX 184(R11), R13 - CMPQ R13, $0x20 + MOVQ 176(R10), R11 + MOVBQZX 184(R10), R12 + CMPQ R12, $0x20 JBE skip_fill3 - MOVQ 168(R11), R14 - SUBQ $0x20, R13 - SUBQ $0x04, R14 - MOVQ 144(R11), R15 + MOVQ 168(R10), R13 + SUBQ $0x20, R12 + SUBQ $0x04, R13 + MOVQ 144(R10), R14 // b.value |= uint64(low) << (b.bitsRead & 63) - MOVL (R14)(R15*1), R15 - MOVQ R13, CX - SHLQ CL, R15 - MOVQ R14, 168(R11) - ORQ R15, R12 + MOVL (R13)(R14*1), R14 + MOVQ R12, CX + SHLQ CL, R14 + MOVQ R13, 168(R10) + ORQ R14, R11 - // exhausted = exhausted || (br3.off < 4) - CMPQ R14, $0x04 - SETLT AL - ORB AL, DL + // exhausted += (br3.off < 4) + CMPQ R13, $0x04 + ADCB $+0, DL skip_fill3: // val0 := br3.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v0 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br3.advance(uint8(v0.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val1 := br3.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v1 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br3.advance(uint8(v1.entry) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 BSWAPL AX // val2 := br3.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v2 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br3.advance(uint8(v2.entry) MOVB CH, AH - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 // val3 := br3.peekTopBits(peekBits) - MOVQ R12, R14 + MOVQ R11, R13 MOVQ DI, CX - SHRQ CL, R14 + SHRQ CL, R13 // v3 := table[val0&mask] - MOVW (R10)(R14*2), CX + MOVW (R9)(R13*2), CX // br3.advance(uint8(v3.entry) MOVB CH, AL - SHLQ CL, R12 - ADDB CL, R13 + SHLQ CL, R11 + ADDB CL, R12 BSWAPL AX // these four writes get coalesced @@ -632,11 +615,12 @@ skip_fill3: // out[id * dstEvery + 1] = uint8(v1.entry >> 8) // out[id * dstEvery + 3] = uint8(v2.entry >> 8) // out[id * dstEvery + 4] = uint8(v3.entry >> 8) - MOVL AX, (R8) + LEAQ (R8)(R8*2), CX + MOVL AX, (BX)(CX*1) // update the bitreader structure - MOVQ R12, 176(R11) - MOVB R13, 184(R11) + MOVQ R11, 176(R10) + MOVB R12, 184(R10) ADDQ $0x04, BX TESTB DL, DL JZ main_loop @@ -652,7 +636,7 @@ TEXT ·decompress1x_main_loop_amd64(SB), $0-8 MOVQ 16(CX), DX MOVQ 24(CX), BX CMPQ BX, $0x04 - JB error_max_decoded_size_exeeded + JB error_max_decoded_size_exceeded LEAQ (DX)(BX*1), BX MOVQ (CX), SI MOVQ (SI), R8 @@ -667,7 +651,7 @@ main_loop: // Check if we have room for 4 bytes in the output buffer LEAQ 4(DX), CX CMPQ CX, BX - JGE error_max_decoded_size_exeeded + JGE error_max_decoded_size_exceeded // Decode 4 values CMPQ R11, $0x20 @@ -744,7 +728,7 @@ loop_condition: RET // Report error -error_max_decoded_size_exeeded: +error_max_decoded_size_exceeded: MOVQ ctx+0(FP), AX MOVQ $-1, CX MOVQ CX, 40(AX) @@ -757,7 +741,7 @@ TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 MOVQ 16(CX), DX MOVQ 24(CX), BX CMPQ BX, $0x04 - JB error_max_decoded_size_exeeded + JB error_max_decoded_size_exceeded LEAQ (DX)(BX*1), BX MOVQ (CX), SI MOVQ (SI), R8 @@ -772,7 +756,7 @@ main_loop: // Check if we have room for 4 bytes in the output buffer LEAQ 4(DX), CX CMPQ CX, BX - JGE error_max_decoded_size_exeeded + JGE error_max_decoded_size_exceeded // Decode 4 values CMPQ R11, $0x20 @@ -839,7 +823,7 @@ loop_condition: RET // Report error -error_max_decoded_size_exeeded: +error_max_decoded_size_exceeded: MOVQ ctx+0(FP), AX MOVQ $-1, CX MOVQ CX, 40(AX) diff --git a/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go b/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go index 298c4f8..05db94d 100644 --- a/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go +++ b/vendor/github.com/klauspost/compress/internal/snapref/encode_other.go @@ -103,6 +103,28 @@ func hash(u, shift uint32) uint32 { return (u * 0x1e35a7bd) >> shift } +// EncodeBlockInto exposes encodeBlock but checks dst size. +func EncodeBlockInto(dst, src []byte) (d int) { + if MaxEncodedLen(len(src)) > len(dst) { + return 0 + } + + // encodeBlock breaks on too big blocks, so split. + for len(src) > 0 { + p := src + src = nil + if len(p) > maxBlockSize { + p, src = p[:maxBlockSize], p[maxBlockSize:] + } + if len(p) < minNonLiteralBlockSize { + d += emitLiteral(dst[d:], p) + } else { + d += encodeBlock(dst[d:], p) + } + } + return d +} + // encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It // assumes that the varint-encoded length of the decompressed bytes has already // been written. diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index da81471..5f272d8 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -9,6 +9,7 @@ import ( "encoding/binary" "errors" "fmt" + "hash/crc32" "io" "os" "path/filepath" @@ -82,8 +83,9 @@ type blockDec struct { err error - // Check against this crc - checkCRC []byte + // Check against this crc, if hasCRC is true. + checkCRC uint32 + hasCRC bool // Frame to use for singlethreaded decoding. // Should not be used by the decoder itself since parent may be another frame. @@ -191,16 +193,14 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { } // Read block data. - if cap(b.dataStorage) < cSize { + if _, ok := br.(*byteBuf); !ok && cap(b.dataStorage) < cSize { + // byteBuf doesn't need a destination buffer. if b.lowMem || cSize > maxCompressedBlockSize { b.dataStorage = make([]byte, 0, cSize+compressedBlockOverAlloc) } else { b.dataStorage = make([]byte, 0, maxCompressedBlockSizeAlloc) } } - if cap(b.dst) <= maxSize { - b.dst = make([]byte, 0, maxSize+1) - } b.data, err = br.readBig(cSize, b.dataStorage) if err != nil { if debugDecoder { @@ -209,6 +209,9 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { } return err } + if cap(b.dst) <= maxSize { + b.dst = make([]byte, 0, maxSize+1) + } return nil } @@ -440,6 +443,9 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err } } var err error + if debugDecoder { + println("huff table input:", len(literals), "CRC:", crc32.ChecksumIEEE(literals)) + } huff, literals, err = huff0.ReadTable(literals, huff) if err != nil { println("reading huffman table:", err) diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go index 12e8f6f..fd4a36f 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockenc.go +++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go @@ -473,7 +473,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { return b.encodeLits(b.literals, rawAllLits) } // We want some difference to at least account for the headers. - saved := b.size - len(b.literals) - (b.size >> 5) + saved := b.size - len(b.literals) - (b.size >> 6) if saved < 16 { if org == nil { return errIncompressible @@ -779,10 +779,13 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { } b.output = wr.out + // Maybe even add a bigger margin. if len(b.output)-3-bhOffset >= b.size { - // Maybe even add a bigger margin. + // Discard and encode as raw block. + b.output = b.encodeRawTo(b.output[:bhOffset], org) + b.popOffsets() b.litEnc.Reuse = huff0.ReusePolicyNone - return errIncompressible + return nil } // Size is output minus block header. diff --git a/vendor/github.com/klauspost/compress/zstd/bytebuf.go b/vendor/github.com/klauspost/compress/zstd/bytebuf.go index 176788f..55a3885 100644 --- a/vendor/github.com/klauspost/compress/zstd/bytebuf.go +++ b/vendor/github.com/klauspost/compress/zstd/bytebuf.go @@ -54,7 +54,7 @@ func (b *byteBuf) readBig(n int, dst []byte) ([]byte, error) { func (b *byteBuf) readByte() (byte, error) { bb := *b if len(bb) < 1 { - return 0, nil + return 0, io.ErrUnexpectedEOF } r := bb[0] *b = bb[1:] @@ -109,7 +109,7 @@ func (r *readerWrapper) readBig(n int, dst []byte) ([]byte, error) { } func (r *readerWrapper) readByte() (byte, error) { - n2, err := r.r.Read(r.tmp[:1]) + n2, err := io.ReadFull(r.r, r.tmp[:1]) if err != nil { if err == io.EOF { err = io.ErrUnexpectedEOF diff --git a/vendor/github.com/klauspost/compress/zstd/decodeheader.go b/vendor/github.com/klauspost/compress/zstd/decodeheader.go index 5022e71..f6a2409 100644 --- a/vendor/github.com/klauspost/compress/zstd/decodeheader.go +++ b/vendor/github.com/klauspost/compress/zstd/decodeheader.go @@ -4,7 +4,6 @@ package zstd import ( - "bytes" "encoding/binary" "errors" "io" @@ -102,8 +101,8 @@ func (h *Header) Decode(in []byte) error { } h.HeaderSize += 4 b, in := in[:4], in[4:] - if !bytes.Equal(b, frameMagic) { - if !bytes.Equal(b[1:4], skippableFrameMagic) || b[0]&0xf0 != 0x50 { + if string(b) != frameMagic { + if string(b[1:4]) != skippableFrameMagic || b[0]&0xf0 != 0x50 { return ErrMagicMismatch } if len(in) < 4 { @@ -153,7 +152,7 @@ func (h *Header) Decode(in []byte) error { } b, in = in[:size], in[size:] h.HeaderSize += int(size) - switch size { + switch len(b) { case 1: h.DictionaryID = uint32(b[0]) case 2: @@ -183,7 +182,7 @@ func (h *Header) Decode(in []byte) error { } b, in = in[:fcsSize], in[fcsSize:] h.HeaderSize += int(fcsSize) - switch fcsSize { + switch len(b) { case 1: h.FrameContentSize = uint64(b[0]) case 2: diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 74d645f..f04aaa2 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -5,7 +5,6 @@ package zstd import ( - "bytes" "context" "encoding/binary" "io" @@ -41,8 +40,7 @@ type Decoder struct { frame *frameDec // Custom dictionaries. - // Always uses copies. - dicts map[uint32]dict + dicts map[uint32]*dict // streamWg is the waitgroup for all streams streamWg sync.WaitGroup @@ -104,7 +102,7 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { } // Transfer option dicts. - d.dicts = make(map[uint32]dict, len(d.o.dicts)) + d.dicts = make(map[uint32]*dict, len(d.o.dicts)) for _, dc := range d.o.dicts { d.dicts[dc.id] = dc } @@ -342,15 +340,8 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { } return dst, err } - if frame.DictionaryID != nil { - dict, ok := d.dicts[*frame.DictionaryID] - if !ok { - return nil, ErrUnknownDictionary - } - if debugDecoder { - println("setting dict", frame.DictionaryID) - } - frame.history.setDict(&dict) + if err = d.setDict(frame); err != nil { + return nil, err } if frame.WindowSize > d.o.maxWindowSize { if debugDecoder { @@ -459,26 +450,23 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) { println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp) } - if !d.o.ignoreChecksum && len(next.b) > 0 { - n, err := d.current.crc.Write(next.b) - if err == nil { - if n != len(next.b) { - d.current.err = io.ErrShortWrite - } - } + if d.o.ignoreChecksum { + return true } - if next.err == nil && next.d != nil && len(next.d.checkCRC) != 0 { - got := d.current.crc.Sum64() - var tmp [4]byte - binary.LittleEndian.PutUint32(tmp[:], uint32(got)) - if !d.o.ignoreChecksum && !bytes.Equal(tmp[:], next.d.checkCRC) { + + if len(next.b) > 0 { + d.current.crc.Write(next.b) + } + if next.err == nil && next.d != nil && next.d.hasCRC { + got := uint32(d.current.crc.Sum64()) + if got != next.d.checkCRC { if debugDecoder { - println("CRC Check Failed:", tmp[:], " (got) !=", next.d.checkCRC, "(on stream)") + printf("CRC Check Failed: %08x (got) != %08x (on stream)\n", got, next.d.checkCRC) } d.current.err = ErrCRCMismatch } else { if debugDecoder { - println("CRC ok", tmp[:]) + printf("CRC ok %08x\n", got) } } } @@ -494,18 +482,12 @@ func (d *Decoder) nextBlockSync() (ok bool) { if !d.syncStream.inFrame { d.frame.history.reset() d.current.err = d.frame.reset(&d.syncStream.br) + if d.current.err == nil { + d.current.err = d.setDict(d.frame) + } if d.current.err != nil { return false } - if d.frame.DictionaryID != nil { - dict, ok := d.dicts[*d.frame.DictionaryID] - if !ok { - d.current.err = ErrUnknownDictionary - return false - } else { - d.frame.history.setDict(&dict) - } - } if d.frame.WindowSize > d.o.maxDecodedSize || d.frame.WindowSize > d.o.maxWindowSize { d.current.err = ErrDecoderSizeExceeded return false @@ -864,13 +846,8 @@ decodeStream: if debugDecoder && err != nil { println("Frame decoder returned", err) } - if err == nil && frame.DictionaryID != nil { - dict, ok := d.dicts[*frame.DictionaryID] - if !ok { - err = ErrUnknownDictionary - } else { - frame.history.setDict(&dict) - } + if err == nil { + err = d.setDict(frame) } if err == nil && d.frame.WindowSize > d.o.maxWindowSize { if debugDecoder { @@ -918,18 +895,22 @@ decodeStream: println("next block returned error:", err) } dec.err = err - dec.checkCRC = nil + dec.hasCRC = false if dec.Last && frame.HasCheckSum && err == nil { crc, err := frame.rawInput.readSmall(4) - if err != nil { + if len(crc) < 4 { + if err == nil { + err = io.ErrUnexpectedEOF + + } println("CRC missing?", err) dec.err = err - } - var tmp [4]byte - copy(tmp[:], crc) - dec.checkCRC = tmp[:] - if debugDecoder { - println("found crc to check:", dec.checkCRC) + } else { + dec.checkCRC = binary.LittleEndian.Uint32(crc) + dec.hasCRC = true + if debugDecoder { + printf("found crc to check: %08x\n", dec.checkCRC) + } } } err = dec.err @@ -948,3 +929,20 @@ decodeStream: hist.reset() d.frame.history.b = frameHistCache } + +func (d *Decoder) setDict(frame *frameDec) (err error) { + dict, ok := d.dicts[frame.DictionaryID] + if ok { + if debugDecoder { + println("setting dict", frame.DictionaryID) + } + frame.history.setDict(dict) + } else if frame.DictionaryID != 0 { + // A zero or missing dictionary id is ambiguous: + // either dictionary zero, or no dictionary. In particular, + // zstd --patch-from uses this id for the source file, + // so only return an error if the dictionary id is not zero. + err = ErrUnknownDictionary + } + return err +} diff --git a/vendor/github.com/klauspost/compress/zstd/decoder_options.go b/vendor/github.com/klauspost/compress/zstd/decoder_options.go index f42448e..07a90dd 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder_options.go @@ -6,6 +6,8 @@ package zstd import ( "errors" + "fmt" + "math/bits" "runtime" ) @@ -18,7 +20,7 @@ type decoderOptions struct { concurrent int maxDecodedSize uint64 maxWindowSize uint64 - dicts []dict + dicts []*dict ignoreChecksum bool limitToCap bool decodeBufsBelow int @@ -85,7 +87,13 @@ func WithDecoderMaxMemory(n uint64) DOption { } // WithDecoderDicts allows to register one or more dictionaries for the decoder. -// If several dictionaries with the same ID is provided the last one will be used. +// +// Each slice in dict must be in the [dictionary format] produced by +// "zstd --train" from the Zstandard reference implementation. +// +// If several dictionaries with the same ID are provided, the last one will be used. +// +// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format func WithDecoderDicts(dicts ...[]byte) DOption { return func(o *decoderOptions) error { for _, b := range dicts { @@ -93,12 +101,24 @@ func WithDecoderDicts(dicts ...[]byte) DOption { if err != nil { return err } - o.dicts = append(o.dicts, *d) + o.dicts = append(o.dicts, d) } return nil } } +// WithEncoderDictRaw registers a dictionary that may be used by the decoder. +// The slice content can be arbitrary data. +func WithDecoderDictRaw(id uint32, content []byte) DOption { + return func(o *decoderOptions) error { + if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { + return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content)) + } + o.dicts = append(o.dicts, &dict{id: id, content: content, offsets: [3]int{1, 4, 8}}) + return nil + } +} + // WithDecoderMaxWindow allows to set a maximum window size for decodes. // This allows rejecting packets that will cause big memory usage. // The Decoder will likely allocate more memory based on the WithDecoderLowmem setting. diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go index a36ae83..ca09514 100644 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -1,7 +1,6 @@ package zstd import ( - "bytes" "encoding/binary" "errors" "fmt" @@ -20,7 +19,10 @@ type dict struct { content []byte } -var dictMagic = [4]byte{0x37, 0xa4, 0x30, 0xec} +const dictMagic = "\x37\xa4\x30\xec" + +// Maximum dictionary size for the reference implementation (1.5.3) is 2 GiB. +const dictMaxLength = 1 << 31 // ID returns the dictionary id or 0 if d is nil. func (d *dict) ID() uint32 { @@ -30,14 +32,38 @@ func (d *dict) ID() uint32 { return d.id } -// DictContentSize returns the dictionary content size or 0 if d is nil. -func (d *dict) DictContentSize() int { +// ContentSize returns the dictionary content size or 0 if d is nil. +func (d *dict) ContentSize() int { if d == nil { return 0 } return len(d.content) } +// Content returns the dictionary content. +func (d *dict) Content() []byte { + if d == nil { + return nil + } + return d.content +} + +// Offsets returns the initial offsets. +func (d *dict) Offsets() [3]int { + if d == nil { + return [3]int{} + } + return d.offsets +} + +// LitEncoder returns the literal encoder. +func (d *dict) LitEncoder() *huff0.Scratch { + if d == nil { + return nil + } + return d.litEnc +} + // Load a dictionary as described in // https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format func loadDict(b []byte) (*dict, error) { @@ -50,7 +76,7 @@ func loadDict(b []byte) (*dict, error) { ofDec: sequenceDec{fse: &fseDecoder{}}, mlDec: sequenceDec{fse: &fseDecoder{}}, } - if !bytes.Equal(b[:4], dictMagic[:]) { + if string(b[:4]) != dictMagic { return nil, ErrMagicMismatch } d.id = binary.LittleEndian.Uint32(b[4:8]) @@ -62,7 +88,7 @@ func loadDict(b []byte) (*dict, error) { var err error d.litEnc, b, err = huff0.ReadTable(b[8:], nil) if err != nil { - return nil, err + return nil, fmt.Errorf("loading literal table: %w", err) } d.litEnc.Reuse = huff0.ReusePolicyMust @@ -120,3 +146,16 @@ func loadDict(b []byte) (*dict, error) { return &d, nil } + +// InspectDictionary loads a zstd dictionary and provides functions to inspect the content. +func InspectDictionary(b []byte) (interface { + ID() uint32 + ContentSize() int + Content() []byte + Offsets() [3]int + LitEncoder() *huff0.Scratch +}, error) { + initPredefined() + d, err := loadDict(b) + return d, err +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go index 15ae8ee..e008b99 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -16,6 +16,7 @@ type fastBase struct { cur int32 // maximum offset. Should be at least 2x block size. maxMatchOff int32 + bufferReset int32 hist []byte crc *xxhash.Digest tmp [8]byte @@ -56,8 +57,8 @@ func (e *fastBase) Block() *blockEnc { } func (e *fastBase) addBlock(src []byte) int32 { - if debugAsserts && e.cur > bufferReset { - panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, bufferReset)) + if debugAsserts && e.cur > e.bufferReset { + panic(fmt.Sprintf("ecur (%d) > buffer reset (%d)", e.cur, e.bufferReset)) } // check if we have space already if len(e.hist)+len(src) > cap(e.hist) { @@ -126,24 +127,7 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 { panic(fmt.Sprintf("len(src)-s (%d) > maxCompressedBlockSize (%d)", len(src)-int(s), maxCompressedBlockSize)) } } - a := src[s:] - b := src[t:] - b = b[:len(a)] - end := int32((len(a) >> 3) << 3) - for i := int32(0); i < end; i += 8 { - if diff := load6432(a, i) ^ load6432(b, i); diff != 0 { - return i + int32(bits.TrailingZeros64(diff)>>3) - } - } - - a = a[end:] - b = b[end:] - for i := range a { - if a[i] != b[i] { - return int32(i) + end - } - } - return int32(len(a)) + end + return int32(matchLen(src[s:], src[t:])) } // Reset the encoding table. @@ -165,13 +149,13 @@ func (e *fastBase) resetBase(d *dict, singleBlock bool) { if singleBlock { e.lowMem = true } - e.ensureHist(d.DictContentSize() + maxCompressedBlockSize) + e.ensureHist(d.ContentSize() + maxCompressedBlockSize) e.lowMem = low } // We offset current position so everything will be out of reach. // If above reset line, history will be purged. - if e.cur < bufferReset { + if e.cur < e.bufferReset { e.cur += e.maxMatchOff + int32(len(e.hist)) } e.hist = e.hist[:0] diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go index dbbb88d..9819d41 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -32,10 +32,9 @@ type match struct { length int32 rep int32 est int32 - _ [12]byte // Aligned size to cache line: 4+4+4+4+4 bytes + 12 bytes padding = 32 bytes } -const highScore = 25000 +const highScore = maxMatchLen * 8 // estBits will estimate output bits from predefined tables. func (m *match) estBits(bitsPerByte int32) { @@ -85,14 +84,10 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = prevEntry{} - } - for i := range e.longTable[:] { - e.longTable[i] = prevEntry{} - } + e.table = [bestShortTableSize]prevEntry{} + e.longTable = [bestLongTableSize]prevEntry{} e.cur = e.maxMatchOff break } @@ -164,7 +159,6 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) { // nextEmit is where in src the next emitLiteral should start from. nextEmit := s - cv := load6432(src, s) // Relative offsets offset1 := int32(blk.recentOffsets[0]) @@ -178,7 +172,6 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) { blk.literals = append(blk.literals, src[nextEmit:until]...) s.litLen = uint32(until - nextEmit) } - _ = addLiterals if debugEncoder { println("recent offsets:", blk.recentOffsets) @@ -193,49 +186,96 @@ encodeLoop: panic("offset0 was 0") } - bestOf := func(a, b match) match { - if a.est+(a.s-b.s)*bitsPerByte>>10 < b.est+(b.s-a.s)*bitsPerByte>>10 { - return a - } - return b - } - const goodEnough = 100 + const goodEnough = 250 + + cv := load6432(src, s) nextHashL := hashLen(cv, bestLongTableBits, bestLongLen) nextHashS := hashLen(cv, bestShortTableBits, bestShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] - matchAt := func(offset int32, s int32, first uint32, rep int32) match { + // Set m to a match at offset if it looks like that will improve compression. + improve := func(m *match, offset int32, s int32, first uint32, rep int32) { if s-offset >= e.maxMatchOff || load3232(src, offset) != first { - return match{s: s, est: highScore} + return } if debugAsserts { + if offset <= 0 { + panic(offset) + } if !bytes.Equal(src[s:s+4], src[offset:offset+4]) { panic(fmt.Sprintf("first match mismatch: %v != %v, first: %08x", src[s:s+4], src[offset:offset+4], first)) } } - m := match{offset: offset, s: s, length: 4 + e.matchlen(s+4, offset+4, src), rep: rep} - m.estBits(bitsPerByte) - return m + // Try to quick reject if we already have a long match. + if m.length > 16 { + left := len(src) - int(m.s+m.length) + // If we are too close to the end, keep as is. + if left <= 0 { + return + } + checkLen := m.length - (s - m.s) - 8 + if left > 2 && checkLen > 4 { + // Check 4 bytes, 4 bytes from the end of the current match. + a := load3232(src, offset+checkLen) + b := load3232(src, s+checkLen) + if a != b { + return + } + } + } + l := 4 + e.matchlen(s+4, offset+4, src) + if rep < 0 { + // Extend candidate match backwards as far as possible. + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for offset > tMin && s > nextEmit && src[offset-1] == src[s-1] && l < maxMatchLength { + s-- + offset-- + l++ + } + } + + cand := match{offset: offset, s: s, length: l, rep: rep} + cand.estBits(bitsPerByte) + if m.est >= highScore || cand.est-m.est+(cand.s-m.s)*bitsPerByte>>10 < 0 { + *m = cand + } } - best := bestOf(matchAt(candidateL.offset-e.cur, s, uint32(cv), -1), matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)) - best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)) - best = bestOf(best, matchAt(candidateS.prev-e.cur, s, uint32(cv), -1)) + best := match{s: s, est: highScore} + improve(&best, candidateL.offset-e.cur, s, uint32(cv), -1) + improve(&best, candidateL.prev-e.cur, s, uint32(cv), -1) + improve(&best, candidateS.offset-e.cur, s, uint32(cv), -1) + improve(&best, candidateS.prev-e.cur, s, uint32(cv), -1) if canRepeat && best.length < goodEnough { - cv32 := uint32(cv >> 8) - spp := s + 1 - best = bestOf(best, matchAt(spp-offset1, spp, cv32, 1)) - best = bestOf(best, matchAt(spp-offset2, spp, cv32, 2)) - best = bestOf(best, matchAt(spp-offset3, spp, cv32, 3)) - if best.length > 0 { - cv32 = uint32(cv >> 24) - spp += 2 - best = bestOf(best, matchAt(spp-offset1, spp, cv32, 1)) - best = bestOf(best, matchAt(spp-offset2, spp, cv32, 2)) - best = bestOf(best, matchAt(spp-offset3, spp, cv32, 3)) + if s == nextEmit { + // Check repeats straight after a match. + improve(&best, s-offset2, s, uint32(cv), 1|4) + improve(&best, s-offset3, s, uint32(cv), 2|4) + if offset1 > 1 { + improve(&best, s-(offset1-1), s, uint32(cv), 3|4) + } + } + + // If either no match or a non-repeat match, check at + 1 + if best.rep <= 0 { + cv32 := uint32(cv >> 8) + spp := s + 1 + improve(&best, spp-offset1, spp, cv32, 1) + improve(&best, spp-offset2, spp, cv32, 2) + improve(&best, spp-offset3, spp, cv32, 3) + if best.rep < 0 { + cv32 = uint32(cv >> 24) + spp += 2 + improve(&best, spp-offset1, spp, cv32, 1) + improve(&best, spp-offset2, spp, cv32, 2) + improve(&best, spp-offset3, spp, cv32, 3) + } } } // Load next and check... @@ -250,40 +290,45 @@ encodeLoop: if s >= sLimit { break encodeLoop } - cv = load6432(src, s) continue } - s++ candidateS = e.table[hashLen(cv>>8, bestShortTableBits, bestShortLen)] - cv = load6432(src, s) - cv2 := load6432(src, s+1) + cv = load6432(src, s+1) + cv2 := load6432(src, s+2) candidateL = e.longTable[hashLen(cv, bestLongTableBits, bestLongLen)] candidateL2 := e.longTable[hashLen(cv2, bestLongTableBits, bestLongLen)] // Short at s+1 - best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)) + improve(&best, candidateS.offset-e.cur, s+1, uint32(cv), -1) // Long at s+1, s+2 - best = bestOf(best, matchAt(candidateL.offset-e.cur, s, uint32(cv), -1)) - best = bestOf(best, matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)) - best = bestOf(best, matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1)) - best = bestOf(best, matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1)) + improve(&best, candidateL.offset-e.cur, s+1, uint32(cv), -1) + improve(&best, candidateL.prev-e.cur, s+1, uint32(cv), -1) + improve(&best, candidateL2.offset-e.cur, s+2, uint32(cv2), -1) + improve(&best, candidateL2.prev-e.cur, s+2, uint32(cv2), -1) if false { // Short at s+3. // Too often worse... - best = bestOf(best, matchAt(e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+2, uint32(cv2>>8), -1)) + improve(&best, e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+3, uint32(cv2>>8), -1) } - // See if we can find a better match by checking where the current best ends. - // Use that offset to see if we can find a better full match. - if sAt := best.s + best.length; sAt < sLimit { - nextHashL := hashLen(load6432(src, sAt), bestLongTableBits, bestLongLen) - candidateEnd := e.longTable[nextHashL] - if pos := candidateEnd.offset - e.cur - best.length; pos >= 0 { - bestEnd := bestOf(best, matchAt(pos, best.s, load3232(src, best.s), -1)) - if pos := candidateEnd.prev - e.cur - best.length; pos >= 0 { - bestEnd = bestOf(bestEnd, matchAt(pos, best.s, load3232(src, best.s), -1)) + + // Start check at a fixed offset to allow for a few mismatches. + // For this compression level 2 yields the best results. + // We cannot do this if we have already indexed this position. + const skipBeginning = 2 + if best.s > s-skipBeginning { + // See if we can find a better match by checking where the current best ends. + // Use that offset to see if we can find a better full match. + if sAt := best.s + best.length; sAt < sLimit { + nextHashL := hashLen(load6432(src, sAt), bestLongTableBits, bestLongLen) + candidateEnd := e.longTable[nextHashL] + + if off := candidateEnd.offset - e.cur - best.length + skipBeginning; off >= 0 { + improve(&best, off, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1) + if off := candidateEnd.prev - e.cur - best.length + skipBeginning; off >= 0 { + improve(&best, off, best.s+skipBeginning, load3232(src, best.s+skipBeginning), -1) + } } - best = bestEnd } } } @@ -296,51 +341,34 @@ encodeLoop: // We have a match, we can store the forward value if best.rep > 0 { - s = best.s var seq seq seq.matchLen = uint32(best.length - zstdMinMatch) - - // We might be able to match backwards. - // Extend as long as we can. - start := best.s - // We end the search early, so we don't risk 0 literals - // and have to do special offset treatment. - startLimit := nextEmit + 1 - - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 + if debugAsserts && s <= nextEmit { + panic("s <= nextEmit") } - repIndex := best.offset - for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { - repIndex-- - start-- - seq.matchLen++ - } - addLiterals(&seq, start) + addLiterals(&seq, best.s) - // rep 0 - seq.offset = uint32(best.rep) + // Repeat. If bit 4 is set, this is a non-lit repeat. + seq.offset = uint32(best.rep & 3) if debugSequences { println("repeat sequence", seq, "next s:", s) } blk.sequences = append(blk.sequences, seq) - // Index match start+1 (long) -> s - 1 - index0 := s + // Index old s + 1 -> s - 1 + index0 := s + 1 s = best.s + best.length nextEmit = s if s >= sLimit { if debugEncoder { println("repeat ended", s, best.length) - } break encodeLoop } // Index skipped... off := index0 + e.cur - for index0 < s-1 { + for index0 < s { cv0 := load6432(src, index0) h0 := hashLen(cv0, bestLongTableBits, bestLongLen) h1 := hashLen(cv0, bestShortTableBits, bestShortLen) @@ -350,17 +378,19 @@ encodeLoop: index0++ } switch best.rep { - case 2: + case 2, 4 | 1: offset1, offset2 = offset2, offset1 - case 3: + case 3, 4 | 2: offset1, offset2, offset3 = offset3, offset1, offset2 + case 4 | 3: + offset1, offset2, offset3 = offset1-1, offset1, offset2 } - cv = load6432(src, s) continue } // A 4-byte match has been found. Update recent offsets. // We'll later see if more than 4 bytes. + index0 := s + 1 s = best.s t := best.offset offset1, offset2, offset3 = s-t, offset1, offset2 @@ -373,22 +403,9 @@ encodeLoop: panic("invalid offset") } - // Extend the n-byte match as long as possible. - l := best.length - - // Extend backwards - tMin := s - e.maxMatchOff - if tMin < 0 { - tMin = 0 - } - for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { - s-- - t-- - l++ - } - // Write our sequence var seq seq + l := best.length seq.litLen = uint32(s - nextEmit) seq.matchLen = uint32(l - zstdMinMatch) if seq.litLen > 0 { @@ -405,10 +422,8 @@ encodeLoop: break encodeLoop } - // Index match start+1 (long) -> s - 1 - index0 := s - l + 1 - // every entry - for index0 < s-1 { + // Index old s + 1 -> s - 1 + for index0 < s { cv0 := load6432(src, index0) h0 := hashLen(cv0, bestLongTableBits, bestLongLen) h1 := hashLen(cv0, bestShortTableBits, bestShortLen) @@ -417,50 +432,6 @@ encodeLoop: e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset} index0++ } - - cv = load6432(src, s) - if !canRepeat { - continue - } - - // Check offset 2 - for { - o2 := s - offset2 - if load3232(src, o2) != uint32(cv) { - // Do regular search - break - } - - // Store this, since we have it. - nextHashS := hashLen(cv, bestShortTableBits, bestShortLen) - nextHashL := hashLen(cv, bestLongTableBits, bestLongLen) - - // We have at least 4 byte match. - // No need to check backwards. We come straight from a match - l := 4 + e.matchlen(s+4, o2+4, src) - - e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: e.longTable[nextHashL].offset} - e.table[nextHashS] = prevEntry{offset: s + e.cur, prev: e.table[nextHashS].offset} - seq.matchLen = uint32(l) - zstdMinMatch - seq.litLen = 0 - - // Since litlen is always 0, this is offset 1. - seq.offset = 1 - s += l - nextEmit = s - if debugSequences { - println("sequence", seq, "next s:", s) - } - blk.sequences = append(blk.sequences, seq) - - // Swap offset 1 and 2. - offset1, offset2 = offset2, offset1 - if s >= sLimit { - // Finished - break encodeLoop - } - cv = load6432(src, s) - } } if int(nextEmit) < len(src) { diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index d70e3fd..8582f31 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -62,14 +62,10 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - for i := range e.longTable[:] { - e.longTable[i] = prevEntry{} - } + e.table = [betterShortTableSize]tableEntry{} + e.longTable = [betterLongTableSize]prevEntry{} e.cur = e.maxMatchOff break } @@ -587,7 +583,7 @@ func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { for i := range e.table[:] { e.table[i] = tableEntry{} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 1f4a9a2..7d42510 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -44,14 +44,10 @@ func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } - for i := range e.longTable[:] { - e.longTable[i] = tableEntry{} - } + e.table = [dFastShortTableSize]tableEntry{} + e.longTable = [dFastLongTableSize]tableEntry{} e.cur = e.maxMatchOff break } @@ -388,7 +384,7 @@ func (e *doubleFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - if e.cur >= bufferReset { + if e.cur >= e.bufferReset { for i := range e.table[:] { e.table[i] = tableEntry{} } @@ -685,7 +681,7 @@ encodeLoop: } // We do not store history, so we must offset e.cur to avoid false matches for next user. - if e.cur < bufferReset { + if e.cur < e.bufferReset { e.cur += int32(len(src)) } } @@ -700,7 +696,7 @@ func (e *doubleFastEncoderDict) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { for i := range e.table[:] { e.table[i] = tableEntry{} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index 202636d..315b1a8 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -43,7 +43,7 @@ func (e *fastEncoder) Encode(blk *blockEnc, src []byte) { ) // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { for i := range e.table[:] { e.table[i] = tableEntry{} @@ -310,7 +310,7 @@ func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { } // Protect against e.cur wraparound. - if e.cur >= bufferReset { + if e.cur >= e.bufferReset { for i := range e.table[:] { e.table[i] = tableEntry{} } @@ -538,7 +538,7 @@ encodeLoop: println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } // We do not store history, so we must offset e.cur to avoid false matches for next user. - if e.cur < bufferReset { + if e.cur < e.bufferReset { e.cur += int32(len(src)) } } @@ -555,11 +555,9 @@ func (e *fastEncoderDict) Encode(blk *blockEnc, src []byte) { return } // Protect against e.cur wraparound. - for e.cur >= bufferReset { + for e.cur >= e.bufferReset-int32(len(e.hist)) { if len(e.hist) == 0 { - for i := range e.table[:] { - e.table[i] = tableEntry{} - } + e.table = [tableSize]tableEntry{} e.cur = e.maxMatchOff break } diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 7aaaedb..4de0aed 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -8,6 +8,7 @@ import ( "crypto/rand" "fmt" "io" + "math" rdebug "runtime/debug" "sync" @@ -276,23 +277,9 @@ func (e *Encoder) nextBlock(final bool) error { s.eofWritten = true } - err := errIncompressible - // If we got the exact same number of literals as input, - // assume the literals cannot be compressed. - if len(src) != len(blk.literals) || len(src) != e.o.blockSize { - err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) - } - switch err { - case errIncompressible: - if debugEncoder { - println("Storing incompressible block as raw") - } - blk.encodeRaw(src) - // In fast mode, we do not transfer offsets, so we don't have to deal with changing the. - case nil: - default: - s.err = err - return err + s.err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) + if s.err != nil { + return s.err } _, s.err = s.w.Write(blk.output) s.nWritten += int64(len(blk.output)) @@ -342,22 +329,8 @@ func (e *Encoder) nextBlock(final bool) error { } s.wWg.Done() }() - err := errIncompressible - // If we got the exact same number of literals as input, - // assume the literals cannot be compressed. - if len(src) != len(blk.literals) || len(src) != e.o.blockSize { - err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) - } - switch err { - case errIncompressible: - if debugEncoder { - println("Storing incompressible block as raw") - } - blk.encodeRaw(src) - // In fast mode, we do not transfer offsets, so we don't have to deal with changing the. - case nil: - default: - s.writeErr = err + s.writeErr = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) + if s.writeErr != nil { return } _, s.writeErr = s.w.Write(blk.output) @@ -567,25 +540,15 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { // If we got the exact same number of literals as input, // assume the literals cannot be compressed. - err := errIncompressible oldout := blk.output - if len(blk.literals) != len(src) || len(src) != e.o.blockSize { - // Output directly to dst - blk.output = dst - err = blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) - } + // Output directly to dst + blk.output = dst - switch err { - case errIncompressible: - if debugEncoder { - println("Storing incompressible block as raw") - } - dst = blk.encodeRawTo(dst, src) - case nil: - dst = blk.output - default: + err := blk.encode(src, e.o.noEntropy, !e.o.allLitEntropy) + if err != nil { panic(err) } + dst = blk.output blk.output = oldout } else { enc.Reset(e.o.dict, false) @@ -604,25 +567,11 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { if len(src) == 0 { blk.last = true } - err := errIncompressible - // If we got the exact same number of literals as input, - // assume the literals cannot be compressed. - if len(blk.literals) != len(todo) || len(todo) != e.o.blockSize { - err = blk.encode(todo, e.o.noEntropy, !e.o.allLitEntropy) - } - - switch err { - case errIncompressible: - if debugEncoder { - println("Storing incompressible block as raw") - } - dst = blk.encodeRawTo(dst, todo) - blk.popOffsets() - case nil: - dst = append(dst, blk.output...) - default: + err := blk.encode(todo, e.o.noEntropy, !e.o.allLitEntropy) + if err != nil { panic(err) } + dst = append(dst, blk.output...) blk.reset(nil) } } @@ -639,3 +588,37 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } return dst } + +// MaxEncodedSize returns the expected maximum +// size of an encoded block or stream. +func (e *Encoder) MaxEncodedSize(size int) int { + frameHeader := 4 + 2 // magic + frame header & window descriptor + if e.o.dict != nil { + frameHeader += 4 + } + // Frame content size: + if size < 256 { + frameHeader++ + } else if size < 65536+256 { + frameHeader += 2 + } else if size < math.MaxInt32 { + frameHeader += 4 + } else { + frameHeader += 8 + } + // Final crc + if e.o.crc { + frameHeader += 4 + } + + // Max overhead is 3 bytes/block. + // There cannot be 0 blocks. + blocks := (size + e.o.blockSize) / e.o.blockSize + + // Combine, add padding. + maxSz := frameHeader + 3*blocks + size + if e.o.pad > 1 { + maxSz += calcSkippableFrame(int64(maxSz), int64(e.o.pad)) + } + return maxSz +} diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index a7c5e1a..50f7053 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -3,6 +3,8 @@ package zstd import ( "errors" "fmt" + "math" + "math/bits" "runtime" "strings" ) @@ -37,7 +39,7 @@ func (o *encoderOptions) setDefault() { blockSize: maxCompressedBlockSize, windowSize: 8 << 20, level: SpeedDefault, - allLitEntropy: true, + allLitEntropy: false, lowMem: false, } } @@ -47,22 +49,22 @@ func (o encoderOptions) encoder() encoder { switch o.level { case SpeedFastest: if o.dict != nil { - return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}} + return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}} } - return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}} + return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}} case SpeedDefault: if o.dict != nil { - return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}} + return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}}} } - return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}} + return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}} case SpeedBetterCompression: if o.dict != nil { - return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}} + return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}}} } - return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}} + return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}} case SpeedBestCompression: - return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}} + return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), bufferReset: math.MaxInt32 - int32(o.windowSize*2), lowMem: o.lowMem}} } panic("unknown compression level") } @@ -236,7 +238,7 @@ func WithEncoderLevel(l EncoderLevel) EOption { } } if !o.customALEntropy { - o.allLitEntropy = l > SpeedFastest + o.allLitEntropy = l > SpeedDefault } return nil @@ -304,7 +306,13 @@ func WithLowerEncoderMem(b bool) EOption { } // WithEncoderDict allows to register a dictionary that will be used for the encode. +// +// The slice dict must be in the [dictionary format] produced by +// "zstd --train" from the Zstandard reference implementation. +// // The encoder *may* choose to use no dictionary instead for certain payloads. +// +// [dictionary format]: https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format func WithEncoderDict(dict []byte) EOption { return func(o *encoderOptions) error { d, err := loadDict(dict) @@ -315,3 +323,17 @@ func WithEncoderDict(dict []byte) EOption { return nil } } + +// WithEncoderDictRaw registers a dictionary that may be used by the encoder. +// +// The slice content may contain arbitrary data. It will be used as an initial +// history. +func WithEncoderDictRaw(id uint32, content []byte) EOption { + return func(o *encoderOptions) error { + if bits.UintSize > 32 && uint(len(content)) > dictMaxLength { + return fmt.Errorf("dictionary of size %d > 2GiB too large", len(content)) + } + o.dict = &dict{id: id, content: content, offsets: [3]int{1, 4, 8}} + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 2c0affc..cc0aa22 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -5,7 +5,7 @@ package zstd import ( - "bytes" + "encoding/binary" "encoding/hex" "errors" "io" @@ -29,7 +29,7 @@ type frameDec struct { FrameContentSize uint64 - DictionaryID *uint32 + DictionaryID uint32 HasCheckSum bool SingleSegment bool } @@ -43,9 +43,9 @@ const ( MaxWindowSize = 1 << 29 ) -var ( - frameMagic = []byte{0x28, 0xb5, 0x2f, 0xfd} - skippableFrameMagic = []byte{0x2a, 0x4d, 0x18} +const ( + frameMagic = "\x28\xb5\x2f\xfd" + skippableFrameMagic = "\x2a\x4d\x18" ) func newFrameDec(o decoderOptions) *frameDec { @@ -89,9 +89,9 @@ func (d *frameDec) reset(br byteBuffer) error { copy(signature[1:], b) } - if !bytes.Equal(signature[1:4], skippableFrameMagic) || signature[0]&0xf0 != 0x50 { + if string(signature[1:4]) != skippableFrameMagic || signature[0]&0xf0 != 0x50 { if debugDecoder { - println("Not skippable", hex.EncodeToString(signature[:]), hex.EncodeToString(skippableFrameMagic)) + println("Not skippable", hex.EncodeToString(signature[:]), hex.EncodeToString([]byte(skippableFrameMagic))) } // Break if not skippable frame. break @@ -114,9 +114,9 @@ func (d *frameDec) reset(br byteBuffer) error { return err } } - if !bytes.Equal(signature[:], frameMagic) { + if string(signature[:]) != frameMagic { if debugDecoder { - println("Got magic numbers: ", signature, "want:", frameMagic) + println("Got magic numbers: ", signature, "want:", []byte(frameMagic)) } return ErrMagicMismatch } @@ -155,7 +155,7 @@ func (d *frameDec) reset(br byteBuffer) error { // Read Dictionary_ID // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id - d.DictionaryID = nil + d.DictionaryID = 0 if size := fhd & 3; size != 0 { if size == 3 { size = 4 @@ -167,7 +167,7 @@ func (d *frameDec) reset(br byteBuffer) error { return err } var id uint32 - switch size { + switch len(b) { case 1: id = uint32(b[0]) case 2: @@ -178,11 +178,7 @@ func (d *frameDec) reset(br byteBuffer) error { if debugDecoder { println("Dict size", size, "ID:", id) } - if id > 0 { - // ID 0 means "sorry, no dictionary anyway". - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format - d.DictionaryID = &id - } + d.DictionaryID = id } // Read Frame_Content_Size @@ -204,7 +200,7 @@ func (d *frameDec) reset(br byteBuffer) error { println("Reading Frame content", err) return err } - switch fcsSize { + switch len(b) { case 1: d.FrameContentSize = uint64(b[0]) case 2: @@ -297,55 +293,38 @@ func (d *frameDec) next(block *blockDec) error { return nil } -// checkCRC will check the checksum if the frame has one. +// checkCRC will check the checksum, assuming the frame has one. // Will return ErrCRCMismatch if crc check failed, otherwise nil. func (d *frameDec) checkCRC() error { - if !d.HasCheckSum { - return nil - } - // We can overwrite upper tmp now - want, err := d.rawInput.readSmall(4) + buf, err := d.rawInput.readSmall(4) if err != nil { println("CRC missing?", err) return err } - if d.o.ignoreChecksum { - return nil - } + want := binary.LittleEndian.Uint32(buf[:4]) + got := uint32(d.crc.Sum64()) - var tmp [4]byte - got := d.crc.Sum64() - // Flip to match file order. - tmp[0] = byte(got >> 0) - tmp[1] = byte(got >> 8) - tmp[2] = byte(got >> 16) - tmp[3] = byte(got >> 24) - - if !bytes.Equal(tmp[:], want) { + if got != want { if debugDecoder { - println("CRC Check Failed:", tmp[:], "!=", want) + printf("CRC check failed: got %08x, want %08x\n", got, want) } return ErrCRCMismatch } if debugDecoder { - println("CRC ok", tmp[:]) + printf("CRC ok %08x\n", got) } return nil } -// consumeCRC reads the checksum data if the frame has one. +// consumeCRC skips over the checksum, assuming the frame has one. func (d *frameDec) consumeCRC() error { - if d.HasCheckSum { - _, err := d.rawInput.readSmall(4) - if err != nil { - println("CRC missing?", err) - return err - } + _, err := d.rawInput.readSmall(4) + if err != nil { + println("CRC missing?", err) } - - return nil + return err } // runDecoder will run the decoder for the remainder of the frame. @@ -424,15 +403,8 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { if d.o.ignoreChecksum { err = d.consumeCRC() } else { - var n int - n, err = d.crc.Write(dst[crcStart:]) - if err == nil { - if n != len(dst)-crcStart { - err = io.ErrShortWrite - } else { - err = d.checkCRC() - } - } + d.crc.Write(dst[crcStart:]) + err = d.checkCRC() } } } diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md index 69aa3bb..777290d 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/README.md @@ -2,12 +2,7 @@ VENDORED: Go to [github.com/cespare/xxhash](https://github.com/cespare/xxhash) for original package. - -[![GoDoc](https://godoc.org/github.com/cespare/xxhash?status.svg)](https://godoc.org/github.com/cespare/xxhash) -[![Build Status](https://travis-ci.org/cespare/xxhash.svg?branch=master)](https://travis-ci.org/cespare/xxhash) - -xxhash is a Go implementation of the 64-bit -[xxHash](http://cyan4973.github.io/xxHash/) algorithm, XXH64. This is a +xxhash is a Go implementation of the 64-bit [xxHash] algorithm, XXH64. This is a high-quality hashing algorithm that is much faster than anything in the Go standard library. @@ -28,31 +23,49 @@ func (*Digest) WriteString(string) (int, error) func (*Digest) Sum64() uint64 ``` -This implementation provides a fast pure-Go implementation and an even faster -assembly implementation for amd64. +The package is written with optimized pure Go and also contains even faster +assembly implementations for amd64 and arm64. If desired, the `purego` build tag +opts into using the Go code even on those architectures. + +[xxHash]: http://cyan4973.github.io/xxHash/ + +## Compatibility + +This package is in a module and the latest code is in version 2 of the module. +You need a version of Go with at least "minimal module compatibility" to use +github.com/cespare/xxhash/v2: + +* 1.9.7+ for Go 1.9 +* 1.10.3+ for Go 1.10 +* Go 1.11 or later + +I recommend using the latest release of Go. ## Benchmarks Here are some quick benchmarks comparing the pure-Go and assembly implementations of Sum64. -| input size | purego | asm | -| --- | --- | --- | -| 5 B | 979.66 MB/s | 1291.17 MB/s | -| 100 B | 7475.26 MB/s | 7973.40 MB/s | -| 4 KB | 17573.46 MB/s | 17602.65 MB/s | -| 10 MB | 17131.46 MB/s | 17142.16 MB/s | +| input size | purego | asm | +| ---------- | --------- | --------- | +| 4 B | 1.3 GB/s | 1.2 GB/s | +| 16 B | 2.9 GB/s | 3.5 GB/s | +| 100 B | 6.9 GB/s | 8.1 GB/s | +| 4 KB | 11.7 GB/s | 16.7 GB/s | +| 10 MB | 12.0 GB/s | 17.3 GB/s | -These numbers were generated on Ubuntu 18.04 with an Intel i7-8700K CPU using -the following commands under Go 1.11.2: +These numbers were generated on Ubuntu 20.04 with an Intel Xeon Platinum 8252C +CPU using the following commands under Go 1.19.2: ``` -$ go test -tags purego -benchtime 10s -bench '/xxhash,direct,bytes' -$ go test -benchtime 10s -bench '/xxhash,direct,bytes' +benchstat <(go test -tags purego -benchtime 500ms -count 15 -bench 'Sum64$') +benchstat <(go test -benchtime 500ms -count 15 -bench 'Sum64$') ``` ## Projects using this package - [InfluxDB](https://github.com/influxdata/influxdb) - [Prometheus](https://github.com/prometheus/prometheus) +- [VictoriaMetrics](https://github.com/VictoriaMetrics/VictoriaMetrics) - [FreeCache](https://github.com/coocood/freecache) +- [FastCache](https://github.com/VictoriaMetrics/fastcache) diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go index 2c112a0..fc40c82 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash.go @@ -18,19 +18,11 @@ const ( prime5 uint64 = 2870177450012600261 ) -// NOTE(caleb): I'm using both consts and vars of the primes. Using consts where -// possible in the Go code is worth a small (but measurable) performance boost -// by avoiding some MOVQs. Vars are needed for the asm and also are useful for -// convenience in the Go code in a few places where we need to intentionally -// avoid constant arithmetic (e.g., v1 := prime1 + prime2 fails because the -// result overflows a uint64). -var ( - prime1v = prime1 - prime2v = prime2 - prime3v = prime3 - prime4v = prime4 - prime5v = prime5 -) +// Store the primes in an array as well. +// +// The consts are used when possible in Go code to avoid MOVs but we need a +// contiguous array of the assembly code. +var primes = [...]uint64{prime1, prime2, prime3, prime4, prime5} // Digest implements hash.Hash64. type Digest struct { @@ -52,10 +44,10 @@ func New() *Digest { // Reset clears the Digest's state so that it can be reused. func (d *Digest) Reset() { - d.v1 = prime1v + prime2 + d.v1 = primes[0] + prime2 d.v2 = prime2 d.v3 = 0 - d.v4 = -prime1v + d.v4 = -primes[0] d.total = 0 d.n = 0 } @@ -71,21 +63,23 @@ func (d *Digest) Write(b []byte) (n int, err error) { n = len(b) d.total += uint64(n) + memleft := d.mem[d.n&(len(d.mem)-1):] + if d.n+n < 32 { // This new data doesn't even fill the current block. - copy(d.mem[d.n:], b) + copy(memleft, b) d.n += n return } if d.n > 0 { // Finish off the partial block. - copy(d.mem[d.n:], b) + c := copy(memleft, b) d.v1 = round(d.v1, u64(d.mem[0:8])) d.v2 = round(d.v2, u64(d.mem[8:16])) d.v3 = round(d.v3, u64(d.mem[16:24])) d.v4 = round(d.v4, u64(d.mem[24:32])) - b = b[32-d.n:] + b = b[c:] d.n = 0 } @@ -135,21 +129,20 @@ func (d *Digest) Sum64() uint64 { h += d.total - i, end := 0, d.n - for ; i+8 <= end; i += 8 { - k1 := round(0, u64(d.mem[i:i+8])) + b := d.mem[:d.n&(len(d.mem)-1)] + for ; len(b) >= 8; b = b[8:] { + k1 := round(0, u64(b[:8])) h ^= k1 h = rol27(h)*prime1 + prime4 } - if i+4 <= end { - h ^= uint64(u32(d.mem[i:i+4])) * prime1 + if len(b) >= 4 { + h ^= uint64(u32(b[:4])) * prime1 h = rol23(h)*prime2 + prime3 - i += 4 + b = b[4:] } - for i < end { - h ^= uint64(d.mem[i]) * prime5 + for ; len(b) > 0; b = b[1:] { + h ^= uint64(b[0]) * prime5 h = rol11(h) * prime1 - i++ } h ^= h >> 33 diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s index cea1785..ddb63aa 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_amd64.s @@ -1,3 +1,4 @@ +//go:build !appengine && gc && !purego && !noasm // +build !appengine // +build gc // +build !purego @@ -5,212 +6,205 @@ #include "textflag.h" -// Register allocation: -// AX h -// SI pointer to advance through b -// DX n -// BX loop end -// R8 v1, k1 -// R9 v2 -// R10 v3 -// R11 v4 -// R12 tmp -// R13 prime1v -// R14 prime2v -// DI prime4v +// Registers: +#define h AX +#define d AX +#define p SI // pointer to advance through b +#define n DX +#define end BX // loop end +#define v1 R8 +#define v2 R9 +#define v3 R10 +#define v4 R11 +#define x R12 +#define prime1 R13 +#define prime2 R14 +#define prime4 DI -// round reads from and advances the buffer pointer in SI. -// It assumes that R13 has prime1v and R14 has prime2v. -#define round(r) \ - MOVQ (SI), R12 \ - ADDQ $8, SI \ - IMULQ R14, R12 \ - ADDQ R12, r \ - ROLQ $31, r \ - IMULQ R13, r +#define round(acc, x) \ + IMULQ prime2, x \ + ADDQ x, acc \ + ROLQ $31, acc \ + IMULQ prime1, acc -// mergeRound applies a merge round on the two registers acc and val. -// It assumes that R13 has prime1v, R14 has prime2v, and DI has prime4v. -#define mergeRound(acc, val) \ - IMULQ R14, val \ - ROLQ $31, val \ - IMULQ R13, val \ - XORQ val, acc \ - IMULQ R13, acc \ - ADDQ DI, acc +// round0 performs the operation x = round(0, x). +#define round0(x) \ + IMULQ prime2, x \ + ROLQ $31, x \ + IMULQ prime1, x + +// mergeRound applies a merge round on the two registers acc and x. +// It assumes that prime1, prime2, and prime4 have been loaded. +#define mergeRound(acc, x) \ + round0(x) \ + XORQ x, acc \ + IMULQ prime1, acc \ + ADDQ prime4, acc + +// blockLoop processes as many 32-byte blocks as possible, +// updating v1, v2, v3, and v4. It assumes that there is at least one block +// to process. +#define blockLoop() \ +loop: \ + MOVQ +0(p), x \ + round(v1, x) \ + MOVQ +8(p), x \ + round(v2, x) \ + MOVQ +16(p), x \ + round(v3, x) \ + MOVQ +24(p), x \ + round(v4, x) \ + ADDQ $32, p \ + CMPQ p, end \ + JLE loop // func Sum64(b []byte) uint64 -TEXT ·Sum64(SB), NOSPLIT, $0-32 +TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32 // Load fixed primes. - MOVQ ·prime1v(SB), R13 - MOVQ ·prime2v(SB), R14 - MOVQ ·prime4v(SB), DI + MOVQ ·primes+0(SB), prime1 + MOVQ ·primes+8(SB), prime2 + MOVQ ·primes+24(SB), prime4 // Load slice. - MOVQ b_base+0(FP), SI - MOVQ b_len+8(FP), DX - LEAQ (SI)(DX*1), BX + MOVQ b_base+0(FP), p + MOVQ b_len+8(FP), n + LEAQ (p)(n*1), end // The first loop limit will be len(b)-32. - SUBQ $32, BX + SUBQ $32, end // Check whether we have at least one block. - CMPQ DX, $32 + CMPQ n, $32 JLT noBlocks // Set up initial state (v1, v2, v3, v4). - MOVQ R13, R8 - ADDQ R14, R8 - MOVQ R14, R9 - XORQ R10, R10 - XORQ R11, R11 - SUBQ R13, R11 + MOVQ prime1, v1 + ADDQ prime2, v1 + MOVQ prime2, v2 + XORQ v3, v3 + XORQ v4, v4 + SUBQ prime1, v4 - // Loop until SI > BX. -blockLoop: - round(R8) - round(R9) - round(R10) - round(R11) + blockLoop() - CMPQ SI, BX - JLE blockLoop + MOVQ v1, h + ROLQ $1, h + MOVQ v2, x + ROLQ $7, x + ADDQ x, h + MOVQ v3, x + ROLQ $12, x + ADDQ x, h + MOVQ v4, x + ROLQ $18, x + ADDQ x, h - MOVQ R8, AX - ROLQ $1, AX - MOVQ R9, R12 - ROLQ $7, R12 - ADDQ R12, AX - MOVQ R10, R12 - ROLQ $12, R12 - ADDQ R12, AX - MOVQ R11, R12 - ROLQ $18, R12 - ADDQ R12, AX - - mergeRound(AX, R8) - mergeRound(AX, R9) - mergeRound(AX, R10) - mergeRound(AX, R11) + mergeRound(h, v1) + mergeRound(h, v2) + mergeRound(h, v3) + mergeRound(h, v4) JMP afterBlocks noBlocks: - MOVQ ·prime5v(SB), AX + MOVQ ·primes+32(SB), h afterBlocks: - ADDQ DX, AX + ADDQ n, h - // Right now BX has len(b)-32, and we want to loop until SI > len(b)-8. - ADDQ $24, BX + ADDQ $24, end + CMPQ p, end + JG try4 - CMPQ SI, BX - JG fourByte +loop8: + MOVQ (p), x + ADDQ $8, p + round0(x) + XORQ x, h + ROLQ $27, h + IMULQ prime1, h + ADDQ prime4, h -wordLoop: - // Calculate k1. - MOVQ (SI), R8 - ADDQ $8, SI - IMULQ R14, R8 - ROLQ $31, R8 - IMULQ R13, R8 + CMPQ p, end + JLE loop8 - XORQ R8, AX - ROLQ $27, AX - IMULQ R13, AX - ADDQ DI, AX +try4: + ADDQ $4, end + CMPQ p, end + JG try1 - CMPQ SI, BX - JLE wordLoop + MOVL (p), x + ADDQ $4, p + IMULQ prime1, x + XORQ x, h -fourByte: - ADDQ $4, BX - CMPQ SI, BX - JG singles + ROLQ $23, h + IMULQ prime2, h + ADDQ ·primes+16(SB), h - MOVL (SI), R8 - ADDQ $4, SI - IMULQ R13, R8 - XORQ R8, AX - - ROLQ $23, AX - IMULQ R14, AX - ADDQ ·prime3v(SB), AX - -singles: - ADDQ $4, BX - CMPQ SI, BX +try1: + ADDQ $4, end + CMPQ p, end JGE finalize -singlesLoop: - MOVBQZX (SI), R12 - ADDQ $1, SI - IMULQ ·prime5v(SB), R12 - XORQ R12, AX +loop1: + MOVBQZX (p), x + ADDQ $1, p + IMULQ ·primes+32(SB), x + XORQ x, h + ROLQ $11, h + IMULQ prime1, h - ROLQ $11, AX - IMULQ R13, AX - - CMPQ SI, BX - JL singlesLoop + CMPQ p, end + JL loop1 finalize: - MOVQ AX, R12 - SHRQ $33, R12 - XORQ R12, AX - IMULQ R14, AX - MOVQ AX, R12 - SHRQ $29, R12 - XORQ R12, AX - IMULQ ·prime3v(SB), AX - MOVQ AX, R12 - SHRQ $32, R12 - XORQ R12, AX + MOVQ h, x + SHRQ $33, x + XORQ x, h + IMULQ prime2, h + MOVQ h, x + SHRQ $29, x + XORQ x, h + IMULQ ·primes+16(SB), h + MOVQ h, x + SHRQ $32, x + XORQ x, h - MOVQ AX, ret+24(FP) + MOVQ h, ret+24(FP) RET -// writeBlocks uses the same registers as above except that it uses AX to store -// the d pointer. - // func writeBlocks(d *Digest, b []byte) int -TEXT ·writeBlocks(SB), NOSPLIT, $0-40 +TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 // Load fixed primes needed for round. - MOVQ ·prime1v(SB), R13 - MOVQ ·prime2v(SB), R14 + MOVQ ·primes+0(SB), prime1 + MOVQ ·primes+8(SB), prime2 // Load slice. - MOVQ b_base+8(FP), SI - MOVQ b_len+16(FP), DX - LEAQ (SI)(DX*1), BX - SUBQ $32, BX + MOVQ b_base+8(FP), p + MOVQ b_len+16(FP), n + LEAQ (p)(n*1), end + SUBQ $32, end // Load vN from d. - MOVQ d+0(FP), AX - MOVQ 0(AX), R8 // v1 - MOVQ 8(AX), R9 // v2 - MOVQ 16(AX), R10 // v3 - MOVQ 24(AX), R11 // v4 + MOVQ s+0(FP), d + MOVQ 0(d), v1 + MOVQ 8(d), v2 + MOVQ 16(d), v3 + MOVQ 24(d), v4 // We don't need to check the loop condition here; this function is // always called with at least one block of data to process. -blockLoop: - round(R8) - round(R9) - round(R10) - round(R11) - - CMPQ SI, BX - JLE blockLoop + blockLoop() // Copy vN back to d. - MOVQ R8, 0(AX) - MOVQ R9, 8(AX) - MOVQ R10, 16(AX) - MOVQ R11, 24(AX) + MOVQ v1, 0(d) + MOVQ v2, 8(d) + MOVQ v3, 16(d) + MOVQ v4, 24(d) - // The number of bytes written is SI minus the old base pointer. - SUBQ b_base+8(FP), SI - MOVQ SI, ret+32(FP) + // The number of bytes written is p minus the old base pointer. + SUBQ b_base+8(FP), p + MOVQ p, ret+32(FP) RET diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s index 4d64a17..17901e0 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_arm64.s @@ -1,13 +1,17 @@ -// +build gc,!purego,!noasm +//go:build !appengine && gc && !purego && !noasm +// +build !appengine +// +build gc +// +build !purego +// +build !noasm #include "textflag.h" -// Register allocation. +// Registers: #define digest R1 -#define h R2 // Return value. -#define p R3 // Input pointer. -#define len R4 -#define nblocks R5 // len / 32. +#define h R2 // return value +#define p R3 // input pointer +#define n R4 // input length +#define nblocks R5 // n / 32 #define prime1 R7 #define prime2 R8 #define prime3 R9 @@ -25,60 +29,52 @@ #define round(acc, x) \ MADD prime2, acc, x, acc \ ROR $64-31, acc \ - MUL prime1, acc \ + MUL prime1, acc -// x = round(0, x). +// round0 performs the operation x = round(0, x). #define round0(x) \ MUL prime2, x \ ROR $64-31, x \ - MUL prime1, x \ + MUL prime1, x -#define mergeRound(x) \ - round0(x) \ - EOR x, h \ - MADD h, prime4, prime1, h \ +#define mergeRound(acc, x) \ + round0(x) \ + EOR x, acc \ + MADD acc, prime4, prime1, acc -// Update v[1-4] with 32-byte blocks. Assumes len >= 32. -#define blocksLoop() \ - LSR $5, len, nblocks \ - PCALIGN $16 \ - loop: \ - LDP.P 32(p), (x1, x2) \ - round(v1, x1) \ - LDP -16(p), (x3, x4) \ - round(v2, x2) \ - SUB $1, nblocks \ - round(v3, x3) \ - round(v4, x4) \ - CBNZ nblocks, loop \ - -// The primes are repeated here to ensure that they're stored -// in a contiguous array, so we can load them with LDP. -DATA primes<> +0(SB)/8, $11400714785074694791 -DATA primes<> +8(SB)/8, $14029467366897019727 -DATA primes<>+16(SB)/8, $1609587929392839161 -DATA primes<>+24(SB)/8, $9650029242287828579 -DATA primes<>+32(SB)/8, $2870177450012600261 -GLOBL primes<>(SB), NOPTR+RODATA, $40 +// blockLoop processes as many 32-byte blocks as possible, +// updating v1, v2, v3, and v4. It assumes that n >= 32. +#define blockLoop() \ + LSR $5, n, nblocks \ + PCALIGN $16 \ + loop: \ + LDP.P 16(p), (x1, x2) \ + LDP.P 16(p), (x3, x4) \ + round(v1, x1) \ + round(v2, x2) \ + round(v3, x3) \ + round(v4, x4) \ + SUB $1, nblocks \ + CBNZ nblocks, loop // func Sum64(b []byte) uint64 -TEXT ·Sum64(SB), NOFRAME+NOSPLIT, $0-32 - LDP b_base+0(FP), (p, len) +TEXT ·Sum64(SB), NOSPLIT|NOFRAME, $0-32 + LDP b_base+0(FP), (p, n) - LDP primes<> +0(SB), (prime1, prime2) - LDP primes<>+16(SB), (prime3, prime4) - MOVD primes<>+32(SB), prime5 + LDP ·primes+0(SB), (prime1, prime2) + LDP ·primes+16(SB), (prime3, prime4) + MOVD ·primes+32(SB), prime5 - CMP $32, len - CSEL LO, prime5, ZR, h // if len < 32 { h = prime5 } else { h = 0 } - BLO afterLoop + CMP $32, n + CSEL LT, prime5, ZR, h // if n < 32 { h = prime5 } else { h = 0 } + BLT afterLoop ADD prime1, prime2, v1 MOVD prime2, v2 MOVD $0, v3 NEG prime1, v4 - blocksLoop() + blockLoop() ROR $64-1, v1, x1 ROR $64-7, v2, x2 @@ -88,71 +84,75 @@ TEXT ·Sum64(SB), NOFRAME+NOSPLIT, $0-32 ADD x3, x4 ADD x2, x4, h - mergeRound(v1) - mergeRound(v2) - mergeRound(v3) - mergeRound(v4) + mergeRound(h, v1) + mergeRound(h, v2) + mergeRound(h, v3) + mergeRound(h, v4) afterLoop: - ADD len, h + ADD n, h - TBZ $4, len, try8 + TBZ $4, n, try8 LDP.P 16(p), (x1, x2) round0(x1) + + // NOTE: here and below, sequencing the EOR after the ROR (using a + // rotated register) is worth a small but measurable speedup for small + // inputs. ROR $64-27, h EOR x1 @> 64-27, h, h MADD h, prime4, prime1, h round0(x2) ROR $64-27, h - EOR x2 @> 64-27, h + EOR x2 @> 64-27, h, h MADD h, prime4, prime1, h try8: - TBZ $3, len, try4 + TBZ $3, n, try4 MOVD.P 8(p), x1 round0(x1) ROR $64-27, h - EOR x1 @> 64-27, h + EOR x1 @> 64-27, h, h MADD h, prime4, prime1, h try4: - TBZ $2, len, try2 + TBZ $2, n, try2 MOVWU.P 4(p), x2 MUL prime1, x2 ROR $64-23, h - EOR x2 @> 64-23, h + EOR x2 @> 64-23, h, h MADD h, prime3, prime2, h try2: - TBZ $1, len, try1 + TBZ $1, n, try1 MOVHU.P 2(p), x3 AND $255, x3, x1 LSR $8, x3, x2 MUL prime5, x1 ROR $64-11, h - EOR x1 @> 64-11, h + EOR x1 @> 64-11, h, h MUL prime1, h MUL prime5, x2 ROR $64-11, h - EOR x2 @> 64-11, h + EOR x2 @> 64-11, h, h MUL prime1, h try1: - TBZ $0, len, end + TBZ $0, n, finalize MOVBU (p), x4 MUL prime5, x4 ROR $64-11, h - EOR x4 @> 64-11, h + EOR x4 @> 64-11, h, h MUL prime1, h -end: +finalize: EOR h >> 33, h MUL prime2, h EOR h >> 29, h @@ -163,24 +163,22 @@ end: RET // func writeBlocks(d *Digest, b []byte) int -// -// Assumes len(b) >= 32. -TEXT ·writeBlocks(SB), NOFRAME+NOSPLIT, $0-40 - LDP primes<>(SB), (prime1, prime2) +TEXT ·writeBlocks(SB), NOSPLIT|NOFRAME, $0-40 + LDP ·primes+0(SB), (prime1, prime2) // Load state. Assume v[1-4] are stored contiguously. MOVD d+0(FP), digest LDP 0(digest), (v1, v2) LDP 16(digest), (v3, v4) - LDP b_base+8(FP), (p, len) + LDP b_base+8(FP), (p, n) - blocksLoop() + blockLoop() // Store updated state. STP (v1, v2), 0(digest) STP (v3, v4), 16(digest) - BIC $31, len - MOVD len, ret+32(FP) + BIC $31, n + MOVD n, ret+32(FP) RET diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go index 1a1fac9..d4221ed 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_asm.go @@ -13,4 +13,4 @@ package xxhash func Sum64(b []byte) uint64 //go:noescape -func writeBlocks(d *Digest, b []byte) int +func writeBlocks(s *Digest, b []byte) int diff --git a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go index 209cb4a..0be16ce 100644 --- a/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go +++ b/vendor/github.com/klauspost/compress/zstd/internal/xxhash/xxhash_other.go @@ -15,10 +15,10 @@ func Sum64(b []byte) uint64 { var h uint64 if n >= 32 { - v1 := prime1v + prime2 + v1 := primes[0] + prime2 v2 := prime2 v3 := uint64(0) - v4 := -prime1v + v4 := -primes[0] for len(b) >= 32 { v1 = round(v1, u64(b[0:8:len(b)])) v2 = round(v2, u64(b[8:16:len(b)])) @@ -37,19 +37,18 @@ func Sum64(b []byte) uint64 { h += uint64(n) - i, end := 0, len(b) - for ; i+8 <= end; i += 8 { - k1 := round(0, u64(b[i:i+8:len(b)])) + for ; len(b) >= 8; b = b[8:] { + k1 := round(0, u64(b[:8])) h ^= k1 h = rol27(h)*prime1 + prime4 } - if i+4 <= end { - h ^= uint64(u32(b[i:i+4:len(b)])) * prime1 + if len(b) >= 4 { + h ^= uint64(u32(b[:4])) * prime1 h = rol23(h)*prime2 + prime3 - i += 4 + b = b[4:] } - for ; i < end; i++ { - h ^= uint64(b[i]) * prime5 + for ; len(b) > 0; b = b[1:] { + h ^= uint64(b[0]) * prime5 h = rol11(h) * prime1 } diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index f833d15..9405fcf 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -236,9 +236,12 @@ func (s *sequenceDecs) decodeSync(hist []byte) error { maxBlockSize = s.windowSize } + if debugDecoder { + println("decodeSync: decoding", seqs, "sequences", br.remain(), "bits remain on stream") + } for i := seqs - 1; i >= 0; i-- { if br.overread() { - printf("reading sequence %d, exceeded available data\n", seqs-i) + printf("reading sequence %d, exceeded available data. Overread by %d\n", seqs-i, -br.remain()) return io.ErrUnexpectedEOF } var ll, mo, ml int @@ -314,9 +317,6 @@ func (s *sequenceDecs) decodeSync(hist []byte) error { } size := ll + ml + len(out) if size-startSize > maxBlockSize { - if size-startSize == 424242 { - panic("here") - } return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) } if size > cap(out) { @@ -427,8 +427,7 @@ func (s *sequenceDecs) decodeSync(hist []byte) error { } } - // Check if space for literals - if size := len(s.literals) + len(s.out) - startSize; size > maxBlockSize { + if size := len(s.literals) + len(out) - startSize; size > maxBlockSize { return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) } diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go index 191384a..8adabd8 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -5,6 +5,7 @@ package zstd import ( "fmt" + "io" "github.com/klauspost/compress/internal/cpuinfo" ) @@ -134,6 +135,9 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { return true, fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ctx.ll, ctx.litRemain+ctx.ll) + case errorOverread: + return true, io.ErrUnexpectedEOF + case errorNotEnoughSpace: size := ctx.outPosition + ctx.ll + ctx.ml if debugDecoder { @@ -148,7 +152,6 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { s.seqSize += ctx.litRemain if s.seqSize > maxBlockSize { return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) - } err := br.close() if err != nil { @@ -203,6 +206,9 @@ const errorNotEnoughLiterals = 4 // error reported when capacity of `out` is too small const errorNotEnoughSpace = 5 +// error reported when bits are overread. +const errorOverread = 6 + // sequenceDecs_decode implements the main loop of sequenceDecs in x86 asm. // // Please refer to seqdec_generic.go for the reference implementation. @@ -248,6 +254,10 @@ func (s *sequenceDecs) decode(seqs []seqVals) error { litRemain: len(s.literals), } + if debugDecoder { + println("decode: decoding", len(seqs), "sequences", br.remain(), "bits remain on stream") + } + s.seqSize = 0 lte56bits := s.maxBits+s.offsets.fse.actualTableLog+s.matchLengths.fse.actualTableLog+s.litLengths.fse.actualTableLog <= 56 var errCode int @@ -278,6 +288,8 @@ func (s *sequenceDecs) decode(seqs []seqVals) error { case errorNotEnoughLiterals: ll := ctx.seqs[i].ll return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, ctx.litRemain+ll) + case errorOverread: + return io.ErrUnexpectedEOF } return fmt.Errorf("sequenceDecs_decode_amd64 returned erronous code %d", errCode) @@ -292,6 +304,9 @@ func (s *sequenceDecs) decode(seqs []seqVals) error { if s.seqSize > maxBlockSize { return fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) } + if debugDecoder { + println("decode: ", br.remain(), "bits remain on stream. code:", errCode) + } err := br.close() if err != nil { printf("Closing sequences: %v, %+v\n", err, *br) diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 52e5703..b6f4ba6 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -38,7 +38,7 @@ sequenceDecs_decode_amd64_main_loop: sequenceDecs_decode_amd64_fill_byte_by_byte: CMPQ SI, $0x00 - JLE sequenceDecs_decode_amd64_fill_end + JLE sequenceDecs_decode_amd64_fill_check_overread CMPQ BX, $0x07 JLE sequenceDecs_decode_amd64_fill_end SHLQ $0x08, DX @@ -49,6 +49,10 @@ sequenceDecs_decode_amd64_fill_byte_by_byte: ORQ AX, DX JMP sequenceDecs_decode_amd64_fill_byte_by_byte +sequenceDecs_decode_amd64_fill_check_overread: + CMPQ BX, $0x40 + JA error_overread + sequenceDecs_decode_amd64_fill_end: // Update offset MOVQ R9, AX @@ -105,7 +109,7 @@ sequenceDecs_decode_amd64_ml_update_zero: sequenceDecs_decode_amd64_fill_2_byte_by_byte: CMPQ SI, $0x00 - JLE sequenceDecs_decode_amd64_fill_2_end + JLE sequenceDecs_decode_amd64_fill_2_check_overread CMPQ BX, $0x07 JLE sequenceDecs_decode_amd64_fill_2_end SHLQ $0x08, DX @@ -116,6 +120,10 @@ sequenceDecs_decode_amd64_fill_2_byte_by_byte: ORQ AX, DX JMP sequenceDecs_decode_amd64_fill_2_byte_by_byte +sequenceDecs_decode_amd64_fill_2_check_overread: + CMPQ BX, $0x40 + JA error_overread + sequenceDecs_decode_amd64_fill_2_end: // Update literal length MOVQ DI, AX @@ -320,8 +328,9 @@ error_not_enough_literals: MOVQ $0x00000004, ret+24(FP) RET - // Return with not enough output space error - MOVQ $0x00000005, ret+24(FP) + // Return with overread error +error_overread: + MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_56_amd64(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int @@ -360,7 +369,7 @@ sequenceDecs_decode_56_amd64_main_loop: sequenceDecs_decode_56_amd64_fill_byte_by_byte: CMPQ SI, $0x00 - JLE sequenceDecs_decode_56_amd64_fill_end + JLE sequenceDecs_decode_56_amd64_fill_check_overread CMPQ BX, $0x07 JLE sequenceDecs_decode_56_amd64_fill_end SHLQ $0x08, DX @@ -371,6 +380,10 @@ sequenceDecs_decode_56_amd64_fill_byte_by_byte: ORQ AX, DX JMP sequenceDecs_decode_56_amd64_fill_byte_by_byte +sequenceDecs_decode_56_amd64_fill_check_overread: + CMPQ BX, $0x40 + JA error_overread + sequenceDecs_decode_56_amd64_fill_end: // Update offset MOVQ R9, AX @@ -617,8 +630,9 @@ error_not_enough_literals: MOVQ $0x00000004, ret+24(FP) RET - // Return with not enough output space error - MOVQ $0x00000005, ret+24(FP) + // Return with overread error +error_overread: + MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int @@ -657,7 +671,7 @@ sequenceDecs_decode_bmi2_main_loop: sequenceDecs_decode_bmi2_fill_byte_by_byte: CMPQ BX, $0x00 - JLE sequenceDecs_decode_bmi2_fill_end + JLE sequenceDecs_decode_bmi2_fill_check_overread CMPQ DX, $0x07 JLE sequenceDecs_decode_bmi2_fill_end SHLQ $0x08, AX @@ -668,6 +682,10 @@ sequenceDecs_decode_bmi2_fill_byte_by_byte: ORQ CX, AX JMP sequenceDecs_decode_bmi2_fill_byte_by_byte +sequenceDecs_decode_bmi2_fill_check_overread: + CMPQ DX, $0x40 + JA error_overread + sequenceDecs_decode_bmi2_fill_end: // Update offset MOVQ $0x00000808, CX @@ -708,7 +726,7 @@ sequenceDecs_decode_bmi2_fill_end: sequenceDecs_decode_bmi2_fill_2_byte_by_byte: CMPQ BX, $0x00 - JLE sequenceDecs_decode_bmi2_fill_2_end + JLE sequenceDecs_decode_bmi2_fill_2_check_overread CMPQ DX, $0x07 JLE sequenceDecs_decode_bmi2_fill_2_end SHLQ $0x08, AX @@ -719,6 +737,10 @@ sequenceDecs_decode_bmi2_fill_2_byte_by_byte: ORQ CX, AX JMP sequenceDecs_decode_bmi2_fill_2_byte_by_byte +sequenceDecs_decode_bmi2_fill_2_check_overread: + CMPQ DX, $0x40 + JA error_overread + sequenceDecs_decode_bmi2_fill_2_end: // Update literal length MOVQ $0x00000808, CX @@ -897,8 +919,9 @@ error_not_enough_literals: MOVQ $0x00000004, ret+24(FP) RET - // Return with not enough output space error - MOVQ $0x00000005, ret+24(FP) + // Return with overread error +error_overread: + MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_decode_56_bmi2(s *sequenceDecs, br *bitReader, ctx *decodeAsmContext) int @@ -937,7 +960,7 @@ sequenceDecs_decode_56_bmi2_main_loop: sequenceDecs_decode_56_bmi2_fill_byte_by_byte: CMPQ BX, $0x00 - JLE sequenceDecs_decode_56_bmi2_fill_end + JLE sequenceDecs_decode_56_bmi2_fill_check_overread CMPQ DX, $0x07 JLE sequenceDecs_decode_56_bmi2_fill_end SHLQ $0x08, AX @@ -948,6 +971,10 @@ sequenceDecs_decode_56_bmi2_fill_byte_by_byte: ORQ CX, AX JMP sequenceDecs_decode_56_bmi2_fill_byte_by_byte +sequenceDecs_decode_56_bmi2_fill_check_overread: + CMPQ DX, $0x40 + JA error_overread + sequenceDecs_decode_56_bmi2_fill_end: // Update offset MOVQ $0x00000808, CX @@ -1152,8 +1179,9 @@ error_not_enough_literals: MOVQ $0x00000004, ret+24(FP) RET - // Return with not enough output space error - MOVQ $0x00000005, ret+24(FP) + // Return with overread error +error_overread: + MOVQ $0x00000006, ret+24(FP) RET // func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool @@ -1389,8 +1417,7 @@ loop_finished: MOVQ ctx+0(FP), AX MOVQ DX, 24(AX) MOVQ DI, 104(AX) - MOVQ 80(AX), CX - SUBQ CX, SI + SUBQ 80(AX), SI MOVQ SI, 112(AX) RET @@ -1402,8 +1429,7 @@ error_match_off_too_big: MOVQ ctx+0(FP), AX MOVQ DX, 24(AX) MOVQ DI, 104(AX) - MOVQ 80(AX), CX - SUBQ CX, SI + SUBQ 80(AX), SI MOVQ SI, 112(AX) RET @@ -1747,8 +1773,7 @@ loop_finished: MOVQ ctx+0(FP), AX MOVQ DX, 24(AX) MOVQ DI, 104(AX) - MOVQ 80(AX), CX - SUBQ CX, SI + SUBQ 80(AX), SI MOVQ SI, 112(AX) RET @@ -1760,8 +1785,7 @@ error_match_off_too_big: MOVQ ctx+0(FP), AX MOVQ DX, 24(AX) MOVQ DI, 104(AX) - MOVQ 80(AX), CX - SUBQ CX, SI + SUBQ 80(AX), SI MOVQ SI, 112(AX) RET @@ -1824,7 +1848,7 @@ sequenceDecs_decodeSync_amd64_main_loop: sequenceDecs_decodeSync_amd64_fill_byte_by_byte: CMPQ SI, $0x00 - JLE sequenceDecs_decodeSync_amd64_fill_end + JLE sequenceDecs_decodeSync_amd64_fill_check_overread CMPQ BX, $0x07 JLE sequenceDecs_decodeSync_amd64_fill_end SHLQ $0x08, DX @@ -1835,6 +1859,10 @@ sequenceDecs_decodeSync_amd64_fill_byte_by_byte: ORQ AX, DX JMP sequenceDecs_decodeSync_amd64_fill_byte_by_byte +sequenceDecs_decodeSync_amd64_fill_check_overread: + CMPQ BX, $0x40 + JA error_overread + sequenceDecs_decodeSync_amd64_fill_end: // Update offset MOVQ R9, AX @@ -1891,7 +1919,7 @@ sequenceDecs_decodeSync_amd64_ml_update_zero: sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte: CMPQ SI, $0x00 - JLE sequenceDecs_decodeSync_amd64_fill_2_end + JLE sequenceDecs_decodeSync_amd64_fill_2_check_overread CMPQ BX, $0x07 JLE sequenceDecs_decodeSync_amd64_fill_2_end SHLQ $0x08, DX @@ -1902,6 +1930,10 @@ sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte: ORQ AX, DX JMP sequenceDecs_decodeSync_amd64_fill_2_byte_by_byte +sequenceDecs_decodeSync_amd64_fill_2_check_overread: + CMPQ BX, $0x40 + JA error_overread + sequenceDecs_decodeSync_amd64_fill_2_end: // Update literal length MOVQ DI, AX @@ -2311,6 +2343,11 @@ error_not_enough_literals: MOVQ $0x00000004, ret+24(FP) RET + // Return with overread error +error_overread: + MOVQ $0x00000006, ret+24(FP) + RET + // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX @@ -2376,7 +2413,7 @@ sequenceDecs_decodeSync_bmi2_main_loop: sequenceDecs_decodeSync_bmi2_fill_byte_by_byte: CMPQ BX, $0x00 - JLE sequenceDecs_decodeSync_bmi2_fill_end + JLE sequenceDecs_decodeSync_bmi2_fill_check_overread CMPQ DX, $0x07 JLE sequenceDecs_decodeSync_bmi2_fill_end SHLQ $0x08, AX @@ -2387,6 +2424,10 @@ sequenceDecs_decodeSync_bmi2_fill_byte_by_byte: ORQ CX, AX JMP sequenceDecs_decodeSync_bmi2_fill_byte_by_byte +sequenceDecs_decodeSync_bmi2_fill_check_overread: + CMPQ DX, $0x40 + JA error_overread + sequenceDecs_decodeSync_bmi2_fill_end: // Update offset MOVQ $0x00000808, CX @@ -2427,7 +2468,7 @@ sequenceDecs_decodeSync_bmi2_fill_end: sequenceDecs_decodeSync_bmi2_fill_2_byte_by_byte: CMPQ BX, $0x00 - JLE sequenceDecs_decodeSync_bmi2_fill_2_end + JLE sequenceDecs_decodeSync_bmi2_fill_2_check_overread CMPQ DX, $0x07 JLE sequenceDecs_decodeSync_bmi2_fill_2_end SHLQ $0x08, AX @@ -2438,6 +2479,10 @@ sequenceDecs_decodeSync_bmi2_fill_2_byte_by_byte: ORQ CX, AX JMP sequenceDecs_decodeSync_bmi2_fill_2_byte_by_byte +sequenceDecs_decodeSync_bmi2_fill_2_check_overread: + CMPQ DX, $0x40 + JA error_overread + sequenceDecs_decodeSync_bmi2_fill_2_end: // Update literal length MOVQ $0x00000808, CX @@ -2821,6 +2866,11 @@ error_not_enough_literals: MOVQ $0x00000004, ret+24(FP) RET + // Return with overread error +error_overread: + MOVQ $0x00000006, ret+24(FP) + RET + // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX @@ -2886,7 +2936,7 @@ sequenceDecs_decodeSync_safe_amd64_main_loop: sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte: CMPQ SI, $0x00 - JLE sequenceDecs_decodeSync_safe_amd64_fill_end + JLE sequenceDecs_decodeSync_safe_amd64_fill_check_overread CMPQ BX, $0x07 JLE sequenceDecs_decodeSync_safe_amd64_fill_end SHLQ $0x08, DX @@ -2897,6 +2947,10 @@ sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte: ORQ AX, DX JMP sequenceDecs_decodeSync_safe_amd64_fill_byte_by_byte +sequenceDecs_decodeSync_safe_amd64_fill_check_overread: + CMPQ BX, $0x40 + JA error_overread + sequenceDecs_decodeSync_safe_amd64_fill_end: // Update offset MOVQ R9, AX @@ -2953,7 +3007,7 @@ sequenceDecs_decodeSync_safe_amd64_ml_update_zero: sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte: CMPQ SI, $0x00 - JLE sequenceDecs_decodeSync_safe_amd64_fill_2_end + JLE sequenceDecs_decodeSync_safe_amd64_fill_2_check_overread CMPQ BX, $0x07 JLE sequenceDecs_decodeSync_safe_amd64_fill_2_end SHLQ $0x08, DX @@ -2964,6 +3018,10 @@ sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte: ORQ AX, DX JMP sequenceDecs_decodeSync_safe_amd64_fill_2_byte_by_byte +sequenceDecs_decodeSync_safe_amd64_fill_2_check_overread: + CMPQ BX, $0x40 + JA error_overread + sequenceDecs_decodeSync_safe_amd64_fill_2_end: // Update literal length MOVQ DI, AX @@ -3475,6 +3533,11 @@ error_not_enough_literals: MOVQ $0x00000004, ret+24(FP) RET + // Return with overread error +error_overread: + MOVQ $0x00000006, ret+24(FP) + RET + // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX @@ -3540,7 +3603,7 @@ sequenceDecs_decodeSync_safe_bmi2_main_loop: sequenceDecs_decodeSync_safe_bmi2_fill_byte_by_byte: CMPQ BX, $0x00 - JLE sequenceDecs_decodeSync_safe_bmi2_fill_end + JLE sequenceDecs_decodeSync_safe_bmi2_fill_check_overread CMPQ DX, $0x07 JLE sequenceDecs_decodeSync_safe_bmi2_fill_end SHLQ $0x08, AX @@ -3551,6 +3614,10 @@ sequenceDecs_decodeSync_safe_bmi2_fill_byte_by_byte: ORQ CX, AX JMP sequenceDecs_decodeSync_safe_bmi2_fill_byte_by_byte +sequenceDecs_decodeSync_safe_bmi2_fill_check_overread: + CMPQ DX, $0x40 + JA error_overread + sequenceDecs_decodeSync_safe_bmi2_fill_end: // Update offset MOVQ $0x00000808, CX @@ -3591,7 +3658,7 @@ sequenceDecs_decodeSync_safe_bmi2_fill_end: sequenceDecs_decodeSync_safe_bmi2_fill_2_byte_by_byte: CMPQ BX, $0x00 - JLE sequenceDecs_decodeSync_safe_bmi2_fill_2_end + JLE sequenceDecs_decodeSync_safe_bmi2_fill_2_check_overread CMPQ DX, $0x07 JLE sequenceDecs_decodeSync_safe_bmi2_fill_2_end SHLQ $0x08, AX @@ -3602,6 +3669,10 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_byte_by_byte: ORQ CX, AX JMP sequenceDecs_decodeSync_safe_bmi2_fill_2_byte_by_byte +sequenceDecs_decodeSync_safe_bmi2_fill_2_check_overread: + CMPQ DX, $0x40 + JA error_overread + sequenceDecs_decodeSync_safe_bmi2_fill_2_end: // Update literal length MOVQ $0x00000808, CX @@ -4087,6 +4158,11 @@ error_not_enough_literals: MOVQ $0x00000004, ret+24(FP) RET + // Return with overread error +error_overread: + MOVQ $0x00000006, ret+24(FP) + RET + // Return with not enough output space error error_not_enough_space: MOVQ ctx+16(FP), AX diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go index 3eb3f1c..8939667 100644 --- a/vendor/github.com/klauspost/compress/zstd/zstd.go +++ b/vendor/github.com/klauspost/compress/zstd/zstd.go @@ -36,9 +36,6 @@ const forcePreDef = false // zstdMinMatch is the minimum zstd match length. const zstdMinMatch = 3 -// Reset the buffer offset when reaching this. -const bufferReset = math.MaxInt32 - MaxWindowSize - // fcsUnknown is used for unknown frame content size. const fcsUnknown = math.MaxUint64 @@ -75,7 +72,6 @@ var ( ErrDecoderSizeExceeded = errors.New("decompressed size exceeds configured limit") // ErrUnknownDictionary is returned if the dictionary ID is unknown. - // For the time being dictionaries are not supported. ErrUnknownDictionary = errors.New("unknown dictionary") // ErrFrameSizeExceeded is returned if the stated frame size is exceeded. @@ -110,38 +106,33 @@ func printf(format string, a ...interface{}) { } } -// matchLen returns the maximum length. +// matchLen returns the maximum common prefix length of a and b. // a must be the shortest of the two. -// The function also returns whether all bytes matched. -func matchLen(a, b []byte) int { - b = b[:len(a)] - for i := 0; i < len(a)-7; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - return i + (bits.TrailingZeros64(diff) >> 3) +func matchLen(a, b []byte) (n int) { + for ; len(a) >= 8 && len(b) >= 8; a, b = a[8:], b[8:] { + diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b) + if diff != 0 { + return n + bits.TrailingZeros64(diff)>>3 } + n += 8 } - checked := (len(a) >> 3) << 3 - a = a[checked:] - b = b[checked:] for i := range a { if a[i] != b[i] { - return i + checked + break } + n++ } - return len(a) + checked + return n + } func load3232(b []byte, i int32) uint32 { - return binary.LittleEndian.Uint32(b[i:]) + return binary.LittleEndian.Uint32(b[:len(b):len(b)][i:]) } func load6432(b []byte, i int32) uint64 { - return binary.LittleEndian.Uint64(b[i:]) -} - -func load64(b []byte, i int) uint64 { - return binary.LittleEndian.Uint64(b[i:]) + return binary.LittleEndian.Uint64(b[:len(b):len(b)][i:]) } type byter interface { diff --git a/vendor/github.com/prometheus/client_golang/prometheus/counter.go b/vendor/github.com/prometheus/client_golang/prometheus/counter.go index 3668a16..62de4dc 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/counter.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/counter.go @@ -59,6 +59,18 @@ type ExemplarAdder interface { // CounterOpts is an alias for Opts. See there for doc comments. type CounterOpts Opts +// CounterVecOpts bundles the options to create a CounterVec metric. +// It is mandatory to set CounterOpts, see there for mandatory fields. VariableLabels +// is optional and can safely be left to its default value. +type CounterVecOpts struct { + CounterOpts + + // VariableLabels are used to partition the metric vector by the given set + // of labels. Each label value will be constrained with the optional Contraint + // function, if provided. + VariableLabels ConstrainableLabels +} + // NewCounter creates a new Counter based on the provided CounterOpts. // // The returned implementation also implements ExemplarAdder. It is safe to @@ -174,16 +186,24 @@ type CounterVec struct { // NewCounterVec creates a new CounterVec based on the provided CounterOpts and // partitioned by the given label names. func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec { - desc := NewDesc( + return V2.NewCounterVec(CounterVecOpts{ + CounterOpts: opts, + VariableLabels: UnconstrainedLabels(labelNames), + }) +} + +// NewCounterVec creates a new CounterVec based on the provided CounterVecOpts. +func (v2) NewCounterVec(opts CounterVecOpts) *CounterVec { + desc := V2.NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, - labelNames, + opts.VariableLabels, opts.ConstLabels, ) return &CounterVec{ MetricVec: NewMetricVec(desc, func(lvs ...string) Metric { if len(lvs) != len(desc.variableLabels) { - panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs)) + panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.labelNames(), lvs)) } result := &counter{desc: desc, labelPairs: MakeLabelPairs(desc, lvs), now: time.Now} result.init(result) // Init self-collection. @@ -246,7 +266,8 @@ func (v *CounterVec) GetMetricWith(labels Labels) (Counter, error) { // WithLabelValues works as GetMetricWithLabelValues, but panics where // GetMetricWithLabelValues would have returned an error. Not returning an // error allows shortcuts like -// myVec.WithLabelValues("404", "GET").Add(42) +// +// myVec.WithLabelValues("404", "GET").Add(42) func (v *CounterVec) WithLabelValues(lvs ...string) Counter { c, err := v.GetMetricWithLabelValues(lvs...) if err != nil { @@ -257,7 +278,8 @@ func (v *CounterVec) WithLabelValues(lvs ...string) Counter { // With works as GetMetricWith, but panics where GetMetricWithLabels would have // returned an error. Not returning an error allows shortcuts like -// myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42) +// +// myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42) func (v *CounterVec) With(labels Labels) Counter { c, err := v.GetMetricWith(labels) if err != nil { diff --git a/vendor/github.com/prometheus/client_golang/prometheus/desc.go b/vendor/github.com/prometheus/client_golang/prometheus/desc.go index 8bc5e44..1233154 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/desc.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/desc.go @@ -14,20 +14,16 @@ package prometheus import ( - "errors" "fmt" "sort" "strings" - "github.com/cespare/xxhash/v2" - "github.com/prometheus/client_golang/prometheus/internal" - //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. - "github.com/golang/protobuf/proto" - "github.com/prometheus/common/model" - + "github.com/cespare/xxhash/v2" dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/model" + "google.golang.org/protobuf/proto" ) // Desc is the descriptor used by every Prometheus Metric. It is essentially @@ -54,9 +50,9 @@ type Desc struct { // constLabelPairs contains precalculated DTO label pairs based on // the constant labels. constLabelPairs []*dto.LabelPair - // variableLabels contains names of labels for which the metric - // maintains variable values. - variableLabels []string + // variableLabels contains names of labels and normalization function for + // which the metric maintains variable values. + variableLabels ConstrainedLabels // id is a hash of the values of the ConstLabels and fqName. This // must be unique among all registered descriptors and can therefore be // used as an identifier of the descriptor. @@ -80,10 +76,24 @@ type Desc struct { // For constLabels, the label values are constant. Therefore, they are fully // specified in the Desc. See the Collector example for a usage pattern. func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) *Desc { + return V2.NewDesc(fqName, help, UnconstrainedLabels(variableLabels), constLabels) +} + +// NewDesc allocates and initializes a new Desc. Errors are recorded in the Desc +// and will be reported on registration time. variableLabels and constLabels can +// be nil if no such labels should be set. fqName must not be empty. +// +// variableLabels only contain the label names and normalization functions. Their +// label values are variable and therefore not part of the Desc. (They are managed +// within the Metric.) +// +// For constLabels, the label values are constant. Therefore, they are fully +// specified in the Desc. See the Collector example for a usage pattern. +func (v2) NewDesc(fqName, help string, variableLabels ConstrainableLabels, constLabels Labels) *Desc { d := &Desc{ fqName: fqName, help: help, - variableLabels: variableLabels, + variableLabels: variableLabels.constrainedLabels(), } if !model.IsValidMetricName(model.LabelValue(fqName)) { d.err = fmt.Errorf("%q is not a valid metric name", fqName) @@ -93,7 +103,7 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) * // their sorted label names) plus the fqName (at position 0). labelValues := make([]string, 1, len(constLabels)+1) labelValues[0] = fqName - labelNames := make([]string, 0, len(constLabels)+len(variableLabels)) + labelNames := make([]string, 0, len(constLabels)+len(d.variableLabels)) labelNameSet := map[string]struct{}{} // First add only the const label names and sort them... for labelName := range constLabels { @@ -118,16 +128,16 @@ func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) * // Now add the variable label names, but prefix them with something that // cannot be in a regular label name. That prevents matching the label // dimension with a different mix between preset and variable labels. - for _, labelName := range variableLabels { - if !checkLabelName(labelName) { - d.err = fmt.Errorf("%q is not a valid label name for metric %q", labelName, fqName) + for _, label := range d.variableLabels { + if !checkLabelName(label.Name) { + d.err = fmt.Errorf("%q is not a valid label name for metric %q", label.Name, fqName) return d } - labelNames = append(labelNames, "$"+labelName) - labelNameSet[labelName] = struct{}{} + labelNames = append(labelNames, "$"+label.Name) + labelNameSet[label.Name] = struct{}{} } if len(labelNames) != len(labelNameSet) { - d.err = errors.New("duplicate label names") + d.err = fmt.Errorf("duplicate label names in constant and variable labels for metric %q", fqName) return d } diff --git a/vendor/github.com/prometheus/client_golang/prometheus/doc.go b/vendor/github.com/prometheus/client_golang/prometheus/doc.go index 9845012..962608f 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/doc.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/doc.go @@ -21,55 +21,66 @@ // All exported functions and methods are safe to be used concurrently unless // specified otherwise. // -// A Basic Example +// # A Basic Example // // As a starting point, a very basic usage example: // -// package main +// package main // -// import ( -// "log" -// "net/http" +// import ( +// "log" +// "net/http" // -// "github.com/prometheus/client_golang/prometheus" -// "github.com/prometheus/client_golang/prometheus/promhttp" -// ) +// "github.com/prometheus/client_golang/prometheus" +// "github.com/prometheus/client_golang/prometheus/promhttp" +// ) // -// var ( -// cpuTemp = prometheus.NewGauge(prometheus.GaugeOpts{ -// Name: "cpu_temperature_celsius", -// Help: "Current temperature of the CPU.", -// }) -// hdFailures = prometheus.NewCounterVec( -// prometheus.CounterOpts{ -// Name: "hd_errors_total", -// Help: "Number of hard-disk errors.", -// }, -// []string{"device"}, -// ) -// ) +// type metrics struct { +// cpuTemp prometheus.Gauge +// hdFailures *prometheus.CounterVec +// } // -// func init() { -// // Metrics have to be registered to be exposed: -// prometheus.MustRegister(cpuTemp) -// prometheus.MustRegister(hdFailures) -// } +// func NewMetrics(reg prometheus.Registerer) *metrics { +// m := &metrics{ +// cpuTemp: prometheus.NewGauge(prometheus.GaugeOpts{ +// Name: "cpu_temperature_celsius", +// Help: "Current temperature of the CPU.", +// }), +// hdFailures: prometheus.NewCounterVec( +// prometheus.CounterOpts{ +// Name: "hd_errors_total", +// Help: "Number of hard-disk errors.", +// }, +// []string{"device"}, +// ), +// } +// reg.MustRegister(m.cpuTemp) +// reg.MustRegister(m.hdFailures) +// return m +// } // -// func main() { -// cpuTemp.Set(65.3) -// hdFailures.With(prometheus.Labels{"device":"/dev/sda"}).Inc() +// func main() { +// // Create a non-global registry. +// reg := prometheus.NewRegistry() // -// // The Handler function provides a default handler to expose metrics -// // via an HTTP server. "/metrics" is the usual endpoint for that. -// http.Handle("/metrics", promhttp.Handler()) -// log.Fatal(http.ListenAndServe(":8080", nil)) -// } +// // Create new metrics and register them using the custom registry. +// m := NewMetrics(reg) +// // Set values for the new created metrics. +// m.cpuTemp.Set(65.3) +// m.hdFailures.With(prometheus.Labels{"device":"/dev/sda"}).Inc() // +// // Expose metrics and custom registry via an HTTP server +// // using the HandleFor function. "/metrics" is the usual endpoint for that. +// http.Handle("/metrics", promhttp.HandlerFor(reg, promhttp.HandlerOpts{Registry: reg})) +// log.Fatal(http.ListenAndServe(":8080", nil)) +// } // // This is a complete program that exports two metrics, a Gauge and a Counter, // the latter with a label attached to turn it into a (one-dimensional) vector. +// It register the metrics using a custom registry and exposes them via an HTTP server +// on the /metrics endpoint. // -// Metrics +// # Metrics // // The number of exported identifiers in this package might appear a bit // overwhelming. However, in addition to the basic plumbing shown in the example @@ -100,7 +111,7 @@ // To create instances of Metrics and their vector versions, you need a suitable // …Opts struct, i.e. GaugeOpts, CounterOpts, SummaryOpts, or HistogramOpts. // -// Custom Collectors and constant Metrics +// # Custom Collectors and constant Metrics // // While you could create your own implementations of Metric, most likely you // will only ever implement the Collector interface on your own. At a first @@ -141,7 +152,7 @@ // a metric, GaugeFunc, CounterFunc, or UntypedFunc might be interesting // shortcuts. // -// Advanced Uses of the Registry +// # Advanced Uses of the Registry // // While MustRegister is the by far most common way of registering a Collector, // sometimes you might want to handle the errors the registration might cause. @@ -176,23 +187,23 @@ // NewProcessCollector). With a custom registry, you are in control and decide // yourself about the Collectors to register. // -// HTTP Exposition +// # HTTP Exposition // // The Registry implements the Gatherer interface. The caller of the Gather // method can then expose the gathered metrics in some way. Usually, the metrics // are served via HTTP on the /metrics endpoint. That's happening in the example // above. The tools to expose metrics via HTTP are in the promhttp sub-package. // -// Pushing to the Pushgateway +// # Pushing to the Pushgateway // // Function for pushing to the Pushgateway can be found in the push sub-package. // -// Graphite Bridge +// # Graphite Bridge // // Functions and examples to push metrics from a Gatherer to Graphite can be // found in the graphite sub-package. // -// Other Means of Exposition +// # Other Means of Exposition // // More ways of exposing metrics can easily be added by following the approaches // of the existing implementations. diff --git a/vendor/github.com/prometheus/client_golang/prometheus/gauge.go b/vendor/github.com/prometheus/client_golang/prometheus/gauge.go index bd0733d..f1ea6c7 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/gauge.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/gauge.go @@ -55,6 +55,18 @@ type Gauge interface { // GaugeOpts is an alias for Opts. See there for doc comments. type GaugeOpts Opts +// GaugeVecOpts bundles the options to create a GaugeVec metric. +// It is mandatory to set GaugeOpts, see there for mandatory fields. VariableLabels +// is optional and can safely be left to its default value. +type GaugeVecOpts struct { + GaugeOpts + + // VariableLabels are used to partition the metric vector by the given set + // of labels. Each label value will be constrained with the optional Contraint + // function, if provided. + VariableLabels ConstrainableLabels +} + // NewGauge creates a new Gauge based on the provided GaugeOpts. // // The returned implementation is optimized for a fast Set method. If you have a @@ -138,16 +150,24 @@ type GaugeVec struct { // NewGaugeVec creates a new GaugeVec based on the provided GaugeOpts and // partitioned by the given label names. func NewGaugeVec(opts GaugeOpts, labelNames []string) *GaugeVec { - desc := NewDesc( + return V2.NewGaugeVec(GaugeVecOpts{ + GaugeOpts: opts, + VariableLabels: UnconstrainedLabels(labelNames), + }) +} + +// NewGaugeVec creates a new GaugeVec based on the provided GaugeVecOpts. +func (v2) NewGaugeVec(opts GaugeVecOpts) *GaugeVec { + desc := V2.NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, - labelNames, + opts.VariableLabels, opts.ConstLabels, ) return &GaugeVec{ MetricVec: NewMetricVec(desc, func(lvs ...string) Metric { if len(lvs) != len(desc.variableLabels) { - panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, lvs)) + panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.labelNames(), lvs)) } result := &gauge{desc: desc, labelPairs: MakeLabelPairs(desc, lvs)} result.init(result) // Init self-collection. @@ -210,7 +230,8 @@ func (v *GaugeVec) GetMetricWith(labels Labels) (Gauge, error) { // WithLabelValues works as GetMetricWithLabelValues, but panics where // GetMetricWithLabelValues would have returned an error. Not returning an // error allows shortcuts like -// myVec.WithLabelValues("404", "GET").Add(42) +// +// myVec.WithLabelValues("404", "GET").Add(42) func (v *GaugeVec) WithLabelValues(lvs ...string) Gauge { g, err := v.GetMetricWithLabelValues(lvs...) if err != nil { @@ -221,7 +242,8 @@ func (v *GaugeVec) WithLabelValues(lvs ...string) Gauge { // With works as GetMetricWith, but panics where GetMetricWithLabels would have // returned an error. Not returning an error allows shortcuts like -// myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42) +// +// myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Add(42) func (v *GaugeVec) With(labels Labels) Gauge { g, err := v.GetMetricWith(labels) if err != nil { diff --git a/vendor/github.com/prometheus/client_golang/prometheus/go_collector_latest.go b/vendor/github.com/prometheus/client_golang/prometheus/go_collector_latest.go index 3a2d55e..2d8d9f6 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/go_collector_latest.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/go_collector_latest.go @@ -23,11 +23,10 @@ import ( "strings" "sync" - //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. - "github.com/golang/protobuf/proto" - dto "github.com/prometheus/client_model/go" - "github.com/prometheus/client_golang/prometheus/internal" + + dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/proto" ) const ( diff --git a/vendor/github.com/prometheus/client_golang/prometheus/histogram.go b/vendor/github.com/prometheus/client_golang/prometheus/histogram.go index 73e814a..5b69965 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/histogram.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/histogram.go @@ -22,25 +22,221 @@ import ( "sync/atomic" "time" - //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. - "github.com/golang/protobuf/proto" - dto "github.com/prometheus/client_model/go" + + "google.golang.org/protobuf/proto" ) +// nativeHistogramBounds for the frac of observed values. Only relevant for +// schema > 0. The position in the slice is the schema. (0 is never used, just +// here for convenience of using the schema directly as the index.) +// +// TODO(beorn7): Currently, we do a binary search into these slices. There are +// ways to turn it into a small number of simple array lookups. It probably only +// matters for schema 5 and beyond, but should be investigated. See this comment +// as a starting point: +// https://github.com/open-telemetry/opentelemetry-specification/issues/1776#issuecomment-870164310 +var nativeHistogramBounds = [][]float64{ + // Schema "0": + {0.5}, + // Schema 1: + {0.5, 0.7071067811865475}, + // Schema 2: + {0.5, 0.5946035575013605, 0.7071067811865475, 0.8408964152537144}, + // Schema 3: + { + 0.5, 0.5452538663326288, 0.5946035575013605, 0.6484197773255048, + 0.7071067811865475, 0.7711054127039704, 0.8408964152537144, 0.9170040432046711, + }, + // Schema 4: + { + 0.5, 0.5221368912137069, 0.5452538663326288, 0.5693943173783458, + 0.5946035575013605, 0.620928906036742, 0.6484197773255048, 0.6771277734684463, + 0.7071067811865475, 0.7384130729697496, 0.7711054127039704, 0.805245165974627, + 0.8408964152537144, 0.8781260801866495, 0.9170040432046711, 0.9576032806985735, + }, + // Schema 5: + { + 0.5, 0.5109485743270583, 0.5221368912137069, 0.5335702003384117, + 0.5452538663326288, 0.5571933712979462, 0.5693943173783458, 0.5818624293887887, + 0.5946035575013605, 0.6076236799902344, 0.620928906036742, 0.6345254785958666, + 0.6484197773255048, 0.6626183215798706, 0.6771277734684463, 0.6919549409819159, + 0.7071067811865475, 0.7225904034885232, 0.7384130729697496, 0.7545822137967112, + 0.7711054127039704, 0.7879904225539431, 0.805245165974627, 0.8228777390769823, + 0.8408964152537144, 0.8593096490612387, 0.8781260801866495, 0.8973545375015533, + 0.9170040432046711, 0.9370838170551498, 0.9576032806985735, 0.9785720620876999, + }, + // Schema 6: + { + 0.5, 0.5054446430258502, 0.5109485743270583, 0.5165124395106142, + 0.5221368912137069, 0.5278225891802786, 0.5335702003384117, 0.5393803988785598, + 0.5452538663326288, 0.5511912916539204, 0.5571933712979462, 0.5632608093041209, + 0.5693943173783458, 0.5755946149764913, 0.5818624293887887, 0.5881984958251406, + 0.5946035575013605, 0.6010783657263515, 0.6076236799902344, 0.6142402680534349, + 0.620928906036742, 0.6276903785123455, 0.6345254785958666, 0.6414350080393891, + 0.6484197773255048, 0.6554806057623822, 0.6626183215798706, 0.6698337620266515, + 0.6771277734684463, 0.6845012114872953, 0.6919549409819159, 0.6994898362691555, + 0.7071067811865475, 0.7148066691959849, 0.7225904034885232, 0.7304588970903234, + 0.7384130729697496, 0.7464538641456323, 0.7545822137967112, 0.762799075372269, + 0.7711054127039704, 0.7795022001189185, 0.7879904225539431, 0.7965710756711334, + 0.805245165974627, 0.8140137109286738, 0.8228777390769823, 0.8318382901633681, + 0.8408964152537144, 0.8500531768592616, 0.8593096490612387, 0.8686669176368529, + 0.8781260801866495, 0.8876882462632604, 0.8973545375015533, 0.9071260877501991, + 0.9170040432046711, 0.9269895625416926, 0.9370838170551498, 0.9472879907934827, + 0.9576032806985735, 0.9680308967461471, 0.9785720620876999, 0.9892280131939752, + }, + // Schema 7: + { + 0.5, 0.5027149505564014, 0.5054446430258502, 0.5081891574554764, + 0.5109485743270583, 0.5137229745593818, 0.5165124395106142, 0.5193170509806894, + 0.5221368912137069, 0.5249720429003435, 0.5278225891802786, 0.5306886136446309, + 0.5335702003384117, 0.5364674337629877, 0.5393803988785598, 0.5423091811066545, + 0.5452538663326288, 0.5482145409081883, 0.5511912916539204, 0.5541842058618393, + 0.5571933712979462, 0.5602188762048033, 0.5632608093041209, 0.5663192597993595, + 0.5693943173783458, 0.572486072215902, 0.5755946149764913, 0.5787200368168754, + 0.5818624293887887, 0.585021884841625, 0.5881984958251406, 0.5913923554921704, + 0.5946035575013605, 0.5978321960199137, 0.6010783657263515, 0.6043421618132907, + 0.6076236799902344, 0.6109230164863786, 0.6142402680534349, 0.6175755319684665, + 0.620928906036742, 0.6243004885946023, 0.6276903785123455, 0.6310986751971253, + 0.6345254785958666, 0.637970889198196, 0.6414350080393891, 0.6449179367033329, + 0.6484197773255048, 0.6519406325959679, 0.6554806057623822, 0.659039800633032, + 0.6626183215798706, 0.6662162735415805, 0.6698337620266515, 0.6734708931164728, + 0.6771277734684463, 0.6808045103191123, 0.6845012114872953, 0.688217985377265, + 0.6919549409819159, 0.6957121878859629, 0.6994898362691555, 0.7032879969095076, + 0.7071067811865475, 0.7109463010845827, 0.7148066691959849, 0.718687998724491, + 0.7225904034885232, 0.7265139979245261, 0.7304588970903234, 0.7344252166684908, + 0.7384130729697496, 0.7424225829363761, 0.7464538641456323, 0.7505070348132126, + 0.7545822137967112, 0.7586795205991071, 0.762799075372269, 0.7669409989204777, + 0.7711054127039704, 0.7752924388424999, 0.7795022001189185, 0.7837348199827764, + 0.7879904225539431, 0.7922691326262467, 0.7965710756711334, 0.8008963778413465, + 0.805245165974627, 0.8096175675974316, 0.8140137109286738, 0.8184337248834821, + 0.8228777390769823, 0.8273458838280969, 0.8318382901633681, 0.8363550898207981, + 0.8408964152537144, 0.8454623996346523, 0.8500531768592616, 0.8546688815502312, + 0.8593096490612387, 0.8639756154809185, 0.8686669176368529, 0.8733836930995842, + 0.8781260801866495, 0.8828942179666361, 0.8876882462632604, 0.8925083056594671, + 0.8973545375015533, 0.9022270839033115, 0.9071260877501991, 0.9120516927035263, + 0.9170040432046711, 0.9219832844793128, 0.9269895625416926, 0.9320230241988943, + 0.9370838170551498, 0.9421720895161669, 0.9472879907934827, 0.9524316709088368, + 0.9576032806985735, 0.9628029718180622, 0.9680308967461471, 0.9732872087896164, + 0.9785720620876999, 0.9838856116165875, 0.9892280131939752, 0.9945994234836328, + }, + // Schema 8: + { + 0.5, 0.5013556375251013, 0.5027149505564014, 0.5040779490592088, + 0.5054446430258502, 0.5068150424757447, 0.5081891574554764, 0.509566998038869, + 0.5109485743270583, 0.5123338964485679, 0.5137229745593818, 0.5151158188430205, + 0.5165124395106142, 0.5179128468009786, 0.5193170509806894, 0.520725062344158, + 0.5221368912137069, 0.5235525479396449, 0.5249720429003435, 0.526395386502313, + 0.5278225891802786, 0.5292536613972564, 0.5306886136446309, 0.5321274564422321, + 0.5335702003384117, 0.5350168559101208, 0.5364674337629877, 0.5379219445313954, + 0.5393803988785598, 0.5408428074966075, 0.5423091811066545, 0.5437795304588847, + 0.5452538663326288, 0.5467321995364429, 0.5482145409081883, 0.549700901315111, + 0.5511912916539204, 0.5526857228508706, 0.5541842058618393, 0.5556867516724088, + 0.5571933712979462, 0.5587040757836845, 0.5602188762048033, 0.5617377836665098, + 0.5632608093041209, 0.564787964283144, 0.5663192597993595, 0.5678547070789026, + 0.5693943173783458, 0.5709381019847808, 0.572486072215902, 0.5740382394200894, + 0.5755946149764913, 0.5771552102951081, 0.5787200368168754, 0.5802891060137493, + 0.5818624293887887, 0.5834400184762408, 0.585021884841625, 0.5866080400818185, + 0.5881984958251406, 0.5897932637314379, 0.5913923554921704, 0.5929957828304968, + 0.5946035575013605, 0.5962156912915756, 0.5978321960199137, 0.5994530835371903, + 0.6010783657263515, 0.6027080545025619, 0.6043421618132907, 0.6059806996384005, + 0.6076236799902344, 0.6092711149137041, 0.6109230164863786, 0.6125793968185725, + 0.6142402680534349, 0.6159056423670379, 0.6175755319684665, 0.6192499490999082, + 0.620928906036742, 0.622612415087629, 0.6243004885946023, 0.6259931389331581, + 0.6276903785123455, 0.6293922197748583, 0.6310986751971253, 0.6328097572894031, + 0.6345254785958666, 0.6362458516947014, 0.637970889198196, 0.6397006037528346, + 0.6414350080393891, 0.6431741147730128, 0.6449179367033329, 0.6466664866145447, + 0.6484197773255048, 0.6501778216898253, 0.6519406325959679, 0.6537082229673385, + 0.6554806057623822, 0.6572577939746774, 0.659039800633032, 0.6608266388015788, + 0.6626183215798706, 0.6644148621029772, 0.6662162735415805, 0.6680225691020727, + 0.6698337620266515, 0.6716498655934177, 0.6734708931164728, 0.6752968579460171, + 0.6771277734684463, 0.6789636531064505, 0.6808045103191123, 0.6826503586020058, + 0.6845012114872953, 0.6863570825438342, 0.688217985377265, 0.690083933630119, + 0.6919549409819159, 0.6938310211492645, 0.6957121878859629, 0.6975984549830999, + 0.6994898362691555, 0.7013863456101023, 0.7032879969095076, 0.7051948041086352, + 0.7071067811865475, 0.7090239421602076, 0.7109463010845827, 0.7128738720527471, + 0.7148066691959849, 0.7167447066838943, 0.718687998724491, 0.7206365595643126, + 0.7225904034885232, 0.7245495448210174, 0.7265139979245261, 0.7284837772007218, + 0.7304588970903234, 0.7324393720732029, 0.7344252166684908, 0.7364164454346837, + 0.7384130729697496, 0.7404151139112358, 0.7424225829363761, 0.7444354947621984, + 0.7464538641456323, 0.7484777058836176, 0.7505070348132126, 0.7525418658117031, + 0.7545822137967112, 0.7566280937263048, 0.7586795205991071, 0.7607365094544071, + 0.762799075372269, 0.7648672334736434, 0.7669409989204777, 0.7690203869158282, + 0.7711054127039704, 0.7731960915705107, 0.7752924388424999, 0.7773944698885442, + 0.7795022001189185, 0.7816156449856788, 0.7837348199827764, 0.7858597406461707, + 0.7879904225539431, 0.7901268813264122, 0.7922691326262467, 0.7944171921585818, + 0.7965710756711334, 0.7987307989543135, 0.8008963778413465, 0.8030678282083853, + 0.805245165974627, 0.8074284071024302, 0.8096175675974316, 0.8118126635086642, + 0.8140137109286738, 0.8162207259936375, 0.8184337248834821, 0.820652723822003, + 0.8228777390769823, 0.8251087869603088, 0.8273458838280969, 0.8295890460808079, + 0.8318382901633681, 0.8340936325652911, 0.8363550898207981, 0.8386226785089391, + 0.8408964152537144, 0.8431763167241966, 0.8454623996346523, 0.8477546807446661, + 0.8500531768592616, 0.8523579048290255, 0.8546688815502312, 0.8569861239649629, + 0.8593096490612387, 0.8616394738731368, 0.8639756154809185, 0.8663180910111553, + 0.8686669176368529, 0.871022112577578, 0.8733836930995842, 0.8757516765159389, + 0.8781260801866495, 0.8805069215187917, 0.8828942179666361, 0.8852879870317771, + 0.8876882462632604, 0.890095013257712, 0.8925083056594671, 0.8949281411607002, + 0.8973545375015533, 0.8997875124702672, 0.9022270839033115, 0.9046732696855155, + 0.9071260877501991, 0.909585556079304, 0.9120516927035263, 0.9145245157024483, + 0.9170040432046711, 0.9194902933879467, 0.9219832844793128, 0.9244830347552253, + 0.9269895625416926, 0.92950288621441, 0.9320230241988943, 0.9345499949706191, + 0.9370838170551498, 0.93962450902828, 0.9421720895161669, 0.9447265771954693, + 0.9472879907934827, 0.9498563490882775, 0.9524316709088368, 0.9550139751351947, + 0.9576032806985735, 0.9601996065815236, 0.9628029718180622, 0.9654133954938133, + 0.9680308967461471, 0.9706554947643201, 0.9732872087896164, 0.9759260581154889, + 0.9785720620876999, 0.9812252401044634, 0.9838856116165875, 0.9865531961276168, + 0.9892280131939752, 0.9919100824251095, 0.9945994234836328, 0.9972960560854698, + }, +} + +// The nativeHistogramBounds above can be generated with the code below. +// +// TODO(beorn7): It's tempting to actually use `go generate` to generate the +// code above. However, this could lead to slightly different numbers on +// different architectures. We still need to come to terms if we are fine with +// that, or if we might prefer to specify precise numbers in the standard. +// +// var nativeHistogramBounds [][]float64 = make([][]float64, 9) +// +// func init() { +// // Populate nativeHistogramBounds. +// numBuckets := 1 +// for i := range nativeHistogramBounds { +// bounds := []float64{0.5} +// factor := math.Exp2(math.Exp2(float64(-i))) +// for j := 0; j < numBuckets-1; j++ { +// var bound float64 +// if (j+1)%2 == 0 { +// // Use previously calculated value for increased precision. +// bound = nativeHistogramBounds[i-1][j/2+1] +// } else { +// bound = bounds[j] * factor +// } +// bounds = append(bounds, bound) +// } +// numBuckets *= 2 +// nativeHistogramBounds[i] = bounds +// } +// } + // A Histogram counts individual observations from an event or sample stream in -// configurable buckets. Similar to a summary, it also provides a sum of -// observations and an observation count. +// configurable static buckets (or in dynamic sparse buckets as part of the +// experimental Native Histograms, see below for more details). Similar to a +// Summary, it also provides a sum of observations and an observation count. // // On the Prometheus server, quantiles can be calculated from a Histogram using -// the histogram_quantile function in the query language. +// the histogram_quantile PromQL function. // -// Note that Histograms, in contrast to Summaries, can be aggregated with the -// Prometheus query language (see the documentation for detailed -// procedures). However, Histograms require the user to pre-define suitable -// buckets, and they are in general less accurate. The Observe method of a -// Histogram has a very low performance overhead in comparison with the Observe -// method of a Summary. +// Note that Histograms, in contrast to Summaries, can be aggregated in PromQL +// (see the documentation for detailed procedures). However, Histograms require +// the user to pre-define suitable buckets, and they are in general less +// accurate. (Both problems are addressed by the experimental Native +// Histograms. To use them, configure a NativeHistogramBucketFactor in the +// HistogramOpts. They also require a Prometheus server v2.40+ with the +// corresponding feature flag enabled.) +// +// The Observe method of a Histogram has a very low performance overhead in +// comparison with the Observe method of a Summary. // // To create Histogram instances, use NewHistogram. type Histogram interface { @@ -50,7 +246,8 @@ type Histogram interface { // Observe adds a single observation to the histogram. Observations are // usually positive or zero. Negative observations are accepted but // prevent current versions of Prometheus from properly detecting - // counter resets in the sum of observations. See + // counter resets in the sum of observations. (The experimental Native + // Histograms handle negative observations properly.) See // https://prometheus.io/docs/practices/histograms/#count-and-sum-of-observations // for details. Observe(float64) @@ -64,18 +261,28 @@ const bucketLabel = "le" // tailored to broadly measure the response time (in seconds) of a network // service. Most likely, however, you will be required to define buckets // customized to your use case. -var ( - DefBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10} +var DefBuckets = []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10} - errBucketLabelNotAllowed = fmt.Errorf( - "%q is not allowed as label name in histograms", bucketLabel, - ) +// DefNativeHistogramZeroThreshold is the default value for +// NativeHistogramZeroThreshold in the HistogramOpts. +// +// The value is 2^-128 (or 0.5*2^-127 in the actual IEEE 754 representation), +// which is a bucket boundary at all possible resolutions. +const DefNativeHistogramZeroThreshold = 2.938735877055719e-39 + +// NativeHistogramZeroThresholdZero can be used as NativeHistogramZeroThreshold +// in the HistogramOpts to create a zero bucket of width zero, i.e. a zero +// bucket that only receives observations of precisely zero. +const NativeHistogramZeroThresholdZero = -1 + +var errBucketLabelNotAllowed = fmt.Errorf( + "%q is not allowed as label name in histograms", bucketLabel, ) -// LinearBuckets creates 'count' buckets, each 'width' wide, where the lowest -// bucket has an upper bound of 'start'. The final +Inf bucket is not counted -// and not included in the returned slice. The returned slice is meant to be -// used for the Buckets field of HistogramOpts. +// LinearBuckets creates 'count' regular buckets, each 'width' wide, where the +// lowest bucket has an upper bound of 'start'. The final +Inf bucket is not +// counted and not included in the returned slice. The returned slice is meant +// to be used for the Buckets field of HistogramOpts. // // The function panics if 'count' is zero or negative. func LinearBuckets(start, width float64, count int) []float64 { @@ -90,11 +297,11 @@ func LinearBuckets(start, width float64, count int) []float64 { return buckets } -// ExponentialBuckets creates 'count' buckets, where the lowest bucket has an -// upper bound of 'start' and each following bucket's upper bound is 'factor' -// times the previous bucket's upper bound. The final +Inf bucket is not counted -// and not included in the returned slice. The returned slice is meant to be -// used for the Buckets field of HistogramOpts. +// ExponentialBuckets creates 'count' regular buckets, where the lowest bucket +// has an upper bound of 'start' and each following bucket's upper bound is +// 'factor' times the previous bucket's upper bound. The final +Inf bucket is +// not counted and not included in the returned slice. The returned slice is +// meant to be used for the Buckets field of HistogramOpts. // // The function panics if 'count' is 0 or negative, if 'start' is 0 or negative, // or if 'factor' is less than or equal 1. @@ -180,8 +387,97 @@ type HistogramOpts struct { // element in the slice is the upper inclusive bound of a bucket. The // values must be sorted in strictly increasing order. There is no need // to add a highest bucket with +Inf bound, it will be added - // implicitly. The default value is DefBuckets. + // implicitly. If Buckets is left as nil or set to a slice of length + // zero, it is replaced by default buckets. The default buckets are + // DefBuckets if no buckets for a native histogram (see below) are used, + // otherwise the default is no buckets. (In other words, if you want to + // use both reguler buckets and buckets for a native histogram, you have + // to define the regular buckets here explicitly.) Buckets []float64 + + // If NativeHistogramBucketFactor is greater than one, so-called sparse + // buckets are used (in addition to the regular buckets, if defined + // above). A Histogram with sparse buckets will be ingested as a Native + // Histogram by a Prometheus server with that feature enabled (requires + // Prometheus v2.40+). Sparse buckets are exponential buckets covering + // the whole float64 range (with the exception of the “zero” bucket, see + // SparseBucketsZeroThreshold below). From any one bucket to the next, + // the width of the bucket grows by a constant + // factor. NativeHistogramBucketFactor provides an upper bound for this + // factor (exception see below). The smaller + // NativeHistogramBucketFactor, the more buckets will be used and thus + // the more costly the histogram will become. A generally good trade-off + // between cost and accuracy is a value of 1.1 (each bucket is at most + // 10% wider than the previous one), which will result in each power of + // two divided into 8 buckets (e.g. there will be 8 buckets between 1 + // and 2, same as between 2 and 4, and 4 and 8, etc.). + // + // Details about the actually used factor: The factor is calculated as + // 2^(2^n), where n is an integer number between (and including) -8 and + // 4. n is chosen so that the resulting factor is the largest that is + // still smaller or equal to NativeHistogramBucketFactor. Note that the + // smallest possible factor is therefore approx. 1.00271 (i.e. 2^(2^-8) + // ). If NativeHistogramBucketFactor is greater than 1 but smaller than + // 2^(2^-8), then the actually used factor is still 2^(2^-8) even though + // it is larger than the provided NativeHistogramBucketFactor. + // + // NOTE: Native Histograms are still an experimental feature. Their + // behavior might still change without a major version + // bump. Subsequently, all NativeHistogram... options here might still + // change their behavior or name (or might completely disappear) without + // a major version bump. + NativeHistogramBucketFactor float64 + // All observations with an absolute value of less or equal + // NativeHistogramZeroThreshold are accumulated into a “zero” + // bucket. For best results, this should be close to a bucket + // boundary. This is usually the case if picking a power of two. If + // NativeHistogramZeroThreshold is left at zero, + // DefSparseBucketsZeroThreshold is used as the threshold. To configure + // a zero bucket with an actual threshold of zero (i.e. only + // observations of precisely zero will go into the zero bucket), set + // NativeHistogramZeroThreshold to the NativeHistogramZeroThresholdZero + // constant (or any negative float value). + NativeHistogramZeroThreshold float64 + + // The remaining fields define a strategy to limit the number of + // populated sparse buckets. If NativeHistogramMaxBucketNumber is left + // at zero, the number of buckets is not limited. (Note that this might + // lead to unbounded memory consumption if the values observed by the + // Histogram are sufficiently wide-spread. In particular, this could be + // used as a DoS attack vector. Where the observed values depend on + // external inputs, it is highly recommended to set a + // NativeHistogramMaxBucketNumber.) Once the set + // NativeHistogramMaxBucketNumber is exceeded, the following strategy is + // enacted: First, if the last reset (or the creation) of the histogram + // is at least NativeHistogramMinResetDuration ago, then the whole + // histogram is reset to its initial state (including regular + // buckets). If less time has passed, or if + // NativeHistogramMinResetDuration is zero, no reset is + // performed. Instead, the zero threshold is increased sufficiently to + // reduce the number of buckets to or below + // NativeHistogramMaxBucketNumber, but not to more than + // NativeHistogramMaxZeroThreshold. Thus, if + // NativeHistogramMaxZeroThreshold is already at or below the current + // zero threshold, nothing happens at this step. After that, if the + // number of buckets still exceeds NativeHistogramMaxBucketNumber, the + // resolution of the histogram is reduced by doubling the width of the + // sparse buckets (up to a growth factor between one bucket to the next + // of 2^(2^4) = 65536, see above). + NativeHistogramMaxBucketNumber uint32 + NativeHistogramMinResetDuration time.Duration + NativeHistogramMaxZeroThreshold float64 +} + +// HistogramVecOpts bundles the options to create a HistogramVec metric. +// It is mandatory to set HistogramOpts, see there for mandatory fields. VariableLabels +// is optional and can safely be left to its default value. +type HistogramVecOpts struct { + HistogramOpts + + // VariableLabels are used to partition the metric vector by the given set + // of labels. Each label value will be constrained with the optional Contraint + // function, if provided. + VariableLabels ConstrainableLabels } // NewHistogram creates a new Histogram based on the provided HistogramOpts. It @@ -204,11 +500,11 @@ func NewHistogram(opts HistogramOpts) Histogram { func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogram { if len(desc.variableLabels) != len(labelValues) { - panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, labelValues)) + panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.labelNames(), labelValues)) } for _, n := range desc.variableLabels { - if n == bucketLabel { + if n.Name == bucketLabel { panic(errBucketLabelNotAllowed) } } @@ -218,16 +514,29 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr } } - if len(opts.Buckets) == 0 { - opts.Buckets = DefBuckets - } - h := &histogram{ - desc: desc, - upperBounds: opts.Buckets, - labelPairs: MakeLabelPairs(desc, labelValues), - counts: [2]*histogramCounts{{}, {}}, - now: time.Now, + desc: desc, + upperBounds: opts.Buckets, + labelPairs: MakeLabelPairs(desc, labelValues), + nativeHistogramMaxBuckets: opts.NativeHistogramMaxBucketNumber, + nativeHistogramMaxZeroThreshold: opts.NativeHistogramMaxZeroThreshold, + nativeHistogramMinResetDuration: opts.NativeHistogramMinResetDuration, + lastResetTime: time.Now(), + now: time.Now, + } + if len(h.upperBounds) == 0 && opts.NativeHistogramBucketFactor <= 1 { + h.upperBounds = DefBuckets + } + if opts.NativeHistogramBucketFactor <= 1 { + h.nativeHistogramSchema = math.MinInt32 // To mark that there are no sparse buckets. + } else { + switch { + case opts.NativeHistogramZeroThreshold > 0: + h.nativeHistogramZeroThreshold = opts.NativeHistogramZeroThreshold + case opts.NativeHistogramZeroThreshold == 0: + h.nativeHistogramZeroThreshold = DefNativeHistogramZeroThreshold + } // Leave h.nativeHistogramZeroThreshold at 0 otherwise. + h.nativeHistogramSchema = pickSchema(opts.NativeHistogramBucketFactor) } for i, upperBound := range h.upperBounds { if i < len(h.upperBounds)-1 { @@ -246,8 +555,12 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr } // Finally we know the final length of h.upperBounds and can make buckets // for both counts as well as exemplars: - h.counts[0].buckets = make([]uint64, len(h.upperBounds)) - h.counts[1].buckets = make([]uint64, len(h.upperBounds)) + h.counts[0] = &histogramCounts{buckets: make([]uint64, len(h.upperBounds))} + atomic.StoreUint64(&h.counts[0].nativeHistogramZeroThresholdBits, math.Float64bits(h.nativeHistogramZeroThreshold)) + atomic.StoreInt32(&h.counts[0].nativeHistogramSchema, h.nativeHistogramSchema) + h.counts[1] = &histogramCounts{buckets: make([]uint64, len(h.upperBounds))} + atomic.StoreUint64(&h.counts[1].nativeHistogramZeroThresholdBits, math.Float64bits(h.nativeHistogramZeroThreshold)) + atomic.StoreInt32(&h.counts[1].nativeHistogramSchema, h.nativeHistogramSchema) h.exemplars = make([]atomic.Value, len(h.upperBounds)+1) h.init(h) // Init self-collection. @@ -255,13 +568,98 @@ func newHistogram(desc *Desc, opts HistogramOpts, labelValues ...string) Histogr } type histogramCounts struct { + // Order in this struct matters for the alignment required by atomic + // operations, see http://golang.org/pkg/sync/atomic/#pkg-note-BUG + // sumBits contains the bits of the float64 representing the sum of all - // observations. sumBits and count have to go first in the struct to - // guarantee alignment for atomic operations. - // http://golang.org/pkg/sync/atomic/#pkg-note-BUG + // observations. sumBits uint64 count uint64 + + // nativeHistogramZeroBucket counts all (positive and negative) + // observations in the zero bucket (with an absolute value less or equal + // the current threshold, see next field. + nativeHistogramZeroBucket uint64 + // nativeHistogramZeroThresholdBits is the bit pattern of the current + // threshold for the zero bucket. It's initially equal to + // nativeHistogramZeroThreshold but may change according to the bucket + // count limitation strategy. + nativeHistogramZeroThresholdBits uint64 + // nativeHistogramSchema may change over time according to the bucket + // count limitation strategy and therefore has to be saved here. + nativeHistogramSchema int32 + // Number of (positive and negative) sparse buckets. + nativeHistogramBucketsNumber uint32 + + // Regular buckets. buckets []uint64 + + // The sparse buckets for native histograms are implemented with a + // sync.Map for now. A dedicated data structure will likely be more + // efficient. There are separate maps for negative and positive + // observations. The map's value is an *int64, counting observations in + // that bucket. (Note that we don't use uint64 as an int64 won't + // overflow in practice, and working with signed numbers from the + // beginning simplifies the handling of deltas.) The map's key is the + // index of the bucket according to the used + // nativeHistogramSchema. Index 0 is for an upper bound of 1. + nativeHistogramBucketsPositive, nativeHistogramBucketsNegative sync.Map +} + +// observe manages the parts of observe that only affects +// histogramCounts. doSparse is true if sparse buckets should be done, +// too. +func (hc *histogramCounts) observe(v float64, bucket int, doSparse bool) { + if bucket < len(hc.buckets) { + atomic.AddUint64(&hc.buckets[bucket], 1) + } + atomicAddFloat(&hc.sumBits, v) + if doSparse && !math.IsNaN(v) { + var ( + key int + schema = atomic.LoadInt32(&hc.nativeHistogramSchema) + zeroThreshold = math.Float64frombits(atomic.LoadUint64(&hc.nativeHistogramZeroThresholdBits)) + bucketCreated, isInf bool + ) + if math.IsInf(v, 0) { + // Pretend v is MaxFloat64 but later increment key by one. + if math.IsInf(v, +1) { + v = math.MaxFloat64 + } else { + v = -math.MaxFloat64 + } + isInf = true + } + frac, exp := math.Frexp(math.Abs(v)) + if schema > 0 { + bounds := nativeHistogramBounds[schema] + key = sort.SearchFloat64s(bounds, frac) + (exp-1)*len(bounds) + } else { + key = exp + if frac == 0.5 { + key-- + } + div := 1 << -schema + key = (key + div - 1) / div + } + if isInf { + key++ + } + switch { + case v > zeroThreshold: + bucketCreated = addToBucket(&hc.nativeHistogramBucketsPositive, key, 1) + case v < -zeroThreshold: + bucketCreated = addToBucket(&hc.nativeHistogramBucketsNegative, key, 1) + default: + atomic.AddUint64(&hc.nativeHistogramZeroBucket, 1) + } + if bucketCreated { + atomic.AddUint32(&hc.nativeHistogramBucketsNumber, 1) + } + } + // Increment count last as we take it as a signal that the observation + // is complete. + atomic.AddUint64(&hc.count, 1) } type histogram struct { @@ -276,7 +674,7 @@ type histogram struct { // perspective of the histogram) swap the hot–cold under the writeMtx // lock. A cooldown is awaited (while locked) by comparing the number of // observations with the initiation count. Once they match, then the - // last observation on the now cool one has completed. All cool fields must + // last observation on the now cool one has completed. All cold fields must // be merged into the new hot before releasing writeMtx. // // Fields with atomic access first! See alignment constraint: @@ -284,8 +682,10 @@ type histogram struct { countAndHotIdx uint64 selfCollector - desc *Desc - writeMtx sync.Mutex // Only used in the Write method. + desc *Desc + + // Only used in the Write method and for sparse bucket management. + mtx sync.Mutex // Two counts, one is "hot" for lock-free observations, the other is // "cold" for writing out a dto.Metric. It has to be an array of @@ -293,9 +693,15 @@ type histogram struct { // http://golang.org/pkg/sync/atomic/#pkg-note-BUG. counts [2]*histogramCounts - upperBounds []float64 - labelPairs []*dto.LabelPair - exemplars []atomic.Value // One more than buckets (to include +Inf), each a *dto.Exemplar. + upperBounds []float64 + labelPairs []*dto.LabelPair + exemplars []atomic.Value // One more than buckets (to include +Inf), each a *dto.Exemplar. + nativeHistogramSchema int32 // The initial schema. Set to math.MinInt32 if no sparse buckets are used. + nativeHistogramZeroThreshold float64 // The initial zero threshold. + nativeHistogramMaxZeroThreshold float64 + nativeHistogramMaxBuckets uint32 + nativeHistogramMinResetDuration time.Duration + lastResetTime time.Time // Protected by mtx. now func() time.Time // To mock out time.Now() for testing. } @@ -319,8 +725,8 @@ func (h *histogram) Write(out *dto.Metric) error { // the hot path, i.e. Observe is called much more often than Write. The // complication of making Write lock-free isn't worth it, if possible at // all. - h.writeMtx.Lock() - defer h.writeMtx.Unlock() + h.mtx.Lock() + defer h.mtx.Unlock() // Adding 1<<63 switches the hot index (from 0 to 1 or from 1 to 0) // without touching the count bits. See the struct comments for a full @@ -333,16 +739,16 @@ func (h *histogram) Write(out *dto.Metric) error { hotCounts := h.counts[n>>63] coldCounts := h.counts[(^n)>>63] - // Await cooldown. - for count != atomic.LoadUint64(&coldCounts.count) { - runtime.Gosched() // Let observations get work done. - } + waitForCooldown(count, coldCounts) his := &dto.Histogram{ Bucket: make([]*dto.Bucket, len(h.upperBounds)), SampleCount: proto.Uint64(count), SampleSum: proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.sumBits))), } + out.Histogram = his + out.Label = h.labelPairs + var cumCount uint64 for i, upperBound := range h.upperBounds { cumCount += atomic.LoadUint64(&coldCounts.buckets[i]) @@ -363,25 +769,21 @@ func (h *histogram) Write(out *dto.Metric) error { } his.Bucket = append(his.Bucket, b) } + if h.nativeHistogramSchema > math.MinInt32 { + his.ZeroThreshold = proto.Float64(math.Float64frombits(atomic.LoadUint64(&coldCounts.nativeHistogramZeroThresholdBits))) + his.Schema = proto.Int32(atomic.LoadInt32(&coldCounts.nativeHistogramSchema)) + zeroBucket := atomic.LoadUint64(&coldCounts.nativeHistogramZeroBucket) - out.Histogram = his - out.Label = h.labelPairs + defer func() { + coldCounts.nativeHistogramBucketsPositive.Range(addAndReset(&hotCounts.nativeHistogramBucketsPositive, &hotCounts.nativeHistogramBucketsNumber)) + coldCounts.nativeHistogramBucketsNegative.Range(addAndReset(&hotCounts.nativeHistogramBucketsNegative, &hotCounts.nativeHistogramBucketsNumber)) + }() - // Finally add all the cold counts to the new hot counts and reset the cold counts. - atomic.AddUint64(&hotCounts.count, count) - atomic.StoreUint64(&coldCounts.count, 0) - for { - oldBits := atomic.LoadUint64(&hotCounts.sumBits) - newBits := math.Float64bits(math.Float64frombits(oldBits) + his.GetSampleSum()) - if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) { - atomic.StoreUint64(&coldCounts.sumBits, 0) - break - } - } - for i := range h.upperBounds { - atomic.AddUint64(&hotCounts.buckets[i], atomic.LoadUint64(&coldCounts.buckets[i])) - atomic.StoreUint64(&coldCounts.buckets[i], 0) + his.ZeroCount = proto.Uint64(zeroBucket) + his.NegativeSpan, his.NegativeDelta = makeBuckets(&coldCounts.nativeHistogramBucketsNegative) + his.PositiveSpan, his.PositiveDelta = makeBuckets(&coldCounts.nativeHistogramBucketsPositive) } + addAndResetCounts(hotCounts, coldCounts) return nil } @@ -402,25 +804,216 @@ func (h *histogram) findBucket(v float64) int { // observe is the implementation for Observe without the findBucket part. func (h *histogram) observe(v float64, bucket int) { + // Do not add to sparse buckets for NaN observations. + doSparse := h.nativeHistogramSchema > math.MinInt32 && !math.IsNaN(v) // We increment h.countAndHotIdx so that the counter in the lower // 63 bits gets incremented. At the same time, we get the new value // back, which we can use to find the currently-hot counts. n := atomic.AddUint64(&h.countAndHotIdx, 1) hotCounts := h.counts[n>>63] - - if bucket < len(h.upperBounds) { - atomic.AddUint64(&hotCounts.buckets[bucket], 1) + hotCounts.observe(v, bucket, doSparse) + if doSparse { + h.limitBuckets(hotCounts, v, bucket) } - for { - oldBits := atomic.LoadUint64(&hotCounts.sumBits) - newBits := math.Float64bits(math.Float64frombits(oldBits) + v) - if atomic.CompareAndSwapUint64(&hotCounts.sumBits, oldBits, newBits) { - break +} + +// limitSparsebuckets applies a strategy to limit the number of populated sparse +// buckets. It's generally best effort, and there are situations where the +// number can go higher (if even the lowest resolution isn't enough to reduce +// the number sufficiently, or if the provided counts aren't fully updated yet +// by a concurrently happening Write call). +func (h *histogram) limitBuckets(counts *histogramCounts, value float64, bucket int) { + if h.nativeHistogramMaxBuckets == 0 { + return // No limit configured. + } + if h.nativeHistogramMaxBuckets >= atomic.LoadUint32(&counts.nativeHistogramBucketsNumber) { + return // Bucket limit not exceeded yet. + } + + h.mtx.Lock() + defer h.mtx.Unlock() + + // The hot counts might have been swapped just before we acquired the + // lock. Re-fetch the hot counts first... + n := atomic.LoadUint64(&h.countAndHotIdx) + hotIdx := n >> 63 + coldIdx := (^n) >> 63 + hotCounts := h.counts[hotIdx] + coldCounts := h.counts[coldIdx] + // ...and then check again if we really have to reduce the bucket count. + if h.nativeHistogramMaxBuckets >= atomic.LoadUint32(&hotCounts.nativeHistogramBucketsNumber) { + return // Bucket limit not exceeded after all. + } + // Try the various strategies in order. + if h.maybeReset(hotCounts, coldCounts, coldIdx, value, bucket) { + return + } + if h.maybeWidenZeroBucket(hotCounts, coldCounts) { + return + } + h.doubleBucketWidth(hotCounts, coldCounts) +} + +// maybeReset resests the whole histogram if at least h.nativeHistogramMinResetDuration +// has been passed. It returns true if the histogram has been reset. The caller +// must have locked h.mtx. +func (h *histogram) maybeReset(hot, cold *histogramCounts, coldIdx uint64, value float64, bucket int) bool { + // We are using the possibly mocked h.now() rather than + // time.Since(h.lastResetTime) to enable testing. + if h.nativeHistogramMinResetDuration == 0 || h.now().Sub(h.lastResetTime) < h.nativeHistogramMinResetDuration { + return false + } + // Completely reset coldCounts. + h.resetCounts(cold) + // Repeat the latest observation to not lose it completely. + cold.observe(value, bucket, true) + // Make coldCounts the new hot counts while ressetting countAndHotIdx. + n := atomic.SwapUint64(&h.countAndHotIdx, (coldIdx<<63)+1) + count := n & ((1 << 63) - 1) + waitForCooldown(count, hot) + // Finally, reset the formerly hot counts, too. + h.resetCounts(hot) + h.lastResetTime = h.now() + return true +} + +// maybeWidenZeroBucket widens the zero bucket until it includes the existing +// buckets closest to the zero bucket (which could be two, if an equidistant +// negative and a positive bucket exists, but usually it's only one bucket to be +// merged into the new wider zero bucket). h.nativeHistogramMaxZeroThreshold +// limits how far the zero bucket can be extended, and if that's not enough to +// include an existing bucket, the method returns false. The caller must have +// locked h.mtx. +func (h *histogram) maybeWidenZeroBucket(hot, cold *histogramCounts) bool { + currentZeroThreshold := math.Float64frombits(atomic.LoadUint64(&hot.nativeHistogramZeroThresholdBits)) + if currentZeroThreshold >= h.nativeHistogramMaxZeroThreshold { + return false + } + // Find the key of the bucket closest to zero. + smallestKey := findSmallestKey(&hot.nativeHistogramBucketsPositive) + smallestNegativeKey := findSmallestKey(&hot.nativeHistogramBucketsNegative) + if smallestNegativeKey < smallestKey { + smallestKey = smallestNegativeKey + } + if smallestKey == math.MaxInt32 { + return false + } + newZeroThreshold := getLe(smallestKey, atomic.LoadInt32(&hot.nativeHistogramSchema)) + if newZeroThreshold > h.nativeHistogramMaxZeroThreshold { + return false // New threshold would exceed the max threshold. + } + atomic.StoreUint64(&cold.nativeHistogramZeroThresholdBits, math.Float64bits(newZeroThreshold)) + // Remove applicable buckets. + if _, loaded := cold.nativeHistogramBucketsNegative.LoadAndDelete(smallestKey); loaded { + atomicDecUint32(&cold.nativeHistogramBucketsNumber) + } + if _, loaded := cold.nativeHistogramBucketsPositive.LoadAndDelete(smallestKey); loaded { + atomicDecUint32(&cold.nativeHistogramBucketsNumber) + } + // Make cold counts the new hot counts. + n := atomic.AddUint64(&h.countAndHotIdx, 1<<63) + count := n & ((1 << 63) - 1) + // Swap the pointer names to represent the new roles and make + // the rest less confusing. + hot, cold = cold, hot + waitForCooldown(count, cold) + // Add all the now cold counts to the new hot counts... + addAndResetCounts(hot, cold) + // ...adjust the new zero threshold in the cold counts, too... + atomic.StoreUint64(&cold.nativeHistogramZeroThresholdBits, math.Float64bits(newZeroThreshold)) + // ...and then merge the newly deleted buckets into the wider zero + // bucket. + mergeAndDeleteOrAddAndReset := func(hotBuckets, coldBuckets *sync.Map) func(k, v interface{}) bool { + return func(k, v interface{}) bool { + key := k.(int) + bucket := v.(*int64) + if key == smallestKey { + // Merge into hot zero bucket... + atomic.AddUint64(&hot.nativeHistogramZeroBucket, uint64(atomic.LoadInt64(bucket))) + // ...and delete from cold counts. + coldBuckets.Delete(key) + atomicDecUint32(&cold.nativeHistogramBucketsNumber) + } else { + // Add to corresponding hot bucket... + if addToBucket(hotBuckets, key, atomic.LoadInt64(bucket)) { + atomic.AddUint32(&hot.nativeHistogramBucketsNumber, 1) + } + // ...and reset cold bucket. + atomic.StoreInt64(bucket, 0) + } + return true } } - // Increment count last as we take it as a signal that the observation - // is complete. - atomic.AddUint64(&hotCounts.count, 1) + + cold.nativeHistogramBucketsPositive.Range(mergeAndDeleteOrAddAndReset(&hot.nativeHistogramBucketsPositive, &cold.nativeHistogramBucketsPositive)) + cold.nativeHistogramBucketsNegative.Range(mergeAndDeleteOrAddAndReset(&hot.nativeHistogramBucketsNegative, &cold.nativeHistogramBucketsNegative)) + return true +} + +// doubleBucketWidth doubles the bucket width (by decrementing the schema +// number). Note that very sparse buckets could lead to a low reduction of the +// bucket count (or even no reduction at all). The method does nothing if the +// schema is already -4. +func (h *histogram) doubleBucketWidth(hot, cold *histogramCounts) { + coldSchema := atomic.LoadInt32(&cold.nativeHistogramSchema) + if coldSchema == -4 { + return // Already at lowest resolution. + } + coldSchema-- + atomic.StoreInt32(&cold.nativeHistogramSchema, coldSchema) + // Play it simple and just delete all cold buckets. + atomic.StoreUint32(&cold.nativeHistogramBucketsNumber, 0) + deleteSyncMap(&cold.nativeHistogramBucketsNegative) + deleteSyncMap(&cold.nativeHistogramBucketsPositive) + // Make coldCounts the new hot counts. + n := atomic.AddUint64(&h.countAndHotIdx, 1<<63) + count := n & ((1 << 63) - 1) + // Swap the pointer names to represent the new roles and make + // the rest less confusing. + hot, cold = cold, hot + waitForCooldown(count, cold) + // Add all the now cold counts to the new hot counts... + addAndResetCounts(hot, cold) + // ...adjust the schema in the cold counts, too... + atomic.StoreInt32(&cold.nativeHistogramSchema, coldSchema) + // ...and then merge the cold buckets into the wider hot buckets. + merge := func(hotBuckets *sync.Map) func(k, v interface{}) bool { + return func(k, v interface{}) bool { + key := k.(int) + bucket := v.(*int64) + // Adjust key to match the bucket to merge into. + if key > 0 { + key++ + } + key /= 2 + // Add to corresponding hot bucket. + if addToBucket(hotBuckets, key, atomic.LoadInt64(bucket)) { + atomic.AddUint32(&hot.nativeHistogramBucketsNumber, 1) + } + return true + } + } + + cold.nativeHistogramBucketsPositive.Range(merge(&hot.nativeHistogramBucketsPositive)) + cold.nativeHistogramBucketsNegative.Range(merge(&hot.nativeHistogramBucketsNegative)) + // Play it simple again and just delete all cold buckets. + atomic.StoreUint32(&cold.nativeHistogramBucketsNumber, 0) + deleteSyncMap(&cold.nativeHistogramBucketsNegative) + deleteSyncMap(&cold.nativeHistogramBucketsPositive) +} + +func (h *histogram) resetCounts(counts *histogramCounts) { + atomic.StoreUint64(&counts.sumBits, 0) + atomic.StoreUint64(&counts.count, 0) + atomic.StoreUint64(&counts.nativeHistogramZeroBucket, 0) + atomic.StoreUint64(&counts.nativeHistogramZeroThresholdBits, math.Float64bits(h.nativeHistogramZeroThreshold)) + atomic.StoreInt32(&counts.nativeHistogramSchema, h.nativeHistogramSchema) + atomic.StoreUint32(&counts.nativeHistogramBucketsNumber, 0) + for i := range h.upperBounds { + atomic.StoreUint64(&counts.buckets[i], 0) + } + deleteSyncMap(&counts.nativeHistogramBucketsNegative) + deleteSyncMap(&counts.nativeHistogramBucketsPositive) } // updateExemplar replaces the exemplar for the provided bucket. With empty @@ -448,15 +1041,23 @@ type HistogramVec struct { // NewHistogramVec creates a new HistogramVec based on the provided HistogramOpts and // partitioned by the given label names. func NewHistogramVec(opts HistogramOpts, labelNames []string) *HistogramVec { - desc := NewDesc( + return V2.NewHistogramVec(HistogramVecOpts{ + HistogramOpts: opts, + VariableLabels: UnconstrainedLabels(labelNames), + }) +} + +// NewHistogramVec creates a new HistogramVec based on the provided HistogramVecOpts. +func (v2) NewHistogramVec(opts HistogramVecOpts) *HistogramVec { + desc := V2.NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, - labelNames, + opts.VariableLabels, opts.ConstLabels, ) return &HistogramVec{ MetricVec: NewMetricVec(desc, func(lvs ...string) Metric { - return newHistogram(desc, opts, lvs...) + return newHistogram(desc, opts.HistogramOpts, lvs...) }), } } @@ -516,7 +1117,8 @@ func (v *HistogramVec) GetMetricWith(labels Labels) (Observer, error) { // WithLabelValues works as GetMetricWithLabelValues, but panics where // GetMetricWithLabelValues would have returned an error. Not returning an // error allows shortcuts like -// myVec.WithLabelValues("404", "GET").Observe(42.21) +// +// myVec.WithLabelValues("404", "GET").Observe(42.21) func (v *HistogramVec) WithLabelValues(lvs ...string) Observer { h, err := v.GetMetricWithLabelValues(lvs...) if err != nil { @@ -527,7 +1129,8 @@ func (v *HistogramVec) WithLabelValues(lvs ...string) Observer { // With works as GetMetricWith but panics where GetMetricWithLabels would have // returned an error. Not returning an error allows shortcuts like -// myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Observe(42.21) +// +// myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Observe(42.21) func (v *HistogramVec) With(labels Labels) Observer { h, err := v.GetMetricWith(labels) if err != nil { @@ -668,3 +1271,229 @@ func (s buckSort) Swap(i, j int) { func (s buckSort) Less(i, j int) bool { return s[i].GetUpperBound() < s[j].GetUpperBound() } + +// pickSchema returns the largest number n between -4 and 8 such that +// 2^(2^-n) is less or equal the provided bucketFactor. +// +// Special cases: +// - bucketFactor <= 1: panics. +// - bucketFactor < 2^(2^-8) (but > 1): still returns 8. +func pickSchema(bucketFactor float64) int32 { + if bucketFactor <= 1 { + panic(fmt.Errorf("bucketFactor %f is <=1", bucketFactor)) + } + floor := math.Floor(math.Log2(math.Log2(bucketFactor))) + switch { + case floor <= -8: + return 8 + case floor >= 4: + return -4 + default: + return -int32(floor) + } +} + +func makeBuckets(buckets *sync.Map) ([]*dto.BucketSpan, []int64) { + var ii []int + buckets.Range(func(k, v interface{}) bool { + ii = append(ii, k.(int)) + return true + }) + sort.Ints(ii) + + if len(ii) == 0 { + return nil, nil + } + + var ( + spans []*dto.BucketSpan + deltas []int64 + prevCount int64 + nextI int + ) + + appendDelta := func(count int64) { + *spans[len(spans)-1].Length++ + deltas = append(deltas, count-prevCount) + prevCount = count + } + + for n, i := range ii { + v, _ := buckets.Load(i) + count := atomic.LoadInt64(v.(*int64)) + // Multiple spans with only small gaps in between are probably + // encoded more efficiently as one larger span with a few empty + // buckets. Needs some research to find the sweet spot. For now, + // we assume that gaps of one ore two buckets should not create + // a new span. + iDelta := int32(i - nextI) + if n == 0 || iDelta > 2 { + // We have to create a new span, either because we are + // at the very beginning, or because we have found a gap + // of more than two buckets. + spans = append(spans, &dto.BucketSpan{ + Offset: proto.Int32(iDelta), + Length: proto.Uint32(0), + }) + } else { + // We have found a small gap (or no gap at all). + // Insert empty buckets as needed. + for j := int32(0); j < iDelta; j++ { + appendDelta(0) + } + } + appendDelta(count) + nextI = i + 1 + } + return spans, deltas +} + +// addToBucket increments the sparse bucket at key by the provided amount. It +// returns true if a new sparse bucket had to be created for that. +func addToBucket(buckets *sync.Map, key int, increment int64) bool { + if existingBucket, ok := buckets.Load(key); ok { + // Fast path without allocation. + atomic.AddInt64(existingBucket.(*int64), increment) + return false + } + // Bucket doesn't exist yet. Slow path allocating new counter. + newBucket := increment // TODO(beorn7): Check if this is sufficient to not let increment escape. + if actualBucket, loaded := buckets.LoadOrStore(key, &newBucket); loaded { + // The bucket was created concurrently in another goroutine. + // Have to increment after all. + atomic.AddInt64(actualBucket.(*int64), increment) + return false + } + return true +} + +// addAndReset returns a function to be used with sync.Map.Range of spare +// buckets in coldCounts. It increments the buckets in the provided hotBuckets +// according to the buckets ranged through. It then resets all buckets ranged +// through to 0 (but leaves them in place so that they don't need to get +// recreated on the next scrape). +func addAndReset(hotBuckets *sync.Map, bucketNumber *uint32) func(k, v interface{}) bool { + return func(k, v interface{}) bool { + bucket := v.(*int64) + if addToBucket(hotBuckets, k.(int), atomic.LoadInt64(bucket)) { + atomic.AddUint32(bucketNumber, 1) + } + atomic.StoreInt64(bucket, 0) + return true + } +} + +func deleteSyncMap(m *sync.Map) { + m.Range(func(k, v interface{}) bool { + m.Delete(k) + return true + }) +} + +func findSmallestKey(m *sync.Map) int { + result := math.MaxInt32 + m.Range(func(k, v interface{}) bool { + key := k.(int) + if key < result { + result = key + } + return true + }) + return result +} + +func getLe(key int, schema int32) float64 { + // Here a bit of context about the behavior for the last bucket counting + // regular numbers (called simply "last bucket" below) and the bucket + // counting observations of ±Inf (called "inf bucket" below, with a key + // one higher than that of the "last bucket"): + // + // If we apply the usual formula to the last bucket, its upper bound + // would be calculated as +Inf. The reason is that the max possible + // regular float64 number (math.MaxFloat64) doesn't coincide with one of + // the calculated bucket boundaries. So the calculated boundary has to + // be larger than math.MaxFloat64, and the only float64 larger than + // math.MaxFloat64 is +Inf. However, we want to count actual + // observations of ±Inf in the inf bucket. Therefore, we have to treat + // the upper bound of the last bucket specially and set it to + // math.MaxFloat64. (The upper bound of the inf bucket, with its key + // being one higher than that of the last bucket, naturally comes out as + // +Inf by the usual formula. So that's fine.) + // + // math.MaxFloat64 has a frac of 0.9999999999999999 and an exp of + // 1024. If there were a float64 number following math.MaxFloat64, it + // would have a frac of 1.0 and an exp of 1024, or equivalently a frac + // of 0.5 and an exp of 1025. However, since frac must be smaller than + // 1, and exp must be smaller than 1025, either representation overflows + // a float64. (Which, in turn, is the reason that math.MaxFloat64 is the + // largest possible float64. Q.E.D.) However, the formula for + // calculating the upper bound from the idx and schema of the last + // bucket results in precisely that. It is either frac=1.0 & exp=1024 + // (for schema < 0) or frac=0.5 & exp=1025 (for schema >=0). (This is, + // by the way, a power of two where the exponent itself is a power of + // two, 2¹⁰ in fact, which coinicides with a bucket boundary in all + // schemas.) So these are the special cases we have to catch below. + if schema < 0 { + exp := key << -schema + if exp == 1024 { + // This is the last bucket before the overflow bucket + // (for ±Inf observations). Return math.MaxFloat64 as + // explained above. + return math.MaxFloat64 + } + return math.Ldexp(1, exp) + } + + fracIdx := key & ((1 << schema) - 1) + frac := nativeHistogramBounds[schema][fracIdx] + exp := (key >> schema) + 1 + if frac == 0.5 && exp == 1025 { + // This is the last bucket before the overflow bucket (for ±Inf + // observations). Return math.MaxFloat64 as explained above. + return math.MaxFloat64 + } + return math.Ldexp(frac, exp) +} + +// waitForCooldown returns after the count field in the provided histogramCounts +// has reached the provided count value. +func waitForCooldown(count uint64, counts *histogramCounts) { + for count != atomic.LoadUint64(&counts.count) { + runtime.Gosched() // Let observations get work done. + } +} + +// atomicAddFloat adds the provided float atomically to another float +// represented by the bit pattern the bits pointer is pointing to. +func atomicAddFloat(bits *uint64, v float64) { + for { + loadedBits := atomic.LoadUint64(bits) + newBits := math.Float64bits(math.Float64frombits(loadedBits) + v) + if atomic.CompareAndSwapUint64(bits, loadedBits, newBits) { + break + } + } +} + +// atomicDecUint32 atomically decrements the uint32 p points to. See +// https://pkg.go.dev/sync/atomic#AddUint32 to understand how this is done. +func atomicDecUint32(p *uint32) { + atomic.AddUint32(p, ^uint32(0)) +} + +// addAndResetCounts adds certain fields (count, sum, conventional buckets, zero +// bucket) from the cold counts to the corresponding fields in the hot +// counts. Those fields are then reset to 0 in the cold counts. +func addAndResetCounts(hot, cold *histogramCounts) { + atomic.AddUint64(&hot.count, atomic.LoadUint64(&cold.count)) + atomic.StoreUint64(&cold.count, 0) + coldSum := math.Float64frombits(atomic.LoadUint64(&cold.sumBits)) + atomicAddFloat(&hot.sumBits, coldSum) + atomic.StoreUint64(&cold.sumBits, 0) + for i := range hot.buckets { + atomic.AddUint64(&hot.buckets[i], atomic.LoadUint64(&cold.buckets[i])) + atomic.StoreUint64(&cold.buckets[i], 0) + } + atomic.AddUint64(&hot.nativeHistogramZeroBucket, atomic.LoadUint64(&cold.nativeHistogramZeroBucket)) + atomic.StoreUint64(&cold.nativeHistogramZeroBucket, 0) +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/internal/almost_equal.go b/vendor/github.com/prometheus/client_golang/prometheus/internal/almost_equal.go new file mode 100644 index 0000000..1ed5abe --- /dev/null +++ b/vendor/github.com/prometheus/client_golang/prometheus/internal/almost_equal.go @@ -0,0 +1,60 @@ +// Copyright (c) 2015 Björn Rabenstein +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. +// +// The code in this package is copy/paste to avoid a dependency. Hence this file +// carries the copyright of the original repo. +// https://github.com/beorn7/floats +package internal + +import ( + "math" +) + +// minNormalFloat64 is the smallest positive normal value of type float64. +var minNormalFloat64 = math.Float64frombits(0x0010000000000000) + +// AlmostEqualFloat64 returns true if a and b are equal within a relative error +// of epsilon. See http://floating-point-gui.de/errors/comparison/ for the +// details of the applied method. +func AlmostEqualFloat64(a, b, epsilon float64) bool { + if a == b { + return true + } + absA := math.Abs(a) + absB := math.Abs(b) + diff := math.Abs(a - b) + if a == 0 || b == 0 || absA+absB < minNormalFloat64 { + return diff < epsilon*minNormalFloat64 + } + return diff/math.Min(absA+absB, math.MaxFloat64) < epsilon +} + +// AlmostEqualFloat64s is the slice form of AlmostEqualFloat64. +func AlmostEqualFloat64s(a, b []float64, epsilon float64) bool { + if len(a) != len(b) { + return false + } + for i := range a { + if !AlmostEqualFloat64(a[i], b[i], epsilon) { + return false + } + } + return true +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/internal/difflib.go b/vendor/github.com/prometheus/client_golang/prometheus/internal/difflib.go index fd45cad..fd0750f 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/internal/difflib.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/internal/difflib.go @@ -201,12 +201,15 @@ func (m *SequenceMatcher) isBJunk(s string) bool { // If IsJunk is not defined: // // Return (i,j,k) such that a[i:i+k] is equal to b[j:j+k], where -// alo <= i <= i+k <= ahi -// blo <= j <= j+k <= bhi +// +// alo <= i <= i+k <= ahi +// blo <= j <= j+k <= bhi +// // and for all (i',j',k') meeting those conditions, -// k >= k' -// i <= i' -// and if i == i', j <= j' +// +// k >= k' +// i <= i' +// and if i == i', j <= j' // // In other words, of all maximal matching blocks, return one that // starts earliest in a, and of all those maximal matching blocks that diff --git a/vendor/github.com/prometheus/client_golang/prometheus/labels.go b/vendor/github.com/prometheus/client_golang/prometheus/labels.go index 6eee198..63ff868 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/labels.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/labels.go @@ -25,12 +25,85 @@ import ( // Labels represents a collection of label name -> value mappings. This type is // commonly used with the With(Labels) and GetMetricWith(Labels) methods of // metric vector Collectors, e.g.: -// myVec.With(Labels{"code": "404", "method": "GET"}).Add(42) +// +// myVec.With(Labels{"code": "404", "method": "GET"}).Add(42) // // The other use-case is the specification of constant label pairs in Opts or to // create a Desc. type Labels map[string]string +// ConstrainedLabels represents a label name and its constrain function +// to normalize label values. This type is commonly used when constructing +// metric vector Collectors. +type ConstrainedLabel struct { + Name string + Constraint func(string) string +} + +func (cl ConstrainedLabel) Constrain(v string) string { + if cl.Constraint == nil { + return v + } + return cl.Constraint(v) +} + +// ConstrainableLabels is an interface that allows creating of labels that can +// be optionally constrained. +// +// prometheus.V2().NewCounterVec(CounterVecOpts{ +// CounterOpts: {...}, // Usual CounterOpts fields +// VariableLabels: []ConstrainedLabels{ +// {Name: "A"}, +// {Name: "B", Constraint: func(v string) string { ... }}, +// }, +// }) +type ConstrainableLabels interface { + constrainedLabels() ConstrainedLabels + labelNames() []string +} + +// ConstrainedLabels represents a collection of label name -> constrain function +// to normalize label values. This type is commonly used when constructing +// metric vector Collectors. +type ConstrainedLabels []ConstrainedLabel + +func (cls ConstrainedLabels) constrainedLabels() ConstrainedLabels { + return cls +} + +func (cls ConstrainedLabels) labelNames() []string { + names := make([]string, len(cls)) + for i, label := range cls { + names[i] = label.Name + } + return names +} + +// UnconstrainedLabels represents collection of label without any constraint on +// their value. Thus, it is simply a collection of label names. +// +// UnconstrainedLabels([]string{ "A", "B" }) +// +// is equivalent to +// +// ConstrainedLabels { +// { Name: "A" }, +// { Name: "B" }, +// } +type UnconstrainedLabels []string + +func (uls UnconstrainedLabels) constrainedLabels() ConstrainedLabels { + constrainedLabels := make([]ConstrainedLabel, len(uls)) + for i, l := range uls { + constrainedLabels[i] = ConstrainedLabel{Name: l} + } + return constrainedLabels +} + +func (uls UnconstrainedLabels) labelNames() []string { + return uls +} + // reservedLabelPrefix is a prefix which is not legal in user-supplied // label names. const reservedLabelPrefix = "__" diff --git a/vendor/github.com/prometheus/client_golang/prometheus/metric.go b/vendor/github.com/prometheus/client_golang/prometheus/metric.go index b5119c5..07bbc9d 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/metric.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/metric.go @@ -20,11 +20,9 @@ import ( "strings" "time" - //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. - "github.com/golang/protobuf/proto" - "github.com/prometheus/common/model" - dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/model" + "google.golang.org/protobuf/proto" ) var separatorByteSlice = []byte{model.SeparatorByte} // For convenient use with xxhash. diff --git a/vendor/github.com/prometheus/client_golang/prometheus/registry.go b/vendor/github.com/prometheus/client_golang/prometheus/registry.go index 325f665..44da943 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/registry.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/registry.go @@ -21,18 +21,17 @@ import ( "path/filepath" "runtime" "sort" + "strconv" "strings" "sync" "unicode/utf8" - "github.com/cespare/xxhash/v2" - //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. - "github.com/golang/protobuf/proto" - "github.com/prometheus/common/expfmt" - - dto "github.com/prometheus/client_model/go" - "github.com/prometheus/client_golang/prometheus/internal" + + "github.com/cespare/xxhash/v2" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "google.golang.org/protobuf/proto" ) const ( @@ -252,9 +251,12 @@ func (errs MultiError) MaybeUnwrap() error { } // Registry registers Prometheus collectors, collects their metrics, and gathers -// them into MetricFamilies for exposition. It implements both Registerer and -// Gatherer. The zero value is not usable. Create instances with NewRegistry or -// NewPedanticRegistry. +// them into MetricFamilies for exposition. It implements Registerer, Gatherer, +// and Collector. The zero value is not usable. Create instances with +// NewRegistry or NewPedanticRegistry. +// +// Registry implements Collector to allow it to be used for creating groups of +// metrics. See the Grouping example for how this can be done. type Registry struct { mtx sync.RWMutex collectorsByID map[uint64]Collector // ID is a hash of the descIDs. @@ -556,6 +558,31 @@ func (r *Registry) Gather() ([]*dto.MetricFamily, error) { return internal.NormalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap() } +// Describe implements Collector. +func (r *Registry) Describe(ch chan<- *Desc) { + r.mtx.RLock() + defer r.mtx.RUnlock() + + // Only report the checked Collectors; unchecked collectors don't report any + // Desc. + for _, c := range r.collectorsByID { + c.Describe(ch) + } +} + +// Collect implements Collector. +func (r *Registry) Collect(ch chan<- Metric) { + r.mtx.RLock() + defer r.mtx.RUnlock() + + for _, c := range r.collectorsByID { + c.Collect(ch) + } + for _, c := range r.uncheckedCollectors { + c.Collect(ch) + } +} + // WriteToTextfile calls Gather on the provided Gatherer, encodes the result in the // Prometheus text format, and writes it to a temporary file. Upon success, the // temporary file is renamed to the provided filename. @@ -905,6 +932,10 @@ func checkMetricConsistency( h.WriteString(lp.GetValue()) h.Write(separatorByteSlice) } + if dtoMetric.TimestampMs != nil { + h.WriteString(strconv.FormatInt(*(dtoMetric.TimestampMs), 10)) + h.Write(separatorByteSlice) + } hSum := h.Sum64() if _, exists := metricHashes[hSum]; exists { return fmt.Errorf( @@ -934,7 +965,7 @@ func checkDescConsistency( copy(lpsFromDesc, desc.constLabelPairs) for _, l := range desc.variableLabels { lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{ - Name: proto.String(l), + Name: proto.String(l.Name), }) } if len(lpsFromDesc) != len(dtoMetric.Label) { diff --git a/vendor/github.com/prometheus/client_golang/prometheus/summary.go b/vendor/github.com/prometheus/client_golang/prometheus/summary.go index c5fa8ed..dd35926 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/summary.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/summary.go @@ -22,11 +22,10 @@ import ( "sync/atomic" "time" - "github.com/beorn7/perks/quantile" - //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. - "github.com/golang/protobuf/proto" - dto "github.com/prometheus/client_model/go" + + "github.com/beorn7/perks/quantile" + "google.golang.org/protobuf/proto" ) // quantileLabel is used for the label that defines the quantile in a @@ -148,6 +147,18 @@ type SummaryOpts struct { BufCap uint32 } +// SummaryVecOpts bundles the options to create a SummaryVec metric. +// It is mandatory to set SummaryOpts, see there for mandatory fields. VariableLabels +// is optional and can safely be left to its default value. +type SummaryVecOpts struct { + SummaryOpts + + // VariableLabels are used to partition the metric vector by the given set + // of labels. Each label value will be constrained with the optional Contraint + // function, if provided. + VariableLabels ConstrainableLabels +} + // Problem with the sliding-window decay algorithm... The Merge method of // perk/quantile is actually not working as advertised - and it might be // unfixable, as the underlying algorithm is apparently not capable of merging @@ -178,11 +189,11 @@ func NewSummary(opts SummaryOpts) Summary { func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary { if len(desc.variableLabels) != len(labelValues) { - panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels, labelValues)) + panic(makeInconsistentCardinalityError(desc.fqName, desc.variableLabels.labelNames(), labelValues)) } for _, n := range desc.variableLabels { - if n == quantileLabel { + if n.Name == quantileLabel { panic(errQuantileLabelNotAllowed) } } @@ -530,20 +541,28 @@ type SummaryVec struct { // it is handled by the Prometheus server internally, “quantile” is an illegal // label name. NewSummaryVec will panic if this label name is used. func NewSummaryVec(opts SummaryOpts, labelNames []string) *SummaryVec { - for _, ln := range labelNames { + return V2.NewSummaryVec(SummaryVecOpts{ + SummaryOpts: opts, + VariableLabels: UnconstrainedLabels(labelNames), + }) +} + +// NewSummaryVec creates a new SummaryVec based on the provided SummaryVecOpts. +func (v2) NewSummaryVec(opts SummaryVecOpts) *SummaryVec { + for _, ln := range opts.VariableLabels.labelNames() { if ln == quantileLabel { panic(errQuantileLabelNotAllowed) } } - desc := NewDesc( + desc := V2.NewDesc( BuildFQName(opts.Namespace, opts.Subsystem, opts.Name), opts.Help, - labelNames, + opts.VariableLabels, opts.ConstLabels, ) return &SummaryVec{ MetricVec: NewMetricVec(desc, func(lvs ...string) Metric { - return newSummary(desc, opts, lvs...) + return newSummary(desc, opts.SummaryOpts, lvs...) }), } } @@ -603,7 +622,8 @@ func (v *SummaryVec) GetMetricWith(labels Labels) (Observer, error) { // WithLabelValues works as GetMetricWithLabelValues, but panics where // GetMetricWithLabelValues would have returned an error. Not returning an // error allows shortcuts like -// myVec.WithLabelValues("404", "GET").Observe(42.21) +// +// myVec.WithLabelValues("404", "GET").Observe(42.21) func (v *SummaryVec) WithLabelValues(lvs ...string) Observer { s, err := v.GetMetricWithLabelValues(lvs...) if err != nil { @@ -614,7 +634,8 @@ func (v *SummaryVec) WithLabelValues(lvs ...string) Observer { // With works as GetMetricWith, but panics where GetMetricWithLabels would have // returned an error. Not returning an error allows shortcuts like -// myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Observe(42.21) +// +// myVec.With(prometheus.Labels{"code": "404", "method": "GET"}).Observe(42.21) func (v *SummaryVec) With(labels Labels) Observer { s, err := v.GetMetricWith(labels) if err != nil { @@ -701,7 +722,8 @@ func (s *constSummary) Write(out *dto.Metric) error { // // quantiles maps ranks to quantile values. For example, a median latency of // 0.23s and a 99th percentile latency of 0.56s would be expressed as: -// map[float64]float64{0.5: 0.23, 0.99: 0.56} +// +// map[float64]float64{0.5: 0.23, 0.99: 0.56} // // NewConstSummary returns an error if the length of labelValues is not // consistent with the variable labels in Desc or if Desc is invalid. diff --git a/vendor/github.com/prometheus/client_golang/prometheus/timer.go b/vendor/github.com/prometheus/client_golang/prometheus/timer.go index 8d5f105..52344fe 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/timer.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/timer.go @@ -23,13 +23,24 @@ type Timer struct { } // NewTimer creates a new Timer. The provided Observer is used to observe a -// duration in seconds. Timer is usually used to time a function call in the +// duration in seconds. If the Observer implements ExemplarObserver, passing exemplar +// later on will be also supported. +// Timer is usually used to time a function call in the // following way: -// func TimeMe() { -// timer := NewTimer(myHistogram) -// defer timer.ObserveDuration() -// // Do actual work. -// } +// +// func TimeMe() { +// timer := NewTimer(myHistogram) +// defer timer.ObserveDuration() +// // Do actual work. +// } +// +// or +// +// func TimeMeWithExemplar() { +// timer := NewTimer(myHistogram) +// defer timer.ObserveDurationWithExemplar(exemplar) +// // Do actual work. +// } func NewTimer(o Observer) *Timer { return &Timer{ begin: time.Now(), @@ -52,3 +63,19 @@ func (t *Timer) ObserveDuration() time.Duration { } return d } + +// ObserveDurationWithExemplar is like ObserveDuration, but it will also +// observe exemplar with the duration unless exemplar is nil or provided Observer can't +// be casted to ExemplarObserver. +func (t *Timer) ObserveDurationWithExemplar(exemplar Labels) time.Duration { + d := time.Since(t.begin) + eo, ok := t.observer.(ExemplarObserver) + if ok && exemplar != nil { + eo.ObserveWithExemplar(d.Seconds(), exemplar) + return d + } + if t.observer != nil { + t.observer.Observe(d.Seconds()) + } + return d +} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/value.go b/vendor/github.com/prometheus/client_golang/prometheus/value.go index 2d3abc1..5f6bb80 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/value.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/value.go @@ -19,13 +19,11 @@ import ( "time" "unicode/utf8" - //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. - "github.com/golang/protobuf/proto" - "google.golang.org/protobuf/types/known/timestamppb" - "github.com/prometheus/client_golang/prometheus/internal" dto "github.com/prometheus/client_model/go" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/timestamppb" ) // ValueType is an enumeration of metric types that represent a simple value. @@ -188,9 +186,9 @@ func MakeLabelPairs(desc *Desc, labelValues []string) []*dto.LabelPair { return desc.constLabelPairs } labelPairs := make([]*dto.LabelPair, 0, totalLen) - for i, n := range desc.variableLabels { + for i, l := range desc.variableLabels { labelPairs = append(labelPairs, &dto.LabelPair{ - Name: proto.String(n), + Name: proto.String(l.Name), Value: proto.String(labelValues[i]), }) } diff --git a/vendor/github.com/prometheus/client_golang/prometheus/vec.go b/vendor/github.com/prometheus/client_golang/prometheus/vec.go index 7ae3225..386fb2d 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/vec.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/vec.go @@ -72,6 +72,7 @@ func NewMetricVec(desc *Desc, newMetric func(lvs ...string) Metric) *MetricVec { // with a performance overhead (for creating and processing the Labels map). // See also the CounterVec example. func (m *MetricVec) DeleteLabelValues(lvs ...string) bool { + lvs = constrainLabelValues(m.desc, lvs, m.curry) h, err := m.hashLabelValues(lvs) if err != nil { return false @@ -91,6 +92,7 @@ func (m *MetricVec) DeleteLabelValues(lvs ...string) bool { // This method is used for the same purpose as DeleteLabelValues(...string). See // there for pros and cons of the two methods. func (m *MetricVec) Delete(labels Labels) bool { + labels = constrainLabels(m.desc, labels) h, err := m.hashLabels(labels) if err != nil { return false @@ -106,6 +108,7 @@ func (m *MetricVec) Delete(labels Labels) bool { // Note that curried labels will never be matched if deleting from the curried vector. // To match curried labels with DeletePartialMatch, it must be called on the base vector. func (m *MetricVec) DeletePartialMatch(labels Labels) int { + labels = constrainLabels(m.desc, labels) return m.metricMap.deleteByLabels(labels, m.curry) } @@ -145,10 +148,10 @@ func (m *MetricVec) CurryWith(labels Labels) (*MetricVec, error) { iCurry int ) for i, label := range m.desc.variableLabels { - val, ok := labels[label] + val, ok := labels[label.Name] if iCurry < len(oldCurry) && oldCurry[iCurry].index == i { if ok { - return nil, fmt.Errorf("label name %q is already curried", label) + return nil, fmt.Errorf("label name %q is already curried", label.Name) } newCurry = append(newCurry, oldCurry[iCurry]) iCurry++ @@ -156,7 +159,7 @@ func (m *MetricVec) CurryWith(labels Labels) (*MetricVec, error) { if !ok { continue // Label stays uncurried. } - newCurry = append(newCurry, curriedLabelValue{i, val}) + newCurry = append(newCurry, curriedLabelValue{i, label.Constrain(val)}) } } if l := len(oldCurry) + len(labels) - len(newCurry); l > 0 { @@ -199,6 +202,7 @@ func (m *MetricVec) CurryWith(labels Labels) (*MetricVec, error) { // a wrapper around MetricVec, implementing a vector for a specific Metric // implementation, for example GaugeVec. func (m *MetricVec) GetMetricWithLabelValues(lvs ...string) (Metric, error) { + lvs = constrainLabelValues(m.desc, lvs, m.curry) h, err := m.hashLabelValues(lvs) if err != nil { return nil, err @@ -224,6 +228,7 @@ func (m *MetricVec) GetMetricWithLabelValues(lvs ...string) (Metric, error) { // around MetricVec, implementing a vector for a specific Metric implementation, // for example GaugeVec. func (m *MetricVec) GetMetricWith(labels Labels) (Metric, error) { + labels = constrainLabels(m.desc, labels) h, err := m.hashLabels(labels) if err != nil { return nil, err @@ -266,16 +271,16 @@ func (m *MetricVec) hashLabels(labels Labels) (uint64, error) { iCurry int ) for i, label := range m.desc.variableLabels { - val, ok := labels[label] + val, ok := labels[label.Name] if iCurry < len(curry) && curry[iCurry].index == i { if ok { - return 0, fmt.Errorf("label name %q is already curried", label) + return 0, fmt.Errorf("label name %q is already curried", label.Name) } h = m.hashAdd(h, curry[iCurry].value) iCurry++ } else { if !ok { - return 0, fmt.Errorf("label name %q missing in label map", label) + return 0, fmt.Errorf("label name %q missing in label map", label.Name) } h = m.hashAdd(h, val) } @@ -453,7 +458,7 @@ func valueMatchesVariableOrCurriedValue(targetValue string, index int, values [] func matchPartialLabels(desc *Desc, values []string, labels Labels, curry []curriedLabelValue) bool { for l, v := range labels { // Check if the target label exists in our metrics and get the index. - varLabelIndex, validLabel := indexOf(l, desc.variableLabels) + varLabelIndex, validLabel := indexOf(l, desc.variableLabels.labelNames()) if validLabel { // Check the value of that label against the target value. // We don't consider curried values in partial matches. @@ -605,7 +610,7 @@ func matchLabels(desc *Desc, values []string, labels Labels, curry []curriedLabe iCurry++ continue } - if values[i] != labels[k] { + if values[i] != labels[k.Name] { return false } } @@ -621,7 +626,7 @@ func extractLabelValues(desc *Desc, labels Labels, curry []curriedLabelValue) [] iCurry++ continue } - labelValues[i] = labels[k] + labelValues[i] = labels[k.Name] } return labelValues } @@ -640,3 +645,34 @@ func inlineLabelValues(lvs []string, curry []curriedLabelValue) []string { } return labelValues } + +func constrainLabels(desc *Desc, labels Labels) Labels { + constrainedValues := make(Labels, len(labels)) + for l, v := range labels { + if i, ok := indexOf(l, desc.variableLabels.labelNames()); ok { + constrainedValues[l] = desc.variableLabels[i].Constrain(v) + continue + } + constrainedValues[l] = v + } + return constrainedValues +} + +func constrainLabelValues(desc *Desc, lvs []string, curry []curriedLabelValue) []string { + constrainedValues := make([]string, len(lvs)) + var iCurry, iLVs int + for i := 0; i < len(lvs)+len(curry); i++ { + if iCurry < len(curry) && curry[iCurry].index == i { + iCurry++ + continue + } + + if i < len(desc.variableLabels) { + constrainedValues[iLVs] = desc.variableLabels[i].Constrain(lvs[iLVs]) + } else { + constrainedValues[iLVs] = lvs[iLVs] + } + iLVs++ + } + return constrainedValues +} diff --git a/vendor/github.com/cockroachdb/pebble/vfs/preallocate_generic.go b/vendor/github.com/prometheus/client_golang/prometheus/vnext.go similarity index 51% rename from vendor/github.com/cockroachdb/pebble/vfs/preallocate_generic.go rename to vendor/github.com/prometheus/client_golang/prometheus/vnext.go index 4b86afb..42bc3a8 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/preallocate_generic.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/vnext.go @@ -1,10 +1,9 @@ -// Copyright 2016 The etcd Authors -// +// Copyright 2022 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -12,14 +11,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build !linux -// +build !linux +package prometheus -package vfs +type v2 struct{} -func preallocExtend(fd uintptr, offset, length int64) error { - // It is ok for correctness to no-op file preallocation. WAL recycling is the - // more important mechanism for WAL sync performance and it doesn't rely on - // fallocate or posix_fallocate in order to be effective. - return nil -} +// V2 is a struct that can be referenced to access experimental API that might +// be present in v2 of client golang someday. It offers extended functionality +// of v1 with slightly changed API. It is acceptable to use some pieces from v1 +// and e.g `prometheus.NewGauge` and some from v2 e.g. `prometheus.V2.NewDesc` +// in the same codebase. +var V2 = v2{} diff --git a/vendor/github.com/prometheus/client_golang/prometheus/wrap.go b/vendor/github.com/prometheus/client_golang/prometheus/wrap.go index 1498ee1..25da157 100644 --- a/vendor/github.com/prometheus/client_golang/prometheus/wrap.go +++ b/vendor/github.com/prometheus/client_golang/prometheus/wrap.go @@ -17,12 +17,10 @@ import ( "fmt" "sort" - //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. - "github.com/golang/protobuf/proto" + "github.com/prometheus/client_golang/prometheus/internal" dto "github.com/prometheus/client_model/go" - - "github.com/prometheus/client_golang/prometheus/internal" + "google.golang.org/protobuf/proto" ) // WrapRegistererWith returns a Registerer wrapping the provided @@ -206,7 +204,7 @@ func wrapDesc(desc *Desc, prefix string, labels Labels) *Desc { constLabels[ln] = lv } // NewDesc will do remaining validations. - newDesc := NewDesc(prefix+desc.fqName, desc.help, desc.variableLabels, constLabels) + newDesc := V2.NewDesc(prefix+desc.fqName, desc.help, desc.variableLabels, constLabels) // Propagate errors if there was any. This will override any errer // created by NewDesc above, i.e. earlier errors get precedence. if desc.err != nil { diff --git a/vendor/github.com/prometheus/client_model/go/metrics.pb.go b/vendor/github.com/prometheus/client_model/go/metrics.pb.go index 35904ea..2b5bca4 100644 --- a/vendor/github.com/prometheus/client_model/go/metrics.pb.go +++ b/vendor/github.com/prometheus/client_model/go/metrics.pb.go @@ -1,25 +1,38 @@ +// Copyright 2013 Prometheus Team +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.30.0 +// protoc v3.20.3 // source: io/prometheus/client/metrics.proto package io_prometheus_client import ( - fmt "fmt" - proto "github.com/golang/protobuf/proto" - timestamp "github.com/golang/protobuf/ptypes/timestamp" - math "math" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" ) -// Reference imports to suppress errors if they are not otherwise used. -var _ = proto.Marshal -var _ = fmt.Errorf -var _ = math.Inf - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the proto package it is being compiled against. -// A compilation error at this line likely means your copy of the -// proto package needs to be updated. -const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) type MetricType int32 @@ -38,23 +51,25 @@ const ( MetricType_GAUGE_HISTOGRAM MetricType = 5 ) -var MetricType_name = map[int32]string{ - 0: "COUNTER", - 1: "GAUGE", - 2: "SUMMARY", - 3: "UNTYPED", - 4: "HISTOGRAM", - 5: "GAUGE_HISTOGRAM", -} - -var MetricType_value = map[string]int32{ - "COUNTER": 0, - "GAUGE": 1, - "SUMMARY": 2, - "UNTYPED": 3, - "HISTOGRAM": 4, - "GAUGE_HISTOGRAM": 5, -} +// Enum value maps for MetricType. +var ( + MetricType_name = map[int32]string{ + 0: "COUNTER", + 1: "GAUGE", + 2: "SUMMARY", + 3: "UNTYPED", + 4: "HISTOGRAM", + 5: "GAUGE_HISTOGRAM", + } + MetricType_value = map[string]int32{ + "COUNTER": 0, + "GAUGE": 1, + "SUMMARY": 2, + "UNTYPED": 3, + "HISTOGRAM": 4, + "GAUGE_HISTOGRAM": 5, + } +) func (x MetricType) Enum() *MetricType { p := new(MetricType) @@ -63,449 +78,519 @@ func (x MetricType) Enum() *MetricType { } func (x MetricType) String() string { - return proto.EnumName(MetricType_name, int32(x)) + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) } -func (x *MetricType) UnmarshalJSON(data []byte) error { - value, err := proto.UnmarshalJSONEnum(MetricType_value, data, "MetricType") +func (MetricType) Descriptor() protoreflect.EnumDescriptor { + return file_io_prometheus_client_metrics_proto_enumTypes[0].Descriptor() +} + +func (MetricType) Type() protoreflect.EnumType { + return &file_io_prometheus_client_metrics_proto_enumTypes[0] +} + +func (x MetricType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *MetricType) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) if err != nil { return err } - *x = MetricType(value) + *x = MetricType(num) return nil } +// Deprecated: Use MetricType.Descriptor instead. func (MetricType) EnumDescriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{0} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{0} } type LabelPair struct { - Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - Value *string `protobuf:"bytes,2,opt,name=value" json:"value,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Value *string `protobuf:"bytes,2,opt,name=value" json:"value,omitempty"` } -func (m *LabelPair) Reset() { *m = LabelPair{} } -func (m *LabelPair) String() string { return proto.CompactTextString(m) } -func (*LabelPair) ProtoMessage() {} +func (x *LabelPair) Reset() { + *x = LabelPair{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *LabelPair) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LabelPair) ProtoMessage() {} + +func (x *LabelPair) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use LabelPair.ProtoReflect.Descriptor instead. func (*LabelPair) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{0} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{0} } -func (m *LabelPair) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_LabelPair.Unmarshal(m, b) -} -func (m *LabelPair) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_LabelPair.Marshal(b, m, deterministic) -} -func (m *LabelPair) XXX_Merge(src proto.Message) { - xxx_messageInfo_LabelPair.Merge(m, src) -} -func (m *LabelPair) XXX_Size() int { - return xxx_messageInfo_LabelPair.Size(m) -} -func (m *LabelPair) XXX_DiscardUnknown() { - xxx_messageInfo_LabelPair.DiscardUnknown(m) -} - -var xxx_messageInfo_LabelPair proto.InternalMessageInfo - -func (m *LabelPair) GetName() string { - if m != nil && m.Name != nil { - return *m.Name +func (x *LabelPair) GetName() string { + if x != nil && x.Name != nil { + return *x.Name } return "" } -func (m *LabelPair) GetValue() string { - if m != nil && m.Value != nil { - return *m.Value +func (x *LabelPair) GetValue() string { + if x != nil && x.Value != nil { + return *x.Value } return "" } type Gauge struct { - Value *float64 `protobuf:"fixed64,1,opt,name=value" json:"value,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Value *float64 `protobuf:"fixed64,1,opt,name=value" json:"value,omitempty"` } -func (m *Gauge) Reset() { *m = Gauge{} } -func (m *Gauge) String() string { return proto.CompactTextString(m) } -func (*Gauge) ProtoMessage() {} +func (x *Gauge) Reset() { + *x = Gauge{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Gauge) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Gauge) ProtoMessage() {} + +func (x *Gauge) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Gauge.ProtoReflect.Descriptor instead. func (*Gauge) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{1} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{1} } -func (m *Gauge) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Gauge.Unmarshal(m, b) -} -func (m *Gauge) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Gauge.Marshal(b, m, deterministic) -} -func (m *Gauge) XXX_Merge(src proto.Message) { - xxx_messageInfo_Gauge.Merge(m, src) -} -func (m *Gauge) XXX_Size() int { - return xxx_messageInfo_Gauge.Size(m) -} -func (m *Gauge) XXX_DiscardUnknown() { - xxx_messageInfo_Gauge.DiscardUnknown(m) -} - -var xxx_messageInfo_Gauge proto.InternalMessageInfo - -func (m *Gauge) GetValue() float64 { - if m != nil && m.Value != nil { - return *m.Value +func (x *Gauge) GetValue() float64 { + if x != nil && x.Value != nil { + return *x.Value } return 0 } type Counter struct { - Value *float64 `protobuf:"fixed64,1,opt,name=value" json:"value,omitempty"` - Exemplar *Exemplar `protobuf:"bytes,2,opt,name=exemplar" json:"exemplar,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Value *float64 `protobuf:"fixed64,1,opt,name=value" json:"value,omitempty"` + Exemplar *Exemplar `protobuf:"bytes,2,opt,name=exemplar" json:"exemplar,omitempty"` } -func (m *Counter) Reset() { *m = Counter{} } -func (m *Counter) String() string { return proto.CompactTextString(m) } -func (*Counter) ProtoMessage() {} +func (x *Counter) Reset() { + *x = Counter{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Counter) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Counter) ProtoMessage() {} + +func (x *Counter) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Counter.ProtoReflect.Descriptor instead. func (*Counter) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{2} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{2} } -func (m *Counter) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Counter.Unmarshal(m, b) -} -func (m *Counter) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Counter.Marshal(b, m, deterministic) -} -func (m *Counter) XXX_Merge(src proto.Message) { - xxx_messageInfo_Counter.Merge(m, src) -} -func (m *Counter) XXX_Size() int { - return xxx_messageInfo_Counter.Size(m) -} -func (m *Counter) XXX_DiscardUnknown() { - xxx_messageInfo_Counter.DiscardUnknown(m) -} - -var xxx_messageInfo_Counter proto.InternalMessageInfo - -func (m *Counter) GetValue() float64 { - if m != nil && m.Value != nil { - return *m.Value +func (x *Counter) GetValue() float64 { + if x != nil && x.Value != nil { + return *x.Value } return 0 } -func (m *Counter) GetExemplar() *Exemplar { - if m != nil { - return m.Exemplar +func (x *Counter) GetExemplar() *Exemplar { + if x != nil { + return x.Exemplar } return nil } type Quantile struct { - Quantile *float64 `protobuf:"fixed64,1,opt,name=quantile" json:"quantile,omitempty"` - Value *float64 `protobuf:"fixed64,2,opt,name=value" json:"value,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Quantile *float64 `protobuf:"fixed64,1,opt,name=quantile" json:"quantile,omitempty"` + Value *float64 `protobuf:"fixed64,2,opt,name=value" json:"value,omitempty"` } -func (m *Quantile) Reset() { *m = Quantile{} } -func (m *Quantile) String() string { return proto.CompactTextString(m) } -func (*Quantile) ProtoMessage() {} +func (x *Quantile) Reset() { + *x = Quantile{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Quantile) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Quantile) ProtoMessage() {} + +func (x *Quantile) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Quantile.ProtoReflect.Descriptor instead. func (*Quantile) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{3} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{3} } -func (m *Quantile) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Quantile.Unmarshal(m, b) -} -func (m *Quantile) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Quantile.Marshal(b, m, deterministic) -} -func (m *Quantile) XXX_Merge(src proto.Message) { - xxx_messageInfo_Quantile.Merge(m, src) -} -func (m *Quantile) XXX_Size() int { - return xxx_messageInfo_Quantile.Size(m) -} -func (m *Quantile) XXX_DiscardUnknown() { - xxx_messageInfo_Quantile.DiscardUnknown(m) -} - -var xxx_messageInfo_Quantile proto.InternalMessageInfo - -func (m *Quantile) GetQuantile() float64 { - if m != nil && m.Quantile != nil { - return *m.Quantile +func (x *Quantile) GetQuantile() float64 { + if x != nil && x.Quantile != nil { + return *x.Quantile } return 0 } -func (m *Quantile) GetValue() float64 { - if m != nil && m.Value != nil { - return *m.Value +func (x *Quantile) GetValue() float64 { + if x != nil && x.Value != nil { + return *x.Value } return 0 } type Summary struct { - SampleCount *uint64 `protobuf:"varint,1,opt,name=sample_count,json=sampleCount" json:"sample_count,omitempty"` - SampleSum *float64 `protobuf:"fixed64,2,opt,name=sample_sum,json=sampleSum" json:"sample_sum,omitempty"` - Quantile []*Quantile `protobuf:"bytes,3,rep,name=quantile" json:"quantile,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + SampleCount *uint64 `protobuf:"varint,1,opt,name=sample_count,json=sampleCount" json:"sample_count,omitempty"` + SampleSum *float64 `protobuf:"fixed64,2,opt,name=sample_sum,json=sampleSum" json:"sample_sum,omitempty"` + Quantile []*Quantile `protobuf:"bytes,3,rep,name=quantile" json:"quantile,omitempty"` } -func (m *Summary) Reset() { *m = Summary{} } -func (m *Summary) String() string { return proto.CompactTextString(m) } -func (*Summary) ProtoMessage() {} +func (x *Summary) Reset() { + *x = Summary{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Summary) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Summary) ProtoMessage() {} + +func (x *Summary) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[4] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Summary.ProtoReflect.Descriptor instead. func (*Summary) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{4} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{4} } -func (m *Summary) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Summary.Unmarshal(m, b) -} -func (m *Summary) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Summary.Marshal(b, m, deterministic) -} -func (m *Summary) XXX_Merge(src proto.Message) { - xxx_messageInfo_Summary.Merge(m, src) -} -func (m *Summary) XXX_Size() int { - return xxx_messageInfo_Summary.Size(m) -} -func (m *Summary) XXX_DiscardUnknown() { - xxx_messageInfo_Summary.DiscardUnknown(m) -} - -var xxx_messageInfo_Summary proto.InternalMessageInfo - -func (m *Summary) GetSampleCount() uint64 { - if m != nil && m.SampleCount != nil { - return *m.SampleCount +func (x *Summary) GetSampleCount() uint64 { + if x != nil && x.SampleCount != nil { + return *x.SampleCount } return 0 } -func (m *Summary) GetSampleSum() float64 { - if m != nil && m.SampleSum != nil { - return *m.SampleSum +func (x *Summary) GetSampleSum() float64 { + if x != nil && x.SampleSum != nil { + return *x.SampleSum } return 0 } -func (m *Summary) GetQuantile() []*Quantile { - if m != nil { - return m.Quantile +func (x *Summary) GetQuantile() []*Quantile { + if x != nil { + return x.Quantile } return nil } type Untyped struct { - Value *float64 `protobuf:"fixed64,1,opt,name=value" json:"value,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Value *float64 `protobuf:"fixed64,1,opt,name=value" json:"value,omitempty"` } -func (m *Untyped) Reset() { *m = Untyped{} } -func (m *Untyped) String() string { return proto.CompactTextString(m) } -func (*Untyped) ProtoMessage() {} +func (x *Untyped) Reset() { + *x = Untyped{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Untyped) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Untyped) ProtoMessage() {} + +func (x *Untyped) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[5] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Untyped.ProtoReflect.Descriptor instead. func (*Untyped) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{5} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{5} } -func (m *Untyped) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Untyped.Unmarshal(m, b) -} -func (m *Untyped) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Untyped.Marshal(b, m, deterministic) -} -func (m *Untyped) XXX_Merge(src proto.Message) { - xxx_messageInfo_Untyped.Merge(m, src) -} -func (m *Untyped) XXX_Size() int { - return xxx_messageInfo_Untyped.Size(m) -} -func (m *Untyped) XXX_DiscardUnknown() { - xxx_messageInfo_Untyped.DiscardUnknown(m) -} - -var xxx_messageInfo_Untyped proto.InternalMessageInfo - -func (m *Untyped) GetValue() float64 { - if m != nil && m.Value != nil { - return *m.Value +func (x *Untyped) GetValue() float64 { + if x != nil && x.Value != nil { + return *x.Value } return 0 } type Histogram struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + SampleCount *uint64 `protobuf:"varint,1,opt,name=sample_count,json=sampleCount" json:"sample_count,omitempty"` - SampleCountFloat *float64 `protobuf:"fixed64,4,opt,name=sample_count_float,json=sampleCountFloat" json:"sample_count_float,omitempty"` + SampleCountFloat *float64 `protobuf:"fixed64,4,opt,name=sample_count_float,json=sampleCountFloat" json:"sample_count_float,omitempty"` // Overrides sample_count if > 0. SampleSum *float64 `protobuf:"fixed64,2,opt,name=sample_sum,json=sampleSum" json:"sample_sum,omitempty"` // Buckets for the conventional histogram. - Bucket []*Bucket `protobuf:"bytes,3,rep,name=bucket" json:"bucket,omitempty"` + Bucket []*Bucket `protobuf:"bytes,3,rep,name=bucket" json:"bucket,omitempty"` // Ordered in increasing order of upper_bound, +Inf bucket is optional. // schema defines the bucket schema. Currently, valid numbers are -4 <= n <= 8. // They are all for base-2 bucket schemas, where 1 is a bucket boundary in each case, and // then each power of two is divided into 2^n logarithmic buckets. // Or in other words, each bucket boundary is the previous boundary times 2^(2^-n). // In the future, more bucket schemas may be added using numbers < -4 or > 8. Schema *int32 `protobuf:"zigzag32,5,opt,name=schema" json:"schema,omitempty"` - ZeroThreshold *float64 `protobuf:"fixed64,6,opt,name=zero_threshold,json=zeroThreshold" json:"zero_threshold,omitempty"` - ZeroCount *uint64 `protobuf:"varint,7,opt,name=zero_count,json=zeroCount" json:"zero_count,omitempty"` - ZeroCountFloat *float64 `protobuf:"fixed64,8,opt,name=zero_count_float,json=zeroCountFloat" json:"zero_count_float,omitempty"` + ZeroThreshold *float64 `protobuf:"fixed64,6,opt,name=zero_threshold,json=zeroThreshold" json:"zero_threshold,omitempty"` // Breadth of the zero bucket. + ZeroCount *uint64 `protobuf:"varint,7,opt,name=zero_count,json=zeroCount" json:"zero_count,omitempty"` // Count in zero bucket. + ZeroCountFloat *float64 `protobuf:"fixed64,8,opt,name=zero_count_float,json=zeroCountFloat" json:"zero_count_float,omitempty"` // Overrides sb_zero_count if > 0. // Negative buckets for the native histogram. NegativeSpan []*BucketSpan `protobuf:"bytes,9,rep,name=negative_span,json=negativeSpan" json:"negative_span,omitempty"` // Use either "negative_delta" or "negative_count", the former for // regular histograms with integer counts, the latter for float // histograms. - NegativeDelta []int64 `protobuf:"zigzag64,10,rep,name=negative_delta,json=negativeDelta" json:"negative_delta,omitempty"` - NegativeCount []float64 `protobuf:"fixed64,11,rep,name=negative_count,json=negativeCount" json:"negative_count,omitempty"` + NegativeDelta []int64 `protobuf:"zigzag64,10,rep,name=negative_delta,json=negativeDelta" json:"negative_delta,omitempty"` // Count delta of each bucket compared to previous one (or to zero for 1st bucket). + NegativeCount []float64 `protobuf:"fixed64,11,rep,name=negative_count,json=negativeCount" json:"negative_count,omitempty"` // Absolute count of each bucket. // Positive buckets for the native histogram. PositiveSpan []*BucketSpan `protobuf:"bytes,12,rep,name=positive_span,json=positiveSpan" json:"positive_span,omitempty"` // Use either "positive_delta" or "positive_count", the former for // regular histograms with integer counts, the latter for float // histograms. - PositiveDelta []int64 `protobuf:"zigzag64,13,rep,name=positive_delta,json=positiveDelta" json:"positive_delta,omitempty"` - PositiveCount []float64 `protobuf:"fixed64,14,rep,name=positive_count,json=positiveCount" json:"positive_count,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + PositiveDelta []int64 `protobuf:"zigzag64,13,rep,name=positive_delta,json=positiveDelta" json:"positive_delta,omitempty"` // Count delta of each bucket compared to previous one (or to zero for 1st bucket). + PositiveCount []float64 `protobuf:"fixed64,14,rep,name=positive_count,json=positiveCount" json:"positive_count,omitempty"` // Absolute count of each bucket. } -func (m *Histogram) Reset() { *m = Histogram{} } -func (m *Histogram) String() string { return proto.CompactTextString(m) } -func (*Histogram) ProtoMessage() {} +func (x *Histogram) Reset() { + *x = Histogram{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Histogram) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Histogram) ProtoMessage() {} + +func (x *Histogram) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[6] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Histogram.ProtoReflect.Descriptor instead. func (*Histogram) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{6} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{6} } -func (m *Histogram) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Histogram.Unmarshal(m, b) -} -func (m *Histogram) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Histogram.Marshal(b, m, deterministic) -} -func (m *Histogram) XXX_Merge(src proto.Message) { - xxx_messageInfo_Histogram.Merge(m, src) -} -func (m *Histogram) XXX_Size() int { - return xxx_messageInfo_Histogram.Size(m) -} -func (m *Histogram) XXX_DiscardUnknown() { - xxx_messageInfo_Histogram.DiscardUnknown(m) -} - -var xxx_messageInfo_Histogram proto.InternalMessageInfo - -func (m *Histogram) GetSampleCount() uint64 { - if m != nil && m.SampleCount != nil { - return *m.SampleCount +func (x *Histogram) GetSampleCount() uint64 { + if x != nil && x.SampleCount != nil { + return *x.SampleCount } return 0 } -func (m *Histogram) GetSampleCountFloat() float64 { - if m != nil && m.SampleCountFloat != nil { - return *m.SampleCountFloat +func (x *Histogram) GetSampleCountFloat() float64 { + if x != nil && x.SampleCountFloat != nil { + return *x.SampleCountFloat } return 0 } -func (m *Histogram) GetSampleSum() float64 { - if m != nil && m.SampleSum != nil { - return *m.SampleSum +func (x *Histogram) GetSampleSum() float64 { + if x != nil && x.SampleSum != nil { + return *x.SampleSum } return 0 } -func (m *Histogram) GetBucket() []*Bucket { - if m != nil { - return m.Bucket +func (x *Histogram) GetBucket() []*Bucket { + if x != nil { + return x.Bucket } return nil } -func (m *Histogram) GetSchema() int32 { - if m != nil && m.Schema != nil { - return *m.Schema +func (x *Histogram) GetSchema() int32 { + if x != nil && x.Schema != nil { + return *x.Schema } return 0 } -func (m *Histogram) GetZeroThreshold() float64 { - if m != nil && m.ZeroThreshold != nil { - return *m.ZeroThreshold +func (x *Histogram) GetZeroThreshold() float64 { + if x != nil && x.ZeroThreshold != nil { + return *x.ZeroThreshold } return 0 } -func (m *Histogram) GetZeroCount() uint64 { - if m != nil && m.ZeroCount != nil { - return *m.ZeroCount +func (x *Histogram) GetZeroCount() uint64 { + if x != nil && x.ZeroCount != nil { + return *x.ZeroCount } return 0 } -func (m *Histogram) GetZeroCountFloat() float64 { - if m != nil && m.ZeroCountFloat != nil { - return *m.ZeroCountFloat +func (x *Histogram) GetZeroCountFloat() float64 { + if x != nil && x.ZeroCountFloat != nil { + return *x.ZeroCountFloat } return 0 } -func (m *Histogram) GetNegativeSpan() []*BucketSpan { - if m != nil { - return m.NegativeSpan +func (x *Histogram) GetNegativeSpan() []*BucketSpan { + if x != nil { + return x.NegativeSpan } return nil } -func (m *Histogram) GetNegativeDelta() []int64 { - if m != nil { - return m.NegativeDelta +func (x *Histogram) GetNegativeDelta() []int64 { + if x != nil { + return x.NegativeDelta } return nil } -func (m *Histogram) GetNegativeCount() []float64 { - if m != nil { - return m.NegativeCount +func (x *Histogram) GetNegativeCount() []float64 { + if x != nil { + return x.NegativeCount } return nil } -func (m *Histogram) GetPositiveSpan() []*BucketSpan { - if m != nil { - return m.PositiveSpan +func (x *Histogram) GetPositiveSpan() []*BucketSpan { + if x != nil { + return x.PositiveSpan } return nil } -func (m *Histogram) GetPositiveDelta() []int64 { - if m != nil { - return m.PositiveDelta +func (x *Histogram) GetPositiveDelta() []int64 { + if x != nil { + return x.PositiveDelta } return nil } -func (m *Histogram) GetPositiveCount() []float64 { - if m != nil { - return m.PositiveCount +func (x *Histogram) GetPositiveCount() []float64 { + if x != nil { + return x.PositiveCount } return nil } @@ -513,64 +598,72 @@ func (m *Histogram) GetPositiveCount() []float64 { // A Bucket of a conventional histogram, each of which is treated as // an individual counter-like time series by Prometheus. type Bucket struct { - CumulativeCount *uint64 `protobuf:"varint,1,opt,name=cumulative_count,json=cumulativeCount" json:"cumulative_count,omitempty"` - CumulativeCountFloat *float64 `protobuf:"fixed64,4,opt,name=cumulative_count_float,json=cumulativeCountFloat" json:"cumulative_count_float,omitempty"` - UpperBound *float64 `protobuf:"fixed64,2,opt,name=upper_bound,json=upperBound" json:"upper_bound,omitempty"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + CumulativeCount *uint64 `protobuf:"varint,1,opt,name=cumulative_count,json=cumulativeCount" json:"cumulative_count,omitempty"` // Cumulative in increasing order. + CumulativeCountFloat *float64 `protobuf:"fixed64,4,opt,name=cumulative_count_float,json=cumulativeCountFloat" json:"cumulative_count_float,omitempty"` // Overrides cumulative_count if > 0. + UpperBound *float64 `protobuf:"fixed64,2,opt,name=upper_bound,json=upperBound" json:"upper_bound,omitempty"` // Inclusive. Exemplar *Exemplar `protobuf:"bytes,3,opt,name=exemplar" json:"exemplar,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` } -func (m *Bucket) Reset() { *m = Bucket{} } -func (m *Bucket) String() string { return proto.CompactTextString(m) } -func (*Bucket) ProtoMessage() {} +func (x *Bucket) Reset() { + *x = Bucket{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Bucket) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Bucket) ProtoMessage() {} + +func (x *Bucket) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[7] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Bucket.ProtoReflect.Descriptor instead. func (*Bucket) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{7} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{7} } -func (m *Bucket) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Bucket.Unmarshal(m, b) -} -func (m *Bucket) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Bucket.Marshal(b, m, deterministic) -} -func (m *Bucket) XXX_Merge(src proto.Message) { - xxx_messageInfo_Bucket.Merge(m, src) -} -func (m *Bucket) XXX_Size() int { - return xxx_messageInfo_Bucket.Size(m) -} -func (m *Bucket) XXX_DiscardUnknown() { - xxx_messageInfo_Bucket.DiscardUnknown(m) -} - -var xxx_messageInfo_Bucket proto.InternalMessageInfo - -func (m *Bucket) GetCumulativeCount() uint64 { - if m != nil && m.CumulativeCount != nil { - return *m.CumulativeCount +func (x *Bucket) GetCumulativeCount() uint64 { + if x != nil && x.CumulativeCount != nil { + return *x.CumulativeCount } return 0 } -func (m *Bucket) GetCumulativeCountFloat() float64 { - if m != nil && m.CumulativeCountFloat != nil { - return *m.CumulativeCountFloat +func (x *Bucket) GetCumulativeCountFloat() float64 { + if x != nil && x.CumulativeCountFloat != nil { + return *x.CumulativeCountFloat } return 0 } -func (m *Bucket) GetUpperBound() float64 { - if m != nil && m.UpperBound != nil { - return *m.UpperBound +func (x *Bucket) GetUpperBound() float64 { + if x != nil && x.UpperBound != nil { + return *x.UpperBound } return 0 } -func (m *Bucket) GetExemplar() *Exemplar { - if m != nil { - return m.Exemplar +func (x *Bucket) GetExemplar() *Exemplar { + if x != nil { + return x.Exemplar } return nil } @@ -582,333 +675,658 @@ func (m *Bucket) GetExemplar() *Exemplar { // structured here (with all the buckets in a single array separate // from the Spans). type BucketSpan struct { - Offset *int32 `protobuf:"zigzag32,1,opt,name=offset" json:"offset,omitempty"` - Length *uint32 `protobuf:"varint,2,opt,name=length" json:"length,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Offset *int32 `protobuf:"zigzag32,1,opt,name=offset" json:"offset,omitempty"` // Gap to previous span, or starting point for 1st span (which can be negative). + Length *uint32 `protobuf:"varint,2,opt,name=length" json:"length,omitempty"` // Length of consecutive buckets. } -func (m *BucketSpan) Reset() { *m = BucketSpan{} } -func (m *BucketSpan) String() string { return proto.CompactTextString(m) } -func (*BucketSpan) ProtoMessage() {} +func (x *BucketSpan) Reset() { + *x = BucketSpan{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BucketSpan) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BucketSpan) ProtoMessage() {} + +func (x *BucketSpan) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[8] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BucketSpan.ProtoReflect.Descriptor instead. func (*BucketSpan) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{8} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{8} } -func (m *BucketSpan) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_BucketSpan.Unmarshal(m, b) -} -func (m *BucketSpan) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_BucketSpan.Marshal(b, m, deterministic) -} -func (m *BucketSpan) XXX_Merge(src proto.Message) { - xxx_messageInfo_BucketSpan.Merge(m, src) -} -func (m *BucketSpan) XXX_Size() int { - return xxx_messageInfo_BucketSpan.Size(m) -} -func (m *BucketSpan) XXX_DiscardUnknown() { - xxx_messageInfo_BucketSpan.DiscardUnknown(m) -} - -var xxx_messageInfo_BucketSpan proto.InternalMessageInfo - -func (m *BucketSpan) GetOffset() int32 { - if m != nil && m.Offset != nil { - return *m.Offset +func (x *BucketSpan) GetOffset() int32 { + if x != nil && x.Offset != nil { + return *x.Offset } return 0 } -func (m *BucketSpan) GetLength() uint32 { - if m != nil && m.Length != nil { - return *m.Length +func (x *BucketSpan) GetLength() uint32 { + if x != nil && x.Length != nil { + return *x.Length } return 0 } type Exemplar struct { - Label []*LabelPair `protobuf:"bytes,1,rep,name=label" json:"label,omitempty"` - Value *float64 `protobuf:"fixed64,2,opt,name=value" json:"value,omitempty"` - Timestamp *timestamp.Timestamp `protobuf:"bytes,3,opt,name=timestamp" json:"timestamp,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Label []*LabelPair `protobuf:"bytes,1,rep,name=label" json:"label,omitempty"` + Value *float64 `protobuf:"fixed64,2,opt,name=value" json:"value,omitempty"` + Timestamp *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=timestamp" json:"timestamp,omitempty"` // OpenMetrics-style. } -func (m *Exemplar) Reset() { *m = Exemplar{} } -func (m *Exemplar) String() string { return proto.CompactTextString(m) } -func (*Exemplar) ProtoMessage() {} +func (x *Exemplar) Reset() { + *x = Exemplar{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Exemplar) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Exemplar) ProtoMessage() {} + +func (x *Exemplar) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Exemplar.ProtoReflect.Descriptor instead. func (*Exemplar) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{9} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{9} } -func (m *Exemplar) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Exemplar.Unmarshal(m, b) -} -func (m *Exemplar) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Exemplar.Marshal(b, m, deterministic) -} -func (m *Exemplar) XXX_Merge(src proto.Message) { - xxx_messageInfo_Exemplar.Merge(m, src) -} -func (m *Exemplar) XXX_Size() int { - return xxx_messageInfo_Exemplar.Size(m) -} -func (m *Exemplar) XXX_DiscardUnknown() { - xxx_messageInfo_Exemplar.DiscardUnknown(m) -} - -var xxx_messageInfo_Exemplar proto.InternalMessageInfo - -func (m *Exemplar) GetLabel() []*LabelPair { - if m != nil { - return m.Label +func (x *Exemplar) GetLabel() []*LabelPair { + if x != nil { + return x.Label } return nil } -func (m *Exemplar) GetValue() float64 { - if m != nil && m.Value != nil { - return *m.Value +func (x *Exemplar) GetValue() float64 { + if x != nil && x.Value != nil { + return *x.Value } return 0 } -func (m *Exemplar) GetTimestamp() *timestamp.Timestamp { - if m != nil { - return m.Timestamp +func (x *Exemplar) GetTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.Timestamp } return nil } type Metric struct { - Label []*LabelPair `protobuf:"bytes,1,rep,name=label" json:"label,omitempty"` - Gauge *Gauge `protobuf:"bytes,2,opt,name=gauge" json:"gauge,omitempty"` - Counter *Counter `protobuf:"bytes,3,opt,name=counter" json:"counter,omitempty"` - Summary *Summary `protobuf:"bytes,4,opt,name=summary" json:"summary,omitempty"` - Untyped *Untyped `protobuf:"bytes,5,opt,name=untyped" json:"untyped,omitempty"` - Histogram *Histogram `protobuf:"bytes,7,opt,name=histogram" json:"histogram,omitempty"` - TimestampMs *int64 `protobuf:"varint,6,opt,name=timestamp_ms,json=timestampMs" json:"timestamp_ms,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Label []*LabelPair `protobuf:"bytes,1,rep,name=label" json:"label,omitempty"` + Gauge *Gauge `protobuf:"bytes,2,opt,name=gauge" json:"gauge,omitempty"` + Counter *Counter `protobuf:"bytes,3,opt,name=counter" json:"counter,omitempty"` + Summary *Summary `protobuf:"bytes,4,opt,name=summary" json:"summary,omitempty"` + Untyped *Untyped `protobuf:"bytes,5,opt,name=untyped" json:"untyped,omitempty"` + Histogram *Histogram `protobuf:"bytes,7,opt,name=histogram" json:"histogram,omitempty"` + TimestampMs *int64 `protobuf:"varint,6,opt,name=timestamp_ms,json=timestampMs" json:"timestamp_ms,omitempty"` } -func (m *Metric) Reset() { *m = Metric{} } -func (m *Metric) String() string { return proto.CompactTextString(m) } -func (*Metric) ProtoMessage() {} +func (x *Metric) Reset() { + *x = Metric{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *Metric) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Metric) ProtoMessage() {} + +func (x *Metric) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Metric.ProtoReflect.Descriptor instead. func (*Metric) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{10} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{10} } -func (m *Metric) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_Metric.Unmarshal(m, b) -} -func (m *Metric) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_Metric.Marshal(b, m, deterministic) -} -func (m *Metric) XXX_Merge(src proto.Message) { - xxx_messageInfo_Metric.Merge(m, src) -} -func (m *Metric) XXX_Size() int { - return xxx_messageInfo_Metric.Size(m) -} -func (m *Metric) XXX_DiscardUnknown() { - xxx_messageInfo_Metric.DiscardUnknown(m) -} - -var xxx_messageInfo_Metric proto.InternalMessageInfo - -func (m *Metric) GetLabel() []*LabelPair { - if m != nil { - return m.Label +func (x *Metric) GetLabel() []*LabelPair { + if x != nil { + return x.Label } return nil } -func (m *Metric) GetGauge() *Gauge { - if m != nil { - return m.Gauge +func (x *Metric) GetGauge() *Gauge { + if x != nil { + return x.Gauge } return nil } -func (m *Metric) GetCounter() *Counter { - if m != nil { - return m.Counter +func (x *Metric) GetCounter() *Counter { + if x != nil { + return x.Counter } return nil } -func (m *Metric) GetSummary() *Summary { - if m != nil { - return m.Summary +func (x *Metric) GetSummary() *Summary { + if x != nil { + return x.Summary } return nil } -func (m *Metric) GetUntyped() *Untyped { - if m != nil { - return m.Untyped +func (x *Metric) GetUntyped() *Untyped { + if x != nil { + return x.Untyped } return nil } -func (m *Metric) GetHistogram() *Histogram { - if m != nil { - return m.Histogram +func (x *Metric) GetHistogram() *Histogram { + if x != nil { + return x.Histogram } return nil } -func (m *Metric) GetTimestampMs() int64 { - if m != nil && m.TimestampMs != nil { - return *m.TimestampMs +func (x *Metric) GetTimestampMs() int64 { + if x != nil && x.TimestampMs != nil { + return *x.TimestampMs } return 0 } type MetricFamily struct { - Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` - Help *string `protobuf:"bytes,2,opt,name=help" json:"help,omitempty"` - Type *MetricType `protobuf:"varint,3,opt,name=type,enum=io.prometheus.client.MetricType" json:"type,omitempty"` - Metric []*Metric `protobuf:"bytes,4,rep,name=metric" json:"metric,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name *string `protobuf:"bytes,1,opt,name=name" json:"name,omitempty"` + Help *string `protobuf:"bytes,2,opt,name=help" json:"help,omitempty"` + Type *MetricType `protobuf:"varint,3,opt,name=type,enum=io.prometheus.client.MetricType" json:"type,omitempty"` + Metric []*Metric `protobuf:"bytes,4,rep,name=metric" json:"metric,omitempty"` } -func (m *MetricFamily) Reset() { *m = MetricFamily{} } -func (m *MetricFamily) String() string { return proto.CompactTextString(m) } -func (*MetricFamily) ProtoMessage() {} +func (x *MetricFamily) Reset() { + *x = MetricFamily{} + if protoimpl.UnsafeEnabled { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *MetricFamily) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*MetricFamily) ProtoMessage() {} + +func (x *MetricFamily) ProtoReflect() protoreflect.Message { + mi := &file_io_prometheus_client_metrics_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use MetricFamily.ProtoReflect.Descriptor instead. func (*MetricFamily) Descriptor() ([]byte, []int) { - return fileDescriptor_d1e5ddb18987a258, []int{11} + return file_io_prometheus_client_metrics_proto_rawDescGZIP(), []int{11} } -func (m *MetricFamily) XXX_Unmarshal(b []byte) error { - return xxx_messageInfo_MetricFamily.Unmarshal(m, b) -} -func (m *MetricFamily) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - return xxx_messageInfo_MetricFamily.Marshal(b, m, deterministic) -} -func (m *MetricFamily) XXX_Merge(src proto.Message) { - xxx_messageInfo_MetricFamily.Merge(m, src) -} -func (m *MetricFamily) XXX_Size() int { - return xxx_messageInfo_MetricFamily.Size(m) -} -func (m *MetricFamily) XXX_DiscardUnknown() { - xxx_messageInfo_MetricFamily.DiscardUnknown(m) -} - -var xxx_messageInfo_MetricFamily proto.InternalMessageInfo - -func (m *MetricFamily) GetName() string { - if m != nil && m.Name != nil { - return *m.Name +func (x *MetricFamily) GetName() string { + if x != nil && x.Name != nil { + return *x.Name } return "" } -func (m *MetricFamily) GetHelp() string { - if m != nil && m.Help != nil { - return *m.Help +func (x *MetricFamily) GetHelp() string { + if x != nil && x.Help != nil { + return *x.Help } return "" } -func (m *MetricFamily) GetType() MetricType { - if m != nil && m.Type != nil { - return *m.Type +func (x *MetricFamily) GetType() MetricType { + if x != nil && x.Type != nil { + return *x.Type } return MetricType_COUNTER } -func (m *MetricFamily) GetMetric() []*Metric { - if m != nil { - return m.Metric +func (x *MetricFamily) GetMetric() []*Metric { + if x != nil { + return x.Metric } return nil } -func init() { - proto.RegisterEnum("io.prometheus.client.MetricType", MetricType_name, MetricType_value) - proto.RegisterType((*LabelPair)(nil), "io.prometheus.client.LabelPair") - proto.RegisterType((*Gauge)(nil), "io.prometheus.client.Gauge") - proto.RegisterType((*Counter)(nil), "io.prometheus.client.Counter") - proto.RegisterType((*Quantile)(nil), "io.prometheus.client.Quantile") - proto.RegisterType((*Summary)(nil), "io.prometheus.client.Summary") - proto.RegisterType((*Untyped)(nil), "io.prometheus.client.Untyped") - proto.RegisterType((*Histogram)(nil), "io.prometheus.client.Histogram") - proto.RegisterType((*Bucket)(nil), "io.prometheus.client.Bucket") - proto.RegisterType((*BucketSpan)(nil), "io.prometheus.client.BucketSpan") - proto.RegisterType((*Exemplar)(nil), "io.prometheus.client.Exemplar") - proto.RegisterType((*Metric)(nil), "io.prometheus.client.Metric") - proto.RegisterType((*MetricFamily)(nil), "io.prometheus.client.MetricFamily") +var File_io_prometheus_client_metrics_proto protoreflect.FileDescriptor + +var file_io_prometheus_client_metrics_proto_rawDesc = []byte{ + 0x0a, 0x22, 0x69, 0x6f, 0x2f, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2f, + 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2f, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x14, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, + 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x1a, 0x1f, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2f, 0x74, 0x69, 0x6d, 0x65, + 0x73, 0x74, 0x61, 0x6d, 0x70, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x22, 0x35, 0x0a, 0x09, 0x4c, + 0x61, 0x62, 0x65, 0x6c, 0x50, 0x61, 0x69, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x22, 0x1d, 0x0a, 0x05, 0x47, 0x61, 0x75, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x22, 0x5b, 0x0a, 0x07, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x14, 0x0a, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x12, 0x3a, 0x0a, 0x08, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, + 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x78, 0x65, 0x6d, + 0x70, 0x6c, 0x61, 0x72, 0x52, 0x08, 0x65, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x22, 0x3c, + 0x0a, 0x08, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x71, 0x75, + 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, 0x52, 0x08, 0x71, 0x75, + 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0x87, 0x01, 0x0a, + 0x07, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x61, 0x6d, 0x70, + 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, + 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, + 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, 0x73, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x09, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x53, 0x75, 0x6d, 0x12, 0x3a, 0x0a, 0x08, 0x71, 0x75, + 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x69, + 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x2e, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x52, 0x08, 0x71, 0x75, + 0x61, 0x6e, 0x74, 0x69, 0x6c, 0x65, 0x22, 0x1f, 0x0a, 0x07, 0x55, 0x6e, 0x74, 0x79, 0x70, 0x65, + 0x64, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x01, + 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x22, 0xe3, 0x04, 0x0a, 0x09, 0x48, 0x69, 0x73, 0x74, + 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x5f, + 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x04, 0x52, 0x0b, 0x73, 0x61, 0x6d, + 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x2c, 0x0a, 0x12, 0x73, 0x61, 0x6d, 0x70, + 0x6c, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x18, 0x04, + 0x20, 0x01, 0x28, 0x01, 0x52, 0x10, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, 0x43, 0x6f, 0x75, 0x6e, + 0x74, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x12, 0x1d, 0x0a, 0x0a, 0x73, 0x61, 0x6d, 0x70, 0x6c, 0x65, + 0x5f, 0x73, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x09, 0x73, 0x61, 0x6d, 0x70, + 0x6c, 0x65, 0x53, 0x75, 0x6d, 0x12, 0x34, 0x0a, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x18, + 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, + 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x42, 0x75, 0x63, + 0x6b, 0x65, 0x74, 0x52, 0x06, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, 0x73, + 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x05, 0x20, 0x01, 0x28, 0x11, 0x52, 0x06, 0x73, 0x63, 0x68, + 0x65, 0x6d, 0x61, 0x12, 0x25, 0x0a, 0x0e, 0x7a, 0x65, 0x72, 0x6f, 0x5f, 0x74, 0x68, 0x72, 0x65, + 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0d, 0x7a, 0x65, 0x72, + 0x6f, 0x54, 0x68, 0x72, 0x65, 0x73, 0x68, 0x6f, 0x6c, 0x64, 0x12, 0x1d, 0x0a, 0x0a, 0x7a, 0x65, + 0x72, 0x6f, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x07, 0x20, 0x01, 0x28, 0x04, 0x52, 0x09, + 0x7a, 0x65, 0x72, 0x6f, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x28, 0x0a, 0x10, 0x7a, 0x65, 0x72, + 0x6f, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x18, 0x08, 0x20, + 0x01, 0x28, 0x01, 0x52, 0x0e, 0x7a, 0x65, 0x72, 0x6f, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x46, 0x6c, + 0x6f, 0x61, 0x74, 0x12, 0x45, 0x0a, 0x0d, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, + 0x73, 0x70, 0x61, 0x6e, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x69, 0x6f, 0x2e, + 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, + 0x74, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x70, 0x61, 0x6e, 0x52, 0x0c, 0x6e, 0x65, + 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x53, 0x70, 0x61, 0x6e, 0x12, 0x25, 0x0a, 0x0e, 0x6e, 0x65, + 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x64, 0x65, 0x6c, 0x74, 0x61, 0x18, 0x0a, 0x20, 0x03, + 0x28, 0x12, 0x52, 0x0d, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x44, 0x65, 0x6c, 0x74, + 0x61, 0x12, 0x25, 0x0a, 0x0e, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x6f, + 0x75, 0x6e, 0x74, 0x18, 0x0b, 0x20, 0x03, 0x28, 0x01, 0x52, 0x0d, 0x6e, 0x65, 0x67, 0x61, 0x74, + 0x69, 0x76, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x12, 0x45, 0x0a, 0x0d, 0x70, 0x6f, 0x73, 0x69, + 0x74, 0x69, 0x76, 0x65, 0x5f, 0x73, 0x70, 0x61, 0x6e, 0x18, 0x0c, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x20, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, + 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x53, 0x70, 0x61, + 0x6e, 0x52, 0x0c, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x53, 0x70, 0x61, 0x6e, 0x12, + 0x25, 0x0a, 0x0e, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x64, 0x65, 0x6c, 0x74, + 0x61, 0x18, 0x0d, 0x20, 0x03, 0x28, 0x12, 0x52, 0x0d, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, + 0x65, 0x44, 0x65, 0x6c, 0x74, 0x61, 0x12, 0x25, 0x0a, 0x0e, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, + 0x76, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x0e, 0x20, 0x03, 0x28, 0x01, 0x52, 0x0d, + 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x22, 0xc6, 0x01, + 0x0a, 0x06, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x75, 0x6d, 0x75, + 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x04, 0x52, 0x0f, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x43, 0x6f, + 0x75, 0x6e, 0x74, 0x12, 0x34, 0x0a, 0x16, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, + 0x65, 0x5f, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x5f, 0x66, 0x6c, 0x6f, 0x61, 0x74, 0x18, 0x04, 0x20, + 0x01, 0x28, 0x01, 0x52, 0x14, 0x63, 0x75, 0x6d, 0x75, 0x6c, 0x61, 0x74, 0x69, 0x76, 0x65, 0x43, + 0x6f, 0x75, 0x6e, 0x74, 0x46, 0x6c, 0x6f, 0x61, 0x74, 0x12, 0x1f, 0x0a, 0x0b, 0x75, 0x70, 0x70, + 0x65, 0x72, 0x5f, 0x62, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0a, + 0x75, 0x70, 0x70, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64, 0x12, 0x3a, 0x0a, 0x08, 0x65, 0x78, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x69, + 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, + 0x65, 0x6e, 0x74, 0x2e, 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x52, 0x08, 0x65, 0x78, + 0x65, 0x6d, 0x70, 0x6c, 0x61, 0x72, 0x22, 0x3c, 0x0a, 0x0a, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, + 0x53, 0x70, 0x61, 0x6e, 0x12, 0x16, 0x0a, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x11, 0x52, 0x06, 0x6f, 0x66, 0x66, 0x73, 0x65, 0x74, 0x12, 0x16, 0x0a, 0x06, + 0x6c, 0x65, 0x6e, 0x67, 0x74, 0x68, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x06, 0x6c, 0x65, + 0x6e, 0x67, 0x74, 0x68, 0x22, 0x91, 0x01, 0x0a, 0x08, 0x45, 0x78, 0x65, 0x6d, 0x70, 0x6c, 0x61, + 0x72, 0x12, 0x35, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x1f, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, + 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x50, 0x61, 0x69, + 0x72, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x38, + 0x0a, 0x09, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x1a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x52, 0x09, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x22, 0xff, 0x02, 0x0a, 0x06, 0x4d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x12, 0x35, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x01, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, + 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x50, + 0x61, 0x69, 0x72, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x31, 0x0a, 0x05, 0x67, 0x61, + 0x75, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1b, 0x2e, 0x69, 0x6f, 0x2e, 0x70, + 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x2e, 0x47, 0x61, 0x75, 0x67, 0x65, 0x52, 0x05, 0x67, 0x61, 0x75, 0x67, 0x65, 0x12, 0x37, 0x0a, + 0x07, 0x63, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, + 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, + 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x52, 0x07, 0x63, + 0x6f, 0x75, 0x6e, 0x74, 0x65, 0x72, 0x12, 0x37, 0x0a, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, + 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, + 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x53, + 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x52, 0x07, 0x73, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x12, + 0x37, 0x0a, 0x07, 0x75, 0x6e, 0x74, 0x79, 0x70, 0x65, 0x64, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1d, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, + 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x55, 0x6e, 0x74, 0x79, 0x70, 0x65, 0x64, 0x52, + 0x07, 0x75, 0x6e, 0x74, 0x79, 0x70, 0x65, 0x64, 0x12, 0x3d, 0x0a, 0x09, 0x68, 0x69, 0x73, 0x74, + 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x69, 0x6f, + 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, + 0x6e, 0x74, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x52, 0x09, 0x68, 0x69, + 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x12, 0x21, 0x0a, 0x0c, 0x74, 0x69, 0x6d, 0x65, 0x73, + 0x74, 0x61, 0x6d, 0x70, 0x5f, 0x6d, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, 0x03, 0x52, 0x0b, 0x74, + 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0x4d, 0x73, 0x22, 0xa2, 0x01, 0x0a, 0x0c, 0x4d, + 0x65, 0x74, 0x72, 0x69, 0x63, 0x46, 0x61, 0x6d, 0x69, 0x6c, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, + 0x12, 0x0a, 0x04, 0x68, 0x65, 0x6c, 0x70, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x68, + 0x65, 0x6c, 0x70, 0x12, 0x34, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x20, 0x2e, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, + 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x34, 0x0a, 0x06, 0x6d, 0x65, 0x74, + 0x72, 0x69, 0x63, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x69, 0x6f, 0x2e, 0x70, + 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, + 0x2e, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x2a, + 0x62, 0x0a, 0x0a, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a, + 0x07, 0x43, 0x4f, 0x55, 0x4e, 0x54, 0x45, 0x52, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x47, 0x41, + 0x55, 0x47, 0x45, 0x10, 0x01, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x55, 0x4d, 0x4d, 0x41, 0x52, 0x59, + 0x10, 0x02, 0x12, 0x0b, 0x0a, 0x07, 0x55, 0x4e, 0x54, 0x59, 0x50, 0x45, 0x44, 0x10, 0x03, 0x12, + 0x0d, 0x0a, 0x09, 0x48, 0x49, 0x53, 0x54, 0x4f, 0x47, 0x52, 0x41, 0x4d, 0x10, 0x04, 0x12, 0x13, + 0x0a, 0x0f, 0x47, 0x41, 0x55, 0x47, 0x45, 0x5f, 0x48, 0x49, 0x53, 0x54, 0x4f, 0x47, 0x52, 0x41, + 0x4d, 0x10, 0x05, 0x42, 0x52, 0x0a, 0x14, 0x69, 0x6f, 0x2e, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, + 0x68, 0x65, 0x75, 0x73, 0x2e, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5a, 0x3a, 0x67, 0x69, 0x74, + 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, + 0x75, 0x73, 0x2f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x6d, 0x6f, 0x64, 0x65, 0x6c, 0x2f, + 0x67, 0x6f, 0x3b, 0x69, 0x6f, 0x5f, 0x70, 0x72, 0x6f, 0x6d, 0x65, 0x74, 0x68, 0x65, 0x75, 0x73, + 0x5f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, } -func init() { - proto.RegisterFile("io/prometheus/client/metrics.proto", fileDescriptor_d1e5ddb18987a258) +var ( + file_io_prometheus_client_metrics_proto_rawDescOnce sync.Once + file_io_prometheus_client_metrics_proto_rawDescData = file_io_prometheus_client_metrics_proto_rawDesc +) + +func file_io_prometheus_client_metrics_proto_rawDescGZIP() []byte { + file_io_prometheus_client_metrics_proto_rawDescOnce.Do(func() { + file_io_prometheus_client_metrics_proto_rawDescData = protoimpl.X.CompressGZIP(file_io_prometheus_client_metrics_proto_rawDescData) + }) + return file_io_prometheus_client_metrics_proto_rawDescData } -var fileDescriptor_d1e5ddb18987a258 = []byte{ - // 896 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x56, 0xdd, 0x8e, 0xdb, 0x44, - 0x18, 0xc5, 0x9b, 0x5f, 0x7f, 0xd9, 0x6c, 0xd3, 0x61, 0x55, 0x59, 0x0b, 0xcb, 0x06, 0x4b, 0x48, - 0x0b, 0x42, 0x8e, 0x40, 0x5b, 0x81, 0x0a, 0x5c, 0xec, 0xb6, 0xe9, 0x16, 0x89, 0xb4, 0x65, 0x92, - 0x5c, 0x14, 0x2e, 0xac, 0x49, 0x32, 0xeb, 0x58, 0x78, 0x3c, 0xc6, 0x1e, 0x57, 0x2c, 0x2f, 0xc0, - 0x35, 0xaf, 0xc0, 0xc3, 0xf0, 0x22, 0x3c, 0x08, 0x68, 0xfe, 0xec, 0xdd, 0xe2, 0x94, 0xd2, 0x3b, - 0x7f, 0x67, 0xce, 0xf7, 0xcd, 0x39, 0xe3, 0xc9, 0x71, 0xc0, 0x8f, 0xf9, 0x24, 0xcb, 0x39, 0xa3, - 0x62, 0x4b, 0xcb, 0x62, 0xb2, 0x4e, 0x62, 0x9a, 0x8a, 0x09, 0xa3, 0x22, 0x8f, 0xd7, 0x45, 0x90, - 0xe5, 0x5c, 0x70, 0x74, 0x18, 0xf3, 0xa0, 0xe6, 0x04, 0x9a, 0x73, 0x74, 0x12, 0x71, 0x1e, 0x25, - 0x74, 0xa2, 0x38, 0xab, 0xf2, 0x6a, 0x22, 0x62, 0x46, 0x0b, 0x41, 0x58, 0xa6, 0xdb, 0xfc, 0xfb, - 0xe0, 0x7e, 0x47, 0x56, 0x34, 0x79, 0x4e, 0xe2, 0x1c, 0x21, 0x68, 0xa7, 0x84, 0x51, 0xcf, 0x19, - 0x3b, 0xa7, 0x2e, 0x56, 0xcf, 0xe8, 0x10, 0x3a, 0x2f, 0x49, 0x52, 0x52, 0x6f, 0x4f, 0x81, 0xba, - 0xf0, 0x8f, 0xa1, 0x73, 0x49, 0xca, 0xe8, 0xc6, 0xb2, 0xec, 0x71, 0xec, 0xf2, 0x8f, 0xd0, 0x7b, - 0xc8, 0xcb, 0x54, 0xd0, 0xbc, 0x99, 0x80, 0x1e, 0x40, 0x9f, 0xfe, 0x42, 0x59, 0x96, 0x90, 0x5c, - 0x0d, 0x1e, 0x7c, 0xfe, 0x41, 0xd0, 0x64, 0x20, 0x98, 0x1a, 0x16, 0xae, 0xf8, 0xfe, 0xd7, 0xd0, - 0xff, 0xbe, 0x24, 0xa9, 0x88, 0x13, 0x8a, 0x8e, 0xa0, 0xff, 0xb3, 0x79, 0x36, 0x1b, 0x54, 0xf5, - 0x6d, 0xe5, 0x95, 0xb4, 0xdf, 0x1c, 0xe8, 0xcd, 0x4b, 0xc6, 0x48, 0x7e, 0x8d, 0x3e, 0x84, 0xfd, - 0x82, 0xb0, 0x2c, 0xa1, 0xe1, 0x5a, 0xaa, 0x55, 0x13, 0xda, 0x78, 0xa0, 0x31, 0x65, 0x00, 0x1d, - 0x03, 0x18, 0x4a, 0x51, 0x32, 0x33, 0xc9, 0xd5, 0xc8, 0xbc, 0x64, 0xd2, 0x47, 0xb5, 0x7f, 0x6b, - 0xdc, 0xda, 0xed, 0xc3, 0x2a, 0xae, 0xf5, 0xf9, 0x27, 0xd0, 0x5b, 0xa6, 0xe2, 0x3a, 0xa3, 0x9b, - 0x1d, 0xa7, 0xf8, 0x57, 0x1b, 0xdc, 0x27, 0x71, 0x21, 0x78, 0x94, 0x13, 0xf6, 0x26, 0x62, 0x3f, - 0x05, 0x74, 0x93, 0x12, 0x5e, 0x25, 0x9c, 0x08, 0xaf, 0xad, 0x66, 0x8e, 0x6e, 0x10, 0x1f, 0x4b, - 0xfc, 0xbf, 0xac, 0x9d, 0x41, 0x77, 0x55, 0xae, 0x7f, 0xa2, 0xc2, 0x18, 0x7b, 0xbf, 0xd9, 0xd8, - 0x85, 0xe2, 0x60, 0xc3, 0x45, 0xf7, 0xa0, 0x5b, 0xac, 0xb7, 0x94, 0x11, 0xaf, 0x33, 0x76, 0x4e, - 0xef, 0x62, 0x53, 0xa1, 0x8f, 0xe0, 0xe0, 0x57, 0x9a, 0xf3, 0x50, 0x6c, 0x73, 0x5a, 0x6c, 0x79, - 0xb2, 0xf1, 0xba, 0x6a, 0xc3, 0xa1, 0x44, 0x17, 0x16, 0x94, 0x9a, 0x14, 0x4d, 0x5b, 0xec, 0x29, - 0x8b, 0xae, 0x44, 0xb4, 0xc1, 0x53, 0x18, 0xd5, 0xcb, 0xc6, 0x5e, 0x5f, 0xcd, 0x39, 0xa8, 0x48, - 0xda, 0xdc, 0x14, 0x86, 0x29, 0x8d, 0x88, 0x88, 0x5f, 0xd2, 0xb0, 0xc8, 0x48, 0xea, 0xb9, 0xca, - 0xc4, 0xf8, 0x75, 0x26, 0xe6, 0x19, 0x49, 0xf1, 0xbe, 0x6d, 0x93, 0x95, 0x94, 0x5d, 0x8d, 0xd9, - 0xd0, 0x44, 0x10, 0x0f, 0xc6, 0xad, 0x53, 0x84, 0xab, 0xe1, 0x8f, 0x24, 0x78, 0x8b, 0xa6, 0xa5, - 0x0f, 0xc6, 0x2d, 0xe9, 0xce, 0xa2, 0x5a, 0xfe, 0x14, 0x86, 0x19, 0x2f, 0xe2, 0x5a, 0xd4, 0xfe, - 0x9b, 0x8a, 0xb2, 0x6d, 0x56, 0x54, 0x35, 0x46, 0x8b, 0x1a, 0x6a, 0x51, 0x16, 0xad, 0x44, 0x55, - 0x34, 0x2d, 0xea, 0x40, 0x8b, 0xb2, 0xa8, 0x12, 0xe5, 0xff, 0xe9, 0x40, 0x57, 0x6f, 0x85, 0x3e, - 0x86, 0xd1, 0xba, 0x64, 0x65, 0x72, 0xd3, 0x88, 0xbe, 0x66, 0x77, 0x6a, 0x5c, 0x5b, 0x39, 0x83, - 0x7b, 0xaf, 0x52, 0x6f, 0x5d, 0xb7, 0xc3, 0x57, 0x1a, 0xf4, 0x5b, 0x39, 0x81, 0x41, 0x99, 0x65, - 0x34, 0x0f, 0x57, 0xbc, 0x4c, 0x37, 0xe6, 0xce, 0x81, 0x82, 0x2e, 0x24, 0x72, 0x2b, 0x17, 0x5a, - 0xff, 0x3b, 0x17, 0xa0, 0x3e, 0x32, 0x79, 0x11, 0xf9, 0xd5, 0x55, 0x41, 0xb5, 0x83, 0xbb, 0xd8, - 0x54, 0x12, 0x4f, 0x68, 0x1a, 0x89, 0xad, 0xda, 0x7d, 0x88, 0x4d, 0xe5, 0xff, 0xee, 0x40, 0xdf, - 0x0e, 0x45, 0xf7, 0xa1, 0x93, 0xc8, 0x54, 0xf4, 0x1c, 0xf5, 0x82, 0x4e, 0x9a, 0x35, 0x54, 0xc1, - 0x89, 0x35, 0xbb, 0x39, 0x71, 0xd0, 0x97, 0xe0, 0x56, 0xa9, 0x6b, 0x4c, 0x1d, 0x05, 0x3a, 0x97, - 0x03, 0x9b, 0xcb, 0xc1, 0xc2, 0x32, 0x70, 0x4d, 0xf6, 0xff, 0xde, 0x83, 0xee, 0x4c, 0xa5, 0xfc, - 0xdb, 0x2a, 0xfa, 0x0c, 0x3a, 0x91, 0xcc, 0x69, 0x13, 0xb2, 0xef, 0x35, 0xb7, 0xa9, 0x28, 0xc7, - 0x9a, 0x89, 0xbe, 0x80, 0xde, 0x5a, 0x67, 0xb7, 0x11, 0x7b, 0xdc, 0xdc, 0x64, 0x02, 0x1e, 0x5b, - 0xb6, 0x6c, 0x2c, 0x74, 0xb0, 0xaa, 0x3b, 0xb0, 0xb3, 0xd1, 0xa4, 0x2f, 0xb6, 0x6c, 0xd9, 0x58, - 0xea, 0x20, 0x54, 0xa1, 0xb1, 0xb3, 0xd1, 0xa4, 0x25, 0xb6, 0x6c, 0xf4, 0x0d, 0xb8, 0x5b, 0x9b, - 0x8f, 0x2a, 0x2c, 0x76, 0x1e, 0x4c, 0x15, 0xa3, 0xb8, 0xee, 0x90, 0x89, 0x5a, 0x9d, 0x75, 0xc8, - 0x0a, 0x95, 0x48, 0x2d, 0x3c, 0xa8, 0xb0, 0x59, 0xe1, 0xff, 0xe1, 0xc0, 0xbe, 0x7e, 0x03, 0x8f, - 0x09, 0x8b, 0x93, 0xeb, 0xc6, 0x4f, 0x24, 0x82, 0xf6, 0x96, 0x26, 0x99, 0xf9, 0x42, 0xaa, 0x67, - 0x74, 0x06, 0x6d, 0xa9, 0x51, 0x1d, 0xe1, 0xc1, 0xae, 0x5f, 0xb8, 0x9e, 0xbc, 0xb8, 0xce, 0x28, - 0x56, 0x6c, 0x99, 0xb9, 0xfa, 0xab, 0xee, 0xb5, 0x5f, 0x97, 0xb9, 0xba, 0x0f, 0x1b, 0xee, 0x27, - 0x2b, 0x80, 0x7a, 0x12, 0x1a, 0x40, 0xef, 0xe1, 0xb3, 0xe5, 0xd3, 0xc5, 0x14, 0x8f, 0xde, 0x41, - 0x2e, 0x74, 0x2e, 0xcf, 0x97, 0x97, 0xd3, 0x91, 0x23, 0xf1, 0xf9, 0x72, 0x36, 0x3b, 0xc7, 0x2f, - 0x46, 0x7b, 0xb2, 0x58, 0x3e, 0x5d, 0xbc, 0x78, 0x3e, 0x7d, 0x34, 0x6a, 0xa1, 0x21, 0xb8, 0x4f, - 0xbe, 0x9d, 0x2f, 0x9e, 0x5d, 0xe2, 0xf3, 0xd9, 0xa8, 0x8d, 0xde, 0x85, 0x3b, 0xaa, 0x27, 0xac, - 0xc1, 0xce, 0x05, 0x86, 0xc6, 0x3f, 0x18, 0x3f, 0x3c, 0x88, 0x62, 0xb1, 0x2d, 0x57, 0xc1, 0x9a, - 0xb3, 0x7f, 0xff, 0x45, 0x09, 0x19, 0xdf, 0xd0, 0x64, 0x12, 0xf1, 0xaf, 0x62, 0x1e, 0xd6, 0xab, - 0xa1, 0x5e, 0xfd, 0x27, 0x00, 0x00, 0xff, 0xff, 0x16, 0x77, 0x81, 0x98, 0xd7, 0x08, 0x00, 0x00, +var file_io_prometheus_client_metrics_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_io_prometheus_client_metrics_proto_msgTypes = make([]protoimpl.MessageInfo, 12) +var file_io_prometheus_client_metrics_proto_goTypes = []interface{}{ + (MetricType)(0), // 0: io.prometheus.client.MetricType + (*LabelPair)(nil), // 1: io.prometheus.client.LabelPair + (*Gauge)(nil), // 2: io.prometheus.client.Gauge + (*Counter)(nil), // 3: io.prometheus.client.Counter + (*Quantile)(nil), // 4: io.prometheus.client.Quantile + (*Summary)(nil), // 5: io.prometheus.client.Summary + (*Untyped)(nil), // 6: io.prometheus.client.Untyped + (*Histogram)(nil), // 7: io.prometheus.client.Histogram + (*Bucket)(nil), // 8: io.prometheus.client.Bucket + (*BucketSpan)(nil), // 9: io.prometheus.client.BucketSpan + (*Exemplar)(nil), // 10: io.prometheus.client.Exemplar + (*Metric)(nil), // 11: io.prometheus.client.Metric + (*MetricFamily)(nil), // 12: io.prometheus.client.MetricFamily + (*timestamppb.Timestamp)(nil), // 13: google.protobuf.Timestamp +} +var file_io_prometheus_client_metrics_proto_depIdxs = []int32{ + 10, // 0: io.prometheus.client.Counter.exemplar:type_name -> io.prometheus.client.Exemplar + 4, // 1: io.prometheus.client.Summary.quantile:type_name -> io.prometheus.client.Quantile + 8, // 2: io.prometheus.client.Histogram.bucket:type_name -> io.prometheus.client.Bucket + 9, // 3: io.prometheus.client.Histogram.negative_span:type_name -> io.prometheus.client.BucketSpan + 9, // 4: io.prometheus.client.Histogram.positive_span:type_name -> io.prometheus.client.BucketSpan + 10, // 5: io.prometheus.client.Bucket.exemplar:type_name -> io.prometheus.client.Exemplar + 1, // 6: io.prometheus.client.Exemplar.label:type_name -> io.prometheus.client.LabelPair + 13, // 7: io.prometheus.client.Exemplar.timestamp:type_name -> google.protobuf.Timestamp + 1, // 8: io.prometheus.client.Metric.label:type_name -> io.prometheus.client.LabelPair + 2, // 9: io.prometheus.client.Metric.gauge:type_name -> io.prometheus.client.Gauge + 3, // 10: io.prometheus.client.Metric.counter:type_name -> io.prometheus.client.Counter + 5, // 11: io.prometheus.client.Metric.summary:type_name -> io.prometheus.client.Summary + 6, // 12: io.prometheus.client.Metric.untyped:type_name -> io.prometheus.client.Untyped + 7, // 13: io.prometheus.client.Metric.histogram:type_name -> io.prometheus.client.Histogram + 0, // 14: io.prometheus.client.MetricFamily.type:type_name -> io.prometheus.client.MetricType + 11, // 15: io.prometheus.client.MetricFamily.metric:type_name -> io.prometheus.client.Metric + 16, // [16:16] is the sub-list for method output_type + 16, // [16:16] is the sub-list for method input_type + 16, // [16:16] is the sub-list for extension type_name + 16, // [16:16] is the sub-list for extension extendee + 0, // [0:16] is the sub-list for field type_name +} + +func init() { file_io_prometheus_client_metrics_proto_init() } +func file_io_prometheus_client_metrics_proto_init() { + if File_io_prometheus_client_metrics_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_io_prometheus_client_metrics_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LabelPair); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_io_prometheus_client_metrics_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Gauge); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_io_prometheus_client_metrics_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Counter); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_io_prometheus_client_metrics_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Quantile); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_io_prometheus_client_metrics_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Summary); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_io_prometheus_client_metrics_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Untyped); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_io_prometheus_client_metrics_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Histogram); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_io_prometheus_client_metrics_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Bucket); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_io_prometheus_client_metrics_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BucketSpan); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_io_prometheus_client_metrics_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Exemplar); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_io_prometheus_client_metrics_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Metric); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_io_prometheus_client_metrics_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*MetricFamily); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_io_prometheus_client_metrics_proto_rawDesc, + NumEnums: 1, + NumMessages: 12, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_io_prometheus_client_metrics_proto_goTypes, + DependencyIndexes: file_io_prometheus_client_metrics_proto_depIdxs, + EnumInfos: file_io_prometheus_client_metrics_proto_enumTypes, + MessageInfos: file_io_prometheus_client_metrics_proto_msgTypes, + }.Build() + File_io_prometheus_client_metrics_proto = out.File + file_io_prometheus_client_metrics_proto_rawDesc = nil + file_io_prometheus_client_metrics_proto_goTypes = nil + file_io_prometheus_client_metrics_proto_depIdxs = nil } diff --git a/vendor/github.com/prometheus/common/expfmt/decode.go b/vendor/github.com/prometheus/common/expfmt/decode.go index 7657f84..9063978 100644 --- a/vendor/github.com/prometheus/common/expfmt/decode.go +++ b/vendor/github.com/prometheus/common/expfmt/decode.go @@ -115,32 +115,31 @@ func (d *protoDecoder) Decode(v *dto.MetricFamily) error { // textDecoder implements the Decoder interface for the text protocol. type textDecoder struct { r io.Reader - p TextParser - fams []*dto.MetricFamily + fams map[string]*dto.MetricFamily + err error } // Decode implements the Decoder interface. func (d *textDecoder) Decode(v *dto.MetricFamily) error { - // TODO(fabxc): Wrap this as a line reader to make streaming safer. - if len(d.fams) == 0 { - // No cached metric families, read everything and parse metrics. - fams, err := d.p.TextToMetricFamilies(d.r) - if err != nil { - return err - } - if len(fams) == 0 { - return io.EOF - } - d.fams = make([]*dto.MetricFamily, 0, len(fams)) - for _, f := range fams { - d.fams = append(d.fams, f) + if d.err == nil { + // Read all metrics in one shot. + var p TextParser + d.fams, d.err = p.TextToMetricFamilies(d.r) + // If we don't get an error, store io.EOF for the end. + if d.err == nil { + d.err = io.EOF } } - - *v = *d.fams[0] - d.fams = d.fams[1:] - - return nil + // Pick off one MetricFamily per Decode until there's nothing left. + for key, fam := range d.fams { + v.Name = fam.Name + v.Help = fam.Help + v.Type = fam.Type + v.Metric = fam.Metric + delete(d.fams, key) + return nil + } + return d.err } // SampleDecoder wraps a Decoder to extract samples from the metric families diff --git a/vendor/github.com/prometheus/common/expfmt/encode.go b/vendor/github.com/prometheus/common/expfmt/encode.go index 64dc0eb..7f611ff 100644 --- a/vendor/github.com/prometheus/common/expfmt/encode.go +++ b/vendor/github.com/prometheus/common/expfmt/encode.go @@ -18,9 +18,9 @@ import ( "io" "net/http" - "github.com/golang/protobuf/proto" //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. "github.com/matttproud/golang_protobuf_extensions/pbutil" "github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg" + "google.golang.org/protobuf/encoding/prototext" dto "github.com/prometheus/client_model/go" ) @@ -99,8 +99,11 @@ func NegotiateIncludingOpenMetrics(h http.Header) Format { if ac.Type == "text" && ac.SubType == "plain" && (ver == TextVersion || ver == "") { return FmtText } - if ac.Type+"/"+ac.SubType == OpenMetricsType && (ver == OpenMetricsVersion || ver == "") { - return FmtOpenMetrics + if ac.Type+"/"+ac.SubType == OpenMetricsType && (ver == OpenMetricsVersion_0_0_1 || ver == OpenMetricsVersion_1_0_0 || ver == "") { + if ver == OpenMetricsVersion_1_0_0 { + return FmtOpenMetrics_1_0_0 + } + return FmtOpenMetrics_0_0_1 } } return FmtText @@ -133,7 +136,7 @@ func NewEncoder(w io.Writer, format Format) Encoder { case FmtProtoText: return encoderCloser{ encode: func(v *dto.MetricFamily) error { - _, err := fmt.Fprintln(w, proto.MarshalTextString(v)) + _, err := fmt.Fprintln(w, prototext.Format(v)) return err }, close: func() error { return nil }, @@ -146,7 +149,7 @@ func NewEncoder(w io.Writer, format Format) Encoder { }, close: func() error { return nil }, } - case FmtOpenMetrics: + case FmtOpenMetrics_0_0_1, FmtOpenMetrics_1_0_0: return encoderCloser{ encode: func(v *dto.MetricFamily) error { _, err := MetricFamilyToOpenMetrics(w, v) diff --git a/vendor/github.com/prometheus/common/expfmt/expfmt.go b/vendor/github.com/prometheus/common/expfmt/expfmt.go index 0f176fa..c4cb20f 100644 --- a/vendor/github.com/prometheus/common/expfmt/expfmt.go +++ b/vendor/github.com/prometheus/common/expfmt/expfmt.go @@ -19,20 +19,22 @@ type Format string // Constants to assemble the Content-Type values for the different wire protocols. const ( - TextVersion = "0.0.4" - ProtoType = `application/vnd.google.protobuf` - ProtoProtocol = `io.prometheus.client.MetricFamily` - ProtoFmt = ProtoType + "; proto=" + ProtoProtocol + ";" - OpenMetricsType = `application/openmetrics-text` - OpenMetricsVersion = "0.0.1" + TextVersion = "0.0.4" + ProtoType = `application/vnd.google.protobuf` + ProtoProtocol = `io.prometheus.client.MetricFamily` + ProtoFmt = ProtoType + "; proto=" + ProtoProtocol + ";" + OpenMetricsType = `application/openmetrics-text` + OpenMetricsVersion_0_0_1 = "0.0.1" + OpenMetricsVersion_1_0_0 = "1.0.0" // The Content-Type values for the different wire protocols. - FmtUnknown Format = `` - FmtText Format = `text/plain; version=` + TextVersion + `; charset=utf-8` - FmtProtoDelim Format = ProtoFmt + ` encoding=delimited` - FmtProtoText Format = ProtoFmt + ` encoding=text` - FmtProtoCompact Format = ProtoFmt + ` encoding=compact-text` - FmtOpenMetrics Format = OpenMetricsType + `; version=` + OpenMetricsVersion + `; charset=utf-8` + FmtUnknown Format = `` + FmtText Format = `text/plain; version=` + TextVersion + `; charset=utf-8` + FmtProtoDelim Format = ProtoFmt + ` encoding=delimited` + FmtProtoText Format = ProtoFmt + ` encoding=text` + FmtProtoCompact Format = ProtoFmt + ` encoding=compact-text` + FmtOpenMetrics_1_0_0 Format = OpenMetricsType + `; version=` + OpenMetricsVersion_1_0_0 + `; charset=utf-8` + FmtOpenMetrics_0_0_1 Format = OpenMetricsType + `; version=` + OpenMetricsVersion_0_0_1 + `; charset=utf-8` ) const ( diff --git a/vendor/github.com/prometheus/common/expfmt/fuzz.go b/vendor/github.com/prometheus/common/expfmt/fuzz.go index f819e4f..dfac962 100644 --- a/vendor/github.com/prometheus/common/expfmt/fuzz.go +++ b/vendor/github.com/prometheus/common/expfmt/fuzz.go @@ -21,8 +21,8 @@ import "bytes" // Fuzz text metric parser with with github.com/dvyukov/go-fuzz: // -// go-fuzz-build github.com/prometheus/common/expfmt -// go-fuzz -bin expfmt-fuzz.zip -workdir fuzz +// go-fuzz-build github.com/prometheus/common/expfmt +// go-fuzz -bin expfmt-fuzz.zip -workdir fuzz // // Further input samples should go in the folder fuzz/corpus. func Fuzz(in []byte) int { diff --git a/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go b/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go index 9d94ae9..21cdddc 100644 --- a/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go +++ b/vendor/github.com/prometheus/common/expfmt/openmetrics_create.go @@ -46,20 +46,20 @@ import ( // missing features and peculiarities to avoid complications when switching from // Prometheus to OpenMetrics or vice versa: // -// - Counters are expected to have the `_total` suffix in their metric name. In -// the output, the suffix will be truncated from the `# TYPE` and `# HELP` -// line. A counter with a missing `_total` suffix is not an error. However, -// its type will be set to `unknown` in that case to avoid invalid OpenMetrics -// output. +// - Counters are expected to have the `_total` suffix in their metric name. In +// the output, the suffix will be truncated from the `# TYPE` and `# HELP` +// line. A counter with a missing `_total` suffix is not an error. However, +// its type will be set to `unknown` in that case to avoid invalid OpenMetrics +// output. // -// - No support for the following (optional) features: `# UNIT` line, `_created` -// line, info type, stateset type, gaugehistogram type. +// - No support for the following (optional) features: `# UNIT` line, `_created` +// line, info type, stateset type, gaugehistogram type. // -// - The size of exemplar labels is not checked (i.e. it's possible to create -// exemplars that are larger than allowed by the OpenMetrics specification). +// - The size of exemplar labels is not checked (i.e. it's possible to create +// exemplars that are larger than allowed by the OpenMetrics specification). // -// - The value of Counters is not checked. (OpenMetrics doesn't allow counters -// with a `NaN` value.) +// - The value of Counters is not checked. (OpenMetrics doesn't allow counters +// with a `NaN` value.) func MetricFamilyToOpenMetrics(out io.Writer, in *dto.MetricFamily) (written int, err error) { name := in.GetName() if name == "" { diff --git a/vendor/github.com/prometheus/common/expfmt/text_create.go b/vendor/github.com/prometheus/common/expfmt/text_create.go index 5ba503b..2946b8f 100644 --- a/vendor/github.com/prometheus/common/expfmt/text_create.go +++ b/vendor/github.com/prometheus/common/expfmt/text_create.go @@ -17,7 +17,6 @@ import ( "bufio" "fmt" "io" - "io/ioutil" "math" "strconv" "strings" @@ -44,7 +43,7 @@ const ( var ( bufPool = sync.Pool{ New: func() interface{} { - return bufio.NewWriter(ioutil.Discard) + return bufio.NewWriter(io.Discard) }, } numBufPool = sync.Pool{ diff --git a/vendor/github.com/prometheus/common/expfmt/text_parse.go b/vendor/github.com/prometheus/common/expfmt/text_parse.go index 84be064..35db1cc 100644 --- a/vendor/github.com/prometheus/common/expfmt/text_parse.go +++ b/vendor/github.com/prometheus/common/expfmt/text_parse.go @@ -24,8 +24,8 @@ import ( dto "github.com/prometheus/client_model/go" - "github.com/golang/protobuf/proto" //nolint:staticcheck // Ignore SA1019. Need to keep deprecated package for compatibility. "github.com/prometheus/common/model" + "google.golang.org/protobuf/proto" ) // A stateFn is a function that represents a state in a state machine. By @@ -142,9 +142,13 @@ func (p *TextParser) reset(in io.Reader) { func (p *TextParser) startOfLine() stateFn { p.lineCount++ if p.skipBlankTab(); p.err != nil { - // End of input reached. This is the only case where - // that is not an error but a signal that we are done. - p.err = nil + // This is the only place that we expect to see io.EOF, + // which is not an error but the signal that we are done. + // Any other error that happens to align with the start of + // a line is still an error. + if p.err == io.EOF { + p.err = nil + } return nil } switch p.currentByte { diff --git a/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/autoneg.go b/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/autoneg.go index 26e9228..a21b9d1 100644 --- a/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/autoneg.go +++ b/vendor/github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg/autoneg.go @@ -11,18 +11,18 @@ Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: - Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. - Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. + Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. - Neither the name of the Open Knowledge Foundation Ltd. nor the - names of its contributors may be used to endorse or promote - products derived from this software without specific prior written - permission. + Neither the name of the Open Knowledge Foundation Ltd. nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT @@ -35,8 +35,6 @@ DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - */ package goautoneg diff --git a/vendor/github.com/prometheus/common/model/time.go b/vendor/github.com/prometheus/common/model/time.go index c909b8a..5727452 100644 --- a/vendor/github.com/prometheus/common/model/time.go +++ b/vendor/github.com/prometheus/common/model/time.go @@ -18,7 +18,6 @@ import ( "errors" "fmt" "math" - "regexp" "strconv" "strings" "time" @@ -183,54 +182,78 @@ func (d *Duration) Type() string { return "duration" } -var durationRE = regexp.MustCompile("^(([0-9]+)y)?(([0-9]+)w)?(([0-9]+)d)?(([0-9]+)h)?(([0-9]+)m)?(([0-9]+)s)?(([0-9]+)ms)?$") +func isdigit(c byte) bool { return c >= '0' && c <= '9' } + +// Units are required to go in order from biggest to smallest. +// This guards against confusion from "1m1d" being 1 minute + 1 day, not 1 month + 1 day. +var unitMap = map[string]struct { + pos int + mult uint64 +}{ + "ms": {7, uint64(time.Millisecond)}, + "s": {6, uint64(time.Second)}, + "m": {5, uint64(time.Minute)}, + "h": {4, uint64(time.Hour)}, + "d": {3, uint64(24 * time.Hour)}, + "w": {2, uint64(7 * 24 * time.Hour)}, + "y": {1, uint64(365 * 24 * time.Hour)}, +} // ParseDuration parses a string into a time.Duration, assuming that a year // always has 365d, a week always has 7d, and a day always has 24h. -func ParseDuration(durationStr string) (Duration, error) { - switch durationStr { +func ParseDuration(s string) (Duration, error) { + switch s { case "0": // Allow 0 without a unit. return 0, nil case "": return 0, errors.New("empty duration string") } - matches := durationRE.FindStringSubmatch(durationStr) - if matches == nil { - return 0, fmt.Errorf("not a valid duration string: %q", durationStr) - } - var dur time.Duration - // Parse the match at pos `pos` in the regex and use `mult` to turn that - // into ms, then add that value to the total parsed duration. - var overflowErr error - m := func(pos int, mult time.Duration) { - if matches[pos] == "" { - return + orig := s + var dur uint64 + lastUnitPos := 0 + + for s != "" { + if !isdigit(s[0]) { + return 0, fmt.Errorf("not a valid duration string: %q", orig) } - n, _ := strconv.Atoi(matches[pos]) + // Consume [0-9]* + i := 0 + for ; i < len(s) && isdigit(s[i]); i++ { + } + v, err := strconv.ParseUint(s[:i], 10, 0) + if err != nil { + return 0, fmt.Errorf("not a valid duration string: %q", orig) + } + s = s[i:] + // Consume unit. + for i = 0; i < len(s) && !isdigit(s[i]); i++ { + } + if i == 0 { + return 0, fmt.Errorf("not a valid duration string: %q", orig) + } + u := s[:i] + s = s[i:] + unit, ok := unitMap[u] + if !ok { + return 0, fmt.Errorf("unknown unit %q in duration %q", u, orig) + } + if unit.pos <= lastUnitPos { // Units must go in order from biggest to smallest. + return 0, fmt.Errorf("not a valid duration string: %q", orig) + } + lastUnitPos = unit.pos // Check if the provided duration overflows time.Duration (> ~ 290years). - if n > int((1<<63-1)/mult/time.Millisecond) { - overflowErr = errors.New("duration out of range") + if v > 1<<63/unit.mult { + return 0, errors.New("duration out of range") } - d := time.Duration(n) * time.Millisecond - dur += d * mult - - if dur < 0 { - overflowErr = errors.New("duration out of range") + dur += v * unit.mult + if dur > 1<<63-1 { + return 0, errors.New("duration out of range") } } - - m(2, 1000*60*60*24*365) // y - m(4, 1000*60*60*24*7) // w - m(6, 1000*60*60*24) // d - m(8, 1000*60*60) // h - m(10, 1000*60) // m - m(12, 1000) // s - m(14, 1) // ms - - return Duration(dur), overflowErr + return Duration(dur), nil } func (d Duration) String() string { diff --git a/vendor/github.com/prometheus/common/model/value.go b/vendor/github.com/prometheus/common/model/value.go index c9d8fb1..9eb4404 100644 --- a/vendor/github.com/prometheus/common/model/value.go +++ b/vendor/github.com/prometheus/common/model/value.go @@ -16,20 +16,12 @@ package model import ( "encoding/json" "fmt" - "math" "sort" "strconv" "strings" ) var ( - // ZeroSamplePair is the pseudo zero-value of SamplePair used to signal a - // non-existing sample pair. It is a SamplePair with timestamp Earliest and - // value 0.0. Note that the natural zero value of SamplePair has a timestamp - // of 0, which is possible to appear in a real SamplePair and thus not - // suitable to signal a non-existing SamplePair. - ZeroSamplePair = SamplePair{Timestamp: Earliest} - // ZeroSample is the pseudo zero-value of Sample used to signal a // non-existing sample. It is a Sample with timestamp Earliest, value 0.0, // and metric nil. Note that the natural zero value of Sample has a timestamp @@ -38,82 +30,14 @@ var ( ZeroSample = Sample{Timestamp: Earliest} ) -// A SampleValue is a representation of a value for a given sample at a given -// time. -type SampleValue float64 - -// MarshalJSON implements json.Marshaler. -func (v SampleValue) MarshalJSON() ([]byte, error) { - return json.Marshal(v.String()) -} - -// UnmarshalJSON implements json.Unmarshaler. -func (v *SampleValue) UnmarshalJSON(b []byte) error { - if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { - return fmt.Errorf("sample value must be a quoted string") - } - f, err := strconv.ParseFloat(string(b[1:len(b)-1]), 64) - if err != nil { - return err - } - *v = SampleValue(f) - return nil -} - -// Equal returns true if the value of v and o is equal or if both are NaN. Note -// that v==o is false if both are NaN. If you want the conventional float -// behavior, use == to compare two SampleValues. -func (v SampleValue) Equal(o SampleValue) bool { - if v == o { - return true - } - return math.IsNaN(float64(v)) && math.IsNaN(float64(o)) -} - -func (v SampleValue) String() string { - return strconv.FormatFloat(float64(v), 'f', -1, 64) -} - -// SamplePair pairs a SampleValue with a Timestamp. -type SamplePair struct { - Timestamp Time - Value SampleValue -} - -// MarshalJSON implements json.Marshaler. -func (s SamplePair) MarshalJSON() ([]byte, error) { - t, err := json.Marshal(s.Timestamp) - if err != nil { - return nil, err - } - v, err := json.Marshal(s.Value) - if err != nil { - return nil, err - } - return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil -} - -// UnmarshalJSON implements json.Unmarshaler. -func (s *SamplePair) UnmarshalJSON(b []byte) error { - v := [...]json.Unmarshaler{&s.Timestamp, &s.Value} - return json.Unmarshal(b, &v) -} - -// Equal returns true if this SamplePair and o have equal Values and equal -// Timestamps. The semantics of Value equality is defined by SampleValue.Equal. -func (s *SamplePair) Equal(o *SamplePair) bool { - return s == o || (s.Value.Equal(o.Value) && s.Timestamp.Equal(o.Timestamp)) -} - -func (s SamplePair) String() string { - return fmt.Sprintf("%s @[%s]", s.Value, s.Timestamp) -} - -// Sample is a sample pair associated with a metric. +// Sample is a sample pair associated with a metric. A single sample must either +// define Value or Histogram but not both. Histogram == nil implies the Value +// field is used, otherwise it should be ignored. type Sample struct { - Metric Metric `json:"metric"` - Value SampleValue `json:"value"` - Timestamp Time `json:"timestamp"` + Metric Metric `json:"metric"` + Value SampleValue `json:"value"` + Timestamp Time `json:"timestamp"` + Histogram *SampleHistogram `json:"histogram"` } // Equal compares first the metrics, then the timestamp, then the value. The @@ -129,11 +53,19 @@ func (s *Sample) Equal(o *Sample) bool { if !s.Timestamp.Equal(o.Timestamp) { return false } - + if s.Histogram != nil { + return s.Histogram.Equal(o.Histogram) + } return s.Value.Equal(o.Value) } func (s Sample) String() string { + if s.Histogram != nil { + return fmt.Sprintf("%s => %s", s.Metric, SampleHistogramPair{ + Timestamp: s.Timestamp, + Histogram: s.Histogram, + }) + } return fmt.Sprintf("%s => %s", s.Metric, SamplePair{ Timestamp: s.Timestamp, Value: s.Value, @@ -142,6 +74,19 @@ func (s Sample) String() string { // MarshalJSON implements json.Marshaler. func (s Sample) MarshalJSON() ([]byte, error) { + if s.Histogram != nil { + v := struct { + Metric Metric `json:"metric"` + Histogram SampleHistogramPair `json:"histogram"` + }{ + Metric: s.Metric, + Histogram: SampleHistogramPair{ + Timestamp: s.Timestamp, + Histogram: s.Histogram, + }, + } + return json.Marshal(&v) + } v := struct { Metric Metric `json:"metric"` Value SamplePair `json:"value"` @@ -152,21 +97,25 @@ func (s Sample) MarshalJSON() ([]byte, error) { Value: s.Value, }, } - return json.Marshal(&v) } // UnmarshalJSON implements json.Unmarshaler. func (s *Sample) UnmarshalJSON(b []byte) error { v := struct { - Metric Metric `json:"metric"` - Value SamplePair `json:"value"` + Metric Metric `json:"metric"` + Value SamplePair `json:"value"` + Histogram SampleHistogramPair `json:"histogram"` }{ Metric: s.Metric, Value: SamplePair{ Timestamp: s.Timestamp, Value: s.Value, }, + Histogram: SampleHistogramPair{ + Timestamp: s.Timestamp, + Histogram: s.Histogram, + }, } if err := json.Unmarshal(b, &v); err != nil { @@ -174,8 +123,13 @@ func (s *Sample) UnmarshalJSON(b []byte) error { } s.Metric = v.Metric - s.Timestamp = v.Value.Timestamp - s.Value = v.Value.Value + if v.Histogram.Histogram != nil { + s.Timestamp = v.Histogram.Timestamp + s.Histogram = v.Histogram.Histogram + } else { + s.Timestamp = v.Value.Timestamp + s.Value = v.Value.Value + } return nil } @@ -221,80 +175,76 @@ func (s Samples) Equal(o Samples) bool { // SampleStream is a stream of Values belonging to an attached COWMetric. type SampleStream struct { - Metric Metric `json:"metric"` - Values []SamplePair `json:"values"` + Metric Metric `json:"metric"` + Values []SamplePair `json:"values"` + Histograms []SampleHistogramPair `json:"histograms"` } func (ss SampleStream) String() string { - vals := make([]string, len(ss.Values)) + valuesLength := len(ss.Values) + vals := make([]string, valuesLength+len(ss.Histograms)) for i, v := range ss.Values { vals[i] = v.String() } + for i, v := range ss.Histograms { + vals[i+valuesLength] = v.String() + } return fmt.Sprintf("%s =>\n%s", ss.Metric, strings.Join(vals, "\n")) } -// Value is a generic interface for values resulting from a query evaluation. -type Value interface { - Type() ValueType - String() string +func (ss SampleStream) MarshalJSON() ([]byte, error) { + if len(ss.Histograms) > 0 && len(ss.Values) > 0 { + v := struct { + Metric Metric `json:"metric"` + Values []SamplePair `json:"values"` + Histograms []SampleHistogramPair `json:"histograms"` + }{ + Metric: ss.Metric, + Values: ss.Values, + Histograms: ss.Histograms, + } + return json.Marshal(&v) + } else if len(ss.Histograms) > 0 { + v := struct { + Metric Metric `json:"metric"` + Histograms []SampleHistogramPair `json:"histograms"` + }{ + Metric: ss.Metric, + Histograms: ss.Histograms, + } + return json.Marshal(&v) + } else { + v := struct { + Metric Metric `json:"metric"` + Values []SamplePair `json:"values"` + }{ + Metric: ss.Metric, + Values: ss.Values, + } + return json.Marshal(&v) + } } -func (Matrix) Type() ValueType { return ValMatrix } -func (Vector) Type() ValueType { return ValVector } -func (*Scalar) Type() ValueType { return ValScalar } -func (*String) Type() ValueType { return ValString } +func (ss *SampleStream) UnmarshalJSON(b []byte) error { + v := struct { + Metric Metric `json:"metric"` + Values []SamplePair `json:"values"` + Histograms []SampleHistogramPair `json:"histograms"` + }{ + Metric: ss.Metric, + Values: ss.Values, + Histograms: ss.Histograms, + } -type ValueType int - -const ( - ValNone ValueType = iota - ValScalar - ValVector - ValMatrix - ValString -) - -// MarshalJSON implements json.Marshaler. -func (et ValueType) MarshalJSON() ([]byte, error) { - return json.Marshal(et.String()) -} - -func (et *ValueType) UnmarshalJSON(b []byte) error { - var s string - if err := json.Unmarshal(b, &s); err != nil { + if err := json.Unmarshal(b, &v); err != nil { return err } - switch s { - case "": - *et = ValNone - case "scalar": - *et = ValScalar - case "vector": - *et = ValVector - case "matrix": - *et = ValMatrix - case "string": - *et = ValString - default: - return fmt.Errorf("unknown value type %q", s) - } - return nil -} -func (e ValueType) String() string { - switch e { - case ValNone: - return "" - case ValScalar: - return "scalar" - case ValVector: - return "vector" - case ValMatrix: - return "matrix" - case ValString: - return "string" - } - panic("ValueType.String: unhandled value type") + ss.Metric = v.Metric + ss.Values = v.Values + ss.Histograms = v.Histograms + + return nil } // Scalar is a scalar value evaluated at the set timestamp. diff --git a/vendor/github.com/prometheus/common/model/value_float.go b/vendor/github.com/prometheus/common/model/value_float.go new file mode 100644 index 0000000..0f615a7 --- /dev/null +++ b/vendor/github.com/prometheus/common/model/value_float.go @@ -0,0 +1,100 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "encoding/json" + "fmt" + "math" + "strconv" +) + +var ( + // ZeroSamplePair is the pseudo zero-value of SamplePair used to signal a + // non-existing sample pair. It is a SamplePair with timestamp Earliest and + // value 0.0. Note that the natural zero value of SamplePair has a timestamp + // of 0, which is possible to appear in a real SamplePair and thus not + // suitable to signal a non-existing SamplePair. + ZeroSamplePair = SamplePair{Timestamp: Earliest} +) + +// A SampleValue is a representation of a value for a given sample at a given +// time. +type SampleValue float64 + +// MarshalJSON implements json.Marshaler. +func (v SampleValue) MarshalJSON() ([]byte, error) { + return json.Marshal(v.String()) +} + +// UnmarshalJSON implements json.Unmarshaler. +func (v *SampleValue) UnmarshalJSON(b []byte) error { + if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { + return fmt.Errorf("sample value must be a quoted string") + } + f, err := strconv.ParseFloat(string(b[1:len(b)-1]), 64) + if err != nil { + return err + } + *v = SampleValue(f) + return nil +} + +// Equal returns true if the value of v and o is equal or if both are NaN. Note +// that v==o is false if both are NaN. If you want the conventional float +// behavior, use == to compare two SampleValues. +func (v SampleValue) Equal(o SampleValue) bool { + if v == o { + return true + } + return math.IsNaN(float64(v)) && math.IsNaN(float64(o)) +} + +func (v SampleValue) String() string { + return strconv.FormatFloat(float64(v), 'f', -1, 64) +} + +// SamplePair pairs a SampleValue with a Timestamp. +type SamplePair struct { + Timestamp Time + Value SampleValue +} + +func (s SamplePair) MarshalJSON() ([]byte, error) { + t, err := json.Marshal(s.Timestamp) + if err != nil { + return nil, err + } + v, err := json.Marshal(s.Value) + if err != nil { + return nil, err + } + return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (s *SamplePair) UnmarshalJSON(b []byte) error { + v := [...]json.Unmarshaler{&s.Timestamp, &s.Value} + return json.Unmarshal(b, &v) +} + +// Equal returns true if this SamplePair and o have equal Values and equal +// Timestamps. The semantics of Value equality is defined by SampleValue.Equal. +func (s *SamplePair) Equal(o *SamplePair) bool { + return s == o || (s.Value.Equal(o.Value) && s.Timestamp.Equal(o.Timestamp)) +} + +func (s SamplePair) String() string { + return fmt.Sprintf("%s @[%s]", s.Value, s.Timestamp) +} diff --git a/vendor/github.com/prometheus/common/model/value_histogram.go b/vendor/github.com/prometheus/common/model/value_histogram.go new file mode 100644 index 0000000..54bb038 --- /dev/null +++ b/vendor/github.com/prometheus/common/model/value_histogram.go @@ -0,0 +1,178 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" +) + +type FloatString float64 + +func (v FloatString) String() string { + return strconv.FormatFloat(float64(v), 'f', -1, 64) +} + +func (v FloatString) MarshalJSON() ([]byte, error) { + return json.Marshal(v.String()) +} + +func (v *FloatString) UnmarshalJSON(b []byte) error { + if len(b) < 2 || b[0] != '"' || b[len(b)-1] != '"' { + return fmt.Errorf("float value must be a quoted string") + } + f, err := strconv.ParseFloat(string(b[1:len(b)-1]), 64) + if err != nil { + return err + } + *v = FloatString(f) + return nil +} + +type HistogramBucket struct { + Boundaries int32 + Lower FloatString + Upper FloatString + Count FloatString +} + +func (s HistogramBucket) MarshalJSON() ([]byte, error) { + b, err := json.Marshal(s.Boundaries) + if err != nil { + return nil, err + } + l, err := json.Marshal(s.Lower) + if err != nil { + return nil, err + } + u, err := json.Marshal(s.Upper) + if err != nil { + return nil, err + } + c, err := json.Marshal(s.Count) + if err != nil { + return nil, err + } + return []byte(fmt.Sprintf("[%s,%s,%s,%s]", b, l, u, c)), nil +} + +func (s *HistogramBucket) UnmarshalJSON(buf []byte) error { + tmp := []interface{}{&s.Boundaries, &s.Lower, &s.Upper, &s.Count} + wantLen := len(tmp) + if err := json.Unmarshal(buf, &tmp); err != nil { + return err + } + if gotLen := len(tmp); gotLen != wantLen { + return fmt.Errorf("wrong number of fields: %d != %d", gotLen, wantLen) + } + return nil +} + +func (s *HistogramBucket) Equal(o *HistogramBucket) bool { + return s == o || (s.Boundaries == o.Boundaries && s.Lower == o.Lower && s.Upper == o.Upper && s.Count == o.Count) +} + +func (b HistogramBucket) String() string { + var sb strings.Builder + lowerInclusive := b.Boundaries == 1 || b.Boundaries == 3 + upperInclusive := b.Boundaries == 0 || b.Boundaries == 3 + if lowerInclusive { + sb.WriteRune('[') + } else { + sb.WriteRune('(') + } + fmt.Fprintf(&sb, "%g,%g", b.Lower, b.Upper) + if upperInclusive { + sb.WriteRune(']') + } else { + sb.WriteRune(')') + } + fmt.Fprintf(&sb, ":%v", b.Count) + return sb.String() +} + +type HistogramBuckets []*HistogramBucket + +func (s HistogramBuckets) Equal(o HistogramBuckets) bool { + if len(s) != len(o) { + return false + } + + for i, bucket := range s { + if !bucket.Equal(o[i]) { + return false + } + } + return true +} + +type SampleHistogram struct { + Count FloatString `json:"count"` + Sum FloatString `json:"sum"` + Buckets HistogramBuckets `json:"buckets"` +} + +func (s SampleHistogram) String() string { + return fmt.Sprintf("Count: %f, Sum: %f, Buckets: %v", s.Count, s.Sum, s.Buckets) +} + +func (s *SampleHistogram) Equal(o *SampleHistogram) bool { + return s == o || (s.Count == o.Count && s.Sum == o.Sum && s.Buckets.Equal(o.Buckets)) +} + +type SampleHistogramPair struct { + Timestamp Time + // Histogram should never be nil, it's only stored as pointer for efficiency. + Histogram *SampleHistogram +} + +func (s SampleHistogramPair) MarshalJSON() ([]byte, error) { + if s.Histogram == nil { + return nil, fmt.Errorf("histogram is nil") + } + t, err := json.Marshal(s.Timestamp) + if err != nil { + return nil, err + } + v, err := json.Marshal(s.Histogram) + if err != nil { + return nil, err + } + return []byte(fmt.Sprintf("[%s,%s]", t, v)), nil +} + +func (s *SampleHistogramPair) UnmarshalJSON(buf []byte) error { + tmp := []interface{}{&s.Timestamp, &s.Histogram} + wantLen := len(tmp) + if err := json.Unmarshal(buf, &tmp); err != nil { + return err + } + if gotLen := len(tmp); gotLen != wantLen { + return fmt.Errorf("wrong number of fields: %d != %d", gotLen, wantLen) + } + if s.Histogram == nil { + return fmt.Errorf("histogram is null") + } + return nil +} + +func (s SampleHistogramPair) String() string { + return fmt.Sprintf("%s @[%s]", s.Histogram, s.Timestamp) +} + +func (s *SampleHistogramPair) Equal(o *SampleHistogramPair) bool { + return s == o || (s.Histogram.Equal(o.Histogram) && s.Timestamp.Equal(o.Timestamp)) +} diff --git a/vendor/github.com/prometheus/common/model/value_type.go b/vendor/github.com/prometheus/common/model/value_type.go new file mode 100644 index 0000000..726c50e --- /dev/null +++ b/vendor/github.com/prometheus/common/model/value_type.go @@ -0,0 +1,83 @@ +// Copyright 2013 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import ( + "encoding/json" + "fmt" +) + +// Value is a generic interface for values resulting from a query evaluation. +type Value interface { + Type() ValueType + String() string +} + +func (Matrix) Type() ValueType { return ValMatrix } +func (Vector) Type() ValueType { return ValVector } +func (*Scalar) Type() ValueType { return ValScalar } +func (*String) Type() ValueType { return ValString } + +type ValueType int + +const ( + ValNone ValueType = iota + ValScalar + ValVector + ValMatrix + ValString +) + +// MarshalJSON implements json.Marshaler. +func (et ValueType) MarshalJSON() ([]byte, error) { + return json.Marshal(et.String()) +} + +func (et *ValueType) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + switch s { + case "": + *et = ValNone + case "scalar": + *et = ValScalar + case "vector": + *et = ValVector + case "matrix": + *et = ValMatrix + case "string": + *et = ValString + default: + return fmt.Errorf("unknown value type %q", s) + } + return nil +} + +func (e ValueType) String() string { + switch e { + case ValNone: + return "" + case ValScalar: + return "scalar" + case ValVector: + return "vector" + case ValMatrix: + return "matrix" + case ValString: + return "string" + } + panic("ValueType.String: unhandled value type") +} diff --git a/vendor/github.com/prometheus/procfs/Makefile.common b/vendor/github.com/prometheus/procfs/Makefile.common index 6c8e3e2..e358db6 100644 --- a/vendor/github.com/prometheus/procfs/Makefile.common +++ b/vendor/github.com/prometheus/procfs/Makefile.common @@ -55,19 +55,22 @@ ifneq ($(shell which gotestsum),) endif endif -PROMU_VERSION ?= 0.13.0 +PROMU_VERSION ?= 0.14.0 PROMU_URL := https://github.com/prometheus/promu/releases/download/v$(PROMU_VERSION)/promu-$(PROMU_VERSION).$(GO_BUILD_PLATFORM).tar.gz +SKIP_GOLANGCI_LINT := GOLANGCI_LINT := GOLANGCI_LINT_OPTS ?= -GOLANGCI_LINT_VERSION ?= v1.45.2 +GOLANGCI_LINT_VERSION ?= v1.49.0 # golangci-lint only supports linux, darwin and windows platforms on i386/amd64. # windows isn't included here because of the path separator being different. ifeq ($(GOHOSTOS),$(filter $(GOHOSTOS),linux darwin)) ifeq ($(GOHOSTARCH),$(filter $(GOHOSTARCH),amd64 i386)) # If we're in CI and there is an Actions file, that means the linter # is being run in Actions, so we don't need to run it here. - ifeq (,$(CIRCLE_JOB)) + ifneq (,$(SKIP_GOLANGCI_LINT)) + GOLANGCI_LINT := + else ifeq (,$(CIRCLE_JOB)) GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint else ifeq (,$(wildcard .github/workflows/golangci-lint.yml)) GOLANGCI_LINT := $(FIRST_GOPATH)/bin/golangci-lint diff --git a/vendor/github.com/prometheus/procfs/cpuinfo.go b/vendor/github.com/prometheus/procfs/cpuinfo.go index ff6b927..06968ca 100644 --- a/vendor/github.com/prometheus/procfs/cpuinfo.go +++ b/vendor/github.com/prometheus/procfs/cpuinfo.go @@ -380,6 +380,42 @@ func parseCPUInfoMips(info []byte) ([]CPUInfo, error) { return cpuinfo, nil } +func parseCPUInfoLoong(info []byte) ([]CPUInfo, error) { + scanner := bufio.NewScanner(bytes.NewReader(info)) + // find the first "processor" line + firstLine := firstNonEmptyLine(scanner) + if !strings.HasPrefix(firstLine, "system type") || !strings.Contains(firstLine, ":") { + return nil, errors.New("invalid cpuinfo file: " + firstLine) + } + field := strings.SplitN(firstLine, ": ", 2) + cpuinfo := []CPUInfo{} + systemType := field[1] + i := 0 + for scanner.Scan() { + line := scanner.Text() + if !strings.Contains(line, ":") { + continue + } + field := strings.SplitN(line, ": ", 2) + switch strings.TrimSpace(field[0]) { + case "processor": + v, err := strconv.ParseUint(field[1], 0, 32) + if err != nil { + return nil, err + } + i = int(v) + cpuinfo = append(cpuinfo, CPUInfo{}) // start of the next processor + cpuinfo[i].Processor = uint(v) + cpuinfo[i].VendorID = systemType + case "CPU Family": + cpuinfo[i].CPUFamily = field[1] + case "Model Name": + cpuinfo[i].ModelName = field[1] + } + } + return cpuinfo, nil +} + func parseCPUInfoPPC(info []byte) ([]CPUInfo, error) { scanner := bufio.NewScanner(bytes.NewReader(info)) diff --git a/vendor/github.com/cockroachdb/pebble/vfs/preallocate_linux.go b/vendor/github.com/prometheus/procfs/cpuinfo_loong64.go similarity index 65% rename from vendor/github.com/cockroachdb/pebble/vfs/preallocate_linux.go rename to vendor/github.com/prometheus/procfs/cpuinfo_loong64.go index b6b36eb..d88442f 100644 --- a/vendor/github.com/cockroachdb/pebble/vfs/preallocate_linux.go +++ b/vendor/github.com/prometheus/procfs/cpuinfo_loong64.go @@ -1,10 +1,9 @@ -// Copyright 2016 The etcd Authors -// +// Copyright 2022 The Prometheus Authors // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -15,14 +14,6 @@ //go:build linux // +build linux -package vfs +package procfs -import ( - "syscall" - - "golang.org/x/sys/unix" -) - -func preallocExtend(fd uintptr, offset, length int64) error { - return syscall.Fallocate(int(fd), unix.FALLOC_FL_KEEP_SIZE, offset, length) -} +var parseCPUInfo = parseCPUInfoLoong diff --git a/vendor/github.com/prometheus/procfs/cpuinfo_others.go b/vendor/github.com/prometheus/procfs/cpuinfo_others.go index ea41bf2..a6b2b31 100644 --- a/vendor/github.com/prometheus/procfs/cpuinfo_others.go +++ b/vendor/github.com/prometheus/procfs/cpuinfo_others.go @@ -11,8 +11,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -//go:build linux && !386 && !amd64 && !arm && !arm64 && !mips && !mips64 && !mips64le && !mipsle && !ppc64 && !ppc64le && !riscv64 && !s390x -// +build linux,!386,!amd64,!arm,!arm64,!mips,!mips64,!mips64le,!mipsle,!ppc64,!ppc64le,!riscv64,!s390x +//go:build linux && !386 && !amd64 && !arm && !arm64 && !loong64 && !mips && !mips64 && !mips64le && !mipsle && !ppc64 && !ppc64le && !riscv64 && !s390x +// +build linux,!386,!amd64,!arm,!arm64,!loong64,!mips,!mips64,!mips64le,!mipsle,!ppc64,!ppc64le,!riscv64,!s390x package procfs diff --git a/vendor/github.com/prometheus/procfs/doc.go b/vendor/github.com/prometheus/procfs/doc.go index d31a826..f9d961e 100644 --- a/vendor/github.com/prometheus/procfs/doc.go +++ b/vendor/github.com/prometheus/procfs/doc.go @@ -16,30 +16,29 @@ // // Example: // -// package main +// package main // -// import ( -// "fmt" -// "log" +// import ( +// "fmt" +// "log" // -// "github.com/prometheus/procfs" -// ) +// "github.com/prometheus/procfs" +// ) // -// func main() { -// p, err := procfs.Self() -// if err != nil { -// log.Fatalf("could not get process: %s", err) -// } +// func main() { +// p, err := procfs.Self() +// if err != nil { +// log.Fatalf("could not get process: %s", err) +// } // -// stat, err := p.Stat() -// if err != nil { -// log.Fatalf("could not get process stat: %s", err) -// } -// -// fmt.Printf("command: %s\n", stat.Comm) -// fmt.Printf("cpu time: %fs\n", stat.CPUTime()) -// fmt.Printf("vsize: %dB\n", stat.VirtualMemory()) -// fmt.Printf("rss: %dB\n", stat.ResidentMemory()) -// } +// stat, err := p.Stat() +// if err != nil { +// log.Fatalf("could not get process stat: %s", err) +// } // +// fmt.Printf("command: %s\n", stat.Comm) +// fmt.Printf("cpu time: %fs\n", stat.CPUTime()) +// fmt.Printf("vsize: %dB\n", stat.VirtualMemory()) +// fmt.Printf("rss: %dB\n", stat.ResidentMemory()) +// } package procfs diff --git a/vendor/github.com/prometheus/procfs/mountstats.go b/vendor/github.com/prometheus/procfs/mountstats.go index f7a828b..0c482c1 100644 --- a/vendor/github.com/prometheus/procfs/mountstats.go +++ b/vendor/github.com/prometheus/procfs/mountstats.go @@ -284,7 +284,8 @@ func parseMountStats(r io.Reader) ([]*Mount, error) { } // parseMount parses an entry in /proc/[pid]/mountstats in the format: -// device [device] mounted on [mount] with fstype [type] +// +// device [device] mounted on [mount] with fstype [type] func parseMount(ss []string) (*Mount, error) { if len(ss) < deviceEntryLen { return nil, fmt.Errorf("invalid device entry: %v", ss) diff --git a/vendor/github.com/prometheus/procfs/net_softnet.go b/vendor/github.com/prometheus/procfs/net_softnet.go index a94f86d..06b7b8f 100644 --- a/vendor/github.com/prometheus/procfs/net_softnet.go +++ b/vendor/github.com/prometheus/procfs/net_softnet.go @@ -27,8 +27,9 @@ import ( // For the proc file format details, // See: // * Linux 2.6.23 https://elixir.bootlin.com/linux/v2.6.23/source/net/core/dev.c#L2343 -// * Linux 4.17 https://elixir.bootlin.com/linux/v4.17/source/net/core/net-procfs.c#L162 -// and https://elixir.bootlin.com/linux/v4.17/source/include/linux/netdevice.h#L2810. +// * Linux 2.6.39 https://elixir.bootlin.com/linux/v2.6.39/source/net/core/dev.c#L4086 +// * Linux 4.18 https://elixir.bootlin.com/linux/v4.18/source/net/core/net-procfs.c#L162 +// * Linux 5.14 https://elixir.bootlin.com/linux/v5.14/source/net/core/net-procfs.c#L169 // SoftnetStat contains a single row of data from /proc/net/softnet_stat. type SoftnetStat struct { @@ -38,6 +39,18 @@ type SoftnetStat struct { Dropped uint32 // Number of times processing packets ran out of quota. TimeSqueezed uint32 + // Number of collision occur while obtaining device lock while transmitting. + CPUCollision uint32 + // Number of times cpu woken up received_rps. + ReceivedRps uint32 + // number of times flow limit has been reached. + FlowLimitCount uint32 + // Softnet backlog status. + SoftnetBacklogLen uint32 + // CPU id owning this softnet_data. + Index uint32 + // softnet_data's Width. + Width int } var softNetProcFile = "net/softnet_stat" @@ -66,22 +79,57 @@ func parseSoftnet(r io.Reader) ([]SoftnetStat, error) { for s.Scan() { columns := strings.Fields(s.Text()) width := len(columns) + softnetStat := SoftnetStat{} if width < minColumns { return nil, fmt.Errorf("%d columns were detected, but at least %d were expected", width, minColumns) } - // We only parse the first three columns at the moment. - us, err := parseHexUint32s(columns[0:3]) - if err != nil { - return nil, err + // Linux 2.6.23 https://elixir.bootlin.com/linux/v2.6.23/source/net/core/dev.c#L2347 + if width >= minColumns { + us, err := parseHexUint32s(columns[0:9]) + if err != nil { + return nil, err + } + + softnetStat.Processed = us[0] + softnetStat.Dropped = us[1] + softnetStat.TimeSqueezed = us[2] + softnetStat.CPUCollision = us[8] } - stats = append(stats, SoftnetStat{ - Processed: us[0], - Dropped: us[1], - TimeSqueezed: us[2], - }) + // Linux 2.6.39 https://elixir.bootlin.com/linux/v2.6.39/source/net/core/dev.c#L4086 + if width >= 10 { + us, err := parseHexUint32s(columns[9:10]) + if err != nil { + return nil, err + } + + softnetStat.ReceivedRps = us[0] + } + + // Linux 4.18 https://elixir.bootlin.com/linux/v4.18/source/net/core/net-procfs.c#L162 + if width >= 11 { + us, err := parseHexUint32s(columns[10:11]) + if err != nil { + return nil, err + } + + softnetStat.FlowLimitCount = us[0] + } + + // Linux 5.14 https://elixir.bootlin.com/linux/v5.14/source/net/core/net-procfs.c#L169 + if width >= 13 { + us, err := parseHexUint32s(columns[11:13]) + if err != nil { + return nil, err + } + + softnetStat.SoftnetBacklogLen = us[0] + softnetStat.Index = us[1] + } + softnetStat.Width = width + stats = append(stats, softnetStat) } return stats, nil diff --git a/vendor/github.com/prometheus/procfs/netstat.go b/vendor/github.com/prometheus/procfs/netstat.go index dcea9c5..5cc40ae 100644 --- a/vendor/github.com/prometheus/procfs/netstat.go +++ b/vendor/github.com/prometheus/procfs/netstat.go @@ -15,6 +15,7 @@ package procfs import ( "bufio" + "io" "os" "path/filepath" "strconv" @@ -42,27 +43,43 @@ func (fs FS) NetStat() ([]NetStat, error) { return nil, err } - netStatFile := NetStat{ - Filename: filepath.Base(filePath), - Stats: make(map[string][]uint64), + procNetstat, err := parseNetstat(file) + if err != nil { + return nil, err } - scanner := bufio.NewScanner(file) - scanner.Scan() - // First string is always a header for stats - var headers []string - headers = append(headers, strings.Fields(scanner.Text())...) + procNetstat.Filename = filepath.Base(filePath) - // Other strings represent per-CPU counters - for scanner.Scan() { - for num, counter := range strings.Fields(scanner.Text()) { - value, err := strconv.ParseUint(counter, 16, 64) - if err != nil { - return nil, err - } - netStatFile.Stats[headers[num]] = append(netStatFile.Stats[headers[num]], value) - } - } - netStatsTotal = append(netStatsTotal, netStatFile) + netStatsTotal = append(netStatsTotal, procNetstat) } return netStatsTotal, nil } + +// parseNetstat parses the metrics from `/proc/net/stat/` file +// and returns a NetStat structure. +func parseNetstat(r io.Reader) (NetStat, error) { + var ( + scanner = bufio.NewScanner(r) + netStat = NetStat{ + Stats: make(map[string][]uint64), + } + ) + + scanner.Scan() + + // First string is always a header for stats + var headers []string + headers = append(headers, strings.Fields(scanner.Text())...) + + // Other strings represent per-CPU counters + for scanner.Scan() { + for num, counter := range strings.Fields(scanner.Text()) { + value, err := strconv.ParseUint(counter, 16, 64) + if err != nil { + return NetStat{}, err + } + netStat.Stats[headers[num]] = append(netStat.Stats[headers[num]], value) + } + } + + return netStat, nil +} diff --git a/vendor/github.com/prometheus/procfs/proc_cgroup.go b/vendor/github.com/prometheus/procfs/proc_cgroup.go index cca0332..ea83a75 100644 --- a/vendor/github.com/prometheus/procfs/proc_cgroup.go +++ b/vendor/github.com/prometheus/procfs/proc_cgroup.go @@ -23,7 +23,7 @@ import ( "github.com/prometheus/procfs/internal/util" ) -// Cgroup models one line from /proc/[pid]/cgroup. Each Cgroup struct describes the the placement of a PID inside a +// Cgroup models one line from /proc/[pid]/cgroup. Each Cgroup struct describes the placement of a PID inside a // specific control hierarchy. The kernel has two cgroup APIs, v1 and v2. v1 has one hierarchy per available resource // controller, while v2 has one unified hierarchy shared by all controllers. Regardless of v1 or v2, all hierarchies // contain all running processes, so the question answerable with a Cgroup struct is 'where is this process in diff --git a/vendor/github.com/prometheus/procfs/proc_interrupts.go b/vendor/github.com/prometheus/procfs/proc_interrupts.go new file mode 100644 index 0000000..9df79c2 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/proc_interrupts.go @@ -0,0 +1,98 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procfs + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "strconv" + "strings" + + "github.com/prometheus/procfs/internal/util" +) + +// Interrupt represents a single interrupt line. +type Interrupt struct { + // Info is the type of interrupt. + Info string + // Devices is the name of the device that is located at that IRQ + Devices string + // Values is the number of interrupts per CPU. + Values []string +} + +// Interrupts models the content of /proc/interrupts. Key is the IRQ number. +// - https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/6/html/deployment_guide/s2-proc-interrupts +// - https://raspberrypi.stackexchange.com/questions/105802/explanation-of-proc-interrupts-output +type Interrupts map[string]Interrupt + +// Interrupts creates a new instance from a given Proc instance. +func (p Proc) Interrupts() (Interrupts, error) { + data, err := util.ReadFileNoStat(p.path("interrupts")) + if err != nil { + return nil, err + } + return parseInterrupts(bytes.NewReader(data)) +} + +func parseInterrupts(r io.Reader) (Interrupts, error) { + var ( + interrupts = Interrupts{} + scanner = bufio.NewScanner(r) + ) + + if !scanner.Scan() { + return nil, errors.New("interrupts empty") + } + cpuNum := len(strings.Fields(scanner.Text())) // one header per cpu + + for scanner.Scan() { + parts := strings.Fields(scanner.Text()) + if len(parts) == 0 { // skip empty lines + continue + } + if len(parts) < 2 { + return nil, fmt.Errorf("not enough fields in interrupts (expected at least 2 fields but got %d): %s", len(parts), parts) + } + intName := parts[0][:len(parts[0])-1] // remove trailing : + + if len(parts) == 2 { + interrupts[intName] = Interrupt{ + Info: "", + Devices: "", + Values: []string{ + parts[1], + }, + } + continue + } + + intr := Interrupt{ + Values: parts[1 : cpuNum+1], + } + + if _, err := strconv.Atoi(intName); err == nil { // numeral interrupt + intr.Info = parts[cpuNum+1] + intr.Devices = strings.Join(parts[cpuNum+2:], " ") + } else { + intr.Info = strings.Join(parts[cpuNum+1:], " ") + } + interrupts[intName] = intr + } + + return interrupts, scanner.Err() +} diff --git a/vendor/github.com/prometheus/procfs/proc_netstat.go b/vendor/github.com/prometheus/procfs/proc_netstat.go index 48b5238..6a43bb2 100644 --- a/vendor/github.com/prometheus/procfs/proc_netstat.go +++ b/vendor/github.com/prometheus/procfs/proc_netstat.go @@ -33,139 +33,140 @@ type ProcNetstat struct { } type TcpExt struct { // nolint:revive - SyncookiesSent float64 - SyncookiesRecv float64 - SyncookiesFailed float64 - EmbryonicRsts float64 - PruneCalled float64 - RcvPruned float64 - OfoPruned float64 - OutOfWindowIcmps float64 - LockDroppedIcmps float64 - ArpFilter float64 - TW float64 - TWRecycled float64 - TWKilled float64 - PAWSActive float64 - PAWSEstab float64 - DelayedACKs float64 - DelayedACKLocked float64 - DelayedACKLost float64 - ListenOverflows float64 - ListenDrops float64 - TCPHPHits float64 - TCPPureAcks float64 - TCPHPAcks float64 - TCPRenoRecovery float64 - TCPSackRecovery float64 - TCPSACKReneging float64 - TCPSACKReorder float64 - TCPRenoReorder float64 - TCPTSReorder float64 - TCPFullUndo float64 - TCPPartialUndo float64 - TCPDSACKUndo float64 - TCPLossUndo float64 - TCPLostRetransmit float64 - TCPRenoFailures float64 - TCPSackFailures float64 - TCPLossFailures float64 - TCPFastRetrans float64 - TCPSlowStartRetrans float64 - TCPTimeouts float64 - TCPLossProbes float64 - TCPLossProbeRecovery float64 - TCPRenoRecoveryFail float64 - TCPSackRecoveryFail float64 - TCPRcvCollapsed float64 - TCPDSACKOldSent float64 - TCPDSACKOfoSent float64 - TCPDSACKRecv float64 - TCPDSACKOfoRecv float64 - TCPAbortOnData float64 - TCPAbortOnClose float64 - TCPAbortOnMemory float64 - TCPAbortOnTimeout float64 - TCPAbortOnLinger float64 - TCPAbortFailed float64 - TCPMemoryPressures float64 - TCPMemoryPressuresChrono float64 - TCPSACKDiscard float64 - TCPDSACKIgnoredOld float64 - TCPDSACKIgnoredNoUndo float64 - TCPSpuriousRTOs float64 - TCPMD5NotFound float64 - TCPMD5Unexpected float64 - TCPMD5Failure float64 - TCPSackShifted float64 - TCPSackMerged float64 - TCPSackShiftFallback float64 - TCPBacklogDrop float64 - PFMemallocDrop float64 - TCPMinTTLDrop float64 - TCPDeferAcceptDrop float64 - IPReversePathFilter float64 - TCPTimeWaitOverflow float64 - TCPReqQFullDoCookies float64 - TCPReqQFullDrop float64 - TCPRetransFail float64 - TCPRcvCoalesce float64 - TCPOFOQueue float64 - TCPOFODrop float64 - TCPOFOMerge float64 - TCPChallengeACK float64 - TCPSYNChallenge float64 - TCPFastOpenActive float64 - TCPFastOpenActiveFail float64 - TCPFastOpenPassive float64 - TCPFastOpenPassiveFail float64 - TCPFastOpenListenOverflow float64 - TCPFastOpenCookieReqd float64 - TCPFastOpenBlackhole float64 - TCPSpuriousRtxHostQueues float64 - BusyPollRxPackets float64 - TCPAutoCorking float64 - TCPFromZeroWindowAdv float64 - TCPToZeroWindowAdv float64 - TCPWantZeroWindowAdv float64 - TCPSynRetrans float64 - TCPOrigDataSent float64 - TCPHystartTrainDetect float64 - TCPHystartTrainCwnd float64 - TCPHystartDelayDetect float64 - TCPHystartDelayCwnd float64 - TCPACKSkippedSynRecv float64 - TCPACKSkippedPAWS float64 - TCPACKSkippedSeq float64 - TCPACKSkippedFinWait2 float64 - TCPACKSkippedTimeWait float64 - TCPACKSkippedChallenge float64 - TCPWinProbe float64 - TCPKeepAlive float64 - TCPMTUPFail float64 - TCPMTUPSuccess float64 - TCPWqueueTooBig float64 + SyncookiesSent *float64 + SyncookiesRecv *float64 + SyncookiesFailed *float64 + EmbryonicRsts *float64 + PruneCalled *float64 + RcvPruned *float64 + OfoPruned *float64 + OutOfWindowIcmps *float64 + LockDroppedIcmps *float64 + ArpFilter *float64 + TW *float64 + TWRecycled *float64 + TWKilled *float64 + PAWSActive *float64 + PAWSEstab *float64 + DelayedACKs *float64 + DelayedACKLocked *float64 + DelayedACKLost *float64 + ListenOverflows *float64 + ListenDrops *float64 + TCPHPHits *float64 + TCPPureAcks *float64 + TCPHPAcks *float64 + TCPRenoRecovery *float64 + TCPSackRecovery *float64 + TCPSACKReneging *float64 + TCPSACKReorder *float64 + TCPRenoReorder *float64 + TCPTSReorder *float64 + TCPFullUndo *float64 + TCPPartialUndo *float64 + TCPDSACKUndo *float64 + TCPLossUndo *float64 + TCPLostRetransmit *float64 + TCPRenoFailures *float64 + TCPSackFailures *float64 + TCPLossFailures *float64 + TCPFastRetrans *float64 + TCPSlowStartRetrans *float64 + TCPTimeouts *float64 + TCPLossProbes *float64 + TCPLossProbeRecovery *float64 + TCPRenoRecoveryFail *float64 + TCPSackRecoveryFail *float64 + TCPRcvCollapsed *float64 + TCPDSACKOldSent *float64 + TCPDSACKOfoSent *float64 + TCPDSACKRecv *float64 + TCPDSACKOfoRecv *float64 + TCPAbortOnData *float64 + TCPAbortOnClose *float64 + TCPAbortOnMemory *float64 + TCPAbortOnTimeout *float64 + TCPAbortOnLinger *float64 + TCPAbortFailed *float64 + TCPMemoryPressures *float64 + TCPMemoryPressuresChrono *float64 + TCPSACKDiscard *float64 + TCPDSACKIgnoredOld *float64 + TCPDSACKIgnoredNoUndo *float64 + TCPSpuriousRTOs *float64 + TCPMD5NotFound *float64 + TCPMD5Unexpected *float64 + TCPMD5Failure *float64 + TCPSackShifted *float64 + TCPSackMerged *float64 + TCPSackShiftFallback *float64 + TCPBacklogDrop *float64 + PFMemallocDrop *float64 + TCPMinTTLDrop *float64 + TCPDeferAcceptDrop *float64 + IPReversePathFilter *float64 + TCPTimeWaitOverflow *float64 + TCPReqQFullDoCookies *float64 + TCPReqQFullDrop *float64 + TCPRetransFail *float64 + TCPRcvCoalesce *float64 + TCPRcvQDrop *float64 + TCPOFOQueue *float64 + TCPOFODrop *float64 + TCPOFOMerge *float64 + TCPChallengeACK *float64 + TCPSYNChallenge *float64 + TCPFastOpenActive *float64 + TCPFastOpenActiveFail *float64 + TCPFastOpenPassive *float64 + TCPFastOpenPassiveFail *float64 + TCPFastOpenListenOverflow *float64 + TCPFastOpenCookieReqd *float64 + TCPFastOpenBlackhole *float64 + TCPSpuriousRtxHostQueues *float64 + BusyPollRxPackets *float64 + TCPAutoCorking *float64 + TCPFromZeroWindowAdv *float64 + TCPToZeroWindowAdv *float64 + TCPWantZeroWindowAdv *float64 + TCPSynRetrans *float64 + TCPOrigDataSent *float64 + TCPHystartTrainDetect *float64 + TCPHystartTrainCwnd *float64 + TCPHystartDelayDetect *float64 + TCPHystartDelayCwnd *float64 + TCPACKSkippedSynRecv *float64 + TCPACKSkippedPAWS *float64 + TCPACKSkippedSeq *float64 + TCPACKSkippedFinWait2 *float64 + TCPACKSkippedTimeWait *float64 + TCPACKSkippedChallenge *float64 + TCPWinProbe *float64 + TCPKeepAlive *float64 + TCPMTUPFail *float64 + TCPMTUPSuccess *float64 + TCPWqueueTooBig *float64 } type IpExt struct { // nolint:revive - InNoRoutes float64 - InTruncatedPkts float64 - InMcastPkts float64 - OutMcastPkts float64 - InBcastPkts float64 - OutBcastPkts float64 - InOctets float64 - OutOctets float64 - InMcastOctets float64 - OutMcastOctets float64 - InBcastOctets float64 - OutBcastOctets float64 - InCsumErrors float64 - InNoECTPkts float64 - InECT1Pkts float64 - InECT0Pkts float64 - InCEPkts float64 - ReasmOverlaps float64 + InNoRoutes *float64 + InTruncatedPkts *float64 + InMcastPkts *float64 + OutMcastPkts *float64 + InBcastPkts *float64 + OutBcastPkts *float64 + InOctets *float64 + OutOctets *float64 + InMcastOctets *float64 + OutMcastOctets *float64 + InBcastOctets *float64 + OutBcastOctets *float64 + InCsumErrors *float64 + InNoECTPkts *float64 + InECT1Pkts *float64 + InECT0Pkts *float64 + InCEPkts *float64 + ReasmOverlaps *float64 } func (p Proc) Netstat() (ProcNetstat, error) { @@ -174,14 +175,14 @@ func (p Proc) Netstat() (ProcNetstat, error) { if err != nil { return ProcNetstat{PID: p.PID}, err } - procNetstat, err := parseNetstat(bytes.NewReader(data), filename) + procNetstat, err := parseProcNetstat(bytes.NewReader(data), filename) procNetstat.PID = p.PID return procNetstat, err } -// parseNetstat parses the metrics from proc//net/netstat file +// parseProcNetstat parses the metrics from proc//net/netstat file // and returns a ProcNetstat structure. -func parseNetstat(r io.Reader, fileName string) (ProcNetstat, error) { +func parseProcNetstat(r io.Reader, fileName string) (ProcNetstat, error) { var ( scanner = bufio.NewScanner(r) procNetstat = ProcNetstat{} @@ -208,230 +209,232 @@ func parseNetstat(r io.Reader, fileName string) (ProcNetstat, error) { case "TcpExt": switch key { case "SyncookiesSent": - procNetstat.TcpExt.SyncookiesSent = value + procNetstat.TcpExt.SyncookiesSent = &value case "SyncookiesRecv": - procNetstat.TcpExt.SyncookiesRecv = value + procNetstat.TcpExt.SyncookiesRecv = &value case "SyncookiesFailed": - procNetstat.TcpExt.SyncookiesFailed = value + procNetstat.TcpExt.SyncookiesFailed = &value case "EmbryonicRsts": - procNetstat.TcpExt.EmbryonicRsts = value + procNetstat.TcpExt.EmbryonicRsts = &value case "PruneCalled": - procNetstat.TcpExt.PruneCalled = value + procNetstat.TcpExt.PruneCalled = &value case "RcvPruned": - procNetstat.TcpExt.RcvPruned = value + procNetstat.TcpExt.RcvPruned = &value case "OfoPruned": - procNetstat.TcpExt.OfoPruned = value + procNetstat.TcpExt.OfoPruned = &value case "OutOfWindowIcmps": - procNetstat.TcpExt.OutOfWindowIcmps = value + procNetstat.TcpExt.OutOfWindowIcmps = &value case "LockDroppedIcmps": - procNetstat.TcpExt.LockDroppedIcmps = value + procNetstat.TcpExt.LockDroppedIcmps = &value case "ArpFilter": - procNetstat.TcpExt.ArpFilter = value + procNetstat.TcpExt.ArpFilter = &value case "TW": - procNetstat.TcpExt.TW = value + procNetstat.TcpExt.TW = &value case "TWRecycled": - procNetstat.TcpExt.TWRecycled = value + procNetstat.TcpExt.TWRecycled = &value case "TWKilled": - procNetstat.TcpExt.TWKilled = value + procNetstat.TcpExt.TWKilled = &value case "PAWSActive": - procNetstat.TcpExt.PAWSActive = value + procNetstat.TcpExt.PAWSActive = &value case "PAWSEstab": - procNetstat.TcpExt.PAWSEstab = value + procNetstat.TcpExt.PAWSEstab = &value case "DelayedACKs": - procNetstat.TcpExt.DelayedACKs = value + procNetstat.TcpExt.DelayedACKs = &value case "DelayedACKLocked": - procNetstat.TcpExt.DelayedACKLocked = value + procNetstat.TcpExt.DelayedACKLocked = &value case "DelayedACKLost": - procNetstat.TcpExt.DelayedACKLost = value + procNetstat.TcpExt.DelayedACKLost = &value case "ListenOverflows": - procNetstat.TcpExt.ListenOverflows = value + procNetstat.TcpExt.ListenOverflows = &value case "ListenDrops": - procNetstat.TcpExt.ListenDrops = value + procNetstat.TcpExt.ListenDrops = &value case "TCPHPHits": - procNetstat.TcpExt.TCPHPHits = value + procNetstat.TcpExt.TCPHPHits = &value case "TCPPureAcks": - procNetstat.TcpExt.TCPPureAcks = value + procNetstat.TcpExt.TCPPureAcks = &value case "TCPHPAcks": - procNetstat.TcpExt.TCPHPAcks = value + procNetstat.TcpExt.TCPHPAcks = &value case "TCPRenoRecovery": - procNetstat.TcpExt.TCPRenoRecovery = value + procNetstat.TcpExt.TCPRenoRecovery = &value case "TCPSackRecovery": - procNetstat.TcpExt.TCPSackRecovery = value + procNetstat.TcpExt.TCPSackRecovery = &value case "TCPSACKReneging": - procNetstat.TcpExt.TCPSACKReneging = value + procNetstat.TcpExt.TCPSACKReneging = &value case "TCPSACKReorder": - procNetstat.TcpExt.TCPSACKReorder = value + procNetstat.TcpExt.TCPSACKReorder = &value case "TCPRenoReorder": - procNetstat.TcpExt.TCPRenoReorder = value + procNetstat.TcpExt.TCPRenoReorder = &value case "TCPTSReorder": - procNetstat.TcpExt.TCPTSReorder = value + procNetstat.TcpExt.TCPTSReorder = &value case "TCPFullUndo": - procNetstat.TcpExt.TCPFullUndo = value + procNetstat.TcpExt.TCPFullUndo = &value case "TCPPartialUndo": - procNetstat.TcpExt.TCPPartialUndo = value + procNetstat.TcpExt.TCPPartialUndo = &value case "TCPDSACKUndo": - procNetstat.TcpExt.TCPDSACKUndo = value + procNetstat.TcpExt.TCPDSACKUndo = &value case "TCPLossUndo": - procNetstat.TcpExt.TCPLossUndo = value + procNetstat.TcpExt.TCPLossUndo = &value case "TCPLostRetransmit": - procNetstat.TcpExt.TCPLostRetransmit = value + procNetstat.TcpExt.TCPLostRetransmit = &value case "TCPRenoFailures": - procNetstat.TcpExt.TCPRenoFailures = value + procNetstat.TcpExt.TCPRenoFailures = &value case "TCPSackFailures": - procNetstat.TcpExt.TCPSackFailures = value + procNetstat.TcpExt.TCPSackFailures = &value case "TCPLossFailures": - procNetstat.TcpExt.TCPLossFailures = value + procNetstat.TcpExt.TCPLossFailures = &value case "TCPFastRetrans": - procNetstat.TcpExt.TCPFastRetrans = value + procNetstat.TcpExt.TCPFastRetrans = &value case "TCPSlowStartRetrans": - procNetstat.TcpExt.TCPSlowStartRetrans = value + procNetstat.TcpExt.TCPSlowStartRetrans = &value case "TCPTimeouts": - procNetstat.TcpExt.TCPTimeouts = value + procNetstat.TcpExt.TCPTimeouts = &value case "TCPLossProbes": - procNetstat.TcpExt.TCPLossProbes = value + procNetstat.TcpExt.TCPLossProbes = &value case "TCPLossProbeRecovery": - procNetstat.TcpExt.TCPLossProbeRecovery = value + procNetstat.TcpExt.TCPLossProbeRecovery = &value case "TCPRenoRecoveryFail": - procNetstat.TcpExt.TCPRenoRecoveryFail = value + procNetstat.TcpExt.TCPRenoRecoveryFail = &value case "TCPSackRecoveryFail": - procNetstat.TcpExt.TCPSackRecoveryFail = value + procNetstat.TcpExt.TCPSackRecoveryFail = &value case "TCPRcvCollapsed": - procNetstat.TcpExt.TCPRcvCollapsed = value + procNetstat.TcpExt.TCPRcvCollapsed = &value case "TCPDSACKOldSent": - procNetstat.TcpExt.TCPDSACKOldSent = value + procNetstat.TcpExt.TCPDSACKOldSent = &value case "TCPDSACKOfoSent": - procNetstat.TcpExt.TCPDSACKOfoSent = value + procNetstat.TcpExt.TCPDSACKOfoSent = &value case "TCPDSACKRecv": - procNetstat.TcpExt.TCPDSACKRecv = value + procNetstat.TcpExt.TCPDSACKRecv = &value case "TCPDSACKOfoRecv": - procNetstat.TcpExt.TCPDSACKOfoRecv = value + procNetstat.TcpExt.TCPDSACKOfoRecv = &value case "TCPAbortOnData": - procNetstat.TcpExt.TCPAbortOnData = value + procNetstat.TcpExt.TCPAbortOnData = &value case "TCPAbortOnClose": - procNetstat.TcpExt.TCPAbortOnClose = value + procNetstat.TcpExt.TCPAbortOnClose = &value case "TCPDeferAcceptDrop": - procNetstat.TcpExt.TCPDeferAcceptDrop = value + procNetstat.TcpExt.TCPDeferAcceptDrop = &value case "IPReversePathFilter": - procNetstat.TcpExt.IPReversePathFilter = value + procNetstat.TcpExt.IPReversePathFilter = &value case "TCPTimeWaitOverflow": - procNetstat.TcpExt.TCPTimeWaitOverflow = value + procNetstat.TcpExt.TCPTimeWaitOverflow = &value case "TCPReqQFullDoCookies": - procNetstat.TcpExt.TCPReqQFullDoCookies = value + procNetstat.TcpExt.TCPReqQFullDoCookies = &value case "TCPReqQFullDrop": - procNetstat.TcpExt.TCPReqQFullDrop = value + procNetstat.TcpExt.TCPReqQFullDrop = &value case "TCPRetransFail": - procNetstat.TcpExt.TCPRetransFail = value + procNetstat.TcpExt.TCPRetransFail = &value case "TCPRcvCoalesce": - procNetstat.TcpExt.TCPRcvCoalesce = value + procNetstat.TcpExt.TCPRcvCoalesce = &value + case "TCPRcvQDrop": + procNetstat.TcpExt.TCPRcvQDrop = &value case "TCPOFOQueue": - procNetstat.TcpExt.TCPOFOQueue = value + procNetstat.TcpExt.TCPOFOQueue = &value case "TCPOFODrop": - procNetstat.TcpExt.TCPOFODrop = value + procNetstat.TcpExt.TCPOFODrop = &value case "TCPOFOMerge": - procNetstat.TcpExt.TCPOFOMerge = value + procNetstat.TcpExt.TCPOFOMerge = &value case "TCPChallengeACK": - procNetstat.TcpExt.TCPChallengeACK = value + procNetstat.TcpExt.TCPChallengeACK = &value case "TCPSYNChallenge": - procNetstat.TcpExt.TCPSYNChallenge = value + procNetstat.TcpExt.TCPSYNChallenge = &value case "TCPFastOpenActive": - procNetstat.TcpExt.TCPFastOpenActive = value + procNetstat.TcpExt.TCPFastOpenActive = &value case "TCPFastOpenActiveFail": - procNetstat.TcpExt.TCPFastOpenActiveFail = value + procNetstat.TcpExt.TCPFastOpenActiveFail = &value case "TCPFastOpenPassive": - procNetstat.TcpExt.TCPFastOpenPassive = value + procNetstat.TcpExt.TCPFastOpenPassive = &value case "TCPFastOpenPassiveFail": - procNetstat.TcpExt.TCPFastOpenPassiveFail = value + procNetstat.TcpExt.TCPFastOpenPassiveFail = &value case "TCPFastOpenListenOverflow": - procNetstat.TcpExt.TCPFastOpenListenOverflow = value + procNetstat.TcpExt.TCPFastOpenListenOverflow = &value case "TCPFastOpenCookieReqd": - procNetstat.TcpExt.TCPFastOpenCookieReqd = value + procNetstat.TcpExt.TCPFastOpenCookieReqd = &value case "TCPFastOpenBlackhole": - procNetstat.TcpExt.TCPFastOpenBlackhole = value + procNetstat.TcpExt.TCPFastOpenBlackhole = &value case "TCPSpuriousRtxHostQueues": - procNetstat.TcpExt.TCPSpuriousRtxHostQueues = value + procNetstat.TcpExt.TCPSpuriousRtxHostQueues = &value case "BusyPollRxPackets": - procNetstat.TcpExt.BusyPollRxPackets = value + procNetstat.TcpExt.BusyPollRxPackets = &value case "TCPAutoCorking": - procNetstat.TcpExt.TCPAutoCorking = value + procNetstat.TcpExt.TCPAutoCorking = &value case "TCPFromZeroWindowAdv": - procNetstat.TcpExt.TCPFromZeroWindowAdv = value + procNetstat.TcpExt.TCPFromZeroWindowAdv = &value case "TCPToZeroWindowAdv": - procNetstat.TcpExt.TCPToZeroWindowAdv = value + procNetstat.TcpExt.TCPToZeroWindowAdv = &value case "TCPWantZeroWindowAdv": - procNetstat.TcpExt.TCPWantZeroWindowAdv = value + procNetstat.TcpExt.TCPWantZeroWindowAdv = &value case "TCPSynRetrans": - procNetstat.TcpExt.TCPSynRetrans = value + procNetstat.TcpExt.TCPSynRetrans = &value case "TCPOrigDataSent": - procNetstat.TcpExt.TCPOrigDataSent = value + procNetstat.TcpExt.TCPOrigDataSent = &value case "TCPHystartTrainDetect": - procNetstat.TcpExt.TCPHystartTrainDetect = value + procNetstat.TcpExt.TCPHystartTrainDetect = &value case "TCPHystartTrainCwnd": - procNetstat.TcpExt.TCPHystartTrainCwnd = value + procNetstat.TcpExt.TCPHystartTrainCwnd = &value case "TCPHystartDelayDetect": - procNetstat.TcpExt.TCPHystartDelayDetect = value + procNetstat.TcpExt.TCPHystartDelayDetect = &value case "TCPHystartDelayCwnd": - procNetstat.TcpExt.TCPHystartDelayCwnd = value + procNetstat.TcpExt.TCPHystartDelayCwnd = &value case "TCPACKSkippedSynRecv": - procNetstat.TcpExt.TCPACKSkippedSynRecv = value + procNetstat.TcpExt.TCPACKSkippedSynRecv = &value case "TCPACKSkippedPAWS": - procNetstat.TcpExt.TCPACKSkippedPAWS = value + procNetstat.TcpExt.TCPACKSkippedPAWS = &value case "TCPACKSkippedSeq": - procNetstat.TcpExt.TCPACKSkippedSeq = value + procNetstat.TcpExt.TCPACKSkippedSeq = &value case "TCPACKSkippedFinWait2": - procNetstat.TcpExt.TCPACKSkippedFinWait2 = value + procNetstat.TcpExt.TCPACKSkippedFinWait2 = &value case "TCPACKSkippedTimeWait": - procNetstat.TcpExt.TCPACKSkippedTimeWait = value + procNetstat.TcpExt.TCPACKSkippedTimeWait = &value case "TCPACKSkippedChallenge": - procNetstat.TcpExt.TCPACKSkippedChallenge = value + procNetstat.TcpExt.TCPACKSkippedChallenge = &value case "TCPWinProbe": - procNetstat.TcpExt.TCPWinProbe = value + procNetstat.TcpExt.TCPWinProbe = &value case "TCPKeepAlive": - procNetstat.TcpExt.TCPKeepAlive = value + procNetstat.TcpExt.TCPKeepAlive = &value case "TCPMTUPFail": - procNetstat.TcpExt.TCPMTUPFail = value + procNetstat.TcpExt.TCPMTUPFail = &value case "TCPMTUPSuccess": - procNetstat.TcpExt.TCPMTUPSuccess = value + procNetstat.TcpExt.TCPMTUPSuccess = &value case "TCPWqueueTooBig": - procNetstat.TcpExt.TCPWqueueTooBig = value + procNetstat.TcpExt.TCPWqueueTooBig = &value } case "IpExt": switch key { case "InNoRoutes": - procNetstat.IpExt.InNoRoutes = value + procNetstat.IpExt.InNoRoutes = &value case "InTruncatedPkts": - procNetstat.IpExt.InTruncatedPkts = value + procNetstat.IpExt.InTruncatedPkts = &value case "InMcastPkts": - procNetstat.IpExt.InMcastPkts = value + procNetstat.IpExt.InMcastPkts = &value case "OutMcastPkts": - procNetstat.IpExt.OutMcastPkts = value + procNetstat.IpExt.OutMcastPkts = &value case "InBcastPkts": - procNetstat.IpExt.InBcastPkts = value + procNetstat.IpExt.InBcastPkts = &value case "OutBcastPkts": - procNetstat.IpExt.OutBcastPkts = value + procNetstat.IpExt.OutBcastPkts = &value case "InOctets": - procNetstat.IpExt.InOctets = value + procNetstat.IpExt.InOctets = &value case "OutOctets": - procNetstat.IpExt.OutOctets = value + procNetstat.IpExt.OutOctets = &value case "InMcastOctets": - procNetstat.IpExt.InMcastOctets = value + procNetstat.IpExt.InMcastOctets = &value case "OutMcastOctets": - procNetstat.IpExt.OutMcastOctets = value + procNetstat.IpExt.OutMcastOctets = &value case "InBcastOctets": - procNetstat.IpExt.InBcastOctets = value + procNetstat.IpExt.InBcastOctets = &value case "OutBcastOctets": - procNetstat.IpExt.OutBcastOctets = value + procNetstat.IpExt.OutBcastOctets = &value case "InCsumErrors": - procNetstat.IpExt.InCsumErrors = value + procNetstat.IpExt.InCsumErrors = &value case "InNoECTPkts": - procNetstat.IpExt.InNoECTPkts = value + procNetstat.IpExt.InNoECTPkts = &value case "InECT1Pkts": - procNetstat.IpExt.InECT1Pkts = value + procNetstat.IpExt.InECT1Pkts = &value case "InECT0Pkts": - procNetstat.IpExt.InECT0Pkts = value + procNetstat.IpExt.InECT0Pkts = &value case "InCEPkts": - procNetstat.IpExt.InCEPkts = value + procNetstat.IpExt.InCEPkts = &value case "ReasmOverlaps": - procNetstat.IpExt.ReasmOverlaps = value + procNetstat.IpExt.ReasmOverlaps = &value } } } diff --git a/vendor/github.com/prometheus/procfs/proc_snmp.go b/vendor/github.com/prometheus/procfs/proc_snmp.go index ae19189..6c46b71 100644 --- a/vendor/github.com/prometheus/procfs/proc_snmp.go +++ b/vendor/github.com/prometheus/procfs/proc_snmp.go @@ -37,100 +37,100 @@ type ProcSnmp struct { } type Ip struct { // nolint:revive - Forwarding float64 - DefaultTTL float64 - InReceives float64 - InHdrErrors float64 - InAddrErrors float64 - ForwDatagrams float64 - InUnknownProtos float64 - InDiscards float64 - InDelivers float64 - OutRequests float64 - OutDiscards float64 - OutNoRoutes float64 - ReasmTimeout float64 - ReasmReqds float64 - ReasmOKs float64 - ReasmFails float64 - FragOKs float64 - FragFails float64 - FragCreates float64 + Forwarding *float64 + DefaultTTL *float64 + InReceives *float64 + InHdrErrors *float64 + InAddrErrors *float64 + ForwDatagrams *float64 + InUnknownProtos *float64 + InDiscards *float64 + InDelivers *float64 + OutRequests *float64 + OutDiscards *float64 + OutNoRoutes *float64 + ReasmTimeout *float64 + ReasmReqds *float64 + ReasmOKs *float64 + ReasmFails *float64 + FragOKs *float64 + FragFails *float64 + FragCreates *float64 } -type Icmp struct { - InMsgs float64 - InErrors float64 - InCsumErrors float64 - InDestUnreachs float64 - InTimeExcds float64 - InParmProbs float64 - InSrcQuenchs float64 - InRedirects float64 - InEchos float64 - InEchoReps float64 - InTimestamps float64 - InTimestampReps float64 - InAddrMasks float64 - InAddrMaskReps float64 - OutMsgs float64 - OutErrors float64 - OutDestUnreachs float64 - OutTimeExcds float64 - OutParmProbs float64 - OutSrcQuenchs float64 - OutRedirects float64 - OutEchos float64 - OutEchoReps float64 - OutTimestamps float64 - OutTimestampReps float64 - OutAddrMasks float64 - OutAddrMaskReps float64 +type Icmp struct { // nolint:revive + InMsgs *float64 + InErrors *float64 + InCsumErrors *float64 + InDestUnreachs *float64 + InTimeExcds *float64 + InParmProbs *float64 + InSrcQuenchs *float64 + InRedirects *float64 + InEchos *float64 + InEchoReps *float64 + InTimestamps *float64 + InTimestampReps *float64 + InAddrMasks *float64 + InAddrMaskReps *float64 + OutMsgs *float64 + OutErrors *float64 + OutDestUnreachs *float64 + OutTimeExcds *float64 + OutParmProbs *float64 + OutSrcQuenchs *float64 + OutRedirects *float64 + OutEchos *float64 + OutEchoReps *float64 + OutTimestamps *float64 + OutTimestampReps *float64 + OutAddrMasks *float64 + OutAddrMaskReps *float64 } type IcmpMsg struct { - InType3 float64 - OutType3 float64 + InType3 *float64 + OutType3 *float64 } type Tcp struct { // nolint:revive - RtoAlgorithm float64 - RtoMin float64 - RtoMax float64 - MaxConn float64 - ActiveOpens float64 - PassiveOpens float64 - AttemptFails float64 - EstabResets float64 - CurrEstab float64 - InSegs float64 - OutSegs float64 - RetransSegs float64 - InErrs float64 - OutRsts float64 - InCsumErrors float64 + RtoAlgorithm *float64 + RtoMin *float64 + RtoMax *float64 + MaxConn *float64 + ActiveOpens *float64 + PassiveOpens *float64 + AttemptFails *float64 + EstabResets *float64 + CurrEstab *float64 + InSegs *float64 + OutSegs *float64 + RetransSegs *float64 + InErrs *float64 + OutRsts *float64 + InCsumErrors *float64 } type Udp struct { // nolint:revive - InDatagrams float64 - NoPorts float64 - InErrors float64 - OutDatagrams float64 - RcvbufErrors float64 - SndbufErrors float64 - InCsumErrors float64 - IgnoredMulti float64 + InDatagrams *float64 + NoPorts *float64 + InErrors *float64 + OutDatagrams *float64 + RcvbufErrors *float64 + SndbufErrors *float64 + InCsumErrors *float64 + IgnoredMulti *float64 } type UdpLite struct { // nolint:revive - InDatagrams float64 - NoPorts float64 - InErrors float64 - OutDatagrams float64 - RcvbufErrors float64 - SndbufErrors float64 - InCsumErrors float64 - IgnoredMulti float64 + InDatagrams *float64 + NoPorts *float64 + InErrors *float64 + OutDatagrams *float64 + RcvbufErrors *float64 + SndbufErrors *float64 + InCsumErrors *float64 + IgnoredMulti *float64 } func (p Proc) Snmp() (ProcSnmp, error) { @@ -173,178 +173,178 @@ func parseSnmp(r io.Reader, fileName string) (ProcSnmp, error) { case "Ip": switch key { case "Forwarding": - procSnmp.Ip.Forwarding = value + procSnmp.Ip.Forwarding = &value case "DefaultTTL": - procSnmp.Ip.DefaultTTL = value + procSnmp.Ip.DefaultTTL = &value case "InReceives": - procSnmp.Ip.InReceives = value + procSnmp.Ip.InReceives = &value case "InHdrErrors": - procSnmp.Ip.InHdrErrors = value + procSnmp.Ip.InHdrErrors = &value case "InAddrErrors": - procSnmp.Ip.InAddrErrors = value + procSnmp.Ip.InAddrErrors = &value case "ForwDatagrams": - procSnmp.Ip.ForwDatagrams = value + procSnmp.Ip.ForwDatagrams = &value case "InUnknownProtos": - procSnmp.Ip.InUnknownProtos = value + procSnmp.Ip.InUnknownProtos = &value case "InDiscards": - procSnmp.Ip.InDiscards = value + procSnmp.Ip.InDiscards = &value case "InDelivers": - procSnmp.Ip.InDelivers = value + procSnmp.Ip.InDelivers = &value case "OutRequests": - procSnmp.Ip.OutRequests = value + procSnmp.Ip.OutRequests = &value case "OutDiscards": - procSnmp.Ip.OutDiscards = value + procSnmp.Ip.OutDiscards = &value case "OutNoRoutes": - procSnmp.Ip.OutNoRoutes = value + procSnmp.Ip.OutNoRoutes = &value case "ReasmTimeout": - procSnmp.Ip.ReasmTimeout = value + procSnmp.Ip.ReasmTimeout = &value case "ReasmReqds": - procSnmp.Ip.ReasmReqds = value + procSnmp.Ip.ReasmReqds = &value case "ReasmOKs": - procSnmp.Ip.ReasmOKs = value + procSnmp.Ip.ReasmOKs = &value case "ReasmFails": - procSnmp.Ip.ReasmFails = value + procSnmp.Ip.ReasmFails = &value case "FragOKs": - procSnmp.Ip.FragOKs = value + procSnmp.Ip.FragOKs = &value case "FragFails": - procSnmp.Ip.FragFails = value + procSnmp.Ip.FragFails = &value case "FragCreates": - procSnmp.Ip.FragCreates = value + procSnmp.Ip.FragCreates = &value } case "Icmp": switch key { case "InMsgs": - procSnmp.Icmp.InMsgs = value + procSnmp.Icmp.InMsgs = &value case "InErrors": - procSnmp.Icmp.InErrors = value + procSnmp.Icmp.InErrors = &value case "InCsumErrors": - procSnmp.Icmp.InCsumErrors = value + procSnmp.Icmp.InCsumErrors = &value case "InDestUnreachs": - procSnmp.Icmp.InDestUnreachs = value + procSnmp.Icmp.InDestUnreachs = &value case "InTimeExcds": - procSnmp.Icmp.InTimeExcds = value + procSnmp.Icmp.InTimeExcds = &value case "InParmProbs": - procSnmp.Icmp.InParmProbs = value + procSnmp.Icmp.InParmProbs = &value case "InSrcQuenchs": - procSnmp.Icmp.InSrcQuenchs = value + procSnmp.Icmp.InSrcQuenchs = &value case "InRedirects": - procSnmp.Icmp.InRedirects = value + procSnmp.Icmp.InRedirects = &value case "InEchos": - procSnmp.Icmp.InEchos = value + procSnmp.Icmp.InEchos = &value case "InEchoReps": - procSnmp.Icmp.InEchoReps = value + procSnmp.Icmp.InEchoReps = &value case "InTimestamps": - procSnmp.Icmp.InTimestamps = value + procSnmp.Icmp.InTimestamps = &value case "InTimestampReps": - procSnmp.Icmp.InTimestampReps = value + procSnmp.Icmp.InTimestampReps = &value case "InAddrMasks": - procSnmp.Icmp.InAddrMasks = value + procSnmp.Icmp.InAddrMasks = &value case "InAddrMaskReps": - procSnmp.Icmp.InAddrMaskReps = value + procSnmp.Icmp.InAddrMaskReps = &value case "OutMsgs": - procSnmp.Icmp.OutMsgs = value + procSnmp.Icmp.OutMsgs = &value case "OutErrors": - procSnmp.Icmp.OutErrors = value + procSnmp.Icmp.OutErrors = &value case "OutDestUnreachs": - procSnmp.Icmp.OutDestUnreachs = value + procSnmp.Icmp.OutDestUnreachs = &value case "OutTimeExcds": - procSnmp.Icmp.OutTimeExcds = value + procSnmp.Icmp.OutTimeExcds = &value case "OutParmProbs": - procSnmp.Icmp.OutParmProbs = value + procSnmp.Icmp.OutParmProbs = &value case "OutSrcQuenchs": - procSnmp.Icmp.OutSrcQuenchs = value + procSnmp.Icmp.OutSrcQuenchs = &value case "OutRedirects": - procSnmp.Icmp.OutRedirects = value + procSnmp.Icmp.OutRedirects = &value case "OutEchos": - procSnmp.Icmp.OutEchos = value + procSnmp.Icmp.OutEchos = &value case "OutEchoReps": - procSnmp.Icmp.OutEchoReps = value + procSnmp.Icmp.OutEchoReps = &value case "OutTimestamps": - procSnmp.Icmp.OutTimestamps = value + procSnmp.Icmp.OutTimestamps = &value case "OutTimestampReps": - procSnmp.Icmp.OutTimestampReps = value + procSnmp.Icmp.OutTimestampReps = &value case "OutAddrMasks": - procSnmp.Icmp.OutAddrMasks = value + procSnmp.Icmp.OutAddrMasks = &value case "OutAddrMaskReps": - procSnmp.Icmp.OutAddrMaskReps = value + procSnmp.Icmp.OutAddrMaskReps = &value } case "IcmpMsg": switch key { case "InType3": - procSnmp.IcmpMsg.InType3 = value + procSnmp.IcmpMsg.InType3 = &value case "OutType3": - procSnmp.IcmpMsg.OutType3 = value + procSnmp.IcmpMsg.OutType3 = &value } case "Tcp": switch key { case "RtoAlgorithm": - procSnmp.Tcp.RtoAlgorithm = value + procSnmp.Tcp.RtoAlgorithm = &value case "RtoMin": - procSnmp.Tcp.RtoMin = value + procSnmp.Tcp.RtoMin = &value case "RtoMax": - procSnmp.Tcp.RtoMax = value + procSnmp.Tcp.RtoMax = &value case "MaxConn": - procSnmp.Tcp.MaxConn = value + procSnmp.Tcp.MaxConn = &value case "ActiveOpens": - procSnmp.Tcp.ActiveOpens = value + procSnmp.Tcp.ActiveOpens = &value case "PassiveOpens": - procSnmp.Tcp.PassiveOpens = value + procSnmp.Tcp.PassiveOpens = &value case "AttemptFails": - procSnmp.Tcp.AttemptFails = value + procSnmp.Tcp.AttemptFails = &value case "EstabResets": - procSnmp.Tcp.EstabResets = value + procSnmp.Tcp.EstabResets = &value case "CurrEstab": - procSnmp.Tcp.CurrEstab = value + procSnmp.Tcp.CurrEstab = &value case "InSegs": - procSnmp.Tcp.InSegs = value + procSnmp.Tcp.InSegs = &value case "OutSegs": - procSnmp.Tcp.OutSegs = value + procSnmp.Tcp.OutSegs = &value case "RetransSegs": - procSnmp.Tcp.RetransSegs = value + procSnmp.Tcp.RetransSegs = &value case "InErrs": - procSnmp.Tcp.InErrs = value + procSnmp.Tcp.InErrs = &value case "OutRsts": - procSnmp.Tcp.OutRsts = value + procSnmp.Tcp.OutRsts = &value case "InCsumErrors": - procSnmp.Tcp.InCsumErrors = value + procSnmp.Tcp.InCsumErrors = &value } case "Udp": switch key { case "InDatagrams": - procSnmp.Udp.InDatagrams = value + procSnmp.Udp.InDatagrams = &value case "NoPorts": - procSnmp.Udp.NoPorts = value + procSnmp.Udp.NoPorts = &value case "InErrors": - procSnmp.Udp.InErrors = value + procSnmp.Udp.InErrors = &value case "OutDatagrams": - procSnmp.Udp.OutDatagrams = value + procSnmp.Udp.OutDatagrams = &value case "RcvbufErrors": - procSnmp.Udp.RcvbufErrors = value + procSnmp.Udp.RcvbufErrors = &value case "SndbufErrors": - procSnmp.Udp.SndbufErrors = value + procSnmp.Udp.SndbufErrors = &value case "InCsumErrors": - procSnmp.Udp.InCsumErrors = value + procSnmp.Udp.InCsumErrors = &value case "IgnoredMulti": - procSnmp.Udp.IgnoredMulti = value + procSnmp.Udp.IgnoredMulti = &value } case "UdpLite": switch key { case "InDatagrams": - procSnmp.UdpLite.InDatagrams = value + procSnmp.UdpLite.InDatagrams = &value case "NoPorts": - procSnmp.UdpLite.NoPorts = value + procSnmp.UdpLite.NoPorts = &value case "InErrors": - procSnmp.UdpLite.InErrors = value + procSnmp.UdpLite.InErrors = &value case "OutDatagrams": - procSnmp.UdpLite.OutDatagrams = value + procSnmp.UdpLite.OutDatagrams = &value case "RcvbufErrors": - procSnmp.UdpLite.RcvbufErrors = value + procSnmp.UdpLite.RcvbufErrors = &value case "SndbufErrors": - procSnmp.UdpLite.SndbufErrors = value + procSnmp.UdpLite.SndbufErrors = &value case "InCsumErrors": - procSnmp.UdpLite.InCsumErrors = value + procSnmp.UdpLite.InCsumErrors = &value case "IgnoredMulti": - procSnmp.UdpLite.IgnoredMulti = value + procSnmp.UdpLite.IgnoredMulti = &value } } } diff --git a/vendor/github.com/prometheus/procfs/proc_snmp6.go b/vendor/github.com/prometheus/procfs/proc_snmp6.go index f611992..3059cc6 100644 --- a/vendor/github.com/prometheus/procfs/proc_snmp6.go +++ b/vendor/github.com/prometheus/procfs/proc_snmp6.go @@ -36,106 +36,106 @@ type ProcSnmp6 struct { } type Ip6 struct { // nolint:revive - InReceives float64 - InHdrErrors float64 - InTooBigErrors float64 - InNoRoutes float64 - InAddrErrors float64 - InUnknownProtos float64 - InTruncatedPkts float64 - InDiscards float64 - InDelivers float64 - OutForwDatagrams float64 - OutRequests float64 - OutDiscards float64 - OutNoRoutes float64 - ReasmTimeout float64 - ReasmReqds float64 - ReasmOKs float64 - ReasmFails float64 - FragOKs float64 - FragFails float64 - FragCreates float64 - InMcastPkts float64 - OutMcastPkts float64 - InOctets float64 - OutOctets float64 - InMcastOctets float64 - OutMcastOctets float64 - InBcastOctets float64 - OutBcastOctets float64 - InNoECTPkts float64 - InECT1Pkts float64 - InECT0Pkts float64 - InCEPkts float64 + InReceives *float64 + InHdrErrors *float64 + InTooBigErrors *float64 + InNoRoutes *float64 + InAddrErrors *float64 + InUnknownProtos *float64 + InTruncatedPkts *float64 + InDiscards *float64 + InDelivers *float64 + OutForwDatagrams *float64 + OutRequests *float64 + OutDiscards *float64 + OutNoRoutes *float64 + ReasmTimeout *float64 + ReasmReqds *float64 + ReasmOKs *float64 + ReasmFails *float64 + FragOKs *float64 + FragFails *float64 + FragCreates *float64 + InMcastPkts *float64 + OutMcastPkts *float64 + InOctets *float64 + OutOctets *float64 + InMcastOctets *float64 + OutMcastOctets *float64 + InBcastOctets *float64 + OutBcastOctets *float64 + InNoECTPkts *float64 + InECT1Pkts *float64 + InECT0Pkts *float64 + InCEPkts *float64 } type Icmp6 struct { - InMsgs float64 - InErrors float64 - OutMsgs float64 - OutErrors float64 - InCsumErrors float64 - InDestUnreachs float64 - InPktTooBigs float64 - InTimeExcds float64 - InParmProblems float64 - InEchos float64 - InEchoReplies float64 - InGroupMembQueries float64 - InGroupMembResponses float64 - InGroupMembReductions float64 - InRouterSolicits float64 - InRouterAdvertisements float64 - InNeighborSolicits float64 - InNeighborAdvertisements float64 - InRedirects float64 - InMLDv2Reports float64 - OutDestUnreachs float64 - OutPktTooBigs float64 - OutTimeExcds float64 - OutParmProblems float64 - OutEchos float64 - OutEchoReplies float64 - OutGroupMembQueries float64 - OutGroupMembResponses float64 - OutGroupMembReductions float64 - OutRouterSolicits float64 - OutRouterAdvertisements float64 - OutNeighborSolicits float64 - OutNeighborAdvertisements float64 - OutRedirects float64 - OutMLDv2Reports float64 - InType1 float64 - InType134 float64 - InType135 float64 - InType136 float64 - InType143 float64 - OutType133 float64 - OutType135 float64 - OutType136 float64 - OutType143 float64 + InMsgs *float64 + InErrors *float64 + OutMsgs *float64 + OutErrors *float64 + InCsumErrors *float64 + InDestUnreachs *float64 + InPktTooBigs *float64 + InTimeExcds *float64 + InParmProblems *float64 + InEchos *float64 + InEchoReplies *float64 + InGroupMembQueries *float64 + InGroupMembResponses *float64 + InGroupMembReductions *float64 + InRouterSolicits *float64 + InRouterAdvertisements *float64 + InNeighborSolicits *float64 + InNeighborAdvertisements *float64 + InRedirects *float64 + InMLDv2Reports *float64 + OutDestUnreachs *float64 + OutPktTooBigs *float64 + OutTimeExcds *float64 + OutParmProblems *float64 + OutEchos *float64 + OutEchoReplies *float64 + OutGroupMembQueries *float64 + OutGroupMembResponses *float64 + OutGroupMembReductions *float64 + OutRouterSolicits *float64 + OutRouterAdvertisements *float64 + OutNeighborSolicits *float64 + OutNeighborAdvertisements *float64 + OutRedirects *float64 + OutMLDv2Reports *float64 + InType1 *float64 + InType134 *float64 + InType135 *float64 + InType136 *float64 + InType143 *float64 + OutType133 *float64 + OutType135 *float64 + OutType136 *float64 + OutType143 *float64 } type Udp6 struct { // nolint:revive - InDatagrams float64 - NoPorts float64 - InErrors float64 - OutDatagrams float64 - RcvbufErrors float64 - SndbufErrors float64 - InCsumErrors float64 - IgnoredMulti float64 + InDatagrams *float64 + NoPorts *float64 + InErrors *float64 + OutDatagrams *float64 + RcvbufErrors *float64 + SndbufErrors *float64 + InCsumErrors *float64 + IgnoredMulti *float64 } type UdpLite6 struct { // nolint:revive - InDatagrams float64 - NoPorts float64 - InErrors float64 - OutDatagrams float64 - RcvbufErrors float64 - SndbufErrors float64 - InCsumErrors float64 + InDatagrams *float64 + NoPorts *float64 + InErrors *float64 + OutDatagrams *float64 + RcvbufErrors *float64 + SndbufErrors *float64 + InCsumErrors *float64 } func (p Proc) Snmp6() (ProcSnmp6, error) { @@ -182,197 +182,197 @@ func parseSNMP6Stats(r io.Reader) (ProcSnmp6, error) { case "Ip6": switch key { case "InReceives": - procSnmp6.Ip6.InReceives = value + procSnmp6.Ip6.InReceives = &value case "InHdrErrors": - procSnmp6.Ip6.InHdrErrors = value + procSnmp6.Ip6.InHdrErrors = &value case "InTooBigErrors": - procSnmp6.Ip6.InTooBigErrors = value + procSnmp6.Ip6.InTooBigErrors = &value case "InNoRoutes": - procSnmp6.Ip6.InNoRoutes = value + procSnmp6.Ip6.InNoRoutes = &value case "InAddrErrors": - procSnmp6.Ip6.InAddrErrors = value + procSnmp6.Ip6.InAddrErrors = &value case "InUnknownProtos": - procSnmp6.Ip6.InUnknownProtos = value + procSnmp6.Ip6.InUnknownProtos = &value case "InTruncatedPkts": - procSnmp6.Ip6.InTruncatedPkts = value + procSnmp6.Ip6.InTruncatedPkts = &value case "InDiscards": - procSnmp6.Ip6.InDiscards = value + procSnmp6.Ip6.InDiscards = &value case "InDelivers": - procSnmp6.Ip6.InDelivers = value + procSnmp6.Ip6.InDelivers = &value case "OutForwDatagrams": - procSnmp6.Ip6.OutForwDatagrams = value + procSnmp6.Ip6.OutForwDatagrams = &value case "OutRequests": - procSnmp6.Ip6.OutRequests = value + procSnmp6.Ip6.OutRequests = &value case "OutDiscards": - procSnmp6.Ip6.OutDiscards = value + procSnmp6.Ip6.OutDiscards = &value case "OutNoRoutes": - procSnmp6.Ip6.OutNoRoutes = value + procSnmp6.Ip6.OutNoRoutes = &value case "ReasmTimeout": - procSnmp6.Ip6.ReasmTimeout = value + procSnmp6.Ip6.ReasmTimeout = &value case "ReasmReqds": - procSnmp6.Ip6.ReasmReqds = value + procSnmp6.Ip6.ReasmReqds = &value case "ReasmOKs": - procSnmp6.Ip6.ReasmOKs = value + procSnmp6.Ip6.ReasmOKs = &value case "ReasmFails": - procSnmp6.Ip6.ReasmFails = value + procSnmp6.Ip6.ReasmFails = &value case "FragOKs": - procSnmp6.Ip6.FragOKs = value + procSnmp6.Ip6.FragOKs = &value case "FragFails": - procSnmp6.Ip6.FragFails = value + procSnmp6.Ip6.FragFails = &value case "FragCreates": - procSnmp6.Ip6.FragCreates = value + procSnmp6.Ip6.FragCreates = &value case "InMcastPkts": - procSnmp6.Ip6.InMcastPkts = value + procSnmp6.Ip6.InMcastPkts = &value case "OutMcastPkts": - procSnmp6.Ip6.OutMcastPkts = value + procSnmp6.Ip6.OutMcastPkts = &value case "InOctets": - procSnmp6.Ip6.InOctets = value + procSnmp6.Ip6.InOctets = &value case "OutOctets": - procSnmp6.Ip6.OutOctets = value + procSnmp6.Ip6.OutOctets = &value case "InMcastOctets": - procSnmp6.Ip6.InMcastOctets = value + procSnmp6.Ip6.InMcastOctets = &value case "OutMcastOctets": - procSnmp6.Ip6.OutMcastOctets = value + procSnmp6.Ip6.OutMcastOctets = &value case "InBcastOctets": - procSnmp6.Ip6.InBcastOctets = value + procSnmp6.Ip6.InBcastOctets = &value case "OutBcastOctets": - procSnmp6.Ip6.OutBcastOctets = value + procSnmp6.Ip6.OutBcastOctets = &value case "InNoECTPkts": - procSnmp6.Ip6.InNoECTPkts = value + procSnmp6.Ip6.InNoECTPkts = &value case "InECT1Pkts": - procSnmp6.Ip6.InECT1Pkts = value + procSnmp6.Ip6.InECT1Pkts = &value case "InECT0Pkts": - procSnmp6.Ip6.InECT0Pkts = value + procSnmp6.Ip6.InECT0Pkts = &value case "InCEPkts": - procSnmp6.Ip6.InCEPkts = value + procSnmp6.Ip6.InCEPkts = &value } case "Icmp6": switch key { case "InMsgs": - procSnmp6.Icmp6.InMsgs = value + procSnmp6.Icmp6.InMsgs = &value case "InErrors": - procSnmp6.Icmp6.InErrors = value + procSnmp6.Icmp6.InErrors = &value case "OutMsgs": - procSnmp6.Icmp6.OutMsgs = value + procSnmp6.Icmp6.OutMsgs = &value case "OutErrors": - procSnmp6.Icmp6.OutErrors = value + procSnmp6.Icmp6.OutErrors = &value case "InCsumErrors": - procSnmp6.Icmp6.InCsumErrors = value + procSnmp6.Icmp6.InCsumErrors = &value case "InDestUnreachs": - procSnmp6.Icmp6.InDestUnreachs = value + procSnmp6.Icmp6.InDestUnreachs = &value case "InPktTooBigs": - procSnmp6.Icmp6.InPktTooBigs = value + procSnmp6.Icmp6.InPktTooBigs = &value case "InTimeExcds": - procSnmp6.Icmp6.InTimeExcds = value + procSnmp6.Icmp6.InTimeExcds = &value case "InParmProblems": - procSnmp6.Icmp6.InParmProblems = value + procSnmp6.Icmp6.InParmProblems = &value case "InEchos": - procSnmp6.Icmp6.InEchos = value + procSnmp6.Icmp6.InEchos = &value case "InEchoReplies": - procSnmp6.Icmp6.InEchoReplies = value + procSnmp6.Icmp6.InEchoReplies = &value case "InGroupMembQueries": - procSnmp6.Icmp6.InGroupMembQueries = value + procSnmp6.Icmp6.InGroupMembQueries = &value case "InGroupMembResponses": - procSnmp6.Icmp6.InGroupMembResponses = value + procSnmp6.Icmp6.InGroupMembResponses = &value case "InGroupMembReductions": - procSnmp6.Icmp6.InGroupMembReductions = value + procSnmp6.Icmp6.InGroupMembReductions = &value case "InRouterSolicits": - procSnmp6.Icmp6.InRouterSolicits = value + procSnmp6.Icmp6.InRouterSolicits = &value case "InRouterAdvertisements": - procSnmp6.Icmp6.InRouterAdvertisements = value + procSnmp6.Icmp6.InRouterAdvertisements = &value case "InNeighborSolicits": - procSnmp6.Icmp6.InNeighborSolicits = value + procSnmp6.Icmp6.InNeighborSolicits = &value case "InNeighborAdvertisements": - procSnmp6.Icmp6.InNeighborAdvertisements = value + procSnmp6.Icmp6.InNeighborAdvertisements = &value case "InRedirects": - procSnmp6.Icmp6.InRedirects = value + procSnmp6.Icmp6.InRedirects = &value case "InMLDv2Reports": - procSnmp6.Icmp6.InMLDv2Reports = value + procSnmp6.Icmp6.InMLDv2Reports = &value case "OutDestUnreachs": - procSnmp6.Icmp6.OutDestUnreachs = value + procSnmp6.Icmp6.OutDestUnreachs = &value case "OutPktTooBigs": - procSnmp6.Icmp6.OutPktTooBigs = value + procSnmp6.Icmp6.OutPktTooBigs = &value case "OutTimeExcds": - procSnmp6.Icmp6.OutTimeExcds = value + procSnmp6.Icmp6.OutTimeExcds = &value case "OutParmProblems": - procSnmp6.Icmp6.OutParmProblems = value + procSnmp6.Icmp6.OutParmProblems = &value case "OutEchos": - procSnmp6.Icmp6.OutEchos = value + procSnmp6.Icmp6.OutEchos = &value case "OutEchoReplies": - procSnmp6.Icmp6.OutEchoReplies = value + procSnmp6.Icmp6.OutEchoReplies = &value case "OutGroupMembQueries": - procSnmp6.Icmp6.OutGroupMembQueries = value + procSnmp6.Icmp6.OutGroupMembQueries = &value case "OutGroupMembResponses": - procSnmp6.Icmp6.OutGroupMembResponses = value + procSnmp6.Icmp6.OutGroupMembResponses = &value case "OutGroupMembReductions": - procSnmp6.Icmp6.OutGroupMembReductions = value + procSnmp6.Icmp6.OutGroupMembReductions = &value case "OutRouterSolicits": - procSnmp6.Icmp6.OutRouterSolicits = value + procSnmp6.Icmp6.OutRouterSolicits = &value case "OutRouterAdvertisements": - procSnmp6.Icmp6.OutRouterAdvertisements = value + procSnmp6.Icmp6.OutRouterAdvertisements = &value case "OutNeighborSolicits": - procSnmp6.Icmp6.OutNeighborSolicits = value + procSnmp6.Icmp6.OutNeighborSolicits = &value case "OutNeighborAdvertisements": - procSnmp6.Icmp6.OutNeighborAdvertisements = value + procSnmp6.Icmp6.OutNeighborAdvertisements = &value case "OutRedirects": - procSnmp6.Icmp6.OutRedirects = value + procSnmp6.Icmp6.OutRedirects = &value case "OutMLDv2Reports": - procSnmp6.Icmp6.OutMLDv2Reports = value + procSnmp6.Icmp6.OutMLDv2Reports = &value case "InType1": - procSnmp6.Icmp6.InType1 = value + procSnmp6.Icmp6.InType1 = &value case "InType134": - procSnmp6.Icmp6.InType134 = value + procSnmp6.Icmp6.InType134 = &value case "InType135": - procSnmp6.Icmp6.InType135 = value + procSnmp6.Icmp6.InType135 = &value case "InType136": - procSnmp6.Icmp6.InType136 = value + procSnmp6.Icmp6.InType136 = &value case "InType143": - procSnmp6.Icmp6.InType143 = value + procSnmp6.Icmp6.InType143 = &value case "OutType133": - procSnmp6.Icmp6.OutType133 = value + procSnmp6.Icmp6.OutType133 = &value case "OutType135": - procSnmp6.Icmp6.OutType135 = value + procSnmp6.Icmp6.OutType135 = &value case "OutType136": - procSnmp6.Icmp6.OutType136 = value + procSnmp6.Icmp6.OutType136 = &value case "OutType143": - procSnmp6.Icmp6.OutType143 = value + procSnmp6.Icmp6.OutType143 = &value } case "Udp6": switch key { case "InDatagrams": - procSnmp6.Udp6.InDatagrams = value + procSnmp6.Udp6.InDatagrams = &value case "NoPorts": - procSnmp6.Udp6.NoPorts = value + procSnmp6.Udp6.NoPorts = &value case "InErrors": - procSnmp6.Udp6.InErrors = value + procSnmp6.Udp6.InErrors = &value case "OutDatagrams": - procSnmp6.Udp6.OutDatagrams = value + procSnmp6.Udp6.OutDatagrams = &value case "RcvbufErrors": - procSnmp6.Udp6.RcvbufErrors = value + procSnmp6.Udp6.RcvbufErrors = &value case "SndbufErrors": - procSnmp6.Udp6.SndbufErrors = value + procSnmp6.Udp6.SndbufErrors = &value case "InCsumErrors": - procSnmp6.Udp6.InCsumErrors = value + procSnmp6.Udp6.InCsumErrors = &value case "IgnoredMulti": - procSnmp6.Udp6.IgnoredMulti = value + procSnmp6.Udp6.IgnoredMulti = &value } case "UdpLite6": switch key { case "InDatagrams": - procSnmp6.UdpLite6.InDatagrams = value + procSnmp6.UdpLite6.InDatagrams = &value case "NoPorts": - procSnmp6.UdpLite6.NoPorts = value + procSnmp6.UdpLite6.NoPorts = &value case "InErrors": - procSnmp6.UdpLite6.InErrors = value + procSnmp6.UdpLite6.InErrors = &value case "OutDatagrams": - procSnmp6.UdpLite6.OutDatagrams = value + procSnmp6.UdpLite6.OutDatagrams = &value case "RcvbufErrors": - procSnmp6.UdpLite6.RcvbufErrors = value + procSnmp6.UdpLite6.RcvbufErrors = &value case "SndbufErrors": - procSnmp6.UdpLite6.SndbufErrors = value + procSnmp6.UdpLite6.SndbufErrors = &value case "InCsumErrors": - procSnmp6.UdpLite6.InCsumErrors = value + procSnmp6.UdpLite6.InCsumErrors = &value } } } diff --git a/vendor/github.com/prometheus/procfs/proc_stat.go b/vendor/github.com/prometheus/procfs/proc_stat.go index 06c556e..b278eb2 100644 --- a/vendor/github.com/prometheus/procfs/proc_stat.go +++ b/vendor/github.com/prometheus/procfs/proc_stat.go @@ -102,6 +102,8 @@ type ProcStat struct { RSS int // Soft limit in bytes on the rss of the process. RSSLimit uint64 + // CPU number last executed on. + Processor uint // Real-time scheduling priority, a number in the range 1 to 99 for processes // scheduled under a real-time policy, or 0, for non-real-time processes. RTPriority uint @@ -184,7 +186,7 @@ func (p Proc) Stat() (ProcStat, error) { &ignoreUint64, &ignoreUint64, &ignoreInt64, - &ignoreInt64, + &s.Processor, &s.RTPriority, &s.Policy, &s.DelayAcctBlkIOTicks, diff --git a/vendor/github.com/prometheus/procfs/proc_status.go b/vendor/github.com/prometheus/procfs/proc_status.go index 594022d..3d8c064 100644 --- a/vendor/github.com/prometheus/procfs/proc_status.go +++ b/vendor/github.com/prometheus/procfs/proc_status.go @@ -96,10 +96,10 @@ func (p Proc) NewStatus() (ProcStatus, error) { kv := strings.SplitN(line, ":", 2) // removes spaces - k := string(strings.TrimSpace(kv[0])) - v := string(strings.TrimSpace(kv[1])) + k := strings.TrimSpace(kv[0]) + v := strings.TrimSpace(kv[1]) // removes "kB" - v = string(bytes.Trim([]byte(v), " kB")) + v = strings.TrimSuffix(v, " kB") // value to int when possible // we can skip error check here, 'cause vKBytes is not used when value is a string diff --git a/vendor/github.com/prometheus/procfs/stat.go b/vendor/github.com/prometheus/procfs/stat.go index 33f97ca..586af48 100644 --- a/vendor/github.com/prometheus/procfs/stat.go +++ b/vendor/github.com/prometheus/procfs/stat.go @@ -62,7 +62,7 @@ type Stat struct { // Summed up cpu statistics. CPUTotal CPUStat // Per-CPU statistics. - CPU []CPUStat + CPU map[int64]CPUStat // Number of times interrupts were handled, which contains numbered and unnumbered IRQs. IRQTotal uint64 // Number of times a numbered IRQ was triggered. @@ -170,10 +170,23 @@ func (fs FS) Stat() (Stat, error) { if err != nil { return Stat{}, err } + procStat, err := parseStat(bytes.NewReader(data), fileName) + if err != nil { + return Stat{}, err + } + return procStat, nil +} - stat := Stat{} +// parseStat parses the metrics from /proc/[pid]/stat. +func parseStat(r io.Reader, fileName string) (Stat, error) { + var ( + scanner = bufio.NewScanner(r) + stat = Stat{ + CPU: make(map[int64]CPUStat), + } + err error + ) - scanner := bufio.NewScanner(bytes.NewReader(data)) for scanner.Scan() { line := scanner.Text() parts := strings.Fields(scanner.Text()) @@ -228,9 +241,6 @@ func (fs FS) Stat() (Stat, error) { if cpuID == -1 { stat.CPUTotal = cpuStat } else { - for int64(len(stat.CPU)) <= cpuID { - stat.CPU = append(stat.CPU, CPUStat{}) - } stat.CPU[cpuID] = cpuStat } } diff --git a/vendor/github.com/prometheus/procfs/thread.go b/vendor/github.com/prometheus/procfs/thread.go new file mode 100644 index 0000000..f08bfc7 --- /dev/null +++ b/vendor/github.com/prometheus/procfs/thread.go @@ -0,0 +1,79 @@ +// Copyright 2022 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package procfs + +import ( + "fmt" + "os" + "strconv" + + fsi "github.com/prometheus/procfs/internal/fs" +) + +// Provide access to /proc/PID/task/TID files, for thread specific values. Since +// such files have the same structure as /proc/PID/ ones, the data structures +// and the parsers for the latter may be reused. + +// AllThreads returns a list of all currently available threads under /proc/PID. +func AllThreads(pid int) (Procs, error) { + fs, err := NewFS(DefaultMountPoint) + if err != nil { + return Procs{}, err + } + return fs.AllThreads(pid) +} + +// AllThreads returns a list of all currently available threads for PID. +func (fs FS) AllThreads(pid int) (Procs, error) { + taskPath := fs.proc.Path(strconv.Itoa(pid), "task") + d, err := os.Open(taskPath) + if err != nil { + return Procs{}, err + } + defer d.Close() + + names, err := d.Readdirnames(-1) + if err != nil { + return Procs{}, fmt.Errorf("could not read %q: %w", d.Name(), err) + } + + t := Procs{} + for _, n := range names { + tid, err := strconv.ParseInt(n, 10, 64) + if err != nil { + continue + } + t = append(t, Proc{PID: int(tid), fs: fsi.FS(taskPath)}) + } + + return t, nil +} + +// Thread returns a process for a given PID, TID. +func (fs FS) Thread(pid, tid int) (Proc, error) { + taskPath := fs.proc.Path(strconv.Itoa(pid), "task") + if _, err := os.Stat(taskPath); err != nil { + return Proc{}, err + } + return Proc{PID: tid, fs: fsi.FS(taskPath)}, nil +} + +// Thread returns a process for a given TID of Proc. +func (proc Proc) Thread(tid int) (Proc, error) { + tfs := fsi.FS(proc.path("task")) + if _, err := os.Stat(tfs.Path(strconv.Itoa(tid))); err != nil { + return Proc{}, err + } + return Proc{PID: tid, fs: tfs}, nil +} diff --git a/vendor/github.com/prometheus/procfs/vm.go b/vendor/github.com/prometheus/procfs/vm.go index 20ceb77..cdedcae 100644 --- a/vendor/github.com/prometheus/procfs/vm.go +++ b/vendor/github.com/prometheus/procfs/vm.go @@ -26,7 +26,9 @@ import ( ) // The VM interface is described at -// https://www.kernel.org/doc/Documentation/sysctl/vm.txt +// +// https://www.kernel.org/doc/Documentation/sysctl/vm.txt +// // Each setting is exposed as a single file. // Each file contains one line with a single numerical value, except lowmem_reserve_ratio which holds an array // and numa_zonelist_order (deprecated) which is a string. diff --git a/vendor/github.com/rogpeppe/go-internal/fmtsort/mapelem.go b/vendor/github.com/rogpeppe/go-internal/fmtsort/mapelem.go index 4e4c294..98e4e38 100644 --- a/vendor/github.com/rogpeppe/go-internal/fmtsort/mapelem.go +++ b/vendor/github.com/rogpeppe/go-internal/fmtsort/mapelem.go @@ -1,6 +1,3 @@ -//go:build go1.12 -// +build go1.12 - package fmtsort import "reflect" diff --git a/vendor/github.com/rogpeppe/go-internal/fmtsort/mapelem_1.11.go b/vendor/github.com/rogpeppe/go-internal/fmtsort/mapelem_1.11.go deleted file mode 100644 index 873bf7f..0000000 --- a/vendor/github.com/rogpeppe/go-internal/fmtsort/mapelem_1.11.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build !go1.12 -// +build !go1.12 - -package fmtsort - -import "reflect" - -const brokenNaNs = true - -func mapElems(mapValue reflect.Value) ([]reflect.Value, []reflect.Value) { - key := mapValue.MapKeys() - value := make([]reflect.Value, 0, len(key)) - for _, k := range key { - v := mapValue.MapIndex(k) - if !v.IsValid() { - // Note: we can't retrieve the value, probably because - // the key is NaN, so just do the best we can and - // add a zero value of the correct type in that case. - v = reflect.Zero(mapValue.Type().Elem()) - } - value = append(value, v) - } - return key, value -} diff --git a/vendor/github.com/rogpeppe/go-internal/fmtsort/sort.go b/vendor/github.com/rogpeppe/go-internal/fmtsort/sort.go index 0fb5187..7f51854 100644 --- a/vendor/github.com/rogpeppe/go-internal/fmtsort/sort.go +++ b/vendor/github.com/rogpeppe/go-internal/fmtsort/sort.go @@ -36,19 +36,18 @@ func (o *SortedMap) Swap(i, j int) { // // The ordering rules are more general than with Go's < operator: // -// - when applicable, nil compares low -// - ints, floats, and strings order by < -// - NaN compares less than non-NaN floats -// - bool compares false before true -// - complex compares real, then imag -// - pointers compare by machine address -// - channel values compare by machine address -// - structs compare each field in turn -// - arrays compare each element in turn. -// Otherwise identical arrays compare by length. -// - interface values compare first by reflect.Type describing the concrete type -// and then by concrete value as described in the previous rules. -// +// - when applicable, nil compares low +// - ints, floats, and strings order by < +// - NaN compares less than non-NaN floats +// - bool compares false before true +// - complex compares real, then imag +// - pointers compare by machine address +// - channel values compare by machine address +// - structs compare each field in turn +// - arrays compare each element in turn. +// Otherwise identical arrays compare by length. +// - interface values compare first by reflect.Type describing the concrete type +// and then by concrete value as described in the previous rules. func Sort(mapValue reflect.Value) *SortedMap { if mapValue.Type().Kind() != reflect.Map { return nil diff --git a/vendor/github.com/sirupsen/logrus/README.md b/vendor/github.com/sirupsen/logrus/README.md index b042c89..d1d4a85 100644 --- a/vendor/github.com/sirupsen/logrus/README.md +++ b/vendor/github.com/sirupsen/logrus/README.md @@ -9,7 +9,7 @@ the last thing you want from your Logging library (again...). This does not mean Logrus is dead. Logrus will continue to be maintained for security, (backwards compatible) bug fixes, and performance (where we are -limited by the interface). +limited by the interface). I believe Logrus' biggest contribution is to have played a part in today's widespread use of structured logging in Golang. There doesn't seem to be a @@ -43,7 +43,7 @@ plain text): With `log.SetFormatter(&log.JSONFormatter{})`, for easy parsing by logstash or Splunk: -```json +```text {"animal":"walrus","level":"info","msg":"A group of walrus emerges from the ocean","size":10,"time":"2014-03-10 19:57:38.562264131 -0400 EDT"} @@ -99,7 +99,7 @@ time="2015-03-26T01:27:38-04:00" level=fatal method=github.com/sirupsen/arcticcr ``` Note that this does add measurable overhead - the cost will depend on the version of Go, but is between 20 and 40% in recent tests with 1.6 and 1.7. You can validate this in your -environment via benchmarks: +environment via benchmarks: ``` go test -bench=.*CallerTracing ``` @@ -317,6 +317,8 @@ log.SetLevel(log.InfoLevel) It may be useful to set `log.Level = logrus.DebugLevel` in a debug or verbose environment if your application has that. +Note: If you want different log levels for global (`log.SetLevel(...)`) and syslog logging, please check the [syslog hook README](hooks/syslog/README.md#different-log-levels-for-local-and-remote-logging). + #### Entries Besides the fields added with `WithField` or `WithFields` some fields are diff --git a/vendor/golang.org/x/sys/execabs/execabs.go b/vendor/golang.org/x/sys/execabs/execabs.go index b981cfb..3bf40fd 100644 --- a/vendor/golang.org/x/sys/execabs/execabs.go +++ b/vendor/golang.org/x/sys/execabs/execabs.go @@ -63,7 +63,7 @@ func LookPath(file string) (string, error) { } func fixCmd(name string, cmd *exec.Cmd) { - if filepath.Base(name) == name && !filepath.IsAbs(cmd.Path) { + if filepath.Base(name) == name && !filepath.IsAbs(cmd.Path) && !isGo119ErrFieldSet(cmd) { // exec.Command was called with a bare binary name and // exec.LookPath returned a path which is not absolute. // Set cmd.lookPathErr and clear cmd.Path so that it diff --git a/vendor/golang.org/x/sys/execabs/execabs_go118.go b/vendor/golang.org/x/sys/execabs/execabs_go118.go index 6ab5f50..2000064 100644 --- a/vendor/golang.org/x/sys/execabs/execabs_go118.go +++ b/vendor/golang.org/x/sys/execabs/execabs_go118.go @@ -7,6 +7,12 @@ package execabs +import "os/exec" + func isGo119ErrDot(err error) bool { return false } + +func isGo119ErrFieldSet(cmd *exec.Cmd) bool { + return false +} diff --git a/vendor/golang.org/x/sys/execabs/execabs_go119.go b/vendor/golang.org/x/sys/execabs/execabs_go119.go index 1e7a9ad..f364b34 100644 --- a/vendor/golang.org/x/sys/execabs/execabs_go119.go +++ b/vendor/golang.org/x/sys/execabs/execabs_go119.go @@ -7,9 +7,15 @@ package execabs -import "strings" +import ( + "errors" + "os/exec" +) func isGo119ErrDot(err error) bool { - // TODO: return errors.Is(err, exec.ErrDot) - return strings.Contains(err.Error(), "current directory") + return errors.Is(err, exec.ErrDot) +} + +func isGo119ErrFieldSet(cmd *exec.Cmd) bool { + return cmd.Err != nil } diff --git a/vendor/golang.org/x/sys/unix/gccgo.go b/vendor/golang.org/x/sys/unix/gccgo.go index 0dee232..b06f52d 100644 --- a/vendor/golang.org/x/sys/unix/gccgo.go +++ b/vendor/golang.org/x/sys/unix/gccgo.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build gccgo && !aix -// +build gccgo,!aix +//go:build gccgo && !aix && !hurd +// +build gccgo,!aix,!hurd package unix diff --git a/vendor/golang.org/x/sys/unix/gccgo_c.c b/vendor/golang.org/x/sys/unix/gccgo_c.c index 2cb1fef..f98a1c5 100644 --- a/vendor/golang.org/x/sys/unix/gccgo_c.c +++ b/vendor/golang.org/x/sys/unix/gccgo_c.c @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build gccgo -// +build !aix +//go:build gccgo && !aix && !hurd +// +build gccgo,!aix,!hurd #include #include diff --git a/vendor/golang.org/x/sys/unix/ioctl_signed.go b/vendor/golang.org/x/sys/unix/ioctl_signed.go new file mode 100644 index 0000000..7def958 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/ioctl_signed.go @@ -0,0 +1,70 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build aix || solaris +// +build aix solaris + +package unix + +import ( + "unsafe" +) + +// ioctl itself should not be exposed directly, but additional get/set +// functions for specific types are permissible. + +// IoctlSetInt performs an ioctl operation which sets an integer value +// on fd, using the specified request number. +func IoctlSetInt(fd int, req int, value int) error { + return ioctl(fd, req, uintptr(value)) +} + +// IoctlSetPointerInt performs an ioctl operation which sets an +// integer value on fd, using the specified request number. The ioctl +// argument is called with a pointer to the integer value, rather than +// passing the integer value directly. +func IoctlSetPointerInt(fd int, req int, value int) error { + v := int32(value) + return ioctlPtr(fd, req, unsafe.Pointer(&v)) +} + +// IoctlSetWinsize performs an ioctl on fd with a *Winsize argument. +// +// To change fd's window size, the req argument should be TIOCSWINSZ. +func IoctlSetWinsize(fd int, req int, value *Winsize) error { + // TODO: if we get the chance, remove the req parameter and + // hardcode TIOCSWINSZ. + return ioctlPtr(fd, req, unsafe.Pointer(value)) +} + +// IoctlSetTermios performs an ioctl on fd with a *Termios. +// +// The req value will usually be TCSETA or TIOCSETA. +func IoctlSetTermios(fd int, req int, value *Termios) error { + // TODO: if we get the chance, remove the req parameter. + return ioctlPtr(fd, req, unsafe.Pointer(value)) +} + +// IoctlGetInt performs an ioctl operation which gets an integer value +// from fd, using the specified request number. +// +// A few ioctl requests use the return value as an output parameter; +// for those, IoctlRetInt should be used instead of this function. +func IoctlGetInt(fd int, req int) (int, error) { + var value int + err := ioctlPtr(fd, req, unsafe.Pointer(&value)) + return value, err +} + +func IoctlGetWinsize(fd int, req int) (*Winsize, error) { + var value Winsize + err := ioctlPtr(fd, req, unsafe.Pointer(&value)) + return &value, err +} + +func IoctlGetTermios(fd int, req int) (*Termios, error) { + var value Termios + err := ioctlPtr(fd, req, unsafe.Pointer(&value)) + return &value, err +} diff --git a/vendor/golang.org/x/sys/unix/ioctl.go b/vendor/golang.org/x/sys/unix/ioctl_unsigned.go similarity index 77% rename from vendor/golang.org/x/sys/unix/ioctl.go rename to vendor/golang.org/x/sys/unix/ioctl_unsigned.go index 6c7ad05..649913d 100644 --- a/vendor/golang.org/x/sys/unix/ioctl.go +++ b/vendor/golang.org/x/sys/unix/ioctl_unsigned.go @@ -2,13 +2,12 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//go:build aix || darwin || dragonfly || freebsd || linux || netbsd || openbsd || solaris -// +build aix darwin dragonfly freebsd linux netbsd openbsd solaris +//go:build darwin || dragonfly || freebsd || hurd || linux || netbsd || openbsd +// +build darwin dragonfly freebsd hurd linux netbsd openbsd package unix import ( - "runtime" "unsafe" ) @@ -27,7 +26,7 @@ func IoctlSetInt(fd int, req uint, value int) error { // passing the integer value directly. func IoctlSetPointerInt(fd int, req uint, value int) error { v := int32(value) - return ioctl(fd, req, uintptr(unsafe.Pointer(&v))) + return ioctlPtr(fd, req, unsafe.Pointer(&v)) } // IoctlSetWinsize performs an ioctl on fd with a *Winsize argument. @@ -36,9 +35,7 @@ func IoctlSetPointerInt(fd int, req uint, value int) error { func IoctlSetWinsize(fd int, req uint, value *Winsize) error { // TODO: if we get the chance, remove the req parameter and // hardcode TIOCSWINSZ. - err := ioctl(fd, req, uintptr(unsafe.Pointer(value))) - runtime.KeepAlive(value) - return err + return ioctlPtr(fd, req, unsafe.Pointer(value)) } // IoctlSetTermios performs an ioctl on fd with a *Termios. @@ -46,9 +43,7 @@ func IoctlSetWinsize(fd int, req uint, value *Winsize) error { // The req value will usually be TCSETA or TIOCSETA. func IoctlSetTermios(fd int, req uint, value *Termios) error { // TODO: if we get the chance, remove the req parameter. - err := ioctl(fd, req, uintptr(unsafe.Pointer(value))) - runtime.KeepAlive(value) - return err + return ioctlPtr(fd, req, unsafe.Pointer(value)) } // IoctlGetInt performs an ioctl operation which gets an integer value @@ -58,18 +53,18 @@ func IoctlSetTermios(fd int, req uint, value *Termios) error { // for those, IoctlRetInt should be used instead of this function. func IoctlGetInt(fd int, req uint) (int, error) { var value int - err := ioctl(fd, req, uintptr(unsafe.Pointer(&value))) + err := ioctlPtr(fd, req, unsafe.Pointer(&value)) return value, err } func IoctlGetWinsize(fd int, req uint) (*Winsize, error) { var value Winsize - err := ioctl(fd, req, uintptr(unsafe.Pointer(&value))) + err := ioctlPtr(fd, req, unsafe.Pointer(&value)) return &value, err } func IoctlGetTermios(fd int, req uint) (*Termios, error) { var value Termios - err := ioctl(fd, req, uintptr(unsafe.Pointer(&value))) + err := ioctlPtr(fd, req, unsafe.Pointer(&value)) return &value, err } diff --git a/vendor/golang.org/x/sys/unix/ioctl_zos.go b/vendor/golang.org/x/sys/unix/ioctl_zos.go index 5384e7d..cdc21bf 100644 --- a/vendor/golang.org/x/sys/unix/ioctl_zos.go +++ b/vendor/golang.org/x/sys/unix/ioctl_zos.go @@ -17,25 +17,23 @@ import ( // IoctlSetInt performs an ioctl operation which sets an integer value // on fd, using the specified request number. -func IoctlSetInt(fd int, req uint, value int) error { +func IoctlSetInt(fd int, req int, value int) error { return ioctl(fd, req, uintptr(value)) } // IoctlSetWinsize performs an ioctl on fd with a *Winsize argument. // // To change fd's window size, the req argument should be TIOCSWINSZ. -func IoctlSetWinsize(fd int, req uint, value *Winsize) error { +func IoctlSetWinsize(fd int, req int, value *Winsize) error { // TODO: if we get the chance, remove the req parameter and // hardcode TIOCSWINSZ. - err := ioctl(fd, req, uintptr(unsafe.Pointer(value))) - runtime.KeepAlive(value) - return err + return ioctlPtr(fd, req, unsafe.Pointer(value)) } // IoctlSetTermios performs an ioctl on fd with a *Termios. // // The req value is expected to be TCSETS, TCSETSW, or TCSETSF -func IoctlSetTermios(fd int, req uint, value *Termios) error { +func IoctlSetTermios(fd int, req int, value *Termios) error { if (req != TCSETS) && (req != TCSETSW) && (req != TCSETSF) { return ENOSYS } @@ -49,22 +47,22 @@ func IoctlSetTermios(fd int, req uint, value *Termios) error { // // A few ioctl requests use the return value as an output parameter; // for those, IoctlRetInt should be used instead of this function. -func IoctlGetInt(fd int, req uint) (int, error) { +func IoctlGetInt(fd int, req int) (int, error) { var value int - err := ioctl(fd, req, uintptr(unsafe.Pointer(&value))) + err := ioctlPtr(fd, req, unsafe.Pointer(&value)) return value, err } -func IoctlGetWinsize(fd int, req uint) (*Winsize, error) { +func IoctlGetWinsize(fd int, req int) (*Winsize, error) { var value Winsize - err := ioctl(fd, req, uintptr(unsafe.Pointer(&value))) + err := ioctlPtr(fd, req, unsafe.Pointer(&value)) return &value, err } // IoctlGetTermios performs an ioctl on fd with a *Termios. // // The req value is expected to be TCGETS -func IoctlGetTermios(fd int, req uint) (*Termios, error) { +func IoctlGetTermios(fd int, req int) (*Termios, error) { var value Termios if req != TCGETS { return &value, ENOSYS diff --git a/vendor/golang.org/x/sys/unix/mkall.sh b/vendor/golang.org/x/sys/unix/mkall.sh index 727cba2..8e3947c 100644 --- a/vendor/golang.org/x/sys/unix/mkall.sh +++ b/vendor/golang.org/x/sys/unix/mkall.sh @@ -174,10 +174,10 @@ openbsd_arm64) mktypes="GOARCH=$GOARCH go tool cgo -godefs -- -fsigned-char" ;; openbsd_mips64) + mkasm="go run mkasm.go" mkerrors="$mkerrors -m64" - mksyscall="go run mksyscall.go -openbsd" + mksyscall="go run mksyscall.go -openbsd -libc" mksysctl="go run mksysctl_openbsd.go" - mksysnum="go run mksysnum.go 'https://cvsweb.openbsd.org/cgi-bin/cvsweb/~checkout~/src/sys/kern/syscalls.master'" # Let the type of C char be signed for making the bare syscall # API consistent across platforms. mktypes="GOARCH=$GOARCH go tool cgo -godefs -- -fsigned-char" diff --git a/vendor/golang.org/x/sys/unix/mkerrors.sh b/vendor/golang.org/x/sys/unix/mkerrors.sh index 7456d9d..be0423e 100644 --- a/vendor/golang.org/x/sys/unix/mkerrors.sh +++ b/vendor/golang.org/x/sys/unix/mkerrors.sh @@ -66,6 +66,7 @@ includes_Darwin=' #include #include #include +#include #include #include #include @@ -203,6 +204,7 @@ struct ltchars { #include #include #include +#include #include #include #include @@ -517,10 +519,11 @@ ccflags="$@" $2 ~ /^LOCK_(SH|EX|NB|UN)$/ || $2 ~ /^LO_(KEY|NAME)_SIZE$/ || $2 ~ /^LOOP_(CLR|CTL|GET|SET)_/ || - $2 ~ /^(AF|SOCK|SO|SOL|IPPROTO|IP|IPV6|TCP|MCAST|EVFILT|NOTE|SHUT|PROT|MAP|MFD|T?PACKET|MSG|SCM|MCL|DT|MADV|PR|LOCAL|TCPOPT)_/ || + $2 ~ /^(AF|SOCK|SO|SOL|IPPROTO|IP|IPV6|TCP|MCAST|EVFILT|NOTE|SHUT|PROT|MAP|MFD|T?PACKET|MSG|SCM|MCL|DT|MADV|PR|LOCAL|TCPOPT|UDP)_/ || $2 ~ /^NFC_(GENL|PROTO|COMM|RF|SE|DIRECTION|LLCP|SOCKPROTO)_/ || $2 ~ /^NFC_.*_(MAX)?SIZE$/ || $2 ~ /^RAW_PAYLOAD_/ || + $2 ~ /^[US]F_/ || $2 ~ /^TP_STATUS_/ || $2 ~ /^FALLOC_/ || $2 ~ /^ICMPV?6?_(FILTER|SEC)/ || diff --git a/vendor/golang.org/x/sys/unix/ptrace_darwin.go b/vendor/golang.org/x/sys/unix/ptrace_darwin.go index 463c3ef..39dba6c 100644 --- a/vendor/golang.org/x/sys/unix/ptrace_darwin.go +++ b/vendor/golang.org/x/sys/unix/ptrace_darwin.go @@ -7,6 +7,12 @@ package unix +import "unsafe" + func ptrace(request int, pid int, addr uintptr, data uintptr) error { return ptrace1(request, pid, addr, data) } + +func ptracePtr(request int, pid int, addr uintptr, data unsafe.Pointer) error { + return ptrace1Ptr(request, pid, addr, data) +} diff --git a/vendor/golang.org/x/sys/unix/ptrace_ios.go b/vendor/golang.org/x/sys/unix/ptrace_ios.go index ed0509a..9ea6633 100644 --- a/vendor/golang.org/x/sys/unix/ptrace_ios.go +++ b/vendor/golang.org/x/sys/unix/ptrace_ios.go @@ -7,6 +7,12 @@ package unix +import "unsafe" + func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) { return ENOTSUP } + +func ptracePtr(request int, pid int, addr uintptr, data unsafe.Pointer) (err error) { + return ENOTSUP +} diff --git a/vendor/golang.org/x/sys/unix/sockcmsg_unix.go b/vendor/golang.org/x/sys/unix/sockcmsg_unix.go index 453a942..3865943 100644 --- a/vendor/golang.org/x/sys/unix/sockcmsg_unix.go +++ b/vendor/golang.org/x/sys/unix/sockcmsg_unix.go @@ -52,6 +52,20 @@ func ParseSocketControlMessage(b []byte) ([]SocketControlMessage, error) { return msgs, nil } +// ParseOneSocketControlMessage parses a single socket control message from b, returning the message header, +// message data (a slice of b), and the remainder of b after that single message. +// When there are no remaining messages, len(remainder) == 0. +func ParseOneSocketControlMessage(b []byte) (hdr Cmsghdr, data []byte, remainder []byte, err error) { + h, dbuf, err := socketControlMessageHeaderAndData(b) + if err != nil { + return Cmsghdr{}, nil, nil, err + } + if i := cmsgAlignOf(int(h.Len)); i < len(b) { + remainder = b[i:] + } + return *h, dbuf, remainder, nil +} + func socketControlMessageHeaderAndData(b []byte) (*Cmsghdr, []byte, error) { h := (*Cmsghdr)(unsafe.Pointer(&b[0])) if h.Len < SizeofCmsghdr || uint64(h.Len) > uint64(len(b)) { diff --git a/vendor/golang.org/x/sys/unix/syscall_aix.go b/vendor/golang.org/x/sys/unix/syscall_aix.go index 2db1b51..c406ae0 100644 --- a/vendor/golang.org/x/sys/unix/syscall_aix.go +++ b/vendor/golang.org/x/sys/unix/syscall_aix.go @@ -292,9 +292,7 @@ func anyToSockaddr(fd int, rsa *RawSockaddrAny) (Sockaddr, error) { break } } - - bytes := (*[len(pp.Path)]byte)(unsafe.Pointer(&pp.Path[0]))[0:n] - sa.Name = string(bytes) + sa.Name = string(unsafe.Slice((*byte)(unsafe.Pointer(&pp.Path[0])), n)) return sa, nil case AF_INET: @@ -410,7 +408,8 @@ func (w WaitStatus) CoreDump() bool { return w&0x80 == 0x80 } func (w WaitStatus) TrapCause() int { return -1 } -//sys ioctl(fd int, req uint, arg uintptr) (err error) +//sys ioctl(fd int, req int, arg uintptr) (err error) +//sys ioctlPtr(fd int, req int, arg unsafe.Pointer) (err error) = ioctl // fcntl must never be called with cmd=F_DUP2FD because it doesn't work on AIX // There is no way to create a custom fcntl and to keep //sys fcntl easily, diff --git a/vendor/golang.org/x/sys/unix/syscall_aix_ppc.go b/vendor/golang.org/x/sys/unix/syscall_aix_ppc.go index e92a0be..f2871fa 100644 --- a/vendor/golang.org/x/sys/unix/syscall_aix_ppc.go +++ b/vendor/golang.org/x/sys/unix/syscall_aix_ppc.go @@ -8,7 +8,6 @@ package unix //sysnb Getrlimit(resource int, rlim *Rlimit) (err error) = getrlimit64 -//sysnb Setrlimit(resource int, rlim *Rlimit) (err error) = setrlimit64 //sys Seek(fd int, offset int64, whence int) (off int64, err error) = lseek64 //sys mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int64) (xaddr uintptr, err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_aix_ppc64.go b/vendor/golang.org/x/sys/unix/syscall_aix_ppc64.go index 16eed17..75718ec 100644 --- a/vendor/golang.org/x/sys/unix/syscall_aix_ppc64.go +++ b/vendor/golang.org/x/sys/unix/syscall_aix_ppc64.go @@ -8,7 +8,6 @@ package unix //sysnb Getrlimit(resource int, rlim *Rlimit) (err error) -//sysnb Setrlimit(resource int, rlim *Rlimit) (err error) //sys Seek(fd int, offset int64, whence int) (off int64, err error) = lseek //sys mmap(addr uintptr, length uintptr, prot int, flags int, fd int, offset int64) (xaddr uintptr, err error) = mmap64 diff --git a/vendor/golang.org/x/sys/unix/syscall_bsd.go b/vendor/golang.org/x/sys/unix/syscall_bsd.go index eda4267..7705c32 100644 --- a/vendor/golang.org/x/sys/unix/syscall_bsd.go +++ b/vendor/golang.org/x/sys/unix/syscall_bsd.go @@ -245,8 +245,7 @@ func anyToSockaddr(fd int, rsa *RawSockaddrAny) (Sockaddr, error) { break } } - bytes := (*[len(pp.Path)]byte)(unsafe.Pointer(&pp.Path[0]))[0:n] - sa.Name = string(bytes) + sa.Name = string(unsafe.Slice((*byte)(unsafe.Pointer(&pp.Path[0])), n)) return sa, nil case AF_INET: diff --git a/vendor/golang.org/x/sys/unix/syscall_darwin.go b/vendor/golang.org/x/sys/unix/syscall_darwin.go index 1f63382..2069215 100644 --- a/vendor/golang.org/x/sys/unix/syscall_darwin.go +++ b/vendor/golang.org/x/sys/unix/syscall_darwin.go @@ -14,7 +14,6 @@ package unix import ( "fmt" - "runtime" "syscall" "unsafe" ) @@ -230,6 +229,7 @@ func direntNamlen(buf []byte) (uint64, bool) { func PtraceAttach(pid int) (err error) { return ptrace(PT_ATTACH, pid, 0, 0) } func PtraceDetach(pid int) (err error) { return ptrace(PT_DETACH, pid, 0, 0) } +func PtraceDenyAttach() (err error) { return ptrace(PT_DENY_ATTACH, 0, 0, 0) } //sysnb pipe(p *[2]int32) (err error) @@ -375,11 +375,10 @@ func Flistxattr(fd int, dest []byte) (sz int, err error) { func Kill(pid int, signum syscall.Signal) (err error) { return kill(pid, int(signum), 1) } //sys ioctl(fd int, req uint, arg uintptr) (err error) +//sys ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) = SYS_IOCTL func IoctlCtlInfo(fd int, ctlInfo *CtlInfo) error { - err := ioctl(fd, CTLIOCGINFO, uintptr(unsafe.Pointer(ctlInfo))) - runtime.KeepAlive(ctlInfo) - return err + return ioctlPtr(fd, CTLIOCGINFO, unsafe.Pointer(ctlInfo)) } // IfreqMTU is struct ifreq used to get or set a network device's MTU. @@ -393,16 +392,14 @@ type IfreqMTU struct { func IoctlGetIfreqMTU(fd int, ifname string) (*IfreqMTU, error) { var ifreq IfreqMTU copy(ifreq.Name[:], ifname) - err := ioctl(fd, SIOCGIFMTU, uintptr(unsafe.Pointer(&ifreq))) + err := ioctlPtr(fd, SIOCGIFMTU, unsafe.Pointer(&ifreq)) return &ifreq, err } // IoctlSetIfreqMTU performs the SIOCSIFMTU ioctl operation on fd to set the MTU // of the network device specified by ifreq.Name. func IoctlSetIfreqMTU(fd int, ifreq *IfreqMTU) error { - err := ioctl(fd, SIOCSIFMTU, uintptr(unsafe.Pointer(ifreq))) - runtime.KeepAlive(ifreq) - return err + return ioctlPtr(fd, SIOCSIFMTU, unsafe.Pointer(ifreq)) } //sys sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) = SYS_SYSCTL @@ -616,6 +613,7 @@ func SysctlKinfoProcSlice(name string, args ...int) ([]KinfoProc, error) { //sys Rmdir(path string) (err error) //sys Seek(fd int, offset int64, whence int) (newoffset int64, err error) = SYS_LSEEK //sys Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err error) +//sys Setattrlist(path string, attrlist *Attrlist, attrBuf []byte, options int) (err error) //sys Setegid(egid int) (err error) //sysnb Seteuid(euid int) (err error) //sysnb Setgid(gid int) (err error) @@ -625,7 +623,6 @@ func SysctlKinfoProcSlice(name string, args ...int) ([]KinfoProc, error) { //sys Setprivexec(flag int) (err error) //sysnb Setregid(rgid int, egid int) (err error) //sysnb Setreuid(ruid int, euid int) (err error) -//sysnb Setrlimit(which int, lim *Rlimit) (err error) //sysnb Setsid() (pid int, err error) //sysnb Settimeofday(tp *Timeval) (err error) //sysnb Setuid(uid int) (err error) @@ -679,7 +676,6 @@ func SysctlKinfoProcSlice(name string, args ...int) ([]KinfoProc, error) { // Kqueue_from_portset_np // Kqueue_portset // Getattrlist -// Setattrlist // Getdirentriesattr // Searchfs // Delete diff --git a/vendor/golang.org/x/sys/unix/syscall_darwin_amd64.go b/vendor/golang.org/x/sys/unix/syscall_darwin_amd64.go index b37310c..9fa8798 100644 --- a/vendor/golang.org/x/sys/unix/syscall_darwin_amd64.go +++ b/vendor/golang.org/x/sys/unix/syscall_darwin_amd64.go @@ -47,5 +47,6 @@ func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, //sys getfsstat(buf unsafe.Pointer, size uintptr, flags int) (n int, err error) = SYS_GETFSSTAT64 //sys Lstat(path string, stat *Stat_t) (err error) = SYS_LSTAT64 //sys ptrace1(request int, pid int, addr uintptr, data uintptr) (err error) = SYS_ptrace +//sys ptrace1Ptr(request int, pid int, addr unsafe.Pointer, data uintptr) (err error) = SYS_ptrace //sys Stat(path string, stat *Stat_t) (err error) = SYS_STAT64 //sys Statfs(path string, stat *Statfs_t) (err error) = SYS_STATFS64 diff --git a/vendor/golang.org/x/sys/unix/syscall_darwin_arm64.go b/vendor/golang.org/x/sys/unix/syscall_darwin_arm64.go index d51ec99..f17b8c5 100644 --- a/vendor/golang.org/x/sys/unix/syscall_darwin_arm64.go +++ b/vendor/golang.org/x/sys/unix/syscall_darwin_arm64.go @@ -47,5 +47,6 @@ func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, //sys getfsstat(buf unsafe.Pointer, size uintptr, flags int) (n int, err error) = SYS_GETFSSTAT //sys Lstat(path string, stat *Stat_t) (err error) //sys ptrace1(request int, pid int, addr uintptr, data uintptr) (err error) = SYS_ptrace +//sys ptrace1Ptr(request int, pid int, addr unsafe.Pointer, data uintptr) (err error) = SYS_ptrace //sys Stat(path string, stat *Stat_t) (err error) //sys Statfs(path string, stat *Statfs_t) (err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_dragonfly.go b/vendor/golang.org/x/sys/unix/syscall_dragonfly.go index 61c0d0d..d4ce988 100644 --- a/vendor/golang.org/x/sys/unix/syscall_dragonfly.go +++ b/vendor/golang.org/x/sys/unix/syscall_dragonfly.go @@ -172,6 +172,7 @@ func Getfsstat(buf []Statfs_t, flags int) (n int, err error) { } //sys ioctl(fd int, req uint, arg uintptr) (err error) +//sys ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) = SYS_IOCTL //sys sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) = SYS___SYSCTL @@ -255,6 +256,7 @@ func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err e //sys Chmod(path string, mode uint32) (err error) //sys Chown(path string, uid int, gid int) (err error) //sys Chroot(path string) (err error) +//sys ClockGettime(clockid int32, time *Timespec) (err error) //sys Close(fd int) (err error) //sys Dup(fd int) (nfd int, err error) //sys Dup2(from int, to int) (err error) @@ -324,7 +326,6 @@ func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err e //sysnb Setreuid(ruid int, euid int) (err error) //sysnb Setresgid(rgid int, egid int, sgid int) (err error) //sysnb Setresuid(ruid int, euid int, suid int) (err error) -//sysnb Setrlimit(which int, lim *Rlimit) (err error) //sysnb Setsid() (pid int, err error) //sysnb Settimeofday(tp *Timeval) (err error) //sysnb Setuid(uid int) (err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_freebsd.go b/vendor/golang.org/x/sys/unix/syscall_freebsd.go index de7c23e..afb1010 100644 --- a/vendor/golang.org/x/sys/unix/syscall_freebsd.go +++ b/vendor/golang.org/x/sys/unix/syscall_freebsd.go @@ -161,7 +161,8 @@ func Getfsstat(buf []Statfs_t, flags int) (n int, err error) { return } -//sys ioctl(fd int, req uint, arg uintptr) (err error) +//sys ioctl(fd int, req uint, arg uintptr) (err error) = SYS_IOCTL +//sys ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) = SYS_IOCTL //sys sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) = SYS___SYSCTL @@ -253,6 +254,7 @@ func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err e } //sys ptrace(request int, pid int, addr uintptr, data int) (err error) +//sys ptracePtr(request int, pid int, addr unsafe.Pointer, data int) (err error) = SYS_PTRACE func PtraceAttach(pid int) (err error) { return ptrace(PT_ATTACH, pid, 0, 0) @@ -267,19 +269,36 @@ func PtraceDetach(pid int) (err error) { } func PtraceGetFpRegs(pid int, fpregsout *FpReg) (err error) { - return ptrace(PT_GETFPREGS, pid, uintptr(unsafe.Pointer(fpregsout)), 0) + return ptracePtr(PT_GETFPREGS, pid, unsafe.Pointer(fpregsout), 0) } func PtraceGetRegs(pid int, regsout *Reg) (err error) { - return ptrace(PT_GETREGS, pid, uintptr(unsafe.Pointer(regsout)), 0) + return ptracePtr(PT_GETREGS, pid, unsafe.Pointer(regsout), 0) +} + +func PtraceIO(req int, pid int, offs uintptr, out []byte, countin int) (count int, err error) { + ioDesc := PtraceIoDesc{ + Op: int32(req), + Offs: offs, + } + if countin > 0 { + _ = out[:countin] // check bounds + ioDesc.Addr = &out[0] + } else if out != nil { + ioDesc.Addr = (*byte)(unsafe.Pointer(&_zero)) + } + ioDesc.SetLen(countin) + + err = ptracePtr(PT_IO, pid, unsafe.Pointer(&ioDesc), 0) + return int(ioDesc.Len), err } func PtraceLwpEvents(pid int, enable int) (err error) { return ptrace(PT_LWP_EVENTS, pid, 0, enable) } -func PtraceLwpInfo(pid int, info uintptr) (err error) { - return ptrace(PT_LWPINFO, pid, info, int(unsafe.Sizeof(PtraceLwpInfoStruct{}))) +func PtraceLwpInfo(pid int, info *PtraceLwpInfoStruct) (err error) { + return ptracePtr(PT_LWPINFO, pid, unsafe.Pointer(info), int(unsafe.Sizeof(*info))) } func PtracePeekData(pid int, addr uintptr, out []byte) (count int, err error) { @@ -299,13 +318,25 @@ func PtracePokeText(pid int, addr uintptr, data []byte) (count int, err error) { } func PtraceSetRegs(pid int, regs *Reg) (err error) { - return ptrace(PT_SETREGS, pid, uintptr(unsafe.Pointer(regs)), 0) + return ptracePtr(PT_SETREGS, pid, unsafe.Pointer(regs), 0) } func PtraceSingleStep(pid int) (err error) { return ptrace(PT_STEP, pid, 1, 0) } +func Dup3(oldfd, newfd, flags int) error { + if oldfd == newfd || flags&^O_CLOEXEC != 0 { + return EINVAL + } + how := F_DUP2FD + if flags&O_CLOEXEC != 0 { + how = F_DUP2FD_CLOEXEC + } + _, err := fcntl(oldfd, how, newfd) + return err +} + /* * Exposed directly */ @@ -319,6 +350,7 @@ func PtraceSingleStep(pid int) (err error) { //sys Chmod(path string, mode uint32) (err error) //sys Chown(path string, uid int, gid int) (err error) //sys Chroot(path string) (err error) +//sys ClockGettime(clockid int32, time *Timespec) (err error) //sys Close(fd int) (err error) //sys Dup(fd int) (nfd int, err error) //sys Dup2(from int, to int) (err error) @@ -401,7 +433,6 @@ func PtraceSingleStep(pid int) (err error) { //sysnb Setreuid(ruid int, euid int) (err error) //sysnb Setresgid(rgid int, egid int, sgid int) (err error) //sysnb Setresuid(ruid int, euid int, suid int) (err error) -//sysnb Setrlimit(which int, lim *Rlimit) (err error) //sysnb Setsid() (pid int, err error) //sysnb Settimeofday(tp *Timeval) (err error) //sysnb Setuid(uid int) (err error) diff --git a/vendor/golang.org/x/sys/unix/syscall_freebsd_386.go b/vendor/golang.org/x/sys/unix/syscall_freebsd_386.go index b11ede8..b8da510 100644 --- a/vendor/golang.org/x/sys/unix/syscall_freebsd_386.go +++ b/vendor/golang.org/x/sys/unix/syscall_freebsd_386.go @@ -42,6 +42,10 @@ func (cmsg *Cmsghdr) SetLen(length int) { cmsg.Len = uint32(length) } +func (d *PtraceIoDesc) SetLen(length int) { + d.Len = uint32(length) +} + func sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) { var writtenOut uint64 = 0 _, _, e1 := Syscall9(SYS_SENDFILE, uintptr(infd), uintptr(outfd), uintptr(*offset), uintptr((*offset)>>32), uintptr(count), 0, uintptr(unsafe.Pointer(&writtenOut)), 0, 0) @@ -57,11 +61,5 @@ func sendfile(outfd int, infd int, offset *int64, count int) (written int, err e func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno) func PtraceGetFsBase(pid int, fsbase *int64) (err error) { - return ptrace(PT_GETFSBASE, pid, uintptr(unsafe.Pointer(fsbase)), 0) -} - -func PtraceIO(req int, pid int, addr uintptr, out []byte, countin int) (count int, err error) { - ioDesc := PtraceIoDesc{Op: int32(req), Offs: uintptr(unsafe.Pointer(addr)), Addr: uintptr(unsafe.Pointer(&out[0])), Len: uint32(countin)} - err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0) - return int(ioDesc.Len), err + return ptracePtr(PT_GETFSBASE, pid, unsafe.Pointer(fsbase), 0) } diff --git a/vendor/golang.org/x/sys/unix/syscall_freebsd_amd64.go b/vendor/golang.org/x/sys/unix/syscall_freebsd_amd64.go index 9ed8eec..47155c4 100644 --- a/vendor/golang.org/x/sys/unix/syscall_freebsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/syscall_freebsd_amd64.go @@ -42,6 +42,10 @@ func (cmsg *Cmsghdr) SetLen(length int) { cmsg.Len = uint32(length) } +func (d *PtraceIoDesc) SetLen(length int) { + d.Len = uint64(length) +} + func sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) { var writtenOut uint64 = 0 _, _, e1 := Syscall9(SYS_SENDFILE, uintptr(infd), uintptr(outfd), uintptr(*offset), uintptr(count), 0, uintptr(unsafe.Pointer(&writtenOut)), 0, 0, 0) @@ -57,11 +61,5 @@ func sendfile(outfd int, infd int, offset *int64, count int) (written int, err e func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno) func PtraceGetFsBase(pid int, fsbase *int64) (err error) { - return ptrace(PT_GETFSBASE, pid, uintptr(unsafe.Pointer(fsbase)), 0) -} - -func PtraceIO(req int, pid int, addr uintptr, out []byte, countin int) (count int, err error) { - ioDesc := PtraceIoDesc{Op: int32(req), Offs: uintptr(unsafe.Pointer(addr)), Addr: uintptr(unsafe.Pointer(&out[0])), Len: uint64(countin)} - err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0) - return int(ioDesc.Len), err + return ptracePtr(PT_GETFSBASE, pid, unsafe.Pointer(fsbase), 0) } diff --git a/vendor/golang.org/x/sys/unix/syscall_freebsd_arm.go b/vendor/golang.org/x/sys/unix/syscall_freebsd_arm.go index f8ac982..0893209 100644 --- a/vendor/golang.org/x/sys/unix/syscall_freebsd_arm.go +++ b/vendor/golang.org/x/sys/unix/syscall_freebsd_arm.go @@ -42,6 +42,10 @@ func (cmsg *Cmsghdr) SetLen(length int) { cmsg.Len = uint32(length) } +func (d *PtraceIoDesc) SetLen(length int) { + d.Len = uint32(length) +} + func sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) { var writtenOut uint64 = 0 _, _, e1 := Syscall9(SYS_SENDFILE, uintptr(infd), uintptr(outfd), uintptr(*offset), uintptr((*offset)>>32), uintptr(count), 0, uintptr(unsafe.Pointer(&writtenOut)), 0, 0) @@ -55,9 +59,3 @@ func sendfile(outfd int, infd int, offset *int64, count int) (written int, err e } func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno) - -func PtraceIO(req int, pid int, addr uintptr, out []byte, countin int) (count int, err error) { - ioDesc := PtraceIoDesc{Op: int32(req), Offs: uintptr(unsafe.Pointer(addr)), Addr: uintptr(unsafe.Pointer(&out[0])), Len: uint32(countin)} - err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0) - return int(ioDesc.Len), err -} diff --git a/vendor/golang.org/x/sys/unix/syscall_freebsd_arm64.go b/vendor/golang.org/x/sys/unix/syscall_freebsd_arm64.go index 8e93203..d151a0d 100644 --- a/vendor/golang.org/x/sys/unix/syscall_freebsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/syscall_freebsd_arm64.go @@ -42,6 +42,10 @@ func (cmsg *Cmsghdr) SetLen(length int) { cmsg.Len = uint32(length) } +func (d *PtraceIoDesc) SetLen(length int) { + d.Len = uint64(length) +} + func sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) { var writtenOut uint64 = 0 _, _, e1 := Syscall9(SYS_SENDFILE, uintptr(infd), uintptr(outfd), uintptr(*offset), uintptr(count), 0, uintptr(unsafe.Pointer(&writtenOut)), 0, 0, 0) @@ -55,9 +59,3 @@ func sendfile(outfd int, infd int, offset *int64, count int) (written int, err e } func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno) - -func PtraceIO(req int, pid int, addr uintptr, out []byte, countin int) (count int, err error) { - ioDesc := PtraceIoDesc{Op: int32(req), Offs: uintptr(unsafe.Pointer(addr)), Addr: uintptr(unsafe.Pointer(&out[0])), Len: uint64(countin)} - err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0) - return int(ioDesc.Len), err -} diff --git a/vendor/golang.org/x/sys/unix/syscall_freebsd_riscv64.go b/vendor/golang.org/x/sys/unix/syscall_freebsd_riscv64.go index cbe1222..d5cd64b 100644 --- a/vendor/golang.org/x/sys/unix/syscall_freebsd_riscv64.go +++ b/vendor/golang.org/x/sys/unix/syscall_freebsd_riscv64.go @@ -42,6 +42,10 @@ func (cmsg *Cmsghdr) SetLen(length int) { cmsg.Len = uint32(length) } +func (d *PtraceIoDesc) SetLen(length int) { + d.Len = uint64(length) +} + func sendfile(outfd int, infd int, offset *int64, count int) (written int, err error) { var writtenOut uint64 = 0 _, _, e1 := Syscall9(SYS_SENDFILE, uintptr(infd), uintptr(outfd), uintptr(*offset), uintptr(count), 0, uintptr(unsafe.Pointer(&writtenOut)), 0, 0, 0) @@ -55,9 +59,3 @@ func sendfile(outfd int, infd int, offset *int64, count int) (written int, err e } func Syscall9(num, a1, a2, a3, a4, a5, a6, a7, a8, a9 uintptr) (r1, r2 uintptr, err syscall.Errno) - -func PtraceIO(req int, pid int, addr uintptr, out []byte, countin int) (count int, err error) { - ioDesc := PtraceIoDesc{Op: int32(req), Offs: uintptr(unsafe.Pointer(addr)), Addr: uintptr(unsafe.Pointer(&out[0])), Len: uint64(countin)} - err = ptrace(PT_IO, pid, uintptr(unsafe.Pointer(&ioDesc)), 0) - return int(ioDesc.Len), err -} diff --git a/vendor/golang.org/x/sys/unix/syscall_hurd.go b/vendor/golang.org/x/sys/unix/syscall_hurd.go new file mode 100644 index 0000000..381fd46 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/syscall_hurd.go @@ -0,0 +1,30 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build hurd +// +build hurd + +package unix + +/* +#include +int ioctl(int, unsigned long int, uintptr_t); +*/ +import "C" + +func ioctl(fd int, req uint, arg uintptr) (err error) { + r0, er := C.ioctl(C.int(fd), C.ulong(req), C.uintptr_t(arg)) + if r0 == -1 && er != nil { + err = er + } + return +} + +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + r0, er := C.ioctl(C.int(fd), C.ulong(req), C.uintptr_t(uintptr(arg))) + if r0 == -1 && er != nil { + err = er + } + return +} diff --git a/vendor/golang.org/x/sys/unix/syscall_hurd_386.go b/vendor/golang.org/x/sys/unix/syscall_hurd_386.go new file mode 100644 index 0000000..7cf54a3 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/syscall_hurd_386.go @@ -0,0 +1,29 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//go:build 386 && hurd +// +build 386,hurd + +package unix + +const ( + TIOCGETA = 0x62251713 +) + +type Winsize struct { + Row uint16 + Col uint16 + Xpixel uint16 + Ypixel uint16 +} + +type Termios struct { + Iflag uint32 + Oflag uint32 + Cflag uint32 + Lflag uint32 + Cc [20]uint8 + Ispeed int32 + Ospeed int32 +} diff --git a/vendor/golang.org/x/sys/unix/syscall_linux.go b/vendor/golang.org/x/sys/unix/syscall_linux.go index e044d5b..fbaeb5f 100644 --- a/vendor/golang.org/x/sys/unix/syscall_linux.go +++ b/vendor/golang.org/x/sys/unix/syscall_linux.go @@ -1015,8 +1015,7 @@ func anyToSockaddr(fd int, rsa *RawSockaddrAny) (Sockaddr, error) { for n < len(pp.Path) && pp.Path[n] != 0 { n++ } - bytes := (*[len(pp.Path)]byte)(unsafe.Pointer(&pp.Path[0]))[0:n] - sa.Name = string(bytes) + sa.Name = string(unsafe.Slice((*byte)(unsafe.Pointer(&pp.Path[0])), n)) return sa, nil case AF_INET: @@ -1365,6 +1364,10 @@ func SetsockoptTCPRepairOpt(fd, level, opt int, o []TCPRepairOpt) (err error) { return setsockopt(fd, level, opt, unsafe.Pointer(&o[0]), uintptr(SizeofTCPRepairOpt*len(o))) } +func SetsockoptTCPMD5Sig(fd, level, opt int, s *TCPMD5Sig) error { + return setsockopt(fd, level, opt, unsafe.Pointer(s), unsafe.Sizeof(*s)) +} + // Keyctl Commands (http://man7.org/linux/man-pages/man2/keyctl.2.html) // KeyctlInt calls keyctl commands in which each argument is an int. @@ -1554,6 +1557,7 @@ func sendmsgN(fd int, iov []Iovec, oob []byte, ptr unsafe.Pointer, salen _Sockle var iova [1]Iovec iova[0].Base = &dummy iova[0].SetLen(1) + iov = iova[:] } } msg.Control = &oob[0] @@ -1578,6 +1582,7 @@ func BindToDevice(fd int, device string) (err error) { } //sys ptrace(request int, pid int, addr uintptr, data uintptr) (err error) +//sys ptracePtr(request int, pid int, addr uintptr, data unsafe.Pointer) (err error) = SYS_PTRACE func ptracePeek(req int, pid int, addr uintptr, out []byte) (count int, err error) { // The peek requests are machine-size oriented, so we wrap it @@ -1595,7 +1600,7 @@ func ptracePeek(req int, pid int, addr uintptr, out []byte) (count int, err erro // boundary. n := 0 if addr%SizeofPtr != 0 { - err = ptrace(req, pid, addr-addr%SizeofPtr, uintptr(unsafe.Pointer(&buf[0]))) + err = ptracePtr(req, pid, addr-addr%SizeofPtr, unsafe.Pointer(&buf[0])) if err != nil { return 0, err } @@ -1607,7 +1612,7 @@ func ptracePeek(req int, pid int, addr uintptr, out []byte) (count int, err erro for len(out) > 0 { // We use an internal buffer to guarantee alignment. // It's not documented if this is necessary, but we're paranoid. - err = ptrace(req, pid, addr+uintptr(n), uintptr(unsafe.Pointer(&buf[0]))) + err = ptracePtr(req, pid, addr+uintptr(n), unsafe.Pointer(&buf[0])) if err != nil { return n, err } @@ -1639,7 +1644,7 @@ func ptracePoke(pokeReq int, peekReq int, pid int, addr uintptr, data []byte) (c n := 0 if addr%SizeofPtr != 0 { var buf [SizeofPtr]byte - err = ptrace(peekReq, pid, addr-addr%SizeofPtr, uintptr(unsafe.Pointer(&buf[0]))) + err = ptracePtr(peekReq, pid, addr-addr%SizeofPtr, unsafe.Pointer(&buf[0])) if err != nil { return 0, err } @@ -1666,7 +1671,7 @@ func ptracePoke(pokeReq int, peekReq int, pid int, addr uintptr, data []byte) (c // Trailing edge. if len(data) > 0 { var buf [SizeofPtr]byte - err = ptrace(peekReq, pid, addr+uintptr(n), uintptr(unsafe.Pointer(&buf[0]))) + err = ptracePtr(peekReq, pid, addr+uintptr(n), unsafe.Pointer(&buf[0])) if err != nil { return n, err } @@ -1695,11 +1700,11 @@ func PtracePokeUser(pid int, addr uintptr, data []byte) (count int, err error) { } func PtraceGetRegs(pid int, regsout *PtraceRegs) (err error) { - return ptrace(PTRACE_GETREGS, pid, 0, uintptr(unsafe.Pointer(regsout))) + return ptracePtr(PTRACE_GETREGS, pid, 0, unsafe.Pointer(regsout)) } func PtraceSetRegs(pid int, regs *PtraceRegs) (err error) { - return ptrace(PTRACE_SETREGS, pid, 0, uintptr(unsafe.Pointer(regs))) + return ptracePtr(PTRACE_SETREGS, pid, 0, unsafe.Pointer(regs)) } func PtraceSetOptions(pid int, options int) (err error) { @@ -1708,7 +1713,7 @@ func PtraceSetOptions(pid int, options int) (err error) { func PtraceGetEventMsg(pid int) (msg uint, err error) { var data _C_long - err = ptrace(PTRACE_GETEVENTMSG, pid, 0, uintptr(unsafe.Pointer(&data))) + err = ptracePtr(PTRACE_GETEVENTMSG, pid, 0, unsafe.Pointer(&data)) msg = uint(data) return } @@ -1799,6 +1804,7 @@ func Sendfile(outfd int, infd int, offset *int64, count int) (written int, err e //sysnb Capset(hdr *CapUserHeader, data *CapUserData) (err error) //sys Chdir(path string) (err error) //sys Chroot(path string) (err error) +//sys ClockAdjtime(clockid int32, buf *Timex) (state int, err error) //sys ClockGetres(clockid int32, res *Timespec) (err error) //sys ClockGettime(clockid int32, time *Timespec) (err error) //sys ClockNanosleep(clockid int32, flags int, request *Timespec, remain *Timespec) (err error) @@ -1867,7 +1873,6 @@ func Getpgrp() (pid int) { //sys OpenTree(dfd int, fileName string, flags uint) (r int, err error) //sys PerfEventOpen(attr *PerfEventAttr, pid int, cpu int, groupFd int, flags int) (fd int, err error) //sys PivotRoot(newroot string, putold string) (err error) = SYS_PIVOT_ROOT -//sysnb Prlimit(pid int, resource int, newlimit *Rlimit, old *Rlimit) (err error) = SYS_PRLIMIT64 //sys Prctl(option int, arg2 uintptr, arg3 uintptr, arg4 uintptr, arg5 uintptr) (err error) //sys Pselect(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timespec, sigmask *Sigset_t) (n int, err error) = SYS_PSELECT6 //sys read(fd int, p []byte) (n int, err error) @@ -1881,6 +1886,15 @@ func Getpgrp() (pid int) { //sysnb Settimeofday(tv *Timeval) (err error) //sys Setns(fd int, nstype int) (err error) +//go:linkname syscall_prlimit syscall.prlimit +func syscall_prlimit(pid, resource int, newlimit, old *syscall.Rlimit) error + +func Prlimit(pid, resource int, newlimit, old *Rlimit) error { + // Just call the syscall version, because as of Go 1.21 + // it will affect starting a new process. + return syscall_prlimit(pid, resource, (*syscall.Rlimit)(newlimit), (*syscall.Rlimit)(old)) +} + // PrctlRetInt performs a prctl operation specified by option and further // optional arguments arg2 through arg5 depending on option. It returns a // non-negative integer that is returned by the prctl syscall. @@ -1972,36 +1986,46 @@ func Signalfd(fd int, sigmask *Sigset_t, flags int) (newfd int, err error) { //sys preadv2(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr, flags int) (n int, err error) = SYS_PREADV2 //sys pwritev2(fd int, iovs []Iovec, offs_l uintptr, offs_h uintptr, flags int) (n int, err error) = SYS_PWRITEV2 -func bytes2iovec(bs [][]byte) []Iovec { - iovecs := make([]Iovec, len(bs)) - for i, b := range bs { - iovecs[i].SetLen(len(b)) +// minIovec is the size of the small initial allocation used by +// Readv, Writev, etc. +// +// This small allocation gets stack allocated, which lets the +// common use case of len(iovs) <= minIovs avoid more expensive +// heap allocations. +const minIovec = 8 + +// appendBytes converts bs to Iovecs and appends them to vecs. +func appendBytes(vecs []Iovec, bs [][]byte) []Iovec { + for _, b := range bs { + var v Iovec + v.SetLen(len(b)) if len(b) > 0 { - iovecs[i].Base = &b[0] + v.Base = &b[0] } else { - iovecs[i].Base = (*byte)(unsafe.Pointer(&_zero)) + v.Base = (*byte)(unsafe.Pointer(&_zero)) } + vecs = append(vecs, v) } - return iovecs + return vecs } -// offs2lohi splits offs into its lower and upper unsigned long. On 64-bit -// systems, hi will always be 0. On 32-bit systems, offs will be split in half. -// preadv/pwritev chose this calling convention so they don't need to add a -// padding-register for alignment on ARM. +// offs2lohi splits offs into its low and high order bits. func offs2lohi(offs int64) (lo, hi uintptr) { - return uintptr(offs), uintptr(uint64(offs) >> SizeofLong) + const longBits = SizeofLong * 8 + return uintptr(offs), uintptr(uint64(offs) >> (longBits - 1) >> 1) // two shifts to avoid false positive in vet } func Readv(fd int, iovs [][]byte) (n int, err error) { - iovecs := bytes2iovec(iovs) + iovecs := make([]Iovec, 0, minIovec) + iovecs = appendBytes(iovecs, iovs) n, err = readv(fd, iovecs) readvRacedetect(iovecs, n, err) return n, err } func Preadv(fd int, iovs [][]byte, offset int64) (n int, err error) { - iovecs := bytes2iovec(iovs) + iovecs := make([]Iovec, 0, minIovec) + iovecs = appendBytes(iovecs, iovs) lo, hi := offs2lohi(offset) n, err = preadv(fd, iovecs, lo, hi) readvRacedetect(iovecs, n, err) @@ -2009,7 +2033,8 @@ func Preadv(fd int, iovs [][]byte, offset int64) (n int, err error) { } func Preadv2(fd int, iovs [][]byte, offset int64, flags int) (n int, err error) { - iovecs := bytes2iovec(iovs) + iovecs := make([]Iovec, 0, minIovec) + iovecs = appendBytes(iovecs, iovs) lo, hi := offs2lohi(offset) n, err = preadv2(fd, iovecs, lo, hi, flags) readvRacedetect(iovecs, n, err) @@ -2036,7 +2061,8 @@ func readvRacedetect(iovecs []Iovec, n int, err error) { } func Writev(fd int, iovs [][]byte) (n int, err error) { - iovecs := bytes2iovec(iovs) + iovecs := make([]Iovec, 0, minIovec) + iovecs = appendBytes(iovecs, iovs) if raceenabled { raceReleaseMerge(unsafe.Pointer(&ioSync)) } @@ -2046,7 +2072,8 @@ func Writev(fd int, iovs [][]byte) (n int, err error) { } func Pwritev(fd int, iovs [][]byte, offset int64) (n int, err error) { - iovecs := bytes2iovec(iovs) + iovecs := make([]Iovec, 0, minIovec) + iovecs = appendBytes(iovecs, iovs) if raceenabled { raceReleaseMerge(unsafe.Pointer(&ioSync)) } @@ -2057,7 +2084,8 @@ func Pwritev(fd int, iovs [][]byte, offset int64) (n int, err error) { } func Pwritev2(fd int, iovs [][]byte, offset int64, flags int) (n int, err error) { - iovecs := bytes2iovec(iovs) + iovecs := make([]Iovec, 0, minIovec) + iovecs = appendBytes(iovecs, iovs) if raceenabled { raceReleaseMerge(unsafe.Pointer(&ioSync)) } @@ -2138,6 +2166,14 @@ func isGroupMember(gid int) bool { return false } +func isCapDacOverrideSet() bool { + hdr := CapUserHeader{Version: LINUX_CAPABILITY_VERSION_3} + data := [2]CapUserData{} + err := Capget(&hdr, &data[0]) + + return err == nil && data[0].Effective&(1< 0 { @@ -346,13 +359,9 @@ func Recvmsg(fd int, p, oob []byte, flags int) (n, oobn int, recvflags int, from return } -// RecvmsgBuffers receives a message from a socket using the recvmsg -// system call. The flags are passed to recvmsg. Any non-control data -// read is scattered into the buffers slices. The results are: -// - n is the number of non-control data read into bufs -// - oobn is the number of control data read into oob; this may be interpreted using [ParseSocketControlMessage] -// - recvflags is flags returned by recvmsg -// - from is the address of the sender +// RecvmsgBuffers receives a message from a socket using the recvmsg system +// call. This function is equivalent to Recvmsg, but non-control data read is +// scattered into the buffers slices. func RecvmsgBuffers(fd int, buffers [][]byte, oob []byte, flags int) (n, oobn int, recvflags int, from Sockaddr, err error) { iov := make([]Iovec, len(buffers)) for i := range buffers { @@ -371,11 +380,38 @@ func RecvmsgBuffers(fd int, buffers [][]byte, oob []byte, flags int) (n, oobn in return } +// Sendmsg sends a message on a socket to an address using the sendmsg system +// call. This function is equivalent to SendmsgN, but does not return the +// number of bytes actually sent. func Sendmsg(fd int, p, oob []byte, to Sockaddr, flags int) (err error) { _, err = SendmsgN(fd, p, oob, to, flags) return } +// SendmsgN sends a message on a socket to an address using the sendmsg system +// call. p contains the non-control data to send, and oob contains the "out of +// band" control data. The flags are passed to sendmsg. The number of +// non-control bytes actually written to the socket is returned. +// +// Some socket types do not support sending control data without accompanying +// non-control data. If p is empty, and oob contains control data, and the +// underlying socket type is not SOCK_DGRAM, p will be treated as containing a +// single '\0' and the return value will indicate zero bytes sent. +// +// The Go function Recvmsg, if called with an empty p and a non-empty oob, +// will read and ignore this additional '\0'. If the message is received by +// code that does not use Recvmsg, or that does not use Go at all, that code +// will need to be written to expect and ignore the additional '\0'. +// +// If you need to send non-empty oob with p actually empty, and if the +// underlying socket type supports it, you can do so via a raw system call as +// follows: +// +// msg := &unix.Msghdr{ +// Control: &oob[0], +// } +// msg.SetControllen(len(oob)) +// n, _, errno := unix.Syscall(unix.SYS_SENDMSG, uintptr(fd), uintptr(unsafe.Pointer(msg)), flags) func SendmsgN(fd int, p, oob []byte, to Sockaddr, flags int) (n int, err error) { var iov [1]Iovec if len(p) > 0 { @@ -394,9 +430,8 @@ func SendmsgN(fd int, p, oob []byte, to Sockaddr, flags int) (n int, err error) } // SendmsgBuffers sends a message on a socket to an address using the sendmsg -// system call. The flags are passed to sendmsg. Any non-control data written -// is gathered from buffers. The function returns the number of bytes written -// to the socket. +// system call. This function is equivalent to SendmsgN, but the non-control +// data is gathered from buffers. func SendmsgBuffers(fd int, buffers [][]byte, oob []byte, to Sockaddr, flags int) (n int, err error) { iov := make([]Iovec, len(buffers)) for i := range buffers { @@ -543,7 +578,7 @@ func Lutimes(path string, tv []Timeval) error { return UtimesNanoAt(AT_FDCWD, path, ts, AT_SYMLINK_NOFOLLOW) } -// emptyIovec reports whether there are no bytes in the slice of Iovec. +// emptyIovecs reports whether there are no bytes in the slice of Iovec. func emptyIovecs(iov []Iovec) bool { for i := range iov { if iov[i].Len > 0 { @@ -552,3 +587,10 @@ func emptyIovecs(iov []Iovec) bool { } return true } + +// Setrlimit sets a resource limit. +func Setrlimit(resource int, rlim *Rlimit) error { + // Just call the syscall version, because as of Go 1.21 + // it will affect starting a new process. + return syscall.Setrlimit(resource, (*syscall.Rlimit)(rlim)) +} diff --git a/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go b/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go index 68b2f3e..d3d49ec 100644 --- a/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go +++ b/vendor/golang.org/x/sys/unix/syscall_zos_s390x.go @@ -139,8 +139,7 @@ func anyToSockaddr(_ int, rsa *RawSockaddrAny) (Sockaddr, error) { for n < int(pp.Len) && pp.Path[n] != 0 { n++ } - bytes := (*[len(pp.Path)]byte)(unsafe.Pointer(&pp.Path[0]))[0:n] - sa.Name = string(bytes) + sa.Name = string(unsafe.Slice((*byte)(unsafe.Pointer(&pp.Path[0])), n)) return sa, nil case AF_INET: @@ -213,7 +212,8 @@ func (cmsg *Cmsghdr) SetLen(length int) { //sys sendmsg(s int, msg *Msghdr, flags int) (n int, err error) = SYS___SENDMSG_A //sys mmap(addr uintptr, length uintptr, prot int, flag int, fd int, pos int64) (ret uintptr, err error) = SYS_MMAP //sys munmap(addr uintptr, length uintptr) (err error) = SYS_MUNMAP -//sys ioctl(fd int, req uint, arg uintptr) (err error) = SYS_IOCTL +//sys ioctl(fd int, req int, arg uintptr) (err error) = SYS_IOCTL +//sys ioctlPtr(fd int, req int, arg unsafe.Pointer) (err error) = SYS_IOCTL //sys Access(path string, mode uint32) (err error) = SYS___ACCESS_A //sys Chdir(path string) (err error) = SYS___CHDIR_A diff --git a/vendor/golang.org/x/sys/unix/timestruct.go b/vendor/golang.org/x/sys/unix/timestruct.go index 3d89304..616b1b2 100644 --- a/vendor/golang.org/x/sys/unix/timestruct.go +++ b/vendor/golang.org/x/sys/unix/timestruct.go @@ -9,7 +9,7 @@ package unix import "time" -// TimespecToNSec returns the time stored in ts as nanoseconds. +// TimespecToNsec returns the time stored in ts as nanoseconds. func TimespecToNsec(ts Timespec) int64 { return ts.Nano() } // NsecToTimespec converts a number of nanoseconds into a Timespec. diff --git a/vendor/golang.org/x/sys/unix/xattr_bsd.go b/vendor/golang.org/x/sys/unix/xattr_bsd.go index 663b377..f5f8e9f 100644 --- a/vendor/golang.org/x/sys/unix/xattr_bsd.go +++ b/vendor/golang.org/x/sys/unix/xattr_bsd.go @@ -36,9 +36,14 @@ func xattrnamespace(fullattr string) (ns int, attr string, err error) { func initxattrdest(dest []byte, idx int) (d unsafe.Pointer) { if len(dest) > idx { return unsafe.Pointer(&dest[idx]) - } else { - return unsafe.Pointer(_zero) } + if dest != nil { + // extattr_get_file and extattr_list_file treat NULL differently from + // a non-NULL pointer of length zero. Preserve the property of nilness, + // even if we can't use dest directly. + return unsafe.Pointer(&_zero) + } + return nil } // FreeBSD and NetBSD implement their own syscalls to handle extended attributes diff --git a/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go index 476a1c7..1430076 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_darwin_amd64.go @@ -1270,6 +1270,16 @@ const ( SEEK_END = 0x2 SEEK_HOLE = 0x3 SEEK_SET = 0x0 + SF_APPEND = 0x40000 + SF_ARCHIVED = 0x10000 + SF_DATALESS = 0x40000000 + SF_FIRMLINK = 0x800000 + SF_IMMUTABLE = 0x20000 + SF_NOUNLINK = 0x100000 + SF_RESTRICTED = 0x80000 + SF_SETTABLE = 0x3fff0000 + SF_SUPPORTED = 0x9f0000 + SF_SYNTHETIC = 0xc0000000 SHUT_RD = 0x0 SHUT_RDWR = 0x2 SHUT_WR = 0x1 @@ -1543,6 +1553,15 @@ const ( TIOCTIMESTAMP = 0x40107459 TIOCUCNTL = 0x80047466 TOSTOP = 0x400000 + UF_APPEND = 0x4 + UF_COMPRESSED = 0x20 + UF_DATAVAULT = 0x80 + UF_HIDDEN = 0x8000 + UF_IMMUTABLE = 0x2 + UF_NODUMP = 0x1 + UF_OPAQUE = 0x8 + UF_SETTABLE = 0xffff + UF_TRACKED = 0x40 VDISCARD = 0xf VDSUSP = 0xb VEOF = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go index e36f517..ab044a7 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_darwin_arm64.go @@ -1270,6 +1270,16 @@ const ( SEEK_END = 0x2 SEEK_HOLE = 0x3 SEEK_SET = 0x0 + SF_APPEND = 0x40000 + SF_ARCHIVED = 0x10000 + SF_DATALESS = 0x40000000 + SF_FIRMLINK = 0x800000 + SF_IMMUTABLE = 0x20000 + SF_NOUNLINK = 0x100000 + SF_RESTRICTED = 0x80000 + SF_SETTABLE = 0x3fff0000 + SF_SUPPORTED = 0x9f0000 + SF_SYNTHETIC = 0xc0000000 SHUT_RD = 0x0 SHUT_RDWR = 0x2 SHUT_WR = 0x1 @@ -1543,6 +1553,15 @@ const ( TIOCTIMESTAMP = 0x40107459 TIOCUCNTL = 0x80047466 TOSTOP = 0x400000 + UF_APPEND = 0x4 + UF_COMPRESSED = 0x20 + UF_DATAVAULT = 0x80 + UF_HIDDEN = 0x8000 + UF_IMMUTABLE = 0x2 + UF_NODUMP = 0x1 + UF_OPAQUE = 0x8 + UF_SETTABLE = 0xffff + UF_TRACKED = 0x40 VDISCARD = 0xf VDSUSP = 0xb VEOF = 0x0 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux.go b/vendor/golang.org/x/sys/unix/zerrors_linux.go index 785d693..de936b6 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux.go @@ -70,6 +70,7 @@ const ( ALG_SET_DRBG_ENTROPY = 0x6 ALG_SET_IV = 0x2 ALG_SET_KEY = 0x1 + ALG_SET_KEY_BY_KEY_SERIAL = 0x7 ALG_SET_OP = 0x3 ANON_INODE_FS_MAGIC = 0x9041934 ARPHRD_6LOWPAN = 0x339 @@ -457,7 +458,6 @@ const ( B600 = 0x8 B75 = 0x2 B9600 = 0xd - BALLOON_KVM_MAGIC = 0x13661366 BDEVFS_MAGIC = 0x62646576 BINDERFS_SUPER_MAGIC = 0x6c6f6f70 BINFMTFS_MAGIC = 0x42494e4d @@ -563,6 +563,7 @@ const ( BUS_USB = 0x3 BUS_VIRTUAL = 0x6 CAN_BCM = 0x2 + CAN_BUS_OFF_THRESHOLD = 0x100 CAN_CTRLMODE_3_SAMPLES = 0x4 CAN_CTRLMODE_BERR_REPORTING = 0x10 CAN_CTRLMODE_CC_LEN8_DLC = 0x100 @@ -577,9 +578,12 @@ const ( CAN_EFF_FLAG = 0x80000000 CAN_EFF_ID_BITS = 0x1d CAN_EFF_MASK = 0x1fffffff + CAN_ERROR_PASSIVE_THRESHOLD = 0x80 + CAN_ERROR_WARNING_THRESHOLD = 0x60 CAN_ERR_ACK = 0x20 CAN_ERR_BUSERROR = 0x80 CAN_ERR_BUSOFF = 0x40 + CAN_ERR_CNT = 0x200 CAN_ERR_CRTL = 0x4 CAN_ERR_CRTL_ACTIVE = 0x40 CAN_ERR_CRTL_RX_OVERFLOW = 0x1 @@ -771,6 +775,8 @@ const ( DEVLINK_GENL_MCGRP_CONFIG_NAME = "config" DEVLINK_GENL_NAME = "devlink" DEVLINK_GENL_VERSION = 0x1 + DEVLINK_PORT_FN_CAP_MIGRATABLE = 0x2 + DEVLINK_PORT_FN_CAP_ROCE = 0x1 DEVLINK_SB_THRESHOLD_TO_ALPHA_MAX = 0x14 DEVLINK_SUPPORTED_FLASH_OVERWRITE_SECTIONS = 0x3 DEVMEM_MAGIC = 0x454d444d @@ -820,9 +826,9 @@ const ( DM_UUID_FLAG = 0x4000 DM_UUID_LEN = 0x81 DM_VERSION = 0xc138fd00 - DM_VERSION_EXTRA = "-ioctl (2022-02-22)" + DM_VERSION_EXTRA = "-ioctl (2022-07-28)" DM_VERSION_MAJOR = 0x4 - DM_VERSION_MINOR = 0x2e + DM_VERSION_MINOR = 0x2f DM_VERSION_PATCHLEVEL = 0x0 DT_BLK = 0x6 DT_CHR = 0x2 @@ -1049,6 +1055,7 @@ const ( ETH_P_CAIF = 0xf7 ETH_P_CAN = 0xc ETH_P_CANFD = 0xd + ETH_P_CANXL = 0xe ETH_P_CFM = 0x8902 ETH_P_CONTROL = 0x16 ETH_P_CUST = 0x6006 @@ -1060,6 +1067,7 @@ const ( ETH_P_DNA_RT = 0x6003 ETH_P_DSA = 0x1b ETH_P_DSA_8021Q = 0xdadb + ETH_P_DSA_A5PSW = 0xe001 ETH_P_ECONET = 0x18 ETH_P_EDSA = 0xdada ETH_P_ERSPAN = 0x88be @@ -1194,8 +1202,10 @@ const ( FAN_MARK_EVICTABLE = 0x200 FAN_MARK_FILESYSTEM = 0x100 FAN_MARK_FLUSH = 0x80 + FAN_MARK_IGNORE = 0x400 FAN_MARK_IGNORED_MASK = 0x20 FAN_MARK_IGNORED_SURV_MODIFY = 0x40 + FAN_MARK_IGNORE_SURV = 0x440 FAN_MARK_INODE = 0x0 FAN_MARK_MOUNT = 0x10 FAN_MARK_ONLYDIR = 0x8 @@ -1253,7 +1263,10 @@ const ( FSCRYPT_MODE_AES_128_CBC = 0x5 FSCRYPT_MODE_AES_128_CTS = 0x6 FSCRYPT_MODE_AES_256_CTS = 0x4 + FSCRYPT_MODE_AES_256_HCTR2 = 0xa FSCRYPT_MODE_AES_256_XTS = 0x1 + FSCRYPT_MODE_SM4_CTS = 0x8 + FSCRYPT_MODE_SM4_XTS = 0x7 FSCRYPT_POLICY_FLAGS_PAD_16 = 0x2 FSCRYPT_POLICY_FLAGS_PAD_32 = 0x3 FSCRYPT_POLICY_FLAGS_PAD_4 = 0x0 @@ -1272,8 +1285,6 @@ const ( FS_ENCRYPTION_MODE_AES_256_GCM = 0x2 FS_ENCRYPTION_MODE_AES_256_XTS = 0x1 FS_ENCRYPTION_MODE_INVALID = 0x0 - FS_ENCRYPTION_MODE_SPECK128_256_CTS = 0x8 - FS_ENCRYPTION_MODE_SPECK128_256_XTS = 0x7 FS_IOC_ADD_ENCRYPTION_KEY = 0xc0506617 FS_IOC_GET_ENCRYPTION_KEY_STATUS = 0xc080661a FS_IOC_GET_ENCRYPTION_POLICY_EX = 0xc0096616 @@ -1430,6 +1441,7 @@ const ( IFF_NOARP = 0x80 IFF_NOFILTER = 0x1000 IFF_NOTRAILERS = 0x20 + IFF_NO_CARRIER = 0x40 IFF_NO_PI = 0x1000 IFF_ONE_QUEUE = 0x2000 IFF_PERSIST = 0x800 @@ -1761,6 +1773,7 @@ const ( LANDLOCK_ACCESS_FS_REFER = 0x2000 LANDLOCK_ACCESS_FS_REMOVE_DIR = 0x10 LANDLOCK_ACCESS_FS_REMOVE_FILE = 0x20 + LANDLOCK_ACCESS_FS_TRUNCATE = 0x4000 LANDLOCK_ACCESS_FS_WRITE_FILE = 0x2 LANDLOCK_CREATE_RULESET_VERSION = 0x1 LINUX_REBOOT_CMD_CAD_OFF = 0x0 @@ -1800,11 +1813,13 @@ const ( LWTUNNEL_IP_OPT_GENEVE_MAX = 0x3 LWTUNNEL_IP_OPT_VXLAN_MAX = 0x1 MADV_COLD = 0x14 + MADV_COLLAPSE = 0x19 MADV_DODUMP = 0x11 MADV_DOFORK = 0xb MADV_DONTDUMP = 0x10 MADV_DONTFORK = 0xa MADV_DONTNEED = 0x4 + MADV_DONTNEED_LOCKED = 0x18 MADV_FREE = 0x8 MADV_HUGEPAGE = 0xe MADV_HWPOISON = 0x64 @@ -1846,7 +1861,7 @@ const ( MFD_ALLOW_SEALING = 0x2 MFD_CLOEXEC = 0x1 MFD_HUGETLB = 0x4 - MFD_HUGE_16GB = -0x78000000 + MFD_HUGE_16GB = 0x88000000 MFD_HUGE_16MB = 0x60000000 MFD_HUGE_1GB = 0x78000000 MFD_HUGE_1MB = 0x50000000 @@ -2153,6 +2168,7 @@ const ( PACKET_FANOUT_DATA = 0x16 PACKET_FANOUT_EBPF = 0x7 PACKET_FANOUT_FLAG_DEFRAG = 0x8000 + PACKET_FANOUT_FLAG_IGNORE_OUTGOING = 0x4000 PACKET_FANOUT_FLAG_ROLLOVER = 0x1000 PACKET_FANOUT_FLAG_UNIQUEID = 0x2000 PACKET_FANOUT_HASH = 0x0 @@ -2212,6 +2228,11 @@ const ( PERF_AUX_FLAG_PARTIAL = 0x4 PERF_AUX_FLAG_PMU_FORMAT_TYPE_MASK = 0xff00 PERF_AUX_FLAG_TRUNCATED = 0x1 + PERF_BR_ARM64_DEBUG_DATA = 0x7 + PERF_BR_ARM64_DEBUG_EXIT = 0x5 + PERF_BR_ARM64_DEBUG_HALT = 0x4 + PERF_BR_ARM64_DEBUG_INST = 0x6 + PERF_BR_ARM64_FIQ = 0x3 PERF_FLAG_FD_CLOEXEC = 0x8 PERF_FLAG_FD_NO_GROUP = 0x1 PERF_FLAG_FD_OUTPUT = 0x2 @@ -2232,6 +2253,8 @@ const ( PERF_MEM_LOCK_NA = 0x1 PERF_MEM_LOCK_SHIFT = 0x18 PERF_MEM_LVLNUM_ANY_CACHE = 0xb + PERF_MEM_LVLNUM_CXL = 0x9 + PERF_MEM_LVLNUM_IO = 0xa PERF_MEM_LVLNUM_L1 = 0x1 PERF_MEM_LVLNUM_L2 = 0x2 PERF_MEM_LVLNUM_L3 = 0x3 @@ -2265,6 +2288,7 @@ const ( PERF_MEM_REMOTE_REMOTE = 0x1 PERF_MEM_REMOTE_SHIFT = 0x25 PERF_MEM_SNOOPX_FWD = 0x1 + PERF_MEM_SNOOPX_PEER = 0x2 PERF_MEM_SNOOPX_SHIFT = 0x26 PERF_MEM_SNOOP_HIT = 0x4 PERF_MEM_SNOOP_HITM = 0x10 @@ -2301,7 +2325,6 @@ const ( PERF_SAMPLE_BRANCH_PLM_ALL = 0x7 PERF_SAMPLE_WEIGHT_TYPE = 0x1004000 PIPEFS_MAGIC = 0x50495045 - PPC_CMM_MAGIC = 0xc7571590 PPPIOCGNPMODE = 0xc008744c PPPIOCNEWUNIT = 0xc004743e PRIO_PGRP = 0x1 @@ -2944,6 +2967,7 @@ const ( SOL_TCP = 0x6 SOL_TIPC = 0x10f SOL_TLS = 0x11a + SOL_UDP = 0x11 SOL_X25 = 0x106 SOL_XDP = 0x11b SOMAXCONN = 0x1000 @@ -2999,6 +3023,7 @@ const ( STATX_BLOCKS = 0x400 STATX_BTIME = 0x800 STATX_CTIME = 0x80 + STATX_DIOALIGN = 0x2000 STATX_GID = 0x10 STATX_INO = 0x100 STATX_MNT_ID = 0x1000 @@ -3227,6 +3252,19 @@ const ( TRACEFS_MAGIC = 0x74726163 TS_COMM_LEN = 0x20 UDF_SUPER_MAGIC = 0x15013346 + UDP_CORK = 0x1 + UDP_ENCAP = 0x64 + UDP_ENCAP_ESPINUDP = 0x2 + UDP_ENCAP_ESPINUDP_NON_IKE = 0x1 + UDP_ENCAP_GTP0 = 0x4 + UDP_ENCAP_GTP1U = 0x5 + UDP_ENCAP_L2TPINUDP = 0x3 + UDP_GRO = 0x68 + UDP_NO_CHECK6_RX = 0x66 + UDP_NO_CHECK6_TX = 0x65 + UDP_SEGMENT = 0x67 + UDP_V4_FLOW = 0x2 + UDP_V6_FLOW = 0x6 UMOUNT_NOFOLLOW = 0x8 USBDEVICE_SUPER_MAGIC = 0x9fa2 UTIME_NOW = 0x3fffffff @@ -3392,9 +3430,7 @@ const ( XDP_ZEROCOPY = 0x4 XENFS_SUPER_MAGIC = 0xabba1974 XFS_SUPER_MAGIC = 0x58465342 - Z3FOLD_MAGIC = 0x33 ZONEFS_MAGIC = 0x5a4f4653 - ZSMALLOC_MAGIC = 0x58295829 _HIDIOCGRAWNAME_LEN = 0x80 _HIDIOCGRAWPHYS_LEN = 0x40 _HIDIOCGRAWUNIQ_LEN = 0x40 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go index 36c0dfc..a46df0f 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_386.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_386.go @@ -133,6 +133,7 @@ const ( MEMGETREGIONCOUNT = 0x80044d07 MEMISLOCKED = 0x80084d17 MEMLOCK = 0x40084d05 + MEMREAD = 0xc03c4d1a MEMREADOOB = 0xc00c4d04 MEMSETBADBLOCK = 0x40084d0c MEMUNLOCK = 0x40084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go index 4ff9427..6cd4a3e 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_amd64.go @@ -133,6 +133,7 @@ const ( MEMGETREGIONCOUNT = 0x80044d07 MEMISLOCKED = 0x80084d17 MEMLOCK = 0x40084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc0104d04 MEMSETBADBLOCK = 0x40084d0c MEMUNLOCK = 0x40084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go index 3eaa0fb..c7ebee2 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_arm.go @@ -131,6 +131,7 @@ const ( MEMGETREGIONCOUNT = 0x80044d07 MEMISLOCKED = 0x80084d17 MEMLOCK = 0x40084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc00c4d04 MEMSETBADBLOCK = 0x40084d0c MEMUNLOCK = 0x40084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go index d7995bd..9d5352c 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_arm64.go @@ -134,6 +134,7 @@ const ( MEMGETREGIONCOUNT = 0x80044d07 MEMISLOCKED = 0x80084d17 MEMLOCK = 0x40084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc0104d04 MEMSETBADBLOCK = 0x40084d0c MEMUNLOCK = 0x40084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go index 928e24c..f26a164 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_loong64.go @@ -132,6 +132,7 @@ const ( MEMGETREGIONCOUNT = 0x80044d07 MEMISLOCKED = 0x80084d17 MEMLOCK = 0x40084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc0104d04 MEMSETBADBLOCK = 0x40084d0c MEMUNLOCK = 0x40084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go index 179bffb..890bc3c 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips.go @@ -131,6 +131,7 @@ const ( MEMGETREGIONCOUNT = 0x40044d07 MEMISLOCKED = 0x40084d17 MEMLOCK = 0x80084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc00c4d04 MEMSETBADBLOCK = 0x80084d0c MEMUNLOCK = 0x80084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go index 1fba17b..549f26a 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64.go @@ -131,6 +131,7 @@ const ( MEMGETREGIONCOUNT = 0x40044d07 MEMISLOCKED = 0x40084d17 MEMLOCK = 0x80084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc0104d04 MEMSETBADBLOCK = 0x80084d0c MEMUNLOCK = 0x80084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go index b77dde3..e0365e3 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mips64le.go @@ -131,6 +131,7 @@ const ( MEMGETREGIONCOUNT = 0x40044d07 MEMISLOCKED = 0x40084d17 MEMLOCK = 0x80084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc0104d04 MEMSETBADBLOCK = 0x80084d0c MEMUNLOCK = 0x80084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go index 78c6c75..fdccce1 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_mipsle.go @@ -131,6 +131,7 @@ const ( MEMGETREGIONCOUNT = 0x40044d07 MEMISLOCKED = 0x40084d17 MEMLOCK = 0x80084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc00c4d04 MEMSETBADBLOCK = 0x80084d0c MEMUNLOCK = 0x80084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go index 1c0d31f..b2205c8 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc.go @@ -131,6 +131,7 @@ const ( MEMGETREGIONCOUNT = 0x40044d07 MEMISLOCKED = 0x40084d17 MEMLOCK = 0x80084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc00c4d04 MEMSETBADBLOCK = 0x80084d0c MEMUNLOCK = 0x80084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go index 959dd9b..81aa5ad 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64.go @@ -131,6 +131,7 @@ const ( MEMGETREGIONCOUNT = 0x40044d07 MEMISLOCKED = 0x40084d17 MEMLOCK = 0x80084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc0104d04 MEMSETBADBLOCK = 0x80084d0c MEMUNLOCK = 0x80084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go index 5a873cd..76807a1 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_ppc64le.go @@ -131,6 +131,7 @@ const ( MEMGETREGIONCOUNT = 0x40044d07 MEMISLOCKED = 0x40084d17 MEMLOCK = 0x80084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc0104d04 MEMSETBADBLOCK = 0x80084d0c MEMUNLOCK = 0x80084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go index e336d14..d4a5ab9 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_riscv64.go @@ -131,6 +131,7 @@ const ( MEMGETREGIONCOUNT = 0x80044d07 MEMISLOCKED = 0x80084d17 MEMLOCK = 0x40084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc0104d04 MEMSETBADBLOCK = 0x40084d0c MEMUNLOCK = 0x40084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go b/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go index 390c01d..66e65db 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_s390x.go @@ -131,6 +131,7 @@ const ( MEMGETREGIONCOUNT = 0x80044d07 MEMISLOCKED = 0x80084d17 MEMLOCK = 0x40084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc0104d04 MEMSETBADBLOCK = 0x40084d0c MEMUNLOCK = 0x40084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go b/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go index 98a6e5f..f619252 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_linux_sparc64.go @@ -136,6 +136,7 @@ const ( MEMGETREGIONCOUNT = 0x40044d07 MEMISLOCKED = 0x40084d17 MEMLOCK = 0x80084d05 + MEMREAD = 0xc0404d1a MEMREADOOB = 0xc0104d04 MEMSETBADBLOCK = 0x80084d0c MEMUNLOCK = 0x80084d06 diff --git a/vendor/golang.org/x/sys/unix/zerrors_openbsd_386.go b/vendor/golang.org/x/sys/unix/zerrors_openbsd_386.go index 6d56edc..af20e47 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_openbsd_386.go +++ b/vendor/golang.org/x/sys/unix/zerrors_openbsd_386.go @@ -46,6 +46,7 @@ const ( AF_SNA = 0xb AF_UNIX = 0x1 AF_UNSPEC = 0x0 + ALTWERASE = 0x200 ARPHRD_ETHER = 0x1 ARPHRD_FRELAY = 0xf ARPHRD_IEEE1394 = 0x18 @@ -108,6 +109,15 @@ const ( BPF_DIRECTION_IN = 0x1 BPF_DIRECTION_OUT = 0x2 BPF_DIV = 0x30 + BPF_FILDROP_CAPTURE = 0x1 + BPF_FILDROP_DROP = 0x2 + BPF_FILDROP_PASS = 0x0 + BPF_F_DIR_IN = 0x10 + BPF_F_DIR_MASK = 0x30 + BPF_F_DIR_OUT = 0x20 + BPF_F_DIR_SHIFT = 0x4 + BPF_F_FLOWID = 0x8 + BPF_F_PRI_MASK = 0x7 BPF_H = 0x8 BPF_IMM = 0x0 BPF_IND = 0x40 @@ -136,6 +146,7 @@ const ( BPF_OR = 0x40 BPF_RELEASE = 0x30bb6 BPF_RET = 0x6 + BPF_RND = 0xc0 BPF_RSH = 0x70 BPF_ST = 0x2 BPF_STX = 0x3 @@ -147,6 +158,12 @@ const ( BRKINT = 0x2 CFLUSH = 0xf CLOCAL = 0x8000 + CLOCK_BOOTTIME = 0x6 + CLOCK_MONOTONIC = 0x3 + CLOCK_PROCESS_CPUTIME_ID = 0x2 + CLOCK_REALTIME = 0x0 + CLOCK_THREAD_CPUTIME_ID = 0x4 + CLOCK_UPTIME = 0x5 CPUSTATES = 0x6 CP_IDLE = 0x5 CP_INTR = 0x4 @@ -170,7 +187,65 @@ const ( CTL_KERN = 0x1 CTL_MAXNAME = 0xc CTL_NET = 0x4 + DIOCADDQUEUE = 0xc100445d + DIOCADDRULE = 0xccc84404 + DIOCADDSTATE = 0xc1084425 + DIOCCHANGERULE = 0xccc8441a + DIOCCLRIFFLAG = 0xc024445a + DIOCCLRSRCNODES = 0x20004455 + DIOCCLRSTATES = 0xc0d04412 + DIOCCLRSTATUS = 0xc0244416 + DIOCGETLIMIT = 0xc0084427 + DIOCGETQSTATS = 0xc1084460 + DIOCGETQUEUE = 0xc100445f + DIOCGETQUEUES = 0xc100445e + DIOCGETRULE = 0xccc84407 + DIOCGETRULES = 0xccc84406 + DIOCGETRULESET = 0xc444443b + DIOCGETRULESETS = 0xc444443a + DIOCGETSRCNODES = 0xc0084454 + DIOCGETSTATE = 0xc1084413 + DIOCGETSTATES = 0xc0084419 + DIOCGETSTATUS = 0xc1e84415 + DIOCGETSYNFLWATS = 0xc0084463 + DIOCGETTIMEOUT = 0xc008441e + DIOCIGETIFACES = 0xc0244457 + DIOCKILLSRCNODES = 0xc068445b + DIOCKILLSTATES = 0xc0d04429 + DIOCNATLOOK = 0xc0504417 + DIOCOSFPADD = 0xc084444f DIOCOSFPFLUSH = 0x2000444e + DIOCOSFPGET = 0xc0844450 + DIOCRADDADDRS = 0xc44c4443 + DIOCRADDTABLES = 0xc44c443d + DIOCRCLRADDRS = 0xc44c4442 + DIOCRCLRASTATS = 0xc44c4448 + DIOCRCLRTABLES = 0xc44c443c + DIOCRCLRTSTATS = 0xc44c4441 + DIOCRDELADDRS = 0xc44c4444 + DIOCRDELTABLES = 0xc44c443e + DIOCRGETADDRS = 0xc44c4446 + DIOCRGETASTATS = 0xc44c4447 + DIOCRGETTABLES = 0xc44c443f + DIOCRGETTSTATS = 0xc44c4440 + DIOCRINADEFINE = 0xc44c444d + DIOCRSETADDRS = 0xc44c4445 + DIOCRSETTFLAGS = 0xc44c444a + DIOCRTSTADDRS = 0xc44c4449 + DIOCSETDEBUG = 0xc0044418 + DIOCSETHOSTID = 0xc0044456 + DIOCSETIFFLAG = 0xc0244459 + DIOCSETLIMIT = 0xc0084428 + DIOCSETREASS = 0xc004445c + DIOCSETSTATUSIF = 0xc0244414 + DIOCSETSYNCOOKIES = 0xc0014462 + DIOCSETSYNFLWATS = 0xc0084461 + DIOCSETTIMEOUT = 0xc008441d + DIOCSTART = 0x20004401 + DIOCSTOP = 0x20004402 + DIOCXBEGIN = 0xc00c4451 + DIOCXCOMMIT = 0xc00c4452 + DIOCXROLLBACK = 0xc00c4453 DLT_ARCNET = 0x7 DLT_ATM_RFC1483 = 0xb DLT_AX25 = 0x3 @@ -186,6 +261,7 @@ const ( DLT_LOOP = 0xc DLT_MPLS = 0xdb DLT_NULL = 0x0 + DLT_OPENFLOW = 0x10b DLT_PFLOG = 0x75 DLT_PFSYNC = 0x12 DLT_PPP = 0x9 @@ -196,6 +272,23 @@ const ( DLT_RAW = 0xe DLT_SLIP = 0x8 DLT_SLIP_BSDOS = 0xf + DLT_USBPCAP = 0xf9 + DLT_USER0 = 0x93 + DLT_USER1 = 0x94 + DLT_USER10 = 0x9d + DLT_USER11 = 0x9e + DLT_USER12 = 0x9f + DLT_USER13 = 0xa0 + DLT_USER14 = 0xa1 + DLT_USER15 = 0xa2 + DLT_USER2 = 0x95 + DLT_USER3 = 0x96 + DLT_USER4 = 0x97 + DLT_USER5 = 0x98 + DLT_USER6 = 0x99 + DLT_USER7 = 0x9a + DLT_USER8 = 0x9b + DLT_USER9 = 0x9c DT_BLK = 0x6 DT_CHR = 0x2 DT_DIR = 0x4 @@ -215,6 +308,8 @@ const ( EMUL_ENABLED = 0x1 EMUL_NATIVE = 0x2 ENDRUNDISC = 0x9 + ETH64_8021_RSVD_MASK = 0xfffffffffff0 + ETH64_8021_RSVD_PREFIX = 0x180c2000000 ETHERMIN = 0x2e ETHERMTU = 0x5dc ETHERTYPE_8023 = 0x4 @@ -267,6 +362,7 @@ const ( ETHERTYPE_DN = 0x6003 ETHERTYPE_DOGFIGHT = 0x1989 ETHERTYPE_DSMD = 0x8039 + ETHERTYPE_EAPOL = 0x888e ETHERTYPE_ECMA = 0x803 ETHERTYPE_ENCRYPT = 0x803d ETHERTYPE_ES = 0x805d @@ -298,6 +394,7 @@ const ( ETHERTYPE_LLDP = 0x88cc ETHERTYPE_LOGICRAFT = 0x8148 ETHERTYPE_LOOPBACK = 0x9000 + ETHERTYPE_MACSEC = 0x88e5 ETHERTYPE_MATRA = 0x807a ETHERTYPE_MAX = 0xffff ETHERTYPE_MERIT = 0x807c @@ -326,15 +423,17 @@ const ( ETHERTYPE_NCD = 0x8149 ETHERTYPE_NESTAR = 0x8006 ETHERTYPE_NETBEUI = 0x8191 + ETHERTYPE_NHRP = 0x2001 ETHERTYPE_NOVELL = 0x8138 ETHERTYPE_NS = 0x600 ETHERTYPE_NSAT = 0x601 ETHERTYPE_NSCOMPAT = 0x807 + ETHERTYPE_NSH = 0x984f ETHERTYPE_NTRAILER = 0x10 ETHERTYPE_OS9 = 0x7007 ETHERTYPE_OS9NET = 0x7009 ETHERTYPE_PACER = 0x80c6 - ETHERTYPE_PAE = 0x888e + ETHERTYPE_PBB = 0x88e7 ETHERTYPE_PCS = 0x4242 ETHERTYPE_PLANNING = 0x8044 ETHERTYPE_PPP = 0x880b @@ -409,28 +508,40 @@ const ( ETHER_CRC_POLY_LE = 0xedb88320 ETHER_HDR_LEN = 0xe ETHER_MAX_DIX_LEN = 0x600 + ETHER_MAX_HARDMTU_LEN = 0xff9b ETHER_MAX_LEN = 0x5ee ETHER_MIN_LEN = 0x40 ETHER_TYPE_LEN = 0x2 ETHER_VLAN_ENCAP_LEN = 0x4 EVFILT_AIO = -0x3 + EVFILT_DEVICE = -0x8 + EVFILT_EXCEPT = -0x9 EVFILT_PROC = -0x5 EVFILT_READ = -0x1 EVFILT_SIGNAL = -0x6 - EVFILT_SYSCOUNT = 0x7 + EVFILT_SYSCOUNT = 0x9 EVFILT_TIMER = -0x7 EVFILT_VNODE = -0x4 EVFILT_WRITE = -0x2 + EVL_ENCAPLEN = 0x4 + EVL_PRIO_BITS = 0xd + EVL_PRIO_MAX = 0x7 + EVL_VLID_MASK = 0xfff + EVL_VLID_MAX = 0xffe + EVL_VLID_MIN = 0x1 + EVL_VLID_NULL = 0x0 EV_ADD = 0x1 EV_CLEAR = 0x20 EV_DELETE = 0x2 EV_DISABLE = 0x8 + EV_DISPATCH = 0x80 EV_ENABLE = 0x4 EV_EOF = 0x8000 EV_ERROR = 0x4000 EV_FLAG1 = 0x2000 EV_ONESHOT = 0x10 - EV_SYSFLAGS = 0xf000 + EV_RECEIPT = 0x40 + EV_SYSFLAGS = 0xf800 EXTA = 0x4b00 EXTB = 0x9600 EXTPROC = 0x800 @@ -443,6 +554,7 @@ const ( F_GETFL = 0x3 F_GETLK = 0x7 F_GETOWN = 0x5 + F_ISATTY = 0xb F_OK = 0x0 F_RDLCK = 0x1 F_SETFD = 0x2 @@ -460,7 +572,6 @@ const ( IEXTEN = 0x400 IFAN_ARRIVAL = 0x0 IFAN_DEPARTURE = 0x1 - IFA_ROUTE = 0x1 IFF_ALLMULTI = 0x200 IFF_BROADCAST = 0x2 IFF_CANTCHANGE = 0x8e52 @@ -471,12 +582,12 @@ const ( IFF_LOOPBACK = 0x8 IFF_MULTICAST = 0x8000 IFF_NOARP = 0x80 - IFF_NOTRAILERS = 0x20 IFF_OACTIVE = 0x400 IFF_POINTOPOINT = 0x10 IFF_PROMISC = 0x100 IFF_RUNNING = 0x40 IFF_SIMPLEX = 0x800 + IFF_STATICARP = 0x20 IFF_UP = 0x1 IFNAMSIZ = 0x10 IFT_1822 = 0x2 @@ -605,6 +716,7 @@ const ( IFT_LINEGROUP = 0xd2 IFT_LOCALTALK = 0x2a IFT_LOOP = 0x18 + IFT_MBIM = 0xfa IFT_MEDIAMAILOVERIP = 0x8b IFT_MFSIGLINK = 0xa7 IFT_MIOX25 = 0x26 @@ -695,6 +807,7 @@ const ( IFT_VOICEOVERCABLE = 0xc6 IFT_VOICEOVERFRAMERELAY = 0x99 IFT_VOICEOVERIP = 0x68 + IFT_WIREGUARD = 0xfb IFT_X213 = 0x5d IFT_X25 = 0x5 IFT_X25DDN = 0x4 @@ -729,8 +842,6 @@ const ( IPPROTO_AH = 0x33 IPPROTO_CARP = 0x70 IPPROTO_DIVERT = 0x102 - IPPROTO_DIVERT_INIT = 0x2 - IPPROTO_DIVERT_RESP = 0x1 IPPROTO_DONE = 0x101 IPPROTO_DSTOPTS = 0x3c IPPROTO_EGP = 0x8 @@ -762,9 +873,11 @@ const ( IPPROTO_RAW = 0xff IPPROTO_ROUTING = 0x2b IPPROTO_RSVP = 0x2e + IPPROTO_SCTP = 0x84 IPPROTO_TCP = 0x6 IPPROTO_TP = 0x1d IPPROTO_UDP = 0x11 + IPPROTO_UDPLITE = 0x88 IPV6_AUTH_LEVEL = 0x35 IPV6_AUTOFLOWLABEL = 0x3b IPV6_CHECKSUM = 0x1a @@ -787,6 +900,7 @@ const ( IPV6_LEAVE_GROUP = 0xd IPV6_MAXHLIM = 0xff IPV6_MAXPACKET = 0xffff + IPV6_MINHOPCOUNT = 0x41 IPV6_MMTU = 0x500 IPV6_MULTICAST_HOPS = 0xa IPV6_MULTICAST_IF = 0x9 @@ -826,12 +940,12 @@ const ( IP_DEFAULT_MULTICAST_LOOP = 0x1 IP_DEFAULT_MULTICAST_TTL = 0x1 IP_DF = 0x4000 - IP_DIVERTFL = 0x1022 IP_DROP_MEMBERSHIP = 0xd IP_ESP_NETWORK_LEVEL = 0x16 IP_ESP_TRANS_LEVEL = 0x15 IP_HDRINCL = 0x2 IP_IPCOMP_LEVEL = 0x1d + IP_IPDEFTTL = 0x25 IP_IPSECFLOWINFO = 0x24 IP_IPSEC_LOCAL_AUTH = 0x1b IP_IPSEC_LOCAL_CRED = 0x19 @@ -865,10 +979,15 @@ const ( IP_RETOPTS = 0x8 IP_RF = 0x8000 IP_RTABLE = 0x1021 + IP_SENDSRCADDR = 0x7 IP_TOS = 0x3 IP_TTL = 0x4 ISIG = 0x80 ISTRIP = 0x20 + ITIMER_PROF = 0x2 + ITIMER_REAL = 0x0 + ITIMER_VIRTUAL = 0x1 + IUCLC = 0x1000 IXANY = 0x800 IXOFF = 0x400 IXON = 0x200 @@ -900,10 +1019,11 @@ const ( MAP_INHERIT_COPY = 0x1 MAP_INHERIT_NONE = 0x2 MAP_INHERIT_SHARE = 0x0 - MAP_NOEXTEND = 0x100 - MAP_NORESERVE = 0x40 + MAP_INHERIT_ZERO = 0x3 + MAP_NOEXTEND = 0x0 + MAP_NORESERVE = 0x0 MAP_PRIVATE = 0x2 - MAP_RENAME = 0x20 + MAP_RENAME = 0x0 MAP_SHARED = 0x1 MAP_STACK = 0x4000 MAP_TRYFIXED = 0x0 @@ -922,6 +1042,7 @@ const ( MNT_NOATIME = 0x8000 MNT_NODEV = 0x10 MNT_NOEXEC = 0x4 + MNT_NOPERM = 0x20 MNT_NOSUID = 0x8 MNT_NOWAIT = 0x2 MNT_QUOTA = 0x2000 @@ -929,13 +1050,29 @@ const ( MNT_RELOAD = 0x40000 MNT_ROOTFS = 0x4000 MNT_SOFTDEP = 0x4000000 + MNT_STALLED = 0x100000 + MNT_SWAPPABLE = 0x200000 MNT_SYNCHRONOUS = 0x2 MNT_UPDATE = 0x10000 MNT_VISFLAGMASK = 0x400ffff MNT_WAIT = 0x1 MNT_WANTRDWR = 0x2000000 MNT_WXALLOWED = 0x800 + MOUNT_AFS = "afs" + MOUNT_CD9660 = "cd9660" + MOUNT_EXT2FS = "ext2fs" + MOUNT_FFS = "ffs" + MOUNT_FUSEFS = "fuse" + MOUNT_MFS = "mfs" + MOUNT_MSDOS = "msdos" + MOUNT_NCPFS = "ncpfs" + MOUNT_NFS = "nfs" + MOUNT_NTFS = "ntfs" + MOUNT_TMPFS = "tmpfs" + MOUNT_UDF = "udf" + MOUNT_UFS = "ffs" MSG_BCAST = 0x100 + MSG_CMSG_CLOEXEC = 0x800 MSG_CTRUNC = 0x20 MSG_DONTROUTE = 0x4 MSG_DONTWAIT = 0x80 @@ -946,6 +1083,7 @@ const ( MSG_PEEK = 0x2 MSG_TRUNC = 0x10 MSG_WAITALL = 0x40 + MSG_WAITFORONE = 0x1000 MS_ASYNC = 0x1 MS_INVALIDATE = 0x4 MS_SYNC = 0x2 @@ -953,12 +1091,16 @@ const ( NET_RT_DUMP = 0x1 NET_RT_FLAGS = 0x2 NET_RT_IFLIST = 0x3 - NET_RT_MAXID = 0x6 + NET_RT_IFNAMES = 0x6 + NET_RT_MAXID = 0x8 + NET_RT_SOURCE = 0x7 NET_RT_STATS = 0x4 NET_RT_TABLE = 0x5 NFDBITS = 0x20 NOFLSH = 0x80000000 + NOKERNINFO = 0x2000000 NOTE_ATTRIB = 0x8 + NOTE_CHANGE = 0x1 NOTE_CHILD = 0x4 NOTE_DELETE = 0x1 NOTE_EOF = 0x2 @@ -968,6 +1110,7 @@ const ( NOTE_FORK = 0x40000000 NOTE_LINK = 0x10 NOTE_LOWAT = 0x1 + NOTE_OOB = 0x4 NOTE_PCTRLMASK = 0xf0000000 NOTE_PDATAMASK = 0xfffff NOTE_RENAME = 0x20 @@ -977,11 +1120,13 @@ const ( NOTE_TRUNCATE = 0x80 NOTE_WRITE = 0x2 OCRNL = 0x10 + OLCUC = 0x20 ONLCR = 0x2 ONLRET = 0x80 ONOCR = 0x40 ONOEOT = 0x8 OPOST = 0x1 + OXTABS = 0x4 O_ACCMODE = 0x3 O_APPEND = 0x8 O_ASYNC = 0x40 @@ -1015,7 +1160,6 @@ const ( PROT_NONE = 0x0 PROT_READ = 0x1 PROT_WRITE = 0x2 - PT_MASK = 0x3ff000 RLIMIT_CORE = 0x4 RLIMIT_CPU = 0x0 RLIMIT_DATA = 0x2 @@ -1027,19 +1171,25 @@ const ( RLIMIT_STACK = 0x3 RLIM_INFINITY = 0x7fffffffffffffff RTAX_AUTHOR = 0x6 + RTAX_BFD = 0xb RTAX_BRD = 0x7 + RTAX_DNS = 0xc RTAX_DST = 0x0 RTAX_GATEWAY = 0x1 RTAX_GENMASK = 0x3 RTAX_IFA = 0x5 RTAX_IFP = 0x4 RTAX_LABEL = 0xa - RTAX_MAX = 0xb + RTAX_MAX = 0xf RTAX_NETMASK = 0x2 + RTAX_SEARCH = 0xe RTAX_SRC = 0x8 RTAX_SRCMASK = 0x9 + RTAX_STATIC = 0xd RTA_AUTHOR = 0x40 + RTA_BFD = 0x800 RTA_BRD = 0x80 + RTA_DNS = 0x1000 RTA_DST = 0x1 RTA_GATEWAY = 0x2 RTA_GENMASK = 0x8 @@ -1047,49 +1197,57 @@ const ( RTA_IFP = 0x10 RTA_LABEL = 0x400 RTA_NETMASK = 0x4 + RTA_SEARCH = 0x4000 RTA_SRC = 0x100 RTA_SRCMASK = 0x200 + RTA_STATIC = 0x2000 RTF_ANNOUNCE = 0x4000 + RTF_BFD = 0x1000000 RTF_BLACKHOLE = 0x1000 + RTF_BROADCAST = 0x400000 + RTF_CACHED = 0x20000 RTF_CLONED = 0x10000 RTF_CLONING = 0x100 + RTF_CONNECTED = 0x800000 RTF_DONE = 0x40 RTF_DYNAMIC = 0x10 - RTF_FMASK = 0x10f808 + RTF_FMASK = 0x110fc08 RTF_GATEWAY = 0x2 RTF_HOST = 0x4 RTF_LLINFO = 0x400 - RTF_MASK = 0x80 + RTF_LOCAL = 0x200000 RTF_MODIFIED = 0x20 RTF_MPATH = 0x40000 RTF_MPLS = 0x100000 + RTF_MULTICAST = 0x200 RTF_PERMANENT_ARP = 0x2000 RTF_PROTO1 = 0x8000 RTF_PROTO2 = 0x4000 RTF_PROTO3 = 0x2000 RTF_REJECT = 0x8 - RTF_SOURCE = 0x20000 RTF_STATIC = 0x800 - RTF_TUNNEL = 0x100000 RTF_UP = 0x1 RTF_USETRAILERS = 0x8000 - RTF_XRESOLVE = 0x200 + RTM_80211INFO = 0x15 RTM_ADD = 0x1 + RTM_BFD = 0x12 RTM_CHANGE = 0x3 + RTM_CHGADDRATTR = 0x14 RTM_DELADDR = 0xd RTM_DELETE = 0x2 RTM_DESYNC = 0x10 RTM_GET = 0x4 RTM_IFANNOUNCE = 0xf RTM_IFINFO = 0xe - RTM_LOCK = 0x8 + RTM_INVALIDATE = 0x11 RTM_LOSING = 0x5 RTM_MAXSIZE = 0x800 RTM_MISS = 0x7 RTM_NEWADDR = 0xc + RTM_PROPOSAL = 0x13 RTM_REDIRECT = 0x6 RTM_RESOLVE = 0xb - RTM_RTTUNIT = 0xf4240 + RTM_SOURCE = 0x16 RTM_VERSION = 0x5 RTV_EXPIRE = 0x4 RTV_HOPCOUNT = 0x2 @@ -1099,67 +1257,74 @@ const ( RTV_RTTVAR = 0x80 RTV_SPIPE = 0x10 RTV_SSTHRESH = 0x20 + RT_TABLEID_BITS = 0x8 + RT_TABLEID_MASK = 0xff RT_TABLEID_MAX = 0xff RUSAGE_CHILDREN = -0x1 RUSAGE_SELF = 0x0 RUSAGE_THREAD = 0x1 SCM_RIGHTS = 0x1 SCM_TIMESTAMP = 0x4 + SEEK_CUR = 0x1 + SEEK_END = 0x2 + SEEK_SET = 0x0 SHUT_RD = 0x0 SHUT_RDWR = 0x2 SHUT_WR = 0x1 SIOCADDMULTI = 0x80206931 SIOCAIFADDR = 0x8040691a SIOCAIFGROUP = 0x80246987 - SIOCALIFADDR = 0x8218691c SIOCATMARK = 0x40047307 - SIOCBRDGADD = 0x8054693c - SIOCBRDGADDS = 0x80546941 - SIOCBRDGARL = 0x806e694d + SIOCBRDGADD = 0x805c693c + SIOCBRDGADDL = 0x805c6949 + SIOCBRDGADDS = 0x805c6941 + SIOCBRDGARL = 0x808c694d SIOCBRDGDADDR = 0x81286947 - SIOCBRDGDEL = 0x8054693d - SIOCBRDGDELS = 0x80546942 - SIOCBRDGFLUSH = 0x80546948 - SIOCBRDGFRL = 0x806e694e + SIOCBRDGDEL = 0x805c693d + SIOCBRDGDELS = 0x805c6942 + SIOCBRDGFLUSH = 0x805c6948 + SIOCBRDGFRL = 0x808c694e SIOCBRDGGCACHE = 0xc0146941 SIOCBRDGGFD = 0xc0146952 SIOCBRDGGHT = 0xc0146951 - SIOCBRDGGIFFLGS = 0xc054693e + SIOCBRDGGIFFLGS = 0xc05c693e SIOCBRDGGMA = 0xc0146953 SIOCBRDGGPARAM = 0xc03c6958 SIOCBRDGGPRI = 0xc0146950 SIOCBRDGGRL = 0xc028694f - SIOCBRDGGSIFS = 0xc054693c SIOCBRDGGTO = 0xc0146946 - SIOCBRDGIFS = 0xc0546942 + SIOCBRDGIFS = 0xc05c6942 SIOCBRDGRTS = 0xc0186943 SIOCBRDGSADDR = 0xc1286944 SIOCBRDGSCACHE = 0x80146940 SIOCBRDGSFD = 0x80146952 SIOCBRDGSHT = 0x80146951 - SIOCBRDGSIFCOST = 0x80546955 - SIOCBRDGSIFFLGS = 0x8054693f - SIOCBRDGSIFPRIO = 0x80546954 + SIOCBRDGSIFCOST = 0x805c6955 + SIOCBRDGSIFFLGS = 0x805c693f + SIOCBRDGSIFPRIO = 0x805c6954 + SIOCBRDGSIFPROT = 0x805c694a SIOCBRDGSMA = 0x80146953 SIOCBRDGSPRI = 0x80146950 SIOCBRDGSPROTO = 0x8014695a SIOCBRDGSTO = 0x80146945 SIOCBRDGSTXHC = 0x80146959 + SIOCDELLABEL = 0x80206997 SIOCDELMULTI = 0x80206932 SIOCDIFADDR = 0x80206919 SIOCDIFGROUP = 0x80246989 + SIOCDIFPARENT = 0x802069b4 SIOCDIFPHYADDR = 0x80206949 - SIOCDLIFADDR = 0x8218691e + SIOCDPWE3NEIGHBOR = 0x802069de + SIOCDVNETID = 0x802069af SIOCGETKALIVE = 0xc01869a4 SIOCGETLABEL = 0x8020699a + SIOCGETMPWCFG = 0xc02069ae SIOCGETPFLOW = 0xc02069fe SIOCGETPFSYNC = 0xc02069f8 SIOCGETSGCNT = 0xc0147534 SIOCGETVIFCNT = 0xc0147533 SIOCGETVLAN = 0xc0206990 - SIOCGHIWAT = 0x40047301 SIOCGIFADDR = 0xc0206921 - SIOCGIFASYNCMAP = 0xc020697c SIOCGIFBRDADDR = 0xc0206923 SIOCGIFCONF = 0xc0086924 SIOCGIFDATA = 0xc020691b @@ -1168,40 +1333,53 @@ const ( SIOCGIFFLAGS = 0xc0206911 SIOCGIFGATTR = 0xc024698b SIOCGIFGENERIC = 0xc020693a + SIOCGIFGLIST = 0xc024698d SIOCGIFGMEMB = 0xc024698a SIOCGIFGROUP = 0xc0246988 SIOCGIFHARDMTU = 0xc02069a5 - SIOCGIFMEDIA = 0xc0286936 + SIOCGIFLLPRIO = 0xc02069b6 + SIOCGIFMEDIA = 0xc0386938 SIOCGIFMETRIC = 0xc0206917 SIOCGIFMTU = 0xc020697e SIOCGIFNETMASK = 0xc0206925 - SIOCGIFPDSTADDR = 0xc0206948 + SIOCGIFPAIR = 0xc02069b1 + SIOCGIFPARENT = 0xc02069b3 SIOCGIFPRIORITY = 0xc020699c - SIOCGIFPSRCADDR = 0xc0206947 SIOCGIFRDOMAIN = 0xc02069a0 SIOCGIFRTLABEL = 0xc0206983 - SIOCGIFTIMESLOT = 0xc0206986 + SIOCGIFRXR = 0x802069aa + SIOCGIFSFFPAGE = 0xc1126939 SIOCGIFXFLAGS = 0xc020699e - SIOCGLIFADDR = 0xc218691d SIOCGLIFPHYADDR = 0xc218694b + SIOCGLIFPHYDF = 0xc02069c2 + SIOCGLIFPHYECN = 0xc02069c8 SIOCGLIFPHYRTABLE = 0xc02069a2 SIOCGLIFPHYTTL = 0xc02069a9 - SIOCGLOWAT = 0x40047303 SIOCGPGRP = 0x40047309 + SIOCGPWE3 = 0xc0206998 + SIOCGPWE3CTRLWORD = 0xc02069dc + SIOCGPWE3FAT = 0xc02069dd + SIOCGPWE3NEIGHBOR = 0xc21869de + SIOCGRXHPRIO = 0xc02069db SIOCGSPPPPARAMS = 0xc0206994 + SIOCGTXHPRIO = 0xc02069c6 + SIOCGUMBINFO = 0xc02069be + SIOCGUMBPARAM = 0xc02069c0 SIOCGVH = 0xc02069f6 + SIOCGVNETFLOWID = 0xc02069c4 SIOCGVNETID = 0xc02069a7 + SIOCIFAFATTACH = 0x801169ab + SIOCIFAFDETACH = 0x801169ac SIOCIFCREATE = 0x8020697a SIOCIFDESTROY = 0x80206979 SIOCIFGCLONERS = 0xc00c6978 SIOCSETKALIVE = 0x801869a3 SIOCSETLABEL = 0x80206999 + SIOCSETMPWCFG = 0x802069ad SIOCSETPFLOW = 0x802069fd SIOCSETPFSYNC = 0x802069f7 SIOCSETVLAN = 0x8020698f - SIOCSHIWAT = 0x80047300 SIOCSIFADDR = 0x8020690c - SIOCSIFASYNCMAP = 0x8020697d SIOCSIFBRDADDR = 0x80206913 SIOCSIFDESCR = 0x80206980 SIOCSIFDSTADDR = 0x8020690e @@ -1209,25 +1387,37 @@ const ( SIOCSIFGATTR = 0x8024698c SIOCSIFGENERIC = 0x80206939 SIOCSIFLLADDR = 0x8020691f - SIOCSIFMEDIA = 0xc0206935 + SIOCSIFLLPRIO = 0x802069b5 + SIOCSIFMEDIA = 0xc0206937 SIOCSIFMETRIC = 0x80206918 SIOCSIFMTU = 0x8020697f SIOCSIFNETMASK = 0x80206916 - SIOCSIFPHYADDR = 0x80406946 + SIOCSIFPAIR = 0x802069b0 + SIOCSIFPARENT = 0x802069b2 SIOCSIFPRIORITY = 0x8020699b SIOCSIFRDOMAIN = 0x8020699f SIOCSIFRTLABEL = 0x80206982 - SIOCSIFTIMESLOT = 0x80206985 SIOCSIFXFLAGS = 0x8020699d SIOCSLIFPHYADDR = 0x8218694a + SIOCSLIFPHYDF = 0x802069c1 + SIOCSLIFPHYECN = 0x802069c7 SIOCSLIFPHYRTABLE = 0x802069a1 SIOCSLIFPHYTTL = 0x802069a8 - SIOCSLOWAT = 0x80047302 SIOCSPGRP = 0x80047308 + SIOCSPWE3CTRLWORD = 0x802069dc + SIOCSPWE3FAT = 0x802069dd + SIOCSPWE3NEIGHBOR = 0x821869de + SIOCSRXHPRIO = 0x802069db SIOCSSPPPPARAMS = 0x80206993 + SIOCSTXHPRIO = 0x802069c5 + SIOCSUMBPARAM = 0x802069bf SIOCSVH = 0xc02069f5 + SIOCSVNETFLOWID = 0x802069c3 SIOCSVNETID = 0x802069a6 + SOCK_CLOEXEC = 0x8000 SOCK_DGRAM = 0x2 + SOCK_DNS = 0x1000 + SOCK_NONBLOCK = 0x4000 SOCK_RAW = 0x3 SOCK_RDM = 0x4 SOCK_SEQPACKET = 0x5 @@ -1238,6 +1428,7 @@ const ( SO_BINDANY = 0x1000 SO_BROADCAST = 0x20 SO_DEBUG = 0x1 + SO_DOMAIN = 0x1024 SO_DONTROUTE = 0x10 SO_ERROR = 0x1007 SO_KEEPALIVE = 0x8 @@ -1245,6 +1436,7 @@ const ( SO_NETPROC = 0x1020 SO_OOBINLINE = 0x100 SO_PEERCRED = 0x1022 + SO_PROTOCOL = 0x1025 SO_RCVBUF = 0x1002 SO_RCVLOWAT = 0x1004 SO_RCVTIMEO = 0x1006 @@ -1258,6 +1450,7 @@ const ( SO_TIMESTAMP = 0x800 SO_TYPE = 0x1008 SO_USELOOPBACK = 0x40 + SO_ZEROIZE = 0x2000 S_BLKSIZE = 0x200 S_IEXEC = 0x40 S_IFBLK = 0x6000 @@ -1287,9 +1480,24 @@ const ( S_IXOTH = 0x1 S_IXUSR = 0x40 TCIFLUSH = 0x1 + TCIOFF = 0x3 TCIOFLUSH = 0x3 + TCION = 0x4 TCOFLUSH = 0x2 - TCP_MAXBURST = 0x4 + TCOOFF = 0x1 + TCOON = 0x2 + TCPOPT_EOL = 0x0 + TCPOPT_MAXSEG = 0x2 + TCPOPT_NOP = 0x1 + TCPOPT_SACK = 0x5 + TCPOPT_SACK_HDR = 0x1010500 + TCPOPT_SACK_PERMITTED = 0x4 + TCPOPT_SACK_PERMIT_HDR = 0x1010402 + TCPOPT_SIGNATURE = 0x13 + TCPOPT_TIMESTAMP = 0x8 + TCPOPT_TSTAMP_HDR = 0x101080a + TCPOPT_WINDOW = 0x3 + TCP_INFO = 0x9 TCP_MAXSEG = 0x2 TCP_MAXWIN = 0xffff TCP_MAX_SACK = 0x3 @@ -1298,11 +1506,15 @@ const ( TCP_MSS = 0x200 TCP_NODELAY = 0x1 TCP_NOPUSH = 0x10 - TCP_NSTATES = 0xb + TCP_SACKHOLE_LIMIT = 0x80 TCP_SACK_ENABLE = 0x8 TCSAFLUSH = 0x2 + TIMER_ABSTIME = 0x1 + TIMER_RELTIME = 0x0 TIOCCBRK = 0x2000747a TIOCCDTR = 0x20007478 + TIOCCHKVERAUTH = 0x2000741e + TIOCCLRVERAUTH = 0x2000741d TIOCCONS = 0x80047462 TIOCDRAIN = 0x2000745e TIOCEXCL = 0x2000740d @@ -1357,17 +1569,21 @@ const ( TIOCSETAF = 0x802c7416 TIOCSETAW = 0x802c7415 TIOCSETD = 0x8004741b + TIOCSETVERAUTH = 0x8004741c TIOCSFLAGS = 0x8004745c TIOCSIG = 0x8004745f TIOCSPGRP = 0x80047476 TIOCSTART = 0x2000746e - TIOCSTAT = 0x80047465 - TIOCSTI = 0x80017472 + TIOCSTAT = 0x20007465 TIOCSTOP = 0x2000746f TIOCSTSTAMP = 0x8008745a TIOCSWINSZ = 0x80087467 TIOCUCNTL = 0x80047466 + TIOCUCNTL_CBRK = 0x7a + TIOCUCNTL_SBRK = 0x7b TOSTOP = 0x400000 + UTIME_NOW = -0x2 + UTIME_OMIT = -0x1 VDISCARD = 0xf VDSUSP = 0xb VEOF = 0x0 @@ -1378,6 +1594,19 @@ const ( VKILL = 0x5 VLNEXT = 0xe VMIN = 0x10 + VM_ANONMIN = 0x7 + VM_LOADAVG = 0x2 + VM_MALLOC_CONF = 0xc + VM_MAXID = 0xd + VM_MAXSLP = 0xa + VM_METER = 0x1 + VM_NKMEMPAGES = 0x6 + VM_PSSTRINGS = 0x3 + VM_SWAPENCRYPT = 0x5 + VM_USPACE = 0xb + VM_UVMEXP = 0x4 + VM_VNODEMIN = 0x9 + VM_VTEXTMIN = 0x8 VQUIT = 0x9 VREPRINT = 0x6 VSTART = 0xc @@ -1390,8 +1619,8 @@ const ( WCONTINUED = 0x8 WCOREFLAG = 0x80 WNOHANG = 0x1 - WSTOPPED = 0x7f WUNTRACED = 0x2 + XCASE = 0x1000000 ) // Errors @@ -1405,6 +1634,7 @@ const ( EALREADY = syscall.Errno(0x25) EAUTH = syscall.Errno(0x50) EBADF = syscall.Errno(0x9) + EBADMSG = syscall.Errno(0x5c) EBADRPC = syscall.Errno(0x48) EBUSY = syscall.Errno(0x10) ECANCELED = syscall.Errno(0x58) @@ -1431,7 +1661,7 @@ const ( EIPSEC = syscall.Errno(0x52) EISCONN = syscall.Errno(0x38) EISDIR = syscall.Errno(0x15) - ELAST = syscall.Errno(0x5b) + ELAST = syscall.Errno(0x5f) ELOOP = syscall.Errno(0x3e) EMEDIUMTYPE = syscall.Errno(0x56) EMFILE = syscall.Errno(0x18) @@ -1459,12 +1689,14 @@ const ( ENOTCONN = syscall.Errno(0x39) ENOTDIR = syscall.Errno(0x14) ENOTEMPTY = syscall.Errno(0x42) + ENOTRECOVERABLE = syscall.Errno(0x5d) ENOTSOCK = syscall.Errno(0x26) ENOTSUP = syscall.Errno(0x5b) ENOTTY = syscall.Errno(0x19) ENXIO = syscall.Errno(0x6) EOPNOTSUPP = syscall.Errno(0x2d) EOVERFLOW = syscall.Errno(0x57) + EOWNERDEAD = syscall.Errno(0x5e) EPERM = syscall.Errno(0x1) EPFNOSUPPORT = syscall.Errno(0x2e) EPIPE = syscall.Errno(0x20) @@ -1472,6 +1704,7 @@ const ( EPROCUNAVAIL = syscall.Errno(0x4c) EPROGMISMATCH = syscall.Errno(0x4b) EPROGUNAVAIL = syscall.Errno(0x4a) + EPROTO = syscall.Errno(0x5f) EPROTONOSUPPORT = syscall.Errno(0x2b) EPROTOTYPE = syscall.Errno(0x29) ERANGE = syscall.Errno(0x22) @@ -1568,7 +1801,7 @@ var errorList = [...]struct { {32, "EPIPE", "broken pipe"}, {33, "EDOM", "numerical argument out of domain"}, {34, "ERANGE", "result too large"}, - {35, "EWOULDBLOCK", "resource temporarily unavailable"}, + {35, "EAGAIN", "resource temporarily unavailable"}, {36, "EINPROGRESS", "operation now in progress"}, {37, "EALREADY", "operation already in progress"}, {38, "ENOTSOCK", "socket operation on non-socket"}, @@ -1624,7 +1857,11 @@ var errorList = [...]struct { {88, "ECANCELED", "operation canceled"}, {89, "EIDRM", "identifier removed"}, {90, "ENOMSG", "no message of desired type"}, - {91, "ELAST", "not supported"}, + {91, "ENOTSUP", "not supported"}, + {92, "EBADMSG", "bad message"}, + {93, "ENOTRECOVERABLE", "state not recoverable"}, + {94, "EOWNERDEAD", "previous owner died"}, + {95, "ELAST", "protocol error"}, } // Signal table @@ -1638,7 +1875,7 @@ var signalList = [...]struct { {3, "SIGQUIT", "quit"}, {4, "SIGILL", "illegal instruction"}, {5, "SIGTRAP", "trace/BPT trap"}, - {6, "SIGABRT", "abort trap"}, + {6, "SIGIOT", "abort trap"}, {7, "SIGEMT", "EMT trap"}, {8, "SIGFPE", "floating point exception"}, {9, "SIGKILL", "killed"}, @@ -1665,4 +1902,5 @@ var signalList = [...]struct { {30, "SIGUSR1", "user defined signal 1"}, {31, "SIGUSR2", "user defined signal 2"}, {32, "SIGTHR", "thread AST"}, + {28672, "SIGSTKSZ", "unknown signal"}, } diff --git a/vendor/golang.org/x/sys/unix/zerrors_openbsd_amd64.go b/vendor/golang.org/x/sys/unix/zerrors_openbsd_amd64.go index 25cb609..6015fcb 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_openbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_openbsd_amd64.go @@ -109,6 +109,15 @@ const ( BPF_DIRECTION_IN = 0x1 BPF_DIRECTION_OUT = 0x2 BPF_DIV = 0x30 + BPF_FILDROP_CAPTURE = 0x1 + BPF_FILDROP_DROP = 0x2 + BPF_FILDROP_PASS = 0x0 + BPF_F_DIR_IN = 0x10 + BPF_F_DIR_MASK = 0x30 + BPF_F_DIR_OUT = 0x20 + BPF_F_DIR_SHIFT = 0x4 + BPF_F_FLOWID = 0x8 + BPF_F_PRI_MASK = 0x7 BPF_H = 0x8 BPF_IMM = 0x0 BPF_IND = 0x40 @@ -137,6 +146,7 @@ const ( BPF_OR = 0x40 BPF_RELEASE = 0x30bb6 BPF_RET = 0x6 + BPF_RND = 0xc0 BPF_RSH = 0x70 BPF_ST = 0x2 BPF_STX = 0x3 @@ -177,7 +187,65 @@ const ( CTL_KERN = 0x1 CTL_MAXNAME = 0xc CTL_NET = 0x4 + DIOCADDQUEUE = 0xc110445d + DIOCADDRULE = 0xcd604404 + DIOCADDSTATE = 0xc1084425 + DIOCCHANGERULE = 0xcd60441a + DIOCCLRIFFLAG = 0xc028445a + DIOCCLRSRCNODES = 0x20004455 + DIOCCLRSTATES = 0xc0e04412 + DIOCCLRSTATUS = 0xc0284416 + DIOCGETLIMIT = 0xc0084427 + DIOCGETQSTATS = 0xc1204460 + DIOCGETQUEUE = 0xc110445f + DIOCGETQUEUES = 0xc110445e + DIOCGETRULE = 0xcd604407 + DIOCGETRULES = 0xcd604406 + DIOCGETRULESET = 0xc444443b + DIOCGETRULESETS = 0xc444443a + DIOCGETSRCNODES = 0xc0104454 + DIOCGETSTATE = 0xc1084413 + DIOCGETSTATES = 0xc0104419 + DIOCGETSTATUS = 0xc1e84415 + DIOCGETSYNFLWATS = 0xc0084463 + DIOCGETTIMEOUT = 0xc008441e + DIOCIGETIFACES = 0xc0284457 + DIOCKILLSRCNODES = 0xc080445b + DIOCKILLSTATES = 0xc0e04429 + DIOCNATLOOK = 0xc0504417 + DIOCOSFPADD = 0xc088444f DIOCOSFPFLUSH = 0x2000444e + DIOCOSFPGET = 0xc0884450 + DIOCRADDADDRS = 0xc4504443 + DIOCRADDTABLES = 0xc450443d + DIOCRCLRADDRS = 0xc4504442 + DIOCRCLRASTATS = 0xc4504448 + DIOCRCLRTABLES = 0xc450443c + DIOCRCLRTSTATS = 0xc4504441 + DIOCRDELADDRS = 0xc4504444 + DIOCRDELTABLES = 0xc450443e + DIOCRGETADDRS = 0xc4504446 + DIOCRGETASTATS = 0xc4504447 + DIOCRGETTABLES = 0xc450443f + DIOCRGETTSTATS = 0xc4504440 + DIOCRINADEFINE = 0xc450444d + DIOCRSETADDRS = 0xc4504445 + DIOCRSETTFLAGS = 0xc450444a + DIOCRTSTADDRS = 0xc4504449 + DIOCSETDEBUG = 0xc0044418 + DIOCSETHOSTID = 0xc0044456 + DIOCSETIFFLAG = 0xc0284459 + DIOCSETLIMIT = 0xc0084428 + DIOCSETREASS = 0xc004445c + DIOCSETSTATUSIF = 0xc0284414 + DIOCSETSYNCOOKIES = 0xc0014462 + DIOCSETSYNFLWATS = 0xc0084461 + DIOCSETTIMEOUT = 0xc008441d + DIOCSTART = 0x20004401 + DIOCSTOP = 0x20004402 + DIOCXBEGIN = 0xc0104451 + DIOCXCOMMIT = 0xc0104452 + DIOCXROLLBACK = 0xc0104453 DLT_ARCNET = 0x7 DLT_ATM_RFC1483 = 0xb DLT_AX25 = 0x3 @@ -240,6 +308,8 @@ const ( EMUL_ENABLED = 0x1 EMUL_NATIVE = 0x2 ENDRUNDISC = 0x9 + ETH64_8021_RSVD_MASK = 0xfffffffffff0 + ETH64_8021_RSVD_PREFIX = 0x180c2000000 ETHERMIN = 0x2e ETHERMTU = 0x5dc ETHERTYPE_8023 = 0x4 @@ -292,6 +362,7 @@ const ( ETHERTYPE_DN = 0x6003 ETHERTYPE_DOGFIGHT = 0x1989 ETHERTYPE_DSMD = 0x8039 + ETHERTYPE_EAPOL = 0x888e ETHERTYPE_ECMA = 0x803 ETHERTYPE_ENCRYPT = 0x803d ETHERTYPE_ES = 0x805d @@ -323,6 +394,7 @@ const ( ETHERTYPE_LLDP = 0x88cc ETHERTYPE_LOGICRAFT = 0x8148 ETHERTYPE_LOOPBACK = 0x9000 + ETHERTYPE_MACSEC = 0x88e5 ETHERTYPE_MATRA = 0x807a ETHERTYPE_MAX = 0xffff ETHERTYPE_MERIT = 0x807c @@ -351,15 +423,17 @@ const ( ETHERTYPE_NCD = 0x8149 ETHERTYPE_NESTAR = 0x8006 ETHERTYPE_NETBEUI = 0x8191 + ETHERTYPE_NHRP = 0x2001 ETHERTYPE_NOVELL = 0x8138 ETHERTYPE_NS = 0x600 ETHERTYPE_NSAT = 0x601 ETHERTYPE_NSCOMPAT = 0x807 + ETHERTYPE_NSH = 0x984f ETHERTYPE_NTRAILER = 0x10 ETHERTYPE_OS9 = 0x7007 ETHERTYPE_OS9NET = 0x7009 ETHERTYPE_PACER = 0x80c6 - ETHERTYPE_PAE = 0x888e + ETHERTYPE_PBB = 0x88e7 ETHERTYPE_PCS = 0x4242 ETHERTYPE_PLANNING = 0x8044 ETHERTYPE_PPP = 0x880b @@ -441,10 +515,11 @@ const ( ETHER_VLAN_ENCAP_LEN = 0x4 EVFILT_AIO = -0x3 EVFILT_DEVICE = -0x8 + EVFILT_EXCEPT = -0x9 EVFILT_PROC = -0x5 EVFILT_READ = -0x1 EVFILT_SIGNAL = -0x6 - EVFILT_SYSCOUNT = 0x8 + EVFILT_SYSCOUNT = 0x9 EVFILT_TIMER = -0x7 EVFILT_VNODE = -0x4 EVFILT_WRITE = -0x2 @@ -466,7 +541,7 @@ const ( EV_FLAG1 = 0x2000 EV_ONESHOT = 0x10 EV_RECEIPT = 0x40 - EV_SYSFLAGS = 0xf000 + EV_SYSFLAGS = 0xf800 EXTA = 0x4b00 EXTB = 0x9600 EXTPROC = 0x800 @@ -732,6 +807,7 @@ const ( IFT_VOICEOVERCABLE = 0xc6 IFT_VOICEOVERFRAMERELAY = 0x99 IFT_VOICEOVERIP = 0x68 + IFT_WIREGUARD = 0xfb IFT_X213 = 0x5d IFT_X25 = 0x5 IFT_X25DDN = 0x4 @@ -797,9 +873,11 @@ const ( IPPROTO_RAW = 0xff IPPROTO_ROUTING = 0x2b IPPROTO_RSVP = 0x2e + IPPROTO_SCTP = 0x84 IPPROTO_TCP = 0x6 IPPROTO_TP = 0x1d IPPROTO_UDP = 0x11 + IPPROTO_UDPLITE = 0x88 IPV6_AUTH_LEVEL = 0x35 IPV6_AUTOFLOWLABEL = 0x3b IPV6_CHECKSUM = 0x1a @@ -906,6 +984,9 @@ const ( IP_TTL = 0x4 ISIG = 0x80 ISTRIP = 0x20 + ITIMER_PROF = 0x2 + ITIMER_REAL = 0x0 + ITIMER_VIRTUAL = 0x1 IUCLC = 0x1000 IXANY = 0x800 IXOFF = 0x400 @@ -970,12 +1051,26 @@ const ( MNT_ROOTFS = 0x4000 MNT_SOFTDEP = 0x4000000 MNT_STALLED = 0x100000 + MNT_SWAPPABLE = 0x200000 MNT_SYNCHRONOUS = 0x2 MNT_UPDATE = 0x10000 MNT_VISFLAGMASK = 0x400ffff MNT_WAIT = 0x1 MNT_WANTRDWR = 0x2000000 MNT_WXALLOWED = 0x800 + MOUNT_AFS = "afs" + MOUNT_CD9660 = "cd9660" + MOUNT_EXT2FS = "ext2fs" + MOUNT_FFS = "ffs" + MOUNT_FUSEFS = "fuse" + MOUNT_MFS = "mfs" + MOUNT_MSDOS = "msdos" + MOUNT_NCPFS = "ncpfs" + MOUNT_NFS = "nfs" + MOUNT_NTFS = "ntfs" + MOUNT_TMPFS = "tmpfs" + MOUNT_UDF = "udf" + MOUNT_UFS = "ffs" MSG_BCAST = 0x100 MSG_CMSG_CLOEXEC = 0x800 MSG_CTRUNC = 0x20 @@ -988,6 +1083,7 @@ const ( MSG_PEEK = 0x2 MSG_TRUNC = 0x10 MSG_WAITALL = 0x40 + MSG_WAITFORONE = 0x1000 MS_ASYNC = 0x1 MS_INVALIDATE = 0x4 MS_SYNC = 0x2 @@ -996,7 +1092,8 @@ const ( NET_RT_FLAGS = 0x2 NET_RT_IFLIST = 0x3 NET_RT_IFNAMES = 0x6 - NET_RT_MAXID = 0x7 + NET_RT_MAXID = 0x8 + NET_RT_SOURCE = 0x7 NET_RT_STATS = 0x4 NET_RT_TABLE = 0x5 NFDBITS = 0x20 @@ -1013,6 +1110,7 @@ const ( NOTE_FORK = 0x40000000 NOTE_LINK = 0x10 NOTE_LOWAT = 0x1 + NOTE_OOB = 0x4 NOTE_PCTRLMASK = 0xf0000000 NOTE_PDATAMASK = 0xfffff NOTE_RENAME = 0x20 @@ -1130,9 +1228,11 @@ const ( RTF_STATIC = 0x800 RTF_UP = 0x1 RTF_USETRAILERS = 0x8000 + RTM_80211INFO = 0x15 RTM_ADD = 0x1 RTM_BFD = 0x12 RTM_CHANGE = 0x3 + RTM_CHGADDRATTR = 0x14 RTM_DELADDR = 0xd RTM_DELETE = 0x2 RTM_DESYNC = 0x10 @@ -1140,7 +1240,6 @@ const ( RTM_IFANNOUNCE = 0xf RTM_IFINFO = 0xe RTM_INVALIDATE = 0x11 - RTM_LOCK = 0x8 RTM_LOSING = 0x5 RTM_MAXSIZE = 0x800 RTM_MISS = 0x7 @@ -1148,7 +1247,7 @@ const ( RTM_PROPOSAL = 0x13 RTM_REDIRECT = 0x6 RTM_RESOLVE = 0xb - RTM_RTTUNIT = 0xf4240 + RTM_SOURCE = 0x16 RTM_VERSION = 0x5 RTV_EXPIRE = 0x4 RTV_HOPCOUNT = 0x2 @@ -1166,6 +1265,9 @@ const ( RUSAGE_THREAD = 0x1 SCM_RIGHTS = 0x1 SCM_TIMESTAMP = 0x4 + SEEK_CUR = 0x1 + SEEK_END = 0x2 + SEEK_SET = 0x0 SHUT_RD = 0x0 SHUT_RDWR = 0x2 SHUT_WR = 0x1 @@ -1182,35 +1284,37 @@ const ( SIOCBRDGDELS = 0x80606942 SIOCBRDGFLUSH = 0x80606948 SIOCBRDGFRL = 0x808c694e - SIOCBRDGGCACHE = 0xc0186941 - SIOCBRDGGFD = 0xc0186952 - SIOCBRDGGHT = 0xc0186951 + SIOCBRDGGCACHE = 0xc0146941 + SIOCBRDGGFD = 0xc0146952 + SIOCBRDGGHT = 0xc0146951 SIOCBRDGGIFFLGS = 0xc060693e - SIOCBRDGGMA = 0xc0186953 + SIOCBRDGGMA = 0xc0146953 SIOCBRDGGPARAM = 0xc0406958 - SIOCBRDGGPRI = 0xc0186950 + SIOCBRDGGPRI = 0xc0146950 SIOCBRDGGRL = 0xc030694f - SIOCBRDGGTO = 0xc0186946 + SIOCBRDGGTO = 0xc0146946 SIOCBRDGIFS = 0xc0606942 SIOCBRDGRTS = 0xc0206943 SIOCBRDGSADDR = 0xc1286944 - SIOCBRDGSCACHE = 0x80186940 - SIOCBRDGSFD = 0x80186952 - SIOCBRDGSHT = 0x80186951 + SIOCBRDGSCACHE = 0x80146940 + SIOCBRDGSFD = 0x80146952 + SIOCBRDGSHT = 0x80146951 SIOCBRDGSIFCOST = 0x80606955 SIOCBRDGSIFFLGS = 0x8060693f SIOCBRDGSIFPRIO = 0x80606954 SIOCBRDGSIFPROT = 0x8060694a - SIOCBRDGSMA = 0x80186953 - SIOCBRDGSPRI = 0x80186950 - SIOCBRDGSPROTO = 0x8018695a - SIOCBRDGSTO = 0x80186945 - SIOCBRDGSTXHC = 0x80186959 + SIOCBRDGSMA = 0x80146953 + SIOCBRDGSPRI = 0x80146950 + SIOCBRDGSPROTO = 0x8014695a + SIOCBRDGSTO = 0x80146945 + SIOCBRDGSTXHC = 0x80146959 + SIOCDELLABEL = 0x80206997 SIOCDELMULTI = 0x80206932 SIOCDIFADDR = 0x80206919 SIOCDIFGROUP = 0x80286989 SIOCDIFPARENT = 0x802069b4 SIOCDIFPHYADDR = 0x80206949 + SIOCDPWE3NEIGHBOR = 0x802069de SIOCDVNETID = 0x802069af SIOCGETKALIVE = 0xc01869a4 SIOCGETLABEL = 0x8020699a @@ -1229,6 +1333,7 @@ const ( SIOCGIFFLAGS = 0xc0206911 SIOCGIFGATTR = 0xc028698b SIOCGIFGENERIC = 0xc020693a + SIOCGIFGLIST = 0xc028698d SIOCGIFGMEMB = 0xc028698a SIOCGIFGROUP = 0xc0286988 SIOCGIFHARDMTU = 0xc02069a5 @@ -1243,13 +1348,21 @@ const ( SIOCGIFRDOMAIN = 0xc02069a0 SIOCGIFRTLABEL = 0xc0206983 SIOCGIFRXR = 0x802069aa + SIOCGIFSFFPAGE = 0xc1126939 SIOCGIFXFLAGS = 0xc020699e SIOCGLIFPHYADDR = 0xc218694b SIOCGLIFPHYDF = 0xc02069c2 + SIOCGLIFPHYECN = 0xc02069c8 SIOCGLIFPHYRTABLE = 0xc02069a2 SIOCGLIFPHYTTL = 0xc02069a9 SIOCGPGRP = 0x40047309 + SIOCGPWE3 = 0xc0206998 + SIOCGPWE3CTRLWORD = 0xc02069dc + SIOCGPWE3FAT = 0xc02069dd + SIOCGPWE3NEIGHBOR = 0xc21869de + SIOCGRXHPRIO = 0xc02069db SIOCGSPPPPARAMS = 0xc0206994 + SIOCGTXHPRIO = 0xc02069c6 SIOCGUMBINFO = 0xc02069be SIOCGUMBPARAM = 0xc02069c0 SIOCGVH = 0xc02069f6 @@ -1287,19 +1400,20 @@ const ( SIOCSIFXFLAGS = 0x8020699d SIOCSLIFPHYADDR = 0x8218694a SIOCSLIFPHYDF = 0x802069c1 + SIOCSLIFPHYECN = 0x802069c7 SIOCSLIFPHYRTABLE = 0x802069a1 SIOCSLIFPHYTTL = 0x802069a8 SIOCSPGRP = 0x80047308 + SIOCSPWE3CTRLWORD = 0x802069dc + SIOCSPWE3FAT = 0x802069dd + SIOCSPWE3NEIGHBOR = 0x821869de + SIOCSRXHPRIO = 0x802069db SIOCSSPPPPARAMS = 0x80206993 + SIOCSTXHPRIO = 0x802069c5 SIOCSUMBPARAM = 0x802069bf SIOCSVH = 0xc02069f5 SIOCSVNETFLOWID = 0x802069c3 SIOCSVNETID = 0x802069a6 - SIOCSWGDPID = 0xc018695b - SIOCSWGMAXFLOW = 0xc0186960 - SIOCSWGMAXGROUP = 0xc018695d - SIOCSWSDPID = 0x8018695c - SIOCSWSPORTNO = 0xc060695f SOCK_CLOEXEC = 0x8000 SOCK_DGRAM = 0x2 SOCK_DNS = 0x1000 @@ -1314,6 +1428,7 @@ const ( SO_BINDANY = 0x1000 SO_BROADCAST = 0x20 SO_DEBUG = 0x1 + SO_DOMAIN = 0x1024 SO_DONTROUTE = 0x10 SO_ERROR = 0x1007 SO_KEEPALIVE = 0x8 @@ -1321,6 +1436,7 @@ const ( SO_NETPROC = 0x1020 SO_OOBINLINE = 0x100 SO_PEERCRED = 0x1022 + SO_PROTOCOL = 0x1025 SO_RCVBUF = 0x1002 SO_RCVLOWAT = 0x1004 SO_RCVTIMEO = 0x1006 @@ -1370,7 +1486,18 @@ const ( TCOFLUSH = 0x2 TCOOFF = 0x1 TCOON = 0x2 - TCP_MAXBURST = 0x4 + TCPOPT_EOL = 0x0 + TCPOPT_MAXSEG = 0x2 + TCPOPT_NOP = 0x1 + TCPOPT_SACK = 0x5 + TCPOPT_SACK_HDR = 0x1010500 + TCPOPT_SACK_PERMITTED = 0x4 + TCPOPT_SACK_PERMIT_HDR = 0x1010402 + TCPOPT_SIGNATURE = 0x13 + TCPOPT_TIMESTAMP = 0x8 + TCPOPT_TSTAMP_HDR = 0x101080a + TCPOPT_WINDOW = 0x3 + TCP_INFO = 0x9 TCP_MAXSEG = 0x2 TCP_MAXWIN = 0xffff TCP_MAX_SACK = 0x3 @@ -1379,8 +1506,11 @@ const ( TCP_MSS = 0x200 TCP_NODELAY = 0x1 TCP_NOPUSH = 0x10 + TCP_SACKHOLE_LIMIT = 0x80 TCP_SACK_ENABLE = 0x8 TCSAFLUSH = 0x2 + TIMER_ABSTIME = 0x1 + TIMER_RELTIME = 0x0 TIOCCBRK = 0x2000747a TIOCCDTR = 0x20007478 TIOCCHKVERAUTH = 0x2000741e @@ -1445,7 +1575,6 @@ const ( TIOCSPGRP = 0x80047476 TIOCSTART = 0x2000746e TIOCSTAT = 0x20007465 - TIOCSTI = 0x80017472 TIOCSTOP = 0x2000746f TIOCSTSTAMP = 0x8008745a TIOCSWINSZ = 0x80087467 @@ -1467,7 +1596,8 @@ const ( VMIN = 0x10 VM_ANONMIN = 0x7 VM_LOADAVG = 0x2 - VM_MAXID = 0xc + VM_MALLOC_CONF = 0xc + VM_MAXID = 0xd VM_MAXSLP = 0xa VM_METER = 0x1 VM_NKMEMPAGES = 0x6 @@ -1745,7 +1875,7 @@ var signalList = [...]struct { {3, "SIGQUIT", "quit"}, {4, "SIGILL", "illegal instruction"}, {5, "SIGTRAP", "trace/BPT trap"}, - {6, "SIGABRT", "abort trap"}, + {6, "SIGIOT", "abort trap"}, {7, "SIGEMT", "EMT trap"}, {8, "SIGFPE", "floating point exception"}, {9, "SIGKILL", "killed"}, @@ -1772,4 +1902,5 @@ var signalList = [...]struct { {30, "SIGUSR1", "user defined signal 1"}, {31, "SIGUSR2", "user defined signal 2"}, {32, "SIGTHR", "thread AST"}, + {28672, "SIGSTKSZ", "unknown signal"}, } diff --git a/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm.go b/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm.go index aef6c08..8d44955 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm.go @@ -46,6 +46,7 @@ const ( AF_SNA = 0xb AF_UNIX = 0x1 AF_UNSPEC = 0x0 + ALTWERASE = 0x200 ARPHRD_ETHER = 0x1 ARPHRD_FRELAY = 0xf ARPHRD_IEEE1394 = 0x18 @@ -82,7 +83,7 @@ const ( BIOCGFILDROP = 0x40044278 BIOCGHDRCMPLT = 0x40044274 BIOCGRSIG = 0x40044273 - BIOCGRTIMEOUT = 0x400c426e + BIOCGRTIMEOUT = 0x4010426e BIOCGSTATS = 0x4008426f BIOCIMMEDIATE = 0x80044270 BIOCLOCK = 0x20004276 @@ -96,7 +97,7 @@ const ( BIOCSFILDROP = 0x80044279 BIOCSHDRCMPLT = 0x80044275 BIOCSRSIG = 0x80044272 - BIOCSRTIMEOUT = 0x800c426d + BIOCSRTIMEOUT = 0x8010426d BIOCVERSION = 0x40044271 BPF_A = 0x10 BPF_ABS = 0x20 @@ -108,6 +109,15 @@ const ( BPF_DIRECTION_IN = 0x1 BPF_DIRECTION_OUT = 0x2 BPF_DIV = 0x30 + BPF_FILDROP_CAPTURE = 0x1 + BPF_FILDROP_DROP = 0x2 + BPF_FILDROP_PASS = 0x0 + BPF_F_DIR_IN = 0x10 + BPF_F_DIR_MASK = 0x30 + BPF_F_DIR_OUT = 0x20 + BPF_F_DIR_SHIFT = 0x4 + BPF_F_FLOWID = 0x8 + BPF_F_PRI_MASK = 0x7 BPF_H = 0x8 BPF_IMM = 0x0 BPF_IND = 0x40 @@ -136,6 +146,7 @@ const ( BPF_OR = 0x40 BPF_RELEASE = 0x30bb6 BPF_RET = 0x6 + BPF_RND = 0xc0 BPF_RSH = 0x70 BPF_ST = 0x2 BPF_STX = 0x3 @@ -147,6 +158,12 @@ const ( BRKINT = 0x2 CFLUSH = 0xf CLOCAL = 0x8000 + CLOCK_BOOTTIME = 0x6 + CLOCK_MONOTONIC = 0x3 + CLOCK_PROCESS_CPUTIME_ID = 0x2 + CLOCK_REALTIME = 0x0 + CLOCK_THREAD_CPUTIME_ID = 0x4 + CLOCK_UPTIME = 0x5 CPUSTATES = 0x6 CP_IDLE = 0x5 CP_INTR = 0x4 @@ -170,7 +187,65 @@ const ( CTL_KERN = 0x1 CTL_MAXNAME = 0xc CTL_NET = 0x4 + DIOCADDQUEUE = 0xc100445d + DIOCADDRULE = 0xcce04404 + DIOCADDSTATE = 0xc1084425 + DIOCCHANGERULE = 0xcce0441a + DIOCCLRIFFLAG = 0xc024445a + DIOCCLRSRCNODES = 0x20004455 + DIOCCLRSTATES = 0xc0d04412 + DIOCCLRSTATUS = 0xc0244416 + DIOCGETLIMIT = 0xc0084427 + DIOCGETQSTATS = 0xc1084460 + DIOCGETQUEUE = 0xc100445f + DIOCGETQUEUES = 0xc100445e + DIOCGETRULE = 0xcce04407 + DIOCGETRULES = 0xcce04406 + DIOCGETRULESET = 0xc444443b + DIOCGETRULESETS = 0xc444443a + DIOCGETSRCNODES = 0xc0084454 + DIOCGETSTATE = 0xc1084413 + DIOCGETSTATES = 0xc0084419 + DIOCGETSTATUS = 0xc1e84415 + DIOCGETSYNFLWATS = 0xc0084463 + DIOCGETTIMEOUT = 0xc008441e + DIOCIGETIFACES = 0xc0244457 + DIOCKILLSRCNODES = 0xc068445b + DIOCKILLSTATES = 0xc0d04429 + DIOCNATLOOK = 0xc0504417 + DIOCOSFPADD = 0xc088444f DIOCOSFPFLUSH = 0x2000444e + DIOCOSFPGET = 0xc0884450 + DIOCRADDADDRS = 0xc44c4443 + DIOCRADDTABLES = 0xc44c443d + DIOCRCLRADDRS = 0xc44c4442 + DIOCRCLRASTATS = 0xc44c4448 + DIOCRCLRTABLES = 0xc44c443c + DIOCRCLRTSTATS = 0xc44c4441 + DIOCRDELADDRS = 0xc44c4444 + DIOCRDELTABLES = 0xc44c443e + DIOCRGETADDRS = 0xc44c4446 + DIOCRGETASTATS = 0xc44c4447 + DIOCRGETTABLES = 0xc44c443f + DIOCRGETTSTATS = 0xc44c4440 + DIOCRINADEFINE = 0xc44c444d + DIOCRSETADDRS = 0xc44c4445 + DIOCRSETTFLAGS = 0xc44c444a + DIOCRTSTADDRS = 0xc44c4449 + DIOCSETDEBUG = 0xc0044418 + DIOCSETHOSTID = 0xc0044456 + DIOCSETIFFLAG = 0xc0244459 + DIOCSETLIMIT = 0xc0084428 + DIOCSETREASS = 0xc004445c + DIOCSETSTATUSIF = 0xc0244414 + DIOCSETSYNCOOKIES = 0xc0014462 + DIOCSETSYNFLWATS = 0xc0084461 + DIOCSETTIMEOUT = 0xc008441d + DIOCSTART = 0x20004401 + DIOCSTOP = 0x20004402 + DIOCXBEGIN = 0xc00c4451 + DIOCXCOMMIT = 0xc00c4452 + DIOCXROLLBACK = 0xc00c4453 DLT_ARCNET = 0x7 DLT_ATM_RFC1483 = 0xb DLT_AX25 = 0x3 @@ -186,6 +261,7 @@ const ( DLT_LOOP = 0xc DLT_MPLS = 0xdb DLT_NULL = 0x0 + DLT_OPENFLOW = 0x10b DLT_PFLOG = 0x75 DLT_PFSYNC = 0x12 DLT_PPP = 0x9 @@ -196,6 +272,23 @@ const ( DLT_RAW = 0xe DLT_SLIP = 0x8 DLT_SLIP_BSDOS = 0xf + DLT_USBPCAP = 0xf9 + DLT_USER0 = 0x93 + DLT_USER1 = 0x94 + DLT_USER10 = 0x9d + DLT_USER11 = 0x9e + DLT_USER12 = 0x9f + DLT_USER13 = 0xa0 + DLT_USER14 = 0xa1 + DLT_USER15 = 0xa2 + DLT_USER2 = 0x95 + DLT_USER3 = 0x96 + DLT_USER4 = 0x97 + DLT_USER5 = 0x98 + DLT_USER6 = 0x99 + DLT_USER7 = 0x9a + DLT_USER8 = 0x9b + DLT_USER9 = 0x9c DT_BLK = 0x6 DT_CHR = 0x2 DT_DIR = 0x4 @@ -215,6 +308,8 @@ const ( EMUL_ENABLED = 0x1 EMUL_NATIVE = 0x2 ENDRUNDISC = 0x9 + ETH64_8021_RSVD_MASK = 0xfffffffffff0 + ETH64_8021_RSVD_PREFIX = 0x180c2000000 ETHERMIN = 0x2e ETHERMTU = 0x5dc ETHERTYPE_8023 = 0x4 @@ -267,6 +362,7 @@ const ( ETHERTYPE_DN = 0x6003 ETHERTYPE_DOGFIGHT = 0x1989 ETHERTYPE_DSMD = 0x8039 + ETHERTYPE_EAPOL = 0x888e ETHERTYPE_ECMA = 0x803 ETHERTYPE_ENCRYPT = 0x803d ETHERTYPE_ES = 0x805d @@ -298,6 +394,7 @@ const ( ETHERTYPE_LLDP = 0x88cc ETHERTYPE_LOGICRAFT = 0x8148 ETHERTYPE_LOOPBACK = 0x9000 + ETHERTYPE_MACSEC = 0x88e5 ETHERTYPE_MATRA = 0x807a ETHERTYPE_MAX = 0xffff ETHERTYPE_MERIT = 0x807c @@ -326,15 +423,17 @@ const ( ETHERTYPE_NCD = 0x8149 ETHERTYPE_NESTAR = 0x8006 ETHERTYPE_NETBEUI = 0x8191 + ETHERTYPE_NHRP = 0x2001 ETHERTYPE_NOVELL = 0x8138 ETHERTYPE_NS = 0x600 ETHERTYPE_NSAT = 0x601 ETHERTYPE_NSCOMPAT = 0x807 + ETHERTYPE_NSH = 0x984f ETHERTYPE_NTRAILER = 0x10 ETHERTYPE_OS9 = 0x7007 ETHERTYPE_OS9NET = 0x7009 ETHERTYPE_PACER = 0x80c6 - ETHERTYPE_PAE = 0x888e + ETHERTYPE_PBB = 0x88e7 ETHERTYPE_PCS = 0x4242 ETHERTYPE_PLANNING = 0x8044 ETHERTYPE_PPP = 0x880b @@ -409,28 +508,40 @@ const ( ETHER_CRC_POLY_LE = 0xedb88320 ETHER_HDR_LEN = 0xe ETHER_MAX_DIX_LEN = 0x600 + ETHER_MAX_HARDMTU_LEN = 0xff9b ETHER_MAX_LEN = 0x5ee ETHER_MIN_LEN = 0x40 ETHER_TYPE_LEN = 0x2 ETHER_VLAN_ENCAP_LEN = 0x4 EVFILT_AIO = -0x3 + EVFILT_DEVICE = -0x8 + EVFILT_EXCEPT = -0x9 EVFILT_PROC = -0x5 EVFILT_READ = -0x1 EVFILT_SIGNAL = -0x6 - EVFILT_SYSCOUNT = 0x7 + EVFILT_SYSCOUNT = 0x9 EVFILT_TIMER = -0x7 EVFILT_VNODE = -0x4 EVFILT_WRITE = -0x2 + EVL_ENCAPLEN = 0x4 + EVL_PRIO_BITS = 0xd + EVL_PRIO_MAX = 0x7 + EVL_VLID_MASK = 0xfff + EVL_VLID_MAX = 0xffe + EVL_VLID_MIN = 0x1 + EVL_VLID_NULL = 0x0 EV_ADD = 0x1 EV_CLEAR = 0x20 EV_DELETE = 0x2 EV_DISABLE = 0x8 + EV_DISPATCH = 0x80 EV_ENABLE = 0x4 EV_EOF = 0x8000 EV_ERROR = 0x4000 EV_FLAG1 = 0x2000 EV_ONESHOT = 0x10 - EV_SYSFLAGS = 0xf000 + EV_RECEIPT = 0x40 + EV_SYSFLAGS = 0xf800 EXTA = 0x4b00 EXTB = 0x9600 EXTPROC = 0x800 @@ -443,6 +554,8 @@ const ( F_GETFL = 0x3 F_GETLK = 0x7 F_GETOWN = 0x5 + F_ISATTY = 0xb + F_OK = 0x0 F_RDLCK = 0x1 F_SETFD = 0x2 F_SETFL = 0x4 @@ -459,7 +572,6 @@ const ( IEXTEN = 0x400 IFAN_ARRIVAL = 0x0 IFAN_DEPARTURE = 0x1 - IFA_ROUTE = 0x1 IFF_ALLMULTI = 0x200 IFF_BROADCAST = 0x2 IFF_CANTCHANGE = 0x8e52 @@ -470,12 +582,12 @@ const ( IFF_LOOPBACK = 0x8 IFF_MULTICAST = 0x8000 IFF_NOARP = 0x80 - IFF_NOTRAILERS = 0x20 IFF_OACTIVE = 0x400 IFF_POINTOPOINT = 0x10 IFF_PROMISC = 0x100 IFF_RUNNING = 0x40 IFF_SIMPLEX = 0x800 + IFF_STATICARP = 0x20 IFF_UP = 0x1 IFNAMSIZ = 0x10 IFT_1822 = 0x2 @@ -604,6 +716,7 @@ const ( IFT_LINEGROUP = 0xd2 IFT_LOCALTALK = 0x2a IFT_LOOP = 0x18 + IFT_MBIM = 0xfa IFT_MEDIAMAILOVERIP = 0x8b IFT_MFSIGLINK = 0xa7 IFT_MIOX25 = 0x26 @@ -694,6 +807,7 @@ const ( IFT_VOICEOVERCABLE = 0xc6 IFT_VOICEOVERFRAMERELAY = 0x99 IFT_VOICEOVERIP = 0x68 + IFT_WIREGUARD = 0xfb IFT_X213 = 0x5d IFT_X25 = 0x5 IFT_X25DDN = 0x4 @@ -728,8 +842,6 @@ const ( IPPROTO_AH = 0x33 IPPROTO_CARP = 0x70 IPPROTO_DIVERT = 0x102 - IPPROTO_DIVERT_INIT = 0x2 - IPPROTO_DIVERT_RESP = 0x1 IPPROTO_DONE = 0x101 IPPROTO_DSTOPTS = 0x3c IPPROTO_EGP = 0x8 @@ -761,9 +873,11 @@ const ( IPPROTO_RAW = 0xff IPPROTO_ROUTING = 0x2b IPPROTO_RSVP = 0x2e + IPPROTO_SCTP = 0x84 IPPROTO_TCP = 0x6 IPPROTO_TP = 0x1d IPPROTO_UDP = 0x11 + IPPROTO_UDPLITE = 0x88 IPV6_AUTH_LEVEL = 0x35 IPV6_AUTOFLOWLABEL = 0x3b IPV6_CHECKSUM = 0x1a @@ -786,6 +900,7 @@ const ( IPV6_LEAVE_GROUP = 0xd IPV6_MAXHLIM = 0xff IPV6_MAXPACKET = 0xffff + IPV6_MINHOPCOUNT = 0x41 IPV6_MMTU = 0x500 IPV6_MULTICAST_HOPS = 0xa IPV6_MULTICAST_IF = 0x9 @@ -825,12 +940,12 @@ const ( IP_DEFAULT_MULTICAST_LOOP = 0x1 IP_DEFAULT_MULTICAST_TTL = 0x1 IP_DF = 0x4000 - IP_DIVERTFL = 0x1022 IP_DROP_MEMBERSHIP = 0xd IP_ESP_NETWORK_LEVEL = 0x16 IP_ESP_TRANS_LEVEL = 0x15 IP_HDRINCL = 0x2 IP_IPCOMP_LEVEL = 0x1d + IP_IPDEFTTL = 0x25 IP_IPSECFLOWINFO = 0x24 IP_IPSEC_LOCAL_AUTH = 0x1b IP_IPSEC_LOCAL_CRED = 0x19 @@ -864,10 +979,15 @@ const ( IP_RETOPTS = 0x8 IP_RF = 0x8000 IP_RTABLE = 0x1021 + IP_SENDSRCADDR = 0x7 IP_TOS = 0x3 IP_TTL = 0x4 ISIG = 0x80 ISTRIP = 0x20 + ITIMER_PROF = 0x2 + ITIMER_REAL = 0x0 + ITIMER_VIRTUAL = 0x1 + IUCLC = 0x1000 IXANY = 0x800 IXOFF = 0x400 IXON = 0x200 @@ -922,6 +1042,7 @@ const ( MNT_NOATIME = 0x8000 MNT_NODEV = 0x10 MNT_NOEXEC = 0x4 + MNT_NOPERM = 0x20 MNT_NOSUID = 0x8 MNT_NOWAIT = 0x2 MNT_QUOTA = 0x2000 @@ -929,12 +1050,27 @@ const ( MNT_RELOAD = 0x40000 MNT_ROOTFS = 0x4000 MNT_SOFTDEP = 0x4000000 + MNT_STALLED = 0x100000 + MNT_SWAPPABLE = 0x200000 MNT_SYNCHRONOUS = 0x2 MNT_UPDATE = 0x10000 MNT_VISFLAGMASK = 0x400ffff MNT_WAIT = 0x1 MNT_WANTRDWR = 0x2000000 MNT_WXALLOWED = 0x800 + MOUNT_AFS = "afs" + MOUNT_CD9660 = "cd9660" + MOUNT_EXT2FS = "ext2fs" + MOUNT_FFS = "ffs" + MOUNT_FUSEFS = "fuse" + MOUNT_MFS = "mfs" + MOUNT_MSDOS = "msdos" + MOUNT_NCPFS = "ncpfs" + MOUNT_NFS = "nfs" + MOUNT_NTFS = "ntfs" + MOUNT_TMPFS = "tmpfs" + MOUNT_UDF = "udf" + MOUNT_UFS = "ffs" MSG_BCAST = 0x100 MSG_CMSG_CLOEXEC = 0x800 MSG_CTRUNC = 0x20 @@ -947,6 +1083,7 @@ const ( MSG_PEEK = 0x2 MSG_TRUNC = 0x10 MSG_WAITALL = 0x40 + MSG_WAITFORONE = 0x1000 MS_ASYNC = 0x1 MS_INVALIDATE = 0x4 MS_SYNC = 0x2 @@ -954,12 +1091,16 @@ const ( NET_RT_DUMP = 0x1 NET_RT_FLAGS = 0x2 NET_RT_IFLIST = 0x3 - NET_RT_MAXID = 0x6 + NET_RT_IFNAMES = 0x6 + NET_RT_MAXID = 0x8 + NET_RT_SOURCE = 0x7 NET_RT_STATS = 0x4 NET_RT_TABLE = 0x5 NFDBITS = 0x20 NOFLSH = 0x80000000 + NOKERNINFO = 0x2000000 NOTE_ATTRIB = 0x8 + NOTE_CHANGE = 0x1 NOTE_CHILD = 0x4 NOTE_DELETE = 0x1 NOTE_EOF = 0x2 @@ -969,6 +1110,7 @@ const ( NOTE_FORK = 0x40000000 NOTE_LINK = 0x10 NOTE_LOWAT = 0x1 + NOTE_OOB = 0x4 NOTE_PCTRLMASK = 0xf0000000 NOTE_PDATAMASK = 0xfffff NOTE_RENAME = 0x20 @@ -978,11 +1120,13 @@ const ( NOTE_TRUNCATE = 0x80 NOTE_WRITE = 0x2 OCRNL = 0x10 + OLCUC = 0x20 ONLCR = 0x2 ONLRET = 0x80 ONOCR = 0x40 ONOEOT = 0x8 OPOST = 0x1 + OXTABS = 0x4 O_ACCMODE = 0x3 O_APPEND = 0x8 O_ASYNC = 0x40 @@ -1027,19 +1171,25 @@ const ( RLIMIT_STACK = 0x3 RLIM_INFINITY = 0x7fffffffffffffff RTAX_AUTHOR = 0x6 + RTAX_BFD = 0xb RTAX_BRD = 0x7 + RTAX_DNS = 0xc RTAX_DST = 0x0 RTAX_GATEWAY = 0x1 RTAX_GENMASK = 0x3 RTAX_IFA = 0x5 RTAX_IFP = 0x4 RTAX_LABEL = 0xa - RTAX_MAX = 0xb + RTAX_MAX = 0xf RTAX_NETMASK = 0x2 + RTAX_SEARCH = 0xe RTAX_SRC = 0x8 RTAX_SRCMASK = 0x9 + RTAX_STATIC = 0xd RTA_AUTHOR = 0x40 + RTA_BFD = 0x800 RTA_BRD = 0x80 + RTA_DNS = 0x1000 RTA_DST = 0x1 RTA_GATEWAY = 0x2 RTA_GENMASK = 0x8 @@ -1047,24 +1197,29 @@ const ( RTA_IFP = 0x10 RTA_LABEL = 0x400 RTA_NETMASK = 0x4 + RTA_SEARCH = 0x4000 RTA_SRC = 0x100 RTA_SRCMASK = 0x200 + RTA_STATIC = 0x2000 RTF_ANNOUNCE = 0x4000 + RTF_BFD = 0x1000000 RTF_BLACKHOLE = 0x1000 RTF_BROADCAST = 0x400000 + RTF_CACHED = 0x20000 RTF_CLONED = 0x10000 RTF_CLONING = 0x100 + RTF_CONNECTED = 0x800000 RTF_DONE = 0x40 RTF_DYNAMIC = 0x10 - RTF_FMASK = 0x70f808 + RTF_FMASK = 0x110fc08 RTF_GATEWAY = 0x2 RTF_HOST = 0x4 RTF_LLINFO = 0x400 RTF_LOCAL = 0x200000 - RTF_MASK = 0x80 RTF_MODIFIED = 0x20 RTF_MPATH = 0x40000 RTF_MPLS = 0x100000 + RTF_MULTICAST = 0x200 RTF_PERMANENT_ARP = 0x2000 RTF_PROTO1 = 0x8000 RTF_PROTO2 = 0x4000 @@ -1073,23 +1228,26 @@ const ( RTF_STATIC = 0x800 RTF_UP = 0x1 RTF_USETRAILERS = 0x8000 - RTF_XRESOLVE = 0x200 + RTM_80211INFO = 0x15 RTM_ADD = 0x1 + RTM_BFD = 0x12 RTM_CHANGE = 0x3 + RTM_CHGADDRATTR = 0x14 RTM_DELADDR = 0xd RTM_DELETE = 0x2 RTM_DESYNC = 0x10 RTM_GET = 0x4 RTM_IFANNOUNCE = 0xf RTM_IFINFO = 0xe - RTM_LOCK = 0x8 + RTM_INVALIDATE = 0x11 RTM_LOSING = 0x5 RTM_MAXSIZE = 0x800 RTM_MISS = 0x7 RTM_NEWADDR = 0xc + RTM_PROPOSAL = 0x13 RTM_REDIRECT = 0x6 RTM_RESOLVE = 0xb - RTM_RTTUNIT = 0xf4240 + RTM_SOURCE = 0x16 RTM_VERSION = 0x5 RTV_EXPIRE = 0x4 RTV_HOPCOUNT = 0x2 @@ -1099,67 +1257,74 @@ const ( RTV_RTTVAR = 0x80 RTV_SPIPE = 0x10 RTV_SSTHRESH = 0x20 + RT_TABLEID_BITS = 0x8 + RT_TABLEID_MASK = 0xff RT_TABLEID_MAX = 0xff RUSAGE_CHILDREN = -0x1 RUSAGE_SELF = 0x0 RUSAGE_THREAD = 0x1 SCM_RIGHTS = 0x1 SCM_TIMESTAMP = 0x4 + SEEK_CUR = 0x1 + SEEK_END = 0x2 + SEEK_SET = 0x0 SHUT_RD = 0x0 SHUT_RDWR = 0x2 SHUT_WR = 0x1 SIOCADDMULTI = 0x80206931 SIOCAIFADDR = 0x8040691a SIOCAIFGROUP = 0x80246987 - SIOCALIFADDR = 0x8218691c SIOCATMARK = 0x40047307 - SIOCBRDGADD = 0x8054693c - SIOCBRDGADDS = 0x80546941 - SIOCBRDGARL = 0x806e694d + SIOCBRDGADD = 0x8060693c + SIOCBRDGADDL = 0x80606949 + SIOCBRDGADDS = 0x80606941 + SIOCBRDGARL = 0x808c694d SIOCBRDGDADDR = 0x81286947 - SIOCBRDGDEL = 0x8054693d - SIOCBRDGDELS = 0x80546942 - SIOCBRDGFLUSH = 0x80546948 - SIOCBRDGFRL = 0x806e694e + SIOCBRDGDEL = 0x8060693d + SIOCBRDGDELS = 0x80606942 + SIOCBRDGFLUSH = 0x80606948 + SIOCBRDGFRL = 0x808c694e SIOCBRDGGCACHE = 0xc0146941 SIOCBRDGGFD = 0xc0146952 SIOCBRDGGHT = 0xc0146951 - SIOCBRDGGIFFLGS = 0xc054693e + SIOCBRDGGIFFLGS = 0xc060693e SIOCBRDGGMA = 0xc0146953 - SIOCBRDGGPARAM = 0xc03c6958 + SIOCBRDGGPARAM = 0xc0406958 SIOCBRDGGPRI = 0xc0146950 SIOCBRDGGRL = 0xc028694f - SIOCBRDGGSIFS = 0xc054693c SIOCBRDGGTO = 0xc0146946 - SIOCBRDGIFS = 0xc0546942 + SIOCBRDGIFS = 0xc0606942 SIOCBRDGRTS = 0xc0186943 SIOCBRDGSADDR = 0xc1286944 SIOCBRDGSCACHE = 0x80146940 SIOCBRDGSFD = 0x80146952 SIOCBRDGSHT = 0x80146951 - SIOCBRDGSIFCOST = 0x80546955 - SIOCBRDGSIFFLGS = 0x8054693f - SIOCBRDGSIFPRIO = 0x80546954 + SIOCBRDGSIFCOST = 0x80606955 + SIOCBRDGSIFFLGS = 0x8060693f + SIOCBRDGSIFPRIO = 0x80606954 + SIOCBRDGSIFPROT = 0x8060694a SIOCBRDGSMA = 0x80146953 SIOCBRDGSPRI = 0x80146950 SIOCBRDGSPROTO = 0x8014695a SIOCBRDGSTO = 0x80146945 SIOCBRDGSTXHC = 0x80146959 + SIOCDELLABEL = 0x80206997 SIOCDELMULTI = 0x80206932 SIOCDIFADDR = 0x80206919 SIOCDIFGROUP = 0x80246989 + SIOCDIFPARENT = 0x802069b4 SIOCDIFPHYADDR = 0x80206949 - SIOCDLIFADDR = 0x8218691e + SIOCDPWE3NEIGHBOR = 0x802069de + SIOCDVNETID = 0x802069af SIOCGETKALIVE = 0xc01869a4 SIOCGETLABEL = 0x8020699a + SIOCGETMPWCFG = 0xc02069ae SIOCGETPFLOW = 0xc02069fe SIOCGETPFSYNC = 0xc02069f8 SIOCGETSGCNT = 0xc0147534 SIOCGETVIFCNT = 0xc0147533 SIOCGETVLAN = 0xc0206990 - SIOCGHIWAT = 0x40047301 SIOCGIFADDR = 0xc0206921 - SIOCGIFASYNCMAP = 0xc020697c SIOCGIFBRDADDR = 0xc0206923 SIOCGIFCONF = 0xc0086924 SIOCGIFDATA = 0xc020691b @@ -1168,41 +1333,53 @@ const ( SIOCGIFFLAGS = 0xc0206911 SIOCGIFGATTR = 0xc024698b SIOCGIFGENERIC = 0xc020693a + SIOCGIFGLIST = 0xc024698d SIOCGIFGMEMB = 0xc024698a SIOCGIFGROUP = 0xc0246988 SIOCGIFHARDMTU = 0xc02069a5 - SIOCGIFMEDIA = 0xc0286936 + SIOCGIFLLPRIO = 0xc02069b6 + SIOCGIFMEDIA = 0xc0386938 SIOCGIFMETRIC = 0xc0206917 SIOCGIFMTU = 0xc020697e SIOCGIFNETMASK = 0xc0206925 - SIOCGIFPDSTADDR = 0xc0206948 + SIOCGIFPAIR = 0xc02069b1 + SIOCGIFPARENT = 0xc02069b3 SIOCGIFPRIORITY = 0xc020699c - SIOCGIFPSRCADDR = 0xc0206947 SIOCGIFRDOMAIN = 0xc02069a0 SIOCGIFRTLABEL = 0xc0206983 SIOCGIFRXR = 0x802069aa - SIOCGIFTIMESLOT = 0xc0206986 + SIOCGIFSFFPAGE = 0xc1126939 SIOCGIFXFLAGS = 0xc020699e - SIOCGLIFADDR = 0xc218691d SIOCGLIFPHYADDR = 0xc218694b + SIOCGLIFPHYDF = 0xc02069c2 + SIOCGLIFPHYECN = 0xc02069c8 SIOCGLIFPHYRTABLE = 0xc02069a2 SIOCGLIFPHYTTL = 0xc02069a9 - SIOCGLOWAT = 0x40047303 SIOCGPGRP = 0x40047309 + SIOCGPWE3 = 0xc0206998 + SIOCGPWE3CTRLWORD = 0xc02069dc + SIOCGPWE3FAT = 0xc02069dd + SIOCGPWE3NEIGHBOR = 0xc21869de + SIOCGRXHPRIO = 0xc02069db SIOCGSPPPPARAMS = 0xc0206994 + SIOCGTXHPRIO = 0xc02069c6 + SIOCGUMBINFO = 0xc02069be + SIOCGUMBPARAM = 0xc02069c0 SIOCGVH = 0xc02069f6 + SIOCGVNETFLOWID = 0xc02069c4 SIOCGVNETID = 0xc02069a7 + SIOCIFAFATTACH = 0x801169ab + SIOCIFAFDETACH = 0x801169ac SIOCIFCREATE = 0x8020697a SIOCIFDESTROY = 0x80206979 SIOCIFGCLONERS = 0xc00c6978 SIOCSETKALIVE = 0x801869a3 SIOCSETLABEL = 0x80206999 + SIOCSETMPWCFG = 0x802069ad SIOCSETPFLOW = 0x802069fd SIOCSETPFSYNC = 0x802069f7 SIOCSETVLAN = 0x8020698f - SIOCSHIWAT = 0x80047300 SIOCSIFADDR = 0x8020690c - SIOCSIFASYNCMAP = 0x8020697d SIOCSIFBRDADDR = 0x80206913 SIOCSIFDESCR = 0x80206980 SIOCSIFDSTADDR = 0x8020690e @@ -1210,26 +1387,36 @@ const ( SIOCSIFGATTR = 0x8024698c SIOCSIFGENERIC = 0x80206939 SIOCSIFLLADDR = 0x8020691f - SIOCSIFMEDIA = 0xc0206935 + SIOCSIFLLPRIO = 0x802069b5 + SIOCSIFMEDIA = 0xc0206937 SIOCSIFMETRIC = 0x80206918 SIOCSIFMTU = 0x8020697f SIOCSIFNETMASK = 0x80206916 - SIOCSIFPHYADDR = 0x80406946 + SIOCSIFPAIR = 0x802069b0 + SIOCSIFPARENT = 0x802069b2 SIOCSIFPRIORITY = 0x8020699b SIOCSIFRDOMAIN = 0x8020699f SIOCSIFRTLABEL = 0x80206982 - SIOCSIFTIMESLOT = 0x80206985 SIOCSIFXFLAGS = 0x8020699d SIOCSLIFPHYADDR = 0x8218694a + SIOCSLIFPHYDF = 0x802069c1 + SIOCSLIFPHYECN = 0x802069c7 SIOCSLIFPHYRTABLE = 0x802069a1 SIOCSLIFPHYTTL = 0x802069a8 - SIOCSLOWAT = 0x80047302 SIOCSPGRP = 0x80047308 + SIOCSPWE3CTRLWORD = 0x802069dc + SIOCSPWE3FAT = 0x802069dd + SIOCSPWE3NEIGHBOR = 0x821869de + SIOCSRXHPRIO = 0x802069db SIOCSSPPPPARAMS = 0x80206993 + SIOCSTXHPRIO = 0x802069c5 + SIOCSUMBPARAM = 0x802069bf SIOCSVH = 0xc02069f5 + SIOCSVNETFLOWID = 0x802069c3 SIOCSVNETID = 0x802069a6 SOCK_CLOEXEC = 0x8000 SOCK_DGRAM = 0x2 + SOCK_DNS = 0x1000 SOCK_NONBLOCK = 0x4000 SOCK_RAW = 0x3 SOCK_RDM = 0x4 @@ -1241,6 +1428,7 @@ const ( SO_BINDANY = 0x1000 SO_BROADCAST = 0x20 SO_DEBUG = 0x1 + SO_DOMAIN = 0x1024 SO_DONTROUTE = 0x10 SO_ERROR = 0x1007 SO_KEEPALIVE = 0x8 @@ -1248,6 +1436,7 @@ const ( SO_NETPROC = 0x1020 SO_OOBINLINE = 0x100 SO_PEERCRED = 0x1022 + SO_PROTOCOL = 0x1025 SO_RCVBUF = 0x1002 SO_RCVLOWAT = 0x1004 SO_RCVTIMEO = 0x1006 @@ -1261,6 +1450,7 @@ const ( SO_TIMESTAMP = 0x800 SO_TYPE = 0x1008 SO_USELOOPBACK = 0x40 + SO_ZEROIZE = 0x2000 S_BLKSIZE = 0x200 S_IEXEC = 0x40 S_IFBLK = 0x6000 @@ -1290,9 +1480,24 @@ const ( S_IXOTH = 0x1 S_IXUSR = 0x40 TCIFLUSH = 0x1 + TCIOFF = 0x3 TCIOFLUSH = 0x3 + TCION = 0x4 TCOFLUSH = 0x2 - TCP_MAXBURST = 0x4 + TCOOFF = 0x1 + TCOON = 0x2 + TCPOPT_EOL = 0x0 + TCPOPT_MAXSEG = 0x2 + TCPOPT_NOP = 0x1 + TCPOPT_SACK = 0x5 + TCPOPT_SACK_HDR = 0x1010500 + TCPOPT_SACK_PERMITTED = 0x4 + TCPOPT_SACK_PERMIT_HDR = 0x1010402 + TCPOPT_SIGNATURE = 0x13 + TCPOPT_TIMESTAMP = 0x8 + TCPOPT_TSTAMP_HDR = 0x101080a + TCPOPT_WINDOW = 0x3 + TCP_INFO = 0x9 TCP_MAXSEG = 0x2 TCP_MAXWIN = 0xffff TCP_MAX_SACK = 0x3 @@ -1301,11 +1506,15 @@ const ( TCP_MSS = 0x200 TCP_NODELAY = 0x1 TCP_NOPUSH = 0x10 - TCP_NSTATES = 0xb + TCP_SACKHOLE_LIMIT = 0x80 TCP_SACK_ENABLE = 0x8 TCSAFLUSH = 0x2 + TIMER_ABSTIME = 0x1 + TIMER_RELTIME = 0x0 TIOCCBRK = 0x2000747a TIOCCDTR = 0x20007478 + TIOCCHKVERAUTH = 0x2000741e + TIOCCLRVERAUTH = 0x2000741d TIOCCONS = 0x80047462 TIOCDRAIN = 0x2000745e TIOCEXCL = 0x2000740d @@ -1321,7 +1530,7 @@ const ( TIOCGFLAGS = 0x4004745d TIOCGPGRP = 0x40047477 TIOCGSID = 0x40047463 - TIOCGTSTAMP = 0x400c745b + TIOCGTSTAMP = 0x4010745b TIOCGWINSZ = 0x40087468 TIOCMBIC = 0x8004746b TIOCMBIS = 0x8004746c @@ -1360,17 +1569,21 @@ const ( TIOCSETAF = 0x802c7416 TIOCSETAW = 0x802c7415 TIOCSETD = 0x8004741b + TIOCSETVERAUTH = 0x8004741c TIOCSFLAGS = 0x8004745c TIOCSIG = 0x8004745f TIOCSPGRP = 0x80047476 TIOCSTART = 0x2000746e - TIOCSTAT = 0x80047465 - TIOCSTI = 0x80017472 + TIOCSTAT = 0x20007465 TIOCSTOP = 0x2000746f TIOCSTSTAMP = 0x8008745a TIOCSWINSZ = 0x80087467 TIOCUCNTL = 0x80047466 + TIOCUCNTL_CBRK = 0x7a + TIOCUCNTL_SBRK = 0x7b TOSTOP = 0x400000 + UTIME_NOW = -0x2 + UTIME_OMIT = -0x1 VDISCARD = 0xf VDSUSP = 0xb VEOF = 0x0 @@ -1381,6 +1594,19 @@ const ( VKILL = 0x5 VLNEXT = 0xe VMIN = 0x10 + VM_ANONMIN = 0x7 + VM_LOADAVG = 0x2 + VM_MALLOC_CONF = 0xc + VM_MAXID = 0xd + VM_MAXSLP = 0xa + VM_METER = 0x1 + VM_NKMEMPAGES = 0x6 + VM_PSSTRINGS = 0x3 + VM_SWAPENCRYPT = 0x5 + VM_USPACE = 0xb + VM_UVMEXP = 0x4 + VM_VNODEMIN = 0x9 + VM_VTEXTMIN = 0x8 VQUIT = 0x9 VREPRINT = 0x6 VSTART = 0xc @@ -1394,6 +1620,7 @@ const ( WCOREFLAG = 0x80 WNOHANG = 0x1 WUNTRACED = 0x2 + XCASE = 0x1000000 ) // Errors @@ -1407,6 +1634,7 @@ const ( EALREADY = syscall.Errno(0x25) EAUTH = syscall.Errno(0x50) EBADF = syscall.Errno(0x9) + EBADMSG = syscall.Errno(0x5c) EBADRPC = syscall.Errno(0x48) EBUSY = syscall.Errno(0x10) ECANCELED = syscall.Errno(0x58) @@ -1433,7 +1661,7 @@ const ( EIPSEC = syscall.Errno(0x52) EISCONN = syscall.Errno(0x38) EISDIR = syscall.Errno(0x15) - ELAST = syscall.Errno(0x5b) + ELAST = syscall.Errno(0x5f) ELOOP = syscall.Errno(0x3e) EMEDIUMTYPE = syscall.Errno(0x56) EMFILE = syscall.Errno(0x18) @@ -1461,12 +1689,14 @@ const ( ENOTCONN = syscall.Errno(0x39) ENOTDIR = syscall.Errno(0x14) ENOTEMPTY = syscall.Errno(0x42) + ENOTRECOVERABLE = syscall.Errno(0x5d) ENOTSOCK = syscall.Errno(0x26) ENOTSUP = syscall.Errno(0x5b) ENOTTY = syscall.Errno(0x19) ENXIO = syscall.Errno(0x6) EOPNOTSUPP = syscall.Errno(0x2d) EOVERFLOW = syscall.Errno(0x57) + EOWNERDEAD = syscall.Errno(0x5e) EPERM = syscall.Errno(0x1) EPFNOSUPPORT = syscall.Errno(0x2e) EPIPE = syscall.Errno(0x20) @@ -1474,6 +1704,7 @@ const ( EPROCUNAVAIL = syscall.Errno(0x4c) EPROGMISMATCH = syscall.Errno(0x4b) EPROGUNAVAIL = syscall.Errno(0x4a) + EPROTO = syscall.Errno(0x5f) EPROTONOSUPPORT = syscall.Errno(0x2b) EPROTOTYPE = syscall.Errno(0x29) ERANGE = syscall.Errno(0x22) @@ -1570,7 +1801,7 @@ var errorList = [...]struct { {32, "EPIPE", "broken pipe"}, {33, "EDOM", "numerical argument out of domain"}, {34, "ERANGE", "result too large"}, - {35, "EWOULDBLOCK", "resource temporarily unavailable"}, + {35, "EAGAIN", "resource temporarily unavailable"}, {36, "EINPROGRESS", "operation now in progress"}, {37, "EALREADY", "operation already in progress"}, {38, "ENOTSOCK", "socket operation on non-socket"}, @@ -1626,7 +1857,11 @@ var errorList = [...]struct { {88, "ECANCELED", "operation canceled"}, {89, "EIDRM", "identifier removed"}, {90, "ENOMSG", "no message of desired type"}, - {91, "ELAST", "not supported"}, + {91, "ENOTSUP", "not supported"}, + {92, "EBADMSG", "bad message"}, + {93, "ENOTRECOVERABLE", "state not recoverable"}, + {94, "EOWNERDEAD", "previous owner died"}, + {95, "ELAST", "protocol error"}, } // Signal table @@ -1640,7 +1875,7 @@ var signalList = [...]struct { {3, "SIGQUIT", "quit"}, {4, "SIGILL", "illegal instruction"}, {5, "SIGTRAP", "trace/BPT trap"}, - {6, "SIGABRT", "abort trap"}, + {6, "SIGIOT", "abort trap"}, {7, "SIGEMT", "EMT trap"}, {8, "SIGFPE", "floating point exception"}, {9, "SIGKILL", "killed"}, @@ -1667,4 +1902,5 @@ var signalList = [...]struct { {30, "SIGUSR1", "user defined signal 1"}, {31, "SIGUSR2", "user defined signal 2"}, {32, "SIGTHR", "thread AST"}, + {28672, "SIGSTKSZ", "unknown signal"}, } diff --git a/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm64.go b/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm64.go index 90de7df..ae16fe7 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_openbsd_arm64.go @@ -112,6 +112,12 @@ const ( BPF_FILDROP_CAPTURE = 0x1 BPF_FILDROP_DROP = 0x2 BPF_FILDROP_PASS = 0x0 + BPF_F_DIR_IN = 0x10 + BPF_F_DIR_MASK = 0x30 + BPF_F_DIR_OUT = 0x20 + BPF_F_DIR_SHIFT = 0x4 + BPF_F_FLOWID = 0x8 + BPF_F_PRI_MASK = 0x7 BPF_H = 0x8 BPF_IMM = 0x0 BPF_IND = 0x40 @@ -140,6 +146,7 @@ const ( BPF_OR = 0x40 BPF_RELEASE = 0x30bb6 BPF_RET = 0x6 + BPF_RND = 0xc0 BPF_RSH = 0x70 BPF_ST = 0x2 BPF_STX = 0x3 @@ -180,7 +187,65 @@ const ( CTL_KERN = 0x1 CTL_MAXNAME = 0xc CTL_NET = 0x4 + DIOCADDQUEUE = 0xc110445d + DIOCADDRULE = 0xcd604404 + DIOCADDSTATE = 0xc1084425 + DIOCCHANGERULE = 0xcd60441a + DIOCCLRIFFLAG = 0xc028445a + DIOCCLRSRCNODES = 0x20004455 + DIOCCLRSTATES = 0xc0e04412 + DIOCCLRSTATUS = 0xc0284416 + DIOCGETLIMIT = 0xc0084427 + DIOCGETQSTATS = 0xc1204460 + DIOCGETQUEUE = 0xc110445f + DIOCGETQUEUES = 0xc110445e + DIOCGETRULE = 0xcd604407 + DIOCGETRULES = 0xcd604406 + DIOCGETRULESET = 0xc444443b + DIOCGETRULESETS = 0xc444443a + DIOCGETSRCNODES = 0xc0104454 + DIOCGETSTATE = 0xc1084413 + DIOCGETSTATES = 0xc0104419 + DIOCGETSTATUS = 0xc1e84415 + DIOCGETSYNFLWATS = 0xc0084463 + DIOCGETTIMEOUT = 0xc008441e + DIOCIGETIFACES = 0xc0284457 + DIOCKILLSRCNODES = 0xc080445b + DIOCKILLSTATES = 0xc0e04429 + DIOCNATLOOK = 0xc0504417 + DIOCOSFPADD = 0xc088444f DIOCOSFPFLUSH = 0x2000444e + DIOCOSFPGET = 0xc0884450 + DIOCRADDADDRS = 0xc4504443 + DIOCRADDTABLES = 0xc450443d + DIOCRCLRADDRS = 0xc4504442 + DIOCRCLRASTATS = 0xc4504448 + DIOCRCLRTABLES = 0xc450443c + DIOCRCLRTSTATS = 0xc4504441 + DIOCRDELADDRS = 0xc4504444 + DIOCRDELTABLES = 0xc450443e + DIOCRGETADDRS = 0xc4504446 + DIOCRGETASTATS = 0xc4504447 + DIOCRGETTABLES = 0xc450443f + DIOCRGETTSTATS = 0xc4504440 + DIOCRINADEFINE = 0xc450444d + DIOCRSETADDRS = 0xc4504445 + DIOCRSETTFLAGS = 0xc450444a + DIOCRTSTADDRS = 0xc4504449 + DIOCSETDEBUG = 0xc0044418 + DIOCSETHOSTID = 0xc0044456 + DIOCSETIFFLAG = 0xc0284459 + DIOCSETLIMIT = 0xc0084428 + DIOCSETREASS = 0xc004445c + DIOCSETSTATUSIF = 0xc0284414 + DIOCSETSYNCOOKIES = 0xc0014462 + DIOCSETSYNFLWATS = 0xc0084461 + DIOCSETTIMEOUT = 0xc008441d + DIOCSTART = 0x20004401 + DIOCSTOP = 0x20004402 + DIOCXBEGIN = 0xc0104451 + DIOCXCOMMIT = 0xc0104452 + DIOCXROLLBACK = 0xc0104453 DLT_ARCNET = 0x7 DLT_ATM_RFC1483 = 0xb DLT_AX25 = 0x3 @@ -243,6 +308,8 @@ const ( EMUL_ENABLED = 0x1 EMUL_NATIVE = 0x2 ENDRUNDISC = 0x9 + ETH64_8021_RSVD_MASK = 0xfffffffffff0 + ETH64_8021_RSVD_PREFIX = 0x180c2000000 ETHERMIN = 0x2e ETHERMTU = 0x5dc ETHERTYPE_8023 = 0x4 @@ -295,6 +362,7 @@ const ( ETHERTYPE_DN = 0x6003 ETHERTYPE_DOGFIGHT = 0x1989 ETHERTYPE_DSMD = 0x8039 + ETHERTYPE_EAPOL = 0x888e ETHERTYPE_ECMA = 0x803 ETHERTYPE_ENCRYPT = 0x803d ETHERTYPE_ES = 0x805d @@ -326,6 +394,7 @@ const ( ETHERTYPE_LLDP = 0x88cc ETHERTYPE_LOGICRAFT = 0x8148 ETHERTYPE_LOOPBACK = 0x9000 + ETHERTYPE_MACSEC = 0x88e5 ETHERTYPE_MATRA = 0x807a ETHERTYPE_MAX = 0xffff ETHERTYPE_MERIT = 0x807c @@ -354,15 +423,16 @@ const ( ETHERTYPE_NCD = 0x8149 ETHERTYPE_NESTAR = 0x8006 ETHERTYPE_NETBEUI = 0x8191 + ETHERTYPE_NHRP = 0x2001 ETHERTYPE_NOVELL = 0x8138 ETHERTYPE_NS = 0x600 ETHERTYPE_NSAT = 0x601 ETHERTYPE_NSCOMPAT = 0x807 + ETHERTYPE_NSH = 0x984f ETHERTYPE_NTRAILER = 0x10 ETHERTYPE_OS9 = 0x7007 ETHERTYPE_OS9NET = 0x7009 ETHERTYPE_PACER = 0x80c6 - ETHERTYPE_PAE = 0x888e ETHERTYPE_PBB = 0x88e7 ETHERTYPE_PCS = 0x4242 ETHERTYPE_PLANNING = 0x8044 @@ -445,10 +515,11 @@ const ( ETHER_VLAN_ENCAP_LEN = 0x4 EVFILT_AIO = -0x3 EVFILT_DEVICE = -0x8 + EVFILT_EXCEPT = -0x9 EVFILT_PROC = -0x5 EVFILT_READ = -0x1 EVFILT_SIGNAL = -0x6 - EVFILT_SYSCOUNT = 0x8 + EVFILT_SYSCOUNT = 0x9 EVFILT_TIMER = -0x7 EVFILT_VNODE = -0x4 EVFILT_WRITE = -0x2 @@ -470,7 +541,7 @@ const ( EV_FLAG1 = 0x2000 EV_ONESHOT = 0x10 EV_RECEIPT = 0x40 - EV_SYSFLAGS = 0xf000 + EV_SYSFLAGS = 0xf800 EXTA = 0x4b00 EXTB = 0x9600 EXTPROC = 0x800 @@ -736,6 +807,7 @@ const ( IFT_VOICEOVERCABLE = 0xc6 IFT_VOICEOVERFRAMERELAY = 0x99 IFT_VOICEOVERIP = 0x68 + IFT_WIREGUARD = 0xfb IFT_X213 = 0x5d IFT_X25 = 0x5 IFT_X25DDN = 0x4 @@ -801,9 +873,11 @@ const ( IPPROTO_RAW = 0xff IPPROTO_ROUTING = 0x2b IPPROTO_RSVP = 0x2e + IPPROTO_SCTP = 0x84 IPPROTO_TCP = 0x6 IPPROTO_TP = 0x1d IPPROTO_UDP = 0x11 + IPPROTO_UDPLITE = 0x88 IPV6_AUTH_LEVEL = 0x35 IPV6_AUTOFLOWLABEL = 0x3b IPV6_CHECKSUM = 0x1a @@ -910,6 +984,9 @@ const ( IP_TTL = 0x4 ISIG = 0x80 ISTRIP = 0x20 + ITIMER_PROF = 0x2 + ITIMER_REAL = 0x0 + ITIMER_VIRTUAL = 0x1 IUCLC = 0x1000 IXANY = 0x800 IXOFF = 0x400 @@ -981,6 +1058,19 @@ const ( MNT_WAIT = 0x1 MNT_WANTRDWR = 0x2000000 MNT_WXALLOWED = 0x800 + MOUNT_AFS = "afs" + MOUNT_CD9660 = "cd9660" + MOUNT_EXT2FS = "ext2fs" + MOUNT_FFS = "ffs" + MOUNT_FUSEFS = "fuse" + MOUNT_MFS = "mfs" + MOUNT_MSDOS = "msdos" + MOUNT_NCPFS = "ncpfs" + MOUNT_NFS = "nfs" + MOUNT_NTFS = "ntfs" + MOUNT_TMPFS = "tmpfs" + MOUNT_UDF = "udf" + MOUNT_UFS = "ffs" MSG_BCAST = 0x100 MSG_CMSG_CLOEXEC = 0x800 MSG_CTRUNC = 0x20 @@ -993,6 +1083,7 @@ const ( MSG_PEEK = 0x2 MSG_TRUNC = 0x10 MSG_WAITALL = 0x40 + MSG_WAITFORONE = 0x1000 MS_ASYNC = 0x1 MS_INVALIDATE = 0x4 MS_SYNC = 0x2 @@ -1001,7 +1092,8 @@ const ( NET_RT_FLAGS = 0x2 NET_RT_IFLIST = 0x3 NET_RT_IFNAMES = 0x6 - NET_RT_MAXID = 0x7 + NET_RT_MAXID = 0x8 + NET_RT_SOURCE = 0x7 NET_RT_STATS = 0x4 NET_RT_TABLE = 0x5 NFDBITS = 0x20 @@ -1018,6 +1110,7 @@ const ( NOTE_FORK = 0x40000000 NOTE_LINK = 0x10 NOTE_LOWAT = 0x1 + NOTE_OOB = 0x4 NOTE_PCTRLMASK = 0xf0000000 NOTE_PDATAMASK = 0xfffff NOTE_RENAME = 0x20 @@ -1154,7 +1247,7 @@ const ( RTM_PROPOSAL = 0x13 RTM_REDIRECT = 0x6 RTM_RESOLVE = 0xb - RTM_RTTUNIT = 0xf4240 + RTM_SOURCE = 0x16 RTM_VERSION = 0x5 RTV_EXPIRE = 0x4 RTV_HOPCOUNT = 0x2 @@ -1172,6 +1265,9 @@ const ( RUSAGE_THREAD = 0x1 SCM_RIGHTS = 0x1 SCM_TIMESTAMP = 0x4 + SEEK_CUR = 0x1 + SEEK_END = 0x2 + SEEK_SET = 0x0 SHUT_RD = 0x0 SHUT_RDWR = 0x2 SHUT_WR = 0x1 @@ -1188,30 +1284,30 @@ const ( SIOCBRDGDELS = 0x80606942 SIOCBRDGFLUSH = 0x80606948 SIOCBRDGFRL = 0x808c694e - SIOCBRDGGCACHE = 0xc0186941 - SIOCBRDGGFD = 0xc0186952 - SIOCBRDGGHT = 0xc0186951 + SIOCBRDGGCACHE = 0xc0146941 + SIOCBRDGGFD = 0xc0146952 + SIOCBRDGGHT = 0xc0146951 SIOCBRDGGIFFLGS = 0xc060693e - SIOCBRDGGMA = 0xc0186953 + SIOCBRDGGMA = 0xc0146953 SIOCBRDGGPARAM = 0xc0406958 - SIOCBRDGGPRI = 0xc0186950 + SIOCBRDGGPRI = 0xc0146950 SIOCBRDGGRL = 0xc030694f - SIOCBRDGGTO = 0xc0186946 + SIOCBRDGGTO = 0xc0146946 SIOCBRDGIFS = 0xc0606942 SIOCBRDGRTS = 0xc0206943 SIOCBRDGSADDR = 0xc1286944 - SIOCBRDGSCACHE = 0x80186940 - SIOCBRDGSFD = 0x80186952 - SIOCBRDGSHT = 0x80186951 + SIOCBRDGSCACHE = 0x80146940 + SIOCBRDGSFD = 0x80146952 + SIOCBRDGSHT = 0x80146951 SIOCBRDGSIFCOST = 0x80606955 SIOCBRDGSIFFLGS = 0x8060693f SIOCBRDGSIFPRIO = 0x80606954 SIOCBRDGSIFPROT = 0x8060694a - SIOCBRDGSMA = 0x80186953 - SIOCBRDGSPRI = 0x80186950 - SIOCBRDGSPROTO = 0x8018695a - SIOCBRDGSTO = 0x80186945 - SIOCBRDGSTXHC = 0x80186959 + SIOCBRDGSMA = 0x80146953 + SIOCBRDGSPRI = 0x80146950 + SIOCBRDGSPROTO = 0x8014695a + SIOCBRDGSTO = 0x80146945 + SIOCBRDGSTXHC = 0x80146959 SIOCDELLABEL = 0x80206997 SIOCDELMULTI = 0x80206932 SIOCDIFADDR = 0x80206919 @@ -1264,6 +1360,7 @@ const ( SIOCGPWE3CTRLWORD = 0xc02069dc SIOCGPWE3FAT = 0xc02069dd SIOCGPWE3NEIGHBOR = 0xc21869de + SIOCGRXHPRIO = 0xc02069db SIOCGSPPPPARAMS = 0xc0206994 SIOCGTXHPRIO = 0xc02069c6 SIOCGUMBINFO = 0xc02069be @@ -1310,17 +1407,13 @@ const ( SIOCSPWE3CTRLWORD = 0x802069dc SIOCSPWE3FAT = 0x802069dd SIOCSPWE3NEIGHBOR = 0x821869de + SIOCSRXHPRIO = 0x802069db SIOCSSPPPPARAMS = 0x80206993 SIOCSTXHPRIO = 0x802069c5 SIOCSUMBPARAM = 0x802069bf SIOCSVH = 0xc02069f5 SIOCSVNETFLOWID = 0x802069c3 SIOCSVNETID = 0x802069a6 - SIOCSWGDPID = 0xc018695b - SIOCSWGMAXFLOW = 0xc0186960 - SIOCSWGMAXGROUP = 0xc018695d - SIOCSWSDPID = 0x8018695c - SIOCSWSPORTNO = 0xc060695f SOCK_CLOEXEC = 0x8000 SOCK_DGRAM = 0x2 SOCK_DNS = 0x1000 @@ -1335,6 +1428,7 @@ const ( SO_BINDANY = 0x1000 SO_BROADCAST = 0x20 SO_DEBUG = 0x1 + SO_DOMAIN = 0x1024 SO_DONTROUTE = 0x10 SO_ERROR = 0x1007 SO_KEEPALIVE = 0x8 @@ -1342,6 +1436,7 @@ const ( SO_NETPROC = 0x1020 SO_OOBINLINE = 0x100 SO_PEERCRED = 0x1022 + SO_PROTOCOL = 0x1025 SO_RCVBUF = 0x1002 SO_RCVLOWAT = 0x1004 SO_RCVTIMEO = 0x1006 @@ -1391,7 +1486,18 @@ const ( TCOFLUSH = 0x2 TCOOFF = 0x1 TCOON = 0x2 - TCP_MAXBURST = 0x4 + TCPOPT_EOL = 0x0 + TCPOPT_MAXSEG = 0x2 + TCPOPT_NOP = 0x1 + TCPOPT_SACK = 0x5 + TCPOPT_SACK_HDR = 0x1010500 + TCPOPT_SACK_PERMITTED = 0x4 + TCPOPT_SACK_PERMIT_HDR = 0x1010402 + TCPOPT_SIGNATURE = 0x13 + TCPOPT_TIMESTAMP = 0x8 + TCPOPT_TSTAMP_HDR = 0x101080a + TCPOPT_WINDOW = 0x3 + TCP_INFO = 0x9 TCP_MAXSEG = 0x2 TCP_MAXWIN = 0xffff TCP_MAX_SACK = 0x3 @@ -1400,6 +1506,7 @@ const ( TCP_MSS = 0x200 TCP_NODELAY = 0x1 TCP_NOPUSH = 0x10 + TCP_SACKHOLE_LIMIT = 0x80 TCP_SACK_ENABLE = 0x8 TCSAFLUSH = 0x2 TIMER_ABSTIME = 0x1 @@ -1768,7 +1875,7 @@ var signalList = [...]struct { {3, "SIGQUIT", "quit"}, {4, "SIGILL", "illegal instruction"}, {5, "SIGTRAP", "trace/BPT trap"}, - {6, "SIGABRT", "abort trap"}, + {6, "SIGIOT", "abort trap"}, {7, "SIGEMT", "EMT trap"}, {8, "SIGFPE", "floating point exception"}, {9, "SIGKILL", "killed"}, @@ -1795,4 +1902,5 @@ var signalList = [...]struct { {30, "SIGUSR1", "user defined signal 1"}, {31, "SIGUSR2", "user defined signal 2"}, {32, "SIGTHR", "thread AST"}, + {28672, "SIGSTKSZ", "unknown signal"}, } diff --git a/vendor/golang.org/x/sys/unix/zerrors_openbsd_mips64.go b/vendor/golang.org/x/sys/unix/zerrors_openbsd_mips64.go index f1154ff..03d90fe 100644 --- a/vendor/golang.org/x/sys/unix/zerrors_openbsd_mips64.go +++ b/vendor/golang.org/x/sys/unix/zerrors_openbsd_mips64.go @@ -112,6 +112,12 @@ const ( BPF_FILDROP_CAPTURE = 0x1 BPF_FILDROP_DROP = 0x2 BPF_FILDROP_PASS = 0x0 + BPF_F_DIR_IN = 0x10 + BPF_F_DIR_MASK = 0x30 + BPF_F_DIR_OUT = 0x20 + BPF_F_DIR_SHIFT = 0x4 + BPF_F_FLOWID = 0x8 + BPF_F_PRI_MASK = 0x7 BPF_H = 0x8 BPF_IMM = 0x0 BPF_IND = 0x40 @@ -140,6 +146,7 @@ const ( BPF_OR = 0x40 BPF_RELEASE = 0x30bb6 BPF_RET = 0x6 + BPF_RND = 0xc0 BPF_RSH = 0x70 BPF_ST = 0x2 BPF_STX = 0x3 @@ -301,6 +308,8 @@ const ( EMUL_ENABLED = 0x1 EMUL_NATIVE = 0x2 ENDRUNDISC = 0x9 + ETH64_8021_RSVD_MASK = 0xfffffffffff0 + ETH64_8021_RSVD_PREFIX = 0x180c2000000 ETHERMIN = 0x2e ETHERMTU = 0x5dc ETHERTYPE_8023 = 0x4 @@ -353,6 +362,7 @@ const ( ETHERTYPE_DN = 0x6003 ETHERTYPE_DOGFIGHT = 0x1989 ETHERTYPE_DSMD = 0x8039 + ETHERTYPE_EAPOL = 0x888e ETHERTYPE_ECMA = 0x803 ETHERTYPE_ENCRYPT = 0x803d ETHERTYPE_ES = 0x805d @@ -413,15 +423,16 @@ const ( ETHERTYPE_NCD = 0x8149 ETHERTYPE_NESTAR = 0x8006 ETHERTYPE_NETBEUI = 0x8191 + ETHERTYPE_NHRP = 0x2001 ETHERTYPE_NOVELL = 0x8138 ETHERTYPE_NS = 0x600 ETHERTYPE_NSAT = 0x601 ETHERTYPE_NSCOMPAT = 0x807 + ETHERTYPE_NSH = 0x984f ETHERTYPE_NTRAILER = 0x10 ETHERTYPE_OS9 = 0x7007 ETHERTYPE_OS9NET = 0x7009 ETHERTYPE_PACER = 0x80c6 - ETHERTYPE_PAE = 0x888e ETHERTYPE_PBB = 0x88e7 ETHERTYPE_PCS = 0x4242 ETHERTYPE_PLANNING = 0x8044 @@ -504,10 +515,11 @@ const ( ETHER_VLAN_ENCAP_LEN = 0x4 EVFILT_AIO = -0x3 EVFILT_DEVICE = -0x8 + EVFILT_EXCEPT = -0x9 EVFILT_PROC = -0x5 EVFILT_READ = -0x1 EVFILT_SIGNAL = -0x6 - EVFILT_SYSCOUNT = 0x8 + EVFILT_SYSCOUNT = 0x9 EVFILT_TIMER = -0x7 EVFILT_VNODE = -0x4 EVFILT_WRITE = -0x2 @@ -529,7 +541,7 @@ const ( EV_FLAG1 = 0x2000 EV_ONESHOT = 0x10 EV_RECEIPT = 0x40 - EV_SYSFLAGS = 0xf000 + EV_SYSFLAGS = 0xf800 EXTA = 0x4b00 EXTB = 0x9600 EXTPROC = 0x800 @@ -795,6 +807,7 @@ const ( IFT_VOICEOVERCABLE = 0xc6 IFT_VOICEOVERFRAMERELAY = 0x99 IFT_VOICEOVERIP = 0x68 + IFT_WIREGUARD = 0xfb IFT_X213 = 0x5d IFT_X25 = 0x5 IFT_X25DDN = 0x4 @@ -860,6 +873,7 @@ const ( IPPROTO_RAW = 0xff IPPROTO_ROUTING = 0x2b IPPROTO_RSVP = 0x2e + IPPROTO_SCTP = 0x84 IPPROTO_TCP = 0x6 IPPROTO_TP = 0x1d IPPROTO_UDP = 0x11 @@ -970,6 +984,9 @@ const ( IP_TTL = 0x4 ISIG = 0x80 ISTRIP = 0x20 + ITIMER_PROF = 0x2 + ITIMER_REAL = 0x0 + ITIMER_VIRTUAL = 0x1 IUCLC = 0x1000 IXANY = 0x800 IXOFF = 0x400 @@ -1041,6 +1058,19 @@ const ( MNT_WAIT = 0x1 MNT_WANTRDWR = 0x2000000 MNT_WXALLOWED = 0x800 + MOUNT_AFS = "afs" + MOUNT_CD9660 = "cd9660" + MOUNT_EXT2FS = "ext2fs" + MOUNT_FFS = "ffs" + MOUNT_FUSEFS = "fuse" + MOUNT_MFS = "mfs" + MOUNT_MSDOS = "msdos" + MOUNT_NCPFS = "ncpfs" + MOUNT_NFS = "nfs" + MOUNT_NTFS = "ntfs" + MOUNT_TMPFS = "tmpfs" + MOUNT_UDF = "udf" + MOUNT_UFS = "ffs" MSG_BCAST = 0x100 MSG_CMSG_CLOEXEC = 0x800 MSG_CTRUNC = 0x20 @@ -1053,6 +1083,7 @@ const ( MSG_PEEK = 0x2 MSG_TRUNC = 0x10 MSG_WAITALL = 0x40 + MSG_WAITFORONE = 0x1000 MS_ASYNC = 0x1 MS_INVALIDATE = 0x4 MS_SYNC = 0x2 @@ -1061,7 +1092,8 @@ const ( NET_RT_FLAGS = 0x2 NET_RT_IFLIST = 0x3 NET_RT_IFNAMES = 0x6 - NET_RT_MAXID = 0x7 + NET_RT_MAXID = 0x8 + NET_RT_SOURCE = 0x7 NET_RT_STATS = 0x4 NET_RT_TABLE = 0x5 NFDBITS = 0x20 @@ -1078,6 +1110,7 @@ const ( NOTE_FORK = 0x40000000 NOTE_LINK = 0x10 NOTE_LOWAT = 0x1 + NOTE_OOB = 0x4 NOTE_PCTRLMASK = 0xf0000000 NOTE_PDATAMASK = 0xfffff NOTE_RENAME = 0x20 @@ -1214,7 +1247,7 @@ const ( RTM_PROPOSAL = 0x13 RTM_REDIRECT = 0x6 RTM_RESOLVE = 0xb - RTM_RTTUNIT = 0xf4240 + RTM_SOURCE = 0x16 RTM_VERSION = 0x5 RTV_EXPIRE = 0x4 RTV_HOPCOUNT = 0x2 @@ -1232,6 +1265,9 @@ const ( RUSAGE_THREAD = 0x1 SCM_RIGHTS = 0x1 SCM_TIMESTAMP = 0x4 + SEEK_CUR = 0x1 + SEEK_END = 0x2 + SEEK_SET = 0x0 SHUT_RD = 0x0 SHUT_RDWR = 0x2 SHUT_WR = 0x1 @@ -1248,30 +1284,30 @@ const ( SIOCBRDGDELS = 0x80606942 SIOCBRDGFLUSH = 0x80606948 SIOCBRDGFRL = 0x808c694e - SIOCBRDGGCACHE = 0xc0186941 - SIOCBRDGGFD = 0xc0186952 - SIOCBRDGGHT = 0xc0186951 + SIOCBRDGGCACHE = 0xc0146941 + SIOCBRDGGFD = 0xc0146952 + SIOCBRDGGHT = 0xc0146951 SIOCBRDGGIFFLGS = 0xc060693e - SIOCBRDGGMA = 0xc0186953 + SIOCBRDGGMA = 0xc0146953 SIOCBRDGGPARAM = 0xc0406958 - SIOCBRDGGPRI = 0xc0186950 + SIOCBRDGGPRI = 0xc0146950 SIOCBRDGGRL = 0xc030694f - SIOCBRDGGTO = 0xc0186946 + SIOCBRDGGTO = 0xc0146946 SIOCBRDGIFS = 0xc0606942 SIOCBRDGRTS = 0xc0206943 SIOCBRDGSADDR = 0xc1286944 - SIOCBRDGSCACHE = 0x80186940 - SIOCBRDGSFD = 0x80186952 - SIOCBRDGSHT = 0x80186951 + SIOCBRDGSCACHE = 0x80146940 + SIOCBRDGSFD = 0x80146952 + SIOCBRDGSHT = 0x80146951 SIOCBRDGSIFCOST = 0x80606955 SIOCBRDGSIFFLGS = 0x8060693f SIOCBRDGSIFPRIO = 0x80606954 SIOCBRDGSIFPROT = 0x8060694a - SIOCBRDGSMA = 0x80186953 - SIOCBRDGSPRI = 0x80186950 - SIOCBRDGSPROTO = 0x8018695a - SIOCBRDGSTO = 0x80186945 - SIOCBRDGSTXHC = 0x80186959 + SIOCBRDGSMA = 0x80146953 + SIOCBRDGSPRI = 0x80146950 + SIOCBRDGSPROTO = 0x8014695a + SIOCBRDGSTO = 0x80146945 + SIOCBRDGSTXHC = 0x80146959 SIOCDELLABEL = 0x80206997 SIOCDELMULTI = 0x80206932 SIOCDIFADDR = 0x80206919 @@ -1378,11 +1414,6 @@ const ( SIOCSVH = 0xc02069f5 SIOCSVNETFLOWID = 0x802069c3 SIOCSVNETID = 0x802069a6 - SIOCSWGDPID = 0xc018695b - SIOCSWGMAXFLOW = 0xc0186960 - SIOCSWGMAXGROUP = 0xc018695d - SIOCSWSDPID = 0x8018695c - SIOCSWSPORTNO = 0xc060695f SOCK_CLOEXEC = 0x8000 SOCK_DGRAM = 0x2 SOCK_DNS = 0x1000 @@ -1455,7 +1486,18 @@ const ( TCOFLUSH = 0x2 TCOOFF = 0x1 TCOON = 0x2 - TCP_MAXBURST = 0x4 + TCPOPT_EOL = 0x0 + TCPOPT_MAXSEG = 0x2 + TCPOPT_NOP = 0x1 + TCPOPT_SACK = 0x5 + TCPOPT_SACK_HDR = 0x1010500 + TCPOPT_SACK_PERMITTED = 0x4 + TCPOPT_SACK_PERMIT_HDR = 0x1010402 + TCPOPT_SIGNATURE = 0x13 + TCPOPT_TIMESTAMP = 0x8 + TCPOPT_TSTAMP_HDR = 0x101080a + TCPOPT_WINDOW = 0x3 + TCP_INFO = 0x9 TCP_MAXSEG = 0x2 TCP_MAXWIN = 0xffff TCP_MAX_SACK = 0x3 @@ -1833,7 +1875,7 @@ var signalList = [...]struct { {3, "SIGQUIT", "quit"}, {4, "SIGILL", "illegal instruction"}, {5, "SIGTRAP", "trace/BPT trap"}, - {6, "SIGABRT", "abort trap"}, + {6, "SIGIOT", "abort trap"}, {7, "SIGEMT", "EMT trap"}, {8, "SIGFPE", "floating point exception"}, {9, "SIGKILL", "killed"}, @@ -1860,4 +1902,5 @@ var signalList = [...]struct { {30, "SIGUSR1", "user defined signal 1"}, {31, "SIGUSR2", "user defined signal 2"}, {32, "SIGTHR", "thread AST"}, + {81920, "SIGSTKSZ", "unknown signal"}, } diff --git a/vendor/golang.org/x/sys/unix/zptrace_armnn_linux.go b/vendor/golang.org/x/sys/unix/zptrace_armnn_linux.go index bd001a6..97f20ca 100644 --- a/vendor/golang.org/x/sys/unix/zptrace_armnn_linux.go +++ b/vendor/golang.org/x/sys/unix/zptrace_armnn_linux.go @@ -15,12 +15,12 @@ type PtraceRegsArm struct { // PtraceGetRegsArm fetches the registers used by arm binaries. func PtraceGetRegsArm(pid int, regsout *PtraceRegsArm) error { - return ptrace(PTRACE_GETREGS, pid, 0, uintptr(unsafe.Pointer(regsout))) + return ptracePtr(PTRACE_GETREGS, pid, 0, unsafe.Pointer(regsout)) } // PtraceSetRegsArm sets the registers used by arm binaries. func PtraceSetRegsArm(pid int, regs *PtraceRegsArm) error { - return ptrace(PTRACE_SETREGS, pid, 0, uintptr(unsafe.Pointer(regs))) + return ptracePtr(PTRACE_SETREGS, pid, 0, unsafe.Pointer(regs)) } // PtraceRegsArm64 is the registers used by arm64 binaries. @@ -33,10 +33,10 @@ type PtraceRegsArm64 struct { // PtraceGetRegsArm64 fetches the registers used by arm64 binaries. func PtraceGetRegsArm64(pid int, regsout *PtraceRegsArm64) error { - return ptrace(PTRACE_GETREGS, pid, 0, uintptr(unsafe.Pointer(regsout))) + return ptracePtr(PTRACE_GETREGS, pid, 0, unsafe.Pointer(regsout)) } // PtraceSetRegsArm64 sets the registers used by arm64 binaries. func PtraceSetRegsArm64(pid int, regs *PtraceRegsArm64) error { - return ptrace(PTRACE_SETREGS, pid, 0, uintptr(unsafe.Pointer(regs))) + return ptracePtr(PTRACE_SETREGS, pid, 0, unsafe.Pointer(regs)) } diff --git a/vendor/golang.org/x/sys/unix/zptrace_linux_arm64.go b/vendor/golang.org/x/sys/unix/zptrace_linux_arm64.go index 6cb6d68..834d285 100644 --- a/vendor/golang.org/x/sys/unix/zptrace_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zptrace_linux_arm64.go @@ -7,11 +7,11 @@ import "unsafe" // PtraceGetRegSetArm64 fetches the registers used by arm64 binaries. func PtraceGetRegSetArm64(pid, addr int, regsout *PtraceRegsArm64) error { iovec := Iovec{(*byte)(unsafe.Pointer(regsout)), uint64(unsafe.Sizeof(*regsout))} - return ptrace(PTRACE_GETREGSET, pid, uintptr(addr), uintptr(unsafe.Pointer(&iovec))) + return ptracePtr(PTRACE_GETREGSET, pid, uintptr(addr), unsafe.Pointer(&iovec)) } // PtraceSetRegSetArm64 sets the registers used by arm64 binaries. func PtraceSetRegSetArm64(pid, addr int, regs *PtraceRegsArm64) error { iovec := Iovec{(*byte)(unsafe.Pointer(regs)), uint64(unsafe.Sizeof(*regs))} - return ptrace(PTRACE_SETREGSET, pid, uintptr(addr), uintptr(unsafe.Pointer(&iovec))) + return ptracePtr(PTRACE_SETREGSET, pid, uintptr(addr), unsafe.Pointer(&iovec)) } diff --git a/vendor/golang.org/x/sys/unix/zptrace_mipsnn_linux.go b/vendor/golang.org/x/sys/unix/zptrace_mipsnn_linux.go index c34d063..0b5f794 100644 --- a/vendor/golang.org/x/sys/unix/zptrace_mipsnn_linux.go +++ b/vendor/golang.org/x/sys/unix/zptrace_mipsnn_linux.go @@ -21,12 +21,12 @@ type PtraceRegsMips struct { // PtraceGetRegsMips fetches the registers used by mips binaries. func PtraceGetRegsMips(pid int, regsout *PtraceRegsMips) error { - return ptrace(PTRACE_GETREGS, pid, 0, uintptr(unsafe.Pointer(regsout))) + return ptracePtr(PTRACE_GETREGS, pid, 0, unsafe.Pointer(regsout)) } // PtraceSetRegsMips sets the registers used by mips binaries. func PtraceSetRegsMips(pid int, regs *PtraceRegsMips) error { - return ptrace(PTRACE_SETREGS, pid, 0, uintptr(unsafe.Pointer(regs))) + return ptracePtr(PTRACE_SETREGS, pid, 0, unsafe.Pointer(regs)) } // PtraceRegsMips64 is the registers used by mips64 binaries. @@ -42,10 +42,10 @@ type PtraceRegsMips64 struct { // PtraceGetRegsMips64 fetches the registers used by mips64 binaries. func PtraceGetRegsMips64(pid int, regsout *PtraceRegsMips64) error { - return ptrace(PTRACE_GETREGS, pid, 0, uintptr(unsafe.Pointer(regsout))) + return ptracePtr(PTRACE_GETREGS, pid, 0, unsafe.Pointer(regsout)) } // PtraceSetRegsMips64 sets the registers used by mips64 binaries. func PtraceSetRegsMips64(pid int, regs *PtraceRegsMips64) error { - return ptrace(PTRACE_SETREGS, pid, 0, uintptr(unsafe.Pointer(regs))) + return ptracePtr(PTRACE_SETREGS, pid, 0, unsafe.Pointer(regs)) } diff --git a/vendor/golang.org/x/sys/unix/zptrace_mipsnnle_linux.go b/vendor/golang.org/x/sys/unix/zptrace_mipsnnle_linux.go index 3ccf0c0..2807f7e 100644 --- a/vendor/golang.org/x/sys/unix/zptrace_mipsnnle_linux.go +++ b/vendor/golang.org/x/sys/unix/zptrace_mipsnnle_linux.go @@ -21,12 +21,12 @@ type PtraceRegsMipsle struct { // PtraceGetRegsMipsle fetches the registers used by mipsle binaries. func PtraceGetRegsMipsle(pid int, regsout *PtraceRegsMipsle) error { - return ptrace(PTRACE_GETREGS, pid, 0, uintptr(unsafe.Pointer(regsout))) + return ptracePtr(PTRACE_GETREGS, pid, 0, unsafe.Pointer(regsout)) } // PtraceSetRegsMipsle sets the registers used by mipsle binaries. func PtraceSetRegsMipsle(pid int, regs *PtraceRegsMipsle) error { - return ptrace(PTRACE_SETREGS, pid, 0, uintptr(unsafe.Pointer(regs))) + return ptracePtr(PTRACE_SETREGS, pid, 0, unsafe.Pointer(regs)) } // PtraceRegsMips64le is the registers used by mips64le binaries. @@ -42,10 +42,10 @@ type PtraceRegsMips64le struct { // PtraceGetRegsMips64le fetches the registers used by mips64le binaries. func PtraceGetRegsMips64le(pid int, regsout *PtraceRegsMips64le) error { - return ptrace(PTRACE_GETREGS, pid, 0, uintptr(unsafe.Pointer(regsout))) + return ptracePtr(PTRACE_GETREGS, pid, 0, unsafe.Pointer(regsout)) } // PtraceSetRegsMips64le sets the registers used by mips64le binaries. func PtraceSetRegsMips64le(pid int, regs *PtraceRegsMips64le) error { - return ptrace(PTRACE_SETREGS, pid, 0, uintptr(unsafe.Pointer(regs))) + return ptracePtr(PTRACE_SETREGS, pid, 0, unsafe.Pointer(regs)) } diff --git a/vendor/golang.org/x/sys/unix/zptrace_x86_linux.go b/vendor/golang.org/x/sys/unix/zptrace_x86_linux.go index 7d65857..281ea64 100644 --- a/vendor/golang.org/x/sys/unix/zptrace_x86_linux.go +++ b/vendor/golang.org/x/sys/unix/zptrace_x86_linux.go @@ -31,12 +31,12 @@ type PtraceRegs386 struct { // PtraceGetRegs386 fetches the registers used by 386 binaries. func PtraceGetRegs386(pid int, regsout *PtraceRegs386) error { - return ptrace(PTRACE_GETREGS, pid, 0, uintptr(unsafe.Pointer(regsout))) + return ptracePtr(PTRACE_GETREGS, pid, 0, unsafe.Pointer(regsout)) } // PtraceSetRegs386 sets the registers used by 386 binaries. func PtraceSetRegs386(pid int, regs *PtraceRegs386) error { - return ptrace(PTRACE_SETREGS, pid, 0, uintptr(unsafe.Pointer(regs))) + return ptracePtr(PTRACE_SETREGS, pid, 0, unsafe.Pointer(regs)) } // PtraceRegsAmd64 is the registers used by amd64 binaries. @@ -72,10 +72,10 @@ type PtraceRegsAmd64 struct { // PtraceGetRegsAmd64 fetches the registers used by amd64 binaries. func PtraceGetRegsAmd64(pid int, regsout *PtraceRegsAmd64) error { - return ptrace(PTRACE_GETREGS, pid, 0, uintptr(unsafe.Pointer(regsout))) + return ptracePtr(PTRACE_GETREGS, pid, 0, unsafe.Pointer(regsout)) } // PtraceSetRegsAmd64 sets the registers used by amd64 binaries. func PtraceSetRegsAmd64(pid int, regs *PtraceRegsAmd64) error { - return ptrace(PTRACE_SETREGS, pid, 0, uintptr(unsafe.Pointer(regs))) + return ptracePtr(PTRACE_SETREGS, pid, 0, unsafe.Pointer(regs)) } diff --git a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc.go b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc.go index 870215d..9a25721 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc.go @@ -124,7 +124,6 @@ int utime(uintptr_t, uintptr_t); unsigned long long getsystemcfg(int); int umount(uintptr_t); int getrlimit64(int, uintptr_t); -int setrlimit64(int, uintptr_t); long long lseek64(int, long long, int); uintptr_t mmap(uintptr_t, uintptr_t, int, int, int, long long); @@ -213,7 +212,7 @@ func wait4(pid Pid_t, status *_C_int, options int, rusage *Rusage) (wpid Pid_t, // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctl(fd int, req uint, arg uintptr) (err error) { +func ioctl(fd int, req int, arg uintptr) (err error) { r0, er := C.ioctl(C.int(fd), C.int(req), C.uintptr_t(arg)) if r0 == -1 && er != nil { err = er @@ -223,6 +222,16 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ioctlPtr(fd int, req int, arg unsafe.Pointer) (err error) { + r0, er := C.ioctl(C.int(fd), C.int(req), C.uintptr_t(uintptr(arg))) + if r0 == -1 && er != nil { + err = er + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func FcntlInt(fd uintptr, cmd int, arg int) (r int, err error) { r0, er := C.fcntl(C.uintptr_t(fd), C.int(cmd), C.uintptr_t(arg)) r = int(r0) @@ -1454,16 +1463,6 @@ func Getrlimit(resource int, rlim *Rlimit) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - r0, er := C.setrlimit64(C.int(resource), C.uintptr_t(uintptr(unsafe.Pointer(rlim)))) - if r0 == -1 && er != nil { - err = er - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Seek(fd int, offset int64, whence int) (off int64, err error) { r0, er := C.lseek64(C.int(fd), C.longlong(offset), C.int(whence)) off = int64(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64.go b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64.go index a89b0bf..6de80c2 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64.go @@ -93,8 +93,18 @@ func wait4(pid Pid_t, status *_C_int, options int, rusage *Rusage) (wpid Pid_t, // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctl(fd int, req uint, arg uintptr) (err error) { - _, e1 := callioctl(fd, int(req), arg) +func ioctl(fd int, req int, arg uintptr) (err error) { + _, e1 := callioctl(fd, req, arg) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func ioctlPtr(fd int, req int, arg unsafe.Pointer) (err error) { + _, e1 := callioctl_ptr(fd, req, arg) if e1 != 0 { err = errnoErr(e1) } @@ -1412,16 +1422,6 @@ func Getrlimit(resource int, rlim *Rlimit) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, e1 := callsetrlimit(resource, uintptr(unsafe.Pointer(rlim))) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Seek(fd int, offset int64, whence int) (off int64, err error) { r0, e1 := calllseek(fd, offset, whence) off = int64(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gc.go b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gc.go index 2caa5ad..c4d50ae 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gc.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gc.go @@ -124,7 +124,6 @@ import ( //go:cgo_import_dynamic libc_getsystemcfg getsystemcfg "libc.a/shr_64.o" //go:cgo_import_dynamic libc_umount umount "libc.a/shr_64.o" //go:cgo_import_dynamic libc_getrlimit getrlimit "libc.a/shr_64.o" -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.a/shr_64.o" //go:cgo_import_dynamic libc_lseek lseek "libc.a/shr_64.o" //go:cgo_import_dynamic libc_mmap64 mmap64 "libc.a/shr_64.o" @@ -242,7 +241,6 @@ import ( //go:linkname libc_getsystemcfg libc_getsystemcfg //go:linkname libc_umount libc_umount //go:linkname libc_getrlimit libc_getrlimit -//go:linkname libc_setrlimit libc_setrlimit //go:linkname libc_lseek libc_lseek //go:linkname libc_mmap64 libc_mmap64 @@ -363,7 +361,6 @@ var ( libc_getsystemcfg, libc_umount, libc_getrlimit, - libc_setrlimit, libc_lseek, libc_mmap64 syscallFunc ) @@ -423,6 +420,13 @@ func callioctl(fd int, req int, arg uintptr) (r1 uintptr, e1 Errno) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func callioctl_ptr(fd int, req int, arg unsafe.Pointer) (r1 uintptr, e1 Errno) { + r1, _, e1 = syscall6(uintptr(unsafe.Pointer(&libc_ioctl)), 3, uintptr(fd), uintptr(req), uintptr(arg), 0, 0, 0) + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func callfcntl(fd uintptr, cmd int, arg uintptr) (r1 uintptr, e1 Errno) { r1, _, e1 = syscall6(uintptr(unsafe.Pointer(&libc_fcntl)), 3, fd, uintptr(cmd), arg, 0, 0, 0) return @@ -1172,13 +1176,6 @@ func callgetrlimit(resource int, rlim uintptr) (r1 uintptr, e1 Errno) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func callsetrlimit(resource int, rlim uintptr) (r1 uintptr, e1 Errno) { - r1, _, e1 = rawSyscall6(uintptr(unsafe.Pointer(&libc_setrlimit)), 2, uintptr(resource), rlim, 0, 0, 0, 0) - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func calllseek(fd int, offset int64, whence int) (r1 uintptr, e1 Errno) { r1, _, e1 = syscall6(uintptr(unsafe.Pointer(&libc_lseek)), 3, uintptr(fd), uintptr(offset), uintptr(whence), 0, 0, 0) return diff --git a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gccgo.go b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gccgo.go index 944a714..6903d3b 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gccgo.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_aix_ppc64_gccgo.go @@ -123,7 +123,6 @@ int utime(uintptr_t, uintptr_t); unsigned long long getsystemcfg(int); int umount(uintptr_t); int getrlimit(int, uintptr_t); -int setrlimit(int, uintptr_t); long long lseek(int, long long, int); uintptr_t mmap64(uintptr_t, uintptr_t, int, int, int, long long); @@ -131,6 +130,7 @@ uintptr_t mmap64(uintptr_t, uintptr_t, int, int, int, long long); import "C" import ( "syscall" + "unsafe" ) // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT @@ -191,6 +191,14 @@ func callioctl(fd int, req int, arg uintptr) (r1 uintptr, e1 Errno) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func callioctl_ptr(fd int, req int, arg unsafe.Pointer) (r1 uintptr, e1 Errno) { + r1 = uintptr(C.ioctl(C.int(fd), C.int(req), C.uintptr_t(uintptr(arg)))) + e1 = syscall.GetErrno() + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func callfcntl(fd uintptr, cmd int, arg uintptr) (r1 uintptr, e1 Errno) { r1 = uintptr(C.fcntl(C.uintptr_t(fd), C.int(cmd), C.uintptr_t(arg))) e1 = syscall.GetErrno() @@ -1047,14 +1055,6 @@ func callgetrlimit(resource int, rlim uintptr) (r1 uintptr, e1 Errno) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func callsetrlimit(resource int, rlim uintptr) (r1 uintptr, e1 Errno) { - r1 = uintptr(C.setrlimit(C.int(resource), C.uintptr_t(rlim))) - e1 = syscall.GetErrno() - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func calllseek(fd int, offset int64, whence int) (r1 uintptr, e1 Errno) { r1 = uintptr(C.lseek(C.int(fd), C.longlong(offset), C.int(whence))) e1 = syscall.GetErrno() diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go index c2461c4..4037ccf 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.go @@ -725,6 +725,14 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { return } +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := syscall_syscall(libc_ioctl_trampoline_addr, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + var libc_ioctl_trampoline_addr uintptr //go:cgo_import_dynamic libc_ioctl ioctl "/usr/lib/libSystem.B.dylib" @@ -1984,6 +1992,31 @@ var libc_select_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func Setattrlist(path string, attrlist *Attrlist, attrBuf []byte, options int) (err error) { + var _p0 *byte + _p0, err = BytePtrFromString(path) + if err != nil { + return + } + var _p1 unsafe.Pointer + if len(attrBuf) > 0 { + _p1 = unsafe.Pointer(&attrBuf[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + _, _, e1 := syscall_syscall6(libc_setattrlist_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(attrlist)), uintptr(_p1), uintptr(len(attrBuf)), uintptr(options), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_setattrlist_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setattrlist setattrlist "/usr/lib/libSystem.B.dylib" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Setegid(egid int) (err error) { _, _, e1 := syscall_syscall(libc_setegid_trampoline_addr, uintptr(egid), 0, 0) if e1 != 0 { @@ -2115,20 +2148,6 @@ var libc_setreuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "/usr/lib/libSystem.B.dylib" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := syscall_rawSyscall(libc_setsid_trampoline_addr, 0, 0, 0) pid = int(r0) @@ -2502,6 +2521,14 @@ func ptrace1(request int, pid int, addr uintptr, data uintptr) (err error) { return } +func ptrace1Ptr(request int, pid int, addr uintptr, data unsafe.Pointer) (err error) { + _, _, e1 := syscall_syscall6(libc_ptrace_trampoline_addr, uintptr(request), uintptr(pid), addr, uintptr(data), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + var libc_ptrace_trampoline_addr uintptr //go:cgo_import_dynamic libc_ptrace ptrace "/usr/lib/libSystem.B.dylib" diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s index 95fe4c0..4baaed0 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_amd64.s @@ -705,6 +705,11 @@ TEXT libc_select_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_select_trampoline_addr(SB), RODATA, $8 DATA ·libc_select_trampoline_addr(SB)/8, $libc_select_trampoline<>(SB) +TEXT libc_setattrlist_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setattrlist(SB) +GLOBL ·libc_setattrlist_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setattrlist_trampoline_addr(SB)/8, $libc_setattrlist_trampoline<>(SB) + TEXT libc_setegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setegid(SB) @@ -759,12 +764,6 @@ TEXT libc_setreuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setreuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setreuid_trampoline_addr(SB)/8, $libc_setreuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) - -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsid(SB) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go index 26a0fdc..51d6f3f 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.go @@ -725,6 +725,14 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { return } +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := syscall_syscall(libc_ioctl_trampoline_addr, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + var libc_ioctl_trampoline_addr uintptr //go:cgo_import_dynamic libc_ioctl ioctl "/usr/lib/libSystem.B.dylib" @@ -1984,6 +1992,31 @@ var libc_select_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func Setattrlist(path string, attrlist *Attrlist, attrBuf []byte, options int) (err error) { + var _p0 *byte + _p0, err = BytePtrFromString(path) + if err != nil { + return + } + var _p1 unsafe.Pointer + if len(attrBuf) > 0 { + _p1 = unsafe.Pointer(&attrBuf[0]) + } else { + _p1 = unsafe.Pointer(&_zero) + } + _, _, e1 := syscall_syscall6(libc_setattrlist_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(attrlist)), uintptr(_p1), uintptr(len(attrBuf)), uintptr(options), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_setattrlist_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setattrlist setattrlist "/usr/lib/libSystem.B.dylib" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Setegid(egid int) (err error) { _, _, e1 := syscall_syscall(libc_setegid_trampoline_addr, uintptr(egid), 0, 0) if e1 != 0 { @@ -2115,20 +2148,6 @@ var libc_setreuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "/usr/lib/libSystem.B.dylib" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := syscall_rawSyscall(libc_setsid_trampoline_addr, 0, 0, 0) pid = int(r0) @@ -2502,6 +2521,14 @@ func ptrace1(request int, pid int, addr uintptr, data uintptr) (err error) { return } +func ptrace1Ptr(request int, pid int, addr uintptr, data unsafe.Pointer) (err error) { + _, _, e1 := syscall_syscall6(libc_ptrace_trampoline_addr, uintptr(request), uintptr(pid), addr, uintptr(data), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + var libc_ptrace_trampoline_addr uintptr //go:cgo_import_dynamic libc_ptrace ptrace "/usr/lib/libSystem.B.dylib" diff --git a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s index efa5b4c..c3b82c0 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_darwin_arm64.s @@ -705,6 +705,11 @@ TEXT libc_select_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_select_trampoline_addr(SB), RODATA, $8 DATA ·libc_select_trampoline_addr(SB)/8, $libc_select_trampoline<>(SB) +TEXT libc_setattrlist_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setattrlist(SB) +GLOBL ·libc_setattrlist_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setattrlist_trampoline_addr(SB)/8, $libc_setattrlist_trampoline<>(SB) + TEXT libc_setegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setegid(SB) @@ -759,12 +764,6 @@ TEXT libc_setreuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setreuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setreuid_trampoline_addr(SB)/8, $libc_setreuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) - -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsid(SB) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go index 1b6eedf..0eabac7 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_dragonfly_amd64.go @@ -436,6 +436,16 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := Syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { var _p0 unsafe.Pointer if len(mib) > 0 { @@ -552,6 +562,16 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := Syscall(SYS_CLOCK_GETTIME, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0) if e1 != 0 { @@ -1390,16 +1410,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go index 039c4aa..ee313eb 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_386.go @@ -388,6 +388,16 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := Syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { var _p0 unsafe.Pointer if len(mib) > 0 { @@ -414,6 +424,16 @@ func ptrace(request int, pid int, addr uintptr, data int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ptracePtr(request int, pid int, addr unsafe.Pointer, data int) (err error) { + _, _, e1 := Syscall6(SYS_PTRACE, uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Access(path string, mode uint32) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) @@ -544,6 +564,16 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := Syscall(SYS_CLOCK_GETTIME, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0) if e1 != 0 { @@ -1615,16 +1645,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go index 0535d3c..4c986e4 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_amd64.go @@ -388,6 +388,16 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := Syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { var _p0 unsafe.Pointer if len(mib) > 0 { @@ -414,6 +424,16 @@ func ptrace(request int, pid int, addr uintptr, data int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ptracePtr(request int, pid int, addr unsafe.Pointer, data int) (err error) { + _, _, e1 := Syscall6(SYS_PTRACE, uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Access(path string, mode uint32) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) @@ -544,6 +564,16 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := Syscall(SYS_CLOCK_GETTIME, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0) if e1 != 0 { @@ -1615,16 +1645,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go index 1018b52..5552169 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm.go @@ -388,6 +388,16 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := Syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { var _p0 unsafe.Pointer if len(mib) > 0 { @@ -414,6 +424,16 @@ func ptrace(request int, pid int, addr uintptr, data int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ptracePtr(request int, pid int, addr unsafe.Pointer, data int) (err error) { + _, _, e1 := Syscall6(SYS_PTRACE, uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Access(path string, mode uint32) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) @@ -544,6 +564,16 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := Syscall(SYS_CLOCK_GETTIME, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0) if e1 != 0 { @@ -1615,16 +1645,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm64.go index 3802f4b..67a226f 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_arm64.go @@ -388,6 +388,16 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := Syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { var _p0 unsafe.Pointer if len(mib) > 0 { @@ -414,6 +424,16 @@ func ptrace(request int, pid int, addr uintptr, data int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ptracePtr(request int, pid int, addr unsafe.Pointer, data int) (err error) { + _, _, e1 := Syscall6(SYS_PTRACE, uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Access(path string, mode uint32) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) @@ -544,6 +564,16 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := Syscall(SYS_CLOCK_GETTIME, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0) if e1 != 0 { @@ -1615,16 +1645,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_riscv64.go b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_riscv64.go index 8a2db7d..f0b9dda 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_freebsd_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_freebsd_riscv64.go @@ -388,6 +388,16 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := Syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { var _p0 unsafe.Pointer if len(mib) > 0 { @@ -414,6 +424,16 @@ func ptrace(request int, pid int, addr uintptr, data int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ptracePtr(request int, pid int, addr unsafe.Pointer, data int) (err error) { + _, _, e1 := Syscall6(SYS_PTRACE, uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Access(path string, mode uint32) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) @@ -544,6 +564,16 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := Syscall(SYS_CLOCK_GETTIME, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0) if e1 != 0 { @@ -1615,16 +1645,6 @@ func Setresuid(ruid int, euid int, suid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux.go b/vendor/golang.org/x/sys/unix/zsyscall_linux.go index 293cf36..da63d9d 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux.go @@ -379,6 +379,16 @@ func ptrace(request int, pid int, addr uintptr, data uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ptracePtr(request int, pid int, addr uintptr, data unsafe.Pointer) (err error) { + _, _, e1 := Syscall6(SYS_PTRACE, uintptr(request), uintptr(pid), uintptr(addr), uintptr(data), 0, 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func reboot(magic1 uint, magic2 uint, cmd int, arg string) (err error) { var _p0 *byte _p0, err = BytePtrFromString(arg) @@ -537,6 +547,17 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockAdjtime(clockid int32, buf *Timex) (state int, err error) { + r0, _, e1 := Syscall(SYS_CLOCK_ADJTIME, uintptr(clockid), uintptr(unsafe.Pointer(buf)), 0) + state = int(r0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func ClockGetres(clockid int32, res *Timespec) (err error) { _, _, e1 := Syscall(SYS_CLOCK_GETRES, uintptr(clockid), uintptr(unsafe.Pointer(res)), 0) if e1 != 0 { @@ -1325,16 +1346,6 @@ func PivotRoot(newroot string, putold string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Prlimit(pid int, resource int, newlimit *Rlimit, old *Rlimit) (err error) { - _, _, e1 := RawSyscall6(SYS_PRLIMIT64, uintptr(pid), uintptr(resource), uintptr(unsafe.Pointer(newlimit)), uintptr(unsafe.Pointer(old)), 0, 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Prctl(option int, arg2 uintptr, arg3 uintptr, arg4 uintptr, arg5 uintptr) (err error) { _, _, e1 := Syscall6(SYS_PRCTL, uintptr(option), uintptr(arg2), uintptr(arg3), uintptr(arg4), uintptr(arg5), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_386.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_386.go index c81b0ad..07b549c 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_386.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_386.go @@ -411,16 +411,6 @@ func getrlimit(resource int, rlim *rlimit32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func futimesat(dirfd int, path string, times *[2]Timeval) (err error) { var _p0 *byte _p0, err = BytePtrFromString(path) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go index 2206bce..5f481bf 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_amd64.go @@ -334,16 +334,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_arm.go index edf6b39..824cd52 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_arm.go @@ -578,16 +578,6 @@ func getrlimit(resource int, rlim *rlimit32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func armSyncFileRange(fd int, flags int, off int64, n int64) (err error) { _, _, e1 := Syscall6(SYS_ARM_SYNC_FILE_RANGE, uintptr(fd), uintptr(flags), uintptr(off), uintptr(off>>32), uintptr(n), uintptr(n>>32)) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_arm64.go index 190609f..e77aecf 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_arm64.go @@ -289,16 +289,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips.go index 5f984cb..961a3af 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips.go @@ -644,16 +644,6 @@ func getrlimit(resource int, rlim *rlimit32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Alarm(seconds uint) (remaining uint, err error) { r0, _, e1 := Syscall(SYS_ALARM, uintptr(seconds), 0, 0) remaining = uint(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64.go index 46fc380..ed05005 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64.go @@ -278,16 +278,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go index cbd0d4d..d365b71 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_mips64le.go @@ -278,16 +278,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_mipsle.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_mipsle.go index 0c13d15..c3f1b8b 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_mipsle.go @@ -644,16 +644,6 @@ func getrlimit(resource int, rlim *rlimit32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Alarm(seconds uint) (remaining uint, err error) { r0, _, e1 := Syscall(SYS_ALARM, uintptr(seconds), 0, 0) remaining = uint(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc.go index e01432a..a6574cf 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc.go @@ -624,16 +624,6 @@ func getrlimit(resource int, rlim *rlimit32) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func setrlimit(resource int, rlim *rlimit32) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func syncFileRange2(fd int, flags int, off int64, n int64) (err error) { _, _, e1 := Syscall6(SYS_SYNC_FILE_RANGE2, uintptr(fd), uintptr(flags), uintptr(off>>32), uintptr(off), uintptr(n>>32), uintptr(n)) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64.go index 13c7ee7..f409902 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64.go @@ -349,16 +349,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64le.go index 02d0c0f..9dfcc29 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_ppc64le.go @@ -349,16 +349,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_riscv64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_riscv64.go index 9fee3b1..0b29239 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_riscv64.go @@ -269,16 +269,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_s390x.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_s390x.go index 647bbfe..6cde322 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_s390x.go @@ -319,16 +319,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Splice(rfd int, roff *int64, wfd int, woff *int64, len int, flags int) (n int64, err error) { r0, _, e1 := Syscall6(SYS_SPLICE, uintptr(rfd), uintptr(unsafe.Pointer(roff)), uintptr(wfd), uintptr(unsafe.Pointer(woff)), uintptr(len), uintptr(flags)) n = int64(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go b/vendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go index ada057f..5253d65 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_linux_sparc64.go @@ -329,16 +329,6 @@ func setfsuid(uid int) (prev int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(resource int, rlim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(resource), uintptr(unsafe.Pointer(rlim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Shutdown(fd int, how int) (err error) { _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(fd), uintptr(how), 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go index 4af561a..cdb2af5 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_386.go @@ -405,6 +405,16 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := Syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { var _p0 unsafe.Pointer if len(mib) > 0 { @@ -521,6 +531,16 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := Syscall(SYS_CLOCK_GETTIME, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0) if e1 != 0 { @@ -1587,16 +1607,6 @@ func Setreuid(ruid int, euid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go index 3b90e94..9d25f76 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_amd64.go @@ -405,6 +405,16 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := Syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { var _p0 unsafe.Pointer if len(mib) > 0 { @@ -521,6 +531,16 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := Syscall(SYS_CLOCK_GETTIME, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0) if e1 != 0 { @@ -1587,16 +1607,6 @@ func Setreuid(ruid int, euid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go index 890f4cc..d3f8035 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm.go @@ -405,6 +405,16 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := Syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { var _p0 unsafe.Pointer if len(mib) > 0 { @@ -521,6 +531,16 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := Syscall(SYS_CLOCK_GETTIME, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0) if e1 != 0 { @@ -1587,16 +1607,6 @@ func Setreuid(ruid int, euid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm64.go index c79f071..887188a 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_netbsd_arm64.go @@ -405,6 +405,16 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := Syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { var _p0 unsafe.Pointer if len(mib) > 0 { @@ -521,6 +531,16 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := Syscall(SYS_CLOCK_GETTIME, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0) if e1 != 0 { @@ -1587,16 +1607,6 @@ func Setreuid(ruid int, euid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go index 2925fe0..6699a78 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.go @@ -527,6 +527,14 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { return } +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := syscall_syscall(libc_ioctl_trampoline_addr, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + var libc_ioctl_trampoline_addr uintptr //go:cgo_import_dynamic libc_ioctl ioctl "libc.so" @@ -696,6 +704,20 @@ var libc_chroot_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := syscall_syscall(libc_clock_gettime_trampoline_addr, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_clock_gettime_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := syscall_syscall(libc_close_trampoline_addr, uintptr(fd), 0, 0) if e1 != 0 { @@ -1872,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s index 75eb2f5..04f0de3 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_386.s @@ -5,792 +5,660 @@ TEXT libc_getgroups_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getgroups(SB) - GLOBL ·libc_getgroups_trampoline_addr(SB), RODATA, $4 DATA ·libc_getgroups_trampoline_addr(SB)/4, $libc_getgroups_trampoline<>(SB) TEXT libc_setgroups_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setgroups(SB) - GLOBL ·libc_setgroups_trampoline_addr(SB), RODATA, $4 DATA ·libc_setgroups_trampoline_addr(SB)/4, $libc_setgroups_trampoline<>(SB) TEXT libc_wait4_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_wait4(SB) - GLOBL ·libc_wait4_trampoline_addr(SB), RODATA, $4 DATA ·libc_wait4_trampoline_addr(SB)/4, $libc_wait4_trampoline<>(SB) TEXT libc_accept_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_accept(SB) - GLOBL ·libc_accept_trampoline_addr(SB), RODATA, $4 DATA ·libc_accept_trampoline_addr(SB)/4, $libc_accept_trampoline<>(SB) TEXT libc_bind_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_bind(SB) - GLOBL ·libc_bind_trampoline_addr(SB), RODATA, $4 DATA ·libc_bind_trampoline_addr(SB)/4, $libc_bind_trampoline<>(SB) TEXT libc_connect_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_connect(SB) - GLOBL ·libc_connect_trampoline_addr(SB), RODATA, $4 DATA ·libc_connect_trampoline_addr(SB)/4, $libc_connect_trampoline<>(SB) TEXT libc_socket_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_socket(SB) - GLOBL ·libc_socket_trampoline_addr(SB), RODATA, $4 DATA ·libc_socket_trampoline_addr(SB)/4, $libc_socket_trampoline<>(SB) TEXT libc_getsockopt_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsockopt(SB) - GLOBL ·libc_getsockopt_trampoline_addr(SB), RODATA, $4 DATA ·libc_getsockopt_trampoline_addr(SB)/4, $libc_getsockopt_trampoline<>(SB) TEXT libc_setsockopt_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsockopt(SB) - GLOBL ·libc_setsockopt_trampoline_addr(SB), RODATA, $4 DATA ·libc_setsockopt_trampoline_addr(SB)/4, $libc_setsockopt_trampoline<>(SB) TEXT libc_getpeername_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpeername(SB) - GLOBL ·libc_getpeername_trampoline_addr(SB), RODATA, $4 DATA ·libc_getpeername_trampoline_addr(SB)/4, $libc_getpeername_trampoline<>(SB) TEXT libc_getsockname_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsockname(SB) - GLOBL ·libc_getsockname_trampoline_addr(SB), RODATA, $4 DATA ·libc_getsockname_trampoline_addr(SB)/4, $libc_getsockname_trampoline<>(SB) TEXT libc_shutdown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_shutdown(SB) - GLOBL ·libc_shutdown_trampoline_addr(SB), RODATA, $4 DATA ·libc_shutdown_trampoline_addr(SB)/4, $libc_shutdown_trampoline<>(SB) TEXT libc_socketpair_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_socketpair(SB) - GLOBL ·libc_socketpair_trampoline_addr(SB), RODATA, $4 DATA ·libc_socketpair_trampoline_addr(SB)/4, $libc_socketpair_trampoline<>(SB) TEXT libc_recvfrom_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_recvfrom(SB) - GLOBL ·libc_recvfrom_trampoline_addr(SB), RODATA, $4 DATA ·libc_recvfrom_trampoline_addr(SB)/4, $libc_recvfrom_trampoline<>(SB) TEXT libc_sendto_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sendto(SB) - GLOBL ·libc_sendto_trampoline_addr(SB), RODATA, $4 DATA ·libc_sendto_trampoline_addr(SB)/4, $libc_sendto_trampoline<>(SB) TEXT libc_recvmsg_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_recvmsg(SB) - GLOBL ·libc_recvmsg_trampoline_addr(SB), RODATA, $4 DATA ·libc_recvmsg_trampoline_addr(SB)/4, $libc_recvmsg_trampoline<>(SB) TEXT libc_sendmsg_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sendmsg(SB) - GLOBL ·libc_sendmsg_trampoline_addr(SB), RODATA, $4 DATA ·libc_sendmsg_trampoline_addr(SB)/4, $libc_sendmsg_trampoline<>(SB) TEXT libc_kevent_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kevent(SB) - GLOBL ·libc_kevent_trampoline_addr(SB), RODATA, $4 DATA ·libc_kevent_trampoline_addr(SB)/4, $libc_kevent_trampoline<>(SB) TEXT libc_utimes_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_utimes(SB) - GLOBL ·libc_utimes_trampoline_addr(SB), RODATA, $4 DATA ·libc_utimes_trampoline_addr(SB)/4, $libc_utimes_trampoline<>(SB) TEXT libc_futimes_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_futimes(SB) - GLOBL ·libc_futimes_trampoline_addr(SB), RODATA, $4 DATA ·libc_futimes_trampoline_addr(SB)/4, $libc_futimes_trampoline<>(SB) TEXT libc_poll_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_poll(SB) - GLOBL ·libc_poll_trampoline_addr(SB), RODATA, $4 DATA ·libc_poll_trampoline_addr(SB)/4, $libc_poll_trampoline<>(SB) TEXT libc_madvise_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_madvise(SB) - GLOBL ·libc_madvise_trampoline_addr(SB), RODATA, $4 DATA ·libc_madvise_trampoline_addr(SB)/4, $libc_madvise_trampoline<>(SB) TEXT libc_mlock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mlock(SB) - GLOBL ·libc_mlock_trampoline_addr(SB), RODATA, $4 DATA ·libc_mlock_trampoline_addr(SB)/4, $libc_mlock_trampoline<>(SB) TEXT libc_mlockall_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mlockall(SB) - GLOBL ·libc_mlockall_trampoline_addr(SB), RODATA, $4 DATA ·libc_mlockall_trampoline_addr(SB)/4, $libc_mlockall_trampoline<>(SB) TEXT libc_mprotect_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mprotect(SB) - GLOBL ·libc_mprotect_trampoline_addr(SB), RODATA, $4 DATA ·libc_mprotect_trampoline_addr(SB)/4, $libc_mprotect_trampoline<>(SB) TEXT libc_msync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_msync(SB) - GLOBL ·libc_msync_trampoline_addr(SB), RODATA, $4 DATA ·libc_msync_trampoline_addr(SB)/4, $libc_msync_trampoline<>(SB) TEXT libc_munlock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munlock(SB) - GLOBL ·libc_munlock_trampoline_addr(SB), RODATA, $4 DATA ·libc_munlock_trampoline_addr(SB)/4, $libc_munlock_trampoline<>(SB) TEXT libc_munlockall_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munlockall(SB) - GLOBL ·libc_munlockall_trampoline_addr(SB), RODATA, $4 DATA ·libc_munlockall_trampoline_addr(SB)/4, $libc_munlockall_trampoline<>(SB) TEXT libc_pipe2_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pipe2(SB) - GLOBL ·libc_pipe2_trampoline_addr(SB), RODATA, $4 DATA ·libc_pipe2_trampoline_addr(SB)/4, $libc_pipe2_trampoline<>(SB) TEXT libc_getdents_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getdents(SB) - GLOBL ·libc_getdents_trampoline_addr(SB), RODATA, $4 DATA ·libc_getdents_trampoline_addr(SB)/4, $libc_getdents_trampoline<>(SB) TEXT libc_getcwd_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getcwd(SB) - GLOBL ·libc_getcwd_trampoline_addr(SB), RODATA, $4 DATA ·libc_getcwd_trampoline_addr(SB)/4, $libc_getcwd_trampoline<>(SB) TEXT libc_ioctl_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ioctl(SB) - GLOBL ·libc_ioctl_trampoline_addr(SB), RODATA, $4 DATA ·libc_ioctl_trampoline_addr(SB)/4, $libc_ioctl_trampoline<>(SB) TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sysctl(SB) - GLOBL ·libc_sysctl_trampoline_addr(SB), RODATA, $4 DATA ·libc_sysctl_trampoline_addr(SB)/4, $libc_sysctl_trampoline<>(SB) TEXT libc_ppoll_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ppoll(SB) - GLOBL ·libc_ppoll_trampoline_addr(SB), RODATA, $4 DATA ·libc_ppoll_trampoline_addr(SB)/4, $libc_ppoll_trampoline<>(SB) TEXT libc_access_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_access(SB) - GLOBL ·libc_access_trampoline_addr(SB), RODATA, $4 DATA ·libc_access_trampoline_addr(SB)/4, $libc_access_trampoline<>(SB) TEXT libc_adjtime_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_adjtime(SB) - GLOBL ·libc_adjtime_trampoline_addr(SB), RODATA, $4 DATA ·libc_adjtime_trampoline_addr(SB)/4, $libc_adjtime_trampoline<>(SB) TEXT libc_chdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chdir(SB) - GLOBL ·libc_chdir_trampoline_addr(SB), RODATA, $4 DATA ·libc_chdir_trampoline_addr(SB)/4, $libc_chdir_trampoline<>(SB) TEXT libc_chflags_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chflags(SB) - GLOBL ·libc_chflags_trampoline_addr(SB), RODATA, $4 DATA ·libc_chflags_trampoline_addr(SB)/4, $libc_chflags_trampoline<>(SB) TEXT libc_chmod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chmod(SB) - GLOBL ·libc_chmod_trampoline_addr(SB), RODATA, $4 DATA ·libc_chmod_trampoline_addr(SB)/4, $libc_chmod_trampoline<>(SB) TEXT libc_chown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chown(SB) - GLOBL ·libc_chown_trampoline_addr(SB), RODATA, $4 DATA ·libc_chown_trampoline_addr(SB)/4, $libc_chown_trampoline<>(SB) TEXT libc_chroot_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chroot(SB) - GLOBL ·libc_chroot_trampoline_addr(SB), RODATA, $4 DATA ·libc_chroot_trampoline_addr(SB)/4, $libc_chroot_trampoline<>(SB) +TEXT libc_clock_gettime_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_clock_gettime(SB) +GLOBL ·libc_clock_gettime_trampoline_addr(SB), RODATA, $4 +DATA ·libc_clock_gettime_trampoline_addr(SB)/4, $libc_clock_gettime_trampoline<>(SB) + TEXT libc_close_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_close(SB) - GLOBL ·libc_close_trampoline_addr(SB), RODATA, $4 DATA ·libc_close_trampoline_addr(SB)/4, $libc_close_trampoline<>(SB) TEXT libc_dup_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup(SB) - GLOBL ·libc_dup_trampoline_addr(SB), RODATA, $4 DATA ·libc_dup_trampoline_addr(SB)/4, $libc_dup_trampoline<>(SB) TEXT libc_dup2_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup2(SB) - GLOBL ·libc_dup2_trampoline_addr(SB), RODATA, $4 DATA ·libc_dup2_trampoline_addr(SB)/4, $libc_dup2_trampoline<>(SB) TEXT libc_dup3_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup3(SB) - GLOBL ·libc_dup3_trampoline_addr(SB), RODATA, $4 DATA ·libc_dup3_trampoline_addr(SB)/4, $libc_dup3_trampoline<>(SB) TEXT libc_exit_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_exit(SB) - GLOBL ·libc_exit_trampoline_addr(SB), RODATA, $4 DATA ·libc_exit_trampoline_addr(SB)/4, $libc_exit_trampoline<>(SB) TEXT libc_faccessat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_faccessat(SB) - GLOBL ·libc_faccessat_trampoline_addr(SB), RODATA, $4 DATA ·libc_faccessat_trampoline_addr(SB)/4, $libc_faccessat_trampoline<>(SB) TEXT libc_fchdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchdir(SB) - GLOBL ·libc_fchdir_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchdir_trampoline_addr(SB)/4, $libc_fchdir_trampoline<>(SB) TEXT libc_fchflags_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchflags(SB) - GLOBL ·libc_fchflags_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchflags_trampoline_addr(SB)/4, $libc_fchflags_trampoline<>(SB) TEXT libc_fchmod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchmod(SB) - GLOBL ·libc_fchmod_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchmod_trampoline_addr(SB)/4, $libc_fchmod_trampoline<>(SB) TEXT libc_fchmodat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchmodat(SB) - GLOBL ·libc_fchmodat_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchmodat_trampoline_addr(SB)/4, $libc_fchmodat_trampoline<>(SB) TEXT libc_fchown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchown(SB) - GLOBL ·libc_fchown_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchown_trampoline_addr(SB)/4, $libc_fchown_trampoline<>(SB) TEXT libc_fchownat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchownat(SB) - GLOBL ·libc_fchownat_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchownat_trampoline_addr(SB)/4, $libc_fchownat_trampoline<>(SB) TEXT libc_flock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_flock(SB) - GLOBL ·libc_flock_trampoline_addr(SB), RODATA, $4 DATA ·libc_flock_trampoline_addr(SB)/4, $libc_flock_trampoline<>(SB) TEXT libc_fpathconf_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fpathconf(SB) - GLOBL ·libc_fpathconf_trampoline_addr(SB), RODATA, $4 DATA ·libc_fpathconf_trampoline_addr(SB)/4, $libc_fpathconf_trampoline<>(SB) TEXT libc_fstat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstat(SB) - GLOBL ·libc_fstat_trampoline_addr(SB), RODATA, $4 DATA ·libc_fstat_trampoline_addr(SB)/4, $libc_fstat_trampoline<>(SB) TEXT libc_fstatat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) - GLOBL ·libc_fstatat_trampoline_addr(SB), RODATA, $4 DATA ·libc_fstatat_trampoline_addr(SB)/4, $libc_fstatat_trampoline<>(SB) TEXT libc_fstatfs_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstatfs(SB) - GLOBL ·libc_fstatfs_trampoline_addr(SB), RODATA, $4 DATA ·libc_fstatfs_trampoline_addr(SB)/4, $libc_fstatfs_trampoline<>(SB) TEXT libc_fsync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fsync(SB) - GLOBL ·libc_fsync_trampoline_addr(SB), RODATA, $4 DATA ·libc_fsync_trampoline_addr(SB)/4, $libc_fsync_trampoline<>(SB) TEXT libc_ftruncate_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ftruncate(SB) - GLOBL ·libc_ftruncate_trampoline_addr(SB), RODATA, $4 DATA ·libc_ftruncate_trampoline_addr(SB)/4, $libc_ftruncate_trampoline<>(SB) TEXT libc_getegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getegid(SB) - GLOBL ·libc_getegid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getegid_trampoline_addr(SB)/4, $libc_getegid_trampoline<>(SB) TEXT libc_geteuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_geteuid(SB) - GLOBL ·libc_geteuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_geteuid_trampoline_addr(SB)/4, $libc_geteuid_trampoline<>(SB) TEXT libc_getgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getgid(SB) - GLOBL ·libc_getgid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getgid_trampoline_addr(SB)/4, $libc_getgid_trampoline<>(SB) TEXT libc_getpgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpgid(SB) - GLOBL ·libc_getpgid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getpgid_trampoline_addr(SB)/4, $libc_getpgid_trampoline<>(SB) TEXT libc_getpgrp_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpgrp(SB) - GLOBL ·libc_getpgrp_trampoline_addr(SB), RODATA, $4 DATA ·libc_getpgrp_trampoline_addr(SB)/4, $libc_getpgrp_trampoline<>(SB) TEXT libc_getpid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpid(SB) - GLOBL ·libc_getpid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getpid_trampoline_addr(SB)/4, $libc_getpid_trampoline<>(SB) TEXT libc_getppid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getppid(SB) - GLOBL ·libc_getppid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getppid_trampoline_addr(SB)/4, $libc_getppid_trampoline<>(SB) TEXT libc_getpriority_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpriority(SB) - GLOBL ·libc_getpriority_trampoline_addr(SB), RODATA, $4 DATA ·libc_getpriority_trampoline_addr(SB)/4, $libc_getpriority_trampoline<>(SB) TEXT libc_getrlimit_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrlimit(SB) - GLOBL ·libc_getrlimit_trampoline_addr(SB), RODATA, $4 DATA ·libc_getrlimit_trampoline_addr(SB)/4, $libc_getrlimit_trampoline<>(SB) TEXT libc_getrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrtable(SB) - GLOBL ·libc_getrtable_trampoline_addr(SB), RODATA, $4 DATA ·libc_getrtable_trampoline_addr(SB)/4, $libc_getrtable_trampoline<>(SB) TEXT libc_getrusage_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrusage(SB) - GLOBL ·libc_getrusage_trampoline_addr(SB), RODATA, $4 DATA ·libc_getrusage_trampoline_addr(SB)/4, $libc_getrusage_trampoline<>(SB) TEXT libc_getsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsid(SB) - GLOBL ·libc_getsid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getsid_trampoline_addr(SB)/4, $libc_getsid_trampoline<>(SB) TEXT libc_gettimeofday_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_gettimeofday(SB) - GLOBL ·libc_gettimeofday_trampoline_addr(SB), RODATA, $4 DATA ·libc_gettimeofday_trampoline_addr(SB)/4, $libc_gettimeofday_trampoline<>(SB) TEXT libc_getuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getuid(SB) - GLOBL ·libc_getuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getuid_trampoline_addr(SB)/4, $libc_getuid_trampoline<>(SB) TEXT libc_issetugid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_issetugid(SB) - GLOBL ·libc_issetugid_trampoline_addr(SB), RODATA, $4 DATA ·libc_issetugid_trampoline_addr(SB)/4, $libc_issetugid_trampoline<>(SB) TEXT libc_kill_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kill(SB) - GLOBL ·libc_kill_trampoline_addr(SB), RODATA, $4 DATA ·libc_kill_trampoline_addr(SB)/4, $libc_kill_trampoline<>(SB) TEXT libc_kqueue_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kqueue(SB) - GLOBL ·libc_kqueue_trampoline_addr(SB), RODATA, $4 DATA ·libc_kqueue_trampoline_addr(SB)/4, $libc_kqueue_trampoline<>(SB) TEXT libc_lchown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lchown(SB) - GLOBL ·libc_lchown_trampoline_addr(SB), RODATA, $4 DATA ·libc_lchown_trampoline_addr(SB)/4, $libc_lchown_trampoline<>(SB) TEXT libc_link_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_link(SB) - GLOBL ·libc_link_trampoline_addr(SB), RODATA, $4 DATA ·libc_link_trampoline_addr(SB)/4, $libc_link_trampoline<>(SB) TEXT libc_linkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_linkat(SB) - GLOBL ·libc_linkat_trampoline_addr(SB), RODATA, $4 DATA ·libc_linkat_trampoline_addr(SB)/4, $libc_linkat_trampoline<>(SB) TEXT libc_listen_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_listen(SB) - GLOBL ·libc_listen_trampoline_addr(SB), RODATA, $4 DATA ·libc_listen_trampoline_addr(SB)/4, $libc_listen_trampoline<>(SB) TEXT libc_lstat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lstat(SB) - GLOBL ·libc_lstat_trampoline_addr(SB), RODATA, $4 DATA ·libc_lstat_trampoline_addr(SB)/4, $libc_lstat_trampoline<>(SB) TEXT libc_mkdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkdir(SB) - GLOBL ·libc_mkdir_trampoline_addr(SB), RODATA, $4 DATA ·libc_mkdir_trampoline_addr(SB)/4, $libc_mkdir_trampoline<>(SB) TEXT libc_mkdirat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkdirat(SB) - GLOBL ·libc_mkdirat_trampoline_addr(SB), RODATA, $4 DATA ·libc_mkdirat_trampoline_addr(SB)/4, $libc_mkdirat_trampoline<>(SB) TEXT libc_mkfifo_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkfifo(SB) - GLOBL ·libc_mkfifo_trampoline_addr(SB), RODATA, $4 DATA ·libc_mkfifo_trampoline_addr(SB)/4, $libc_mkfifo_trampoline<>(SB) TEXT libc_mkfifoat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkfifoat(SB) - GLOBL ·libc_mkfifoat_trampoline_addr(SB), RODATA, $4 DATA ·libc_mkfifoat_trampoline_addr(SB)/4, $libc_mkfifoat_trampoline<>(SB) TEXT libc_mknod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mknod(SB) - GLOBL ·libc_mknod_trampoline_addr(SB), RODATA, $4 DATA ·libc_mknod_trampoline_addr(SB)/4, $libc_mknod_trampoline<>(SB) TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mknodat(SB) - GLOBL ·libc_mknodat_trampoline_addr(SB), RODATA, $4 DATA ·libc_mknodat_trampoline_addr(SB)/4, $libc_mknodat_trampoline<>(SB) TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_nanosleep(SB) - GLOBL ·libc_nanosleep_trampoline_addr(SB), RODATA, $4 DATA ·libc_nanosleep_trampoline_addr(SB)/4, $libc_nanosleep_trampoline<>(SB) TEXT libc_open_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_open(SB) - GLOBL ·libc_open_trampoline_addr(SB), RODATA, $4 DATA ·libc_open_trampoline_addr(SB)/4, $libc_open_trampoline<>(SB) TEXT libc_openat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_openat(SB) - GLOBL ·libc_openat_trampoline_addr(SB), RODATA, $4 DATA ·libc_openat_trampoline_addr(SB)/4, $libc_openat_trampoline<>(SB) TEXT libc_pathconf_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pathconf(SB) - GLOBL ·libc_pathconf_trampoline_addr(SB), RODATA, $4 DATA ·libc_pathconf_trampoline_addr(SB)/4, $libc_pathconf_trampoline<>(SB) TEXT libc_pread_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pread(SB) - GLOBL ·libc_pread_trampoline_addr(SB), RODATA, $4 DATA ·libc_pread_trampoline_addr(SB)/4, $libc_pread_trampoline<>(SB) TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pwrite(SB) - GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $4 DATA ·libc_pwrite_trampoline_addr(SB)/4, $libc_pwrite_trampoline<>(SB) TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_read(SB) - GLOBL ·libc_read_trampoline_addr(SB), RODATA, $4 DATA ·libc_read_trampoline_addr(SB)/4, $libc_read_trampoline<>(SB) TEXT libc_readlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_readlink(SB) - GLOBL ·libc_readlink_trampoline_addr(SB), RODATA, $4 DATA ·libc_readlink_trampoline_addr(SB)/4, $libc_readlink_trampoline<>(SB) TEXT libc_readlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_readlinkat(SB) - GLOBL ·libc_readlinkat_trampoline_addr(SB), RODATA, $4 DATA ·libc_readlinkat_trampoline_addr(SB)/4, $libc_readlinkat_trampoline<>(SB) TEXT libc_rename_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_rename(SB) - GLOBL ·libc_rename_trampoline_addr(SB), RODATA, $4 DATA ·libc_rename_trampoline_addr(SB)/4, $libc_rename_trampoline<>(SB) TEXT libc_renameat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_renameat(SB) - GLOBL ·libc_renameat_trampoline_addr(SB), RODATA, $4 DATA ·libc_renameat_trampoline_addr(SB)/4, $libc_renameat_trampoline<>(SB) TEXT libc_revoke_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_revoke(SB) - GLOBL ·libc_revoke_trampoline_addr(SB), RODATA, $4 DATA ·libc_revoke_trampoline_addr(SB)/4, $libc_revoke_trampoline<>(SB) TEXT libc_rmdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_rmdir(SB) - GLOBL ·libc_rmdir_trampoline_addr(SB), RODATA, $4 DATA ·libc_rmdir_trampoline_addr(SB)/4, $libc_rmdir_trampoline<>(SB) TEXT libc_lseek_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lseek(SB) - GLOBL ·libc_lseek_trampoline_addr(SB), RODATA, $4 DATA ·libc_lseek_trampoline_addr(SB)/4, $libc_lseek_trampoline<>(SB) TEXT libc_select_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_select(SB) - GLOBL ·libc_select_trampoline_addr(SB), RODATA, $4 DATA ·libc_select_trampoline_addr(SB)/4, $libc_select_trampoline<>(SB) TEXT libc_setegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setegid(SB) - GLOBL ·libc_setegid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setegid_trampoline_addr(SB)/4, $libc_setegid_trampoline<>(SB) TEXT libc_seteuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_seteuid(SB) - GLOBL ·libc_seteuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_seteuid_trampoline_addr(SB)/4, $libc_seteuid_trampoline<>(SB) TEXT libc_setgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setgid(SB) - GLOBL ·libc_setgid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setgid_trampoline_addr(SB)/4, $libc_setgid_trampoline<>(SB) TEXT libc_setlogin_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setlogin(SB) - GLOBL ·libc_setlogin_trampoline_addr(SB), RODATA, $4 DATA ·libc_setlogin_trampoline_addr(SB)/4, $libc_setlogin_trampoline<>(SB) TEXT libc_setpgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setpgid(SB) - GLOBL ·libc_setpgid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setpgid_trampoline_addr(SB)/4, $libc_setpgid_trampoline<>(SB) TEXT libc_setpriority_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setpriority(SB) - GLOBL ·libc_setpriority_trampoline_addr(SB), RODATA, $4 DATA ·libc_setpriority_trampoline_addr(SB)/4, $libc_setpriority_trampoline<>(SB) TEXT libc_setregid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setregid(SB) - GLOBL ·libc_setregid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setregid_trampoline_addr(SB)/4, $libc_setregid_trampoline<>(SB) TEXT libc_setreuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setreuid(SB) - GLOBL ·libc_setreuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setreuid_trampoline_addr(SB)/4, $libc_setreuid_trampoline<>(SB) TEXT libc_setresgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setresgid(SB) - GLOBL ·libc_setresgid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setresgid_trampoline_addr(SB)/4, $libc_setresgid_trampoline<>(SB) TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setresuid(SB) - GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setresuid_trampoline_addr(SB)/4, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) - -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $4 -DATA ·libc_setrlimit_trampoline_addr(SB)/4, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setrtable(SB) - GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $4 DATA ·libc_setrtable_trampoline_addr(SB)/4, $libc_setrtable_trampoline<>(SB) TEXT libc_setsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsid(SB) - GLOBL ·libc_setsid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setsid_trampoline_addr(SB)/4, $libc_setsid_trampoline<>(SB) TEXT libc_settimeofday_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_settimeofday(SB) - GLOBL ·libc_settimeofday_trampoline_addr(SB), RODATA, $4 DATA ·libc_settimeofday_trampoline_addr(SB)/4, $libc_settimeofday_trampoline<>(SB) TEXT libc_setuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setuid(SB) - GLOBL ·libc_setuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setuid_trampoline_addr(SB)/4, $libc_setuid_trampoline<>(SB) TEXT libc_stat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_stat(SB) - GLOBL ·libc_stat_trampoline_addr(SB), RODATA, $4 DATA ·libc_stat_trampoline_addr(SB)/4, $libc_stat_trampoline<>(SB) TEXT libc_statfs_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_statfs(SB) - GLOBL ·libc_statfs_trampoline_addr(SB), RODATA, $4 DATA ·libc_statfs_trampoline_addr(SB)/4, $libc_statfs_trampoline<>(SB) TEXT libc_symlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_symlink(SB) - GLOBL ·libc_symlink_trampoline_addr(SB), RODATA, $4 DATA ·libc_symlink_trampoline_addr(SB)/4, $libc_symlink_trampoline<>(SB) TEXT libc_symlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_symlinkat(SB) - GLOBL ·libc_symlinkat_trampoline_addr(SB), RODATA, $4 DATA ·libc_symlinkat_trampoline_addr(SB)/4, $libc_symlinkat_trampoline<>(SB) TEXT libc_sync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sync(SB) - GLOBL ·libc_sync_trampoline_addr(SB), RODATA, $4 DATA ·libc_sync_trampoline_addr(SB)/4, $libc_sync_trampoline<>(SB) TEXT libc_truncate_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_truncate(SB) - GLOBL ·libc_truncate_trampoline_addr(SB), RODATA, $4 DATA ·libc_truncate_trampoline_addr(SB)/4, $libc_truncate_trampoline<>(SB) TEXT libc_umask_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_umask(SB) - GLOBL ·libc_umask_trampoline_addr(SB), RODATA, $4 DATA ·libc_umask_trampoline_addr(SB)/4, $libc_umask_trampoline<>(SB) TEXT libc_unlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unlink(SB) - GLOBL ·libc_unlink_trampoline_addr(SB), RODATA, $4 DATA ·libc_unlink_trampoline_addr(SB)/4, $libc_unlink_trampoline<>(SB) TEXT libc_unlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unlinkat(SB) - GLOBL ·libc_unlinkat_trampoline_addr(SB), RODATA, $4 DATA ·libc_unlinkat_trampoline_addr(SB)/4, $libc_unlinkat_trampoline<>(SB) TEXT libc_unmount_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unmount(SB) - GLOBL ·libc_unmount_trampoline_addr(SB), RODATA, $4 DATA ·libc_unmount_trampoline_addr(SB)/4, $libc_unmount_trampoline<>(SB) TEXT libc_write_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_write(SB) - GLOBL ·libc_write_trampoline_addr(SB), RODATA, $4 DATA ·libc_write_trampoline_addr(SB)/4, $libc_write_trampoline<>(SB) TEXT libc_mmap_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mmap(SB) - GLOBL ·libc_mmap_trampoline_addr(SB), RODATA, $4 DATA ·libc_mmap_trampoline_addr(SB)/4, $libc_mmap_trampoline<>(SB) TEXT libc_munmap_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munmap(SB) - GLOBL ·libc_munmap_trampoline_addr(SB), RODATA, $4 DATA ·libc_munmap_trampoline_addr(SB)/4, $libc_munmap_trampoline<>(SB) TEXT libc_utimensat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_utimensat(SB) - GLOBL ·libc_utimensat_trampoline_addr(SB), RODATA, $4 DATA ·libc_utimensat_trampoline_addr(SB)/4, $libc_utimensat_trampoline<>(SB) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go index 98446d2..1e775fe 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.go @@ -527,6 +527,14 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { return } +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := syscall_syscall(libc_ioctl_trampoline_addr, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + var libc_ioctl_trampoline_addr uintptr //go:cgo_import_dynamic libc_ioctl ioctl "libc.so" @@ -696,6 +704,20 @@ var libc_chroot_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := syscall_syscall(libc_clock_gettime_trampoline_addr, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_clock_gettime_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := syscall_syscall(libc_close_trampoline_addr, uintptr(fd), 0, 0) if e1 != 0 { @@ -1872,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s index 243a666..27b6f4d 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_amd64.s @@ -5,792 +5,660 @@ TEXT libc_getgroups_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getgroups(SB) - GLOBL ·libc_getgroups_trampoline_addr(SB), RODATA, $8 DATA ·libc_getgroups_trampoline_addr(SB)/8, $libc_getgroups_trampoline<>(SB) TEXT libc_setgroups_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setgroups(SB) - GLOBL ·libc_setgroups_trampoline_addr(SB), RODATA, $8 DATA ·libc_setgroups_trampoline_addr(SB)/8, $libc_setgroups_trampoline<>(SB) TEXT libc_wait4_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_wait4(SB) - GLOBL ·libc_wait4_trampoline_addr(SB), RODATA, $8 DATA ·libc_wait4_trampoline_addr(SB)/8, $libc_wait4_trampoline<>(SB) TEXT libc_accept_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_accept(SB) - GLOBL ·libc_accept_trampoline_addr(SB), RODATA, $8 DATA ·libc_accept_trampoline_addr(SB)/8, $libc_accept_trampoline<>(SB) TEXT libc_bind_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_bind(SB) - GLOBL ·libc_bind_trampoline_addr(SB), RODATA, $8 DATA ·libc_bind_trampoline_addr(SB)/8, $libc_bind_trampoline<>(SB) TEXT libc_connect_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_connect(SB) - GLOBL ·libc_connect_trampoline_addr(SB), RODATA, $8 DATA ·libc_connect_trampoline_addr(SB)/8, $libc_connect_trampoline<>(SB) TEXT libc_socket_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_socket(SB) - GLOBL ·libc_socket_trampoline_addr(SB), RODATA, $8 DATA ·libc_socket_trampoline_addr(SB)/8, $libc_socket_trampoline<>(SB) TEXT libc_getsockopt_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsockopt(SB) - GLOBL ·libc_getsockopt_trampoline_addr(SB), RODATA, $8 DATA ·libc_getsockopt_trampoline_addr(SB)/8, $libc_getsockopt_trampoline<>(SB) TEXT libc_setsockopt_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsockopt(SB) - GLOBL ·libc_setsockopt_trampoline_addr(SB), RODATA, $8 DATA ·libc_setsockopt_trampoline_addr(SB)/8, $libc_setsockopt_trampoline<>(SB) TEXT libc_getpeername_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpeername(SB) - GLOBL ·libc_getpeername_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpeername_trampoline_addr(SB)/8, $libc_getpeername_trampoline<>(SB) TEXT libc_getsockname_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsockname(SB) - GLOBL ·libc_getsockname_trampoline_addr(SB), RODATA, $8 DATA ·libc_getsockname_trampoline_addr(SB)/8, $libc_getsockname_trampoline<>(SB) TEXT libc_shutdown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_shutdown(SB) - GLOBL ·libc_shutdown_trampoline_addr(SB), RODATA, $8 DATA ·libc_shutdown_trampoline_addr(SB)/8, $libc_shutdown_trampoline<>(SB) TEXT libc_socketpair_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_socketpair(SB) - GLOBL ·libc_socketpair_trampoline_addr(SB), RODATA, $8 DATA ·libc_socketpair_trampoline_addr(SB)/8, $libc_socketpair_trampoline<>(SB) TEXT libc_recvfrom_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_recvfrom(SB) - GLOBL ·libc_recvfrom_trampoline_addr(SB), RODATA, $8 DATA ·libc_recvfrom_trampoline_addr(SB)/8, $libc_recvfrom_trampoline<>(SB) TEXT libc_sendto_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sendto(SB) - GLOBL ·libc_sendto_trampoline_addr(SB), RODATA, $8 DATA ·libc_sendto_trampoline_addr(SB)/8, $libc_sendto_trampoline<>(SB) TEXT libc_recvmsg_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_recvmsg(SB) - GLOBL ·libc_recvmsg_trampoline_addr(SB), RODATA, $8 DATA ·libc_recvmsg_trampoline_addr(SB)/8, $libc_recvmsg_trampoline<>(SB) TEXT libc_sendmsg_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sendmsg(SB) - GLOBL ·libc_sendmsg_trampoline_addr(SB), RODATA, $8 DATA ·libc_sendmsg_trampoline_addr(SB)/8, $libc_sendmsg_trampoline<>(SB) TEXT libc_kevent_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kevent(SB) - GLOBL ·libc_kevent_trampoline_addr(SB), RODATA, $8 DATA ·libc_kevent_trampoline_addr(SB)/8, $libc_kevent_trampoline<>(SB) TEXT libc_utimes_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_utimes(SB) - GLOBL ·libc_utimes_trampoline_addr(SB), RODATA, $8 DATA ·libc_utimes_trampoline_addr(SB)/8, $libc_utimes_trampoline<>(SB) TEXT libc_futimes_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_futimes(SB) - GLOBL ·libc_futimes_trampoline_addr(SB), RODATA, $8 DATA ·libc_futimes_trampoline_addr(SB)/8, $libc_futimes_trampoline<>(SB) TEXT libc_poll_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_poll(SB) - GLOBL ·libc_poll_trampoline_addr(SB), RODATA, $8 DATA ·libc_poll_trampoline_addr(SB)/8, $libc_poll_trampoline<>(SB) TEXT libc_madvise_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_madvise(SB) - GLOBL ·libc_madvise_trampoline_addr(SB), RODATA, $8 DATA ·libc_madvise_trampoline_addr(SB)/8, $libc_madvise_trampoline<>(SB) TEXT libc_mlock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mlock(SB) - GLOBL ·libc_mlock_trampoline_addr(SB), RODATA, $8 DATA ·libc_mlock_trampoline_addr(SB)/8, $libc_mlock_trampoline<>(SB) TEXT libc_mlockall_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mlockall(SB) - GLOBL ·libc_mlockall_trampoline_addr(SB), RODATA, $8 DATA ·libc_mlockall_trampoline_addr(SB)/8, $libc_mlockall_trampoline<>(SB) TEXT libc_mprotect_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mprotect(SB) - GLOBL ·libc_mprotect_trampoline_addr(SB), RODATA, $8 DATA ·libc_mprotect_trampoline_addr(SB)/8, $libc_mprotect_trampoline<>(SB) TEXT libc_msync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_msync(SB) - GLOBL ·libc_msync_trampoline_addr(SB), RODATA, $8 DATA ·libc_msync_trampoline_addr(SB)/8, $libc_msync_trampoline<>(SB) TEXT libc_munlock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munlock(SB) - GLOBL ·libc_munlock_trampoline_addr(SB), RODATA, $8 DATA ·libc_munlock_trampoline_addr(SB)/8, $libc_munlock_trampoline<>(SB) TEXT libc_munlockall_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munlockall(SB) - GLOBL ·libc_munlockall_trampoline_addr(SB), RODATA, $8 DATA ·libc_munlockall_trampoline_addr(SB)/8, $libc_munlockall_trampoline<>(SB) TEXT libc_pipe2_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pipe2(SB) - GLOBL ·libc_pipe2_trampoline_addr(SB), RODATA, $8 DATA ·libc_pipe2_trampoline_addr(SB)/8, $libc_pipe2_trampoline<>(SB) TEXT libc_getdents_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getdents(SB) - GLOBL ·libc_getdents_trampoline_addr(SB), RODATA, $8 DATA ·libc_getdents_trampoline_addr(SB)/8, $libc_getdents_trampoline<>(SB) TEXT libc_getcwd_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getcwd(SB) - GLOBL ·libc_getcwd_trampoline_addr(SB), RODATA, $8 DATA ·libc_getcwd_trampoline_addr(SB)/8, $libc_getcwd_trampoline<>(SB) TEXT libc_ioctl_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ioctl(SB) - GLOBL ·libc_ioctl_trampoline_addr(SB), RODATA, $8 DATA ·libc_ioctl_trampoline_addr(SB)/8, $libc_ioctl_trampoline<>(SB) TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sysctl(SB) - GLOBL ·libc_sysctl_trampoline_addr(SB), RODATA, $8 DATA ·libc_sysctl_trampoline_addr(SB)/8, $libc_sysctl_trampoline<>(SB) TEXT libc_ppoll_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ppoll(SB) - GLOBL ·libc_ppoll_trampoline_addr(SB), RODATA, $8 DATA ·libc_ppoll_trampoline_addr(SB)/8, $libc_ppoll_trampoline<>(SB) TEXT libc_access_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_access(SB) - GLOBL ·libc_access_trampoline_addr(SB), RODATA, $8 DATA ·libc_access_trampoline_addr(SB)/8, $libc_access_trampoline<>(SB) TEXT libc_adjtime_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_adjtime(SB) - GLOBL ·libc_adjtime_trampoline_addr(SB), RODATA, $8 DATA ·libc_adjtime_trampoline_addr(SB)/8, $libc_adjtime_trampoline<>(SB) TEXT libc_chdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chdir(SB) - GLOBL ·libc_chdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_chdir_trampoline_addr(SB)/8, $libc_chdir_trampoline<>(SB) TEXT libc_chflags_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chflags(SB) - GLOBL ·libc_chflags_trampoline_addr(SB), RODATA, $8 DATA ·libc_chflags_trampoline_addr(SB)/8, $libc_chflags_trampoline<>(SB) TEXT libc_chmod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chmod(SB) - GLOBL ·libc_chmod_trampoline_addr(SB), RODATA, $8 DATA ·libc_chmod_trampoline_addr(SB)/8, $libc_chmod_trampoline<>(SB) TEXT libc_chown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chown(SB) - GLOBL ·libc_chown_trampoline_addr(SB), RODATA, $8 DATA ·libc_chown_trampoline_addr(SB)/8, $libc_chown_trampoline<>(SB) TEXT libc_chroot_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chroot(SB) - GLOBL ·libc_chroot_trampoline_addr(SB), RODATA, $8 DATA ·libc_chroot_trampoline_addr(SB)/8, $libc_chroot_trampoline<>(SB) +TEXT libc_clock_gettime_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_clock_gettime(SB) +GLOBL ·libc_clock_gettime_trampoline_addr(SB), RODATA, $8 +DATA ·libc_clock_gettime_trampoline_addr(SB)/8, $libc_clock_gettime_trampoline<>(SB) + TEXT libc_close_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_close(SB) - GLOBL ·libc_close_trampoline_addr(SB), RODATA, $8 DATA ·libc_close_trampoline_addr(SB)/8, $libc_close_trampoline<>(SB) TEXT libc_dup_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup(SB) - GLOBL ·libc_dup_trampoline_addr(SB), RODATA, $8 DATA ·libc_dup_trampoline_addr(SB)/8, $libc_dup_trampoline<>(SB) TEXT libc_dup2_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup2(SB) - GLOBL ·libc_dup2_trampoline_addr(SB), RODATA, $8 DATA ·libc_dup2_trampoline_addr(SB)/8, $libc_dup2_trampoline<>(SB) TEXT libc_dup3_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup3(SB) - GLOBL ·libc_dup3_trampoline_addr(SB), RODATA, $8 DATA ·libc_dup3_trampoline_addr(SB)/8, $libc_dup3_trampoline<>(SB) TEXT libc_exit_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_exit(SB) - GLOBL ·libc_exit_trampoline_addr(SB), RODATA, $8 DATA ·libc_exit_trampoline_addr(SB)/8, $libc_exit_trampoline<>(SB) TEXT libc_faccessat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_faccessat(SB) - GLOBL ·libc_faccessat_trampoline_addr(SB), RODATA, $8 DATA ·libc_faccessat_trampoline_addr(SB)/8, $libc_faccessat_trampoline<>(SB) TEXT libc_fchdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchdir(SB) - GLOBL ·libc_fchdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchdir_trampoline_addr(SB)/8, $libc_fchdir_trampoline<>(SB) TEXT libc_fchflags_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchflags(SB) - GLOBL ·libc_fchflags_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchflags_trampoline_addr(SB)/8, $libc_fchflags_trampoline<>(SB) TEXT libc_fchmod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchmod(SB) - GLOBL ·libc_fchmod_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchmod_trampoline_addr(SB)/8, $libc_fchmod_trampoline<>(SB) TEXT libc_fchmodat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchmodat(SB) - GLOBL ·libc_fchmodat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchmodat_trampoline_addr(SB)/8, $libc_fchmodat_trampoline<>(SB) TEXT libc_fchown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchown(SB) - GLOBL ·libc_fchown_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchown_trampoline_addr(SB)/8, $libc_fchown_trampoline<>(SB) TEXT libc_fchownat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchownat(SB) - GLOBL ·libc_fchownat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchownat_trampoline_addr(SB)/8, $libc_fchownat_trampoline<>(SB) TEXT libc_flock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_flock(SB) - GLOBL ·libc_flock_trampoline_addr(SB), RODATA, $8 DATA ·libc_flock_trampoline_addr(SB)/8, $libc_flock_trampoline<>(SB) TEXT libc_fpathconf_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fpathconf(SB) - GLOBL ·libc_fpathconf_trampoline_addr(SB), RODATA, $8 DATA ·libc_fpathconf_trampoline_addr(SB)/8, $libc_fpathconf_trampoline<>(SB) TEXT libc_fstat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstat(SB) - GLOBL ·libc_fstat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fstat_trampoline_addr(SB)/8, $libc_fstat_trampoline<>(SB) TEXT libc_fstatat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) - GLOBL ·libc_fstatat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fstatat_trampoline_addr(SB)/8, $libc_fstatat_trampoline<>(SB) TEXT libc_fstatfs_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstatfs(SB) - GLOBL ·libc_fstatfs_trampoline_addr(SB), RODATA, $8 DATA ·libc_fstatfs_trampoline_addr(SB)/8, $libc_fstatfs_trampoline<>(SB) TEXT libc_fsync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fsync(SB) - GLOBL ·libc_fsync_trampoline_addr(SB), RODATA, $8 DATA ·libc_fsync_trampoline_addr(SB)/8, $libc_fsync_trampoline<>(SB) TEXT libc_ftruncate_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ftruncate(SB) - GLOBL ·libc_ftruncate_trampoline_addr(SB), RODATA, $8 DATA ·libc_ftruncate_trampoline_addr(SB)/8, $libc_ftruncate_trampoline<>(SB) TEXT libc_getegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getegid(SB) - GLOBL ·libc_getegid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getegid_trampoline_addr(SB)/8, $libc_getegid_trampoline<>(SB) TEXT libc_geteuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_geteuid(SB) - GLOBL ·libc_geteuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_geteuid_trampoline_addr(SB)/8, $libc_geteuid_trampoline<>(SB) TEXT libc_getgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getgid(SB) - GLOBL ·libc_getgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getgid_trampoline_addr(SB)/8, $libc_getgid_trampoline<>(SB) TEXT libc_getpgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpgid(SB) - GLOBL ·libc_getpgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpgid_trampoline_addr(SB)/8, $libc_getpgid_trampoline<>(SB) TEXT libc_getpgrp_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpgrp(SB) - GLOBL ·libc_getpgrp_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpgrp_trampoline_addr(SB)/8, $libc_getpgrp_trampoline<>(SB) TEXT libc_getpid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpid(SB) - GLOBL ·libc_getpid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpid_trampoline_addr(SB)/8, $libc_getpid_trampoline<>(SB) TEXT libc_getppid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getppid(SB) - GLOBL ·libc_getppid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getppid_trampoline_addr(SB)/8, $libc_getppid_trampoline<>(SB) TEXT libc_getpriority_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpriority(SB) - GLOBL ·libc_getpriority_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpriority_trampoline_addr(SB)/8, $libc_getpriority_trampoline<>(SB) TEXT libc_getrlimit_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrlimit(SB) - GLOBL ·libc_getrlimit_trampoline_addr(SB), RODATA, $8 DATA ·libc_getrlimit_trampoline_addr(SB)/8, $libc_getrlimit_trampoline<>(SB) TEXT libc_getrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrtable(SB) - GLOBL ·libc_getrtable_trampoline_addr(SB), RODATA, $8 DATA ·libc_getrtable_trampoline_addr(SB)/8, $libc_getrtable_trampoline<>(SB) TEXT libc_getrusage_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrusage(SB) - GLOBL ·libc_getrusage_trampoline_addr(SB), RODATA, $8 DATA ·libc_getrusage_trampoline_addr(SB)/8, $libc_getrusage_trampoline<>(SB) TEXT libc_getsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsid(SB) - GLOBL ·libc_getsid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getsid_trampoline_addr(SB)/8, $libc_getsid_trampoline<>(SB) TEXT libc_gettimeofday_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_gettimeofday(SB) - GLOBL ·libc_gettimeofday_trampoline_addr(SB), RODATA, $8 DATA ·libc_gettimeofday_trampoline_addr(SB)/8, $libc_gettimeofday_trampoline<>(SB) TEXT libc_getuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getuid(SB) - GLOBL ·libc_getuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getuid_trampoline_addr(SB)/8, $libc_getuid_trampoline<>(SB) TEXT libc_issetugid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_issetugid(SB) - GLOBL ·libc_issetugid_trampoline_addr(SB), RODATA, $8 DATA ·libc_issetugid_trampoline_addr(SB)/8, $libc_issetugid_trampoline<>(SB) TEXT libc_kill_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kill(SB) - GLOBL ·libc_kill_trampoline_addr(SB), RODATA, $8 DATA ·libc_kill_trampoline_addr(SB)/8, $libc_kill_trampoline<>(SB) TEXT libc_kqueue_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kqueue(SB) - GLOBL ·libc_kqueue_trampoline_addr(SB), RODATA, $8 DATA ·libc_kqueue_trampoline_addr(SB)/8, $libc_kqueue_trampoline<>(SB) TEXT libc_lchown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lchown(SB) - GLOBL ·libc_lchown_trampoline_addr(SB), RODATA, $8 DATA ·libc_lchown_trampoline_addr(SB)/8, $libc_lchown_trampoline<>(SB) TEXT libc_link_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_link(SB) - GLOBL ·libc_link_trampoline_addr(SB), RODATA, $8 DATA ·libc_link_trampoline_addr(SB)/8, $libc_link_trampoline<>(SB) TEXT libc_linkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_linkat(SB) - GLOBL ·libc_linkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_linkat_trampoline_addr(SB)/8, $libc_linkat_trampoline<>(SB) TEXT libc_listen_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_listen(SB) - GLOBL ·libc_listen_trampoline_addr(SB), RODATA, $8 DATA ·libc_listen_trampoline_addr(SB)/8, $libc_listen_trampoline<>(SB) TEXT libc_lstat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lstat(SB) - GLOBL ·libc_lstat_trampoline_addr(SB), RODATA, $8 DATA ·libc_lstat_trampoline_addr(SB)/8, $libc_lstat_trampoline<>(SB) TEXT libc_mkdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkdir(SB) - GLOBL ·libc_mkdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkdir_trampoline_addr(SB)/8, $libc_mkdir_trampoline<>(SB) TEXT libc_mkdirat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkdirat(SB) - GLOBL ·libc_mkdirat_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkdirat_trampoline_addr(SB)/8, $libc_mkdirat_trampoline<>(SB) TEXT libc_mkfifo_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkfifo(SB) - GLOBL ·libc_mkfifo_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkfifo_trampoline_addr(SB)/8, $libc_mkfifo_trampoline<>(SB) TEXT libc_mkfifoat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkfifoat(SB) - GLOBL ·libc_mkfifoat_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkfifoat_trampoline_addr(SB)/8, $libc_mkfifoat_trampoline<>(SB) TEXT libc_mknod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mknod(SB) - GLOBL ·libc_mknod_trampoline_addr(SB), RODATA, $8 DATA ·libc_mknod_trampoline_addr(SB)/8, $libc_mknod_trampoline<>(SB) TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mknodat(SB) - GLOBL ·libc_mknodat_trampoline_addr(SB), RODATA, $8 DATA ·libc_mknodat_trampoline_addr(SB)/8, $libc_mknodat_trampoline<>(SB) TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_nanosleep(SB) - GLOBL ·libc_nanosleep_trampoline_addr(SB), RODATA, $8 DATA ·libc_nanosleep_trampoline_addr(SB)/8, $libc_nanosleep_trampoline<>(SB) TEXT libc_open_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_open(SB) - GLOBL ·libc_open_trampoline_addr(SB), RODATA, $8 DATA ·libc_open_trampoline_addr(SB)/8, $libc_open_trampoline<>(SB) TEXT libc_openat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_openat(SB) - GLOBL ·libc_openat_trampoline_addr(SB), RODATA, $8 DATA ·libc_openat_trampoline_addr(SB)/8, $libc_openat_trampoline<>(SB) TEXT libc_pathconf_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pathconf(SB) - GLOBL ·libc_pathconf_trampoline_addr(SB), RODATA, $8 DATA ·libc_pathconf_trampoline_addr(SB)/8, $libc_pathconf_trampoline<>(SB) TEXT libc_pread_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pread(SB) - GLOBL ·libc_pread_trampoline_addr(SB), RODATA, $8 DATA ·libc_pread_trampoline_addr(SB)/8, $libc_pread_trampoline<>(SB) TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pwrite(SB) - GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $8 DATA ·libc_pwrite_trampoline_addr(SB)/8, $libc_pwrite_trampoline<>(SB) TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_read(SB) - GLOBL ·libc_read_trampoline_addr(SB), RODATA, $8 DATA ·libc_read_trampoline_addr(SB)/8, $libc_read_trampoline<>(SB) TEXT libc_readlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_readlink(SB) - GLOBL ·libc_readlink_trampoline_addr(SB), RODATA, $8 DATA ·libc_readlink_trampoline_addr(SB)/8, $libc_readlink_trampoline<>(SB) TEXT libc_readlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_readlinkat(SB) - GLOBL ·libc_readlinkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_readlinkat_trampoline_addr(SB)/8, $libc_readlinkat_trampoline<>(SB) TEXT libc_rename_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_rename(SB) - GLOBL ·libc_rename_trampoline_addr(SB), RODATA, $8 DATA ·libc_rename_trampoline_addr(SB)/8, $libc_rename_trampoline<>(SB) TEXT libc_renameat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_renameat(SB) - GLOBL ·libc_renameat_trampoline_addr(SB), RODATA, $8 DATA ·libc_renameat_trampoline_addr(SB)/8, $libc_renameat_trampoline<>(SB) TEXT libc_revoke_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_revoke(SB) - GLOBL ·libc_revoke_trampoline_addr(SB), RODATA, $8 DATA ·libc_revoke_trampoline_addr(SB)/8, $libc_revoke_trampoline<>(SB) TEXT libc_rmdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_rmdir(SB) - GLOBL ·libc_rmdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_rmdir_trampoline_addr(SB)/8, $libc_rmdir_trampoline<>(SB) TEXT libc_lseek_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lseek(SB) - GLOBL ·libc_lseek_trampoline_addr(SB), RODATA, $8 DATA ·libc_lseek_trampoline_addr(SB)/8, $libc_lseek_trampoline<>(SB) TEXT libc_select_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_select(SB) - GLOBL ·libc_select_trampoline_addr(SB), RODATA, $8 DATA ·libc_select_trampoline_addr(SB)/8, $libc_select_trampoline<>(SB) TEXT libc_setegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setegid(SB) - GLOBL ·libc_setegid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setegid_trampoline_addr(SB)/8, $libc_setegid_trampoline<>(SB) TEXT libc_seteuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_seteuid(SB) - GLOBL ·libc_seteuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_seteuid_trampoline_addr(SB)/8, $libc_seteuid_trampoline<>(SB) TEXT libc_setgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setgid(SB) - GLOBL ·libc_setgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setgid_trampoline_addr(SB)/8, $libc_setgid_trampoline<>(SB) TEXT libc_setlogin_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setlogin(SB) - GLOBL ·libc_setlogin_trampoline_addr(SB), RODATA, $8 DATA ·libc_setlogin_trampoline_addr(SB)/8, $libc_setlogin_trampoline<>(SB) TEXT libc_setpgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setpgid(SB) - GLOBL ·libc_setpgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setpgid_trampoline_addr(SB)/8, $libc_setpgid_trampoline<>(SB) TEXT libc_setpriority_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setpriority(SB) - GLOBL ·libc_setpriority_trampoline_addr(SB), RODATA, $8 DATA ·libc_setpriority_trampoline_addr(SB)/8, $libc_setpriority_trampoline<>(SB) TEXT libc_setregid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setregid(SB) - GLOBL ·libc_setregid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setregid_trampoline_addr(SB)/8, $libc_setregid_trampoline<>(SB) TEXT libc_setreuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setreuid(SB) - GLOBL ·libc_setreuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setreuid_trampoline_addr(SB)/8, $libc_setreuid_trampoline<>(SB) TEXT libc_setresgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setresgid(SB) - GLOBL ·libc_setresgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresgid_trampoline_addr(SB)/8, $libc_setresgid_trampoline<>(SB) TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setresuid(SB) - GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresuid_trampoline_addr(SB)/8, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) - -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setrtable(SB) - GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $8 DATA ·libc_setrtable_trampoline_addr(SB)/8, $libc_setrtable_trampoline<>(SB) TEXT libc_setsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsid(SB) - GLOBL ·libc_setsid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setsid_trampoline_addr(SB)/8, $libc_setsid_trampoline<>(SB) TEXT libc_settimeofday_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_settimeofday(SB) - GLOBL ·libc_settimeofday_trampoline_addr(SB), RODATA, $8 DATA ·libc_settimeofday_trampoline_addr(SB)/8, $libc_settimeofday_trampoline<>(SB) TEXT libc_setuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setuid(SB) - GLOBL ·libc_setuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setuid_trampoline_addr(SB)/8, $libc_setuid_trampoline<>(SB) TEXT libc_stat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_stat(SB) - GLOBL ·libc_stat_trampoline_addr(SB), RODATA, $8 DATA ·libc_stat_trampoline_addr(SB)/8, $libc_stat_trampoline<>(SB) TEXT libc_statfs_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_statfs(SB) - GLOBL ·libc_statfs_trampoline_addr(SB), RODATA, $8 DATA ·libc_statfs_trampoline_addr(SB)/8, $libc_statfs_trampoline<>(SB) TEXT libc_symlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_symlink(SB) - GLOBL ·libc_symlink_trampoline_addr(SB), RODATA, $8 DATA ·libc_symlink_trampoline_addr(SB)/8, $libc_symlink_trampoline<>(SB) TEXT libc_symlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_symlinkat(SB) - GLOBL ·libc_symlinkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_symlinkat_trampoline_addr(SB)/8, $libc_symlinkat_trampoline<>(SB) TEXT libc_sync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sync(SB) - GLOBL ·libc_sync_trampoline_addr(SB), RODATA, $8 DATA ·libc_sync_trampoline_addr(SB)/8, $libc_sync_trampoline<>(SB) TEXT libc_truncate_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_truncate(SB) - GLOBL ·libc_truncate_trampoline_addr(SB), RODATA, $8 DATA ·libc_truncate_trampoline_addr(SB)/8, $libc_truncate_trampoline<>(SB) TEXT libc_umask_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_umask(SB) - GLOBL ·libc_umask_trampoline_addr(SB), RODATA, $8 DATA ·libc_umask_trampoline_addr(SB)/8, $libc_umask_trampoline<>(SB) TEXT libc_unlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unlink(SB) - GLOBL ·libc_unlink_trampoline_addr(SB), RODATA, $8 DATA ·libc_unlink_trampoline_addr(SB)/8, $libc_unlink_trampoline<>(SB) TEXT libc_unlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unlinkat(SB) - GLOBL ·libc_unlinkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_unlinkat_trampoline_addr(SB)/8, $libc_unlinkat_trampoline<>(SB) TEXT libc_unmount_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unmount(SB) - GLOBL ·libc_unmount_trampoline_addr(SB), RODATA, $8 DATA ·libc_unmount_trampoline_addr(SB)/8, $libc_unmount_trampoline<>(SB) TEXT libc_write_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_write(SB) - GLOBL ·libc_write_trampoline_addr(SB), RODATA, $8 DATA ·libc_write_trampoline_addr(SB)/8, $libc_write_trampoline<>(SB) TEXT libc_mmap_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mmap(SB) - GLOBL ·libc_mmap_trampoline_addr(SB), RODATA, $8 DATA ·libc_mmap_trampoline_addr(SB)/8, $libc_mmap_trampoline<>(SB) TEXT libc_munmap_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munmap(SB) - GLOBL ·libc_munmap_trampoline_addr(SB), RODATA, $8 DATA ·libc_munmap_trampoline_addr(SB)/8, $libc_munmap_trampoline<>(SB) TEXT libc_utimensat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_utimensat(SB) - GLOBL ·libc_utimensat_trampoline_addr(SB), RODATA, $8 DATA ·libc_utimensat_trampoline_addr(SB)/8, $libc_utimensat_trampoline<>(SB) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go index 8da6791..7f64278 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.go @@ -527,6 +527,14 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { return } +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := syscall_syscall(libc_ioctl_trampoline_addr, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + var libc_ioctl_trampoline_addr uintptr //go:cgo_import_dynamic libc_ioctl ioctl "libc.so" @@ -696,6 +704,20 @@ var libc_chroot_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := syscall_syscall(libc_clock_gettime_trampoline_addr, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_clock_gettime_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := syscall_syscall(libc_close_trampoline_addr, uintptr(fd), 0, 0) if e1 != 0 { @@ -1872,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s index 9ad116d..b797045 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm.s @@ -5,792 +5,660 @@ TEXT libc_getgroups_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getgroups(SB) - GLOBL ·libc_getgroups_trampoline_addr(SB), RODATA, $4 DATA ·libc_getgroups_trampoline_addr(SB)/4, $libc_getgroups_trampoline<>(SB) TEXT libc_setgroups_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setgroups(SB) - GLOBL ·libc_setgroups_trampoline_addr(SB), RODATA, $4 DATA ·libc_setgroups_trampoline_addr(SB)/4, $libc_setgroups_trampoline<>(SB) TEXT libc_wait4_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_wait4(SB) - GLOBL ·libc_wait4_trampoline_addr(SB), RODATA, $4 DATA ·libc_wait4_trampoline_addr(SB)/4, $libc_wait4_trampoline<>(SB) TEXT libc_accept_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_accept(SB) - GLOBL ·libc_accept_trampoline_addr(SB), RODATA, $4 DATA ·libc_accept_trampoline_addr(SB)/4, $libc_accept_trampoline<>(SB) TEXT libc_bind_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_bind(SB) - GLOBL ·libc_bind_trampoline_addr(SB), RODATA, $4 DATA ·libc_bind_trampoline_addr(SB)/4, $libc_bind_trampoline<>(SB) TEXT libc_connect_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_connect(SB) - GLOBL ·libc_connect_trampoline_addr(SB), RODATA, $4 DATA ·libc_connect_trampoline_addr(SB)/4, $libc_connect_trampoline<>(SB) TEXT libc_socket_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_socket(SB) - GLOBL ·libc_socket_trampoline_addr(SB), RODATA, $4 DATA ·libc_socket_trampoline_addr(SB)/4, $libc_socket_trampoline<>(SB) TEXT libc_getsockopt_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsockopt(SB) - GLOBL ·libc_getsockopt_trampoline_addr(SB), RODATA, $4 DATA ·libc_getsockopt_trampoline_addr(SB)/4, $libc_getsockopt_trampoline<>(SB) TEXT libc_setsockopt_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsockopt(SB) - GLOBL ·libc_setsockopt_trampoline_addr(SB), RODATA, $4 DATA ·libc_setsockopt_trampoline_addr(SB)/4, $libc_setsockopt_trampoline<>(SB) TEXT libc_getpeername_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpeername(SB) - GLOBL ·libc_getpeername_trampoline_addr(SB), RODATA, $4 DATA ·libc_getpeername_trampoline_addr(SB)/4, $libc_getpeername_trampoline<>(SB) TEXT libc_getsockname_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsockname(SB) - GLOBL ·libc_getsockname_trampoline_addr(SB), RODATA, $4 DATA ·libc_getsockname_trampoline_addr(SB)/4, $libc_getsockname_trampoline<>(SB) TEXT libc_shutdown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_shutdown(SB) - GLOBL ·libc_shutdown_trampoline_addr(SB), RODATA, $4 DATA ·libc_shutdown_trampoline_addr(SB)/4, $libc_shutdown_trampoline<>(SB) TEXT libc_socketpair_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_socketpair(SB) - GLOBL ·libc_socketpair_trampoline_addr(SB), RODATA, $4 DATA ·libc_socketpair_trampoline_addr(SB)/4, $libc_socketpair_trampoline<>(SB) TEXT libc_recvfrom_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_recvfrom(SB) - GLOBL ·libc_recvfrom_trampoline_addr(SB), RODATA, $4 DATA ·libc_recvfrom_trampoline_addr(SB)/4, $libc_recvfrom_trampoline<>(SB) TEXT libc_sendto_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sendto(SB) - GLOBL ·libc_sendto_trampoline_addr(SB), RODATA, $4 DATA ·libc_sendto_trampoline_addr(SB)/4, $libc_sendto_trampoline<>(SB) TEXT libc_recvmsg_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_recvmsg(SB) - GLOBL ·libc_recvmsg_trampoline_addr(SB), RODATA, $4 DATA ·libc_recvmsg_trampoline_addr(SB)/4, $libc_recvmsg_trampoline<>(SB) TEXT libc_sendmsg_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sendmsg(SB) - GLOBL ·libc_sendmsg_trampoline_addr(SB), RODATA, $4 DATA ·libc_sendmsg_trampoline_addr(SB)/4, $libc_sendmsg_trampoline<>(SB) TEXT libc_kevent_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kevent(SB) - GLOBL ·libc_kevent_trampoline_addr(SB), RODATA, $4 DATA ·libc_kevent_trampoline_addr(SB)/4, $libc_kevent_trampoline<>(SB) TEXT libc_utimes_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_utimes(SB) - GLOBL ·libc_utimes_trampoline_addr(SB), RODATA, $4 DATA ·libc_utimes_trampoline_addr(SB)/4, $libc_utimes_trampoline<>(SB) TEXT libc_futimes_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_futimes(SB) - GLOBL ·libc_futimes_trampoline_addr(SB), RODATA, $4 DATA ·libc_futimes_trampoline_addr(SB)/4, $libc_futimes_trampoline<>(SB) TEXT libc_poll_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_poll(SB) - GLOBL ·libc_poll_trampoline_addr(SB), RODATA, $4 DATA ·libc_poll_trampoline_addr(SB)/4, $libc_poll_trampoline<>(SB) TEXT libc_madvise_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_madvise(SB) - GLOBL ·libc_madvise_trampoline_addr(SB), RODATA, $4 DATA ·libc_madvise_trampoline_addr(SB)/4, $libc_madvise_trampoline<>(SB) TEXT libc_mlock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mlock(SB) - GLOBL ·libc_mlock_trampoline_addr(SB), RODATA, $4 DATA ·libc_mlock_trampoline_addr(SB)/4, $libc_mlock_trampoline<>(SB) TEXT libc_mlockall_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mlockall(SB) - GLOBL ·libc_mlockall_trampoline_addr(SB), RODATA, $4 DATA ·libc_mlockall_trampoline_addr(SB)/4, $libc_mlockall_trampoline<>(SB) TEXT libc_mprotect_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mprotect(SB) - GLOBL ·libc_mprotect_trampoline_addr(SB), RODATA, $4 DATA ·libc_mprotect_trampoline_addr(SB)/4, $libc_mprotect_trampoline<>(SB) TEXT libc_msync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_msync(SB) - GLOBL ·libc_msync_trampoline_addr(SB), RODATA, $4 DATA ·libc_msync_trampoline_addr(SB)/4, $libc_msync_trampoline<>(SB) TEXT libc_munlock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munlock(SB) - GLOBL ·libc_munlock_trampoline_addr(SB), RODATA, $4 DATA ·libc_munlock_trampoline_addr(SB)/4, $libc_munlock_trampoline<>(SB) TEXT libc_munlockall_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munlockall(SB) - GLOBL ·libc_munlockall_trampoline_addr(SB), RODATA, $4 DATA ·libc_munlockall_trampoline_addr(SB)/4, $libc_munlockall_trampoline<>(SB) TEXT libc_pipe2_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pipe2(SB) - GLOBL ·libc_pipe2_trampoline_addr(SB), RODATA, $4 DATA ·libc_pipe2_trampoline_addr(SB)/4, $libc_pipe2_trampoline<>(SB) TEXT libc_getdents_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getdents(SB) - GLOBL ·libc_getdents_trampoline_addr(SB), RODATA, $4 DATA ·libc_getdents_trampoline_addr(SB)/4, $libc_getdents_trampoline<>(SB) TEXT libc_getcwd_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getcwd(SB) - GLOBL ·libc_getcwd_trampoline_addr(SB), RODATA, $4 DATA ·libc_getcwd_trampoline_addr(SB)/4, $libc_getcwd_trampoline<>(SB) TEXT libc_ioctl_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ioctl(SB) - GLOBL ·libc_ioctl_trampoline_addr(SB), RODATA, $4 DATA ·libc_ioctl_trampoline_addr(SB)/4, $libc_ioctl_trampoline<>(SB) TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sysctl(SB) - GLOBL ·libc_sysctl_trampoline_addr(SB), RODATA, $4 DATA ·libc_sysctl_trampoline_addr(SB)/4, $libc_sysctl_trampoline<>(SB) TEXT libc_ppoll_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ppoll(SB) - GLOBL ·libc_ppoll_trampoline_addr(SB), RODATA, $4 DATA ·libc_ppoll_trampoline_addr(SB)/4, $libc_ppoll_trampoline<>(SB) TEXT libc_access_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_access(SB) - GLOBL ·libc_access_trampoline_addr(SB), RODATA, $4 DATA ·libc_access_trampoline_addr(SB)/4, $libc_access_trampoline<>(SB) TEXT libc_adjtime_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_adjtime(SB) - GLOBL ·libc_adjtime_trampoline_addr(SB), RODATA, $4 DATA ·libc_adjtime_trampoline_addr(SB)/4, $libc_adjtime_trampoline<>(SB) TEXT libc_chdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chdir(SB) - GLOBL ·libc_chdir_trampoline_addr(SB), RODATA, $4 DATA ·libc_chdir_trampoline_addr(SB)/4, $libc_chdir_trampoline<>(SB) TEXT libc_chflags_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chflags(SB) - GLOBL ·libc_chflags_trampoline_addr(SB), RODATA, $4 DATA ·libc_chflags_trampoline_addr(SB)/4, $libc_chflags_trampoline<>(SB) TEXT libc_chmod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chmod(SB) - GLOBL ·libc_chmod_trampoline_addr(SB), RODATA, $4 DATA ·libc_chmod_trampoline_addr(SB)/4, $libc_chmod_trampoline<>(SB) TEXT libc_chown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chown(SB) - GLOBL ·libc_chown_trampoline_addr(SB), RODATA, $4 DATA ·libc_chown_trampoline_addr(SB)/4, $libc_chown_trampoline<>(SB) TEXT libc_chroot_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chroot(SB) - GLOBL ·libc_chroot_trampoline_addr(SB), RODATA, $4 DATA ·libc_chroot_trampoline_addr(SB)/4, $libc_chroot_trampoline<>(SB) +TEXT libc_clock_gettime_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_clock_gettime(SB) +GLOBL ·libc_clock_gettime_trampoline_addr(SB), RODATA, $4 +DATA ·libc_clock_gettime_trampoline_addr(SB)/4, $libc_clock_gettime_trampoline<>(SB) + TEXT libc_close_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_close(SB) - GLOBL ·libc_close_trampoline_addr(SB), RODATA, $4 DATA ·libc_close_trampoline_addr(SB)/4, $libc_close_trampoline<>(SB) TEXT libc_dup_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup(SB) - GLOBL ·libc_dup_trampoline_addr(SB), RODATA, $4 DATA ·libc_dup_trampoline_addr(SB)/4, $libc_dup_trampoline<>(SB) TEXT libc_dup2_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup2(SB) - GLOBL ·libc_dup2_trampoline_addr(SB), RODATA, $4 DATA ·libc_dup2_trampoline_addr(SB)/4, $libc_dup2_trampoline<>(SB) TEXT libc_dup3_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup3(SB) - GLOBL ·libc_dup3_trampoline_addr(SB), RODATA, $4 DATA ·libc_dup3_trampoline_addr(SB)/4, $libc_dup3_trampoline<>(SB) TEXT libc_exit_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_exit(SB) - GLOBL ·libc_exit_trampoline_addr(SB), RODATA, $4 DATA ·libc_exit_trampoline_addr(SB)/4, $libc_exit_trampoline<>(SB) TEXT libc_faccessat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_faccessat(SB) - GLOBL ·libc_faccessat_trampoline_addr(SB), RODATA, $4 DATA ·libc_faccessat_trampoline_addr(SB)/4, $libc_faccessat_trampoline<>(SB) TEXT libc_fchdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchdir(SB) - GLOBL ·libc_fchdir_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchdir_trampoline_addr(SB)/4, $libc_fchdir_trampoline<>(SB) TEXT libc_fchflags_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchflags(SB) - GLOBL ·libc_fchflags_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchflags_trampoline_addr(SB)/4, $libc_fchflags_trampoline<>(SB) TEXT libc_fchmod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchmod(SB) - GLOBL ·libc_fchmod_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchmod_trampoline_addr(SB)/4, $libc_fchmod_trampoline<>(SB) TEXT libc_fchmodat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchmodat(SB) - GLOBL ·libc_fchmodat_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchmodat_trampoline_addr(SB)/4, $libc_fchmodat_trampoline<>(SB) TEXT libc_fchown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchown(SB) - GLOBL ·libc_fchown_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchown_trampoline_addr(SB)/4, $libc_fchown_trampoline<>(SB) TEXT libc_fchownat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchownat(SB) - GLOBL ·libc_fchownat_trampoline_addr(SB), RODATA, $4 DATA ·libc_fchownat_trampoline_addr(SB)/4, $libc_fchownat_trampoline<>(SB) TEXT libc_flock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_flock(SB) - GLOBL ·libc_flock_trampoline_addr(SB), RODATA, $4 DATA ·libc_flock_trampoline_addr(SB)/4, $libc_flock_trampoline<>(SB) TEXT libc_fpathconf_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fpathconf(SB) - GLOBL ·libc_fpathconf_trampoline_addr(SB), RODATA, $4 DATA ·libc_fpathconf_trampoline_addr(SB)/4, $libc_fpathconf_trampoline<>(SB) TEXT libc_fstat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstat(SB) - GLOBL ·libc_fstat_trampoline_addr(SB), RODATA, $4 DATA ·libc_fstat_trampoline_addr(SB)/4, $libc_fstat_trampoline<>(SB) TEXT libc_fstatat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) - GLOBL ·libc_fstatat_trampoline_addr(SB), RODATA, $4 DATA ·libc_fstatat_trampoline_addr(SB)/4, $libc_fstatat_trampoline<>(SB) TEXT libc_fstatfs_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstatfs(SB) - GLOBL ·libc_fstatfs_trampoline_addr(SB), RODATA, $4 DATA ·libc_fstatfs_trampoline_addr(SB)/4, $libc_fstatfs_trampoline<>(SB) TEXT libc_fsync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fsync(SB) - GLOBL ·libc_fsync_trampoline_addr(SB), RODATA, $4 DATA ·libc_fsync_trampoline_addr(SB)/4, $libc_fsync_trampoline<>(SB) TEXT libc_ftruncate_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ftruncate(SB) - GLOBL ·libc_ftruncate_trampoline_addr(SB), RODATA, $4 DATA ·libc_ftruncate_trampoline_addr(SB)/4, $libc_ftruncate_trampoline<>(SB) TEXT libc_getegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getegid(SB) - GLOBL ·libc_getegid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getegid_trampoline_addr(SB)/4, $libc_getegid_trampoline<>(SB) TEXT libc_geteuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_geteuid(SB) - GLOBL ·libc_geteuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_geteuid_trampoline_addr(SB)/4, $libc_geteuid_trampoline<>(SB) TEXT libc_getgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getgid(SB) - GLOBL ·libc_getgid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getgid_trampoline_addr(SB)/4, $libc_getgid_trampoline<>(SB) TEXT libc_getpgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpgid(SB) - GLOBL ·libc_getpgid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getpgid_trampoline_addr(SB)/4, $libc_getpgid_trampoline<>(SB) TEXT libc_getpgrp_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpgrp(SB) - GLOBL ·libc_getpgrp_trampoline_addr(SB), RODATA, $4 DATA ·libc_getpgrp_trampoline_addr(SB)/4, $libc_getpgrp_trampoline<>(SB) TEXT libc_getpid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpid(SB) - GLOBL ·libc_getpid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getpid_trampoline_addr(SB)/4, $libc_getpid_trampoline<>(SB) TEXT libc_getppid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getppid(SB) - GLOBL ·libc_getppid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getppid_trampoline_addr(SB)/4, $libc_getppid_trampoline<>(SB) TEXT libc_getpriority_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpriority(SB) - GLOBL ·libc_getpriority_trampoline_addr(SB), RODATA, $4 DATA ·libc_getpriority_trampoline_addr(SB)/4, $libc_getpriority_trampoline<>(SB) TEXT libc_getrlimit_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrlimit(SB) - GLOBL ·libc_getrlimit_trampoline_addr(SB), RODATA, $4 DATA ·libc_getrlimit_trampoline_addr(SB)/4, $libc_getrlimit_trampoline<>(SB) TEXT libc_getrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrtable(SB) - GLOBL ·libc_getrtable_trampoline_addr(SB), RODATA, $4 DATA ·libc_getrtable_trampoline_addr(SB)/4, $libc_getrtable_trampoline<>(SB) TEXT libc_getrusage_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrusage(SB) - GLOBL ·libc_getrusage_trampoline_addr(SB), RODATA, $4 DATA ·libc_getrusage_trampoline_addr(SB)/4, $libc_getrusage_trampoline<>(SB) TEXT libc_getsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsid(SB) - GLOBL ·libc_getsid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getsid_trampoline_addr(SB)/4, $libc_getsid_trampoline<>(SB) TEXT libc_gettimeofday_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_gettimeofday(SB) - GLOBL ·libc_gettimeofday_trampoline_addr(SB), RODATA, $4 DATA ·libc_gettimeofday_trampoline_addr(SB)/4, $libc_gettimeofday_trampoline<>(SB) TEXT libc_getuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getuid(SB) - GLOBL ·libc_getuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_getuid_trampoline_addr(SB)/4, $libc_getuid_trampoline<>(SB) TEXT libc_issetugid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_issetugid(SB) - GLOBL ·libc_issetugid_trampoline_addr(SB), RODATA, $4 DATA ·libc_issetugid_trampoline_addr(SB)/4, $libc_issetugid_trampoline<>(SB) TEXT libc_kill_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kill(SB) - GLOBL ·libc_kill_trampoline_addr(SB), RODATA, $4 DATA ·libc_kill_trampoline_addr(SB)/4, $libc_kill_trampoline<>(SB) TEXT libc_kqueue_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kqueue(SB) - GLOBL ·libc_kqueue_trampoline_addr(SB), RODATA, $4 DATA ·libc_kqueue_trampoline_addr(SB)/4, $libc_kqueue_trampoline<>(SB) TEXT libc_lchown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lchown(SB) - GLOBL ·libc_lchown_trampoline_addr(SB), RODATA, $4 DATA ·libc_lchown_trampoline_addr(SB)/4, $libc_lchown_trampoline<>(SB) TEXT libc_link_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_link(SB) - GLOBL ·libc_link_trampoline_addr(SB), RODATA, $4 DATA ·libc_link_trampoline_addr(SB)/4, $libc_link_trampoline<>(SB) TEXT libc_linkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_linkat(SB) - GLOBL ·libc_linkat_trampoline_addr(SB), RODATA, $4 DATA ·libc_linkat_trampoline_addr(SB)/4, $libc_linkat_trampoline<>(SB) TEXT libc_listen_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_listen(SB) - GLOBL ·libc_listen_trampoline_addr(SB), RODATA, $4 DATA ·libc_listen_trampoline_addr(SB)/4, $libc_listen_trampoline<>(SB) TEXT libc_lstat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lstat(SB) - GLOBL ·libc_lstat_trampoline_addr(SB), RODATA, $4 DATA ·libc_lstat_trampoline_addr(SB)/4, $libc_lstat_trampoline<>(SB) TEXT libc_mkdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkdir(SB) - GLOBL ·libc_mkdir_trampoline_addr(SB), RODATA, $4 DATA ·libc_mkdir_trampoline_addr(SB)/4, $libc_mkdir_trampoline<>(SB) TEXT libc_mkdirat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkdirat(SB) - GLOBL ·libc_mkdirat_trampoline_addr(SB), RODATA, $4 DATA ·libc_mkdirat_trampoline_addr(SB)/4, $libc_mkdirat_trampoline<>(SB) TEXT libc_mkfifo_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkfifo(SB) - GLOBL ·libc_mkfifo_trampoline_addr(SB), RODATA, $4 DATA ·libc_mkfifo_trampoline_addr(SB)/4, $libc_mkfifo_trampoline<>(SB) TEXT libc_mkfifoat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkfifoat(SB) - GLOBL ·libc_mkfifoat_trampoline_addr(SB), RODATA, $4 DATA ·libc_mkfifoat_trampoline_addr(SB)/4, $libc_mkfifoat_trampoline<>(SB) TEXT libc_mknod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mknod(SB) - GLOBL ·libc_mknod_trampoline_addr(SB), RODATA, $4 DATA ·libc_mknod_trampoline_addr(SB)/4, $libc_mknod_trampoline<>(SB) TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mknodat(SB) - GLOBL ·libc_mknodat_trampoline_addr(SB), RODATA, $4 DATA ·libc_mknodat_trampoline_addr(SB)/4, $libc_mknodat_trampoline<>(SB) TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_nanosleep(SB) - GLOBL ·libc_nanosleep_trampoline_addr(SB), RODATA, $4 DATA ·libc_nanosleep_trampoline_addr(SB)/4, $libc_nanosleep_trampoline<>(SB) TEXT libc_open_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_open(SB) - GLOBL ·libc_open_trampoline_addr(SB), RODATA, $4 DATA ·libc_open_trampoline_addr(SB)/4, $libc_open_trampoline<>(SB) TEXT libc_openat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_openat(SB) - GLOBL ·libc_openat_trampoline_addr(SB), RODATA, $4 DATA ·libc_openat_trampoline_addr(SB)/4, $libc_openat_trampoline<>(SB) TEXT libc_pathconf_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pathconf(SB) - GLOBL ·libc_pathconf_trampoline_addr(SB), RODATA, $4 DATA ·libc_pathconf_trampoline_addr(SB)/4, $libc_pathconf_trampoline<>(SB) TEXT libc_pread_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pread(SB) - GLOBL ·libc_pread_trampoline_addr(SB), RODATA, $4 DATA ·libc_pread_trampoline_addr(SB)/4, $libc_pread_trampoline<>(SB) TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pwrite(SB) - GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $4 DATA ·libc_pwrite_trampoline_addr(SB)/4, $libc_pwrite_trampoline<>(SB) TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_read(SB) - GLOBL ·libc_read_trampoline_addr(SB), RODATA, $4 DATA ·libc_read_trampoline_addr(SB)/4, $libc_read_trampoline<>(SB) TEXT libc_readlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_readlink(SB) - GLOBL ·libc_readlink_trampoline_addr(SB), RODATA, $4 DATA ·libc_readlink_trampoline_addr(SB)/4, $libc_readlink_trampoline<>(SB) TEXT libc_readlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_readlinkat(SB) - GLOBL ·libc_readlinkat_trampoline_addr(SB), RODATA, $4 DATA ·libc_readlinkat_trampoline_addr(SB)/4, $libc_readlinkat_trampoline<>(SB) TEXT libc_rename_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_rename(SB) - GLOBL ·libc_rename_trampoline_addr(SB), RODATA, $4 DATA ·libc_rename_trampoline_addr(SB)/4, $libc_rename_trampoline<>(SB) TEXT libc_renameat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_renameat(SB) - GLOBL ·libc_renameat_trampoline_addr(SB), RODATA, $4 DATA ·libc_renameat_trampoline_addr(SB)/4, $libc_renameat_trampoline<>(SB) TEXT libc_revoke_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_revoke(SB) - GLOBL ·libc_revoke_trampoline_addr(SB), RODATA, $4 DATA ·libc_revoke_trampoline_addr(SB)/4, $libc_revoke_trampoline<>(SB) TEXT libc_rmdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_rmdir(SB) - GLOBL ·libc_rmdir_trampoline_addr(SB), RODATA, $4 DATA ·libc_rmdir_trampoline_addr(SB)/4, $libc_rmdir_trampoline<>(SB) TEXT libc_lseek_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lseek(SB) - GLOBL ·libc_lseek_trampoline_addr(SB), RODATA, $4 DATA ·libc_lseek_trampoline_addr(SB)/4, $libc_lseek_trampoline<>(SB) TEXT libc_select_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_select(SB) - GLOBL ·libc_select_trampoline_addr(SB), RODATA, $4 DATA ·libc_select_trampoline_addr(SB)/4, $libc_select_trampoline<>(SB) TEXT libc_setegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setegid(SB) - GLOBL ·libc_setegid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setegid_trampoline_addr(SB)/4, $libc_setegid_trampoline<>(SB) TEXT libc_seteuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_seteuid(SB) - GLOBL ·libc_seteuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_seteuid_trampoline_addr(SB)/4, $libc_seteuid_trampoline<>(SB) TEXT libc_setgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setgid(SB) - GLOBL ·libc_setgid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setgid_trampoline_addr(SB)/4, $libc_setgid_trampoline<>(SB) TEXT libc_setlogin_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setlogin(SB) - GLOBL ·libc_setlogin_trampoline_addr(SB), RODATA, $4 DATA ·libc_setlogin_trampoline_addr(SB)/4, $libc_setlogin_trampoline<>(SB) TEXT libc_setpgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setpgid(SB) - GLOBL ·libc_setpgid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setpgid_trampoline_addr(SB)/4, $libc_setpgid_trampoline<>(SB) TEXT libc_setpriority_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setpriority(SB) - GLOBL ·libc_setpriority_trampoline_addr(SB), RODATA, $4 DATA ·libc_setpriority_trampoline_addr(SB)/4, $libc_setpriority_trampoline<>(SB) TEXT libc_setregid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setregid(SB) - GLOBL ·libc_setregid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setregid_trampoline_addr(SB)/4, $libc_setregid_trampoline<>(SB) TEXT libc_setreuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setreuid(SB) - GLOBL ·libc_setreuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setreuid_trampoline_addr(SB)/4, $libc_setreuid_trampoline<>(SB) TEXT libc_setresgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setresgid(SB) - GLOBL ·libc_setresgid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setresgid_trampoline_addr(SB)/4, $libc_setresgid_trampoline<>(SB) TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setresuid(SB) - GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setresuid_trampoline_addr(SB)/4, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) - -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $4 -DATA ·libc_setrlimit_trampoline_addr(SB)/4, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setrtable(SB) - GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $4 DATA ·libc_setrtable_trampoline_addr(SB)/4, $libc_setrtable_trampoline<>(SB) TEXT libc_setsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsid(SB) - GLOBL ·libc_setsid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setsid_trampoline_addr(SB)/4, $libc_setsid_trampoline<>(SB) TEXT libc_settimeofday_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_settimeofday(SB) - GLOBL ·libc_settimeofday_trampoline_addr(SB), RODATA, $4 DATA ·libc_settimeofday_trampoline_addr(SB)/4, $libc_settimeofday_trampoline<>(SB) TEXT libc_setuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setuid(SB) - GLOBL ·libc_setuid_trampoline_addr(SB), RODATA, $4 DATA ·libc_setuid_trampoline_addr(SB)/4, $libc_setuid_trampoline<>(SB) TEXT libc_stat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_stat(SB) - GLOBL ·libc_stat_trampoline_addr(SB), RODATA, $4 DATA ·libc_stat_trampoline_addr(SB)/4, $libc_stat_trampoline<>(SB) TEXT libc_statfs_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_statfs(SB) - GLOBL ·libc_statfs_trampoline_addr(SB), RODATA, $4 DATA ·libc_statfs_trampoline_addr(SB)/4, $libc_statfs_trampoline<>(SB) TEXT libc_symlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_symlink(SB) - GLOBL ·libc_symlink_trampoline_addr(SB), RODATA, $4 DATA ·libc_symlink_trampoline_addr(SB)/4, $libc_symlink_trampoline<>(SB) TEXT libc_symlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_symlinkat(SB) - GLOBL ·libc_symlinkat_trampoline_addr(SB), RODATA, $4 DATA ·libc_symlinkat_trampoline_addr(SB)/4, $libc_symlinkat_trampoline<>(SB) TEXT libc_sync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sync(SB) - GLOBL ·libc_sync_trampoline_addr(SB), RODATA, $4 DATA ·libc_sync_trampoline_addr(SB)/4, $libc_sync_trampoline<>(SB) TEXT libc_truncate_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_truncate(SB) - GLOBL ·libc_truncate_trampoline_addr(SB), RODATA, $4 DATA ·libc_truncate_trampoline_addr(SB)/4, $libc_truncate_trampoline<>(SB) TEXT libc_umask_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_umask(SB) - GLOBL ·libc_umask_trampoline_addr(SB), RODATA, $4 DATA ·libc_umask_trampoline_addr(SB)/4, $libc_umask_trampoline<>(SB) TEXT libc_unlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unlink(SB) - GLOBL ·libc_unlink_trampoline_addr(SB), RODATA, $4 DATA ·libc_unlink_trampoline_addr(SB)/4, $libc_unlink_trampoline<>(SB) TEXT libc_unlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unlinkat(SB) - GLOBL ·libc_unlinkat_trampoline_addr(SB), RODATA, $4 DATA ·libc_unlinkat_trampoline_addr(SB)/4, $libc_unlinkat_trampoline<>(SB) TEXT libc_unmount_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unmount(SB) - GLOBL ·libc_unmount_trampoline_addr(SB), RODATA, $4 DATA ·libc_unmount_trampoline_addr(SB)/4, $libc_unmount_trampoline<>(SB) TEXT libc_write_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_write(SB) - GLOBL ·libc_write_trampoline_addr(SB), RODATA, $4 DATA ·libc_write_trampoline_addr(SB)/4, $libc_write_trampoline<>(SB) TEXT libc_mmap_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mmap(SB) - GLOBL ·libc_mmap_trampoline_addr(SB), RODATA, $4 DATA ·libc_mmap_trampoline_addr(SB)/4, $libc_mmap_trampoline<>(SB) TEXT libc_munmap_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munmap(SB) - GLOBL ·libc_munmap_trampoline_addr(SB), RODATA, $4 DATA ·libc_munmap_trampoline_addr(SB)/4, $libc_munmap_trampoline<>(SB) TEXT libc_utimensat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_utimensat(SB) - GLOBL ·libc_utimensat_trampoline_addr(SB), RODATA, $4 DATA ·libc_utimensat_trampoline_addr(SB)/4, $libc_utimensat_trampoline<>(SB) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go index 800aab6..756ef7b 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.go @@ -527,6 +527,14 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { return } +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := syscall_syscall(libc_ioctl_trampoline_addr, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + var libc_ioctl_trampoline_addr uintptr //go:cgo_import_dynamic libc_ioctl ioctl "libc.so" @@ -696,6 +704,20 @@ var libc_chroot_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := syscall_syscall(libc_clock_gettime_trampoline_addr, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_clock_gettime_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := syscall_syscall(libc_close_trampoline_addr, uintptr(fd), 0, 0) if e1 != 0 { @@ -1872,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s index 4efeff9..a871266 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_arm64.s @@ -5,792 +5,660 @@ TEXT libc_getgroups_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getgroups(SB) - GLOBL ·libc_getgroups_trampoline_addr(SB), RODATA, $8 DATA ·libc_getgroups_trampoline_addr(SB)/8, $libc_getgroups_trampoline<>(SB) TEXT libc_setgroups_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setgroups(SB) - GLOBL ·libc_setgroups_trampoline_addr(SB), RODATA, $8 DATA ·libc_setgroups_trampoline_addr(SB)/8, $libc_setgroups_trampoline<>(SB) TEXT libc_wait4_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_wait4(SB) - GLOBL ·libc_wait4_trampoline_addr(SB), RODATA, $8 DATA ·libc_wait4_trampoline_addr(SB)/8, $libc_wait4_trampoline<>(SB) TEXT libc_accept_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_accept(SB) - GLOBL ·libc_accept_trampoline_addr(SB), RODATA, $8 DATA ·libc_accept_trampoline_addr(SB)/8, $libc_accept_trampoline<>(SB) TEXT libc_bind_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_bind(SB) - GLOBL ·libc_bind_trampoline_addr(SB), RODATA, $8 DATA ·libc_bind_trampoline_addr(SB)/8, $libc_bind_trampoline<>(SB) TEXT libc_connect_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_connect(SB) - GLOBL ·libc_connect_trampoline_addr(SB), RODATA, $8 DATA ·libc_connect_trampoline_addr(SB)/8, $libc_connect_trampoline<>(SB) TEXT libc_socket_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_socket(SB) - GLOBL ·libc_socket_trampoline_addr(SB), RODATA, $8 DATA ·libc_socket_trampoline_addr(SB)/8, $libc_socket_trampoline<>(SB) TEXT libc_getsockopt_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsockopt(SB) - GLOBL ·libc_getsockopt_trampoline_addr(SB), RODATA, $8 DATA ·libc_getsockopt_trampoline_addr(SB)/8, $libc_getsockopt_trampoline<>(SB) TEXT libc_setsockopt_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsockopt(SB) - GLOBL ·libc_setsockopt_trampoline_addr(SB), RODATA, $8 DATA ·libc_setsockopt_trampoline_addr(SB)/8, $libc_setsockopt_trampoline<>(SB) TEXT libc_getpeername_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpeername(SB) - GLOBL ·libc_getpeername_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpeername_trampoline_addr(SB)/8, $libc_getpeername_trampoline<>(SB) TEXT libc_getsockname_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsockname(SB) - GLOBL ·libc_getsockname_trampoline_addr(SB), RODATA, $8 DATA ·libc_getsockname_trampoline_addr(SB)/8, $libc_getsockname_trampoline<>(SB) TEXT libc_shutdown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_shutdown(SB) - GLOBL ·libc_shutdown_trampoline_addr(SB), RODATA, $8 DATA ·libc_shutdown_trampoline_addr(SB)/8, $libc_shutdown_trampoline<>(SB) TEXT libc_socketpair_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_socketpair(SB) - GLOBL ·libc_socketpair_trampoline_addr(SB), RODATA, $8 DATA ·libc_socketpair_trampoline_addr(SB)/8, $libc_socketpair_trampoline<>(SB) TEXT libc_recvfrom_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_recvfrom(SB) - GLOBL ·libc_recvfrom_trampoline_addr(SB), RODATA, $8 DATA ·libc_recvfrom_trampoline_addr(SB)/8, $libc_recvfrom_trampoline<>(SB) TEXT libc_sendto_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sendto(SB) - GLOBL ·libc_sendto_trampoline_addr(SB), RODATA, $8 DATA ·libc_sendto_trampoline_addr(SB)/8, $libc_sendto_trampoline<>(SB) TEXT libc_recvmsg_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_recvmsg(SB) - GLOBL ·libc_recvmsg_trampoline_addr(SB), RODATA, $8 DATA ·libc_recvmsg_trampoline_addr(SB)/8, $libc_recvmsg_trampoline<>(SB) TEXT libc_sendmsg_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sendmsg(SB) - GLOBL ·libc_sendmsg_trampoline_addr(SB), RODATA, $8 DATA ·libc_sendmsg_trampoline_addr(SB)/8, $libc_sendmsg_trampoline<>(SB) TEXT libc_kevent_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kevent(SB) - GLOBL ·libc_kevent_trampoline_addr(SB), RODATA, $8 DATA ·libc_kevent_trampoline_addr(SB)/8, $libc_kevent_trampoline<>(SB) TEXT libc_utimes_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_utimes(SB) - GLOBL ·libc_utimes_trampoline_addr(SB), RODATA, $8 DATA ·libc_utimes_trampoline_addr(SB)/8, $libc_utimes_trampoline<>(SB) TEXT libc_futimes_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_futimes(SB) - GLOBL ·libc_futimes_trampoline_addr(SB), RODATA, $8 DATA ·libc_futimes_trampoline_addr(SB)/8, $libc_futimes_trampoline<>(SB) TEXT libc_poll_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_poll(SB) - GLOBL ·libc_poll_trampoline_addr(SB), RODATA, $8 DATA ·libc_poll_trampoline_addr(SB)/8, $libc_poll_trampoline<>(SB) TEXT libc_madvise_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_madvise(SB) - GLOBL ·libc_madvise_trampoline_addr(SB), RODATA, $8 DATA ·libc_madvise_trampoline_addr(SB)/8, $libc_madvise_trampoline<>(SB) TEXT libc_mlock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mlock(SB) - GLOBL ·libc_mlock_trampoline_addr(SB), RODATA, $8 DATA ·libc_mlock_trampoline_addr(SB)/8, $libc_mlock_trampoline<>(SB) TEXT libc_mlockall_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mlockall(SB) - GLOBL ·libc_mlockall_trampoline_addr(SB), RODATA, $8 DATA ·libc_mlockall_trampoline_addr(SB)/8, $libc_mlockall_trampoline<>(SB) TEXT libc_mprotect_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mprotect(SB) - GLOBL ·libc_mprotect_trampoline_addr(SB), RODATA, $8 DATA ·libc_mprotect_trampoline_addr(SB)/8, $libc_mprotect_trampoline<>(SB) TEXT libc_msync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_msync(SB) - GLOBL ·libc_msync_trampoline_addr(SB), RODATA, $8 DATA ·libc_msync_trampoline_addr(SB)/8, $libc_msync_trampoline<>(SB) TEXT libc_munlock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munlock(SB) - GLOBL ·libc_munlock_trampoline_addr(SB), RODATA, $8 DATA ·libc_munlock_trampoline_addr(SB)/8, $libc_munlock_trampoline<>(SB) TEXT libc_munlockall_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munlockall(SB) - GLOBL ·libc_munlockall_trampoline_addr(SB), RODATA, $8 DATA ·libc_munlockall_trampoline_addr(SB)/8, $libc_munlockall_trampoline<>(SB) TEXT libc_pipe2_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pipe2(SB) - GLOBL ·libc_pipe2_trampoline_addr(SB), RODATA, $8 DATA ·libc_pipe2_trampoline_addr(SB)/8, $libc_pipe2_trampoline<>(SB) TEXT libc_getdents_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getdents(SB) - GLOBL ·libc_getdents_trampoline_addr(SB), RODATA, $8 DATA ·libc_getdents_trampoline_addr(SB)/8, $libc_getdents_trampoline<>(SB) TEXT libc_getcwd_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getcwd(SB) - GLOBL ·libc_getcwd_trampoline_addr(SB), RODATA, $8 DATA ·libc_getcwd_trampoline_addr(SB)/8, $libc_getcwd_trampoline<>(SB) TEXT libc_ioctl_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ioctl(SB) - GLOBL ·libc_ioctl_trampoline_addr(SB), RODATA, $8 DATA ·libc_ioctl_trampoline_addr(SB)/8, $libc_ioctl_trampoline<>(SB) TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sysctl(SB) - GLOBL ·libc_sysctl_trampoline_addr(SB), RODATA, $8 DATA ·libc_sysctl_trampoline_addr(SB)/8, $libc_sysctl_trampoline<>(SB) TEXT libc_ppoll_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ppoll(SB) - GLOBL ·libc_ppoll_trampoline_addr(SB), RODATA, $8 DATA ·libc_ppoll_trampoline_addr(SB)/8, $libc_ppoll_trampoline<>(SB) TEXT libc_access_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_access(SB) - GLOBL ·libc_access_trampoline_addr(SB), RODATA, $8 DATA ·libc_access_trampoline_addr(SB)/8, $libc_access_trampoline<>(SB) TEXT libc_adjtime_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_adjtime(SB) - GLOBL ·libc_adjtime_trampoline_addr(SB), RODATA, $8 DATA ·libc_adjtime_trampoline_addr(SB)/8, $libc_adjtime_trampoline<>(SB) TEXT libc_chdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chdir(SB) - GLOBL ·libc_chdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_chdir_trampoline_addr(SB)/8, $libc_chdir_trampoline<>(SB) TEXT libc_chflags_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chflags(SB) - GLOBL ·libc_chflags_trampoline_addr(SB), RODATA, $8 DATA ·libc_chflags_trampoline_addr(SB)/8, $libc_chflags_trampoline<>(SB) TEXT libc_chmod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chmod(SB) - GLOBL ·libc_chmod_trampoline_addr(SB), RODATA, $8 DATA ·libc_chmod_trampoline_addr(SB)/8, $libc_chmod_trampoline<>(SB) TEXT libc_chown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chown(SB) - GLOBL ·libc_chown_trampoline_addr(SB), RODATA, $8 DATA ·libc_chown_trampoline_addr(SB)/8, $libc_chown_trampoline<>(SB) TEXT libc_chroot_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chroot(SB) - GLOBL ·libc_chroot_trampoline_addr(SB), RODATA, $8 DATA ·libc_chroot_trampoline_addr(SB)/8, $libc_chroot_trampoline<>(SB) +TEXT libc_clock_gettime_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_clock_gettime(SB) +GLOBL ·libc_clock_gettime_trampoline_addr(SB), RODATA, $8 +DATA ·libc_clock_gettime_trampoline_addr(SB)/8, $libc_clock_gettime_trampoline<>(SB) + TEXT libc_close_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_close(SB) - GLOBL ·libc_close_trampoline_addr(SB), RODATA, $8 DATA ·libc_close_trampoline_addr(SB)/8, $libc_close_trampoline<>(SB) TEXT libc_dup_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup(SB) - GLOBL ·libc_dup_trampoline_addr(SB), RODATA, $8 DATA ·libc_dup_trampoline_addr(SB)/8, $libc_dup_trampoline<>(SB) TEXT libc_dup2_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup2(SB) - GLOBL ·libc_dup2_trampoline_addr(SB), RODATA, $8 DATA ·libc_dup2_trampoline_addr(SB)/8, $libc_dup2_trampoline<>(SB) TEXT libc_dup3_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup3(SB) - GLOBL ·libc_dup3_trampoline_addr(SB), RODATA, $8 DATA ·libc_dup3_trampoline_addr(SB)/8, $libc_dup3_trampoline<>(SB) TEXT libc_exit_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_exit(SB) - GLOBL ·libc_exit_trampoline_addr(SB), RODATA, $8 DATA ·libc_exit_trampoline_addr(SB)/8, $libc_exit_trampoline<>(SB) TEXT libc_faccessat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_faccessat(SB) - GLOBL ·libc_faccessat_trampoline_addr(SB), RODATA, $8 DATA ·libc_faccessat_trampoline_addr(SB)/8, $libc_faccessat_trampoline<>(SB) TEXT libc_fchdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchdir(SB) - GLOBL ·libc_fchdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchdir_trampoline_addr(SB)/8, $libc_fchdir_trampoline<>(SB) TEXT libc_fchflags_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchflags(SB) - GLOBL ·libc_fchflags_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchflags_trampoline_addr(SB)/8, $libc_fchflags_trampoline<>(SB) TEXT libc_fchmod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchmod(SB) - GLOBL ·libc_fchmod_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchmod_trampoline_addr(SB)/8, $libc_fchmod_trampoline<>(SB) TEXT libc_fchmodat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchmodat(SB) - GLOBL ·libc_fchmodat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchmodat_trampoline_addr(SB)/8, $libc_fchmodat_trampoline<>(SB) TEXT libc_fchown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchown(SB) - GLOBL ·libc_fchown_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchown_trampoline_addr(SB)/8, $libc_fchown_trampoline<>(SB) TEXT libc_fchownat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchownat(SB) - GLOBL ·libc_fchownat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchownat_trampoline_addr(SB)/8, $libc_fchownat_trampoline<>(SB) TEXT libc_flock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_flock(SB) - GLOBL ·libc_flock_trampoline_addr(SB), RODATA, $8 DATA ·libc_flock_trampoline_addr(SB)/8, $libc_flock_trampoline<>(SB) TEXT libc_fpathconf_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fpathconf(SB) - GLOBL ·libc_fpathconf_trampoline_addr(SB), RODATA, $8 DATA ·libc_fpathconf_trampoline_addr(SB)/8, $libc_fpathconf_trampoline<>(SB) TEXT libc_fstat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstat(SB) - GLOBL ·libc_fstat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fstat_trampoline_addr(SB)/8, $libc_fstat_trampoline<>(SB) TEXT libc_fstatat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) - GLOBL ·libc_fstatat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fstatat_trampoline_addr(SB)/8, $libc_fstatat_trampoline<>(SB) TEXT libc_fstatfs_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstatfs(SB) - GLOBL ·libc_fstatfs_trampoline_addr(SB), RODATA, $8 DATA ·libc_fstatfs_trampoline_addr(SB)/8, $libc_fstatfs_trampoline<>(SB) TEXT libc_fsync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fsync(SB) - GLOBL ·libc_fsync_trampoline_addr(SB), RODATA, $8 DATA ·libc_fsync_trampoline_addr(SB)/8, $libc_fsync_trampoline<>(SB) TEXT libc_ftruncate_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ftruncate(SB) - GLOBL ·libc_ftruncate_trampoline_addr(SB), RODATA, $8 DATA ·libc_ftruncate_trampoline_addr(SB)/8, $libc_ftruncate_trampoline<>(SB) TEXT libc_getegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getegid(SB) - GLOBL ·libc_getegid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getegid_trampoline_addr(SB)/8, $libc_getegid_trampoline<>(SB) TEXT libc_geteuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_geteuid(SB) - GLOBL ·libc_geteuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_geteuid_trampoline_addr(SB)/8, $libc_geteuid_trampoline<>(SB) TEXT libc_getgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getgid(SB) - GLOBL ·libc_getgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getgid_trampoline_addr(SB)/8, $libc_getgid_trampoline<>(SB) TEXT libc_getpgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpgid(SB) - GLOBL ·libc_getpgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpgid_trampoline_addr(SB)/8, $libc_getpgid_trampoline<>(SB) TEXT libc_getpgrp_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpgrp(SB) - GLOBL ·libc_getpgrp_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpgrp_trampoline_addr(SB)/8, $libc_getpgrp_trampoline<>(SB) TEXT libc_getpid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpid(SB) - GLOBL ·libc_getpid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpid_trampoline_addr(SB)/8, $libc_getpid_trampoline<>(SB) TEXT libc_getppid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getppid(SB) - GLOBL ·libc_getppid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getppid_trampoline_addr(SB)/8, $libc_getppid_trampoline<>(SB) TEXT libc_getpriority_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpriority(SB) - GLOBL ·libc_getpriority_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpriority_trampoline_addr(SB)/8, $libc_getpriority_trampoline<>(SB) TEXT libc_getrlimit_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrlimit(SB) - GLOBL ·libc_getrlimit_trampoline_addr(SB), RODATA, $8 DATA ·libc_getrlimit_trampoline_addr(SB)/8, $libc_getrlimit_trampoline<>(SB) TEXT libc_getrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrtable(SB) - GLOBL ·libc_getrtable_trampoline_addr(SB), RODATA, $8 DATA ·libc_getrtable_trampoline_addr(SB)/8, $libc_getrtable_trampoline<>(SB) TEXT libc_getrusage_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrusage(SB) - GLOBL ·libc_getrusage_trampoline_addr(SB), RODATA, $8 DATA ·libc_getrusage_trampoline_addr(SB)/8, $libc_getrusage_trampoline<>(SB) TEXT libc_getsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsid(SB) - GLOBL ·libc_getsid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getsid_trampoline_addr(SB)/8, $libc_getsid_trampoline<>(SB) TEXT libc_gettimeofday_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_gettimeofday(SB) - GLOBL ·libc_gettimeofday_trampoline_addr(SB), RODATA, $8 DATA ·libc_gettimeofday_trampoline_addr(SB)/8, $libc_gettimeofday_trampoline<>(SB) TEXT libc_getuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getuid(SB) - GLOBL ·libc_getuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getuid_trampoline_addr(SB)/8, $libc_getuid_trampoline<>(SB) TEXT libc_issetugid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_issetugid(SB) - GLOBL ·libc_issetugid_trampoline_addr(SB), RODATA, $8 DATA ·libc_issetugid_trampoline_addr(SB)/8, $libc_issetugid_trampoline<>(SB) TEXT libc_kill_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kill(SB) - GLOBL ·libc_kill_trampoline_addr(SB), RODATA, $8 DATA ·libc_kill_trampoline_addr(SB)/8, $libc_kill_trampoline<>(SB) TEXT libc_kqueue_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kqueue(SB) - GLOBL ·libc_kqueue_trampoline_addr(SB), RODATA, $8 DATA ·libc_kqueue_trampoline_addr(SB)/8, $libc_kqueue_trampoline<>(SB) TEXT libc_lchown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lchown(SB) - GLOBL ·libc_lchown_trampoline_addr(SB), RODATA, $8 DATA ·libc_lchown_trampoline_addr(SB)/8, $libc_lchown_trampoline<>(SB) TEXT libc_link_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_link(SB) - GLOBL ·libc_link_trampoline_addr(SB), RODATA, $8 DATA ·libc_link_trampoline_addr(SB)/8, $libc_link_trampoline<>(SB) TEXT libc_linkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_linkat(SB) - GLOBL ·libc_linkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_linkat_trampoline_addr(SB)/8, $libc_linkat_trampoline<>(SB) TEXT libc_listen_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_listen(SB) - GLOBL ·libc_listen_trampoline_addr(SB), RODATA, $8 DATA ·libc_listen_trampoline_addr(SB)/8, $libc_listen_trampoline<>(SB) TEXT libc_lstat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lstat(SB) - GLOBL ·libc_lstat_trampoline_addr(SB), RODATA, $8 DATA ·libc_lstat_trampoline_addr(SB)/8, $libc_lstat_trampoline<>(SB) TEXT libc_mkdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkdir(SB) - GLOBL ·libc_mkdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkdir_trampoline_addr(SB)/8, $libc_mkdir_trampoline<>(SB) TEXT libc_mkdirat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkdirat(SB) - GLOBL ·libc_mkdirat_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkdirat_trampoline_addr(SB)/8, $libc_mkdirat_trampoline<>(SB) TEXT libc_mkfifo_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkfifo(SB) - GLOBL ·libc_mkfifo_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkfifo_trampoline_addr(SB)/8, $libc_mkfifo_trampoline<>(SB) TEXT libc_mkfifoat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkfifoat(SB) - GLOBL ·libc_mkfifoat_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkfifoat_trampoline_addr(SB)/8, $libc_mkfifoat_trampoline<>(SB) TEXT libc_mknod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mknod(SB) - GLOBL ·libc_mknod_trampoline_addr(SB), RODATA, $8 DATA ·libc_mknod_trampoline_addr(SB)/8, $libc_mknod_trampoline<>(SB) TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mknodat(SB) - GLOBL ·libc_mknodat_trampoline_addr(SB), RODATA, $8 DATA ·libc_mknodat_trampoline_addr(SB)/8, $libc_mknodat_trampoline<>(SB) TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_nanosleep(SB) - GLOBL ·libc_nanosleep_trampoline_addr(SB), RODATA, $8 DATA ·libc_nanosleep_trampoline_addr(SB)/8, $libc_nanosleep_trampoline<>(SB) TEXT libc_open_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_open(SB) - GLOBL ·libc_open_trampoline_addr(SB), RODATA, $8 DATA ·libc_open_trampoline_addr(SB)/8, $libc_open_trampoline<>(SB) TEXT libc_openat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_openat(SB) - GLOBL ·libc_openat_trampoline_addr(SB), RODATA, $8 DATA ·libc_openat_trampoline_addr(SB)/8, $libc_openat_trampoline<>(SB) TEXT libc_pathconf_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pathconf(SB) - GLOBL ·libc_pathconf_trampoline_addr(SB), RODATA, $8 DATA ·libc_pathconf_trampoline_addr(SB)/8, $libc_pathconf_trampoline<>(SB) TEXT libc_pread_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pread(SB) - GLOBL ·libc_pread_trampoline_addr(SB), RODATA, $8 DATA ·libc_pread_trampoline_addr(SB)/8, $libc_pread_trampoline<>(SB) TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pwrite(SB) - GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $8 DATA ·libc_pwrite_trampoline_addr(SB)/8, $libc_pwrite_trampoline<>(SB) TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_read(SB) - GLOBL ·libc_read_trampoline_addr(SB), RODATA, $8 DATA ·libc_read_trampoline_addr(SB)/8, $libc_read_trampoline<>(SB) TEXT libc_readlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_readlink(SB) - GLOBL ·libc_readlink_trampoline_addr(SB), RODATA, $8 DATA ·libc_readlink_trampoline_addr(SB)/8, $libc_readlink_trampoline<>(SB) TEXT libc_readlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_readlinkat(SB) - GLOBL ·libc_readlinkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_readlinkat_trampoline_addr(SB)/8, $libc_readlinkat_trampoline<>(SB) TEXT libc_rename_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_rename(SB) - GLOBL ·libc_rename_trampoline_addr(SB), RODATA, $8 DATA ·libc_rename_trampoline_addr(SB)/8, $libc_rename_trampoline<>(SB) TEXT libc_renameat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_renameat(SB) - GLOBL ·libc_renameat_trampoline_addr(SB), RODATA, $8 DATA ·libc_renameat_trampoline_addr(SB)/8, $libc_renameat_trampoline<>(SB) TEXT libc_revoke_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_revoke(SB) - GLOBL ·libc_revoke_trampoline_addr(SB), RODATA, $8 DATA ·libc_revoke_trampoline_addr(SB)/8, $libc_revoke_trampoline<>(SB) TEXT libc_rmdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_rmdir(SB) - GLOBL ·libc_rmdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_rmdir_trampoline_addr(SB)/8, $libc_rmdir_trampoline<>(SB) TEXT libc_lseek_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lseek(SB) - GLOBL ·libc_lseek_trampoline_addr(SB), RODATA, $8 DATA ·libc_lseek_trampoline_addr(SB)/8, $libc_lseek_trampoline<>(SB) TEXT libc_select_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_select(SB) - GLOBL ·libc_select_trampoline_addr(SB), RODATA, $8 DATA ·libc_select_trampoline_addr(SB)/8, $libc_select_trampoline<>(SB) TEXT libc_setegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setegid(SB) - GLOBL ·libc_setegid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setegid_trampoline_addr(SB)/8, $libc_setegid_trampoline<>(SB) TEXT libc_seteuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_seteuid(SB) - GLOBL ·libc_seteuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_seteuid_trampoline_addr(SB)/8, $libc_seteuid_trampoline<>(SB) TEXT libc_setgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setgid(SB) - GLOBL ·libc_setgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setgid_trampoline_addr(SB)/8, $libc_setgid_trampoline<>(SB) TEXT libc_setlogin_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setlogin(SB) - GLOBL ·libc_setlogin_trampoline_addr(SB), RODATA, $8 DATA ·libc_setlogin_trampoline_addr(SB)/8, $libc_setlogin_trampoline<>(SB) TEXT libc_setpgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setpgid(SB) - GLOBL ·libc_setpgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setpgid_trampoline_addr(SB)/8, $libc_setpgid_trampoline<>(SB) TEXT libc_setpriority_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setpriority(SB) - GLOBL ·libc_setpriority_trampoline_addr(SB), RODATA, $8 DATA ·libc_setpriority_trampoline_addr(SB)/8, $libc_setpriority_trampoline<>(SB) TEXT libc_setregid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setregid(SB) - GLOBL ·libc_setregid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setregid_trampoline_addr(SB)/8, $libc_setregid_trampoline<>(SB) TEXT libc_setreuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setreuid(SB) - GLOBL ·libc_setreuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setreuid_trampoline_addr(SB)/8, $libc_setreuid_trampoline<>(SB) TEXT libc_setresgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setresgid(SB) - GLOBL ·libc_setresgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresgid_trampoline_addr(SB)/8, $libc_setresgid_trampoline<>(SB) TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setresuid(SB) - GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresuid_trampoline_addr(SB)/8, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) - -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setrtable(SB) - GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $8 DATA ·libc_setrtable_trampoline_addr(SB)/8, $libc_setrtable_trampoline<>(SB) TEXT libc_setsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsid(SB) - GLOBL ·libc_setsid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setsid_trampoline_addr(SB)/8, $libc_setsid_trampoline<>(SB) TEXT libc_settimeofday_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_settimeofday(SB) - GLOBL ·libc_settimeofday_trampoline_addr(SB), RODATA, $8 DATA ·libc_settimeofday_trampoline_addr(SB)/8, $libc_settimeofday_trampoline<>(SB) TEXT libc_setuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setuid(SB) - GLOBL ·libc_setuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setuid_trampoline_addr(SB)/8, $libc_setuid_trampoline<>(SB) TEXT libc_stat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_stat(SB) - GLOBL ·libc_stat_trampoline_addr(SB), RODATA, $8 DATA ·libc_stat_trampoline_addr(SB)/8, $libc_stat_trampoline<>(SB) TEXT libc_statfs_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_statfs(SB) - GLOBL ·libc_statfs_trampoline_addr(SB), RODATA, $8 DATA ·libc_statfs_trampoline_addr(SB)/8, $libc_statfs_trampoline<>(SB) TEXT libc_symlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_symlink(SB) - GLOBL ·libc_symlink_trampoline_addr(SB), RODATA, $8 DATA ·libc_symlink_trampoline_addr(SB)/8, $libc_symlink_trampoline<>(SB) TEXT libc_symlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_symlinkat(SB) - GLOBL ·libc_symlinkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_symlinkat_trampoline_addr(SB)/8, $libc_symlinkat_trampoline<>(SB) TEXT libc_sync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sync(SB) - GLOBL ·libc_sync_trampoline_addr(SB), RODATA, $8 DATA ·libc_sync_trampoline_addr(SB)/8, $libc_sync_trampoline<>(SB) TEXT libc_truncate_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_truncate(SB) - GLOBL ·libc_truncate_trampoline_addr(SB), RODATA, $8 DATA ·libc_truncate_trampoline_addr(SB)/8, $libc_truncate_trampoline<>(SB) TEXT libc_umask_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_umask(SB) - GLOBL ·libc_umask_trampoline_addr(SB), RODATA, $8 DATA ·libc_umask_trampoline_addr(SB)/8, $libc_umask_trampoline<>(SB) TEXT libc_unlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unlink(SB) - GLOBL ·libc_unlink_trampoline_addr(SB), RODATA, $8 DATA ·libc_unlink_trampoline_addr(SB)/8, $libc_unlink_trampoline<>(SB) TEXT libc_unlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unlinkat(SB) - GLOBL ·libc_unlinkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_unlinkat_trampoline_addr(SB)/8, $libc_unlinkat_trampoline<>(SB) TEXT libc_unmount_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unmount(SB) - GLOBL ·libc_unmount_trampoline_addr(SB), RODATA, $8 DATA ·libc_unmount_trampoline_addr(SB)/8, $libc_unmount_trampoline<>(SB) TEXT libc_write_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_write(SB) - GLOBL ·libc_write_trampoline_addr(SB), RODATA, $8 DATA ·libc_write_trampoline_addr(SB)/8, $libc_write_trampoline<>(SB) TEXT libc_mmap_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mmap(SB) - GLOBL ·libc_mmap_trampoline_addr(SB), RODATA, $8 DATA ·libc_mmap_trampoline_addr(SB)/8, $libc_mmap_trampoline<>(SB) TEXT libc_munmap_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munmap(SB) - GLOBL ·libc_munmap_trampoline_addr(SB), RODATA, $8 DATA ·libc_munmap_trampoline_addr(SB)/8, $libc_munmap_trampoline<>(SB) TEXT libc_utimensat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_utimensat(SB) - GLOBL ·libc_utimensat_trampoline_addr(SB), RODATA, $8 DATA ·libc_utimensat_trampoline_addr(SB)/8, $libc_utimensat_trampoline<>(SB) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go index 016d959..7bc2e24 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.go @@ -1,4 +1,4 @@ -// go run mksyscall.go -openbsd -tags openbsd,mips64 syscall_bsd.go syscall_openbsd.go syscall_openbsd_mips64.go +// go run mksyscall.go -openbsd -libc -tags openbsd,mips64 syscall_bsd.go syscall_openbsd.go syscall_openbsd_mips64.go // Code generated by the command above; see README.md. DO NOT EDIT. //go:build openbsd && mips64 @@ -16,7 +16,7 @@ var _ syscall.Errno // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func getgroups(ngid int, gid *_Gid_t) (n int, err error) { - r0, _, e1 := RawSyscall(SYS_GETGROUPS, uintptr(ngid), uintptr(unsafe.Pointer(gid)), 0) + r0, _, e1 := syscall_rawSyscall(libc_getgroups_trampoline_addr, uintptr(ngid), uintptr(unsafe.Pointer(gid)), 0) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -24,20 +24,28 @@ func getgroups(ngid int, gid *_Gid_t) (n int, err error) { return } +var libc_getgroups_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getgroups getgroups "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func setgroups(ngid int, gid *_Gid_t) (err error) { - _, _, e1 := RawSyscall(SYS_SETGROUPS, uintptr(ngid), uintptr(unsafe.Pointer(gid)), 0) + _, _, e1 := syscall_rawSyscall(libc_setgroups_trampoline_addr, uintptr(ngid), uintptr(unsafe.Pointer(gid)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setgroups_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setgroups setgroups "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func wait4(pid int, wstatus *_C_int, options int, rusage *Rusage) (wpid int, err error) { - r0, _, e1 := Syscall6(SYS_WAIT4, uintptr(pid), uintptr(unsafe.Pointer(wstatus)), uintptr(options), uintptr(unsafe.Pointer(rusage)), 0, 0) + r0, _, e1 := syscall_syscall6(libc_wait4_trampoline_addr, uintptr(pid), uintptr(unsafe.Pointer(wstatus)), uintptr(options), uintptr(unsafe.Pointer(rusage)), 0, 0) wpid = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -45,10 +53,14 @@ func wait4(pid int, wstatus *_C_int, options int, rusage *Rusage) (wpid int, err return } +var libc_wait4_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_wait4 wait4 "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func accept(s int, rsa *RawSockaddrAny, addrlen *_Socklen) (fd int, err error) { - r0, _, e1 := Syscall(SYS_ACCEPT, uintptr(s), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) + r0, _, e1 := syscall_syscall(libc_accept_trampoline_addr, uintptr(s), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) fd = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -56,30 +68,42 @@ func accept(s int, rsa *RawSockaddrAny, addrlen *_Socklen) (fd int, err error) { return } +var libc_accept_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_accept accept "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func bind(s int, addr unsafe.Pointer, addrlen _Socklen) (err error) { - _, _, e1 := Syscall(SYS_BIND, uintptr(s), uintptr(addr), uintptr(addrlen)) + _, _, e1 := syscall_syscall(libc_bind_trampoline_addr, uintptr(s), uintptr(addr), uintptr(addrlen)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_bind_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_bind bind "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func connect(s int, addr unsafe.Pointer, addrlen _Socklen) (err error) { - _, _, e1 := Syscall(SYS_CONNECT, uintptr(s), uintptr(addr), uintptr(addrlen)) + _, _, e1 := syscall_syscall(libc_connect_trampoline_addr, uintptr(s), uintptr(addr), uintptr(addrlen)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_connect_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_connect connect "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func socket(domain int, typ int, proto int) (fd int, err error) { - r0, _, e1 := RawSyscall(SYS_SOCKET, uintptr(domain), uintptr(typ), uintptr(proto)) + r0, _, e1 := syscall_rawSyscall(libc_socket_trampoline_addr, uintptr(domain), uintptr(typ), uintptr(proto)) fd = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -87,66 +111,94 @@ func socket(domain int, typ int, proto int) (fd int, err error) { return } +var libc_socket_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_socket socket "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func getsockopt(s int, level int, name int, val unsafe.Pointer, vallen *_Socklen) (err error) { - _, _, e1 := Syscall6(SYS_GETSOCKOPT, uintptr(s), uintptr(level), uintptr(name), uintptr(val), uintptr(unsafe.Pointer(vallen)), 0) + _, _, e1 := syscall_syscall6(libc_getsockopt_trampoline_addr, uintptr(s), uintptr(level), uintptr(name), uintptr(val), uintptr(unsafe.Pointer(vallen)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_getsockopt_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getsockopt getsockopt "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func setsockopt(s int, level int, name int, val unsafe.Pointer, vallen uintptr) (err error) { - _, _, e1 := Syscall6(SYS_SETSOCKOPT, uintptr(s), uintptr(level), uintptr(name), uintptr(val), uintptr(vallen), 0) + _, _, e1 := syscall_syscall6(libc_setsockopt_trampoline_addr, uintptr(s), uintptr(level), uintptr(name), uintptr(val), uintptr(vallen), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setsockopt_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setsockopt setsockopt "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func getpeername(fd int, rsa *RawSockaddrAny, addrlen *_Socklen) (err error) { - _, _, e1 := RawSyscall(SYS_GETPEERNAME, uintptr(fd), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) + _, _, e1 := syscall_rawSyscall(libc_getpeername_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) if e1 != 0 { err = errnoErr(e1) } return } +var libc_getpeername_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getpeername getpeername "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func getsockname(fd int, rsa *RawSockaddrAny, addrlen *_Socklen) (err error) { - _, _, e1 := RawSyscall(SYS_GETSOCKNAME, uintptr(fd), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) + _, _, e1 := syscall_rawSyscall(libc_getsockname_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(rsa)), uintptr(unsafe.Pointer(addrlen))) if e1 != 0 { err = errnoErr(e1) } return } +var libc_getsockname_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getsockname getsockname "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Shutdown(s int, how int) (err error) { - _, _, e1 := Syscall(SYS_SHUTDOWN, uintptr(s), uintptr(how), 0) + _, _, e1 := syscall_syscall(libc_shutdown_trampoline_addr, uintptr(s), uintptr(how), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_shutdown_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_shutdown shutdown "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func socketpair(domain int, typ int, proto int, fd *[2]int32) (err error) { - _, _, e1 := RawSyscall6(SYS_SOCKETPAIR, uintptr(domain), uintptr(typ), uintptr(proto), uintptr(unsafe.Pointer(fd)), 0, 0) + _, _, e1 := syscall_rawSyscall6(libc_socketpair_trampoline_addr, uintptr(domain), uintptr(typ), uintptr(proto), uintptr(unsafe.Pointer(fd)), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_socketpair_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_socketpair socketpair "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func recvfrom(fd int, p []byte, flags int, from *RawSockaddrAny, fromlen *_Socklen) (n int, err error) { @@ -156,7 +208,7 @@ func recvfrom(fd int, p []byte, flags int, from *RawSockaddrAny, fromlen *_Sockl } else { _p0 = unsafe.Pointer(&_zero) } - r0, _, e1 := Syscall6(SYS_RECVFROM, uintptr(fd), uintptr(_p0), uintptr(len(p)), uintptr(flags), uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(fromlen))) + r0, _, e1 := syscall_syscall6(libc_recvfrom_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(p)), uintptr(flags), uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(fromlen))) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -164,6 +216,10 @@ func recvfrom(fd int, p []byte, flags int, from *RawSockaddrAny, fromlen *_Sockl return } +var libc_recvfrom_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_recvfrom recvfrom "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func sendto(s int, buf []byte, flags int, to unsafe.Pointer, addrlen _Socklen) (err error) { @@ -173,17 +229,21 @@ func sendto(s int, buf []byte, flags int, to unsafe.Pointer, addrlen _Socklen) ( } else { _p0 = unsafe.Pointer(&_zero) } - _, _, e1 := Syscall6(SYS_SENDTO, uintptr(s), uintptr(_p0), uintptr(len(buf)), uintptr(flags), uintptr(to), uintptr(addrlen)) + _, _, e1 := syscall_syscall6(libc_sendto_trampoline_addr, uintptr(s), uintptr(_p0), uintptr(len(buf)), uintptr(flags), uintptr(to), uintptr(addrlen)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_sendto_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_sendto sendto "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func recvmsg(s int, msg *Msghdr, flags int) (n int, err error) { - r0, _, e1 := Syscall(SYS_RECVMSG, uintptr(s), uintptr(unsafe.Pointer(msg)), uintptr(flags)) + r0, _, e1 := syscall_syscall(libc_recvmsg_trampoline_addr, uintptr(s), uintptr(unsafe.Pointer(msg)), uintptr(flags)) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -191,10 +251,14 @@ func recvmsg(s int, msg *Msghdr, flags int) (n int, err error) { return } +var libc_recvmsg_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_recvmsg recvmsg "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func sendmsg(s int, msg *Msghdr, flags int) (n int, err error) { - r0, _, e1 := Syscall(SYS_SENDMSG, uintptr(s), uintptr(unsafe.Pointer(msg)), uintptr(flags)) + r0, _, e1 := syscall_syscall(libc_sendmsg_trampoline_addr, uintptr(s), uintptr(unsafe.Pointer(msg)), uintptr(flags)) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -202,10 +266,14 @@ func sendmsg(s int, msg *Msghdr, flags int) (n int, err error) { return } +var libc_sendmsg_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_sendmsg sendmsg "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func kevent(kq int, change unsafe.Pointer, nchange int, event unsafe.Pointer, nevent int, timeout *Timespec) (n int, err error) { - r0, _, e1 := Syscall6(SYS_KEVENT, uintptr(kq), uintptr(change), uintptr(nchange), uintptr(event), uintptr(nevent), uintptr(unsafe.Pointer(timeout))) + r0, _, e1 := syscall_syscall6(libc_kevent_trampoline_addr, uintptr(kq), uintptr(change), uintptr(nchange), uintptr(event), uintptr(nevent), uintptr(unsafe.Pointer(timeout))) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -213,6 +281,10 @@ func kevent(kq int, change unsafe.Pointer, nchange int, event unsafe.Pointer, ne return } +var libc_kevent_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_kevent kevent "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func utimes(path string, timeval *[2]Timeval) (err error) { @@ -221,27 +293,35 @@ func utimes(path string, timeval *[2]Timeval) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_UTIMES, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(timeval)), 0) + _, _, e1 := syscall_syscall(libc_utimes_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(timeval)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_utimes_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_utimes utimes "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func futimes(fd int, timeval *[2]Timeval) (err error) { - _, _, e1 := Syscall(SYS_FUTIMES, uintptr(fd), uintptr(unsafe.Pointer(timeval)), 0) + _, _, e1 := syscall_syscall(libc_futimes_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(timeval)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_futimes_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_futimes futimes "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func poll(fds *PollFd, nfds int, timeout int) (n int, err error) { - r0, _, e1 := Syscall(SYS_POLL, uintptr(unsafe.Pointer(fds)), uintptr(nfds), uintptr(timeout)) + r0, _, e1 := syscall_syscall(libc_poll_trampoline_addr, uintptr(unsafe.Pointer(fds)), uintptr(nfds), uintptr(timeout)) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -249,6 +329,10 @@ func poll(fds *PollFd, nfds int, timeout int) (n int, err error) { return } +var libc_poll_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_poll poll "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Madvise(b []byte, behav int) (err error) { @@ -258,13 +342,17 @@ func Madvise(b []byte, behav int) (err error) { } else { _p0 = unsafe.Pointer(&_zero) } - _, _, e1 := Syscall(SYS_MADVISE, uintptr(_p0), uintptr(len(b)), uintptr(behav)) + _, _, e1 := syscall_syscall(libc_madvise_trampoline_addr, uintptr(_p0), uintptr(len(b)), uintptr(behav)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_madvise_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_madvise madvise "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Mlock(b []byte) (err error) { @@ -274,23 +362,31 @@ func Mlock(b []byte) (err error) { } else { _p0 = unsafe.Pointer(&_zero) } - _, _, e1 := Syscall(SYS_MLOCK, uintptr(_p0), uintptr(len(b)), 0) + _, _, e1 := syscall_syscall(libc_mlock_trampoline_addr, uintptr(_p0), uintptr(len(b)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_mlock_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_mlock mlock "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Mlockall(flags int) (err error) { - _, _, e1 := Syscall(SYS_MLOCKALL, uintptr(flags), 0, 0) + _, _, e1 := syscall_syscall(libc_mlockall_trampoline_addr, uintptr(flags), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_mlockall_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_mlockall mlockall "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Mprotect(b []byte, prot int) (err error) { @@ -300,13 +396,17 @@ func Mprotect(b []byte, prot int) (err error) { } else { _p0 = unsafe.Pointer(&_zero) } - _, _, e1 := Syscall(SYS_MPROTECT, uintptr(_p0), uintptr(len(b)), uintptr(prot)) + _, _, e1 := syscall_syscall(libc_mprotect_trampoline_addr, uintptr(_p0), uintptr(len(b)), uintptr(prot)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_mprotect_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_mprotect mprotect "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Msync(b []byte, flags int) (err error) { @@ -316,13 +416,17 @@ func Msync(b []byte, flags int) (err error) { } else { _p0 = unsafe.Pointer(&_zero) } - _, _, e1 := Syscall(SYS_MSYNC, uintptr(_p0), uintptr(len(b)), uintptr(flags)) + _, _, e1 := syscall_syscall(libc_msync_trampoline_addr, uintptr(_p0), uintptr(len(b)), uintptr(flags)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_msync_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_msync msync "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Munlock(b []byte) (err error) { @@ -332,33 +436,45 @@ func Munlock(b []byte) (err error) { } else { _p0 = unsafe.Pointer(&_zero) } - _, _, e1 := Syscall(SYS_MUNLOCK, uintptr(_p0), uintptr(len(b)), 0) + _, _, e1 := syscall_syscall(libc_munlock_trampoline_addr, uintptr(_p0), uintptr(len(b)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_munlock_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_munlock munlock "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Munlockall() (err error) { - _, _, e1 := Syscall(SYS_MUNLOCKALL, 0, 0, 0) + _, _, e1 := syscall_syscall(libc_munlockall_trampoline_addr, 0, 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_munlockall_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_munlockall munlockall "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func pipe2(p *[2]_C_int, flags int) (err error) { - _, _, e1 := RawSyscall(SYS_PIPE2, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) + _, _, e1 := syscall_rawSyscall(libc_pipe2_trampoline_addr, uintptr(unsafe.Pointer(p)), uintptr(flags), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_pipe2_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_pipe2 pipe2 "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getdents(fd int, buf []byte) (n int, err error) { @@ -368,7 +484,7 @@ func Getdents(fd int, buf []byte) (n int, err error) { } else { _p0 = unsafe.Pointer(&_zero) } - r0, _, e1 := Syscall(SYS_GETDENTS, uintptr(fd), uintptr(_p0), uintptr(len(buf))) + r0, _, e1 := syscall_syscall(libc_getdents_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(buf))) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -376,6 +492,10 @@ func Getdents(fd int, buf []byte) (n int, err error) { return } +var libc_getdents_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getdents getdents "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getcwd(buf []byte) (n int, err error) { @@ -385,7 +505,7 @@ func Getcwd(buf []byte) (n int, err error) { } else { _p0 = unsafe.Pointer(&_zero) } - r0, _, e1 := Syscall(SYS___GETCWD, uintptr(_p0), uintptr(len(buf)), 0) + r0, _, e1 := syscall_syscall(libc_getcwd_trampoline_addr, uintptr(_p0), uintptr(len(buf)), 0) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -393,16 +513,32 @@ func Getcwd(buf []byte) (n int, err error) { return } +var libc_getcwd_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getcwd getcwd "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func ioctl(fd int, req uint, arg uintptr) (err error) { - _, _, e1 := Syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + _, _, e1 := syscall_syscall(libc_ioctl_trampoline_addr, uintptr(fd), uintptr(req), uintptr(arg)) if e1 != 0 { err = errnoErr(e1) } return } +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := syscall_syscall(libc_ioctl_trampoline_addr, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_ioctl_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_ioctl ioctl "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) (err error) { @@ -412,17 +548,21 @@ func sysctl(mib []_C_int, old *byte, oldlen *uintptr, new *byte, newlen uintptr) } else { _p0 = unsafe.Pointer(&_zero) } - _, _, e1 := Syscall6(SYS___SYSCTL, uintptr(_p0), uintptr(len(mib)), uintptr(unsafe.Pointer(old)), uintptr(unsafe.Pointer(oldlen)), uintptr(unsafe.Pointer(new)), uintptr(newlen)) + _, _, e1 := syscall_syscall6(libc_sysctl_trampoline_addr, uintptr(_p0), uintptr(len(mib)), uintptr(unsafe.Pointer(old)), uintptr(unsafe.Pointer(oldlen)), uintptr(unsafe.Pointer(new)), uintptr(newlen)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_sysctl_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_sysctl sysctl "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func ppoll(fds *PollFd, nfds int, timeout *Timespec, sigmask *Sigset_t) (n int, err error) { - r0, _, e1 := Syscall6(SYS_PPOLL, uintptr(unsafe.Pointer(fds)), uintptr(nfds), uintptr(unsafe.Pointer(timeout)), uintptr(unsafe.Pointer(sigmask)), 0, 0) + r0, _, e1 := syscall_syscall6(libc_ppoll_trampoline_addr, uintptr(unsafe.Pointer(fds)), uintptr(nfds), uintptr(unsafe.Pointer(timeout)), uintptr(unsafe.Pointer(sigmask)), 0, 0) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -430,6 +570,10 @@ func ppoll(fds *PollFd, nfds int, timeout *Timespec, sigmask *Sigset_t) (n int, return } +var libc_ppoll_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_ppoll ppoll "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Access(path string, mode uint32) (err error) { @@ -438,23 +582,31 @@ func Access(path string, mode uint32) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_ACCESS, uintptr(unsafe.Pointer(_p0)), uintptr(mode), 0) + _, _, e1 := syscall_syscall(libc_access_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(mode), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_access_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_access access "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Adjtime(delta *Timeval, olddelta *Timeval) (err error) { - _, _, e1 := Syscall(SYS_ADJTIME, uintptr(unsafe.Pointer(delta)), uintptr(unsafe.Pointer(olddelta)), 0) + _, _, e1 := syscall_syscall(libc_adjtime_trampoline_addr, uintptr(unsafe.Pointer(delta)), uintptr(unsafe.Pointer(olddelta)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_adjtime_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_adjtime adjtime "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Chdir(path string) (err error) { @@ -463,13 +615,17 @@ func Chdir(path string) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_CHDIR, uintptr(unsafe.Pointer(_p0)), 0, 0) + _, _, e1 := syscall_syscall(libc_chdir_trampoline_addr, uintptr(unsafe.Pointer(_p0)), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_chdir_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_chdir chdir "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Chflags(path string, flags int) (err error) { @@ -478,13 +634,17 @@ func Chflags(path string, flags int) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_CHFLAGS, uintptr(unsafe.Pointer(_p0)), uintptr(flags), 0) + _, _, e1 := syscall_syscall(libc_chflags_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(flags), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_chflags_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_chflags chflags "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Chmod(path string, mode uint32) (err error) { @@ -493,13 +653,17 @@ func Chmod(path string, mode uint32) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_CHMOD, uintptr(unsafe.Pointer(_p0)), uintptr(mode), 0) + _, _, e1 := syscall_syscall(libc_chmod_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(mode), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_chmod_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_chmod chmod "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Chown(path string, uid int, gid int) (err error) { @@ -508,13 +672,17 @@ func Chown(path string, uid int, gid int) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_CHOWN, uintptr(unsafe.Pointer(_p0)), uintptr(uid), uintptr(gid)) + _, _, e1 := syscall_syscall(libc_chown_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(uid), uintptr(gid)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_chown_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_chown chown "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Chroot(path string) (err error) { @@ -523,27 +691,49 @@ func Chroot(path string) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_CHROOT, uintptr(unsafe.Pointer(_p0)), 0, 0) + _, _, e1 := syscall_syscall(libc_chroot_trampoline_addr, uintptr(unsafe.Pointer(_p0)), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_chroot_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_chroot chroot "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := syscall_syscall(libc_clock_gettime_trampoline_addr, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_clock_gettime_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Close(fd int) (err error) { - _, _, e1 := Syscall(SYS_CLOSE, uintptr(fd), 0, 0) + _, _, e1 := syscall_syscall(libc_close_trampoline_addr, uintptr(fd), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_close_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_close close "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Dup(fd int) (nfd int, err error) { - r0, _, e1 := Syscall(SYS_DUP, uintptr(fd), 0, 0) + r0, _, e1 := syscall_syscall(libc_dup_trampoline_addr, uintptr(fd), 0, 0) nfd = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -551,33 +741,49 @@ func Dup(fd int) (nfd int, err error) { return } +var libc_dup_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_dup dup "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Dup2(from int, to int) (err error) { - _, _, e1 := Syscall(SYS_DUP2, uintptr(from), uintptr(to), 0) + _, _, e1 := syscall_syscall(libc_dup2_trampoline_addr, uintptr(from), uintptr(to), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_dup2_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_dup2 dup2 "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Dup3(from int, to int, flags int) (err error) { - _, _, e1 := Syscall(SYS_DUP3, uintptr(from), uintptr(to), uintptr(flags)) + _, _, e1 := syscall_syscall(libc_dup3_trampoline_addr, uintptr(from), uintptr(to), uintptr(flags)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_dup3_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_dup3 dup3 "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Exit(code int) { - Syscall(SYS_EXIT, uintptr(code), 0, 0) + syscall_syscall(libc_exit_trampoline_addr, uintptr(code), 0, 0) return } +var libc_exit_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_exit exit "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Faccessat(dirfd int, path string, mode uint32, flags int) (err error) { @@ -586,43 +792,59 @@ func Faccessat(dirfd int, path string, mode uint32, flags int) (err error) { if err != nil { return } - _, _, e1 := Syscall6(SYS_FACCESSAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0) + _, _, e1 := syscall_syscall6(libc_faccessat_trampoline_addr, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_faccessat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_faccessat faccessat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Fchdir(fd int) (err error) { - _, _, e1 := Syscall(SYS_FCHDIR, uintptr(fd), 0, 0) + _, _, e1 := syscall_syscall(libc_fchdir_trampoline_addr, uintptr(fd), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_fchdir_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_fchdir fchdir "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Fchflags(fd int, flags int) (err error) { - _, _, e1 := Syscall(SYS_FCHFLAGS, uintptr(fd), uintptr(flags), 0) + _, _, e1 := syscall_syscall(libc_fchflags_trampoline_addr, uintptr(fd), uintptr(flags), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_fchflags_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_fchflags fchflags "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Fchmod(fd int, mode uint32) (err error) { - _, _, e1 := Syscall(SYS_FCHMOD, uintptr(fd), uintptr(mode), 0) + _, _, e1 := syscall_syscall(libc_fchmod_trampoline_addr, uintptr(fd), uintptr(mode), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_fchmod_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_fchmod fchmod "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Fchmodat(dirfd int, path string, mode uint32, flags int) (err error) { @@ -631,23 +853,31 @@ func Fchmodat(dirfd int, path string, mode uint32, flags int) (err error) { if err != nil { return } - _, _, e1 := Syscall6(SYS_FCHMODAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0) + _, _, e1 := syscall_syscall6(libc_fchmodat_trampoline_addr, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(flags), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_fchmodat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_fchmodat fchmodat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Fchown(fd int, uid int, gid int) (err error) { - _, _, e1 := Syscall(SYS_FCHOWN, uintptr(fd), uintptr(uid), uintptr(gid)) + _, _, e1 := syscall_syscall(libc_fchown_trampoline_addr, uintptr(fd), uintptr(uid), uintptr(gid)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_fchown_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_fchown fchown "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Fchownat(dirfd int, path string, uid int, gid int, flags int) (err error) { @@ -656,27 +886,35 @@ func Fchownat(dirfd int, path string, uid int, gid int, flags int) (err error) { if err != nil { return } - _, _, e1 := Syscall6(SYS_FCHOWNAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(uid), uintptr(gid), uintptr(flags), 0) + _, _, e1 := syscall_syscall6(libc_fchownat_trampoline_addr, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(uid), uintptr(gid), uintptr(flags), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_fchownat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_fchownat fchownat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Flock(fd int, how int) (err error) { - _, _, e1 := Syscall(SYS_FLOCK, uintptr(fd), uintptr(how), 0) + _, _, e1 := syscall_syscall(libc_flock_trampoline_addr, uintptr(fd), uintptr(how), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_flock_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_flock flock "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Fpathconf(fd int, name int) (val int, err error) { - r0, _, e1 := Syscall(SYS_FPATHCONF, uintptr(fd), uintptr(name), 0) + r0, _, e1 := syscall_syscall(libc_fpathconf_trampoline_addr, uintptr(fd), uintptr(name), 0) val = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -684,16 +922,24 @@ func Fpathconf(fd int, name int) (val int, err error) { return } +var libc_fpathconf_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_fpathconf fpathconf "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Fstat(fd int, stat *Stat_t) (err error) { - _, _, e1 := Syscall(SYS_FSTAT, uintptr(fd), uintptr(unsafe.Pointer(stat)), 0) + _, _, e1 := syscall_syscall(libc_fstat_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(stat)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_fstat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_fstat fstat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { @@ -702,71 +948,99 @@ func Fstatat(fd int, path string, stat *Stat_t, flags int) (err error) { if err != nil { return } - _, _, e1 := Syscall6(SYS_FSTATAT, uintptr(fd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) + _, _, e1 := syscall_syscall6(libc_fstatat_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), uintptr(flags), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_fstatat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_fstatat fstatat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Fstatfs(fd int, stat *Statfs_t) (err error) { - _, _, e1 := Syscall(SYS_FSTATFS, uintptr(fd), uintptr(unsafe.Pointer(stat)), 0) + _, _, e1 := syscall_syscall(libc_fstatfs_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(stat)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_fstatfs_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_fstatfs fstatfs "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Fsync(fd int) (err error) { - _, _, e1 := Syscall(SYS_FSYNC, uintptr(fd), 0, 0) + _, _, e1 := syscall_syscall(libc_fsync_trampoline_addr, uintptr(fd), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_fsync_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_fsync fsync "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Ftruncate(fd int, length int64) (err error) { - _, _, e1 := Syscall(SYS_FTRUNCATE, uintptr(fd), 0, uintptr(length)) + _, _, e1 := syscall_syscall(libc_ftruncate_trampoline_addr, uintptr(fd), uintptr(length), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_ftruncate_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_ftruncate ftruncate "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getegid() (egid int) { - r0, _, _ := RawSyscall(SYS_GETEGID, 0, 0, 0) + r0, _, _ := syscall_rawSyscall(libc_getegid_trampoline_addr, 0, 0, 0) egid = int(r0) return } +var libc_getegid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getegid getegid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Geteuid() (uid int) { - r0, _, _ := RawSyscall(SYS_GETEUID, 0, 0, 0) + r0, _, _ := syscall_rawSyscall(libc_geteuid_trampoline_addr, 0, 0, 0) uid = int(r0) return } +var libc_geteuid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_geteuid geteuid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getgid() (gid int) { - r0, _, _ := RawSyscall(SYS_GETGID, 0, 0, 0) + r0, _, _ := syscall_rawSyscall(libc_getgid_trampoline_addr, 0, 0, 0) gid = int(r0) return } +var libc_getgid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getgid getgid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getpgid(pid int) (pgid int, err error) { - r0, _, e1 := RawSyscall(SYS_GETPGID, uintptr(pid), 0, 0) + r0, _, e1 := syscall_rawSyscall(libc_getpgid_trampoline_addr, uintptr(pid), 0, 0) pgid = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -774,34 +1048,50 @@ func Getpgid(pid int) (pgid int, err error) { return } +var libc_getpgid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getpgid getpgid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getpgrp() (pgrp int) { - r0, _, _ := RawSyscall(SYS_GETPGRP, 0, 0, 0) + r0, _, _ := syscall_rawSyscall(libc_getpgrp_trampoline_addr, 0, 0, 0) pgrp = int(r0) return } +var libc_getpgrp_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getpgrp getpgrp "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getpid() (pid int) { - r0, _, _ := RawSyscall(SYS_GETPID, 0, 0, 0) + r0, _, _ := syscall_rawSyscall(libc_getpid_trampoline_addr, 0, 0, 0) pid = int(r0) return } +var libc_getpid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getpid getpid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getppid() (ppid int) { - r0, _, _ := RawSyscall(SYS_GETPPID, 0, 0, 0) + r0, _, _ := syscall_rawSyscall(libc_getppid_trampoline_addr, 0, 0, 0) ppid = int(r0) return } +var libc_getppid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getppid getppid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getpriority(which int, who int) (prio int, err error) { - r0, _, e1 := Syscall(SYS_GETPRIORITY, uintptr(which), uintptr(who), 0) + r0, _, e1 := syscall_syscall(libc_getpriority_trampoline_addr, uintptr(which), uintptr(who), 0) prio = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -809,20 +1099,28 @@ func Getpriority(which int, who int) (prio int, err error) { return } +var libc_getpriority_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getpriority getpriority "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_GETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) + _, _, e1 := syscall_rawSyscall(libc_getrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_getrlimit_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getrlimit getrlimit "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getrtable() (rtable int, err error) { - r0, _, e1 := RawSyscall(SYS_GETRTABLE, 0, 0, 0) + r0, _, e1 := syscall_rawSyscall(libc_getrtable_trampoline_addr, 0, 0, 0) rtable = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -830,20 +1128,28 @@ func Getrtable() (rtable int, err error) { return } +var libc_getrtable_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getrtable getrtable "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getrusage(who int, rusage *Rusage) (err error) { - _, _, e1 := RawSyscall(SYS_GETRUSAGE, uintptr(who), uintptr(unsafe.Pointer(rusage)), 0) + _, _, e1 := syscall_rawSyscall(libc_getrusage_trampoline_addr, uintptr(who), uintptr(unsafe.Pointer(rusage)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_getrusage_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getrusage getrusage "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getsid(pid int) (sid int, err error) { - r0, _, e1 := RawSyscall(SYS_GETSID, uintptr(pid), 0, 0) + r0, _, e1 := syscall_rawSyscall(libc_getsid_trampoline_addr, uintptr(pid), 0, 0) sid = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -851,46 +1157,66 @@ func Getsid(pid int) (sid int, err error) { return } +var libc_getsid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getsid getsid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Gettimeofday(tv *Timeval) (err error) { - _, _, e1 := RawSyscall(SYS_GETTIMEOFDAY, uintptr(unsafe.Pointer(tv)), 0, 0) + _, _, e1 := syscall_rawSyscall(libc_gettimeofday_trampoline_addr, uintptr(unsafe.Pointer(tv)), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_gettimeofday_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_gettimeofday gettimeofday "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Getuid() (uid int) { - r0, _, _ := RawSyscall(SYS_GETUID, 0, 0, 0) + r0, _, _ := syscall_rawSyscall(libc_getuid_trampoline_addr, 0, 0, 0) uid = int(r0) return } +var libc_getuid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_getuid getuid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Issetugid() (tainted bool) { - r0, _, _ := Syscall(SYS_ISSETUGID, 0, 0, 0) + r0, _, _ := syscall_syscall(libc_issetugid_trampoline_addr, 0, 0, 0) tainted = bool(r0 != 0) return } +var libc_issetugid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_issetugid issetugid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Kill(pid int, signum syscall.Signal) (err error) { - _, _, e1 := Syscall(SYS_KILL, uintptr(pid), uintptr(signum), 0) + _, _, e1 := syscall_syscall(libc_kill_trampoline_addr, uintptr(pid), uintptr(signum), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_kill_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_kill kill "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Kqueue() (fd int, err error) { - r0, _, e1 := Syscall(SYS_KQUEUE, 0, 0, 0) + r0, _, e1 := syscall_syscall(libc_kqueue_trampoline_addr, 0, 0, 0) fd = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -898,6 +1224,10 @@ func Kqueue() (fd int, err error) { return } +var libc_kqueue_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_kqueue kqueue "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Lchown(path string, uid int, gid int) (err error) { @@ -906,13 +1236,17 @@ func Lchown(path string, uid int, gid int) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_LCHOWN, uintptr(unsafe.Pointer(_p0)), uintptr(uid), uintptr(gid)) + _, _, e1 := syscall_syscall(libc_lchown_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(uid), uintptr(gid)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_lchown_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_lchown lchown "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Link(path string, link string) (err error) { @@ -926,13 +1260,17 @@ func Link(path string, link string) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_LINK, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), 0) + _, _, e1 := syscall_syscall(libc_link_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_link_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_link link "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Linkat(pathfd int, path string, linkfd int, link string, flags int) (err error) { @@ -946,23 +1284,31 @@ func Linkat(pathfd int, path string, linkfd int, link string, flags int) (err er if err != nil { return } - _, _, e1 := Syscall6(SYS_LINKAT, uintptr(pathfd), uintptr(unsafe.Pointer(_p0)), uintptr(linkfd), uintptr(unsafe.Pointer(_p1)), uintptr(flags), 0) + _, _, e1 := syscall_syscall6(libc_linkat_trampoline_addr, uintptr(pathfd), uintptr(unsafe.Pointer(_p0)), uintptr(linkfd), uintptr(unsafe.Pointer(_p1)), uintptr(flags), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_linkat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_linkat linkat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Listen(s int, backlog int) (err error) { - _, _, e1 := Syscall(SYS_LISTEN, uintptr(s), uintptr(backlog), 0) + _, _, e1 := syscall_syscall(libc_listen_trampoline_addr, uintptr(s), uintptr(backlog), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_listen_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_listen listen "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Lstat(path string, stat *Stat_t) (err error) { @@ -971,13 +1317,17 @@ func Lstat(path string, stat *Stat_t) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_LSTAT, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), 0) + _, _, e1 := syscall_syscall(libc_lstat_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_lstat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_lstat lstat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Mkdir(path string, mode uint32) (err error) { @@ -986,13 +1336,17 @@ func Mkdir(path string, mode uint32) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_MKDIR, uintptr(unsafe.Pointer(_p0)), uintptr(mode), 0) + _, _, e1 := syscall_syscall(libc_mkdir_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(mode), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_mkdir_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_mkdir mkdir "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Mkdirat(dirfd int, path string, mode uint32) (err error) { @@ -1001,13 +1355,17 @@ func Mkdirat(dirfd int, path string, mode uint32) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_MKDIRAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode)) + _, _, e1 := syscall_syscall(libc_mkdirat_trampoline_addr, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_mkdirat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_mkdirat mkdirat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Mkfifo(path string, mode uint32) (err error) { @@ -1016,13 +1374,17 @@ func Mkfifo(path string, mode uint32) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_MKFIFO, uintptr(unsafe.Pointer(_p0)), uintptr(mode), 0) + _, _, e1 := syscall_syscall(libc_mkfifo_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(mode), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_mkfifo_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_mkfifo mkfifo "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Mkfifoat(dirfd int, path string, mode uint32) (err error) { @@ -1031,13 +1393,17 @@ func Mkfifoat(dirfd int, path string, mode uint32) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_MKFIFOAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode)) + _, _, e1 := syscall_syscall(libc_mkfifoat_trampoline_addr, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_mkfifoat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_mkfifoat mkfifoat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Mknod(path string, mode uint32, dev int) (err error) { @@ -1046,13 +1412,17 @@ func Mknod(path string, mode uint32, dev int) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_MKNOD, uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(dev)) + _, _, e1 := syscall_syscall(libc_mknod_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(dev)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_mknod_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_mknod mknod "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Mknodat(dirfd int, path string, mode uint32, dev int) (err error) { @@ -1061,23 +1431,31 @@ func Mknodat(dirfd int, path string, mode uint32, dev int) (err error) { if err != nil { return } - _, _, e1 := Syscall6(SYS_MKNODAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(dev), 0, 0) + _, _, e1 := syscall_syscall6(libc_mknodat_trampoline_addr, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(dev), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_mknodat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_mknodat mknodat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Nanosleep(time *Timespec, leftover *Timespec) (err error) { - _, _, e1 := Syscall(SYS_NANOSLEEP, uintptr(unsafe.Pointer(time)), uintptr(unsafe.Pointer(leftover)), 0) + _, _, e1 := syscall_syscall(libc_nanosleep_trampoline_addr, uintptr(unsafe.Pointer(time)), uintptr(unsafe.Pointer(leftover)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_nanosleep_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_nanosleep nanosleep "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Open(path string, mode int, perm uint32) (fd int, err error) { @@ -1086,7 +1464,7 @@ func Open(path string, mode int, perm uint32) (fd int, err error) { if err != nil { return } - r0, _, e1 := Syscall(SYS_OPEN, uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(perm)) + r0, _, e1 := syscall_syscall(libc_open_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(perm)) fd = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1094,6 +1472,10 @@ func Open(path string, mode int, perm uint32) (fd int, err error) { return } +var libc_open_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_open open "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Openat(dirfd int, path string, mode int, perm uint32) (fd int, err error) { @@ -1102,7 +1484,7 @@ func Openat(dirfd int, path string, mode int, perm uint32) (fd int, err error) { if err != nil { return } - r0, _, e1 := Syscall6(SYS_OPENAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(perm), 0, 0) + r0, _, e1 := syscall_syscall6(libc_openat_trampoline_addr, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(mode), uintptr(perm), 0, 0) fd = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1110,6 +1492,10 @@ func Openat(dirfd int, path string, mode int, perm uint32) (fd int, err error) { return } +var libc_openat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_openat openat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Pathconf(path string, name int) (val int, err error) { @@ -1118,7 +1504,7 @@ func Pathconf(path string, name int) (val int, err error) { if err != nil { return } - r0, _, e1 := Syscall(SYS_PATHCONF, uintptr(unsafe.Pointer(_p0)), uintptr(name), 0) + r0, _, e1 := syscall_syscall(libc_pathconf_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(name), 0) val = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1126,6 +1512,10 @@ func Pathconf(path string, name int) (val int, err error) { return } +var libc_pathconf_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_pathconf pathconf "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func pread(fd int, p []byte, offset int64) (n int, err error) { @@ -1135,7 +1525,7 @@ func pread(fd int, p []byte, offset int64) (n int, err error) { } else { _p0 = unsafe.Pointer(&_zero) } - r0, _, e1 := Syscall6(SYS_PREAD, uintptr(fd), uintptr(_p0), uintptr(len(p)), 0, uintptr(offset), 0) + r0, _, e1 := syscall_syscall6(libc_pread_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(p)), uintptr(offset), 0, 0) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1143,6 +1533,10 @@ func pread(fd int, p []byte, offset int64) (n int, err error) { return } +var libc_pread_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_pread pread "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func pwrite(fd int, p []byte, offset int64) (n int, err error) { @@ -1152,7 +1546,7 @@ func pwrite(fd int, p []byte, offset int64) (n int, err error) { } else { _p0 = unsafe.Pointer(&_zero) } - r0, _, e1 := Syscall6(SYS_PWRITE, uintptr(fd), uintptr(_p0), uintptr(len(p)), 0, uintptr(offset), 0) + r0, _, e1 := syscall_syscall6(libc_pwrite_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(p)), uintptr(offset), 0, 0) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1160,6 +1554,10 @@ func pwrite(fd int, p []byte, offset int64) (n int, err error) { return } +var libc_pwrite_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_pwrite pwrite "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func read(fd int, p []byte) (n int, err error) { @@ -1169,7 +1567,7 @@ func read(fd int, p []byte) (n int, err error) { } else { _p0 = unsafe.Pointer(&_zero) } - r0, _, e1 := Syscall(SYS_READ, uintptr(fd), uintptr(_p0), uintptr(len(p))) + r0, _, e1 := syscall_syscall(libc_read_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(p))) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1177,6 +1575,10 @@ func read(fd int, p []byte) (n int, err error) { return } +var libc_read_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_read read "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Readlink(path string, buf []byte) (n int, err error) { @@ -1191,7 +1593,7 @@ func Readlink(path string, buf []byte) (n int, err error) { } else { _p1 = unsafe.Pointer(&_zero) } - r0, _, e1 := Syscall(SYS_READLINK, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(buf))) + r0, _, e1 := syscall_syscall(libc_readlink_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(buf))) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1199,6 +1601,10 @@ func Readlink(path string, buf []byte) (n int, err error) { return } +var libc_readlink_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readlink readlink "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Readlinkat(dirfd int, path string, buf []byte) (n int, err error) { @@ -1213,7 +1619,7 @@ func Readlinkat(dirfd int, path string, buf []byte) (n int, err error) { } else { _p1 = unsafe.Pointer(&_zero) } - r0, _, e1 := Syscall6(SYS_READLINKAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(buf)), 0, 0) + r0, _, e1 := syscall_syscall6(libc_readlinkat_trampoline_addr, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(_p1), uintptr(len(buf)), 0, 0) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1221,6 +1627,10 @@ func Readlinkat(dirfd int, path string, buf []byte) (n int, err error) { return } +var libc_readlinkat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_readlinkat readlinkat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Rename(from string, to string) (err error) { @@ -1234,13 +1644,17 @@ func Rename(from string, to string) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_RENAME, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), 0) + _, _, e1 := syscall_syscall(libc_rename_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_rename_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_rename rename "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Renameat(fromfd int, from string, tofd int, to string) (err error) { @@ -1254,13 +1668,17 @@ func Renameat(fromfd int, from string, tofd int, to string) (err error) { if err != nil { return } - _, _, e1 := Syscall6(SYS_RENAMEAT, uintptr(fromfd), uintptr(unsafe.Pointer(_p0)), uintptr(tofd), uintptr(unsafe.Pointer(_p1)), 0, 0) + _, _, e1 := syscall_syscall6(libc_renameat_trampoline_addr, uintptr(fromfd), uintptr(unsafe.Pointer(_p0)), uintptr(tofd), uintptr(unsafe.Pointer(_p1)), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_renameat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_renameat renameat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Revoke(path string) (err error) { @@ -1269,13 +1687,17 @@ func Revoke(path string) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_REVOKE, uintptr(unsafe.Pointer(_p0)), 0, 0) + _, _, e1 := syscall_syscall(libc_revoke_trampoline_addr, uintptr(unsafe.Pointer(_p0)), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_revoke_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_revoke revoke "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Rmdir(path string) (err error) { @@ -1284,17 +1706,21 @@ func Rmdir(path string) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_RMDIR, uintptr(unsafe.Pointer(_p0)), 0, 0) + _, _, e1 := syscall_syscall(libc_rmdir_trampoline_addr, uintptr(unsafe.Pointer(_p0)), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_rmdir_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_rmdir rmdir "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Seek(fd int, offset int64, whence int) (newoffset int64, err error) { - r0, _, e1 := Syscall6(SYS_LSEEK, uintptr(fd), 0, uintptr(offset), uintptr(whence), 0, 0) + r0, _, e1 := syscall_syscall(libc_lseek_trampoline_addr, uintptr(fd), uintptr(offset), uintptr(whence)) newoffset = int64(r0) if e1 != 0 { err = errnoErr(e1) @@ -1302,10 +1728,14 @@ func Seek(fd int, offset int64, whence int) (newoffset int64, err error) { return } +var libc_lseek_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_lseek lseek "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err error) { - r0, _, e1 := Syscall6(SYS_SELECT, uintptr(nfd), uintptr(unsafe.Pointer(r)), uintptr(unsafe.Pointer(w)), uintptr(unsafe.Pointer(e)), uintptr(unsafe.Pointer(timeout)), 0) + r0, _, e1 := syscall_syscall6(libc_select_trampoline_addr, uintptr(nfd), uintptr(unsafe.Pointer(r)), uintptr(unsafe.Pointer(w)), uintptr(unsafe.Pointer(e)), uintptr(unsafe.Pointer(timeout)), 0) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1313,36 +1743,52 @@ func Select(nfd int, r *FdSet, w *FdSet, e *FdSet, timeout *Timeval) (n int, err return } +var libc_select_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_select select "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setegid(egid int) (err error) { - _, _, e1 := RawSyscall(SYS_SETEGID, uintptr(egid), 0, 0) + _, _, e1 := syscall_rawSyscall(libc_setegid_trampoline_addr, uintptr(egid), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setegid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setegid setegid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Seteuid(euid int) (err error) { - _, _, e1 := RawSyscall(SYS_SETEUID, uintptr(euid), 0, 0) + _, _, e1 := syscall_rawSyscall(libc_seteuid_trampoline_addr, uintptr(euid), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_seteuid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_seteuid seteuid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setgid(gid int) (err error) { - _, _, e1 := RawSyscall(SYS_SETGID, uintptr(gid), 0, 0) + _, _, e1 := syscall_rawSyscall(libc_setgid_trampoline_addr, uintptr(gid), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setgid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setgid setgid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setlogin(name string) (err error) { @@ -1351,97 +1797,119 @@ func Setlogin(name string) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_SETLOGIN, uintptr(unsafe.Pointer(_p0)), 0, 0) + _, _, e1 := syscall_syscall(libc_setlogin_trampoline_addr, uintptr(unsafe.Pointer(_p0)), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setlogin_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setlogin setlogin "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setpgid(pid int, pgid int) (err error) { - _, _, e1 := RawSyscall(SYS_SETPGID, uintptr(pid), uintptr(pgid), 0) + _, _, e1 := syscall_rawSyscall(libc_setpgid_trampoline_addr, uintptr(pid), uintptr(pgid), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setpgid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setpgid setpgid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setpriority(which int, who int, prio int) (err error) { - _, _, e1 := Syscall(SYS_SETPRIORITY, uintptr(which), uintptr(who), uintptr(prio)) + _, _, e1 := syscall_syscall(libc_setpriority_trampoline_addr, uintptr(which), uintptr(who), uintptr(prio)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setpriority_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setpriority setpriority "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setregid(rgid int, egid int) (err error) { - _, _, e1 := RawSyscall(SYS_SETREGID, uintptr(rgid), uintptr(egid), 0) + _, _, e1 := syscall_rawSyscall(libc_setregid_trampoline_addr, uintptr(rgid), uintptr(egid), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setregid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setregid setregid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setreuid(ruid int, euid int) (err error) { - _, _, e1 := RawSyscall(SYS_SETREUID, uintptr(ruid), uintptr(euid), 0) + _, _, e1 := syscall_rawSyscall(libc_setreuid_trampoline_addr, uintptr(ruid), uintptr(euid), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setreuid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setreuid setreuid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setresgid(rgid int, egid int, sgid int) (err error) { - _, _, e1 := RawSyscall(SYS_SETRESGID, uintptr(rgid), uintptr(egid), uintptr(sgid)) + _, _, e1 := syscall_rawSyscall(libc_setresgid_trampoline_addr, uintptr(rgid), uintptr(egid), uintptr(sgid)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setresgid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setresgid setresgid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setresuid(ruid int, euid int, suid int) (err error) { - _, _, e1 := RawSyscall(SYS_SETRESUID, uintptr(ruid), uintptr(euid), uintptr(suid)) + _, _, e1 := syscall_rawSyscall(libc_setresuid_trampoline_addr, uintptr(ruid), uintptr(euid), uintptr(suid)) if e1 != 0 { err = errnoErr(e1) } return } -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +var libc_setresuid_trampoline_addr uintptr -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := RawSyscall(SYS_SETRLIMIT, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} +//go:cgo_import_dynamic libc_setresuid setresuid "libc.so" // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setrtable(rtable int) (err error) { - _, _, e1 := RawSyscall(SYS_SETRTABLE, uintptr(rtable), 0, 0) + _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setrtable_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setrtable setrtable "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setsid() (pid int, err error) { - r0, _, e1 := RawSyscall(SYS_SETSID, 0, 0, 0) + r0, _, e1 := syscall_rawSyscall(libc_setsid_trampoline_addr, 0, 0, 0) pid = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1449,26 +1917,38 @@ func Setsid() (pid int, err error) { return } +var libc_setsid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setsid setsid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Settimeofday(tp *Timeval) (err error) { - _, _, e1 := RawSyscall(SYS_SETTIMEOFDAY, uintptr(unsafe.Pointer(tp)), 0, 0) + _, _, e1 := syscall_rawSyscall(libc_settimeofday_trampoline_addr, uintptr(unsafe.Pointer(tp)), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_settimeofday_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_settimeofday settimeofday "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Setuid(uid int) (err error) { - _, _, e1 := RawSyscall(SYS_SETUID, uintptr(uid), 0, 0) + _, _, e1 := syscall_rawSyscall(libc_setuid_trampoline_addr, uintptr(uid), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_setuid_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_setuid setuid "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Stat(path string, stat *Stat_t) (err error) { @@ -1477,13 +1957,17 @@ func Stat(path string, stat *Stat_t) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_STAT, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), 0) + _, _, e1 := syscall_syscall(libc_stat_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_stat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_stat stat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Statfs(path string, stat *Statfs_t) (err error) { @@ -1492,13 +1976,17 @@ func Statfs(path string, stat *Statfs_t) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_STATFS, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), 0) + _, _, e1 := syscall_syscall(libc_statfs_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(stat)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_statfs_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_statfs statfs "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Symlink(path string, link string) (err error) { @@ -1512,13 +2000,17 @@ func Symlink(path string, link string) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_SYMLINK, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), 0) + _, _, e1 := syscall_syscall(libc_symlink_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(_p1)), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_symlink_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_symlink symlink "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Symlinkat(oldpath string, newdirfd int, newpath string) (err error) { @@ -1532,23 +2024,31 @@ func Symlinkat(oldpath string, newdirfd int, newpath string) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_SYMLINKAT, uintptr(unsafe.Pointer(_p0)), uintptr(newdirfd), uintptr(unsafe.Pointer(_p1))) + _, _, e1 := syscall_syscall(libc_symlinkat_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(newdirfd), uintptr(unsafe.Pointer(_p1))) if e1 != 0 { err = errnoErr(e1) } return } +var libc_symlinkat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_symlinkat symlinkat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Sync() (err error) { - _, _, e1 := Syscall(SYS_SYNC, 0, 0, 0) + _, _, e1 := syscall_syscall(libc_sync_trampoline_addr, 0, 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_sync_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_sync sync "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Truncate(path string, length int64) (err error) { @@ -1557,21 +2057,29 @@ func Truncate(path string, length int64) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_TRUNCATE, uintptr(unsafe.Pointer(_p0)), 0, uintptr(length)) + _, _, e1 := syscall_syscall(libc_truncate_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(length), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_truncate_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_truncate truncate "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Umask(newmask int) (oldmask int) { - r0, _, _ := Syscall(SYS_UMASK, uintptr(newmask), 0, 0) + r0, _, _ := syscall_syscall(libc_umask_trampoline_addr, uintptr(newmask), 0, 0) oldmask = int(r0) return } +var libc_umask_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_umask umask "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Unlink(path string) (err error) { @@ -1580,13 +2088,17 @@ func Unlink(path string) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_UNLINK, uintptr(unsafe.Pointer(_p0)), 0, 0) + _, _, e1 := syscall_syscall(libc_unlink_trampoline_addr, uintptr(unsafe.Pointer(_p0)), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_unlink_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_unlink unlink "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Unlinkat(dirfd int, path string, flags int) (err error) { @@ -1595,13 +2107,17 @@ func Unlinkat(dirfd int, path string, flags int) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_UNLINKAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(flags)) + _, _, e1 := syscall_syscall(libc_unlinkat_trampoline_addr, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(flags)) if e1 != 0 { err = errnoErr(e1) } return } +var libc_unlinkat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_unlinkat unlinkat "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func Unmount(path string, flags int) (err error) { @@ -1610,13 +2126,17 @@ func Unmount(path string, flags int) (err error) { if err != nil { return } - _, _, e1 := Syscall(SYS_UNMOUNT, uintptr(unsafe.Pointer(_p0)), uintptr(flags), 0) + _, _, e1 := syscall_syscall(libc_unmount_trampoline_addr, uintptr(unsafe.Pointer(_p0)), uintptr(flags), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_unmount_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_unmount unmount "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func write(fd int, p []byte) (n int, err error) { @@ -1626,7 +2146,7 @@ func write(fd int, p []byte) (n int, err error) { } else { _p0 = unsafe.Pointer(&_zero) } - r0, _, e1 := Syscall(SYS_WRITE, uintptr(fd), uintptr(_p0), uintptr(len(p))) + r0, _, e1 := syscall_syscall(libc_write_trampoline_addr, uintptr(fd), uintptr(_p0), uintptr(len(p))) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1634,10 +2154,14 @@ func write(fd int, p []byte) (n int, err error) { return } +var libc_write_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_write write "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func mmap(addr uintptr, length uintptr, prot int, flag int, fd int, pos int64) (ret uintptr, err error) { - r0, _, e1 := Syscall9(SYS_MMAP, uintptr(addr), uintptr(length), uintptr(prot), uintptr(flag), uintptr(fd), 0, uintptr(pos), 0, 0) + r0, _, e1 := syscall_syscall6(libc_mmap_trampoline_addr, uintptr(addr), uintptr(length), uintptr(prot), uintptr(flag), uintptr(fd), uintptr(pos)) ret = uintptr(r0) if e1 != 0 { err = errnoErr(e1) @@ -1645,20 +2169,28 @@ func mmap(addr uintptr, length uintptr, prot int, flag int, fd int, pos int64) ( return } +var libc_mmap_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_mmap mmap "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func munmap(addr uintptr, length uintptr) (err error) { - _, _, e1 := Syscall(SYS_MUNMAP, uintptr(addr), uintptr(length), 0) + _, _, e1 := syscall_syscall(libc_munmap_trampoline_addr, uintptr(addr), uintptr(length), 0) if e1 != 0 { err = errnoErr(e1) } return } +var libc_munmap_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_munmap munmap "libc.so" + // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func readlen(fd int, buf *byte, nbuf int) (n int, err error) { - r0, _, e1 := Syscall(SYS_READ, uintptr(fd), uintptr(unsafe.Pointer(buf)), uintptr(nbuf)) + r0, _, e1 := syscall_syscall(libc_read_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(buf)), uintptr(nbuf)) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1669,7 +2201,7 @@ func readlen(fd int, buf *byte, nbuf int) (n int, err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT func writelen(fd int, buf *byte, nbuf int) (n int, err error) { - r0, _, e1 := Syscall(SYS_WRITE, uintptr(fd), uintptr(unsafe.Pointer(buf)), uintptr(nbuf)) + r0, _, e1 := syscall_syscall(libc_write_trampoline_addr, uintptr(fd), uintptr(unsafe.Pointer(buf)), uintptr(nbuf)) n = int(r0) if e1 != 0 { err = errnoErr(e1) @@ -1685,9 +2217,13 @@ func utimensat(dirfd int, path string, times *[2]Timespec, flags int) (err error if err != nil { return } - _, _, e1 := Syscall6(SYS_UTIMENSAT, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(times)), uintptr(flags), 0, 0) + _, _, e1 := syscall_syscall6(libc_utimensat_trampoline_addr, uintptr(dirfd), uintptr(unsafe.Pointer(_p0)), uintptr(unsafe.Pointer(times)), uintptr(flags), 0, 0) if e1 != 0 { err = errnoErr(e1) } return } + +var libc_utimensat_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_utimensat utimensat "libc.so" diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s new file mode 100644 index 0000000..05d4bff --- /dev/null +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_mips64.s @@ -0,0 +1,664 @@ +// go run mkasm.go openbsd mips64 +// Code generated by the command above; DO NOT EDIT. + +#include "textflag.h" + +TEXT libc_getgroups_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getgroups(SB) +GLOBL ·libc_getgroups_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getgroups_trampoline_addr(SB)/8, $libc_getgroups_trampoline<>(SB) + +TEXT libc_setgroups_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setgroups(SB) +GLOBL ·libc_setgroups_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setgroups_trampoline_addr(SB)/8, $libc_setgroups_trampoline<>(SB) + +TEXT libc_wait4_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_wait4(SB) +GLOBL ·libc_wait4_trampoline_addr(SB), RODATA, $8 +DATA ·libc_wait4_trampoline_addr(SB)/8, $libc_wait4_trampoline<>(SB) + +TEXT libc_accept_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_accept(SB) +GLOBL ·libc_accept_trampoline_addr(SB), RODATA, $8 +DATA ·libc_accept_trampoline_addr(SB)/8, $libc_accept_trampoline<>(SB) + +TEXT libc_bind_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_bind(SB) +GLOBL ·libc_bind_trampoline_addr(SB), RODATA, $8 +DATA ·libc_bind_trampoline_addr(SB)/8, $libc_bind_trampoline<>(SB) + +TEXT libc_connect_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_connect(SB) +GLOBL ·libc_connect_trampoline_addr(SB), RODATA, $8 +DATA ·libc_connect_trampoline_addr(SB)/8, $libc_connect_trampoline<>(SB) + +TEXT libc_socket_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_socket(SB) +GLOBL ·libc_socket_trampoline_addr(SB), RODATA, $8 +DATA ·libc_socket_trampoline_addr(SB)/8, $libc_socket_trampoline<>(SB) + +TEXT libc_getsockopt_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getsockopt(SB) +GLOBL ·libc_getsockopt_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getsockopt_trampoline_addr(SB)/8, $libc_getsockopt_trampoline<>(SB) + +TEXT libc_setsockopt_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setsockopt(SB) +GLOBL ·libc_setsockopt_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setsockopt_trampoline_addr(SB)/8, $libc_setsockopt_trampoline<>(SB) + +TEXT libc_getpeername_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getpeername(SB) +GLOBL ·libc_getpeername_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getpeername_trampoline_addr(SB)/8, $libc_getpeername_trampoline<>(SB) + +TEXT libc_getsockname_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getsockname(SB) +GLOBL ·libc_getsockname_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getsockname_trampoline_addr(SB)/8, $libc_getsockname_trampoline<>(SB) + +TEXT libc_shutdown_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_shutdown(SB) +GLOBL ·libc_shutdown_trampoline_addr(SB), RODATA, $8 +DATA ·libc_shutdown_trampoline_addr(SB)/8, $libc_shutdown_trampoline<>(SB) + +TEXT libc_socketpair_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_socketpair(SB) +GLOBL ·libc_socketpair_trampoline_addr(SB), RODATA, $8 +DATA ·libc_socketpair_trampoline_addr(SB)/8, $libc_socketpair_trampoline<>(SB) + +TEXT libc_recvfrom_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_recvfrom(SB) +GLOBL ·libc_recvfrom_trampoline_addr(SB), RODATA, $8 +DATA ·libc_recvfrom_trampoline_addr(SB)/8, $libc_recvfrom_trampoline<>(SB) + +TEXT libc_sendto_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sendto(SB) +GLOBL ·libc_sendto_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sendto_trampoline_addr(SB)/8, $libc_sendto_trampoline<>(SB) + +TEXT libc_recvmsg_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_recvmsg(SB) +GLOBL ·libc_recvmsg_trampoline_addr(SB), RODATA, $8 +DATA ·libc_recvmsg_trampoline_addr(SB)/8, $libc_recvmsg_trampoline<>(SB) + +TEXT libc_sendmsg_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sendmsg(SB) +GLOBL ·libc_sendmsg_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sendmsg_trampoline_addr(SB)/8, $libc_sendmsg_trampoline<>(SB) + +TEXT libc_kevent_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_kevent(SB) +GLOBL ·libc_kevent_trampoline_addr(SB), RODATA, $8 +DATA ·libc_kevent_trampoline_addr(SB)/8, $libc_kevent_trampoline<>(SB) + +TEXT libc_utimes_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_utimes(SB) +GLOBL ·libc_utimes_trampoline_addr(SB), RODATA, $8 +DATA ·libc_utimes_trampoline_addr(SB)/8, $libc_utimes_trampoline<>(SB) + +TEXT libc_futimes_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_futimes(SB) +GLOBL ·libc_futimes_trampoline_addr(SB), RODATA, $8 +DATA ·libc_futimes_trampoline_addr(SB)/8, $libc_futimes_trampoline<>(SB) + +TEXT libc_poll_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_poll(SB) +GLOBL ·libc_poll_trampoline_addr(SB), RODATA, $8 +DATA ·libc_poll_trampoline_addr(SB)/8, $libc_poll_trampoline<>(SB) + +TEXT libc_madvise_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_madvise(SB) +GLOBL ·libc_madvise_trampoline_addr(SB), RODATA, $8 +DATA ·libc_madvise_trampoline_addr(SB)/8, $libc_madvise_trampoline<>(SB) + +TEXT libc_mlock_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_mlock(SB) +GLOBL ·libc_mlock_trampoline_addr(SB), RODATA, $8 +DATA ·libc_mlock_trampoline_addr(SB)/8, $libc_mlock_trampoline<>(SB) + +TEXT libc_mlockall_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_mlockall(SB) +GLOBL ·libc_mlockall_trampoline_addr(SB), RODATA, $8 +DATA ·libc_mlockall_trampoline_addr(SB)/8, $libc_mlockall_trampoline<>(SB) + +TEXT libc_mprotect_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_mprotect(SB) +GLOBL ·libc_mprotect_trampoline_addr(SB), RODATA, $8 +DATA ·libc_mprotect_trampoline_addr(SB)/8, $libc_mprotect_trampoline<>(SB) + +TEXT libc_msync_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_msync(SB) +GLOBL ·libc_msync_trampoline_addr(SB), RODATA, $8 +DATA ·libc_msync_trampoline_addr(SB)/8, $libc_msync_trampoline<>(SB) + +TEXT libc_munlock_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_munlock(SB) +GLOBL ·libc_munlock_trampoline_addr(SB), RODATA, $8 +DATA ·libc_munlock_trampoline_addr(SB)/8, $libc_munlock_trampoline<>(SB) + +TEXT libc_munlockall_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_munlockall(SB) +GLOBL ·libc_munlockall_trampoline_addr(SB), RODATA, $8 +DATA ·libc_munlockall_trampoline_addr(SB)/8, $libc_munlockall_trampoline<>(SB) + +TEXT libc_pipe2_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_pipe2(SB) +GLOBL ·libc_pipe2_trampoline_addr(SB), RODATA, $8 +DATA ·libc_pipe2_trampoline_addr(SB)/8, $libc_pipe2_trampoline<>(SB) + +TEXT libc_getdents_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getdents(SB) +GLOBL ·libc_getdents_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getdents_trampoline_addr(SB)/8, $libc_getdents_trampoline<>(SB) + +TEXT libc_getcwd_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getcwd(SB) +GLOBL ·libc_getcwd_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getcwd_trampoline_addr(SB)/8, $libc_getcwd_trampoline<>(SB) + +TEXT libc_ioctl_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_ioctl(SB) +GLOBL ·libc_ioctl_trampoline_addr(SB), RODATA, $8 +DATA ·libc_ioctl_trampoline_addr(SB)/8, $libc_ioctl_trampoline<>(SB) + +TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sysctl(SB) +GLOBL ·libc_sysctl_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sysctl_trampoline_addr(SB)/8, $libc_sysctl_trampoline<>(SB) + +TEXT libc_ppoll_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_ppoll(SB) +GLOBL ·libc_ppoll_trampoline_addr(SB), RODATA, $8 +DATA ·libc_ppoll_trampoline_addr(SB)/8, $libc_ppoll_trampoline<>(SB) + +TEXT libc_access_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_access(SB) +GLOBL ·libc_access_trampoline_addr(SB), RODATA, $8 +DATA ·libc_access_trampoline_addr(SB)/8, $libc_access_trampoline<>(SB) + +TEXT libc_adjtime_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_adjtime(SB) +GLOBL ·libc_adjtime_trampoline_addr(SB), RODATA, $8 +DATA ·libc_adjtime_trampoline_addr(SB)/8, $libc_adjtime_trampoline<>(SB) + +TEXT libc_chdir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_chdir(SB) +GLOBL ·libc_chdir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_chdir_trampoline_addr(SB)/8, $libc_chdir_trampoline<>(SB) + +TEXT libc_chflags_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_chflags(SB) +GLOBL ·libc_chflags_trampoline_addr(SB), RODATA, $8 +DATA ·libc_chflags_trampoline_addr(SB)/8, $libc_chflags_trampoline<>(SB) + +TEXT libc_chmod_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_chmod(SB) +GLOBL ·libc_chmod_trampoline_addr(SB), RODATA, $8 +DATA ·libc_chmod_trampoline_addr(SB)/8, $libc_chmod_trampoline<>(SB) + +TEXT libc_chown_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_chown(SB) +GLOBL ·libc_chown_trampoline_addr(SB), RODATA, $8 +DATA ·libc_chown_trampoline_addr(SB)/8, $libc_chown_trampoline<>(SB) + +TEXT libc_chroot_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_chroot(SB) +GLOBL ·libc_chroot_trampoline_addr(SB), RODATA, $8 +DATA ·libc_chroot_trampoline_addr(SB)/8, $libc_chroot_trampoline<>(SB) + +TEXT libc_clock_gettime_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_clock_gettime(SB) +GLOBL ·libc_clock_gettime_trampoline_addr(SB), RODATA, $8 +DATA ·libc_clock_gettime_trampoline_addr(SB)/8, $libc_clock_gettime_trampoline<>(SB) + +TEXT libc_close_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_close(SB) +GLOBL ·libc_close_trampoline_addr(SB), RODATA, $8 +DATA ·libc_close_trampoline_addr(SB)/8, $libc_close_trampoline<>(SB) + +TEXT libc_dup_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_dup(SB) +GLOBL ·libc_dup_trampoline_addr(SB), RODATA, $8 +DATA ·libc_dup_trampoline_addr(SB)/8, $libc_dup_trampoline<>(SB) + +TEXT libc_dup2_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_dup2(SB) +GLOBL ·libc_dup2_trampoline_addr(SB), RODATA, $8 +DATA ·libc_dup2_trampoline_addr(SB)/8, $libc_dup2_trampoline<>(SB) + +TEXT libc_dup3_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_dup3(SB) +GLOBL ·libc_dup3_trampoline_addr(SB), RODATA, $8 +DATA ·libc_dup3_trampoline_addr(SB)/8, $libc_dup3_trampoline<>(SB) + +TEXT libc_exit_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_exit(SB) +GLOBL ·libc_exit_trampoline_addr(SB), RODATA, $8 +DATA ·libc_exit_trampoline_addr(SB)/8, $libc_exit_trampoline<>(SB) + +TEXT libc_faccessat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_faccessat(SB) +GLOBL ·libc_faccessat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_faccessat_trampoline_addr(SB)/8, $libc_faccessat_trampoline<>(SB) + +TEXT libc_fchdir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_fchdir(SB) +GLOBL ·libc_fchdir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_fchdir_trampoline_addr(SB)/8, $libc_fchdir_trampoline<>(SB) + +TEXT libc_fchflags_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_fchflags(SB) +GLOBL ·libc_fchflags_trampoline_addr(SB), RODATA, $8 +DATA ·libc_fchflags_trampoline_addr(SB)/8, $libc_fchflags_trampoline<>(SB) + +TEXT libc_fchmod_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_fchmod(SB) +GLOBL ·libc_fchmod_trampoline_addr(SB), RODATA, $8 +DATA ·libc_fchmod_trampoline_addr(SB)/8, $libc_fchmod_trampoline<>(SB) + +TEXT libc_fchmodat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_fchmodat(SB) +GLOBL ·libc_fchmodat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_fchmodat_trampoline_addr(SB)/8, $libc_fchmodat_trampoline<>(SB) + +TEXT libc_fchown_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_fchown(SB) +GLOBL ·libc_fchown_trampoline_addr(SB), RODATA, $8 +DATA ·libc_fchown_trampoline_addr(SB)/8, $libc_fchown_trampoline<>(SB) + +TEXT libc_fchownat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_fchownat(SB) +GLOBL ·libc_fchownat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_fchownat_trampoline_addr(SB)/8, $libc_fchownat_trampoline<>(SB) + +TEXT libc_flock_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_flock(SB) +GLOBL ·libc_flock_trampoline_addr(SB), RODATA, $8 +DATA ·libc_flock_trampoline_addr(SB)/8, $libc_flock_trampoline<>(SB) + +TEXT libc_fpathconf_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_fpathconf(SB) +GLOBL ·libc_fpathconf_trampoline_addr(SB), RODATA, $8 +DATA ·libc_fpathconf_trampoline_addr(SB)/8, $libc_fpathconf_trampoline<>(SB) + +TEXT libc_fstat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_fstat(SB) +GLOBL ·libc_fstat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_fstat_trampoline_addr(SB)/8, $libc_fstat_trampoline<>(SB) + +TEXT libc_fstatat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_fstatat(SB) +GLOBL ·libc_fstatat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_fstatat_trampoline_addr(SB)/8, $libc_fstatat_trampoline<>(SB) + +TEXT libc_fstatfs_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_fstatfs(SB) +GLOBL ·libc_fstatfs_trampoline_addr(SB), RODATA, $8 +DATA ·libc_fstatfs_trampoline_addr(SB)/8, $libc_fstatfs_trampoline<>(SB) + +TEXT libc_fsync_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_fsync(SB) +GLOBL ·libc_fsync_trampoline_addr(SB), RODATA, $8 +DATA ·libc_fsync_trampoline_addr(SB)/8, $libc_fsync_trampoline<>(SB) + +TEXT libc_ftruncate_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_ftruncate(SB) +GLOBL ·libc_ftruncate_trampoline_addr(SB), RODATA, $8 +DATA ·libc_ftruncate_trampoline_addr(SB)/8, $libc_ftruncate_trampoline<>(SB) + +TEXT libc_getegid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getegid(SB) +GLOBL ·libc_getegid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getegid_trampoline_addr(SB)/8, $libc_getegid_trampoline<>(SB) + +TEXT libc_geteuid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_geteuid(SB) +GLOBL ·libc_geteuid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_geteuid_trampoline_addr(SB)/8, $libc_geteuid_trampoline<>(SB) + +TEXT libc_getgid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getgid(SB) +GLOBL ·libc_getgid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getgid_trampoline_addr(SB)/8, $libc_getgid_trampoline<>(SB) + +TEXT libc_getpgid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getpgid(SB) +GLOBL ·libc_getpgid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getpgid_trampoline_addr(SB)/8, $libc_getpgid_trampoline<>(SB) + +TEXT libc_getpgrp_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getpgrp(SB) +GLOBL ·libc_getpgrp_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getpgrp_trampoline_addr(SB)/8, $libc_getpgrp_trampoline<>(SB) + +TEXT libc_getpid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getpid(SB) +GLOBL ·libc_getpid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getpid_trampoline_addr(SB)/8, $libc_getpid_trampoline<>(SB) + +TEXT libc_getppid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getppid(SB) +GLOBL ·libc_getppid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getppid_trampoline_addr(SB)/8, $libc_getppid_trampoline<>(SB) + +TEXT libc_getpriority_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getpriority(SB) +GLOBL ·libc_getpriority_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getpriority_trampoline_addr(SB)/8, $libc_getpriority_trampoline<>(SB) + +TEXT libc_getrlimit_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getrlimit(SB) +GLOBL ·libc_getrlimit_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getrlimit_trampoline_addr(SB)/8, $libc_getrlimit_trampoline<>(SB) + +TEXT libc_getrtable_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getrtable(SB) +GLOBL ·libc_getrtable_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getrtable_trampoline_addr(SB)/8, $libc_getrtable_trampoline<>(SB) + +TEXT libc_getrusage_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getrusage(SB) +GLOBL ·libc_getrusage_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getrusage_trampoline_addr(SB)/8, $libc_getrusage_trampoline<>(SB) + +TEXT libc_getsid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getsid(SB) +GLOBL ·libc_getsid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getsid_trampoline_addr(SB)/8, $libc_getsid_trampoline<>(SB) + +TEXT libc_gettimeofday_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_gettimeofday(SB) +GLOBL ·libc_gettimeofday_trampoline_addr(SB), RODATA, $8 +DATA ·libc_gettimeofday_trampoline_addr(SB)/8, $libc_gettimeofday_trampoline<>(SB) + +TEXT libc_getuid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_getuid(SB) +GLOBL ·libc_getuid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_getuid_trampoline_addr(SB)/8, $libc_getuid_trampoline<>(SB) + +TEXT libc_issetugid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_issetugid(SB) +GLOBL ·libc_issetugid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_issetugid_trampoline_addr(SB)/8, $libc_issetugid_trampoline<>(SB) + +TEXT libc_kill_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_kill(SB) +GLOBL ·libc_kill_trampoline_addr(SB), RODATA, $8 +DATA ·libc_kill_trampoline_addr(SB)/8, $libc_kill_trampoline<>(SB) + +TEXT libc_kqueue_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_kqueue(SB) +GLOBL ·libc_kqueue_trampoline_addr(SB), RODATA, $8 +DATA ·libc_kqueue_trampoline_addr(SB)/8, $libc_kqueue_trampoline<>(SB) + +TEXT libc_lchown_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_lchown(SB) +GLOBL ·libc_lchown_trampoline_addr(SB), RODATA, $8 +DATA ·libc_lchown_trampoline_addr(SB)/8, $libc_lchown_trampoline<>(SB) + +TEXT libc_link_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_link(SB) +GLOBL ·libc_link_trampoline_addr(SB), RODATA, $8 +DATA ·libc_link_trampoline_addr(SB)/8, $libc_link_trampoline<>(SB) + +TEXT libc_linkat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_linkat(SB) +GLOBL ·libc_linkat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_linkat_trampoline_addr(SB)/8, $libc_linkat_trampoline<>(SB) + +TEXT libc_listen_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_listen(SB) +GLOBL ·libc_listen_trampoline_addr(SB), RODATA, $8 +DATA ·libc_listen_trampoline_addr(SB)/8, $libc_listen_trampoline<>(SB) + +TEXT libc_lstat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_lstat(SB) +GLOBL ·libc_lstat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_lstat_trampoline_addr(SB)/8, $libc_lstat_trampoline<>(SB) + +TEXT libc_mkdir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_mkdir(SB) +GLOBL ·libc_mkdir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_mkdir_trampoline_addr(SB)/8, $libc_mkdir_trampoline<>(SB) + +TEXT libc_mkdirat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_mkdirat(SB) +GLOBL ·libc_mkdirat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_mkdirat_trampoline_addr(SB)/8, $libc_mkdirat_trampoline<>(SB) + +TEXT libc_mkfifo_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_mkfifo(SB) +GLOBL ·libc_mkfifo_trampoline_addr(SB), RODATA, $8 +DATA ·libc_mkfifo_trampoline_addr(SB)/8, $libc_mkfifo_trampoline<>(SB) + +TEXT libc_mkfifoat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_mkfifoat(SB) +GLOBL ·libc_mkfifoat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_mkfifoat_trampoline_addr(SB)/8, $libc_mkfifoat_trampoline<>(SB) + +TEXT libc_mknod_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_mknod(SB) +GLOBL ·libc_mknod_trampoline_addr(SB), RODATA, $8 +DATA ·libc_mknod_trampoline_addr(SB)/8, $libc_mknod_trampoline<>(SB) + +TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_mknodat(SB) +GLOBL ·libc_mknodat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_mknodat_trampoline_addr(SB)/8, $libc_mknodat_trampoline<>(SB) + +TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_nanosleep(SB) +GLOBL ·libc_nanosleep_trampoline_addr(SB), RODATA, $8 +DATA ·libc_nanosleep_trampoline_addr(SB)/8, $libc_nanosleep_trampoline<>(SB) + +TEXT libc_open_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_open(SB) +GLOBL ·libc_open_trampoline_addr(SB), RODATA, $8 +DATA ·libc_open_trampoline_addr(SB)/8, $libc_open_trampoline<>(SB) + +TEXT libc_openat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_openat(SB) +GLOBL ·libc_openat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_openat_trampoline_addr(SB)/8, $libc_openat_trampoline<>(SB) + +TEXT libc_pathconf_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_pathconf(SB) +GLOBL ·libc_pathconf_trampoline_addr(SB), RODATA, $8 +DATA ·libc_pathconf_trampoline_addr(SB)/8, $libc_pathconf_trampoline<>(SB) + +TEXT libc_pread_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_pread(SB) +GLOBL ·libc_pread_trampoline_addr(SB), RODATA, $8 +DATA ·libc_pread_trampoline_addr(SB)/8, $libc_pread_trampoline<>(SB) + +TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_pwrite(SB) +GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $8 +DATA ·libc_pwrite_trampoline_addr(SB)/8, $libc_pwrite_trampoline<>(SB) + +TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_read(SB) +GLOBL ·libc_read_trampoline_addr(SB), RODATA, $8 +DATA ·libc_read_trampoline_addr(SB)/8, $libc_read_trampoline<>(SB) + +TEXT libc_readlink_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readlink(SB) +GLOBL ·libc_readlink_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readlink_trampoline_addr(SB)/8, $libc_readlink_trampoline<>(SB) + +TEXT libc_readlinkat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_readlinkat(SB) +GLOBL ·libc_readlinkat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_readlinkat_trampoline_addr(SB)/8, $libc_readlinkat_trampoline<>(SB) + +TEXT libc_rename_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_rename(SB) +GLOBL ·libc_rename_trampoline_addr(SB), RODATA, $8 +DATA ·libc_rename_trampoline_addr(SB)/8, $libc_rename_trampoline<>(SB) + +TEXT libc_renameat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_renameat(SB) +GLOBL ·libc_renameat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_renameat_trampoline_addr(SB)/8, $libc_renameat_trampoline<>(SB) + +TEXT libc_revoke_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_revoke(SB) +GLOBL ·libc_revoke_trampoline_addr(SB), RODATA, $8 +DATA ·libc_revoke_trampoline_addr(SB)/8, $libc_revoke_trampoline<>(SB) + +TEXT libc_rmdir_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_rmdir(SB) +GLOBL ·libc_rmdir_trampoline_addr(SB), RODATA, $8 +DATA ·libc_rmdir_trampoline_addr(SB)/8, $libc_rmdir_trampoline<>(SB) + +TEXT libc_lseek_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_lseek(SB) +GLOBL ·libc_lseek_trampoline_addr(SB), RODATA, $8 +DATA ·libc_lseek_trampoline_addr(SB)/8, $libc_lseek_trampoline<>(SB) + +TEXT libc_select_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_select(SB) +GLOBL ·libc_select_trampoline_addr(SB), RODATA, $8 +DATA ·libc_select_trampoline_addr(SB)/8, $libc_select_trampoline<>(SB) + +TEXT libc_setegid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setegid(SB) +GLOBL ·libc_setegid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setegid_trampoline_addr(SB)/8, $libc_setegid_trampoline<>(SB) + +TEXT libc_seteuid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_seteuid(SB) +GLOBL ·libc_seteuid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_seteuid_trampoline_addr(SB)/8, $libc_seteuid_trampoline<>(SB) + +TEXT libc_setgid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setgid(SB) +GLOBL ·libc_setgid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setgid_trampoline_addr(SB)/8, $libc_setgid_trampoline<>(SB) + +TEXT libc_setlogin_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setlogin(SB) +GLOBL ·libc_setlogin_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setlogin_trampoline_addr(SB)/8, $libc_setlogin_trampoline<>(SB) + +TEXT libc_setpgid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setpgid(SB) +GLOBL ·libc_setpgid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setpgid_trampoline_addr(SB)/8, $libc_setpgid_trampoline<>(SB) + +TEXT libc_setpriority_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setpriority(SB) +GLOBL ·libc_setpriority_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setpriority_trampoline_addr(SB)/8, $libc_setpriority_trampoline<>(SB) + +TEXT libc_setregid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setregid(SB) +GLOBL ·libc_setregid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setregid_trampoline_addr(SB)/8, $libc_setregid_trampoline<>(SB) + +TEXT libc_setreuid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setreuid(SB) +GLOBL ·libc_setreuid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setreuid_trampoline_addr(SB)/8, $libc_setreuid_trampoline<>(SB) + +TEXT libc_setresgid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setresgid(SB) +GLOBL ·libc_setresgid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setresgid_trampoline_addr(SB)/8, $libc_setresgid_trampoline<>(SB) + +TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setresuid(SB) +GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setresuid_trampoline_addr(SB)/8, $libc_setresuid_trampoline<>(SB) + +TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setrtable(SB) +GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setrtable_trampoline_addr(SB)/8, $libc_setrtable_trampoline<>(SB) + +TEXT libc_setsid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setsid(SB) +GLOBL ·libc_setsid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setsid_trampoline_addr(SB)/8, $libc_setsid_trampoline<>(SB) + +TEXT libc_settimeofday_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_settimeofday(SB) +GLOBL ·libc_settimeofday_trampoline_addr(SB), RODATA, $8 +DATA ·libc_settimeofday_trampoline_addr(SB)/8, $libc_settimeofday_trampoline<>(SB) + +TEXT libc_setuid_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_setuid(SB) +GLOBL ·libc_setuid_trampoline_addr(SB), RODATA, $8 +DATA ·libc_setuid_trampoline_addr(SB)/8, $libc_setuid_trampoline<>(SB) + +TEXT libc_stat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_stat(SB) +GLOBL ·libc_stat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_stat_trampoline_addr(SB)/8, $libc_stat_trampoline<>(SB) + +TEXT libc_statfs_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_statfs(SB) +GLOBL ·libc_statfs_trampoline_addr(SB), RODATA, $8 +DATA ·libc_statfs_trampoline_addr(SB)/8, $libc_statfs_trampoline<>(SB) + +TEXT libc_symlink_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_symlink(SB) +GLOBL ·libc_symlink_trampoline_addr(SB), RODATA, $8 +DATA ·libc_symlink_trampoline_addr(SB)/8, $libc_symlink_trampoline<>(SB) + +TEXT libc_symlinkat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_symlinkat(SB) +GLOBL ·libc_symlinkat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_symlinkat_trampoline_addr(SB)/8, $libc_symlinkat_trampoline<>(SB) + +TEXT libc_sync_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_sync(SB) +GLOBL ·libc_sync_trampoline_addr(SB), RODATA, $8 +DATA ·libc_sync_trampoline_addr(SB)/8, $libc_sync_trampoline<>(SB) + +TEXT libc_truncate_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_truncate(SB) +GLOBL ·libc_truncate_trampoline_addr(SB), RODATA, $8 +DATA ·libc_truncate_trampoline_addr(SB)/8, $libc_truncate_trampoline<>(SB) + +TEXT libc_umask_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_umask(SB) +GLOBL ·libc_umask_trampoline_addr(SB), RODATA, $8 +DATA ·libc_umask_trampoline_addr(SB)/8, $libc_umask_trampoline<>(SB) + +TEXT libc_unlink_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_unlink(SB) +GLOBL ·libc_unlink_trampoline_addr(SB), RODATA, $8 +DATA ·libc_unlink_trampoline_addr(SB)/8, $libc_unlink_trampoline<>(SB) + +TEXT libc_unlinkat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_unlinkat(SB) +GLOBL ·libc_unlinkat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_unlinkat_trampoline_addr(SB)/8, $libc_unlinkat_trampoline<>(SB) + +TEXT libc_unmount_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_unmount(SB) +GLOBL ·libc_unmount_trampoline_addr(SB), RODATA, $8 +DATA ·libc_unmount_trampoline_addr(SB)/8, $libc_unmount_trampoline<>(SB) + +TEXT libc_write_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_write(SB) +GLOBL ·libc_write_trampoline_addr(SB), RODATA, $8 +DATA ·libc_write_trampoline_addr(SB)/8, $libc_write_trampoline<>(SB) + +TEXT libc_mmap_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_mmap(SB) +GLOBL ·libc_mmap_trampoline_addr(SB), RODATA, $8 +DATA ·libc_mmap_trampoline_addr(SB)/8, $libc_mmap_trampoline<>(SB) + +TEXT libc_munmap_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_munmap(SB) +GLOBL ·libc_munmap_trampoline_addr(SB), RODATA, $8 +DATA ·libc_munmap_trampoline_addr(SB)/8, $libc_munmap_trampoline<>(SB) + +TEXT libc_utimensat_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_utimensat(SB) +GLOBL ·libc_utimensat_trampoline_addr(SB), RODATA, $8 +DATA ·libc_utimensat_trampoline_addr(SB)/8, $libc_utimensat_trampoline<>(SB) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go index c85de2d..739be62 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.go @@ -527,6 +527,14 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { return } +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := syscall_syscall(libc_ioctl_trampoline_addr, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + var libc_ioctl_trampoline_addr uintptr //go:cgo_import_dynamic libc_ioctl ioctl "libc.so" @@ -696,6 +704,20 @@ var libc_chroot_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := syscall_syscall(libc_clock_gettime_trampoline_addr, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_clock_gettime_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := syscall_syscall(libc_close_trampoline_addr, uintptr(fd), 0, 0) if e1 != 0 { @@ -1872,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s index 7c9223b..74a25f8 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_ppc64.s @@ -249,6 +249,12 @@ TEXT libc_chroot_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_chroot_trampoline_addr(SB), RODATA, $8 DATA ·libc_chroot_trampoline_addr(SB)/8, $libc_chroot_trampoline<>(SB) +TEXT libc_clock_gettime_trampoline<>(SB),NOSPLIT,$0-0 + CALL libc_clock_gettime(SB) + RET +GLOBL ·libc_clock_gettime_trampoline_addr(SB), RODATA, $8 +DATA ·libc_clock_gettime_trampoline_addr(SB)/8, $libc_clock_gettime_trampoline<>(SB) + TEXT libc_close_trampoline<>(SB),NOSPLIT,$0-0 CALL libc_close(SB) RET @@ -681,12 +687,6 @@ TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresuid_trampoline_addr(SB)/8, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - CALL libc_setrlimit(SB) - RET -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 CALL libc_setrtable(SB) RET diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go index 8e3e787..7d95a19 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.go @@ -527,6 +527,14 @@ func ioctl(fd int, req uint, arg uintptr) (err error) { return } +func ioctlPtr(fd int, req uint, arg unsafe.Pointer) (err error) { + _, _, e1 := syscall_syscall(libc_ioctl_trampoline_addr, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + var libc_ioctl_trampoline_addr uintptr //go:cgo_import_dynamic libc_ioctl ioctl "libc.so" @@ -696,6 +704,20 @@ var libc_chroot_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := syscall_syscall(libc_clock_gettime_trampoline_addr, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +var libc_clock_gettime_trampoline_addr uintptr + +//go:cgo_import_dynamic libc_clock_gettime clock_gettime "libc.so" + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := syscall_syscall(libc_close_trampoline_addr, uintptr(fd), 0, 0) if e1 != 0 { @@ -1872,20 +1894,6 @@ var libc_setresuid_trampoline_addr uintptr // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := syscall_rawSyscall(libc_setrlimit_trampoline_addr, uintptr(which), uintptr(unsafe.Pointer(lim)), 0) - if e1 != 0 { - err = errnoErr(e1) - } - return -} - -var libc_setrlimit_trampoline_addr uintptr - -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setrtable(rtable int) (err error) { _, _, e1 := syscall_rawSyscall(libc_setrtable_trampoline_addr, uintptr(rtable), 0, 0) if e1 != 0 { diff --git a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s index 7dba789..990be24 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s +++ b/vendor/golang.org/x/sys/unix/zsyscall_openbsd_riscv64.s @@ -5,792 +5,660 @@ TEXT libc_getgroups_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getgroups(SB) - GLOBL ·libc_getgroups_trampoline_addr(SB), RODATA, $8 DATA ·libc_getgroups_trampoline_addr(SB)/8, $libc_getgroups_trampoline<>(SB) TEXT libc_setgroups_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setgroups(SB) - GLOBL ·libc_setgroups_trampoline_addr(SB), RODATA, $8 DATA ·libc_setgroups_trampoline_addr(SB)/8, $libc_setgroups_trampoline<>(SB) TEXT libc_wait4_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_wait4(SB) - GLOBL ·libc_wait4_trampoline_addr(SB), RODATA, $8 DATA ·libc_wait4_trampoline_addr(SB)/8, $libc_wait4_trampoline<>(SB) TEXT libc_accept_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_accept(SB) - GLOBL ·libc_accept_trampoline_addr(SB), RODATA, $8 DATA ·libc_accept_trampoline_addr(SB)/8, $libc_accept_trampoline<>(SB) TEXT libc_bind_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_bind(SB) - GLOBL ·libc_bind_trampoline_addr(SB), RODATA, $8 DATA ·libc_bind_trampoline_addr(SB)/8, $libc_bind_trampoline<>(SB) TEXT libc_connect_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_connect(SB) - GLOBL ·libc_connect_trampoline_addr(SB), RODATA, $8 DATA ·libc_connect_trampoline_addr(SB)/8, $libc_connect_trampoline<>(SB) TEXT libc_socket_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_socket(SB) - GLOBL ·libc_socket_trampoline_addr(SB), RODATA, $8 DATA ·libc_socket_trampoline_addr(SB)/8, $libc_socket_trampoline<>(SB) TEXT libc_getsockopt_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsockopt(SB) - GLOBL ·libc_getsockopt_trampoline_addr(SB), RODATA, $8 DATA ·libc_getsockopt_trampoline_addr(SB)/8, $libc_getsockopt_trampoline<>(SB) TEXT libc_setsockopt_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsockopt(SB) - GLOBL ·libc_setsockopt_trampoline_addr(SB), RODATA, $8 DATA ·libc_setsockopt_trampoline_addr(SB)/8, $libc_setsockopt_trampoline<>(SB) TEXT libc_getpeername_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpeername(SB) - GLOBL ·libc_getpeername_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpeername_trampoline_addr(SB)/8, $libc_getpeername_trampoline<>(SB) TEXT libc_getsockname_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsockname(SB) - GLOBL ·libc_getsockname_trampoline_addr(SB), RODATA, $8 DATA ·libc_getsockname_trampoline_addr(SB)/8, $libc_getsockname_trampoline<>(SB) TEXT libc_shutdown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_shutdown(SB) - GLOBL ·libc_shutdown_trampoline_addr(SB), RODATA, $8 DATA ·libc_shutdown_trampoline_addr(SB)/8, $libc_shutdown_trampoline<>(SB) TEXT libc_socketpair_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_socketpair(SB) - GLOBL ·libc_socketpair_trampoline_addr(SB), RODATA, $8 DATA ·libc_socketpair_trampoline_addr(SB)/8, $libc_socketpair_trampoline<>(SB) TEXT libc_recvfrom_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_recvfrom(SB) - GLOBL ·libc_recvfrom_trampoline_addr(SB), RODATA, $8 DATA ·libc_recvfrom_trampoline_addr(SB)/8, $libc_recvfrom_trampoline<>(SB) TEXT libc_sendto_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sendto(SB) - GLOBL ·libc_sendto_trampoline_addr(SB), RODATA, $8 DATA ·libc_sendto_trampoline_addr(SB)/8, $libc_sendto_trampoline<>(SB) TEXT libc_recvmsg_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_recvmsg(SB) - GLOBL ·libc_recvmsg_trampoline_addr(SB), RODATA, $8 DATA ·libc_recvmsg_trampoline_addr(SB)/8, $libc_recvmsg_trampoline<>(SB) TEXT libc_sendmsg_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sendmsg(SB) - GLOBL ·libc_sendmsg_trampoline_addr(SB), RODATA, $8 DATA ·libc_sendmsg_trampoline_addr(SB)/8, $libc_sendmsg_trampoline<>(SB) TEXT libc_kevent_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kevent(SB) - GLOBL ·libc_kevent_trampoline_addr(SB), RODATA, $8 DATA ·libc_kevent_trampoline_addr(SB)/8, $libc_kevent_trampoline<>(SB) TEXT libc_utimes_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_utimes(SB) - GLOBL ·libc_utimes_trampoline_addr(SB), RODATA, $8 DATA ·libc_utimes_trampoline_addr(SB)/8, $libc_utimes_trampoline<>(SB) TEXT libc_futimes_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_futimes(SB) - GLOBL ·libc_futimes_trampoline_addr(SB), RODATA, $8 DATA ·libc_futimes_trampoline_addr(SB)/8, $libc_futimes_trampoline<>(SB) TEXT libc_poll_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_poll(SB) - GLOBL ·libc_poll_trampoline_addr(SB), RODATA, $8 DATA ·libc_poll_trampoline_addr(SB)/8, $libc_poll_trampoline<>(SB) TEXT libc_madvise_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_madvise(SB) - GLOBL ·libc_madvise_trampoline_addr(SB), RODATA, $8 DATA ·libc_madvise_trampoline_addr(SB)/8, $libc_madvise_trampoline<>(SB) TEXT libc_mlock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mlock(SB) - GLOBL ·libc_mlock_trampoline_addr(SB), RODATA, $8 DATA ·libc_mlock_trampoline_addr(SB)/8, $libc_mlock_trampoline<>(SB) TEXT libc_mlockall_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mlockall(SB) - GLOBL ·libc_mlockall_trampoline_addr(SB), RODATA, $8 DATA ·libc_mlockall_trampoline_addr(SB)/8, $libc_mlockall_trampoline<>(SB) TEXT libc_mprotect_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mprotect(SB) - GLOBL ·libc_mprotect_trampoline_addr(SB), RODATA, $8 DATA ·libc_mprotect_trampoline_addr(SB)/8, $libc_mprotect_trampoline<>(SB) TEXT libc_msync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_msync(SB) - GLOBL ·libc_msync_trampoline_addr(SB), RODATA, $8 DATA ·libc_msync_trampoline_addr(SB)/8, $libc_msync_trampoline<>(SB) TEXT libc_munlock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munlock(SB) - GLOBL ·libc_munlock_trampoline_addr(SB), RODATA, $8 DATA ·libc_munlock_trampoline_addr(SB)/8, $libc_munlock_trampoline<>(SB) TEXT libc_munlockall_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munlockall(SB) - GLOBL ·libc_munlockall_trampoline_addr(SB), RODATA, $8 DATA ·libc_munlockall_trampoline_addr(SB)/8, $libc_munlockall_trampoline<>(SB) TEXT libc_pipe2_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pipe2(SB) - GLOBL ·libc_pipe2_trampoline_addr(SB), RODATA, $8 DATA ·libc_pipe2_trampoline_addr(SB)/8, $libc_pipe2_trampoline<>(SB) TEXT libc_getdents_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getdents(SB) - GLOBL ·libc_getdents_trampoline_addr(SB), RODATA, $8 DATA ·libc_getdents_trampoline_addr(SB)/8, $libc_getdents_trampoline<>(SB) TEXT libc_getcwd_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getcwd(SB) - GLOBL ·libc_getcwd_trampoline_addr(SB), RODATA, $8 DATA ·libc_getcwd_trampoline_addr(SB)/8, $libc_getcwd_trampoline<>(SB) TEXT libc_ioctl_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ioctl(SB) - GLOBL ·libc_ioctl_trampoline_addr(SB), RODATA, $8 DATA ·libc_ioctl_trampoline_addr(SB)/8, $libc_ioctl_trampoline<>(SB) TEXT libc_sysctl_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sysctl(SB) - GLOBL ·libc_sysctl_trampoline_addr(SB), RODATA, $8 DATA ·libc_sysctl_trampoline_addr(SB)/8, $libc_sysctl_trampoline<>(SB) TEXT libc_ppoll_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ppoll(SB) - GLOBL ·libc_ppoll_trampoline_addr(SB), RODATA, $8 DATA ·libc_ppoll_trampoline_addr(SB)/8, $libc_ppoll_trampoline<>(SB) TEXT libc_access_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_access(SB) - GLOBL ·libc_access_trampoline_addr(SB), RODATA, $8 DATA ·libc_access_trampoline_addr(SB)/8, $libc_access_trampoline<>(SB) TEXT libc_adjtime_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_adjtime(SB) - GLOBL ·libc_adjtime_trampoline_addr(SB), RODATA, $8 DATA ·libc_adjtime_trampoline_addr(SB)/8, $libc_adjtime_trampoline<>(SB) TEXT libc_chdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chdir(SB) - GLOBL ·libc_chdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_chdir_trampoline_addr(SB)/8, $libc_chdir_trampoline<>(SB) TEXT libc_chflags_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chflags(SB) - GLOBL ·libc_chflags_trampoline_addr(SB), RODATA, $8 DATA ·libc_chflags_trampoline_addr(SB)/8, $libc_chflags_trampoline<>(SB) TEXT libc_chmod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chmod(SB) - GLOBL ·libc_chmod_trampoline_addr(SB), RODATA, $8 DATA ·libc_chmod_trampoline_addr(SB)/8, $libc_chmod_trampoline<>(SB) TEXT libc_chown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chown(SB) - GLOBL ·libc_chown_trampoline_addr(SB), RODATA, $8 DATA ·libc_chown_trampoline_addr(SB)/8, $libc_chown_trampoline<>(SB) TEXT libc_chroot_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_chroot(SB) - GLOBL ·libc_chroot_trampoline_addr(SB), RODATA, $8 DATA ·libc_chroot_trampoline_addr(SB)/8, $libc_chroot_trampoline<>(SB) +TEXT libc_clock_gettime_trampoline<>(SB),NOSPLIT,$0-0 + JMP libc_clock_gettime(SB) +GLOBL ·libc_clock_gettime_trampoline_addr(SB), RODATA, $8 +DATA ·libc_clock_gettime_trampoline_addr(SB)/8, $libc_clock_gettime_trampoline<>(SB) + TEXT libc_close_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_close(SB) - GLOBL ·libc_close_trampoline_addr(SB), RODATA, $8 DATA ·libc_close_trampoline_addr(SB)/8, $libc_close_trampoline<>(SB) TEXT libc_dup_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup(SB) - GLOBL ·libc_dup_trampoline_addr(SB), RODATA, $8 DATA ·libc_dup_trampoline_addr(SB)/8, $libc_dup_trampoline<>(SB) TEXT libc_dup2_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup2(SB) - GLOBL ·libc_dup2_trampoline_addr(SB), RODATA, $8 DATA ·libc_dup2_trampoline_addr(SB)/8, $libc_dup2_trampoline<>(SB) TEXT libc_dup3_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_dup3(SB) - GLOBL ·libc_dup3_trampoline_addr(SB), RODATA, $8 DATA ·libc_dup3_trampoline_addr(SB)/8, $libc_dup3_trampoline<>(SB) TEXT libc_exit_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_exit(SB) - GLOBL ·libc_exit_trampoline_addr(SB), RODATA, $8 DATA ·libc_exit_trampoline_addr(SB)/8, $libc_exit_trampoline<>(SB) TEXT libc_faccessat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_faccessat(SB) - GLOBL ·libc_faccessat_trampoline_addr(SB), RODATA, $8 DATA ·libc_faccessat_trampoline_addr(SB)/8, $libc_faccessat_trampoline<>(SB) TEXT libc_fchdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchdir(SB) - GLOBL ·libc_fchdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchdir_trampoline_addr(SB)/8, $libc_fchdir_trampoline<>(SB) TEXT libc_fchflags_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchflags(SB) - GLOBL ·libc_fchflags_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchflags_trampoline_addr(SB)/8, $libc_fchflags_trampoline<>(SB) TEXT libc_fchmod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchmod(SB) - GLOBL ·libc_fchmod_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchmod_trampoline_addr(SB)/8, $libc_fchmod_trampoline<>(SB) TEXT libc_fchmodat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchmodat(SB) - GLOBL ·libc_fchmodat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchmodat_trampoline_addr(SB)/8, $libc_fchmodat_trampoline<>(SB) TEXT libc_fchown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchown(SB) - GLOBL ·libc_fchown_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchown_trampoline_addr(SB)/8, $libc_fchown_trampoline<>(SB) TEXT libc_fchownat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fchownat(SB) - GLOBL ·libc_fchownat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fchownat_trampoline_addr(SB)/8, $libc_fchownat_trampoline<>(SB) TEXT libc_flock_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_flock(SB) - GLOBL ·libc_flock_trampoline_addr(SB), RODATA, $8 DATA ·libc_flock_trampoline_addr(SB)/8, $libc_flock_trampoline<>(SB) TEXT libc_fpathconf_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fpathconf(SB) - GLOBL ·libc_fpathconf_trampoline_addr(SB), RODATA, $8 DATA ·libc_fpathconf_trampoline_addr(SB)/8, $libc_fpathconf_trampoline<>(SB) TEXT libc_fstat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstat(SB) - GLOBL ·libc_fstat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fstat_trampoline_addr(SB)/8, $libc_fstat_trampoline<>(SB) TEXT libc_fstatat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstatat(SB) - GLOBL ·libc_fstatat_trampoline_addr(SB), RODATA, $8 DATA ·libc_fstatat_trampoline_addr(SB)/8, $libc_fstatat_trampoline<>(SB) TEXT libc_fstatfs_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fstatfs(SB) - GLOBL ·libc_fstatfs_trampoline_addr(SB), RODATA, $8 DATA ·libc_fstatfs_trampoline_addr(SB)/8, $libc_fstatfs_trampoline<>(SB) TEXT libc_fsync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_fsync(SB) - GLOBL ·libc_fsync_trampoline_addr(SB), RODATA, $8 DATA ·libc_fsync_trampoline_addr(SB)/8, $libc_fsync_trampoline<>(SB) TEXT libc_ftruncate_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_ftruncate(SB) - GLOBL ·libc_ftruncate_trampoline_addr(SB), RODATA, $8 DATA ·libc_ftruncate_trampoline_addr(SB)/8, $libc_ftruncate_trampoline<>(SB) TEXT libc_getegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getegid(SB) - GLOBL ·libc_getegid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getegid_trampoline_addr(SB)/8, $libc_getegid_trampoline<>(SB) TEXT libc_geteuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_geteuid(SB) - GLOBL ·libc_geteuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_geteuid_trampoline_addr(SB)/8, $libc_geteuid_trampoline<>(SB) TEXT libc_getgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getgid(SB) - GLOBL ·libc_getgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getgid_trampoline_addr(SB)/8, $libc_getgid_trampoline<>(SB) TEXT libc_getpgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpgid(SB) - GLOBL ·libc_getpgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpgid_trampoline_addr(SB)/8, $libc_getpgid_trampoline<>(SB) TEXT libc_getpgrp_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpgrp(SB) - GLOBL ·libc_getpgrp_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpgrp_trampoline_addr(SB)/8, $libc_getpgrp_trampoline<>(SB) TEXT libc_getpid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpid(SB) - GLOBL ·libc_getpid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpid_trampoline_addr(SB)/8, $libc_getpid_trampoline<>(SB) TEXT libc_getppid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getppid(SB) - GLOBL ·libc_getppid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getppid_trampoline_addr(SB)/8, $libc_getppid_trampoline<>(SB) TEXT libc_getpriority_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getpriority(SB) - GLOBL ·libc_getpriority_trampoline_addr(SB), RODATA, $8 DATA ·libc_getpriority_trampoline_addr(SB)/8, $libc_getpriority_trampoline<>(SB) TEXT libc_getrlimit_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrlimit(SB) - GLOBL ·libc_getrlimit_trampoline_addr(SB), RODATA, $8 DATA ·libc_getrlimit_trampoline_addr(SB)/8, $libc_getrlimit_trampoline<>(SB) TEXT libc_getrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrtable(SB) - GLOBL ·libc_getrtable_trampoline_addr(SB), RODATA, $8 DATA ·libc_getrtable_trampoline_addr(SB)/8, $libc_getrtable_trampoline<>(SB) TEXT libc_getrusage_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getrusage(SB) - GLOBL ·libc_getrusage_trampoline_addr(SB), RODATA, $8 DATA ·libc_getrusage_trampoline_addr(SB)/8, $libc_getrusage_trampoline<>(SB) TEXT libc_getsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getsid(SB) - GLOBL ·libc_getsid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getsid_trampoline_addr(SB)/8, $libc_getsid_trampoline<>(SB) TEXT libc_gettimeofday_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_gettimeofday(SB) - GLOBL ·libc_gettimeofday_trampoline_addr(SB), RODATA, $8 DATA ·libc_gettimeofday_trampoline_addr(SB)/8, $libc_gettimeofday_trampoline<>(SB) TEXT libc_getuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_getuid(SB) - GLOBL ·libc_getuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_getuid_trampoline_addr(SB)/8, $libc_getuid_trampoline<>(SB) TEXT libc_issetugid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_issetugid(SB) - GLOBL ·libc_issetugid_trampoline_addr(SB), RODATA, $8 DATA ·libc_issetugid_trampoline_addr(SB)/8, $libc_issetugid_trampoline<>(SB) TEXT libc_kill_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kill(SB) - GLOBL ·libc_kill_trampoline_addr(SB), RODATA, $8 DATA ·libc_kill_trampoline_addr(SB)/8, $libc_kill_trampoline<>(SB) TEXT libc_kqueue_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_kqueue(SB) - GLOBL ·libc_kqueue_trampoline_addr(SB), RODATA, $8 DATA ·libc_kqueue_trampoline_addr(SB)/8, $libc_kqueue_trampoline<>(SB) TEXT libc_lchown_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lchown(SB) - GLOBL ·libc_lchown_trampoline_addr(SB), RODATA, $8 DATA ·libc_lchown_trampoline_addr(SB)/8, $libc_lchown_trampoline<>(SB) TEXT libc_link_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_link(SB) - GLOBL ·libc_link_trampoline_addr(SB), RODATA, $8 DATA ·libc_link_trampoline_addr(SB)/8, $libc_link_trampoline<>(SB) TEXT libc_linkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_linkat(SB) - GLOBL ·libc_linkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_linkat_trampoline_addr(SB)/8, $libc_linkat_trampoline<>(SB) TEXT libc_listen_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_listen(SB) - GLOBL ·libc_listen_trampoline_addr(SB), RODATA, $8 DATA ·libc_listen_trampoline_addr(SB)/8, $libc_listen_trampoline<>(SB) TEXT libc_lstat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lstat(SB) - GLOBL ·libc_lstat_trampoline_addr(SB), RODATA, $8 DATA ·libc_lstat_trampoline_addr(SB)/8, $libc_lstat_trampoline<>(SB) TEXT libc_mkdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkdir(SB) - GLOBL ·libc_mkdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkdir_trampoline_addr(SB)/8, $libc_mkdir_trampoline<>(SB) TEXT libc_mkdirat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkdirat(SB) - GLOBL ·libc_mkdirat_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkdirat_trampoline_addr(SB)/8, $libc_mkdirat_trampoline<>(SB) TEXT libc_mkfifo_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkfifo(SB) - GLOBL ·libc_mkfifo_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkfifo_trampoline_addr(SB)/8, $libc_mkfifo_trampoline<>(SB) TEXT libc_mkfifoat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mkfifoat(SB) - GLOBL ·libc_mkfifoat_trampoline_addr(SB), RODATA, $8 DATA ·libc_mkfifoat_trampoline_addr(SB)/8, $libc_mkfifoat_trampoline<>(SB) TEXT libc_mknod_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mknod(SB) - GLOBL ·libc_mknod_trampoline_addr(SB), RODATA, $8 DATA ·libc_mknod_trampoline_addr(SB)/8, $libc_mknod_trampoline<>(SB) TEXT libc_mknodat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mknodat(SB) - GLOBL ·libc_mknodat_trampoline_addr(SB), RODATA, $8 DATA ·libc_mknodat_trampoline_addr(SB)/8, $libc_mknodat_trampoline<>(SB) TEXT libc_nanosleep_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_nanosleep(SB) - GLOBL ·libc_nanosleep_trampoline_addr(SB), RODATA, $8 DATA ·libc_nanosleep_trampoline_addr(SB)/8, $libc_nanosleep_trampoline<>(SB) TEXT libc_open_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_open(SB) - GLOBL ·libc_open_trampoline_addr(SB), RODATA, $8 DATA ·libc_open_trampoline_addr(SB)/8, $libc_open_trampoline<>(SB) TEXT libc_openat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_openat(SB) - GLOBL ·libc_openat_trampoline_addr(SB), RODATA, $8 DATA ·libc_openat_trampoline_addr(SB)/8, $libc_openat_trampoline<>(SB) TEXT libc_pathconf_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pathconf(SB) - GLOBL ·libc_pathconf_trampoline_addr(SB), RODATA, $8 DATA ·libc_pathconf_trampoline_addr(SB)/8, $libc_pathconf_trampoline<>(SB) TEXT libc_pread_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pread(SB) - GLOBL ·libc_pread_trampoline_addr(SB), RODATA, $8 DATA ·libc_pread_trampoline_addr(SB)/8, $libc_pread_trampoline<>(SB) TEXT libc_pwrite_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_pwrite(SB) - GLOBL ·libc_pwrite_trampoline_addr(SB), RODATA, $8 DATA ·libc_pwrite_trampoline_addr(SB)/8, $libc_pwrite_trampoline<>(SB) TEXT libc_read_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_read(SB) - GLOBL ·libc_read_trampoline_addr(SB), RODATA, $8 DATA ·libc_read_trampoline_addr(SB)/8, $libc_read_trampoline<>(SB) TEXT libc_readlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_readlink(SB) - GLOBL ·libc_readlink_trampoline_addr(SB), RODATA, $8 DATA ·libc_readlink_trampoline_addr(SB)/8, $libc_readlink_trampoline<>(SB) TEXT libc_readlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_readlinkat(SB) - GLOBL ·libc_readlinkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_readlinkat_trampoline_addr(SB)/8, $libc_readlinkat_trampoline<>(SB) TEXT libc_rename_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_rename(SB) - GLOBL ·libc_rename_trampoline_addr(SB), RODATA, $8 DATA ·libc_rename_trampoline_addr(SB)/8, $libc_rename_trampoline<>(SB) TEXT libc_renameat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_renameat(SB) - GLOBL ·libc_renameat_trampoline_addr(SB), RODATA, $8 DATA ·libc_renameat_trampoline_addr(SB)/8, $libc_renameat_trampoline<>(SB) TEXT libc_revoke_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_revoke(SB) - GLOBL ·libc_revoke_trampoline_addr(SB), RODATA, $8 DATA ·libc_revoke_trampoline_addr(SB)/8, $libc_revoke_trampoline<>(SB) TEXT libc_rmdir_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_rmdir(SB) - GLOBL ·libc_rmdir_trampoline_addr(SB), RODATA, $8 DATA ·libc_rmdir_trampoline_addr(SB)/8, $libc_rmdir_trampoline<>(SB) TEXT libc_lseek_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_lseek(SB) - GLOBL ·libc_lseek_trampoline_addr(SB), RODATA, $8 DATA ·libc_lseek_trampoline_addr(SB)/8, $libc_lseek_trampoline<>(SB) TEXT libc_select_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_select(SB) - GLOBL ·libc_select_trampoline_addr(SB), RODATA, $8 DATA ·libc_select_trampoline_addr(SB)/8, $libc_select_trampoline<>(SB) TEXT libc_setegid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setegid(SB) - GLOBL ·libc_setegid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setegid_trampoline_addr(SB)/8, $libc_setegid_trampoline<>(SB) TEXT libc_seteuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_seteuid(SB) - GLOBL ·libc_seteuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_seteuid_trampoline_addr(SB)/8, $libc_seteuid_trampoline<>(SB) TEXT libc_setgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setgid(SB) - GLOBL ·libc_setgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setgid_trampoline_addr(SB)/8, $libc_setgid_trampoline<>(SB) TEXT libc_setlogin_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setlogin(SB) - GLOBL ·libc_setlogin_trampoline_addr(SB), RODATA, $8 DATA ·libc_setlogin_trampoline_addr(SB)/8, $libc_setlogin_trampoline<>(SB) TEXT libc_setpgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setpgid(SB) - GLOBL ·libc_setpgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setpgid_trampoline_addr(SB)/8, $libc_setpgid_trampoline<>(SB) TEXT libc_setpriority_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setpriority(SB) - GLOBL ·libc_setpriority_trampoline_addr(SB), RODATA, $8 DATA ·libc_setpriority_trampoline_addr(SB)/8, $libc_setpriority_trampoline<>(SB) TEXT libc_setregid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setregid(SB) - GLOBL ·libc_setregid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setregid_trampoline_addr(SB)/8, $libc_setregid_trampoline<>(SB) TEXT libc_setreuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setreuid(SB) - GLOBL ·libc_setreuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setreuid_trampoline_addr(SB)/8, $libc_setreuid_trampoline<>(SB) TEXT libc_setresgid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setresgid(SB) - GLOBL ·libc_setresgid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresgid_trampoline_addr(SB)/8, $libc_setresgid_trampoline<>(SB) TEXT libc_setresuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setresuid(SB) - GLOBL ·libc_setresuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setresuid_trampoline_addr(SB)/8, $libc_setresuid_trampoline<>(SB) -TEXT libc_setrlimit_trampoline<>(SB),NOSPLIT,$0-0 - JMP libc_setrlimit(SB) - -GLOBL ·libc_setrlimit_trampoline_addr(SB), RODATA, $8 -DATA ·libc_setrlimit_trampoline_addr(SB)/8, $libc_setrlimit_trampoline<>(SB) - TEXT libc_setrtable_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setrtable(SB) - GLOBL ·libc_setrtable_trampoline_addr(SB), RODATA, $8 DATA ·libc_setrtable_trampoline_addr(SB)/8, $libc_setrtable_trampoline<>(SB) TEXT libc_setsid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setsid(SB) - GLOBL ·libc_setsid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setsid_trampoline_addr(SB)/8, $libc_setsid_trampoline<>(SB) TEXT libc_settimeofday_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_settimeofday(SB) - GLOBL ·libc_settimeofday_trampoline_addr(SB), RODATA, $8 DATA ·libc_settimeofday_trampoline_addr(SB)/8, $libc_settimeofday_trampoline<>(SB) TEXT libc_setuid_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_setuid(SB) - GLOBL ·libc_setuid_trampoline_addr(SB), RODATA, $8 DATA ·libc_setuid_trampoline_addr(SB)/8, $libc_setuid_trampoline<>(SB) TEXT libc_stat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_stat(SB) - GLOBL ·libc_stat_trampoline_addr(SB), RODATA, $8 DATA ·libc_stat_trampoline_addr(SB)/8, $libc_stat_trampoline<>(SB) TEXT libc_statfs_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_statfs(SB) - GLOBL ·libc_statfs_trampoline_addr(SB), RODATA, $8 DATA ·libc_statfs_trampoline_addr(SB)/8, $libc_statfs_trampoline<>(SB) TEXT libc_symlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_symlink(SB) - GLOBL ·libc_symlink_trampoline_addr(SB), RODATA, $8 DATA ·libc_symlink_trampoline_addr(SB)/8, $libc_symlink_trampoline<>(SB) TEXT libc_symlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_symlinkat(SB) - GLOBL ·libc_symlinkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_symlinkat_trampoline_addr(SB)/8, $libc_symlinkat_trampoline<>(SB) TEXT libc_sync_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_sync(SB) - GLOBL ·libc_sync_trampoline_addr(SB), RODATA, $8 DATA ·libc_sync_trampoline_addr(SB)/8, $libc_sync_trampoline<>(SB) TEXT libc_truncate_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_truncate(SB) - GLOBL ·libc_truncate_trampoline_addr(SB), RODATA, $8 DATA ·libc_truncate_trampoline_addr(SB)/8, $libc_truncate_trampoline<>(SB) TEXT libc_umask_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_umask(SB) - GLOBL ·libc_umask_trampoline_addr(SB), RODATA, $8 DATA ·libc_umask_trampoline_addr(SB)/8, $libc_umask_trampoline<>(SB) TEXT libc_unlink_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unlink(SB) - GLOBL ·libc_unlink_trampoline_addr(SB), RODATA, $8 DATA ·libc_unlink_trampoline_addr(SB)/8, $libc_unlink_trampoline<>(SB) TEXT libc_unlinkat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unlinkat(SB) - GLOBL ·libc_unlinkat_trampoline_addr(SB), RODATA, $8 DATA ·libc_unlinkat_trampoline_addr(SB)/8, $libc_unlinkat_trampoline<>(SB) TEXT libc_unmount_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_unmount(SB) - GLOBL ·libc_unmount_trampoline_addr(SB), RODATA, $8 DATA ·libc_unmount_trampoline_addr(SB)/8, $libc_unmount_trampoline<>(SB) TEXT libc_write_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_write(SB) - GLOBL ·libc_write_trampoline_addr(SB), RODATA, $8 DATA ·libc_write_trampoline_addr(SB)/8, $libc_write_trampoline<>(SB) TEXT libc_mmap_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_mmap(SB) - GLOBL ·libc_mmap_trampoline_addr(SB), RODATA, $8 DATA ·libc_mmap_trampoline_addr(SB)/8, $libc_mmap_trampoline<>(SB) TEXT libc_munmap_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_munmap(SB) - GLOBL ·libc_munmap_trampoline_addr(SB), RODATA, $8 DATA ·libc_munmap_trampoline_addr(SB)/8, $libc_munmap_trampoline<>(SB) TEXT libc_utimensat_trampoline<>(SB),NOSPLIT,$0-0 JMP libc_utimensat(SB) - GLOBL ·libc_utimensat_trampoline_addr(SB), RODATA, $8 DATA ·libc_utimensat_trampoline_addr(SB)/8, $libc_utimensat_trampoline<>(SB) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_solaris_amd64.go b/vendor/golang.org/x/sys/unix/zsyscall_solaris_amd64.go index 91f5a2b..609d1c5 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_solaris_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_solaris_amd64.go @@ -38,6 +38,7 @@ import ( //go:cgo_import_dynamic libc_chmod chmod "libc.so" //go:cgo_import_dynamic libc_chown chown "libc.so" //go:cgo_import_dynamic libc_chroot chroot "libc.so" +//go:cgo_import_dynamic libc_clockgettime clockgettime "libc.so" //go:cgo_import_dynamic libc_close close "libc.so" //go:cgo_import_dynamic libc_creat creat "libc.so" //go:cgo_import_dynamic libc_dup dup "libc.so" @@ -109,7 +110,6 @@ import ( //go:cgo_import_dynamic libc_setpriority setpriority "libc.so" //go:cgo_import_dynamic libc_setregid setregid "libc.so" //go:cgo_import_dynamic libc_setreuid setreuid "libc.so" -//go:cgo_import_dynamic libc_setrlimit setrlimit "libc.so" //go:cgo_import_dynamic libc_setsid setsid "libc.so" //go:cgo_import_dynamic libc_setuid setuid "libc.so" //go:cgo_import_dynamic libc_shutdown shutdown "libsocket.so" @@ -177,6 +177,7 @@ import ( //go:linkname procChmod libc_chmod //go:linkname procChown libc_chown //go:linkname procChroot libc_chroot +//go:linkname procClockGettime libc_clockgettime //go:linkname procClose libc_close //go:linkname procCreat libc_creat //go:linkname procDup libc_dup @@ -248,7 +249,6 @@ import ( //go:linkname procSetpriority libc_setpriority //go:linkname procSetregid libc_setregid //go:linkname procSetreuid libc_setreuid -//go:linkname procSetrlimit libc_setrlimit //go:linkname procSetsid libc_setsid //go:linkname procSetuid libc_setuid //go:linkname procshutdown libc_shutdown @@ -317,6 +317,7 @@ var ( procChmod, procChown, procChroot, + procClockGettime, procClose, procCreat, procDup, @@ -388,7 +389,6 @@ var ( procSetpriority, procSetregid, procSetreuid, - procSetrlimit, procSetsid, procSetuid, procshutdown, @@ -643,7 +643,18 @@ func __minor(version int, dev uint64) (val uint) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctlRet(fd int, req uint, arg uintptr) (ret int, err error) { +func ioctlRet(fd int, req int, arg uintptr) (ret int, err error) { + r0, _, e1 := sysvicall6(uintptr(unsafe.Pointer(&procioctl)), 3, uintptr(fd), uintptr(req), uintptr(arg), 0, 0, 0) + ret = int(r0) + if e1 != 0 { + err = e1 + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func ioctlPtrRet(fd int, req int, arg unsafe.Pointer) (ret int, err error) { r0, _, e1 := sysvicall6(uintptr(unsafe.Pointer(&procioctl)), 3, uintptr(fd), uintptr(req), uintptr(arg), 0, 0, 0) ret = int(r0) if e1 != 0 { @@ -750,6 +761,16 @@ func Chroot(path string) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT +func ClockGettime(clockid int32, time *Timespec) (err error) { + _, _, e1 := sysvicall6(uintptr(unsafe.Pointer(&procClockGettime)), 2, uintptr(clockid), uintptr(unsafe.Pointer(time)), 0, 0, 0, 0) + if e1 != 0 { + err = e1 + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + func Close(fd int) (err error) { _, _, e1 := sysvicall6(uintptr(unsafe.Pointer(&procClose)), 1, uintptr(fd), 0, 0, 0, 0, 0) if e1 != 0 { @@ -1626,16 +1647,6 @@ func Setreuid(ruid int, euid int) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func Setrlimit(which int, lim *Rlimit) (err error) { - _, _, e1 := rawSysvicall6(uintptr(unsafe.Pointer(&procSetrlimit)), 2, uintptr(which), uintptr(unsafe.Pointer(lim)), 0, 0, 0, 0) - if e1 != 0 { - err = e1 - } - return -} - -// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT - func Setsid() (pid int, err error) { r0, _, e1 := rawSysvicall6(uintptr(unsafe.Pointer(&procSetsid)), 0, 0, 0, 0, 0, 0, 0) pid = int(r0) diff --git a/vendor/golang.org/x/sys/unix/zsyscall_zos_s390x.go b/vendor/golang.org/x/sys/unix/zsyscall_zos_s390x.go index f207945..c316817 100644 --- a/vendor/golang.org/x/sys/unix/zsyscall_zos_s390x.go +++ b/vendor/golang.org/x/sys/unix/zsyscall_zos_s390x.go @@ -257,7 +257,17 @@ func munmap(addr uintptr, length uintptr) (err error) { // THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT -func ioctl(fd int, req uint, arg uintptr) (err error) { +func ioctl(fd int, req int, arg uintptr) (err error) { + _, _, e1 := syscall_syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) + if e1 != 0 { + err = errnoErr(e1) + } + return +} + +// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT + +func ioctlPtr(fd int, req int, arg unsafe.Pointer) (err error) { _, _, e1 := syscall_syscall(SYS_IOCTL, uintptr(fd), uintptr(req), uintptr(arg)) if e1 != 0 { err = errnoErr(e1) diff --git a/vendor/golang.org/x/sys/unix/zsysctl_openbsd_386.go b/vendor/golang.org/x/sys/unix/zsysctl_openbsd_386.go index 9e9d0b2..55e0484 100644 --- a/vendor/golang.org/x/sys/unix/zsysctl_openbsd_386.go +++ b/vendor/golang.org/x/sys/unix/zsysctl_openbsd_386.go @@ -17,6 +17,7 @@ var sysctlMib = []mibentry{ {"ddb.max_line", []_C_int{9, 3}}, {"ddb.max_width", []_C_int{9, 2}}, {"ddb.panic", []_C_int{9, 5}}, + {"ddb.profile", []_C_int{9, 9}}, {"ddb.radix", []_C_int{9, 1}}, {"ddb.tab_stop_width", []_C_int{9, 4}}, {"ddb.trigger", []_C_int{9, 8}}, @@ -33,29 +34,37 @@ var sysctlMib = []mibentry{ {"hw.ncpufound", []_C_int{6, 21}}, {"hw.ncpuonline", []_C_int{6, 25}}, {"hw.pagesize", []_C_int{6, 7}}, + {"hw.perfpolicy", []_C_int{6, 23}}, {"hw.physmem", []_C_int{6, 19}}, + {"hw.power", []_C_int{6, 26}}, {"hw.product", []_C_int{6, 15}}, {"hw.serialno", []_C_int{6, 17}}, {"hw.setperf", []_C_int{6, 13}}, + {"hw.smt", []_C_int{6, 24}}, {"hw.usermem", []_C_int{6, 20}}, {"hw.uuid", []_C_int{6, 18}}, {"hw.vendor", []_C_int{6, 14}}, {"hw.version", []_C_int{6, 16}}, - {"kern.arandom", []_C_int{1, 37}}, + {"kern.allowdt", []_C_int{1, 65}}, + {"kern.allowkmem", []_C_int{1, 52}}, {"kern.argmax", []_C_int{1, 8}}, + {"kern.audio", []_C_int{1, 84}}, {"kern.boottime", []_C_int{1, 21}}, {"kern.bufcachepercent", []_C_int{1, 72}}, {"kern.ccpu", []_C_int{1, 45}}, {"kern.clockrate", []_C_int{1, 12}}, + {"kern.consbuf", []_C_int{1, 83}}, + {"kern.consbufsize", []_C_int{1, 82}}, {"kern.consdev", []_C_int{1, 75}}, {"kern.cp_time", []_C_int{1, 40}}, {"kern.cp_time2", []_C_int{1, 71}}, - {"kern.cryptodevallowsoft", []_C_int{1, 53}}, + {"kern.cpustats", []_C_int{1, 85}}, {"kern.domainname", []_C_int{1, 22}}, {"kern.file", []_C_int{1, 73}}, {"kern.forkstat", []_C_int{1, 42}}, {"kern.fscale", []_C_int{1, 46}}, {"kern.fsync", []_C_int{1, 33}}, + {"kern.global_ptrace", []_C_int{1, 81}}, {"kern.hostid", []_C_int{1, 11}}, {"kern.hostname", []_C_int{1, 10}}, {"kern.intrcnt.nintrcnt", []_C_int{1, 63, 1}}, @@ -78,17 +87,16 @@ var sysctlMib = []mibentry{ {"kern.ngroups", []_C_int{1, 18}}, {"kern.nosuidcoredump", []_C_int{1, 32}}, {"kern.nprocs", []_C_int{1, 47}}, - {"kern.nselcoll", []_C_int{1, 43}}, {"kern.nthreads", []_C_int{1, 26}}, {"kern.numvnodes", []_C_int{1, 58}}, {"kern.osrelease", []_C_int{1, 2}}, {"kern.osrevision", []_C_int{1, 3}}, {"kern.ostype", []_C_int{1, 1}}, {"kern.osversion", []_C_int{1, 27}}, + {"kern.pfstatus", []_C_int{1, 86}}, {"kern.pool_debug", []_C_int{1, 77}}, {"kern.posix1version", []_C_int{1, 17}}, {"kern.proc", []_C_int{1, 66}}, - {"kern.random", []_C_int{1, 31}}, {"kern.rawpartition", []_C_int{1, 24}}, {"kern.saved_ids", []_C_int{1, 20}}, {"kern.securelevel", []_C_int{1, 9}}, @@ -106,21 +114,20 @@ var sysctlMib = []mibentry{ {"kern.timecounter.hardware", []_C_int{1, 69, 3}}, {"kern.timecounter.tick", []_C_int{1, 69, 1}}, {"kern.timecounter.timestepwarnings", []_C_int{1, 69, 2}}, - {"kern.tty.maxptys", []_C_int{1, 44, 6}}, - {"kern.tty.nptys", []_C_int{1, 44, 7}}, + {"kern.timeout_stats", []_C_int{1, 87}}, {"kern.tty.tk_cancc", []_C_int{1, 44, 4}}, {"kern.tty.tk_nin", []_C_int{1, 44, 1}}, {"kern.tty.tk_nout", []_C_int{1, 44, 2}}, {"kern.tty.tk_rawcc", []_C_int{1, 44, 3}}, {"kern.tty.ttyinfo", []_C_int{1, 44, 5}}, {"kern.ttycount", []_C_int{1, 57}}, - {"kern.userasymcrypto", []_C_int{1, 60}}, - {"kern.usercrypto", []_C_int{1, 52}}, - {"kern.usermount", []_C_int{1, 30}}, + {"kern.utc_offset", []_C_int{1, 88}}, {"kern.version", []_C_int{1, 4}}, - {"kern.vnode", []_C_int{1, 13}}, + {"kern.video", []_C_int{1, 89}}, {"kern.watchdog.auto", []_C_int{1, 64, 2}}, {"kern.watchdog.period", []_C_int{1, 64, 1}}, + {"kern.witnesswatch", []_C_int{1, 53}}, + {"kern.wxabort", []_C_int{1, 74}}, {"net.bpf.bufsize", []_C_int{4, 31, 1}}, {"net.bpf.maxbufsize", []_C_int{4, 31, 2}}, {"net.inet.ah.enable", []_C_int{4, 2, 51, 1}}, @@ -148,7 +155,9 @@ var sysctlMib = []mibentry{ {"net.inet.icmp.stats", []_C_int{4, 2, 1, 7}}, {"net.inet.icmp.tstamprepl", []_C_int{4, 2, 1, 6}}, {"net.inet.igmp.stats", []_C_int{4, 2, 2, 1}}, + {"net.inet.ip.arpdown", []_C_int{4, 2, 0, 40}}, {"net.inet.ip.arpqueued", []_C_int{4, 2, 0, 36}}, + {"net.inet.ip.arptimeout", []_C_int{4, 2, 0, 39}}, {"net.inet.ip.encdebug", []_C_int{4, 2, 0, 12}}, {"net.inet.ip.forwarding", []_C_int{4, 2, 0, 1}}, {"net.inet.ip.ifq.congestion", []_C_int{4, 2, 0, 30, 4}}, @@ -157,8 +166,10 @@ var sysctlMib = []mibentry{ {"net.inet.ip.ifq.maxlen", []_C_int{4, 2, 0, 30, 2}}, {"net.inet.ip.maxqueue", []_C_int{4, 2, 0, 11}}, {"net.inet.ip.mforwarding", []_C_int{4, 2, 0, 31}}, + {"net.inet.ip.mrtmfc", []_C_int{4, 2, 0, 37}}, {"net.inet.ip.mrtproto", []_C_int{4, 2, 0, 34}}, {"net.inet.ip.mrtstats", []_C_int{4, 2, 0, 35}}, + {"net.inet.ip.mrtvif", []_C_int{4, 2, 0, 38}}, {"net.inet.ip.mtu", []_C_int{4, 2, 0, 4}}, {"net.inet.ip.mtudisc", []_C_int{4, 2, 0, 27}}, {"net.inet.ip.mtudisctimeout", []_C_int{4, 2, 0, 28}}, @@ -175,9 +186,7 @@ var sysctlMib = []mibentry{ {"net.inet.ipcomp.stats", []_C_int{4, 2, 108, 2}}, {"net.inet.ipip.allow", []_C_int{4, 2, 4, 1}}, {"net.inet.ipip.stats", []_C_int{4, 2, 4, 2}}, - {"net.inet.mobileip.allow", []_C_int{4, 2, 55, 1}}, {"net.inet.pfsync.stats", []_C_int{4, 2, 240, 1}}, - {"net.inet.pim.stats", []_C_int{4, 2, 103, 1}}, {"net.inet.tcp.ackonpush", []_C_int{4, 2, 6, 13}}, {"net.inet.tcp.always_keepalive", []_C_int{4, 2, 6, 22}}, {"net.inet.tcp.baddynamic", []_C_int{4, 2, 6, 6}}, @@ -191,6 +200,7 @@ var sysctlMib = []mibentry{ {"net.inet.tcp.reasslimit", []_C_int{4, 2, 6, 18}}, {"net.inet.tcp.rfc1323", []_C_int{4, 2, 6, 1}}, {"net.inet.tcp.rfc3390", []_C_int{4, 2, 6, 17}}, + {"net.inet.tcp.rootonly", []_C_int{4, 2, 6, 24}}, {"net.inet.tcp.rstppslimit", []_C_int{4, 2, 6, 12}}, {"net.inet.tcp.sack", []_C_int{4, 2, 6, 10}}, {"net.inet.tcp.sackholelimit", []_C_int{4, 2, 6, 20}}, @@ -198,9 +208,12 @@ var sysctlMib = []mibentry{ {"net.inet.tcp.stats", []_C_int{4, 2, 6, 21}}, {"net.inet.tcp.synbucketlimit", []_C_int{4, 2, 6, 16}}, {"net.inet.tcp.syncachelimit", []_C_int{4, 2, 6, 15}}, + {"net.inet.tcp.synhashsize", []_C_int{4, 2, 6, 25}}, + {"net.inet.tcp.synuselimit", []_C_int{4, 2, 6, 23}}, {"net.inet.udp.baddynamic", []_C_int{4, 2, 17, 2}}, {"net.inet.udp.checksum", []_C_int{4, 2, 17, 1}}, {"net.inet.udp.recvspace", []_C_int{4, 2, 17, 3}}, + {"net.inet.udp.rootonly", []_C_int{4, 2, 17, 6}}, {"net.inet.udp.sendspace", []_C_int{4, 2, 17, 4}}, {"net.inet.udp.stats", []_C_int{4, 2, 17, 5}}, {"net.inet6.divert.recvspace", []_C_int{4, 24, 86, 1}}, @@ -213,13 +226,8 @@ var sysctlMib = []mibentry{ {"net.inet6.icmp6.nd6_delay", []_C_int{4, 24, 30, 8}}, {"net.inet6.icmp6.nd6_maxnudhint", []_C_int{4, 24, 30, 15}}, {"net.inet6.icmp6.nd6_mmaxtries", []_C_int{4, 24, 30, 10}}, - {"net.inet6.icmp6.nd6_prune", []_C_int{4, 24, 30, 6}}, {"net.inet6.icmp6.nd6_umaxtries", []_C_int{4, 24, 30, 9}}, - {"net.inet6.icmp6.nd6_useloopback", []_C_int{4, 24, 30, 11}}, - {"net.inet6.icmp6.nodeinfo", []_C_int{4, 24, 30, 13}}, - {"net.inet6.icmp6.rediraccept", []_C_int{4, 24, 30, 2}}, {"net.inet6.icmp6.redirtimeout", []_C_int{4, 24, 30, 3}}, - {"net.inet6.ip6.accept_rtadv", []_C_int{4, 24, 17, 12}}, {"net.inet6.ip6.auto_flowlabel", []_C_int{4, 24, 17, 17}}, {"net.inet6.ip6.dad_count", []_C_int{4, 24, 17, 16}}, {"net.inet6.ip6.dad_pending", []_C_int{4, 24, 17, 49}}, @@ -232,20 +240,19 @@ var sysctlMib = []mibentry{ {"net.inet6.ip6.maxdynroutes", []_C_int{4, 24, 17, 48}}, {"net.inet6.ip6.maxfragpackets", []_C_int{4, 24, 17, 9}}, {"net.inet6.ip6.maxfrags", []_C_int{4, 24, 17, 41}}, - {"net.inet6.ip6.maxifdefrouters", []_C_int{4, 24, 17, 47}}, - {"net.inet6.ip6.maxifprefixes", []_C_int{4, 24, 17, 46}}, {"net.inet6.ip6.mforwarding", []_C_int{4, 24, 17, 42}}, + {"net.inet6.ip6.mrtmfc", []_C_int{4, 24, 17, 53}}, + {"net.inet6.ip6.mrtmif", []_C_int{4, 24, 17, 52}}, {"net.inet6.ip6.mrtproto", []_C_int{4, 24, 17, 8}}, {"net.inet6.ip6.mtudisctimeout", []_C_int{4, 24, 17, 50}}, {"net.inet6.ip6.multicast_mtudisc", []_C_int{4, 24, 17, 44}}, {"net.inet6.ip6.multipath", []_C_int{4, 24, 17, 43}}, {"net.inet6.ip6.neighborgcthresh", []_C_int{4, 24, 17, 45}}, {"net.inet6.ip6.redirect", []_C_int{4, 24, 17, 2}}, - {"net.inet6.ip6.rr_prune", []_C_int{4, 24, 17, 22}}, + {"net.inet6.ip6.soiikey", []_C_int{4, 24, 17, 54}}, {"net.inet6.ip6.sourcecheck", []_C_int{4, 24, 17, 10}}, {"net.inet6.ip6.sourcecheck_logint", []_C_int{4, 24, 17, 11}}, {"net.inet6.ip6.use_deprecated", []_C_int{4, 24, 17, 21}}, - {"net.inet6.ip6.v6only", []_C_int{4, 24, 17, 24}}, {"net.key.sadb_dump", []_C_int{4, 30, 1}}, {"net.key.spd_dump", []_C_int{4, 30, 2}}, {"net.mpls.ifq.congestion", []_C_int{4, 33, 3, 4}}, @@ -254,12 +261,12 @@ var sysctlMib = []mibentry{ {"net.mpls.ifq.maxlen", []_C_int{4, 33, 3, 2}}, {"net.mpls.mapttl_ip", []_C_int{4, 33, 5}}, {"net.mpls.mapttl_ip6", []_C_int{4, 33, 6}}, - {"net.mpls.maxloop_inkernel", []_C_int{4, 33, 4}}, {"net.mpls.ttl", []_C_int{4, 33, 2}}, {"net.pflow.stats", []_C_int{4, 34, 1}}, {"net.pipex.enable", []_C_int{4, 35, 1}}, {"vm.anonmin", []_C_int{2, 7}}, {"vm.loadavg", []_C_int{2, 2}}, + {"vm.malloc_conf", []_C_int{2, 12}}, {"vm.maxslp", []_C_int{2, 10}}, {"vm.nkmempages", []_C_int{2, 6}}, {"vm.psstrings", []_C_int{2, 3}}, diff --git a/vendor/golang.org/x/sys/unix/zsysctl_openbsd_amd64.go b/vendor/golang.org/x/sys/unix/zsysctl_openbsd_amd64.go index adecd09..d2243cf 100644 --- a/vendor/golang.org/x/sys/unix/zsysctl_openbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/zsysctl_openbsd_amd64.go @@ -36,23 +36,29 @@ var sysctlMib = []mibentry{ {"hw.pagesize", []_C_int{6, 7}}, {"hw.perfpolicy", []_C_int{6, 23}}, {"hw.physmem", []_C_int{6, 19}}, + {"hw.power", []_C_int{6, 26}}, {"hw.product", []_C_int{6, 15}}, {"hw.serialno", []_C_int{6, 17}}, {"hw.setperf", []_C_int{6, 13}}, + {"hw.smt", []_C_int{6, 24}}, {"hw.usermem", []_C_int{6, 20}}, {"hw.uuid", []_C_int{6, 18}}, {"hw.vendor", []_C_int{6, 14}}, {"hw.version", []_C_int{6, 16}}, + {"kern.allowdt", []_C_int{1, 65}}, {"kern.allowkmem", []_C_int{1, 52}}, {"kern.argmax", []_C_int{1, 8}}, + {"kern.audio", []_C_int{1, 84}}, {"kern.boottime", []_C_int{1, 21}}, {"kern.bufcachepercent", []_C_int{1, 72}}, {"kern.ccpu", []_C_int{1, 45}}, {"kern.clockrate", []_C_int{1, 12}}, + {"kern.consbuf", []_C_int{1, 83}}, + {"kern.consbufsize", []_C_int{1, 82}}, {"kern.consdev", []_C_int{1, 75}}, {"kern.cp_time", []_C_int{1, 40}}, {"kern.cp_time2", []_C_int{1, 71}}, - {"kern.dnsjackport", []_C_int{1, 13}}, + {"kern.cpustats", []_C_int{1, 85}}, {"kern.domainname", []_C_int{1, 22}}, {"kern.file", []_C_int{1, 73}}, {"kern.forkstat", []_C_int{1, 42}}, @@ -81,13 +87,13 @@ var sysctlMib = []mibentry{ {"kern.ngroups", []_C_int{1, 18}}, {"kern.nosuidcoredump", []_C_int{1, 32}}, {"kern.nprocs", []_C_int{1, 47}}, - {"kern.nselcoll", []_C_int{1, 43}}, {"kern.nthreads", []_C_int{1, 26}}, {"kern.numvnodes", []_C_int{1, 58}}, {"kern.osrelease", []_C_int{1, 2}}, {"kern.osrevision", []_C_int{1, 3}}, {"kern.ostype", []_C_int{1, 1}}, {"kern.osversion", []_C_int{1, 27}}, + {"kern.pfstatus", []_C_int{1, 86}}, {"kern.pool_debug", []_C_int{1, 77}}, {"kern.posix1version", []_C_int{1, 17}}, {"kern.proc", []_C_int{1, 66}}, @@ -108,15 +114,19 @@ var sysctlMib = []mibentry{ {"kern.timecounter.hardware", []_C_int{1, 69, 3}}, {"kern.timecounter.tick", []_C_int{1, 69, 1}}, {"kern.timecounter.timestepwarnings", []_C_int{1, 69, 2}}, + {"kern.timeout_stats", []_C_int{1, 87}}, {"kern.tty.tk_cancc", []_C_int{1, 44, 4}}, {"kern.tty.tk_nin", []_C_int{1, 44, 1}}, {"kern.tty.tk_nout", []_C_int{1, 44, 2}}, {"kern.tty.tk_rawcc", []_C_int{1, 44, 3}}, {"kern.tty.ttyinfo", []_C_int{1, 44, 5}}, {"kern.ttycount", []_C_int{1, 57}}, + {"kern.utc_offset", []_C_int{1, 88}}, {"kern.version", []_C_int{1, 4}}, + {"kern.video", []_C_int{1, 89}}, {"kern.watchdog.auto", []_C_int{1, 64, 2}}, {"kern.watchdog.period", []_C_int{1, 64, 1}}, + {"kern.witnesswatch", []_C_int{1, 53}}, {"kern.wxabort", []_C_int{1, 74}}, {"net.bpf.bufsize", []_C_int{4, 31, 1}}, {"net.bpf.maxbufsize", []_C_int{4, 31, 2}}, @@ -176,7 +186,6 @@ var sysctlMib = []mibentry{ {"net.inet.ipcomp.stats", []_C_int{4, 2, 108, 2}}, {"net.inet.ipip.allow", []_C_int{4, 2, 4, 1}}, {"net.inet.ipip.stats", []_C_int{4, 2, 4, 2}}, - {"net.inet.mobileip.allow", []_C_int{4, 2, 55, 1}}, {"net.inet.pfsync.stats", []_C_int{4, 2, 240, 1}}, {"net.inet.tcp.ackonpush", []_C_int{4, 2, 6, 13}}, {"net.inet.tcp.always_keepalive", []_C_int{4, 2, 6, 22}}, @@ -252,12 +261,12 @@ var sysctlMib = []mibentry{ {"net.mpls.ifq.maxlen", []_C_int{4, 33, 3, 2}}, {"net.mpls.mapttl_ip", []_C_int{4, 33, 5}}, {"net.mpls.mapttl_ip6", []_C_int{4, 33, 6}}, - {"net.mpls.maxloop_inkernel", []_C_int{4, 33, 4}}, {"net.mpls.ttl", []_C_int{4, 33, 2}}, {"net.pflow.stats", []_C_int{4, 34, 1}}, {"net.pipex.enable", []_C_int{4, 35, 1}}, {"vm.anonmin", []_C_int{2, 7}}, {"vm.loadavg", []_C_int{2, 2}}, + {"vm.malloc_conf", []_C_int{2, 12}}, {"vm.maxslp", []_C_int{2, 10}}, {"vm.nkmempages", []_C_int{2, 6}}, {"vm.psstrings", []_C_int{2, 3}}, diff --git a/vendor/golang.org/x/sys/unix/zsysctl_openbsd_arm.go b/vendor/golang.org/x/sys/unix/zsysctl_openbsd_arm.go index 8ea52a4..82dc51b 100644 --- a/vendor/golang.org/x/sys/unix/zsysctl_openbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/zsysctl_openbsd_arm.go @@ -17,6 +17,7 @@ var sysctlMib = []mibentry{ {"ddb.max_line", []_C_int{9, 3}}, {"ddb.max_width", []_C_int{9, 2}}, {"ddb.panic", []_C_int{9, 5}}, + {"ddb.profile", []_C_int{9, 9}}, {"ddb.radix", []_C_int{9, 1}}, {"ddb.tab_stop_width", []_C_int{9, 4}}, {"ddb.trigger", []_C_int{9, 8}}, @@ -33,29 +34,37 @@ var sysctlMib = []mibentry{ {"hw.ncpufound", []_C_int{6, 21}}, {"hw.ncpuonline", []_C_int{6, 25}}, {"hw.pagesize", []_C_int{6, 7}}, + {"hw.perfpolicy", []_C_int{6, 23}}, {"hw.physmem", []_C_int{6, 19}}, + {"hw.power", []_C_int{6, 26}}, {"hw.product", []_C_int{6, 15}}, {"hw.serialno", []_C_int{6, 17}}, {"hw.setperf", []_C_int{6, 13}}, + {"hw.smt", []_C_int{6, 24}}, {"hw.usermem", []_C_int{6, 20}}, {"hw.uuid", []_C_int{6, 18}}, {"hw.vendor", []_C_int{6, 14}}, {"hw.version", []_C_int{6, 16}}, - {"kern.arandom", []_C_int{1, 37}}, + {"kern.allowdt", []_C_int{1, 65}}, + {"kern.allowkmem", []_C_int{1, 52}}, {"kern.argmax", []_C_int{1, 8}}, + {"kern.audio", []_C_int{1, 84}}, {"kern.boottime", []_C_int{1, 21}}, {"kern.bufcachepercent", []_C_int{1, 72}}, {"kern.ccpu", []_C_int{1, 45}}, {"kern.clockrate", []_C_int{1, 12}}, + {"kern.consbuf", []_C_int{1, 83}}, + {"kern.consbufsize", []_C_int{1, 82}}, {"kern.consdev", []_C_int{1, 75}}, {"kern.cp_time", []_C_int{1, 40}}, {"kern.cp_time2", []_C_int{1, 71}}, - {"kern.cryptodevallowsoft", []_C_int{1, 53}}, + {"kern.cpustats", []_C_int{1, 85}}, {"kern.domainname", []_C_int{1, 22}}, {"kern.file", []_C_int{1, 73}}, {"kern.forkstat", []_C_int{1, 42}}, {"kern.fscale", []_C_int{1, 46}}, {"kern.fsync", []_C_int{1, 33}}, + {"kern.global_ptrace", []_C_int{1, 81}}, {"kern.hostid", []_C_int{1, 11}}, {"kern.hostname", []_C_int{1, 10}}, {"kern.intrcnt.nintrcnt", []_C_int{1, 63, 1}}, @@ -78,17 +87,16 @@ var sysctlMib = []mibentry{ {"kern.ngroups", []_C_int{1, 18}}, {"kern.nosuidcoredump", []_C_int{1, 32}}, {"kern.nprocs", []_C_int{1, 47}}, - {"kern.nselcoll", []_C_int{1, 43}}, {"kern.nthreads", []_C_int{1, 26}}, {"kern.numvnodes", []_C_int{1, 58}}, {"kern.osrelease", []_C_int{1, 2}}, {"kern.osrevision", []_C_int{1, 3}}, {"kern.ostype", []_C_int{1, 1}}, {"kern.osversion", []_C_int{1, 27}}, + {"kern.pfstatus", []_C_int{1, 86}}, {"kern.pool_debug", []_C_int{1, 77}}, {"kern.posix1version", []_C_int{1, 17}}, {"kern.proc", []_C_int{1, 66}}, - {"kern.random", []_C_int{1, 31}}, {"kern.rawpartition", []_C_int{1, 24}}, {"kern.saved_ids", []_C_int{1, 20}}, {"kern.securelevel", []_C_int{1, 9}}, @@ -106,21 +114,20 @@ var sysctlMib = []mibentry{ {"kern.timecounter.hardware", []_C_int{1, 69, 3}}, {"kern.timecounter.tick", []_C_int{1, 69, 1}}, {"kern.timecounter.timestepwarnings", []_C_int{1, 69, 2}}, - {"kern.tty.maxptys", []_C_int{1, 44, 6}}, - {"kern.tty.nptys", []_C_int{1, 44, 7}}, + {"kern.timeout_stats", []_C_int{1, 87}}, {"kern.tty.tk_cancc", []_C_int{1, 44, 4}}, {"kern.tty.tk_nin", []_C_int{1, 44, 1}}, {"kern.tty.tk_nout", []_C_int{1, 44, 2}}, {"kern.tty.tk_rawcc", []_C_int{1, 44, 3}}, {"kern.tty.ttyinfo", []_C_int{1, 44, 5}}, {"kern.ttycount", []_C_int{1, 57}}, - {"kern.userasymcrypto", []_C_int{1, 60}}, - {"kern.usercrypto", []_C_int{1, 52}}, - {"kern.usermount", []_C_int{1, 30}}, + {"kern.utc_offset", []_C_int{1, 88}}, {"kern.version", []_C_int{1, 4}}, - {"kern.vnode", []_C_int{1, 13}}, + {"kern.video", []_C_int{1, 89}}, {"kern.watchdog.auto", []_C_int{1, 64, 2}}, {"kern.watchdog.period", []_C_int{1, 64, 1}}, + {"kern.witnesswatch", []_C_int{1, 53}}, + {"kern.wxabort", []_C_int{1, 74}}, {"net.bpf.bufsize", []_C_int{4, 31, 1}}, {"net.bpf.maxbufsize", []_C_int{4, 31, 2}}, {"net.inet.ah.enable", []_C_int{4, 2, 51, 1}}, @@ -148,7 +155,9 @@ var sysctlMib = []mibentry{ {"net.inet.icmp.stats", []_C_int{4, 2, 1, 7}}, {"net.inet.icmp.tstamprepl", []_C_int{4, 2, 1, 6}}, {"net.inet.igmp.stats", []_C_int{4, 2, 2, 1}}, + {"net.inet.ip.arpdown", []_C_int{4, 2, 0, 40}}, {"net.inet.ip.arpqueued", []_C_int{4, 2, 0, 36}}, + {"net.inet.ip.arptimeout", []_C_int{4, 2, 0, 39}}, {"net.inet.ip.encdebug", []_C_int{4, 2, 0, 12}}, {"net.inet.ip.forwarding", []_C_int{4, 2, 0, 1}}, {"net.inet.ip.ifq.congestion", []_C_int{4, 2, 0, 30, 4}}, @@ -157,8 +166,10 @@ var sysctlMib = []mibentry{ {"net.inet.ip.ifq.maxlen", []_C_int{4, 2, 0, 30, 2}}, {"net.inet.ip.maxqueue", []_C_int{4, 2, 0, 11}}, {"net.inet.ip.mforwarding", []_C_int{4, 2, 0, 31}}, + {"net.inet.ip.mrtmfc", []_C_int{4, 2, 0, 37}}, {"net.inet.ip.mrtproto", []_C_int{4, 2, 0, 34}}, {"net.inet.ip.mrtstats", []_C_int{4, 2, 0, 35}}, + {"net.inet.ip.mrtvif", []_C_int{4, 2, 0, 38}}, {"net.inet.ip.mtu", []_C_int{4, 2, 0, 4}}, {"net.inet.ip.mtudisc", []_C_int{4, 2, 0, 27}}, {"net.inet.ip.mtudisctimeout", []_C_int{4, 2, 0, 28}}, @@ -175,9 +186,7 @@ var sysctlMib = []mibentry{ {"net.inet.ipcomp.stats", []_C_int{4, 2, 108, 2}}, {"net.inet.ipip.allow", []_C_int{4, 2, 4, 1}}, {"net.inet.ipip.stats", []_C_int{4, 2, 4, 2}}, - {"net.inet.mobileip.allow", []_C_int{4, 2, 55, 1}}, {"net.inet.pfsync.stats", []_C_int{4, 2, 240, 1}}, - {"net.inet.pim.stats", []_C_int{4, 2, 103, 1}}, {"net.inet.tcp.ackonpush", []_C_int{4, 2, 6, 13}}, {"net.inet.tcp.always_keepalive", []_C_int{4, 2, 6, 22}}, {"net.inet.tcp.baddynamic", []_C_int{4, 2, 6, 6}}, @@ -191,6 +200,7 @@ var sysctlMib = []mibentry{ {"net.inet.tcp.reasslimit", []_C_int{4, 2, 6, 18}}, {"net.inet.tcp.rfc1323", []_C_int{4, 2, 6, 1}}, {"net.inet.tcp.rfc3390", []_C_int{4, 2, 6, 17}}, + {"net.inet.tcp.rootonly", []_C_int{4, 2, 6, 24}}, {"net.inet.tcp.rstppslimit", []_C_int{4, 2, 6, 12}}, {"net.inet.tcp.sack", []_C_int{4, 2, 6, 10}}, {"net.inet.tcp.sackholelimit", []_C_int{4, 2, 6, 20}}, @@ -198,9 +208,12 @@ var sysctlMib = []mibentry{ {"net.inet.tcp.stats", []_C_int{4, 2, 6, 21}}, {"net.inet.tcp.synbucketlimit", []_C_int{4, 2, 6, 16}}, {"net.inet.tcp.syncachelimit", []_C_int{4, 2, 6, 15}}, + {"net.inet.tcp.synhashsize", []_C_int{4, 2, 6, 25}}, + {"net.inet.tcp.synuselimit", []_C_int{4, 2, 6, 23}}, {"net.inet.udp.baddynamic", []_C_int{4, 2, 17, 2}}, {"net.inet.udp.checksum", []_C_int{4, 2, 17, 1}}, {"net.inet.udp.recvspace", []_C_int{4, 2, 17, 3}}, + {"net.inet.udp.rootonly", []_C_int{4, 2, 17, 6}}, {"net.inet.udp.sendspace", []_C_int{4, 2, 17, 4}}, {"net.inet.udp.stats", []_C_int{4, 2, 17, 5}}, {"net.inet6.divert.recvspace", []_C_int{4, 24, 86, 1}}, @@ -213,13 +226,8 @@ var sysctlMib = []mibentry{ {"net.inet6.icmp6.nd6_delay", []_C_int{4, 24, 30, 8}}, {"net.inet6.icmp6.nd6_maxnudhint", []_C_int{4, 24, 30, 15}}, {"net.inet6.icmp6.nd6_mmaxtries", []_C_int{4, 24, 30, 10}}, - {"net.inet6.icmp6.nd6_prune", []_C_int{4, 24, 30, 6}}, {"net.inet6.icmp6.nd6_umaxtries", []_C_int{4, 24, 30, 9}}, - {"net.inet6.icmp6.nd6_useloopback", []_C_int{4, 24, 30, 11}}, - {"net.inet6.icmp6.nodeinfo", []_C_int{4, 24, 30, 13}}, - {"net.inet6.icmp6.rediraccept", []_C_int{4, 24, 30, 2}}, {"net.inet6.icmp6.redirtimeout", []_C_int{4, 24, 30, 3}}, - {"net.inet6.ip6.accept_rtadv", []_C_int{4, 24, 17, 12}}, {"net.inet6.ip6.auto_flowlabel", []_C_int{4, 24, 17, 17}}, {"net.inet6.ip6.dad_count", []_C_int{4, 24, 17, 16}}, {"net.inet6.ip6.dad_pending", []_C_int{4, 24, 17, 49}}, @@ -232,20 +240,19 @@ var sysctlMib = []mibentry{ {"net.inet6.ip6.maxdynroutes", []_C_int{4, 24, 17, 48}}, {"net.inet6.ip6.maxfragpackets", []_C_int{4, 24, 17, 9}}, {"net.inet6.ip6.maxfrags", []_C_int{4, 24, 17, 41}}, - {"net.inet6.ip6.maxifdefrouters", []_C_int{4, 24, 17, 47}}, - {"net.inet6.ip6.maxifprefixes", []_C_int{4, 24, 17, 46}}, {"net.inet6.ip6.mforwarding", []_C_int{4, 24, 17, 42}}, + {"net.inet6.ip6.mrtmfc", []_C_int{4, 24, 17, 53}}, + {"net.inet6.ip6.mrtmif", []_C_int{4, 24, 17, 52}}, {"net.inet6.ip6.mrtproto", []_C_int{4, 24, 17, 8}}, {"net.inet6.ip6.mtudisctimeout", []_C_int{4, 24, 17, 50}}, {"net.inet6.ip6.multicast_mtudisc", []_C_int{4, 24, 17, 44}}, {"net.inet6.ip6.multipath", []_C_int{4, 24, 17, 43}}, {"net.inet6.ip6.neighborgcthresh", []_C_int{4, 24, 17, 45}}, {"net.inet6.ip6.redirect", []_C_int{4, 24, 17, 2}}, - {"net.inet6.ip6.rr_prune", []_C_int{4, 24, 17, 22}}, + {"net.inet6.ip6.soiikey", []_C_int{4, 24, 17, 54}}, {"net.inet6.ip6.sourcecheck", []_C_int{4, 24, 17, 10}}, {"net.inet6.ip6.sourcecheck_logint", []_C_int{4, 24, 17, 11}}, {"net.inet6.ip6.use_deprecated", []_C_int{4, 24, 17, 21}}, - {"net.inet6.ip6.v6only", []_C_int{4, 24, 17, 24}}, {"net.key.sadb_dump", []_C_int{4, 30, 1}}, {"net.key.spd_dump", []_C_int{4, 30, 2}}, {"net.mpls.ifq.congestion", []_C_int{4, 33, 3, 4}}, @@ -254,12 +261,12 @@ var sysctlMib = []mibentry{ {"net.mpls.ifq.maxlen", []_C_int{4, 33, 3, 2}}, {"net.mpls.mapttl_ip", []_C_int{4, 33, 5}}, {"net.mpls.mapttl_ip6", []_C_int{4, 33, 6}}, - {"net.mpls.maxloop_inkernel", []_C_int{4, 33, 4}}, {"net.mpls.ttl", []_C_int{4, 33, 2}}, {"net.pflow.stats", []_C_int{4, 34, 1}}, {"net.pipex.enable", []_C_int{4, 35, 1}}, {"vm.anonmin", []_C_int{2, 7}}, {"vm.loadavg", []_C_int{2, 2}}, + {"vm.malloc_conf", []_C_int{2, 12}}, {"vm.maxslp", []_C_int{2, 10}}, {"vm.nkmempages", []_C_int{2, 6}}, {"vm.psstrings", []_C_int{2, 3}}, diff --git a/vendor/golang.org/x/sys/unix/zsysctl_openbsd_arm64.go b/vendor/golang.org/x/sys/unix/zsysctl_openbsd_arm64.go index 154b57a..cbdda1a 100644 --- a/vendor/golang.org/x/sys/unix/zsysctl_openbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/zsysctl_openbsd_arm64.go @@ -36,6 +36,7 @@ var sysctlMib = []mibentry{ {"hw.pagesize", []_C_int{6, 7}}, {"hw.perfpolicy", []_C_int{6, 23}}, {"hw.physmem", []_C_int{6, 19}}, + {"hw.power", []_C_int{6, 26}}, {"hw.product", []_C_int{6, 15}}, {"hw.serialno", []_C_int{6, 17}}, {"hw.setperf", []_C_int{6, 13}}, @@ -44,6 +45,7 @@ var sysctlMib = []mibentry{ {"hw.uuid", []_C_int{6, 18}}, {"hw.vendor", []_C_int{6, 14}}, {"hw.version", []_C_int{6, 16}}, + {"kern.allowdt", []_C_int{1, 65}}, {"kern.allowkmem", []_C_int{1, 52}}, {"kern.argmax", []_C_int{1, 8}}, {"kern.audio", []_C_int{1, 84}}, @@ -51,6 +53,8 @@ var sysctlMib = []mibentry{ {"kern.bufcachepercent", []_C_int{1, 72}}, {"kern.ccpu", []_C_int{1, 45}}, {"kern.clockrate", []_C_int{1, 12}}, + {"kern.consbuf", []_C_int{1, 83}}, + {"kern.consbufsize", []_C_int{1, 82}}, {"kern.consdev", []_C_int{1, 75}}, {"kern.cp_time", []_C_int{1, 40}}, {"kern.cp_time2", []_C_int{1, 71}}, @@ -83,13 +87,13 @@ var sysctlMib = []mibentry{ {"kern.ngroups", []_C_int{1, 18}}, {"kern.nosuidcoredump", []_C_int{1, 32}}, {"kern.nprocs", []_C_int{1, 47}}, - {"kern.nselcoll", []_C_int{1, 43}}, {"kern.nthreads", []_C_int{1, 26}}, {"kern.numvnodes", []_C_int{1, 58}}, {"kern.osrelease", []_C_int{1, 2}}, {"kern.osrevision", []_C_int{1, 3}}, {"kern.ostype", []_C_int{1, 1}}, {"kern.osversion", []_C_int{1, 27}}, + {"kern.pfstatus", []_C_int{1, 86}}, {"kern.pool_debug", []_C_int{1, 77}}, {"kern.posix1version", []_C_int{1, 17}}, {"kern.proc", []_C_int{1, 66}}, @@ -110,13 +114,16 @@ var sysctlMib = []mibentry{ {"kern.timecounter.hardware", []_C_int{1, 69, 3}}, {"kern.timecounter.tick", []_C_int{1, 69, 1}}, {"kern.timecounter.timestepwarnings", []_C_int{1, 69, 2}}, + {"kern.timeout_stats", []_C_int{1, 87}}, {"kern.tty.tk_cancc", []_C_int{1, 44, 4}}, {"kern.tty.tk_nin", []_C_int{1, 44, 1}}, {"kern.tty.tk_nout", []_C_int{1, 44, 2}}, {"kern.tty.tk_rawcc", []_C_int{1, 44, 3}}, {"kern.tty.ttyinfo", []_C_int{1, 44, 5}}, {"kern.ttycount", []_C_int{1, 57}}, + {"kern.utc_offset", []_C_int{1, 88}}, {"kern.version", []_C_int{1, 4}}, + {"kern.video", []_C_int{1, 89}}, {"kern.watchdog.auto", []_C_int{1, 64, 2}}, {"kern.watchdog.period", []_C_int{1, 64, 1}}, {"kern.witnesswatch", []_C_int{1, 53}}, @@ -179,7 +186,6 @@ var sysctlMib = []mibentry{ {"net.inet.ipcomp.stats", []_C_int{4, 2, 108, 2}}, {"net.inet.ipip.allow", []_C_int{4, 2, 4, 1}}, {"net.inet.ipip.stats", []_C_int{4, 2, 4, 2}}, - {"net.inet.mobileip.allow", []_C_int{4, 2, 55, 1}}, {"net.inet.pfsync.stats", []_C_int{4, 2, 240, 1}}, {"net.inet.tcp.ackonpush", []_C_int{4, 2, 6, 13}}, {"net.inet.tcp.always_keepalive", []_C_int{4, 2, 6, 22}}, @@ -255,7 +261,6 @@ var sysctlMib = []mibentry{ {"net.mpls.ifq.maxlen", []_C_int{4, 33, 3, 2}}, {"net.mpls.mapttl_ip", []_C_int{4, 33, 5}}, {"net.mpls.mapttl_ip6", []_C_int{4, 33, 6}}, - {"net.mpls.maxloop_inkernel", []_C_int{4, 33, 4}}, {"net.mpls.ttl", []_C_int{4, 33, 2}}, {"net.pflow.stats", []_C_int{4, 34, 1}}, {"net.pipex.enable", []_C_int{4, 35, 1}}, diff --git a/vendor/golang.org/x/sys/unix/zsysctl_openbsd_mips64.go b/vendor/golang.org/x/sys/unix/zsysctl_openbsd_mips64.go index d96bb2b..f55eae1 100644 --- a/vendor/golang.org/x/sys/unix/zsysctl_openbsd_mips64.go +++ b/vendor/golang.org/x/sys/unix/zsysctl_openbsd_mips64.go @@ -36,6 +36,7 @@ var sysctlMib = []mibentry{ {"hw.pagesize", []_C_int{6, 7}}, {"hw.perfpolicy", []_C_int{6, 23}}, {"hw.physmem", []_C_int{6, 19}}, + {"hw.power", []_C_int{6, 26}}, {"hw.product", []_C_int{6, 15}}, {"hw.serialno", []_C_int{6, 17}}, {"hw.setperf", []_C_int{6, 13}}, @@ -86,7 +87,6 @@ var sysctlMib = []mibentry{ {"kern.ngroups", []_C_int{1, 18}}, {"kern.nosuidcoredump", []_C_int{1, 32}}, {"kern.nprocs", []_C_int{1, 47}}, - {"kern.nselcoll", []_C_int{1, 43}}, {"kern.nthreads", []_C_int{1, 26}}, {"kern.numvnodes", []_C_int{1, 58}}, {"kern.osrelease", []_C_int{1, 2}}, @@ -123,6 +123,7 @@ var sysctlMib = []mibentry{ {"kern.ttycount", []_C_int{1, 57}}, {"kern.utc_offset", []_C_int{1, 88}}, {"kern.version", []_C_int{1, 4}}, + {"kern.video", []_C_int{1, 89}}, {"kern.watchdog.auto", []_C_int{1, 64, 2}}, {"kern.watchdog.period", []_C_int{1, 64, 1}}, {"kern.witnesswatch", []_C_int{1, 53}}, diff --git a/vendor/golang.org/x/sys/unix/zsysnum_openbsd_mips64.go b/vendor/golang.org/x/sys/unix/zsysnum_openbsd_mips64.go index a37f773..01c43a0 100644 --- a/vendor/golang.org/x/sys/unix/zsysnum_openbsd_mips64.go +++ b/vendor/golang.org/x/sys/unix/zsysnum_openbsd_mips64.go @@ -6,6 +6,7 @@ package unix +// Deprecated: Use libc wrappers instead of direct syscalls. const ( SYS_EXIT = 1 // { void sys_exit(int rval); } SYS_FORK = 2 // { int sys_fork(void); } diff --git a/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go index e2a64f0..690cefc 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_darwin_amd64.go @@ -151,6 +151,16 @@ type Dirent struct { _ [3]byte } +type Attrlist struct { + Bitmapcount uint16 + Reserved uint16 + Commonattr uint32 + Volattr uint32 + Dirattr uint32 + Fileattr uint32 + Forkattr uint32 +} + const ( PathMax = 0x400 ) @@ -610,6 +620,7 @@ const ( AT_REMOVEDIR = 0x80 AT_SYMLINK_FOLLOW = 0x40 AT_SYMLINK_NOFOLLOW = 0x20 + AT_EACCESS = 0x10 ) type PollFd struct { diff --git a/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go index 34aa775..5bffc10 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_darwin_arm64.go @@ -151,6 +151,16 @@ type Dirent struct { _ [3]byte } +type Attrlist struct { + Bitmapcount uint16 + Reserved uint16 + Commonattr uint32 + Volattr uint32 + Dirattr uint32 + Fileattr uint32 + Forkattr uint32 +} + const ( PathMax = 0x400 ) @@ -610,6 +620,7 @@ const ( AT_REMOVEDIR = 0x80 AT_SYMLINK_FOLLOW = 0x40 AT_SYMLINK_NOFOLLOW = 0x20 + AT_EACCESS = 0x10 ) type PollFd struct { diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go index d9c78cd..29dc483 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go +++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_386.go @@ -362,7 +362,7 @@ type FpExtendedPrecision struct{} type PtraceIoDesc struct { Op int32 Offs uintptr - Addr uintptr + Addr *byte Len uint32 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go index 26991b1..0a89b28 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_amd64.go @@ -367,7 +367,7 @@ type FpExtendedPrecision struct{} type PtraceIoDesc struct { Op int32 Offs uintptr - Addr uintptr + Addr *byte Len uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go index f8324e7..c8666bb 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go +++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm.go @@ -350,7 +350,7 @@ type FpExtendedPrecision struct { type PtraceIoDesc struct { Op int32 Offs uintptr - Addr uintptr + Addr *byte Len uint32 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go index 4220411..88fb48a 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_arm64.go @@ -347,7 +347,7 @@ type FpExtendedPrecision struct{} type PtraceIoDesc struct { Op int32 Offs uintptr - Addr uintptr + Addr *byte Len uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go b/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go index 0660fd4..698dc97 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_freebsd_riscv64.go @@ -348,7 +348,7 @@ type FpExtendedPrecision struct{} type PtraceIoDesc struct { Op int32 Offs uintptr - Addr uintptr + Addr *byte Len uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux.go b/vendor/golang.org/x/sys/unix/ztypes_linux.go index ff68811..ca84727 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux.go @@ -29,6 +29,41 @@ type Itimerval struct { Value Timeval } +const ( + ADJ_OFFSET = 0x1 + ADJ_FREQUENCY = 0x2 + ADJ_MAXERROR = 0x4 + ADJ_ESTERROR = 0x8 + ADJ_STATUS = 0x10 + ADJ_TIMECONST = 0x20 + ADJ_TAI = 0x80 + ADJ_SETOFFSET = 0x100 + ADJ_MICRO = 0x1000 + ADJ_NANO = 0x2000 + ADJ_TICK = 0x4000 + ADJ_OFFSET_SINGLESHOT = 0x8001 + ADJ_OFFSET_SS_READ = 0xa001 +) + +const ( + STA_PLL = 0x1 + STA_PPSFREQ = 0x2 + STA_PPSTIME = 0x4 + STA_FLL = 0x8 + STA_INS = 0x10 + STA_DEL = 0x20 + STA_UNSYNC = 0x40 + STA_FREQHOLD = 0x80 + STA_PPSSIGNAL = 0x100 + STA_PPSJITTER = 0x200 + STA_PPSWANDER = 0x400 + STA_PPSERROR = 0x800 + STA_CLOCKERR = 0x1000 + STA_NANO = 0x2000 + STA_MODE = 0x4000 + STA_CLK = 0x8000 +) + const ( TIME_OK = 0x0 TIME_INS = 0x1 @@ -53,29 +88,30 @@ type StatxTimestamp struct { } type Statx_t struct { - Mask uint32 - Blksize uint32 - Attributes uint64 - Nlink uint32 - Uid uint32 - Gid uint32 - Mode uint16 - _ [1]uint16 - Ino uint64 - Size uint64 - Blocks uint64 - Attributes_mask uint64 - Atime StatxTimestamp - Btime StatxTimestamp - Ctime StatxTimestamp - Mtime StatxTimestamp - Rdev_major uint32 - Rdev_minor uint32 - Dev_major uint32 - Dev_minor uint32 - Mnt_id uint64 - _ uint64 - _ [12]uint64 + Mask uint32 + Blksize uint32 + Attributes uint64 + Nlink uint32 + Uid uint32 + Gid uint32 + Mode uint16 + _ [1]uint16 + Ino uint64 + Size uint64 + Blocks uint64 + Attributes_mask uint64 + Atime StatxTimestamp + Btime StatxTimestamp + Ctime StatxTimestamp + Mtime StatxTimestamp + Rdev_major uint32 + Rdev_minor uint32 + Dev_major uint32 + Dev_minor uint32 + Mnt_id uint64 + Dio_mem_align uint32 + Dio_offset_align uint32 + _ [12]uint64 } type Fsid struct { @@ -420,36 +456,60 @@ type Ucred struct { } type TCPInfo struct { - State uint8 - Ca_state uint8 - Retransmits uint8 - Probes uint8 - Backoff uint8 - Options uint8 - Rto uint32 - Ato uint32 - Snd_mss uint32 - Rcv_mss uint32 - Unacked uint32 - Sacked uint32 - Lost uint32 - Retrans uint32 - Fackets uint32 - Last_data_sent uint32 - Last_ack_sent uint32 - Last_data_recv uint32 - Last_ack_recv uint32 - Pmtu uint32 - Rcv_ssthresh uint32 - Rtt uint32 - Rttvar uint32 - Snd_ssthresh uint32 - Snd_cwnd uint32 - Advmss uint32 - Reordering uint32 - Rcv_rtt uint32 - Rcv_space uint32 - Total_retrans uint32 + State uint8 + Ca_state uint8 + Retransmits uint8 + Probes uint8 + Backoff uint8 + Options uint8 + Rto uint32 + Ato uint32 + Snd_mss uint32 + Rcv_mss uint32 + Unacked uint32 + Sacked uint32 + Lost uint32 + Retrans uint32 + Fackets uint32 + Last_data_sent uint32 + Last_ack_sent uint32 + Last_data_recv uint32 + Last_ack_recv uint32 + Pmtu uint32 + Rcv_ssthresh uint32 + Rtt uint32 + Rttvar uint32 + Snd_ssthresh uint32 + Snd_cwnd uint32 + Advmss uint32 + Reordering uint32 + Rcv_rtt uint32 + Rcv_space uint32 + Total_retrans uint32 + Pacing_rate uint64 + Max_pacing_rate uint64 + Bytes_acked uint64 + Bytes_received uint64 + Segs_out uint32 + Segs_in uint32 + Notsent_bytes uint32 + Min_rtt uint32 + Data_segs_in uint32 + Data_segs_out uint32 + Delivery_rate uint64 + Busy_time uint64 + Rwnd_limited uint64 + Sndbuf_limited uint64 + Delivered uint32 + Delivered_ce uint32 + Bytes_sent uint64 + Bytes_retrans uint64 + Dsack_dups uint32 + Reord_seen uint32 + Rcv_ooopack uint32 + Snd_wnd uint32 + Rcv_wnd uint32 + Rehash uint32 } type CanFilter struct { @@ -492,7 +552,7 @@ const ( SizeofIPv6MTUInfo = 0x20 SizeofICMPv6Filter = 0x20 SizeofUcred = 0xc - SizeofTCPInfo = 0x68 + SizeofTCPInfo = 0xf0 SizeofCanFilter = 0x8 SizeofTCPRepairOpt = 0x8 ) @@ -1007,6 +1067,7 @@ const ( PerfBitCommExec = CBitFieldMaskBit24 PerfBitUseClockID = CBitFieldMaskBit25 PerfBitContextSwitch = CBitFieldMaskBit26 + PerfBitWriteBackward = CBitFieldMaskBit27 ) const ( @@ -1099,7 +1160,8 @@ const ( PERF_SAMPLE_BRANCH_NO_CYCLES_SHIFT = 0xf PERF_SAMPLE_BRANCH_TYPE_SAVE_SHIFT = 0x10 PERF_SAMPLE_BRANCH_HW_INDEX_SHIFT = 0x11 - PERF_SAMPLE_BRANCH_MAX_SHIFT = 0x12 + PERF_SAMPLE_BRANCH_PRIV_SAVE_SHIFT = 0x12 + PERF_SAMPLE_BRANCH_MAX_SHIFT = 0x13 PERF_SAMPLE_BRANCH_USER = 0x1 PERF_SAMPLE_BRANCH_KERNEL = 0x2 PERF_SAMPLE_BRANCH_HV = 0x4 @@ -1118,7 +1180,8 @@ const ( PERF_SAMPLE_BRANCH_NO_CYCLES = 0x8000 PERF_SAMPLE_BRANCH_TYPE_SAVE = 0x10000 PERF_SAMPLE_BRANCH_HW_INDEX = 0x20000 - PERF_SAMPLE_BRANCH_MAX = 0x40000 + PERF_SAMPLE_BRANCH_PRIV_SAVE = 0x40000 + PERF_SAMPLE_BRANCH_MAX = 0x80000 PERF_BR_UNKNOWN = 0x0 PERF_BR_COND = 0x1 PERF_BR_UNCOND = 0x2 @@ -1132,7 +1195,10 @@ const ( PERF_BR_COND_RET = 0xa PERF_BR_ERET = 0xb PERF_BR_IRQ = 0xc - PERF_BR_MAX = 0xd + PERF_BR_SERROR = 0xd + PERF_BR_NO_TX = 0xe + PERF_BR_EXTEND_ABI = 0xf + PERF_BR_MAX = 0x10 PERF_SAMPLE_REGS_ABI_NONE = 0x0 PERF_SAMPLE_REGS_ABI_32 = 0x1 PERF_SAMPLE_REGS_ABI_64 = 0x2 @@ -1151,7 +1217,8 @@ const ( PERF_FORMAT_TOTAL_TIME_RUNNING = 0x2 PERF_FORMAT_ID = 0x4 PERF_FORMAT_GROUP = 0x8 - PERF_FORMAT_MAX = 0x10 + PERF_FORMAT_LOST = 0x10 + PERF_FORMAT_MAX = 0x20 PERF_IOC_FLAG_GROUP = 0x1 PERF_RECORD_MMAP = 0x1 PERF_RECORD_LOST = 0x2 @@ -1197,7 +1264,7 @@ type TCPMD5Sig struct { Flags uint8 Prefixlen uint8 Keylen uint16 - _ uint32 + Ifindex int32 Key [80]uint8 } @@ -1897,7 +1964,11 @@ const ( NFT_MSG_GETOBJ = 0x13 NFT_MSG_DELOBJ = 0x14 NFT_MSG_GETOBJ_RESET = 0x15 - NFT_MSG_MAX = 0x19 + NFT_MSG_NEWFLOWTABLE = 0x16 + NFT_MSG_GETFLOWTABLE = 0x17 + NFT_MSG_DELFLOWTABLE = 0x18 + NFT_MSG_GETRULE_RESET = 0x19 + NFT_MSG_MAX = 0x1a NFTA_LIST_UNSPEC = 0x0 NFTA_LIST_ELEM = 0x1 NFTA_HOOK_UNSPEC = 0x0 @@ -2401,9 +2472,11 @@ const ( SOF_TIMESTAMPING_OPT_STATS = 0x1000 SOF_TIMESTAMPING_OPT_PKTINFO = 0x2000 SOF_TIMESTAMPING_OPT_TX_SWHW = 0x4000 + SOF_TIMESTAMPING_BIND_PHC = 0x8000 + SOF_TIMESTAMPING_OPT_ID_TCP = 0x10000 - SOF_TIMESTAMPING_LAST = 0x8000 - SOF_TIMESTAMPING_MASK = 0xffff + SOF_TIMESTAMPING_LAST = 0x10000 + SOF_TIMESTAMPING_MASK = 0x1ffff SCM_TSTAMP_SND = 0x0 SCM_TSTAMP_SCHED = 0x1 @@ -2979,7 +3052,16 @@ const ( DEVLINK_CMD_TRAP_POLICER_NEW = 0x47 DEVLINK_CMD_TRAP_POLICER_DEL = 0x48 DEVLINK_CMD_HEALTH_REPORTER_TEST = 0x49 - DEVLINK_CMD_MAX = 0x51 + DEVLINK_CMD_RATE_GET = 0x4a + DEVLINK_CMD_RATE_SET = 0x4b + DEVLINK_CMD_RATE_NEW = 0x4c + DEVLINK_CMD_RATE_DEL = 0x4d + DEVLINK_CMD_LINECARD_GET = 0x4e + DEVLINK_CMD_LINECARD_SET = 0x4f + DEVLINK_CMD_LINECARD_NEW = 0x50 + DEVLINK_CMD_LINECARD_DEL = 0x51 + DEVLINK_CMD_SELFTESTS_GET = 0x52 + DEVLINK_CMD_MAX = 0x53 DEVLINK_PORT_TYPE_NOTSET = 0x0 DEVLINK_PORT_TYPE_AUTO = 0x1 DEVLINK_PORT_TYPE_ETH = 0x2 @@ -3208,7 +3290,13 @@ const ( DEVLINK_ATTR_RATE_NODE_NAME = 0xa8 DEVLINK_ATTR_RATE_PARENT_NODE_NAME = 0xa9 DEVLINK_ATTR_REGION_MAX_SNAPSHOTS = 0xaa - DEVLINK_ATTR_MAX = 0xae + DEVLINK_ATTR_LINECARD_INDEX = 0xab + DEVLINK_ATTR_LINECARD_STATE = 0xac + DEVLINK_ATTR_LINECARD_TYPE = 0xad + DEVLINK_ATTR_LINECARD_SUPPORTED_TYPES = 0xae + DEVLINK_ATTR_NESTED_DEVLINK = 0xaf + DEVLINK_ATTR_SELFTESTS = 0xb0 + DEVLINK_ATTR_MAX = 0xb3 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_NONE = 0x0 DEVLINK_DPIPE_FIELD_MAPPING_TYPE_IFINDEX = 0x1 DEVLINK_DPIPE_MATCH_TYPE_FIELD_EXACT = 0x0 @@ -3224,7 +3312,8 @@ const ( DEVLINK_PORT_FUNCTION_ATTR_HW_ADDR = 0x1 DEVLINK_PORT_FN_ATTR_STATE = 0x2 DEVLINK_PORT_FN_ATTR_OPSTATE = 0x3 - DEVLINK_PORT_FUNCTION_ATTR_MAX = 0x3 + DEVLINK_PORT_FN_ATTR_CAPS = 0x4 + DEVLINK_PORT_FUNCTION_ATTR_MAX = 0x4 ) type FsverityDigest struct { @@ -3317,7 +3406,8 @@ const ( LWTUNNEL_ENCAP_SEG6_LOCAL = 0x7 LWTUNNEL_ENCAP_RPL = 0x8 LWTUNNEL_ENCAP_IOAM6 = 0x9 - LWTUNNEL_ENCAP_MAX = 0x9 + LWTUNNEL_ENCAP_XFRM = 0xa + LWTUNNEL_ENCAP_MAX = 0xa MPLS_IPTUNNEL_UNSPEC = 0x0 MPLS_IPTUNNEL_DST = 0x1 @@ -3512,7 +3602,10 @@ const ( ETHTOOL_MSG_PHC_VCLOCKS_GET = 0x21 ETHTOOL_MSG_MODULE_GET = 0x22 ETHTOOL_MSG_MODULE_SET = 0x23 - ETHTOOL_MSG_USER_MAX = 0x23 + ETHTOOL_MSG_PSE_GET = 0x24 + ETHTOOL_MSG_PSE_SET = 0x25 + ETHTOOL_MSG_RSS_GET = 0x26 + ETHTOOL_MSG_USER_MAX = 0x26 ETHTOOL_MSG_KERNEL_NONE = 0x0 ETHTOOL_MSG_STRSET_GET_REPLY = 0x1 ETHTOOL_MSG_LINKINFO_GET_REPLY = 0x2 @@ -3550,7 +3643,9 @@ const ( ETHTOOL_MSG_PHC_VCLOCKS_GET_REPLY = 0x22 ETHTOOL_MSG_MODULE_GET_REPLY = 0x23 ETHTOOL_MSG_MODULE_NTF = 0x24 - ETHTOOL_MSG_KERNEL_MAX = 0x24 + ETHTOOL_MSG_PSE_GET_REPLY = 0x25 + ETHTOOL_MSG_RSS_GET_REPLY = 0x26 + ETHTOOL_MSG_KERNEL_MAX = 0x26 ETHTOOL_A_HEADER_UNSPEC = 0x0 ETHTOOL_A_HEADER_DEV_INDEX = 0x1 ETHTOOL_A_HEADER_DEV_NAME = 0x2 @@ -3609,7 +3704,8 @@ const ( ETHTOOL_A_LINKMODES_MASTER_SLAVE_CFG = 0x7 ETHTOOL_A_LINKMODES_MASTER_SLAVE_STATE = 0x8 ETHTOOL_A_LINKMODES_LANES = 0x9 - ETHTOOL_A_LINKMODES_MAX = 0x9 + ETHTOOL_A_LINKMODES_RATE_MATCHING = 0xa + ETHTOOL_A_LINKMODES_MAX = 0xa ETHTOOL_A_LINKSTATE_UNSPEC = 0x0 ETHTOOL_A_LINKSTATE_HEADER = 0x1 ETHTOOL_A_LINKSTATE_LINK = 0x2 @@ -3617,7 +3713,8 @@ const ( ETHTOOL_A_LINKSTATE_SQI_MAX = 0x4 ETHTOOL_A_LINKSTATE_EXT_STATE = 0x5 ETHTOOL_A_LINKSTATE_EXT_SUBSTATE = 0x6 - ETHTOOL_A_LINKSTATE_MAX = 0x6 + ETHTOOL_A_LINKSTATE_EXT_DOWN_CNT = 0x7 + ETHTOOL_A_LINKSTATE_MAX = 0x7 ETHTOOL_A_DEBUG_UNSPEC = 0x0 ETHTOOL_A_DEBUG_HEADER = 0x1 ETHTOOL_A_DEBUG_MSGMASK = 0x2 @@ -4201,6 +4298,9 @@ const ( NL80211_ACL_POLICY_DENY_UNLESS_LISTED = 0x1 NL80211_AC_VI = 0x1 NL80211_AC_VO = 0x0 + NL80211_AP_SETTINGS_EXTERNAL_AUTH_SUPPORT = 0x1 + NL80211_AP_SETTINGS_SA_QUERY_OFFLOAD_SUPPORT = 0x2 + NL80211_AP_SME_SA_QUERY_OFFLOAD = 0x1 NL80211_ATTR_4ADDR = 0x53 NL80211_ATTR_ACK = 0x5c NL80211_ATTR_ACK_SIGNAL = 0x107 @@ -4209,6 +4309,7 @@ const ( NL80211_ATTR_AIRTIME_WEIGHT = 0x112 NL80211_ATTR_AKM_SUITES = 0x4c NL80211_ATTR_AP_ISOLATE = 0x60 + NL80211_ATTR_AP_SETTINGS_FLAGS = 0x135 NL80211_ATTR_AUTH_DATA = 0x9c NL80211_ATTR_AUTH_TYPE = 0x35 NL80211_ATTR_BANDS = 0xef @@ -4240,6 +4341,9 @@ const ( NL80211_ATTR_COALESCE_RULE_DELAY = 0x1 NL80211_ATTR_COALESCE_RULE_MAX = 0x3 NL80211_ATTR_COALESCE_RULE_PKT_PATTERN = 0x3 + NL80211_ATTR_COLOR_CHANGE_COLOR = 0x130 + NL80211_ATTR_COLOR_CHANGE_COUNT = 0x12f + NL80211_ATTR_COLOR_CHANGE_ELEMS = 0x131 NL80211_ATTR_CONN_FAILED_REASON = 0x9b NL80211_ATTR_CONTROL_PORT = 0x44 NL80211_ATTR_CONTROL_PORT_ETHERTYPE = 0x66 @@ -4266,6 +4370,7 @@ const ( NL80211_ATTR_DEVICE_AP_SME = 0x8d NL80211_ATTR_DFS_CAC_TIME = 0x7 NL80211_ATTR_DFS_REGION = 0x92 + NL80211_ATTR_DISABLE_EHT = 0x137 NL80211_ATTR_DISABLE_HE = 0x12d NL80211_ATTR_DISABLE_HT = 0x93 NL80211_ATTR_DISABLE_VHT = 0xaf @@ -4273,6 +4378,8 @@ const ( NL80211_ATTR_DONT_WAIT_FOR_ACK = 0x8e NL80211_ATTR_DTIM_PERIOD = 0xd NL80211_ATTR_DURATION = 0x57 + NL80211_ATTR_EHT_CAPABILITY = 0x136 + NL80211_ATTR_EML_CAPABILITY = 0x13d NL80211_ATTR_EXT_CAPA = 0xa9 NL80211_ATTR_EXT_CAPA_MASK = 0xaa NL80211_ATTR_EXTERNAL_AUTH_ACTION = 0x104 @@ -4337,10 +4444,11 @@ const ( NL80211_ATTR_MAC_HINT = 0xc8 NL80211_ATTR_MAC_MASK = 0xd7 NL80211_ATTR_MAX_AP_ASSOC_STA = 0xca - NL80211_ATTR_MAX = 0x137 + NL80211_ATTR_MAX = 0x141 NL80211_ATTR_MAX_CRIT_PROT_DURATION = 0xb4 NL80211_ATTR_MAX_CSA_COUNTERS = 0xce NL80211_ATTR_MAX_MATCH_SETS = 0x85 + NL80211_ATTR_MAX_NUM_AKM_SUITES = 0x13c NL80211_ATTR_MAX_NUM_PMKIDS = 0x56 NL80211_ATTR_MAX_NUM_SCAN_SSIDS = 0x2b NL80211_ATTR_MAX_NUM_SCHED_SCAN_PLANS = 0xde @@ -4350,6 +4458,8 @@ const ( NL80211_ATTR_MAX_SCAN_PLAN_INTERVAL = 0xdf NL80211_ATTR_MAX_SCAN_PLAN_ITERATIONS = 0xe0 NL80211_ATTR_MAX_SCHED_SCAN_IE_LEN = 0x7c + NL80211_ATTR_MBSSID_CONFIG = 0x132 + NL80211_ATTR_MBSSID_ELEMS = 0x133 NL80211_ATTR_MCAST_RATE = 0x6b NL80211_ATTR_MDID = 0xb1 NL80211_ATTR_MEASUREMENT_DURATION = 0xeb @@ -4359,6 +4469,11 @@ const ( NL80211_ATTR_MESH_PEER_AID = 0xed NL80211_ATTR_MESH_SETUP = 0x70 NL80211_ATTR_MGMT_SUBTYPE = 0x29 + NL80211_ATTR_MLD_ADDR = 0x13a + NL80211_ATTR_MLD_CAPA_AND_OPS = 0x13e + NL80211_ATTR_MLO_LINK_ID = 0x139 + NL80211_ATTR_MLO_LINKS = 0x138 + NL80211_ATTR_MLO_SUPPORT = 0x13b NL80211_ATTR_MNTR_FLAGS = 0x17 NL80211_ATTR_MPATH_INFO = 0x1b NL80211_ATTR_MPATH_NEXT_HOP = 0x1a @@ -4371,6 +4486,7 @@ const ( NL80211_ATTR_NETNS_FD = 0xdb NL80211_ATTR_NOACK_MAP = 0x95 NL80211_ATTR_NSS = 0x106 + NL80211_ATTR_OBSS_COLOR_BITMAP = 0x12e NL80211_ATTR_OFFCHANNEL_TX_OK = 0x6c NL80211_ATTR_OPER_CLASS = 0xd6 NL80211_ATTR_OPMODE_NOTIF = 0xc2 @@ -4397,6 +4513,7 @@ const ( NL80211_ATTR_PROTOCOL_FEATURES = 0xad NL80211_ATTR_PS_STATE = 0x5d NL80211_ATTR_QOS_MAP = 0xc7 + NL80211_ATTR_RADAR_BACKGROUND = 0x134 NL80211_ATTR_RADAR_EVENT = 0xa8 NL80211_ATTR_REASON_CODE = 0x36 NL80211_ATTR_RECEIVE_MULTICAST = 0x121 @@ -4412,6 +4529,7 @@ const ( NL80211_ATTR_RESP_IE = 0x4e NL80211_ATTR_ROAM_SUPPORT = 0x83 NL80211_ATTR_RX_FRAME_TYPES = 0x64 + NL80211_ATTR_RX_HW_TIMESTAMP = 0x140 NL80211_ATTR_RXMGMT_FLAGS = 0xbc NL80211_ATTR_RX_SIGNAL_DBM = 0x97 NL80211_ATTR_S1G_CAPABILITY = 0x128 @@ -4469,6 +4587,7 @@ const ( NL80211_ATTR_SUPPORT_MESH_AUTH = 0x73 NL80211_ATTR_SURVEY_INFO = 0x54 NL80211_ATTR_SURVEY_RADIO_STATS = 0xda + NL80211_ATTR_TD_BITMAP = 0x141 NL80211_ATTR_TDLS_ACTION = 0x88 NL80211_ATTR_TDLS_DIALOG_TOKEN = 0x89 NL80211_ATTR_TDLS_EXTERNAL_SETUP = 0x8c @@ -4484,6 +4603,7 @@ const ( NL80211_ATTR_TSID = 0xd2 NL80211_ATTR_TWT_RESPONDER = 0x116 NL80211_ATTR_TX_FRAME_TYPES = 0x63 + NL80211_ATTR_TX_HW_TIMESTAMP = 0x13f NL80211_ATTR_TX_NO_CCK_RATE = 0x87 NL80211_ATTR_TXQ_LIMIT = 0x10a NL80211_ATTR_TXQ_MEMORY_LIMIT = 0x10b @@ -4557,6 +4677,10 @@ const ( NL80211_BAND_ATTR_RATES = 0x2 NL80211_BAND_ATTR_VHT_CAPA = 0x8 NL80211_BAND_ATTR_VHT_MCS_SET = 0x7 + NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MAC = 0x8 + NL80211_BAND_IFTYPE_ATTR_EHT_CAP_MCS_SET = 0xa + NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PHY = 0x9 + NL80211_BAND_IFTYPE_ATTR_EHT_CAP_PPE = 0xb NL80211_BAND_IFTYPE_ATTR_HE_6GHZ_CAPA = 0x6 NL80211_BAND_IFTYPE_ATTR_HE_CAP_MAC = 0x2 NL80211_BAND_IFTYPE_ATTR_HE_CAP_MCS_SET = 0x4 @@ -4564,6 +4688,8 @@ const ( NL80211_BAND_IFTYPE_ATTR_HE_CAP_PPE = 0x5 NL80211_BAND_IFTYPE_ATTR_IFTYPES = 0x1 NL80211_BAND_IFTYPE_ATTR_MAX = 0xb + NL80211_BAND_IFTYPE_ATTR_VENDOR_ELEMS = 0x7 + NL80211_BAND_LC = 0x5 NL80211_BAND_S1GHZ = 0x4 NL80211_BITRATE_ATTR_2GHZ_SHORTPREAMBLE = 0x2 NL80211_BITRATE_ATTR_MAX = 0x2 @@ -4584,7 +4710,9 @@ const ( NL80211_BSS_FREQUENCY_OFFSET = 0x14 NL80211_BSS_INFORMATION_ELEMENTS = 0x6 NL80211_BSS_LAST_SEEN_BOOTTIME = 0xf - NL80211_BSS_MAX = 0x14 + NL80211_BSS_MAX = 0x16 + NL80211_BSS_MLD_ADDR = 0x16 + NL80211_BSS_MLO_LINK_ID = 0x15 NL80211_BSS_PAD = 0x10 NL80211_BSS_PARENT_BSSID = 0x12 NL80211_BSS_PARENT_TSF = 0x11 @@ -4612,6 +4740,7 @@ const ( NL80211_CHAN_WIDTH_20 = 0x1 NL80211_CHAN_WIDTH_20_NOHT = 0x0 NL80211_CHAN_WIDTH_2 = 0x9 + NL80211_CHAN_WIDTH_320 = 0xd NL80211_CHAN_WIDTH_40 = 0x2 NL80211_CHAN_WIDTH_4 = 0xa NL80211_CHAN_WIDTH_5 = 0x6 @@ -4621,8 +4750,11 @@ const ( NL80211_CMD_ABORT_SCAN = 0x72 NL80211_CMD_ACTION = 0x3b NL80211_CMD_ACTION_TX_STATUS = 0x3c + NL80211_CMD_ADD_LINK = 0x94 + NL80211_CMD_ADD_LINK_STA = 0x96 NL80211_CMD_ADD_NAN_FUNCTION = 0x75 NL80211_CMD_ADD_TX_TS = 0x69 + NL80211_CMD_ASSOC_COMEBACK = 0x93 NL80211_CMD_ASSOCIATE = 0x26 NL80211_CMD_AUTHENTICATE = 0x25 NL80211_CMD_CANCEL_REMAIN_ON_CHANNEL = 0x38 @@ -4630,6 +4762,10 @@ const ( NL80211_CMD_CHANNEL_SWITCH = 0x66 NL80211_CMD_CH_SWITCH_NOTIFY = 0x58 NL80211_CMD_CH_SWITCH_STARTED_NOTIFY = 0x6e + NL80211_CMD_COLOR_CHANGE_ABORTED = 0x90 + NL80211_CMD_COLOR_CHANGE_COMPLETED = 0x91 + NL80211_CMD_COLOR_CHANGE_REQUEST = 0x8e + NL80211_CMD_COLOR_CHANGE_STARTED = 0x8f NL80211_CMD_CONNECT = 0x2e NL80211_CMD_CONN_FAILED = 0x5b NL80211_CMD_CONTROL_PORT_FRAME = 0x81 @@ -4678,8 +4814,9 @@ const ( NL80211_CMD_LEAVE_IBSS = 0x2c NL80211_CMD_LEAVE_MESH = 0x45 NL80211_CMD_LEAVE_OCB = 0x6d - NL80211_CMD_MAX = 0x93 + NL80211_CMD_MAX = 0x98 NL80211_CMD_MICHAEL_MIC_FAILURE = 0x29 + NL80211_CMD_MODIFY_LINK_STA = 0x97 NL80211_CMD_NAN_MATCH = 0x78 NL80211_CMD_NEW_BEACON = 0xf NL80211_CMD_NEW_INTERFACE = 0x7 @@ -4692,6 +4829,7 @@ const ( NL80211_CMD_NEW_WIPHY = 0x3 NL80211_CMD_NOTIFY_CQM = 0x40 NL80211_CMD_NOTIFY_RADAR = 0x86 + NL80211_CMD_OBSS_COLOR_COLLISION = 0x8d NL80211_CMD_PEER_MEASUREMENT_COMPLETE = 0x85 NL80211_CMD_PEER_MEASUREMENT_RESULT = 0x84 NL80211_CMD_PEER_MEASUREMENT_START = 0x83 @@ -4707,6 +4845,8 @@ const ( NL80211_CMD_REGISTER_FRAME = 0x3a NL80211_CMD_RELOAD_REGDB = 0x7e NL80211_CMD_REMAIN_ON_CHANNEL = 0x37 + NL80211_CMD_REMOVE_LINK = 0x95 + NL80211_CMD_REMOVE_LINK_STA = 0x98 NL80211_CMD_REQ_SET_REG = 0x1b NL80211_CMD_ROAM = 0x2f NL80211_CMD_SCAN_ABORTED = 0x23 @@ -4717,6 +4857,7 @@ const ( NL80211_CMD_SET_CHANNEL = 0x41 NL80211_CMD_SET_COALESCE = 0x65 NL80211_CMD_SET_CQM = 0x3f + NL80211_CMD_SET_FILS_AAD = 0x92 NL80211_CMD_SET_INTERFACE = 0x6 NL80211_CMD_SET_KEY = 0xa NL80211_CMD_SET_MAC_ACL = 0x5d @@ -4791,6 +4932,8 @@ const ( NL80211_EDMG_BW_CONFIG_MIN = 0x4 NL80211_EDMG_CHANNELS_MAX = 0x3c NL80211_EDMG_CHANNELS_MIN = 0x1 + NL80211_EHT_MAX_CAPABILITY_LEN = 0x33 + NL80211_EHT_MIN_CAPABILITY_LEN = 0xd NL80211_EXTERNAL_AUTH_ABORT = 0x1 NL80211_EXTERNAL_AUTH_START = 0x0 NL80211_EXT_FEATURE_4WAY_HANDSHAKE_AP_PSK = 0x32 @@ -4807,6 +4950,7 @@ const ( NL80211_EXT_FEATURE_BEACON_RATE_HT = 0x7 NL80211_EXT_FEATURE_BEACON_RATE_LEGACY = 0x6 NL80211_EXT_FEATURE_BEACON_RATE_VHT = 0x8 + NL80211_EXT_FEATURE_BSS_COLOR = 0x3a NL80211_EXT_FEATURE_BSS_PARENT_TSF = 0x4 NL80211_EXT_FEATURE_CAN_REPLACE_PTK0 = 0x1f NL80211_EXT_FEATURE_CONTROL_PORT_NO_PREAUTH = 0x2a @@ -4818,6 +4962,7 @@ const ( NL80211_EXT_FEATURE_DFS_OFFLOAD = 0x19 NL80211_EXT_FEATURE_ENABLE_FTM_RESPONDER = 0x20 NL80211_EXT_FEATURE_EXT_KEY_ID = 0x24 + NL80211_EXT_FEATURE_FILS_CRYPTO_OFFLOAD = 0x3b NL80211_EXT_FEATURE_FILS_DISCOVERY = 0x34 NL80211_EXT_FEATURE_FILS_MAX_CHANNEL_TIME = 0x11 NL80211_EXT_FEATURE_FILS_SK_OFFLOAD = 0xe @@ -4833,8 +4978,10 @@ const ( NL80211_EXT_FEATURE_OCE_PROBE_REQ_DEFERRAL_SUPPRESSION = 0x14 NL80211_EXT_FEATURE_OCE_PROBE_REQ_HIGH_TX_RATE = 0x13 NL80211_EXT_FEATURE_OPERATING_CHANNEL_VALIDATION = 0x31 + NL80211_EXT_FEATURE_POWERED_ADDR_CHANGE = 0x3d NL80211_EXT_FEATURE_PROTECTED_TWT = 0x2b NL80211_EXT_FEATURE_PROT_RANGE_NEGO_AND_MEASURE = 0x39 + NL80211_EXT_FEATURE_RADAR_BACKGROUND = 0x3c NL80211_EXT_FEATURE_RRM = 0x1 NL80211_EXT_FEATURE_SAE_OFFLOAD_AP = 0x33 NL80211_EXT_FEATURE_SAE_OFFLOAD = 0x26 @@ -4906,7 +5053,9 @@ const ( NL80211_FREQUENCY_ATTR_NO_10MHZ = 0x11 NL80211_FREQUENCY_ATTR_NO_160MHZ = 0xc NL80211_FREQUENCY_ATTR_NO_20MHZ = 0x10 + NL80211_FREQUENCY_ATTR_NO_320MHZ = 0x1a NL80211_FREQUENCY_ATTR_NO_80MHZ = 0xb + NL80211_FREQUENCY_ATTR_NO_EHT = 0x1b NL80211_FREQUENCY_ATTR_NO_HE = 0x13 NL80211_FREQUENCY_ATTR_NO_HT40_MINUS = 0x9 NL80211_FREQUENCY_ATTR_NO_HT40_PLUS = 0xa @@ -5006,6 +5155,12 @@ const ( NL80211_MAX_SUPP_HT_RATES = 0x4d NL80211_MAX_SUPP_RATES = 0x20 NL80211_MAX_SUPP_REG_RULES = 0x80 + NL80211_MBSSID_CONFIG_ATTR_EMA = 0x5 + NL80211_MBSSID_CONFIG_ATTR_INDEX = 0x3 + NL80211_MBSSID_CONFIG_ATTR_MAX = 0x5 + NL80211_MBSSID_CONFIG_ATTR_MAX_EMA_PROFILE_PERIODICITY = 0x2 + NL80211_MBSSID_CONFIG_ATTR_MAX_INTERFACES = 0x1 + NL80211_MBSSID_CONFIG_ATTR_TX_IFINDEX = 0x4 NL80211_MESHCONF_ATTR_MAX = 0x1f NL80211_MESHCONF_AUTO_OPEN_PLINKS = 0x7 NL80211_MESHCONF_AWAKE_WINDOW = 0x1b @@ -5168,6 +5323,7 @@ const ( NL80211_PMSR_FTM_FAILURE_UNSPECIFIED = 0x0 NL80211_PMSR_FTM_FAILURE_WRONG_CHANNEL = 0x3 NL80211_PMSR_FTM_REQ_ATTR_ASAP = 0x1 + NL80211_PMSR_FTM_REQ_ATTR_BSS_COLOR = 0xd NL80211_PMSR_FTM_REQ_ATTR_BURST_DURATION = 0x5 NL80211_PMSR_FTM_REQ_ATTR_BURST_PERIOD = 0x4 NL80211_PMSR_FTM_REQ_ATTR_FTMS_PER_BURST = 0x6 @@ -5244,12 +5400,36 @@ const ( NL80211_RADAR_PRE_CAC_EXPIRED = 0x4 NL80211_RATE_INFO_10_MHZ_WIDTH = 0xb NL80211_RATE_INFO_160_MHZ_WIDTH = 0xa + NL80211_RATE_INFO_320_MHZ_WIDTH = 0x12 NL80211_RATE_INFO_40_MHZ_WIDTH = 0x3 NL80211_RATE_INFO_5_MHZ_WIDTH = 0xc NL80211_RATE_INFO_80_MHZ_WIDTH = 0x8 NL80211_RATE_INFO_80P80_MHZ_WIDTH = 0x9 NL80211_RATE_INFO_BITRATE32 = 0x5 NL80211_RATE_INFO_BITRATE = 0x1 + NL80211_RATE_INFO_EHT_GI_0_8 = 0x0 + NL80211_RATE_INFO_EHT_GI_1_6 = 0x1 + NL80211_RATE_INFO_EHT_GI_3_2 = 0x2 + NL80211_RATE_INFO_EHT_GI = 0x15 + NL80211_RATE_INFO_EHT_MCS = 0x13 + NL80211_RATE_INFO_EHT_NSS = 0x14 + NL80211_RATE_INFO_EHT_RU_ALLOC_106 = 0x3 + NL80211_RATE_INFO_EHT_RU_ALLOC_106P26 = 0x4 + NL80211_RATE_INFO_EHT_RU_ALLOC_242 = 0x5 + NL80211_RATE_INFO_EHT_RU_ALLOC_26 = 0x0 + NL80211_RATE_INFO_EHT_RU_ALLOC_2x996 = 0xb + NL80211_RATE_INFO_EHT_RU_ALLOC_2x996P484 = 0xc + NL80211_RATE_INFO_EHT_RU_ALLOC_3x996 = 0xd + NL80211_RATE_INFO_EHT_RU_ALLOC_3x996P484 = 0xe + NL80211_RATE_INFO_EHT_RU_ALLOC_484 = 0x6 + NL80211_RATE_INFO_EHT_RU_ALLOC_484P242 = 0x7 + NL80211_RATE_INFO_EHT_RU_ALLOC_4x996 = 0xf + NL80211_RATE_INFO_EHT_RU_ALLOC_52 = 0x1 + NL80211_RATE_INFO_EHT_RU_ALLOC_52P26 = 0x2 + NL80211_RATE_INFO_EHT_RU_ALLOC_996 = 0x8 + NL80211_RATE_INFO_EHT_RU_ALLOC_996P484 = 0x9 + NL80211_RATE_INFO_EHT_RU_ALLOC_996P484P242 = 0xa + NL80211_RATE_INFO_EHT_RU_ALLOC = 0x16 NL80211_RATE_INFO_HE_1XLTF = 0x0 NL80211_RATE_INFO_HE_2XLTF = 0x1 NL80211_RATE_INFO_HE_4XLTF = 0x2 @@ -5292,6 +5472,7 @@ const ( NL80211_RRF_GO_CONCURRENT = 0x1000 NL80211_RRF_IR_CONCURRENT = 0x1000 NL80211_RRF_NO_160MHZ = 0x10000 + NL80211_RRF_NO_320MHZ = 0x40000 NL80211_RRF_NO_80MHZ = 0x8000 NL80211_RRF_NO_CCK = 0x2 NL80211_RRF_NO_HE = 0x20000 @@ -5607,3 +5788,25 @@ const ( AUDIT_NLGRP_NONE = 0x0 AUDIT_NLGRP_READLOG = 0x1 ) + +const ( + TUN_F_CSUM = 0x1 + TUN_F_TSO4 = 0x2 + TUN_F_TSO6 = 0x4 + TUN_F_TSO_ECN = 0x8 + TUN_F_UFO = 0x10 +) + +const ( + VIRTIO_NET_HDR_F_NEEDS_CSUM = 0x1 + VIRTIO_NET_HDR_F_DATA_VALID = 0x2 + VIRTIO_NET_HDR_F_RSC_INFO = 0x4 +) + +const ( + VIRTIO_NET_HDR_GSO_NONE = 0x0 + VIRTIO_NET_HDR_GSO_TCPV4 = 0x1 + VIRTIO_NET_HDR_GSO_UDP = 0x3 + VIRTIO_NET_HDR_GSO_TCPV6 = 0x4 + VIRTIO_NET_HDR_GSO_ECN = 0x80 +) diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_386.go b/vendor/golang.org/x/sys/unix/ztypes_linux_386.go index 89c516a..4ecc149 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_386.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_386.go @@ -414,7 +414,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [122]int8 + Data [122]byte _ uint32 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go index 62b4fb2..34fddff 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_amd64.go @@ -427,7 +427,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [118]int8 + Data [118]byte _ uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go b/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go index e86b358..3b14a60 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_arm.go @@ -405,7 +405,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [122]uint8 + Data [122]byte _ uint32 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go index 6c6be4c..0517651 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_arm64.go @@ -406,7 +406,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [118]int8 + Data [118]byte _ uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_loong64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_loong64.go index 4982ea3..3b0c518 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_loong64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_loong64.go @@ -407,7 +407,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [118]int8 + Data [118]byte _ uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go index 173141a..fccdf4d 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mips.go @@ -410,7 +410,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [122]int8 + Data [122]byte _ uint32 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go index 93ae4c5..500de8f 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64.go @@ -409,7 +409,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [118]int8 + Data [118]byte _ uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go index 4e4e510..d0434cd 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mips64le.go @@ -409,7 +409,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [118]int8 + Data [118]byte _ uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go b/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go index 3f5ba01..84206ba 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_mipsle.go @@ -410,7 +410,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [122]int8 + Data [122]byte _ uint32 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc.go b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc.go index 71dfe7c..ab078cf 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc.go @@ -417,7 +417,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [122]uint8 + Data [122]byte _ uint32 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go index 3a2b7f0..42eb2c4 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64.go @@ -416,7 +416,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [118]uint8 + Data [118]byte _ uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go index a52d627..31304a4 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_ppc64le.go @@ -416,7 +416,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [118]uint8 + Data [118]byte _ uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go index dfc007d..c311f96 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_riscv64.go @@ -434,7 +434,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [118]uint8 + Data [118]byte _ uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go b/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go index b53cb91..bba3cef 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_s390x.go @@ -429,7 +429,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [118]int8 + Data [118]byte _ uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go b/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go index fe0aa35..ad8a013 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux_sparc64.go @@ -411,7 +411,7 @@ const ( type SockaddrStorage struct { Family uint16 - _ [118]int8 + Data [118]byte _ uint64 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_netbsd_386.go b/vendor/golang.org/x/sys/unix/ztypes_netbsd_386.go index 2fd2060..9bc4c8f 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_netbsd_386.go +++ b/vendor/golang.org/x/sys/unix/ztypes_netbsd_386.go @@ -491,6 +491,90 @@ type Utsname struct { Machine [256]byte } +const SizeofUvmexp = 0x278 + +type Uvmexp struct { + Pagesize int64 + Pagemask int64 + Pageshift int64 + Npages int64 + Free int64 + Active int64 + Inactive int64 + Paging int64 + Wired int64 + Zeropages int64 + Reserve_pagedaemon int64 + Reserve_kernel int64 + Freemin int64 + Freetarg int64 + Inactarg int64 + Wiredmax int64 + Nswapdev int64 + Swpages int64 + Swpginuse int64 + Swpgonly int64 + Nswget int64 + Unused1 int64 + Cpuhit int64 + Cpumiss int64 + Faults int64 + Traps int64 + Intrs int64 + Swtch int64 + Softs int64 + Syscalls int64 + Pageins int64 + Swapins int64 + Swapouts int64 + Pgswapin int64 + Pgswapout int64 + Forks int64 + Forks_ppwait int64 + Forks_sharevm int64 + Pga_zerohit int64 + Pga_zeromiss int64 + Zeroaborts int64 + Fltnoram int64 + Fltnoanon int64 + Fltpgwait int64 + Fltpgrele int64 + Fltrelck int64 + Fltrelckok int64 + Fltanget int64 + Fltanretry int64 + Fltamcopy int64 + Fltnamap int64 + Fltnomap int64 + Fltlget int64 + Fltget int64 + Flt_anon int64 + Flt_acow int64 + Flt_obj int64 + Flt_prcopy int64 + Flt_przero int64 + Pdwoke int64 + Pdrevs int64 + Unused4 int64 + Pdfreed int64 + Pdscans int64 + Pdanscan int64 + Pdobscan int64 + Pdreact int64 + Pdbusy int64 + Pdpageouts int64 + Pdpending int64 + Pddeact int64 + Anonpages int64 + Filepages int64 + Execpages int64 + Colorhit int64 + Colormiss int64 + Ncolors int64 + Bootpages int64 + Poolpages int64 +} + const SizeofClockinfo = 0x14 type Clockinfo struct { diff --git a/vendor/golang.org/x/sys/unix/ztypes_netbsd_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_netbsd_amd64.go index 6a5a1a8..bb05f65 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_netbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_netbsd_amd64.go @@ -499,6 +499,90 @@ type Utsname struct { Machine [256]byte } +const SizeofUvmexp = 0x278 + +type Uvmexp struct { + Pagesize int64 + Pagemask int64 + Pageshift int64 + Npages int64 + Free int64 + Active int64 + Inactive int64 + Paging int64 + Wired int64 + Zeropages int64 + Reserve_pagedaemon int64 + Reserve_kernel int64 + Freemin int64 + Freetarg int64 + Inactarg int64 + Wiredmax int64 + Nswapdev int64 + Swpages int64 + Swpginuse int64 + Swpgonly int64 + Nswget int64 + Unused1 int64 + Cpuhit int64 + Cpumiss int64 + Faults int64 + Traps int64 + Intrs int64 + Swtch int64 + Softs int64 + Syscalls int64 + Pageins int64 + Swapins int64 + Swapouts int64 + Pgswapin int64 + Pgswapout int64 + Forks int64 + Forks_ppwait int64 + Forks_sharevm int64 + Pga_zerohit int64 + Pga_zeromiss int64 + Zeroaborts int64 + Fltnoram int64 + Fltnoanon int64 + Fltpgwait int64 + Fltpgrele int64 + Fltrelck int64 + Fltrelckok int64 + Fltanget int64 + Fltanretry int64 + Fltamcopy int64 + Fltnamap int64 + Fltnomap int64 + Fltlget int64 + Fltget int64 + Flt_anon int64 + Flt_acow int64 + Flt_obj int64 + Flt_prcopy int64 + Flt_przero int64 + Pdwoke int64 + Pdrevs int64 + Unused4 int64 + Pdfreed int64 + Pdscans int64 + Pdanscan int64 + Pdobscan int64 + Pdreact int64 + Pdbusy int64 + Pdpageouts int64 + Pdpending int64 + Pddeact int64 + Anonpages int64 + Filepages int64 + Execpages int64 + Colorhit int64 + Colormiss int64 + Ncolors int64 + Bootpages int64 + Poolpages int64 +} + const SizeofClockinfo = 0x14 type Clockinfo struct { diff --git a/vendor/golang.org/x/sys/unix/ztypes_netbsd_arm.go b/vendor/golang.org/x/sys/unix/ztypes_netbsd_arm.go index 84cc8d0..db40e3a 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_netbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/ztypes_netbsd_arm.go @@ -496,6 +496,90 @@ type Utsname struct { Machine [256]byte } +const SizeofUvmexp = 0x278 + +type Uvmexp struct { + Pagesize int64 + Pagemask int64 + Pageshift int64 + Npages int64 + Free int64 + Active int64 + Inactive int64 + Paging int64 + Wired int64 + Zeropages int64 + Reserve_pagedaemon int64 + Reserve_kernel int64 + Freemin int64 + Freetarg int64 + Inactarg int64 + Wiredmax int64 + Nswapdev int64 + Swpages int64 + Swpginuse int64 + Swpgonly int64 + Nswget int64 + Unused1 int64 + Cpuhit int64 + Cpumiss int64 + Faults int64 + Traps int64 + Intrs int64 + Swtch int64 + Softs int64 + Syscalls int64 + Pageins int64 + Swapins int64 + Swapouts int64 + Pgswapin int64 + Pgswapout int64 + Forks int64 + Forks_ppwait int64 + Forks_sharevm int64 + Pga_zerohit int64 + Pga_zeromiss int64 + Zeroaborts int64 + Fltnoram int64 + Fltnoanon int64 + Fltpgwait int64 + Fltpgrele int64 + Fltrelck int64 + Fltrelckok int64 + Fltanget int64 + Fltanretry int64 + Fltamcopy int64 + Fltnamap int64 + Fltnomap int64 + Fltlget int64 + Fltget int64 + Flt_anon int64 + Flt_acow int64 + Flt_obj int64 + Flt_prcopy int64 + Flt_przero int64 + Pdwoke int64 + Pdrevs int64 + Unused4 int64 + Pdfreed int64 + Pdscans int64 + Pdanscan int64 + Pdobscan int64 + Pdreact int64 + Pdbusy int64 + Pdpageouts int64 + Pdpending int64 + Pddeact int64 + Anonpages int64 + Filepages int64 + Execpages int64 + Colorhit int64 + Colormiss int64 + Ncolors int64 + Bootpages int64 + Poolpages int64 +} + const SizeofClockinfo = 0x14 type Clockinfo struct { diff --git a/vendor/golang.org/x/sys/unix/ztypes_netbsd_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_netbsd_arm64.go index c844e70..1112115 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_netbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_netbsd_arm64.go @@ -499,6 +499,90 @@ type Utsname struct { Machine [256]byte } +const SizeofUvmexp = 0x278 + +type Uvmexp struct { + Pagesize int64 + Pagemask int64 + Pageshift int64 + Npages int64 + Free int64 + Active int64 + Inactive int64 + Paging int64 + Wired int64 + Zeropages int64 + Reserve_pagedaemon int64 + Reserve_kernel int64 + Freemin int64 + Freetarg int64 + Inactarg int64 + Wiredmax int64 + Nswapdev int64 + Swpages int64 + Swpginuse int64 + Swpgonly int64 + Nswget int64 + Unused1 int64 + Cpuhit int64 + Cpumiss int64 + Faults int64 + Traps int64 + Intrs int64 + Swtch int64 + Softs int64 + Syscalls int64 + Pageins int64 + Swapins int64 + Swapouts int64 + Pgswapin int64 + Pgswapout int64 + Forks int64 + Forks_ppwait int64 + Forks_sharevm int64 + Pga_zerohit int64 + Pga_zeromiss int64 + Zeroaborts int64 + Fltnoram int64 + Fltnoanon int64 + Fltpgwait int64 + Fltpgrele int64 + Fltrelck int64 + Fltrelckok int64 + Fltanget int64 + Fltanretry int64 + Fltamcopy int64 + Fltnamap int64 + Fltnomap int64 + Fltlget int64 + Fltget int64 + Flt_anon int64 + Flt_acow int64 + Flt_obj int64 + Flt_prcopy int64 + Flt_przero int64 + Pdwoke int64 + Pdrevs int64 + Unused4 int64 + Pdfreed int64 + Pdscans int64 + Pdanscan int64 + Pdobscan int64 + Pdreact int64 + Pdbusy int64 + Pdpageouts int64 + Pdpending int64 + Pddeact int64 + Anonpages int64 + Filepages int64 + Execpages int64 + Colorhit int64 + Colormiss int64 + Ncolors int64 + Bootpages int64 + Poolpages int64 +} + const SizeofClockinfo = 0x14 type Clockinfo struct { diff --git a/vendor/golang.org/x/sys/unix/ztypes_openbsd_386.go b/vendor/golang.org/x/sys/unix/ztypes_openbsd_386.go index 2ed718c..26eba23 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_openbsd_386.go +++ b/vendor/golang.org/x/sys/unix/ztypes_openbsd_386.go @@ -58,22 +58,22 @@ type Rlimit struct { type _Gid_t uint32 type Stat_t struct { - Mode uint32 - Dev int32 - Ino uint64 - Nlink uint32 - Uid uint32 - Gid uint32 - Rdev int32 - Atim Timespec - Mtim Timespec - Ctim Timespec - Size int64 - Blocks int64 - Blksize uint32 - Flags uint32 - Gen uint32 - X__st_birthtim Timespec + Mode uint32 + Dev int32 + Ino uint64 + Nlink uint32 + Uid uint32 + Gid uint32 + Rdev int32 + Atim Timespec + Mtim Timespec + Ctim Timespec + Size int64 + Blocks int64 + Blksize int32 + Flags uint32 + Gen uint32 + _ Timespec } type Statfs_t struct { @@ -98,7 +98,7 @@ type Statfs_t struct { F_mntonname [90]byte F_mntfromname [90]byte F_mntfromspec [90]byte - Pad_cgo_0 [2]byte + _ [2]byte Mount_info [160]byte } @@ -111,13 +111,13 @@ type Flock_t struct { } type Dirent struct { - Fileno uint64 - Off int64 - Reclen uint16 - Type uint8 - Namlen uint8 - X__d_padding [4]uint8 - Name [256]int8 + Fileno uint64 + Off int64 + Reclen uint16 + Type uint8 + Namlen uint8 + _ [4]uint8 + Name [256]int8 } type Fsid struct { @@ -262,8 +262,8 @@ type FdSet struct { } const ( - SizeofIfMsghdr = 0xec - SizeofIfData = 0xd4 + SizeofIfMsghdr = 0xa0 + SizeofIfData = 0x88 SizeofIfaMsghdr = 0x18 SizeofIfAnnounceMsghdr = 0x1a SizeofRtMsghdr = 0x60 @@ -292,7 +292,7 @@ type IfData struct { Link_state uint8 Mtu uint32 Metric uint32 - Pad uint32 + Rdomain uint32 Baudrate uint64 Ipackets uint64 Ierrors uint64 @@ -304,10 +304,10 @@ type IfData struct { Imcasts uint64 Omcasts uint64 Iqdrops uint64 + Oqdrops uint64 Noproto uint64 Capabilities uint32 Lastchange Timeval - Mclpool [7]Mclpool } type IfaMsghdr struct { @@ -368,20 +368,12 @@ type RtMetrics struct { Pad uint32 } -type Mclpool struct { - Grown int32 - Alive uint16 - Hwm uint16 - Cwm uint16 - Lwm uint16 -} - const ( SizeofBpfVersion = 0x4 SizeofBpfStat = 0x8 SizeofBpfProgram = 0x8 SizeofBpfInsn = 0x8 - SizeofBpfHdr = 0x14 + SizeofBpfHdr = 0x18 ) type BpfVersion struct { @@ -407,11 +399,14 @@ type BpfInsn struct { } type BpfHdr struct { - Tstamp BpfTimeval - Caplen uint32 - Datalen uint32 - Hdrlen uint16 - Pad_cgo_0 [2]byte + Tstamp BpfTimeval + Caplen uint32 + Datalen uint32 + Hdrlen uint16 + Ifidx uint16 + Flowid uint16 + Flags uint8 + Drops uint8 } type BpfTimeval struct { @@ -488,7 +483,7 @@ type Uvmexp struct { Zeropages int32 Reserve_pagedaemon int32 Reserve_kernel int32 - Anonpages int32 + Unused01 int32 Vnodepages int32 Vtextpages int32 Freemin int32 @@ -507,8 +502,8 @@ type Uvmexp struct { Swpgonly int32 Nswget int32 Nanon int32 - Nanonneeded int32 - Nfreeanon int32 + Unused05 int32 + Unused06 int32 Faults int32 Traps int32 Intrs int32 @@ -516,8 +511,8 @@ type Uvmexp struct { Softs int32 Syscalls int32 Pageins int32 - Obsolete_swapins int32 - Obsolete_swapouts int32 + Unused07 int32 + Unused08 int32 Pgswapin int32 Pgswapout int32 Forks int32 @@ -525,7 +520,7 @@ type Uvmexp struct { Forks_sharevm int32 Pga_zerohit int32 Pga_zeromiss int32 - Zeroaborts int32 + Unused09 int32 Fltnoram int32 Fltnoanon int32 Fltnoamap int32 @@ -557,9 +552,9 @@ type Uvmexp struct { Pdpageouts int32 Pdpending int32 Pddeact int32 - Pdreanon int32 - Pdrevnode int32 - Pdrevtext int32 + Unused11 int32 + Unused12 int32 + Unused13 int32 Fpswtch int32 Kmapent int32 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_openbsd_amd64.go b/vendor/golang.org/x/sys/unix/ztypes_openbsd_amd64.go index b4fb97e..5a54798 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_openbsd_amd64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_openbsd_amd64.go @@ -73,7 +73,6 @@ type Stat_t struct { Blksize int32 Flags uint32 Gen uint32 - _ [4]byte _ Timespec } @@ -81,7 +80,6 @@ type Statfs_t struct { F_flags uint32 F_bsize uint32 F_iosize uint32 - _ [4]byte F_blocks uint64 F_bfree uint64 F_bavail int64 @@ -200,10 +198,8 @@ type IPv6Mreq struct { type Msghdr struct { Name *byte Namelen uint32 - _ [4]byte Iov *Iovec Iovlen uint32 - _ [4]byte Control *byte Controllen uint32 Flags int32 @@ -311,7 +307,6 @@ type IfData struct { Oqdrops uint64 Noproto uint64 Capabilities uint32 - _ [4]byte Lastchange Timeval } @@ -373,14 +368,12 @@ type RtMetrics struct { Pad uint32 } -type Mclpool struct{} - const ( SizeofBpfVersion = 0x4 SizeofBpfStat = 0x8 SizeofBpfProgram = 0x10 SizeofBpfInsn = 0x8 - SizeofBpfHdr = 0x14 + SizeofBpfHdr = 0x18 ) type BpfVersion struct { @@ -395,7 +388,6 @@ type BpfStat struct { type BpfProgram struct { Len uint32 - _ [4]byte Insns *BpfInsn } @@ -411,7 +403,10 @@ type BpfHdr struct { Caplen uint32 Datalen uint32 Hdrlen uint16 - _ [2]byte + Ifidx uint16 + Flowid uint16 + Flags uint8 + Drops uint8 } type BpfTimeval struct { @@ -488,7 +483,7 @@ type Uvmexp struct { Zeropages int32 Reserve_pagedaemon int32 Reserve_kernel int32 - Anonpages int32 + Unused01 int32 Vnodepages int32 Vtextpages int32 Freemin int32 @@ -507,8 +502,8 @@ type Uvmexp struct { Swpgonly int32 Nswget int32 Nanon int32 - Nanonneeded int32 - Nfreeanon int32 + Unused05 int32 + Unused06 int32 Faults int32 Traps int32 Intrs int32 @@ -516,8 +511,8 @@ type Uvmexp struct { Softs int32 Syscalls int32 Pageins int32 - Obsolete_swapins int32 - Obsolete_swapouts int32 + Unused07 int32 + Unused08 int32 Pgswapin int32 Pgswapout int32 Forks int32 @@ -525,7 +520,7 @@ type Uvmexp struct { Forks_sharevm int32 Pga_zerohit int32 Pga_zeromiss int32 - Zeroaborts int32 + Unused09 int32 Fltnoram int32 Fltnoanon int32 Fltnoamap int32 @@ -557,9 +552,9 @@ type Uvmexp struct { Pdpageouts int32 Pdpending int32 Pddeact int32 - Pdreanon int32 - Pdrevnode int32 - Pdrevtext int32 + Unused11 int32 + Unused12 int32 + Unused13 int32 Fpswtch int32 Kmapent int32 } diff --git a/vendor/golang.org/x/sys/unix/ztypes_openbsd_arm.go b/vendor/golang.org/x/sys/unix/ztypes_openbsd_arm.go index 2c46750..be58c4e 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_openbsd_arm.go +++ b/vendor/golang.org/x/sys/unix/ztypes_openbsd_arm.go @@ -375,14 +375,12 @@ type RtMetrics struct { Pad uint32 } -type Mclpool struct{} - const ( SizeofBpfVersion = 0x4 SizeofBpfStat = 0x8 SizeofBpfProgram = 0x8 SizeofBpfInsn = 0x8 - SizeofBpfHdr = 0x14 + SizeofBpfHdr = 0x18 ) type BpfVersion struct { @@ -412,7 +410,10 @@ type BpfHdr struct { Caplen uint32 Datalen uint32 Hdrlen uint16 - _ [2]byte + Ifidx uint16 + Flowid uint16 + Flags uint8 + Drops uint8 } type BpfTimeval struct { diff --git a/vendor/golang.org/x/sys/unix/ztypes_openbsd_arm64.go b/vendor/golang.org/x/sys/unix/ztypes_openbsd_arm64.go index ddee045..5233826 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_openbsd_arm64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_openbsd_arm64.go @@ -368,14 +368,12 @@ type RtMetrics struct { Pad uint32 } -type Mclpool struct{} - const ( SizeofBpfVersion = 0x4 SizeofBpfStat = 0x8 SizeofBpfProgram = 0x10 SizeofBpfInsn = 0x8 - SizeofBpfHdr = 0x14 + SizeofBpfHdr = 0x18 ) type BpfVersion struct { @@ -405,7 +403,10 @@ type BpfHdr struct { Caplen uint32 Datalen uint32 Hdrlen uint16 - _ [2]byte + Ifidx uint16 + Flowid uint16 + Flags uint8 + Drops uint8 } type BpfTimeval struct { diff --git a/vendor/golang.org/x/sys/unix/ztypes_openbsd_mips64.go b/vendor/golang.org/x/sys/unix/ztypes_openbsd_mips64.go index eb13d4e..605cfdb 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_openbsd_mips64.go +++ b/vendor/golang.org/x/sys/unix/ztypes_openbsd_mips64.go @@ -368,14 +368,12 @@ type RtMetrics struct { Pad uint32 } -type Mclpool struct{} - const ( SizeofBpfVersion = 0x4 SizeofBpfStat = 0x8 SizeofBpfProgram = 0x10 SizeofBpfInsn = 0x8 - SizeofBpfHdr = 0x14 + SizeofBpfHdr = 0x18 ) type BpfVersion struct { @@ -405,7 +403,10 @@ type BpfHdr struct { Caplen uint32 Datalen uint32 Hdrlen uint16 - _ [2]byte + Ifidx uint16 + Flowid uint16 + Flags uint8 + Drops uint8 } type BpfTimeval struct { diff --git a/vendor/golang.org/x/sys/windows/env_windows.go b/vendor/golang.org/x/sys/windows/env_windows.go index 92ac05f..b8ad192 100644 --- a/vendor/golang.org/x/sys/windows/env_windows.go +++ b/vendor/golang.org/x/sys/windows/env_windows.go @@ -37,14 +37,14 @@ func (token Token) Environ(inheritExisting bool) (env []string, err error) { return nil, err } defer DestroyEnvironmentBlock(block) - blockp := uintptr(unsafe.Pointer(block)) + blockp := unsafe.Pointer(block) for { - entry := UTF16PtrToString((*uint16)(unsafe.Pointer(blockp))) + entry := UTF16PtrToString((*uint16)(blockp)) if len(entry) == 0 { break } env = append(env, entry) - blockp += 2 * (uintptr(len(entry)) + 1) + blockp = unsafe.Add(blockp, 2*(len(entry)+1)) } return env, nil } diff --git a/vendor/golang.org/x/sys/windows/exec_windows.go b/vendor/golang.org/x/sys/windows/exec_windows.go index 75980fd..a52e033 100644 --- a/vendor/golang.org/x/sys/windows/exec_windows.go +++ b/vendor/golang.org/x/sys/windows/exec_windows.go @@ -95,12 +95,17 @@ func ComposeCommandLine(args []string) string { // DecomposeCommandLine breaks apart its argument command line into unescaped parts using CommandLineToArgv, // as gathered from GetCommandLine, QUERY_SERVICE_CONFIG's BinaryPathName argument, or elsewhere that // command lines are passed around. +// DecomposeCommandLine returns error if commandLine contains NUL. func DecomposeCommandLine(commandLine string) ([]string, error) { if len(commandLine) == 0 { return []string{}, nil } + utf16CommandLine, err := UTF16FromString(commandLine) + if err != nil { + return nil, errorspkg.New("string with NUL passed to DecomposeCommandLine") + } var argc int32 - argv, err := CommandLineToArgv(StringToUTF16Ptr(commandLine), &argc) + argv, err := CommandLineToArgv(&utf16CommandLine[0], &argc) if err != nil { return nil, err } diff --git a/vendor/golang.org/x/sys/windows/service.go b/vendor/golang.org/x/sys/windows/service.go index f8deca8..c964b68 100644 --- a/vendor/golang.org/x/sys/windows/service.go +++ b/vendor/golang.org/x/sys/windows/service.go @@ -141,6 +141,12 @@ const ( SERVICE_DYNAMIC_INFORMATION_LEVEL_START_REASON = 1 ) +type ENUM_SERVICE_STATUS struct { + ServiceName *uint16 + DisplayName *uint16 + ServiceStatus SERVICE_STATUS +} + type SERVICE_STATUS struct { ServiceType uint32 CurrentState uint32 @@ -245,3 +251,4 @@ type QUERY_SERVICE_LOCK_STATUS struct { //sys UnsubscribeServiceChangeNotifications(subscription uintptr) = sechost.UnsubscribeServiceChangeNotifications? //sys RegisterServiceCtrlHandlerEx(serviceName *uint16, handlerProc uintptr, context uintptr) (handle Handle, err error) = advapi32.RegisterServiceCtrlHandlerExW //sys QueryServiceDynamicInformation(service Handle, infoLevel uint32, dynamicInfo unsafe.Pointer) (err error) = advapi32.QueryServiceDynamicInformation? +//sys EnumDependentServices(service Handle, activityState uint32, services *ENUM_SERVICE_STATUS, buffSize uint32, bytesNeeded *uint32, servicesReturned *uint32) (err error) = advapi32.EnumDependentServicesW diff --git a/vendor/golang.org/x/sys/windows/syscall_windows.go b/vendor/golang.org/x/sys/windows/syscall_windows.go index 7a6ba43..3723b2c 100644 --- a/vendor/golang.org/x/sys/windows/syscall_windows.go +++ b/vendor/golang.org/x/sys/windows/syscall_windows.go @@ -10,7 +10,6 @@ import ( errorspkg "errors" "fmt" "runtime" - "strings" "sync" "syscall" "time" @@ -87,22 +86,13 @@ func StringToUTF16(s string) []uint16 { // s, with a terminating NUL added. If s contains a NUL byte at any // location, it returns (nil, syscall.EINVAL). func UTF16FromString(s string) ([]uint16, error) { - if strings.IndexByte(s, 0) != -1 { - return nil, syscall.EINVAL - } - return utf16.Encode([]rune(s + "\x00")), nil + return syscall.UTF16FromString(s) } // UTF16ToString returns the UTF-8 encoding of the UTF-16 sequence s, // with a terminating NUL and any bytes after the NUL removed. func UTF16ToString(s []uint16) string { - for i, v := range s { - if v == 0 { - s = s[:i] - break - } - } - return string(utf16.Decode(s)) + return syscall.UTF16ToString(s) } // StringToUTF16Ptr is deprecated. Use UTF16PtrFromString instead. @@ -367,6 +357,7 @@ func NewCallbackCDecl(fn interface{}) uintptr { //sys IsWindowUnicode(hwnd HWND) (isUnicode bool) = user32.IsWindowUnicode //sys IsWindowVisible(hwnd HWND) (isVisible bool) = user32.IsWindowVisible //sys GetGUIThreadInfo(thread uint32, info *GUIThreadInfo) (err error) = user32.GetGUIThreadInfo +//sys GetLargePageMinimum() (size uintptr) // Volume Management Functions //sys DefineDosDevice(flags uint32, deviceName *uint16, targetPath *uint16) (err error) = DefineDosDeviceW @@ -833,6 +824,9 @@ const socket_error = uintptr(^uint32(0)) //sys WSAStartup(verreq uint32, data *WSAData) (sockerr error) = ws2_32.WSAStartup //sys WSACleanup() (err error) [failretval==socket_error] = ws2_32.WSACleanup //sys WSAIoctl(s Handle, iocc uint32, inbuf *byte, cbif uint32, outbuf *byte, cbob uint32, cbbr *uint32, overlapped *Overlapped, completionRoutine uintptr) (err error) [failretval==socket_error] = ws2_32.WSAIoctl +//sys WSALookupServiceBegin(querySet *WSAQUERYSET, flags uint32, handle *Handle) (err error) [failretval==socket_error] = ws2_32.WSALookupServiceBeginW +//sys WSALookupServiceNext(handle Handle, flags uint32, size *int32, querySet *WSAQUERYSET) (err error) [failretval==socket_error] = ws2_32.WSALookupServiceNextW +//sys WSALookupServiceEnd(handle Handle) (err error) [failretval==socket_error] = ws2_32.WSALookupServiceEnd //sys socket(af int32, typ int32, protocol int32) (handle Handle, err error) [failretval==InvalidHandle] = ws2_32.socket //sys sendto(s Handle, buf []byte, flags int32, to unsafe.Pointer, tolen int32) (err error) [failretval==socket_error] = ws2_32.sendto //sys recvfrom(s Handle, buf []byte, flags int32, from *RawSockaddrAny, fromlen *int32) (n int32, err error) [failretval==-1] = ws2_32.recvfrom @@ -1028,8 +1022,7 @@ func (rsa *RawSockaddrAny) Sockaddr() (Sockaddr, error) { for n < len(pp.Path) && pp.Path[n] != 0 { n++ } - bytes := (*[len(pp.Path)]byte)(unsafe.Pointer(&pp.Path[0]))[0:n] - sa.Name = string(bytes) + sa.Name = string(unsafe.Slice((*byte)(unsafe.Pointer(&pp.Path[0])), n)) return sa, nil case AF_INET: diff --git a/vendor/golang.org/x/sys/windows/types_windows.go b/vendor/golang.org/x/sys/windows/types_windows.go index 0c4add9..88e62a6 100644 --- a/vendor/golang.org/x/sys/windows/types_windows.go +++ b/vendor/golang.org/x/sys/windows/types_windows.go @@ -1243,6 +1243,51 @@ const ( DnsSectionAdditional = 0x0003 ) +const ( + // flags of WSALookupService + LUP_DEEP = 0x0001 + LUP_CONTAINERS = 0x0002 + LUP_NOCONTAINERS = 0x0004 + LUP_NEAREST = 0x0008 + LUP_RETURN_NAME = 0x0010 + LUP_RETURN_TYPE = 0x0020 + LUP_RETURN_VERSION = 0x0040 + LUP_RETURN_COMMENT = 0x0080 + LUP_RETURN_ADDR = 0x0100 + LUP_RETURN_BLOB = 0x0200 + LUP_RETURN_ALIASES = 0x0400 + LUP_RETURN_QUERY_STRING = 0x0800 + LUP_RETURN_ALL = 0x0FF0 + LUP_RES_SERVICE = 0x8000 + + LUP_FLUSHCACHE = 0x1000 + LUP_FLUSHPREVIOUS = 0x2000 + + LUP_NON_AUTHORITATIVE = 0x4000 + LUP_SECURE = 0x8000 + LUP_RETURN_PREFERRED_NAMES = 0x10000 + LUP_DNS_ONLY = 0x20000 + + LUP_ADDRCONFIG = 0x100000 + LUP_DUAL_ADDR = 0x200000 + LUP_FILESERVER = 0x400000 + LUP_DISABLE_IDN_ENCODING = 0x00800000 + LUP_API_ANSI = 0x01000000 + + LUP_RESOLUTION_HANDLE = 0x80000000 +) + +const ( + // values of WSAQUERYSET's namespace + NS_ALL = 0 + NS_DNS = 12 + NS_NLA = 15 + NS_BTH = 16 + NS_EMAIL = 37 + NS_PNRPNAME = 38 + NS_PNRPCLOUD = 39 +) + type DNSSRVData struct { Target *uint16 Priority uint16 @@ -2175,19 +2220,23 @@ type JOBOBJECT_BASIC_UI_RESTRICTIONS struct { } const ( - // JobObjectInformationClass + // JobObjectInformationClass for QueryInformationJobObject and SetInformationJobObject JobObjectAssociateCompletionPortInformation = 7 + JobObjectBasicAccountingInformation = 1 + JobObjectBasicAndIoAccountingInformation = 8 JobObjectBasicLimitInformation = 2 + JobObjectBasicProcessIdList = 3 JobObjectBasicUIRestrictions = 4 JobObjectCpuRateControlInformation = 15 JobObjectEndOfJobTimeInformation = 6 JobObjectExtendedLimitInformation = 9 JobObjectGroupInformation = 11 JobObjectGroupInformationEx = 14 - JobObjectLimitViolationInformation2 = 35 + JobObjectLimitViolationInformation = 13 + JobObjectLimitViolationInformation2 = 34 JobObjectNetRateControlInformation = 32 JobObjectNotificationLimitInformation = 12 - JobObjectNotificationLimitInformation2 = 34 + JobObjectNotificationLimitInformation2 = 33 JobObjectSecurityLimitInformation = 5 ) @@ -3258,3 +3307,43 @@ const ( DWMWA_TEXT_COLOR = 36 DWMWA_VISIBLE_FRAME_BORDER_THICKNESS = 37 ) + +type WSAQUERYSET struct { + Size uint32 + ServiceInstanceName *uint16 + ServiceClassId *GUID + Version *WSAVersion + Comment *uint16 + NameSpace uint32 + NSProviderId *GUID + Context *uint16 + NumberOfProtocols uint32 + AfpProtocols *AFProtocols + QueryString *uint16 + NumberOfCsAddrs uint32 + SaBuffer *CSAddrInfo + OutputFlags uint32 + Blob *BLOB +} + +type WSAVersion struct { + Version uint32 + EnumerationOfComparison int32 +} + +type AFProtocols struct { + AddressFamily int32 + Protocol int32 +} + +type CSAddrInfo struct { + LocalAddr SocketAddress + RemoteAddr SocketAddress + SocketType int32 + Protocol int32 +} + +type BLOB struct { + Size uint32 + BlobData *byte +} diff --git a/vendor/golang.org/x/sys/windows/zsyscall_windows.go b/vendor/golang.org/x/sys/windows/zsyscall_windows.go index 96ba855..a81ea2c 100644 --- a/vendor/golang.org/x/sys/windows/zsyscall_windows.go +++ b/vendor/golang.org/x/sys/windows/zsyscall_windows.go @@ -86,6 +86,7 @@ var ( procDeleteService = modadvapi32.NewProc("DeleteService") procDeregisterEventSource = modadvapi32.NewProc("DeregisterEventSource") procDuplicateTokenEx = modadvapi32.NewProc("DuplicateTokenEx") + procEnumDependentServicesW = modadvapi32.NewProc("EnumDependentServicesW") procEnumServicesStatusExW = modadvapi32.NewProc("EnumServicesStatusExW") procEqualSid = modadvapi32.NewProc("EqualSid") procFreeSid = modadvapi32.NewProc("FreeSid") @@ -252,6 +253,7 @@ var ( procGetFileType = modkernel32.NewProc("GetFileType") procGetFinalPathNameByHandleW = modkernel32.NewProc("GetFinalPathNameByHandleW") procGetFullPathNameW = modkernel32.NewProc("GetFullPathNameW") + procGetLargePageMinimum = modkernel32.NewProc("GetLargePageMinimum") procGetLastError = modkernel32.NewProc("GetLastError") procGetLogicalDriveStringsW = modkernel32.NewProc("GetLogicalDriveStringsW") procGetLogicalDrives = modkernel32.NewProc("GetLogicalDrives") @@ -473,6 +475,9 @@ var ( procWSAEnumProtocolsW = modws2_32.NewProc("WSAEnumProtocolsW") procWSAGetOverlappedResult = modws2_32.NewProc("WSAGetOverlappedResult") procWSAIoctl = modws2_32.NewProc("WSAIoctl") + procWSALookupServiceBeginW = modws2_32.NewProc("WSALookupServiceBeginW") + procWSALookupServiceEnd = modws2_32.NewProc("WSALookupServiceEnd") + procWSALookupServiceNextW = modws2_32.NewProc("WSALookupServiceNextW") procWSARecv = modws2_32.NewProc("WSARecv") procWSARecvFrom = modws2_32.NewProc("WSARecvFrom") procWSASend = modws2_32.NewProc("WSASend") @@ -730,6 +735,14 @@ func DuplicateTokenEx(existingToken Token, desiredAccess uint32, tokenAttributes return } +func EnumDependentServices(service Handle, activityState uint32, services *ENUM_SERVICE_STATUS, buffSize uint32, bytesNeeded *uint32, servicesReturned *uint32) (err error) { + r1, _, e1 := syscall.Syscall6(procEnumDependentServicesW.Addr(), 6, uintptr(service), uintptr(activityState), uintptr(unsafe.Pointer(services)), uintptr(buffSize), uintptr(unsafe.Pointer(bytesNeeded)), uintptr(unsafe.Pointer(servicesReturned))) + if r1 == 0 { + err = errnoErr(e1) + } + return +} + func EnumServicesStatusEx(mgr Handle, infoLevel uint32, serviceType uint32, serviceState uint32, services *byte, bufSize uint32, bytesNeeded *uint32, servicesReturned *uint32, resumeHandle *uint32, groupName *uint16) (err error) { r1, _, e1 := syscall.Syscall12(procEnumServicesStatusExW.Addr(), 10, uintptr(mgr), uintptr(infoLevel), uintptr(serviceType), uintptr(serviceState), uintptr(unsafe.Pointer(services)), uintptr(bufSize), uintptr(unsafe.Pointer(bytesNeeded)), uintptr(unsafe.Pointer(servicesReturned)), uintptr(unsafe.Pointer(resumeHandle)), uintptr(unsafe.Pointer(groupName)), 0, 0) if r1 == 0 { @@ -2180,6 +2193,12 @@ func GetFullPathName(path *uint16, buflen uint32, buf *uint16, fname **uint16) ( return } +func GetLargePageMinimum() (size uintptr) { + r0, _, _ := syscall.Syscall(procGetLargePageMinimum.Addr(), 0, 0, 0, 0) + size = uintptr(r0) + return +} + func GetLastError() (lasterr error) { r0, _, _ := syscall.Syscall(procGetLastError.Addr(), 0, 0, 0, 0) if r0 != 0 { @@ -4060,6 +4079,30 @@ func WSAIoctl(s Handle, iocc uint32, inbuf *byte, cbif uint32, outbuf *byte, cbo return } +func WSALookupServiceBegin(querySet *WSAQUERYSET, flags uint32, handle *Handle) (err error) { + r1, _, e1 := syscall.Syscall(procWSALookupServiceBeginW.Addr(), 3, uintptr(unsafe.Pointer(querySet)), uintptr(flags), uintptr(unsafe.Pointer(handle))) + if r1 == socket_error { + err = errnoErr(e1) + } + return +} + +func WSALookupServiceEnd(handle Handle) (err error) { + r1, _, e1 := syscall.Syscall(procWSALookupServiceEnd.Addr(), 1, uintptr(handle), 0, 0) + if r1 == socket_error { + err = errnoErr(e1) + } + return +} + +func WSALookupServiceNext(handle Handle, flags uint32, size *int32, querySet *WSAQUERYSET) (err error) { + r1, _, e1 := syscall.Syscall6(procWSALookupServiceNextW.Addr(), 4, uintptr(handle), uintptr(flags), uintptr(unsafe.Pointer(size)), uintptr(unsafe.Pointer(querySet)), 0, 0) + if r1 == socket_error { + err = errnoErr(e1) + } + return +} + func WSARecv(s Handle, bufs *WSABuf, bufcnt uint32, recvd *uint32, flags *uint32, overlapped *Overlapped, croutine *byte) (err error) { r1, _, e1 := syscall.Syscall9(procWSARecv.Addr(), 7, uintptr(s), uintptr(unsafe.Pointer(bufs)), uintptr(bufcnt), uintptr(unsafe.Pointer(recvd)), uintptr(unsafe.Pointer(flags)), uintptr(unsafe.Pointer(overlapped)), uintptr(unsafe.Pointer(croutine)), 0, 0) if r1 == socket_error { diff --git a/vendor/golang.org/x/text/internal/language/compact/language.go b/vendor/golang.org/x/text/internal/language/compact/language.go index 83816a7..8c1b666 100644 --- a/vendor/golang.org/x/text/internal/language/compact/language.go +++ b/vendor/golang.org/x/text/internal/language/compact/language.go @@ -118,7 +118,7 @@ func (t Tag) Parent() Tag { return Tag{language: lang, locale: lang} } -// returns token t and the rest of the string. +// nextToken returns token t and the rest of the string. func nextToken(s string) (t, tail string) { p := strings.Index(s[1:], "-") if p == -1 { diff --git a/vendor/golang.org/x/text/internal/language/language.go b/vendor/golang.org/x/text/internal/language/language.go index 6105bc7..09d41c7 100644 --- a/vendor/golang.org/x/text/internal/language/language.go +++ b/vendor/golang.org/x/text/internal/language/language.go @@ -409,7 +409,7 @@ func (t Tag) SetTypeForKey(key, value string) (Tag, error) { return t, nil } -// findKeyAndType returns the start and end position for the type corresponding +// findTypeForKey returns the start and end position for the type corresponding // to key or the point at which to insert the key-value pair if the type // wasn't found. The hasExt return value reports whether an -u extension was present. // Note: the extensions are typically very small and are likely to contain diff --git a/vendor/golang.org/x/text/language/language.go b/vendor/golang.org/x/text/language/language.go index 289b3a3..4d9c661 100644 --- a/vendor/golang.org/x/text/language/language.go +++ b/vendor/golang.org/x/text/language/language.go @@ -344,7 +344,7 @@ func (t Tag) Parent() Tag { return Tag(compact.Tag(t).Parent()) } -// returns token t and the rest of the string. +// nextToken returns token t and the rest of the string. func nextToken(s string) (t, tail string) { p := strings.Index(s[1:], "-") if p == -1 { diff --git a/vendor/golang.org/x/text/unicode/norm/forminfo.go b/vendor/golang.org/x/text/unicode/norm/forminfo.go index d69ccb4..487335d 100644 --- a/vendor/golang.org/x/text/unicode/norm/forminfo.go +++ b/vendor/golang.org/x/text/unicode/norm/forminfo.go @@ -13,7 +13,7 @@ import "encoding/binary" // a rune to a uint16. The values take two forms. For v >= 0x8000: // bits // 15: 1 (inverse of NFD_QC bit of qcInfo) -// 13..7: qcInfo (see below). isYesD is always true (no decompostion). +// 13..7: qcInfo (see below). isYesD is always true (no decomposition). // 6..0: ccc (compressed CCC value). // For v < 0x8000, the respective rune has a decomposition and v is an index // into a byte array of UTF-8 decomposition sequences and additional info and diff --git a/vendor/google.golang.org/protobuf/encoding/protowire/wire.go b/vendor/google.golang.org/protobuf/encoding/protowire/wire.go index ce57f57..f4b4686 100644 --- a/vendor/google.golang.org/protobuf/encoding/protowire/wire.go +++ b/vendor/google.golang.org/protobuf/encoding/protowire/wire.go @@ -3,7 +3,7 @@ // license that can be found in the LICENSE file. // Package protowire parses and formats the raw wire encoding. -// See https://developers.google.com/protocol-buffers/docs/encoding. +// See https://protobuf.dev/programming-guides/encoding. // // For marshaling and unmarshaling entire protobuf messages, // use the "google.golang.org/protobuf/proto" package instead. @@ -29,12 +29,8 @@ const ( ) // IsValid reports whether the field number is semantically valid. -// -// Note that while numbers within the reserved range are semantically invalid, -// they are syntactically valid in the wire format. -// Implementations may treat records with reserved field numbers as unknown. func (n Number) IsValid() bool { - return MinValidNumber <= n && n < FirstReservedNumber || LastReservedNumber < n && n <= MaxValidNumber + return MinValidNumber <= n && n <= MaxValidNumber } // Type represents the wire type. diff --git a/vendor/google.golang.org/protobuf/internal/encoding/text/decode.go b/vendor/google.golang.org/protobuf/internal/encoding/text/decode.go index 427c62d..87853e7 100644 --- a/vendor/google.golang.org/protobuf/internal/encoding/text/decode.go +++ b/vendor/google.golang.org/protobuf/internal/encoding/text/decode.go @@ -412,12 +412,13 @@ func (d *Decoder) parseFieldName() (tok Token, err error) { // Field number. Identify if input is a valid number that is not negative // and is decimal integer within 32-bit range. if num := parseNumber(d.in); num.size > 0 { + str := num.string(d.in) if !num.neg && num.kind == numDec { - if _, err := strconv.ParseInt(string(d.in[:num.size]), 10, 32); err == nil { + if _, err := strconv.ParseInt(str, 10, 32); err == nil { return d.consumeToken(Name, num.size, uint8(FieldNumber)), nil } } - return Token{}, d.newSyntaxError("invalid field number: %s", d.in[:num.size]) + return Token{}, d.newSyntaxError("invalid field number: %s", str) } return Token{}, d.newSyntaxError("invalid field name: %s", errId(d.in)) diff --git a/vendor/google.golang.org/protobuf/internal/encoding/text/decode_number.go b/vendor/google.golang.org/protobuf/internal/encoding/text/decode_number.go index 81a5d8c..45c81f0 100644 --- a/vendor/google.golang.org/protobuf/internal/encoding/text/decode_number.go +++ b/vendor/google.golang.org/protobuf/internal/encoding/text/decode_number.go @@ -15,17 +15,12 @@ func (d *Decoder) parseNumberValue() (Token, bool) { if num.neg { numAttrs |= isNegative } - strSize := num.size - last := num.size - 1 - if num.kind == numFloat && (d.in[last] == 'f' || d.in[last] == 'F') { - strSize = last - } tok := Token{ kind: Scalar, attrs: numberValue, pos: len(d.orig) - len(d.in), raw: d.in[:num.size], - str: string(d.in[:strSize]), + str: num.string(d.in), numAttrs: numAttrs, } d.consume(num.size) @@ -46,6 +41,27 @@ type number struct { kind uint8 neg bool size int + // if neg, this is the length of whitespace and comments between + // the minus sign and the rest fo the number literal + sep int +} + +func (num number) string(data []byte) string { + strSize := num.size + last := num.size - 1 + if num.kind == numFloat && (data[last] == 'f' || data[last] == 'F') { + strSize = last + } + if num.neg && num.sep > 0 { + // strip whitespace/comments between negative sign and the rest + strLen := strSize - num.sep + str := make([]byte, strLen) + str[0] = data[0] + copy(str[1:], data[num.sep+1:strSize]) + return string(str) + } + return string(data[:strSize]) + } // parseNumber constructs a number object from given input. It allows for the @@ -67,19 +83,22 @@ func parseNumber(input []byte) number { } // Optional - + var sep int if s[0] == '-' { neg = true s = s[1:] size++ + // Consume any whitespace or comments between the + // negative sign and the rest of the number + lenBefore := len(s) + s = consume(s, 0) + sep = lenBefore - len(s) + size += sep if len(s) == 0 { return number{} } } - // C++ allows for whitespace and comments in between the negative sign and - // the rest of the number. This logic currently does not but is consistent - // with v1. - switch { case s[0] == '0': if len(s) > 1 { @@ -116,7 +135,7 @@ func parseNumber(input []byte) number { if len(s) > 0 && !isDelim(s[0]) { return number{} } - return number{kind: kind, neg: neg, size: size} + return number{kind: kind, neg: neg, size: size, sep: sep} } } s = s[1:] @@ -188,5 +207,5 @@ func parseNumber(input []byte) number { return number{} } - return number{kind: kind, neg: neg, size: size} + return number{kind: kind, neg: neg, size: size, sep: sep} } diff --git a/vendor/google.golang.org/protobuf/internal/genid/descriptor_gen.go b/vendor/google.golang.org/protobuf/internal/genid/descriptor_gen.go index e3cdf1c..5c0e8f7 100644 --- a/vendor/google.golang.org/protobuf/internal/genid/descriptor_gen.go +++ b/vendor/google.golang.org/protobuf/internal/genid/descriptor_gen.go @@ -50,6 +50,7 @@ const ( FileDescriptorProto_Options_field_name protoreflect.Name = "options" FileDescriptorProto_SourceCodeInfo_field_name protoreflect.Name = "source_code_info" FileDescriptorProto_Syntax_field_name protoreflect.Name = "syntax" + FileDescriptorProto_Edition_field_name protoreflect.Name = "edition" FileDescriptorProto_Name_field_fullname protoreflect.FullName = "google.protobuf.FileDescriptorProto.name" FileDescriptorProto_Package_field_fullname protoreflect.FullName = "google.protobuf.FileDescriptorProto.package" @@ -63,6 +64,7 @@ const ( FileDescriptorProto_Options_field_fullname protoreflect.FullName = "google.protobuf.FileDescriptorProto.options" FileDescriptorProto_SourceCodeInfo_field_fullname protoreflect.FullName = "google.protobuf.FileDescriptorProto.source_code_info" FileDescriptorProto_Syntax_field_fullname protoreflect.FullName = "google.protobuf.FileDescriptorProto.syntax" + FileDescriptorProto_Edition_field_fullname protoreflect.FullName = "google.protobuf.FileDescriptorProto.edition" ) // Field numbers for google.protobuf.FileDescriptorProto. @@ -79,6 +81,7 @@ const ( FileDescriptorProto_Options_field_number protoreflect.FieldNumber = 8 FileDescriptorProto_SourceCodeInfo_field_number protoreflect.FieldNumber = 9 FileDescriptorProto_Syntax_field_number protoreflect.FieldNumber = 12 + FileDescriptorProto_Edition_field_number protoreflect.FieldNumber = 13 ) // Names for google.protobuf.DescriptorProto. @@ -494,26 +497,29 @@ const ( // Field names for google.protobuf.MessageOptions. const ( - MessageOptions_MessageSetWireFormat_field_name protoreflect.Name = "message_set_wire_format" - MessageOptions_NoStandardDescriptorAccessor_field_name protoreflect.Name = "no_standard_descriptor_accessor" - MessageOptions_Deprecated_field_name protoreflect.Name = "deprecated" - MessageOptions_MapEntry_field_name protoreflect.Name = "map_entry" - MessageOptions_UninterpretedOption_field_name protoreflect.Name = "uninterpreted_option" + MessageOptions_MessageSetWireFormat_field_name protoreflect.Name = "message_set_wire_format" + MessageOptions_NoStandardDescriptorAccessor_field_name protoreflect.Name = "no_standard_descriptor_accessor" + MessageOptions_Deprecated_field_name protoreflect.Name = "deprecated" + MessageOptions_MapEntry_field_name protoreflect.Name = "map_entry" + MessageOptions_DeprecatedLegacyJsonFieldConflicts_field_name protoreflect.Name = "deprecated_legacy_json_field_conflicts" + MessageOptions_UninterpretedOption_field_name protoreflect.Name = "uninterpreted_option" - MessageOptions_MessageSetWireFormat_field_fullname protoreflect.FullName = "google.protobuf.MessageOptions.message_set_wire_format" - MessageOptions_NoStandardDescriptorAccessor_field_fullname protoreflect.FullName = "google.protobuf.MessageOptions.no_standard_descriptor_accessor" - MessageOptions_Deprecated_field_fullname protoreflect.FullName = "google.protobuf.MessageOptions.deprecated" - MessageOptions_MapEntry_field_fullname protoreflect.FullName = "google.protobuf.MessageOptions.map_entry" - MessageOptions_UninterpretedOption_field_fullname protoreflect.FullName = "google.protobuf.MessageOptions.uninterpreted_option" + MessageOptions_MessageSetWireFormat_field_fullname protoreflect.FullName = "google.protobuf.MessageOptions.message_set_wire_format" + MessageOptions_NoStandardDescriptorAccessor_field_fullname protoreflect.FullName = "google.protobuf.MessageOptions.no_standard_descriptor_accessor" + MessageOptions_Deprecated_field_fullname protoreflect.FullName = "google.protobuf.MessageOptions.deprecated" + MessageOptions_MapEntry_field_fullname protoreflect.FullName = "google.protobuf.MessageOptions.map_entry" + MessageOptions_DeprecatedLegacyJsonFieldConflicts_field_fullname protoreflect.FullName = "google.protobuf.MessageOptions.deprecated_legacy_json_field_conflicts" + MessageOptions_UninterpretedOption_field_fullname protoreflect.FullName = "google.protobuf.MessageOptions.uninterpreted_option" ) // Field numbers for google.protobuf.MessageOptions. const ( - MessageOptions_MessageSetWireFormat_field_number protoreflect.FieldNumber = 1 - MessageOptions_NoStandardDescriptorAccessor_field_number protoreflect.FieldNumber = 2 - MessageOptions_Deprecated_field_number protoreflect.FieldNumber = 3 - MessageOptions_MapEntry_field_number protoreflect.FieldNumber = 7 - MessageOptions_UninterpretedOption_field_number protoreflect.FieldNumber = 999 + MessageOptions_MessageSetWireFormat_field_number protoreflect.FieldNumber = 1 + MessageOptions_NoStandardDescriptorAccessor_field_number protoreflect.FieldNumber = 2 + MessageOptions_Deprecated_field_number protoreflect.FieldNumber = 3 + MessageOptions_MapEntry_field_number protoreflect.FieldNumber = 7 + MessageOptions_DeprecatedLegacyJsonFieldConflicts_field_number protoreflect.FieldNumber = 11 + MessageOptions_UninterpretedOption_field_number protoreflect.FieldNumber = 999 ) // Names for google.protobuf.FieldOptions. @@ -528,16 +534,24 @@ const ( FieldOptions_Packed_field_name protoreflect.Name = "packed" FieldOptions_Jstype_field_name protoreflect.Name = "jstype" FieldOptions_Lazy_field_name protoreflect.Name = "lazy" + FieldOptions_UnverifiedLazy_field_name protoreflect.Name = "unverified_lazy" FieldOptions_Deprecated_field_name protoreflect.Name = "deprecated" FieldOptions_Weak_field_name protoreflect.Name = "weak" + FieldOptions_DebugRedact_field_name protoreflect.Name = "debug_redact" + FieldOptions_Retention_field_name protoreflect.Name = "retention" + FieldOptions_Target_field_name protoreflect.Name = "target" FieldOptions_UninterpretedOption_field_name protoreflect.Name = "uninterpreted_option" FieldOptions_Ctype_field_fullname protoreflect.FullName = "google.protobuf.FieldOptions.ctype" FieldOptions_Packed_field_fullname protoreflect.FullName = "google.protobuf.FieldOptions.packed" FieldOptions_Jstype_field_fullname protoreflect.FullName = "google.protobuf.FieldOptions.jstype" FieldOptions_Lazy_field_fullname protoreflect.FullName = "google.protobuf.FieldOptions.lazy" + FieldOptions_UnverifiedLazy_field_fullname protoreflect.FullName = "google.protobuf.FieldOptions.unverified_lazy" FieldOptions_Deprecated_field_fullname protoreflect.FullName = "google.protobuf.FieldOptions.deprecated" FieldOptions_Weak_field_fullname protoreflect.FullName = "google.protobuf.FieldOptions.weak" + FieldOptions_DebugRedact_field_fullname protoreflect.FullName = "google.protobuf.FieldOptions.debug_redact" + FieldOptions_Retention_field_fullname protoreflect.FullName = "google.protobuf.FieldOptions.retention" + FieldOptions_Target_field_fullname protoreflect.FullName = "google.protobuf.FieldOptions.target" FieldOptions_UninterpretedOption_field_fullname protoreflect.FullName = "google.protobuf.FieldOptions.uninterpreted_option" ) @@ -547,8 +561,12 @@ const ( FieldOptions_Packed_field_number protoreflect.FieldNumber = 2 FieldOptions_Jstype_field_number protoreflect.FieldNumber = 6 FieldOptions_Lazy_field_number protoreflect.FieldNumber = 5 + FieldOptions_UnverifiedLazy_field_number protoreflect.FieldNumber = 15 FieldOptions_Deprecated_field_number protoreflect.FieldNumber = 3 FieldOptions_Weak_field_number protoreflect.FieldNumber = 10 + FieldOptions_DebugRedact_field_number protoreflect.FieldNumber = 16 + FieldOptions_Retention_field_number protoreflect.FieldNumber = 17 + FieldOptions_Target_field_number protoreflect.FieldNumber = 18 FieldOptions_UninterpretedOption_field_number protoreflect.FieldNumber = 999 ) @@ -564,6 +582,18 @@ const ( FieldOptions_JSType_enum_name = "JSType" ) +// Full and short names for google.protobuf.FieldOptions.OptionRetention. +const ( + FieldOptions_OptionRetention_enum_fullname = "google.protobuf.FieldOptions.OptionRetention" + FieldOptions_OptionRetention_enum_name = "OptionRetention" +) + +// Full and short names for google.protobuf.FieldOptions.OptionTargetType. +const ( + FieldOptions_OptionTargetType_enum_fullname = "google.protobuf.FieldOptions.OptionTargetType" + FieldOptions_OptionTargetType_enum_name = "OptionTargetType" +) + // Names for google.protobuf.OneofOptions. const ( OneofOptions_message_name protoreflect.Name = "OneofOptions" @@ -590,20 +620,23 @@ const ( // Field names for google.protobuf.EnumOptions. const ( - EnumOptions_AllowAlias_field_name protoreflect.Name = "allow_alias" - EnumOptions_Deprecated_field_name protoreflect.Name = "deprecated" - EnumOptions_UninterpretedOption_field_name protoreflect.Name = "uninterpreted_option" + EnumOptions_AllowAlias_field_name protoreflect.Name = "allow_alias" + EnumOptions_Deprecated_field_name protoreflect.Name = "deprecated" + EnumOptions_DeprecatedLegacyJsonFieldConflicts_field_name protoreflect.Name = "deprecated_legacy_json_field_conflicts" + EnumOptions_UninterpretedOption_field_name protoreflect.Name = "uninterpreted_option" - EnumOptions_AllowAlias_field_fullname protoreflect.FullName = "google.protobuf.EnumOptions.allow_alias" - EnumOptions_Deprecated_field_fullname protoreflect.FullName = "google.protobuf.EnumOptions.deprecated" - EnumOptions_UninterpretedOption_field_fullname protoreflect.FullName = "google.protobuf.EnumOptions.uninterpreted_option" + EnumOptions_AllowAlias_field_fullname protoreflect.FullName = "google.protobuf.EnumOptions.allow_alias" + EnumOptions_Deprecated_field_fullname protoreflect.FullName = "google.protobuf.EnumOptions.deprecated" + EnumOptions_DeprecatedLegacyJsonFieldConflicts_field_fullname protoreflect.FullName = "google.protobuf.EnumOptions.deprecated_legacy_json_field_conflicts" + EnumOptions_UninterpretedOption_field_fullname protoreflect.FullName = "google.protobuf.EnumOptions.uninterpreted_option" ) // Field numbers for google.protobuf.EnumOptions. const ( - EnumOptions_AllowAlias_field_number protoreflect.FieldNumber = 2 - EnumOptions_Deprecated_field_number protoreflect.FieldNumber = 3 - EnumOptions_UninterpretedOption_field_number protoreflect.FieldNumber = 999 + EnumOptions_AllowAlias_field_number protoreflect.FieldNumber = 2 + EnumOptions_Deprecated_field_number protoreflect.FieldNumber = 3 + EnumOptions_DeprecatedLegacyJsonFieldConflicts_field_number protoreflect.FieldNumber = 6 + EnumOptions_UninterpretedOption_field_number protoreflect.FieldNumber = 999 ) // Names for google.protobuf.EnumValueOptions. @@ -813,11 +846,13 @@ const ( GeneratedCodeInfo_Annotation_SourceFile_field_name protoreflect.Name = "source_file" GeneratedCodeInfo_Annotation_Begin_field_name protoreflect.Name = "begin" GeneratedCodeInfo_Annotation_End_field_name protoreflect.Name = "end" + GeneratedCodeInfo_Annotation_Semantic_field_name protoreflect.Name = "semantic" GeneratedCodeInfo_Annotation_Path_field_fullname protoreflect.FullName = "google.protobuf.GeneratedCodeInfo.Annotation.path" GeneratedCodeInfo_Annotation_SourceFile_field_fullname protoreflect.FullName = "google.protobuf.GeneratedCodeInfo.Annotation.source_file" GeneratedCodeInfo_Annotation_Begin_field_fullname protoreflect.FullName = "google.protobuf.GeneratedCodeInfo.Annotation.begin" GeneratedCodeInfo_Annotation_End_field_fullname protoreflect.FullName = "google.protobuf.GeneratedCodeInfo.Annotation.end" + GeneratedCodeInfo_Annotation_Semantic_field_fullname protoreflect.FullName = "google.protobuf.GeneratedCodeInfo.Annotation.semantic" ) // Field numbers for google.protobuf.GeneratedCodeInfo.Annotation. @@ -826,4 +861,11 @@ const ( GeneratedCodeInfo_Annotation_SourceFile_field_number protoreflect.FieldNumber = 2 GeneratedCodeInfo_Annotation_Begin_field_number protoreflect.FieldNumber = 3 GeneratedCodeInfo_Annotation_End_field_number protoreflect.FieldNumber = 4 + GeneratedCodeInfo_Annotation_Semantic_field_number protoreflect.FieldNumber = 5 +) + +// Full and short names for google.protobuf.GeneratedCodeInfo.Annotation.Semantic. +const ( + GeneratedCodeInfo_Annotation_Semantic_enum_fullname = "google.protobuf.GeneratedCodeInfo.Annotation.Semantic" + GeneratedCodeInfo_Annotation_Semantic_enum_name = "Semantic" ) diff --git a/vendor/google.golang.org/protobuf/internal/impl/convert.go b/vendor/google.golang.org/protobuf/internal/impl/convert.go index 11a6128..185ef2e 100644 --- a/vendor/google.golang.org/protobuf/internal/impl/convert.go +++ b/vendor/google.golang.org/protobuf/internal/impl/convert.go @@ -59,7 +59,6 @@ func NewConverter(t reflect.Type, fd protoreflect.FieldDescriptor) Converter { default: return newSingularConverter(t, fd) } - panic(fmt.Sprintf("invalid Go type %v for field %v", t, fd.FullName())) } var ( diff --git a/vendor/google.golang.org/protobuf/internal/strs/strings_unsafe.go b/vendor/google.golang.org/protobuf/internal/strs/strings_unsafe.go index fea589c..61a84d3 100644 --- a/vendor/google.golang.org/protobuf/internal/strs/strings_unsafe.go +++ b/vendor/google.golang.org/protobuf/internal/strs/strings_unsafe.go @@ -87,7 +87,7 @@ func (sb *Builder) grow(n int) { // Unlike strings.Builder, we do not need to copy over the contents // of the old buffer since our builder provides no API for // retrieving previously created strings. - sb.buf = make([]byte, 2*(cap(sb.buf)+n)) + sb.buf = make([]byte, 0, 2*(cap(sb.buf)+n)) } func (sb *Builder) last(n int) string { diff --git a/vendor/google.golang.org/protobuf/internal/version/version.go b/vendor/google.golang.org/protobuf/internal/version/version.go index b480c50..f7014cd 100644 --- a/vendor/google.golang.org/protobuf/internal/version/version.go +++ b/vendor/google.golang.org/protobuf/internal/version/version.go @@ -51,8 +51,8 @@ import ( // 10. Send out the CL for review and submit it. const ( Major = 1 - Minor = 28 - Patch = 1 + Minor = 30 + Patch = 0 PreRelease = "" ) diff --git a/vendor/google.golang.org/protobuf/proto/doc.go b/vendor/google.golang.org/protobuf/proto/doc.go index 08d2a46..ec71e71 100644 --- a/vendor/google.golang.org/protobuf/proto/doc.go +++ b/vendor/google.golang.org/protobuf/proto/doc.go @@ -5,16 +5,13 @@ // Package proto provides functions operating on protocol buffer messages. // // For documentation on protocol buffers in general, see: -// -// https://developers.google.com/protocol-buffers +// https://protobuf.dev. // // For a tutorial on using protocol buffers with Go, see: -// -// https://developers.google.com/protocol-buffers/docs/gotutorial +// https://protobuf.dev/getting-started/gotutorial. // // For a guide to generated Go protocol buffer code, see: -// -// https://developers.google.com/protocol-buffers/docs/reference/go-generated +// https://protobuf.dev/reference/go/go-generated. // // # Binary serialization // diff --git a/vendor/google.golang.org/protobuf/proto/equal.go b/vendor/google.golang.org/protobuf/proto/equal.go index 67948dd..1a0be1b 100644 --- a/vendor/google.golang.org/protobuf/proto/equal.go +++ b/vendor/google.golang.org/protobuf/proto/equal.go @@ -5,30 +5,39 @@ package proto import ( - "bytes" - "math" "reflect" - "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/reflect/protoreflect" ) -// Equal reports whether two messages are equal. -// If two messages marshal to the same bytes under deterministic serialization, -// then Equal is guaranteed to report true. +// Equal reports whether two messages are equal, +// by recursively comparing the fields of the message. // -// Two messages are equal if they belong to the same message descriptor, -// have the same set of populated known and extension field values, -// and the same set of unknown fields values. If either of the top-level -// messages are invalid, then Equal reports true only if both are invalid. +// - Bytes fields are equal if they contain identical bytes. +// Empty bytes (regardless of nil-ness) are considered equal. // -// Scalar values are compared with the equivalent of the == operator in Go, -// except bytes values which are compared using bytes.Equal and -// floating point values which specially treat NaNs as equal. -// Message values are compared by recursively calling Equal. -// Lists are equal if each element value is also equal. -// Maps are equal if they have the same set of keys, where the pair of values -// for each key is also equal. +// - Floating-point fields are equal if they contain the same value. +// Unlike the == operator, a NaN is equal to another NaN. +// +// - Other scalar fields are equal if they contain the same value. +// +// - Message fields are equal if they have +// the same set of populated known and extension field values, and +// the same set of unknown fields values. +// +// - Lists are equal if they are the same length and +// each corresponding element is equal. +// +// - Maps are equal if they have the same set of keys and +// the corresponding value for each key is equal. +// +// An invalid message is not equal to a valid message. +// An invalid message is only equal to another invalid message of the +// same type. An invalid message often corresponds to a nil pointer +// of the concrete message type. For example, (*pb.M)(nil) is not equal +// to &pb.M{}. +// If two valid messages marshal to the same bytes under deterministic +// serialization, then Equal is guaranteed to report true. func Equal(x, y Message) bool { if x == nil || y == nil { return x == nil && y == nil @@ -42,130 +51,7 @@ func Equal(x, y Message) bool { if mx.IsValid() != my.IsValid() { return false } - return equalMessage(mx, my) -} - -// equalMessage compares two messages. -func equalMessage(mx, my protoreflect.Message) bool { - if mx.Descriptor() != my.Descriptor() { - return false - } - - nx := 0 - equal := true - mx.Range(func(fd protoreflect.FieldDescriptor, vx protoreflect.Value) bool { - nx++ - vy := my.Get(fd) - equal = my.Has(fd) && equalField(fd, vx, vy) - return equal - }) - if !equal { - return false - } - ny := 0 - my.Range(func(fd protoreflect.FieldDescriptor, vx protoreflect.Value) bool { - ny++ - return true - }) - if nx != ny { - return false - } - - return equalUnknown(mx.GetUnknown(), my.GetUnknown()) -} - -// equalField compares two fields. -func equalField(fd protoreflect.FieldDescriptor, x, y protoreflect.Value) bool { - switch { - case fd.IsList(): - return equalList(fd, x.List(), y.List()) - case fd.IsMap(): - return equalMap(fd, x.Map(), y.Map()) - default: - return equalValue(fd, x, y) - } -} - -// equalMap compares two maps. -func equalMap(fd protoreflect.FieldDescriptor, x, y protoreflect.Map) bool { - if x.Len() != y.Len() { - return false - } - equal := true - x.Range(func(k protoreflect.MapKey, vx protoreflect.Value) bool { - vy := y.Get(k) - equal = y.Has(k) && equalValue(fd.MapValue(), vx, vy) - return equal - }) - return equal -} - -// equalList compares two lists. -func equalList(fd protoreflect.FieldDescriptor, x, y protoreflect.List) bool { - if x.Len() != y.Len() { - return false - } - for i := x.Len() - 1; i >= 0; i-- { - if !equalValue(fd, x.Get(i), y.Get(i)) { - return false - } - } - return true -} - -// equalValue compares two singular values. -func equalValue(fd protoreflect.FieldDescriptor, x, y protoreflect.Value) bool { - switch fd.Kind() { - case protoreflect.BoolKind: - return x.Bool() == y.Bool() - case protoreflect.EnumKind: - return x.Enum() == y.Enum() - case protoreflect.Int32Kind, protoreflect.Sint32Kind, - protoreflect.Int64Kind, protoreflect.Sint64Kind, - protoreflect.Sfixed32Kind, protoreflect.Sfixed64Kind: - return x.Int() == y.Int() - case protoreflect.Uint32Kind, protoreflect.Uint64Kind, - protoreflect.Fixed32Kind, protoreflect.Fixed64Kind: - return x.Uint() == y.Uint() - case protoreflect.FloatKind, protoreflect.DoubleKind: - fx := x.Float() - fy := y.Float() - if math.IsNaN(fx) || math.IsNaN(fy) { - return math.IsNaN(fx) && math.IsNaN(fy) - } - return fx == fy - case protoreflect.StringKind: - return x.String() == y.String() - case protoreflect.BytesKind: - return bytes.Equal(x.Bytes(), y.Bytes()) - case protoreflect.MessageKind, protoreflect.GroupKind: - return equalMessage(x.Message(), y.Message()) - default: - return x.Interface() == y.Interface() - } -} - -// equalUnknown compares unknown fields by direct comparison on the raw bytes -// of each individual field number. -func equalUnknown(x, y protoreflect.RawFields) bool { - if len(x) != len(y) { - return false - } - if bytes.Equal([]byte(x), []byte(y)) { - return true - } - - mx := make(map[protoreflect.FieldNumber]protoreflect.RawFields) - my := make(map[protoreflect.FieldNumber]protoreflect.RawFields) - for len(x) > 0 { - fnum, _, n := protowire.ConsumeField(x) - mx[fnum] = append(mx[fnum], x[:n]...) - x = x[n:] - } - for len(y) > 0 { - fnum, _, n := protowire.ConsumeField(y) - my[fnum] = append(my[fnum], y[:n]...) - y = y[n:] - } - return reflect.DeepEqual(mx, my) + vx := protoreflect.ValueOfMessage(mx) + vy := protoreflect.ValueOfMessage(my) + return vx.Equal(vy) } diff --git a/vendor/google.golang.org/protobuf/reflect/protoreflect/source_gen.go b/vendor/google.golang.org/protobuf/reflect/protoreflect/source_gen.go index b03c122..54ce326 100644 --- a/vendor/google.golang.org/protobuf/reflect/protoreflect/source_gen.go +++ b/vendor/google.golang.org/protobuf/reflect/protoreflect/source_gen.go @@ -35,6 +35,8 @@ func (p *SourcePath) appendFileDescriptorProto(b []byte) []byte { b = p.appendSingularField(b, "source_code_info", (*SourcePath).appendSourceCodeInfo) case 12: b = p.appendSingularField(b, "syntax", nil) + case 13: + b = p.appendSingularField(b, "edition", nil) } return b } @@ -236,6 +238,8 @@ func (p *SourcePath) appendMessageOptions(b []byte) []byte { b = p.appendSingularField(b, "deprecated", nil) case 7: b = p.appendSingularField(b, "map_entry", nil) + case 11: + b = p.appendSingularField(b, "deprecated_legacy_json_field_conflicts", nil) case 999: b = p.appendRepeatedField(b, "uninterpreted_option", (*SourcePath).appendUninterpretedOption) } @@ -279,6 +283,8 @@ func (p *SourcePath) appendEnumOptions(b []byte) []byte { b = p.appendSingularField(b, "allow_alias", nil) case 3: b = p.appendSingularField(b, "deprecated", nil) + case 6: + b = p.appendSingularField(b, "deprecated_legacy_json_field_conflicts", nil) case 999: b = p.appendRepeatedField(b, "uninterpreted_option", (*SourcePath).appendUninterpretedOption) } @@ -345,10 +351,18 @@ func (p *SourcePath) appendFieldOptions(b []byte) []byte { b = p.appendSingularField(b, "jstype", nil) case 5: b = p.appendSingularField(b, "lazy", nil) + case 15: + b = p.appendSingularField(b, "unverified_lazy", nil) case 3: b = p.appendSingularField(b, "deprecated", nil) case 10: b = p.appendSingularField(b, "weak", nil) + case 16: + b = p.appendSingularField(b, "debug_redact", nil) + case 17: + b = p.appendSingularField(b, "retention", nil) + case 18: + b = p.appendSingularField(b, "target", nil) case 999: b = p.appendRepeatedField(b, "uninterpreted_option", (*SourcePath).appendUninterpretedOption) } diff --git a/vendor/google.golang.org/protobuf/reflect/protoreflect/value.go b/vendor/google.golang.org/protobuf/reflect/protoreflect/value.go index f319810..37601b7 100644 --- a/vendor/google.golang.org/protobuf/reflect/protoreflect/value.go +++ b/vendor/google.golang.org/protobuf/reflect/protoreflect/value.go @@ -148,7 +148,7 @@ type Message interface { // be preserved in marshaling or other operations. IsValid() bool - // ProtoMethods returns optional fast-path implementions of various operations. + // ProtoMethods returns optional fast-path implementations of various operations. // This method may return nil. // // The returned methods type is identical to diff --git a/vendor/google.golang.org/protobuf/reflect/protoreflect/value_equal.go b/vendor/google.golang.org/protobuf/reflect/protoreflect/value_equal.go new file mode 100644 index 0000000..5916525 --- /dev/null +++ b/vendor/google.golang.org/protobuf/reflect/protoreflect/value_equal.go @@ -0,0 +1,168 @@ +// Copyright 2022 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package protoreflect + +import ( + "bytes" + "fmt" + "math" + "reflect" + + "google.golang.org/protobuf/encoding/protowire" +) + +// Equal reports whether v1 and v2 are recursively equal. +// +// - Values of different types are always unequal. +// +// - Bytes values are equal if they contain identical bytes. +// Empty bytes (regardless of nil-ness) are considered equal. +// +// - Floating point values are equal if they contain the same value. +// Unlike the == operator, a NaN is equal to another NaN. +// +// - Enums are equal if they contain the same number. +// Since Value does not contain an enum descriptor, +// enum values do not consider the type of the enum. +// +// - Other scalar values are equal if they contain the same value. +// +// - Message values are equal if they belong to the same message descriptor, +// have the same set of populated known and extension field values, +// and the same set of unknown fields values. +// +// - Lists are equal if they are the same length and +// each corresponding element is equal. +// +// - Maps are equal if they have the same set of keys and +// the corresponding value for each key is equal. +func (v1 Value) Equal(v2 Value) bool { + return equalValue(v1, v2) +} + +func equalValue(x, y Value) bool { + eqType := x.typ == y.typ + switch x.typ { + case nilType: + return eqType + case boolType: + return eqType && x.Bool() == y.Bool() + case int32Type, int64Type: + return eqType && x.Int() == y.Int() + case uint32Type, uint64Type: + return eqType && x.Uint() == y.Uint() + case float32Type, float64Type: + return eqType && equalFloat(x.Float(), y.Float()) + case stringType: + return eqType && x.String() == y.String() + case bytesType: + return eqType && bytes.Equal(x.Bytes(), y.Bytes()) + case enumType: + return eqType && x.Enum() == y.Enum() + default: + switch x := x.Interface().(type) { + case Message: + y, ok := y.Interface().(Message) + return ok && equalMessage(x, y) + case List: + y, ok := y.Interface().(List) + return ok && equalList(x, y) + case Map: + y, ok := y.Interface().(Map) + return ok && equalMap(x, y) + default: + panic(fmt.Sprintf("unknown type: %T", x)) + } + } +} + +// equalFloat compares two floats, where NaNs are treated as equal. +func equalFloat(x, y float64) bool { + if math.IsNaN(x) || math.IsNaN(y) { + return math.IsNaN(x) && math.IsNaN(y) + } + return x == y +} + +// equalMessage compares two messages. +func equalMessage(mx, my Message) bool { + if mx.Descriptor() != my.Descriptor() { + return false + } + + nx := 0 + equal := true + mx.Range(func(fd FieldDescriptor, vx Value) bool { + nx++ + vy := my.Get(fd) + equal = my.Has(fd) && equalValue(vx, vy) + return equal + }) + if !equal { + return false + } + ny := 0 + my.Range(func(fd FieldDescriptor, vx Value) bool { + ny++ + return true + }) + if nx != ny { + return false + } + + return equalUnknown(mx.GetUnknown(), my.GetUnknown()) +} + +// equalList compares two lists. +func equalList(x, y List) bool { + if x.Len() != y.Len() { + return false + } + for i := x.Len() - 1; i >= 0; i-- { + if !equalValue(x.Get(i), y.Get(i)) { + return false + } + } + return true +} + +// equalMap compares two maps. +func equalMap(x, y Map) bool { + if x.Len() != y.Len() { + return false + } + equal := true + x.Range(func(k MapKey, vx Value) bool { + vy := y.Get(k) + equal = y.Has(k) && equalValue(vx, vy) + return equal + }) + return equal +} + +// equalUnknown compares unknown fields by direct comparison on the raw bytes +// of each individual field number. +func equalUnknown(x, y RawFields) bool { + if len(x) != len(y) { + return false + } + if bytes.Equal([]byte(x), []byte(y)) { + return true + } + + mx := make(map[FieldNumber]RawFields) + my := make(map[FieldNumber]RawFields) + for len(x) > 0 { + fnum, _, n := protowire.ConsumeField(x) + mx[fnum] = append(mx[fnum], x[:n]...) + x = x[n:] + } + for len(y) > 0 { + fnum, _, n := protowire.ConsumeField(y) + my[fnum] = append(my[fnum], y[:n]...) + y = y[n:] + } + return reflect.DeepEqual(mx, my) +} diff --git a/vendor/google.golang.org/protobuf/reflect/protoreflect/value_union.go b/vendor/google.golang.org/protobuf/reflect/protoreflect/value_union.go index ca8e28c..08e5ef7 100644 --- a/vendor/google.golang.org/protobuf/reflect/protoreflect/value_union.go +++ b/vendor/google.golang.org/protobuf/reflect/protoreflect/value_union.go @@ -54,11 +54,11 @@ import ( // // Append a 0 to a "repeated int32" field. // // Since the Value returned by Mutable is guaranteed to alias // // the source message, modifying the Value modifies the message. -// message.Mutable(fieldDesc).(List).Append(protoreflect.ValueOfInt32(0)) +// message.Mutable(fieldDesc).List().Append(protoreflect.ValueOfInt32(0)) // // // Assign [0] to a "repeated int32" field by creating a new Value, // // modifying it, and assigning it. -// list := message.NewField(fieldDesc).(List) +// list := message.NewField(fieldDesc).List() // list.Append(protoreflect.ValueOfInt32(0)) // message.Set(fieldDesc, list) // // ERROR: Since it is not defined whether Set aliases the source, diff --git a/vendor/google.golang.org/protobuf/reflect/protoregistry/registry.go b/vendor/google.golang.org/protobuf/reflect/protoregistry/registry.go index 58352a6..aeb5597 100644 --- a/vendor/google.golang.org/protobuf/reflect/protoregistry/registry.go +++ b/vendor/google.golang.org/protobuf/reflect/protoregistry/registry.go @@ -46,7 +46,7 @@ var conflictPolicy = "panic" // "panic" | "warn" | "ignore" // It is a variable so that the behavior is easily overridden in another file. var ignoreConflict = func(d protoreflect.Descriptor, err error) bool { const env = "GOLANG_PROTOBUF_REGISTRATION_CONFLICT" - const faq = "https://developers.google.com/protocol-buffers/docs/reference/go/faq#namespace-conflict" + const faq = "https://protobuf.dev/reference/go/faq#namespace-conflict" policy := conflictPolicy if v := os.Getenv(env); v != "" { policy = v diff --git a/vendor/google.golang.org/protobuf/types/descriptorpb/descriptor.pb.go b/vendor/google.golang.org/protobuf/types/descriptorpb/descriptor.pb.go index abe4ab5..dac5671 100644 --- a/vendor/google.golang.org/protobuf/types/descriptorpb/descriptor.pb.go +++ b/vendor/google.golang.org/protobuf/types/descriptorpb/descriptor.pb.go @@ -406,6 +406,152 @@ func (FieldOptions_JSType) EnumDescriptor() ([]byte, []int) { return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{12, 1} } +// If set to RETENTION_SOURCE, the option will be omitted from the binary. +// Note: as of January 2023, support for this is in progress and does not yet +// have an effect (b/264593489). +type FieldOptions_OptionRetention int32 + +const ( + FieldOptions_RETENTION_UNKNOWN FieldOptions_OptionRetention = 0 + FieldOptions_RETENTION_RUNTIME FieldOptions_OptionRetention = 1 + FieldOptions_RETENTION_SOURCE FieldOptions_OptionRetention = 2 +) + +// Enum value maps for FieldOptions_OptionRetention. +var ( + FieldOptions_OptionRetention_name = map[int32]string{ + 0: "RETENTION_UNKNOWN", + 1: "RETENTION_RUNTIME", + 2: "RETENTION_SOURCE", + } + FieldOptions_OptionRetention_value = map[string]int32{ + "RETENTION_UNKNOWN": 0, + "RETENTION_RUNTIME": 1, + "RETENTION_SOURCE": 2, + } +) + +func (x FieldOptions_OptionRetention) Enum() *FieldOptions_OptionRetention { + p := new(FieldOptions_OptionRetention) + *p = x + return p +} + +func (x FieldOptions_OptionRetention) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FieldOptions_OptionRetention) Descriptor() protoreflect.EnumDescriptor { + return file_google_protobuf_descriptor_proto_enumTypes[5].Descriptor() +} + +func (FieldOptions_OptionRetention) Type() protoreflect.EnumType { + return &file_google_protobuf_descriptor_proto_enumTypes[5] +} + +func (x FieldOptions_OptionRetention) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *FieldOptions_OptionRetention) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = FieldOptions_OptionRetention(num) + return nil +} + +// Deprecated: Use FieldOptions_OptionRetention.Descriptor instead. +func (FieldOptions_OptionRetention) EnumDescriptor() ([]byte, []int) { + return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{12, 2} +} + +// This indicates the types of entities that the field may apply to when used +// as an option. If it is unset, then the field may be freely used as an +// option on any kind of entity. Note: as of January 2023, support for this is +// in progress and does not yet have an effect (b/264593489). +type FieldOptions_OptionTargetType int32 + +const ( + FieldOptions_TARGET_TYPE_UNKNOWN FieldOptions_OptionTargetType = 0 + FieldOptions_TARGET_TYPE_FILE FieldOptions_OptionTargetType = 1 + FieldOptions_TARGET_TYPE_EXTENSION_RANGE FieldOptions_OptionTargetType = 2 + FieldOptions_TARGET_TYPE_MESSAGE FieldOptions_OptionTargetType = 3 + FieldOptions_TARGET_TYPE_FIELD FieldOptions_OptionTargetType = 4 + FieldOptions_TARGET_TYPE_ONEOF FieldOptions_OptionTargetType = 5 + FieldOptions_TARGET_TYPE_ENUM FieldOptions_OptionTargetType = 6 + FieldOptions_TARGET_TYPE_ENUM_ENTRY FieldOptions_OptionTargetType = 7 + FieldOptions_TARGET_TYPE_SERVICE FieldOptions_OptionTargetType = 8 + FieldOptions_TARGET_TYPE_METHOD FieldOptions_OptionTargetType = 9 +) + +// Enum value maps for FieldOptions_OptionTargetType. +var ( + FieldOptions_OptionTargetType_name = map[int32]string{ + 0: "TARGET_TYPE_UNKNOWN", + 1: "TARGET_TYPE_FILE", + 2: "TARGET_TYPE_EXTENSION_RANGE", + 3: "TARGET_TYPE_MESSAGE", + 4: "TARGET_TYPE_FIELD", + 5: "TARGET_TYPE_ONEOF", + 6: "TARGET_TYPE_ENUM", + 7: "TARGET_TYPE_ENUM_ENTRY", + 8: "TARGET_TYPE_SERVICE", + 9: "TARGET_TYPE_METHOD", + } + FieldOptions_OptionTargetType_value = map[string]int32{ + "TARGET_TYPE_UNKNOWN": 0, + "TARGET_TYPE_FILE": 1, + "TARGET_TYPE_EXTENSION_RANGE": 2, + "TARGET_TYPE_MESSAGE": 3, + "TARGET_TYPE_FIELD": 4, + "TARGET_TYPE_ONEOF": 5, + "TARGET_TYPE_ENUM": 6, + "TARGET_TYPE_ENUM_ENTRY": 7, + "TARGET_TYPE_SERVICE": 8, + "TARGET_TYPE_METHOD": 9, + } +) + +func (x FieldOptions_OptionTargetType) Enum() *FieldOptions_OptionTargetType { + p := new(FieldOptions_OptionTargetType) + *p = x + return p +} + +func (x FieldOptions_OptionTargetType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (FieldOptions_OptionTargetType) Descriptor() protoreflect.EnumDescriptor { + return file_google_protobuf_descriptor_proto_enumTypes[6].Descriptor() +} + +func (FieldOptions_OptionTargetType) Type() protoreflect.EnumType { + return &file_google_protobuf_descriptor_proto_enumTypes[6] +} + +func (x FieldOptions_OptionTargetType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *FieldOptions_OptionTargetType) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = FieldOptions_OptionTargetType(num) + return nil +} + +// Deprecated: Use FieldOptions_OptionTargetType.Descriptor instead. +func (FieldOptions_OptionTargetType) EnumDescriptor() ([]byte, []int) { + return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{12, 3} +} + // Is this method side-effect-free (or safe in HTTP parlance), or idempotent, // or neither? HTTP based RPC implementation may choose GET verb for safe // methods, and PUT verb for idempotent methods instead of the default POST. @@ -442,11 +588,11 @@ func (x MethodOptions_IdempotencyLevel) String() string { } func (MethodOptions_IdempotencyLevel) Descriptor() protoreflect.EnumDescriptor { - return file_google_protobuf_descriptor_proto_enumTypes[5].Descriptor() + return file_google_protobuf_descriptor_proto_enumTypes[7].Descriptor() } func (MethodOptions_IdempotencyLevel) Type() protoreflect.EnumType { - return &file_google_protobuf_descriptor_proto_enumTypes[5] + return &file_google_protobuf_descriptor_proto_enumTypes[7] } func (x MethodOptions_IdempotencyLevel) Number() protoreflect.EnumNumber { @@ -468,6 +614,70 @@ func (MethodOptions_IdempotencyLevel) EnumDescriptor() ([]byte, []int) { return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{17, 0} } +// Represents the identified object's effect on the element in the original +// .proto file. +type GeneratedCodeInfo_Annotation_Semantic int32 + +const ( + // There is no effect or the effect is indescribable. + GeneratedCodeInfo_Annotation_NONE GeneratedCodeInfo_Annotation_Semantic = 0 + // The element is set or otherwise mutated. + GeneratedCodeInfo_Annotation_SET GeneratedCodeInfo_Annotation_Semantic = 1 + // An alias to the element is returned. + GeneratedCodeInfo_Annotation_ALIAS GeneratedCodeInfo_Annotation_Semantic = 2 +) + +// Enum value maps for GeneratedCodeInfo_Annotation_Semantic. +var ( + GeneratedCodeInfo_Annotation_Semantic_name = map[int32]string{ + 0: "NONE", + 1: "SET", + 2: "ALIAS", + } + GeneratedCodeInfo_Annotation_Semantic_value = map[string]int32{ + "NONE": 0, + "SET": 1, + "ALIAS": 2, + } +) + +func (x GeneratedCodeInfo_Annotation_Semantic) Enum() *GeneratedCodeInfo_Annotation_Semantic { + p := new(GeneratedCodeInfo_Annotation_Semantic) + *p = x + return p +} + +func (x GeneratedCodeInfo_Annotation_Semantic) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GeneratedCodeInfo_Annotation_Semantic) Descriptor() protoreflect.EnumDescriptor { + return file_google_protobuf_descriptor_proto_enumTypes[8].Descriptor() +} + +func (GeneratedCodeInfo_Annotation_Semantic) Type() protoreflect.EnumType { + return &file_google_protobuf_descriptor_proto_enumTypes[8] +} + +func (x GeneratedCodeInfo_Annotation_Semantic) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Do not use. +func (x *GeneratedCodeInfo_Annotation_Semantic) UnmarshalJSON(b []byte) error { + num, err := protoimpl.X.UnmarshalJSONEnum(x.Descriptor(), b) + if err != nil { + return err + } + *x = GeneratedCodeInfo_Annotation_Semantic(num) + return nil +} + +// Deprecated: Use GeneratedCodeInfo_Annotation_Semantic.Descriptor instead. +func (GeneratedCodeInfo_Annotation_Semantic) EnumDescriptor() ([]byte, []int) { + return file_google_protobuf_descriptor_proto_rawDescGZIP(), []int{20, 0, 0} +} + // The protocol compiler can output a FileDescriptorSet containing the .proto // files it parses. type FileDescriptorSet struct { @@ -544,8 +754,12 @@ type FileDescriptorProto struct { // development tools. SourceCodeInfo *SourceCodeInfo `protobuf:"bytes,9,opt,name=source_code_info,json=sourceCodeInfo" json:"source_code_info,omitempty"` // The syntax of the proto file. - // The supported values are "proto2" and "proto3". + // The supported values are "proto2", "proto3", and "editions". + // + // If `edition` is present, this value must be "editions". Syntax *string `protobuf:"bytes,12,opt,name=syntax" json:"syntax,omitempty"` + // The edition of the proto file, which is an opaque string. + Edition *string `protobuf:"bytes,13,opt,name=edition" json:"edition,omitempty"` } func (x *FileDescriptorProto) Reset() { @@ -664,6 +878,13 @@ func (x *FileDescriptorProto) GetSyntax() string { return "" } +func (x *FileDescriptorProto) GetEdition() string { + if x != nil && x.Edition != nil { + return *x.Edition + } + return "" +} + // Describes a message type. type DescriptorProto struct { state protoimpl.MessageState @@ -860,7 +1081,6 @@ type FieldDescriptorProto struct { // For booleans, "true" or "false". // For strings, contains the default text contents (not escaped in any way). // For bytes, contains the C escaped value. All bytes >= 128 are escaped. - // TODO(kenton): Base-64 encode? DefaultValue *string `protobuf:"bytes,7,opt,name=default_value,json=defaultValue" json:"default_value,omitempty"` // If set, gives the index of a oneof in the containing type's oneof_decl // list. This field is a member of that oneof. @@ -1382,22 +1602,22 @@ type FileOptions struct { // inappropriate because proto packages do not normally start with backwards // domain names. JavaPackage *string `protobuf:"bytes,1,opt,name=java_package,json=javaPackage" json:"java_package,omitempty"` - // If set, all the classes from the .proto file are wrapped in a single - // outer class with the given name. This applies to both Proto1 - // (equivalent to the old "--one_java_file" option) and Proto2 (where - // a .proto always translates to a single class, but you may want to - // explicitly choose the class name). + // Controls the name of the wrapper Java class generated for the .proto file. + // That class will always contain the .proto file's getDescriptor() method as + // well as any top-level extensions defined in the .proto file. + // If java_multiple_files is disabled, then all the other classes from the + // .proto file will be nested inside the single wrapper outer class. JavaOuterClassname *string `protobuf:"bytes,8,opt,name=java_outer_classname,json=javaOuterClassname" json:"java_outer_classname,omitempty"` - // If set true, then the Java code generator will generate a separate .java + // If enabled, then the Java code generator will generate a separate .java // file for each top-level message, enum, and service defined in the .proto - // file. Thus, these types will *not* be nested inside the outer class - // named by java_outer_classname. However, the outer class will still be + // file. Thus, these types will *not* be nested inside the wrapper class + // named by java_outer_classname. However, the wrapper class will still be // generated to contain the file's getDescriptor() method as well as any // top-level extensions defined in the file. JavaMultipleFiles *bool `protobuf:"varint,10,opt,name=java_multiple_files,json=javaMultipleFiles,def=0" json:"java_multiple_files,omitempty"` // This option does nothing. // - // Deprecated: Do not use. + // Deprecated: Marked as deprecated in google/protobuf/descriptor.proto. JavaGenerateEqualsAndHash *bool `protobuf:"varint,20,opt,name=java_generate_equals_and_hash,json=javaGenerateEqualsAndHash" json:"java_generate_equals_and_hash,omitempty"` // If set true, then the Java2 code generator will generate code that // throws an exception whenever an attempt is made to assign a non-UTF-8 @@ -1531,7 +1751,7 @@ func (x *FileOptions) GetJavaMultipleFiles() bool { return Default_FileOptions_JavaMultipleFiles } -// Deprecated: Do not use. +// Deprecated: Marked as deprecated in google/protobuf/descriptor.proto. func (x *FileOptions) GetJavaGenerateEqualsAndHash() bool { if x != nil && x.JavaGenerateEqualsAndHash != nil { return *x.JavaGenerateEqualsAndHash @@ -1670,10 +1890,12 @@ type MessageOptions struct { // efficient, has fewer features, and is more complicated. // // The message must be defined exactly as follows: - // message Foo { - // option message_set_wire_format = true; - // extensions 4 to max; - // } + // + // message Foo { + // option message_set_wire_format = true; + // extensions 4 to max; + // } + // // Note that the message cannot have any defined fields; MessageSets only // have extensions. // @@ -1692,28 +1914,44 @@ type MessageOptions struct { // for the message, or it will be completely ignored; in the very least, // this is a formalization for deprecating messages. Deprecated *bool `protobuf:"varint,3,opt,name=deprecated,def=0" json:"deprecated,omitempty"` + // NOTE: Do not set the option in .proto files. Always use the maps syntax + // instead. The option should only be implicitly set by the proto compiler + // parser. + // // Whether the message is an automatically generated map entry type for the // maps field. // // For maps fields: - // map map_field = 1; + // + // map map_field = 1; + // // The parsed descriptor looks like: - // message MapFieldEntry { - // option map_entry = true; - // optional KeyType key = 1; - // optional ValueType value = 2; - // } - // repeated MapFieldEntry map_field = 1; + // + // message MapFieldEntry { + // option map_entry = true; + // optional KeyType key = 1; + // optional ValueType value = 2; + // } + // repeated MapFieldEntry map_field = 1; // // Implementations may choose not to generate the map_entry=true message, but // use a native map in the target language to hold the keys and values. // The reflection APIs in such implementations still need to work as // if the field is a repeated message field. - // - // NOTE: Do not set the option in .proto files. Always use the maps syntax - // instead. The option should only be implicitly set by the proto compiler - // parser. MapEntry *bool `protobuf:"varint,7,opt,name=map_entry,json=mapEntry" json:"map_entry,omitempty"` + // Enable the legacy handling of JSON field name conflicts. This lowercases + // and strips underscored from the fields before comparison in proto3 only. + // The new behavior takes `json_name` into account and applies to proto2 as + // well. + // + // This should only be used as a temporary measure against broken builds due + // to the change in behavior for JSON field name conflicts. + // + // TODO(b/261750190) This is legacy behavior we plan to remove once downstream + // teams have had time to migrate. + // + // Deprecated: Marked as deprecated in google/protobuf/descriptor.proto. + DeprecatedLegacyJsonFieldConflicts *bool `protobuf:"varint,11,opt,name=deprecated_legacy_json_field_conflicts,json=deprecatedLegacyJsonFieldConflicts" json:"deprecated_legacy_json_field_conflicts,omitempty"` // The parser stores options it doesn't recognize here. See above. UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` } @@ -1785,6 +2023,14 @@ func (x *MessageOptions) GetMapEntry() bool { return false } +// Deprecated: Marked as deprecated in google/protobuf/descriptor.proto. +func (x *MessageOptions) GetDeprecatedLegacyJsonFieldConflicts() bool { + if x != nil && x.DeprecatedLegacyJsonFieldConflicts != nil { + return *x.DeprecatedLegacyJsonFieldConflicts + } + return false +} + func (x *MessageOptions) GetUninterpretedOption() []*UninterpretedOption { if x != nil { return x.UninterpretedOption @@ -1838,7 +2084,6 @@ type FieldOptions struct { // call from multiple threads concurrently, while non-const methods continue // to require exclusive access. // - // // Note that implementations may choose not to check required fields within // a lazy sub-message. That is, calling IsInitialized() on the outer message // may return true even if the inner message has missing required fields. @@ -1849,7 +2094,14 @@ type FieldOptions struct { // implementation must either *always* check its required fields, or *never* // check its required fields, regardless of whether or not the message has // been parsed. + // + // As of May 2022, lazy verifies the contents of the byte stream during + // parsing. An invalid byte stream will cause the overall parsing to fail. Lazy *bool `protobuf:"varint,5,opt,name=lazy,def=0" json:"lazy,omitempty"` + // unverified_lazy does no correctness checks on the byte stream. This should + // only be used where lazy with verification is prohibitive for performance + // reasons. + UnverifiedLazy *bool `protobuf:"varint,15,opt,name=unverified_lazy,json=unverifiedLazy,def=0" json:"unverified_lazy,omitempty"` // Is this field deprecated? // Depending on the target platform, this can emit Deprecated annotations // for accessors, or it will be completely ignored; in the very least, this @@ -1857,17 +2109,24 @@ type FieldOptions struct { Deprecated *bool `protobuf:"varint,3,opt,name=deprecated,def=0" json:"deprecated,omitempty"` // For Google-internal migration only. Do not use. Weak *bool `protobuf:"varint,10,opt,name=weak,def=0" json:"weak,omitempty"` + // Indicate that the field value should not be printed out when using debug + // formats, e.g. when the field contains sensitive credentials. + DebugRedact *bool `protobuf:"varint,16,opt,name=debug_redact,json=debugRedact,def=0" json:"debug_redact,omitempty"` + Retention *FieldOptions_OptionRetention `protobuf:"varint,17,opt,name=retention,enum=google.protobuf.FieldOptions_OptionRetention" json:"retention,omitempty"` + Target *FieldOptions_OptionTargetType `protobuf:"varint,18,opt,name=target,enum=google.protobuf.FieldOptions_OptionTargetType" json:"target,omitempty"` // The parser stores options it doesn't recognize here. See above. UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` } // Default values for FieldOptions fields. const ( - Default_FieldOptions_Ctype = FieldOptions_STRING - Default_FieldOptions_Jstype = FieldOptions_JS_NORMAL - Default_FieldOptions_Lazy = bool(false) - Default_FieldOptions_Deprecated = bool(false) - Default_FieldOptions_Weak = bool(false) + Default_FieldOptions_Ctype = FieldOptions_STRING + Default_FieldOptions_Jstype = FieldOptions_JS_NORMAL + Default_FieldOptions_Lazy = bool(false) + Default_FieldOptions_UnverifiedLazy = bool(false) + Default_FieldOptions_Deprecated = bool(false) + Default_FieldOptions_Weak = bool(false) + Default_FieldOptions_DebugRedact = bool(false) ) func (x *FieldOptions) Reset() { @@ -1930,6 +2189,13 @@ func (x *FieldOptions) GetLazy() bool { return Default_FieldOptions_Lazy } +func (x *FieldOptions) GetUnverifiedLazy() bool { + if x != nil && x.UnverifiedLazy != nil { + return *x.UnverifiedLazy + } + return Default_FieldOptions_UnverifiedLazy +} + func (x *FieldOptions) GetDeprecated() bool { if x != nil && x.Deprecated != nil { return *x.Deprecated @@ -1944,6 +2210,27 @@ func (x *FieldOptions) GetWeak() bool { return Default_FieldOptions_Weak } +func (x *FieldOptions) GetDebugRedact() bool { + if x != nil && x.DebugRedact != nil { + return *x.DebugRedact + } + return Default_FieldOptions_DebugRedact +} + +func (x *FieldOptions) GetRetention() FieldOptions_OptionRetention { + if x != nil && x.Retention != nil { + return *x.Retention + } + return FieldOptions_RETENTION_UNKNOWN +} + +func (x *FieldOptions) GetTarget() FieldOptions_OptionTargetType { + if x != nil && x.Target != nil { + return *x.Target + } + return FieldOptions_TARGET_TYPE_UNKNOWN +} + func (x *FieldOptions) GetUninterpretedOption() []*UninterpretedOption { if x != nil { return x.UninterpretedOption @@ -2014,6 +2301,15 @@ type EnumOptions struct { // for the enum, or it will be completely ignored; in the very least, this // is a formalization for deprecating enums. Deprecated *bool `protobuf:"varint,3,opt,name=deprecated,def=0" json:"deprecated,omitempty"` + // Enable the legacy handling of JSON field name conflicts. This lowercases + // and strips underscored from the fields before comparison in proto3 only. + // The new behavior takes `json_name` into account and applies to proto2 as + // well. + // TODO(b/261750190) Remove this legacy behavior once downstream teams have + // had time to migrate. + // + // Deprecated: Marked as deprecated in google/protobuf/descriptor.proto. + DeprecatedLegacyJsonFieldConflicts *bool `protobuf:"varint,6,opt,name=deprecated_legacy_json_field_conflicts,json=deprecatedLegacyJsonFieldConflicts" json:"deprecated_legacy_json_field_conflicts,omitempty"` // The parser stores options it doesn't recognize here. See above. UninterpretedOption []*UninterpretedOption `protobuf:"bytes,999,rep,name=uninterpreted_option,json=uninterpretedOption" json:"uninterpreted_option,omitempty"` } @@ -2069,6 +2365,14 @@ func (x *EnumOptions) GetDeprecated() bool { return Default_EnumOptions_Deprecated } +// Deprecated: Marked as deprecated in google/protobuf/descriptor.proto. +func (x *EnumOptions) GetDeprecatedLegacyJsonFieldConflicts() bool { + if x != nil && x.DeprecatedLegacyJsonFieldConflicts != nil { + return *x.DeprecatedLegacyJsonFieldConflicts + } + return false +} + func (x *EnumOptions) GetUninterpretedOption() []*UninterpretedOption { if x != nil { return x.UninterpretedOption @@ -2399,43 +2703,48 @@ type SourceCodeInfo struct { // tools. // // For example, say we have a file like: - // message Foo { - // optional string foo = 1; - // } + // + // message Foo { + // optional string foo = 1; + // } + // // Let's look at just the field definition: - // optional string foo = 1; - // ^ ^^ ^^ ^ ^^^ - // a bc de f ghi + // + // optional string foo = 1; + // ^ ^^ ^^ ^ ^^^ + // a bc de f ghi + // // We have the following locations: - // span path represents - // [a,i) [ 4, 0, 2, 0 ] The whole field definition. - // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). - // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). - // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). - // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). + // + // span path represents + // [a,i) [ 4, 0, 2, 0 ] The whole field definition. + // [a,b) [ 4, 0, 2, 0, 4 ] The label (optional). + // [c,d) [ 4, 0, 2, 0, 5 ] The type (string). + // [e,f) [ 4, 0, 2, 0, 1 ] The name (foo). + // [g,h) [ 4, 0, 2, 0, 3 ] The number (1). // // Notes: - // - A location may refer to a repeated field itself (i.e. not to any - // particular index within it). This is used whenever a set of elements are - // logically enclosed in a single code segment. For example, an entire - // extend block (possibly containing multiple extension definitions) will - // have an outer location whose path refers to the "extensions" repeated - // field without an index. - // - Multiple locations may have the same path. This happens when a single - // logical declaration is spread out across multiple places. The most - // obvious example is the "extend" block again -- there may be multiple - // extend blocks in the same scope, each of which will have the same path. - // - A location's span is not always a subset of its parent's span. For - // example, the "extendee" of an extension declaration appears at the - // beginning of the "extend" block and is shared by all extensions within - // the block. - // - Just because a location's span is a subset of some other location's span - // does not mean that it is a descendant. For example, a "group" defines - // both a type and a field in a single declaration. Thus, the locations - // corresponding to the type and field and their components will overlap. - // - Code which tries to interpret locations should probably be designed to - // ignore those that it doesn't understand, as more types of locations could - // be recorded in the future. + // - A location may refer to a repeated field itself (i.e. not to any + // particular index within it). This is used whenever a set of elements are + // logically enclosed in a single code segment. For example, an entire + // extend block (possibly containing multiple extension definitions) will + // have an outer location whose path refers to the "extensions" repeated + // field without an index. + // - Multiple locations may have the same path. This happens when a single + // logical declaration is spread out across multiple places. The most + // obvious example is the "extend" block again -- there may be multiple + // extend blocks in the same scope, each of which will have the same path. + // - A location's span is not always a subset of its parent's span. For + // example, the "extendee" of an extension declaration appears at the + // beginning of the "extend" block and is shared by all extensions within + // the block. + // - Just because a location's span is a subset of some other location's span + // does not mean that it is a descendant. For example, a "group" defines + // both a type and a field in a single declaration. Thus, the locations + // corresponding to the type and field and their components will overlap. + // - Code which tries to interpret locations should probably be designed to + // ignore those that it doesn't understand, as more types of locations could + // be recorded in the future. Location []*SourceCodeInfo_Location `protobuf:"bytes,1,rep,name=location" json:"location,omitempty"` } @@ -2715,8 +3024,8 @@ func (x *EnumDescriptorProto_EnumReservedRange) GetEnd() int32 { // The name of the uninterpreted option. Each string represents a segment in // a dot-separated name. is_extension is true iff a segment represents an // extension (denoted with parentheses in options specs in .proto files). -// E.g.,{ ["foo", false], ["bar.baz", true], ["qux", false] } represents -// "foo.(bar.baz).qux". +// E.g.,{ ["foo", false], ["bar.baz", true], ["moo", false] } represents +// "foo.(bar.baz).moo". type UninterpretedOption_NamePart struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2781,23 +3090,34 @@ type SourceCodeInfo_Location struct { // location. // // Each element is a field number or an index. They form a path from - // the root FileDescriptorProto to the place where the definition. For - // example, this path: - // [ 4, 3, 2, 7, 1 ] + // the root FileDescriptorProto to the place where the definition occurs. + // For example, this path: + // + // [ 4, 3, 2, 7, 1 ] + // // refers to: - // file.message_type(3) // 4, 3 - // .field(7) // 2, 7 - // .name() // 1 + // + // file.message_type(3) // 4, 3 + // .field(7) // 2, 7 + // .name() // 1 + // // This is because FileDescriptorProto.message_type has field number 4: - // repeated DescriptorProto message_type = 4; + // + // repeated DescriptorProto message_type = 4; + // // and DescriptorProto.field has field number 2: - // repeated FieldDescriptorProto field = 2; + // + // repeated FieldDescriptorProto field = 2; + // // and FieldDescriptorProto.name has field number 1: - // optional string name = 1; + // + // optional string name = 1; // // Thus, the above path gives the location of a field name. If we removed // the last element: - // [ 4, 3, 2, 7 ] + // + // [ 4, 3, 2, 7 ] + // // this path refers to the whole field declaration (from the beginning // of the label to the terminating semicolon). Path []int32 `protobuf:"varint,1,rep,packed,name=path" json:"path,omitempty"` @@ -2826,34 +3146,34 @@ type SourceCodeInfo_Location struct { // // Examples: // - // optional int32 foo = 1; // Comment attached to foo. - // // Comment attached to bar. - // optional int32 bar = 2; + // optional int32 foo = 1; // Comment attached to foo. + // // Comment attached to bar. + // optional int32 bar = 2; // - // optional string baz = 3; - // // Comment attached to baz. - // // Another line attached to baz. + // optional string baz = 3; + // // Comment attached to baz. + // // Another line attached to baz. // - // // Comment attached to qux. - // // - // // Another line attached to qux. - // optional double qux = 4; + // // Comment attached to moo. + // // + // // Another line attached to moo. + // optional double moo = 4; // - // // Detached comment for corge. This is not leading or trailing comments - // // to qux or corge because there are blank lines separating it from - // // both. + // // Detached comment for corge. This is not leading or trailing comments + // // to moo or corge because there are blank lines separating it from + // // both. // - // // Detached comment for corge paragraph 2. + // // Detached comment for corge paragraph 2. // - // optional string corge = 5; - // /* Block comment attached - // * to corge. Leading asterisks - // * will be removed. */ - // /* Block comment attached to - // * grault. */ - // optional int32 grault = 6; + // optional string corge = 5; + // /* Block comment attached + // * to corge. Leading asterisks + // * will be removed. */ + // /* Block comment attached to + // * grault. */ + // optional int32 grault = 6; // - // // ignored detached comments. + // // ignored detached comments. LeadingComments *string `protobuf:"bytes,3,opt,name=leading_comments,json=leadingComments" json:"leading_comments,omitempty"` TrailingComments *string `protobuf:"bytes,4,opt,name=trailing_comments,json=trailingComments" json:"trailing_comments,omitempty"` LeadingDetachedComments []string `protobuf:"bytes,6,rep,name=leading_detached_comments,json=leadingDetachedComments" json:"leading_detached_comments,omitempty"` @@ -2940,9 +3260,10 @@ type GeneratedCodeInfo_Annotation struct { // that relates to the identified object. Begin *int32 `protobuf:"varint,3,opt,name=begin" json:"begin,omitempty"` // Identifies the ending offset in bytes in the generated code that - // relates to the identified offset. The end offset should be one past + // relates to the identified object. The end offset should be one past // the last relevant byte (so the length of the text = end - begin). - End *int32 `protobuf:"varint,4,opt,name=end" json:"end,omitempty"` + End *int32 `protobuf:"varint,4,opt,name=end" json:"end,omitempty"` + Semantic *GeneratedCodeInfo_Annotation_Semantic `protobuf:"varint,5,opt,name=semantic,enum=google.protobuf.GeneratedCodeInfo_Annotation_Semantic" json:"semantic,omitempty"` } func (x *GeneratedCodeInfo_Annotation) Reset() { @@ -3005,6 +3326,13 @@ func (x *GeneratedCodeInfo_Annotation) GetEnd() int32 { return 0 } +func (x *GeneratedCodeInfo_Annotation) GetSemantic() GeneratedCodeInfo_Annotation_Semantic { + if x != nil && x.Semantic != nil { + return *x.Semantic + } + return GeneratedCodeInfo_Annotation_NONE +} + var File_google_protobuf_descriptor_proto protoreflect.FileDescriptor var file_google_protobuf_descriptor_proto_rawDesc = []byte{ @@ -3016,7 +3344,7 @@ var file_google_protobuf_descriptor_proto_rawDesc = []byte{ 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x04, 0x66, 0x69, - 0x6c, 0x65, 0x22, 0xe4, 0x04, 0x0a, 0x13, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x6c, 0x65, 0x22, 0xfe, 0x04, 0x0a, 0x13, 0x46, 0x69, 0x6c, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, @@ -3054,330 +3382,391 @@ var file_google_protobuf_descriptor_proto_rawDesc = []byte{ 0x75, 0x66, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x0e, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x16, 0x0a, 0x06, 0x73, 0x79, 0x6e, 0x74, 0x61, 0x78, 0x18, 0x0c, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x06, 0x73, 0x79, 0x6e, 0x74, 0x61, 0x78, 0x22, 0xb9, 0x06, 0x0a, 0x0f, 0x44, 0x65, + 0x09, 0x52, 0x06, 0x73, 0x79, 0x6e, 0x74, 0x61, 0x78, 0x12, 0x18, 0x0a, 0x07, 0x65, 0x64, 0x69, + 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x65, 0x64, 0x69, 0x74, + 0x69, 0x6f, 0x6e, 0x22, 0xb9, 0x06, 0x0a, 0x0f, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, + 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x66, + 0x69, 0x65, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x43, 0x0a, 0x09, 0x65, 0x78, 0x74, 0x65, + 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, + 0x65, 0x6c, 0x64, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, + 0x74, 0x6f, 0x52, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x41, 0x0a, + 0x0b, 0x6e, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x03, + 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x0a, 0x6e, 0x65, 0x73, 0x74, 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x41, 0x0a, 0x09, 0x65, 0x6e, 0x75, 0x6d, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x08, 0x65, 0x6e, 0x75, 0x6d, 0x54, + 0x79, 0x70, 0x65, 0x12, 0x58, 0x0a, 0x0f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, + 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x05, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, + 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, + 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x0e, 0x65, + 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x44, 0x0a, + 0x0a, 0x6f, 0x6e, 0x65, 0x6f, 0x66, 0x5f, 0x64, 0x65, 0x63, 0x6c, 0x18, 0x08, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x25, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x4f, 0x6e, 0x65, 0x6f, 0x66, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x09, 0x6f, 0x6e, 0x65, 0x6f, 0x66, 0x44, + 0x65, 0x63, 0x6c, 0x12, 0x39, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x07, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x4f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x55, + 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, + 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x64, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, + 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, + 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x7a, 0x0a, 0x0e, 0x45, 0x78, + 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x73, 0x74, 0x61, + 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, + 0x03, 0x65, 0x6e, 0x64, 0x12, 0x40, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, + 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, + 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x37, 0x0a, 0x0d, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, + 0x65, 0x64, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x10, 0x0a, + 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x22, + 0x7c, 0x0a, 0x15, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x6e, 0x67, + 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, + 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, + 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, + 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x22, 0xc1, 0x06, + 0x0a, 0x14, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, + 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x75, + 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x62, + 0x65, 0x72, 0x12, 0x41, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x2b, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x52, 0x05, + 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x3e, 0x0a, 0x04, 0x74, 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x44, 0x65, 0x73, 0x63, 0x72, + 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, + 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x74, 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, + 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x64, 0x65, 0x65, 0x12, 0x23, + 0x0a, 0x0d, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x56, 0x61, + 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x6e, 0x65, 0x6f, 0x66, 0x5f, 0x69, 0x6e, 0x64, + 0x65, 0x78, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, 0x6f, 0x6e, 0x65, 0x6f, 0x66, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x12, 0x1b, 0x0a, 0x09, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6a, 0x73, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, + 0x65, 0x12, 0x37, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, + 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x33, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x18, 0x11, 0x20, + 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x61, 0x6c, 0x22, 0xb6, 0x02, 0x0a, 0x04, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, + 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x55, 0x42, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x0e, 0x0a, + 0x0a, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x4c, 0x4f, 0x41, 0x54, 0x10, 0x02, 0x12, 0x0e, 0x0a, + 0x0a, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10, 0x03, 0x12, 0x0f, 0x0a, + 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10, 0x04, 0x12, 0x0e, + 0x0a, 0x0a, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x33, 0x32, 0x10, 0x05, 0x12, 0x10, + 0x0a, 0x0c, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x58, 0x45, 0x44, 0x36, 0x34, 0x10, 0x06, + 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x58, 0x45, 0x44, 0x33, 0x32, + 0x10, 0x07, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x4f, 0x4f, 0x4c, 0x10, + 0x08, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, + 0x10, 0x09, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, + 0x10, 0x0a, 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4d, 0x45, 0x53, 0x53, 0x41, + 0x47, 0x45, 0x10, 0x0b, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x42, 0x59, 0x54, + 0x45, 0x53, 0x10, 0x0c, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x49, 0x4e, + 0x54, 0x33, 0x32, 0x10, 0x0d, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x4e, + 0x55, 0x4d, 0x10, 0x0e, 0x12, 0x11, 0x0a, 0x0d, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x46, 0x49, + 0x58, 0x45, 0x44, 0x33, 0x32, 0x10, 0x0f, 0x12, 0x11, 0x0a, 0x0d, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x53, 0x46, 0x49, 0x58, 0x45, 0x44, 0x36, 0x34, 0x10, 0x10, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, + 0x50, 0x45, 0x5f, 0x53, 0x49, 0x4e, 0x54, 0x33, 0x32, 0x10, 0x11, 0x12, 0x0f, 0x0a, 0x0b, 0x54, + 0x59, 0x50, 0x45, 0x5f, 0x53, 0x49, 0x4e, 0x54, 0x36, 0x34, 0x10, 0x12, 0x22, 0x43, 0x0a, 0x05, + 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x12, 0x0a, 0x0e, 0x4c, 0x41, 0x42, 0x45, 0x4c, 0x5f, 0x4f, + 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x41, 0x4c, 0x10, 0x01, 0x12, 0x12, 0x0a, 0x0e, 0x4c, 0x41, 0x42, + 0x45, 0x4c, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, 0x44, 0x10, 0x02, 0x12, 0x12, 0x0a, + 0x0e, 0x4c, 0x41, 0x42, 0x45, 0x4c, 0x5f, 0x52, 0x45, 0x50, 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, + 0x03, 0x22, 0x63, 0x0a, 0x14, 0x4f, 0x6e, 0x65, 0x6f, 0x66, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x37, 0x0a, + 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, + 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, + 0x2e, 0x4f, 0x6e, 0x65, 0x6f, 0x66, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xe3, 0x02, 0x0a, 0x13, 0x45, 0x6e, 0x75, 0x6d, 0x44, + 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, + 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x3f, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x29, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x44, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x05, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, + 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x5d, 0x0a, 0x0e, 0x72, + 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x04, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x65, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x0d, 0x72, 0x65, 0x73, + 0x65, 0x72, 0x76, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, + 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x03, 0x28, + 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x1a, + 0x3b, 0x0a, 0x11, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x52, + 0x61, 0x6e, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, + 0x01, 0x28, 0x05, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, + 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x22, 0x83, 0x01, 0x0a, + 0x18, 0x45, 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, + 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, + 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x06, 0x6e, + 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x3b, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, + 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x22, 0xa7, 0x01, 0x0a, 0x16, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x3b, 0x0a, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, - 0x32, 0x25, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, - 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, - 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x05, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x12, 0x43, - 0x0a, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x06, 0x20, 0x03, 0x28, - 0x0b, 0x32, 0x25, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x09, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, - 0x69, 0x6f, 0x6e, 0x12, 0x41, 0x0a, 0x0b, 0x6e, 0x65, 0x73, 0x74, 0x65, 0x64, 0x5f, 0x74, 0x79, - 0x70, 0x65, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x20, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, - 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x0a, 0x6e, 0x65, 0x73, 0x74, - 0x65, 0x64, 0x54, 0x79, 0x70, 0x65, 0x12, 0x41, 0x0a, 0x09, 0x65, 0x6e, 0x75, 0x6d, 0x5f, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, - 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, - 0x08, 0x65, 0x6e, 0x75, 0x6d, 0x54, 0x79, 0x70, 0x65, 0x12, 0x58, 0x0a, 0x0f, 0x65, 0x78, 0x74, - 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x05, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x2f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, - 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x61, - 0x6e, 0x67, 0x65, 0x52, 0x0e, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x61, - 0x6e, 0x67, 0x65, 0x12, 0x44, 0x0a, 0x0a, 0x6f, 0x6e, 0x65, 0x6f, 0x66, 0x5f, 0x64, 0x65, 0x63, - 0x6c, 0x18, 0x08, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4f, 0x6e, 0x65, 0x6f, 0x66, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x09, - 0x6f, 0x6e, 0x65, 0x6f, 0x66, 0x44, 0x65, 0x63, 0x6c, 0x12, 0x39, 0x0a, 0x07, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x73, - 0x73, 0x61, 0x67, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x55, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, - 0x5f, 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2e, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x52, - 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x52, 0x0d, 0x72, 0x65, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x72, - 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x03, - 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x4e, 0x61, 0x6d, 0x65, - 0x1a, 0x7a, 0x0a, 0x0e, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x6e, - 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x05, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x65, 0x6e, 0x64, 0x12, 0x40, 0x0a, 0x07, 0x6f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x6f, - 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x78, - 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x1a, 0x37, 0x0a, 0x0d, - 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x14, 0x0a, - 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x03, 0x65, 0x6e, 0x64, 0x22, 0x7c, 0x0a, 0x15, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, - 0x6f, 0x6e, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x58, - 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, - 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, - 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, - 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, - 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, - 0x80, 0x80, 0x02, 0x22, 0xc1, 0x06, 0x0a, 0x14, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x44, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, - 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x41, 0x0a, 0x05, 0x6c, 0x61, 0x62, 0x65, - 0x6c, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2b, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x4c, - 0x61, 0x62, 0x65, 0x6c, 0x52, 0x05, 0x6c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x3e, 0x0a, 0x04, 0x74, - 0x79, 0x70, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2a, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, - 0x64, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, - 0x2e, 0x54, 0x79, 0x70, 0x65, 0x52, 0x04, 0x74, 0x79, 0x70, 0x65, 0x12, 0x1b, 0x0a, 0x09, 0x74, - 0x79, 0x70, 0x65, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, - 0x74, 0x79, 0x70, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x65, 0x78, 0x74, 0x65, - 0x6e, 0x64, 0x65, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x65, 0x78, 0x74, 0x65, - 0x6e, 0x64, 0x65, 0x65, 0x12, 0x23, 0x0a, 0x0d, 0x64, 0x65, 0x66, 0x61, 0x75, 0x6c, 0x74, 0x5f, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x64, 0x65, 0x66, - 0x61, 0x75, 0x6c, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x6e, 0x65, - 0x6f, 0x66, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x18, 0x09, 0x20, 0x01, 0x28, 0x05, 0x52, 0x0a, - 0x6f, 0x6e, 0x65, 0x6f, 0x66, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x12, 0x1b, 0x0a, 0x09, 0x6a, 0x73, - 0x6f, 0x6e, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x6a, - 0x73, 0x6f, 0x6e, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x37, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x27, 0x0a, 0x0f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x61, 0x6c, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x33, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x61, 0x6c, 0x22, 0xb6, 0x02, 0x0a, 0x04, 0x54, 0x79, - 0x70, 0x65, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x44, 0x4f, 0x55, 0x42, 0x4c, - 0x45, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x4c, 0x4f, 0x41, - 0x54, 0x10, 0x02, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x54, 0x36, - 0x34, 0x10, 0x03, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x49, 0x4e, 0x54, - 0x36, 0x34, 0x10, 0x04, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x49, 0x4e, 0x54, - 0x33, 0x32, 0x10, 0x05, 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, 0x49, 0x58, - 0x45, 0x44, 0x36, 0x34, 0x10, 0x06, 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x46, - 0x49, 0x58, 0x45, 0x44, 0x33, 0x32, 0x10, 0x07, 0x12, 0x0d, 0x0a, 0x09, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x42, 0x4f, 0x4f, 0x4c, 0x10, 0x08, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, - 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x09, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x47, 0x52, 0x4f, 0x55, 0x50, 0x10, 0x0a, 0x12, 0x10, 0x0a, 0x0c, 0x54, 0x59, 0x50, 0x45, - 0x5f, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, 0x10, 0x0b, 0x12, 0x0e, 0x0a, 0x0a, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x42, 0x59, 0x54, 0x45, 0x53, 0x10, 0x0c, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x55, 0x49, 0x4e, 0x54, 0x33, 0x32, 0x10, 0x0d, 0x12, 0x0d, 0x0a, 0x09, 0x54, - 0x59, 0x50, 0x45, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x10, 0x0e, 0x12, 0x11, 0x0a, 0x0d, 0x54, 0x59, - 0x50, 0x45, 0x5f, 0x53, 0x46, 0x49, 0x58, 0x45, 0x44, 0x33, 0x32, 0x10, 0x0f, 0x12, 0x11, 0x0a, - 0x0d, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x46, 0x49, 0x58, 0x45, 0x44, 0x36, 0x34, 0x10, 0x10, - 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x49, 0x4e, 0x54, 0x33, 0x32, 0x10, - 0x11, 0x12, 0x0f, 0x0a, 0x0b, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x53, 0x49, 0x4e, 0x54, 0x36, 0x34, - 0x10, 0x12, 0x22, 0x43, 0x0a, 0x05, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x12, 0x12, 0x0a, 0x0e, 0x4c, - 0x41, 0x42, 0x45, 0x4c, 0x5f, 0x4f, 0x50, 0x54, 0x49, 0x4f, 0x4e, 0x41, 0x4c, 0x10, 0x01, 0x12, - 0x12, 0x0a, 0x0e, 0x4c, 0x41, 0x42, 0x45, 0x4c, 0x5f, 0x52, 0x45, 0x51, 0x55, 0x49, 0x52, 0x45, - 0x44, 0x10, 0x02, 0x12, 0x12, 0x0a, 0x0e, 0x4c, 0x41, 0x42, 0x45, 0x4c, 0x5f, 0x52, 0x45, 0x50, - 0x45, 0x41, 0x54, 0x45, 0x44, 0x10, 0x03, 0x22, 0x63, 0x0a, 0x14, 0x4f, 0x6e, 0x65, 0x6f, 0x66, - 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, - 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x12, 0x37, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, - 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4f, 0x6e, 0x65, 0x6f, 0x66, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xe3, 0x02, 0x0a, - 0x13, 0x45, 0x6e, 0x75, 0x6d, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, - 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3f, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x29, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, 0x56, 0x61, - 0x6c, 0x75, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, - 0x74, 0x6f, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x36, 0x0a, 0x07, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, - 0x6d, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x12, 0x5d, 0x0a, 0x0e, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x5f, 0x72, 0x61, - 0x6e, 0x67, 0x65, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x36, 0x2e, 0x67, 0x6f, 0x6f, 0x67, - 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, 0x6e, 0x75, 0x6d, - 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x2e, - 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x67, - 0x65, 0x52, 0x0d, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x67, 0x65, - 0x12, 0x23, 0x0a, 0x0d, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, 0x64, 0x5f, 0x6e, 0x61, 0x6d, - 0x65, 0x18, 0x05, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0c, 0x72, 0x65, 0x73, 0x65, 0x72, 0x76, 0x65, - 0x64, 0x4e, 0x61, 0x6d, 0x65, 0x1a, 0x3b, 0x0a, 0x11, 0x45, 0x6e, 0x75, 0x6d, 0x52, 0x65, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x64, 0x52, 0x61, 0x6e, 0x67, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x73, 0x74, - 0x61, 0x72, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x73, 0x74, 0x61, 0x72, 0x74, - 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x65, - 0x6e, 0x64, 0x22, 0x83, 0x01, 0x0a, 0x18, 0x45, 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, 0x75, 0x65, - 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, - 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, - 0x61, 0x6d, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x05, 0x52, 0x06, 0x6e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x12, 0x3b, 0x0a, 0x07, 0x6f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x21, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x45, - 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, - 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xa7, 0x01, 0x0a, 0x16, 0x53, 0x65, 0x72, - 0x76, 0x69, 0x63, 0x65, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, - 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, - 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x3e, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, - 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, - 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, - 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x12, 0x39, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, - 0x63, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x73, 0x22, 0x89, 0x02, 0x0a, 0x15, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x44, 0x65, 0x73, - 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, - 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, - 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, - 0x1f, 0x0a, 0x0b, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x38, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, - 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x30, 0x0a, 0x10, 0x63, 0x6c, - 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x18, 0x05, - 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0f, 0x63, 0x6c, 0x69, - 0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x12, 0x30, 0x0a, 0x10, - 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, - 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0f, 0x73, - 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x22, 0x91, - 0x09, 0x0a, 0x0b, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x21, - 0x0a, 0x0c, 0x6a, 0x61, 0x76, 0x61, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x18, 0x01, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x6a, 0x61, 0x76, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, - 0x65, 0x12, 0x30, 0x0a, 0x14, 0x6a, 0x61, 0x76, 0x61, 0x5f, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x5f, - 0x63, 0x6c, 0x61, 0x73, 0x73, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x12, 0x6a, 0x61, 0x76, 0x61, 0x4f, 0x75, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x6e, - 0x61, 0x6d, 0x65, 0x12, 0x35, 0x0a, 0x13, 0x6a, 0x61, 0x76, 0x61, 0x5f, 0x6d, 0x75, 0x6c, 0x74, - 0x69, 0x70, 0x6c, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, - 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x11, 0x6a, 0x61, 0x76, 0x61, 0x4d, 0x75, 0x6c, - 0x74, 0x69, 0x70, 0x6c, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x73, 0x12, 0x44, 0x0a, 0x1d, 0x6a, 0x61, - 0x76, 0x61, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x65, 0x71, 0x75, 0x61, - 0x6c, 0x73, 0x5f, 0x61, 0x6e, 0x64, 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x14, 0x20, 0x01, 0x28, - 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x19, 0x6a, 0x61, 0x76, 0x61, 0x47, 0x65, 0x6e, 0x65, 0x72, - 0x61, 0x74, 0x65, 0x45, 0x71, 0x75, 0x61, 0x6c, 0x73, 0x41, 0x6e, 0x64, 0x48, 0x61, 0x73, 0x68, - 0x12, 0x3a, 0x0a, 0x16, 0x6a, 0x61, 0x76, 0x61, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, - 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, 0x75, 0x74, 0x66, 0x38, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x08, - 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x13, 0x6a, 0x61, 0x76, 0x61, 0x53, 0x74, 0x72, - 0x69, 0x6e, 0x67, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x55, 0x74, 0x66, 0x38, 0x12, 0x53, 0x0a, 0x0c, - 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x69, 0x7a, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x18, 0x09, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x29, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6d, 0x69, 0x7a, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x3a, 0x05, 0x53, - 0x50, 0x45, 0x45, 0x44, 0x52, 0x0b, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x69, 0x7a, 0x65, 0x46, 0x6f, - 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x67, 0x6f, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x18, - 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x67, 0x6f, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, - 0x12, 0x35, 0x0a, 0x13, 0x63, 0x63, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x5f, 0x73, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, - 0x61, 0x6c, 0x73, 0x65, 0x52, 0x11, 0x63, 0x63, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x53, - 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x39, 0x0a, 0x15, 0x6a, 0x61, 0x76, 0x61, 0x5f, - 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, - 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x13, 0x6a, - 0x61, 0x76, 0x61, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x12, 0x35, 0x0a, 0x13, 0x70, 0x79, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, - 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x08, 0x3a, - 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x11, 0x70, 0x79, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x69, - 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x37, 0x0a, 0x14, 0x70, 0x68, 0x70, + 0x65, 0x12, 0x3e, 0x0a, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x18, 0x02, 0x20, 0x03, 0x28, + 0x0b, 0x32, 0x26, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, + 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, + 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x52, 0x06, 0x6d, 0x65, 0x74, 0x68, 0x6f, + 0x64, 0x12, 0x39, 0x0a, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x1f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4f, 0x70, 0x74, 0x69, + 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0x89, 0x02, 0x0a, + 0x15, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, + 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x6e, + 0x70, 0x75, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, + 0x69, 0x6e, 0x70, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x1f, 0x0a, 0x0b, 0x6f, 0x75, 0x74, + 0x70, 0x75, 0x74, 0x5f, 0x74, 0x79, 0x70, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, + 0x6f, 0x75, 0x74, 0x70, 0x75, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x38, 0x0a, 0x07, 0x6f, 0x70, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1e, 0x2e, 0x67, 0x6f, + 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, + 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x07, 0x6f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x30, 0x0a, 0x10, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x5f, 0x73, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, + 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0f, 0x63, 0x6c, 0x69, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x72, + 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x12, 0x30, 0x0a, 0x10, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, + 0x5f, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x18, 0x06, 0x20, 0x01, 0x28, 0x08, + 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0f, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x53, + 0x74, 0x72, 0x65, 0x61, 0x6d, 0x69, 0x6e, 0x67, 0x22, 0x91, 0x09, 0x0a, 0x0b, 0x46, 0x69, 0x6c, + 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x21, 0x0a, 0x0c, 0x6a, 0x61, 0x76, 0x61, + 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, + 0x6a, 0x61, 0x76, 0x61, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x30, 0x0a, 0x14, 0x6a, + 0x61, 0x76, 0x61, 0x5f, 0x6f, 0x75, 0x74, 0x65, 0x72, 0x5f, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x6e, + 0x61, 0x6d, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x12, 0x6a, 0x61, 0x76, 0x61, 0x4f, + 0x75, 0x74, 0x65, 0x72, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x35, 0x0a, + 0x13, 0x6a, 0x61, 0x76, 0x61, 0x5f, 0x6d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x5f, 0x66, + 0x69, 0x6c, 0x65, 0x73, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, + 0x65, 0x52, 0x11, 0x6a, 0x61, 0x76, 0x61, 0x4d, 0x75, 0x6c, 0x74, 0x69, 0x70, 0x6c, 0x65, 0x46, + 0x69, 0x6c, 0x65, 0x73, 0x12, 0x44, 0x0a, 0x1d, 0x6a, 0x61, 0x76, 0x61, 0x5f, 0x67, 0x65, 0x6e, + 0x65, 0x72, 0x61, 0x74, 0x65, 0x5f, 0x65, 0x71, 0x75, 0x61, 0x6c, 0x73, 0x5f, 0x61, 0x6e, 0x64, + 0x5f, 0x68, 0x61, 0x73, 0x68, 0x18, 0x14, 0x20, 0x01, 0x28, 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, + 0x19, 0x6a, 0x61, 0x76, 0x61, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x45, 0x71, 0x75, + 0x61, 0x6c, 0x73, 0x41, 0x6e, 0x64, 0x48, 0x61, 0x73, 0x68, 0x12, 0x3a, 0x0a, 0x16, 0x6a, 0x61, + 0x76, 0x61, 0x5f, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x63, 0x68, 0x65, 0x63, 0x6b, 0x5f, + 0x75, 0x74, 0x66, 0x38, 0x18, 0x1b, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, + 0x65, 0x52, 0x13, 0x6a, 0x61, 0x76, 0x61, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x43, 0x68, 0x65, + 0x63, 0x6b, 0x55, 0x74, 0x66, 0x38, 0x12, 0x53, 0x0a, 0x0c, 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x69, + 0x7a, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x18, 0x09, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x29, 0x2e, 0x67, + 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, + 0x69, 0x6c, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6d, + 0x69, 0x7a, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x3a, 0x05, 0x53, 0x50, 0x45, 0x45, 0x44, 0x52, 0x0b, + 0x6f, 0x70, 0x74, 0x69, 0x6d, 0x69, 0x7a, 0x65, 0x46, 0x6f, 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x67, + 0x6f, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x09, 0x67, 0x6f, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x35, 0x0a, 0x13, 0x63, 0x63, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x73, 0x18, 0x2a, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x12, - 0x70, 0x68, 0x70, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, - 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, - 0x18, 0x17, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0a, 0x64, - 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x2e, 0x0a, 0x10, 0x63, 0x63, 0x5f, - 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x5f, 0x61, 0x72, 0x65, 0x6e, 0x61, 0x73, 0x18, 0x1f, 0x20, - 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, 0x72, 0x75, 0x65, 0x52, 0x0e, 0x63, 0x63, 0x45, 0x6e, 0x61, - 0x62, 0x6c, 0x65, 0x41, 0x72, 0x65, 0x6e, 0x61, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x6f, 0x62, 0x6a, - 0x63, 0x5f, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x24, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x6f, 0x62, 0x6a, 0x63, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x50, - 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x29, 0x0a, 0x10, 0x63, 0x73, 0x68, 0x61, 0x72, 0x70, 0x5f, - 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x25, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x0f, 0x63, 0x73, 0x68, 0x61, 0x72, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, - 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x77, 0x69, 0x66, 0x74, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, - 0x18, 0x27, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x73, 0x77, 0x69, 0x66, 0x74, 0x50, 0x72, 0x65, - 0x66, 0x69, 0x78, 0x12, 0x28, 0x0a, 0x10, 0x70, 0x68, 0x70, 0x5f, 0x63, 0x6c, 0x61, 0x73, 0x73, - 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, - 0x68, 0x70, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x23, 0x0a, - 0x0d, 0x70, 0x68, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x29, - 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, 0x70, 0x68, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, - 0x63, 0x65, 0x12, 0x34, 0x0a, 0x16, 0x70, 0x68, 0x70, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x2c, 0x20, 0x01, - 0x28, 0x09, 0x52, 0x14, 0x70, 0x68, 0x70, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4e, - 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x75, 0x62, 0x79, - 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x18, 0x2d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, - 0x72, 0x75, 0x62, 0x79, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x58, 0x0a, 0x14, 0x75, - 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, - 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x69, - 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x52, 0x13, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x3a, 0x0a, 0x0c, 0x4f, 0x70, 0x74, 0x69, 0x6d, 0x69, 0x7a, - 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, 0x09, 0x0a, 0x05, 0x53, 0x50, 0x45, 0x45, 0x44, 0x10, 0x01, - 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, 0x44, 0x45, 0x5f, 0x53, 0x49, 0x5a, 0x45, 0x10, 0x02, 0x12, - 0x10, 0x0a, 0x0c, 0x4c, 0x49, 0x54, 0x45, 0x5f, 0x52, 0x55, 0x4e, 0x54, 0x49, 0x4d, 0x45, 0x10, - 0x03, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x4a, 0x04, 0x08, 0x26, - 0x10, 0x27, 0x22, 0xd1, 0x02, 0x0a, 0x0e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x3c, 0x0a, 0x17, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, - 0x5f, 0x73, 0x65, 0x74, 0x5f, 0x77, 0x69, 0x72, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x14, 0x6d, - 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x53, 0x65, 0x74, 0x57, 0x69, 0x72, 0x65, 0x46, 0x6f, 0x72, - 0x6d, 0x61, 0x74, 0x12, 0x4c, 0x0a, 0x1f, 0x6e, 0x6f, 0x5f, 0x73, 0x74, 0x61, 0x6e, 0x64, 0x61, - 0x72, 0x64, 0x5f, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x5f, 0x61, 0x63, - 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, - 0x6c, 0x73, 0x65, 0x52, 0x1c, 0x6e, 0x6f, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x44, - 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, - 0x72, 0x12, 0x25, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0a, 0x64, 0x65, - 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x61, 0x70, 0x5f, - 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6d, 0x61, 0x70, - 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x73, 0x18, 0x10, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x11, + 0x63, 0x63, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, + 0x73, 0x12, 0x39, 0x0a, 0x15, 0x6a, 0x61, 0x76, 0x61, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, + 0x63, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x11, 0x20, 0x01, 0x28, 0x08, + 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x13, 0x6a, 0x61, 0x76, 0x61, 0x47, 0x65, 0x6e, + 0x65, 0x72, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x35, 0x0a, 0x13, + 0x70, 0x79, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x18, 0x12, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, + 0x52, 0x11, 0x70, 0x79, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, + 0x63, 0x65, 0x73, 0x12, 0x37, 0x0a, 0x14, 0x70, 0x68, 0x70, 0x5f, 0x67, 0x65, 0x6e, 0x65, 0x72, + 0x69, 0x63, 0x5f, 0x73, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x18, 0x2a, 0x20, 0x01, 0x28, + 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x12, 0x70, 0x68, 0x70, 0x47, 0x65, 0x6e, + 0x65, 0x72, 0x69, 0x63, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x73, 0x12, 0x25, 0x0a, 0x0a, + 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x17, 0x20, 0x01, 0x28, 0x08, + 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, + 0x74, 0x65, 0x64, 0x12, 0x2e, 0x0a, 0x10, 0x63, 0x63, 0x5f, 0x65, 0x6e, 0x61, 0x62, 0x6c, 0x65, + 0x5f, 0x61, 0x72, 0x65, 0x6e, 0x61, 0x73, 0x18, 0x1f, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x04, 0x74, + 0x72, 0x75, 0x65, 0x52, 0x0e, 0x63, 0x63, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x41, 0x72, 0x65, + 0x6e, 0x61, 0x73, 0x12, 0x2a, 0x0a, 0x11, 0x6f, 0x62, 0x6a, 0x63, 0x5f, 0x63, 0x6c, 0x61, 0x73, + 0x73, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x24, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, + 0x6f, 0x62, 0x6a, 0x63, 0x43, 0x6c, 0x61, 0x73, 0x73, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, + 0x29, 0x0a, 0x10, 0x63, 0x73, 0x68, 0x61, 0x72, 0x70, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x73, 0x70, + 0x61, 0x63, 0x65, 0x18, 0x25, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x63, 0x73, 0x68, 0x61, 0x72, + 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x73, 0x77, + 0x69, 0x66, 0x74, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, 0x78, 0x18, 0x27, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x0b, 0x73, 0x77, 0x69, 0x66, 0x74, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x28, 0x0a, + 0x10, 0x70, 0x68, 0x70, 0x5f, 0x63, 0x6c, 0x61, 0x73, 0x73, 0x5f, 0x70, 0x72, 0x65, 0x66, 0x69, + 0x78, 0x18, 0x28, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x68, 0x70, 0x43, 0x6c, 0x61, 0x73, + 0x73, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x12, 0x23, 0x0a, 0x0d, 0x70, 0x68, 0x70, 0x5f, 0x6e, + 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x29, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0c, + 0x70, 0x68, 0x70, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x12, 0x34, 0x0a, 0x16, + 0x70, 0x68, 0x70, 0x5f, 0x6d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x5f, 0x6e, 0x61, 0x6d, + 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x18, 0x2c, 0x20, 0x01, 0x28, 0x09, 0x52, 0x14, 0x70, 0x68, + 0x70, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, + 0x63, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x72, 0x75, 0x62, 0x79, 0x5f, 0x70, 0x61, 0x63, 0x6b, 0x61, + 0x67, 0x65, 0x18, 0x2d, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x72, 0x75, 0x62, 0x79, 0x50, 0x61, + 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, 0x6e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x2a, - 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x4a, 0x04, 0x08, 0x08, 0x10, 0x09, - 0x4a, 0x04, 0x08, 0x09, 0x10, 0x0a, 0x22, 0xe2, 0x03, 0x0a, 0x0c, 0x46, 0x69, 0x65, 0x6c, 0x64, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x41, 0x0a, 0x05, 0x63, 0x74, 0x79, 0x70, 0x65, - 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x22, + 0x3a, 0x0a, 0x0c, 0x4f, 0x70, 0x74, 0x69, 0x6d, 0x69, 0x7a, 0x65, 0x4d, 0x6f, 0x64, 0x65, 0x12, + 0x09, 0x0a, 0x05, 0x53, 0x50, 0x45, 0x45, 0x44, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x4f, + 0x44, 0x45, 0x5f, 0x53, 0x49, 0x5a, 0x45, 0x10, 0x02, 0x12, 0x10, 0x0a, 0x0c, 0x4c, 0x49, 0x54, + 0x45, 0x5f, 0x52, 0x55, 0x4e, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x03, 0x2a, 0x09, 0x08, 0xe8, 0x07, + 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x4a, 0x04, 0x08, 0x26, 0x10, 0x27, 0x22, 0xbb, 0x03, 0x0a, + 0x0e, 0x4d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, + 0x3c, 0x0a, 0x17, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x5f, 0x73, 0x65, 0x74, 0x5f, 0x77, + 0x69, 0x72, 0x65, 0x5f, 0x66, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, + 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x14, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, + 0x53, 0x65, 0x74, 0x57, 0x69, 0x72, 0x65, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74, 0x12, 0x4c, 0x0a, + 0x1f, 0x6e, 0x6f, 0x5f, 0x73, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x5f, 0x64, 0x65, 0x73, + 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x5f, 0x61, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x1c, 0x6e, + 0x6f, 0x53, 0x74, 0x61, 0x6e, 0x64, 0x61, 0x72, 0x64, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, + 0x74, 0x6f, 0x72, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x12, 0x25, 0x0a, 0x0a, 0x64, + 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x3a, + 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, + 0x65, 0x64, 0x12, 0x1b, 0x0a, 0x09, 0x6d, 0x61, 0x70, 0x5f, 0x65, 0x6e, 0x74, 0x72, 0x79, 0x18, + 0x07, 0x20, 0x01, 0x28, 0x08, 0x52, 0x08, 0x6d, 0x61, 0x70, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, + 0x56, 0x0a, 0x26, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x5f, 0x6c, 0x65, + 0x67, 0x61, 0x63, 0x79, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x66, 0x69, 0x65, 0x6c, 0x64, 0x5f, + 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x73, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x08, 0x42, + 0x02, 0x18, 0x01, 0x52, 0x22, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x4c, + 0x65, 0x67, 0x61, 0x63, 0x79, 0x4a, 0x73, 0x6f, 0x6e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x43, 0x6f, + 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x73, 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, + 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, + 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, 0x6e, + 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x4a, 0x04, 0x08, 0x04, + 0x10, 0x05, 0x4a, 0x04, 0x08, 0x05, 0x10, 0x06, 0x4a, 0x04, 0x08, 0x06, 0x10, 0x07, 0x4a, 0x04, + 0x08, 0x08, 0x10, 0x09, 0x4a, 0x04, 0x08, 0x09, 0x10, 0x0a, 0x22, 0xb7, 0x08, 0x0a, 0x0c, 0x46, + 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x41, 0x0a, 0x05, 0x63, + 0x74, 0x79, 0x70, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x23, 0x2e, 0x67, 0x6f, 0x6f, + 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, + 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x43, 0x54, 0x79, 0x70, 0x65, 0x3a, + 0x06, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x52, 0x05, 0x63, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, + 0x0a, 0x06, 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, + 0x70, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x12, 0x47, 0x0a, 0x06, 0x6a, 0x73, 0x74, 0x79, 0x70, 0x65, + 0x18, 0x06, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x43, 0x54, 0x79, 0x70, 0x65, 0x3a, 0x06, 0x53, 0x54, 0x52, - 0x49, 0x4e, 0x47, 0x52, 0x05, 0x63, 0x74, 0x79, 0x70, 0x65, 0x12, 0x16, 0x0a, 0x06, 0x70, 0x61, - 0x63, 0x6b, 0x65, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x70, 0x61, 0x63, 0x6b, - 0x65, 0x64, 0x12, 0x47, 0x0a, 0x06, 0x6a, 0x73, 0x74, 0x79, 0x70, 0x65, 0x18, 0x06, 0x20, 0x01, - 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, - 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x73, 0x2e, 0x4a, 0x53, 0x54, 0x79, 0x70, 0x65, 0x3a, 0x09, 0x4a, 0x53, 0x5f, 0x4e, 0x4f, 0x52, - 0x4d, 0x41, 0x4c, 0x52, 0x06, 0x6a, 0x73, 0x74, 0x79, 0x70, 0x65, 0x12, 0x19, 0x0a, 0x04, 0x6c, - 0x61, 0x7a, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, - 0x52, 0x04, 0x6c, 0x61, 0x7a, 0x79, 0x12, 0x25, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, - 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, - 0x65, 0x52, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x19, 0x0a, - 0x04, 0x77, 0x65, 0x61, 0x6b, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, - 0x73, 0x65, 0x52, 0x04, 0x77, 0x65, 0x61, 0x6b, 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, - 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x22, 0x2f, 0x0a, 0x05, 0x43, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, 0x0a, 0x06, 0x53, - 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x43, 0x4f, 0x52, 0x44, 0x10, - 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x50, 0x49, 0x45, 0x43, - 0x45, 0x10, 0x02, 0x22, 0x35, 0x0a, 0x06, 0x4a, 0x53, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0d, 0x0a, - 0x09, 0x4a, 0x53, 0x5f, 0x4e, 0x4f, 0x52, 0x4d, 0x41, 0x4c, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, - 0x4a, 0x53, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, 0x0a, 0x09, 0x4a, - 0x53, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x02, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, - 0x80, 0x80, 0x80, 0x80, 0x02, 0x4a, 0x04, 0x08, 0x04, 0x10, 0x05, 0x22, 0x73, 0x0a, 0x0c, 0x4f, - 0x6e, 0x65, 0x6f, 0x66, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x58, 0x0a, 0x14, 0x75, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4a, 0x53, 0x54, 0x79, 0x70, 0x65, 0x3a, 0x09, 0x4a, 0x53, + 0x5f, 0x4e, 0x4f, 0x52, 0x4d, 0x41, 0x4c, 0x52, 0x06, 0x6a, 0x73, 0x74, 0x79, 0x70, 0x65, 0x12, + 0x19, 0x0a, 0x04, 0x6c, 0x61, 0x7a, 0x79, 0x18, 0x05, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, + 0x61, 0x6c, 0x73, 0x65, 0x52, 0x04, 0x6c, 0x61, 0x7a, 0x79, 0x12, 0x2e, 0x0a, 0x0f, 0x75, 0x6e, + 0x76, 0x65, 0x72, 0x69, 0x66, 0x69, 0x65, 0x64, 0x5f, 0x6c, 0x61, 0x7a, 0x79, 0x18, 0x0f, 0x20, + 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0e, 0x75, 0x6e, 0x76, 0x65, + 0x72, 0x69, 0x66, 0x69, 0x65, 0x64, 0x4c, 0x61, 0x7a, 0x79, 0x12, 0x25, 0x0a, 0x0a, 0x64, 0x65, + 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, + 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, + 0x64, 0x12, 0x19, 0x0a, 0x04, 0x77, 0x65, 0x61, 0x6b, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x08, 0x3a, + 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x04, 0x77, 0x65, 0x61, 0x6b, 0x12, 0x28, 0x0a, 0x0c, + 0x64, 0x65, 0x62, 0x75, 0x67, 0x5f, 0x72, 0x65, 0x64, 0x61, 0x63, 0x74, 0x18, 0x10, 0x20, 0x01, + 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0b, 0x64, 0x65, 0x62, 0x75, 0x67, + 0x52, 0x65, 0x64, 0x61, 0x63, 0x74, 0x12, 0x4b, 0x0a, 0x09, 0x72, 0x65, 0x74, 0x65, 0x6e, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x11, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, + 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, + 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x09, 0x72, 0x65, 0x74, 0x65, 0x6e, 0x74, + 0x69, 0x6f, 0x6e, 0x12, 0x46, 0x0a, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x18, 0x12, 0x20, + 0x01, 0x28, 0x0e, 0x32, 0x2e, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x46, 0x69, 0x65, 0x6c, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x73, 0x2e, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, + 0x79, 0x70, 0x65, 0x52, 0x06, 0x74, 0x61, 0x72, 0x67, 0x65, 0x74, 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, - 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, - 0x22, 0xc0, 0x01, 0x0a, 0x0b, 0x45, 0x6e, 0x75, 0x6d, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, - 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x41, 0x6c, 0x69, 0x61, - 0x73, 0x12, 0x25, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, - 0x03, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0a, 0x64, 0x65, - 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, - 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, - 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, - 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, - 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x4a, 0x04, 0x08, - 0x05, 0x10, 0x06, 0x22, 0x9e, 0x01, 0x0a, 0x10, 0x45, 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, 0x75, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x2f, 0x0a, 0x05, 0x43, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0a, + 0x0a, 0x06, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x08, 0x0a, 0x04, 0x43, 0x4f, + 0x52, 0x44, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x5f, 0x50, + 0x49, 0x45, 0x43, 0x45, 0x10, 0x02, 0x22, 0x35, 0x0a, 0x06, 0x4a, 0x53, 0x54, 0x79, 0x70, 0x65, + 0x12, 0x0d, 0x0a, 0x09, 0x4a, 0x53, 0x5f, 0x4e, 0x4f, 0x52, 0x4d, 0x41, 0x4c, 0x10, 0x00, 0x12, + 0x0d, 0x0a, 0x09, 0x4a, 0x53, 0x5f, 0x53, 0x54, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x01, 0x12, 0x0d, + 0x0a, 0x09, 0x4a, 0x53, 0x5f, 0x4e, 0x55, 0x4d, 0x42, 0x45, 0x52, 0x10, 0x02, 0x22, 0x55, 0x0a, + 0x0f, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x65, 0x74, 0x65, 0x6e, 0x74, 0x69, 0x6f, 0x6e, + 0x12, 0x15, 0x0a, 0x11, 0x52, 0x45, 0x54, 0x45, 0x4e, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x55, 0x4e, + 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x15, 0x0a, 0x11, 0x52, 0x45, 0x54, 0x45, 0x4e, + 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x52, 0x55, 0x4e, 0x54, 0x49, 0x4d, 0x45, 0x10, 0x01, 0x12, 0x14, + 0x0a, 0x10, 0x52, 0x45, 0x54, 0x45, 0x4e, 0x54, 0x49, 0x4f, 0x4e, 0x5f, 0x53, 0x4f, 0x55, 0x52, + 0x43, 0x45, 0x10, 0x02, 0x22, 0x8c, 0x02, 0x0a, 0x10, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x54, + 0x61, 0x72, 0x67, 0x65, 0x74, 0x54, 0x79, 0x70, 0x65, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x41, 0x52, + 0x47, 0x45, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, + 0x10, 0x00, 0x12, 0x14, 0x0a, 0x10, 0x54, 0x41, 0x52, 0x47, 0x45, 0x54, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x46, 0x49, 0x4c, 0x45, 0x10, 0x01, 0x12, 0x1f, 0x0a, 0x1b, 0x54, 0x41, 0x52, 0x47, + 0x45, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x58, 0x54, 0x45, 0x4e, 0x53, 0x49, 0x4f, + 0x4e, 0x5f, 0x52, 0x41, 0x4e, 0x47, 0x45, 0x10, 0x02, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x41, 0x52, + 0x47, 0x45, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4d, 0x45, 0x53, 0x53, 0x41, 0x47, 0x45, + 0x10, 0x03, 0x12, 0x15, 0x0a, 0x11, 0x54, 0x41, 0x52, 0x47, 0x45, 0x54, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x46, 0x49, 0x45, 0x4c, 0x44, 0x10, 0x04, 0x12, 0x15, 0x0a, 0x11, 0x54, 0x41, 0x52, + 0x47, 0x45, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4f, 0x4e, 0x45, 0x4f, 0x46, 0x10, 0x05, + 0x12, 0x14, 0x0a, 0x10, 0x54, 0x41, 0x52, 0x47, 0x45, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, + 0x45, 0x4e, 0x55, 0x4d, 0x10, 0x06, 0x12, 0x1a, 0x0a, 0x16, 0x54, 0x41, 0x52, 0x47, 0x45, 0x54, + 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x45, 0x4e, 0x55, 0x4d, 0x5f, 0x45, 0x4e, 0x54, 0x52, 0x59, + 0x10, 0x07, 0x12, 0x17, 0x0a, 0x13, 0x54, 0x41, 0x52, 0x47, 0x45, 0x54, 0x5f, 0x54, 0x59, 0x50, + 0x45, 0x5f, 0x53, 0x45, 0x52, 0x56, 0x49, 0x43, 0x45, 0x10, 0x08, 0x12, 0x16, 0x0a, 0x12, 0x54, + 0x41, 0x52, 0x47, 0x45, 0x54, 0x5f, 0x54, 0x59, 0x50, 0x45, 0x5f, 0x4d, 0x45, 0x54, 0x48, 0x4f, + 0x44, 0x10, 0x09, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x4a, 0x04, + 0x08, 0x04, 0x10, 0x05, 0x22, 0x73, 0x0a, 0x0c, 0x4f, 0x6e, 0x65, 0x6f, 0x66, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, + 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0xe7, 0x07, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, + 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, 0x6e, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x2a, 0x09, + 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x22, 0x98, 0x02, 0x0a, 0x0b, 0x45, 0x6e, + 0x75, 0x6d, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1f, 0x0a, 0x0b, 0x61, 0x6c, 0x6c, + 0x6f, 0x77, 0x5f, 0x61, 0x6c, 0x69, 0x61, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, + 0x61, 0x6c, 0x6c, 0x6f, 0x77, 0x41, 0x6c, 0x69, 0x61, 0x73, 0x12, 0x25, 0x0a, 0x0a, 0x64, 0x65, + 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, + 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, + 0x64, 0x12, 0x56, 0x0a, 0x26, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x5f, + 0x6c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x5f, 0x6a, 0x73, 0x6f, 0x6e, 0x5f, 0x66, 0x69, 0x65, 0x6c, + 0x64, 0x5f, 0x63, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x73, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x08, 0x42, 0x02, 0x18, 0x01, 0x52, 0x22, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, + 0x64, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79, 0x4a, 0x73, 0x6f, 0x6e, 0x46, 0x69, 0x65, 0x6c, 0x64, + 0x43, 0x6f, 0x6e, 0x66, 0x6c, 0x69, 0x63, 0x74, 0x73, 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, + 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, + 0x6e, 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, + 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, + 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, + 0x69, 0x6f, 0x6e, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x4a, 0x04, + 0x08, 0x05, 0x10, 0x06, 0x22, 0x9e, 0x01, 0x0a, 0x10, 0x45, 0x6e, 0x75, 0x6d, 0x56, 0x61, 0x6c, + 0x75, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x25, 0x0a, 0x0a, 0x64, 0x65, 0x70, + 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, + 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, + 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, + 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, + 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, + 0x80, 0x80, 0x80, 0x80, 0x02, 0x22, 0x9c, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x25, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x72, - 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, + 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x21, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, 0x52, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, @@ -3385,97 +3774,95 @@ var file_google_protobuf_descriptor_proto_rawDesc = []byte{ 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, - 0x80, 0x80, 0x80, 0x02, 0x22, 0x9c, 0x01, 0x0a, 0x0e, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x25, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, - 0x63, 0x61, 0x74, 0x65, 0x64, 0x18, 0x21, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, - 0x73, 0x65, 0x52, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x58, - 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x5f, - 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, 0x2e, + 0x80, 0x80, 0x80, 0x02, 0x22, 0xe0, 0x02, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x25, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, + 0x61, 0x74, 0x65, 0x64, 0x18, 0x21, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, + 0x65, 0x52, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x71, 0x0a, + 0x11, 0x69, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x6c, 0x65, 0x76, + 0x65, 0x6c, 0x18, 0x22, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, + 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x49, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, + 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x3a, 0x13, 0x49, 0x44, 0x45, 0x4d, 0x50, + 0x4f, 0x54, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x52, 0x10, + 0x69, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, + 0x12, 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, + 0x64, 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, + 0x24, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, + 0x66, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, + 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, + 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x50, 0x0a, 0x10, 0x49, 0x64, + 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x17, + 0x0a, 0x13, 0x49, 0x44, 0x45, 0x4d, 0x50, 0x4f, 0x54, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x55, 0x4e, + 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x4e, 0x4f, 0x5f, 0x53, 0x49, + 0x44, 0x45, 0x5f, 0x45, 0x46, 0x46, 0x45, 0x43, 0x54, 0x53, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, + 0x49, 0x44, 0x45, 0x4d, 0x50, 0x4f, 0x54, 0x45, 0x4e, 0x54, 0x10, 0x02, 0x2a, 0x09, 0x08, 0xe8, + 0x07, 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x22, 0x9a, 0x03, 0x0a, 0x13, 0x55, 0x6e, 0x69, 0x6e, + 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, + 0x41, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, - 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, - 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x2a, 0x09, 0x08, 0xe8, 0x07, 0x10, 0x80, 0x80, - 0x80, 0x80, 0x02, 0x22, 0xe0, 0x02, 0x0a, 0x0d, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x25, 0x0a, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, - 0x74, 0x65, 0x64, 0x18, 0x21, 0x20, 0x01, 0x28, 0x08, 0x3a, 0x05, 0x66, 0x61, 0x6c, 0x73, 0x65, - 0x52, 0x0a, 0x64, 0x65, 0x70, 0x72, 0x65, 0x63, 0x61, 0x74, 0x65, 0x64, 0x12, 0x71, 0x0a, 0x11, - 0x69, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x5f, 0x6c, 0x65, 0x76, 0x65, - 0x6c, 0x18, 0x22, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x2f, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x4d, 0x65, 0x74, 0x68, 0x6f, 0x64, - 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x2e, 0x49, 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, - 0x6e, 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x3a, 0x13, 0x49, 0x44, 0x45, 0x4d, 0x50, 0x4f, - 0x54, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x52, 0x10, 0x69, - 0x64, 0x65, 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, - 0x58, 0x0a, 0x14, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, - 0x5f, 0x6f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0xe7, 0x07, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x24, - 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, - 0x2e, 0x55, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x13, 0x75, 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, - 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x22, 0x50, 0x0a, 0x10, 0x49, 0x64, 0x65, - 0x6d, 0x70, 0x6f, 0x74, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x65, 0x76, 0x65, 0x6c, 0x12, 0x17, 0x0a, - 0x13, 0x49, 0x44, 0x45, 0x4d, 0x50, 0x4f, 0x54, 0x45, 0x4e, 0x43, 0x59, 0x5f, 0x55, 0x4e, 0x4b, - 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x13, 0x0a, 0x0f, 0x4e, 0x4f, 0x5f, 0x53, 0x49, 0x44, - 0x45, 0x5f, 0x45, 0x46, 0x46, 0x45, 0x43, 0x54, 0x53, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x49, - 0x44, 0x45, 0x4d, 0x50, 0x4f, 0x54, 0x45, 0x4e, 0x54, 0x10, 0x02, 0x2a, 0x09, 0x08, 0xe8, 0x07, - 0x10, 0x80, 0x80, 0x80, 0x80, 0x02, 0x22, 0x9a, 0x03, 0x0a, 0x13, 0x55, 0x6e, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x41, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, - 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x55, - 0x6e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x70, 0x72, 0x65, 0x74, 0x65, 0x64, 0x4f, 0x70, 0x74, 0x69, - 0x6f, 0x6e, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x50, 0x61, 0x72, 0x74, 0x52, 0x04, 0x6e, 0x61, 0x6d, - 0x65, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x5f, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x64, 0x65, - 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x2c, 0x0a, 0x12, - 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, - 0x76, 0x65, 0x49, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x6e, 0x65, - 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, - 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, - 0x49, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x6f, 0x75, 0x62, - 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0b, - 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x73, - 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, - 0x0c, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x27, - 0x0a, 0x0f, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, - 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, - 0x74, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x4a, 0x0a, 0x08, 0x4e, 0x61, 0x6d, 0x65, 0x50, - 0x61, 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x70, 0x61, 0x72, 0x74, - 0x18, 0x01, 0x20, 0x02, 0x28, 0x09, 0x52, 0x08, 0x6e, 0x61, 0x6d, 0x65, 0x50, 0x61, 0x72, 0x74, - 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x73, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, 0x6e, - 0x18, 0x02, 0x20, 0x02, 0x28, 0x08, 0x52, 0x0b, 0x69, 0x73, 0x45, 0x78, 0x74, 0x65, 0x6e, 0x73, - 0x69, 0x6f, 0x6e, 0x22, 0xa7, 0x02, 0x0a, 0x0e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, 0x6f, - 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x44, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, - 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x6f, 0x75, 0x72, 0x63, - 0x65, 0x43, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xce, 0x01, 0x0a, - 0x08, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x04, 0x70, 0x61, 0x74, - 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x01, 0x52, 0x04, 0x70, 0x61, 0x74, - 0x68, 0x12, 0x16, 0x0a, 0x04, 0x73, 0x70, 0x61, 0x6e, 0x18, 0x02, 0x20, 0x03, 0x28, 0x05, 0x42, - 0x02, 0x10, 0x01, 0x52, 0x04, 0x73, 0x70, 0x61, 0x6e, 0x12, 0x29, 0x0a, 0x10, 0x6c, 0x65, 0x61, - 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x0f, 0x6c, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6d, 0x6d, - 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x74, 0x72, 0x61, 0x69, 0x6c, 0x69, 0x6e, 0x67, - 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, - 0x10, 0x74, 0x72, 0x61, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, - 0x73, 0x12, 0x3a, 0x0a, 0x19, 0x6c, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x65, 0x74, - 0x61, 0x63, 0x68, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x06, - 0x20, 0x03, 0x28, 0x09, 0x52, 0x17, 0x6c, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x65, 0x74, - 0x61, 0x63, 0x68, 0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xd1, 0x01, - 0x0a, 0x11, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x64, 0x65, 0x49, - 0x6e, 0x66, 0x6f, 0x12, 0x4d, 0x0a, 0x0a, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, - 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, - 0x74, 0x65, 0x64, 0x43, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, - 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x1a, 0x6d, 0x0a, 0x0a, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, - 0x12, 0x16, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, - 0x10, 0x01, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6f, 0x75, 0x72, - 0x63, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0a, 0x73, - 0x6f, 0x75, 0x72, 0x63, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x62, 0x65, 0x67, - 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x62, 0x65, 0x67, 0x69, 0x6e, 0x12, - 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, 0x65, 0x6e, - 0x64, 0x42, 0x7e, 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, + 0x69, 0x6f, 0x6e, 0x2e, 0x4e, 0x61, 0x6d, 0x65, 0x50, 0x61, 0x72, 0x74, 0x52, 0x04, 0x6e, 0x61, + 0x6d, 0x65, 0x12, 0x29, 0x0a, 0x10, 0x69, 0x64, 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, + 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x69, 0x64, + 0x65, 0x6e, 0x74, 0x69, 0x66, 0x69, 0x65, 0x72, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x2c, 0x0a, + 0x12, 0x70, 0x6f, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x18, 0x04, 0x20, 0x01, 0x28, 0x04, 0x52, 0x10, 0x70, 0x6f, 0x73, 0x69, 0x74, + 0x69, 0x76, 0x65, 0x49, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x2c, 0x0a, 0x12, 0x6e, + 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, 0x65, 0x5f, 0x69, 0x6e, 0x74, 0x5f, 0x76, 0x61, 0x6c, 0x75, + 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x03, 0x52, 0x10, 0x6e, 0x65, 0x67, 0x61, 0x74, 0x69, 0x76, + 0x65, 0x49, 0x6e, 0x74, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0c, 0x64, 0x6f, 0x75, + 0x62, 0x6c, 0x65, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x01, 0x52, + 0x0b, 0x64, 0x6f, 0x75, 0x62, 0x6c, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x21, 0x0a, 0x0c, + 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x5f, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x07, 0x20, 0x01, + 0x28, 0x0c, 0x52, 0x0b, 0x73, 0x74, 0x72, 0x69, 0x6e, 0x67, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, + 0x27, 0x0a, 0x0f, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, 0x61, 0x74, 0x65, 0x5f, 0x76, 0x61, 0x6c, + 0x75, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x61, 0x67, 0x67, 0x72, 0x65, 0x67, + 0x61, 0x74, 0x65, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x1a, 0x4a, 0x0a, 0x08, 0x4e, 0x61, 0x6d, 0x65, + 0x50, 0x61, 0x72, 0x74, 0x12, 0x1b, 0x0a, 0x09, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x70, 0x61, 0x72, + 0x74, 0x18, 0x01, 0x20, 0x02, 0x28, 0x09, 0x52, 0x08, 0x6e, 0x61, 0x6d, 0x65, 0x50, 0x61, 0x72, + 0x74, 0x12, 0x21, 0x0a, 0x0c, 0x69, 0x73, 0x5f, 0x65, 0x78, 0x74, 0x65, 0x6e, 0x73, 0x69, 0x6f, + 0x6e, 0x18, 0x02, 0x20, 0x02, 0x28, 0x08, 0x52, 0x0b, 0x69, 0x73, 0x45, 0x78, 0x74, 0x65, 0x6e, + 0x73, 0x69, 0x6f, 0x6e, 0x22, 0xa7, 0x02, 0x0a, 0x0e, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x43, + 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x44, 0x0a, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x67, 0x6f, 0x6f, 0x67, + 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x53, 0x6f, 0x75, 0x72, + 0x63, 0x65, 0x43, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x4c, 0x6f, 0x63, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x52, 0x08, 0x6c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x1a, 0xce, 0x01, + 0x0a, 0x08, 0x4c, 0x6f, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x05, 0x42, 0x02, 0x10, 0x01, 0x52, 0x04, 0x70, 0x61, + 0x74, 0x68, 0x12, 0x16, 0x0a, 0x04, 0x73, 0x70, 0x61, 0x6e, 0x18, 0x02, 0x20, 0x03, 0x28, 0x05, + 0x42, 0x02, 0x10, 0x01, 0x52, 0x04, 0x73, 0x70, 0x61, 0x6e, 0x12, 0x29, 0x0a, 0x10, 0x6c, 0x65, + 0x61, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x03, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x0f, 0x6c, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6d, + 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x2b, 0x0a, 0x11, 0x74, 0x72, 0x61, 0x69, 0x6c, 0x69, 0x6e, + 0x67, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x10, 0x74, 0x72, 0x61, 0x69, 0x6c, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, + 0x74, 0x73, 0x12, 0x3a, 0x0a, 0x19, 0x6c, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x5f, 0x64, 0x65, + 0x74, 0x61, 0x63, 0x68, 0x65, 0x64, 0x5f, 0x63, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x18, + 0x06, 0x20, 0x03, 0x28, 0x09, 0x52, 0x17, 0x6c, 0x65, 0x61, 0x64, 0x69, 0x6e, 0x67, 0x44, 0x65, + 0x74, 0x61, 0x63, 0x68, 0x65, 0x64, 0x43, 0x6f, 0x6d, 0x6d, 0x65, 0x6e, 0x74, 0x73, 0x22, 0xd0, + 0x02, 0x0a, 0x11, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x64, 0x65, + 0x49, 0x6e, 0x66, 0x6f, 0x12, 0x4d, 0x0a, 0x0a, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, + 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, + 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x6e, 0x6e, + 0x6f, 0x74, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0a, 0x61, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x1a, 0xeb, 0x01, 0x0a, 0x0a, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, 0x74, 0x69, + 0x6f, 0x6e, 0x12, 0x16, 0x0a, 0x04, 0x70, 0x61, 0x74, 0x68, 0x18, 0x01, 0x20, 0x03, 0x28, 0x05, + 0x42, 0x02, 0x10, 0x01, 0x52, 0x04, 0x70, 0x61, 0x74, 0x68, 0x12, 0x1f, 0x0a, 0x0b, 0x73, 0x6f, + 0x75, 0x72, 0x63, 0x65, 0x5f, 0x66, 0x69, 0x6c, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0a, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x12, 0x14, 0x0a, 0x05, 0x62, + 0x65, 0x67, 0x69, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x05, 0x52, 0x05, 0x62, 0x65, 0x67, 0x69, + 0x6e, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x6e, 0x64, 0x18, 0x04, 0x20, 0x01, 0x28, 0x05, 0x52, 0x03, + 0x65, 0x6e, 0x64, 0x12, 0x52, 0x0a, 0x08, 0x73, 0x65, 0x6d, 0x61, 0x6e, 0x74, 0x69, 0x63, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x36, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x2e, 0x47, 0x65, 0x6e, 0x65, 0x72, 0x61, 0x74, 0x65, + 0x64, 0x43, 0x6f, 0x64, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x2e, 0x41, 0x6e, 0x6e, 0x6f, 0x74, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x53, 0x65, 0x6d, 0x61, 0x6e, 0x74, 0x69, 0x63, 0x52, 0x08, 0x73, + 0x65, 0x6d, 0x61, 0x6e, 0x74, 0x69, 0x63, 0x22, 0x28, 0x0a, 0x08, 0x53, 0x65, 0x6d, 0x61, 0x6e, + 0x74, 0x69, 0x63, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x4f, 0x4e, 0x45, 0x10, 0x00, 0x12, 0x07, 0x0a, + 0x03, 0x53, 0x45, 0x54, 0x10, 0x01, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x4c, 0x49, 0x41, 0x53, 0x10, + 0x02, 0x42, 0x7e, 0x0a, 0x13, 0x63, 0x6f, 0x6d, 0x2e, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x75, 0x66, 0x42, 0x10, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x6f, 0x72, 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x48, 0x01, 0x5a, 0x2d, 0x67, 0x6f, 0x6f, 0x67, 0x6c, 0x65, 0x2e, 0x67, 0x6f, 0x6c, 0x61, 0x6e, 0x67, 0x2e, 0x6f, 0x72, 0x67, 0x2f, @@ -3498,7 +3885,7 @@ func file_google_protobuf_descriptor_proto_rawDescGZIP() []byte { return file_google_protobuf_descriptor_proto_rawDescData } -var file_google_protobuf_descriptor_proto_enumTypes = make([]protoimpl.EnumInfo, 6) +var file_google_protobuf_descriptor_proto_enumTypes = make([]protoimpl.EnumInfo, 9) var file_google_protobuf_descriptor_proto_msgTypes = make([]protoimpl.MessageInfo, 27) var file_google_protobuf_descriptor_proto_goTypes = []interface{}{ (FieldDescriptorProto_Type)(0), // 0: google.protobuf.FieldDescriptorProto.Type @@ -3506,84 +3893,90 @@ var file_google_protobuf_descriptor_proto_goTypes = []interface{}{ (FileOptions_OptimizeMode)(0), // 2: google.protobuf.FileOptions.OptimizeMode (FieldOptions_CType)(0), // 3: google.protobuf.FieldOptions.CType (FieldOptions_JSType)(0), // 4: google.protobuf.FieldOptions.JSType - (MethodOptions_IdempotencyLevel)(0), // 5: google.protobuf.MethodOptions.IdempotencyLevel - (*FileDescriptorSet)(nil), // 6: google.protobuf.FileDescriptorSet - (*FileDescriptorProto)(nil), // 7: google.protobuf.FileDescriptorProto - (*DescriptorProto)(nil), // 8: google.protobuf.DescriptorProto - (*ExtensionRangeOptions)(nil), // 9: google.protobuf.ExtensionRangeOptions - (*FieldDescriptorProto)(nil), // 10: google.protobuf.FieldDescriptorProto - (*OneofDescriptorProto)(nil), // 11: google.protobuf.OneofDescriptorProto - (*EnumDescriptorProto)(nil), // 12: google.protobuf.EnumDescriptorProto - (*EnumValueDescriptorProto)(nil), // 13: google.protobuf.EnumValueDescriptorProto - (*ServiceDescriptorProto)(nil), // 14: google.protobuf.ServiceDescriptorProto - (*MethodDescriptorProto)(nil), // 15: google.protobuf.MethodDescriptorProto - (*FileOptions)(nil), // 16: google.protobuf.FileOptions - (*MessageOptions)(nil), // 17: google.protobuf.MessageOptions - (*FieldOptions)(nil), // 18: google.protobuf.FieldOptions - (*OneofOptions)(nil), // 19: google.protobuf.OneofOptions - (*EnumOptions)(nil), // 20: google.protobuf.EnumOptions - (*EnumValueOptions)(nil), // 21: google.protobuf.EnumValueOptions - (*ServiceOptions)(nil), // 22: google.protobuf.ServiceOptions - (*MethodOptions)(nil), // 23: google.protobuf.MethodOptions - (*UninterpretedOption)(nil), // 24: google.protobuf.UninterpretedOption - (*SourceCodeInfo)(nil), // 25: google.protobuf.SourceCodeInfo - (*GeneratedCodeInfo)(nil), // 26: google.protobuf.GeneratedCodeInfo - (*DescriptorProto_ExtensionRange)(nil), // 27: google.protobuf.DescriptorProto.ExtensionRange - (*DescriptorProto_ReservedRange)(nil), // 28: google.protobuf.DescriptorProto.ReservedRange - (*EnumDescriptorProto_EnumReservedRange)(nil), // 29: google.protobuf.EnumDescriptorProto.EnumReservedRange - (*UninterpretedOption_NamePart)(nil), // 30: google.protobuf.UninterpretedOption.NamePart - (*SourceCodeInfo_Location)(nil), // 31: google.protobuf.SourceCodeInfo.Location - (*GeneratedCodeInfo_Annotation)(nil), // 32: google.protobuf.GeneratedCodeInfo.Annotation + (FieldOptions_OptionRetention)(0), // 5: google.protobuf.FieldOptions.OptionRetention + (FieldOptions_OptionTargetType)(0), // 6: google.protobuf.FieldOptions.OptionTargetType + (MethodOptions_IdempotencyLevel)(0), // 7: google.protobuf.MethodOptions.IdempotencyLevel + (GeneratedCodeInfo_Annotation_Semantic)(0), // 8: google.protobuf.GeneratedCodeInfo.Annotation.Semantic + (*FileDescriptorSet)(nil), // 9: google.protobuf.FileDescriptorSet + (*FileDescriptorProto)(nil), // 10: google.protobuf.FileDescriptorProto + (*DescriptorProto)(nil), // 11: google.protobuf.DescriptorProto + (*ExtensionRangeOptions)(nil), // 12: google.protobuf.ExtensionRangeOptions + (*FieldDescriptorProto)(nil), // 13: google.protobuf.FieldDescriptorProto + (*OneofDescriptorProto)(nil), // 14: google.protobuf.OneofDescriptorProto + (*EnumDescriptorProto)(nil), // 15: google.protobuf.EnumDescriptorProto + (*EnumValueDescriptorProto)(nil), // 16: google.protobuf.EnumValueDescriptorProto + (*ServiceDescriptorProto)(nil), // 17: google.protobuf.ServiceDescriptorProto + (*MethodDescriptorProto)(nil), // 18: google.protobuf.MethodDescriptorProto + (*FileOptions)(nil), // 19: google.protobuf.FileOptions + (*MessageOptions)(nil), // 20: google.protobuf.MessageOptions + (*FieldOptions)(nil), // 21: google.protobuf.FieldOptions + (*OneofOptions)(nil), // 22: google.protobuf.OneofOptions + (*EnumOptions)(nil), // 23: google.protobuf.EnumOptions + (*EnumValueOptions)(nil), // 24: google.protobuf.EnumValueOptions + (*ServiceOptions)(nil), // 25: google.protobuf.ServiceOptions + (*MethodOptions)(nil), // 26: google.protobuf.MethodOptions + (*UninterpretedOption)(nil), // 27: google.protobuf.UninterpretedOption + (*SourceCodeInfo)(nil), // 28: google.protobuf.SourceCodeInfo + (*GeneratedCodeInfo)(nil), // 29: google.protobuf.GeneratedCodeInfo + (*DescriptorProto_ExtensionRange)(nil), // 30: google.protobuf.DescriptorProto.ExtensionRange + (*DescriptorProto_ReservedRange)(nil), // 31: google.protobuf.DescriptorProto.ReservedRange + (*EnumDescriptorProto_EnumReservedRange)(nil), // 32: google.protobuf.EnumDescriptorProto.EnumReservedRange + (*UninterpretedOption_NamePart)(nil), // 33: google.protobuf.UninterpretedOption.NamePart + (*SourceCodeInfo_Location)(nil), // 34: google.protobuf.SourceCodeInfo.Location + (*GeneratedCodeInfo_Annotation)(nil), // 35: google.protobuf.GeneratedCodeInfo.Annotation } var file_google_protobuf_descriptor_proto_depIdxs = []int32{ - 7, // 0: google.protobuf.FileDescriptorSet.file:type_name -> google.protobuf.FileDescriptorProto - 8, // 1: google.protobuf.FileDescriptorProto.message_type:type_name -> google.protobuf.DescriptorProto - 12, // 2: google.protobuf.FileDescriptorProto.enum_type:type_name -> google.protobuf.EnumDescriptorProto - 14, // 3: google.protobuf.FileDescriptorProto.service:type_name -> google.protobuf.ServiceDescriptorProto - 10, // 4: google.protobuf.FileDescriptorProto.extension:type_name -> google.protobuf.FieldDescriptorProto - 16, // 5: google.protobuf.FileDescriptorProto.options:type_name -> google.protobuf.FileOptions - 25, // 6: google.protobuf.FileDescriptorProto.source_code_info:type_name -> google.protobuf.SourceCodeInfo - 10, // 7: google.protobuf.DescriptorProto.field:type_name -> google.protobuf.FieldDescriptorProto - 10, // 8: google.protobuf.DescriptorProto.extension:type_name -> google.protobuf.FieldDescriptorProto - 8, // 9: google.protobuf.DescriptorProto.nested_type:type_name -> google.protobuf.DescriptorProto - 12, // 10: google.protobuf.DescriptorProto.enum_type:type_name -> google.protobuf.EnumDescriptorProto - 27, // 11: google.protobuf.DescriptorProto.extension_range:type_name -> google.protobuf.DescriptorProto.ExtensionRange - 11, // 12: google.protobuf.DescriptorProto.oneof_decl:type_name -> google.protobuf.OneofDescriptorProto - 17, // 13: google.protobuf.DescriptorProto.options:type_name -> google.protobuf.MessageOptions - 28, // 14: google.protobuf.DescriptorProto.reserved_range:type_name -> google.protobuf.DescriptorProto.ReservedRange - 24, // 15: google.protobuf.ExtensionRangeOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption + 10, // 0: google.protobuf.FileDescriptorSet.file:type_name -> google.protobuf.FileDescriptorProto + 11, // 1: google.protobuf.FileDescriptorProto.message_type:type_name -> google.protobuf.DescriptorProto + 15, // 2: google.protobuf.FileDescriptorProto.enum_type:type_name -> google.protobuf.EnumDescriptorProto + 17, // 3: google.protobuf.FileDescriptorProto.service:type_name -> google.protobuf.ServiceDescriptorProto + 13, // 4: google.protobuf.FileDescriptorProto.extension:type_name -> google.protobuf.FieldDescriptorProto + 19, // 5: google.protobuf.FileDescriptorProto.options:type_name -> google.protobuf.FileOptions + 28, // 6: google.protobuf.FileDescriptorProto.source_code_info:type_name -> google.protobuf.SourceCodeInfo + 13, // 7: google.protobuf.DescriptorProto.field:type_name -> google.protobuf.FieldDescriptorProto + 13, // 8: google.protobuf.DescriptorProto.extension:type_name -> google.protobuf.FieldDescriptorProto + 11, // 9: google.protobuf.DescriptorProto.nested_type:type_name -> google.protobuf.DescriptorProto + 15, // 10: google.protobuf.DescriptorProto.enum_type:type_name -> google.protobuf.EnumDescriptorProto + 30, // 11: google.protobuf.DescriptorProto.extension_range:type_name -> google.protobuf.DescriptorProto.ExtensionRange + 14, // 12: google.protobuf.DescriptorProto.oneof_decl:type_name -> google.protobuf.OneofDescriptorProto + 20, // 13: google.protobuf.DescriptorProto.options:type_name -> google.protobuf.MessageOptions + 31, // 14: google.protobuf.DescriptorProto.reserved_range:type_name -> google.protobuf.DescriptorProto.ReservedRange + 27, // 15: google.protobuf.ExtensionRangeOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption 1, // 16: google.protobuf.FieldDescriptorProto.label:type_name -> google.protobuf.FieldDescriptorProto.Label 0, // 17: google.protobuf.FieldDescriptorProto.type:type_name -> google.protobuf.FieldDescriptorProto.Type - 18, // 18: google.protobuf.FieldDescriptorProto.options:type_name -> google.protobuf.FieldOptions - 19, // 19: google.protobuf.OneofDescriptorProto.options:type_name -> google.protobuf.OneofOptions - 13, // 20: google.protobuf.EnumDescriptorProto.value:type_name -> google.protobuf.EnumValueDescriptorProto - 20, // 21: google.protobuf.EnumDescriptorProto.options:type_name -> google.protobuf.EnumOptions - 29, // 22: google.protobuf.EnumDescriptorProto.reserved_range:type_name -> google.protobuf.EnumDescriptorProto.EnumReservedRange - 21, // 23: google.protobuf.EnumValueDescriptorProto.options:type_name -> google.protobuf.EnumValueOptions - 15, // 24: google.protobuf.ServiceDescriptorProto.method:type_name -> google.protobuf.MethodDescriptorProto - 22, // 25: google.protobuf.ServiceDescriptorProto.options:type_name -> google.protobuf.ServiceOptions - 23, // 26: google.protobuf.MethodDescriptorProto.options:type_name -> google.protobuf.MethodOptions + 21, // 18: google.protobuf.FieldDescriptorProto.options:type_name -> google.protobuf.FieldOptions + 22, // 19: google.protobuf.OneofDescriptorProto.options:type_name -> google.protobuf.OneofOptions + 16, // 20: google.protobuf.EnumDescriptorProto.value:type_name -> google.protobuf.EnumValueDescriptorProto + 23, // 21: google.protobuf.EnumDescriptorProto.options:type_name -> google.protobuf.EnumOptions + 32, // 22: google.protobuf.EnumDescriptorProto.reserved_range:type_name -> google.protobuf.EnumDescriptorProto.EnumReservedRange + 24, // 23: google.protobuf.EnumValueDescriptorProto.options:type_name -> google.protobuf.EnumValueOptions + 18, // 24: google.protobuf.ServiceDescriptorProto.method:type_name -> google.protobuf.MethodDescriptorProto + 25, // 25: google.protobuf.ServiceDescriptorProto.options:type_name -> google.protobuf.ServiceOptions + 26, // 26: google.protobuf.MethodDescriptorProto.options:type_name -> google.protobuf.MethodOptions 2, // 27: google.protobuf.FileOptions.optimize_for:type_name -> google.protobuf.FileOptions.OptimizeMode - 24, // 28: google.protobuf.FileOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption - 24, // 29: google.protobuf.MessageOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption + 27, // 28: google.protobuf.FileOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption + 27, // 29: google.protobuf.MessageOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption 3, // 30: google.protobuf.FieldOptions.ctype:type_name -> google.protobuf.FieldOptions.CType 4, // 31: google.protobuf.FieldOptions.jstype:type_name -> google.protobuf.FieldOptions.JSType - 24, // 32: google.protobuf.FieldOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption - 24, // 33: google.protobuf.OneofOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption - 24, // 34: google.protobuf.EnumOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption - 24, // 35: google.protobuf.EnumValueOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption - 24, // 36: google.protobuf.ServiceOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption - 5, // 37: google.protobuf.MethodOptions.idempotency_level:type_name -> google.protobuf.MethodOptions.IdempotencyLevel - 24, // 38: google.protobuf.MethodOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption - 30, // 39: google.protobuf.UninterpretedOption.name:type_name -> google.protobuf.UninterpretedOption.NamePart - 31, // 40: google.protobuf.SourceCodeInfo.location:type_name -> google.protobuf.SourceCodeInfo.Location - 32, // 41: google.protobuf.GeneratedCodeInfo.annotation:type_name -> google.protobuf.GeneratedCodeInfo.Annotation - 9, // 42: google.protobuf.DescriptorProto.ExtensionRange.options:type_name -> google.protobuf.ExtensionRangeOptions - 43, // [43:43] is the sub-list for method output_type - 43, // [43:43] is the sub-list for method input_type - 43, // [43:43] is the sub-list for extension type_name - 43, // [43:43] is the sub-list for extension extendee - 0, // [0:43] is the sub-list for field type_name + 5, // 32: google.protobuf.FieldOptions.retention:type_name -> google.protobuf.FieldOptions.OptionRetention + 6, // 33: google.protobuf.FieldOptions.target:type_name -> google.protobuf.FieldOptions.OptionTargetType + 27, // 34: google.protobuf.FieldOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption + 27, // 35: google.protobuf.OneofOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption + 27, // 36: google.protobuf.EnumOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption + 27, // 37: google.protobuf.EnumValueOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption + 27, // 38: google.protobuf.ServiceOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption + 7, // 39: google.protobuf.MethodOptions.idempotency_level:type_name -> google.protobuf.MethodOptions.IdempotencyLevel + 27, // 40: google.protobuf.MethodOptions.uninterpreted_option:type_name -> google.protobuf.UninterpretedOption + 33, // 41: google.protobuf.UninterpretedOption.name:type_name -> google.protobuf.UninterpretedOption.NamePart + 34, // 42: google.protobuf.SourceCodeInfo.location:type_name -> google.protobuf.SourceCodeInfo.Location + 35, // 43: google.protobuf.GeneratedCodeInfo.annotation:type_name -> google.protobuf.GeneratedCodeInfo.Annotation + 12, // 44: google.protobuf.DescriptorProto.ExtensionRange.options:type_name -> google.protobuf.ExtensionRangeOptions + 8, // 45: google.protobuf.GeneratedCodeInfo.Annotation.semantic:type_name -> google.protobuf.GeneratedCodeInfo.Annotation.Semantic + 46, // [46:46] is the sub-list for method output_type + 46, // [46:46] is the sub-list for method input_type + 46, // [46:46] is the sub-list for extension type_name + 46, // [46:46] is the sub-list for extension extendee + 0, // [0:46] is the sub-list for field type_name } func init() { file_google_protobuf_descriptor_proto_init() } @@ -3940,7 +4333,7 @@ func file_google_protobuf_descriptor_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_google_protobuf_descriptor_proto_rawDesc, - NumEnums: 6, + NumEnums: 9, NumMessages: 27, NumExtensions: 0, NumServices: 0, diff --git a/vendor/google.golang.org/protobuf/types/known/timestamppb/timestamp.pb.go b/vendor/google.golang.org/protobuf/types/known/timestamppb/timestamp.pb.go index c9ae921..61f69fc 100644 --- a/vendor/google.golang.org/protobuf/types/known/timestamppb/timestamp.pb.go +++ b/vendor/google.golang.org/protobuf/types/known/timestamppb/timestamp.pb.go @@ -36,8 +36,7 @@ // The Timestamp message represents a timestamp, // an instant in time since the Unix epoch (January 1st, 1970). // -// -// Conversion to a Go Time +// # Conversion to a Go Time // // The AsTime method can be used to convert a Timestamp message to a // standard Go time.Time value in UTC: @@ -59,8 +58,7 @@ // ... // handle error // } // -// -// Conversion from a Go Time +// # Conversion from a Go Time // // The timestamppb.New function can be used to construct a Timestamp message // from a standard Go time.Time value: @@ -72,7 +70,6 @@ // // ts := timestamppb.Now() // ... // make use of ts as a *timestamppb.Timestamp -// package timestamppb import ( @@ -101,52 +98,50 @@ import ( // // Example 1: Compute Timestamp from POSIX `time()`. // -// Timestamp timestamp; -// timestamp.set_seconds(time(NULL)); -// timestamp.set_nanos(0); +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); // // Example 2: Compute Timestamp from POSIX `gettimeofday()`. // -// struct timeval tv; -// gettimeofday(&tv, NULL); +// struct timeval tv; +// gettimeofday(&tv, NULL); // -// Timestamp timestamp; -// timestamp.set_seconds(tv.tv_sec); -// timestamp.set_nanos(tv.tv_usec * 1000); +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); // // Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. // -// FILETIME ft; -// GetSystemTimeAsFileTime(&ft); -// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; // -// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z -// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. -// Timestamp timestamp; -// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); -// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); // // Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. // -// long millis = System.currentTimeMillis(); -// -// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) -// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// long millis = System.currentTimeMillis(); // +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); // // Example 5: Compute Timestamp from Java `Instant.now()`. // -// Instant now = Instant.now(); -// -// Timestamp timestamp = -// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) -// .setNanos(now.getNano()).build(); +// Instant now = Instant.now(); // +// Timestamp timestamp = +// Timestamp.newBuilder().setSeconds(now.getEpochSecond()) +// .setNanos(now.getNano()).build(); // // Example 6: Compute Timestamp from current time in Python. // -// timestamp = Timestamp() -// timestamp.GetCurrentTime() +// timestamp = Timestamp() +// timestamp.GetCurrentTime() // // # JSON Mapping // @@ -174,8 +169,6 @@ import ( // the Joda Time's [`ISODateTimeFormat.dateTime()`]( // http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D // ) to obtain a formatter capable of generating timestamps in this format. -// -// type Timestamp struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache diff --git a/vendor/modules.txt b/vendor/modules.txt index 13ebebf..3764869 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1,13 +1,13 @@ -# github.com/DataDog/zstd v1.5.2 +# github.com/DataDog/zstd v1.5.5 ## explicit; go 1.14 github.com/DataDog/zstd # github.com/beorn7/perks v1.0.1 ## explicit; go 1.11 github.com/beorn7/perks/quantile -# github.com/cespare/xxhash/v2 v2.1.2 +# github.com/cespare/xxhash/v2 v2.2.0 ## explicit; go 1.11 github.com/cespare/xxhash/v2 -# github.com/cockroachdb/errors v1.9.0 +# github.com/cockroachdb/errors v1.9.1 ## explicit; go 1.13 github.com/cockroachdb/errors github.com/cockroachdb/errors/assert @@ -27,11 +27,11 @@ github.com/cockroachdb/errors/secondary github.com/cockroachdb/errors/stdstrings github.com/cockroachdb/errors/telemetrykeys github.com/cockroachdb/errors/withstack -# github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f +# github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b ## explicit; go 1.16 github.com/cockroachdb/logtags -# github.com/cockroachdb/pebble v0.0.0-20221103174942-7486bbcf2861 -## explicit; go 1.17 +# github.com/cockroachdb/pebble v0.0.0-20230517202617-f6eaf9a696e6 +## explicit; go 1.19 github.com/cockroachdb/pebble github.com/cockroachdb/pebble/internal/arenaskl github.com/cockroachdb/pebble/internal/base @@ -52,11 +52,16 @@ github.com/cockroachdb/pebble/internal/rangekey github.com/cockroachdb/pebble/internal/rate github.com/cockroachdb/pebble/internal/rawalloc github.com/cockroachdb/pebble/internal/testkeys +github.com/cockroachdb/pebble/objstorage +github.com/cockroachdb/pebble/objstorage/objstorageprovider +github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing +github.com/cockroachdb/pebble/objstorage/objstorageprovider/sharedobjcat +github.com/cockroachdb/pebble/objstorage/shared github.com/cockroachdb/pebble/record github.com/cockroachdb/pebble/sstable github.com/cockroachdb/pebble/vfs github.com/cockroachdb/pebble/vfs/atomicfs -# github.com/cockroachdb/redact v1.1.3 +# github.com/cockroachdb/redact v1.1.4 ## explicit; go 1.14 github.com/cockroachdb/redact github.com/cockroachdb/redact/builder @@ -68,18 +73,15 @@ github.com/cockroachdb/redact/internal/markers github.com/cockroachdb/redact/internal/redact github.com/cockroachdb/redact/internal/rfmt github.com/cockroachdb/redact/internal/rfmt/fmtsort -# github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 -## explicit; go 1.12 -# github.com/codahale/hdrhistogram v0.9.0 -## explicit # github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f ## explicit github.com/dgryski/go-rendezvous -# github.com/getsentry/sentry-go v0.14.0 -## explicit; go 1.19 +# github.com/getsentry/sentry-go v0.21.0 +## explicit; go 1.18 github.com/getsentry/sentry-go -github.com/getsentry/sentry-go/internal/crypto/randutil github.com/getsentry/sentry-go/internal/debug +github.com/getsentry/sentry-go/internal/otel/baggage +github.com/getsentry/sentry-go/internal/otel/baggage/internal/baggage github.com/getsentry/sentry-go/internal/ratelimit # github.com/go-redis/redis/v8 v8.11.5 ## explicit; go 1.17 @@ -98,18 +100,17 @@ github.com/gogo/protobuf/proto github.com/gogo/protobuf/protoc-gen-gogo/descriptor github.com/gogo/protobuf/sortkeys github.com/gogo/protobuf/types -# github.com/golang/protobuf v1.5.2 +# github.com/golang/protobuf v1.5.3 ## explicit; go 1.9 github.com/golang/protobuf/proto -github.com/golang/protobuf/ptypes/timestamp # github.com/golang/snappy v0.0.4 ## explicit github.com/golang/snappy # github.com/hjson/hjson-go v3.3.0+incompatible ## explicit github.com/hjson/hjson-go -# github.com/klauspost/compress v1.15.12 -## explicit; go 1.17 +# github.com/klauspost/compress v1.16.5 +## explicit; go 1.18 github.com/klauspost/compress github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -129,39 +130,39 @@ github.com/matttproud/golang_protobuf_extensions/pbutil # github.com/pkg/errors v0.9.1 ## explicit github.com/pkg/errors -# github.com/prometheus/client_golang v1.13.1 +# github.com/prometheus/client_golang v1.15.1 ## explicit; go 1.17 github.com/prometheus/client_golang/prometheus github.com/prometheus/client_golang/prometheus/internal -# github.com/prometheus/client_model v0.3.0 -## explicit; go 1.9 +# github.com/prometheus/client_model v0.4.0 +## explicit; go 1.18 github.com/prometheus/client_model/go -# github.com/prometheus/common v0.37.0 -## explicit; go 1.16 +# github.com/prometheus/common v0.43.0 +## explicit; go 1.18 github.com/prometheus/common/expfmt github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg github.com/prometheus/common/model -# github.com/prometheus/procfs v0.8.0 -## explicit; go 1.17 +# github.com/prometheus/procfs v0.9.0 +## explicit; go 1.18 github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util -# github.com/rogpeppe/go-internal v1.9.0 -## explicit; go 1.17 +# github.com/rogpeppe/go-internal v1.10.0 +## explicit; go 1.19 github.com/rogpeppe/go-internal/fmtsort -# github.com/sirupsen/logrus v1.9.0 +# github.com/sirupsen/logrus v1.9.2 ## explicit; go 1.13 github.com/sirupsen/logrus -# golang.org/x/exp v0.0.0-20221031165847-c99f073a8326 -## explicit; go 1.18 +# golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc +## explicit; go 1.20 golang.org/x/exp/rand -# golang.org/x/sys v0.1.0 +# golang.org/x/sys v0.8.0 ## explicit; go 1.17 golang.org/x/sys/execabs golang.org/x/sys/internal/unsafeheader golang.org/x/sys/unix golang.org/x/sys/windows -# golang.org/x/text v0.4.0 +# golang.org/x/text v0.9.0 ## explicit; go 1.17 golang.org/x/text/cases golang.org/x/text/internal @@ -171,7 +172,7 @@ golang.org/x/text/internal/tag golang.org/x/text/language golang.org/x/text/transform golang.org/x/text/unicode/norm -# google.golang.org/protobuf v1.28.1 +# google.golang.org/protobuf v1.30.0 ## explicit; go 1.11 google.golang.org/protobuf/encoding/prototext google.golang.org/protobuf/encoding/protowire