Update deps

vendor/github.com/cockroachdb/pebble/sstable/block_property.go

@@ -9,6 +9,7 @@ import (
 	"fmt"
 	"math"
 	"sync"
+	"unsafe"
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/rangekey"
@@ -643,15 +644,21 @@ func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit(
 			// considered intersecting.
 			continue
 		}
-		byteProps := []byte(props)
-		if len(byteProps) < 1 {
+		if len(props) < 1 {
 			return false, base.CorruptionErrorf(
 				"block properties for %s is corrupted", f.filters[i].Name())
 		}
-		shortID := shortID(byteProps[0])
-		intersects, err := f.filters[i].Intersects(byteProps[1:])
-		if err != nil || !intersects {
-			return false, err
+		shortID := shortID(props[0])
+		{
+			// Use an unsafe conversion to avoid allocating. Intersects() is not
+			// supposed to modify the given slice.
+			// Note that unsafe.StringData only works if the string is not empty
+			// (which we already checked).
+			byteProps := unsafe.Slice(unsafe.StringData(props), len(props))
+			intersects, err := f.filters[i].Intersects(byteProps[1:])
+			if err != nil || !intersects {
+				return false, err
+			}
 		}
 		// Intersects the sstable, so need to use this filter when
 		// deciding whether to read blocks.
@@ -684,12 +691,11 @@ func (f *BlockPropertiesFilterer) intersectsUserPropsAndFinishInit(
 		// be unused within this file.
 		return true, nil
 	}
-	byteProps := []byte(props)
-	if len(byteProps) < 1 {
+	if len(props) < 1 {
 		return false, base.CorruptionErrorf(
 			"block properties for %s is corrupted", f.boundLimitedFilter.Name())
 	}
-	f.boundLimitedShortID = int(byteProps[0])
+	f.boundLimitedShortID = int(props[0])
 
 	// We don't check for table-level intersection for the bound-limited filter.
 	// The bound-limited filter is treated as vacuously intersecting.

vendor/github.com/cockroachdb/pebble/sstable/properties.go

@@ -25,10 +25,20 @@ var propBoolFalse = []byte{'0'}
 
 var propOffsetTagMap = make(map[uintptr]string)
 
-func init() {
-	t := reflect.TypeOf(Properties{})
+func generateTagMaps(t reflect.Type, indexPrefix []int) {
 	for i := 0; i < t.NumField(); i++ {
 		f := t.Field(i)
+		if f.Type.Kind() == reflect.Struct {
+			if tag := f.Tag.Get("prop"); i == 0 && tag == "pebble.embbeded_common_properties" {
+				// CommonProperties struct embedded in Properties. Note that since
+				// CommonProperties is placed at the top of properties we can use
+				// the offsets of the fields within CommonProperties to determine
+				// the offsets of those fields within Properties.
+				generateTagMaps(f.Type, []int{i})
+				continue
+			}
+			panic("pebble: unknown struct type in Properties")
+		}
 		if tag := f.Tag.Get("prop"); tag != "" {
 			switch f.Type.Kind() {
 			case reflect.Bool:
@@ -38,16 +48,84 @@ func init() {
 			default:
 				panic(fmt.Sprintf("unsupported property field type: %s %s", f.Name, f.Type))
 			}
+			if len(indexPrefix) > 0 {
+				// Prepend the index prefix so that we can use FieldByIndex on the top-level struct.
+				f.Index = append(indexPrefix[:len(indexPrefix):len(indexPrefix)], f.Index...)
+			}
 			propTagMap[tag] = f
 			propOffsetTagMap[f.Offset] = tag
 		}
 	}
 }
 
+func init() {
+	generateTagMaps(reflect.TypeOf(Properties{}), nil)
+}
+
+// CommonProperties holds properties for either a virtual or a physical sstable. This
+// can be used by code which doesn't care to make the distinction between physical
+// and virtual sstables properties.
+//
+// For virtual sstables, fields are constructed through extrapolation upon virtual
+// reader construction. See MakeVirtualReader for implementation details.
+//
+// NB: The values of these properties can affect correctness. For example,
+// if NumRangeKeySets == 0, but the sstable actually contains range keys, then
+// the iterators will behave incorrectly.
+type CommonProperties struct {
+	// The number of entries in this table.
+	NumEntries uint64 `prop:"rocksdb.num.entries"`
+	// Total raw key size.
+	RawKeySize uint64 `prop:"rocksdb.raw.key.size"`
+	// Total raw value size.
+	RawValueSize uint64 `prop:"rocksdb.raw.value.size"`
+	// Total raw key size of point deletion tombstones. This value is comparable
+	// to RawKeySize.
+	RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"`
+	// Sum of the raw value sizes carried by point deletion tombstones
+	// containing size estimates. See the DeleteSized key kind. This value is
+	// comparable to Raw{Key,Value}Size.
+	RawPointTombstoneValueSize uint64 `prop:"pebble.raw.point-tombstone.value.size"`
+	// The number of point deletion entries ("tombstones") in this table that
+	// carry a size hint indicating the size of the value the tombstone deletes.
+	NumSizedDeletions uint64 `prop:"pebble.num.deletions.sized"`
+	// The number of deletion entries in this table, including both point and
+	// range deletions.
+	NumDeletions uint64 `prop:"rocksdb.deleted.keys"`
+	// The number of range deletions in this table.
+	NumRangeDeletions uint64 `prop:"rocksdb.num.range-deletions"`
+	// The number of RANGEKEYDELs in this table.
+	NumRangeKeyDels uint64 `prop:"pebble.num.range-key-dels"`
+	// The number of RANGEKEYSETs in this table.
+	NumRangeKeySets uint64 `prop:"pebble.num.range-key-sets"`
+	// Total size of value blocks and value index block. Only serialized if > 0.
+	ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"`
+}
+
+// String is only used for testing purposes.
+func (c *CommonProperties) String() string {
+	var buf bytes.Buffer
+	v := reflect.ValueOf(*c)
+	loaded := make(map[uintptr]struct{})
+	writeProperties(loaded, v, &buf)
+	return buf.String()
+}
+
+// NumPointDeletions is the number of point deletions in the sstable. For virtual
+// sstables, this is an estimate.
+func (c *CommonProperties) NumPointDeletions() uint64 {
+	return c.NumDeletions - c.NumRangeDeletions
+}
+
 // Properties holds the sstable property values. The properties are
 // automatically populated during sstable creation and load from the properties
 // meta block when an sstable is opened.
 type Properties struct {
+	// CommonProperties needs to be at the top of the Properties struct so that the
+	// offsets of the fields in CommonProperties match the offsets of the embedded
+	// fields of CommonProperties in Properties.
+	CommonProperties `prop:"pebble.embbeded_common_properties"`
+
 	// The name of the comparer used in this table.
 	ComparerName string `prop:"rocksdb.comparator"`
 	// The compression algorithm used to compress blocks.
@@ -81,22 +159,8 @@ type Properties struct {
 	MergerName string `prop:"rocksdb.merge.operator"`
 	// The number of blocks in this table.
 	NumDataBlocks uint64 `prop:"rocksdb.num.data.blocks"`
-	// The number of deletion entries in this table, including both point and
-	// range deletions.
-	NumDeletions uint64 `prop:"rocksdb.deleted.keys"`
-	// The number of point deletion entries ("tombstones") in this table that
-	// carry a size hint indicating the size of the value the tombstone deletes.
-	NumSizedDeletions uint64 `prop:"pebble.num.deletions.sized"`
-	// The number of entries in this table.
-	NumEntries uint64 `prop:"rocksdb.num.entries"`
 	// The number of merge operands in the table.
 	NumMergeOperands uint64 `prop:"rocksdb.merge.operands"`
-	// The number of range deletions in this table.
-	NumRangeDeletions uint64 `prop:"rocksdb.num.range-deletions"`
-	// The number of RANGEKEYDELs in this table.
-	NumRangeKeyDels uint64 `prop:"pebble.num.range-key-dels"`
-	// The number of RANGEKEYSETs in this table.
-	NumRangeKeySets uint64 `prop:"pebble.num.range-key-sets"`
 	// The number of RANGEKEYUNSETs in this table.
 	NumRangeKeyUnsets uint64 `prop:"pebble.num.range-key-unsets"`
 	// The number of value blocks in this table. Only serialized if > 0.
@@ -111,21 +175,10 @@ type Properties struct {
 	// A comma separated list of names of the property collectors used in this
 	// table.
 	PropertyCollectorNames string `prop:"rocksdb.property.collectors"`
-	// Total raw key size.
-	RawKeySize uint64 `prop:"rocksdb.raw.key.size"`
-	// Total raw key size of point deletion tombstones. This value is comparable
-	// to RawKeySize.
-	RawPointTombstoneKeySize uint64 `prop:"pebble.raw.point-tombstone.key.size"`
-	// Sum of the raw value sizes carried by point deletion tombstones
-	// containing size estimates. See the DeleteSized key kind. This value is
-	// comparable to Raw{Key,Value}Size.
-	RawPointTombstoneValueSize uint64 `prop:"pebble.raw.point-tombstone.value.size"`
 	// Total raw rangekey key size.
 	RawRangeKeyKeySize uint64 `prop:"pebble.raw.range-key.key.size"`
 	// Total raw rangekey value size.
 	RawRangeKeyValueSize uint64 `prop:"pebble.raw.range-key.value.size"`
-	// Total raw value size.
-	RawValueSize uint64 `prop:"rocksdb.raw.value.size"`
 	// The total number of keys in this table that were pinned by open snapshots.
 	SnapshotPinnedKeys uint64 `prop:"pebble.num.snapshot-pinned-keys"`
 	// The cumulative bytes of keys in this table that were pinned by
@@ -138,8 +191,6 @@ type Properties struct {
 	TopLevelIndexSize uint64 `prop:"rocksdb.top-level.index.size"`
 	// User collected properties.
 	UserProperties map[string]string
-	// Total size of value blocks and value index block. Only serialized if > 0.
-	ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"`
 	// If filtering is enabled, was the filter created on the whole key.
 	WholeKeyFiltering bool `prop:"rocksdb.block.based.table.whole.key.filtering"`
 
@@ -160,12 +211,15 @@ func (p *Properties) NumRangeKeys() uint64 {
 	return p.NumRangeKeyDels + p.NumRangeKeySets + p.NumRangeKeyUnsets
 }
 
-func (p *Properties) String() string {
-	var buf bytes.Buffer
-	v := reflect.ValueOf(*p)
+func writeProperties(loaded map[uintptr]struct{}, v reflect.Value, buf *bytes.Buffer) {
 	vt := v.Type()
 	for i := 0; i < v.NumField(); i++ {
 		ft := vt.Field(i)
+		if ft.Type.Kind() == reflect.Struct {
+			// Embedded struct within the properties.
+			writeProperties(loaded, v.Field(i), buf)
+			continue
+		}
 		tag := ft.Tag.Get("prop")
 		if tag == "" {
 			continue
@@ -175,25 +229,33 @@ func (p *Properties) String() string {
 		// TODO(peter): Use f.IsZero() when we can rely on go1.13.
 		if zero := reflect.Zero(f.Type()); zero.Interface() == f.Interface() {
 			// Skip printing of zero values which were not loaded from disk.
-			if _, ok := p.Loaded[ft.Offset]; !ok {
+			if _, ok := loaded[ft.Offset]; !ok {
 				continue
 			}
 		}
 
-		fmt.Fprintf(&buf, "%s: ", tag)
+		fmt.Fprintf(buf, "%s: ", tag)
 		switch ft.Type.Kind() {
 		case reflect.Bool:
-			fmt.Fprintf(&buf, "%t\n", f.Bool())
+			fmt.Fprintf(buf, "%t\n", f.Bool())
 		case reflect.Uint32:
-			fmt.Fprintf(&buf, "%d\n", f.Uint())
+			fmt.Fprintf(buf, "%d\n", f.Uint())
 		case reflect.Uint64:
-			fmt.Fprintf(&buf, "%d\n", f.Uint())
+			fmt.Fprintf(buf, "%d\n", f.Uint())
 		case reflect.String:
-			fmt.Fprintf(&buf, "%s\n", f.String())
+			fmt.Fprintf(buf, "%s\n", f.String())
 		default:
 			panic("not reached")
 		}
 	}
+}
+
+func (p *Properties) String() string {
+	var buf bytes.Buffer
+	v := reflect.ValueOf(*p)
+	writeProperties(p.Loaded, v, &buf)
+
+	// Write the UserProperties.
 	keys := make([]string, 0, len(p.UserProperties))
 	for key := range p.UserProperties {
 		keys = append(keys, key)
@@ -214,6 +276,7 @@ func (p *Properties) load(
 	}
 	p.Loaded = make(map[uintptr]struct{})
 	v := reflect.ValueOf(p).Elem()
+
 	for valid := i.First(); valid; valid = i.Next() {
 		if f, ok := propTagMap[string(i.Key().UserKey)]; ok {
 			p.Loaded[f.Offset] = struct{}{}

vendor/github.com/cockroachdb/pebble/sstable/reader.go

@@ -174,6 +174,28 @@ func init() {
 	private.SSTableRawTombstonesOpt = rawTombstonesOpt{}
 }
 
+// CommonReader abstracts functionality over a Reader or a VirtualReader. This
+// can be used by code which doesn't care to distinguish between a reader and a
+// virtual reader.
+type CommonReader interface {
+	NewRawRangeKeyIter() (keyspan.FragmentIterator, error)
+	NewRawRangeDelIter() (keyspan.FragmentIterator, error)
+	NewIterWithBlockPropertyFiltersAndContextEtc(
+		ctx context.Context, lower, upper []byte,
+		filterer *BlockPropertiesFilterer,
+		hideObsoletePoints, useFilterBlock bool,
+		stats *base.InternalIteratorStats,
+		rp ReaderProvider,
+	) (Iterator, error)
+	NewCompactionIter(
+		bytesIterated *uint64,
+		rp ReaderProvider,
+		bufferPool *BufferPool,
+	) (Iterator, error)
+	EstimateDiskUsage(start, end []byte) (uint64, error)
+	CommonProperties() *CommonProperties
+}
+
 // Reader is a table reader.
 type Reader struct {
 	readable          objstorage.Readable
@@ -337,7 +359,7 @@ func (r *Reader) newCompactionIter(
 		err := i.init(
 			context.Background(),
 			r, v, nil /* lower */, nil /* upper */, nil,
-			false /* useFilter */, false, /* hideObsoletePoints */
+			false /* useFilter */, v != nil && v.isForeign, /* hideObsoletePoints */
 			nil /* stats */, rp, bufferPool,
 		)
 		if err != nil {
@@ -352,7 +374,7 @@ func (r *Reader) newCompactionIter(
 	i := singleLevelIterPool.Get().(*singleLevelIterator)
 	err := i.init(
 		context.Background(), r, v, nil /* lower */, nil, /* upper */
-		nil, false /* useFilter */, false, /* hideObsoletePoints */
+		nil, false /* useFilter */, v != nil && v.isForeign, /* hideObsoletePoints */
 		nil /* stats */, rp, bufferPool,
 	)
 	if err != nil {
@@ -927,6 +949,11 @@ func (r *Reader) ValidateBlockChecksums() error {
 	return nil
 }
 
+// CommonProperties implemented the CommonReader interface.
+func (r *Reader) CommonProperties() *CommonProperties {
+	return &r.Properties.CommonProperties
+}
+
 // EstimateDiskUsage returns the total size of data blocks overlapping the range
 // `[start, end]`. Even if a data block partially overlaps, or we cannot
 // determine overlap due to abbreviated index keys, the full data block size is

vendor/github.com/cockroachdb/pebble/sstable/reader_iter_two_lvl.go

@@ -810,6 +810,11 @@ func (i *twoLevelIterator) NextPrefix(succKey []byte) (*InternalKey, base.LazyVa
 	if key, val := i.singleLevelIterator.NextPrefix(succKey); key != nil {
 		return key, val
 	}
+	// key == nil
+	if i.err != nil {
+		return nil, base.LazyValue{}
+	}
+
 	// Did not find prefix in the existing second-level index block. This is the
 	// slow-path where we seek the iterator.
 	var ikey *InternalKey

vendor/github.com/cockroachdb/pebble/sstable/reader_virtual.go

@@ -22,46 +22,58 @@ import (
 type VirtualReader struct {
 	vState     virtualState
 	reader     *Reader
-	Properties struct {
-		// RawKeySize, RawValueSize are set upon construction of a
-		// VirtualReader. The values of the fields is extrapolated. See
-		// MakeVirtualReader for implementation details.
-		RawKeySize   uint64
-		RawValueSize uint64
-	}
+	Properties CommonProperties
 }
 
 // Lightweight virtual sstable state which can be passed to sstable iterators.
 type virtualState struct {
-	lower   InternalKey
-	upper   InternalKey
-	fileNum base.FileNum
-	Compare Compare
+	lower     InternalKey
+	upper     InternalKey
+	fileNum   base.FileNum
+	Compare   Compare
+	isForeign bool
+}
+
+func ceilDiv(a, b uint64) uint64 {
+	return (a + b - 1) / b
 }
 
 // MakeVirtualReader is used to contruct a reader which can read from virtual
 // sstables.
-func MakeVirtualReader(reader *Reader, meta manifest.VirtualFileMeta) VirtualReader {
+func MakeVirtualReader(
+	reader *Reader, meta manifest.VirtualFileMeta, isForeign bool,
+) VirtualReader {
 	if reader.fileNum != meta.FileBacking.DiskFileNum {
 		panic("pebble: invalid call to MakeVirtualReader")
 	}
 
 	vState := virtualState{
-		lower:   meta.Smallest,
-		upper:   meta.Largest,
-		fileNum: meta.FileNum,
-		Compare: reader.Compare,
+		lower:     meta.Smallest,
+		upper:     meta.Largest,
+		fileNum:   meta.FileNum,
+		Compare:   reader.Compare,
+		isForeign: isForeign,
 	}
 	v := VirtualReader{
 		vState: vState,
 		reader: reader,
 	}
 
-	v.Properties.RawKeySize =
-		(reader.Properties.RawKeySize * meta.Size) / meta.FileBacking.Size
-	v.Properties.RawValueSize =
-		(reader.Properties.RawValueSize * meta.Size) / meta.FileBacking.Size
+	v.Properties.RawKeySize = ceilDiv(reader.Properties.RawKeySize*meta.Size, meta.FileBacking.Size)
+	v.Properties.RawValueSize = ceilDiv(reader.Properties.RawValueSize*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumEntries = ceilDiv(reader.Properties.NumEntries*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumDeletions = ceilDiv(reader.Properties.NumDeletions*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumRangeDeletions = ceilDiv(reader.Properties.NumRangeDeletions*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumRangeKeyDels = ceilDiv(reader.Properties.NumRangeKeyDels*meta.Size, meta.FileBacking.Size)
 
+	// Note that we rely on NumRangeKeySets for correctness. If the sstable may
+	// contain range keys, then NumRangeKeySets must be > 0. ceilDiv works because
+	// meta.Size will not be 0 for virtual sstables.
+	v.Properties.NumRangeKeySets = ceilDiv(reader.Properties.NumRangeKeySets*meta.Size, meta.FileBacking.Size)
+	v.Properties.ValueBlocksSize = ceilDiv(reader.Properties.ValueBlocksSize*meta.Size, meta.FileBacking.Size)
+	v.Properties.NumSizedDeletions = ceilDiv(reader.Properties.NumSizedDeletions*meta.Size, meta.FileBacking.Size)
+	v.Properties.RawPointTombstoneKeySize = ceilDiv(reader.Properties.RawPointTombstoneKeySize*meta.Size, meta.FileBacking.Size)
+	v.Properties.RawPointTombstoneValueSize = ceilDiv(reader.Properties.RawPointTombstoneValueSize*meta.Size, meta.FileBacking.Size)
 	return v
 }
 
@@ -89,6 +101,12 @@ func (v *VirtualReader) NewIterWithBlockPropertyFiltersAndContextEtc(
 	)
 }
 
+// ValidateBlockChecksumsOnBacking will call ValidateBlockChecksumsOnBacking on the underlying reader.
+// Note that block checksum validation is NOT restricted to virtual sstable bounds.
+func (v *VirtualReader) ValidateBlockChecksumsOnBacking() error {
+	return v.reader.ValidateBlockChecksums()
+}
+
 // NewRawRangeDelIter wraps Reader.NewRawRangeDelIter.
 func (v *VirtualReader) NewRawRangeDelIter() (keyspan.FragmentIterator, error) {
 	iter, err := v.reader.NewRawRangeDelIter()
@@ -181,3 +199,8 @@ func (v *VirtualReader) EstimateDiskUsage(start, end []byte) (uint64, error) {
 	_, f, l := v.vState.constrainBounds(start, end, true /* endInclusive */)
 	return v.reader.EstimateDiskUsage(f, l)
 }
+
+// CommonProperties implements the CommonReader interface.
+func (v *VirtualReader) CommonProperties() *CommonProperties {
+	return &v.Properties
+}