[WIP] lint a bit, update deps

vendor/github.com/cockroachdb/pebble/internal/base/filenames.go

@@ -20,6 +20,41 @@ type FileNum uint64
 // String returns a string representation of the file number.
 func (fn FileNum) String() string { return fmt.Sprintf("%06d", fn) }
 
+// SafeFormat implements redact.SafeFormatter.
+func (fn FileNum) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Printf("%06d", redact.SafeUint(fn))
+}
+
+// DiskFileNum converts a FileNum to a DiskFileNum. DiskFileNum should only be
+// called if the caller can ensure that the FileNum belongs to a physical file
+// on disk. These could be manifests, log files, physical sstables on disk, the
+// options file, but not virtual sstables.
+func (fn FileNum) DiskFileNum() DiskFileNum {
+	return DiskFileNum{fn}
+}
+
+// A DiskFileNum is just a FileNum belonging to a file which exists on disk.
+// Note that a FileNum is an internal DB identifier and it could belong to files
+// which don't exist on disk. An example would be virtual sstable FileNums.
+// Converting a DiskFileNum to a FileNum is always valid, whereas converting a
+// FileNum to DiskFileNum may not be valid and care should be taken to prove
+// that the FileNum actually exists on disk.
+type DiskFileNum struct {
+	fn FileNum
+}
+
+func (dfn DiskFileNum) String() string { return dfn.fn.String() }
+
+// SafeFormat implements redact.SafeFormatter.
+func (dfn DiskFileNum) SafeFormat(w redact.SafePrinter, verb rune) {
+	dfn.fn.SafeFormat(w, verb)
+}
+
+// FileNum converts a DiskFileNum to a FileNum. This conversion is always valid.
+func (dfn DiskFileNum) FileNum() FileNum {
+	return dfn.fn
+}
+
 // FileType enumerates the types of files found in a DB.
 type FileType int
 
@@ -36,92 +71,92 @@ const (
 )
 
 // MakeFilename builds a filename from components.
-func MakeFilename(fileType FileType, fileNum FileNum) string {
+func MakeFilename(fileType FileType, dfn DiskFileNum) string {
 	switch fileType {
 	case FileTypeLog:
-		return fmt.Sprintf("%s.log", fileNum)
+		return fmt.Sprintf("%s.log", dfn)
 	case FileTypeLock:
 		return "LOCK"
 	case FileTypeTable:
-		return fmt.Sprintf("%s.sst", fileNum)
+		return fmt.Sprintf("%s.sst", dfn)
 	case FileTypeManifest:
-		return fmt.Sprintf("MANIFEST-%s", fileNum)
+		return fmt.Sprintf("MANIFEST-%s", dfn)
 	case FileTypeCurrent:
 		return "CURRENT"
 	case FileTypeOptions:
-		return fmt.Sprintf("OPTIONS-%s", fileNum)
+		return fmt.Sprintf("OPTIONS-%s", dfn)
 	case FileTypeOldTemp:
-		return fmt.Sprintf("CURRENT.%s.dbtmp", fileNum)
+		return fmt.Sprintf("CURRENT.%s.dbtmp", dfn)
 	case FileTypeTemp:
-		return fmt.Sprintf("temporary.%s.dbtmp", fileNum)
+		return fmt.Sprintf("temporary.%s.dbtmp", dfn)
 	}
 	panic("unreachable")
 }
 
 // MakeFilepath builds a filepath from components.
-func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, fileNum FileNum) string {
-	return fs.PathJoin(dirname, MakeFilename(fileType, fileNum))
+func MakeFilepath(fs vfs.FS, dirname string, fileType FileType, dfn DiskFileNum) string {
+	return fs.PathJoin(dirname, MakeFilename(fileType, dfn))
 }
 
 // ParseFilename parses the components from a filename.
-func ParseFilename(fs vfs.FS, filename string) (fileType FileType, fileNum FileNum, ok bool) {
+func ParseFilename(fs vfs.FS, filename string) (fileType FileType, dfn DiskFileNum, ok bool) {
 	filename = fs.PathBase(filename)
 	switch {
 	case filename == "CURRENT":
-		return FileTypeCurrent, 0, true
+		return FileTypeCurrent, DiskFileNum{0}, true
 	case filename == "LOCK":
-		return FileTypeLock, 0, true
+		return FileTypeLock, DiskFileNum{0}, true
 	case strings.HasPrefix(filename, "MANIFEST-"):
-		fileNum, ok = parseFileNum(filename[len("MANIFEST-"):])
+		dfn, ok = parseDiskFileNum(filename[len("MANIFEST-"):])
 		if !ok {
 			break
 		}
-		return FileTypeManifest, fileNum, true
+		return FileTypeManifest, dfn, true
 	case strings.HasPrefix(filename, "OPTIONS-"):
-		fileNum, ok = parseFileNum(filename[len("OPTIONS-"):])
+		dfn, ok = parseDiskFileNum(filename[len("OPTIONS-"):])
 		if !ok {
 			break
 		}
-		return FileTypeOptions, fileNum, ok
+		return FileTypeOptions, dfn, ok
 	case strings.HasPrefix(filename, "CURRENT.") && strings.HasSuffix(filename, ".dbtmp"):
 		s := strings.TrimSuffix(filename[len("CURRENT."):], ".dbtmp")
-		fileNum, ok = parseFileNum(s)
+		dfn, ok = parseDiskFileNum(s)
 		if !ok {
 			break
 		}
-		return FileTypeOldTemp, fileNum, ok
+		return FileTypeOldTemp, dfn, ok
 	case strings.HasPrefix(filename, "temporary.") && strings.HasSuffix(filename, ".dbtmp"):
 		s := strings.TrimSuffix(filename[len("temporary."):], ".dbtmp")
-		fileNum, ok = parseFileNum(s)
+		dfn, ok = parseDiskFileNum(s)
 		if !ok {
 			break
 		}
-		return FileTypeTemp, fileNum, ok
+		return FileTypeTemp, dfn, ok
 	default:
 		i := strings.IndexByte(filename, '.')
 		if i < 0 {
 			break
 		}
-		fileNum, ok = parseFileNum(filename[:i])
+		dfn, ok = parseDiskFileNum(filename[:i])
 		if !ok {
 			break
 		}
 		switch filename[i+1:] {
 		case "sst":
-			return FileTypeTable, fileNum, true
+			return FileTypeTable, dfn, true
 		case "log":
-			return FileTypeLog, fileNum, true
+			return FileTypeLog, dfn, true
 		}
 	}
-	return 0, fileNum, false
+	return 0, dfn, false
 }
 
-func parseFileNum(s string) (fileNum FileNum, ok bool) {
+func parseDiskFileNum(s string) (dfn DiskFileNum, ok bool) {
 	u, err := strconv.ParseUint(s, 10, 64)
 	if err != nil {
-		return fileNum, false
+		return dfn, false
 	}
-	return FileNum(u), true
+	return DiskFileNum{FileNum(u)}, true
 }
 
 // A Fataler fatals a process with a message when called.

vendor/github.com/cockroachdb/pebble/internal/base/internal.go

@@ -9,6 +9,17 @@ import (
 	"fmt"
 	"strconv"
 	"strings"
+
+	"github.com/cockroachdb/redact"
+)
+
+const (
+	// SeqNumZero is the zero sequence number, set by compactions if they can
+	// guarantee there are no keys underneath an internal key.
+	SeqNumZero = uint64(0)
+	// SeqNumStart is the first sequence number assigned to a key. Sequence
+	// numbers 1-9 are reserved for potential future use.
+	SeqNumStart = uint64(10)
 )
 
 // InternalKeyKind enumerates the kind of key: a deletion tombstone, a set
@@ -24,6 +35,14 @@ const (
 	//InternalKeyKindColumnFamilyDeletion     InternalKeyKind = 4
 	//InternalKeyKindColumnFamilyValue        InternalKeyKind = 5
 	//InternalKeyKindColumnFamilyMerge        InternalKeyKind = 6
+
+	// InternalKeyKindSingleDelete (SINGLEDEL) is a performance optimization
+	// solely for compactions (to reduce write amp and space amp). Readers other
+	// than compactions should treat SINGLEDEL as equivalent to a DEL.
+	// Historically, it was simpler for readers other than compactions to treat
+	// SINGLEDEL as equivalent to DEL, but as of the introduction of
+	// InternalKeyKindSSTableInternalObsoleteBit, this is also necessary for
+	// correctness.
 	InternalKeyKindSingleDelete InternalKeyKind = 7
 	//InternalKeyKindColumnFamilySingleDelete InternalKeyKind = 8
 	//InternalKeyKindBeginPrepareXID          InternalKeyKind = 9
@@ -60,14 +79,21 @@ const (
 	InternalKeyKindRangeKeyUnset InternalKeyKind = 20
 	InternalKeyKindRangeKeySet   InternalKeyKind = 21
 
-	// InternalKeyIngestSST is used to distinguish a batch that corresponds to
+	// InternalKeyKindIngestSST is used to distinguish a batch that corresponds to
 	// the WAL entry for ingested sstables that are added to the flushable
 	// queue. This InternalKeyKind cannot appear, amongst other key kinds in a
 	// batch, or in an sstable.
-	InternalKeyIngestSST InternalKeyKind = 22
+	InternalKeyKindIngestSST InternalKeyKind = 22
 
-	// This maximum value isn't part of the file format. It's unlikely,
-	// but future extensions may increase this value.
+	// InternalKeyKindDeleteSized keys behave identically to
+	// InternalKeyKindDelete keys, except that they hold an associated uint64
+	// value indicating the (len(key)+len(value)) of the shadowed entry the
+	// tombstone is expected to delete. This value is used to inform compaction
+	// heuristics, but is not required to be accurate for correctness.
+	InternalKeyKindDeleteSized InternalKeyKind = 23
+
+	// This maximum value isn't part of the file format. Future extensions may
+	// increase this value.
 	//
 	// When constructing an internal key to pass to DB.Seek{GE,LE},
 	// internalKeyComparer sorts decreasing by kind (after sorting increasing by
@@ -75,14 +101,19 @@ const (
 	// which sorts 'less than or equal to' any other valid internalKeyKind, when
 	// searching for any kind of internal key formed by a certain user key and
 	// seqNum.
-	InternalKeyKindMax InternalKeyKind = 22
+	InternalKeyKindMax InternalKeyKind = 23
+
+	// Internal to the sstable format. Not exposed by any sstable iterator.
+	// Declared here to prevent definition of valid key kinds that set this bit.
+	InternalKeyKindSSTableInternalObsoleteBit  InternalKeyKind = 64
+	InternalKeyKindSSTableInternalObsoleteMask InternalKeyKind = 191
 
 	// InternalKeyZeroSeqnumMaxTrailer is the largest trailer with a
 	// zero sequence number.
-	InternalKeyZeroSeqnumMaxTrailer = uint64(InternalKeyKindInvalid)
+	InternalKeyZeroSeqnumMaxTrailer = uint64(255)
 
 	// A marker for an invalid key.
-	InternalKeyKindInvalid InternalKeyKind = 255
+	InternalKeyKindInvalid InternalKeyKind = InternalKeyKindSSTableInternalObsoleteMask
 
 	// InternalKeySeqNumBatch is a bit that is set on batch sequence numbers
 	// which prevents those entries from being excluded from iteration.
@@ -105,6 +136,9 @@ const (
 	InternalKeyBoundaryRangeKey = (InternalKeySeqNumMax << 8) | uint64(InternalKeyKindRangeKeySet)
 )
 
+// Assert InternalKeyKindSSTableInternalObsoleteBit > InternalKeyKindMax
+const _ = uint(InternalKeyKindSSTableInternalObsoleteBit - InternalKeyKindMax - 1)
+
 var internalKeyKindNames = []string{
 	InternalKeyKindDelete:         "DEL",
 	InternalKeyKindSet:            "SET",
@@ -117,7 +151,8 @@ var internalKeyKindNames = []string{
 	InternalKeyKindRangeKeySet:    "RANGEKEYSET",
 	InternalKeyKindRangeKeyUnset:  "RANGEKEYUNSET",
 	InternalKeyKindRangeKeyDelete: "RANGEKEYDEL",
-	InternalKeyIngestSST:          "INGESTSST",
+	InternalKeyKindIngestSST:      "INGESTSST",
+	InternalKeyKindDeleteSized:    "DELSIZED",
 	InternalKeyKindInvalid:        "INVALID",
 }
 
@@ -128,6 +163,11 @@ func (k InternalKeyKind) String() string {
 	return fmt.Sprintf("UNKNOWN:%d", k)
 }
 
+// SafeFormat implements redact.SafeFormatter.
+func (k InternalKeyKind) SafeFormat(w redact.SafePrinter, _ rune) {
+	w.Print(redact.SafeString(k.String()))
+}
+
 // InternalKey is a key used for the in-memory and on-disk partial DBs that
 // make up a pebble DB.
 //
@@ -194,6 +234,7 @@ var kindsMap = map[string]InternalKeyKind{
 	"DEL":           InternalKeyKindDelete,
 	"SINGLEDEL":     InternalKeyKindSingleDelete,
 	"RANGEDEL":      InternalKeyKindRangeDelete,
+	"LOGDATA":       InternalKeyKindLogData,
 	"SET":           InternalKeyKindSet,
 	"MERGE":         InternalKeyKindMerge,
 	"INVALID":       InternalKeyKindInvalid,
@@ -202,6 +243,8 @@ var kindsMap = map[string]InternalKeyKind{
 	"RANGEKEYSET":   InternalKeyKindRangeKeySet,
 	"RANGEKEYUNSET": InternalKeyKindRangeKeyUnset,
 	"RANGEKEYDEL":   InternalKeyKindRangeKeyDelete,
+	"INGESTSST":     InternalKeyKindIngestSST,
+	"DELSIZED":      InternalKeyKindDeleteSized,
 }
 
 // ParseInternalKey parses the string representation of an internal key. The
@@ -338,6 +381,11 @@ func (k InternalKey) SeqNum() uint64 {
 	return k.Trailer >> 8
 }
 
+// SeqNumFromTrailer returns the sequence number component of a trailer.
+func SeqNumFromTrailer(t uint64) uint64 {
+	return t >> 8
+}
+
 // Visible returns true if the key is visible at the specified snapshot
 // sequence number.
 func (k InternalKey) Visible(snapshot, batchSnapshot uint64) bool {
@@ -395,6 +443,13 @@ func (k InternalKey) Clone() InternalKey {
 	}
 }
 
+// CopyFrom converts this InternalKey into a clone of the passed-in InternalKey,
+// reusing any space already used for the current UserKey.
+func (k *InternalKey) CopyFrom(k2 InternalKey) {
+	k.UserKey = append(k.UserKey[:0], k2.UserKey...)
+	k.Trailer = k2.Trailer
+}
+
 // String returns a string representation of the key.
 func (k InternalKey) String() string {
 	return fmt.Sprintf("%s#%d,%d", FormatBytes(k.UserKey), k.SeqNum(), k.Kind())

vendor/github.com/cockroachdb/pebble/internal/base/metrics.go

@@ -53,6 +53,15 @@ func (tm *ThroughputMetric) Rate() int64 {
 		float64(time.Second))
 }
 
+// Utilization returns a percent [0, 1.0] indicating the percent of time
+// work was performed.
+func (tm *ThroughputMetric) Utilization() float64 {
+	if tm.WorkDuration == 0 {
+		return 0
+	}
+	return float64(tm.WorkDuration) / float64(tm.WorkDuration+tm.IdleDuration)
+}
+
 // GaugeSampleMetric is used to measure a gauge value (e.g. queue length) by
 // accumulating samples of that gauge.
 type GaugeSampleMetric struct {