// Package hdrhistogram provides an implementation of Gil Tene's HDR Histogram // data structure. The HDR Histogram allows for fast and accurate analysis of // the extreme ranges of data with non-normal distributions, like latency. package hdrhistogram import ( "fmt" "io" "math" "math/bits" "sort" ) // A Bracket is a part of a cumulative distribution. type Bracket struct { Quantile float64 Count, ValueAt int64 } // A Snapshot is an exported view of a Histogram, useful for serializing them. // A Histogram can be constructed from it by passing it to Import. type Snapshot struct { LowestTrackableValue int64 HighestTrackableValue int64 SignificantFigures int64 Counts []int64 } // A Histogram is a lossy data structure used to record the distribution of // non-normally distributed data (like latency) with a high degree of accuracy // and a bounded degree of precision. type Histogram struct { lowestDiscernibleValue int64 highestTrackableValue int64 unitMagnitude int64 significantFigures int64 subBucketHalfCountMagnitude int32 subBucketHalfCount int32 subBucketMask int64 subBucketCount int32 bucketCount int32 countsLen int32 totalCount int64 counts []int64 startTimeMs int64 endTimeMs int64 tag string } func (h *Histogram) Tag() string { return h.tag } func (h *Histogram) SetTag(tag string) { h.tag = tag } func (h *Histogram) EndTimeMs() int64 { return h.endTimeMs } func (h *Histogram) SetEndTimeMs(endTimeMs int64) { h.endTimeMs = endTimeMs } func (h *Histogram) StartTimeMs() int64 { return h.startTimeMs } func (h *Histogram) SetStartTimeMs(startTimeMs int64) { h.startTimeMs = startTimeMs } // Construct a Histogram given the Lowest and Highest values to be tracked and a number of significant decimal digits. // // Providing a lowestDiscernibleValue is useful in situations where the units used for the histogram's values are // much smaller that the minimal accuracy required. // E.g. when tracking time values stated in nanosecond units, where the minimal accuracy required is a microsecond, // the proper value for lowestDiscernibleValue would be 1000. // // Note: the numberOfSignificantValueDigits must be [1,5]. If lower than 1 the numberOfSignificantValueDigits will be // forced to 1, and if higher than 5 the numberOfSignificantValueDigits will be forced to 5. func New(lowestDiscernibleValue, highestTrackableValue int64, numberOfSignificantValueDigits int) *Histogram { if numberOfSignificantValueDigits < 1 { numberOfSignificantValueDigits = 1 } else if numberOfSignificantValueDigits > 5 { numberOfSignificantValueDigits = 5 } if lowestDiscernibleValue < 1 { lowestDiscernibleValue = 1 } // Given a 3 decimal point accuracy, the expectation is obviously for "+/- 1 unit at 1000". It also means that // it's "ok to be +/- 2 units at 2000". The "tricky" thing is that it is NOT ok to be +/- 2 units at 1999. Only // starting at 2000. So internally, we need to maintain single unit resolution to 2x 10^decimalPoints. largestValueWithSingleUnitResolution := 2 * math.Pow10(numberOfSignificantValueDigits) // We need to maintain power-of-two subBucketCount (for clean direct indexing) that is large enough to // provide unit resolution to at least largestValueWithSingleUnitResolution. So figure out // largestValueWithSingleUnitResolution's nearest power-of-two (rounded up), and use that: subBucketCountMagnitude := int32(math.Ceil(math.Log2(float64(largestValueWithSingleUnitResolution)))) subBucketHalfCountMagnitude := subBucketCountMagnitude if subBucketHalfCountMagnitude < 1 { subBucketHalfCountMagnitude = 1 } subBucketHalfCountMagnitude-- unitMagnitude := int32(math.Floor(math.Log2(float64(lowestDiscernibleValue)))) if unitMagnitude < 0 { unitMagnitude = 0 } subBucketCount := int32(math.Pow(2, float64(subBucketHalfCountMagnitude)+1)) subBucketHalfCount := subBucketCount / 2 subBucketMask := int64(subBucketCount-1) << uint(unitMagnitude) // determine exponent range needed to support the trackable value with no // overflow: smallestUntrackableValue := int64(subBucketCount) << uint(unitMagnitude) bucketsNeeded := getBucketsNeededToCoverValue(smallestUntrackableValue, highestTrackableValue) bucketCount := bucketsNeeded countsLen := (bucketCount + 1) * (subBucketCount / 2) return &Histogram{ lowestDiscernibleValue: lowestDiscernibleValue, highestTrackableValue: highestTrackableValue, unitMagnitude: int64(unitMagnitude), significantFigures: int64(numberOfSignificantValueDigits), subBucketHalfCountMagnitude: subBucketHalfCountMagnitude, subBucketHalfCount: subBucketHalfCount, subBucketMask: subBucketMask, subBucketCount: subBucketCount, bucketCount: bucketCount, countsLen: countsLen, totalCount: 0, counts: make([]int64, countsLen), startTimeMs: 0, endTimeMs: 0, tag: "", } } func getBucketsNeededToCoverValue(smallestUntrackableValue int64, maxValue int64) int32 { // always have at least 1 bucket bucketsNeeded := int32(1) for smallestUntrackableValue < maxValue { if smallestUntrackableValue > (math.MaxInt64 / 2) { // next shift will overflow, meaning that bucket could represent values up to ones greater than // math.MaxInt64, so it's the last bucket return bucketsNeeded + 1 } smallestUntrackableValue <<= 1 bucketsNeeded++ } return bucketsNeeded } // ByteSize returns an estimate of the amount of memory allocated to the // histogram in bytes. // // N.B.: This does not take into account the overhead for slices, which are // small, constant, and specific to the compiler version. func (h *Histogram) ByteSize() int { return 6*8 + 5*4 + len(h.counts)*8 } func (h *Histogram) getNormalizingIndexOffset() int32 { return 1 } // Merge merges the data stored in the given histogram with the receiver, // returning the number of recorded values which had to be dropped. func (h *Histogram) Merge(from *Histogram) (dropped int64) { i := from.rIterator() for i.next() { v := i.valueFromIdx c := i.countAtIdx if h.RecordValues(v, c) != nil { dropped += c } } return } // TotalCount returns total number of values recorded. func (h *Histogram) TotalCount() int64 { return h.totalCount } // Max returns the approximate maximum recorded value. func (h *Histogram) Max() int64 { var max int64 i := h.iterator() for i.next() { if i.countAtIdx != 0 { max = i.highestEquivalentValue } } return h.highestEquivalentValue(max) } // Min returns the approximate minimum recorded value. func (h *Histogram) Min() int64 { var min int64 i := h.iterator() for i.next() { if i.countAtIdx != 0 && min == 0 { min = i.highestEquivalentValue break } } return h.lowestEquivalentValue(min) } // Mean returns the approximate arithmetic mean of the recorded values. func (h *Histogram) Mean() float64 { if h.totalCount == 0 { return 0 } var total int64 i := h.iterator() for i.next() { if i.countAtIdx != 0 { total += i.countAtIdx * h.medianEquivalentValue(i.valueFromIdx) } } return float64(total) / float64(h.totalCount) } // StdDev returns the approximate standard deviation of the recorded values. func (h *Histogram) StdDev() float64 { if h.totalCount == 0 { return 0 } mean := h.Mean() geometricDevTotal := 0.0 i := h.iterator() for i.next() { if i.countAtIdx != 0 { dev := float64(h.medianEquivalentValue(i.valueFromIdx)) - mean geometricDevTotal += (dev * dev) * float64(i.countAtIdx) } } return math.Sqrt(geometricDevTotal / float64(h.totalCount)) } // Reset deletes all recorded values and restores the histogram to its original // state. func (h *Histogram) Reset() { h.totalCount = 0 for i := range h.counts { h.counts[i] = 0 } } // RecordValue records the given value, returning an error if the value is out // of range. func (h *Histogram) RecordValue(v int64) error { return h.RecordValues(v, 1) } // RecordCorrectedValue records the given value, correcting for stalls in the // recording process. This only works for processes which are recording values // at an expected interval (e.g., doing jitter analysis). Processes which are // recording ad-hoc values (e.g., latency for incoming requests) can't take // advantage of this. func (h *Histogram) RecordCorrectedValue(v, expectedInterval int64) error { if err := h.RecordValue(v); err != nil { return err } if expectedInterval <= 0 || v <= expectedInterval { return nil } missingValue := v - expectedInterval for missingValue >= expectedInterval { if err := h.RecordValue(missingValue); err != nil { return err } missingValue -= expectedInterval } return nil } // RecordValues records n occurrences of the given value, returning an error if // the value is out of range. func (h *Histogram) RecordValues(v, n int64) error { idx := h.countsIndexFor(v) if idx < 0 || int(h.countsLen) <= idx { return fmt.Errorf("value %d is too large to be recorded", v) } h.setCountAtIndex(idx, n) return nil } func (h *Histogram) setCountAtIndex(idx int, n int64) { h.counts[idx] += n h.totalCount += n } // ValueAtQuantile returns the largest value that (100% - percentile) of the overall recorded value entries // in the histogram are either larger than or equivalent to. // // The passed quantile must be a float64 value in [0.0 .. 100.0] // Note that two values are "equivalent" if `ValuesAreEquivalent(value1,value2)` would return true. // // Returns 0 if no recorded values exist. func (h *Histogram) ValueAtQuantile(q float64) int64 { return h.ValueAtPercentile(q) } // ValueAtPercentile returns the largest value that (100% - percentile) of the overall recorded value entries // in the histogram are either larger than or equivalent to. // // The passed percentile must be a float64 value in [0.0 .. 100.0] // Note that two values are "equivalent" if `ValuesAreEquivalent(value1,value2)` would return true. // // Returns 0 if no recorded values exist. func (h *Histogram) ValueAtPercentile(percentile float64) int64 { if percentile > 100 { percentile = 100 } countAtPercentile := int64(((percentile / 100) * float64(h.totalCount)) + 0.5) valueFromIdx := h.getValueFromIdxUpToCount(countAtPercentile) if percentile == 0.0 { return h.lowestEquivalentValue(valueFromIdx) } return h.highestEquivalentValue(valueFromIdx) } func (h *Histogram) getValueFromIdxUpToCount(countAtPercentile int64) int64 { var countToIdx int64 var valueFromIdx int64 var subBucketIdx int32 = -1 var bucketIdx int32 bucketBaseIdx := h.getBucketBaseIdx(bucketIdx) for { if countToIdx >= countAtPercentile { break } // increment bucket subBucketIdx++ if subBucketIdx >= h.subBucketCount { subBucketIdx = h.subBucketHalfCount bucketIdx++ bucketBaseIdx = h.getBucketBaseIdx(bucketIdx) } countToIdx += h.getCountAtIndexGivenBucketBaseIdx(bucketBaseIdx, subBucketIdx) valueFromIdx = int64(subBucketIdx) << uint(int64(bucketIdx)+h.unitMagnitude) } return valueFromIdx } // ValueAtPercentiles, given an slice of percentiles returns a map containing for each passed percentile, // the largest value that (100% - percentile) of the overall recorded value entries // in the histogram are either larger than or equivalent to. // // Each element in the given an slice of percentiles must be a float64 value in [0.0 .. 100.0] // Note that two values are "equivalent" if `ValuesAreEquivalent(value1,value2)` would return true. // // Returns a map of 0's if no recorded values exist. func (h *Histogram) ValueAtPercentiles(percentiles []float64) (values map[float64]int64) { sort.Float64s(percentiles) totalQuantilesToCalculate := len(percentiles) values = make(map[float64]int64, totalQuantilesToCalculate) countAtPercentiles := make([]int64, totalQuantilesToCalculate) for i, percentile := range percentiles { if percentile > 100 { percentile = 100 } values[percentile] = 0 countAtPercentiles[i] = int64(((percentile / 100) * float64(h.totalCount)) + 0.5) } total := int64(0) currentQuantileSlicePos := 0 i := h.iterator() for currentQuantileSlicePos < totalQuantilesToCalculate && i.nextCountAtIdx(h.totalCount) { total += i.countAtIdx for currentQuantileSlicePos < totalQuantilesToCalculate && total >= countAtPercentiles[currentQuantileSlicePos] { currentPercentile := percentiles[currentQuantileSlicePos] if currentPercentile == 0.0 { values[currentPercentile] = h.lowestEquivalentValue(i.valueFromIdx) } else { values[currentPercentile] = h.highestEquivalentValue(i.valueFromIdx) } currentQuantileSlicePos++ } } return } // Determine if two values are equivalent with the histogram's resolution. // Where "equivalent" means that value samples recorded for any two // equivalent values are counted in a common total count. func (h *Histogram) ValuesAreEquivalent(value1, value2 int64) (result bool) { result = h.lowestEquivalentValue(value1) == h.lowestEquivalentValue(value2) return } // CumulativeDistribution returns an ordered list of brackets of the // distribution of recorded values. func (h *Histogram) CumulativeDistribution() []Bracket { var result []Bracket i := h.pIterator(1) for i.next() { result = append(result, Bracket{ Quantile: i.percentile, Count: i.countToIdx, ValueAt: i.highestEquivalentValue, }) } return result } // SignificantFigures returns the significant figures used to create the // histogram func (h *Histogram) SignificantFigures() int64 { return h.significantFigures } // LowestTrackableValue returns the lower bound on values that will be added // to the histogram func (h *Histogram) LowestTrackableValue() int64 { return h.lowestDiscernibleValue } // HighestTrackableValue returns the upper bound on values that will be added // to the histogram func (h *Histogram) HighestTrackableValue() int64 { return h.highestTrackableValue } // Histogram bar for plotting type Bar struct { From, To, Count int64 } // Pretty print as csv for easy plotting func (b Bar) String() string { return fmt.Sprintf("%v, %v, %v\n", b.From, b.To, b.Count) } // Distribution returns an ordered list of bars of the // distribution of recorded values, counts can be normalized to a probability func (h *Histogram) Distribution() (result []Bar) { i := h.iterator() for i.next() { result = append(result, Bar{ Count: i.countAtIdx, From: h.lowestEquivalentValue(i.valueFromIdx), To: i.highestEquivalentValue, }) } return result } // Equals returns true if the two Histograms are equivalent, false if not. func (h *Histogram) Equals(other *Histogram) bool { switch { case h.lowestDiscernibleValue != other.lowestDiscernibleValue, h.highestTrackableValue != other.highestTrackableValue, h.unitMagnitude != other.unitMagnitude, h.significantFigures != other.significantFigures, h.subBucketHalfCountMagnitude != other.subBucketHalfCountMagnitude, h.subBucketHalfCount != other.subBucketHalfCount, h.subBucketMask != other.subBucketMask, h.subBucketCount != other.subBucketCount, h.bucketCount != other.bucketCount, h.countsLen != other.countsLen, h.totalCount != other.totalCount: return false default: for i, c := range h.counts { if c != other.counts[i] { return false } } } return true } // Export returns a snapshot view of the Histogram. This can be later passed to // Import to construct a new Histogram with the same state. func (h *Histogram) Export() *Snapshot { return &Snapshot{ LowestTrackableValue: h.lowestDiscernibleValue, HighestTrackableValue: h.highestTrackableValue, SignificantFigures: h.significantFigures, Counts: append([]int64(nil), h.counts...), // copy } } // Import returns a new Histogram populated from the Snapshot data (which the // caller must stop accessing). func Import(s *Snapshot) *Histogram { h := New(s.LowestTrackableValue, s.HighestTrackableValue, int(s.SignificantFigures)) h.counts = s.Counts totalCount := int64(0) for i := int32(0); i < h.countsLen; i++ { countAtIndex := h.counts[i] if countAtIndex > 0 { totalCount += countAtIndex } } h.totalCount = totalCount return h } func (h *Histogram) iterator() *iterator { return &iterator{ h: h, subBucketIdx: -1, } } func (h *Histogram) rIterator() *rIterator { return &rIterator{ iterator: iterator{ h: h, subBucketIdx: -1, }, } } func (h *Histogram) pIterator(ticksPerHalfDistance int32) *pIterator { return &pIterator{ iterator: iterator{ h: h, subBucketIdx: -1, }, ticksPerHalfDistance: ticksPerHalfDistance, } } func (h *Histogram) sizeOfEquivalentValueRange(v int64) int64 { bucketIdx := h.getBucketIndex(v) return h.sizeOfEquivalentValueRangeGivenBucketIdx(v, bucketIdx) } func (h *Histogram) sizeOfEquivalentValueRangeGivenBucketIdx(v int64, bucketIdx int32) int64 { subBucketIdx := h.getSubBucketIdx(v, bucketIdx) adjustedBucket := bucketIdx if subBucketIdx >= h.subBucketCount { adjustedBucket++ } return int64(1) << uint(h.unitMagnitude+int64(adjustedBucket)) } func (h *Histogram) valueFromIndex(bucketIdx, subBucketIdx int32) int64 { return int64(subBucketIdx) << uint(int64(bucketIdx)+h.unitMagnitude) } func (h *Histogram) lowestEquivalentValue(v int64) int64 { bucketIdx := h.getBucketIndex(v) return h.lowestEquivalentValueGivenBucketIdx(v, bucketIdx) } func (h *Histogram) lowestEquivalentValueGivenBucketIdx(v int64, bucketIdx int32) int64 { subBucketIdx := h.getSubBucketIdx(v, bucketIdx) return h.valueFromIndex(bucketIdx, subBucketIdx) } func (h *Histogram) nextNonEquivalentValue(v int64) int64 { bucketIdx := h.getBucketIndex(v) return h.lowestEquivalentValueGivenBucketIdx(v, bucketIdx) + h.sizeOfEquivalentValueRangeGivenBucketIdx(v, bucketIdx) } func (h *Histogram) highestEquivalentValue(v int64) int64 { return h.nextNonEquivalentValue(v) - 1 } func (h *Histogram) medianEquivalentValue(v int64) int64 { return h.lowestEquivalentValue(v) + (h.sizeOfEquivalentValueRange(v) >> 1) } func (h *Histogram) getCountAtIndex(bucketIdx, subBucketIdx int32) int64 { return h.counts[h.countsIndex(bucketIdx, subBucketIdx)] } func (h *Histogram) getCountAtIndexGivenBucketBaseIdx(bucketBaseIdx, subBucketIdx int32) int64 { return h.counts[bucketBaseIdx+subBucketIdx-h.subBucketHalfCount] } func (h *Histogram) countsIndex(bucketIdx, subBucketIdx int32) int32 { return h.getBucketBaseIdx(bucketIdx) + subBucketIdx - h.subBucketHalfCount } func (h *Histogram) getBucketBaseIdx(bucketIdx int32) int32 { return (bucketIdx + 1) << uint(h.subBucketHalfCountMagnitude) } // return the lowest (and therefore highest precision) bucket index that can represent the value // Calculates the number of powers of two by which the value is greater than the biggest value that fits in // bucket 0. This is the bucket index since each successive bucket can hold a value 2x greater. func (h *Histogram) getBucketIndex(v int64) int32 { var pow2Ceiling = int64(64 - bits.LeadingZeros64(uint64(v|h.subBucketMask))) return int32(pow2Ceiling - int64(h.unitMagnitude) - int64(h.subBucketHalfCountMagnitude+1)) } // For bucketIndex 0, this is just value, so it may be anywhere in 0 to subBucketCount. // For other bucketIndex, this will always end up in the top half of subBucketCount: assume that for some bucket // k > 0, this calculation will yield a value in the bottom half of 0 to subBucketCount. Then, because of how // buckets overlap, it would have also been in the top half of bucket k-1, and therefore would have // returned k-1 in getBucketIndex(). Since we would then shift it one fewer bits here, it would be twice as big, // and therefore in the top half of subBucketCount. func (h *Histogram) getSubBucketIdx(v int64, idx int32) int32 { return int32(v >> uint(int64(idx)+int64(h.unitMagnitude))) } func (h *Histogram) countsIndexFor(v int64) int { bucketIdx := h.getBucketIndex(v) subBucketIdx := h.getSubBucketIdx(v, bucketIdx) return int(h.countsIndex(bucketIdx, subBucketIdx)) } func (h *Histogram) getIntegerToDoubleValueConversionRatio() float64 { return 1.0 } type iterator struct { h *Histogram bucketIdx, subBucketIdx int32 countAtIdx, countToIdx, valueFromIdx int64 highestEquivalentValue int64 } // nextCountAtIdx does not update the iterator highestEquivalentValue in order to optimize cpu usage. func (i *iterator) nextCountAtIdx(limit int64) bool { if i.countToIdx >= limit { return false } // increment bucket i.subBucketIdx++ if i.subBucketIdx >= i.h.subBucketCount { i.subBucketIdx = i.h.subBucketHalfCount i.bucketIdx++ } if i.bucketIdx >= i.h.bucketCount { return false } i.countAtIdx = i.h.getCountAtIndex(i.bucketIdx, i.subBucketIdx) i.countToIdx += i.countAtIdx i.valueFromIdx = i.h.valueFromIndex(i.bucketIdx, i.subBucketIdx) return true } // Returns the next element in the iteration. func (i *iterator) next() bool { if !i.nextCountAtIdx(i.h.totalCount) { return false } i.highestEquivalentValue = i.h.highestEquivalentValue(i.valueFromIdx) return true } type rIterator struct { iterator countAddedThisStep int64 } func (r *rIterator) next() bool { for r.iterator.next() { if r.countAtIdx != 0 { r.countAddedThisStep = r.countAtIdx return true } } return false } type pIterator struct { iterator seenLastValue bool ticksPerHalfDistance int32 percentileToIteratorTo float64 percentile float64 } func (p *pIterator) next() bool { if !(p.countToIdx < p.h.totalCount) { if p.seenLastValue { return false } p.seenLastValue = true p.percentile = 100 return true } if p.subBucketIdx == -1 && !p.iterator.next() { return false } var done = false for !done { currentPercentile := (100.0 * float64(p.countToIdx)) / float64(p.h.totalCount) if p.countAtIdx != 0 && p.percentileToIteratorTo <= currentPercentile { p.percentile = p.percentileToIteratorTo halfDistance := math.Trunc(math.Pow(2, math.Trunc(math.Log2(100.0/(100.0-p.percentileToIteratorTo)))+1)) percentileReportingTicks := float64(p.ticksPerHalfDistance) * halfDistance p.percentileToIteratorTo += 100.0 / percentileReportingTicks return true } done = !p.iterator.next() } return true } // CumulativeDistribution returns an ordered list of brackets of the // distribution of recorded values. func (h *Histogram) CumulativeDistributionWithTicks(ticksPerHalfDistance int32) []Bracket { var result []Bracket i := h.pIterator(ticksPerHalfDistance) for i.next() { result = append(result, Bracket{ Quantile: i.percentile, Count: i.countToIdx, ValueAt: int64(i.highestEquivalentValue), }) } return result } // Output the percentiles distribution in a text format func (h *Histogram) PercentilesPrint(writer io.Writer, ticksPerHalfDistance int32, valueScale float64) (outputWriter io.Writer, err error) { outputWriter = writer dist := h.CumulativeDistributionWithTicks(ticksPerHalfDistance) _, err = outputWriter.Write([]byte(" Value\tPercentile\tTotalCount\t1/(1-Percentile)\n\n")) if err != nil { return } for _, slice := range dist { percentile := slice.Quantile / 100.0 inverted_percentile := 1.0 / (1.0 - percentile) var inverted_percentile_string = fmt.Sprintf("%12.2f", inverted_percentile) // Given that other language implementations display inf (instead of Go's +Inf) // we want to be as close as possible to them if math.IsInf(inverted_percentile, 1) { inverted_percentile_string = fmt.Sprintf("%12s", "inf") } _, err = outputWriter.Write([]byte(fmt.Sprintf("%12.3f %12f %12d %s\n", float64(slice.ValueAt)/valueScale, percentile, slice.Count, inverted_percentile_string))) if err != nil { return } } footer := fmt.Sprintf("#[Mean = %12.3f, StdDeviation = %12.3f]\n#[Max = %12.3f, Total count = %12d]\n#[Buckets = %12d, SubBuckets = %12d]\n", h.Mean()/valueScale, h.StdDev()/valueScale, float64(h.Max())/valueScale, h.TotalCount(), h.bucketCount, h.subBucketCount, ) _, err = outputWriter.Write([]byte(footer)) return }