From a7619df2936b7f0ef81098f4d44fc03361543d30 Mon Sep 17 00:00:00 2001 From: Ivan Daniluk Date: Sat, 12 Nov 2016 23:59:56 +0100 Subject: [PATCH] Add GCPauses var kind --- histogram.go | 150 +++++++++++++++++++++++++++++++++++++++++++++++++++ var.go | 70 ++++++++++++++++++------ var_test.go | 15 ++++++ 3 files changed, 218 insertions(+), 17 deletions(-) create mode 100644 histogram.go diff --git a/histogram.go b/histogram.go new file mode 100644 index 0000000..320adad --- /dev/null +++ b/histogram.go @@ -0,0 +1,150 @@ +package main + +type Histogram struct { + Bins []Bin + Maxbins int + Total uint64 +} + +type Bin struct { + Value, Count uint64 +} + +// NewHistogram returns a new Histogram with a maximum of n bins. +// +// There is no "optimal" bin count, but somewhere between 20 and 80 bins +// should be sufficient. +func NewHistogram(n int) *Histogram { + return &Histogram{ + Bins: make([]Bin, 0), + Maxbins: n, + Total: 0, + } +} + +func (h *Histogram) Add(n uint64) { + defer h.trim() + h.Total++ + for i := range h.Bins { + if h.Bins[i].Value == n { + h.Bins[i].Count++ + return + } + + if h.Bins[i].Value > n { + newbin := Bin{Value: n, Count: 1} + head := append(make([]Bin, 0), h.Bins[0:i]...) + + head = append(head, newbin) + tail := h.Bins[i:] + h.Bins = append(head, tail...) + return + } + } + + h.Bins = append(h.Bins, Bin{Count: 1, Value: n}) +} + +func (h *Histogram) Quantile(q uint64) int64 { + count := q * h.Total + for i := range h.Bins { + count -= h.Bins[i].Count + + if count <= 0 { + return int64(h.Bins[i].Value) + } + } + + return -1 +} + +// CDF returns the value of the cumulative distribution function +// at x +func (h *Histogram) CDF(x uint64) uint64 { + var count uint64 + for i := range h.Bins { + if h.Bins[i].Value <= x { + count += h.Bins[i].Count + } + } + + return count / h.Total +} + +// Mean returns the sample mean of the distribution +func (h *Histogram) Mean() float64 { + if h.Total == 0 { + return 0 + } + + sum := 0.0 + + for i := range h.Bins { + sum += float64(h.Bins[i].Value * h.Bins[i].Count) + } + + return sum / float64(h.Total) +} + +// Variance returns the variance of the distribution +func (h *Histogram) Variance() float64 { + if h.Total == 0 { + return 0 + } + + sum := 0.0 + mean := h.Mean() + + for i := range h.Bins { + sum += float64(h.Bins[i].Count * (h.Bins[i].Value - uint64(mean)) * (h.Bins[i].Value - uint64(mean))) + } + + return sum / float64(h.Total) +} + +func (h *Histogram) Count() uint64 { + return h.Total +} + +// trim merges adjacent bins to decrease the bin count to the maximum value +func (h *Histogram) trim() { + for len(h.Bins) > h.Maxbins { + // Find closest bins in terms of value + minDelta := uint64(1e10) + minDeltaIndex := 0 + for i := range h.Bins { + if i == 0 { + continue + } + + if delta := h.Bins[i].Value - h.Bins[i-1].Value; delta < minDelta { + minDelta = delta + minDeltaIndex = i + } + } + + // We need to merge bins minDeltaIndex-1 and minDeltaIndex + totalCount := h.Bins[minDeltaIndex-1].Count + h.Bins[minDeltaIndex].Count + mergedbin := Bin{ + Value: (h.Bins[minDeltaIndex-1].Value* + h.Bins[minDeltaIndex-1].Count + + h.Bins[minDeltaIndex].Value* + h.Bins[minDeltaIndex].Count) / + totalCount, // weighted average + Count: totalCount, // summed heights + } + head := append(make([]Bin, 0), h.Bins[0:minDeltaIndex-1]...) + tail := append([]Bin{mergedbin}, h.Bins[minDeltaIndex+1:]...) + h.Bins = append(head, tail...) + } +} + +func (h *Histogram) BarchartData() ([]uint64, []int) { + values := make([]uint64, len(h.Bins)) + counts := make([]int, len(h.Bins)) + for i := 0; i < len(h.Bins); i++ { + values[i] = h.Bins[i].Value + counts[i] = int(h.Bins[i].Count) + } + return values, counts +} diff --git a/var.go b/var.go index e5bf6b4..0a396c3 100644 --- a/var.go +++ b/var.go @@ -25,6 +25,7 @@ const ( KindMemory KindDuration KindString + KindGCPauses ) // Var represents arbitrary value for variable. @@ -41,6 +42,26 @@ type IntVar interface { Value() int } +// NewVar inits new Var object with the given name. +func NewVar(name VarName) Var { + kind := name.Kind() + + switch kind { + case KindDefault: + return &Number{} + case KindMemory: + return &Memory{} + case KindDuration: + return &Duration{} + case KindString: + return &String{} + case KindGCPauses: + return &GCPauses{} + default: + return &Number{} + } +} + // Number is a type for numeric values, obtained from JSON. // In JSON it's always float64, so there is no straightforward way // to separate float from int, so let's keep everything as float. @@ -131,25 +152,35 @@ func (v *String) Set(j *jason.Value) { } } -// TODO: add boolean, timestamp, gcpauses, gcendtimes types +// GCPauses represents GC pauses data. +// +// It uses memstat.PauseNS circular buffer, but lacks +// NumGC information, so we don't know what the start +// and the end. It's enough for most stats, though. +type GCPauses struct { + pauses [256]uint64 +} -// NewVar inits new Var object with the given name. -func NewVar(name VarName) Var { - kind := name.Kind() - - switch kind { - case KindDefault: - return &Number{} - case KindMemory: - return &Memory{} - case KindDuration: - return &Duration{} - case KindString: - return &String{} - default: - return &Number{} +func (v *GCPauses) Kind() VarKind { return KindGCPauses } +func (v *GCPauses) String() string { return "" } +func (v *GCPauses) Set(j *jason.Value) { + v.pauses = [256]uint64{} + if arr, err := j.Array(); err == nil { + for i := 0; i < len(arr); i++ { + p, _ := arr[i].Int64() + v.pauses[i] = uint64(p) + } } } +func (v *GCPauses) Histogram(bins int) *Histogram { + hist := NewHistogram(bins) + for i := 0; i < 256; i++ { + hist.Add(v.pauses[i]) + } + return hist +} + +// TODO: add boolean, timestamp, gcpauses, gcendtimes types // ToSlice converts "dot-separated" notation into the "slice of strings". // @@ -184,8 +215,13 @@ func (v VarName) Long() string { return string(v)[start:] } -// Kind returns kind of variable, based on it's name modifiers ("mem:") +// Kind returns kind of variable, based on it's name +// modifiers ("mem:") or full names for special cases. func (v VarName) Kind() VarKind { + if v.Long() == "memstats.PauseNs" { + return KindGCPauses + } + start := strings.IndexRune(string(v), ':') if start == -1 { return KindDefault diff --git a/var_test.go b/var_test.go index 6c3bf08..9fd4b26 100644 --- a/var_test.go +++ b/var_test.go @@ -191,3 +191,18 @@ func TestVarString(t *testing.T) { t.Fatalf("Expect value to be %s, got %s", want, v.String()) } } + +func TestVarGCPauses(t *testing.T) { + v := &GCPauses{} + v.Set(str2val(t, pauseNSTest)) + hist := v.Histogram(20) + if values, _ := hist.BarchartData(); len(values) != 20 { + t.Fatalf("Expect len of values to be 20, got %v", len(values)) + } + // TODO: check if it's true mean :) + if want, mean := 67383.87109375, hist.Mean(); mean != want { + t.Fatalf("Expect mean to be be %v, got %v", want, mean) + } +} + +const pauseNSTest = "[65916, 92412, 67016, 59076, 55161, 53428, 128675, 90476, 78093, 60473, 64353, 58214, 83926, 64390, 103391, 71275, 76651, 56475, 367180, 184505, 307648, 175680, 129120, 102616, 127322, 224862, 83092, 148607, 122833, 139011, 494885, 97452, 95129, 115403, 119657, 122214, 111744, 115824, 95834, 81927, 91120, 131541, 75511, 135424, 125637, 85784, 107094, 101551, 110081, 80628, 123030, 130343, 128940, 114670, 111470, 75146, 101250, 117553, 112062, 106360, 101543, 108607, 245857, 106147, 108091, 84570, 78700, 117863, 74284, 102977, 83952, 108068, 89709, 115250, 108062, 135150, 84460, 389962, 109881, 79255, 88669, 106366, 90551, 115548, 93409, 124459, 93660, 132709, 70662, 119209, 86984, 118776, 114768, 107875, 70117, 95590, 90558, 86439, 85069, 83155, 89212, 115581, 61221, 78387, 67468, 82099, 107160, 83947, 109817, 113753, 121822, 87682, 104144, 88659, 82247, 91591, 138847, 498527, 121882, 114585, 135840, 111263, 101143, 106915, 100841, 110974, 71145, 97220, 118328, 103716, 115043, 74672, 86126, 106929, 115845, 97969, 118960, 103949, 96019, 80543, 106717, 115346, 114901, 88455, 76337, 107155, 141398, 92871, 120444, 90579, 110057, 94518, 115869, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]"