From 1b669ee5f1f7b8e1d972e809f339cb0d9ec1fe88 Mon Sep 17 00:00:00 2001 From: Stephen McQuay Date: Wed, 1 Jan 2014 12:00:15 -0800 Subject: [PATCH] Added support for starting mid-stream --- ostat.go | 48 ++++++++++++++++-- ostat_test.go | 137 ++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 176 insertions(+), 9 deletions(-) diff --git a/ostat.go b/ostat.go index 956a843..ac1c2a0 100644 --- a/ostat.go +++ b/ostat.go @@ -1,25 +1,31 @@ package ostat import ( + "fmt" "math" ) // from http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance#Online_algorithm +const ( + Population = iota + Sample +) + type OnlineStat struct { - n int64 + n uint64 mean float64 m2 float64 - Max float64 Min float64 - typ int64 + Max float64 + typ uint64 } func NewSampleStat() *OnlineStat { return &OnlineStat{ Min: math.Inf(1), Max: math.Inf(-1), - typ: 1, + typ: Sample, } } @@ -27,7 +33,18 @@ func NewPopulationStat() *OnlineStat { return &OnlineStat{ Min: math.Inf(1), Max: math.Inf(-1), - typ: 0, + typ: Population, + } +} + +func MidStreamStat(n uint64, mean, stddev, min, max float64, typ uint64) *OnlineStat { + return &OnlineStat{ + n: n, + mean: mean, + m2: stddev * stddev * float64(n), + Min: min, + Max: max, + typ: typ, } } @@ -62,3 +79,24 @@ func (os *OnlineStat) Variance() float64 { func (os *OnlineStat) StdDev() float64 { return math.Sqrt(os.Variance()) } + +func (os *OnlineStat) String() string { + return fmt.Sprintf( + "%+v", + struct { + n uint64 + min float64 + max float64 + mean float64 + variance float64 + stdDev float64 + }{ + n: os.n, + min: os.Min, + max: os.Max, + mean: os.Mean(), + variance: os.Variance(), + stdDev: os.StdDev(), + }, + ) +} diff --git a/ostat_test.go b/ostat_test.go index 52c428d..ef1ce1f 100644 --- a/ostat_test.go +++ b/ostat_test.go @@ -1,7 +1,6 @@ package ostat import ( - "log" "math" "testing" ) @@ -130,11 +129,141 @@ func TestEmpty(t *testing.T) { } } -func TestReconstitutedStat(t *testing.T) { - os := OnlineStat{ +func TestMidStreamStat(t *testing.T) { + ps := OnlineStat{ n: 3, mean: 8, m2: 21 * 3, + typ: Population, + } + rps := MidStreamStat(3, 8, 4.5825756949558, 4, 13, Population) + if math.Abs(ps.Mean()-rps.Mean()) > tolerance { + t.Errorf("Incorrect Mean Calc: %f %f", ps.Mean(), rps.Mean()) + } + if math.Abs(ps.Variance()-rps.Variance()) > tolerance { + t.Errorf("Incorrect Variance Calc: %f %f", ps.Variance(), rps.Variance()) + } +} + +// probably spent too much time gathering this data and hard-coding it all in +// here. Alas, this test is to verify that we can start a stat mid-stream, if +// given the right info at the outset. +func TestMidStreamStatWithData(t *testing.T) { + ps := NewPopulationStat() + first := []float64{7633., 8550., -7874., 586., 7488., 839., 8379., 8716., + -9612., -4309., -3923., -9159., 8131., 7392., -1587., 2004., + 7975., -6135., -3863., 686., -7862., 1707., -7967., -9421., + -5618., 2419., -8948., -184., 4450., 7189., 3150., -1879., + -6845., -8551., 5629., -6370., 4949., 6865., 118., 2302., + -4406., 8384., 5643., -7794., 5752., -1361., 6765., 6258., + 5602., -4659., 8268., 2154., 345., 123., 7843., -9841., + 2889., -3665., 6545., -2164., -8135., 8377., 7645., 9759., + 601., 5924., 3348., -3217., 5076., -9145., -2973., 6195., + -8138., -1093., -1432., 813., -6668., -8527., -8646., -670., + -940., -911., 1717., -1388., -786., 2819., 6902., -8566., + 4684., 4809., -4446., -806., 3805., -7975., 1138., 518., + -3416., -1797., -2625., -3202.} + + var n int + var val float64 + for n, val = range first { + ps.Push(val) + } + mss := MidStreamStat( + uint64(n+1), + 83.590000000000003, + 5722.0924356305177, + -9841.0, + 9759.0, + Population, + ) + mean1 := 83.590000000000003 + variance1 := 32742341.841899995 + min1 := -9841.0 + max1 := 9759.0 + + if math.Abs(ps.Mean()-mean1) > tolerance { + t.Errorf("mean: %f != %f", ps.Mean(), mean1) + } + if math.Abs(mss.Mean()-mean1) > tolerance { + t.Errorf("mean: %f != %f", mss.Mean(), mean1) + } + if math.Abs(ps.Variance()-variance1) > tolerance { + t.Errorf("mean: %f != %f", ps.Variance(), variance1) + } + if math.Abs(mss.Variance()-variance1) > tolerance { + t.Errorf("mean: %f != %f", mss.Variance(), variance1) + } + if (ps.Min != mss.Min) || (mss.Min != min1) { + t.Errorf("min: %f != %f != %f", mss.Min, mss.Min, min1) + } + if (ps.Max != mss.Max) || (mss.Max != max1) { + t.Errorf("max: %f != %f != %f", mss.Max, mss.Max, max1) + } + + second := []float64{-3.87000000e+02, -4.26800000e+03, + -1.39900000e+03, 2.00300000e+03, 1.53100000e+03, + -7.91000000e+02, 4.27300000e+03, 4.22100000e+03, + -4.84400000e+03, -3.09100000e+03, 4.69300000e+03, + 4.69100000e+03, -9.78700000e+03, -8.23900000e+03, + 2.60100000e+03, -9.01100000e+03, 4.32700000e+03, + 4.62500000e+03, -1.67800000e+03, -9.76300000e+03, + -9.26700000e+03, -5.80800000e+03, -2.41400000e+03, + -6.62900000e+03, 3.99500000e+03, 6.71800000e+03, + -5.48500000e+03, 1.91300000e+03, -2.21000000e+03, + 6.29400000e+03, 9.35500000e+03, -3.24800000e+03, + 4.58200000e+03, -2.91000000e+02, 1.42500000e+03, + -1.13700000e+03, 5.16900000e+03, -8.36700000e+03, + -5.49500000e+03, 8.93000000e+02, 9.74800000e+03, + -8.50100000e+03, 6.47000000e+02, -4.93800000e+03, + -2.31800000e+03, -5.39100000e+03, 4.55000000e+02, + 9.62000000e+02, -4.39000000e+02, 3.18100000e+03, + 7.39500000e+03, -3.16100000e+03, 6.31400000e+03, + 9.47500000e+03, 3.73500000e+03, 9.48000000e+03, + -4.14700000e+03, -2.48000000e+02, 2.94000000e+02, + -9.95100000e+03, 3.82500000e+03, -1.03700000e+03, + 7.33200000e+03, 9.97800000e+03, -3.36200000e+03, + 9.67200000e+03, 5.62600000e+03, -1.93000000e+02, + -3.41300000e+03, -1.50000000e+01, -2.96000000e+03, + -5.14800000e+03, 8.41900000e+03, -9.07900000e+03, + 7.37500000e+03, -7.05400000e+03, -8.59400000e+03, + 2.66900000e+03, -7.37600000e+03, -6.99500000e+03, + 2.81800000e+03, 2.87700000e+03, 1.83000000e+02, + -9.27100000e+03, 5.83400000e+03, 3.00000000e+00, + -6.02700000e+03, 7.78600000e+03, 1.64700000e+03, + -5.96500000e+03, -8.60000000e+02, -3.67700000e+03, + -6.36600000e+03, -2.06000000e+02, -5.33600000e+03, + -2.31000000e+02, -6.70300000e+03, -4.87900000e+03, + 5.27600000e+03, 9.12600000e+03} + + for _, val = range second { + ps.Push(val) + mss.Push(val) + } + + mean2 := -118.25 + variance2 := 31713809.697500002 + min2 := -9951.0 + max2 := 9978.0 + + if math.Abs(ps.Mean()-mean2) > tolerance { + t.Errorf("mean: %f != %f", ps.Mean(), mean2) + } + if math.Abs(mss.Mean()-mean2) > tolerance { + t.Errorf("mean: %f != %f", mss.Mean(), mean2) + } + + if math.Abs(ps.Variance()-variance2) > tolerance { + t.Errorf("mean: %f != %f", ps.Variance(), variance2) + } + if math.Abs(mss.Variance()-variance2) > tolerance { + t.Errorf("mean: %f != %f", mss.Variance(), variance2) + } + + if (ps.Min != mss.Min) || (mss.Min != min2) { + t.Errorf("min: %f != %f != %f", mss.Min, mss.Min, min2) + } + if (ps.Max != mss.Max) || (mss.Max != max2) { + t.Errorf("max: %f != %f != %f", mss.Max, mss.Max, max2) } - log.Printf("%+v: mean: %f, stddev: %f", os, os.Mean(), os.StdDev()) }