Browse Source

added metrics

Change-Id: I7727bd43911339f48e2d096f8bb43729eb65b3ad
master
Stephen McQuay 3 years ago
parent
commit
aa1ef99295
No known key found for this signature in database
100 changed files with 35350 additions and 0 deletions
  1. +2
    -0
      db.go
  2. +66
    -0
      metrics/metrics.go
  3. +14
    -0
      server.go
  4. +20
    -0
      vendor/github.com/beorn7/perks/LICENSE
  5. +63
    -0
      vendor/github.com/beorn7/perks/quantile/bench_test.go
  6. +121
    -0
      vendor/github.com/beorn7/perks/quantile/example_test.go
  7. +2388
    -0
      vendor/github.com/beorn7/perks/quantile/exampledata.txt
  8. +292
    -0
      vendor/github.com/beorn7/perks/quantile/stream.go
  9. +188
    -0
      vendor/github.com/beorn7/perks/quantile/stream_test.go
  10. +31
    -0
      vendor/github.com/golang/protobuf/LICENSE
  11. +43
    -0
      vendor/github.com/golang/protobuf/proto/Makefile
  12. +2212
    -0
      vendor/github.com/golang/protobuf/proto/all_test.go
  13. +272
    -0
      vendor/github.com/golang/protobuf/proto/any_test.go
  14. +223
    -0
      vendor/github.com/golang/protobuf/proto/clone.go
  15. +267
    -0
      vendor/github.com/golang/protobuf/proto/clone_test.go
  16. +868
    -0
      vendor/github.com/golang/protobuf/proto/decode.go
  17. +1331
    -0
      vendor/github.com/golang/protobuf/proto/encode.go
  18. +276
    -0
      vendor/github.com/golang/protobuf/proto/equal.go
  19. +212
    -0
      vendor/github.com/golang/protobuf/proto/equal_test.go
  20. +399
    -0
      vendor/github.com/golang/protobuf/proto/extensions.go
  21. +430
    -0
      vendor/github.com/golang/protobuf/proto/extensions_test.go
  22. +894
    -0
      vendor/github.com/golang/protobuf/proto/lib.go
  23. +280
    -0
      vendor/github.com/golang/protobuf/proto/message_set.go
  24. +66
    -0
      vendor/github.com/golang/protobuf/proto/message_set_test.go
  25. +479
    -0
      vendor/github.com/golang/protobuf/proto/pointer_reflect.go
  26. +266
    -0
      vendor/github.com/golang/protobuf/proto/pointer_unsafe.go
  27. +846
    -0
      vendor/github.com/golang/protobuf/proto/properties.go
  28. +196
    -0
      vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.pb.go
  29. +72
    -0
      vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.proto
  30. +125
    -0
      vendor/github.com/golang/protobuf/proto/proto3_test.go
  31. +63
    -0
      vendor/github.com/golang/protobuf/proto/size2_test.go
  32. +164
    -0
      vendor/github.com/golang/protobuf/proto/size_test.go
  33. +50
    -0
      vendor/github.com/golang/protobuf/proto/testdata/Makefile
  34. +86
    -0
      vendor/github.com/golang/protobuf/proto/testdata/golden_test.go
  35. +3992
    -0
      vendor/github.com/golang/protobuf/proto/testdata/test.pb.go
  36. +535
    -0
      vendor/github.com/golang/protobuf/proto/testdata/test.proto
  37. +849
    -0
      vendor/github.com/golang/protobuf/proto/text.go
  38. +871
    -0
      vendor/github.com/golang/protobuf/proto/text_parser.go
  39. +557
    -0
      vendor/github.com/golang/protobuf/proto/text_parser_test.go
  40. +474
    -0
      vendor/github.com/golang/protobuf/proto/text_test.go
  41. +111
    -0
      vendor/github.com/golang/protobuf/ptypes/any/any.pb.go
  42. +100
    -0
      vendor/github.com/golang/protobuf/ptypes/any/any.proto
  43. +201
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/LICENSE
  44. +1
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/NOTICE
  45. +7
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/Makefile
  46. +178
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/all_test.go
  47. +75
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/decode.go
  48. +99
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/decode_test.go
  49. +16
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/doc.go
  50. +46
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/encode.go
  51. +67
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/encode_test.go
  52. +4
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/testdata/README.THIRD_PARTY
  53. +4029
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/testdata/test.pb.go
  54. +540
    -0
      vendor/github.com/matttproud/golang_protobuf_extensions/testdata/test.proto
  55. +201
    -0
      vendor/github.com/prometheus/client_golang/LICENSE
  56. +28
    -0
      vendor/github.com/prometheus/client_golang/NOTICE
  57. +53
    -0
      vendor/github.com/prometheus/client_golang/prometheus/README.md
  58. +183
    -0
      vendor/github.com/prometheus/client_golang/prometheus/benchmark_test.go
  59. +75
    -0
      vendor/github.com/prometheus/client_golang/prometheus/collector.go
  60. +173
    -0
      vendor/github.com/prometheus/client_golang/prometheus/counter.go
  61. +58
    -0
      vendor/github.com/prometheus/client_golang/prometheus/counter_test.go
  62. +192
    -0
      vendor/github.com/prometheus/client_golang/prometheus/desc.go
  63. +111
    -0
      vendor/github.com/prometheus/client_golang/prometheus/doc.go
  64. +130
    -0
      vendor/github.com/prometheus/client_golang/prometheus/example_clustermanager_test.go
  65. +87
    -0
      vendor/github.com/prometheus/client_golang/prometheus/example_memstats_test.go
  66. +69
    -0
      vendor/github.com/prometheus/client_golang/prometheus/example_selfcollector_test.go
  67. +640
    -0
      vendor/github.com/prometheus/client_golang/prometheus/examples_test.go
  68. +119
    -0
      vendor/github.com/prometheus/client_golang/prometheus/expvar.go
  69. +97
    -0
      vendor/github.com/prometheus/client_golang/prometheus/expvar_test.go
  70. +29
    -0
      vendor/github.com/prometheus/client_golang/prometheus/fnv.go
  71. +144
    -0
      vendor/github.com/prometheus/client_golang/prometheus/gauge.go
  72. +182
    -0
      vendor/github.com/prometheus/client_golang/prometheus/gauge_test.go
  73. +263
    -0
      vendor/github.com/prometheus/client_golang/prometheus/go_collector.go
  74. +123
    -0
      vendor/github.com/prometheus/client_golang/prometheus/go_collector_test.go
  75. +448
    -0
      vendor/github.com/prometheus/client_golang/prometheus/histogram.go
  76. +326
    -0
      vendor/github.com/prometheus/client_golang/prometheus/histogram_test.go
  77. +381
    -0
      vendor/github.com/prometheus/client_golang/prometheus/http.go
  78. +121
    -0
      vendor/github.com/prometheus/client_golang/prometheus/http_test.go
  79. +166
    -0
      vendor/github.com/prometheus/client_golang/prometheus/metric.go
  80. +35
    -0
      vendor/github.com/prometheus/client_golang/prometheus/metric_test.go
  81. +142
    -0
      vendor/github.com/prometheus/client_golang/prometheus/process_collector.go
  82. +54
    -0
      vendor/github.com/prometheus/client_golang/prometheus/process_collector_test.go
  83. +65
    -0
      vendor/github.com/prometheus/client_golang/prometheus/push.go
  84. +739
    -0
      vendor/github.com/prometheus/client_golang/prometheus/registry.go
  85. +535
    -0
      vendor/github.com/prometheus/client_golang/prometheus/registry_test.go
  86. +538
    -0
      vendor/github.com/prometheus/client_golang/prometheus/summary.go
  87. +347
    -0
      vendor/github.com/prometheus/client_golang/prometheus/summary_test.go
  88. +142
    -0
      vendor/github.com/prometheus/client_golang/prometheus/untyped.go
  89. +234
    -0
      vendor/github.com/prometheus/client_golang/prometheus/value.go
  90. +249
    -0
      vendor/github.com/prometheus/client_golang/prometheus/vec.go
  91. +88
    -0
      vendor/github.com/prometheus/client_golang/prometheus/vec_test.go
  92. +201
    -0
      vendor/github.com/prometheus/client_model/LICENSE
  93. +5
    -0
      vendor/github.com/prometheus/client_model/NOTICE
  94. +364
    -0
      vendor/github.com/prometheus/client_model/go/metrics.pb.go
  95. +201
    -0
      vendor/github.com/prometheus/common/LICENSE
  96. +5
    -0
      vendor/github.com/prometheus/common/NOTICE
  97. +171
    -0
      vendor/github.com/prometheus/common/expfmt/bench_test.go
  98. +412
    -0
      vendor/github.com/prometheus/common/expfmt/decode.go
  99. +367
    -0
      vendor/github.com/prometheus/common/expfmt/decode_test.go
  100. +0
    -0
      vendor/github.com/prometheus/common/expfmt/encode.go

+ 2
- 0
db.go View File

@@ -9,6 +9,7 @@ import (
"time"

verrors "mcquay.me/vain/errors"
"mcquay.me/vain/metrics"
)

// NewMemDB returns a functional MemDB.
@@ -206,6 +207,7 @@ func (m *MemDB) Sync() error {

// flush writes to disk, but expects the user to have taken the lock.
func (m *MemDB) flush(p string) error {
defer metrics.DBTime("flush")()
f, err := os.Create(p)
if err != nil {
return err

+ 66
- 0
metrics/metrics.go View File

@@ -0,0 +1,66 @@
package metrics

import (
"runtime"
"time"

"github.com/prometheus/client_golang/prometheus"
)

var (
// Errors tracks http status codes for problematic requests.
Errors = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "errors_total",
Help: "Number of upstream errors",
},
[]string{"status"},
)

// Func tracks time spent in a function.
Func = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "function_microseconds",
Help: "function timing.",
},
[]string{"route"},
)

// DB tracks timing of interactions with the file system.
DB = prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Name: "db_microseconds",
Help: "db timing.",
},
[]string{"what"},
)
)

func init() {
prometheus.MustRegister(Errors)
prometheus.MustRegister(Func)
prometheus.MustRegister(DB)
}

// Time is a function that makes it simple to add one-line timings to function
// calls.
func Time() func() {
start := time.Now()
return func() {
elapsed := time.Since(start)
pc := make([]uintptr, 10)
runtime.Callers(2, pc)
f := runtime.FuncForPC(pc[0])

Func.WithLabelValues(f.Name()).Observe(float64(elapsed / time.Microsecond))
}
}

// DBTime makes it simple to add one-line timings to db interactions.
func DBTime(name string) func() {
start := time.Now()
return func() {
elapsed := time.Since(start)
DB.WithLabelValues(name).Observe(float64(elapsed / time.Microsecond))
}
}

+ 14
- 0
server.go View File

@@ -9,8 +9,10 @@ import (
"time"

"github.com/elazarl/go-bindata-assetfs"
"github.com/prometheus/client_golang/prometheus"

verrors "mcquay.me/vain/errors"
"mcquay.me/vain/metrics"
"mcquay.me/vain/static"
)

@@ -52,6 +54,7 @@ func NewServer(sm *http.ServeMux, store Storer, m Mailer, static string, emailTi
}

func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
defer metrics.Time()()
if req.Method == "GET" {
req.ParseForm()
if _, ok := req.Form["go-get"]; !ok {
@@ -67,6 +70,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
} else {
p, err := s.db.Package(req.Host + req.URL.Path)
if err := verrors.ToHTTP(err); err != nil {
metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", err.Code, http.StatusText(err.Code))).Add(1)
http.Error(w, err.Message, err.Code)
return
}
@@ -92,6 +96,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}

if err := verrors.ToHTTP(s.db.NSForToken(ns, Token(tok))); err != nil {
metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", err.Code, http.StatusText(err.Code))).Add(1)
http.Error(w, err.Message, err.Code)
return
}
@@ -104,6 +109,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}
p := Package{}
if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", http.StatusBadRequest, http.StatusText(http.StatusBadRequest))).Add(1)
http.Error(w, fmt.Sprintf("unable to parse json from body: %v", err), http.StatusBadRequest)
return
}
@@ -145,6 +151,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
}

func (s *Server) register(w http.ResponseWriter, req *http.Request) {
defer metrics.Time()()
req.ParseForm()
email, ok := req.Form["email"]
if !ok || len(email) != 1 {
@@ -160,6 +167,7 @@ func (s *Server) register(w http.ResponseWriter, req *http.Request) {

tok, err := s.db.Register(Email(addr.Address))
if err := verrors.ToHTTP(err); err != nil {
metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", err.Code, http.StatusText(err.Code))).Add(1)
http.Error(w, err.Message, err.Code)
return
}
@@ -188,6 +196,7 @@ func (s *Server) register(w http.ResponseWriter, req *http.Request) {
}

func (s *Server) confirm(w http.ResponseWriter, req *http.Request) {
defer metrics.Time()()
tok := req.URL.Path[len(prefix["confirm"]):]
tok = strings.TrimRight(tok, "/")
if tok == "" {
@@ -196,6 +205,7 @@ func (s *Server) confirm(w http.ResponseWriter, req *http.Request) {
}
ttok, err := s.db.Confirm(Token(tok))
if err := verrors.ToHTTP(err); err != nil {
metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", err.Code, http.StatusText(err.Code))).Add(1)
http.Error(w, err.Message, err.Code)
return
}
@@ -203,6 +213,7 @@ func (s *Server) confirm(w http.ResponseWriter, req *http.Request) {
}

func (s *Server) forgot(w http.ResponseWriter, req *http.Request) {
defer metrics.Time()()
req.ParseForm()
email, ok := req.Form["email"]
if !ok || len(email) != 1 {
@@ -218,6 +229,7 @@ func (s *Server) forgot(w http.ResponseWriter, req *http.Request) {

tok, err := s.db.Forgot(Email(addr.Address), s.emailTimeout)
if err := verrors.ToHTTP(err); err != nil {
metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", err.Code, http.StatusText(err.Code))).Add(1)
http.Error(w, err.Message, err.Code)
return
}
@@ -245,12 +257,14 @@ func (s *Server) forgot(w http.ResponseWriter, req *http.Request) {
}

func (s *Server) pkgs(w http.ResponseWriter, req *http.Request) {
defer metrics.Time()()
w.Header().Set("Content-type", "application/json")
json.NewEncoder(w).Encode(s.db.Pkgs())
}

func addRoutes(sm *http.ServeMux, s *Server) {
sm.Handle("/", s)
sm.Handle("/metrics", prometheus.Handler())

if s.static == "" {
sm.Handle(

+ 20
- 0
vendor/github.com/beorn7/perks/LICENSE View File

@@ -0,0 +1,20 @@
Copyright (C) 2013 Blake Mizerany

Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:

The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

+ 63
- 0
vendor/github.com/beorn7/perks/quantile/bench_test.go View File

@@ -0,0 +1,63 @@
package quantile

import (
"testing"
)

func BenchmarkInsertTargeted(b *testing.B) {
b.ReportAllocs()

s := NewTargeted(Targets)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}

func BenchmarkInsertTargetedSmallEpsilon(b *testing.B) {
s := NewTargeted(TargetsSmallEpsilon)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}

func BenchmarkInsertBiased(b *testing.B) {
s := NewLowBiased(0.01)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}

func BenchmarkInsertBiasedSmallEpsilon(b *testing.B) {
s := NewLowBiased(0.0001)
b.ResetTimer()
for i := float64(0); i < float64(b.N); i++ {
s.Insert(i)
}
}

func BenchmarkQuery(b *testing.B) {
s := NewTargeted(Targets)
for i := float64(0); i < 1e6; i++ {
s.Insert(i)
}
b.ResetTimer()
n := float64(b.N)
for i := float64(0); i < n; i++ {
s.Query(i / n)
}
}

func BenchmarkQuerySmallEpsilon(b *testing.B) {
s := NewTargeted(TargetsSmallEpsilon)
for i := float64(0); i < 1e6; i++ {
s.Insert(i)
}
b.ResetTimer()
n := float64(b.N)
for i := float64(0); i < n; i++ {
s.Query(i / n)
}
}

+ 121
- 0
vendor/github.com/beorn7/perks/quantile/example_test.go View File

@@ -0,0 +1,121 @@
// +build go1.1

package quantile_test

import (
"bufio"
"fmt"
"log"
"os"
"strconv"
"time"

"github.com/beorn7/perks/quantile"
)

func Example_simple() {
ch := make(chan float64)
go sendFloats(ch)

// Compute the 50th, 90th, and 99th percentile.
q := quantile.NewTargeted(map[float64]float64{
0.50: 0.005,
0.90: 0.001,
0.99: 0.0001,
})
for v := range ch {
q.Insert(v)
}

fmt.Println("perc50:", q.Query(0.50))
fmt.Println("perc90:", q.Query(0.90))
fmt.Println("perc99:", q.Query(0.99))
fmt.Println("count:", q.Count())
// Output:
// perc50: 5
// perc90: 16
// perc99: 223
// count: 2388
}

func Example_mergeMultipleStreams() {
// Scenario:
// We have multiple database shards. On each shard, there is a process
// collecting query response times from the database logs and inserting
// them into a Stream (created via NewTargeted(0.90)), much like the
// Simple example. These processes expose a network interface for us to
// ask them to serialize and send us the results of their
// Stream.Samples so we may Merge and Query them.
//
// NOTES:
// * These sample sets are small, allowing us to get them
// across the network much faster than sending the entire list of data
// points.
//
// * For this to work correctly, we must supply the same quantiles
// a priori the process collecting the samples supplied to NewTargeted,
// even if we do not plan to query them all here.
ch := make(chan quantile.Samples)
getDBQuerySamples(ch)
q := quantile.NewTargeted(map[float64]float64{0.90: 0.001})
for samples := range ch {
q.Merge(samples)
}
fmt.Println("perc90:", q.Query(0.90))
}

func Example_window() {
// Scenario: We want the 90th, 95th, and 99th percentiles for each
// minute.

ch := make(chan float64)
go sendStreamValues(ch)

tick := time.NewTicker(1 * time.Minute)
q := quantile.NewTargeted(map[float64]float64{
0.90: 0.001,
0.95: 0.0005,
0.99: 0.0001,
})
for {
select {
case t := <-tick.C:
flushToDB(t, q.Samples())
q.Reset()
case v := <-ch:
q.Insert(v)
}
}
}

func sendStreamValues(ch chan float64) {
// Use your imagination
}

func flushToDB(t time.Time, samples quantile.Samples) {
// Use your imagination
}

// This is a stub for the above example. In reality this would hit the remote
// servers via http or something like it.
func getDBQuerySamples(ch chan quantile.Samples) {}

func sendFloats(ch chan<- float64) {
f, err := os.Open("exampledata.txt")
if err != nil {
log.Fatal(err)
}
sc := bufio.NewScanner(f)
for sc.Scan() {
b := sc.Bytes()
v, err := strconv.ParseFloat(string(b), 64)
if err != nil {
log.Fatal(err)
}
ch <- v
}
if sc.Err() != nil {
log.Fatal(sc.Err())
}
close(ch)
}

+ 2388
- 0
vendor/github.com/beorn7/perks/quantile/exampledata.txt
File diff suppressed because it is too large
View File


+ 292
- 0
vendor/github.com/beorn7/perks/quantile/stream.go View File

@@ -0,0 +1,292 @@
// Package quantile computes approximate quantiles over an unbounded data
// stream within low memory and CPU bounds.
//
// A small amount of accuracy is traded to achieve the above properties.
//
// Multiple streams can be merged before calling Query to generate a single set
// of results. This is meaningful when the streams represent the same type of
// data. See Merge and Samples.
//
// For more detailed information about the algorithm used, see:
//
// Effective Computation of Biased Quantiles over Data Streams
//
// http://www.cs.rutgers.edu/~muthu/bquant.pdf
package quantile

import (
"math"
"sort"
)

// Sample holds an observed value and meta information for compression. JSON
// tags have been added for convenience.
type Sample struct {
Value float64 `json:",string"`
Width float64 `json:",string"`
Delta float64 `json:",string"`
}

// Samples represents a slice of samples. It implements sort.Interface.
type Samples []Sample

func (a Samples) Len() int { return len(a) }
func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
func (a Samples) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

type invariant func(s *stream, r float64) float64

// NewLowBiased returns an initialized Stream for low-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the lower ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewLowBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * r
}
return newStream(ƒ)
}

// NewHighBiased returns an initialized Stream for high-biased quantiles
// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
// error guarantees can still be given even for the higher ranks of the data
// distribution.
//
// The provided epsilon is a relative error, i.e. the true quantile of a value
// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
// properties.
func NewHighBiased(epsilon float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
return 2 * epsilon * (s.n - r)
}
return newStream(ƒ)
}

// NewTargeted returns an initialized Stream concerned with a particular set of
// quantile values that are supplied a priori. Knowing these a priori reduces
// space and computation time. The targets map maps the desired quantiles to
// their absolute errors, i.e. the true quantile of a value returned by a query
// is guaranteed to be within (Quantile±Epsilon).
//
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func NewTargeted(targets map[float64]float64) *Stream {
ƒ := func(s *stream, r float64) float64 {
var m = math.MaxFloat64
var f float64
for quantile, epsilon := range targets {
if quantile*s.n <= r {
f = (2 * epsilon * r) / quantile
} else {
f = (2 * epsilon * (s.n - r)) / (1 - quantile)
}
if f < m {
m = f
}
}
return m
}
return newStream(ƒ)
}

// Stream computes quantiles for a stream of float64s. It is not thread-safe by
// design. Take care when using across multiple goroutines.
type Stream struct {
*stream
b Samples
sorted bool
}

func newStream(ƒ invariant) *Stream {
x := &stream{ƒ: ƒ}
return &Stream{x, make(Samples, 0, 500), true}
}

// Insert inserts v into the stream.
func (s *Stream) Insert(v float64) {
s.insert(Sample{Value: v, Width: 1})
}

func (s *Stream) insert(sample Sample) {
s.b = append(s.b, sample)
s.sorted = false
if len(s.b) == cap(s.b) {
s.flush()
}
}

// Query returns the computed qth percentiles value. If s was created with
// NewTargeted, and q is not in the set of quantiles provided a priori, Query
// will return an unspecified result.
func (s *Stream) Query(q float64) float64 {
if !s.flushed() {
// Fast path when there hasn't been enough data for a flush;
// this also yields better accuracy for small sets of data.
l := len(s.b)
if l == 0 {
return 0
}
i := int(float64(l) * q)
if i > 0 {
i -= 1
}
s.maybeSort()
return s.b[i].Value
}
s.flush()
return s.stream.query(q)
}

// Merge merges samples into the underlying streams samples. This is handy when
// merging multiple streams from separate threads, database shards, etc.
//
// ATTENTION: This method is broken and does not yield correct results. The
// underlying algorithm is not capable of merging streams correctly.
func (s *Stream) Merge(samples Samples) {
sort.Sort(samples)
s.stream.merge(samples)
}

// Reset reinitializes and clears the list reusing the samples buffer memory.
func (s *Stream) Reset() {
s.stream.reset()
s.b = s.b[:0]
}

// Samples returns stream samples held by s.
func (s *Stream) Samples() Samples {
if !s.flushed() {
return s.b
}
s.flush()
return s.stream.samples()
}

// Count returns the total number of samples observed in the stream
// since initialization.
func (s *Stream) Count() int {
return len(s.b) + s.stream.count()
}

func (s *Stream) flush() {
s.maybeSort()
s.stream.merge(s.b)
s.b = s.b[:0]
}

func (s *Stream) maybeSort() {
if !s.sorted {
s.sorted = true
sort.Sort(s.b)
}
}

func (s *Stream) flushed() bool {
return len(s.stream.l) > 0
}

type stream struct {
n float64
l []Sample
ƒ invariant
}

func (s *stream) reset() {
s.l = s.l[:0]
s.n = 0
}

func (s *stream) insert(v float64) {
s.merge(Samples{{v, 1, 0}})
}

func (s *stream) merge(samples Samples) {
// TODO(beorn7): This tries to merge not only individual samples, but
// whole summaries. The paper doesn't mention merging summaries at
// all. Unittests show that the merging is inaccurate. Find out how to
// do merges properly.
var r float64
i := 0
for _, sample := range samples {
for ; i < len(s.l); i++ {
c := s.l[i]
if c.Value > sample.Value {
// Insert at position i.
s.l = append(s.l, Sample{})
copy(s.l[i+1:], s.l[i:])
s.l[i] = Sample{
sample.Value,
sample.Width,
math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
// TODO(beorn7): How to calculate delta correctly?
}
i++
goto inserted
}
r += c.Width
}
s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
i++
inserted:
s.n += sample.Width
r += sample.Width
}
s.compress()
}

func (s *stream) count() int {
return int(s.n)
}

func (s *stream) query(q float64) float64 {
t := math.Ceil(q * s.n)
t += math.Ceil(s.ƒ(s, t) / 2)
p := s.l[0]
var r float64
for _, c := range s.l[1:] {
r += p.Width
if r+c.Width+c.Delta > t {
return p.Value
}
p = c
}
return p.Value
}

func (s *stream) compress() {
if len(s.l) < 2 {
return
}
x := s.l[len(s.l)-1]
xi := len(s.l) - 1
r := s.n - 1 - x.Width

for i := len(s.l) - 2; i >= 0; i-- {
c := s.l[i]
if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
x.Width += c.Width
s.l[xi] = x
// Remove element at i.
copy(s.l[i:], s.l[i+1:])
s.l = s.l[:len(s.l)-1]
xi -= 1
} else {
x = c
xi = i
}
r -= c.Width
}
}

func (s *stream) samples() Samples {
samples := make(Samples, len(s.l))
copy(samples, s.l)
return samples
}

+ 188
- 0
vendor/github.com/beorn7/perks/quantile/stream_test.go View File

@@ -0,0 +1,188 @@
package quantile

import (
"math"
"math/rand"
"sort"
"testing"
)

var (
Targets = map[float64]float64{
0.01: 0.001,
0.10: 0.01,
0.50: 0.05,
0.90: 0.01,
0.99: 0.001,
}
TargetsSmallEpsilon = map[float64]float64{
0.01: 0.0001,
0.10: 0.001,
0.50: 0.005,
0.90: 0.001,
0.99: 0.0001,
}
LowQuantiles = []float64{0.01, 0.1, 0.5}
HighQuantiles = []float64{0.99, 0.9, 0.5}
)

const RelativeEpsilon = 0.01

func verifyPercsWithAbsoluteEpsilon(t *testing.T, a []float64, s *Stream) {
sort.Float64s(a)
for quantile, epsilon := range Targets {
n := float64(len(a))
k := int(quantile * n)
lower := int((quantile - epsilon) * n)
if lower < 1 {
lower = 1
}
upper := int(math.Ceil((quantile + epsilon) * n))
if upper > len(a) {
upper = len(a)
}
w, min, max := a[k-1], a[lower-1], a[upper-1]
if g := s.Query(quantile); g < min || g > max {
t.Errorf("q=%f: want %v [%f,%f], got %v", quantile, w, min, max, g)
}
}
}

func verifyLowPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
sort.Float64s(a)
for _, qu := range LowQuantiles {
n := float64(len(a))
k := int(qu * n)

lowerRank := int((1 - RelativeEpsilon) * qu * n)
upperRank := int(math.Ceil((1 + RelativeEpsilon) * qu * n))
w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
if g := s.Query(qu); g < min || g > max {
t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
}
}
}

func verifyHighPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
sort.Float64s(a)
for _, qu := range HighQuantiles {
n := float64(len(a))
k := int(qu * n)

lowerRank := int((1 - (1+RelativeEpsilon)*(1-qu)) * n)
upperRank := int(math.Ceil((1 - (1-RelativeEpsilon)*(1-qu)) * n))
w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
if g := s.Query(qu); g < min || g > max {
t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
}
}
}

func populateStream(s *Stream) []float64 {
a := make([]float64, 0, 1e5+100)
for i := 0; i < cap(a); i++ {
v := rand.NormFloat64()
// Add 5% asymmetric outliers.
if i%20 == 0 {
v = v*v + 1
}
s.Insert(v)
a = append(a, v)
}
return a
}

func TestTargetedQuery(t *testing.T) {
rand.Seed(42)
s := NewTargeted(Targets)
a := populateStream(s)
verifyPercsWithAbsoluteEpsilon(t, a, s)
}

func TestLowBiasedQuery(t *testing.T) {
rand.Seed(42)
s := NewLowBiased(RelativeEpsilon)
a := populateStream(s)
verifyLowPercsWithRelativeEpsilon(t, a, s)
}

func TestHighBiasedQuery(t *testing.T) {
rand.Seed(42)
s := NewHighBiased(RelativeEpsilon)
a := populateStream(s)
verifyHighPercsWithRelativeEpsilon(t, a, s)
}

// BrokenTestTargetedMerge is broken, see Merge doc comment.
func BrokenTestTargetedMerge(t *testing.T) {
rand.Seed(42)
s1 := NewTargeted(Targets)
s2 := NewTargeted(Targets)
a := populateStream(s1)
a = append(a, populateStream(s2)...)
s1.Merge(s2.Samples())
verifyPercsWithAbsoluteEpsilon(t, a, s1)
}

// BrokenTestLowBiasedMerge is broken, see Merge doc comment.
func BrokenTestLowBiasedMerge(t *testing.T) {
rand.Seed(42)
s1 := NewLowBiased(RelativeEpsilon)
s2 := NewLowBiased(RelativeEpsilon)
a := populateStream(s1)
a = append(a, populateStream(s2)...)
s1.Merge(s2.Samples())
verifyLowPercsWithRelativeEpsilon(t, a, s2)
}

// BrokenTestHighBiasedMerge is broken, see Merge doc comment.
func BrokenTestHighBiasedMerge(t *testing.T) {
rand.Seed(42)
s1 := NewHighBiased(RelativeEpsilon)
s2 := NewHighBiased(RelativeEpsilon)
a := populateStream(s1)
a = append(a, populateStream(s2)...)
s1.Merge(s2.Samples())
verifyHighPercsWithRelativeEpsilon(t, a, s2)
}

func TestUncompressed(t *testing.T) {
q := NewTargeted(Targets)
for i := 100; i > 0; i-- {
q.Insert(float64(i))
}
if g := q.Count(); g != 100 {
t.Errorf("want count 100, got %d", g)
}
// Before compression, Query should have 100% accuracy.
for quantile := range Targets {
w := quantile * 100
if g := q.Query(quantile); g != w {
t.Errorf("want %f, got %f", w, g)
}
}
}

func TestUncompressedSamples(t *testing.T) {
q := NewTargeted(map[float64]float64{0.99: 0.001})
for i := 1; i <= 100; i++ {
q.Insert(float64(i))
}
if g := q.Samples().Len(); g != 100 {
t.Errorf("want count 100, got %d", g)
}
}

func TestUncompressedOne(t *testing.T) {
q := NewTargeted(map[float64]float64{0.99: 0.01})
q.Insert(3.14)
if g := q.Query(0.90); g != 3.14 {
t.Error("want PI, got", g)
}
}

func TestDefaults(t *testing.T) {
if g := NewTargeted(map[float64]float64{0.99: 0.001}).Query(0.99); g != 0 {
t.Errorf("want 0, got %f", g)
}
}

+ 31
- 0
vendor/github.com/golang/protobuf/LICENSE View File

@@ -0,0 +1,31 @@
Go support for Protocol Buffers - Google's data interchange format

Copyright 2010 The Go Authors. All rights reserved.
https://github.com/golang/protobuf

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


+ 43
- 0
vendor/github.com/golang/protobuf/proto/Makefile View File

@@ -0,0 +1,43 @@
# Go support for Protocol Buffers - Google's data interchange format
#
# Copyright 2010 The Go Authors. All rights reserved.
# https://github.com/golang/protobuf
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

install:
go install

test: install generate-test-pbs
go test


generate-test-pbs:
make install
make -C testdata
protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata,Mgoogle/protobuf/any.proto=github.com/golang/protobuf/ptypes/any:. proto3_proto/proto3.proto
make

+ 2212
- 0
vendor/github.com/golang/protobuf/proto/all_test.go
File diff suppressed because it is too large
View File


+ 272
- 0
vendor/github.com/golang/protobuf/proto/any_test.go View File

@@ -0,0 +1,272 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2016 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package proto_test

import (
"strings"
"testing"

"github.com/golang/protobuf/proto"

pb "github.com/golang/protobuf/proto/proto3_proto"
testpb "github.com/golang/protobuf/proto/testdata"
anypb "github.com/golang/protobuf/ptypes/any"
)

var (
expandedMarshaler = proto.TextMarshaler{ExpandAny: true}
expandedCompactMarshaler = proto.TextMarshaler{Compact: true, ExpandAny: true}
)

// anyEqual reports whether two messages which may be google.protobuf.Any or may
// contain google.protobuf.Any fields are equal. We can't use proto.Equal for
// comparison, because semantically equivalent messages may be marshaled to
// binary in different tag order. Instead, trust that TextMarshaler with
// ExpandAny option works and compare the text marshaling results.
func anyEqual(got, want proto.Message) bool {
// if messages are proto.Equal, no need to marshal.
if proto.Equal(got, want) {
return true
}
g := expandedMarshaler.Text(got)
w := expandedMarshaler.Text(want)
return g == w
}

type golden struct {
m proto.Message
t, c string
}

var goldenMessages = makeGolden()

func makeGolden() []golden {
nested := &pb.Nested{Bunny: "Monty"}
nb, err := proto.Marshal(nested)
if err != nil {
panic(err)
}
m1 := &pb.Message{
Name: "David",
ResultCount: 47,
Anything: &anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(nested), Value: nb},
}
m2 := &pb.Message{
Name: "David",
ResultCount: 47,
Anything: &anypb.Any{TypeUrl: "http://[::1]/type.googleapis.com/" + proto.MessageName(nested), Value: nb},
}
m3 := &pb.Message{
Name: "David",
ResultCount: 47,
Anything: &anypb.Any{TypeUrl: `type.googleapis.com/"/` + proto.MessageName(nested), Value: nb},
}
m4 := &pb.Message{
Name: "David",
ResultCount: 47,
Anything: &anypb.Any{TypeUrl: "type.googleapis.com/a/path/" + proto.MessageName(nested), Value: nb},
}
m5 := &anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(nested), Value: nb}

any1 := &testpb.MyMessage{Count: proto.Int32(47), Name: proto.String("David")}
proto.SetExtension(any1, testpb.E_Ext_More, &testpb.Ext{Data: proto.String("foo")})
proto.SetExtension(any1, testpb.E_Ext_Text, proto.String("bar"))
any1b, err := proto.Marshal(any1)
if err != nil {
panic(err)
}
any2 := &testpb.MyMessage{Count: proto.Int32(42), Bikeshed: testpb.MyMessage_GREEN.Enum(), RepBytes: [][]byte{[]byte("roboto")}}
proto.SetExtension(any2, testpb.E_Ext_More, &testpb.Ext{Data: proto.String("baz")})
any2b, err := proto.Marshal(any2)
if err != nil {
panic(err)
}
m6 := &pb.Message{
Name: "David",
ResultCount: 47,
Anything: &anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(any1), Value: any1b},
ManyThings: []*anypb.Any{
&anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(any2), Value: any2b},
&anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(any1), Value: any1b},
},
}

const (
m1Golden = `
name: "David"
result_count: 47
anything: <
[type.googleapis.com/proto3_proto.Nested]: <
bunny: "Monty"
>
>
`
m2Golden = `
name: "David"
result_count: 47
anything: <
["http://[::1]/type.googleapis.com/proto3_proto.Nested"]: <
bunny: "Monty"
>
>
`
m3Golden = `
name: "David"
result_count: 47
anything: <
["type.googleapis.com/\"/proto3_proto.Nested"]: <
bunny: "Monty"
>
>
`
m4Golden = `
name: "David"
result_count: 47
anything: <
[type.googleapis.com/a/path/proto3_proto.Nested]: <
bunny: "Monty"
>
>
`
m5Golden = `
[type.googleapis.com/proto3_proto.Nested]: <
bunny: "Monty"
>
`
m6Golden = `
name: "David"
result_count: 47
anything: <
[type.googleapis.com/testdata.MyMessage]: <
count: 47
name: "David"
[testdata.Ext.more]: <
data: "foo"
>
[testdata.Ext.text]: "bar"
>
>
many_things: <
[type.googleapis.com/testdata.MyMessage]: <
count: 42
bikeshed: GREEN
rep_bytes: "roboto"
[testdata.Ext.more]: <
data: "baz"
>
>
>
many_things: <
[type.googleapis.com/testdata.MyMessage]: <
count: 47
name: "David"
[testdata.Ext.more]: <
data: "foo"
>
[testdata.Ext.text]: "bar"
>
>
`
)
return []golden{
{m1, strings.TrimSpace(m1Golden) + "\n", strings.TrimSpace(compact(m1Golden)) + " "},
{m2, strings.TrimSpace(m2Golden) + "\n", strings.TrimSpace(compact(m2Golden)) + " "},
{m3, strings.TrimSpace(m3Golden) + "\n", strings.TrimSpace(compact(m3Golden)) + " "},
{m4, strings.TrimSpace(m4Golden) + "\n", strings.TrimSpace(compact(m4Golden)) + " "},
{m5, strings.TrimSpace(m5Golden) + "\n", strings.TrimSpace(compact(m5Golden)) + " "},
{m6, strings.TrimSpace(m6Golden) + "\n", strings.TrimSpace(compact(m6Golden)) + " "},
}
}

func TestMarshalGolden(t *testing.T) {
for _, tt := range goldenMessages {
if got, want := expandedMarshaler.Text(tt.m), tt.t; got != want {
t.Errorf("message %v: got:\n%s\nwant:\n%s", tt.m, got, want)
}
if got, want := expandedCompactMarshaler.Text(tt.m), tt.c; got != want {
t.Errorf("message %v: got:\n`%s`\nwant:\n`%s`", tt.m, got, want)
}
}
}

func TestUnmarshalGolden(t *testing.T) {
for _, tt := range goldenMessages {
want := tt.m
got := proto.Clone(tt.m)
got.Reset()
if err := proto.UnmarshalText(tt.t, got); err != nil {
t.Errorf("failed to unmarshal\n%s\nerror: %v", tt.t, err)
}
if !anyEqual(got, want) {
t.Errorf("message:\n%s\ngot:\n%s\nwant:\n%s", tt.t, got, want)
}
got.Reset()
if err := proto.UnmarshalText(tt.c, got); err != nil {
t.Errorf("failed to unmarshal\n%s\nerror: %v", tt.c, err)
}
if !anyEqual(got, want) {
t.Errorf("message:\n%s\ngot:\n%s\nwant:\n%s", tt.c, got, want)
}
}
}

func TestMarsahlUnknownAny(t *testing.T) {
m := &pb.Message{
Anything: &anypb.Any{
TypeUrl: "foo",
Value: []byte("bar"),
},
}
want := `anything: <
type_url: "foo"
value: "bar"
>
`
got := expandedMarshaler.Text(m)
if got != want {
t.Errorf("got\n`%s`\nwant\n`%s`", got, want)
}
}

func TestAmbiguousAny(t *testing.T) {
pb := &anypb.Any{}
err := proto.UnmarshalText(`
[type.googleapis.com/proto3_proto.Nested]: <
bunny: "Monty"
>
type_url: "ttt/proto3_proto.Nested"
`, pb)
t.Logf("result: %v (error: %v)", expandedMarshaler.Text(pb), err)
if err != nil {
t.Errorf("failed to parse ambiguous Any message: %v", err)
}
}

+ 223
- 0
vendor/github.com/golang/protobuf/proto/clone.go View File

@@ -0,0 +1,223 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2011 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

// Protocol buffer deep copy and merge.
// TODO: RawMessage.

package proto

import (
"log"
"reflect"
"strings"
)

// Clone returns a deep copy of a protocol buffer.
func Clone(pb Message) Message {
in := reflect.ValueOf(pb)
if in.IsNil() {
return pb
}

out := reflect.New(in.Type().Elem())
// out is empty so a merge is a deep copy.
mergeStruct(out.Elem(), in.Elem())
return out.Interface().(Message)
}

// Merge merges src into dst.
// Required and optional fields that are set in src will be set to that value in dst.
// Elements of repeated fields will be appended.
// Merge panics if src and dst are not the same type, or if dst is nil.
func Merge(dst, src Message) {
in := reflect.ValueOf(src)
out := reflect.ValueOf(dst)
if out.IsNil() {
panic("proto: nil destination")
}
if in.Type() != out.Type() {
// Explicit test prior to mergeStruct so that mistyped nils will fail
panic("proto: type mismatch")
}
if in.IsNil() {
// Merging nil into non-nil is a quiet no-op
return
}
mergeStruct(out.Elem(), in.Elem())
}

func mergeStruct(out, in reflect.Value) {
sprop := GetProperties(in.Type())
for i := 0; i < in.NumField(); i++ {
f := in.Type().Field(i)
if strings.HasPrefix(f.Name, "XXX_") {
continue
}
mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i])
}

if emIn, ok := in.Addr().Interface().(extendableProto); ok {
emOut := out.Addr().Interface().(extendableProto)
mergeExtension(emOut.ExtensionMap(), emIn.ExtensionMap())
}

uf := in.FieldByName("XXX_unrecognized")
if !uf.IsValid() {
return
}
uin := uf.Bytes()
if len(uin) > 0 {
out.FieldByName("XXX_unrecognized").SetBytes(append([]byte(nil), uin...))
}
}

// mergeAny performs a merge between two values of the same type.
// viaPtr indicates whether the values were indirected through a pointer (implying proto2).
// prop is set if this is a struct field (it may be nil).
func mergeAny(out, in reflect.Value, viaPtr bool, prop *Properties) {
if in.Type() == protoMessageType {
if !in.IsNil() {
if out.IsNil() {
out.Set(reflect.ValueOf(Clone(in.Interface().(Message))))
} else {
Merge(out.Interface().(Message), in.Interface().(Message))
}
}
return
}
switch in.Kind() {
case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int32, reflect.Int64,
reflect.String, reflect.Uint32, reflect.Uint64:
if !viaPtr && isProto3Zero(in) {
return
}
out.Set(in)
case reflect.Interface:
// Probably a oneof field; copy non-nil values.
if in.IsNil() {
return
}
// Allocate destination if it is not set, or set to a different type.
// Otherwise we will merge as normal.
if out.IsNil() || out.Elem().Type() != in.Elem().Type() {
out.Set(reflect.New(in.Elem().Elem().Type())) // interface -> *T -> T -> new(T)
}
mergeAny(out.Elem(), in.Elem(), false, nil)
case reflect.Map:
if in.Len() == 0 {
return
}
if out.IsNil() {
out.Set(reflect.MakeMap(in.Type()))
}
// For maps with value types of *T or []byte we need to deep copy each value.
elemKind := in.Type().Elem().Kind()
for _, key := range in.MapKeys() {
var val reflect.Value
switch elemKind {
case reflect.Ptr:
val = reflect.New(in.Type().Elem().Elem())
mergeAny(val, in.MapIndex(key), false, nil)
case reflect.Slice:
val = in.MapIndex(key)
val = reflect.ValueOf(append([]byte{}, val.Bytes()...))
default:
val = in.MapIndex(key)
}
out.SetMapIndex(key, val)
}
case reflect.Ptr:
if in.IsNil() {
return
}
if out.IsNil() {
out.Set(reflect.New(in.Elem().Type()))
}
mergeAny(out.Elem(), in.Elem(), true, nil)
case reflect.Slice:
if in.IsNil() {
return
}
if in.Type().Elem().Kind() == reflect.Uint8 {
// []byte is a scalar bytes field, not a repeated field.

// Edge case: if this is in a proto3 message, a zero length
// bytes field is considered the zero value, and should not
// be merged.
if prop != nil && prop.proto3 && in.Len() == 0 {
return
}

// Make a deep copy.
// Append to []byte{} instead of []byte(nil) so that we never end up
// with a nil result.
out.SetBytes(append([]byte{}, in.Bytes()...))
return
}
n := in.Len()
if out.IsNil() {
out.Set(reflect.MakeSlice(in.Type(), 0, n))
}
switch in.Type().Elem().Kind() {
case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int32, reflect.Int64,
reflect.String, reflect.Uint32, reflect.Uint64:
out.Set(reflect.AppendSlice(out, in))
default:
for i := 0; i < n; i++ {
x := reflect.Indirect(reflect.New(in.Type().Elem()))
mergeAny(x, in.Index(i), false, nil)
out.Set(reflect.Append(out, x))
}
}
case reflect.Struct:
mergeStruct(out, in)
default:
// unknown type, so not a protocol buffer
log.Printf("proto: don't know how to copy %v", in)
}
}

func mergeExtension(out, in map[int32]Extension) {
for extNum, eIn := range in {
eOut := Extension{desc: eIn.desc}
if eIn.value != nil {
v := reflect.New(reflect.TypeOf(eIn.value)).Elem()
mergeAny(v, reflect.ValueOf(eIn.value), false, nil)
eOut.value = v.Interface()
}
if eIn.enc != nil {
eOut.enc = make([]byte, len(eIn.enc))
copy(eOut.enc, eIn.enc)
}

out[extNum] = eOut
}
}

+ 267
- 0
vendor/github.com/golang/protobuf/proto/clone_test.go View File

@@ -0,0 +1,267 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2011 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package proto_test

import (
"testing"

"github.com/golang/protobuf/proto"

proto3pb "github.com/golang/protobuf/proto/proto3_proto"
pb "github.com/golang/protobuf/proto/testdata"
)

var cloneTestMessage = &pb.MyMessage{
Count: proto.Int32(42),
Name: proto.String("Dave"),
Pet: []string{"bunny", "kitty", "horsey"},
Inner: &pb.InnerMessage{
Host: proto.String("niles"),
Port: proto.Int32(9099),
Connected: proto.Bool(true),
},
Others: []*pb.OtherMessage{
{
Value: []byte("some bytes"),
},
},
Somegroup: &pb.MyMessage_SomeGroup{
GroupField: proto.Int32(6),
},
RepBytes: [][]byte{[]byte("sham"), []byte("wow")},
}

func init() {
ext := &pb.Ext{
Data: proto.String("extension"),
}
if err := proto.SetExtension(cloneTestMessage, pb.E_Ext_More, ext); err != nil {
panic("SetExtension: " + err.Error())
}
}

func TestClone(t *testing.T) {
m := proto.Clone(cloneTestMessage).(*pb.MyMessage)
if !proto.Equal(m, cloneTestMessage) {
t.Errorf("Clone(%v) = %v", cloneTestMessage, m)
}

// Verify it was a deep copy.
*m.Inner.Port++
if proto.Equal(m, cloneTestMessage) {
t.Error("Mutating clone changed the original")
}
// Byte fields and repeated fields should be copied.
if &m.Pet[0] == &cloneTestMessage.Pet[0] {
t.Error("Pet: repeated field not copied")
}
if &m.Others[0] == &cloneTestMessage.Others[0] {
t.Error("Others: repeated field not copied")
}
if &m.Others[0].Value[0] == &cloneTestMessage.Others[0].Value[0] {
t.Error("Others[0].Value: bytes field not copied")
}
if &m.RepBytes[0] == &cloneTestMessage.RepBytes[0] {
t.Error("RepBytes: repeated field not copied")
}
if &m.RepBytes[0][0] == &cloneTestMessage.RepBytes[0][0] {
t.Error("RepBytes[0]: bytes field not copied")
}
}

func TestCloneNil(t *testing.T) {
var m *pb.MyMessage
if c := proto.Clone(m); !proto.Equal(m, c) {
t.Errorf("Clone(%v) = %v", m, c)
}
}

var mergeTests = []struct {
src, dst, want proto.Message
}{
{
src: &pb.MyMessage{
Count: proto.Int32(42),
},
dst: &pb.MyMessage{
Name: proto.String("Dave"),
},
want: &pb.MyMessage{
Count: proto.Int32(42),
Name: proto.String("Dave"),
},
},
{
src: &pb.MyMessage{
Inner: &pb.InnerMessage{
Host: proto.String("hey"),
Connected: proto.Bool(true),
},
Pet: []string{"horsey"},
Others: []*pb.OtherMessage{
{
Value: []byte("some bytes"),
},
},
},
dst: &pb.MyMessage{
Inner: &pb.InnerMessage{
Host: proto.String("niles"),
Port: proto.Int32(9099),
},
Pet: []string{"bunny", "kitty"},
Others: []*pb.OtherMessage{
{
Key: proto.Int64(31415926535),
},
{
// Explicitly test a src=nil field
Inner: nil,
},
},
},
want: &pb.MyMessage{
Inner: &pb.InnerMessage{
Host: proto.String("hey"),
Connected: proto.Bool(true),
Port: proto.Int32(9099),
},
Pet: []string{"bunny", "kitty", "horsey"},
Others: []*pb.OtherMessage{
{
Key: proto.Int64(31415926535),
},
{},
{
Value: []byte("some bytes"),
},
},
},
},
{
src: &pb.MyMessage{
RepBytes: [][]byte{[]byte("wow")},
},
dst: &pb.MyMessage{
Somegroup: &pb.MyMessage_SomeGroup{
GroupField: proto.Int32(6),
},
RepBytes: [][]byte{[]byte("sham")},
},
want: &pb.MyMessage{
Somegroup: &pb.MyMessage_SomeGroup{
GroupField: proto.Int32(6),
},
RepBytes: [][]byte{[]byte("sham"), []byte("wow")},
},
},
// Check that a scalar bytes field replaces rather than appends.
{
src: &pb.OtherMessage{Value: []byte("foo")},
dst: &pb.OtherMessage{Value: []byte("bar")},
want: &pb.OtherMessage{Value: []byte("foo")},
},
{
src: &pb.MessageWithMap{
NameMapping: map[int32]string{6: "Nigel"},
MsgMapping: map[int64]*pb.FloatingPoint{
0x4001: &pb.FloatingPoint{F: proto.Float64(2.0)},
},
ByteMapping: map[bool][]byte{true: []byte("wowsa")},
},
dst: &pb.MessageWithMap{
NameMapping: map[int32]string{
6: "Bruce", // should be overwritten
7: "Andrew",
},
},
want: &pb.MessageWithMap{
NameMapping: map[int32]string{
6: "Nigel",
7: "Andrew",
},
MsgMapping: map[int64]*pb.FloatingPoint{
0x4001: &pb.FloatingPoint{F: proto.Float64(2.0)},
},
ByteMapping: map[bool][]byte{true: []byte("wowsa")},
},
},
// proto3 shouldn't merge zero values,
// in the same way that proto2 shouldn't merge nils.
{
src: &proto3pb.Message{
Name: "Aaron",
Data: []byte(""), // zero value, but not nil
},
dst: &proto3pb.Message{
HeightInCm: 176,
Data: []byte("texas!"),
},
want: &proto3pb.Message{
Name: "Aaron",
HeightInCm: 176,
Data: []byte("texas!"),
},
},
// Oneof fields should merge by assignment.
{
src: &pb.Communique{
Union: &pb.Communique_Number{41},
},
dst: &pb.Communique{
Union: &pb.Communique_Name{"Bobby Tables"},
},
want: &pb.Communique{
Union: &pb.Communique_Number{41},
},
},
// Oneof nil is the same as not set.
{
src: &pb.Communique{},
dst: &pb.Communique{
Union: &pb.Communique_Name{"Bobby Tables"},
},
want: &pb.Communique{
Union: &pb.Communique_Name{"Bobby Tables"},
},
},
}

func TestMerge(t *testing.T) {
for _, m := range mergeTests {
got := proto.Clone(m.dst)
proto.Merge(got, m.src)
if !proto.Equal(got, m.want) {
t.Errorf("Merge(%v, %v)\n got %v\nwant %v\n", m.dst, m.src, got, m.want)
}
}
}

+ 868
- 0
vendor/github.com/golang/protobuf/proto/decode.go View File

@@ -0,0 +1,868 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2010 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

package proto

/*
* Routines for decoding protocol buffer data to construct in-memory representations.
*/

import (
"errors"
"fmt"
"io"
"os"
"reflect"
)

// errOverflow is returned when an integer is too large to be represented.
var errOverflow = errors.New("proto: integer overflow")

// ErrInternalBadWireType is returned by generated code when an incorrect
// wire type is encountered. It does not get returned to user code.
var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for oneof")

// The fundamental decoders that interpret bytes on the wire.
// Those that take integer types all return uint64 and are
// therefore of type valueDecoder.

// DecodeVarint reads a varint-encoded integer from the slice.
// It returns the integer and the number of bytes consumed, or
// zero if there is not enough.
// This is the format for the
// int32, int64, uint32, uint64, bool, and enum
// protocol buffer types.
func DecodeVarint(buf []byte) (x uint64, n int) {
// x, n already 0
for shift := uint(0); shift < 64; shift += 7 {
if n >= len(buf) {
return 0, 0
}
b := uint64(buf[n])
n++
x |= (b & 0x7F) << shift
if (b & 0x80) == 0 {
return x, n
}
}

// The number is too large to represent in a 64-bit value.
return 0, 0
}

// DecodeVarint reads a varint-encoded integer from the Buffer.
// This is the format for the
// int32, int64, uint32, uint64, bool, and enum
// protocol buffer types.
func (p *Buffer) DecodeVarint() (x uint64, err error) {
// x, err already 0

i := p.index
l := len(p.buf)

for shift := uint(0); shift < 64; shift += 7 {
if i >= l {
err = io.ErrUnexpectedEOF
return
}
b := p.buf[i]
i++
x |= (uint64(b) & 0x7F) << shift
if b < 0x80 {
p.index = i
return
}
}

// The number is too large to represent in a 64-bit value.
err = errOverflow
return
}

// DecodeFixed64 reads a 64-bit integer from the Buffer.
// This is the format for the
// fixed64, sfixed64, and double protocol buffer types.
func (p *Buffer) DecodeFixed64() (x uint64, err error) {
// x, err already 0
i := p.index + 8
if i < 0 || i > len(p.buf) {
err = io.ErrUnexpectedEOF
return
}
p.index = i

x = uint64(p.buf[i-8])
x |= uint64(p.buf[i-7]) << 8
x |= uint64(p.buf[i-6]) << 16
x |= uint64(p.buf[i-5]) << 24
x |= uint64(p.buf[i-4]) << 32
x |= uint64(p.buf[i-3]) << 40
x |= uint64(p.buf[i-2]) << 48
x |= uint64(p.buf[i-1]) << 56
return
}

// DecodeFixed32 reads a 32-bit integer from the Buffer.
// This is the format for the
// fixed32, sfixed32, and float protocol buffer types.
func (p *Buffer) DecodeFixed32() (x uint64, err error) {
// x, err already 0
i := p.index + 4
if i < 0 || i > len(p.buf) {
err = io.ErrUnexpectedEOF
return
}
p.index = i

x = uint64(p.buf[i-4])
x |= uint64(p.buf[i-3]) << 8
x |= uint64(p.buf[i-2]) << 16
x |= uint64(p.buf[i-1]) << 24
return
}

// DecodeZigzag64 reads a zigzag-encoded 64-bit integer
// from the Buffer.
// This is the format used for the sint64 protocol buffer type.
func (p *Buffer) DecodeZigzag64() (x uint64, err error) {
x, err = p.DecodeVarint()
if err != nil {
return
}
x = (x >> 1) ^ uint64((int64(x&1)<<63)>>63)
return
}

// DecodeZigzag32 reads a zigzag-encoded 32-bit integer
// from the Buffer.
// This is the format used for the sint32 protocol buffer type.
func (p *Buffer) DecodeZigzag32() (x uint64, err error) {
x, err = p.DecodeVarint()
if err != nil {
return
}
x = uint64((uint32(x) >> 1) ^ uint32((int32(x&1)<<31)>>31))
return
}

// These are not ValueDecoders: they produce an array of bytes or a string.
// bytes, embedded messages

// DecodeRawBytes reads a count-delimited byte buffer from the Buffer.
// This is the format used for the bytes protocol buffer
// type and for embedded messages.
func (p *Buffer) DecodeRawBytes(alloc bool) (buf []byte, err error) {
n, err := p.DecodeVarint()
if err != nil {
return nil, err
}

nb := int(n)
if nb < 0 {
return nil, fmt.Errorf("proto: bad byte length %d", nb)
}
end := p.index + nb
if end < p.index || end > len(p.buf) {
return nil, io.ErrUnexpectedEOF
}

if !alloc {
// todo: check if can get more uses of alloc=false
buf = p.buf[p.index:end]
p.index += nb
return
}

buf = make([]byte, nb)
copy(buf, p.buf[p.index:])
p.index += nb
return
}

// DecodeStringBytes reads an encoded string from the Buffer.
// This is the format used for the proto2 string type.
func (p *Buffer) DecodeStringBytes() (s string, err error) {
buf, err := p.DecodeRawBytes(false)
if err != nil {
return
}
return string(buf), nil
}

// Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
// If the protocol buffer has extensions, and the field matches, add it as an extension.
// Otherwise, if the XXX_unrecognized field exists, append the skipped data there.
func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base structPointer, unrecField field) error {
oi := o.index

err := o.skip(t, tag, wire)
if err != nil {
return err
}

if !unrecField.IsValid() {
return nil
}

ptr := structPointer_Bytes(base, unrecField)

// Add the skipped field to struct field
obuf := o.buf

o.buf = *ptr
o.EncodeVarint(uint64(tag<<3 | wire))
*ptr = append(o.buf, obuf[oi:o.index]...)

o.buf = obuf

return nil
}

// Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
func (o *Buffer) skip(t reflect.Type, tag, wire int) error {

var u uint64
var err error

switch wire {
case WireVarint:
_, err = o.DecodeVarint()
case WireFixed64:
_, err = o.DecodeFixed64()
case WireBytes:
_, err = o.DecodeRawBytes(false)
case WireFixed32:
_, err = o.DecodeFixed32()
case WireStartGroup:
for {
u, err = o.DecodeVarint()
if err != nil {
break
}
fwire := int(u & 0x7)
if fwire == WireEndGroup {
break
}
ftag := int(u >> 3)
err = o.skip(t, ftag, fwire)
if err != nil {
break
}
}
default:
err = fmt.Errorf("proto: can't skip unknown wire type %d for %s", wire, t)
}
return err
}

// Unmarshaler is the interface representing objects that can
// unmarshal themselves. The method should reset the receiver before
// decoding starts. The argument points to data that may be
// overwritten, so implementations should not keep references to the
// buffer.
type Unmarshaler interface {
Unmarshal([]byte) error
}

// Unmarshal parses the protocol buffer representation in buf and places the
// decoded result in pb. If the struct underlying pb does not match
// the data in buf, the results can be unpredictable.
//
// Unmarshal resets pb before starting to unmarshal, so any
// existing data in pb is always removed. Use UnmarshalMerge
// to preserve and append to existing data.
func Unmarshal(buf []byte, pb Message) error {
pb.Reset()
return UnmarshalMerge(buf, pb)
}

// UnmarshalMerge parses the protocol buffer representation in buf and
// writes the decoded result to pb. If the struct underlying pb does not match
// the data in buf, the results can be unpredictable.
//
// UnmarshalMerge merges into existing data in pb.
// Most code should use Unmarshal instead.
func UnmarshalMerge(buf []byte, pb Message) error {
// If the object can unmarshal itself, let it.
if u, ok := pb.(Unmarshaler); ok {
return u.Unmarshal(buf)
}
return NewBuffer(buf).Unmarshal(pb)
}

// DecodeMessage reads a count-delimited message from the Buffer.
func (p *Buffer) DecodeMessage(pb Message) error {
enc, err := p.DecodeRawBytes(false)
if err != nil {
return err
}
return NewBuffer(enc).Unmarshal(pb)
}

// DecodeGroup reads a tag-delimited group from the Buffer.
func (p *Buffer) DecodeGroup(pb Message) error {
typ, base, err := getbase(pb)
if err != nil {
return err
}
return p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), true, base)
}

// Unmarshal parses the protocol buffer representation in the
// Buffer and places the decoded result in pb. If the struct
// underlying pb does not match the data in the buffer, the results can be
// unpredictable.
func (p *Buffer) Unmarshal(pb Message) error {
// If the object can unmarshal itself, let it.
if u, ok := pb.(Unmarshaler); ok {
err := u.Unmarshal(p.buf[p.index:])
p.index = len(p.buf)
return err
}
<