Browse Source

added metrics

Change-Id: I7727bd43911339f48e2d096f8bb43729eb65b3ad
Stephen McQuay 3 years ago
parent
commit
aa1ef99295
No known key found for this signature in database
100 changed files with 35350 additions and 0 deletions
  1. 2
    0
      db.go
  2. 66
    0
      metrics/metrics.go
  3. 14
    0
      server.go
  4. 20
    0
      vendor/github.com/beorn7/perks/LICENSE
  5. 63
    0
      vendor/github.com/beorn7/perks/quantile/bench_test.go
  6. 121
    0
      vendor/github.com/beorn7/perks/quantile/example_test.go
  7. 2388
    0
      vendor/github.com/beorn7/perks/quantile/exampledata.txt
  8. 292
    0
      vendor/github.com/beorn7/perks/quantile/stream.go
  9. 188
    0
      vendor/github.com/beorn7/perks/quantile/stream_test.go
  10. 31
    0
      vendor/github.com/golang/protobuf/LICENSE
  11. 43
    0
      vendor/github.com/golang/protobuf/proto/Makefile
  12. 2212
    0
      vendor/github.com/golang/protobuf/proto/all_test.go
  13. 272
    0
      vendor/github.com/golang/protobuf/proto/any_test.go
  14. 223
    0
      vendor/github.com/golang/protobuf/proto/clone.go
  15. 267
    0
      vendor/github.com/golang/protobuf/proto/clone_test.go
  16. 868
    0
      vendor/github.com/golang/protobuf/proto/decode.go
  17. 1331
    0
      vendor/github.com/golang/protobuf/proto/encode.go
  18. 276
    0
      vendor/github.com/golang/protobuf/proto/equal.go
  19. 212
    0
      vendor/github.com/golang/protobuf/proto/equal_test.go
  20. 399
    0
      vendor/github.com/golang/protobuf/proto/extensions.go
  21. 430
    0
      vendor/github.com/golang/protobuf/proto/extensions_test.go
  22. 894
    0
      vendor/github.com/golang/protobuf/proto/lib.go
  23. 280
    0
      vendor/github.com/golang/protobuf/proto/message_set.go
  24. 66
    0
      vendor/github.com/golang/protobuf/proto/message_set_test.go
  25. 479
    0
      vendor/github.com/golang/protobuf/proto/pointer_reflect.go
  26. 266
    0
      vendor/github.com/golang/protobuf/proto/pointer_unsafe.go
  27. 846
    0
      vendor/github.com/golang/protobuf/proto/properties.go
  28. 196
    0
      vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.pb.go
  29. 72
    0
      vendor/github.com/golang/protobuf/proto/proto3_proto/proto3.proto
  30. 125
    0
      vendor/github.com/golang/protobuf/proto/proto3_test.go
  31. 63
    0
      vendor/github.com/golang/protobuf/proto/size2_test.go
  32. 164
    0
      vendor/github.com/golang/protobuf/proto/size_test.go
  33. 50
    0
      vendor/github.com/golang/protobuf/proto/testdata/Makefile
  34. 86
    0
      vendor/github.com/golang/protobuf/proto/testdata/golden_test.go
  35. 3992
    0
      vendor/github.com/golang/protobuf/proto/testdata/test.pb.go
  36. 535
    0
      vendor/github.com/golang/protobuf/proto/testdata/test.proto
  37. 849
    0
      vendor/github.com/golang/protobuf/proto/text.go
  38. 871
    0
      vendor/github.com/golang/protobuf/proto/text_parser.go
  39. 557
    0
      vendor/github.com/golang/protobuf/proto/text_parser_test.go
  40. 474
    0
      vendor/github.com/golang/protobuf/proto/text_test.go
  41. 111
    0
      vendor/github.com/golang/protobuf/ptypes/any/any.pb.go
  42. 100
    0
      vendor/github.com/golang/protobuf/ptypes/any/any.proto
  43. 201
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/LICENSE
  44. 1
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/NOTICE
  45. 7
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/Makefile
  46. 178
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/all_test.go
  47. 75
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/decode.go
  48. 99
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/decode_test.go
  49. 16
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/doc.go
  50. 46
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/encode.go
  51. 67
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/pbutil/encode_test.go
  52. 4
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/testdata/README.THIRD_PARTY
  53. 4029
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/testdata/test.pb.go
  54. 540
    0
      vendor/github.com/matttproud/golang_protobuf_extensions/testdata/test.proto
  55. 201
    0
      vendor/github.com/prometheus/client_golang/LICENSE
  56. 28
    0
      vendor/github.com/prometheus/client_golang/NOTICE
  57. 53
    0
      vendor/github.com/prometheus/client_golang/prometheus/README.md
  58. 183
    0
      vendor/github.com/prometheus/client_golang/prometheus/benchmark_test.go
  59. 75
    0
      vendor/github.com/prometheus/client_golang/prometheus/collector.go
  60. 173
    0
      vendor/github.com/prometheus/client_golang/prometheus/counter.go
  61. 58
    0
      vendor/github.com/prometheus/client_golang/prometheus/counter_test.go
  62. 192
    0
      vendor/github.com/prometheus/client_golang/prometheus/desc.go
  63. 111
    0
      vendor/github.com/prometheus/client_golang/prometheus/doc.go
  64. 130
    0
      vendor/github.com/prometheus/client_golang/prometheus/example_clustermanager_test.go
  65. 87
    0
      vendor/github.com/prometheus/client_golang/prometheus/example_memstats_test.go
  66. 69
    0
      vendor/github.com/prometheus/client_golang/prometheus/example_selfcollector_test.go
  67. 640
    0
      vendor/github.com/prometheus/client_golang/prometheus/examples_test.go
  68. 119
    0
      vendor/github.com/prometheus/client_golang/prometheus/expvar.go
  69. 97
    0
      vendor/github.com/prometheus/client_golang/prometheus/expvar_test.go
  70. 29
    0
      vendor/github.com/prometheus/client_golang/prometheus/fnv.go
  71. 144
    0
      vendor/github.com/prometheus/client_golang/prometheus/gauge.go
  72. 182
    0
      vendor/github.com/prometheus/client_golang/prometheus/gauge_test.go
  73. 263
    0
      vendor/github.com/prometheus/client_golang/prometheus/go_collector.go
  74. 123
    0
      vendor/github.com/prometheus/client_golang/prometheus/go_collector_test.go
  75. 448
    0
      vendor/github.com/prometheus/client_golang/prometheus/histogram.go
  76. 326
    0
      vendor/github.com/prometheus/client_golang/prometheus/histogram_test.go
  77. 381
    0
      vendor/github.com/prometheus/client_golang/prometheus/http.go
  78. 121
    0
      vendor/github.com/prometheus/client_golang/prometheus/http_test.go
  79. 166
    0
      vendor/github.com/prometheus/client_golang/prometheus/metric.go
  80. 35
    0
      vendor/github.com/prometheus/client_golang/prometheus/metric_test.go
  81. 142
    0
      vendor/github.com/prometheus/client_golang/prometheus/process_collector.go
  82. 54
    0
      vendor/github.com/prometheus/client_golang/prometheus/process_collector_test.go
  83. 65
    0
      vendor/github.com/prometheus/client_golang/prometheus/push.go
  84. 739
    0
      vendor/github.com/prometheus/client_golang/prometheus/registry.go
  85. 535
    0
      vendor/github.com/prometheus/client_golang/prometheus/registry_test.go
  86. 538
    0
      vendor/github.com/prometheus/client_golang/prometheus/summary.go
  87. 347
    0
      vendor/github.com/prometheus/client_golang/prometheus/summary_test.go
  88. 142
    0
      vendor/github.com/prometheus/client_golang/prometheus/untyped.go
  89. 234
    0
      vendor/github.com/prometheus/client_golang/prometheus/value.go
  90. 249
    0
      vendor/github.com/prometheus/client_golang/prometheus/vec.go
  91. 88
    0
      vendor/github.com/prometheus/client_golang/prometheus/vec_test.go
  92. 201
    0
      vendor/github.com/prometheus/client_model/LICENSE
  93. 5
    0
      vendor/github.com/prometheus/client_model/NOTICE
  94. 364
    0
      vendor/github.com/prometheus/client_model/go/metrics.pb.go
  95. 201
    0
      vendor/github.com/prometheus/common/LICENSE
  96. 5
    0
      vendor/github.com/prometheus/common/NOTICE
  97. 171
    0
      vendor/github.com/prometheus/common/expfmt/bench_test.go
  98. 412
    0
      vendor/github.com/prometheus/common/expfmt/decode.go
  99. 367
    0
      vendor/github.com/prometheus/common/expfmt/decode_test.go
  100. 0
    0
      vendor/github.com/prometheus/common/expfmt/encode.go

+ 2
- 0
db.go View File

@@ -9,6 +9,7 @@ import (
9 9
 	"time"
10 10
 
11 11
 	verrors "mcquay.me/vain/errors"
12
+	"mcquay.me/vain/metrics"
12 13
 )
13 14
 
14 15
 // NewMemDB returns a functional MemDB.
@@ -206,6 +207,7 @@ func (m *MemDB) Sync() error {
206 207
 
207 208
 // flush writes to disk, but expects the user to have taken the lock.
208 209
 func (m *MemDB) flush(p string) error {
210
+	defer metrics.DBTime("flush")()
209 211
 	f, err := os.Create(p)
210 212
 	if err != nil {
211 213
 		return err

+ 66
- 0
metrics/metrics.go View File

@@ -0,0 +1,66 @@
1
+package metrics
2
+
3
+import (
4
+	"runtime"
5
+	"time"
6
+
7
+	"github.com/prometheus/client_golang/prometheus"
8
+)
9
+
10
+var (
11
+	// Errors tracks http status codes for problematic requests.
12
+	Errors = prometheus.NewCounterVec(
13
+		prometheus.CounterOpts{
14
+			Name: "errors_total",
15
+			Help: "Number of upstream errors",
16
+		},
17
+		[]string{"status"},
18
+	)
19
+
20
+	// Func tracks time spent in a function.
21
+	Func = prometheus.NewSummaryVec(
22
+		prometheus.SummaryOpts{
23
+			Name: "function_microseconds",
24
+			Help: "function timing.",
25
+		},
26
+		[]string{"route"},
27
+	)
28
+
29
+	// DB tracks timing of interactions with the file system.
30
+	DB = prometheus.NewSummaryVec(
31
+		prometheus.SummaryOpts{
32
+			Name: "db_microseconds",
33
+			Help: "db timing.",
34
+		},
35
+		[]string{"what"},
36
+	)
37
+)
38
+
39
+func init() {
40
+	prometheus.MustRegister(Errors)
41
+	prometheus.MustRegister(Func)
42
+	prometheus.MustRegister(DB)
43
+}
44
+
45
+// Time is a function that makes it simple to add one-line timings to function
46
+// calls.
47
+func Time() func() {
48
+	start := time.Now()
49
+	return func() {
50
+		elapsed := time.Since(start)
51
+		pc := make([]uintptr, 10)
52
+		runtime.Callers(2, pc)
53
+		f := runtime.FuncForPC(pc[0])
54
+
55
+		Func.WithLabelValues(f.Name()).Observe(float64(elapsed / time.Microsecond))
56
+	}
57
+}
58
+
59
+// DBTime makes it simple to add one-line timings to db interactions.
60
+func DBTime(name string) func() {
61
+	start := time.Now()
62
+	return func() {
63
+		elapsed := time.Since(start)
64
+		DB.WithLabelValues(name).Observe(float64(elapsed / time.Microsecond))
65
+	}
66
+}

+ 14
- 0
server.go View File

@@ -9,8 +9,10 @@ import (
9 9
 	"time"
10 10
 
11 11
 	"github.com/elazarl/go-bindata-assetfs"
12
+	"github.com/prometheus/client_golang/prometheus"
12 13
 
13 14
 	verrors "mcquay.me/vain/errors"
15
+	"mcquay.me/vain/metrics"
14 16
 	"mcquay.me/vain/static"
15 17
 )
16 18
 
@@ -52,6 +54,7 @@ func NewServer(sm *http.ServeMux, store Storer, m Mailer, static string, emailTi
52 54
 }
53 55
 
54 56
 func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
57
+	defer metrics.Time()()
55 58
 	if req.Method == "GET" {
56 59
 		req.ParseForm()
57 60
 		if _, ok := req.Form["go-get"]; !ok {
@@ -67,6 +70,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
67 70
 		} else {
68 71
 			p, err := s.db.Package(req.Host + req.URL.Path)
69 72
 			if err := verrors.ToHTTP(err); err != nil {
73
+				metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", err.Code, http.StatusText(err.Code))).Add(1)
70 74
 				http.Error(w, err.Message, err.Code)
71 75
 				return
72 76
 			}
@@ -92,6 +96,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
92 96
 	}
93 97
 
94 98
 	if err := verrors.ToHTTP(s.db.NSForToken(ns, Token(tok))); err != nil {
99
+		metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", err.Code, http.StatusText(err.Code))).Add(1)
95 100
 		http.Error(w, err.Message, err.Code)
96 101
 		return
97 102
 	}
@@ -104,6 +109,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
104 109
 		}
105 110
 		p := Package{}
106 111
 		if err := json.NewDecoder(req.Body).Decode(&p); err != nil {
112
+			metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", http.StatusBadRequest, http.StatusText(http.StatusBadRequest))).Add(1)
107 113
 			http.Error(w, fmt.Sprintf("unable to parse json from body: %v", err), http.StatusBadRequest)
108 114
 			return
109 115
 		}
@@ -145,6 +151,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) {
145 151
 }
146 152
 
147 153
 func (s *Server) register(w http.ResponseWriter, req *http.Request) {
154
+	defer metrics.Time()()
148 155
 	req.ParseForm()
149 156
 	email, ok := req.Form["email"]
150 157
 	if !ok || len(email) != 1 {
@@ -160,6 +167,7 @@ func (s *Server) register(w http.ResponseWriter, req *http.Request) {
160 167
 
161 168
 	tok, err := s.db.Register(Email(addr.Address))
162 169
 	if err := verrors.ToHTTP(err); err != nil {
170
+		metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", err.Code, http.StatusText(err.Code))).Add(1)
163 171
 		http.Error(w, err.Message, err.Code)
164 172
 		return
165 173
 	}
@@ -188,6 +196,7 @@ func (s *Server) register(w http.ResponseWriter, req *http.Request) {
188 196
 }
189 197
 
190 198
 func (s *Server) confirm(w http.ResponseWriter, req *http.Request) {
199
+	defer metrics.Time()()
191 200
 	tok := req.URL.Path[len(prefix["confirm"]):]
192 201
 	tok = strings.TrimRight(tok, "/")
193 202
 	if tok == "" {
@@ -196,6 +205,7 @@ func (s *Server) confirm(w http.ResponseWriter, req *http.Request) {
196 205
 	}
197 206
 	ttok, err := s.db.Confirm(Token(tok))
198 207
 	if err := verrors.ToHTTP(err); err != nil {
208
+		metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", err.Code, http.StatusText(err.Code))).Add(1)
199 209
 		http.Error(w, err.Message, err.Code)
200 210
 		return
201 211
 	}
@@ -203,6 +213,7 @@ func (s *Server) confirm(w http.ResponseWriter, req *http.Request) {
203 213
 }
204 214
 
205 215
 func (s *Server) forgot(w http.ResponseWriter, req *http.Request) {
216
+	defer metrics.Time()()
206 217
 	req.ParseForm()
207 218
 	email, ok := req.Form["email"]
208 219
 	if !ok || len(email) != 1 {
@@ -218,6 +229,7 @@ func (s *Server) forgot(w http.ResponseWriter, req *http.Request) {
218 229
 
219 230
 	tok, err := s.db.Forgot(Email(addr.Address), s.emailTimeout)
220 231
 	if err := verrors.ToHTTP(err); err != nil {
232
+		metrics.Errors.WithLabelValues(fmt.Sprintf("%d: %s", err.Code, http.StatusText(err.Code))).Add(1)
221 233
 		http.Error(w, err.Message, err.Code)
222 234
 		return
223 235
 	}
@@ -245,12 +257,14 @@ func (s *Server) forgot(w http.ResponseWriter, req *http.Request) {
245 257
 }
246 258
 
247 259
 func (s *Server) pkgs(w http.ResponseWriter, req *http.Request) {
260
+	defer metrics.Time()()
248 261
 	w.Header().Set("Content-type", "application/json")
249 262
 	json.NewEncoder(w).Encode(s.db.Pkgs())
250 263
 }
251 264
 
252 265
 func addRoutes(sm *http.ServeMux, s *Server) {
253 266
 	sm.Handle("/", s)
267
+	sm.Handle("/metrics", prometheus.Handler())
254 268
 
255 269
 	if s.static == "" {
256 270
 		sm.Handle(

+ 20
- 0
vendor/github.com/beorn7/perks/LICENSE View File

@@ -0,0 +1,20 @@
1
+Copyright (C) 2013 Blake Mizerany
2
+
3
+Permission is hereby granted, free of charge, to any person obtaining
4
+a copy of this software and associated documentation files (the
5
+"Software"), to deal in the Software without restriction, including
6
+without limitation the rights to use, copy, modify, merge, publish,
7
+distribute, sublicense, and/or sell copies of the Software, and to
8
+permit persons to whom the Software is furnished to do so, subject to
9
+the following conditions:
10
+
11
+The above copyright notice and this permission notice shall be
12
+included in all copies or substantial portions of the Software.
13
+
14
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

+ 63
- 0
vendor/github.com/beorn7/perks/quantile/bench_test.go View File

@@ -0,0 +1,63 @@
1
+package quantile
2
+
3
+import (
4
+	"testing"
5
+)
6
+
7
+func BenchmarkInsertTargeted(b *testing.B) {
8
+	b.ReportAllocs()
9
+
10
+	s := NewTargeted(Targets)
11
+	b.ResetTimer()
12
+	for i := float64(0); i < float64(b.N); i++ {
13
+		s.Insert(i)
14
+	}
15
+}
16
+
17
+func BenchmarkInsertTargetedSmallEpsilon(b *testing.B) {
18
+	s := NewTargeted(TargetsSmallEpsilon)
19
+	b.ResetTimer()
20
+	for i := float64(0); i < float64(b.N); i++ {
21
+		s.Insert(i)
22
+	}
23
+}
24
+
25
+func BenchmarkInsertBiased(b *testing.B) {
26
+	s := NewLowBiased(0.01)
27
+	b.ResetTimer()
28
+	for i := float64(0); i < float64(b.N); i++ {
29
+		s.Insert(i)
30
+	}
31
+}
32
+
33
+func BenchmarkInsertBiasedSmallEpsilon(b *testing.B) {
34
+	s := NewLowBiased(0.0001)
35
+	b.ResetTimer()
36
+	for i := float64(0); i < float64(b.N); i++ {
37
+		s.Insert(i)
38
+	}
39
+}
40
+
41
+func BenchmarkQuery(b *testing.B) {
42
+	s := NewTargeted(Targets)
43
+	for i := float64(0); i < 1e6; i++ {
44
+		s.Insert(i)
45
+	}
46
+	b.ResetTimer()
47
+	n := float64(b.N)
48
+	for i := float64(0); i < n; i++ {
49
+		s.Query(i / n)
50
+	}
51
+}
52
+
53
+func BenchmarkQuerySmallEpsilon(b *testing.B) {
54
+	s := NewTargeted(TargetsSmallEpsilon)
55
+	for i := float64(0); i < 1e6; i++ {
56
+		s.Insert(i)
57
+	}
58
+	b.ResetTimer()
59
+	n := float64(b.N)
60
+	for i := float64(0); i < n; i++ {
61
+		s.Query(i / n)
62
+	}
63
+}

+ 121
- 0
vendor/github.com/beorn7/perks/quantile/example_test.go View File

@@ -0,0 +1,121 @@
1
+// +build go1.1
2
+
3
+package quantile_test
4
+
5
+import (
6
+	"bufio"
7
+	"fmt"
8
+	"log"
9
+	"os"
10
+	"strconv"
11
+	"time"
12
+
13
+	"github.com/beorn7/perks/quantile"
14
+)
15
+
16
+func Example_simple() {
17
+	ch := make(chan float64)
18
+	go sendFloats(ch)
19
+
20
+	// Compute the 50th, 90th, and 99th percentile.
21
+	q := quantile.NewTargeted(map[float64]float64{
22
+		0.50: 0.005,
23
+		0.90: 0.001,
24
+		0.99: 0.0001,
25
+	})
26
+	for v := range ch {
27
+		q.Insert(v)
28
+	}
29
+
30
+	fmt.Println("perc50:", q.Query(0.50))
31
+	fmt.Println("perc90:", q.Query(0.90))
32
+	fmt.Println("perc99:", q.Query(0.99))
33
+	fmt.Println("count:", q.Count())
34
+	// Output:
35
+	// perc50: 5
36
+	// perc90: 16
37
+	// perc99: 223
38
+	// count: 2388
39
+}
40
+
41
+func Example_mergeMultipleStreams() {
42
+	// Scenario:
43
+	// We have multiple database shards. On each shard, there is a process
44
+	// collecting query response times from the database logs and inserting
45
+	// them into a Stream (created via NewTargeted(0.90)), much like the
46
+	// Simple example. These processes expose a network interface for us to
47
+	// ask them to serialize and send us the results of their
48
+	// Stream.Samples so we may Merge and Query them.
49
+	//
50
+	// NOTES:
51
+	// * These sample sets are small, allowing us to get them
52
+	// across the network much faster than sending the entire list of data
53
+	// points.
54
+	//
55
+	// * For this to work correctly, we must supply the same quantiles
56
+	// a priori the process collecting the samples supplied to NewTargeted,
57
+	// even if we do not plan to query them all here.
58
+	ch := make(chan quantile.Samples)
59
+	getDBQuerySamples(ch)
60
+	q := quantile.NewTargeted(map[float64]float64{0.90: 0.001})
61
+	for samples := range ch {
62
+		q.Merge(samples)
63
+	}
64
+	fmt.Println("perc90:", q.Query(0.90))
65
+}
66
+
67
+func Example_window() {
68
+	// Scenario: We want the 90th, 95th, and 99th percentiles for each
69
+	// minute.
70
+
71
+	ch := make(chan float64)
72
+	go sendStreamValues(ch)
73
+
74
+	tick := time.NewTicker(1 * time.Minute)
75
+	q := quantile.NewTargeted(map[float64]float64{
76
+		0.90: 0.001,
77
+		0.95: 0.0005,
78
+		0.99: 0.0001,
79
+	})
80
+	for {
81
+		select {
82
+		case t := <-tick.C:
83
+			flushToDB(t, q.Samples())
84
+			q.Reset()
85
+		case v := <-ch:
86
+			q.Insert(v)
87
+		}
88
+	}
89
+}
90
+
91
+func sendStreamValues(ch chan float64) {
92
+	// Use your imagination
93
+}
94
+
95
+func flushToDB(t time.Time, samples quantile.Samples) {
96
+	// Use your imagination
97
+}
98
+
99
+// This is a stub for the above example. In reality this would hit the remote
100
+// servers via http or something like it.
101
+func getDBQuerySamples(ch chan quantile.Samples) {}
102
+
103
+func sendFloats(ch chan<- float64) {
104
+	f, err := os.Open("exampledata.txt")
105
+	if err != nil {
106
+		log.Fatal(err)
107
+	}
108
+	sc := bufio.NewScanner(f)
109
+	for sc.Scan() {
110
+		b := sc.Bytes()
111
+		v, err := strconv.ParseFloat(string(b), 64)
112
+		if err != nil {
113
+			log.Fatal(err)
114
+		}
115
+		ch <- v
116
+	}
117
+	if sc.Err() != nil {
118
+		log.Fatal(sc.Err())
119
+	}
120
+	close(ch)
121
+}

+ 2388
- 0
vendor/github.com/beorn7/perks/quantile/exampledata.txt
File diff suppressed because it is too large
View File


+ 292
- 0
vendor/github.com/beorn7/perks/quantile/stream.go View File

@@ -0,0 +1,292 @@
1
+// Package quantile computes approximate quantiles over an unbounded data
2
+// stream within low memory and CPU bounds.
3
+//
4
+// A small amount of accuracy is traded to achieve the above properties.
5
+//
6
+// Multiple streams can be merged before calling Query to generate a single set
7
+// of results. This is meaningful when the streams represent the same type of
8
+// data. See Merge and Samples.
9
+//
10
+// For more detailed information about the algorithm used, see:
11
+//
12
+// Effective Computation of Biased Quantiles over Data Streams
13
+//
14
+// http://www.cs.rutgers.edu/~muthu/bquant.pdf
15
+package quantile
16
+
17
+import (
18
+	"math"
19
+	"sort"
20
+)
21
+
22
+// Sample holds an observed value and meta information for compression. JSON
23
+// tags have been added for convenience.
24
+type Sample struct {
25
+	Value float64 `json:",string"`
26
+	Width float64 `json:",string"`
27
+	Delta float64 `json:",string"`
28
+}
29
+
30
+// Samples represents a slice of samples. It implements sort.Interface.
31
+type Samples []Sample
32
+
33
+func (a Samples) Len() int           { return len(a) }
34
+func (a Samples) Less(i, j int) bool { return a[i].Value < a[j].Value }
35
+func (a Samples) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
36
+
37
+type invariant func(s *stream, r float64) float64
38
+
39
+// NewLowBiased returns an initialized Stream for low-biased quantiles
40
+// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
41
+// error guarantees can still be given even for the lower ranks of the data
42
+// distribution.
43
+//
44
+// The provided epsilon is a relative error, i.e. the true quantile of a value
45
+// returned by a query is guaranteed to be within (1±Epsilon)*Quantile.
46
+//
47
+// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
48
+// properties.
49
+func NewLowBiased(epsilon float64) *Stream {
50
+	ƒ := func(s *stream, r float64) float64 {
51
+		return 2 * epsilon * r
52
+	}
53
+	return newStream(ƒ)
54
+}
55
+
56
+// NewHighBiased returns an initialized Stream for high-biased quantiles
57
+// (e.g. 0.01, 0.1, 0.5) where the needed quantiles are not known a priori, but
58
+// error guarantees can still be given even for the higher ranks of the data
59
+// distribution.
60
+//
61
+// The provided epsilon is a relative error, i.e. the true quantile of a value
62
+// returned by a query is guaranteed to be within 1-(1±Epsilon)*(1-Quantile).
63
+//
64
+// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error
65
+// properties.
66
+func NewHighBiased(epsilon float64) *Stream {
67
+	ƒ := func(s *stream, r float64) float64 {
68
+		return 2 * epsilon * (s.n - r)
69
+	}
70
+	return newStream(ƒ)
71
+}
72
+
73
+// NewTargeted returns an initialized Stream concerned with a particular set of
74
+// quantile values that are supplied a priori. Knowing these a priori reduces
75
+// space and computation time. The targets map maps the desired quantiles to
76
+// their absolute errors, i.e. the true quantile of a value returned by a query
77
+// is guaranteed to be within (Quantile±Epsilon).
78
+//
79
+// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
80
+func NewTargeted(targets map[float64]float64) *Stream {
81
+	ƒ := func(s *stream, r float64) float64 {
82
+		var m = math.MaxFloat64
83
+		var f float64
84
+		for quantile, epsilon := range targets {
85
+			if quantile*s.n <= r {
86
+				f = (2 * epsilon * r) / quantile
87
+			} else {
88
+				f = (2 * epsilon * (s.n - r)) / (1 - quantile)
89
+			}
90
+			if f < m {
91
+				m = f
92
+			}
93
+		}
94
+		return m
95
+	}
96
+	return newStream(ƒ)
97
+}
98
+
99
+// Stream computes quantiles for a stream of float64s. It is not thread-safe by
100
+// design. Take care when using across multiple goroutines.
101
+type Stream struct {
102
+	*stream
103
+	b      Samples
104
+	sorted bool
105
+}
106
+
107
+func newStream(ƒ invariant) *Stream {
108
+	x := &stream{ƒ: ƒ}
109
+	return &Stream{x, make(Samples, 0, 500), true}
110
+}
111
+
112
+// Insert inserts v into the stream.
113
+func (s *Stream) Insert(v float64) {
114
+	s.insert(Sample{Value: v, Width: 1})
115
+}
116
+
117
+func (s *Stream) insert(sample Sample) {
118
+	s.b = append(s.b, sample)
119
+	s.sorted = false
120
+	if len(s.b) == cap(s.b) {
121
+		s.flush()
122
+	}
123
+}
124
+
125
+// Query returns the computed qth percentiles value. If s was created with
126
+// NewTargeted, and q is not in the set of quantiles provided a priori, Query
127
+// will return an unspecified result.
128
+func (s *Stream) Query(q float64) float64 {
129
+	if !s.flushed() {
130
+		// Fast path when there hasn't been enough data for a flush;
131
+		// this also yields better accuracy for small sets of data.
132
+		l := len(s.b)
133
+		if l == 0 {
134
+			return 0
135
+		}
136
+		i := int(float64(l) * q)
137
+		if i > 0 {
138
+			i -= 1
139
+		}
140
+		s.maybeSort()
141
+		return s.b[i].Value
142
+	}
143
+	s.flush()
144
+	return s.stream.query(q)
145
+}
146
+
147
+// Merge merges samples into the underlying streams samples. This is handy when
148
+// merging multiple streams from separate threads, database shards, etc.
149
+//
150
+// ATTENTION: This method is broken and does not yield correct results. The
151
+// underlying algorithm is not capable of merging streams correctly.
152
+func (s *Stream) Merge(samples Samples) {
153
+	sort.Sort(samples)
154
+	s.stream.merge(samples)
155
+}
156
+
157
+// Reset reinitializes and clears the list reusing the samples buffer memory.
158
+func (s *Stream) Reset() {
159
+	s.stream.reset()
160
+	s.b = s.b[:0]
161
+}
162
+
163
+// Samples returns stream samples held by s.
164
+func (s *Stream) Samples() Samples {
165
+	if !s.flushed() {
166
+		return s.b
167
+	}
168
+	s.flush()
169
+	return s.stream.samples()
170
+}
171
+
172
+// Count returns the total number of samples observed in the stream
173
+// since initialization.
174
+func (s *Stream) Count() int {
175
+	return len(s.b) + s.stream.count()
176
+}
177
+
178
+func (s *Stream) flush() {
179
+	s.maybeSort()
180
+	s.stream.merge(s.b)
181
+	s.b = s.b[:0]
182
+}
183
+
184
+func (s *Stream) maybeSort() {
185
+	if !s.sorted {
186
+		s.sorted = true
187
+		sort.Sort(s.b)
188
+	}
189
+}
190
+
191
+func (s *Stream) flushed() bool {
192
+	return len(s.stream.l) > 0
193
+}
194
+
195
+type stream struct {
196
+	n float64
197
+	l []Sample
198
+	ƒ invariant
199
+}
200
+
201
+func (s *stream) reset() {
202
+	s.l = s.l[:0]
203
+	s.n = 0
204
+}
205
+
206
+func (s *stream) insert(v float64) {
207
+	s.merge(Samples{{v, 1, 0}})
208
+}
209
+
210
+func (s *stream) merge(samples Samples) {
211
+	// TODO(beorn7): This tries to merge not only individual samples, but
212
+	// whole summaries. The paper doesn't mention merging summaries at
213
+	// all. Unittests show that the merging is inaccurate. Find out how to
214
+	// do merges properly.
215
+	var r float64
216
+	i := 0
217
+	for _, sample := range samples {
218
+		for ; i < len(s.l); i++ {
219
+			c := s.l[i]
220
+			if c.Value > sample.Value {
221
+				// Insert at position i.
222
+				s.l = append(s.l, Sample{})
223
+				copy(s.l[i+1:], s.l[i:])
224
+				s.l[i] = Sample{
225
+					sample.Value,
226
+					sample.Width,
227
+					math.Max(sample.Delta, math.Floor(s.ƒ(s, r))-1),
228
+					// TODO(beorn7): How to calculate delta correctly?
229
+				}
230
+				i++
231
+				goto inserted
232
+			}
233
+			r += c.Width
234
+		}
235
+		s.l = append(s.l, Sample{sample.Value, sample.Width, 0})
236
+		i++
237
+	inserted:
238
+		s.n += sample.Width
239
+		r += sample.Width
240
+	}
241
+	s.compress()
242
+}
243
+
244
+func (s *stream) count() int {
245
+	return int(s.n)
246
+}
247
+
248
+func (s *stream) query(q float64) float64 {
249
+	t := math.Ceil(q * s.n)
250
+	t += math.Ceil(s.ƒ(s, t) / 2)
251
+	p := s.l[0]
252
+	var r float64
253
+	for _, c := range s.l[1:] {
254
+		r += p.Width
255
+		if r+c.Width+c.Delta > t {
256
+			return p.Value
257
+		}
258
+		p = c
259
+	}
260
+	return p.Value
261
+}
262
+
263
+func (s *stream) compress() {
264
+	if len(s.l) < 2 {
265
+		return
266
+	}
267
+	x := s.l[len(s.l)-1]
268
+	xi := len(s.l) - 1
269
+	r := s.n - 1 - x.Width
270
+
271
+	for i := len(s.l) - 2; i >= 0; i-- {
272
+		c := s.l[i]
273
+		if c.Width+x.Width+x.Delta <= s.ƒ(s, r) {
274
+			x.Width += c.Width
275
+			s.l[xi] = x
276
+			// Remove element at i.
277
+			copy(s.l[i:], s.l[i+1:])
278
+			s.l = s.l[:len(s.l)-1]
279
+			xi -= 1
280
+		} else {
281
+			x = c
282
+			xi = i
283
+		}
284
+		r -= c.Width
285
+	}
286
+}
287
+
288
+func (s *stream) samples() Samples {
289
+	samples := make(Samples, len(s.l))
290
+	copy(samples, s.l)
291
+	return samples
292
+}

+ 188
- 0
vendor/github.com/beorn7/perks/quantile/stream_test.go View File

@@ -0,0 +1,188 @@
1
+package quantile
2
+
3
+import (
4
+	"math"
5
+	"math/rand"
6
+	"sort"
7
+	"testing"
8
+)
9
+
10
+var (
11
+	Targets = map[float64]float64{
12
+		0.01: 0.001,
13
+		0.10: 0.01,
14
+		0.50: 0.05,
15
+		0.90: 0.01,
16
+		0.99: 0.001,
17
+	}
18
+	TargetsSmallEpsilon = map[float64]float64{
19
+		0.01: 0.0001,
20
+		0.10: 0.001,
21
+		0.50: 0.005,
22
+		0.90: 0.001,
23
+		0.99: 0.0001,
24
+	}
25
+	LowQuantiles  = []float64{0.01, 0.1, 0.5}
26
+	HighQuantiles = []float64{0.99, 0.9, 0.5}
27
+)
28
+
29
+const RelativeEpsilon = 0.01
30
+
31
+func verifyPercsWithAbsoluteEpsilon(t *testing.T, a []float64, s *Stream) {
32
+	sort.Float64s(a)
33
+	for quantile, epsilon := range Targets {
34
+		n := float64(len(a))
35
+		k := int(quantile * n)
36
+		lower := int((quantile - epsilon) * n)
37
+		if lower < 1 {
38
+			lower = 1
39
+		}
40
+		upper := int(math.Ceil((quantile + epsilon) * n))
41
+		if upper > len(a) {
42
+			upper = len(a)
43
+		}
44
+		w, min, max := a[k-1], a[lower-1], a[upper-1]
45
+		if g := s.Query(quantile); g < min || g > max {
46
+			t.Errorf("q=%f: want %v [%f,%f], got %v", quantile, w, min, max, g)
47
+		}
48
+	}
49
+}
50
+
51
+func verifyLowPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
52
+	sort.Float64s(a)
53
+	for _, qu := range LowQuantiles {
54
+		n := float64(len(a))
55
+		k := int(qu * n)
56
+
57
+		lowerRank := int((1 - RelativeEpsilon) * qu * n)
58
+		upperRank := int(math.Ceil((1 + RelativeEpsilon) * qu * n))
59
+		w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
60
+		if g := s.Query(qu); g < min || g > max {
61
+			t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
62
+		}
63
+	}
64
+}
65
+
66
+func verifyHighPercsWithRelativeEpsilon(t *testing.T, a []float64, s *Stream) {
67
+	sort.Float64s(a)
68
+	for _, qu := range HighQuantiles {
69
+		n := float64(len(a))
70
+		k := int(qu * n)
71
+
72
+		lowerRank := int((1 - (1+RelativeEpsilon)*(1-qu)) * n)
73
+		upperRank := int(math.Ceil((1 - (1-RelativeEpsilon)*(1-qu)) * n))
74
+		w, min, max := a[k-1], a[lowerRank-1], a[upperRank-1]
75
+		if g := s.Query(qu); g < min || g > max {
76
+			t.Errorf("q=%f: want %v [%f,%f], got %v", qu, w, min, max, g)
77
+		}
78
+	}
79
+}
80
+
81
+func populateStream(s *Stream) []float64 {
82
+	a := make([]float64, 0, 1e5+100)
83
+	for i := 0; i < cap(a); i++ {
84
+		v := rand.NormFloat64()
85
+		// Add 5% asymmetric outliers.
86
+		if i%20 == 0 {
87
+			v = v*v + 1
88
+		}
89
+		s.Insert(v)
90
+		a = append(a, v)
91
+	}
92
+	return a
93
+}
94
+
95
+func TestTargetedQuery(t *testing.T) {
96
+	rand.Seed(42)
97
+	s := NewTargeted(Targets)
98
+	a := populateStream(s)
99
+	verifyPercsWithAbsoluteEpsilon(t, a, s)
100
+}
101
+
102
+func TestLowBiasedQuery(t *testing.T) {
103
+	rand.Seed(42)
104
+	s := NewLowBiased(RelativeEpsilon)
105
+	a := populateStream(s)
106
+	verifyLowPercsWithRelativeEpsilon(t, a, s)
107
+}
108
+
109
+func TestHighBiasedQuery(t *testing.T) {
110
+	rand.Seed(42)
111
+	s := NewHighBiased(RelativeEpsilon)
112
+	a := populateStream(s)
113
+	verifyHighPercsWithRelativeEpsilon(t, a, s)
114
+}
115
+
116
+// BrokenTestTargetedMerge is broken, see Merge doc comment.
117
+func BrokenTestTargetedMerge(t *testing.T) {
118
+	rand.Seed(42)
119
+	s1 := NewTargeted(Targets)
120
+	s2 := NewTargeted(Targets)
121
+	a := populateStream(s1)
122
+	a = append(a, populateStream(s2)...)
123
+	s1.Merge(s2.Samples())
124
+	verifyPercsWithAbsoluteEpsilon(t, a, s1)
125
+}
126
+
127
+// BrokenTestLowBiasedMerge is broken, see Merge doc comment.
128
+func BrokenTestLowBiasedMerge(t *testing.T) {
129
+	rand.Seed(42)
130
+	s1 := NewLowBiased(RelativeEpsilon)
131
+	s2 := NewLowBiased(RelativeEpsilon)
132
+	a := populateStream(s1)
133
+	a = append(a, populateStream(s2)...)
134
+	s1.Merge(s2.Samples())
135
+	verifyLowPercsWithRelativeEpsilon(t, a, s2)
136
+}
137
+
138
+// BrokenTestHighBiasedMerge is broken, see Merge doc comment.
139
+func BrokenTestHighBiasedMerge(t *testing.T) {
140
+	rand.Seed(42)
141
+	s1 := NewHighBiased(RelativeEpsilon)
142
+	s2 := NewHighBiased(RelativeEpsilon)
143
+	a := populateStream(s1)
144
+	a = append(a, populateStream(s2)...)
145
+	s1.Merge(s2.Samples())
146
+	verifyHighPercsWithRelativeEpsilon(t, a, s2)
147
+}
148
+
149
+func TestUncompressed(t *testing.T) {
150
+	q := NewTargeted(Targets)
151
+	for i := 100; i > 0; i-- {
152
+		q.Insert(float64(i))
153
+	}
154
+	if g := q.Count(); g != 100 {
155
+		t.Errorf("want count 100, got %d", g)
156
+	}
157
+	// Before compression, Query should have 100% accuracy.
158
+	for quantile := range Targets {
159
+		w := quantile * 100
160
+		if g := q.Query(quantile); g != w {
161
+			t.Errorf("want %f, got %f", w, g)
162
+		}
163
+	}
164
+}
165
+
166
+func TestUncompressedSamples(t *testing.T) {
167
+	q := NewTargeted(map[float64]float64{0.99: 0.001})
168
+	for i := 1; i <= 100; i++ {
169
+		q.Insert(float64(i))
170
+	}
171
+	if g := q.Samples().Len(); g != 100 {
172
+		t.Errorf("want count 100, got %d", g)
173
+	}
174
+}
175
+
176
+func TestUncompressedOne(t *testing.T) {
177
+	q := NewTargeted(map[float64]float64{0.99: 0.01})
178
+	q.Insert(3.14)
179
+	if g := q.Query(0.90); g != 3.14 {
180
+		t.Error("want PI, got", g)
181
+	}
182
+}
183
+
184
+func TestDefaults(t *testing.T) {
185
+	if g := NewTargeted(map[float64]float64{0.99: 0.001}).Query(0.99); g != 0 {
186
+		t.Errorf("want 0, got %f", g)
187
+	}
188
+}

+ 31
- 0
vendor/github.com/golang/protobuf/LICENSE View File

@@ -0,0 +1,31 @@
1
+Go support for Protocol Buffers - Google's data interchange format
2
+
3
+Copyright 2010 The Go Authors.  All rights reserved.
4
+https://github.com/golang/protobuf
5
+
6
+Redistribution and use in source and binary forms, with or without
7
+modification, are permitted provided that the following conditions are
8
+met:
9
+
10
+    * Redistributions of source code must retain the above copyright
11
+notice, this list of conditions and the following disclaimer.
12
+    * Redistributions in binary form must reproduce the above
13
+copyright notice, this list of conditions and the following disclaimer
14
+in the documentation and/or other materials provided with the
15
+distribution.
16
+    * Neither the name of Google Inc. nor the names of its
17
+contributors may be used to endorse or promote products derived from
18
+this software without specific prior written permission.
19
+
20
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
+

+ 43
- 0
vendor/github.com/golang/protobuf/proto/Makefile View File

@@ -0,0 +1,43 @@
1
+# Go support for Protocol Buffers - Google's data interchange format
2
+#
3
+# Copyright 2010 The Go Authors.  All rights reserved.
4
+# https://github.com/golang/protobuf
5
+#
6
+# Redistribution and use in source and binary forms, with or without
7
+# modification, are permitted provided that the following conditions are
8
+# met:
9
+#
10
+#     * Redistributions of source code must retain the above copyright
11
+# notice, this list of conditions and the following disclaimer.
12
+#     * Redistributions in binary form must reproduce the above
13
+# copyright notice, this list of conditions and the following disclaimer
14
+# in the documentation and/or other materials provided with the
15
+# distribution.
16
+#     * Neither the name of Google Inc. nor the names of its
17
+# contributors may be used to endorse or promote products derived from
18
+# this software without specific prior written permission.
19
+#
20
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21
+# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22
+# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23
+# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24
+# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25
+# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26
+# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30
+# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
+
32
+install:
33
+	go install
34
+
35
+test: install generate-test-pbs
36
+	go test
37
+
38
+
39
+generate-test-pbs:
40
+	make install
41
+	make -C testdata
42
+	protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata,Mgoogle/protobuf/any.proto=github.com/golang/protobuf/ptypes/any:. proto3_proto/proto3.proto
43
+	make

+ 2212
- 0
vendor/github.com/golang/protobuf/proto/all_test.go
File diff suppressed because it is too large
View File


+ 272
- 0
vendor/github.com/golang/protobuf/proto/any_test.go View File

@@ -0,0 +1,272 @@
1
+// Go support for Protocol Buffers - Google's data interchange format
2
+//
3
+// Copyright 2016 The Go Authors.  All rights reserved.
4
+// https://github.com/golang/protobuf
5
+//
6
+// Redistribution and use in source and binary forms, with or without
7
+// modification, are permitted provided that the following conditions are
8
+// met:
9
+//
10
+//     * Redistributions of source code must retain the above copyright
11
+// notice, this list of conditions and the following disclaimer.
12
+//     * Redistributions in binary form must reproduce the above
13
+// copyright notice, this list of conditions and the following disclaimer
14
+// in the documentation and/or other materials provided with the
15
+// distribution.
16
+//     * Neither the name of Google Inc. nor the names of its
17
+// contributors may be used to endorse or promote products derived from
18
+// this software without specific prior written permission.
19
+//
20
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
+
32
+package proto_test
33
+
34
+import (
35
+	"strings"
36
+	"testing"
37
+
38
+	"github.com/golang/protobuf/proto"
39
+
40
+	pb "github.com/golang/protobuf/proto/proto3_proto"
41
+	testpb "github.com/golang/protobuf/proto/testdata"
42
+	anypb "github.com/golang/protobuf/ptypes/any"
43
+)
44
+
45
+var (
46
+	expandedMarshaler        = proto.TextMarshaler{ExpandAny: true}
47
+	expandedCompactMarshaler = proto.TextMarshaler{Compact: true, ExpandAny: true}
48
+)
49
+
50
+// anyEqual reports whether two messages which may be google.protobuf.Any or may
51
+// contain google.protobuf.Any fields are equal. We can't use proto.Equal for
52
+// comparison, because semantically equivalent messages may be marshaled to
53
+// binary in different tag order. Instead, trust that TextMarshaler with
54
+// ExpandAny option works and compare the text marshaling results.
55
+func anyEqual(got, want proto.Message) bool {
56
+	// if messages are proto.Equal, no need to marshal.
57
+	if proto.Equal(got, want) {
58
+		return true
59
+	}
60
+	g := expandedMarshaler.Text(got)
61
+	w := expandedMarshaler.Text(want)
62
+	return g == w
63
+}
64
+
65
+type golden struct {
66
+	m    proto.Message
67
+	t, c string
68
+}
69
+
70
+var goldenMessages = makeGolden()
71
+
72
+func makeGolden() []golden {
73
+	nested := &pb.Nested{Bunny: "Monty"}
74
+	nb, err := proto.Marshal(nested)
75
+	if err != nil {
76
+		panic(err)
77
+	}
78
+	m1 := &pb.Message{
79
+		Name:        "David",
80
+		ResultCount: 47,
81
+		Anything:    &anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(nested), Value: nb},
82
+	}
83
+	m2 := &pb.Message{
84
+		Name:        "David",
85
+		ResultCount: 47,
86
+		Anything:    &anypb.Any{TypeUrl: "http://[::1]/type.googleapis.com/" + proto.MessageName(nested), Value: nb},
87
+	}
88
+	m3 := &pb.Message{
89
+		Name:        "David",
90
+		ResultCount: 47,
91
+		Anything:    &anypb.Any{TypeUrl: `type.googleapis.com/"/` + proto.MessageName(nested), Value: nb},
92
+	}
93
+	m4 := &pb.Message{
94
+		Name:        "David",
95
+		ResultCount: 47,
96
+		Anything:    &anypb.Any{TypeUrl: "type.googleapis.com/a/path/" + proto.MessageName(nested), Value: nb},
97
+	}
98
+	m5 := &anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(nested), Value: nb}
99
+
100
+	any1 := &testpb.MyMessage{Count: proto.Int32(47), Name: proto.String("David")}
101
+	proto.SetExtension(any1, testpb.E_Ext_More, &testpb.Ext{Data: proto.String("foo")})
102
+	proto.SetExtension(any1, testpb.E_Ext_Text, proto.String("bar"))
103
+	any1b, err := proto.Marshal(any1)
104
+	if err != nil {
105
+		panic(err)
106
+	}
107
+	any2 := &testpb.MyMessage{Count: proto.Int32(42), Bikeshed: testpb.MyMessage_GREEN.Enum(), RepBytes: [][]byte{[]byte("roboto")}}
108
+	proto.SetExtension(any2, testpb.E_Ext_More, &testpb.Ext{Data: proto.String("baz")})
109
+	any2b, err := proto.Marshal(any2)
110
+	if err != nil {
111
+		panic(err)
112
+	}
113
+	m6 := &pb.Message{
114
+		Name:        "David",
115
+		ResultCount: 47,
116
+		Anything:    &anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(any1), Value: any1b},
117
+		ManyThings: []*anypb.Any{
118
+			&anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(any2), Value: any2b},
119
+			&anypb.Any{TypeUrl: "type.googleapis.com/" + proto.MessageName(any1), Value: any1b},
120
+		},
121
+	}
122
+
123
+	const (
124
+		m1Golden = `
125
+name: "David"
126
+result_count: 47
127
+anything: <
128
+  [type.googleapis.com/proto3_proto.Nested]: <
129
+    bunny: "Monty"
130
+  >
131
+>
132
+`
133
+		m2Golden = `
134
+name: "David"
135
+result_count: 47
136
+anything: <
137
+  ["http://[::1]/type.googleapis.com/proto3_proto.Nested"]: <
138
+    bunny: "Monty"
139
+  >
140
+>
141
+`
142
+		m3Golden = `
143
+name: "David"
144
+result_count: 47
145
+anything: <
146
+  ["type.googleapis.com/\"/proto3_proto.Nested"]: <
147
+    bunny: "Monty"
148
+  >
149
+>
150
+`
151
+		m4Golden = `
152
+name: "David"
153
+result_count: 47
154
+anything: <
155
+  [type.googleapis.com/a/path/proto3_proto.Nested]: <
156
+    bunny: "Monty"
157
+  >
158
+>
159
+`
160
+		m5Golden = `
161
+[type.googleapis.com/proto3_proto.Nested]: <
162
+  bunny: "Monty"
163
+>
164
+`
165
+		m6Golden = `
166
+name: "David"
167
+result_count: 47
168
+anything: <
169
+  [type.googleapis.com/testdata.MyMessage]: <
170
+    count: 47
171
+    name: "David"
172
+    [testdata.Ext.more]: <
173
+      data: "foo"
174
+    >
175
+    [testdata.Ext.text]: "bar"
176
+  >
177
+>
178
+many_things: <
179
+  [type.googleapis.com/testdata.MyMessage]: <
180
+    count: 42
181
+    bikeshed: GREEN
182
+    rep_bytes: "roboto"
183
+    [testdata.Ext.more]: <
184
+      data: "baz"
185
+    >
186
+  >
187
+>
188
+many_things: <
189
+  [type.googleapis.com/testdata.MyMessage]: <
190
+    count: 47
191
+    name: "David"
192
+    [testdata.Ext.more]: <
193
+      data: "foo"
194
+    >
195
+    [testdata.Ext.text]: "bar"
196
+  >
197
+>
198
+`
199
+	)
200
+	return []golden{
201
+		{m1, strings.TrimSpace(m1Golden) + "\n", strings.TrimSpace(compact(m1Golden)) + " "},
202
+		{m2, strings.TrimSpace(m2Golden) + "\n", strings.TrimSpace(compact(m2Golden)) + " "},
203
+		{m3, strings.TrimSpace(m3Golden) + "\n", strings.TrimSpace(compact(m3Golden)) + " "},
204
+		{m4, strings.TrimSpace(m4Golden) + "\n", strings.TrimSpace(compact(m4Golden)) + " "},
205
+		{m5, strings.TrimSpace(m5Golden) + "\n", strings.TrimSpace(compact(m5Golden)) + " "},
206
+		{m6, strings.TrimSpace(m6Golden) + "\n", strings.TrimSpace(compact(m6Golden)) + " "},
207
+	}
208
+}
209
+
210
+func TestMarshalGolden(t *testing.T) {
211
+	for _, tt := range goldenMessages {
212
+		if got, want := expandedMarshaler.Text(tt.m), tt.t; got != want {
213
+			t.Errorf("message %v: got:\n%s\nwant:\n%s", tt.m, got, want)
214
+		}
215
+		if got, want := expandedCompactMarshaler.Text(tt.m), tt.c; got != want {
216
+			t.Errorf("message %v: got:\n`%s`\nwant:\n`%s`", tt.m, got, want)
217
+		}
218
+	}
219
+}
220
+
221
+func TestUnmarshalGolden(t *testing.T) {
222
+	for _, tt := range goldenMessages {
223
+		want := tt.m
224
+		got := proto.Clone(tt.m)
225
+		got.Reset()
226
+		if err := proto.UnmarshalText(tt.t, got); err != nil {
227
+			t.Errorf("failed to unmarshal\n%s\nerror: %v", tt.t, err)
228
+		}
229
+		if !anyEqual(got, want) {
230
+			t.Errorf("message:\n%s\ngot:\n%s\nwant:\n%s", tt.t, got, want)
231
+		}
232
+		got.Reset()
233
+		if err := proto.UnmarshalText(tt.c, got); err != nil {
234
+			t.Errorf("failed to unmarshal\n%s\nerror: %v", tt.c, err)
235
+		}
236
+		if !anyEqual(got, want) {
237
+			t.Errorf("message:\n%s\ngot:\n%s\nwant:\n%s", tt.c, got, want)
238
+		}
239
+	}
240
+}
241
+
242
+func TestMarsahlUnknownAny(t *testing.T) {
243
+	m := &pb.Message{
244
+		Anything: &anypb.Any{
245
+			TypeUrl: "foo",
246
+			Value:   []byte("bar"),
247
+		},
248
+	}
249
+	want := `anything: <
250
+  type_url: "foo"
251
+  value: "bar"
252
+>
253
+`
254
+	got := expandedMarshaler.Text(m)
255
+	if got != want {
256
+		t.Errorf("got\n`%s`\nwant\n`%s`", got, want)
257
+	}
258
+}
259
+
260
+func TestAmbiguousAny(t *testing.T) {
261
+	pb := &anypb.Any{}
262
+	err := proto.UnmarshalText(`
263
+	[type.googleapis.com/proto3_proto.Nested]: <
264
+	  bunny: "Monty"
265
+	>
266
+	type_url: "ttt/proto3_proto.Nested"
267
+	`, pb)
268
+	t.Logf("result: %v (error: %v)", expandedMarshaler.Text(pb), err)
269
+	if err != nil {
270
+		t.Errorf("failed to parse ambiguous Any message: %v", err)
271
+	}
272
+}

+ 223
- 0
vendor/github.com/golang/protobuf/proto/clone.go View File

@@ -0,0 +1,223 @@
1
+// Go support for Protocol Buffers - Google's data interchange format
2
+//
3
+// Copyright 2011 The Go Authors.  All rights reserved.
4
+// https://github.com/golang/protobuf
5
+//
6
+// Redistribution and use in source and binary forms, with or without
7
+// modification, are permitted provided that the following conditions are
8
+// met:
9
+//
10
+//     * Redistributions of source code must retain the above copyright
11
+// notice, this list of conditions and the following disclaimer.
12
+//     * Redistributions in binary form must reproduce the above
13
+// copyright notice, this list of conditions and the following disclaimer
14
+// in the documentation and/or other materials provided with the
15
+// distribution.
16
+//     * Neither the name of Google Inc. nor the names of its
17
+// contributors may be used to endorse or promote products derived from
18
+// this software without specific prior written permission.
19
+//
20
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
+
32
+// Protocol buffer deep copy and merge.
33
+// TODO: RawMessage.
34
+
35
+package proto
36
+
37
+import (
38
+	"log"
39
+	"reflect"
40
+	"strings"
41
+)
42
+
43
+// Clone returns a deep copy of a protocol buffer.
44
+func Clone(pb Message) Message {
45
+	in := reflect.ValueOf(pb)
46
+	if in.IsNil() {
47
+		return pb
48
+	}
49
+
50
+	out := reflect.New(in.Type().Elem())
51
+	// out is empty so a merge is a deep copy.
52
+	mergeStruct(out.Elem(), in.Elem())
53
+	return out.Interface().(Message)
54
+}
55
+
56
+// Merge merges src into dst.
57
+// Required and optional fields that are set in src will be set to that value in dst.
58
+// Elements of repeated fields will be appended.
59
+// Merge panics if src and dst are not the same type, or if dst is nil.
60
+func Merge(dst, src Message) {
61
+	in := reflect.ValueOf(src)
62
+	out := reflect.ValueOf(dst)
63
+	if out.IsNil() {
64
+		panic("proto: nil destination")
65
+	}
66
+	if in.Type() != out.Type() {
67
+		// Explicit test prior to mergeStruct so that mistyped nils will fail
68
+		panic("proto: type mismatch")
69
+	}
70
+	if in.IsNil() {
71
+		// Merging nil into non-nil is a quiet no-op
72
+		return
73
+	}
74
+	mergeStruct(out.Elem(), in.Elem())
75
+}
76
+
77
+func mergeStruct(out, in reflect.Value) {
78
+	sprop := GetProperties(in.Type())
79
+	for i := 0; i < in.NumField(); i++ {
80
+		f := in.Type().Field(i)
81
+		if strings.HasPrefix(f.Name, "XXX_") {
82
+			continue
83
+		}
84
+		mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i])
85
+	}
86
+
87
+	if emIn, ok := in.Addr().Interface().(extendableProto); ok {
88
+		emOut := out.Addr().Interface().(extendableProto)
89
+		mergeExtension(emOut.ExtensionMap(), emIn.ExtensionMap())
90
+	}
91
+
92
+	uf := in.FieldByName("XXX_unrecognized")
93
+	if !uf.IsValid() {
94
+		return
95
+	}
96
+	uin := uf.Bytes()
97
+	if len(uin) > 0 {
98
+		out.FieldByName("XXX_unrecognized").SetBytes(append([]byte(nil), uin...))
99
+	}
100
+}
101
+
102
+// mergeAny performs a merge between two values of the same type.
103
+// viaPtr indicates whether the values were indirected through a pointer (implying proto2).
104
+// prop is set if this is a struct field (it may be nil).
105
+func mergeAny(out, in reflect.Value, viaPtr bool, prop *Properties) {
106
+	if in.Type() == protoMessageType {
107
+		if !in.IsNil() {
108
+			if out.IsNil() {
109
+				out.Set(reflect.ValueOf(Clone(in.Interface().(Message))))
110
+			} else {
111
+				Merge(out.Interface().(Message), in.Interface().(Message))
112
+			}
113
+		}
114
+		return
115
+	}
116
+	switch in.Kind() {
117
+	case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int32, reflect.Int64,
118
+		reflect.String, reflect.Uint32, reflect.Uint64:
119
+		if !viaPtr && isProto3Zero(in) {
120
+			return
121
+		}
122
+		out.Set(in)
123
+	case reflect.Interface:
124
+		// Probably a oneof field; copy non-nil values.
125
+		if in.IsNil() {
126
+			return
127
+		}
128
+		// Allocate destination if it is not set, or set to a different type.
129
+		// Otherwise we will merge as normal.
130
+		if out.IsNil() || out.Elem().Type() != in.Elem().Type() {
131
+			out.Set(reflect.New(in.Elem().Elem().Type())) // interface -> *T -> T -> new(T)
132
+		}
133
+		mergeAny(out.Elem(), in.Elem(), false, nil)
134
+	case reflect.Map:
135
+		if in.Len() == 0 {
136
+			return
137
+		}
138
+		if out.IsNil() {
139
+			out.Set(reflect.MakeMap(in.Type()))
140
+		}
141
+		// For maps with value types of *T or []byte we need to deep copy each value.
142
+		elemKind := in.Type().Elem().Kind()
143
+		for _, key := range in.MapKeys() {
144
+			var val reflect.Value
145
+			switch elemKind {
146
+			case reflect.Ptr:
147
+				val = reflect.New(in.Type().Elem().Elem())
148
+				mergeAny(val, in.MapIndex(key), false, nil)
149
+			case reflect.Slice:
150
+				val = in.MapIndex(key)
151
+				val = reflect.ValueOf(append([]byte{}, val.Bytes()...))
152
+			default:
153
+				val = in.MapIndex(key)
154
+			}
155
+			out.SetMapIndex(key, val)
156
+		}
157
+	case reflect.Ptr:
158
+		if in.IsNil() {
159
+			return
160
+		}
161
+		if out.IsNil() {
162
+			out.Set(reflect.New(in.Elem().Type()))
163
+		}
164
+		mergeAny(out.Elem(), in.Elem(), true, nil)
165
+	case reflect.Slice:
166
+		if in.IsNil() {
167
+			return
168
+		}
169
+		if in.Type().Elem().Kind() == reflect.Uint8 {
170
+			// []byte is a scalar bytes field, not a repeated field.
171
+
172
+			// Edge case: if this is in a proto3 message, a zero length
173
+			// bytes field is considered the zero value, and should not
174
+			// be merged.
175
+			if prop != nil && prop.proto3 && in.Len() == 0 {
176
+				return
177
+			}
178
+
179
+			// Make a deep copy.
180
+			// Append to []byte{} instead of []byte(nil) so that we never end up
181
+			// with a nil result.
182
+			out.SetBytes(append([]byte{}, in.Bytes()...))
183
+			return
184
+		}
185
+		n := in.Len()
186
+		if out.IsNil() {
187
+			out.Set(reflect.MakeSlice(in.Type(), 0, n))
188
+		}
189
+		switch in.Type().Elem().Kind() {
190
+		case reflect.Bool, reflect.Float32, reflect.Float64, reflect.Int32, reflect.Int64,
191
+			reflect.String, reflect.Uint32, reflect.Uint64:
192
+			out.Set(reflect.AppendSlice(out, in))
193
+		default:
194
+			for i := 0; i < n; i++ {
195
+				x := reflect.Indirect(reflect.New(in.Type().Elem()))
196
+				mergeAny(x, in.Index(i), false, nil)
197
+				out.Set(reflect.Append(out, x))
198
+			}
199
+		}
200
+	case reflect.Struct:
201
+		mergeStruct(out, in)
202
+	default:
203
+		// unknown type, so not a protocol buffer
204
+		log.Printf("proto: don't know how to copy %v", in)
205
+	}
206
+}
207
+
208
+func mergeExtension(out, in map[int32]Extension) {
209
+	for extNum, eIn := range in {
210
+		eOut := Extension{desc: eIn.desc}
211
+		if eIn.value != nil {
212
+			v := reflect.New(reflect.TypeOf(eIn.value)).Elem()
213
+			mergeAny(v, reflect.ValueOf(eIn.value), false, nil)
214
+			eOut.value = v.Interface()
215
+		}
216
+		if eIn.enc != nil {
217
+			eOut.enc = make([]byte, len(eIn.enc))
218
+			copy(eOut.enc, eIn.enc)
219
+		}
220
+
221
+		out[extNum] = eOut
222
+	}
223
+}

+ 267
- 0
vendor/github.com/golang/protobuf/proto/clone_test.go View File

@@ -0,0 +1,267 @@
1
+// Go support for Protocol Buffers - Google's data interchange format
2
+//
3
+// Copyright 2011 The Go Authors.  All rights reserved.
4
+// https://github.com/golang/protobuf
5
+//
6
+// Redistribution and use in source and binary forms, with or without
7
+// modification, are permitted provided that the following conditions are
8
+// met:
9
+//
10
+//     * Redistributions of source code must retain the above copyright
11
+// notice, this list of conditions and the following disclaimer.
12
+//     * Redistributions in binary form must reproduce the above
13
+// copyright notice, this list of conditions and the following disclaimer
14
+// in the documentation and/or other materials provided with the
15
+// distribution.
16
+//     * Neither the name of Google Inc. nor the names of its
17
+// contributors may be used to endorse or promote products derived from
18
+// this software without specific prior written permission.
19
+//
20
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
+
32
+package proto_test
33
+
34
+import (
35
+	"testing"
36
+
37
+	"github.com/golang/protobuf/proto"
38
+
39
+	proto3pb "github.com/golang/protobuf/proto/proto3_proto"
40
+	pb "github.com/golang/protobuf/proto/testdata"
41
+)
42
+
43
+var cloneTestMessage = &pb.MyMessage{
44
+	Count: proto.Int32(42),
45
+	Name:  proto.String("Dave"),
46
+	Pet:   []string{"bunny", "kitty", "horsey"},
47
+	Inner: &pb.InnerMessage{
48
+		Host:      proto.String("niles"),
49
+		Port:      proto.Int32(9099),
50
+		Connected: proto.Bool(true),
51
+	},
52
+	Others: []*pb.OtherMessage{
53
+		{
54
+			Value: []byte("some bytes"),
55
+		},
56
+	},
57
+	Somegroup: &pb.MyMessage_SomeGroup{
58
+		GroupField: proto.Int32(6),
59
+	},
60
+	RepBytes: [][]byte{[]byte("sham"), []byte("wow")},
61
+}
62
+
63
+func init() {
64
+	ext := &pb.Ext{
65
+		Data: proto.String("extension"),
66
+	}
67
+	if err := proto.SetExtension(cloneTestMessage, pb.E_Ext_More, ext); err != nil {
68
+		panic("SetExtension: " + err.Error())
69
+	}
70
+}
71
+
72
+func TestClone(t *testing.T) {
73
+	m := proto.Clone(cloneTestMessage).(*pb.MyMessage)
74
+	if !proto.Equal(m, cloneTestMessage) {
75
+		t.Errorf("Clone(%v) = %v", cloneTestMessage, m)
76
+	}
77
+
78
+	// Verify it was a deep copy.
79
+	*m.Inner.Port++
80
+	if proto.Equal(m, cloneTestMessage) {
81
+		t.Error("Mutating clone changed the original")
82
+	}
83
+	// Byte fields and repeated fields should be copied.
84
+	if &m.Pet[0] == &cloneTestMessage.Pet[0] {
85
+		t.Error("Pet: repeated field not copied")
86
+	}
87
+	if &m.Others[0] == &cloneTestMessage.Others[0] {
88
+		t.Error("Others: repeated field not copied")
89
+	}
90
+	if &m.Others[0].Value[0] == &cloneTestMessage.Others[0].Value[0] {
91
+		t.Error("Others[0].Value: bytes field not copied")
92
+	}
93
+	if &m.RepBytes[0] == &cloneTestMessage.RepBytes[0] {
94
+		t.Error("RepBytes: repeated field not copied")
95
+	}
96
+	if &m.RepBytes[0][0] == &cloneTestMessage.RepBytes[0][0] {
97
+		t.Error("RepBytes[0]: bytes field not copied")
98
+	}
99
+}
100
+
101
+func TestCloneNil(t *testing.T) {
102
+	var m *pb.MyMessage
103
+	if c := proto.Clone(m); !proto.Equal(m, c) {
104
+		t.Errorf("Clone(%v) = %v", m, c)
105
+	}
106
+}
107
+
108
+var mergeTests = []struct {
109
+	src, dst, want proto.Message
110
+}{
111
+	{
112
+		src: &pb.MyMessage{
113
+			Count: proto.Int32(42),
114
+		},
115
+		dst: &pb.MyMessage{
116
+			Name: proto.String("Dave"),
117
+		},
118
+		want: &pb.MyMessage{
119
+			Count: proto.Int32(42),
120
+			Name:  proto.String("Dave"),
121
+		},
122
+	},
123
+	{
124
+		src: &pb.MyMessage{
125
+			Inner: &pb.InnerMessage{
126
+				Host:      proto.String("hey"),
127
+				Connected: proto.Bool(true),
128
+			},
129
+			Pet: []string{"horsey"},
130
+			Others: []*pb.OtherMessage{
131
+				{
132
+					Value: []byte("some bytes"),
133
+				},
134
+			},
135
+		},
136
+		dst: &pb.MyMessage{
137
+			Inner: &pb.InnerMessage{
138
+				Host: proto.String("niles"),
139
+				Port: proto.Int32(9099),
140
+			},
141
+			Pet: []string{"bunny", "kitty"},
142
+			Others: []*pb.OtherMessage{
143
+				{
144
+					Key: proto.Int64(31415926535),
145
+				},
146
+				{
147
+					// Explicitly test a src=nil field
148
+					Inner: nil,
149
+				},
150
+			},
151
+		},
152
+		want: &pb.MyMessage{
153
+			Inner: &pb.InnerMessage{
154
+				Host:      proto.String("hey"),
155
+				Connected: proto.Bool(true),
156
+				Port:      proto.Int32(9099),
157
+			},
158
+			Pet: []string{"bunny", "kitty", "horsey"},
159
+			Others: []*pb.OtherMessage{
160
+				{
161
+					Key: proto.Int64(31415926535),
162
+				},
163
+				{},
164
+				{
165
+					Value: []byte("some bytes"),
166
+				},
167
+			},
168
+		},
169
+	},
170
+	{
171
+		src: &pb.MyMessage{
172
+			RepBytes: [][]byte{[]byte("wow")},
173
+		},
174
+		dst: &pb.MyMessage{
175
+			Somegroup: &pb.MyMessage_SomeGroup{
176
+				GroupField: proto.Int32(6),
177
+			},
178
+			RepBytes: [][]byte{[]byte("sham")},
179
+		},
180
+		want: &pb.MyMessage{
181
+			Somegroup: &pb.MyMessage_SomeGroup{
182
+				GroupField: proto.Int32(6),
183
+			},
184
+			RepBytes: [][]byte{[]byte("sham"), []byte("wow")},
185
+		},
186
+	},
187
+	// Check that a scalar bytes field replaces rather than appends.
188
+	{
189
+		src:  &pb.OtherMessage{Value: []byte("foo")},
190
+		dst:  &pb.OtherMessage{Value: []byte("bar")},
191
+		want: &pb.OtherMessage{Value: []byte("foo")},
192
+	},
193
+	{
194
+		src: &pb.MessageWithMap{
195
+			NameMapping: map[int32]string{6: "Nigel"},
196
+			MsgMapping: map[int64]*pb.FloatingPoint{
197
+				0x4001: &pb.FloatingPoint{F: proto.Float64(2.0)},
198
+			},
199
+			ByteMapping: map[bool][]byte{true: []byte("wowsa")},
200
+		},
201
+		dst: &pb.MessageWithMap{
202
+			NameMapping: map[int32]string{
203
+				6: "Bruce", // should be overwritten
204
+				7: "Andrew",
205
+			},
206
+		},
207
+		want: &pb.MessageWithMap{
208
+			NameMapping: map[int32]string{
209
+				6: "Nigel",
210
+				7: "Andrew",
211
+			},
212
+			MsgMapping: map[int64]*pb.FloatingPoint{
213
+				0x4001: &pb.FloatingPoint{F: proto.Float64(2.0)},
214
+			},
215
+			ByteMapping: map[bool][]byte{true: []byte("wowsa")},
216
+		},
217
+	},
218
+	// proto3 shouldn't merge zero values,
219
+	// in the same way that proto2 shouldn't merge nils.
220
+	{
221
+		src: &proto3pb.Message{
222
+			Name: "Aaron",
223
+			Data: []byte(""), // zero value, but not nil
224
+		},
225
+		dst: &proto3pb.Message{
226
+			HeightInCm: 176,
227
+			Data:       []byte("texas!"),
228
+		},
229
+		want: &proto3pb.Message{
230
+			Name:       "Aaron",
231
+			HeightInCm: 176,
232
+			Data:       []byte("texas!"),
233
+		},
234
+	},
235
+	// Oneof fields should merge by assignment.
236
+	{
237
+		src: &pb.Communique{
238
+			Union: &pb.Communique_Number{41},
239
+		},
240
+		dst: &pb.Communique{
241
+			Union: &pb.Communique_Name{"Bobby Tables"},
242
+		},
243
+		want: &pb.Communique{
244
+			Union: &pb.Communique_Number{41},
245
+		},
246
+	},
247
+	// Oneof nil is the same as not set.
248
+	{
249
+		src: &pb.Communique{},
250
+		dst: &pb.Communique{
251
+			Union: &pb.Communique_Name{"Bobby Tables"},
252
+		},
253
+		want: &pb.Communique{
254
+			Union: &pb.Communique_Name{"Bobby Tables"},
255
+		},
256
+	},
257
+}
258
+
259
+func TestMerge(t *testing.T) {
260
+	for _, m := range mergeTests {
261
+		got := proto.Clone(m.dst)
262
+		proto.Merge(got, m.src)
263
+		if !proto.Equal(got, m.want) {
264
+			t.Errorf("Merge(%v, %v)\n got %v\nwant %v\n", m.dst, m.src, got, m.want)
265
+		}
266
+	}
267
+}

+ 868
- 0
vendor/github.com/golang/protobuf/proto/decode.go View File

@@ -0,0 +1,868 @@
1
+// Go support for Protocol Buffers - Google's data interchange format
2
+//
3
+// Copyright 2010 The Go Authors.  All rights reserved.
4
+// https://github.com/golang/protobuf
5
+//
6
+// Redistribution and use in source and binary forms, with or without
7
+// modification, are permitted provided that the following conditions are
8
+// met:
9
+//
10
+//     * Redistributions of source code must retain the above copyright
11
+// notice, this list of conditions and the following disclaimer.
12
+//     * Redistributions in binary form must reproduce the above
13
+// copyright notice, this list of conditions and the following disclaimer
14
+// in the documentation and/or other materials provided with the
15
+// distribution.
16
+//     * Neither the name of Google Inc. nor the names of its
17
+// contributors may be used to endorse or promote products derived from
18
+// this software without specific prior written permission.
19
+//
20
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
+
32
+package proto
33
+
34
+/*
35
+ * Routines for decoding protocol buffer data to construct in-memory representations.
36
+ */
37
+
38
+import (
39
+	"errors"
40
+	"fmt"
41
+	"io"
42
+	"os"
43
+	"reflect"
44
+)
45
+
46
+// errOverflow is returned when an integer is too large to be represented.
47
+var errOverflow = errors.New("proto: integer overflow")
48
+
49
+// ErrInternalBadWireType is returned by generated code when an incorrect
50
+// wire type is encountered. It does not get returned to user code.
51
+var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for oneof")
52
+
53
+// The fundamental decoders that interpret bytes on the wire.
54
+// Those that take integer types all return uint64 and are
55
+// therefore of type valueDecoder.
56
+
57
+// DecodeVarint reads a varint-encoded integer from the slice.
58
+// It returns the integer and the number of bytes consumed, or
59
+// zero if there is not enough.
60
+// This is the format for the
61
+// int32, int64, uint32, uint64, bool, and enum
62
+// protocol buffer types.
63
+func DecodeVarint(buf []byte) (x uint64, n int) {
64
+	// x, n already 0
65
+	for shift := uint(0); shift < 64; shift += 7 {
66
+		if n >= len(buf) {
67
+			return 0, 0
68
+		}
69
+		b := uint64(buf[n])
70
+		n++
71
+		x |= (b & 0x7F) << shift
72
+		if (b & 0x80) == 0 {
73
+			return x, n
74
+		}
75
+	}
76
+
77
+	// The number is too large to represent in a 64-bit value.
78
+	return 0, 0
79
+}
80
+
81
+// DecodeVarint reads a varint-encoded integer from the Buffer.
82
+// This is the format for the
83
+// int32, int64, uint32, uint64, bool, and enum
84
+// protocol buffer types.
85
+func (p *Buffer) DecodeVarint() (x uint64, err error) {
86
+	// x, err already 0
87
+
88
+	i := p.index
89
+	l := len(p.buf)
90
+
91
+	for shift := uint(0); shift < 64; shift += 7 {
92
+		if i >= l {
93
+			err = io.ErrUnexpectedEOF
94
+			return
95
+		}
96
+		b := p.buf[i]
97
+		i++
98
+		x |= (uint64(b) & 0x7F) << shift
99
+		if b < 0x80 {
100
+			p.index = i
101
+			return
102
+		}
103
+	}
104
+
105
+	// The number is too large to represent in a 64-bit value.
106
+	err = errOverflow
107
+	return
108
+}
109
+
110
+// DecodeFixed64 reads a 64-bit integer from the Buffer.
111
+// This is the format for the
112
+// fixed64, sfixed64, and double protocol buffer types.
113
+func (p *Buffer) DecodeFixed64() (x uint64, err error) {
114
+	// x, err already 0
115
+	i := p.index + 8
116
+	if i < 0 || i > len(p.buf) {
117
+		err = io.ErrUnexpectedEOF
118
+		return
119
+	}
120
+	p.index = i
121
+
122
+	x = uint64(p.buf[i-8])
123
+	x |= uint64(p.buf[i-7]) << 8
124
+	x |= uint64(p.buf[i-6]) << 16
125
+	x |= uint64(p.buf[i-5]) << 24
126
+	x |= uint64(p.buf[i-4]) << 32
127
+	x |= uint64(p.buf[i-3]) << 40
128
+	x |= uint64(p.buf[i-2]) << 48
129
+	x |= uint64(p.buf[i-1]) << 56
130
+	return
131
+}
132
+
133
+// DecodeFixed32 reads a 32-bit integer from the Buffer.
134
+// This is the format for the
135
+// fixed32, sfixed32, and float protocol buffer types.
136
+func (p *Buffer) DecodeFixed32() (x uint64, err error) {
137
+	// x, err already 0
138
+	i := p.index + 4
139
+	if i < 0 || i > len(p.buf) {
140
+		err = io.ErrUnexpectedEOF
141
+		return
142
+	}
143
+	p.index = i
144
+
145
+	x = uint64(p.buf[i-4])
146
+	x |= uint64(p.buf[i-3]) << 8
147
+	x |= uint64(p.buf[i-2]) << 16
148
+	x |= uint64(p.buf[i-1]) << 24
149
+	return
150
+}
151
+
152
+// DecodeZigzag64 reads a zigzag-encoded 64-bit integer
153
+// from the Buffer.
154
+// This is the format used for the sint64 protocol buffer type.
155
+func (p *Buffer) DecodeZigzag64() (x uint64, err error) {
156
+	x, err = p.DecodeVarint()
157
+	if err != nil {
158
+		return
159
+	}
160
+	x = (x >> 1) ^ uint64((int64(x&1)<<63)>>63)
161
+	return
162
+}
163
+
164
+// DecodeZigzag32 reads a zigzag-encoded 32-bit integer
165
+// from  the Buffer.
166
+// This is the format used for the sint32 protocol buffer type.
167
+func (p *Buffer) DecodeZigzag32() (x uint64, err error) {
168
+	x, err = p.DecodeVarint()
169
+	if err != nil {
170
+		return
171
+	}
172
+	x = uint64((uint32(x) >> 1) ^ uint32((int32(x&1)<<31)>>31))
173
+	return
174
+}
175
+
176
+// These are not ValueDecoders: they produce an array of bytes or a string.
177
+// bytes, embedded messages
178
+
179
+// DecodeRawBytes reads a count-delimited byte buffer from the Buffer.
180
+// This is the format used for the bytes protocol buffer
181
+// type and for embedded messages.
182
+func (p *Buffer) DecodeRawBytes(alloc bool) (buf []byte, err error) {
183
+	n, err := p.DecodeVarint()
184
+	if err != nil {
185
+		return nil, err
186
+	}
187
+
188
+	nb := int(n)
189
+	if nb < 0 {
190
+		return nil, fmt.Errorf("proto: bad byte length %d", nb)
191
+	}
192
+	end := p.index + nb
193
+	if end < p.index || end > len(p.buf) {
194
+		return nil, io.ErrUnexpectedEOF
195
+	}
196
+
197
+	if !alloc {
198
+		// todo: check if can get more uses of alloc=false
199
+		buf = p.buf[p.index:end]
200
+		p.index += nb
201
+		return
202
+	}
203
+
204
+	buf = make([]byte, nb)
205
+	copy(buf, p.buf[p.index:])
206
+	p.index += nb
207
+	return
208
+}
209
+
210
+// DecodeStringBytes reads an encoded string from the Buffer.
211
+// This is the format used for the proto2 string type.
212
+func (p *Buffer) DecodeStringBytes() (s string, err error) {
213
+	buf, err := p.DecodeRawBytes(false)
214
+	if err != nil {
215
+		return
216
+	}
217
+	return string(buf), nil
218
+}
219
+
220
+// Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
221
+// If the protocol buffer has extensions, and the field matches, add it as an extension.
222
+// Otherwise, if the XXX_unrecognized field exists, append the skipped data there.
223
+func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base structPointer, unrecField field) error {
224
+	oi := o.index
225
+
226
+	err := o.skip(t, tag, wire)
227
+	if err != nil {
228
+		return err
229
+	}
230
+
231
+	if !unrecField.IsValid() {
232
+		return nil
233
+	}
234
+
235
+	ptr := structPointer_Bytes(base, unrecField)
236
+
237
+	// Add the skipped field to struct field
238
+	obuf := o.buf
239
+
240
+	o.buf = *ptr
241
+	o.EncodeVarint(uint64(tag<<3 | wire))
242
+	*ptr = append(o.buf, obuf[oi:o.index]...)
243
+
244
+	o.buf = obuf
245
+
246
+	return nil
247
+}
248
+
249
+// Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
250
+func (o *Buffer) skip(t reflect.Type, tag, wire int) error {
251
+
252
+	var u uint64
253
+	var err error
254
+
255
+	switch wire {
256
+	case WireVarint:
257
+		_, err = o.DecodeVarint()
258
+	case WireFixed64:
259
+		_, err = o.DecodeFixed64()
260
+	case WireBytes:
261
+		_, err = o.DecodeRawBytes(false)
262
+	case WireFixed32:
263
+		_, err = o.DecodeFixed32()
264
+	case WireStartGroup:
265
+		for {
266
+			u, err = o.DecodeVarint()
267
+			if err != nil {
268
+				break
269
+			}
270
+			fwire := int(u & 0x7)
271
+			if fwire == WireEndGroup {
272
+				break
273
+			}
274
+			ftag := int(u >> 3)
275
+			err = o.skip(t, ftag, fwire)
276
+			if err != nil {
277
+				break
278
+			}
279
+		}
280
+	default:
281
+		err = fmt.Errorf("proto: can't skip unknown wire type %d for %s", wire, t)
282
+	}
283
+	return err
284
+}
285
+
286
+// Unmarshaler is the interface representing objects that can
287
+// unmarshal themselves.  The method should reset the receiver before
288
+// decoding starts.  The argument points to data that may be
289
+// overwritten, so implementations should not keep references to the
290
+// buffer.
291
+type Unmarshaler interface {
292
+	Unmarshal([]byte) error
293
+}
294
+
295
+// Unmarshal parses the protocol buffer representation in buf and places the
296
+// decoded result in pb.  If the struct underlying pb does not match
297
+// the data in buf, the results can be unpredictable.
298
+//
299
+// Unmarshal resets pb before starting to unmarshal, so any
300
+// existing data in pb is always removed. Use UnmarshalMerge
301
+// to preserve and append to existing data.
302
+func Unmarshal(buf []byte, pb Message) error {
303
+	pb.Reset()
304
+	return UnmarshalMerge(buf, pb)
305
+}
306
+
307
+// UnmarshalMerge parses the protocol buffer representation in buf and
308
+// writes the decoded result to pb.  If the struct underlying pb does not match
309
+// the data in buf, the results can be unpredictable.
310
+//
311
+// UnmarshalMerge merges into existing data in pb.
312
+// Most code should use Unmarshal instead.
313
+func UnmarshalMerge(buf []byte, pb Message) error {
314
+	// If the object can unmarshal itself, let it.
315
+	if u, ok := pb.(Unmarshaler); ok {
316
+		return u.Unmarshal(buf)
317
+	}
318
+	return NewBuffer(buf).Unmarshal(pb)
319
+}
320
+
321
+// DecodeMessage reads a count-delimited message from the Buffer.
322
+func (p *Buffer) DecodeMessage(pb Message) error {
323
+	enc, err := p.DecodeRawBytes(false)
324
+	if err != nil {
325
+		return err
326
+	}
327
+	return NewBuffer(enc).Unmarshal(pb)
328
+}
329
+
330
+// DecodeGroup reads a tag-delimited group from the Buffer.
331
+func (p *Buffer) DecodeGroup(pb Message) error {
332
+	typ, base, err := getbase(pb)
333
+	if err != nil {
334
+		return err
335
+	}
336
+	return p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), true, base)
337
+}
338
+
339
+// Unmarshal parses the protocol buffer representation in the
340
+// Buffer and places the decoded result in pb.  If the struct
341
+// underlying pb does not match the data in the buffer, the results can be
342
+// unpredictable.
343
+func (p *Buffer) Unmarshal(pb Message) error {
344
+	// If the object can unmarshal itself, let it.
345
+	if u, ok := pb.(Unmarshaler); ok {
346
+		err := u.Unmarshal(p.buf[p.index:])
347
+		p.index = len(p.buf)
348
+		return err
349
+	}
350
+
351
+	typ, base, err := getbase(pb)
352
+	if err != nil {
353
+		return err
354
+	}
355
+
356
+	err = p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), false, base)
357
+
358
+	if collectStats {
359
+		stats.Decode++
360
+	}
361
+
362
+	return err
363
+}
364
+
365
+// unmarshalType does the work of unmarshaling a structure.
366
+func (o *Buffer) unmarshalType(st reflect.Type, prop *StructProperties, is_group bool, base structPointer) error {
367
+	var state errorState
368
+	required, reqFields := prop.reqCount, uint64(0)
369
+
370
+	var err error
371
+	for err == nil && o.index < len(o.buf) {
372
+		oi := o.index
373
+		var u uint64
374
+		u, err = o.DecodeVarint()
375
+		if err != nil {
376
+			break
377
+		}
378
+		wire := int(u & 0x7)
379
+		if wire == WireEndGroup {
380
+			if is_group {
381
+				return nil // input is satisfied
382
+			}
383
+			return fmt.Errorf("proto: %s: wiretype end group for non-group", st)
384
+		}
385
+		tag := int(u >> 3)
386
+		if tag <= 0 {
387
+			return fmt.Errorf("proto: %s: illegal tag %d (wire type %d)", st, tag, wire)
388
+		}
389
+		fieldnum, ok := prop.decoderTags.get(tag)
390
+		if !ok {
391
+			// Maybe it's an extension?
392
+			if prop.extendable {
393
+				if e := structPointer_Interface(base, st).(extendableProto); isExtensionField(e, int32(tag)) {
394
+					if err = o.skip(st, tag, wire); err == nil {
395
+						ext := e.ExtensionMap()[int32(tag)] // may be missing
396
+						ext.enc = append(ext.enc, o.buf[oi:o.index]...)
397
+						e.ExtensionMap()[int32(tag)] = ext
398
+					}
399
+					continue
400
+				}
401
+			}
402
+			// Maybe it's a oneof?
403
+			if prop.oneofUnmarshaler != nil {
404
+				m := structPointer_Interface(base, st).(Message)
405
+				// First return value indicates whether tag is a oneof field.
406
+				ok, err = prop.oneofUnmarshaler(m, tag, wire, o)
407
+				if err == ErrInternalBadWireType {
408
+					// Map the error to something more descriptive.
409
+					// Do the formatting here to save generated code space.
410
+					err = fmt.Errorf("bad wiretype for oneof field in %T", m)
411
+				}
412
+				if ok {
413
+					continue
414
+				}
415
+			}
416
+			err = o.skipAndSave(st, tag, wire, base, prop.unrecField)
417
+			continue
418
+		}
419
+		p := prop.Prop[fieldnum]
420
+
421
+		if p.dec == nil {
422
+			fmt.Fprintf(os.Stderr, "proto: no protobuf decoder for %s.%s\n", st, st.Field(fieldnum).Name)
423
+			continue
424
+		}
425
+		dec := p.dec
426
+		if wire != WireStartGroup && wire != p.WireType {
427
+			if wire == WireBytes && p.packedDec != nil {
428
+				// a packable field
429
+				dec = p.packedDec
430
+			} else {
431
+				err = fmt.Errorf("proto: bad wiretype for field %s.%s: got wiretype %d, want %d", st, st.Field(fieldnum).Name, wire, p.WireType)
432
+				continue
433
+			}
434
+		}
435
+		decErr := dec(o, p, base)
436
+		if decErr != nil && !state.shouldContinue(decErr, p) {
437
+			err = decErr
438
+		}
439
+		if err == nil && p.Required {
440
+			// Successfully decoded a required field.
441
+			if tag <= 64 {
442
+				// use bitmap for fields 1-64 to catch field reuse.
443
+				var mask uint64 = 1 << uint64(tag-1)
444
+				if reqFields&mask == 0 {
445
+					// new required field
446
+					reqFields |= mask
447
+					required--
448
+				}
449
+			} else {
450
+				// This is imprecise. It can be fooled by a required field
451
+				// with a tag > 64 that is encoded twice; that's very rare.
452
+				// A fully correct implementation would require allocating
453
+				// a data structure, which we would like to avoid.
454
+				required--
455
+			}
456
+		}
457
+	}
458
+	if err == nil {
459
+		if is_group {
460
+			return io.ErrUnexpectedEOF
461
+		}
462
+		if state.err != nil {
463
+			return state.err
464
+		}
465
+		if required > 0 {
466
+			// Not enough information to determine the exact field. If we use extra
467
+			// CPU, we could determine the field only if the missing required field
468
+			// has a tag <= 64 and we check reqFields.
469
+			return &RequiredNotSetError{"{Unknown}"}
470
+		}
471
+	}
472
+	return err
473
+}
474
+
475
+// Individual type decoders
476
+// For each,
477
+//	u is the decoded value,
478
+//	v is a pointer to the field (pointer) in the struct
479
+
480
+// Sizes of the pools to allocate inside the Buffer.
481
+// The goal is modest amortization and allocation
482
+// on at least 16-byte boundaries.
483
+const (
484
+	boolPoolSize   = 16
485
+	uint32PoolSize = 8
486
+	uint64PoolSize = 4
487
+)
488
+
489
+// Decode a bool.
490
+func (o *Buffer) dec_bool(p *Properties, base structPointer) error {
491
+	u, err := p.valDec(o)
492
+	if err != nil {
493
+		return err
494
+	}
495
+	if len(o.bools) == 0 {
496
+		o.bools = make([]bool, boolPoolSize)
497
+	}
498
+	o.bools[0] = u != 0
499
+	*structPointer_Bool(base, p.field) = &o.bools[0]
500
+	o.bools = o.bools[1:]
501
+	return nil
502
+}
503
+
504
+func (o *Buffer) dec_proto3_bool(p *Properties, base structPointer) error {
505
+	u, err := p.valDec(o)
506
+	if err != nil {
507
+		return err
508
+	}
509
+	*structPointer_BoolVal(base, p.field) = u != 0
510
+	return nil
511
+}
512
+
513
+// Decode an int32.
514
+func (o *Buffer) dec_int32(p *Properties, base structPointer) error {
515
+	u, err := p.valDec(o)
516
+	if err != nil {
517
+		return err
518
+	}
519
+	word32_Set(structPointer_Word32(base, p.field), o, uint32(u))
520
+	return nil
521
+}
522
+
523
+func (o *Buffer) dec_proto3_int32(p *Properties, base structPointer) error {
524
+	u, err := p.valDec(o)
525
+	if err != nil {
526
+		return err
527
+	}
528
+	word32Val_Set(structPointer_Word32Val(base, p.field), uint32(u))
529
+	return nil
530
+}
531
+
532
+// Decode an int64.
533
+func (o *Buffer) dec_int64(p *Properties, base structPointer) error {
534
+	u, err := p.valDec(o)
535
+	if err != nil {
536
+		return err
537
+	}
538
+	word64_Set(structPointer_Word64(base, p.field), o, u)
539
+	return nil
540
+}
541
+
542
+func (o *Buffer) dec_proto3_int64(p *Properties, base structPointer) error {
543
+	u, err := p.valDec(o)
544
+	if err != nil {
545
+		return err
546
+	}
547
+	word64Val_Set(structPointer_Word64Val(base, p.field), o, u)
548
+	return nil
549
+}
550
+
551
+// Decode a string.
552
+func (o *Buffer) dec_string(p *Properties, base structPointer) error {
553
+	s, err := o.DecodeStringBytes()
554
+	if err != nil {
555
+		return err
556
+	}
557
+	*structPointer_String(base, p.field) = &s
558
+	return nil
559
+}
560
+
561
+func (o *Buffer) dec_proto3_string(p *Properties, base structPointer) error {
562
+	s, err := o.DecodeStringBytes()
563
+	if err != nil {
564
+		return err
565
+	}
566
+	*structPointer_StringVal(base, p.field) = s
567
+	return nil
568
+}
569
+
570
+// Decode a slice of bytes ([]byte).
571
+func (o *Buffer) dec_slice_byte(p *Properties, base structPointer) error {
572
+	b, err := o.DecodeRawBytes(true)
573
+	if err != nil {
574
+		return err
575
+	}
576
+	*structPointer_Bytes(base, p.field) = b
577
+	return nil
578
+}
579
+
580
+// Decode a slice of bools ([]bool).
581
+func (o *Buffer) dec_slice_bool(p *Properties, base structPointer) error {
582
+	u, err := p.valDec(o)
583
+	if err != nil {
584
+		return err
585
+	}
586
+	v := structPointer_BoolSlice(base, p.field)
587
+	*v = append(*v, u != 0)
588
+	return nil
589
+}
590
+
591
+// Decode a slice of bools ([]bool) in packed format.
592
+func (o *Buffer) dec_slice_packed_bool(p *Properties, base structPointer) error {
593
+	v := structPointer_BoolSlice(base, p.field)
594
+
595
+	nn, err := o.DecodeVarint()
596
+	if err != nil {
597
+		return err
598
+	}
599
+	nb := int(nn) // number of bytes of encoded bools
600
+	fin := o.index + nb
601
+	if fin < o.index {
602
+		return errOverflow
603
+	}
604
+
605
+	y := *v
606
+	for o.index < fin {
607
+		u, err := p.valDec(o)
608
+		if err != nil {
609
+			return err
610
+		}
611
+		y = append(y, u != 0)
612
+	}
613
+
614
+	*v = y
615
+	return nil
616
+}
617
+
618
+// Decode a slice of int32s ([]int32).
619
+func (o *Buffer) dec_slice_int32(p *Properties, base structPointer) error {
620
+	u, err := p.valDec(o)
621
+	if err != nil {
622
+		return err
623
+	}
624
+	structPointer_Word32Slice(base, p.field).Append(uint32(u))
625
+	return nil
626
+}
627
+
628
+// Decode a slice of int32s ([]int32) in packed format.
629
+func (o *Buffer) dec_slice_packed_int32(p *Properties, base structPointer) error {
630
+	v := structPointer_Word32Slice(base, p.field)
631
+
632
+	nn, err := o.DecodeVarint()
633
+	if err != nil {
634
+		return err
635
+	}
636
+	nb := int(nn) // number of bytes of encoded int32s
637
+
638
+	fin := o.index + nb
639
+	if fin < o.index {
640
+		return errOverflow
641
+	}
642
+	for o.index < fin {
643
+		u, err := p.valDec(o)
644
+		if err != nil {
645
+			return err
646
+		}
647
+		v.Append(uint32(u))
648
+	}
649
+	return nil
650
+}
651
+
652
+// Decode a slice of int64s ([]int64).
653
+func (o *Buffer) dec_slice_int64(p *Properties, base structPointer) error {
654
+	u, err := p.valDec(o)
655
+	if err != nil {
656
+		return err
657
+	}
658
+
659
+	structPointer_Word64Slice(base, p.field).Append(u)
660
+	return nil
661
+}
662
+
663
+// Decode a slice of int64s ([]int64) in packed format.
664
+func (o *Buffer) dec_slice_packed_int64(p *Properties, base structPointer) error {
665
+	v := structPointer_Word64Slice(base, p.field)
666
+
667
+	nn, err := o.DecodeVarint()
668
+	if err != nil {
669
+		return err
670
+	}
671
+	nb := int(nn) // number of bytes of encoded int64s
672
+
673
+	fin := o.index + nb
674
+	if fin < o.index {
675
+		return errOverflow
676
+	}
677
+	for o.index < fin {
678
+		u, err := p.valDec(o)
679
+		if err != nil {
680
+			return err
681
+		}
682
+		v.Append(u)
683
+	}
684
+	return nil
685
+}
686
+
687
+// Decode a slice of strings ([]string).
688
+func (o *Buffer) dec_slice_string(p *Properties, base structPointer) error {
689
+	s, err := o.DecodeStringBytes()
690
+	if err != nil {
691
+		return err
692
+	}
693
+	v := structPointer_StringSlice(base, p.field)
694
+	*v = append(*v, s)
695
+	return nil
696
+}
697
+
698
+// Decode a slice of slice of bytes ([][]byte).
699
+func (o *Buffer) dec_slice_slice_byte(p *Properties, base structPointer) error {
700
+	b, err := o.DecodeRawBytes(true)
701
+	if err != nil {
702
+		return err
703
+	}
704
+	v := structPointer_BytesSlice(base, p.field)
705
+	*v = append(*v, b)
706
+	return nil
707
+}
708
+
709
+// Decode a map field.
710
+func (o *Buffer) dec_new_map(p *Properties, base structPointer) error {
711
+	raw, err := o.DecodeRawBytes(false)
712
+	if err != nil {
713
+		return err
714
+	}
715
+	oi := o.index       // index at the end of this map entry
716
+	o.index -= len(raw) // move buffer back to start of map entry
717
+
718
+	mptr := structPointer_NewAt(base, p.field, p.mtype) // *map[K]V
719
+	if mptr.Elem().IsNil() {
720
+		mptr.Elem().Set(reflect.MakeMap(mptr.Type().Elem()))
721
+	}
722
+	v := mptr.Elem() // map[K]V
723
+
724
+	// Prepare addressable doubly-indirect placeholders for the key and value types.
725
+	// See enc_new_map for why.
726
+	keyptr := reflect.New(reflect.PtrTo(p.mtype.Key())).Elem() // addressable *K
727
+	keybase := toStructPointer(keyptr.Addr())                  // **K
728
+
729
+	var valbase structPointer
730
+	var valptr reflect.Value
731
+	switch p.mtype.Elem().Kind() {
732
+	case reflect.Slice:
733
+		// []byte
734
+		var dummy []byte
735
+		valptr = reflect.ValueOf(&dummy)  // *[]byte
736
+		valbase = toStructPointer(valptr) // *[]byte
737
+	case reflect.Ptr:
738
+		// message; valptr is **Msg; need to allocate the intermediate pointer
739
+		valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V
740
+		valptr.Set(reflect.New(valptr.Type().Elem()))
741
+		valbase = toStructPointer(valptr)
742
+	default:
743
+		// everything else
744
+		valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V
745
+		valbase = toStructPointer(valptr.Addr())                   // **V
746
+	}
747
+
748
+	// Decode.
749
+	// This parses a restricted wire format, namely the encoding of a message
750
+	// with two fields. See enc_new_map for the format.
751
+	for o.index < oi {
752
+		// tagcode for key and value properties are always a single byte
753
+		// because they have tags 1 and 2.
754
+		tagcode := o.buf[o.index]
755
+		o.index++
756
+		switch tagcode {
757
+		case p.mkeyprop.tagcode[0]:
758
+			if err := p.mkeyprop.dec(o, p.mkeyprop, keybase); err != nil {
759
+				return err
760
+			}
761
+		case p.mvalprop.tagcode[0]:
762
+			if err := p.mvalprop.dec(o, p.mvalprop, valbase); err != nil {
763
+				return err
764
+			}
765
+		default:
766
+			// TODO: Should we silently skip this instead?
767
+			return fmt.Errorf("proto: bad map data tag %d", raw[0])
768
+		}
769
+	}
770
+	keyelem, valelem := keyptr.Elem(), valptr.Elem()
771
+	if !keyelem.IsValid() {
772
+		keyelem = reflect.Zero(p.mtype.Key())
773
+	}
774
+	if !valelem.IsValid() {
775
+		valelem = reflect.Zero(p.mtype.Elem())
776
+	}
777
+
778
+	v.SetMapIndex(keyelem, valelem)
779
+	return nil
780
+}
781
+
782
+// Decode a group.
783
+func (o *Buffer) dec_struct_group(p *Properties, base structPointer) error {
784
+	bas := structPointer_GetStructPointer(base, p.field)
785
+	if structPointer_IsNil(bas) {
786
+		// allocate new nested message
787
+		bas = toStructPointer(reflect.New(p.stype))
788
+		structPointer_SetStructPointer(base, p.field, bas)
789
+	}
790
+	return o.unmarshalType(p.stype, p.sprop, true, bas)
791
+}
792
+
793
+// Decode an embedded message.
794
+func (o *Buffer) dec_struct_message(p *Properties, base structPointer) (err error) {
795
+	raw, e := o.DecodeRawBytes(false)
796
+	if e != nil {
797
+		return e
798
+	}
799
+
800
+	bas := structPointer_GetStructPointer(base, p.field)
801
+	if structPointer_IsNil(bas) {
802
+		// allocate new nested message
803
+		bas = toStructPointer(reflect.New(p.stype))
804
+		structPointer_SetStructPointer(base, p.field, bas)
805
+	}
806
+
807
+	// If the object can unmarshal itself, let it.
808
+	if p.isUnmarshaler {
809