Merge branch 'deps'

,/`.
        ,'/ __`.
      ,'_/_  _ _`.
    ,'__/_ ___ _  `.
  ,'_  /___ __ _ __ `.
 '-.._/___...-"-.-..__`.
This commit is contained in:
Stephen McQuay 2018-05-11 09:15:47 -07:00
commit 10d195621e
Signed by: sm
GPG Key ID: C383C74875475AC8
28 changed files with 6475 additions and 3105 deletions

12
Gopkg.lock generated
View File

@ -5,13 +5,13 @@
branch = "master" branch = "master"
name = "github.com/beorn7/perks" name = "github.com/beorn7/perks"
packages = ["quantile"] packages = ["quantile"]
revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9" revision = "3a771d992973f24aa725d07868b467d1ddfceafb"
[[projects]] [[projects]]
name = "github.com/golang/protobuf" name = "github.com/golang/protobuf"
packages = ["proto"] packages = ["proto"]
revision = "925541529c1fa6821df4e44ce2723319eb2be768" revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265"
version = "v1.0.0" version = "v1.1.0"
[[projects]] [[projects]]
name = "github.com/matttproud/golang_protobuf_extensions" name = "github.com/matttproud/golang_protobuf_extensions"
@ -48,7 +48,7 @@
"internal/bitbucket.org/ww/goautoneg", "internal/bitbucket.org/ww/goautoneg",
"model" "model"
] ]
revision = "6fb6fce6f8b75884b92e1889c150403fc0872c5e" revision = "d811d2e9bf898806ecfb6ef6296774b13ffc314c"
[[projects]] [[projects]]
branch = "master" branch = "master"
@ -59,7 +59,7 @@
"nfs", "nfs",
"xfs" "xfs"
] ]
revision = "1c7ff3de94ae006f58cba483a4c9c6d7c61e1d98" revision = "8b1c2da0d56deffdbb9e48d4414b4e674bd8083e"
[[projects]] [[projects]]
name = "mcquay.me/metrics" name = "mcquay.me/metrics"
@ -70,6 +70,6 @@
[solve-meta] [solve-meta]
analyzer-name = "dep" analyzer-name = "dep"
analyzer-version = 1 analyzer-version = 1
inputs-digest = "6ca9d144505195ce3da40f2598509d79c7205b05dddfa38c4a18acc7200a3755" inputs-digest = "18edb7690ec444c7d05e02b43bdea2b8cdf52a712a058d1f11f1eb67ee272e89"
solver-name = "gps-cdcl" solver-name = "gps-cdcl"
solver-version = 1 solver-version = 1

View File

@ -6,27 +6,30 @@ V_DIRTY := $(shell git describe --exact-match HEAD 2> /dev/null > /dev/null || e
GIT := $(shell git rev-parse --short HEAD) GIT := $(shell git rev-parse --short HEAD)
DIRTY := $(shell git diff-index --quiet HEAD 2> /dev/null > /dev/null || echo "-dirty") DIRTY := $(shell git diff-index --quiet HEAD 2> /dev/null > /dev/null || echo "-dirty")
bin/hw: $(shell ls *.go) cmd/hw/main.go bin bin/hw: vendor $(shell ls *.go) cmd/hw/main.go bin
@echo hw @echo hw
@GOOS=linux go build -ldflags \ @GOOS=linux go build -ldflags \
"-X mcquay.me/hw.Version=$(VERSION)$(V_DIRTY) \ "-X mcquay.me/hw.Version=$(VERSION)$(V_DIRTY) \
-X mcquay.me/hw.Git=$(GIT)$(DIRTY)" \ -X mcquay.me/hw.Git=$(GIT)$(DIRTY)" \
-v -o bin/hw ./cmd/hw -v -o bin/hw ./cmd/hw
bin/hwc: $(shell ls *.go ) cmd/hwc/main.go bin bin/hwc: vendor $(shell ls *.go ) cmd/hwc/main.go bin
@echo hwc @echo hwc
@GOOS=linux go build -ldflags \ @GOOS=linux go build -ldflags \
"-X mcquay.me/hw.Version=$(VERSION)$(V_DIRTY) \ "-X mcquay.me/hw.Version=$(VERSION)$(V_DIRTY) \
-X mcquay.me/hw.Git=$(GIT)$(DIRTY)" \ -X mcquay.me/hw.Git=$(GIT)$(DIRTY)" \
-v -o bin/hwc ./cmd/hwc -v -o bin/hwc ./cmd/hwc
bin/hwl: $(shell ls *.go) cmd/hwl/main.go bin bin/hwl: vendor $(shell ls *.go) cmd/hwl/main.go bin
@echo hwl @echo hwl
@GOOS=linux go build -ldflags \ @GOOS=linux go build -ldflags \
"-X mcquay.me/hw.Version=$(VERSION)$(V_DIRTY) \ "-X mcquay.me/hw.Version=$(VERSION)$(V_DIRTY) \
-X mcquay.me/hw.Git=$(GIT)$(DIRTY)" \ -X mcquay.me/hw.Git=$(GIT)$(DIRTY)" \
-v -o bin/hwl ./cmd/hwl -v -o bin/hwl ./cmd/hwl
vendor: $(shell find vendor -type f | grep go$$ ) Gopkg.toml Gopkg.lock
dep ensure
bin: bin:
mkdir bin mkdir bin

View File

@ -77,15 +77,20 @@ func NewHighBiased(epsilon float64) *Stream {
// is guaranteed to be within (Quantile±Epsilon). // is guaranteed to be within (Quantile±Epsilon).
// //
// See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties. // See http://www.cs.rutgers.edu/~muthu/bquant.pdf for time, space, and error properties.
func NewTargeted(targets map[float64]float64) *Stream { func NewTargeted(targetMap map[float64]float64) *Stream {
// Convert map to slice to avoid slow iterations on a map.
// ƒ is called on the hot path, so converting the map to a slice
// beforehand results in significant CPU savings.
targets := targetMapToSlice(targetMap)
ƒ := func(s *stream, r float64) float64 { ƒ := func(s *stream, r float64) float64 {
var m = math.MaxFloat64 var m = math.MaxFloat64
var f float64 var f float64
for quantile, epsilon := range targets { for _, t := range targets {
if quantile*s.n <= r { if t.quantile*s.n <= r {
f = (2 * epsilon * r) / quantile f = (2 * t.epsilon * r) / t.quantile
} else { } else {
f = (2 * epsilon * (s.n - r)) / (1 - quantile) f = (2 * t.epsilon * (s.n - r)) / (1 - t.quantile)
} }
if f < m { if f < m {
m = f m = f
@ -96,6 +101,25 @@ func NewTargeted(targets map[float64]float64) *Stream {
return newStream(ƒ) return newStream(ƒ)
} }
type target struct {
quantile float64
epsilon float64
}
func targetMapToSlice(targetMap map[float64]float64) []target {
targets := make([]target, 0, len(targetMap))
for quantile, epsilon := range targetMap {
t := target{
quantile: quantile,
epsilon: epsilon,
}
targets = append(targets, t)
}
return targets
}
// Stream computes quantiles for a stream of float64s. It is not thread-safe by // Stream computes quantiles for a stream of float64s. It is not thread-safe by
// design. Take care when using across multiple goroutines. // design. Take care when using across multiple goroutines.
type Stream struct { type Stream struct {

View File

@ -1,43 +0,0 @@
# Go support for Protocol Buffers - Google's data interchange format
#
# Copyright 2010 The Go Authors. All rights reserved.
# https://github.com/golang/protobuf
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following disclaimer
# in the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived from
# this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
install:
go install
test: install generate-test-pbs
go test
generate-test-pbs:
make install
make -C testdata
protoc --go_out=Mtestdata/test.proto=github.com/golang/protobuf/proto/testdata,Mgoogle/protobuf/any.proto=github.com/golang/protobuf/ptypes/any:. proto3_proto/proto3.proto
make

View File

@ -35,22 +35,39 @@
package proto package proto
import ( import (
"fmt"
"log" "log"
"reflect" "reflect"
"strings" "strings"
) )
// Clone returns a deep copy of a protocol buffer. // Clone returns a deep copy of a protocol buffer.
func Clone(pb Message) Message { func Clone(src Message) Message {
in := reflect.ValueOf(pb) in := reflect.ValueOf(src)
if in.IsNil() { if in.IsNil() {
return pb return src
} }
out := reflect.New(in.Type().Elem()) out := reflect.New(in.Type().Elem())
// out is empty so a merge is a deep copy. dst := out.Interface().(Message)
mergeStruct(out.Elem(), in.Elem()) Merge(dst, src)
return out.Interface().(Message) return dst
}
// Merger is the interface representing objects that can merge messages of the same type.
type Merger interface {
// Merge merges src into this message.
// Required and optional fields that are set in src will be set to that value in dst.
// Elements of repeated fields will be appended.
//
// Merge may panic if called with a different argument type than the receiver.
Merge(src Message)
}
// generatedMerger is the custom merge method that generated protos will have.
// We must add this method since a generate Merge method will conflict with
// many existing protos that have a Merge data field already defined.
type generatedMerger interface {
XXX_Merge(src Message)
} }
// Merge merges src into dst. // Merge merges src into dst.
@ -58,17 +75,24 @@ func Clone(pb Message) Message {
// Elements of repeated fields will be appended. // Elements of repeated fields will be appended.
// Merge panics if src and dst are not the same type, or if dst is nil. // Merge panics if src and dst are not the same type, or if dst is nil.
func Merge(dst, src Message) { func Merge(dst, src Message) {
if m, ok := dst.(Merger); ok {
m.Merge(src)
return
}
in := reflect.ValueOf(src) in := reflect.ValueOf(src)
out := reflect.ValueOf(dst) out := reflect.ValueOf(dst)
if out.IsNil() { if out.IsNil() {
panic("proto: nil destination") panic("proto: nil destination")
} }
if in.Type() != out.Type() { if in.Type() != out.Type() {
// Explicit test prior to mergeStruct so that mistyped nils will fail panic(fmt.Sprintf("proto.Merge(%T, %T) type mismatch", dst, src))
panic("proto: type mismatch")
} }
if in.IsNil() { if in.IsNil() {
// Merging nil into non-nil is a quiet no-op return // Merge from nil src is a noop
}
if m, ok := dst.(generatedMerger); ok {
m.XXX_Merge(src)
return return
} }
mergeStruct(out.Elem(), in.Elem()) mergeStruct(out.Elem(), in.Elem())
@ -84,7 +108,7 @@ func mergeStruct(out, in reflect.Value) {
mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i]) mergeAny(out.Field(i), in.Field(i), false, sprop.Prop[i])
} }
if emIn, ok := extendable(in.Addr().Interface()); ok { if emIn, err := extendable(in.Addr().Interface()); err == nil {
emOut, _ := extendable(out.Addr().Interface()) emOut, _ := extendable(out.Addr().Interface())
mIn, muIn := emIn.extensionsRead() mIn, muIn := emIn.extensionsRead()
if mIn != nil { if mIn != nil {

View File

@ -39,8 +39,6 @@ import (
"errors" "errors"
"fmt" "fmt"
"io" "io"
"os"
"reflect"
) )
// errOverflow is returned when an integer is too large to be represented. // errOverflow is returned when an integer is too large to be represented.
@ -50,10 +48,6 @@ var errOverflow = errors.New("proto: integer overflow")
// wire type is encountered. It does not get returned to user code. // wire type is encountered. It does not get returned to user code.
var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for oneof") var ErrInternalBadWireType = errors.New("proto: internal error: bad wiretype for oneof")
// The fundamental decoders that interpret bytes on the wire.
// Those that take integer types all return uint64 and are
// therefore of type valueDecoder.
// DecodeVarint reads a varint-encoded integer from the slice. // DecodeVarint reads a varint-encoded integer from the slice.
// It returns the integer and the number of bytes consumed, or // It returns the integer and the number of bytes consumed, or
// zero if there is not enough. // zero if there is not enough.
@ -267,9 +261,6 @@ func (p *Buffer) DecodeZigzag32() (x uint64, err error) {
return return
} }
// These are not ValueDecoders: they produce an array of bytes or a string.
// bytes, embedded messages
// DecodeRawBytes reads a count-delimited byte buffer from the Buffer. // DecodeRawBytes reads a count-delimited byte buffer from the Buffer.
// This is the format used for the bytes protocol buffer // This is the format used for the bytes protocol buffer
// type and for embedded messages. // type and for embedded messages.
@ -311,81 +302,29 @@ func (p *Buffer) DecodeStringBytes() (s string, err error) {
return string(buf), nil return string(buf), nil
} }
// Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
// If the protocol buffer has extensions, and the field matches, add it as an extension.
// Otherwise, if the XXX_unrecognized field exists, append the skipped data there.
func (o *Buffer) skipAndSave(t reflect.Type, tag, wire int, base structPointer, unrecField field) error {
oi := o.index
err := o.skip(t, tag, wire)
if err != nil {
return err
}
if !unrecField.IsValid() {
return nil
}
ptr := structPointer_Bytes(base, unrecField)
// Add the skipped field to struct field
obuf := o.buf
o.buf = *ptr
o.EncodeVarint(uint64(tag<<3 | wire))
*ptr = append(o.buf, obuf[oi:o.index]...)
o.buf = obuf
return nil
}
// Skip the next item in the buffer. Its wire type is decoded and presented as an argument.
func (o *Buffer) skip(t reflect.Type, tag, wire int) error {
var u uint64
var err error
switch wire {
case WireVarint:
_, err = o.DecodeVarint()
case WireFixed64:
_, err = o.DecodeFixed64()
case WireBytes:
_, err = o.DecodeRawBytes(false)
case WireFixed32:
_, err = o.DecodeFixed32()
case WireStartGroup:
for {
u, err = o.DecodeVarint()
if err != nil {
break
}
fwire := int(u & 0x7)
if fwire == WireEndGroup {
break
}
ftag := int(u >> 3)
err = o.skip(t, ftag, fwire)
if err != nil {
break
}
}
default:
err = fmt.Errorf("proto: can't skip unknown wire type %d for %s", wire, t)
}
return err
}
// Unmarshaler is the interface representing objects that can // Unmarshaler is the interface representing objects that can
// unmarshal themselves. The method should reset the receiver before // unmarshal themselves. The argument points to data that may be
// decoding starts. The argument points to data that may be
// overwritten, so implementations should not keep references to the // overwritten, so implementations should not keep references to the
// buffer. // buffer.
// Unmarshal implementations should not clear the receiver.
// Any unmarshaled data should be merged into the receiver.
// Callers of Unmarshal that do not want to retain existing data
// should Reset the receiver before calling Unmarshal.
type Unmarshaler interface { type Unmarshaler interface {
Unmarshal([]byte) error Unmarshal([]byte) error
} }
// newUnmarshaler is the interface representing objects that can
// unmarshal themselves. The semantics are identical to Unmarshaler.
//
// This exists to support protoc-gen-go generated messages.
// The proto package will stop type-asserting to this interface in the future.
//
// DO NOT DEPEND ON THIS.
type newUnmarshaler interface {
XXX_Unmarshal([]byte) error
}
// Unmarshal parses the protocol buffer representation in buf and places the // Unmarshal parses the protocol buffer representation in buf and places the
// decoded result in pb. If the struct underlying pb does not match // decoded result in pb. If the struct underlying pb does not match
// the data in buf, the results can be unpredictable. // the data in buf, the results can be unpredictable.
@ -395,7 +334,13 @@ type Unmarshaler interface {
// to preserve and append to existing data. // to preserve and append to existing data.
func Unmarshal(buf []byte, pb Message) error { func Unmarshal(buf []byte, pb Message) error {
pb.Reset() pb.Reset()
return UnmarshalMerge(buf, pb) if u, ok := pb.(newUnmarshaler); ok {
return u.XXX_Unmarshal(buf)
}
if u, ok := pb.(Unmarshaler); ok {
return u.Unmarshal(buf)
}
return NewBuffer(buf).Unmarshal(pb)
} }
// UnmarshalMerge parses the protocol buffer representation in buf and // UnmarshalMerge parses the protocol buffer representation in buf and
@ -405,8 +350,16 @@ func Unmarshal(buf []byte, pb Message) error {
// UnmarshalMerge merges into existing data in pb. // UnmarshalMerge merges into existing data in pb.
// Most code should use Unmarshal instead. // Most code should use Unmarshal instead.
func UnmarshalMerge(buf []byte, pb Message) error { func UnmarshalMerge(buf []byte, pb Message) error {
// If the object can unmarshal itself, let it. if u, ok := pb.(newUnmarshaler); ok {
return u.XXX_Unmarshal(buf)
}
if u, ok := pb.(Unmarshaler); ok { if u, ok := pb.(Unmarshaler); ok {
// NOTE: The history of proto have unfortunately been inconsistent
// whether Unmarshaler should or should not implicitly clear itself.
// Some implementations do, most do not.
// Thus, calling this here may or may not do what people want.
//
// See https://github.com/golang/protobuf/issues/424
return u.Unmarshal(buf) return u.Unmarshal(buf)
} }
return NewBuffer(buf).Unmarshal(pb) return NewBuffer(buf).Unmarshal(pb)
@ -422,12 +375,17 @@ func (p *Buffer) DecodeMessage(pb Message) error {
} }
// DecodeGroup reads a tag-delimited group from the Buffer. // DecodeGroup reads a tag-delimited group from the Buffer.
// StartGroup tag is already consumed. This function consumes
// EndGroup tag.
func (p *Buffer) DecodeGroup(pb Message) error { func (p *Buffer) DecodeGroup(pb Message) error {
typ, base, err := getbase(pb) b := p.buf[p.index:]
if err != nil { x, y := findEndGroup(b)
return err if x < 0 {
return io.ErrUnexpectedEOF
} }
return p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), true, base) err := Unmarshal(b[:x], pb)
p.index += y
return err
} }
// Unmarshal parses the protocol buffer representation in the // Unmarshal parses the protocol buffer representation in the
@ -438,533 +396,33 @@ func (p *Buffer) DecodeGroup(pb Message) error {
// Unlike proto.Unmarshal, this does not reset pb before starting to unmarshal. // Unlike proto.Unmarshal, this does not reset pb before starting to unmarshal.
func (p *Buffer) Unmarshal(pb Message) error { func (p *Buffer) Unmarshal(pb Message) error {
// If the object can unmarshal itself, let it. // If the object can unmarshal itself, let it.
if u, ok := pb.(newUnmarshaler); ok {
err := u.XXX_Unmarshal(p.buf[p.index:])
p.index = len(p.buf)
return err
}
if u, ok := pb.(Unmarshaler); ok { if u, ok := pb.(Unmarshaler); ok {
// NOTE: The history of proto have unfortunately been inconsistent
// whether Unmarshaler should or should not implicitly clear itself.
// Some implementations do, most do not.
// Thus, calling this here may or may not do what people want.
//
// See https://github.com/golang/protobuf/issues/424
err := u.Unmarshal(p.buf[p.index:]) err := u.Unmarshal(p.buf[p.index:])
p.index = len(p.buf) p.index = len(p.buf)
return err return err
} }
typ, base, err := getbase(pb) // Slow workaround for messages that aren't Unmarshalers.
if err != nil { // This includes some hand-coded .pb.go files and
return err // bootstrap protos.
} // TODO: fix all of those and then add Unmarshal to
// the Message interface. Then:
err = p.unmarshalType(typ.Elem(), GetProperties(typ.Elem()), false, base) // The cast above and code below can be deleted.
// The old unmarshaler can be deleted.
if collectStats { // Clients can call Unmarshal directly (can already do that, actually).
stats.Decode++ var info InternalMessageInfo
} err := info.Unmarshal(pb, p.buf[p.index:])
p.index = len(p.buf)
return err
}
// unmarshalType does the work of unmarshaling a structure.
func (o *Buffer) unmarshalType(st reflect.Type, prop *StructProperties, is_group bool, base structPointer) error {
var state errorState
required, reqFields := prop.reqCount, uint64(0)
var err error
for err == nil && o.index < len(o.buf) {
oi := o.index
var u uint64
u, err = o.DecodeVarint()
if err != nil {
break
}
wire := int(u & 0x7)
if wire == WireEndGroup {
if is_group {
if required > 0 {
// Not enough information to determine the exact field.
// (See below.)
return &RequiredNotSetError{"{Unknown}"}
}
return nil // input is satisfied
}
return fmt.Errorf("proto: %s: wiretype end group for non-group", st)
}
tag := int(u >> 3)
if tag <= 0 {
return fmt.Errorf("proto: %s: illegal tag %d (wire type %d)", st, tag, wire)
}
fieldnum, ok := prop.decoderTags.get(tag)
if !ok {
// Maybe it's an extension?
if prop.extendable {
if e, _ := extendable(structPointer_Interface(base, st)); isExtensionField(e, int32(tag)) {
if err = o.skip(st, tag, wire); err == nil {
extmap := e.extensionsWrite()
ext := extmap[int32(tag)] // may be missing
ext.enc = append(ext.enc, o.buf[oi:o.index]...)
extmap[int32(tag)] = ext
}
continue
}
}
// Maybe it's a oneof?
if prop.oneofUnmarshaler != nil {
m := structPointer_Interface(base, st).(Message)
// First return value indicates whether tag is a oneof field.
ok, err = prop.oneofUnmarshaler(m, tag, wire, o)
if err == ErrInternalBadWireType {
// Map the error to something more descriptive.
// Do the formatting here to save generated code space.
err = fmt.Errorf("bad wiretype for oneof field in %T", m)
}
if ok {
continue
}
}
err = o.skipAndSave(st, tag, wire, base, prop.unrecField)
continue
}
p := prop.Prop[fieldnum]
if p.dec == nil {
fmt.Fprintf(os.Stderr, "proto: no protobuf decoder for %s.%s\n", st, st.Field(fieldnum).Name)
continue
}
dec := p.dec
if wire != WireStartGroup && wire != p.WireType {
if wire == WireBytes && p.packedDec != nil {
// a packable field
dec = p.packedDec
} else {
err = fmt.Errorf("proto: bad wiretype for field %s.%s: got wiretype %d, want %d", st, st.Field(fieldnum).Name, wire, p.WireType)
continue
}
}
decErr := dec(o, p, base)
if decErr != nil && !state.shouldContinue(decErr, p) {
err = decErr
}
if err == nil && p.Required {
// Successfully decoded a required field.
if tag <= 64 {
// use bitmap for fields 1-64 to catch field reuse.
var mask uint64 = 1 << uint64(tag-1)
if reqFields&mask == 0 {
// new required field
reqFields |= mask
required--
}
} else {
// This is imprecise. It can be fooled by a required field
// with a tag > 64 that is encoded twice; that's very rare.
// A fully correct implementation would require allocating
// a data structure, which we would like to avoid.
required--
}
}
}
if err == nil {
if is_group {
return io.ErrUnexpectedEOF
}
if state.err != nil {
return state.err
}
if required > 0 {
// Not enough information to determine the exact field. If we use extra
// CPU, we could determine the field only if the missing required field
// has a tag <= 64 and we check reqFields.
return &RequiredNotSetError{"{Unknown}"}
}
}
return err
}
// Individual type decoders
// For each,
// u is the decoded value,
// v is a pointer to the field (pointer) in the struct
// Sizes of the pools to allocate inside the Buffer.
// The goal is modest amortization and allocation
// on at least 16-byte boundaries.
const (
boolPoolSize = 16
uint32PoolSize = 8
uint64PoolSize = 4
)
// Decode a bool.
func (o *Buffer) dec_bool(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
if len(o.bools) == 0 {
o.bools = make([]bool, boolPoolSize)
}
o.bools[0] = u != 0
*structPointer_Bool(base, p.field) = &o.bools[0]
o.bools = o.bools[1:]
return nil
}
func (o *Buffer) dec_proto3_bool(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
*structPointer_BoolVal(base, p.field) = u != 0
return nil
}
// Decode an int32.
func (o *Buffer) dec_int32(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
word32_Set(structPointer_Word32(base, p.field), o, uint32(u))
return nil
}
func (o *Buffer) dec_proto3_int32(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
word32Val_Set(structPointer_Word32Val(base, p.field), uint32(u))
return nil
}
// Decode an int64.
func (o *Buffer) dec_int64(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
word64_Set(structPointer_Word64(base, p.field), o, u)
return nil
}
func (o *Buffer) dec_proto3_int64(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
word64Val_Set(structPointer_Word64Val(base, p.field), o, u)
return nil
}
// Decode a string.
func (o *Buffer) dec_string(p *Properties, base structPointer) error {
s, err := o.DecodeStringBytes()
if err != nil {
return err
}
*structPointer_String(base, p.field) = &s
return nil
}
func (o *Buffer) dec_proto3_string(p *Properties, base structPointer) error {
s, err := o.DecodeStringBytes()
if err != nil {
return err
}
*structPointer_StringVal(base, p.field) = s
return nil
}
// Decode a slice of bytes ([]byte).
func (o *Buffer) dec_slice_byte(p *Properties, base structPointer) error {
b, err := o.DecodeRawBytes(true)
if err != nil {
return err
}
*structPointer_Bytes(base, p.field) = b
return nil
}
// Decode a slice of bools ([]bool).
func (o *Buffer) dec_slice_bool(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
v := structPointer_BoolSlice(base, p.field)
*v = append(*v, u != 0)
return nil
}
// Decode a slice of bools ([]bool) in packed format.
func (o *Buffer) dec_slice_packed_bool(p *Properties, base structPointer) error {
v := structPointer_BoolSlice(base, p.field)
nn, err := o.DecodeVarint()
if err != nil {
return err
}
nb := int(nn) // number of bytes of encoded bools
fin := o.index + nb
if fin < o.index {
return errOverflow
}
y := *v
for o.index < fin {
u, err := p.valDec(o)
if err != nil {
return err
}
y = append(y, u != 0)
}
*v = y
return nil
}
// Decode a slice of int32s ([]int32).
func (o *Buffer) dec_slice_int32(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
structPointer_Word32Slice(base, p.field).Append(uint32(u))
return nil
}
// Decode a slice of int32s ([]int32) in packed format.
func (o *Buffer) dec_slice_packed_int32(p *Properties, base structPointer) error {
v := structPointer_Word32Slice(base, p.field)
nn, err := o.DecodeVarint()
if err != nil {
return err
}
nb := int(nn) // number of bytes of encoded int32s
fin := o.index + nb
if fin < o.index {
return errOverflow
}
for o.index < fin {
u, err := p.valDec(o)
if err != nil {
return err
}
v.Append(uint32(u))
}
return nil
}
// Decode a slice of int64s ([]int64).
func (o *Buffer) dec_slice_int64(p *Properties, base structPointer) error {
u, err := p.valDec(o)
if err != nil {
return err
}
structPointer_Word64Slice(base, p.field).Append(u)
return nil
}
// Decode a slice of int64s ([]int64) in packed format.
func (o *Buffer) dec_slice_packed_int64(p *Properties, base structPointer) error {
v := structPointer_Word64Slice(base, p.field)
nn, err := o.DecodeVarint()
if err != nil {
return err
}
nb := int(nn) // number of bytes of encoded int64s
fin := o.index + nb
if fin < o.index {
return errOverflow
}
for o.index < fin {
u, err := p.valDec(o)
if err != nil {
return err
}
v.Append(u)
}
return nil
}
// Decode a slice of strings ([]string).
func (o *Buffer) dec_slice_string(p *Properties, base structPointer) error {
s, err := o.DecodeStringBytes()
if err != nil {
return err
}
v := structPointer_StringSlice(base, p.field)
*v = append(*v, s)
return nil
}
// Decode a slice of slice of bytes ([][]byte).
func (o *Buffer) dec_slice_slice_byte(p *Properties, base structPointer) error {
b, err := o.DecodeRawBytes(true)
if err != nil {
return err
}
v := structPointer_BytesSlice(base, p.field)
*v = append(*v, b)
return nil
}
// Decode a map field.
func (o *Buffer) dec_new_map(p *Properties, base structPointer) error {
raw, err := o.DecodeRawBytes(false)
if err != nil {
return err
}
oi := o.index // index at the end of this map entry
o.index -= len(raw) // move buffer back to start of map entry
mptr := structPointer_NewAt(base, p.field, p.mtype) // *map[K]V
if mptr.Elem().IsNil() {
mptr.Elem().Set(reflect.MakeMap(mptr.Type().Elem()))
}
v := mptr.Elem() // map[K]V
// Prepare addressable doubly-indirect placeholders for the key and value types.
// See enc_new_map for why.
keyptr := reflect.New(reflect.PtrTo(p.mtype.Key())).Elem() // addressable *K
keybase := toStructPointer(keyptr.Addr()) // **K
var valbase structPointer
var valptr reflect.Value
switch p.mtype.Elem().Kind() {
case reflect.Slice:
// []byte
var dummy []byte
valptr = reflect.ValueOf(&dummy) // *[]byte
valbase = toStructPointer(valptr) // *[]byte
case reflect.Ptr:
// message; valptr is **Msg; need to allocate the intermediate pointer
valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V
valptr.Set(reflect.New(valptr.Type().Elem()))
valbase = toStructPointer(valptr)
default:
// everything else
valptr = reflect.New(reflect.PtrTo(p.mtype.Elem())).Elem() // addressable *V
valbase = toStructPointer(valptr.Addr()) // **V
}
// Decode.
// This parses a restricted wire format, namely the encoding of a message
// with two fields. See enc_new_map for the format.
for o.index < oi {
// tagcode for key and value properties are always a single byte
// because they have tags 1 and 2.
tagcode := o.buf[o.index]
o.index++
switch tagcode {
case p.mkeyprop.tagcode[0]:
if err := p.mkeyprop.dec(o, p.mkeyprop, keybase); err != nil {
return err
}
case p.mvalprop.tagcode[0]:
if err := p.mvalprop.dec(o, p.mvalprop, valbase); err != nil {
return err
}
default:
// TODO: Should we silently skip this instead?
return fmt.Errorf("proto: bad map data tag %d", raw[0])
}
}
keyelem, valelem := keyptr.Elem(), valptr.Elem()
if !keyelem.IsValid() {
keyelem = reflect.Zero(p.mtype.Key())
}
if !valelem.IsValid() {
valelem = reflect.Zero(p.mtype.Elem())
}
v.SetMapIndex(keyelem, valelem)
return nil
}
// Decode a group.
func (o *Buffer) dec_struct_group(p *Properties, base structPointer) error {
bas := structPointer_GetStructPointer(base, p.field)
if structPointer_IsNil(bas) {
// allocate new nested message
bas = toStructPointer(reflect.New(p.stype))
structPointer_SetStructPointer(base, p.field, bas)
}
return o.unmarshalType(p.stype, p.sprop, true, bas)
}
// Decode an embedded message.
func (o *Buffer) dec_struct_message(p *Properties, base structPointer) (err error) {
raw, e := o.DecodeRawBytes(false)
if e != nil {
return e
}
bas := structPointer_GetStructPointer(base, p.field)
if structPointer_IsNil(bas) {
// allocate new nested message
bas = toStructPointer(reflect.New(p.stype))
structPointer_SetStructPointer(base, p.field, bas)
}
// If the object can unmarshal itself, let it.
if p.isUnmarshaler {
iv := structPointer_Interface(bas, p.stype)
return iv.(Unmarshaler).Unmarshal(raw)
}
obuf := o.buf
oi := o.index
o.buf = raw
o.index = 0
err = o.unmarshalType(p.stype, p.sprop, false, bas)
o.buf = obuf
o.index = oi
return err
}
// Decode a slice of embedded messages.
func (o *Buffer) dec_slice_struct_message(p *Properties, base structPointer) error {
return o.dec_slice_struct(p, false, base)
}
// Decode a slice of embedded groups.
func (o *Buffer) dec_slice_struct_group(p *Properties, base structPointer) error {
return o.dec_slice_struct(p, true, base)
}
// Decode a slice of structs ([]*struct).
func (o *Buffer) dec_slice_struct(p *Properties, is_group bool, base structPointer) error {
v := reflect.New(p.stype)
bas := toStructPointer(v)
structPointer_StructPointerSlice(base, p.field).Append(bas)
if is_group {
err := o.unmarshalType(p.stype, p.sprop, is_group, bas)
return err
}
raw, err := o.DecodeRawBytes(false)
if err != nil {
return err
}
// If the object can unmarshal itself, let it.
if p.isUnmarshaler {
iv := v.Interface()
return iv.(Unmarshaler).Unmarshal(raw)
}
obuf := o.buf
oi := o.index
o.buf = raw
o.index = 0
err = o.unmarshalType(p.stype, p.sprop, is_group, bas)
o.buf = obuf
o.index = oi
return err return err
} }

View File

@ -35,8 +35,14 @@ import (
"fmt" "fmt"
"reflect" "reflect"
"strings" "strings"
"sync"
"sync/atomic"
) )
type generatedDiscarder interface {
XXX_DiscardUnknown()
}
// DiscardUnknown recursively discards all unknown fields from this message // DiscardUnknown recursively discards all unknown fields from this message
// and all embedded messages. // and all embedded messages.
// //
@ -49,9 +55,202 @@ import (
// For proto2 messages, the unknown fields of message extensions are only // For proto2 messages, the unknown fields of message extensions are only
// discarded from messages that have been accessed via GetExtension. // discarded from messages that have been accessed via GetExtension.
func DiscardUnknown(m Message) { func DiscardUnknown(m Message) {
if m, ok := m.(generatedDiscarder); ok {
m.XXX_DiscardUnknown()
return
}
// TODO: Dynamically populate a InternalMessageInfo for legacy messages,
// but the master branch has no implementation for InternalMessageInfo,
// so it would be more work to replicate that approach.
discardLegacy(m) discardLegacy(m)
} }
// DiscardUnknown recursively discards all unknown fields.
func (a *InternalMessageInfo) DiscardUnknown(m Message) {
di := atomicLoadDiscardInfo(&a.discard)
if di == nil {
di = getDiscardInfo(reflect.TypeOf(m).Elem())
atomicStoreDiscardInfo(&a.discard, di)
}
di.discard(toPointer(&m))
}
type discardInfo struct {
typ reflect.Type
initialized int32 // 0: only typ is valid, 1: everything is valid
lock sync.Mutex
fields []discardFieldInfo
unrecognized field
}
type discardFieldInfo struct {
field field // Offset of field, guaranteed to be valid
discard func(src pointer)
}
var (
discardInfoMap = map[reflect.Type]*discardInfo{}
discardInfoLock sync.Mutex
)
func getDiscardInfo(t reflect.Type) *discardInfo {
discardInfoLock.Lock()
defer discardInfoLock.Unlock()
di := discardInfoMap[t]
if di == nil {
di = &discardInfo{typ: t}
discardInfoMap[t] = di
}
return di
}
func (di *discardInfo) discard(src pointer) {
if src.isNil() {
return // Nothing to do.
}
if atomic.LoadInt32(&di.initialized) == 0 {
di.computeDiscardInfo()
}
for _, fi := range di.fields {
sfp := src.offset(fi.field)
fi.discard(sfp)
}
// For proto2 messages, only discard unknown fields in message extensions
// that have been accessed via GetExtension.
if em, err := extendable(src.asPointerTo(di.typ).Interface()); err == nil {
// Ignore lock since DiscardUnknown is not concurrency safe.
emm, _ := em.extensionsRead()
for _, mx := range emm {
if m, ok := mx.value.(Message); ok {
DiscardUnknown(m)
}
}
}
if di.unrecognized.IsValid() {
*src.offset(di.unrecognized).toBytes() = nil
}
}
func (di *discardInfo) computeDiscardInfo() {
di.lock.Lock()
defer di.lock.Unlock()
if di.initialized != 0 {
return
}
t := di.typ
n := t.NumField()
for i := 0; i < n; i++ {
f := t.Field(i)
if strings.HasPrefix(f.Name, "XXX_") {
continue
}
dfi := discardFieldInfo{field: toField(&f)}
tf := f.Type
// Unwrap tf to get its most basic type.
var isPointer, isSlice bool
if tf.Kind() == reflect.Slice && tf.Elem().Kind() != reflect.Uint8 {
isSlice = true
tf = tf.Elem()
}
if tf.Kind() == reflect.Ptr {
isPointer = true
tf = tf.Elem()
}
if isPointer && isSlice && tf.Kind() != reflect.Struct {
panic(fmt.Sprintf("%v.%s cannot be a slice of pointers to primitive types", t, f.Name))
}
switch tf.Kind() {
case reflect.Struct:
switch {
case !isPointer:
panic(fmt.Sprintf("%v.%s cannot be a direct struct value", t, f.Name))
case isSlice: // E.g., []*pb.T
di := getDiscardInfo(tf)
dfi.discard = func(src pointer) {
sps := src.getPointerSlice()
for _, sp := range sps {
if !sp.isNil() {
di.discard(sp)
}
}
}
default: // E.g., *pb.T
di := getDiscardInfo(tf)
dfi.discard = func(src pointer) {
sp := src.getPointer()
if !sp.isNil() {
di.discard(sp)
}
}
}
case reflect.Map:
switch {
case isPointer || isSlice:
panic(fmt.Sprintf("%v.%s cannot be a pointer to a map or a slice of map values", t, f.Name))
default: // E.g., map[K]V
if tf.Elem().Kind() == reflect.Ptr { // Proto struct (e.g., *T)
dfi.discard = func(src pointer) {
sm := src.asPointerTo(tf).Elem()
if sm.Len() == 0 {
return
}
for _, key := range sm.MapKeys() {
val := sm.MapIndex(key)
DiscardUnknown(val.Interface().(Message))
}
}
} else {
dfi.discard = func(pointer) {} // Noop
}
}
case reflect.Interface:
// Must be oneof field.
switch {
case isPointer || isSlice:
panic(fmt.Sprintf("%v.%s cannot be a pointer to a interface or a slice of interface values", t, f.Name))
default: // E.g., interface{}
// TODO: Make this faster?
dfi.discard = func(src pointer) {
su := src.asPointerTo(tf).Elem()
if !su.IsNil() {
sv := su.Elem().Elem().Field(0)
if sv.Kind() == reflect.Ptr && sv.IsNil() {
return
}
switch sv.Type().Kind() {
case reflect.Ptr: // Proto struct (e.g., *T)
DiscardUnknown(sv.Interface().(Message))
}
}
}
}
default:
continue
}
di.fields = append(di.fields, dfi)
}
di.unrecognized = invalidField
if f, ok := t.FieldByName("XXX_unrecognized"); ok {
if f.Type != reflect.TypeOf([]byte{}) {
panic("expected XXX_unrecognized to be of type []byte")
}
di.unrecognized = toField(&f)
}
atomic.StoreInt32(&di.initialized, 1)
}
func discardLegacy(m Message) { func discardLegacy(m Message) {
v := reflect.ValueOf(m) v := reflect.ValueOf(m)
if v.Kind() != reflect.Ptr || v.IsNil() { if v.Kind() != reflect.Ptr || v.IsNil() {
@ -139,7 +338,7 @@ func discardLegacy(m Message) {
// For proto2 messages, only discard unknown fields in message extensions // For proto2 messages, only discard unknown fields in message extensions
// that have been accessed via GetExtension. // that have been accessed via GetExtension.
if em, ok := extendable(m); ok { if em, err := extendable(m); err == nil {
// Ignore lock since discardLegacy is not concurrency safe. // Ignore lock since discardLegacy is not concurrency safe.
emm, _ := em.extensionsRead() emm, _ := em.extensionsRead()
for _, mx := range emm { for _, mx := range emm {

File diff suppressed because it is too large Load Diff

View File

@ -109,15 +109,6 @@ func equalStruct(v1, v2 reflect.Value) bool {
// set/unset mismatch // set/unset mismatch
return false return false
} }
b1, ok := f1.Interface().(raw)
if ok {
b2 := f2.Interface().(raw)
// RawMessage
if !bytes.Equal(b1.Bytes(), b2.Bytes()) {
return false
}
continue
}
f1, f2 = f1.Elem(), f2.Elem() f1, f2 = f1.Elem(), f2.Elem()
} }
if !equalAny(f1, f2, sprop.Prop[i]) { if !equalAny(f1, f2, sprop.Prop[i]) {
@ -146,11 +137,7 @@ func equalStruct(v1, v2 reflect.Value) bool {
u1 := uf.Bytes() u1 := uf.Bytes()
u2 := v2.FieldByName("XXX_unrecognized").Bytes() u2 := v2.FieldByName("XXX_unrecognized").Bytes()
if !bytes.Equal(u1, u2) { return bytes.Equal(u1, u2)
return false
}
return true
} }
// v1 and v2 are known to have the same type. // v1 and v2 are known to have the same type.
@ -261,6 +248,15 @@ func equalExtMap(base reflect.Type, em1, em2 map[int32]Extension) bool {
m1, m2 := e1.value, e2.value m1, m2 := e1.value, e2.value
if m1 == nil && m2 == nil {
// Both have only encoded form.
if bytes.Equal(e1.enc, e2.enc) {
continue
}
// The bytes are different, but the extensions might still be
// equal. We need to decode them to compare.
}
if m1 != nil && m2 != nil { if m1 != nil && m2 != nil {
// Both are unencoded. // Both are unencoded.
if !equalAny(reflect.ValueOf(m1), reflect.ValueOf(m2), nil) { if !equalAny(reflect.ValueOf(m1), reflect.ValueOf(m2), nil) {
@ -276,8 +272,12 @@ func equalExtMap(base reflect.Type, em1, em2 map[int32]Extension) bool {
desc = m[extNum] desc = m[extNum]
} }
if desc == nil { if desc == nil {
// If both have only encoded form and the bytes are the same,
// it is handled above. We get here when the bytes are different.
// We don't know how to decode it, so just compare them as byte
// slices.
log.Printf("proto: don't know how to compare extension %d of %v", extNum, base) log.Printf("proto: don't know how to compare extension %d of %v", extNum, base)
continue return false
} }
var err error var err error
if m1 == nil { if m1 == nil {

View File

@ -38,6 +38,7 @@ package proto
import ( import (
"errors" "errors"
"fmt" "fmt"
"io"
"reflect" "reflect"
"strconv" "strconv"
"sync" "sync"
@ -91,14 +92,29 @@ func (n notLocker) Unlock() {}
// extendable returns the extendableProto interface for the given generated proto message. // extendable returns the extendableProto interface for the given generated proto message.
// If the proto message has the old extension format, it returns a wrapper that implements // If the proto message has the old extension format, it returns a wrapper that implements
// the extendableProto interface. // the extendableProto interface.
func extendable(p interface{}) (extendableProto, bool) { func extendable(p interface{}) (extendableProto, error) {
if ep, ok := p.(extendableProto); ok { switch p := p.(type) {
return ep, ok case extendableProto:
if isNilPtr(p) {
return nil, fmt.Errorf("proto: nil %T is not extendable", p)
}
return p, nil
case extendableProtoV1:
if isNilPtr(p) {
return nil, fmt.Errorf("proto: nil %T is not extendable", p)
}
return extensionAdapter{p}, nil
} }
if ep, ok := p.(extendableProtoV1); ok { // Don't allocate a specific error containing %T:
return extensionAdapter{ep}, ok // this is the hot path for Clone and MarshalText.
} return nil, errNotExtendable
return nil, false }
var errNotExtendable = errors.New("proto: not an extendable proto.Message")
func isNilPtr(x interface{}) bool {
v := reflect.ValueOf(x)
return v.Kind() == reflect.Ptr && v.IsNil()
} }
// XXX_InternalExtensions is an internal representation of proto extensions. // XXX_InternalExtensions is an internal representation of proto extensions.
@ -143,9 +159,6 @@ func (e *XXX_InternalExtensions) extensionsRead() (map[int32]Extension, sync.Loc
return e.p.extensionMap, &e.p.mu return e.p.extensionMap, &e.p.mu
} }
var extendableProtoType = reflect.TypeOf((*extendableProto)(nil)).Elem()
var extendableProtoV1Type = reflect.TypeOf((*extendableProtoV1)(nil)).Elem()
// ExtensionDesc represents an extension specification. // ExtensionDesc represents an extension specification.
// Used in generated code from the protocol compiler. // Used in generated code from the protocol compiler.
type ExtensionDesc struct { type ExtensionDesc struct {
@ -179,8 +192,8 @@ type Extension struct {
// SetRawExtension is for testing only. // SetRawExtension is for testing only.
func SetRawExtension(base Message, id int32, b []byte) { func SetRawExtension(base Message, id int32, b []byte) {
epb, ok := extendable(base) epb, err := extendable(base)
if !ok { if err != nil {
return return
} }
extmap := epb.extensionsWrite() extmap := epb.extensionsWrite()
@ -205,7 +218,7 @@ func checkExtensionTypes(pb extendableProto, extension *ExtensionDesc) error {
pbi = ea.extendableProtoV1 pbi = ea.extendableProtoV1
} }
if a, b := reflect.TypeOf(pbi), reflect.TypeOf(extension.ExtendedType); a != b { if a, b := reflect.TypeOf(pbi), reflect.TypeOf(extension.ExtendedType); a != b {
return errors.New("proto: bad extended type; " + b.String() + " does not extend " + a.String()) return fmt.Errorf("proto: bad extended type; %v does not extend %v", b, a)
} }
// Check the range. // Check the range.
if !isExtensionField(pb, extension.Field) { if !isExtensionField(pb, extension.Field) {
@ -250,85 +263,11 @@ func extensionProperties(ed *ExtensionDesc) *Properties {
return prop return prop
} }
// encode encodes any unmarshaled (unencoded) extensions in e.
func encodeExtensions(e *XXX_InternalExtensions) error {
m, mu := e.extensionsRead()
if m == nil {
return nil // fast path
}
mu.Lock()
defer mu.Unlock()
return encodeExtensionsMap(m)
}
// encode encodes any unmarshaled (unencoded) extensions in e.
func encodeExtensionsMap(m map[int32]Extension) error {
for k, e := range m {
if e.value == nil || e.desc == nil {
// Extension is only in its encoded form.
continue
}
// We don't skip extensions that have an encoded form set,
// because the extension value may have been mutated after
// the last time this function was called.
et := reflect.TypeOf(e.desc.ExtensionType)
props := extensionProperties(e.desc)
p := NewBuffer(nil)
// If e.value has type T, the encoder expects a *struct{ X T }.
// Pass a *T with a zero field and hope it all works out.
x := reflect.New(et)
x.Elem().Set(reflect.ValueOf(e.value))
if err := props.enc(p, props, toStructPointer(x)); err != nil {
return err
}
e.enc = p.buf
m[k] = e
}
return nil
}
func extensionsSize(e *XXX_InternalExtensions) (n int) {
m, mu := e.extensionsRead()
if m == nil {
return 0
}
mu.Lock()
defer mu.Unlock()
return extensionsMapSize(m)
}
func extensionsMapSize(m map[int32]Extension) (n int) {
for _, e := range m {
if e.value == nil || e.desc == nil {
// Extension is only in its encoded form.
n += len(e.enc)
continue
}
// We don't skip extensions that have an encoded form set,
// because the extension value may have been mutated after
// the last time this function was called.
et := reflect.TypeOf(e.desc.ExtensionType)
props := extensionProperties(e.desc)
// If e.value has type T, the encoder expects a *struct{ X T }.
// Pass a *T with a zero field and hope it all works out.
x := reflect.New(et)
x.Elem().Set(reflect.ValueOf(e.value))
n += props.size(props, toStructPointer(x))
}
return
}
// HasExtension returns whether the given extension is present in pb. // HasExtension returns whether the given extension is present in pb.
func HasExtension(pb Message, extension *ExtensionDesc) bool { func HasExtension(pb Message, extension *ExtensionDesc) bool {
// TODO: Check types, field numbers, etc.? // TODO: Check types, field numbers, etc.?
epb, ok := extendable(pb) epb, err := extendable(pb)
if !ok { if err != nil {
return false return false
} }
extmap, mu := epb.extensionsRead() extmap, mu := epb.extensionsRead()
@ -336,15 +275,15 @@ func HasExtension(pb Message, extension *ExtensionDesc) bool {
return false return false
} }
mu.Lock() mu.Lock()
_, ok = extmap[extension.Field] _, ok := extmap[extension.Field]
mu.Unlock() mu.Unlock()
return ok return ok
} }
// ClearExtension removes the given extension from pb. // ClearExtension removes the given extension from pb.
func ClearExtension(pb Message, extension *ExtensionDesc) { func ClearExtension(pb Message, extension *ExtensionDesc) {
epb, ok := extendable(pb) epb, err := extendable(pb)
if !ok { if err != nil {
return return
} }
// TODO: Check types, field numbers, etc.? // TODO: Check types, field numbers, etc.?
@ -352,16 +291,26 @@ func ClearExtension(pb Message, extension *ExtensionDesc) {
delete(extmap, extension.Field) delete(extmap, extension.Field)
} }
// GetExtension parses and returns the given extension of pb. // GetExtension retrieves a proto2 extended field from pb.
// If the extension is not present and has no default value it returns ErrMissingExtension. //
// If the descriptor is type complete (i.e., ExtensionDesc.ExtensionType is non-nil),
// then GetExtension parses the encoded field and returns a Go value of the specified type.
// If the field is not present, then the default value is returned (if one is specified),
// otherwise ErrMissingExtension is reported.
//
// If the descriptor is not type complete (i.e., ExtensionDesc.ExtensionType is nil),
// then GetExtension returns the raw encoded bytes of the field extension.
func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) { func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) {
epb, ok := extendable(pb) epb, err := extendable(pb)
if !ok { if err != nil {
return nil, errors.New("proto: not an extendable proto") return nil, err
} }
if err := checkExtensionTypes(epb, extension); err != nil { if extension.ExtendedType != nil {
return nil, err // can only check type if this is a complete descriptor
if err := checkExtensionTypes(epb, extension); err != nil {
return nil, err
}
} }
emap, mu := epb.extensionsRead() emap, mu := epb.extensionsRead()
@ -388,6 +337,11 @@ func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) {
return e.value, nil return e.value, nil
} }
if extension.ExtensionType == nil {
// incomplete descriptor
return e.enc, nil
}
v, err := decodeExtension(e.enc, extension) v, err := decodeExtension(e.enc, extension)
if err != nil { if err != nil {
return nil, err return nil, err
@ -405,6 +359,11 @@ func GetExtension(pb Message, extension *ExtensionDesc) (interface{}, error) {
// defaultExtensionValue returns the default value for extension. // defaultExtensionValue returns the default value for extension.
// If no default for an extension is defined ErrMissingExtension is returned. // If no default for an extension is defined ErrMissingExtension is returned.
func defaultExtensionValue(extension *ExtensionDesc) (interface{}, error) { func defaultExtensionValue(extension *ExtensionDesc) (interface{}, error) {
if extension.ExtensionType == nil {
// incomplete descriptor, so no default
return nil, ErrMissingExtension
}
t := reflect.TypeOf(extension.ExtensionType) t := reflect.TypeOf(extension.ExtensionType)
props := extensionProperties(extension) props := extensionProperties(extension)
@ -439,31 +398,28 @@ func defaultExtensionValue(extension *ExtensionDesc) (interface{}, error) {
// decodeExtension decodes an extension encoded in b. // decodeExtension decodes an extension encoded in b.
func decodeExtension(b []byte, extension *ExtensionDesc) (interface{}, error) { func decodeExtension(b []byte, extension *ExtensionDesc) (interface{}, error) {
o := NewBuffer(b)
t := reflect.TypeOf(extension.ExtensionType) t := reflect.TypeOf(extension.ExtensionType)
unmarshal := typeUnmarshaler(t, extension.Tag)
props := extensionProperties(extension)
// t is a pointer to a struct, pointer to basic type or a slice. // t is a pointer to a struct, pointer to basic type or a slice.
// Allocate a "field" to store the pointer/slice itself; the // Allocate space to store the pointer/slice.
// pointer/slice will be stored here. We pass
// the address of this field to props.dec.
// This passes a zero field and a *t and lets props.dec
// interpret it as a *struct{ x t }.
value := reflect.New(t).Elem() value := reflect.New(t).Elem()
var err error
for { for {
// Discard wire type and field number varint. It isn't needed. x, n := decodeVarint(b)
if _, err := o.DecodeVarint(); err != nil { if n == 0 {
return nil, io.ErrUnexpectedEOF
}
b = b[n:]
wire := int(x) & 7
b, err = unmarshal(b, valToPointer(value.Addr()), wire)
if err != nil {
return nil, err return nil, err
} }
if err := props.dec(o, props, toStructPointer(value.Addr())); err != nil { if len(b) == 0 {
return nil, err
}
if o.index >= len(o.buf) {
break break
} }
} }
@ -473,9 +429,9 @@ func decodeExtension(b []byte, extension *ExtensionDesc) (interface{}, error) {
// GetExtensions returns a slice of the extensions present in pb that are also listed in es. // GetExtensions returns a slice of the extensions present in pb that are also listed in es.
// The returned slice has the same length as es; missing extensions will appear as nil elements. // The returned slice has the same length as es; missing extensions will appear as nil elements.
func GetExtensions(pb Message, es []*ExtensionDesc) (extensions []interface{}, err error) { func GetExtensions(pb Message, es []*ExtensionDesc) (extensions []interface{}, err error) {
epb, ok := extendable(pb) epb, err := extendable(pb)
if !ok { if err != nil {
return nil, errors.New("proto: not an extendable proto") return nil, err
} }
extensions = make([]interface{}, len(es)) extensions = make([]interface{}, len(es))
for i, e := range es { for i, e := range es {
@ -494,9 +450,9 @@ func GetExtensions(pb Message, es []*ExtensionDesc) (extensions []interface{}, e
// For non-registered extensions, ExtensionDescs returns an incomplete descriptor containing // For non-registered extensions, ExtensionDescs returns an incomplete descriptor containing
// just the Field field, which defines the extension's field number. // just the Field field, which defines the extension's field number.
func ExtensionDescs(pb Message) ([]*ExtensionDesc, error) { func ExtensionDescs(pb Message) ([]*ExtensionDesc, error) {
epb, ok := extendable(pb) epb, err := extendable(pb)
if !ok { if err != nil {
return nil, fmt.Errorf("proto: %T is not an extendable proto.Message", pb) return nil, err
} }
registeredExtensions := RegisteredExtensions(pb) registeredExtensions := RegisteredExtensions(pb)
@ -523,9 +479,9 @@ func ExtensionDescs(pb Message) ([]*ExtensionDesc, error) {
// SetExtension sets the specified extension of pb to the specified value. // SetExtension sets the specified extension of pb to the specified value.
func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error { func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error {
epb, ok := extendable(pb) epb, err := extendable(pb)
if !ok { if err != nil {
return errors.New("proto: not an extendable proto") return err
} }
if err := checkExtensionTypes(epb, extension); err != nil { if err := checkExtensionTypes(epb, extension); err != nil {
return err return err
@ -550,8 +506,8 @@ func SetExtension(pb Message, extension *ExtensionDesc, value interface{}) error
// ClearAllExtensions clears all extensions from pb. // ClearAllExtensions clears all extensions from pb.
func ClearAllExtensions(pb Message) { func ClearAllExtensions(pb Message) {
epb, ok := extendable(pb) epb, err := extendable(pb)
if !ok { if err != nil {
return return
} }
m := epb.extensionsWrite() m := epb.extensionsWrite()

View File

@ -265,6 +265,7 @@ package proto
import ( import (
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"log" "log"
"reflect" "reflect"
@ -273,6 +274,8 @@ import (
"sync" "sync"
) )
var errInvalidUTF8 = errors.New("proto: invalid UTF-8 string")
// Message is implemented by generated protocol buffer messages. // Message is implemented by generated protocol buffer messages.
type Message interface { type Message interface {
Reset() Reset()
@ -309,16 +312,7 @@ type Buffer struct {
buf []byte // encode/decode byte stream buf []byte // encode/decode byte stream
index int // read point index int // read point
// pools of basic types to amortize allocation. deterministic bool
bools []bool
uint32s []uint32
uint64s []uint64
// extra pools, only used with pointer_reflect.go
int32s []int32
int64s []int64
float32s []float32
float64s []float64
} }
// NewBuffer allocates a new Buffer and initializes its internal data to // NewBuffer allocates a new Buffer and initializes its internal data to
@ -343,6 +337,30 @@ func (p *Buffer) SetBuf(s []byte) {
// Bytes returns the contents of the Buffer. // Bytes returns the contents of the Buffer.
func (p *Buffer) Bytes() []byte { return p.buf } func (p *Buffer) Bytes() []byte { return p.buf }
// SetDeterministic sets whether to use deterministic serialization.
//
// Deterministic serialization guarantees that for a given binary, equal
// messages will always be serialized to the same bytes. This implies:
//
// - Repeated serialization of a message will return the same bytes.
// - Different processes of the same binary (which may be executing on
// different machines) will serialize equal messages to the same bytes.
//
// Note that the deterministic serialization is NOT canonical across
// languages. It is not guaranteed to remain stable over time. It is unstable
// across different builds with schema changes due to unknown fields.
// Users who need canonical serialization (e.g., persistent storage in a
// canonical form, fingerprinting, etc.) should define their own
// canonicalization specification and implement their own serializer rather
// than relying on this API.
//
// If deterministic serialization is requested, map entries will be sorted
// by keys in lexographical order. This is an implementation detail and
// subject to change.
func (p *Buffer) SetDeterministic(deterministic bool) {
p.deterministic = deterministic
}
/* /*
* Helper routines for simplifying the creation of optional fields of basic type. * Helper routines for simplifying the creation of optional fields of basic type.
*/ */
@ -831,22 +849,12 @@ func fieldDefault(ft reflect.Type, prop *Properties) (sf *scalarField, nestedMes
return sf, false, nil return sf, false, nil
} }
// mapKeys returns a sort.Interface to be used for sorting the map keys.
// Map fields may have key types of non-float scalars, strings and enums. // Map fields may have key types of non-float scalars, strings and enums.
// The easiest way to sort them in some deterministic order is to use fmt.
// If this turns out to be inefficient we can always consider other options,
// such as doing a Schwartzian transform.
func mapKeys(vs []reflect.Value) sort.Interface { func mapKeys(vs []reflect.Value) sort.Interface {
s := mapKeySorter{ s := mapKeySorter{vs: vs}
vs: vs,
// default Less function: textual comparison
less: func(a, b reflect.Value) bool {
return fmt.Sprint(a.Interface()) < fmt.Sprint(b.Interface())
},
}
// Type specialization per https://developers.google.com/protocol-buffers/docs/proto#maps; // Type specialization per https://developers.google.com/protocol-buffers/docs/proto#maps.
// numeric keys are sorted numerically.
if len(vs) == 0 { if len(vs) == 0 {
return s return s
} }
@ -855,6 +863,12 @@ func mapKeys(vs []reflect.Value) sort.Interface {
s.less = func(a, b reflect.Value) bool { return a.Int() < b.Int() } s.less = func(a, b reflect.Value) bool { return a.Int() < b.Int() }
case reflect.Uint32, reflect.Uint64: case reflect.Uint32, reflect.Uint64:
s.less = func(a, b reflect.Value) bool { return a.Uint() < b.Uint() } s.less = func(a, b reflect.Value) bool { return a.Uint() < b.Uint() }
case reflect.Bool:
s.less = func(a, b reflect.Value) bool { return !a.Bool() && b.Bool() } // false < true
case reflect.String:
s.less = func(a, b reflect.Value) bool { return a.String() < b.String() }
default:
panic(fmt.Sprintf("unsupported map key type: %v", vs[0].Kind()))
} }
return s return s
@ -895,3 +909,13 @@ const ProtoPackageIsVersion2 = true
// ProtoPackageIsVersion1 is referenced from generated protocol buffer files // ProtoPackageIsVersion1 is referenced from generated protocol buffer files
// to assert that that code is compatible with this version of the proto package. // to assert that that code is compatible with this version of the proto package.
const ProtoPackageIsVersion1 = true const ProtoPackageIsVersion1 = true
// InternalMessageInfo is a type used internally by generated .pb.go files.
// This type is not intended to be used by non-generated code.
// This type is not subject to any compatibility guarantee.
type InternalMessageInfo struct {
marshal *marshalInfo
unmarshal *unmarshalInfo
merge *mergeInfo
discard *discardInfo
}

View File

@ -42,6 +42,7 @@ import (
"fmt" "fmt"
"reflect" "reflect"
"sort" "sort"
"sync"
) )
// errNoMessageTypeID occurs when a protocol buffer does not have a message type ID. // errNoMessageTypeID occurs when a protocol buffer does not have a message type ID.
@ -94,10 +95,7 @@ func (ms *messageSet) find(pb Message) *_MessageSet_Item {
} }
func (ms *messageSet) Has(pb Message) bool { func (ms *messageSet) Has(pb Message) bool {
if ms.find(pb) != nil { return ms.find(pb) != nil
return true
}
return false
} }
func (ms *messageSet) Unmarshal(pb Message) error { func (ms *messageSet) Unmarshal(pb Message) error {
@ -150,46 +148,42 @@ func skipVarint(buf []byte) []byte {
// MarshalMessageSet encodes the extension map represented by m in the message set wire format. // MarshalMessageSet encodes the extension map represented by m in the message set wire format.
// It is called by generated Marshal methods on protocol buffer messages with the message_set_wire_format option. // It is called by generated Marshal methods on protocol buffer messages with the message_set_wire_format option.
func MarshalMessageSet(exts interface{}) ([]byte, error) { func MarshalMessageSet(exts interface{}) ([]byte, error) {
var m map[int32]Extension return marshalMessageSet(exts, false)
}
// marshaMessageSet implements above function, with the opt to turn on / off deterministic during Marshal.
func marshalMessageSet(exts interface{}, deterministic bool) ([]byte, error) {
switch exts := exts.(type) { switch exts := exts.(type) {
case *XXX_InternalExtensions: case *XXX_InternalExtensions:
if err := encodeExtensions(exts); err != nil { var u marshalInfo
return nil, err siz := u.sizeMessageSet(exts)
} b := make([]byte, 0, siz)
m, _ = exts.extensionsRead() return u.appendMessageSet(b, exts, deterministic)
case map[int32]Extension: case map[int32]Extension:
if err := encodeExtensionsMap(exts); err != nil { // This is an old-style extension map.
return nil, err // Wrap it in a new-style XXX_InternalExtensions.
ie := XXX_InternalExtensions{
p: &struct {
mu sync.Mutex
extensionMap map[int32]Extension
}{
extensionMap: exts,
},
} }
m = exts
var u marshalInfo
siz := u.sizeMessageSet(&ie)
b := make([]byte, 0, siz)
return u.appendMessageSet(b, &ie, deterministic)
default: default:
return nil, errors.New("proto: not an extension map") return nil, errors.New("proto: not an extension map")
} }
// Sort extension IDs to provide a deterministic encoding.
// See also enc_map in encode.go.
ids := make([]int, 0, len(m))
for id := range m {
ids = append(ids, int(id))
}
sort.Ints(ids)
ms := &messageSet{Item: make([]*_MessageSet_Item, 0, len(m))}
for _, id := range ids {
e := m[int32(id)]
// Remove the wire type and field number varint, as well as the length varint.
msg := skipVarint(skipVarint(e.enc))
ms.Item = append(ms.Item, &_MessageSet_Item{
TypeId: Int32(int32(id)),
Message: msg,
})
}
return Marshal(ms)
} }
// UnmarshalMessageSet decodes the extension map encoded in buf in the message set wire format. // UnmarshalMessageSet decodes the extension map encoded in buf in the message set wire format.
// It is called by generated Unmarshal methods on protocol buffer messages with the message_set_wire_format option. // It is called by Unmarshal methods on protocol buffer messages with the message_set_wire_format option.
func UnmarshalMessageSet(buf []byte, exts interface{}) error { func UnmarshalMessageSet(buf []byte, exts interface{}) error {
var m map[int32]Extension var m map[int32]Extension
switch exts := exts.(type) { switch exts := exts.(type) {
@ -235,7 +229,15 @@ func MarshalMessageSetJSON(exts interface{}) ([]byte, error) {
var m map[int32]Extension var m map[int32]Extension
switch exts := exts.(type) { switch exts := exts.(type) {
case *XXX_InternalExtensions: case *XXX_InternalExtensions:
m, _ = exts.extensionsRead() var mu sync.Locker
m, mu = exts.extensionsRead()
if m != nil {
// Keep the extensions map locked until we're done marshaling to prevent
// races between marshaling and unmarshaling the lazily-{en,de}coded
// values.
mu.Lock()
defer mu.Unlock()
}
case map[int32]Extension: case map[int32]Extension:
m = exts m = exts
default: default:
@ -253,15 +255,16 @@ func MarshalMessageSetJSON(exts interface{}) ([]byte, error) {
for i, id := range ids { for i, id := range ids {
ext := m[id] ext := m[id]
if i > 0 {
b.WriteByte(',')
}
msd, ok := messageSetMap[id] msd, ok := messageSetMap[id]
if !ok { if !ok {
// Unknown type; we can't render it, so skip it. // Unknown type; we can't render it, so skip it.
continue continue
} }
if i > 0 && b.Len() > 1 {
b.WriteByte(',')
}
fmt.Fprintf(&b, `"[%s]":`, msd.name) fmt.Fprintf(&b, `"[%s]":`, msd.name)
x := ext.value x := ext.value

View File

@ -29,7 +29,7 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// +build appengine js // +build purego appengine js
// This file contains an implementation of proto field accesses using package reflect. // This file contains an implementation of proto field accesses using package reflect.
// It is slower than the code in pointer_unsafe.go but it avoids package unsafe and can // It is slower than the code in pointer_unsafe.go but it avoids package unsafe and can
@ -38,32 +38,13 @@
package proto package proto
import ( import (
"math"
"reflect" "reflect"
"sync"
) )
// A structPointer is a pointer to a struct. const unsafeAllowed = false
type structPointer struct {
v reflect.Value
}
// toStructPointer returns a structPointer equivalent to the given reflect value. // A field identifies a field in a struct, accessible from a pointer.
// The reflect value must itself be a pointer to a struct.
func toStructPointer(v reflect.Value) structPointer {
return structPointer{v}
}
// IsNil reports whether p is nil.
func structPointer_IsNil(p structPointer) bool {
return p.v.IsNil()
}
// Interface returns the struct pointer as an interface value.
func structPointer_Interface(p structPointer, _ reflect.Type) interface{} {
return p.v.Interface()
}
// A field identifies a field in a struct, accessible from a structPointer.
// In this implementation, a field is identified by the sequence of field indices // In this implementation, a field is identified by the sequence of field indices
// passed to reflect's FieldByIndex. // passed to reflect's FieldByIndex.
type field []int type field []int
@ -76,409 +57,301 @@ func toField(f *reflect.StructField) field {
// invalidField is an invalid field identifier. // invalidField is an invalid field identifier.
var invalidField = field(nil) var invalidField = field(nil)
// zeroField is a noop when calling pointer.offset.
var zeroField = field([]int{})
// IsValid reports whether the field identifier is valid. // IsValid reports whether the field identifier is valid.
func (f field) IsValid() bool { return f != nil } func (f field) IsValid() bool { return f != nil }
// field returns the given field in the struct as a reflect value. // The pointer type is for the table-driven decoder.
func structPointer_field(p structPointer, f field) reflect.Value { // The implementation here uses a reflect.Value of pointer type to
// Special case: an extension map entry with a value of type T // create a generic pointer. In pointer_unsafe.go we use unsafe
// passes a *T to the struct-handling code with a zero field, // instead of reflect to implement the same (but faster) interface.
// expecting that it will be treated as equivalent to *struct{ X T }, type pointer struct {
// which has the same memory layout. We have to handle that case
// specially, because reflect will panic if we call FieldByIndex on a
// non-struct.
if f == nil {
return p.v.Elem()
}
return p.v.Elem().FieldByIndex(f)
}
// ifield returns the given field in the struct as an interface value.
func structPointer_ifield(p structPointer, f field) interface{} {
return structPointer_field(p, f).Addr().Interface()
}
// Bytes returns the address of a []byte field in the struct.
func structPointer_Bytes(p structPointer, f field) *[]byte {
return structPointer_ifield(p, f).(*[]byte)
}
// BytesSlice returns the address of a [][]byte field in the struct.
func structPointer_BytesSlice(p structPointer, f field) *[][]byte {
return structPointer_ifield(p, f).(*[][]byte)
}
// Bool returns the address of a *bool field in the struct.
func structPointer_Bool(p structPointer, f field) **bool {
return structPointer_ifield(p, f).(**bool)
}
// BoolVal returns the address of a bool field in the struct.
func structPointer_BoolVal(p structPointer, f field) *bool {
return structPointer_ifield(p, f).(*bool)
}
// BoolSlice returns the address of a []bool field in the struct.
func structPointer_BoolSlice(p structPointer, f field) *[]bool {
return structPointer_ifield(p, f).(*[]bool)
}
// String returns the address of a *string field in the struct.
func structPointer_String(p structPointer, f field) **string {
return structPointer_ifield(p, f).(**string)
}
// StringVal returns the address of a string field in the struct.
func structPointer_StringVal(p structPointer, f field) *string {
return structPointer_ifield(p, f).(*string)
}
// StringSlice returns the address of a []string field in the struct.
func structPointer_StringSlice(p structPointer, f field) *[]string {
return structPointer_ifield(p, f).(*[]string)
}
// Extensions returns the address of an extension map field in the struct.
func structPointer_Extensions(p structPointer, f field) *XXX_InternalExtensions {
return structPointer_ifield(p, f).(*XXX_InternalExtensions)
}
// ExtMap returns the address of an extension map field in the struct.
func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension {
return structPointer_ifield(p, f).(*map[int32]Extension)
}
// NewAt returns the reflect.Value for a pointer to a field in the struct.
func structPointer_NewAt(p structPointer, f field, typ reflect.Type) reflect.Value {
return structPointer_field(p, f).Addr()
}
// SetStructPointer writes a *struct field in the struct.
func structPointer_SetStructPointer(p structPointer, f field, q structPointer) {
structPointer_field(p, f).Set(q.v)
}
// GetStructPointer reads a *struct field in the struct.
func structPointer_GetStructPointer(p structPointer, f field) structPointer {
return structPointer{structPointer_field(p, f)}
}
// StructPointerSlice the address of a []*struct field in the struct.
func structPointer_StructPointerSlice(p structPointer, f field) structPointerSlice {
return structPointerSlice{structPointer_field(p, f)}
}
// A structPointerSlice represents the address of a slice of pointers to structs
// (themselves messages or groups). That is, v.Type() is *[]*struct{...}.
type structPointerSlice struct {
v reflect.Value v reflect.Value
} }
func (p structPointerSlice) Len() int { return p.v.Len() } // toPointer converts an interface of pointer type to a pointer
func (p structPointerSlice) Index(i int) structPointer { return structPointer{p.v.Index(i)} } // that points to the same target.
func (p structPointerSlice) Append(q structPointer) { func toPointer(i *Message) pointer {
p.v.Set(reflect.Append(p.v, q.v)) return pointer{v: reflect.ValueOf(*i)}
} }
var ( // toAddrPointer converts an interface to a pointer that points to
int32Type = reflect.TypeOf(int32(0)) // the interface data.
uint32Type = reflect.TypeOf(uint32(0)) func toAddrPointer(i *interface{}, isptr bool) pointer {
float32Type = reflect.TypeOf(float32(0)) v := reflect.ValueOf(*i)
int64Type = reflect.TypeOf(int64(0)) u := reflect.New(v.Type())
uint64Type = reflect.TypeOf(uint64(0)) u.Elem().Set(v)
float64Type = reflect.TypeOf(float64(0)) return pointer{v: u}
)
// A word32 represents a field of type *int32, *uint32, *float32, or *enum.
// That is, v.Type() is *int32, *uint32, *float32, or *enum and v is assignable.
type word32 struct {
v reflect.Value
} }
// IsNil reports whether p is nil. // valToPointer converts v to a pointer. v must be of pointer type.
func word32_IsNil(p word32) bool { func valToPointer(v reflect.Value) pointer {
return pointer{v: v}
}
// offset converts from a pointer to a structure to a pointer to
// one of its fields.
func (p pointer) offset(f field) pointer {
return pointer{v: p.v.Elem().FieldByIndex(f).Addr()}
}
func (p pointer) isNil() bool {
return p.v.IsNil() return p.v.IsNil()
} }
// Set sets p to point at a newly allocated word with bits set to x. // grow updates the slice s in place to make it one element longer.
func word32_Set(p word32, o *Buffer, x uint32) { // s must be addressable.
t := p.v.Type().Elem() // Returns the (addressable) new element.
switch t { func grow(s reflect.Value) reflect.Value {
case int32Type: n, m := s.Len(), s.Cap()
if len(o.int32s) == 0 {
o.int32s = make([]int32, uint32PoolSize)
}
o.int32s[0] = int32(x)
p.v.Set(reflect.ValueOf(&o.int32s[0]))
o.int32s = o.int32s[1:]
return
case uint32Type:
if len(o.uint32s) == 0 {
o.uint32s = make([]uint32, uint32PoolSize)
}
o.uint32s[0] = x
p.v.Set(reflect.ValueOf(&o.uint32s[0]))
o.uint32s = o.uint32s[1:]
return
case float32Type:
if len(o.float32s) == 0 {
o.float32s = make([]float32, uint32PoolSize)
}
o.float32s[0] = math.Float32frombits(x)
p.v.Set(reflect.ValueOf(&o.float32s[0]))
o.float32s = o.float32s[1:]
return
}
// must be enum
p.v.Set(reflect.New(t))
p.v.Elem().SetInt(int64(int32(x)))
}
// Get gets the bits pointed at by p, as a uint32.
func word32_Get(p word32) uint32 {
elem := p.v.Elem()
switch elem.Kind() {
case reflect.Int32:
return uint32(elem.Int())
case reflect.Uint32:
return uint32(elem.Uint())
case reflect.Float32:
return math.Float32bits(float32(elem.Float()))
}
panic("unreachable")
}
// Word32 returns a reference to a *int32, *uint32, *float32, or *enum field in the struct.
func structPointer_Word32(p structPointer, f field) word32 {
return word32{structPointer_field(p, f)}
}
// A word32Val represents a field of type int32, uint32, float32, or enum.
// That is, v.Type() is int32, uint32, float32, or enum and v is assignable.
type word32Val struct {
v reflect.Value
}
// Set sets *p to x.
func word32Val_Set(p word32Val, x uint32) {
switch p.v.Type() {
case int32Type:
p.v.SetInt(int64(x))
return
case uint32Type:
p.v.SetUint(uint64(x))
return
case float32Type:
p.v.SetFloat(float64(math.Float32frombits(x)))
return
}
// must be enum
p.v.SetInt(int64(int32(x)))
}
// Get gets the bits pointed at by p, as a uint32.
func word32Val_Get(p word32Val) uint32 {
elem := p.v
switch elem.Kind() {
case reflect.Int32:
return uint32(elem.Int())
case reflect.Uint32:
return uint32(elem.Uint())
case reflect.Float32:
return math.Float32bits(float32(elem.Float()))
}
panic("unreachable")
}
// Word32Val returns a reference to a int32, uint32, float32, or enum field in the struct.
func structPointer_Word32Val(p structPointer, f field) word32Val {
return word32Val{structPointer_field(p, f)}
}
// A word32Slice is a slice of 32-bit values.
// That is, v.Type() is []int32, []uint32, []float32, or []enum.
type word32Slice struct {
v reflect.Value
}
func (p word32Slice) Append(x uint32) {
n, m := p.v.Len(), p.v.Cap()
if n < m { if n < m {
p.v.SetLen(n + 1) s.SetLen(n + 1)
} else { } else {
t := p.v.Type().Elem() s.Set(reflect.Append(s, reflect.Zero(s.Type().Elem())))
p.v.Set(reflect.Append(p.v, reflect.Zero(t)))
} }
elem := p.v.Index(n) return s.Index(n)
switch elem.Kind() { }
case reflect.Int32:
elem.SetInt(int64(int32(x))) func (p pointer) toInt64() *int64 {
case reflect.Uint32: return p.v.Interface().(*int64)
elem.SetUint(uint64(x)) }
case reflect.Float32: func (p pointer) toInt64Ptr() **int64 {
elem.SetFloat(float64(math.Float32frombits(x))) return p.v.Interface().(**int64)
}
func (p pointer) toInt64Slice() *[]int64 {
return p.v.Interface().(*[]int64)
}
var int32ptr = reflect.TypeOf((*int32)(nil))
func (p pointer) toInt32() *int32 {
return p.v.Convert(int32ptr).Interface().(*int32)
}
// The toInt32Ptr/Slice methods don't work because of enums.
// Instead, we must use set/get methods for the int32ptr/slice case.
/*
func (p pointer) toInt32Ptr() **int32 {
return p.v.Interface().(**int32)
}
func (p pointer) toInt32Slice() *[]int32 {
return p.v.Interface().(*[]int32)
}
*/
func (p pointer) getInt32Ptr() *int32 {
if p.v.Type().Elem().Elem() == reflect.TypeOf(int32(0)) {
// raw int32 type
return p.v.Elem().Interface().(*int32)
} }
// an enum
return p.v.Elem().Convert(int32PtrType).Interface().(*int32)
}
func (p pointer) setInt32Ptr(v int32) {
// Allocate value in a *int32. Possibly convert that to a *enum.
// Then assign it to a **int32 or **enum.
// Note: we can convert *int32 to *enum, but we can't convert
// **int32 to **enum!
p.v.Elem().Set(reflect.ValueOf(&v).Convert(p.v.Type().Elem()))
} }
func (p word32Slice) Len() int { // getInt32Slice copies []int32 from p as a new slice.
return p.v.Len() // This behavior differs from the implementation in pointer_unsafe.go.
} func (p pointer) getInt32Slice() []int32 {
if p.v.Type().Elem().Elem() == reflect.TypeOf(int32(0)) {
func (p word32Slice) Index(i int) uint32 { // raw int32 type
elem := p.v.Index(i) return p.v.Elem().Interface().([]int32)
switch elem.Kind() {
case reflect.Int32:
return uint32(elem.Int())
case reflect.Uint32:
return uint32(elem.Uint())
case reflect.Float32:
return math.Float32bits(float32(elem.Float()))
} }
panic("unreachable") // an enum
// Allocate a []int32, then assign []enum's values into it.
// Note: we can't convert []enum to []int32.
slice := p.v.Elem()
s := make([]int32, slice.Len())
for i := 0; i < slice.Len(); i++ {
s[i] = int32(slice.Index(i).Int())
}
return s
} }
// Word32Slice returns a reference to a []int32, []uint32, []float32, or []enum field in the struct. // setInt32Slice copies []int32 into p as a new slice.
func structPointer_Word32Slice(p structPointer, f field) word32Slice { // This behavior differs from the implementation in pointer_unsafe.go.
return word32Slice{structPointer_field(p, f)} func (p pointer) setInt32Slice(v []int32) {
} if p.v.Type().Elem().Elem() == reflect.TypeOf(int32(0)) {
// raw int32 type
// word64 is like word32 but for 64-bit values. p.v.Elem().Set(reflect.ValueOf(v))
type word64 struct {
v reflect.Value
}
func word64_Set(p word64, o *Buffer, x uint64) {
t := p.v.Type().Elem()
switch t {
case int64Type:
if len(o.int64s) == 0 {
o.int64s = make([]int64, uint64PoolSize)
}
o.int64s[0] = int64(x)
p.v.Set(reflect.ValueOf(&o.int64s[0]))
o.int64s = o.int64s[1:]
return
case uint64Type:
if len(o.uint64s) == 0 {
o.uint64s = make([]uint64, uint64PoolSize)
}
o.uint64s[0] = x
p.v.Set(reflect.ValueOf(&o.uint64s[0]))
o.uint64s = o.uint64s[1:]
return
case float64Type:
if len(o.float64s) == 0 {
o.float64s = make([]float64, uint64PoolSize)
}
o.float64s[0] = math.Float64frombits(x)
p.v.Set(reflect.ValueOf(&o.float64s[0]))
o.float64s = o.float64s[1:]
return return
} }
panic("unreachable") // an enum
} // Allocate a []enum, then assign []int32's values into it.
// Note: we can't convert []enum to []int32.
func word64_IsNil(p word64) bool { slice := reflect.MakeSlice(p.v.Type().Elem(), len(v), cap(v))
return p.v.IsNil() for i, x := range v {
} slice.Index(i).SetInt(int64(x))
func word64_Get(p word64) uint64 {
elem := p.v.Elem()
switch elem.Kind() {
case reflect.Int64:
return uint64(elem.Int())
case reflect.Uint64:
return elem.Uint()
case reflect.Float64:
return math.Float64bits(elem.Float())
} }
panic("unreachable") p.v.Elem().Set(slice)
}
func (p pointer) appendInt32Slice(v int32) {
grow(p.v.Elem()).SetInt(int64(v))
} }
func structPointer_Word64(p structPointer, f field) word64 { func (p pointer) toUint64() *uint64 {
return word64{structPointer_field(p, f)} return p.v.Interface().(*uint64)
}
func (p pointer) toUint64Ptr() **uint64 {
return p.v.Interface().(**uint64)
}
func (p pointer) toUint64Slice() *[]uint64 {
return p.v.Interface().(*[]uint64)
}
func (p pointer) toUint32() *uint32 {
return p.v.Interface().(*uint32)
}
func (p pointer) toUint32Ptr() **uint32 {
return p.v.Interface().(**uint32)
}
func (p pointer) toUint32Slice() *[]uint32 {
return p.v.Interface().(*[]uint32)
}
func (p pointer) toBool() *bool {
return p.v.Interface().(*bool)
}
func (p pointer) toBoolPtr() **bool {
return p.v.Interface().(**bool)
}
func (p pointer) toBoolSlice() *[]bool {
return p.v.Interface().(*[]bool)
}
func (p pointer) toFloat64() *float64 {
return p.v.Interface().(*float64)
}
func (p pointer) toFloat64Ptr() **float64 {
return p.v.Interface().(**float64)
}
func (p pointer) toFloat64Slice() *[]float64 {
return p.v.Interface().(*[]float64)
}
func (p pointer) toFloat32() *float32 {
return p.v.Interface().(*float32)
}
func (p pointer) toFloat32Ptr() **float32 {
return p.v.Interface().(**float32)
}
func (p pointer) toFloat32Slice() *[]float32 {
return p.v.Interface().(*[]float32)
}
func (p pointer) toString() *string {
return p.v.Interface().(*string)
}
func (p pointer) toStringPtr() **string {
return p.v.Interface().(**string)
}
func (p pointer) toStringSlice() *[]string {
return p.v.Interface().(*[]string)
}
func (p pointer) toBytes() *[]byte {
return p.v.Interface().(*[]byte)
}
func (p pointer) toBytesSlice() *[][]byte {
return p.v.Interface().(*[][]byte)
}
func (p pointer) toExtensions() *XXX_InternalExtensions {
return p.v.Interface().(*XXX_InternalExtensions)
}
func (p pointer) toOldExtensions() *map[int32]Extension {
return p.v.Interface().(*map[int32]Extension)
}
func (p pointer) getPointer() pointer {
return pointer{v: p.v.Elem()}
}
func (p pointer) setPointer(q pointer) {
p.v.Elem().Set(q.v)
}
func (p pointer) appendPointer(q pointer) {
grow(p.v.Elem()).Set(q.v)
} }
// word64Val is like word32Val but for 64-bit values. // getPointerSlice copies []*T from p as a new []pointer.
type word64Val struct { // This behavior differs from the implementation in pointer_unsafe.go.
v reflect.Value func (p pointer) getPointerSlice() []pointer {
if p.v.IsNil() {
return nil
}
n := p.v.Elem().Len()
s := make([]pointer, n)
for i := 0; i < n; i++ {
s[i] = pointer{v: p.v.Elem().Index(i)}
}
return s
} }
func word64Val_Set(p word64Val, o *Buffer, x uint64) { // setPointerSlice copies []pointer into p as a new []*T.
switch p.v.Type() { // This behavior differs from the implementation in pointer_unsafe.go.
case int64Type: func (p pointer) setPointerSlice(v []pointer) {
p.v.SetInt(int64(x)) if v == nil {
return p.v.Elem().Set(reflect.New(p.v.Elem().Type()).Elem())
case uint64Type:
p.v.SetUint(x)
return
case float64Type:
p.v.SetFloat(math.Float64frombits(x))
return return
} }
panic("unreachable") s := reflect.MakeSlice(p.v.Elem().Type(), 0, len(v))
} for _, p := range v {
s = reflect.Append(s, p.v)
func word64Val_Get(p word64Val) uint64 {
elem := p.v
switch elem.Kind() {
case reflect.Int64:
return uint64(elem.Int())
case reflect.Uint64:
return elem.Uint()
case reflect.Float64:
return math.Float64bits(elem.Float())
} }
panic("unreachable") p.v.Elem().Set(s)
} }
func structPointer_Word64Val(p structPointer, f field) word64Val { // getInterfacePointer returns a pointer that points to the
return word64Val{structPointer_field(p, f)} // interface data of the interface pointed by p.
} func (p pointer) getInterfacePointer() pointer {
if p.v.Elem().IsNil() {
type word64Slice struct { return pointer{v: p.v.Elem()}
v reflect.Value
}
func (p word64Slice) Append(x uint64) {
n, m := p.v.Len(), p.v.Cap()
if n < m {
p.v.SetLen(n + 1)
} else {
t := p.v.Type().Elem()
p.v.Set(reflect.Append(p.v, reflect.Zero(t)))
}
elem := p.v.Index(n)
switch elem.Kind() {
case reflect.Int64:
elem.SetInt(int64(int64(x)))
case reflect.Uint64:
elem.SetUint(uint64(x))
case reflect.Float64:
elem.SetFloat(float64(math.Float64frombits(x)))
} }
return pointer{v: p.v.Elem().Elem().Elem().Field(0).Addr()} // *interface -> interface -> *struct -> struct
} }
func (p word64Slice) Len() int { func (p pointer) asPointerTo(t reflect.Type) reflect.Value {
return p.v.Len() // TODO: check that p.v.Type().Elem() == t?
return p.v
} }
func (p word64Slice) Index(i int) uint64 { func atomicLoadUnmarshalInfo(p **unmarshalInfo) *unmarshalInfo {
elem := p.v.Index(i) atomicLock.Lock()
switch elem.Kind() { defer atomicLock.Unlock()
case reflect.Int64: return *p
return uint64(elem.Int()) }
case reflect.Uint64: func atomicStoreUnmarshalInfo(p **unmarshalInfo, v *unmarshalInfo) {
return uint64(elem.Uint()) atomicLock.Lock()
case reflect.Float64: defer atomicLock.Unlock()
return math.Float64bits(float64(elem.Float())) *p = v
} }
panic("unreachable") func atomicLoadMarshalInfo(p **marshalInfo) *marshalInfo {
atomicLock.Lock()
defer atomicLock.Unlock()
return *p
}
func atomicStoreMarshalInfo(p **marshalInfo, v *marshalInfo) {
atomicLock.Lock()
defer atomicLock.Unlock()
*p = v
}
func atomicLoadMergeInfo(p **mergeInfo) *mergeInfo {
atomicLock.Lock()
defer atomicLock.Unlock()
return *p
}
func atomicStoreMergeInfo(p **mergeInfo, v *mergeInfo) {
atomicLock.Lock()
defer atomicLock.Unlock()
*p = v
}
func atomicLoadDiscardInfo(p **discardInfo) *discardInfo {
atomicLock.Lock()
defer atomicLock.Unlock()
return *p
}
func atomicStoreDiscardInfo(p **discardInfo, v *discardInfo) {
atomicLock.Lock()
defer atomicLock.Unlock()
*p = v
} }
func structPointer_Word64Slice(p structPointer, f field) word64Slice { var atomicLock sync.Mutex
return word64Slice{structPointer_field(p, f)}
}

View File

@ -29,7 +29,7 @@
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// +build !appengine,!js // +build !purego,!appengine,!js
// This file contains the implementation of the proto field accesses using package unsafe. // This file contains the implementation of the proto field accesses using package unsafe.
@ -37,38 +37,13 @@ package proto
import ( import (
"reflect" "reflect"
"sync/atomic"
"unsafe" "unsafe"
) )
// NOTE: These type_Foo functions would more idiomatically be methods, const unsafeAllowed = true
// but Go does not allow methods on pointer types, and we must preserve
// some pointer type for the garbage collector. We use these
// funcs with clunky names as our poor approximation to methods.
//
// An alternative would be
// type structPointer struct { p unsafe.Pointer }
// but that does not registerize as well.
// A structPointer is a pointer to a struct. // A field identifies a field in a struct, accessible from a pointer.
type structPointer unsafe.Pointer
// toStructPointer returns a structPointer equivalent to the given reflect value.
func toStructPointer(v reflect.Value) structPointer {
return structPointer(unsafe.Pointer(v.Pointer()))
}
// IsNil reports whether p is nil.
func structPointer_IsNil(p structPointer) bool {
return p == nil
}
// Interface returns the struct pointer, assumed to have element type t,
// as an interface value.
func structPointer_Interface(p structPointer, t reflect.Type) interface{} {
return reflect.NewAt(t, unsafe.Pointer(p)).Interface()
}
// A field identifies a field in a struct, accessible from a structPointer.
// In this implementation, a field is identified by its byte offset from the start of the struct. // In this implementation, a field is identified by its byte offset from the start of the struct.
type field uintptr type field uintptr
@ -80,191 +55,254 @@ func toField(f *reflect.StructField) field {
// invalidField is an invalid field identifier. // invalidField is an invalid field identifier.
const invalidField = ^field(0) const invalidField = ^field(0)
// zeroField is a noop when calling pointer.offset.
const zeroField = field(0)
// IsValid reports whether the field identifier is valid. // IsValid reports whether the field identifier is valid.
func (f field) IsValid() bool { func (f field) IsValid() bool {
return f != ^field(0) return f != invalidField
} }
// Bytes returns the address of a []byte field in the struct. // The pointer type below is for the new table-driven encoder/decoder.
func structPointer_Bytes(p structPointer, f field) *[]byte { // The implementation here uses unsafe.Pointer to create a generic pointer.
return (*[]byte)(unsafe.Pointer(uintptr(p) + uintptr(f))) // In pointer_reflect.go we use reflect instead of unsafe to implement
// the same (but slower) interface.
type pointer struct {
p unsafe.Pointer
} }
// BytesSlice returns the address of a [][]byte field in the struct. // size of pointer
func structPointer_BytesSlice(p structPointer, f field) *[][]byte { var ptrSize = unsafe.Sizeof(uintptr(0))
return (*[][]byte)(unsafe.Pointer(uintptr(p) + uintptr(f)))
// toPointer converts an interface of pointer type to a pointer
// that points to the same target.
func toPointer(i *Message) pointer {
// Super-tricky - read pointer out of data word of interface value.
// Saves ~25ns over the equivalent:
// return valToPointer(reflect.ValueOf(*i))
return pointer{p: (*[2]unsafe.Pointer)(unsafe.Pointer(i))[1]}
} }
// Bool returns the address of a *bool field in the struct. // toAddrPointer converts an interface to a pointer that points to
func structPointer_Bool(p structPointer, f field) **bool { // the interface data.
return (**bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) func toAddrPointer(i *interface{}, isptr bool) pointer {
} // Super-tricky - read or get the address of data word of interface value.
if isptr {
// BoolVal returns the address of a bool field in the struct. // The interface is of pointer type, thus it is a direct interface.
func structPointer_BoolVal(p structPointer, f field) *bool { // The data word is the pointer data itself. We take its address.
return (*bool)(unsafe.Pointer(uintptr(p) + uintptr(f))) return pointer{p: unsafe.Pointer(uintptr(unsafe.Pointer(i)) + ptrSize)}
}
// BoolSlice returns the address of a []bool field in the struct.
func structPointer_BoolSlice(p structPointer, f field) *[]bool {
return (*[]bool)(unsafe.Pointer(uintptr(p) + uintptr(f)))
}
// String returns the address of a *string field in the struct.
func structPointer_String(p structPointer, f field) **string {
return (**string)(unsafe.Pointer(uintptr(p) + uintptr(f)))
}
// StringVal returns the address of a string field in the struct.
func structPointer_StringVal(p structPointer, f field) *string {
return (*string)(unsafe.Pointer(uintptr(p) + uintptr(f)))
}
// StringSlice returns the address of a []string field in the struct.
func structPointer_StringSlice(p structPointer, f field) *[]string {
return (*[]string)(unsafe.Pointer(uintptr(p) + uintptr(f)))
}
// ExtMap returns the address of an extension map field in the struct.
func structPointer_Extensions(p structPointer, f field) *XXX_InternalExtensions {
return (*XXX_InternalExtensions)(unsafe.Pointer(uintptr(p) + uintptr(f)))
}
func structPointer_ExtMap(p structPointer, f field) *map[int32]Extension {
return (*map[int32]Extension)(unsafe.Pointer(uintptr(p) + uintptr(f)))
}
// NewAt returns the reflect.Value for a pointer to a field in the struct.
func structPointer_NewAt(p structPointer, f field, typ reflect.Type) reflect.Value {
return reflect.NewAt(typ, unsafe.Pointer(uintptr(p)+uintptr(f)))
}
// SetStructPointer writes a *struct field in the struct.
func structPointer_SetStructPointer(p structPointer, f field, q structPointer) {
*(*structPointer)(unsafe.Pointer(uintptr(p) + uintptr(f))) = q
}
// GetStructPointer reads a *struct field in the struct.
func structPointer_GetStructPointer(p structPointer, f field) structPointer {
return *(*structPointer)(unsafe.Pointer(uintptr(p) + uintptr(f)))
}
// StructPointerSlice the address of a []*struct field in the struct.
func structPointer_StructPointerSlice(p structPointer, f field) *structPointerSlice {
return (*structPointerSlice)(unsafe.Pointer(uintptr(p) + uintptr(f)))
}
// A structPointerSlice represents a slice of pointers to structs (themselves submessages or groups).
type structPointerSlice []structPointer
func (v *structPointerSlice) Len() int { return len(*v) }
func (v *structPointerSlice) Index(i int) structPointer { return (*v)[i] }
func (v *structPointerSlice) Append(p structPointer) { *v = append(*v, p) }
// A word32 is the address of a "pointer to 32-bit value" field.
type word32 **uint32
// IsNil reports whether *v is nil.
func word32_IsNil(p word32) bool {
return *p == nil
}
// Set sets *v to point at a newly allocated word set to x.
func word32_Set(p word32, o *Buffer, x uint32) {
if len(o.uint32s) == 0 {
o.uint32s = make([]uint32, uint32PoolSize)
} }
o.uint32s[0] = x // The interface is not of pointer type. The data word is the pointer
*p = &o.uint32s[0] // to the data.
o.uint32s = o.uint32s[1:] return pointer{p: (*[2]unsafe.Pointer)(unsafe.Pointer(i))[1]}
} }
// Get gets the value pointed at by *v. // valToPointer converts v to a pointer. v must be of pointer type.
func word32_Get(p word32) uint32 { func valToPointer(v reflect.Value) pointer {
return **p return pointer{p: unsafe.Pointer(v.Pointer())}
} }
// Word32 returns the address of a *int32, *uint32, *float32, or *enum field in the struct. // offset converts from a pointer to a structure to a pointer to
func structPointer_Word32(p structPointer, f field) word32 { // one of its fields.
return word32((**uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) func (p pointer) offset(f field) pointer {
// For safety, we should panic if !f.IsValid, however calling panic causes
// this to no longer be inlineable, which is a serious performance cost.
/*
if !f.IsValid() {
panic("invalid field")
}
*/
return pointer{p: unsafe.Pointer(uintptr(p.p) + uintptr(f))}
} }
// A word32Val is the address of a 32-bit value field. func (p pointer) isNil() bool {
type word32Val *uint32 return p.p == nil
// Set sets *p to x.
func word32Val_Set(p word32Val, x uint32) {
*p = x
} }
// Get gets the value pointed at by p. func (p pointer) toInt64() *int64 {
func word32Val_Get(p word32Val) uint32 { return (*int64)(p.p)
return *p }
func (p pointer) toInt64Ptr() **int64 {
return (**int64)(p.p)
}
func (p pointer) toInt64Slice() *[]int64 {
return (*[]int64)(p.p)
}
func (p pointer) toInt32() *int32 {
return (*int32)(p.p)
} }
// Word32Val returns the address of a *int32, *uint32, *float32, or *enum field in the struct. // See pointer_reflect.go for why toInt32Ptr/Slice doesn't exist.
func structPointer_Word32Val(p structPointer, f field) word32Val { /*
return word32Val((*uint32)(unsafe.Pointer(uintptr(p) + uintptr(f)))) func (p pointer) toInt32Ptr() **int32 {
} return (**int32)(p.p)
// A word32Slice is a slice of 32-bit values.
type word32Slice []uint32
func (v *word32Slice) Append(x uint32) { *v = append(*v, x) }
func (v *word32Slice) Len() int { return len(*v) }
func (v *word32Slice) Index(i int) uint32 { return (*v)[i] }
// Word32Slice returns the address of a []int32, []uint32, []float32, or []enum field in the struct.
func structPointer_Word32Slice(p structPointer, f field) *word32Slice {
return (*word32Slice)(unsafe.Pointer(uintptr(p) + uintptr(f)))
}
// word64 is like word32 but for 64-bit values.
type word64 **uint64
func word64_Set(p word64, o *Buffer, x uint64) {
if len(o.uint64s) == 0 {
o.uint64s = make([]uint64, uint64PoolSize)
} }
o.uint64s[0] = x func (p pointer) toInt32Slice() *[]int32 {
*p = &o.uint64s[0] return (*[]int32)(p.p)
o.uint64s = o.uint64s[1:] }
*/
func (p pointer) getInt32Ptr() *int32 {
return *(**int32)(p.p)
}
func (p pointer) setInt32Ptr(v int32) {
*(**int32)(p.p) = &v
} }
func word64_IsNil(p word64) bool { // getInt32Slice loads a []int32 from p.
return *p == nil // The value returned is aliased with the original slice.
// This behavior differs from the implementation in pointer_reflect.go.
func (p pointer) getInt32Slice() []int32 {
return *(*[]int32)(p.p)
} }
func word64_Get(p word64) uint64 { // setInt32Slice stores a []int32 to p.
return **p // The value set is aliased with the input slice.
// This behavior differs from the implementation in pointer_reflect.go.
func (p pointer) setInt32Slice(v []int32) {
*(*[]int32)(p.p) = v
} }
func structPointer_Word64(p structPointer, f field) word64 { // TODO: Can we get rid of appendInt32Slice and use setInt32Slice instead?
return word64((**uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) func (p pointer) appendInt32Slice(v int32) {
s := (*[]int32)(p.p)
*s = append(*s, v)
} }
// word64Val is like word32Val but for 64-bit values. func (p pointer) toUint64() *uint64 {
type word64Val *uint64 return (*uint64)(p.p)
}
func word64Val_Set(p word64Val, o *Buffer, x uint64) { func (p pointer) toUint64Ptr() **uint64 {
*p = x return (**uint64)(p.p)
}
func (p pointer) toUint64Slice() *[]uint64 {
return (*[]uint64)(p.p)
}
func (p pointer) toUint32() *uint32 {
return (*uint32)(p.p)
}
func (p pointer) toUint32Ptr() **uint32 {
return (**uint32)(p.p)
}
func (p pointer) toUint32Slice() *[]uint32 {
return (*[]uint32)(p.p)
}
func (p pointer) toBool() *bool {
return (*bool)(p.p)
}
func (p pointer) toBoolPtr() **bool {
return (**bool)(p.p)
}
func (p pointer) toBoolSlice() *[]bool {
return (*[]bool)(p.p)
}
func (p pointer) toFloat64() *float64 {
return (*float64)(p.p)
}
func (p pointer) toFloat64Ptr() **float64 {
return (**float64)(p.p)
}
func (p pointer) toFloat64Slice() *[]float64 {
return (*[]float64)(p.p)
}
func (p pointer) toFloat32() *float32 {
return (*float32)(p.p)
}
func (p pointer) toFloat32Ptr() **float32 {
return (**float32)(p.p)
}
func (p pointer) toFloat32Slice() *[]float32 {
return (*[]float32)(p.p)
}
func (p pointer) toString() *string {
return (*string)(p.p)
}
func (p pointer) toStringPtr() **string {
return (**string)(p.p)
}
func (p pointer) toStringSlice() *[]string {
return (*[]string)(p.p)
}
func (p pointer) toBytes() *[]byte {
return (*[]byte)(p.p)
}
func (p pointer) toBytesSlice() *[][]byte {
return (*[][]byte)(p.p)
}
func (p pointer) toExtensions() *XXX_InternalExtensions {
return (*XXX_InternalExtensions)(p.p)
}
func (p pointer) toOldExtensions() *map[int32]Extension {
return (*map[int32]Extension)(p.p)
} }
func word64Val_Get(p word64Val) uint64 { // getPointerSlice loads []*T from p as a []pointer.
return *p // The value returned is aliased with the original slice.
// This behavior differs from the implementation in pointer_reflect.go.
func (p pointer) getPointerSlice() []pointer {
// Super-tricky - p should point to a []*T where T is a
// message type. We load it as []pointer.
return *(*[]pointer)(p.p)
} }
func structPointer_Word64Val(p structPointer, f field) word64Val { // setPointerSlice stores []pointer into p as a []*T.
return word64Val((*uint64)(unsafe.Pointer(uintptr(p) + uintptr(f)))) // The value set is aliased with the input slice.
// This behavior differs from the implementation in pointer_reflect.go.
func (p pointer) setPointerSlice(v []pointer) {
// Super-tricky - p should point to a []*T where T is a
// message type. We store it as []pointer.
*(*[]pointer)(p.p) = v
} }
// word64Slice is like word32Slice but for 64-bit values. // getPointer loads the pointer at p and returns it.
type word64Slice []uint64 func (p pointer) getPointer() pointer {
return pointer{p: *(*unsafe.Pointer)(p.p)}
func (v *word64Slice) Append(x uint64) { *v = append(*v, x) } }
func (v *word64Slice) Len() int { return len(*v) }
func (v *word64Slice) Index(i int) uint64 { return (*v)[i] } // setPointer stores the pointer q at p.
func (p pointer) setPointer(q pointer) {
func structPointer_Word64Slice(p structPointer, f field) *word64Slice { *(*unsafe.Pointer)(p.p) = q.p
return (*word64Slice)(unsafe.Pointer(uintptr(p) + uintptr(f))) }
// append q to the slice pointed to by p.
func (p pointer) appendPointer(q pointer) {
s := (*[]unsafe.Pointer)(p.p)
*s = append(*s, q.p)
}
// getInterfacePointer returns a pointer that points to the
// interface data of the interface pointed by p.
func (p pointer) getInterfacePointer() pointer {
// Super-tricky - read pointer out of data word of interface value.
return pointer{p: (*(*[2]unsafe.Pointer)(p.p))[1]}
}
// asPointerTo returns a reflect.Value that is a pointer to an
// object of type t stored at p.
func (p pointer) asPointerTo(t reflect.Type) reflect.Value {
return reflect.NewAt(t, p.p)
}
func atomicLoadUnmarshalInfo(p **unmarshalInfo) *unmarshalInfo {
return (*unmarshalInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p))))
}
func atomicStoreUnmarshalInfo(p **unmarshalInfo, v *unmarshalInfo) {
atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v))
}
func atomicLoadMarshalInfo(p **marshalInfo) *marshalInfo {
return (*marshalInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p))))
}
func atomicStoreMarshalInfo(p **marshalInfo, v *marshalInfo) {
atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v))
}
func atomicLoadMergeInfo(p **mergeInfo) *mergeInfo {
return (*mergeInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p))))
}
func atomicStoreMergeInfo(p **mergeInfo, v *mergeInfo) {
atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v))
}
func atomicLoadDiscardInfo(p **discardInfo) *discardInfo {
return (*discardInfo)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(p))))
}
func atomicStoreDiscardInfo(p **discardInfo, v *discardInfo) {
atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(p)), unsafe.Pointer(v))
} }

View File

@ -58,42 +58,6 @@ const (
WireFixed32 = 5 WireFixed32 = 5
) )
const startSize = 10 // initial slice/string sizes
// Encoders are defined in encode.go
// An encoder outputs the full representation of a field, including its
// tag and encoder type.
type encoder func(p *Buffer, prop *Properties, base structPointer) error
// A valueEncoder encodes a single integer in a particular encoding.
type valueEncoder func(o *Buffer, x uint64) error
// Sizers are defined in encode.go
// A sizer returns the encoded size of a field, including its tag and encoder
// type.
type sizer func(prop *Properties, base structPointer) int
// A valueSizer returns the encoded size of a single integer in a particular
// encoding.
type valueSizer func(x uint64) int
// Decoders are defined in decode.go
// A decoder creates a value from its wire representation.
// Unrecognized subelements are saved in unrec.
type decoder func(p *Buffer, prop *Properties, base structPointer) error
// A valueDecoder decodes a single integer in a particular encoding.
type valueDecoder func(o *Buffer) (x uint64, err error)
// A oneofMarshaler does the marshaling for all oneof fields in a message.
type oneofMarshaler func(Message, *Buffer) error
// A oneofUnmarshaler does the unmarshaling for a oneof field in a message.
type oneofUnmarshaler func(Message, int, int, *Buffer) (bool, error)
// A oneofSizer does the sizing for all oneof fields in a message.
type oneofSizer func(Message) int
// tagMap is an optimization over map[int]int for typical protocol buffer // tagMap is an optimization over map[int]int for typical protocol buffer
// use-cases. Encoded protocol buffers are often in tag order with small tag // use-cases. Encoded protocol buffers are often in tag order with small tag
// numbers. // numbers.
@ -140,13 +104,6 @@ type StructProperties struct {
decoderTags tagMap // map from proto tag to struct field number decoderTags tagMap // map from proto tag to struct field number
decoderOrigNames map[string]int // map from original name to struct field number decoderOrigNames map[string]int // map from original name to struct field number
order []int // list of struct field numbers in tag order order []int // list of struct field numbers in tag order
unrecField field // field id of the XXX_unrecognized []byte field
extendable bool // is this an extendable proto
oneofMarshaler oneofMarshaler
oneofUnmarshaler oneofUnmarshaler
oneofSizer oneofSizer
stype reflect.Type
// OneofTypes contains information about the oneof fields in this message. // OneofTypes contains information about the oneof fields in this message.
// It is keyed by the original name of a field. // It is keyed by the original name of a field.
@ -187,36 +144,19 @@ type Properties struct {
Default string // default value Default string // default value
HasDefault bool // whether an explicit default was provided HasDefault bool // whether an explicit default was provided
def_uint64 uint64
enc encoder stype reflect.Type // set for struct types only
valEnc valueEncoder // set for bool and numeric types only sprop *StructProperties // set for struct types only
field field
tagcode []byte // encoding of EncodeVarint((Tag<<3)|WireType)
tagbuf [8]byte
stype reflect.Type // set for struct types only
sprop *StructProperties // set for struct types only
isMarshaler bool
isUnmarshaler bool
mtype reflect.Type // set for map types only mtype reflect.Type // set for map types only
mkeyprop *Properties // set for map types only mkeyprop *Properties // set for map types only
mvalprop *Properties // set for map types only mvalprop *Properties // set for map types only
size sizer
valSize valueSizer // set for bool and numeric types only
dec decoder
valDec valueDecoder // set for bool and numeric types only
// If this is a packable field, this will be the decoder for the packed version of the field.
packedDec decoder
} }
// String formats the properties in the protobuf struct field tag style. // String formats the properties in the protobuf struct field tag style.
func (p *Properties) String() string { func (p *Properties) String() string {
s := p.Wire s := p.Wire
s = "," s += ","
s += strconv.Itoa(p.Tag) s += strconv.Itoa(p.Tag)
if p.Required { if p.Required {
s += ",req" s += ",req"
@ -262,29 +202,14 @@ func (p *Properties) Parse(s string) {
switch p.Wire { switch p.Wire {
case "varint": case "varint":
p.WireType = WireVarint p.WireType = WireVarint
p.valEnc = (*Buffer).EncodeVarint
p.valDec = (*Buffer).DecodeVarint
p.valSize = sizeVarint
case "fixed32": case "fixed32":
p.WireType = WireFixed32 p.WireType = WireFixed32
p.valEnc = (*Buffer).EncodeFixed32
p.valDec = (*Buffer).DecodeFixed32
p.valSize = sizeFixed32
case "fixed64": case "fixed64":
p.WireType = WireFixed64 p.WireType = WireFixed64
p.valEnc = (*Buffer).EncodeFixed64
p.valDec = (*Buffer).DecodeFixed64
p.valSize = sizeFixed64
case "zigzag32": case "zigzag32":
p.WireType = WireVarint p.WireType = WireVarint
p.valEnc = (*Buffer).EncodeZigzag32
p.valDec = (*Buffer).DecodeZigzag32
p.valSize = sizeZigzag32
case "zigzag64": case "zigzag64":
p.WireType = WireVarint p.WireType = WireVarint
p.valEnc = (*Buffer).EncodeZigzag64
p.valDec = (*Buffer).DecodeZigzag64
p.valSize = sizeZigzag64
case "bytes", "group": case "bytes", "group":
p.WireType = WireBytes p.WireType = WireBytes
// no numeric converter for non-numeric types // no numeric converter for non-numeric types
@ -299,6 +224,7 @@ func (p *Properties) Parse(s string) {
return return
} }
outer:
for i := 2; i < len(fields); i++ { for i := 2; i < len(fields); i++ {
f := fields[i] f := fields[i]
switch { switch {
@ -326,229 +252,28 @@ func (p *Properties) Parse(s string) {
if i+1 < len(fields) { if i+1 < len(fields) {
// Commas aren't escaped, and def is always last. // Commas aren't escaped, and def is always last.
p.Default += "," + strings.Join(fields[i+1:], ",") p.Default += "," + strings.Join(fields[i+1:], ",")
break break outer
} }
} }
} }
} }
func logNoSliceEnc(t1, t2 reflect.Type) {
fmt.Fprintf(os.Stderr, "proto: no slice oenc for %T = []%T\n", t1, t2)
}
var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem() var protoMessageType = reflect.TypeOf((*Message)(nil)).Elem()
// Initialize the fields for encoding and decoding. // setFieldProps initializes the field properties for submessages and maps.
func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lockGetProp bool) { func (p *Properties) setFieldProps(typ reflect.Type, f *reflect.StructField, lockGetProp bool) {
p.enc = nil
p.dec = nil
p.size = nil
switch t1 := typ; t1.Kind() { switch t1 := typ; t1.Kind() {
default:
fmt.Fprintf(os.Stderr, "proto: no coders for %v\n", t1)
// proto3 scalar types
case reflect.Bool:
p.enc = (*Buffer).enc_proto3_bool
p.dec = (*Buffer).dec_proto3_bool
p.size = size_proto3_bool
case reflect.Int32:
p.enc = (*Buffer).enc_proto3_int32
p.dec = (*Buffer).dec_proto3_int32
p.size = size_proto3_int32
case reflect.Uint32:
p.enc = (*Buffer).enc_proto3_uint32
p.dec = (*Buffer).dec_proto3_int32 // can reuse
p.size = size_proto3_uint32
case reflect.Int64, reflect.Uint64:
p.enc = (*Buffer).enc_proto3_int64
p.dec = (*Buffer).dec_proto3_int64
p.size = size_proto3_int64
case reflect.Float32:
p.enc = (*Buffer).enc_proto3_uint32 // can just treat them as bits
p.dec = (*Buffer).dec_proto3_int32
p.size = size_proto3_uint32
case reflect.Float64:
p.enc = (*Buffer).enc_proto3_int64 // can just treat them as bits
p.dec = (*Buffer).dec_proto3_int64
p.size = size_proto3_int64
case reflect.String:
p.enc = (*Buffer).enc_proto3_string
p.dec = (*Buffer).dec_proto3_string
p.size = size_proto3_string
case reflect.Ptr: case reflect.Ptr:
switch t2 := t1.Elem(); t2.Kind() { if t1.Elem().Kind() == reflect.Struct {
default:
fmt.Fprintf(os.Stderr, "proto: no encoder function for %v -> %v\n", t1, t2)
break
case reflect.Bool:
p.enc = (*Buffer).enc_bool
p.dec = (*Buffer).dec_bool
p.size = size_bool
case reflect.Int32:
p.enc = (*Buffer).enc_int32
p.dec = (*Buffer).dec_int32
p.size = size_int32
case reflect.Uint32:
p.enc = (*Buffer).enc_uint32
p.dec = (*Buffer).dec_int32 // can reuse
p.size = size_uint32
case reflect.Int64, reflect.Uint64:
p.enc = (*Buffer).enc_int64
p.dec = (*Buffer).dec_int64
p.size = size_int64
case reflect.Float32:
p.enc = (*Buffer).enc_uint32 // can just treat them as bits
p.dec = (*Buffer).dec_int32
p.size = size_uint32
case reflect.Float64:
p.enc = (*Buffer).enc_int64 // can just treat them as bits
p.dec = (*Buffer).dec_int64
p.size = size_int64
case reflect.String:
p.enc = (*Buffer).enc_string
p.dec = (*Buffer).dec_string
p.size = size_string
case reflect.Struct:
p.stype = t1.Elem() p.stype = t1.Elem()
p.isMarshaler = isMarshaler(t1)
p.isUnmarshaler = isUnmarshaler(t1)
if p.Wire == "bytes" {
p.enc = (*Buffer).enc_struct_message
p.dec = (*Buffer).dec_struct_message
p.size = size_struct_message
} else {
p.enc = (*Buffer).enc_struct_group
p.dec = (*Buffer).dec_struct_group
p.size = size_struct_group
}
} }
case reflect.Slice: case reflect.Slice:
switch t2 := t1.Elem(); t2.Kind() { if t2 := t1.Elem(); t2.Kind() == reflect.Ptr && t2.Elem().Kind() == reflect.Struct {
default: p.stype = t2.Elem()
logNoSliceEnc(t1, t2)
break
case reflect.Bool:
if p.Packed {
p.enc = (*Buffer).enc_slice_packed_bool
p.size = size_slice_packed_bool
} else {
p.enc = (*Buffer).enc_slice_bool
p.size = size_slice_bool
}
p.dec = (*Buffer).dec_slice_bool
p.packedDec = (*Buffer).dec_slice_packed_bool
case reflect.Int32:
if p.Packed {
p.enc = (*Buffer).enc_slice_packed_int32
p.size = size_slice_packed_int32
} else {
p.enc = (*Buffer).enc_slice_int32
p.size = size_slice_int32
}
p.dec = (*Buffer).dec_slice_int32
p.packedDec = (*Buffer).dec_slice_packed_int32
case reflect.Uint32:
if p.Packed {
p.enc = (*Buffer).enc_slice_packed_uint32
p.size = size_slice_packed_uint32
} else {
p.enc = (*Buffer).enc_slice_uint32
p.size = size_slice_uint32
}
p.dec = (*Buffer).dec_slice_int32
p.packedDec = (*Buffer).dec_slice_packed_int32
case reflect.Int64, reflect.Uint64:
if p.Packed {
p.enc = (*Buffer).enc_slice_packed_int64
p.size = size_slice_packed_int64
} else {
p.enc = (*Buffer).enc_slice_int64
p.size = size_slice_int64
}
p.dec = (*Buffer).dec_slice_int64
p.packedDec = (*Buffer).dec_slice_packed_int64
case reflect.Uint8:
p.dec = (*Buffer).dec_slice_byte
if p.proto3 {
p.enc = (*Buffer).enc_proto3_slice_byte
p.size = size_proto3_slice_byte
} else {
p.enc = (*Buffer).enc_slice_byte
p.size = size_slice_byte
}
case reflect.Float32, reflect.Float64:
switch t2.Bits() {
case 32:
// can just treat them as bits
if p.Packed {
p.enc = (*Buffer).enc_slice_packed_uint32
p.size = size_slice_packed_uint32
} else {
p.enc = (*Buffer).enc_slice_uint32
p.size = size_slice_uint32
}
p.dec = (*Buffer).dec_slice_int32
p.packedDec = (*Buffer).dec_slice_packed_int32
case 64:
// can just treat them as bits
if p.Packed {
p.enc = (*Buffer).enc_slice_packed_int64
p.size = size_slice_packed_int64
} else {
p.enc = (*Buffer).enc_slice_int64
p.size = size_slice_int64
}
p.dec = (*Buffer).dec_slice_int64
p.packedDec = (*Buffer).dec_slice_packed_int64
default:
logNoSliceEnc(t1, t2)
break
}
case reflect.String:
p.enc = (*Buffer).enc_slice_string
p.dec = (*Buffer).dec_slice_string
p.size = size_slice_string
case reflect.Ptr:
switch t3 := t2.Elem(); t3.Kind() {
default:
fmt.Fprintf(os.Stderr, "proto: no ptr oenc for %T -> %T -> %T\n", t1, t2, t3)
break
case reflect.Struct:
p.stype = t2.Elem()
p.isMarshaler = isMarshaler(t2)
p.isUnmarshaler = isUnmarshaler(t2)
if p.Wire == "bytes" {
p.enc = (*Buffer).enc_slice_struct_message
p.dec = (*Buffer).dec_slice_struct_message
p.size = size_slice_struct_message
} else {
p.enc = (*Buffer).enc_slice_struct_group
p.dec = (*Buffer).dec_slice_struct_group
p.size = size_slice_struct_group
}
}
case reflect.Slice:
switch t2.Elem().Kind() {
default:
fmt.Fprintf(os.Stderr, "proto: no slice elem oenc for %T -> %T -> %T\n", t1, t2, t2.Elem())
break
case reflect.Uint8:
p.enc = (*Buffer).enc_slice_slice_byte
p.dec = (*Buffer).dec_slice_slice_byte
p.size = size_slice_slice_byte
}
} }
case reflect.Map: case reflect.Map:
p.enc = (*Buffer).enc_new_map
p.dec = (*Buffer).dec_new_map
p.size = size_new_map
p.mtype = t1 p.mtype = t1
p.mkeyprop = &Properties{} p.mkeyprop = &Properties{}
p.mkeyprop.init(reflect.PtrTo(p.mtype.Key()), "Key", f.Tag.Get("protobuf_key"), nil, lockGetProp) p.mkeyprop.init(reflect.PtrTo(p.mtype.Key()), "Key", f.Tag.Get("protobuf_key"), nil, lockGetProp)
@ -562,20 +287,6 @@ func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lock
p.mvalprop.init(vtype, "Value", f.Tag.Get("protobuf_val"), nil, lockGetProp) p.mvalprop.init(vtype, "Value", f.Tag.Get("protobuf_val"), nil, lockGetProp)
} }
// precalculate tag code
wire := p.WireType
if p.Packed {
wire = WireBytes
}
x := uint32(p.Tag)<<3 | uint32(wire)
i := 0
for i = 0; x > 127; i++ {
p.tagbuf[i] = 0x80 | uint8(x&0x7F)
x >>= 7
}
p.tagbuf[i] = uint8(x)
p.tagcode = p.tagbuf[0 : i+1]
if p.stype != nil { if p.stype != nil {
if lockGetProp { if lockGetProp {
p.sprop = GetProperties(p.stype) p.sprop = GetProperties(p.stype)
@ -586,32 +297,9 @@ func (p *Properties) setEncAndDec(typ reflect.Type, f *reflect.StructField, lock
} }
var ( var (
marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem() marshalerType = reflect.TypeOf((*Marshaler)(nil)).Elem()
unmarshalerType = reflect.TypeOf((*Unmarshaler)(nil)).Elem()
) )
// isMarshaler reports whether type t implements Marshaler.
func isMarshaler(t reflect.Type) bool {
// We're checking for (likely) pointer-receiver methods
// so if t is not a pointer, something is very wrong.
// The calls above only invoke isMarshaler on pointer types.
if t.Kind() != reflect.Ptr {
panic("proto: misuse of isMarshaler")
}
return t.Implements(marshalerType)
}
// isUnmarshaler reports whether type t implements Unmarshaler.
func isUnmarshaler(t reflect.Type) bool {
// We're checking for (likely) pointer-receiver methods
// so if t is not a pointer, something is very wrong.
// The calls above only invoke isUnmarshaler on pointer types.
if t.Kind() != reflect.Ptr {
panic("proto: misuse of isUnmarshaler")
}
return t.Implements(unmarshalerType)
}
// Init populates the properties from a protocol buffer struct tag. // Init populates the properties from a protocol buffer struct tag.
func (p *Properties) Init(typ reflect.Type, name, tag string, f *reflect.StructField) { func (p *Properties) Init(typ reflect.Type, name, tag string, f *reflect.StructField) {
p.init(typ, name, tag, f, true) p.init(typ, name, tag, f, true)
@ -621,14 +309,11 @@ func (p *Properties) init(typ reflect.Type, name, tag string, f *reflect.StructF
// "bytes,49,opt,def=hello!" // "bytes,49,opt,def=hello!"
p.Name = name p.Name = name
p.OrigName = name p.OrigName = name
if f != nil {
p.field = toField(f)
}
if tag == "" { if tag == "" {
return return
} }
p.Parse(tag) p.Parse(tag)
p.setEncAndDec(typ, f, lockGetProp) p.setFieldProps(typ, f, lockGetProp)
} }
var ( var (
@ -678,9 +363,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties {
propertiesMap[t] = prop propertiesMap[t] = prop
// build properties // build properties
prop.extendable = reflect.PtrTo(t).Implements(extendableProtoType) ||
reflect.PtrTo(t).Implements(extendableProtoV1Type)
prop.unrecField = invalidField
prop.Prop = make([]*Properties, t.NumField()) prop.Prop = make([]*Properties, t.NumField())
prop.order = make([]int, t.NumField()) prop.order = make([]int, t.NumField())
@ -690,17 +372,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties {
name := f.Name name := f.Name
p.init(f.Type, name, f.Tag.Get("protobuf"), &f, false) p.init(f.Type, name, f.Tag.Get("protobuf"), &f, false)
if f.Name == "XXX_InternalExtensions" { // special case
p.enc = (*Buffer).enc_exts
p.dec = nil // not needed
p.size = size_exts
} else if f.Name == "XXX_extensions" { // special case
p.enc = (*Buffer).enc_map
p.dec = nil // not needed
p.size = size_map
} else if f.Name == "XXX_unrecognized" { // special case
prop.unrecField = toField(&f)
}
oneof := f.Tag.Get("protobuf_oneof") // special case oneof := f.Tag.Get("protobuf_oneof") // special case
if oneof != "" { if oneof != "" {
// Oneof fields don't use the traditional protobuf tag. // Oneof fields don't use the traditional protobuf tag.
@ -715,9 +386,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties {
} }
print("\n") print("\n")
} }
if p.enc == nil && !strings.HasPrefix(f.Name, "XXX_") && oneof == "" {
fmt.Fprintln(os.Stderr, "proto: no encoder for", f.Name, f.Type.String(), "[GetProperties]")
}
} }
// Re-order prop.order. // Re-order prop.order.
@ -728,8 +396,7 @@ func getPropertiesLocked(t reflect.Type) *StructProperties {
} }
if om, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); ok { if om, ok := reflect.Zero(reflect.PtrTo(t)).Interface().(oneofMessage); ok {
var oots []interface{} var oots []interface{}
prop.oneofMarshaler, prop.oneofUnmarshaler, prop.oneofSizer, oots = om.XXX_OneofFuncs() _, _, _, oots = om.XXX_OneofFuncs()
prop.stype = t
// Interpret oneof metadata. // Interpret oneof metadata.
prop.OneofTypes = make(map[string]*OneofProperties) prop.OneofTypes = make(map[string]*OneofProperties)
@ -779,30 +446,6 @@ func getPropertiesLocked(t reflect.Type) *StructProperties {
return prop return prop
} }
// Return the Properties object for the x[0]'th field of the structure.
func propByIndex(t reflect.Type, x []int) *Properties {
if len(x) != 1 {
fmt.Fprintf(os.Stderr, "proto: field index dimension %d (not 1) for type %s\n", len(x), t)
return nil
}
prop := GetProperties(t)
return prop.Prop[x[0]]
}
// Get the address and type of a pointer to a struct from an interface.
func getbase(pb Message) (t reflect.Type, b structPointer, err error) {
if pb == nil {
err = ErrNil
return
}
// get the reflect type of the pointer to the struct.
t = reflect.TypeOf(pb)
// get the address of the struct.
value := reflect.ValueOf(pb)
b = toStructPointer(value)
return
}
// A global registry of enum types. // A global registry of enum types.
// The generated code will register the generated maps by calling RegisterEnum. // The generated code will register the generated maps by calling RegisterEnum.
@ -826,20 +469,42 @@ func EnumValueMap(enumType string) map[string]int32 {
// A registry of all linked message types. // A registry of all linked message types.
// The string is a fully-qualified proto name ("pkg.Message"). // The string is a fully-qualified proto name ("pkg.Message").
var ( var (
protoTypes = make(map[string]reflect.Type) protoTypedNils = make(map[string]Message) // a map from proto names to typed nil pointers
revProtoTypes = make(map[reflect.Type]string) protoMapTypes = make(map[string]reflect.Type) // a map from proto names to map types
revProtoTypes = make(map[reflect.Type]string)
) )
// RegisterType is called from generated code and maps from the fully qualified // RegisterType is called from generated code and maps from the fully qualified
// proto name to the type (pointer to struct) of the protocol buffer. // proto name to the type (pointer to struct) of the protocol buffer.
func RegisterType(x Message, name string) { func RegisterType(x Message, name string) {
if _, ok := protoTypes[name]; ok { if _, ok := protoTypedNils[name]; ok {
// TODO: Some day, make this a panic. // TODO: Some day, make this a panic.
log.Printf("proto: duplicate proto type registered: %s", name) log.Printf("proto: duplicate proto type registered: %s", name)
return return
} }
t := reflect.TypeOf(x) t := reflect.TypeOf(x)
protoTypes[name] = t if v := reflect.ValueOf(x); v.Kind() == reflect.Ptr && v.Pointer() == 0 {
// Generated code always calls RegisterType with nil x.
// This check is just for extra safety.
protoTypedNils[name] = x
} else {
protoTypedNils[name] = reflect.Zero(t).Interface().(Message)
}
revProtoTypes[t] = name
}
// RegisterMapType is called from generated code and maps from the fully qualified
// proto name to the native map type of the proto map definition.
func RegisterMapType(x interface{}, name string) {
if reflect.TypeOf(x).Kind() != reflect.Map {
panic(fmt.Sprintf("RegisterMapType(%T, %q); want map", x, name))
}
if _, ok := protoMapTypes[name]; ok {
log.Printf("proto: duplicate proto type registered: %s", name)
return
}
t := reflect.TypeOf(x)
protoMapTypes[name] = t
revProtoTypes[t] = name revProtoTypes[t] = name
} }
@ -855,7 +520,14 @@ func MessageName(x Message) string {
} }
// MessageType returns the message type (pointer to struct) for a named message. // MessageType returns the message type (pointer to struct) for a named message.
func MessageType(name string) reflect.Type { return protoTypes[name] } // The type is not guaranteed to implement proto.Message if the name refers to a
// map entry.
func MessageType(name string) reflect.Type {
if t, ok := protoTypedNils[name]; ok {
return reflect.TypeOf(t)
}
return protoMapTypes[name]
}
// A registry of all linked proto files. // A registry of all linked proto files.
var ( var (

2681
vendor/github.com/golang/protobuf/proto/table_marshal.go generated vendored Normal file

File diff suppressed because it is too large Load Diff

654
vendor/github.com/golang/protobuf/proto/table_merge.go generated vendored Normal file
View File

@ -0,0 +1,654 @@
// Go support for Protocol Buffers - Google's data interchange format
//
// Copyright 2016 The Go Authors. All rights reserved.
// https://github.com/golang/protobuf
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// * Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
// * Redistributions in binary form must reproduce the above
// copyright notice, this list of conditions and the following disclaimer
// in the documentation and/or other materials provided with the
// distribution.
// * Neither the name of Google Inc. nor the names of its
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package proto
import (
"fmt"
"reflect"
"strings"
"sync"
"sync/atomic"
)
// Merge merges the src message into dst.
// This assumes that dst and src of the same type and are non-nil.
func (a *InternalMessageInfo) Merge(dst, src Message) {
mi := atomicLoadMergeInfo(&a.merge)
if mi == nil {
mi = getMergeInfo(reflect.TypeOf(dst).Elem())
atomicStoreMergeInfo(&a.merge, mi)
}
mi.merge(toPointer(&dst), toPointer(&src))
}
type mergeInfo struct {
typ reflect.Type
initialized int32 // 0: only typ is valid, 1: everything is valid
lock sync.Mutex
fields []mergeFieldInfo
unrecognized field // Offset of XXX_unrecognized
}
type mergeFieldInfo struct {
field field // Offset of field, guaranteed to be valid
// isPointer reports whether the value in the field is a pointer.
// This is true for the following situations:
// * Pointer to struct
// * Pointer to basic type (proto2 only)
// * Slice (first value in slice header is a pointer)
// * String (first value in string header is a pointer)
isPointer bool
// basicWidth reports the width of the field assuming that it is directly
// embedded in the struct (as is the case for basic types in proto3).
// The possible values are:
// 0: invalid
// 1: bool
// 4: int32, uint32, float32
// 8: int64, uint64, float64
basicWidth int
// Where dst and src are pointers to the types being merged.
merge func(dst, src pointer)
}
var (
mergeInfoMap = map[reflect.Type]*mergeInfo{}
mergeInfoLock sync.Mutex
)
func getMergeInfo(t reflect.Type) *mergeInfo {
mergeInfoLock.Lock()
defer mergeInfoLock.Unlock()
mi := mergeInfoMap[t]
if mi == nil {
mi = &mergeInfo{typ: t}
mergeInfoMap[t] = mi
}
return mi
}
// merge merges src into dst assuming they are both of type *mi.typ.
func (mi *mergeInfo) merge(dst, src pointer) {
if dst.isNil() {
panic("proto: nil destination")
}
if src.isNil() {
return // Nothing to do.
}
if atomic.LoadInt32(&mi.initialized) == 0 {
mi.computeMergeInfo()
}
for _, fi := range mi.fields {
sfp := src.offset(fi.field)
// As an optimization, we can avoid the merge function call cost
// if we know for sure that the source will have no effect
// by checking if it is the zero value.
if unsafeAllowed {
if fi.isPointer && sfp.getPointer().isNil() { // Could be slice or string
continue
}
if fi.basicWidth > 0 {
switch {
case fi.basicWidth == 1 && !*sfp.toBool():
continue
case fi.basicWidth == 4 && *sfp.toUint32() == 0:
continue
case fi.basicWidth == 8 && *sfp.toUint64() == 0:
continue
}
}
}
dfp := dst.offset(fi.field)
fi.merge(dfp, sfp)
}
// TODO: Make this faster?
out := dst.asPointerTo(mi.typ).Elem()
in := src.asPointerTo(mi.typ).Elem()
if emIn, err := extendable(in.Addr().Interface()); err == nil {
emOut, _ := extendable(out.Addr().Interface())
mIn, muIn := emIn.extensionsRead()
if mIn != nil {
mOut := emOut.extensionsWrite()
muIn.Lock()
mergeExtension(mOut, mIn)
muIn.Unlock()
}
}
if mi.unrecognized.IsValid() {
if b := *src.offset(mi.unrecognized).toBytes(); len(b) > 0 {
*dst.offset(mi.unrecognized).toBytes() = append([]byte(nil), b...)
}
}
}
func (mi *mergeInfo) computeMergeInfo() {
mi.lock.Lock()
defer mi.lock.Unlock()
if mi.initialized != 0 {
return
}
t := mi.typ
n := t.NumField()
props := GetProperties(t)
for i := 0; i < n; i++ {
f := t.Field(i)
if strings.HasPrefix(f.Name, "XXX_") {
continue
}
mfi := mergeFieldInfo{field: toField(&f)}
tf := f.Type
// As an optimization, we can avoid the merge function call cost
// if we know for sure that the source will have no effect
// by checking if it is the zero value.
if unsafeAllowed {
switch tf.Kind() {
case reflect.Ptr, reflect.Slice, reflect.String:
// As a special case, we assume slices and strings are pointers
// since we know that the first field in the SliceSlice or
// StringHeader is a data pointer.
mfi.isPointer = true
case reflect.Bool:
mfi.basicWidth = 1
case reflect.Int32, reflect.Uint32, reflect.Float32:
mfi.basicWidth = 4
case reflect.Int64, reflect.Uint64, reflect.Float64:
mfi.basicWidth = 8
}
}
// Unwrap tf to get at its most basic type.
var isPointer, isSlice bool
if tf.Kind() == reflect.Slice && tf.Elem().Kind() != reflect.Uint8 {
isSlice = true
tf = tf.Elem()
}
if tf.Kind() == reflect.Ptr {
isPointer = true
tf = tf.Elem()
}
if isPointer && isSlice && tf.Kind() != reflect.Struct {
panic("both pointer and slice for basic type in " + tf.Name())
}
switch tf.Kind() {
case reflect.Int32:
switch {
case isSlice: // E.g., []int32
mfi.merge = func(dst, src pointer) {
// NOTE: toInt32Slice is not defined (see pointer_reflect.go).
/*
sfsp := src.toInt32Slice()
if *sfsp != nil {
dfsp := dst.toInt32Slice()
*dfsp = append(*dfsp, *sfsp...)
if *dfsp == nil {
*dfsp = []int64{}
}
}
*/
sfs := src.getInt32Slice()
if sfs != nil {
dfs := dst.getInt32Slice()
dfs = append(dfs, sfs...)
if dfs == nil {
dfs = []int32{}
}
dst.setInt32Slice(dfs)
}
}
case isPointer: // E.g., *int32
mfi.merge = func(dst, src pointer) {
// NOTE: toInt32Ptr is not defined (see pointer_reflect.go).
/*
sfpp := src.toInt32Ptr()
if *sfpp != nil {
dfpp := dst.toInt32Ptr()
if *dfpp == nil {
*dfpp = Int32(**sfpp)
} else {
**dfpp = **sfpp
}
}
*/
sfp := src.getInt32Ptr()
if sfp != nil {
dfp := dst.getInt32Ptr()
if dfp == nil {
dst.setInt32Ptr(*sfp)
} else {
*dfp = *sfp
}
}
}
default: // E.g., int32
mfi.merge = func(dst, src pointer) {
if v := *src.toInt32(); v != 0 {
*dst.toInt32() = v
}
}
}
case reflect.Int64:
switch {
case isSlice: // E.g., []int64
mfi.merge = func(dst, src pointer) {
sfsp := src.toInt64Slice()
if *sfsp != nil {
dfsp := dst.toInt64Slice()
*dfsp = append(*dfsp, *sfsp...)
if *dfsp == nil {
*dfsp = []int64{}
}
}
}
case isPointer: // E.g., *int64
mfi.merge = func(dst, src pointer) {
sfpp := src.toInt64Ptr()
if *sfpp != nil {
dfpp := dst.toInt64Ptr()
if *dfpp == nil {
*dfpp = Int64(**sfpp)
} else {
**dfpp = **sfpp
}
}
}
default: // E.g., int64
mfi.merge = func(dst, src pointer) {
if v := *src.toInt64(); v != 0 {
*dst.toInt64() = v
}
}
}
case reflect.Uint32:
switch {
case isSlice: // E.g., []uint32
mfi.merge = func(dst, src pointer) {
sfsp := src.toUint32Slice()
if *sfsp != nil {
dfsp := dst.toUint32Slice()
*dfsp = append(*dfsp, *sfsp...)
if *dfsp == nil {
*dfsp = []uint32{}
}
}
}
case isPointer: // E.g., *uint32
mfi.merge = func(dst, src pointer) {
sfpp := src.toUint32Ptr()
if *sfpp != nil {
dfpp := dst.toUint32Ptr()
if *dfpp == nil {
*dfpp = Uint32(**sfpp)
} else {
**dfpp = **sfpp
}
}
}
default: // E.g., uint32
mfi.merge = func(dst, src pointer) {
if v := *src.toUint32(); v != 0 {
*dst.toUint32() = v
}
}
}
case reflect.Uint64:
switch {
case isSlice: // E.g., []uint64
mfi.merge = func(dst, src pointer) {
sfsp := src.toUint64Slice()
if *sfsp != nil {
dfsp := dst.toUint64Slice()
*dfsp = append(*dfsp, *sfsp...)
if *dfsp == nil {
*dfsp = []uint64{}
}
}
}
case isPointer: // E.g., *uint64
mfi.merge = func(dst, src pointer) {
sfpp := src.toUint64Ptr()
if *sfpp != nil {
dfpp := dst.toUint64Ptr()
if *dfpp == nil {
*dfpp = Uint64(**sfpp)
} else {
**dfpp = **sfpp
}
}
}
default: // E.g., uint64
mfi.merge = func(dst, src pointer) {
if v := *src.toUint64(); v != 0 {
*dst.toUint64() = v
}
}
}
case reflect.Float32:
switch {
case isSlice: // E.g., []float32
mfi.merge = func(dst, src pointer) {
sfsp := src.toFloat32Slice()
if *sfsp != nil {
dfsp := dst.toFloat32Slice()
*dfsp = append(*dfsp, *sfsp...)
if *dfsp == nil {
*dfsp = []float32{}
}
}
}
case isPointer: // E.g., *float32
mfi.merge = func(dst, src pointer) {
sfpp := src.toFloat32Ptr()
if *sfpp != nil {
dfpp := dst.toFloat32Ptr()
if *dfpp == nil {
*dfpp = Float32(**sfpp)
} else {
**dfpp = **sfpp
}
}
}
default: // E.g., float32
mfi.merge = func(dst, src pointer) {
if v := *src.toFloat32(); v != 0 {
*dst.toFloat32() = v
}
}
}
case reflect.Float64:
switch {
case isSlice: // E.g., []float64
mfi.merge = func(dst, src pointer) {
sfsp := src.toFloat64Slice()
if *sfsp != nil {
dfsp := dst.toFloat64Slice()
*dfsp = append(*dfsp, *sfsp...)
if *dfsp == nil {
*dfsp = []float64{}
}
}
}
case isPointer: // E.g., *float64
mfi.merge = func(dst, src pointer) {
sfpp := src.toFloat64Ptr()
if *sfpp != nil {
dfpp := dst.toFloat64Ptr()
if *dfpp == nil {
*dfpp = Float64(**sfpp)
} else {
**dfpp = **sfpp
}
}
}
default: // E.g., float64
mfi.merge = func(dst, src pointer) {
if v := *src.toFloat64(); v != 0 {
*dst.toFloat64() = v
}
}
}
case reflect.Bool:
switch {
case isSlice: // E.g., []bool
mfi.merge = func(dst, src pointer) {
sfsp := src.toBoolSlice()
if *sfsp != nil {
dfsp := dst.toBoolSlice()
*dfsp = append(*dfsp, *sfsp...)
if *dfsp == nil {
*dfsp = []bool{}
}
}
}
case isPointer: // E.g., *bool
mfi.merge = func(dst, src pointer) {
sfpp := src.toBoolPtr()
if *sfpp != nil {
dfpp := dst.toBoolPtr()
if *dfpp == nil {
*dfpp = Bool(**sfpp)
} else {
**dfpp = **sfpp
}
}
}
default: // E.g., bool
mfi.merge = func(dst, src pointer) {
if v := *src.toBool(); v {
*dst.toBool() = v
}
}
}
case reflect.String:
switch {
case isSlice: // E.g., []string
mfi.merge = func(dst, src pointer) {
sfsp := src.toStringSlice()
if *sfsp != nil {
dfsp := dst.toStringSlice()
*dfsp = append(*dfsp, *sfsp...)
if *dfsp == nil {
*dfsp = []string{}
}
}
}
case isPointer: // E.g., *string
mfi.merge = func(dst, src pointer) {
sfpp := src.toStringPtr()
if *sfpp != nil {
dfpp := dst.toStringPtr()
if *dfpp == nil {
*dfpp = String(**sfpp)
} else {
**dfpp = **sfpp
}
}
}
default: // E.g., string
mfi.merge = func(dst, src pointer) {
if v := *src.toString(); v != "" {
*dst.toString() = v
}
}
}
case reflect.Slice:
isProto3 := props.Prop[i].proto3
switch {
case isPointer:
panic("bad pointer in byte slice case in " + tf.Name())
case tf.Elem().Kind() != reflect.Uint8:
panic("bad element kind in byte slice case in " + tf.Name())
case isSlice: // E.g., [][]byte
mfi.merge = func(dst, src pointer) {
sbsp := src.toBytesSlice()
if *sbsp != nil {
dbsp := dst.toBytesSlice()
for _, sb := range *sbsp {
if sb == nil {
*dbsp = append(*dbsp, nil)
} else {
*dbsp = append(*dbsp, append([]byte{}, sb...))
}
}
if *dbsp == nil {
*dbsp = [][]byte{}
}
}
}
default: // E.g., []byte
mfi.merge = func(dst, src pointer) {
sbp := src.toBytes()
if *sbp != nil {
dbp := dst.toBytes()
if !isProto3 || len(*sbp) > 0 {
*dbp = append([]byte{}, *sbp...)
}
}
}
}
case reflect.Struct:
switch {
case !isPointer:
panic(fmt.Sprintf("message field %s without pointer", tf))
case isSlice: // E.g., []*pb.T
mi := getMergeInfo(tf)
mfi.merge = func(dst, src pointer) {
sps := src.getPointerSlice()
if sps != nil {
dps := dst.getPointerSlice()
for _, sp := range sps {
var dp pointer
if !sp.isNil() {
dp = valToPointer(reflect.New(tf))
mi.merge(dp, sp)
}
dps = append(dps, dp)
}
if dps == nil {
dps = []pointer{}
}
dst.setPointerSlice(dps)
}
}
default: // E.g., *pb.T
mi := getMergeInfo(tf)
mfi.merge = func(dst, src pointer) {
sp := src.getPointer()
if !sp.isNil() {
dp := dst.getPointer()
if dp.isNil() {
dp = valToPointer(reflect.New(tf))
dst.setPointer(dp)
}
mi.merge(dp, sp)
}
}
}
case reflect.Map:
switch {
case isPointer || isSlice:
panic("bad pointer or slice in map case in " + tf.Name())
default: // E.g., map[K]V
mfi.merge = func(dst, src pointer) {
sm := src.asPointerTo(tf).Elem()
if sm.Len() == 0 {
return
}
dm := dst.asPointerTo(tf).Elem()
if dm.IsNil() {
dm.Set(reflect.MakeMap(tf))
}
switch tf.Elem().Kind() {
case reflect.Ptr: // Proto struct (e.g., *T)
for _, key := range sm.MapKeys() {
val := sm.MapIndex(key)
val = reflect.ValueOf(Clone(val.Interface().(Message)))
dm.SetMapIndex(key, val)
}
case reflect.Slice: // E.g. Bytes type (e.g., []byte)
for _, key := range sm.MapKeys() {
val := sm.MapIndex(key)
val = reflect.ValueOf(append([]byte{}, val.Bytes()...))
dm.SetMapIndex(key, val)
}
default: // Basic type (e.g., string)
for _, key := range sm.MapKeys() {
val := sm.MapIndex(key)
dm.SetMapIndex(key, val)
}
}
}
}
case reflect.Interface:
// Must be oneof field.
switch {
case isPointer || isSlice:
panic("bad pointer or slice in interface case in " + tf.Name())
default: // E.g., interface{}
// TODO: Make this faster?
mfi.merge = func(dst, src pointer) {
su := src.asPointerTo(tf).Elem()
if !su.IsNil() {
du := dst.asPointerTo(tf).Elem()
typ := su.Elem().Type()
if du.IsNil() || du.Elem().Type() != typ {
du.Set(reflect.New(typ.Elem())) // Initialize interface if empty
}
sv := su.Elem().Elem().Field(0)
if sv.Kind() == reflect.Ptr && sv.IsNil() {
return
}
dv := du.Elem().Elem().Field(0)
if dv.Kind() == reflect.Ptr && dv.IsNil() {
dv.Set(reflect.New(sv.Type().Elem())) // Initialize proto message if empty
}
switch sv.Type().Kind() {
case reflect.Ptr: // Proto struct (e.g., *T)
Merge(dv.Interface().(Message), sv.Interface().(Message))
case reflect.Slice: // E.g. Bytes type (e.g., []byte)
dv.Set(reflect.ValueOf(append([]byte{}, sv.Bytes()...)))
default: // Basic type (e.g., string)
dv.Set(sv)
}
}
}
}
default:
panic(fmt.Sprintf("merger not found for type:%s", tf))
}
mi.fields = append(mi.fields, mfi)
}
mi.unrecognized = invalidField
if f, ok := t.FieldByName("XXX_unrecognized"); ok {
if f.Type != reflect.TypeOf([]byte{}) {
panic("expected XXX_unrecognized to be of type []byte")
}
mi.unrecognized = toField(&f)
}
atomic.StoreInt32(&mi.initialized, 1)
}

File diff suppressed because it is too large Load Diff

View File

@ -50,7 +50,6 @@ import (
var ( var (
newline = []byte("\n") newline = []byte("\n")
spaces = []byte(" ") spaces = []byte(" ")
gtNewline = []byte(">\n")
endBraceNewline = []byte("}\n") endBraceNewline = []byte("}\n")
backslashN = []byte{'\\', 'n'} backslashN = []byte{'\\', 'n'}
backslashR = []byte{'\\', 'r'} backslashR = []byte{'\\', 'r'}
@ -170,11 +169,6 @@ func writeName(w *textWriter, props *Properties) error {
return nil return nil
} }
// raw is the interface satisfied by RawMessage.
type raw interface {
Bytes() []byte
}
func requiresQuotes(u string) bool { func requiresQuotes(u string) bool {
// When type URL contains any characters except [0-9A-Za-z./\-]*, it must be quoted. // When type URL contains any characters except [0-9A-Za-z./\-]*, it must be quoted.
for _, ch := range u { for _, ch := range u {
@ -269,6 +263,10 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error {
props := sprops.Prop[i] props := sprops.Prop[i]
name := st.Field(i).Name name := st.Field(i).Name
if name == "XXX_NoUnkeyedLiteral" {
continue
}
if strings.HasPrefix(name, "XXX_") { if strings.HasPrefix(name, "XXX_") {
// There are two XXX_ fields: // There are two XXX_ fields:
// XXX_unrecognized []byte // XXX_unrecognized []byte
@ -436,12 +434,6 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error {
return err return err
} }
} }
if b, ok := fv.Interface().(raw); ok {
if err := writeRaw(w, b.Bytes()); err != nil {
return err
}
continue
}
// Enums have a String method, so writeAny will work fine. // Enums have a String method, so writeAny will work fine.
if err := tm.writeAny(w, fv, props); err != nil { if err := tm.writeAny(w, fv, props); err != nil {
@ -455,7 +447,7 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error {
// Extensions (the XXX_extensions field). // Extensions (the XXX_extensions field).
pv := sv.Addr() pv := sv.Addr()
if _, ok := extendable(pv.Interface()); ok { if _, err := extendable(pv.Interface()); err == nil {
if err := tm.writeExtensions(w, pv); err != nil { if err := tm.writeExtensions(w, pv); err != nil {
return err return err
} }
@ -464,27 +456,6 @@ func (tm *TextMarshaler) writeStruct(w *textWriter, sv reflect.Value) error {
return nil return nil
} }
// writeRaw writes an uninterpreted raw message.
func writeRaw(w *textWriter, b []byte) error {
if err := w.WriteByte('<'); err != nil {
return err
}
if !w.compact {
if err := w.WriteByte('\n'); err != nil {
return err
}
}
w.indent()
if err := writeUnknownStruct(w, b); err != nil {
return err
}
w.unindent()
if err := w.WriteByte('>'); err != nil {
return err
}
return nil
}
// writeAny writes an arbitrary field. // writeAny writes an arbitrary field.
func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Properties) error { func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Properties) error {
v = reflect.Indirect(v) v = reflect.Indirect(v)
@ -535,6 +506,19 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert
} }
} }
w.indent() w.indent()
if v.CanAddr() {
// Calling v.Interface on a struct causes the reflect package to
// copy the entire struct. This is racy with the new Marshaler
// since we atomically update the XXX_sizecache.
//
// Thus, we retrieve a pointer to the struct if possible to avoid
// a race since v.Interface on the pointer doesn't copy the struct.
//
// If v is not addressable, then we are not worried about a race
// since it implies that the binary Marshaler cannot possibly be
// mutating this value.
v = v.Addr()
}
if etm, ok := v.Interface().(encoding.TextMarshaler); ok { if etm, ok := v.Interface().(encoding.TextMarshaler); ok {
text, err := etm.MarshalText() text, err := etm.MarshalText()
if err != nil { if err != nil {
@ -543,8 +527,13 @@ func (tm *TextMarshaler) writeAny(w *textWriter, v reflect.Value, props *Propert
if _, err = w.Write(text); err != nil { if _, err = w.Write(text); err != nil {
return err return err
} }
} else if err := tm.writeStruct(w, v); err != nil { } else {
return err if v.Kind() == reflect.Ptr {
v = v.Elem()
}
if err := tm.writeStruct(w, v); err != nil {
return err
}
} }
w.unindent() w.unindent()
if err := w.WriteByte(ket); err != nil { if err := w.WriteByte(ket); err != nil {

View File

@ -206,7 +206,6 @@ func (p *textParser) advance() {
var ( var (
errBadUTF8 = errors.New("proto: bad UTF-8") errBadUTF8 = errors.New("proto: bad UTF-8")
errBadHex = errors.New("proto: bad hexadecimal")
) )
func unquoteC(s string, quote rune) (string, error) { func unquoteC(s string, quote rune) (string, error) {
@ -277,60 +276,47 @@ func unescape(s string) (ch string, tail string, err error) {
return "?", s, nil // trigraph workaround return "?", s, nil // trigraph workaround
case '\'', '"', '\\': case '\'', '"', '\\':
return string(r), s, nil return string(r), s, nil
case '0', '1', '2', '3', '4', '5', '6', '7', 'x', 'X': case '0', '1', '2', '3', '4', '5', '6', '7':
if len(s) < 2 { if len(s) < 2 {
return "", "", fmt.Errorf(`\%c requires 2 following digits`, r) return "", "", fmt.Errorf(`\%c requires 2 following digits`, r)
} }
base := 8 ss := string(r) + s[:2]
ss := s[:2]
s = s[2:] s = s[2:]
if r == 'x' || r == 'X' { i, err := strconv.ParseUint(ss, 8, 8)
base = 16
} else {
ss = string(r) + ss
}
i, err := strconv.ParseUint(ss, base, 8)
if err != nil { if err != nil {
return "", "", err return "", "", fmt.Errorf(`\%s contains non-octal digits`, ss)
} }
return string([]byte{byte(i)}), s, nil return string([]byte{byte(i)}), s, nil
case 'u', 'U': case 'x', 'X', 'u', 'U':
n := 4 var n int
if r == 'U' { switch r {
case 'x', 'X':
n = 2
case 'u':
n = 4
case 'U':
n = 8 n = 8
} }
if len(s) < n { if len(s) < n {
return "", "", fmt.Errorf(`\%c requires %d digits`, r, n) return "", "", fmt.Errorf(`\%c requires %d following digits`, r, n)
}
bs := make([]byte, n/2)
for i := 0; i < n; i += 2 {
a, ok1 := unhex(s[i])
b, ok2 := unhex(s[i+1])
if !ok1 || !ok2 {
return "", "", errBadHex
}
bs[i/2] = a<<4 | b
} }
ss := s[:n]
s = s[n:] s = s[n:]
return string(bs), s, nil i, err := strconv.ParseUint(ss, 16, 64)
if err != nil {
return "", "", fmt.Errorf(`\%c%s contains non-hexadecimal digits`, r, ss)
}
if r == 'x' || r == 'X' {
return string([]byte{byte(i)}), s, nil
}
if i > utf8.MaxRune {
return "", "", fmt.Errorf(`\%c%s is not a valid Unicode code point`, r, ss)
}
return string(i), s, nil
} }
return "", "", fmt.Errorf(`unknown escape \%c`, r) return "", "", fmt.Errorf(`unknown escape \%c`, r)
} }
// Adapted from src/pkg/strconv/quote.go.
func unhex(b byte) (v byte, ok bool) {
switch {
case '0' <= b && b <= '9':
return b - '0', true
case 'a' <= b && b <= 'f':
return b - 'a' + 10, true
case 'A' <= b && b <= 'F':
return b - 'A' + 10, true
}
return 0, false
}
// Back off the parser by one token. Can only be done between calls to next(). // Back off the parser by one token. Can only be done between calls to next().
// It makes the next advance() a no-op. // It makes the next advance() a no-op.
func (p *textParser) back() { p.backed = true } func (p *textParser) back() { p.backed = true }
@ -728,6 +714,9 @@ func (p *textParser) consumeExtName() (string, error) {
if tok.err != nil { if tok.err != nil {
return "", p.errorf("unrecognized type_url or extension name: %s", tok.err) return "", p.errorf("unrecognized type_url or extension name: %s", tok.err)
} }
if p.done && tok.value != "]" {
return "", p.errorf("unclosed type_url or extension name")
}
} }
return strings.Join(parts, ""), nil return strings.Join(parts, ""), nil
} }
@ -865,7 +854,7 @@ func (p *textParser) readAny(v reflect.Value, props *Properties) error {
return p.readStruct(fv, terminator) return p.readStruct(fv, terminator)
case reflect.Uint32: case reflect.Uint32:
if x, err := strconv.ParseUint(tok.value, 0, 32); err == nil { if x, err := strconv.ParseUint(tok.value, 0, 32); err == nil {
fv.SetUint(x) fv.SetUint(uint64(x))
return nil return nil
} }
case reflect.Uint64: case reflect.Uint64:
@ -883,13 +872,9 @@ func (p *textParser) readAny(v reflect.Value, props *Properties) error {
// UnmarshalText returns *RequiredNotSetError. // UnmarshalText returns *RequiredNotSetError.
func UnmarshalText(s string, pb Message) error { func UnmarshalText(s string, pb Message) error {
if um, ok := pb.(encoding.TextUnmarshaler); ok { if um, ok := pb.(encoding.TextUnmarshaler); ok {
err := um.UnmarshalText([]byte(s)) return um.UnmarshalText([]byte(s))
return err
} }
pb.Reset() pb.Reset()
v := reflect.ValueOf(pb) v := reflect.ValueOf(pb)
if pe := newTextParser(s).readStruct(v.Elem(), ""); pe != nil { return newTextParser(s).readStruct(v.Elem(), "")
return pe
}
return nil
} }

View File

@ -164,9 +164,9 @@ func (sd *SampleDecoder) Decode(s *model.Vector) error {
} }
// ExtractSamples builds a slice of samples from the provided metric // ExtractSamples builds a slice of samples from the provided metric
// families. If an error occurs during sample extraction, it continues to // families. If an error occurrs during sample extraction, it continues to
// extract from the remaining metric families. The returned error is the last // extract from the remaining metric families. The returned error is the last
// error that has occured. // error that has occurred.
func ExtractSamples(o *DecodeOptions, fams ...*dto.MetricFamily) (model.Vector, error) { func ExtractSamples(o *DecodeOptions, fams ...*dto.MetricFamily) (model.Vector, error) {
var ( var (
all model.Vector all model.Vector

View File

@ -556,8 +556,8 @@ func (p *TextParser) readTokenUntilWhitespace() {
// byte considered is the byte already read (now in p.currentByte). The first // byte considered is the byte already read (now in p.currentByte). The first
// newline byte encountered is still copied into p.currentByte, but not into // newline byte encountered is still copied into p.currentByte, but not into
// p.currentToken. If recognizeEscapeSequence is true, two escape sequences are // p.currentToken. If recognizeEscapeSequence is true, two escape sequences are
// recognized: '\\' tranlates into '\', and '\n' into a line-feed character. All // recognized: '\\' translates into '\', and '\n' into a line-feed character.
// other escape sequences are invalid and cause an error. // All other escape sequences are invalid and cause an error.
func (p *TextParser) readTokenUntilNewline(recognizeEscapeSequence bool) { func (p *TextParser) readTokenUntilNewline(recognizeEscapeSequence bool) {
p.currentToken.Reset() p.currentToken.Reset()
escaped := false escaped := false

View File

@ -59,8 +59,8 @@ func (m *Matcher) Validate() error {
return nil return nil
} }
// Silence defines the representation of a silence definiton // Silence defines the representation of a silence definition in the Prometheus
// in the Prometheus eco-system. // eco-system.
type Silence struct { type Silence struct {
ID uint64 `json:"id,omitempty"` ID uint64 `json:"id,omitempty"`

View File

@ -100,7 +100,7 @@ func (s *SamplePair) UnmarshalJSON(b []byte) error {
} }
// Equal returns true if this SamplePair and o have equal Values and equal // Equal returns true if this SamplePair and o have equal Values and equal
// Timestamps. The sematics of Value equality is defined by SampleValue.Equal. // Timestamps. The semantics of Value equality is defined by SampleValue.Equal.
func (s *SamplePair) Equal(o *SamplePair) bool { func (s *SamplePair) Equal(o *SamplePair) bool {
return s == o || (s.Value.Equal(o.Value) && s.Timestamp.Equal(o.Timestamp)) return s == o || (s.Value.Equal(o.Value) && s.Timestamp.Equal(o.Timestamp))
} }
@ -117,7 +117,7 @@ type Sample struct {
} }
// Equal compares first the metrics, then the timestamp, then the value. The // Equal compares first the metrics, then the timestamp, then the value. The
// sematics of value equality is defined by SampleValue.Equal. // semantics of value equality is defined by SampleValue.Equal.
func (s *Sample) Equal(o *Sample) bool { func (s *Sample) Equal(o *Sample) bool {
if s == o { if s == o {
return true return true

View File

@ -184,7 +184,7 @@ func (nd NetDev) parseLine(rawLine string) (*NetDevLine, error) {
} }
// Total aggregates the values across interfaces and returns a new NetDevLine. // Total aggregates the values across interfaces and returns a new NetDevLine.
// The Name field will be a sorted comma seperated list of interface names. // The Name field will be a sorted comma separated list of interface names.
func (nd NetDev) Total() NetDevLine { func (nd NetDev) Total() NetDevLine {
total := NetDevLine{} total := NetDevLine{}

View File

@ -11,7 +11,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// Package nfsd implements parsing of /proc/net/rpc/nfsd. // Package nfs implements parsing of /proc/net/rpc/nfsd.
// Fields are documented in https://www.svennd.be/nfsd-stats-explained-procnetrpcnfsd/ // Fields are documented in https://www.svennd.be/nfsd-stats-explained-procnetrpcnfsd/
package nfs package nfs
@ -136,8 +136,8 @@ type ClientV4Stats struct {
Setattr uint64 Setattr uint64
FsInfo uint64 FsInfo uint64
Renew uint64 Renew uint64
SetClientId uint64 SetClientID uint64
SetClientIdConfirm uint64 SetClientIDConfirm uint64
Lock uint64 Lock uint64
Lockt uint64 Lockt uint64
Locku uint64 Locku uint64
@ -156,13 +156,13 @@ type ClientV4Stats struct {
ReadDir uint64 ReadDir uint64
ServerCaps uint64 ServerCaps uint64
DelegReturn uint64 DelegReturn uint64
GetAcl uint64 GetACL uint64
SetAcl uint64 SetACL uint64
FsLocations uint64 FsLocations uint64
ReleaseLockowner uint64 ReleaseLockowner uint64
Secinfo uint64 Secinfo uint64
FsidPresent uint64 FsidPresent uint64
ExchangeId uint64 ExchangeID uint64
CreateSession uint64 CreateSession uint64
DestroySession uint64 DestroySession uint64
Sequence uint64 Sequence uint64
@ -173,11 +173,11 @@ type ClientV4Stats struct {
LayoutCommit uint64 LayoutCommit uint64
LayoutReturn uint64 LayoutReturn uint64
SecinfoNoName uint64 SecinfoNoName uint64
TestStateId uint64 TestStateID uint64
FreeStateId uint64 FreeStateID uint64
GetDeviceList uint64 GetDeviceList uint64
BindConnToSession uint64 BindConnToSession uint64
DestroyClientId uint64 DestroyClientID uint64
Seek uint64 Seek uint64
Allocate uint64 Allocate uint64
DeAllocate uint64 DeAllocate uint64
@ -238,7 +238,7 @@ type V4Ops struct {
RelLockOwner uint64 RelLockOwner uint64
} }
// RPCStats models all stats from /proc/net/rpc/nfs. // ClientRPCStats models all stats from /proc/net/rpc/nfs.
type ClientRPCStats struct { type ClientRPCStats struct {
Network Network Network Network
ClientRPC ClientRPC ClientRPC ClientRPC

View File

@ -204,8 +204,8 @@ func parseClientV4Stats(v []uint64) (ClientV4Stats, error) {
Setattr: v[10], Setattr: v[10],
FsInfo: v[11], FsInfo: v[11],
Renew: v[12], Renew: v[12],
SetClientId: v[13], SetClientID: v[13],
SetClientIdConfirm: v[14], SetClientIDConfirm: v[14],
Lock: v[15], Lock: v[15],
Lockt: v[16], Lockt: v[16],
Locku: v[17], Locku: v[17],
@ -224,13 +224,13 @@ func parseClientV4Stats(v []uint64) (ClientV4Stats, error) {
ReadDir: v[30], ReadDir: v[30],
ServerCaps: v[31], ServerCaps: v[31],
DelegReturn: v[32], DelegReturn: v[32],
GetAcl: v[33], GetACL: v[33],
SetAcl: v[34], SetACL: v[34],
FsLocations: v[35], FsLocations: v[35],
ReleaseLockowner: v[36], ReleaseLockowner: v[36],
Secinfo: v[37], Secinfo: v[37],
FsidPresent: v[38], FsidPresent: v[38],
ExchangeId: v[39], ExchangeID: v[39],
CreateSession: v[40], CreateSession: v[40],
DestroySession: v[41], DestroySession: v[41],
Sequence: v[42], Sequence: v[42],
@ -241,11 +241,11 @@ func parseClientV4Stats(v []uint64) (ClientV4Stats, error) {
LayoutCommit: v[47], LayoutCommit: v[47],
LayoutReturn: v[48], LayoutReturn: v[48],
SecinfoNoName: v[49], SecinfoNoName: v[49],
TestStateId: v[50], TestStateID: v[50],
FreeStateId: v[51], FreeStateID: v[51],
GetDeviceList: v[52], GetDeviceList: v[52],
BindConnToSession: v[53], BindConnToSession: v[53],
DestroyClientId: v[54], DestroyClientID: v[54],
Seek: v[55], Seek: v[55],
Allocate: v[56], Allocate: v[56],
DeAllocate: v[57], DeAllocate: v[57],

View File

@ -229,13 +229,13 @@ function extract {
# The repeated pattern makes up for sed's lack of negative # The repeated pattern makes up for sed's lack of negative
# lookbehind assertions (for consecutive null bytes). # lookbehind assertions (for consecutive null bytes).
echo -n "$line" | \ echo -n "$line" | \
sed 's/^NULLBYTE/\x0/g' | \ sed -e 's/^NULLBYTE/\x0/g;
sed 's/\([^\\]\)NULLBYTE/\1\x0/g' | \ s/\([^\\]\)NULLBYTE/\1\x0/g;
sed 's/\([^\\]\)NULLBYTE/\1\x0/g' | \ s/\([^\\]\)NULLBYTE/\1\x0/g;
sed 's/\\NULLBYTE/NULLBYTE/g' | \ s/\\NULLBYTE/NULLBYTE/g;
sed 's/\([^\\]\)EOF/\1/g' | \ s/\([^\\]\)EOF/\1/g;
sed 's/\\EOF/EOF/g' \ s/\\EOF/EOF/g;
>> "$path" ' >> "$path"
fi fi
if [[ "$eof_without_newline" -eq 0 ]]; then if [[ "$eof_without_newline" -eq 0 ]]; then
echo >> "$path" echo >> "$path"
@ -283,11 +283,14 @@ function get_mode {
local mfile=$1 local mfile=$1
if [ -z "${STAT_OPTION:-}" ]; then if [ -z "${STAT_OPTION:-}" ]; then
if stat -c '%a' "$mfile" >/dev/null 2>&1; then if stat -c '%a' "$mfile" >/dev/null 2>&1; then
# GNU stat
STAT_OPTION='-c' STAT_OPTION='-c'
STAT_FORMAT='%a' STAT_FORMAT='%a'
else else
# BSD stat
STAT_OPTION='-f' STAT_OPTION='-f'
STAT_FORMAT='%A' # Octal output, user/group/other (omit file type, sticky bit)
STAT_FORMAT='%OLp'
fi fi
fi fi
stat "${STAT_OPTION}" "${STAT_FORMAT}" "$mfile" stat "${STAT_OPTION}" "${STAT_FORMAT}" "$mfile"
@ -334,9 +337,10 @@ function _create {
< "$file" python -c "$PYTHON_CREATE_FILTER" < "$file" python -c "$PYTHON_CREATE_FILTER"
else else
< "$file" \ < "$file" \
sed 's/EOF/\\EOF/g' | \ sed 's/EOF/\\EOF/g;
sed 's/NULLBYTE/\\NULLBYTE/g' | \ s/NULLBYTE/\\NULLBYTE/g;
sed 's/\x0/NULLBYTE/g' s/\x0/NULLBYTE/g;
'
fi fi
if [[ "$eof_without_newline" -eq 1 ]]; then if [[ "$eof_without_newline" -eq 1 ]]; then
# Finish line with EOF to indicate that the original line did # Finish line with EOF to indicate that the original line did