go-bot

1
// Copyright 2019 The Go Authors. All rights reserved.
2
// Use of this source code is governed by a BSD-style
3
// license that can be found in the LICENSE file.
4

5
package proto
6

7
import (
8
	"google.golang.org/protobuf/encoding/protowire"
9
	"google.golang.org/protobuf/internal/encoding/messageset"
10
	"google.golang.org/protobuf/internal/order"
11
	"google.golang.org/protobuf/internal/pragma"
12
	"google.golang.org/protobuf/reflect/protoreflect"
13
	"google.golang.org/protobuf/runtime/protoiface"
14
)
15

16
// MarshalOptions configures the marshaler.
17
//
18
// Example usage:
19
//
20
//	b, err := MarshalOptions{Deterministic: true}.Marshal(m)
21
type MarshalOptions struct {
22
	pragma.NoUnkeyedLiterals
23

24
	// AllowPartial allows messages that have missing required fields to marshal
25
	// without returning an error. If AllowPartial is false (the default),
26
	// Marshal will return an error if there are any missing required fields.
27
	AllowPartial bool
28

29
	// Deterministic controls whether the same message will always be
30
	// serialized to the same bytes within the same binary.
31
	//
32
	// Setting this option guarantees that repeated serialization of
33
	// the same message will return the same bytes, and that different
34
	// processes of the same binary (which may be executing on different
35
	// machines) will serialize equal messages to the same bytes.
36
	// It has no effect on the resulting size of the encoded message compared
37
	// to a non-deterministic marshal.
38
	//
39
	// Note that the deterministic serialization is NOT canonical across
40
	// languages. It is not guaranteed to remain stable over time. It is
41
	// unstable across different builds with schema changes due to unknown
42
	// fields. Users who need canonical serialization (e.g., persistent
43
	// storage in a canonical form, fingerprinting, etc.) must define
44
	// their own canonicalization specification and implement their own
45
	// serializer rather than relying on this API.
46
	//
47
	// If deterministic serialization is requested, map entries will be
48
	// sorted by keys in lexographical order. This is an implementation
49
	// detail and subject to change.
50
	Deterministic bool
51

52
	// UseCachedSize indicates that the result of a previous Size call
53
	// may be reused.
54
	//
55
	// Setting this option asserts that:
56
	//
57
	// 1. Size has previously been called on this message with identical
58
	// options (except for UseCachedSize itself).
59
	//
60
	// 2. The message and all its submessages have not changed in any
61
	// way since the Size call.
62
	//
63
	// If either of these invariants is violated,
64
	// the results are undefined and may include panics or corrupted output.
65
	//
66
	// Implementations MAY take this option into account to provide
67
	// better performance, but there is no guarantee that they will do so.
68
	// There is absolutely no guarantee that Size followed by Marshal with
69
	// UseCachedSize set will perform equivalently to Marshal alone.
70
	UseCachedSize bool
71
}
72

73
// Marshal returns the wire-format encoding of m.
74
func Marshal(m Message) ([]byte, error) {
75
	// Treat nil message interface as an empty message; nothing to output.
76
	if m == nil {
77
		return nil, nil
78
	}
79

80
	out, err := MarshalOptions{}.marshal(nil, m.ProtoReflect())
81
	if len(out.Buf) == 0 && err == nil {
82
		out.Buf = emptyBytesForMessage(m)
83
	}
84
	return out.Buf, err
85
}
86

87
// Marshal returns the wire-format encoding of m.
88
func (o MarshalOptions) Marshal(m Message) ([]byte, error) {
89
	// Treat nil message interface as an empty message; nothing to output.
90
	if m == nil {
91
		return nil, nil
92
	}
93

94
	out, err := o.marshal(nil, m.ProtoReflect())
95
	if len(out.Buf) == 0 && err == nil {
96
		out.Buf = emptyBytesForMessage(m)
97
	}
98
	return out.Buf, err
99
}
100

101
// emptyBytesForMessage returns a nil buffer if and only if m is invalid,
102
// otherwise it returns a non-nil empty buffer.
103
//
104
// This is to assist the edge-case where user-code does the following:
105
//
106
//	m1.OptionalBytes, _ = proto.Marshal(m2)
107
//
108
// where they expect the proto2 "optional_bytes" field to be populated
109
// if any only if m2 is a valid message.
110
func emptyBytesForMessage(m Message) []byte {
111
	if m == nil || !m.ProtoReflect().IsValid() {
112
		return nil
113
	}
114
	return emptyBuf[:]
115
}
116

117
// MarshalAppend appends the wire-format encoding of m to b,
118
// returning the result.
119
func (o MarshalOptions) MarshalAppend(b []byte, m Message) ([]byte, error) {
120
	// Treat nil message interface as an empty message; nothing to append.
121
	if m == nil {
122
		return b, nil
123
	}
124

125
	out, err := o.marshal(b, m.ProtoReflect())
126
	return out.Buf, err
127
}
128

129
// MarshalState returns the wire-format encoding of a message.
130
//
131
// This method permits fine-grained control over the marshaler.
132
// Most users should use Marshal instead.
133
func (o MarshalOptions) MarshalState(in protoiface.MarshalInput) (protoiface.MarshalOutput, error) {
134
	return o.marshal(in.Buf, in.Message)
135
}
136

137
// marshal is a centralized function that all marshal operations go through.
138
// For profiling purposes, avoid changing the name of this function or
139
// introducing other code paths for marshal that do not go through this.
140
func (o MarshalOptions) marshal(b []byte, m protoreflect.Message) (out protoiface.MarshalOutput, err error) {
141
	allowPartial := o.AllowPartial
142
	o.AllowPartial = true
143
	if methods := protoMethods(m); methods != nil && methods.Marshal != nil &&
144
		!(o.Deterministic && methods.Flags&protoiface.SupportMarshalDeterministic == 0) {
145
		in := protoiface.MarshalInput{
146
			Message: m,
147
			Buf:     b,
148
		}
149
		if o.Deterministic {
150
			in.Flags |= protoiface.MarshalDeterministic
151
		}
152
		if o.UseCachedSize {
153
			in.Flags |= protoiface.MarshalUseCachedSize
154
		}
155
		if methods.Size != nil {
156
			sout := methods.Size(protoiface.SizeInput{
157
				Message: m,
158
				Flags:   in.Flags,
159
			})
160
			if cap(b) < len(b)+sout.Size {
161
				in.Buf = make([]byte, len(b), growcap(cap(b), len(b)+sout.Size))
162
				copy(in.Buf, b)
163
			}
164
			in.Flags |= protoiface.MarshalUseCachedSize
165
		}
166
		out, err = methods.Marshal(in)
167
	} else {
168
		out.Buf, err = o.marshalMessageSlow(b, m)
169
	}
170
	if err != nil {
171
		return out, err
172
	}
173
	if allowPartial {
174
		return out, nil
175
	}
176
	return out, checkInitialized(m)
177
}
178

179
func (o MarshalOptions) marshalMessage(b []byte, m protoreflect.Message) ([]byte, error) {
180
	out, err := o.marshal(b, m)
181
	return out.Buf, err
182
}
183

184
// growcap scales up the capacity of a slice.
185
//
186
// Given a slice with a current capacity of oldcap and a desired
187
// capacity of wantcap, growcap returns a new capacity >= wantcap.
188
//
189
// The algorithm is mostly identical to the one used by append as of Go 1.14.
190
func growcap(oldcap, wantcap int) (newcap int) {
191
	if wantcap > oldcap*2 {
192
		newcap = wantcap
193
	} else if oldcap < 1024 {
194
		// The Go 1.14 runtime takes this case when len(s) < 1024,
195
		// not when cap(s) < 1024. The difference doesn't seem
196
		// significant here.
197
		newcap = oldcap * 2
198
	} else {
199
		newcap = oldcap
200
		for 0 < newcap && newcap < wantcap {
201
			newcap += newcap / 4
202
		}
203
		if newcap <= 0 {
204
			newcap = wantcap
205
		}
206
	}
207
	return newcap
208
}
209

210
func (o MarshalOptions) marshalMessageSlow(b []byte, m protoreflect.Message) ([]byte, error) {
211
	if messageset.IsMessageSet(m.Descriptor()) {
212
		return o.marshalMessageSet(b, m)
213
	}
214
	fieldOrder := order.AnyFieldOrder
215
	if o.Deterministic {
216
		// TODO: This should use a more natural ordering like NumberFieldOrder,
217
		// but doing so breaks golden tests that make invalid assumption about
218
		// output stability of this implementation.
219
		fieldOrder = order.LegacyFieldOrder
220
	}
221
	var err error
222
	order.RangeFields(m, fieldOrder, func(fd protoreflect.FieldDescriptor, v protoreflect.Value) bool {
223
		b, err = o.marshalField(b, fd, v)
224
		return err == nil
225
	})
226
	if err != nil {
227
		return b, err
228
	}
229
	b = append(b, m.GetUnknown()...)
230
	return b, nil
231
}
232

233
func (o MarshalOptions) marshalField(b []byte, fd protoreflect.FieldDescriptor, value protoreflect.Value) ([]byte, error) {
234
	switch {
235
	case fd.IsList():
236
		return o.marshalList(b, fd, value.List())
237
	case fd.IsMap():
238
		return o.marshalMap(b, fd, value.Map())
239
	default:
240
		b = protowire.AppendTag(b, fd.Number(), wireTypes[fd.Kind()])
241
		return o.marshalSingular(b, fd, value)
242
	}
243
}
244

245
func (o MarshalOptions) marshalList(b []byte, fd protoreflect.FieldDescriptor, list protoreflect.List) ([]byte, error) {
246
	if fd.IsPacked() && list.Len() > 0 {
247
		b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
248
		b, pos := appendSpeculativeLength(b)
249
		for i, llen := 0, list.Len(); i < llen; i++ {
250
			var err error
251
			b, err = o.marshalSingular(b, fd, list.Get(i))
252
			if err != nil {
253
				return b, err
254
			}
255
		}
256
		b = finishSpeculativeLength(b, pos)
257
		return b, nil
258
	}
259

260
	kind := fd.Kind()
261
	for i, llen := 0, list.Len(); i < llen; i++ {
262
		var err error
263
		b = protowire.AppendTag(b, fd.Number(), wireTypes[kind])
264
		b, err = o.marshalSingular(b, fd, list.Get(i))
265
		if err != nil {
266
			return b, err
267
		}
268
	}
269
	return b, nil
270
}
271

272
func (o MarshalOptions) marshalMap(b []byte, fd protoreflect.FieldDescriptor, mapv protoreflect.Map) ([]byte, error) {
273
	keyf := fd.MapKey()
274
	valf := fd.MapValue()
275
	keyOrder := order.AnyKeyOrder
276
	if o.Deterministic {
277
		keyOrder = order.GenericKeyOrder
278
	}
279
	var err error
280
	order.RangeEntries(mapv, keyOrder, func(key protoreflect.MapKey, value protoreflect.Value) bool {
281
		b = protowire.AppendTag(b, fd.Number(), protowire.BytesType)
282
		var pos int
283
		b, pos = appendSpeculativeLength(b)
284

285
		b, err = o.marshalField(b, keyf, key.Value())
286
		if err != nil {
287
			return false
288
		}
289
		b, err = o.marshalField(b, valf, value)
290
		if err != nil {
291
			return false
292
		}
293
		b = finishSpeculativeLength(b, pos)
294
		return true
295
	})
296
	return b, err
297
}
298

299
// When encoding length-prefixed fields, we speculatively set aside some number of bytes
300
// for the length, encode the data, and then encode the length (shifting the data if necessary
301
// to make room).
302
const speculativeLength = 1
303

304
func appendSpeculativeLength(b []byte) ([]byte, int) {
305
	pos := len(b)
306
	b = append(b, "\x00\x00\x00\x00"[:speculativeLength]...)
307
	return b, pos
308
}
309

310
func finishSpeculativeLength(b []byte, pos int) []byte {
311
	mlen := len(b) - pos - speculativeLength
312
	msiz := protowire.SizeVarint(uint64(mlen))
313
	if msiz != speculativeLength {
314
		for i := 0; i < msiz-speculativeLength; i++ {
315
			b = append(b, 0)
316
		}
317
		copy(b[pos+msiz:], b[pos+speculativeLength:])
318
		b = b[:pos+msiz+mlen]
319
	}
320
	protowire.AppendVarint(b[:pos], uint64(mlen))
321
	return b
322
}
323