talos

1
// This Source Code Form is subject to the terms of the Mozilla Public
2
// License, v. 2.0. If a copy of the MPL was not distributed with this
3
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
4

5
package vm
6

7
import (
8
	"bytes"
9
	"context"
10
	"crypto/sha256"
11
	"encoding/hex"
12
	"errors"
13
	"fmt"
14
	"net"
15
	"net/netip"
16
	"strconv"
17
	"strings"
18
	"text/template"
19

20
	"github.com/containernetworking/cni/libcni"
21
	"github.com/containernetworking/plugins/pkg/testutils"
22
	"github.com/coreos/go-iptables/iptables"
23
	"github.com/google/uuid"
24
	"github.com/jsimonetti/rtnetlink/v2"
25
	"github.com/siderolabs/gen/xslices"
26
	sideronet "github.com/siderolabs/net"
27
	"github.com/vishvananda/netlink"
28

29
	"github.com/siderolabs/talos/pkg/provision"
30
)
31

32
// CreateNetwork builds bridge interface name by taking part of checksum of the network name
33
// so that interface name is defined by network name, and different networks have
34
// different bridge interfaces.
35
//
36
//nolint:gocyclo
37
func (p *Provisioner) CreateNetwork(ctx context.Context, state *State, network provision.NetworkRequest, options provision.Options) error {
38
	networkNameHash := sha256.Sum256([]byte(network.Name))
39
	state.BridgeName = fmt.Sprintf("%s%s", "talos", hex.EncodeToString(networkNameHash[:])[:8])
40

41
	// bring up the bridge interface for the first time to get gateway IP assigned
42
	t := template.Must(template.New("bridge").Parse(bridgeTemplate))
43

44
	var buf bytes.Buffer
45

46
	err := t.Execute(&buf, struct {
47
		NetworkName   string
48
		InterfaceName string
49
		MTU           string
50
	}{
51
		NetworkName:   network.Name,
52
		InterfaceName: state.BridgeName,
53
		MTU:           strconv.Itoa(network.MTU),
54
	})
55
	if err != nil {
56
		return fmt.Errorf("error templating bridge CNI config: %w", err)
57
	}
58

59
	bridgeConfig, err := libcni.ConfFromBytes(buf.Bytes())
60
	if err != nil {
61
		return fmt.Errorf("error parsing bridge CNI config: %w", err)
62
	}
63

64
	cniConfig := libcni.NewCNIConfigWithCacheDir(network.CNI.BinPath, network.CNI.CacheDir, nil)
65

66
	ns, err := testutils.NewNS()
67
	if err != nil {
68
		return err
69
	}
70

71
	defer func() {
72
		ns.Close()              //nolint:errcheck
73
		testutils.UnmountNS(ns) //nolint:errcheck
74
	}()
75

76
	// pick a fake address to use for provisioning an interface
77
	fakeIPs := make([]string, len(network.CIDRs))
78
	for j := range fakeIPs {
79
		var fakeIP netip.Addr
80

81
		fakeIP, err = sideronet.NthIPInNetwork(network.CIDRs[j], 2)
82
		if err != nil {
83
			return err
84
		}
85

86
		fakeIPs[j] = sideronet.FormatCIDR(fakeIP, network.CIDRs[j])
87
	}
88

89
	gatewayAddrs := xslices.Map(network.GatewayAddrs, netip.Addr.String)
90

91
	containerID := uuid.New().String()
92
	runtimeConf := libcni.RuntimeConf{
93
		ContainerID: containerID,
94
		NetNS:       ns.Path(),
95
		IfName:      "veth0",
96
		Args: [][2]string{
97
			{"IP", strings.Join(fakeIPs, ",")},
98
			{"GATEWAY", strings.Join(gatewayAddrs, ",")},
99
			{"IgnoreUnknown", "1"},
100
		},
101
	}
102

103
	_, err = cniConfig.AddNetwork(ctx, bridgeConfig, &runtimeConf)
104
	if err != nil {
105
		return fmt.Errorf("error provisioning bridge CNI network: %w", err)
106
	}
107

108
	err = cniConfig.DelNetwork(ctx, bridgeConfig, &runtimeConf)
109
	if err != nil {
110
		return fmt.Errorf("error deleting bridge CNI network: %w", err)
111
	}
112

113
	// prepare an actual network config to be used by the VMs
114
	t = template.Must(template.New("network").Parse(networkTemplate))
115

116
	buf.Reset()
117

118
	err = t.Execute(&buf, struct {
119
		NetworkName   string
120
		InterfaceName string
121
		MTU           string
122
	}{
123
		NetworkName:   network.Name,
124
		InterfaceName: state.BridgeName,
125
		MTU:           strconv.Itoa(network.MTU),
126
	})
127
	if err != nil {
128
		return fmt.Errorf("error templating VM CNI config: %w", err)
129
	}
130

131
	if state.VMCNIConfig, err = libcni.ConfListFromBytes(buf.Bytes()); err != nil {
132
		return fmt.Errorf("error parsing VM CNI config: %w", err)
133
	}
134

135
	// allow traffic on the bridge via `DOCKER-USER` chain
136
	// Docker enables br-netfilter which causes layer2 packets to be filtered with iptables, but we'd like to skip that
137
	// if Docker is not running, this will be no-op
138
	//
139
	// See https://serverfault.com/questions/963759/docker-breaks-libvirt-bridge-network for more details
140
	if err = p.allowBridgeTraffic(state.BridgeName); err != nil {
141
		return fmt.Errorf("error configuring DOCKER-USER chain: %w", err)
142
	}
143

144
	// configure bridge interface with network chaos if flag is set
145
	if network.NetworkChaos {
146
		if err = p.configureNetworkChaos(network, state, options); err != nil {
147
			return err
148
		}
149
	}
150

151
	return nil
152
}
153

154
func (p *Provisioner) allowBridgeTraffic(bridgeName string) error {
155
	ipt, err := iptables.New()
156
	if err != nil {
157
		return fmt.Errorf("error initializing iptables: %w", err)
158
	}
159

160
	chainExists, err := ipt.ChainExists("filter", "DOCKER-USER")
161
	if err != nil {
162
		return fmt.Errorf("error checking chain existence: %w", err)
163
	}
164

165
	if !chainExists {
166
		if err = ipt.NewChain("filter", "DOCKER-USER"); err != nil {
167
			return fmt.Errorf("error creating DOCKER-USER chain: %w", err)
168
		}
169
	}
170

171
	if err := ipt.InsertUnique("filter", "DOCKER-USER", 1, "-i", bridgeName, "-o", bridgeName, "-j", "ACCEPT"); err != nil {
172
		return fmt.Errorf("error inserting rule into DOCKER-USER chain: %w", err)
173
	}
174

175
	return nil
176
}
177

178
func (p *Provisioner) dropBridgeTrafficRule(bridgeName string) error {
179
	ipt, err := iptables.New()
180
	if err != nil {
181
		return fmt.Errorf("error initializing iptables: %w", err)
182
	}
183

184
	chainExists, err := ipt.ChainExists("filter", "DOCKER-USER")
185
	if err != nil {
186
		return fmt.Errorf("error checking chain existence: %w", err)
187
	}
188

189
	if !chainExists {
190
		return nil
191
	}
192

193
	if err := ipt.DeleteIfExists("filter", "DOCKER-USER", "-i", bridgeName, "-o", bridgeName, "-j", "ACCEPT"); err != nil {
194
		return fmt.Errorf("error deleting rule in DOCKER-USER chain: %w", err)
195
	}
196

197
	return nil
198
}
199

200
//nolint:gocyclo
201
func (p *Provisioner) configureNetworkChaos(network provision.NetworkRequest, state *State, options provision.Options) error {
202
	if (network.Bandwidth != 0) && (network.Latency != 0 || network.Jitter != 0 || network.PacketLoss != 0 || network.PacketReorder != 0 || network.PacketCorrupt != 0) {
203
		return errors.New("bandwidth and other chaos options cannot be used together")
204
	}
205

206
	link, err := netlink.LinkByName(state.BridgeName)
207
	if err != nil {
208
		return fmt.Errorf("could not get link: %v", err)
209
	}
210

211
	fmt.Fprintln(options.LogWriter, "network chaos enabled on interface:", state.BridgeName)
212

213
	if network.Bandwidth != 0 {
214
		fmt.Fprintf(options.LogWriter, "  bandwidth: %4d kbps\n", network.Bandwidth)
215

216
		rate := network.Bandwidth * 1000 / 8
217

218
		buffer := rate / 10
219

220
		limit := buffer * 5
221

222
		qdisc := &netlink.Tbf{
223
			QdiscAttrs: netlink.QdiscAttrs{
224
				LinkIndex: link.Attrs().Index,
225
				Handle:    netlink.MakeHandle(1, 0),
226
				Parent:    netlink.HANDLE_ROOT,
227
			},
228
			Rate:   uint64(rate),
229
			Buffer: uint32(buffer),
230
			Limit:  uint32(limit),
231
		}
232

233
		if err := netlink.QdiscAdd(qdisc); err != nil {
234
			return fmt.Errorf("could not add netem qdisc: %v", err)
235
		}
236
	} else {
237
		packetLoss := network.PacketLoss * 100
238
		packetReorder := network.PacketReorder * 100
239
		packetCorrupt := network.PacketCorrupt * 100
240

241
		fmt.Fprintf(options.LogWriter, "  jitter:            %4dms\n", network.Jitter.Milliseconds())
242
		fmt.Fprintf(options.LogWriter, "  latency:           %4dms\n", network.Latency.Milliseconds())
243
		fmt.Fprintf(options.LogWriter, "  packet loss:       %4v%%\n", packetLoss)
244
		fmt.Fprintf(options.LogWriter, "  packet reordering: %4v%%\n", packetReorder)
245
		fmt.Fprintf(options.LogWriter, "  packet corruption: %4v%%\n", packetCorrupt)
246

247
		qdisc := netlink.NewNetem(
248
			netlink.QdiscAttrs{
249
				LinkIndex: link.Attrs().Index,
250
				Handle:    netlink.MakeHandle(1, 0),
251
				Parent:    netlink.HANDLE_ROOT,
252
			},
253
			netlink.NetemQdiscAttrs{
254
				Jitter:      uint32(network.Jitter / 1000),
255
				Latency:     uint32(network.Latency / 1000),
256
				Loss:        float32(packetLoss),
257
				ReorderProb: float32(packetReorder),
258
				CorruptProb: float32(packetCorrupt),
259
			},
260
		)
261
		if err := netlink.QdiscAdd(qdisc); err != nil {
262
			return fmt.Errorf("could not add netem qdisc: %v", err)
263
		}
264
	}
265

266
	return nil
267
}
268

269
// DestroyNetwork destroy bridge interface by name to clean up.
270
func (p *Provisioner) DestroyNetwork(state *State) error {
271
	iface, err := net.InterfaceByName(state.BridgeName)
272
	if err != nil {
273
		return fmt.Errorf("error looking up bridge interface %q: %w", state.BridgeName, err)
274
	}
275

276
	rtconn, err := rtnetlink.Dial(nil)
277
	if err != nil {
278
		return fmt.Errorf("error dialing rnetlink: %w", err)
279
	}
280

281
	if err = rtconn.Link.Delete(uint32(iface.Index)); err != nil {
282
		return fmt.Errorf("error deleting bridge interface: %w", err)
283
	}
284

285
	if err = p.dropBridgeTrafficRule(state.BridgeName); err != nil {
286
		return fmt.Errorf("error dropping bridge traffic rule: %w", err)
287
	}
288

289
	return nil
290
}
291

292
const bridgeTemplate = `
293
{
294
	"name": "{{ .NetworkName }}",
295
	"cniVersion": "0.4.0",
296
	"type": "bridge",
297
	"bridge": "{{ .InterfaceName }}",
298
	"ipMasq": true,
299
	"isGateway": true,
300
	"isDefaultGateway": true,
301
	"ipam": {
302
		  "type": "static"
303
	},
304
	"mtu": {{ .MTU }}
305
}
306
`
307

308
const networkTemplate = `
309
{
310
	"name": "{{ .NetworkName }}",
311
	"cniVersion": "0.4.0",
312
	"plugins": [
313
		{
314
			"type": "bridge",
315
			"bridge": "{{ .InterfaceName }}",
316
			"ipMasq": true,
317
			"isGateway": true,
318
			"isDefaultGateway": true,
319
			"ipam": {
320
				"type": "static"
321
			},
322
			"mtu": {{ .MTU }}
323
		},
324
		{
325
			"type": "firewall"
326
		},
327
		{
328
			"type": "tc-redirect-tap"
329
		}
330
	]
331
}
332
`
333