istio

1
// Copyright Istio Authors
2
//
3
// Licensed under the Apache License, Version 2.0 (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//     http://www.apache.org/licenses/LICENSE-2.0
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14

15
package app
16

17
import (
18
	"context"
19
	"fmt"
20
	"net"
21
	"net/netip"
22
	"strings"
23

24
	"github.com/spf13/cobra"
25

26
	"istio.io/api/annotation"
27
	meshconfig "istio.io/api/mesh/v1alpha1"
28
	"istio.io/istio/pilot/cmd/pilot-agent/config"
29
	"istio.io/istio/pilot/cmd/pilot-agent/options"
30
	"istio.io/istio/pilot/cmd/pilot-agent/status"
31
	"istio.io/istio/pilot/pkg/model"
32
	"istio.io/istio/pilot/pkg/util/network"
33
	"istio.io/istio/pkg/bootstrap"
34
	"istio.io/istio/pkg/cmd"
35
	"istio.io/istio/pkg/collateral"
36
	"istio.io/istio/pkg/config/constants"
37
	"istio.io/istio/pkg/envoy"
38
	istio_agent "istio.io/istio/pkg/istio-agent"
39
	"istio.io/istio/pkg/log"
40
	"istio.io/istio/pkg/security"
41
	"istio.io/istio/pkg/slices"
42
	"istio.io/istio/pkg/util/protomarshal"
43
	"istio.io/istio/pkg/util/sets"
44
	"istio.io/istio/pkg/version"
45
	stsserver "istio.io/istio/security/pkg/stsservice/server"
46
	"istio.io/istio/security/pkg/stsservice/tokenmanager"
47
	cleaniptables "istio.io/istio/tools/istio-clean-iptables/pkg/cmd"
48
	iptables "istio.io/istio/tools/istio-iptables/pkg/cmd"
49
	iptableslog "istio.io/istio/tools/istio-iptables/pkg/log"
50
)
51

52
const (
53
	localHostIPv4 = "127.0.0.1"
54
	localHostIPv6 = "::1"
55
)
56

57
var (
58
	loggingOptions = log.DefaultOptions()
59
	proxyArgs      options.ProxyArgs
60
)
61

62
func NewRootCommand() *cobra.Command {
63
	rootCmd := &cobra.Command{
64
		Use:          "pilot-agent",
65
		Short:        "Istio Pilot agent.",
66
		Long:         "Istio Pilot agent runs in the sidecar or gateway container and bootstraps Envoy.",
67
		SilenceUsage: true,
68
		FParseErrWhitelist: cobra.FParseErrWhitelist{
69
			// Allow unknown flags for backward-compatibility.
70
			UnknownFlags: true,
71
		},
72
	}
73

74
	// Attach the Istio logging options to the command.
75
	loggingOptions.AttachCobraFlags(rootCmd)
76

77
	cmd.AddFlags(rootCmd)
78

79
	proxyCmd := newProxyCommand()
80
	addFlags(proxyCmd)
81
	rootCmd.AddCommand(proxyCmd)
82
	rootCmd.AddCommand(requestCmd)
83
	rootCmd.AddCommand(waitCmd)
84
	rootCmd.AddCommand(version.CobraCommand())
85
	rootCmd.AddCommand(iptables.GetCommand(loggingOptions))
86
	rootCmd.AddCommand(cleaniptables.GetCommand(loggingOptions))
87

88
	rootCmd.AddCommand(collateral.CobraCommand(rootCmd, collateral.Metadata{
89
		Title:   "Istio Pilot Agent",
90
		Section: "pilot-agent CLI",
91
		Manual:  "Istio Pilot Agent",
92
	}))
93

94
	return rootCmd
95
}
96

97
func newProxyCommand() *cobra.Command {
98
	return &cobra.Command{
99
		Use:   "proxy",
100
		Short: "XDS proxy agent",
101
		FParseErrWhitelist: cobra.FParseErrWhitelist{
102
			// Allow unknown flags for backward-compatibility.
103
			UnknownFlags: true,
104
		},
105
		PersistentPreRunE: configureLogging,
106
		RunE: func(c *cobra.Command, args []string) error {
107
			cmd.PrintFlags(c.Flags())
108
			log.Infof("Version %s", version.Info.String())
109

110
			raiseLimits()
111

112
			proxy, err := initProxy(args)
113
			if err != nil {
114
				return err
115
			}
116
			proxyConfig, err := config.ConstructProxyConfig(proxyArgs.MeshConfigFile, proxyArgs.ServiceCluster, options.ProxyConfigEnv, proxyArgs.Concurrency)
117
			if err != nil {
118
				return fmt.Errorf("failed to get proxy config: %v", err)
119
			}
120
			if out, err := protomarshal.ToYAML(proxyConfig); err != nil {
121
				log.Infof("Failed to serialize to YAML: %v", err)
122
			} else {
123
				log.Infof("Effective config: %s", out)
124
			}
125

126
			secOpts, err := options.NewSecurityOptions(proxyConfig, proxyArgs.StsPort, proxyArgs.TokenManagerPlugin)
127
			if err != nil {
128
				return err
129
			}
130

131
			// If security token service (STS) port is not zero, start STS server and
132
			// listen on STS port for STS requests. For STS, see
133
			// https://tools.ietf.org/html/draft-ietf-oauth-token-exchange-16.
134
			// STS is used for stackdriver or other Envoy services using google gRPC.
135
			if proxyArgs.StsPort > 0 {
136
				stsServer, err := initStsServer(proxy, secOpts.TokenManager)
137
				if err != nil {
138
					return err
139
				}
140
				defer stsServer.Stop()
141
			}
142

143
			// If we are using a custom template file (for control plane proxy, for example), configure this.
144
			if proxyArgs.TemplateFile != "" && proxyConfig.CustomConfigFile == "" {
145
				proxyConfig.ProxyBootstrapTemplatePath = proxyArgs.TemplateFile
146
			}
147

148
			envoyOptions := envoy.ProxyConfig{
149
				LogLevel:          proxyArgs.ProxyLogLevel,
150
				ComponentLogLevel: proxyArgs.ProxyComponentLogLevel,
151
				LogAsJSON:         loggingOptions.JSONEncoding,
152
				NodeIPs:           proxy.IPAddresses,
153
				Sidecar:           proxy.Type == model.SidecarProxy,
154
				OutlierLogPath:    proxyArgs.OutlierLogPath,
155
			}
156
			agentOptions := options.NewAgentOptions(proxy, proxyConfig)
157
			agent := istio_agent.NewAgent(proxyConfig, agentOptions, secOpts, envoyOptions)
158
			ctx, cancel := context.WithCancel(context.Background())
159
			defer cancel()
160
			defer agent.Close()
161

162
			// If a status port was provided, start handling status probes.
163
			if proxyConfig.StatusPort > 0 {
164
				if err := initStatusServer(ctx, proxy, proxyConfig,
165
					agentOptions.EnvoyPrometheusPort, proxyArgs.EnableProfiling, agent, cancel); err != nil {
166
					return err
167
				}
168
			}
169

170
			go iptableslog.ReadNFLOGSocket(ctx)
171

172
			// On SIGINT or SIGTERM, cancel the context, triggering a graceful shutdown
173
			go cmd.WaitSignalFunc(cancel)
174

175
			// Start in process SDS, dns server, xds proxy, and Envoy.
176
			wait, err := agent.Run(ctx)
177
			if err != nil {
178
				return err
179
			}
180
			wait()
181
			return nil
182
		},
183
	}
184
}
185

186
func addFlags(proxyCmd *cobra.Command) {
187
	proxyArgs = options.NewProxyArgs()
188
	proxyCmd.PersistentFlags().StringVar(&proxyArgs.DNSDomain, "domain", "",
189
		"DNS domain suffix. If not provided uses ${POD_NAMESPACE}.svc.cluster.local")
190
	proxyCmd.PersistentFlags().StringVar(&proxyArgs.MeshConfigFile, "meshConfig", "./etc/istio/config/mesh",
191
		"File name for Istio mesh configuration. If not specified, a default mesh will be used. This may be overridden by "+
192
			"PROXY_CONFIG environment variable or proxy.istio.io/config annotation.")
193
	proxyCmd.PersistentFlags().IntVar(&proxyArgs.StsPort, "stsPort", 0,
194
		"HTTP Port on which to serve Security Token Service (STS). If zero, STS service will not be provided.")
195
	proxyCmd.PersistentFlags().StringVar(&proxyArgs.TokenManagerPlugin, "tokenManagerPlugin", tokenmanager.GoogleTokenExchange,
196
		"Token provider specific plugin name.")
197
	// DEPRECATED. Flags for proxy configuration
198
	proxyCmd.PersistentFlags().StringVar(&proxyArgs.ServiceCluster, "serviceCluster", constants.ServiceClusterName, "Service cluster")
199
	// Log levels are provided by the library https://github.com/gabime/spdlog, used by Envoy.
200
	proxyCmd.PersistentFlags().StringVar(&proxyArgs.ProxyLogLevel, "proxyLogLevel", "warning,misc:error",
201
		fmt.Sprintf("The log level used to start the Envoy proxy (choose from {%s, %s, %s, %s, %s, %s, %s})."+
202
			"Level may also include one or more scopes, such as 'info,misc:error,upstream:debug'",
203
			"trace", "debug", "info", "warning", "error", "critical", "off"))
204
	proxyCmd.PersistentFlags().IntVar(&proxyArgs.Concurrency, "concurrency", 0, "number of worker threads to run")
205
	// See https://www.envoyproxy.io/docs/envoy/latest/operations/cli#cmdoption-component-log-level
206
	proxyCmd.PersistentFlags().StringVar(&proxyArgs.ProxyComponentLogLevel, "proxyComponentLogLevel", "",
207
		"The component log level used to start the Envoy proxy. Deprecated, use proxyLogLevel instead")
208
	proxyCmd.PersistentFlags().StringVar(&proxyArgs.TemplateFile, "templateFile", "",
209
		"Go template bootstrap config")
210
	proxyCmd.PersistentFlags().StringVar(&proxyArgs.OutlierLogPath, "outlierLogPath", "",
211
		"The log path for outlier detection")
212
	proxyCmd.PersistentFlags().BoolVar(&proxyArgs.EnableProfiling, "profiling", true,
213
		"Enable profiling via web interface host:port/debug/pprof/.")
214
}
215

216
func initStatusServer(
217
	ctx context.Context,
218
	proxy *model.Proxy,
219
	proxyConfig *meshconfig.ProxyConfig,
220
	envoyPrometheusPort int,
221
	enableProfiling bool,
222
	agent *istio_agent.Agent,
223
	shutdown context.CancelFunc,
224
) error {
225
	o := options.NewStatusServerOptions(proxy, proxyConfig, agent)
226
	o.EnvoyPrometheusPort = envoyPrometheusPort
227
	o.EnableProfiling = enableProfiling
228
	o.Context = ctx
229
	o.Shutdown = shutdown
230
	statusServer, err := status.NewServer(*o)
231
	if err != nil {
232
		return err
233
	}
234
	go statusServer.Run(ctx)
235
	return nil
236
}
237

238
func initStsServer(proxy *model.Proxy, tokenManager security.TokenManager) (*stsserver.Server, error) {
239
	localHostAddr := localHostIPv4
240
	if proxy.IsIPv6() {
241
		localHostAddr = localHostIPv6
242
	} else {
243
		// if not ipv6-only, it can be ipv4-only or dual-stack
244
		// let InstanceIP decide the localhost
245
		netIP, _ := netip.ParseAddr(options.InstanceIPVar.Get())
246
		if netIP.Is6() && !netIP.IsLinkLocalUnicast() {
247
			localHostAddr = localHostIPv6
248
		}
249
	}
250
	stsServer, err := stsserver.NewServer(stsserver.Config{
251
		LocalHostAddr: localHostAddr,
252
		LocalPort:     proxyArgs.StsPort,
253
	}, tokenManager)
254
	if err != nil {
255
		return nil, err
256
	}
257
	return stsServer, nil
258
}
259

260
func getDNSDomain(podNamespace, domain string) string {
261
	if len(domain) == 0 {
262
		domain = podNamespace + ".svc." + constants.DefaultClusterLocalDomain
263
	}
264
	return domain
265
}
266

267
func configureLogging(_ *cobra.Command, _ []string) error {
268
	if err := log.Configure(loggingOptions); err != nil {
269
		return err
270
	}
271
	return nil
272
}
273

274
func initProxy(args []string) (*model.Proxy, error) {
275
	proxy := &model.Proxy{
276
		Type: model.SidecarProxy,
277
	}
278
	if len(args) > 0 {
279
		proxy.Type = model.NodeType(args[0])
280
		if !model.IsApplicationNodeType(proxy.Type) {
281
			return nil, fmt.Errorf("Invalid proxy Type: " + string(proxy.Type))
282
		}
283
	}
284

285
	podIP, _ := netip.ParseAddr(options.InstanceIPVar.Get()) // protobuf encoding of IP_ADDRESS type
286
	if podIP.IsValid() {
287
		// The first one must be the pod ip as we pick the first ip as pod ip in istiod.
288
		proxy.IPAddresses = []string{podIP.String()}
289
	}
290

291
	// Obtain all the IPs from the node
292
	proxyAddrs := make([]string, 0)
293
	if ipAddrs, ok := network.GetPrivateIPs(context.Background()); ok {
294
		proxyAddrs = append(proxyAddrs, ipAddrs...)
295
	}
296

297
	// No IP addresses provided, append 127.0.0.1 for ipv4 and ::1 for ipv6
298
	if len(proxyAddrs) == 0 {
299
		proxyAddrs = append(proxyAddrs, localHostIPv4, localHostIPv6)
300
	}
301

302
	// Get exclusions from traffic.sidecar.istio.io/excludeInterfaces
303
	excludeAddrs := getExcludeInterfaces()
304
	excludeAddrs.InsertAll(proxy.IPAddresses...) // prevent duplicate IPs
305
	proxyAddrs = slices.FilterInPlace(proxyAddrs, func(s string) bool {
306
		return !excludeAddrs.Contains(s)
307
	})
308

309
	proxy.IPAddresses = append(proxy.IPAddresses, proxyAddrs...)
310
	log.Debugf("proxy IPAddresses: %v", proxy.IPAddresses)
311

312
	// After IP addresses are set, let us discover IPMode.
313
	proxy.DiscoverIPMode()
314

315
	// Extract pod variables.
316
	proxy.ID = proxyArgs.PodName + "." + proxyArgs.PodNamespace
317

318
	// If not set, set a default based on platform - podNamespace.svc.cluster.local for
319
	// K8S
320
	proxy.DNSDomain = getDNSDomain(proxyArgs.PodNamespace, proxyArgs.DNSDomain)
321
	log.WithLabels("ips", proxy.IPAddresses, "type", proxy.Type, "id", proxy.ID, "domain", proxy.DNSDomain).Info("Proxy role")
322

323
	return proxy, nil
324
}
325

326
func getExcludeInterfaces() sets.String {
327
	excludeAddrs := sets.New[string]()
328

329
	// Get list of excluded interfaces from pod annotation
330
	// TODO: Discuss other input methods such as env, flag (ssuvasanth)
331
	annotations, err := bootstrap.ReadPodAnnotations("")
332
	if err != nil {
333
		log.Debugf("Reading podInfoAnnotations file to get excludeInterfaces was unsuccessful. Continuing without exclusions. msg: %v", err)
334
		return excludeAddrs
335
	}
336
	value, ok := annotations[annotation.SidecarTrafficExcludeInterfaces.Name]
337
	if !ok {
338
		log.Debugf("%s annotation is not present", annotation.SidecarTrafficExcludeInterfaces.Name)
339
		return excludeAddrs
340
	}
341
	exclusions := strings.Split(value, ",")
342

343
	// Find IP addr of excluded interfaces and add to a map for instant lookup
344
	for _, ifaceName := range exclusions {
345
		iface, err := net.InterfaceByName(ifaceName)
346
		if err != nil {
347
			log.Warnf("Unable to get interface %s: %v", ifaceName, err)
348
			continue
349
		}
350
		addrs, err := iface.Addrs()
351
		if err != nil {
352
			log.Warnf("Unable to get IP addr(s) of interface %s: %v", ifaceName, err)
353
			continue
354
		}
355

356
		for _, addr := range addrs {
357
			// Get IP only
358
			var ip net.IP
359
			switch v := addr.(type) {
360
			case *net.IPNet:
361
				ip = v.IP
362
			case *net.IPAddr:
363
				ip = v.IP
364
			default:
365
				continue
366
			}
367

368
			// handling ipv4 wrapping in ipv6
369
			ipAddr, okay := netip.AddrFromSlice(ip)
370
			if !okay {
371
				continue
372
			}
373
			unwrapAddr := ipAddr.Unmap()
374
			if !unwrapAddr.IsValid() || unwrapAddr.IsLoopback() || unwrapAddr.IsLinkLocalUnicast() || unwrapAddr.IsLinkLocalMulticast() || unwrapAddr.IsUnspecified() {
375
				continue
376
			}
377

378
			// Add to map
379
			excludeAddrs.Insert(unwrapAddr.String())
380
		}
381
	}
382

383
	log.Infof("Exclude IPs %v based on %s annotation", excludeAddrs, annotation.SidecarTrafficExcludeInterfaces.Name)
384
	return excludeAddrs
385
}
386

387
func raiseLimits() {
388
	limit, err := RaiseFileLimits()
389
	if err != nil {
390
		log.Warnf("failed setting file limit: %v", err)
391
	} else {
392
		log.Infof("Set max file descriptors (ulimit -n) to: %d", limit)
393
	}
394
}
395