istio

1
// Copyright Istio Authors
2
//
3
// Licensed under the Apache License, Version 2.0 (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//     http://www.apache.org/licenses/LICENSE-2.0
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
// See the License for the specific language governing permissions and
13
// limitations under the License.
14

15
package helmreconciler
16

17
import (
18
	"context"
19
	"fmt"
20
	"strings"
21

22
	kerrors "k8s.io/apimachinery/pkg/api/errors"
23
	"k8s.io/apimachinery/pkg/api/meta"
24
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
25
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
26
	klabels "k8s.io/apimachinery/pkg/labels"
27
	"k8s.io/apimachinery/pkg/runtime/schema"
28
	"k8s.io/apimachinery/pkg/selection"
29
	"sigs.k8s.io/controller-runtime/pkg/client"
30

31
	"istio.io/api/label"
32
	"istio.io/api/operator/v1alpha1"
33
	iopv1alpha1 "istio.io/istio/operator/pkg/apis/istio/v1alpha1"
34
	"istio.io/istio/operator/pkg/cache"
35
	"istio.io/istio/operator/pkg/metrics"
36
	"istio.io/istio/operator/pkg/name"
37
	"istio.io/istio/operator/pkg/object"
38
	"istio.io/istio/operator/pkg/translate"
39
	"istio.io/istio/operator/pkg/util"
40
	"istio.io/istio/pkg/config/constants"
41
	"istio.io/istio/pkg/config/schema/gvk"
42
	"istio.io/istio/pkg/kube"
43
	"istio.io/istio/pkg/proxy"
44
)
45

46
var (
47
	// ClusterResources are resource types the operator prunes, ordered by which types should be deleted, first to last.
48
	ClusterResources = []schema.GroupVersionKind{
49
		{Group: "admissionregistration.k8s.io", Version: "v1", Kind: name.MutatingWebhookConfigurationStr},
50
		{Group: "admissionregistration.k8s.io", Version: "v1", Kind: name.ValidatingWebhookConfigurationStr},
51
		{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: name.ClusterRoleStr},
52
		{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: name.ClusterRoleBindingStr},
53
		// Cannot currently prune CRDs because this will also wipe out user config.
54
		// {Group: "apiextensions.k8s.io", Version: "v1beta1", Kind: name.CRDStr},
55
	}
56
	// ClusterCPResources lists cluster scope resources types which should be deleted during uninstall command.
57
	ClusterCPResources = []schema.GroupVersionKind{
58
		{Group: "admissionregistration.k8s.io", Version: "v1", Kind: name.MutatingWebhookConfigurationStr},
59
		{Group: "admissionregistration.k8s.io", Version: "v1", Kind: name.ValidatingWebhookConfigurationStr},
60
		{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: name.ClusterRoleStr},
61
		{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: name.ClusterRoleBindingStr},
62
	}
63
	// AllClusterResources lists all cluster scope resources types which should be deleted in purge case, including CRD.
64
	AllClusterResources = append(ClusterResources,
65
		schema.GroupVersionKind{Group: "apiextensions.k8s.io", Version: "v1", Kind: name.CRDStr},
66
		schema.GroupVersionKind{Group: "k8s.cni.cncf.io", Version: "v1", Kind: name.NetworkAttachmentDefinitionStr},
67
	)
68
)
69

70
// NamespacedResources gets specific pruning resources based on the k8s version
71
func NamespacedResources() []schema.GroupVersionKind {
72
	res := []schema.GroupVersionKind{
73
		{Group: "apps", Version: "v1", Kind: name.DeploymentStr},
74
		{Group: "apps", Version: "v1", Kind: name.DaemonSetStr},
75
		{Group: "", Version: "v1", Kind: name.ServiceStr},
76
		{Group: "", Version: "v1", Kind: name.CMStr},
77
		{Group: "", Version: "v1", Kind: name.PodStr},
78
		{Group: "", Version: "v1", Kind: name.SecretStr},
79
		{Group: "", Version: "v1", Kind: name.SAStr},
80
		{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: name.RoleBindingStr},
81
		{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: name.RoleStr},
82
		{Group: "policy", Version: "v1", Kind: name.PDBStr},
83
		{Group: "autoscaling", Version: "v2", Kind: name.HPAStr},
84
		gvk.EnvoyFilter.Kubernetes(),
85
	}
86
	return res
87
}
88

89
// Prune removes any resources not specified in manifests generated by HelmReconciler h.
90
func (h *HelmReconciler) Prune(manifests name.ManifestMap, all bool) error {
91
	return h.runForAllTypes(func(labels map[string]string, objects *unstructured.UnstructuredList) error {
92
		var errs util.Errors
93
		if all {
94
			errs = util.AppendErr(errs, h.deleteResources(nil, labels, "", objects, all))
95
		} else {
96
			for cname, manifest := range manifests.Consolidated() {
97
				errs = util.AppendErr(errs, h.deleteResources(object.AllObjectHashes(manifest), labels, cname, objects, all))
98
			}
99
		}
100
		return errs.ToError()
101
	})
102
}
103

104
// PruneControlPlaneByRevisionWithController is called to remove specific control plane revision
105
// during reconciliation process of controller.
106
// It returns the install status and any error encountered.
107
func (h *HelmReconciler) PruneControlPlaneByRevisionWithController(iopSpec *v1alpha1.IstioOperatorSpec) (*v1alpha1.InstallStatus, error) {
108
	ns := iopv1alpha1.Namespace(iopSpec)
109
	if ns == "" {
110
		ns = constants.IstioSystemNamespace
111
	}
112
	errStatus := &v1alpha1.InstallStatus{Status: v1alpha1.InstallStatus_ERROR}
113
	enabledComponents, err := translate.GetEnabledComponents(iopSpec)
114
	if err != nil {
115
		return errStatus,
116
			fmt.Errorf("failed to get enabled components: %v", err)
117
	}
118
	pilotEnabled := false
119
	// check whether the istiod is enabled
120
	for _, c := range enabledComponents {
121
		if c == string(name.PilotComponentName) {
122
			pilotEnabled = true
123
			break
124
		}
125
	}
126
	// If istiod is enabled, check if it has any proxies connected.
127
	if pilotEnabled {
128
		cfg := h.kubeClient.RESTConfig()
129
		kubeClient, err := kube.NewCLIClient(kube.NewClientConfigForRestConfig(cfg), iopSpec.Revision)
130
		if err != nil {
131
			return errStatus, err
132
		}
133

134
		pilotExists, err := h.pilotExists(kubeClient, ns)
135
		if err != nil {
136
			return errStatus, fmt.Errorf("failed to check istiod extist: %v", err)
137
		}
138

139
		if pilotExists {
140
			// TODO(ramaraochavali): Find a better alternative instead of using debug interface
141
			// of istiod as it is typically not recommended in production environments.
142
			pids, err := proxy.GetIDsFromProxyInfo(kubeClient, ns)
143
			if err != nil {
144
				return errStatus, fmt.Errorf("failed to check proxy infos: %v", err)
145
			}
146
			if len(pids) != 0 {
147
				msg := fmt.Sprintf("there are proxies still pointing to the pruned control plane: %s.",
148
					strings.Join(pids, " "))
149
				st := &v1alpha1.InstallStatus{Status: v1alpha1.InstallStatus_ACTION_REQUIRED, Message: msg}
150
				return st, nil
151
			}
152
		}
153
	}
154

155
	for _, c := range enabledComponents {
156
		uslist, err := h.GetPrunedResources(iopSpec.Revision, false, c)
157
		if err != nil {
158
			return errStatus, err
159
		}
160
		err = h.DeleteObjectsList(uslist, c)
161
		if err != nil {
162
			return errStatus, err
163
		}
164
	}
165
	return &v1alpha1.InstallStatus{Status: v1alpha1.InstallStatus_HEALTHY}, nil
166
}
167

168
func (h *HelmReconciler) pilotExists(cliClient kube.CLIClient, istioNamespace string) (bool, error) {
169
	istiodPods, err := cliClient.GetIstioPods(context.TODO(), istioNamespace, metav1.ListOptions{
170
		LabelSelector: "app=istiod",
171
		FieldSelector: "status.phase=Running",
172
	})
173
	if err != nil {
174
		return false, err
175
	}
176

177
	return len(istiodPods) > 0, nil
178
}
179

180
// DeleteObjectsList removed resources that are in the slice of UnstructuredList.
181
func (h *HelmReconciler) DeleteObjectsList(objectsList []*unstructured.UnstructuredList, componentName string) error {
182
	var errs util.Errors
183
	deletedObjects := make(map[string]bool)
184
	for _, ul := range objectsList {
185
		for _, o := range ul.Items {
186
			obj := object.NewK8sObject(&o, nil, nil)
187
			oh := obj.Hash()
188

189
			// kube client does not differentiate API version when listing, added this check to deduplicate.
190
			if deletedObjects[oh] {
191
				continue
192
			}
193
			if err := h.deleteResource(obj, componentName, oh); err != nil {
194
				errs = append(errs, err)
195
			}
196
			deletedObjects[oh] = true
197
		}
198
	}
199

200
	return errs.ToError()
201
}
202

203
// GetPrunedResources get the list of resources to be removed
204
// 1. if includeClusterResources is false, we list the namespaced resources by matching revision and component labels.
205
// 2. if includeClusterResources is true, we list the namespaced and cluster resources by component labels only.
206
// If componentName is not empty, only resources associated with specific components would be returned
207
// UnstructuredList of objects and corresponding list of name kind hash of k8sObjects would be returned
208
func (h *HelmReconciler) GetPrunedResources(revision string, includeClusterResources bool, componentName string) (
209
	[]*unstructured.UnstructuredList, error,
210
) {
211
	var usList []*unstructured.UnstructuredList
212
	labels := make(map[string]string)
213
	if revision != "" {
214
		labels[label.IoIstioRev.Name] = revision
215
	}
216
	if componentName != "" {
217
		labels[IstioComponentLabelStr] = componentName
218
	}
219
	if h.iop.GetName() != "" {
220
		labels[OwningResourceName] = h.iop.GetName()
221
	}
222
	if h.iop.GetNamespace() != "" {
223
		labels[OwningResourceNamespace] = h.iop.GetNamespace()
224
	}
225
	selector := klabels.Set(labels).AsSelectorPreValidated()
226
	resources := NamespacedResources()
227
	gvkList := append(resources, ClusterCPResources...)
228
	if includeClusterResources {
229
		gvkList = append(resources, AllClusterResources...)
230
		// Cleanup IstioOperator, which may be used with in-cluster operator.
231
		if ioplist := h.getIstioOperatorCR(); ioplist != nil && len(ioplist.Items) > 0 {
232
			usList = append(usList, ioplist)
233
		}
234
	}
235
	for _, gvk := range gvkList {
236
		objects := &unstructured.UnstructuredList{}
237
		objects.SetGroupVersionKind(gvk)
238
		componentRequirement, err := klabels.NewRequirement(IstioComponentLabelStr, selection.Exists, nil)
239
		if err != nil {
240
			return usList, err
241
		}
242
		if includeClusterResources {
243
			s := klabels.NewSelector()
244
			err = h.client.List(context.TODO(), objects,
245
				client.MatchingLabelsSelector{Selector: s.Add(*componentRequirement)})
246
		} else {
247
			// do not prune base components or unknown components
248
			includeCN := []string{
249
				string(name.PilotComponentName),
250
				string(name.IngressComponentName), string(name.EgressComponentName),
251
				string(name.CNIComponentName), string(name.IstioOperatorComponentName),
252
				string(name.IstiodRemoteComponentName),
253
				string(name.ZtunnelComponentName),
254
			}
255
			includeRequirement, err := klabels.NewRequirement(IstioComponentLabelStr, selection.In, includeCN)
256
			if err != nil {
257
				return usList, err
258
			}
259
			if err = h.client.List(context.TODO(), objects,
260
				client.MatchingLabelsSelector{
261
					Selector: selector.Add(*includeRequirement, *componentRequirement),
262
				},
263
			); err != nil {
264
				continue
265
			}
266
		}
267
		if err != nil {
268
			continue
269
		}
270
		for _, obj := range objects.Items {
271
			objName := fmt.Sprintf("%s/%s", obj.GetNamespace(), obj.GetName())
272
			metrics.AddResource(objName, gvk.GroupKind())
273
		}
274
		if len(objects.Items) == 0 {
275
			continue
276
		}
277
		usList = append(usList, objects)
278
	}
279

280
	return usList, nil
281
}
282

283
// getIstioOperatorCR is a helper function to get IstioOperator CR during purge,
284
// otherwise the resources would be reconciled back later if there is in-cluster operator deployment.
285
// And it is needed to remove the IstioOperator CRD.
286
func (h *HelmReconciler) getIstioOperatorCR() *unstructured.UnstructuredList {
287
	iopGVR := iopv1alpha1.IstioOperatorGVR
288
	objects, err := h.kubeClient.Dynamic().Resource(iopGVR).List(context.TODO(), metav1.ListOptions{})
289
	if err != nil {
290
		if kerrors.IsNotFound(err) {
291
			return nil
292
		}
293
		scope.Errorf("failed to list IstioOperator CR: %v", err)
294
	}
295
	return objects
296
}
297

298
// runForAllTypes will collect all existing resource types we care about. For each type, the callback function
299
// will be called with the labels used to select this type, and all objects.
300
// This is in internal function meant to support prune and delete
301
func (h *HelmReconciler) runForAllTypes(callback func(labels map[string]string, objects *unstructured.UnstructuredList) error) error {
302
	var errs util.Errors
303
	// Ultimately, we want to prune based on component labels. Each of these share a common set of labels
304
	// Rather than do N List() calls for each component, we will just filter for the common subset here
305
	// and each component will do its own filtering
306
	// Because we are filtering by the core labels, List() will only return items that some components will care
307
	// about, so we are not querying for an overly broad set of resources.
308
	labels, err := h.getCoreOwnerLabels()
309
	if err != nil {
310
		return err
311
	}
312
	selector := klabels.Set(labels).AsSelectorPreValidated()
313
	componentRequirement, err := klabels.NewRequirement(IstioComponentLabelStr, selection.Exists, nil)
314
	if err != nil {
315
		return err
316
	}
317
	selector = selector.Add(*componentRequirement)
318

319
	resources := PrunedResourcesSchemas()
320
	for _, gvk := range resources {
321
		// First, we collect all objects for the provided GVK
322
		objects := &unstructured.UnstructuredList{}
323
		objects.SetGroupVersionKind(gvk)
324
		if err := h.client.List(context.TODO(), objects, client.MatchingLabelsSelector{Selector: selector}); err != nil {
325
			// we only want to retrieve resources clusters
326
			if !(h.opts.DryRun && meta.IsNoMatchError(err)) {
327
				scope.Debugf("retrieving resources to prune type %s: %s", gvk.String(), err)
328
			}
329
			continue
330
		}
331
		for _, obj := range objects.Items {
332
			objName := fmt.Sprintf("%s/%s", obj.GetNamespace(), obj.GetName())
333
			metrics.AddResource(objName, gvk.GroupKind())
334
		}
335
		errs = util.AppendErr(errs, callback(labels, objects))
336
	}
337
	return errs.ToError()
338
}
339

340
func PrunedResourcesSchemas() []schema.GroupVersionKind {
341
	return append(NamespacedResources(), ClusterResources...)
342
}
343

344
// deleteResources delete any resources from the given component that are not in the excluded map. Resource
345
// labels are used to identify the resources belonging to the component.
346
func (h *HelmReconciler) deleteResources(excluded map[string]bool, coreLabels map[string]string,
347
	componentName string, objects *unstructured.UnstructuredList, all bool,
348
) error {
349
	var errs util.Errors
350
	labels := h.addComponentLabels(coreLabels, componentName)
351
	selector := klabels.Set(labels).AsSelectorPreValidated()
352
	for _, o := range objects.Items {
353
		obj := object.NewK8sObject(&o, nil, nil)
354
		oh := obj.Hash()
355
		if !all {
356
			// Label mismatch. Provided objects don't select against the component, so this likely means the object
357
			// is for another component.
358
			if !selector.Matches(klabels.Set(o.GetLabels())) {
359
				continue
360
			}
361
			if excluded[oh] {
362
				continue
363
			}
364
			if o.GetLabels()[OwningResourceNotPruned] == "true" {
365
				continue
366
			}
367
		}
368
		if err := h.deleteResource(obj, componentName, oh); err != nil {
369
			errs = append(errs, err)
370
		}
371
	}
372
	if all {
373
		cache.FlushObjectCaches()
374
	}
375

376
	return errs.ToError()
377
}
378

379
func (h *HelmReconciler) deleteResource(obj *object.K8sObject, componentName, oh string) error {
380
	if h.opts.DryRun {
381
		h.opts.Log.LogAndPrintf("Not pruning object %s because of dry run.", oh)
382
		return nil
383
	}
384
	u := obj.UnstructuredObject()
385
	if u.GetKind() == name.IstioOperatorStr {
386
		u.SetFinalizers([]string{})
387
		if err := h.client.Patch(context.TODO(), u, client.Merge); err != nil {
388
			scope.Errorf("failed to patch IstioOperator CR: %s, %v", u.GetName(), err)
389
		}
390
	}
391
	err := h.client.Delete(context.TODO(), u, client.PropagationPolicy(metav1.DeletePropagationBackground))
392
	scope.Debugf("Deleting %s (%s/%v)", oh, h.iop.Name, h.iop.Spec.Revision)
393
	objGvk := u.GroupVersionKind()
394
	if err != nil {
395
		if !kerrors.IsNotFound(err) {
396
			return err
397
		}
398
		// do not return error if resources are not found
399
		h.opts.Log.LogAndPrintf("object: %s is not being deleted because it no longer exists", obj.Hash())
400
		return nil
401
	}
402
	if componentName != "" {
403
		h.removeFromObjectCache(componentName, oh)
404
	} else {
405
		cache.FlushObjectCaches()
406
	}
407
	metrics.ResourceDeletionTotal.
408
		With(metrics.ResourceKindLabel.Value(util.GKString(objGvk.GroupKind()))).
409
		Increment()
410
	h.addPrunedKind(objGvk.GroupKind())
411
	metrics.RemoveResource(obj.FullName(), objGvk.GroupKind())
412
	h.opts.Log.LogAndPrintf("  Removed %s.", oh)
413
	return nil
414
}
415

416
// RemoveObject removes object with objHash in componentName from the object cache.
417
func (h *HelmReconciler) removeFromObjectCache(componentName, objHash string) {
418
	crHash, err := h.getCRHash(componentName)
419
	if err != nil {
420
		scope.Error(err.Error())
421
	}
422
	cache.RemoveObject(crHash, objHash)
423
	scope.Infof("Removed object %s from Cache.", objHash)
424
}
425