argo-cd

1
package generator
2

3
import (
4
	"bytes"
5
	"context"
6
	"encoding/base64"
7
	"errors"
8
	"log"
9
	"strings"
10
	"time"
11

12
	v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
13

14
	"github.com/argoproj/argo-cd/v2/util/helm"
15

16
	"gopkg.in/yaml.v2"
17

18
	"k8s.io/client-go/kubernetes/scheme"
19

20
	v1 "k8s.io/api/core/v1"
21
	"k8s.io/client-go/rest"
22
	"k8s.io/client-go/tools/remotecommand"
23

24
	"k8s.io/client-go/kubernetes"
25

26
	"github.com/argoproj/argo-cd/v2/hack/gen-resources/util"
27
	argoappv1 "github.com/argoproj/argo-cd/v2/pkg/apis/application/v1alpha1"
28
	"github.com/argoproj/argo-cd/v2/util/db"
29
)
30

31
const POD_PREFIX = "vcluster"
32

33
type Cluster struct {
34
	Server                   string `yaml:"server"`
35
	CertificateAuthorityData string `yaml:"certificate-authority-data,omitempty"`
36
}
37

38
type AuthInfo struct {
39
	ClientCertificateData string `yaml:"client-certificate-data,omitempty"`
40
	ClientKeyData         string `yaml:"client-key-data,omitempty"`
41
}
42

43
type NamedCluster struct {
44
	// Name is the nickname for this Cluster
45
	Name string `yaml:"name"`
46
	// Cluster holds the cluster information
47
	Cluster Cluster `yaml:"cluster"`
48
}
49

50
type NamedAuthInfo struct {
51
	// Name is the nickname for this AuthInfo
52
	Name string `yaml:"name"`
53
	// AuthInfo holds the auth information
54
	AuthInfo AuthInfo `yaml:"user"`
55
}
56

57
type Config struct {
58
	Clusters  []NamedCluster  `yaml:"clusters"`
59
	AuthInfos []NamedAuthInfo `yaml:"users"`
60
}
61

62
type ClusterGenerator struct {
63
	db        db.ArgoDB
64
	clientSet *kubernetes.Clientset
65
	config    *rest.Config
66
}
67

68
func NewClusterGenerator(db db.ArgoDB, clientSet *kubernetes.Clientset, config *rest.Config) Generator {
69
	return &ClusterGenerator{db, clientSet, config}
70
}
71

72
func (cg *ClusterGenerator) getClusterCredentials(namespace string, releaseSuffix string) ([]byte, []byte, []byte, error) {
73
	cmd := []string{
74
		"sh",
75
		"-c",
76
		"cat /root/.kube/config",
77
	}
78

79
	var stdout, stderr, stdin bytes.Buffer
80
	option := &v1.PodExecOptions{
81
		Command:   cmd,
82
		Container: "syncer",
83
		Stdin:     true,
84
		Stdout:    true,
85
		Stderr:    true,
86
		TTY:       true,
87
	}
88

89
	req := cg.clientSet.CoreV1().RESTClient().Post().Resource("pods").Name(POD_PREFIX + "-" + releaseSuffix + "-0").
90
		Namespace(namespace).SubResource("exec")
91

92
	req.VersionedParams(
93
		option,
94
		scheme.ParameterCodec,
95
	)
96

97
	exec, err := remotecommand.NewSPDYExecutor(cg.config, "POST", req.URL())
98
	if err != nil {
99
		return nil, nil, nil, err
100
	}
101

102
	err = exec.StreamWithContext(context.Background(), remotecommand.StreamOptions{
103
		Stdin:  &stdin,
104
		Stdout: &stdout,
105
		Stderr: &stderr,
106
	})
107
	if err != nil {
108
		return nil, nil, nil, err
109
	}
110

111
	var config Config
112

113
	err = yaml.Unmarshal(stdout.Bytes(), &config)
114
	if err != nil {
115
		return nil, nil, nil, err
116
	}
117

118
	if len(config.Clusters) == 0 {
119
		return nil, nil, nil, errors.New("clusters empty")
120
	}
121

122
	caData, err := base64.StdEncoding.DecodeString(config.Clusters[0].Cluster.CertificateAuthorityData)
123
	if err != nil {
124
		return nil, nil, nil, err
125
	}
126

127
	cert, err := base64.StdEncoding.DecodeString(config.AuthInfos[0].AuthInfo.ClientCertificateData)
128
	if err != nil {
129
		return nil, nil, nil, err
130
	}
131

132
	key, err := base64.StdEncoding.DecodeString(config.AuthInfos[0].AuthInfo.ClientKeyData)
133
	if err != nil {
134
		return nil, nil, nil, err
135
	}
136

137
	return caData, cert, key, nil
138
}
139

140
// TODO: also should provision service for vcluster pod
141
func (cg *ClusterGenerator) installVCluster(opts *util.GenerateOpts, namespace string, releaseName string) error {
142
	cmd, err := helm.NewCmd("/tmp", "v3", "")
143
	if err != nil {
144
		return err
145
	}
146
	log.Print("Execute helm install command")
147
	_, err = cmd.Freestyle("upgrade", "--install", releaseName, "vcluster", "--values", opts.ClusterOpts.ValuesFilePath, "--repo", "https://charts.loft.sh", "--namespace", namespace, "--repository-config", "", "--create-namespace", "--wait")
148
	if err != nil {
149
		return err
150
	}
151
	return nil
152
}
153

154
func (cg *ClusterGenerator) getClusterServerUri(namespace string, releaseSuffix string) (string, error) {
155
	pod, err := cg.clientSet.CoreV1().Pods(namespace).Get(context.TODO(), POD_PREFIX+"-"+releaseSuffix+"-0", v12.GetOptions{})
156
	if err != nil {
157
		return "", err
158
	}
159
	// TODO: should be moved to service instead pod
160
	log.Printf("Get service for https://" + pod.Status.PodIP + ":8443")
161
	return "https://" + pod.Status.PodIP + ":8443", nil
162
}
163

164
func (cg *ClusterGenerator) retrieveClusterUri(namespace, releaseSuffix string) (string, error) {
165
	for i := 0; i < 10; i++ {
166
		log.Printf("Attempting to get cluster uri")
167
		uri, err := cg.getClusterServerUri(namespace, releaseSuffix)
168
		if err != nil {
169
			log.Printf("Failed to get cluster uri due to %s", err.Error())
170
			time.Sleep(10 * time.Second)
171
			continue
172
		}
173
		return uri, nil
174
	}
175
	return "", nil
176
}
177

178
func (cg *ClusterGenerator) generate(i int, opts *util.GenerateOpts) error {
179
	log.Printf("Generate cluster #%v of #%v", i, opts.ClusterOpts.Samples)
180

181
	namespace := opts.ClusterOpts.NamespacePrefix + "-" + util.GetRandomString()
182

183
	log.Printf("Namespace is %s", namespace)
184

185
	releaseSuffix := util.GetRandomString()
186

187
	log.Printf("Release suffix is %s", namespace)
188

189
	err := cg.installVCluster(opts, namespace, POD_PREFIX+"-"+releaseSuffix)
190
	if err != nil {
191
		log.Printf("Skip cluster installation due error %v", err.Error())
192
	}
193

194
	log.Print("Get cluster credentials")
195
	caData, cert, key, err := cg.getClusterCredentials(namespace, releaseSuffix)
196

197
	for o := 0; o < 5; o++ {
198
		if err == nil {
199
			break
200
		}
201
		log.Printf("Failed to get cluster credentials %s, retrying...", releaseSuffix)
202
		time.Sleep(10 * time.Second)
203
		caData, cert, key, err = cg.getClusterCredentials(namespace, releaseSuffix)
204
	}
205
	if err != nil {
206
		return err
207
	}
208

209
	log.Print("Get cluster server uri")
210

211
	uri, err := cg.retrieveClusterUri(namespace, releaseSuffix)
212
	if err != nil {
213
		return err
214
	}
215

216
	log.Printf("Cluster server uri is %s", uri)
217

218
	log.Print("Create cluster")
219
	_, err = cg.db.CreateCluster(context.TODO(), &argoappv1.Cluster{
220
		Server: uri,
221
		Name:   opts.ClusterOpts.ClusterNamePrefix + "-" + util.GetRandomString(),
222
		Config: argoappv1.ClusterConfig{
223
			TLSClientConfig: argoappv1.TLSClientConfig{
224
				Insecure:   false,
225
				ServerName: "kubernetes.default.svc",
226
				CAData:     caData,
227
				CertData:   cert,
228
				KeyData:    key,
229
			},
230
		},
231
		ConnectionState: argoappv1.ConnectionState{},
232
		ServerVersion:   "1.18",
233
		Namespaces:      []string{opts.ClusterOpts.DestinationNamespace},
234
		Labels:          labels,
235
	})
236
	if err != nil {
237
		return err
238
	}
239
	return nil
240
}
241

242
func (cg *ClusterGenerator) Generate(opts *util.GenerateOpts) error {
243
	log.Printf("Excute in parallel with %v", opts.ClusterOpts.Concurrency)
244

245
	wg := util.New(opts.ClusterOpts.Concurrency)
246
	for l := 1; l <= opts.ClusterOpts.Samples; l++ {
247
		wg.Add()
248
		go func(i int) {
249
			defer wg.Done()
250
			err := cg.generate(i, opts)
251
			if err != nil {
252
				log.Printf("Failed to generate cluster #%v due to : %s", i, err.Error())
253
			}
254
		}(l)
255
	}
256
	wg.Wait()
257
	return nil
258
}
259

260
func (cg *ClusterGenerator) Clean(opts *util.GenerateOpts) error {
261
	log.Printf("Clean clusters")
262
	namespaces, err := cg.clientSet.CoreV1().Namespaces().List(context.TODO(), v12.ListOptions{})
263
	if err != nil {
264
		return err
265
	}
266

267
	for _, ns := range namespaces.Items {
268
		if strings.HasPrefix(ns.Name, POD_PREFIX) {
269
			log.Printf("Delete namespace %s", ns.Name)
270
			err = cg.clientSet.CoreV1().Namespaces().Delete(context.TODO(), ns.Name, v12.DeleteOptions{})
271
			if err != nil {
272
				log.Printf("Delete namespace failed due: %s", err.Error())
273
			}
274
		}
275
	}
276

277
	secrets := cg.clientSet.CoreV1().Secrets(opts.Namespace)
278
	return secrets.DeleteCollection(context.TODO(), v12.DeleteOptions{}, v12.ListOptions{
279
		LabelSelector: "app.kubernetes.io/generated-by=argocd-generator",
280
	})
281
}
282