talm

1
// This Source Code Form is subject to the terms of the Mozilla Public
2
// License, v. 2.0. If a copy of the MPL was not distributed with this
3
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
4

5
package cri_test
6

7
import (
8
	"context"
9
	"os"
10
	"path/filepath"
11
	"sync"
12
	"testing"
13
	"time"
14

15
	"github.com/containerd/cgroups/v3"
16
	"github.com/containerd/cgroups/v3/cgroup1"
17
	"github.com/containerd/cgroups/v3/cgroup2"
18
	"github.com/opencontainers/runtime-spec/specs-go"
19
	"github.com/stretchr/testify/suite"
20
	runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1"
21

22
	"github.com/aenix-io/talm/internal/app/machined/pkg/runtime/logging"
23
	"github.com/aenix-io/talm/internal/app/machined/pkg/system/events"
24
	"github.com/aenix-io/talm/internal/app/machined/pkg/system/runner"
25
	"github.com/aenix-io/talm/internal/app/machined/pkg/system/runner/process"
26
	"github.com/aenix-io/talm/internal/pkg/cri"
27
	"github.com/siderolabs/talos/pkg/machinery/constants"
28
)
29

30
const (
31
	busyboxImage = "docker.io/library/busybox:1.30.1"
32
)
33

34
func MockEventSink(t *testing.T) func(state events.ServiceState, message string, args ...interface{}) {
35
	return func(state events.ServiceState, message string, args ...interface{}) {
36
		t.Logf(message, args...)
37
	}
38
}
39

40
type CRISuite struct {
41
	suite.Suite
42

43
	tmpDir string
44

45
	containerdRunner  runner.Runner
46
	containerdWg      sync.WaitGroup
47
	containerdAddress string
48

49
	client    *cri.Client
50
	ctx       context.Context //nolint:containedctx
51
	ctxCancel context.CancelFunc
52
}
53

54
func (suite *CRISuite) SetupSuite() {
55
	if cgroups.Mode() == cgroups.Unified {
56
		suite.T().Skip("test doesn't pass under cgroupsv2")
57
	}
58

59
	var err error
60

61
	suite.tmpDir = suite.T().TempDir()
62

63
	stateDir, rootDir := filepath.Join(suite.tmpDir, "state"), filepath.Join(suite.tmpDir, "root")
64
	suite.Require().NoError(os.Mkdir(stateDir, 0o777))
65
	suite.Require().NoError(os.Mkdir(rootDir, 0o777))
66

67
	if cgroups.Mode() == cgroups.Unified {
68
		var (
69
			groupPath string
70
			manager   *cgroup2.Manager
71
		)
72

73
		groupPath, err = cgroup2.NestedGroupPath(suite.tmpDir)
74
		suite.Require().NoError(err)
75

76
		manager, err = cgroup2.NewManager(constants.CgroupMountPath, groupPath, &cgroup2.Resources{})
77
		suite.Require().NoError(err)
78

79
		defer manager.Delete() //nolint:errcheck
80
	} else {
81
		var manager cgroup1.Cgroup
82

83
		manager, err = cgroup1.New(cgroup1.NestedPath(suite.tmpDir), &specs.LinuxResources{})
84
		suite.Require().NoError(err)
85

86
		defer manager.Delete() //nolint:errcheck
87
	}
88

89
	suite.containerdAddress = filepath.Join(suite.tmpDir, "run.sock")
90

91
	args := &runner.Args{
92
		ID: "containerd",
93
		ProcessArgs: []string{
94
			"/bin/containerd",
95
			"--address", suite.containerdAddress,
96
			"--state", stateDir,
97
			"--root", rootDir,
98
			"--config", constants.CRIContainerdConfig,
99
		},
100
	}
101

102
	suite.containerdRunner = process.NewRunner(
103
		false,
104
		args,
105
		runner.WithLoggingManager(logging.NewFileLoggingManager(suite.tmpDir)),
106
		runner.WithEnv([]string{"PATH=/bin:" + constants.PATH}),
107
		runner.WithCgroupPath(suite.tmpDir),
108
	)
109
	suite.Require().NoError(suite.containerdRunner.Open())
110
	suite.containerdWg.Add(1)
111

112
	go func() {
113
		defer suite.containerdWg.Done()
114
		defer suite.containerdRunner.Close()                 //nolint:errcheck
115
		suite.containerdRunner.Run(MockEventSink(suite.T())) //nolint:errcheck
116
	}()
117

118
	suite.client, err = cri.NewClient("unix:"+suite.containerdAddress, 30*time.Second)
119
	suite.Require().NoError(err)
120
}
121

122
func (suite *CRISuite) TearDownSuite() {
123
	suite.ctxCancel()
124

125
	suite.Require().NoError(suite.client.Close())
126

127
	suite.Require().NoError(suite.containerdRunner.Stop())
128
	suite.containerdWg.Wait()
129
}
130

131
func (suite *CRISuite) SetupTest() {
132
	suite.ctx, suite.ctxCancel = context.WithTimeout(context.Background(), 30*time.Second)
133
}
134

135
func (suite *CRISuite) TearDownTest() {
136
	suite.ctxCancel()
137
}
138

139
func (suite *CRISuite) TestRunSandboxContainer() {
140
	podSandboxConfig := &runtimeapi.PodSandboxConfig{
141
		Metadata: &runtimeapi.PodSandboxMetadata{
142
			Name:      "etcd-master-1",
143
			Uid:       "ed1a599a53090941c9b4025c7e3e883d",
144
			Namespace: "kube-system",
145
			Attempt:   0,
146
		},
147
		Labels: map[string]string{
148
			"io.kubernetes.pod.name":      "etcd-master-1",
149
			"io.kubernetes.pod.namespace": "kube-system",
150
		},
151
		LogDirectory: suite.tmpDir,
152
		Linux: &runtimeapi.LinuxPodSandboxConfig{
153
			SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{
154
				NamespaceOptions: &runtimeapi.NamespaceOption{
155
					Network: runtimeapi.NamespaceMode_NODE,
156
				},
157
			},
158
		},
159
	}
160

161
	podSandboxID, err := suite.client.RunPodSandbox(suite.ctx, podSandboxConfig, "")
162
	suite.Require().NoError(err)
163
	suite.Require().Len(podSandboxID, 64)
164

165
	imageRef, err := suite.client.PullImage(
166
		suite.ctx, &runtimeapi.ImageSpec{
167
			Image: busyboxImage,
168
		}, podSandboxConfig,
169
	)
170
	suite.Require().NoError(err)
171

172
	_, err = suite.client.ImageStatus(
173
		suite.ctx, &runtimeapi.ImageSpec{
174
			Image: imageRef,
175
		},
176
	)
177
	suite.Require().NoError(err)
178

179
	ctrID, err := suite.client.CreateContainer(
180
		suite.ctx, podSandboxID,
181
		&runtimeapi.ContainerConfig{
182
			Metadata: &runtimeapi.ContainerMetadata{
183
				Name: "etcd",
184
			},
185
			Labels: map[string]string{
186
				"io.kubernetes.container.name": "etcd",
187
				"io.kubernetes.pod.name":       "etcd-master-1",
188
				"io.kubernetes.pod.namespace":  "kube-system",
189
			},
190
			Annotations: map[string]string{
191
				"io.kubernetes.container.restartCount": "1",
192
			},
193
			Image: &runtimeapi.ImageSpec{
194
				Image: imageRef,
195
			},
196
			Command: []string{"/bin/sh", "-c", "sleep 3600"},
197
		}, podSandboxConfig,
198
	)
199
	suite.Require().NoError(err)
200
	suite.Require().Len(ctrID, 64)
201

202
	err = suite.client.StartContainer(suite.ctx, ctrID)
203
	suite.Require().NoError(err)
204

205
	_, err = suite.client.ContainerStats(suite.ctx, ctrID)
206
	suite.Require().NoError(err)
207

208
	_, _, err = suite.client.ContainerStatus(suite.ctx, ctrID, true)
209
	suite.Require().NoError(err)
210

211
	err = suite.client.StopContainer(suite.ctx, ctrID, 10)
212
	suite.Require().NoError(err)
213

214
	err = suite.client.RemoveContainer(suite.ctx, ctrID)
215
	suite.Require().NoError(err)
216

217
	err = suite.client.StopPodSandbox(suite.ctx, podSandboxID)
218
	suite.Require().NoError(err)
219

220
	err = suite.client.RemovePodSandbox(suite.ctx, podSandboxID)
221
	suite.Require().NoError(err)
222
}
223

224
func (suite *CRISuite) TestList() {
225
	pods, err := suite.client.ListPodSandbox(suite.ctx, &runtimeapi.PodSandboxFilter{})
226
	suite.Require().NoError(err)
227
	suite.Require().Len(pods, 0)
228

229
	containers, err := suite.client.ListContainers(suite.ctx, &runtimeapi.ContainerFilter{})
230
	suite.Require().NoError(err)
231
	suite.Require().Len(containers, 0)
232

233
	containerStats, err := suite.client.ListContainerStats(suite.ctx, &runtimeapi.ContainerStatsFilter{})
234
	suite.Require().NoError(err)
235
	suite.Require().Len(containerStats, 0)
236

237
	_, err = suite.client.ListImages(suite.ctx, &runtimeapi.ImageFilter{})
238
	suite.Require().NoError(err)
239
}
240

241
func TestCRISuite(t *testing.T) {
242
	if os.Getuid() != 0 {
243
		t.Skip("can't run the test as non-root")
244
	}
245

246
	_, err := os.Stat("/bin/containerd")
247
	if err != nil {
248
		t.Skip("containerd binary is not available, skipping the test")
249
	}
250

251
	suite.Run(t, new(CRISuite))
252
}
253