podman

1
package integration
2

3
import (
4
	"encoding/json"
5
	"fmt"
6
	"net"
7
	"os"
8
	"os/exec"
9
	"path/filepath"
10
	"strings"
11
	"time"
12

13
	"github.com/checkpoint-restore/go-criu/v7/stats"
14
	"github.com/containers/podman/v5/pkg/checkpoint/crutils"
15
	"github.com/containers/podman/v5/pkg/criu"
16
	"github.com/containers/podman/v5/pkg/domain/entities"
17
	. "github.com/containers/podman/v5/test/utils"
18
	"github.com/containers/podman/v5/utils"
19
	. "github.com/onsi/ginkgo/v2"
20
	. "github.com/onsi/gomega"
21
	. "github.com/onsi/gomega/gexec"
22
)
23

24
var netname string
25

26
func getRunString(input []string) []string {
27
	runString := []string{"run", "-d", "--network", netname}
28
	return append(runString, input...)
29
}
30

31
var _ = Describe("Podman checkpoint", func() {
32

33
	BeforeEach(func() {
34
		SkipIfRootless("checkpoint not supported in rootless mode")
35

36
		// Check if the runtime implements checkpointing. Currently only
37
		// runc's checkpoint/restore implementation is supported.
38
		cmd := exec.Command(podmanTest.OCIRuntime, "checkpoint", "--help")
39
		if err := cmd.Start(); err != nil {
40
			Skip("OCI runtime does not support checkpoint/restore")
41
		}
42
		if err := cmd.Wait(); err != nil {
43
			Skip("OCI runtime does not support checkpoint/restore")
44
		}
45

46
		if err := criu.CheckForCriu(criu.MinCriuVersion); err != nil {
47
			Skip(fmt.Sprintf("check CRIU version error: %v", err))
48
		}
49

50
		session := podmanTest.Podman([]string{"network", "create"})
51
		session.WaitWithDefaultTimeout()
52
		Expect(session).Should(ExitCleanly())
53
		netname = session.OutputToString()
54
	})
55

56
	AfterEach(func() {
57
		if netname != "" {
58
			session := podmanTest.Podman([]string{"network", "rm", "-f", netname})
59
			session.WaitWithDefaultTimeout()
60
			Expect(session).Should(ExitCleanly())
61
		}
62
	})
63

64
	It("podman checkpoint bogus container", func() {
65
		session := podmanTest.Podman([]string{"container", "checkpoint", "foobar"})
66
		session.WaitWithDefaultTimeout()
67
		Expect(session).Should(ExitWithError(125, "no such container"))
68
	})
69

70
	It("podman restore bogus container", func() {
71
		session := podmanTest.Podman([]string{"container", "restore", "foobar"})
72
		session.WaitWithDefaultTimeout()
73
		Expect(session).Should(ExitWithError(125, "no such container or image"))
74
	})
75

76
	It("podman checkpoint a running container by id", func() {
77
		localRunString := getRunString([]string{ALPINE, "top"})
78
		session := podmanTest.Podman(localRunString)
79
		session.WaitWithDefaultTimeout()
80
		Expect(session).Should(ExitCleanly())
81
		cid := session.OutputToString()
82

83
		// Check if none of the checkpoint/restore specific information is displayed
84
		// for newly started containers.
85
		inspect := podmanTest.Podman([]string{"inspect", cid})
86
		inspect.WaitWithDefaultTimeout()
87
		Expect(inspect).Should(ExitCleanly())
88
		inspectOut := inspect.InspectContainerToJSON()
89
		Expect(inspectOut[0].State.Checkpointed).To(BeFalse(), ".State.Checkpointed")
90
		Expect(inspectOut[0].State.Restored).To(BeFalse(), ".State.Restored")
91
		Expect(inspectOut[0].State).To(HaveField("CheckpointPath", ""))
92
		Expect(inspectOut[0].State).To(HaveField("CheckpointLog", ""))
93
		Expect(inspectOut[0].State).To(HaveField("RestoreLog", ""))
94

95
		result := podmanTest.Podman([]string{
96
			"container",
97
			"checkpoint",
98
			"--keep",
99
			cid,
100
		})
101
		result.WaitWithDefaultTimeout()
102

103
		Expect(result).Should(ExitCleanly())
104
		Expect(result.OutputToString()).To(Equal(cid))
105
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
106
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
107

108
		// For a checkpointed container we expect the checkpoint related information
109
		// to be populated.
110
		inspect = podmanTest.Podman([]string{"inspect", cid})
111
		inspect.WaitWithDefaultTimeout()
112
		Expect(inspect).Should(ExitCleanly())
113
		inspectOut = inspect.InspectContainerToJSON()
114
		Expect(inspectOut[0].State.Checkpointed).To(BeTrue(), ".State.Checkpointed")
115
		Expect(inspectOut[0].State.Restored).To(BeFalse(), ".State.Restored")
116
		Expect(inspectOut[0].State.CheckpointPath).To(ContainSubstring("userdata/checkpoint"))
117
		Expect(inspectOut[0].State.CheckpointLog).To(ContainSubstring("userdata/dump.log"))
118
		Expect(inspectOut[0].State).To(HaveField("RestoreLog", ""))
119

120
		result = podmanTest.Podman([]string{
121
			"container",
122
			"restore",
123
			"--keep",
124
			cid,
125
		})
126
		result.WaitWithDefaultTimeout()
127

128
		Expect(result).Should(ExitCleanly())
129
		Expect(result.OutputToString()).To(Equal(cid))
130
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
131
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
132

133
		inspect = podmanTest.Podman([]string{"inspect", cid})
134
		inspect.WaitWithDefaultTimeout()
135
		Expect(inspect).Should(ExitCleanly())
136
		inspectOut = inspect.InspectContainerToJSON()
137
		Expect(inspectOut[0].State.Restored).To(BeTrue(), ".State.Restored")
138
		Expect(inspectOut[0].State.Checkpointed).To(BeFalse(), ".State.Checkpointed")
139
		Expect(inspectOut[0].State.CheckpointPath).To(ContainSubstring("userdata/checkpoint"))
140
		Expect(inspectOut[0].State.CheckpointLog).To(ContainSubstring("userdata/dump.log"))
141
		Expect(inspectOut[0].State.RestoreLog).To(ContainSubstring("userdata/restore.log"))
142

143
		podmanTest.StopContainer(cid)
144
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
145

146
		result = podmanTest.Podman([]string{
147
			"container",
148
			"start",
149
			cid,
150
		})
151
		result.WaitWithDefaultTimeout()
152

153
		Expect(result).Should(ExitCleanly())
154
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
155

156
		// Stopping and starting the container should remove all checkpoint
157
		// related information from inspect again.
158
		inspect = podmanTest.Podman([]string{"inspect", cid})
159
		inspect.WaitWithDefaultTimeout()
160
		Expect(inspect).Should(ExitCleanly())
161
		inspectOut = inspect.InspectContainerToJSON()
162
		Expect(inspectOut[0].State.Checkpointed).To(BeFalse(), ".State.Checkpointed")
163
		Expect(inspectOut[0].State.Restored).To(BeFalse(), ".State.Restored")
164
		Expect(inspectOut[0].State).To(HaveField("CheckpointPath", ""))
165
		Expect(inspectOut[0].State).To(HaveField("CheckpointLog", ""))
166
		Expect(inspectOut[0].State).To(HaveField("RestoreLog", ""))
167
	})
168

169
	It("podman checkpoint a running container by name", func() {
170
		localRunString := getRunString([]string{"--name", "test_name", ALPINE, "top"})
171
		session := podmanTest.Podman(localRunString)
172
		session.WaitWithDefaultTimeout()
173
		Expect(session).Should(ExitCleanly())
174

175
		result := podmanTest.Podman([]string{"container", "checkpoint", "test_name"})
176
		result.WaitWithDefaultTimeout()
177

178
		Expect(result).Should(ExitCleanly())
179
		Expect(result.OutputToString()).To(Equal("test_name"))
180
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
181
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
182

183
		result = podmanTest.Podman([]string{"container", "restore", "test_name"})
184
		result.WaitWithDefaultTimeout()
185

186
		Expect(result).Should(ExitCleanly())
187
		Expect(result.OutputToString()).To(Equal("test_name"))
188
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
189
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
190

191
		// Restore a container which name is equal to an image name (#15055)
192
		localRunString = getRunString([]string{"--name", "alpine", "quay.io/libpod/alpine:latest", "top"})
193
		session = podmanTest.Podman(localRunString)
194
		session.WaitWithDefaultTimeout()
195
		Expect(session).Should(ExitCleanly())
196

197
		result = podmanTest.Podman([]string{"container", "checkpoint", "alpine"})
198
		result.WaitWithDefaultTimeout()
199

200
		Expect(result).Should(ExitCleanly())
201
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
202
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
203

204
		result = podmanTest.Podman([]string{"container", "restore", "alpine"})
205
		result.WaitWithDefaultTimeout()
206

207
		Expect(result).Should(ExitCleanly())
208
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(2))
209
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
210
	})
211

212
	It("podman pause a checkpointed container by id", func() {
213
		localRunString := getRunString([]string{ALPINE, "top"})
214
		session := podmanTest.Podman(localRunString)
215
		session.WaitWithDefaultTimeout()
216
		Expect(session).Should(ExitCleanly())
217
		cid := session.OutputToString()
218

219
		result := podmanTest.Podman([]string{"container", "checkpoint", cid})
220
		result.WaitWithDefaultTimeout()
221

222
		Expect(result).Should(ExitCleanly())
223
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
224
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
225

226
		result = podmanTest.Podman([]string{"pause", cid})
227
		result.WaitWithDefaultTimeout()
228

229
		Expect(result).Should(ExitWithError(125, `"exited" is not running, can't pause: container state improper`))
230
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
231
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
232

233
		result = podmanTest.Podman([]string{"container", "restore", cid})
234
		result.WaitWithDefaultTimeout()
235
		Expect(result).Should(ExitCleanly())
236
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
237

238
		result = podmanTest.Podman([]string{"rm", cid})
239
		result.WaitWithDefaultTimeout()
240
		Expect(result).Should(ExitWithError(2, " as it is running - running or paused containers cannot be removed without force: container state improper"))
241
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
242

243
		result = podmanTest.Podman([]string{"rm", "-t", "1", "-f", cid})
244
		result.WaitWithDefaultTimeout()
245
		Expect(result).Should(ExitCleanly())
246
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
247

248
	})
249

250
	It("podman checkpoint latest running container", func() {
251
		localRunString := getRunString([]string{"--name", "first", ALPINE, "top"})
252
		session1 := podmanTest.Podman(localRunString)
253
		session1.WaitWithDefaultTimeout()
254
		Expect(session1).Should(ExitCleanly())
255

256
		localRunString = getRunString([]string{"--name", "second", ALPINE, "top"})
257
		session2 := podmanTest.Podman(localRunString)
258
		session2.WaitWithDefaultTimeout()
259
		Expect(session2).Should(ExitCleanly())
260

261
		result := podmanTest.Podman([]string{"container", "checkpoint", "second"})
262
		result.WaitWithDefaultTimeout()
263

264
		Expect(result).Should(ExitCleanly())
265
		Expect(result.OutputToString()).To(Equal("second"))
266
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
267

268
		ps := podmanTest.Podman([]string{"ps", "-q", "--no-trunc"})
269
		ps.WaitWithDefaultTimeout()
270
		Expect(ps).Should(ExitCleanly())
271
		Expect(ps.OutputToString()).To(ContainSubstring(session1.OutputToString()))
272
		Expect(ps.OutputToString()).To(Not(ContainSubstring(session2.OutputToString())))
273

274
		result = podmanTest.Podman([]string{"container", "restore", "second"})
275
		result.WaitWithDefaultTimeout()
276

277
		Expect(result).Should(ExitCleanly())
278
		Expect(result.OutputToString()).To(Equal("second"))
279
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(2))
280
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
281
		Expect(podmanTest.GetContainerStatus()).To(Not(ContainSubstring("Exited")))
282

283
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-fa"})
284
		result.WaitWithDefaultTimeout()
285
		Expect(result).Should(ExitCleanly())
286
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
287
	})
288

289
	It("podman checkpoint all running container", func() {
290
		localRunString := getRunString([]string{"--name", "first", ALPINE, "top"})
291
		session1 := podmanTest.Podman(localRunString)
292
		session1.WaitWithDefaultTimeout()
293
		Expect(session1).Should(ExitCleanly())
294
		cid1 := session1.OutputToString()
295

296
		localRunString = getRunString([]string{"--name", "second", ALPINE, "top"})
297
		session2 := podmanTest.Podman(localRunString)
298
		session2.WaitWithDefaultTimeout()
299
		Expect(session2).Should(ExitCleanly())
300
		cid2 := session2.OutputToString()
301

302
		result := podmanTest.Podman([]string{"container", "checkpoint", "-a"})
303
		result.WaitWithDefaultTimeout()
304

305
		Expect(result).Should(ExitCleanly())
306
		Expect(result.OutputToString()).To(ContainSubstring(cid1))
307
		Expect(result.OutputToString()).To(ContainSubstring(cid2))
308
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
309

310
		ps := podmanTest.Podman([]string{"ps", "-q", "--no-trunc"})
311
		ps.WaitWithDefaultTimeout()
312
		Expect(ps).Should(ExitCleanly())
313
		Expect(ps.OutputToString()).To(Not(ContainSubstring(session1.OutputToString())))
314
		Expect(ps.OutputToString()).To(Not(ContainSubstring(session2.OutputToString())))
315

316
		result = podmanTest.Podman([]string{"container", "restore", "-a"})
317
		result.WaitWithDefaultTimeout()
318

319
		Expect(result).Should(ExitCleanly())
320
		Expect(result.OutputToString()).To(ContainSubstring(cid1))
321
		Expect(result.OutputToString()).To(ContainSubstring(cid2))
322
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(2))
323
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
324
		Expect(podmanTest.GetContainerStatus()).To(Not(ContainSubstring("Exited")))
325

326
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-fa"})
327
		result.WaitWithDefaultTimeout()
328
		Expect(result).Should(ExitCleanly())
329
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
330
	})
331

332
	It("podman checkpoint container with established tcp connections", func() {
333
		localRunString := getRunString([]string{REDIS_IMAGE})
334
		session := podmanTest.Podman(localRunString)
335
		session.WaitWithDefaultTimeout()
336
		Expect(session).Should(ExitCleanly())
337
		cid := session.OutputToString()
338
		if !WaitContainerReady(podmanTest, cid, "Ready to accept connections", 20, 1) {
339
			Fail("Container failed to get ready")
340
		}
341

342
		// clunky format needed because CNI uses dashes in net names
343
		IP := podmanTest.Podman([]string{"inspect", cid, fmt.Sprintf("--format={{(index .NetworkSettings.Networks \"%s\").IPAddress}}", netname)})
344
		IP.WaitWithDefaultTimeout()
345
		Expect(IP).Should(ExitCleanly())
346

347
		// Open a network connection to the redis server
348
		conn, err := net.DialTimeout("tcp4", IP.OutputToString()+":6379", time.Duration(3)*time.Second)
349
		Expect(err).ToNot(HaveOccurred())
350

351
		// This should fail as the container has established TCP connections
352
		result := podmanTest.Podman([]string{"container", "checkpoint", cid})
353
		result.WaitWithDefaultTimeout()
354

355
		// FIXME: criu emits an error message, but podman never sees it:
356
		//   "CRIU checkpointing failed -52.  Please check CRIU logfile /...."
357
		Expect(result).Should(ExitWithError(125, "failed: exit status 1"))
358
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
359
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
360

361
		// Now it should work thanks to "--tcp-established"
362
		result = podmanTest.Podman([]string{"container", "checkpoint", cid, "--tcp-established"})
363
		result.WaitWithDefaultTimeout()
364

365
		Expect(result).Should(ExitCleanly())
366
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
367
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
368

369
		// Restore should fail as the checkpoint image contains established TCP connections
370
		result = podmanTest.Podman([]string{"container", "restore", cid})
371
		result.WaitWithDefaultTimeout()
372

373
		// FIXME: CRIU failure message not seen by podman (same as above)
374
		Expect(result).Should(ExitWithError(125))
375
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
376
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
377

378
		// Now it should work thanks to "--tcp-established"
379
		result = podmanTest.Podman([]string{"container", "restore", cid, "--tcp-established"})
380
		result.WaitWithDefaultTimeout()
381

382
		Expect(result).Should(ExitCleanly())
383
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
384
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
385

386
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-fa"})
387
		result.WaitWithDefaultTimeout()
388
		Expect(result).Should(ExitCleanly())
389
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
390

391
		conn.Close()
392
	})
393

394
	It("podman checkpoint with --leave-running", func() {
395
		localRunString := getRunString([]string{ALPINE, "top"})
396
		session := podmanTest.Podman(localRunString)
397
		session.WaitWithDefaultTimeout()
398
		Expect(session).Should(ExitCleanly())
399
		cid := session.OutputToString()
400

401
		// Checkpoint container, but leave it running
402
		result := podmanTest.Podman([]string{"container", "checkpoint", "--leave-running", cid})
403
		result.WaitWithDefaultTimeout()
404

405
		Expect(result).Should(ExitCleanly())
406
		// Make sure it is still running
407
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
408
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
409

410
		// Stop the container
411
		podmanTest.StopContainer(cid)
412
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
413
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
414

415
		// Restore the stopped container from the previous checkpoint
416
		result = podmanTest.Podman([]string{"container", "restore", cid})
417
		result.WaitWithDefaultTimeout()
418

419
		Expect(result).Should(ExitCleanly())
420
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
421
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
422

423
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-fa"})
424
		result.WaitWithDefaultTimeout()
425
		Expect(result).Should(ExitCleanly())
426
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
427
	})
428

429
	It("podman checkpoint and restore container with same IP", func() {
430
		localRunString := getRunString([]string{"--name", "test_name", ALPINE, "top"})
431
		session := podmanTest.Podman(localRunString)
432
		session.WaitWithDefaultTimeout()
433
		Expect(session).Should(ExitCleanly())
434

435
		// clunky format needed because CNI uses dashes in net names
436
		IPBefore := podmanTest.Podman([]string{"inspect", "test_name", fmt.Sprintf("--format={{(index .NetworkSettings.Networks \"%s\").IPAddress}}", netname)})
437
		IPBefore.WaitWithDefaultTimeout()
438
		Expect(IPBefore).Should(ExitCleanly())
439
		Expect(IPBefore.OutputToString()).To(MatchRegexp("^[0-9]+(\\.[0-9]+){3}$"))
440

441
		MACBefore := podmanTest.Podman([]string{"inspect", "test_name", fmt.Sprintf("--format={{(index .NetworkSettings.Networks \"%s\").MacAddress}}", netname)})
442
		MACBefore.WaitWithDefaultTimeout()
443
		Expect(MACBefore).Should(ExitCleanly())
444
		Expect(MACBefore.OutputToString()).To(MatchRegexp("^[0-9a-f]{2}(:[0-9a-f]{2}){5}$"))
445

446
		result := podmanTest.Podman([]string{"container", "checkpoint", "test_name"})
447
		result.WaitWithDefaultTimeout()
448

449
		Expect(result).Should(ExitCleanly())
450
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
451
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
452

453
		result = podmanTest.Podman([]string{"container", "restore", "test_name"})
454
		result.WaitWithDefaultTimeout()
455

456
		IPAfter := podmanTest.Podman([]string{"inspect", "test_name", fmt.Sprintf("--format={{(index .NetworkSettings.Networks \"%s\").IPAddress}}", netname)})
457
		IPAfter.WaitWithDefaultTimeout()
458
		Expect(IPAfter).Should(ExitCleanly())
459

460
		MACAfter := podmanTest.Podman([]string{"inspect", "test_name", fmt.Sprintf("--format={{(index .NetworkSettings.Networks \"%s\").MacAddress}}", netname)})
461
		MACAfter.WaitWithDefaultTimeout()
462
		Expect(MACAfter).Should(ExitCleanly())
463

464
		// Check that IP address did not change between checkpointing and restoring
465
		Expect(IPAfter.OutputToString()).To(Equal(IPBefore.OutputToString()))
466

467
		// Check that MAC address did not change between checkpointing and restoring
468
		Expect(MACAfter.OutputToString()).To(Equal(MACBefore.OutputToString()))
469

470
		Expect(result).Should(ExitCleanly())
471
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
472
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
473

474
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-fa"})
475
		result.WaitWithDefaultTimeout()
476
		Expect(result).Should(ExitCleanly())
477
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
478
	})
479

480
	// This test does the same steps which are necessary for migrating
481
	// a container from one host to another
482
	It("podman checkpoint container with export (migration)", func() {
483
		localRunString := getRunString([]string{"--rm", ALPINE, "top"})
484
		session := podmanTest.Podman(localRunString)
485
		session.WaitWithDefaultTimeout()
486
		Expect(session).Should(ExitCleanly())
487
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
488
		cid := session.OutputToString()
489
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
490

491
		result := podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName})
492
		result.WaitWithDefaultTimeout()
493

494
		// As the container has been started with '--rm' it will be completely
495
		// cleaned up after checkpointing.
496
		Expect(result).Should(ExitCleanly())
497
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
498
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
499

500
		// Restore container the first time with different name.
501
		// Using '--ignore-static-ip' as for parallel test runs
502
		// each containers gets a random IP address via '--ip'.
503
		// '--ignore-static-ip' tells the restore to use the next
504
		// available IP address.
505
		// First restore the container with a new name/ID to make
506
		// sure nothing in the restored container depends on the
507
		// original container.
508
		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName, "-n", "restore_again", "--ignore-static-ip"})
509
		result.WaitWithDefaultTimeout()
510

511
		Expect(result).Should(ExitCleanly())
512
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
513
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
514

515
		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
516
		result.WaitWithDefaultTimeout()
517

518
		Expect(result).Should(ExitCleanly())
519
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(2))
520
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
521

522
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-fa"})
523
		result.WaitWithDefaultTimeout()
524
		Expect(result).Should(ExitCleanly())
525
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
526
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
527

528
		// Remove exported checkpoint
529
		os.Remove(fileName)
530
	})
531
	// This test does the same steps which are necessary for migrating
532
	// a container from one host to another
533
	It("podman checkpoint container with export and different compression algorithms", func() {
534
		localRunString := getRunString([]string{"--rm", ALPINE, "top"})
535
		session := podmanTest.Podman(localRunString)
536
		session.WaitWithDefaultTimeout()
537
		Expect(session).Should(ExitCleanly())
538
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
539
		cid := session.OutputToString()
540
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
541

542
		// Checkpoint with the default algorithm
543
		result := podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName})
544
		result.WaitWithDefaultTimeout()
545

546
		// As the container has been started with '--rm' it will be completely
547
		// cleaned up after checkpointing.
548
		Expect(result).Should(ExitCleanly())
549
		Expect(result.OutputToString()).To(ContainSubstring(cid))
550
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
551
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
552

553
		// Restore container
554
		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
555
		result.WaitWithDefaultTimeout()
556

557
		Expect(result).Should(ExitCleanly())
558
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
559
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
560

561
		// Checkpoint with the zstd algorithm
562
		result = podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName, "--compress", "zstd"})
563
		result.WaitWithDefaultTimeout()
564

565
		// As the container has been started with '--rm' it will be completely
566
		// cleaned up after checkpointing.
567
		Expect(result).Should(ExitCleanly())
568
		Expect(result.OutputToString()).To(ContainSubstring(cid))
569
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
570
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
571

572
		// Restore container
573
		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
574
		result.WaitWithDefaultTimeout()
575

576
		Expect(result).Should(ExitCleanly())
577
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
578
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
579

580
		// Checkpoint with the none algorithm
581
		result = podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName, "-c", "none"})
582
		result.WaitWithDefaultTimeout()
583

584
		// As the container has been started with '--rm' it will be completely
585
		// cleaned up after checkpointing.
586
		Expect(result).Should(ExitCleanly())
587
		Expect(result.OutputToString()).To(ContainSubstring(cid))
588
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
589
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
590

591
		// Restore container
592
		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
593
		result.WaitWithDefaultTimeout()
594

595
		Expect(result).Should(ExitCleanly())
596
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
597
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
598

599
		// Checkpoint with the gzip algorithm
600
		result = podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName, "-c", "gzip"})
601
		result.WaitWithDefaultTimeout()
602

603
		// As the container has been started with '--rm' it will be completely
604
		// cleaned up after checkpointing.
605
		Expect(result).Should(ExitCleanly())
606
		Expect(result.OutputToString()).To(ContainSubstring(cid))
607
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
608
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
609

610
		// Restore container
611
		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
612
		result.WaitWithDefaultTimeout()
613

614
		Expect(result).Should(ExitCleanly())
615
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
616
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
617

618
		// Checkpoint with the non-existing algorithm
619
		result = podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName, "-c", "non-existing"})
620
		result.WaitWithDefaultTimeout()
621

622
		Expect(result).Should(ExitWithError(125, `selected compression algorithm ("non-existing") not supported. Please select one from`))
623
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
624
		Expect(podmanTest.NumberOfContainers()).To(Equal(1))
625

626
		result = podmanTest.Podman([]string{"rm", "--time", "0", "-fa"})
627
		result.WaitWithDefaultTimeout()
628
		Expect(result).Should(ExitCleanly())
629
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
630
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
631

632
		// Remove exported checkpoint
633
		os.Remove(fileName)
634
	})
635

636
	It("podman checkpoint and restore container with root file-system changes", func() {
637
		// Start the container
638
		localRunString := getRunString([]string{"--rm", ALPINE, "top"})
639
		session := podmanTest.Podman(localRunString)
640
		session.WaitWithDefaultTimeout()
641
		Expect(session).Should(ExitCleanly())
642
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
643
		cid := session.OutputToString()
644
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
645

646
		// Change the container's root file-system
647
		result := podmanTest.Podman([]string{"exec", cid, "/bin/sh", "-c", "echo test" + cid + "test > /test.output"})
648
		result.WaitWithDefaultTimeout()
649
		Expect(result).Should(ExitCleanly())
650

651
		result = podmanTest.Podman([]string{"exec", cid, "/bin/sh", "-c", "rm /etc/motd"})
652
		result.WaitWithDefaultTimeout()
653
		Expect(result).Should(ExitCleanly())
654

655
		result = podmanTest.Podman([]string{"diff", cid})
656
		result.WaitWithDefaultTimeout()
657
		Expect(result).Should(ExitCleanly())
658
		Expect(result.OutputToString()).To(ContainSubstring("C /etc"))
659
		Expect(result.OutputToString()).To(ContainSubstring("A /test.output"))
660
		Expect(result.OutputToString()).To(ContainSubstring("D /etc/motd"))
661
		Expect(result.OutputToStringArray()).To(HaveLen(3))
662

663
		// Checkpoint the container
664
		result = podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName})
665
		result.WaitWithDefaultTimeout()
666

667
		Expect(result).Should(ExitCleanly())
668
		Expect(result.OutputToString()).To(ContainSubstring(cid))
669
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
670
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
671

672
		// Restore the container
673
		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
674
		result.WaitWithDefaultTimeout()
675

676
		Expect(result).Should(ExitCleanly())
677
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
678
		Expect(podmanTest.NumberOfContainers()).To(Equal(1))
679
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
680

681
		// Verify the changes to the container's root file-system
682
		result = podmanTest.Podman([]string{"exec", cid, "cat", "/test.output"})
683
		result.WaitWithDefaultTimeout()
684
		Expect(result).Should(ExitCleanly())
685
		Expect(result.OutputToString()).To(ContainSubstring("test" + cid + "test"))
686

687
		result = podmanTest.Podman([]string{"diff", cid})
688
		result.WaitWithDefaultTimeout()
689
		Expect(result).Should(ExitCleanly())
690
		Expect(result.OutputToString()).To(ContainSubstring("C /etc"))
691
		Expect(result.OutputToString()).To(ContainSubstring("A /test.output"))
692
		Expect(result.OutputToString()).To(ContainSubstring("D /etc/motd"))
693
		Expect(result.OutputToStringArray()).To(HaveLen(3))
694

695
		// Remove exported checkpoint
696
		os.Remove(fileName)
697
	})
698
	It("podman checkpoint and restore container with root file-system changes using --ignore-rootfs during restore", func() {
699
		// Start the container
700
		// test that restore works without network namespace (https://github.com/containers/podman/issues/14389)
701
		session := podmanTest.Podman([]string{"run", "--network=none", "-d", "--rm", ALPINE, "top"})
702
		session.WaitWithDefaultTimeout()
703
		Expect(session).Should(ExitCleanly())
704
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
705
		cid := session.OutputToString()
706
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
707

708
		// Change the container's root file-system
709
		result := podmanTest.Podman([]string{"exec", cid, "/bin/sh", "-c", "echo test" + cid + "test > /test.output"})
710
		result.WaitWithDefaultTimeout()
711
		Expect(result).Should(ExitCleanly())
712

713
		// Checkpoint the container
714
		result = podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName})
715
		result.WaitWithDefaultTimeout()
716

717
		Expect(result).Should(ExitCleanly())
718
		Expect(result.OutputToString()).To(ContainSubstring(cid))
719
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
720
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
721

722
		// Restore the container
723
		result = podmanTest.Podman([]string{"container", "restore", "--ignore-rootfs", "-i", fileName})
724
		result.WaitWithDefaultTimeout()
725

726
		Expect(result).Should(ExitCleanly())
727
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
728
		Expect(podmanTest.NumberOfContainers()).To(Equal(1))
729
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
730

731
		// Verify the changes to the container's root file-system
732
		result = podmanTest.Podman([]string{"exec", cid, "cat", "/test.output"})
733
		result.WaitWithDefaultTimeout()
734
		Expect(result).Should(ExitWithError(1, "cat: can't open '/test.output': No such file or directory"))
735

736
		// Remove exported checkpoint
737
		os.Remove(fileName)
738
	})
739
	It("podman checkpoint and restore container with root file-system changes using --ignore-rootfs during checkpoint", func() {
740
		// Start the container
741
		localRunString := getRunString([]string{"--rm", ALPINE, "top"})
742
		session := podmanTest.Podman(localRunString)
743
		session.WaitWithDefaultTimeout()
744
		Expect(session).Should(ExitCleanly())
745
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
746
		cid := session.OutputToString()
747
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
748

749
		// Change the container's root file-system
750
		result := podmanTest.Podman([]string{"exec", cid, "/bin/sh", "-c", "echo test" + cid + "test > /test.output"})
751
		result.WaitWithDefaultTimeout()
752
		Expect(result).Should(ExitCleanly())
753

754
		// Checkpoint the container
755
		result = podmanTest.Podman([]string{"container", "checkpoint", "--ignore-rootfs", cid, "-e", fileName})
756
		result.WaitWithDefaultTimeout()
757

758
		Expect(result).Should(ExitCleanly())
759
		Expect(result.OutputToString()).To(ContainSubstring(cid))
760
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
761
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
762

763
		// Restore the container
764
		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
765
		result.WaitWithDefaultTimeout()
766

767
		Expect(result).Should(ExitCleanly())
768
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
769
		Expect(podmanTest.NumberOfContainers()).To(Equal(1))
770
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
771

772
		// Verify the changes to the container's root file-system
773
		result = podmanTest.Podman([]string{"exec", cid, "cat", "/test.output"})
774
		result.WaitWithDefaultTimeout()
775
		Expect(result).Should(ExitWithError(1, "cat: can't open '/test.output': No such file or directory"))
776

777
		// Remove exported checkpoint
778
		os.Remove(fileName)
779
	})
780

781
	It("podman checkpoint and run exec in restored container", func() {
782
		// Start the container
783
		localRunString := getRunString([]string{"--rm", ALPINE, "top"})
784
		session := podmanTest.Podman(localRunString)
785
		session.WaitWithDefaultTimeout()
786
		Expect(session).Should(ExitCleanly())
787
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
788
		cid := session.OutputToString()
789
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
790

791
		// Checkpoint the container
792
		result := podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName})
793
		result.WaitWithDefaultTimeout()
794

795
		Expect(result).Should(ExitCleanly())
796
		Expect(result.OutputToString()).To(ContainSubstring(cid))
797
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
798
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
799

800
		// Restore the container
801
		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
802
		result.WaitWithDefaultTimeout()
803

804
		Expect(result).Should(ExitCleanly())
805
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
806
		Expect(podmanTest.NumberOfContainers()).To(Equal(1))
807
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
808

809
		// Exec in the container
810
		result = podmanTest.Podman([]string{"exec", cid, "/bin/sh", "-c", "echo " + cid + " > /test.output"})
811
		result.WaitWithDefaultTimeout()
812
		Expect(result).Should(ExitCleanly())
813

814
		result = podmanTest.Podman([]string{"exec", cid, "cat", "/test.output"})
815
		result.WaitWithDefaultTimeout()
816
		Expect(result).Should(ExitCleanly())
817
		Expect(result.OutputToString()).To(ContainSubstring(cid))
818

819
		// Remove exported checkpoint
820
		os.Remove(fileName)
821
	})
822

823
	It("podman checkpoint a container started with --rm", func() {
824
		// Start the container
825
		localRunString := getRunString([]string{"--rm", ALPINE, "top"})
826
		session := podmanTest.Podman(localRunString)
827
		session.WaitWithDefaultTimeout()
828
		cid := session.OutputToString()
829
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
830

831
		// Checkpoint the container - this should fail as it was started with --rm
832
		result := podmanTest.Podman([]string{"container", "checkpoint", cid})
833
		result.WaitWithDefaultTimeout()
834
		Expect(result).To(ExitWithError(125, "cannot checkpoint containers that have been started with '--rm'"))
835

836
		// Checkpointing with --export should still work
837
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
838

839
		result = podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName})
840
		result.WaitWithDefaultTimeout()
841

842
		// As the container has been started with '--rm' it will be completely
843
		// cleaned up after checkpointing.
844
		Expect(result).Should(ExitCleanly())
845
		Expect(result.OutputToString()).To(ContainSubstring(cid))
846
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
847
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
848

849
		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
850
		result.WaitWithDefaultTimeout()
851

852
		Expect(result).Should(ExitCleanly())
853
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
854
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
855

856
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-fa"})
857
		result.WaitWithDefaultTimeout()
858
		Expect(result).Should(ExitCleanly())
859
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
860
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
861

862
		// Remove exported checkpoint
863
		os.Remove(fileName)
864
	})
865

866
	It("podman checkpoint a container with volumes", func() {
867
		session := podmanTest.Podman([]string{
868
			"build", "-f", "build/basicalpine/Containerfile.volume", "-t", "test-cr-volume",
869
		})
870
		session.WaitWithDefaultTimeout()
871
		Expect(session).Should(ExitCleanly())
872

873
		// Start the container
874
		localRunString := getRunString([]string{
875
			"--rm",
876
			"-v", "/volume1",
877
			"-v", "my-test-vol:/volume2",
878
			"test-cr-volume",
879
			"top",
880
		})
881
		session = podmanTest.Podman(localRunString)
882
		session.WaitWithDefaultTimeout()
883
		Expect(session).Should(ExitCleanly())
884
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
885

886
		cid := session.OutputToString()
887

888
		// Add file in volume0
889
		result := podmanTest.Podman([]string{
890
			"exec", cid, "/bin/sh", "-c", "echo " + cid + " > /volume0/test.output",
891
		})
892
		result.WaitWithDefaultTimeout()
893
		Expect(result).Should(ExitCleanly())
894

895
		// Add file in volume1
896
		result = podmanTest.Podman([]string{
897
			"exec", cid, "/bin/sh", "-c", "echo " + cid + " > /volume1/test.output",
898
		})
899
		result.WaitWithDefaultTimeout()
900
		Expect(result).Should(ExitCleanly())
901

902
		// Add file in volume2
903
		result = podmanTest.Podman([]string{
904
			"exec", cid, "/bin/sh", "-c", "echo " + cid + " > /volume2/test.output",
905
		})
906
		result.WaitWithDefaultTimeout()
907
		Expect(result).Should(ExitCleanly())
908

909
		checkpointFileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
910

911
		// Checkpoint the container
912
		result = podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", checkpointFileName})
913
		result.WaitWithDefaultTimeout()
914
		Expect(result).Should(ExitCleanly())
915
		Expect(result.OutputToString()).To(ContainSubstring(cid))
916
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
917
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
918

919
		// Restore container should fail because named volume still exists
920
		result = podmanTest.Podman([]string{"container", "restore", "-i", checkpointFileName})
921
		result.WaitWithDefaultTimeout()
922
		Expect(result).To(ExitWithError(125, "volume with name my-test-vol already exists. Use --ignore-volumes to not restore content of volumes"))
923

924
		// Remove named volume
925
		session = podmanTest.Podman([]string{"volume", "rm", "my-test-vol"})
926
		session.WaitWithDefaultTimeout()
927
		Expect(session).Should(ExitCleanly())
928

929
		// Restoring container
930
		result = podmanTest.Podman([]string{"container", "restore", "-i", checkpointFileName})
931
		result.WaitWithDefaultTimeout()
932
		Expect(result).Should(ExitCleanly())
933
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
934
		Expect(podmanTest.NumberOfContainers()).To(Equal(1))
935
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
936

937
		// Validate volume0 content
938
		result = podmanTest.Podman([]string{"exec", cid, "cat", "/volume0/test.output"})
939
		result.WaitWithDefaultTimeout()
940
		Expect(result).Should(ExitCleanly())
941
		Expect(result.OutputToString()).To(ContainSubstring(cid))
942

943
		// Validate volume1 content
944
		result = podmanTest.Podman([]string{"exec", cid, "cat", "/volume1/test.output"})
945
		result.WaitWithDefaultTimeout()
946
		Expect(result).Should(ExitCleanly())
947
		Expect(result.OutputToString()).To(ContainSubstring(cid))
948

949
		// Validate volume2 content
950
		result = podmanTest.Podman([]string{"exec", cid, "cat", "/volume2/test.output"})
951
		result.WaitWithDefaultTimeout()
952
		Expect(result).Should(ExitCleanly())
953
		Expect(result.OutputToString()).To(ContainSubstring(cid))
954

955
		// Remove exported checkpoint
956
		os.Remove(checkpointFileName)
957
	})
958

959
	It("podman checkpoint container with --pre-checkpoint", func() {
960
		if !criu.MemTrack() {
961
			Skip("system (architecture/kernel/CRIU) does not support memory tracking")
962
		}
963
		localRunString := getRunString([]string{ALPINE, "top"})
964
		session := podmanTest.Podman(localRunString)
965
		session.WaitWithDefaultTimeout()
966
		Expect(session).Should(ExitCleanly())
967
		cid := session.OutputToString()
968

969
		result := podmanTest.Podman([]string{"container", "checkpoint", "-P", cid})
970
		result.WaitWithDefaultTimeout()
971

972
		Expect(result).Should(ExitCleanly())
973
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
974
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
975

976
		result = podmanTest.Podman([]string{"container", "checkpoint", "--with-previous", cid})
977
		result.WaitWithDefaultTimeout()
978

979
		Expect(result).Should(ExitCleanly())
980
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
981
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
982

983
		result = podmanTest.Podman([]string{"container", "restore", cid})
984
		result.WaitWithDefaultTimeout()
985

986
		Expect(result).Should(ExitCleanly())
987
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
988
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
989
	})
990

991
	It("podman checkpoint container with --pre-checkpoint and export (migration)", func() {
992
		SkipIfRemote("--import-previous is not yet supported on the remote client")
993
		if !criu.MemTrack() {
994
			Skip("system (architecture/kernel/CRIU) does not support memory tracking")
995
		}
996
		localRunString := getRunString([]string{ALPINE, "top"})
997
		session := podmanTest.Podman(localRunString)
998
		session.WaitWithDefaultTimeout()
999
		Expect(session).Should(ExitCleanly())
1000
		cid := session.OutputToString()
1001
		preCheckpointFileName := filepath.Join(podmanTest.TempDir, "/pre-checkpoint-"+cid+".tar.gz")
1002
		checkpointFileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
1003

1004
		result := podmanTest.Podman([]string{"container", "checkpoint", "-P", "-e", preCheckpointFileName, cid})
1005
		result.WaitWithDefaultTimeout()
1006

1007
		Expect(result).Should(ExitCleanly())
1008
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1009
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
1010

1011
		result = podmanTest.Podman([]string{"container", "checkpoint", "--with-previous", "-e", checkpointFileName, cid})
1012
		result.WaitWithDefaultTimeout()
1013

1014
		Expect(result).Should(ExitCleanly())
1015
		Expect(result.OutputToString()).To(ContainSubstring(cid))
1016
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1017
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
1018

1019
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-f", cid})
1020
		result.WaitWithDefaultTimeout()
1021
		Expect(result).Should(ExitCleanly())
1022
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1023

1024
		result = podmanTest.Podman([]string{"container", "restore", "-i", checkpointFileName, "--import-previous", preCheckpointFileName})
1025
		result.WaitWithDefaultTimeout()
1026

1027
		Expect(result).Should(ExitCleanly())
1028
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1029
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
1030

1031
		os.Remove(checkpointFileName)
1032
		os.Remove(preCheckpointFileName)
1033
	})
1034

1035
	It("podman checkpoint and restore container with different port mappings", func() {
1036
		randomPort, err := utils.GetRandomPort()
1037
		Expect(err).ShouldNot(HaveOccurred())
1038
		localRunString := getRunString([]string{"-p", fmt.Sprintf("%d:6379", randomPort), "--rm", REDIS_IMAGE})
1039
		session := podmanTest.Podman(localRunString)
1040
		session.WaitWithDefaultTimeout()
1041
		Expect(session).Should(ExitCleanly())
1042
		cid := session.OutputToString()
1043
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
1044

1045
		if !WaitContainerReady(podmanTest, cid, "Ready to accept connections", 20, 1) {
1046
			Fail("Container failed to get ready")
1047
		}
1048

1049
		GinkgoWriter.Printf("Trying to connect to redis server at localhost:%d\n", randomPort)
1050
		// Open a network connection to the redis server via initial port mapping
1051
		conn, err := net.DialTimeout("tcp4", fmt.Sprintf("localhost:%d", randomPort), time.Duration(3)*time.Second)
1052
		Expect(err).ShouldNot(HaveOccurred())
1053
		conn.Close()
1054

1055
		// Checkpoint the container
1056
		result := podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName})
1057
		result.WaitWithDefaultTimeout()
1058

1059
		// As the container has been started with '--rm' it will be completely
1060
		// cleaned up after checkpointing.
1061
		Expect(result).Should(ExitCleanly())
1062
		Expect(result.OutputToString()).To(ContainSubstring(cid))
1063
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1064
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
1065

1066
		// Restore container with different port mapping
1067
		newRandomPort, err := utils.GetRandomPort()
1068
		Expect(err).ShouldNot(HaveOccurred())
1069
		result = podmanTest.Podman([]string{"container", "restore", "-p", fmt.Sprintf("%d:6379", newRandomPort), "-i", fileName})
1070
		result.WaitWithDefaultTimeout()
1071

1072
		Expect(result).Should(ExitCleanly())
1073
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1074
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
1075

1076
		// Open a network connection to the redis server via initial port mapping
1077
		// This should fail
1078
		_, err = net.DialTimeout("tcp4", fmt.Sprintf("localhost:%d", randomPort), time.Duration(3)*time.Second)
1079
		Expect(err).To(HaveOccurred())
1080
		Expect(err.Error()).To(ContainSubstring("connection refused"))
1081
		// Open a network connection to the redis server via new port mapping
1082
		GinkgoWriter.Printf("Trying to reconnect to redis server at localhost:%d\n", newRandomPort)
1083
		conn, err = net.DialTimeout("tcp4", fmt.Sprintf("localhost:%d", newRandomPort), time.Duration(3)*time.Second)
1084
		Expect(err).ShouldNot(HaveOccurred())
1085
		conn.Close()
1086

1087
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-fa"})
1088
		result.WaitWithDefaultTimeout()
1089
		Expect(result).Should(ExitCleanly())
1090
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1091
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
1092

1093
		// Remove exported checkpoint
1094
		os.Remove(fileName)
1095
	})
1096

1097
	namespaceCombination := []string{
1098
		"ipc,net,uts,pid",
1099
		"ipc,net,uts",
1100
		"ipc,net",
1101
		"net,uts,pid",
1102
		"net,uts",
1103
		"uts,pid",
1104
	}
1105
	for _, share := range namespaceCombination {
1106
		testName := fmt.Sprintf(
1107
			"podman checkpoint and restore container out of and into pod (%s)",
1108
			share,
1109
		)
1110

1111
		share := share // copy into local scope, for use inside function
1112

1113
		It(testName, func() {
1114
			if err := criu.CheckForCriu(criu.PodCriuVersion); err != nil {
1115
				Skip(fmt.Sprintf("check CRIU pod version error: %v", err))
1116
			}
1117
			if !crutils.CRRuntimeSupportsPodCheckpointRestore(podmanTest.OCIRuntime) {
1118
				Skip("runtime does not support pod restore: " + podmanTest.OCIRuntime)
1119
			}
1120
			// Create a pod
1121
			session := podmanTest.Podman([]string{
1122
				"pod",
1123
				"create",
1124
				"--share",
1125
				share,
1126
			})
1127
			session.WaitWithDefaultTimeout()
1128
			Expect(session).To(ExitCleanly())
1129
			podID := session.OutputToString()
1130

1131
			session = podmanTest.Podman([]string{
1132
				"run",
1133
				"-d",
1134
				"--rm",
1135
				"--pod",
1136
				podID,
1137
				ALPINE,
1138
				"top",
1139
			})
1140
			session.WaitWithDefaultTimeout()
1141
			Expect(session).To(ExitCleanly())
1142
			cid := session.OutputToString()
1143

1144
			fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
1145

1146
			// Checkpoint the container
1147
			result := podmanTest.Podman([]string{
1148
				"container",
1149
				"checkpoint",
1150
				"-e",
1151
				fileName,
1152
				cid,
1153
			})
1154
			result.WaitWithDefaultTimeout()
1155

1156
			// As the container has been started with '--rm' it will be completely
1157
			// cleaned up after checkpointing.
1158
			// #11784 (closed wontfix): runc warns "lstat /sys/.../machine.slice/...: ENOENT"
1159
			// so we can't use ExitCleanly()
1160
			if podmanTest.OCIRuntime == "runc" {
1161
				Expect(result).To(Exit(0))
1162
			} else {
1163
				Expect(result).To(ExitCleanly())
1164
			}
1165
			Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1166
			Expect(podmanTest.NumberOfContainers()).To(Equal(1))
1167

1168
			// Remove the pod and create a new pod
1169
			result = podmanTest.Podman([]string{
1170
				"pod",
1171
				"rm",
1172
				podID,
1173
			})
1174
			result.WaitWithDefaultTimeout()
1175
			Expect(result).To(ExitCleanly())
1176

1177
			// First create a pod with different shared namespaces.
1178
			// Restore should fail
1179

1180
			wrongShare := share[:strings.LastIndex(share, ",")]
1181

1182
			session = podmanTest.Podman([]string{
1183
				"pod",
1184
				"create",
1185
				"--share",
1186
				wrongShare,
1187
			})
1188
			session.WaitWithDefaultTimeout()
1189
			Expect(session).To(ExitCleanly())
1190
			podID = session.OutputToString()
1191

1192
			// Restore container with different port mapping
1193
			result = podmanTest.Podman([]string{
1194
				"container",
1195
				"restore",
1196
				"--pod",
1197
				podID,
1198
				"-i",
1199
				fileName,
1200
			})
1201
			result.WaitWithDefaultTimeout()
1202
			Expect(result).To(ExitWithError(125, "does not share the "))
1203

1204
			// Remove the pod and create a new pod
1205
			result = podmanTest.Podman([]string{
1206
				"pod",
1207
				"rm",
1208
				podID,
1209
			})
1210
			result.WaitWithDefaultTimeout()
1211
			Expect(result).To(ExitCleanly())
1212

1213
			session = podmanTest.Podman([]string{
1214
				"pod",
1215
				"create",
1216
				"--share",
1217
				share,
1218
			})
1219
			session.WaitWithDefaultTimeout()
1220
			Expect(session).To(ExitCleanly())
1221
			podID = session.OutputToString()
1222

1223
			// Restore container with different port mapping
1224
			result = podmanTest.Podman([]string{
1225
				"container",
1226
				"restore",
1227
				"--pod",
1228
				podID,
1229
				"-i",
1230
				fileName,
1231
			})
1232
			result.WaitWithDefaultTimeout()
1233

1234
			Expect(result).To(ExitCleanly())
1235
			Expect(podmanTest.NumberOfContainersRunning()).To(Equal(2))
1236
			Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
1237

1238
			result = podmanTest.Podman([]string{
1239
				"rm",
1240
				"-f",
1241
				result.OutputToString(),
1242
			})
1243
			result.WaitWithDefaultTimeout()
1244
			// #11784 (closed wontfix): runc warns "lstat /sys/.../machine.slice/...: ENOENT"
1245
			// so we can't use ExitCleanly()
1246
			if podmanTest.OCIRuntime == "runc" {
1247
				Expect(result).To(Exit(0))
1248
			} else {
1249
				Expect(result).To(ExitCleanly())
1250
			}
1251
			Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1252
			Expect(podmanTest.NumberOfContainers()).To(Equal(1))
1253

1254
			result = podmanTest.Podman([]string{
1255
				"pod",
1256
				"rm",
1257
				"-fa",
1258
			})
1259
			result.WaitWithDefaultTimeout()
1260
			Expect(result).To(ExitCleanly())
1261

1262
			// Remove exported checkpoint
1263
			os.Remove(fileName)
1264
		})
1265
	}
1266

1267
	It("podman checkpoint container with export (migration) and --ipc host", func() {
1268
		localRunString := getRunString([]string{"--rm", "--ipc", "host", ALPINE, "top"})
1269
		session := podmanTest.Podman(localRunString)
1270
		session.WaitWithDefaultTimeout()
1271
		Expect(session).Should(ExitCleanly())
1272
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1273
		cid := session.OutputToString()
1274
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
1275

1276
		result := podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", fileName})
1277
		result.WaitWithDefaultTimeout()
1278

1279
		// As the container has been started with '--rm' it will be completely
1280
		// cleaned up after checkpointing.
1281
		// Cannot use ExitCleanly() because "skipping [ssh-agent-path] since it is a socket"
1282
		Expect(result).Should(Exit(0))
1283
		Expect(result.OutputToString()).To(ContainSubstring(cid))
1284
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1285
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
1286

1287
		result = podmanTest.Podman([]string{"container", "restore", "-i", fileName})
1288
		result.WaitWithDefaultTimeout()
1289

1290
		Expect(result).Should(ExitCleanly())
1291
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1292
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
1293

1294
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-fa"})
1295
		result.WaitWithDefaultTimeout()
1296
		Expect(result).Should(ExitCleanly())
1297
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1298
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
1299

1300
		// Remove exported checkpoint
1301
		os.Remove(fileName)
1302
	})
1303

1304
	It("podman checkpoint container with export and statistics", func() {
1305
		localRunString := getRunString([]string{
1306
			"--rm",
1307
			ALPINE,
1308
			"top",
1309
		})
1310
		session := podmanTest.Podman(localRunString)
1311
		session.WaitWithDefaultTimeout()
1312
		Expect(session).Should(ExitCleanly())
1313
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1314
		cid := session.OutputToString()
1315
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
1316
		defer os.Remove(fileName)
1317

1318
		result := podmanTest.Podman([]string{
1319
			"container",
1320
			"checkpoint",
1321
			cid, "-e",
1322
			fileName,
1323
		})
1324
		result.WaitWithDefaultTimeout()
1325

1326
		// As the container has been started with '--rm' it will be completely
1327
		// cleaned up after checkpointing.
1328
		Expect(result).Should(ExitCleanly())
1329
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1330
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
1331

1332
		// Extract checkpoint archive
1333
		destinationDirectory := filepath.Join(podmanTest.TempDir, "dest")
1334
		err = os.MkdirAll(destinationDirectory, os.ModePerm)
1335
		Expect(err).ToNot(HaveOccurred())
1336

1337
		tarsession := SystemExec(
1338
			"tar",
1339
			[]string{
1340
				"xf",
1341
				fileName,
1342
				"-C",
1343
				destinationDirectory,
1344
			},
1345
		)
1346
		Expect(tarsession).Should(ExitCleanly())
1347

1348
		_, err = os.Stat(filepath.Join(destinationDirectory, stats.StatsDump))
1349
		Expect(err).ShouldNot(HaveOccurred())
1350
	})
1351

1352
	It("podman checkpoint and restore containers with --print-stats", func() {
1353
		session1 := podmanTest.Podman(getRunString([]string{REDIS_IMAGE}))
1354
		session1.WaitWithDefaultTimeout()
1355
		Expect(session1).Should(ExitCleanly())
1356

1357
		session2 := podmanTest.Podman(getRunString([]string{REDIS_IMAGE, "top"}))
1358
		session2.WaitWithDefaultTimeout()
1359
		Expect(session2).Should(ExitCleanly())
1360

1361
		result := podmanTest.Podman([]string{
1362
			"container",
1363
			"checkpoint",
1364
			"-a",
1365
			"--print-stats",
1366
		})
1367
		result.WaitWithDefaultTimeout()
1368

1369
		Expect(result).Should(ExitCleanly())
1370
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1371

1372
		type checkpointStatistics struct {
1373
			PodmanDuration      int64                        `json:"podman_checkpoint_duration"`
1374
			ContainerStatistics []*entities.CheckpointReport `json:"container_statistics"`
1375
		}
1376

1377
		cS := new(checkpointStatistics)
1378
		err := json.Unmarshal([]byte(result.OutputToString()), cS)
1379
		Expect(err).ShouldNot(HaveOccurred())
1380

1381
		Expect(cS.ContainerStatistics).To(HaveLen(2))
1382
		Expect(cS.PodmanDuration).To(BeNumerically(">", cS.ContainerStatistics[0].RuntimeDuration))
1383
		Expect(cS.PodmanDuration).To(BeNumerically(">", cS.ContainerStatistics[1].RuntimeDuration))
1384
		Expect(cS.ContainerStatistics[0].RuntimeDuration).To(
1385
			BeNumerically(">", cS.ContainerStatistics[0].CRIUStatistics.FrozenTime),
1386
		)
1387
		Expect(cS.ContainerStatistics[1].RuntimeDuration).To(
1388
			BeNumerically(">", cS.ContainerStatistics[1].CRIUStatistics.FrozenTime),
1389
		)
1390

1391
		ps := podmanTest.Podman([]string{
1392
			"ps",
1393
			"-q",
1394
			"--no-trunc",
1395
		})
1396
		ps.WaitWithDefaultTimeout()
1397
		Expect(ps).Should(ExitCleanly())
1398
		Expect(ps.OutputToString()).To(Not(ContainSubstring(session1.OutputToString())))
1399
		Expect(ps.OutputToString()).To(Not(ContainSubstring(session2.OutputToString())))
1400

1401
		result = podmanTest.Podman([]string{
1402
			"container",
1403
			"restore",
1404
			"-a",
1405
			"--print-stats",
1406
		})
1407
		result.WaitWithDefaultTimeout()
1408

1409
		Expect(result).Should(ExitCleanly())
1410
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(2))
1411
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
1412
		Expect(podmanTest.GetContainerStatus()).To(Not(ContainSubstring("Exited")))
1413

1414
		type restoreStatistics struct {
1415
			PodmanDuration      int64                     `json:"podman_restore_duration"`
1416
			ContainerStatistics []*entities.RestoreReport `json:"container_statistics"`
1417
		}
1418

1419
		rS := new(restoreStatistics)
1420
		err = json.Unmarshal([]byte(result.OutputToString()), rS)
1421
		Expect(err).ShouldNot(HaveOccurred())
1422

1423
		Expect(cS.ContainerStatistics).To(HaveLen(2))
1424
		Expect(cS.PodmanDuration).To(BeNumerically(">", cS.ContainerStatistics[0].RuntimeDuration))
1425
		Expect(cS.PodmanDuration).To(BeNumerically(">", cS.ContainerStatistics[1].RuntimeDuration))
1426
		Expect(cS.ContainerStatistics[0].RuntimeDuration).To(
1427
			BeNumerically(">", cS.ContainerStatistics[0].CRIUStatistics.RestoreTime),
1428
		)
1429
		Expect(cS.ContainerStatistics[1].RuntimeDuration).To(
1430
			BeNumerically(">", cS.ContainerStatistics[1].CRIUStatistics.RestoreTime),
1431
		)
1432

1433
		result = podmanTest.Podman([]string{
1434
			"rm",
1435
			"-t",
1436
			"0",
1437
			"-fa",
1438
		})
1439
		result.WaitWithDefaultTimeout()
1440
		Expect(result).Should(ExitCleanly())
1441
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1442
	})
1443

1444
	It("podman checkpoint and restore container with --file-locks", func() {
1445
		localRunString := getRunString([]string{"--name", "test_name", ALPINE, "flock", "test.lock", "sh", "-c", "echo READY;sleep 100"})
1446
		session := podmanTest.Podman(localRunString)
1447
		session.WaitWithDefaultTimeout()
1448
		Expect(session).Should(ExitCleanly())
1449
		Expect(WaitContainerReady(podmanTest, "test_name", "READY", 5, 1)).To(BeTrue(), "Timed out waiting for READY")
1450

1451
		// Checkpoint is expected to fail without --file-locks
1452
		result := podmanTest.Podman([]string{"container", "checkpoint", "test_name"})
1453
		result.WaitWithDefaultTimeout()
1454
		Expect(result).Should(ExitWithError(125, "failed: exit status 1"))
1455
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1456

1457
		// Checkpoint is expected to succeed with --file-locks
1458
		result = podmanTest.Podman([]string{"container", "checkpoint", "--file-locks", "test_name"})
1459
		result.WaitWithDefaultTimeout()
1460
		Expect(result).Should(ExitCleanly())
1461
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1462
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
1463

1464
		result = podmanTest.Podman([]string{"container", "restore", "--file-locks", "test_name"})
1465
		result.WaitWithDefaultTimeout()
1466

1467
		Expect(result).Should(ExitCleanly())
1468
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1469
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
1470

1471
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-f", "test_name"})
1472
		result.WaitWithDefaultTimeout()
1473
		Expect(result).Should(ExitCleanly())
1474
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1475
	})
1476

1477
	It("podman checkpoint container with export and verify runtime", func() {
1478
		SkipIfRemote("podman-remote does not support --runtime flag")
1479
		localRunString := getRunString([]string{
1480
			"--rm",
1481
			ALPINE,
1482
			"top",
1483
		})
1484
		session := podmanTest.Podman(localRunString)
1485
		session.WaitWithDefaultTimeout()
1486
		Expect(session).Should(ExitCleanly())
1487
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1488
		cid := session.OutputToString()
1489

1490
		session = podmanTest.Podman([]string{
1491
			"inspect",
1492
			"--format",
1493
			"{{.OCIRuntime}}",
1494
			cid,
1495
		})
1496
		session.WaitWithDefaultTimeout()
1497
		Expect(session).Should(ExitCleanly())
1498
		runtime := session.OutputToString()
1499

1500
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
1501

1502
		result := podmanTest.Podman([]string{
1503
			"container",
1504
			"checkpoint",
1505
			cid, "-e",
1506
			fileName,
1507
		})
1508
		result.WaitWithDefaultTimeout()
1509

1510
		// As the container has been started with '--rm' it will be completely
1511
		// cleaned up after checkpointing.
1512
		Expect(result).Should(ExitCleanly())
1513
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1514
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
1515

1516
		result = podmanTest.Podman([]string{
1517
			"container",
1518
			"restore",
1519
			"-i",
1520
			fileName,
1521
		})
1522
		result.WaitWithDefaultTimeout()
1523
		Expect(result).Should(ExitCleanly())
1524
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1525
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
1526

1527
		// The restored container should have the same runtime as the original container
1528
		result = podmanTest.Podman([]string{
1529
			"inspect",
1530
			"--format",
1531
			"{{.OCIRuntime}}",
1532
			cid,
1533
		})
1534
		result.WaitWithDefaultTimeout()
1535
		Expect(result).Should(ExitCleanly())
1536
		Expect(session.OutputToString()).To(Equal(runtime))
1537

1538
		// Remove exported checkpoint
1539
		os.Remove(fileName)
1540
	})
1541

1542
	It("podman checkpoint container with export and verify non-default runtime", func() {
1543
		SkipIfRemote("podman-remote does not support --runtime flag")
1544
		// This test triggers the edge case where:
1545
		// 1. Default runtime is crun
1546
		// 2. Container is created with runc
1547
		// 3. Checkpoint without setting --runtime into archive
1548
		// 4. Restore without setting --runtime from archive
1549
		// It should be expected that podman identifies runtime
1550
		// from the checkpoint archive.
1551

1552
		// Prevent --runtime arg from being set to force using default
1553
		// runtime unless explicitly set through passed args.
1554
		preservedMakeOptions := podmanTest.PodmanMakeOptions
1555
		podmanTest.PodmanMakeOptions = func(args []string, noEvents, noCache bool) []string {
1556
			defaultArgs := preservedMakeOptions(args, noEvents, noCache)
1557
			for i := range args {
1558
				// Runtime is set explicitly, so we should keep --runtime arg.
1559
				if args[i] == "--runtime" {
1560
					return defaultArgs
1561
				}
1562
			}
1563
			updatedArgs := make([]string, 0)
1564
			for i := 0; i < len(defaultArgs); i++ {
1565
				// Remove --runtime arg, letting podman fall back to its default
1566
				if defaultArgs[i] == "--runtime" {
1567
					i++
1568
				} else {
1569
					updatedArgs = append(updatedArgs, defaultArgs[i])
1570
				}
1571
			}
1572
			return updatedArgs
1573
		}
1574

1575
		for _, runtime := range []string{"runc", "crun"} {
1576
			if err := exec.Command(runtime, "--help").Run(); err != nil {
1577
				Skip(fmt.Sprintf("%s not found in PATH; this test requires both runc and crun", runtime))
1578
			}
1579
		}
1580

1581
		// Detect default runtime
1582
		session := podmanTest.Podman([]string{"info", "--format", "{{.Host.OCIRuntime.Name}}"})
1583
		session.WaitWithDefaultTimeout()
1584
		Expect(session).Should(ExitCleanly())
1585
		if defaultRuntime := session.OutputToString(); defaultRuntime != "crun" {
1586
			Skip(fmt.Sprintf("Default runtime is %q; this test requires crun to be default", defaultRuntime))
1587
		}
1588

1589
		// Force non-default runtime "runc"
1590
		localRunString := getRunString([]string{"--runtime", "runc", "--rm", ALPINE, "top"})
1591
		session = podmanTest.Podman(localRunString)
1592
		session.WaitWithDefaultTimeout()
1593
		Expect(session).Should(ExitCleanly())
1594
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1595
		cid := session.OutputToString()
1596

1597
		session = podmanTest.Podman([]string{"inspect", "--format", "{{.OCIRuntime}}", cid})
1598
		session.WaitWithDefaultTimeout()
1599
		Expect(session).Should(ExitCleanly())
1600
		Expect(session.OutputToString()).To(Equal("runc"))
1601

1602
		checkpointExportPath := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
1603

1604
		session = podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", checkpointExportPath})
1605
		session.WaitWithDefaultTimeout()
1606
		// As the container has been started with '--rm' it will be completely
1607
		// cleaned up after checkpointing.
1608
		Expect(session).Should(ExitCleanly())
1609
		Expect(session.OutputToString()).To(ContainSubstring(cid))
1610
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1611
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
1612

1613
		session = podmanTest.Podman([]string{"container", "restore", "-i", checkpointExportPath})
1614
		session.WaitWithDefaultTimeout()
1615
		Expect(session).Should(ExitCleanly())
1616
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1617
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
1618

1619
		// The restored container should have the same runtime as the original container
1620
		session = podmanTest.Podman([]string{"inspect", "--format", "{{.OCIRuntime}}", cid})
1621
		session.WaitWithDefaultTimeout()
1622
		Expect(session).Should(ExitCleanly())
1623
		Expect(session.OutputToString()).To(Equal("runc"))
1624

1625
		// Remove exported checkpoint
1626
		os.Remove(checkpointExportPath)
1627
	})
1628

1629
	It("podman checkpoint container with export and try to change the runtime", func() {
1630
		SkipIfRemote("podman-remote does not support --runtime flag")
1631
		// This test will only run if runc and crun both exist
1632
		if !strings.Contains(podmanTest.OCIRuntime, "crun") {
1633
			Skip("Test requires crun and runc")
1634
		}
1635
		cmd := exec.Command("runc")
1636
		if err := cmd.Start(); err != nil {
1637
			Skip("Test requires crun and runc")
1638
		}
1639
		if err := cmd.Wait(); err != nil {
1640
			Skip("Test requires crun and runc")
1641
		}
1642
		localRunString := getRunString([]string{
1643
			"--rm",
1644
			ALPINE,
1645
			"top",
1646
		})
1647
		// Let's start a container with runc and try to restore it with crun (expected to fail)
1648
		localRunString = append(
1649
			[]string{
1650
				"--runtime",
1651
				"runc",
1652
			},
1653
			localRunString...,
1654
		)
1655
		session := podmanTest.Podman(localRunString)
1656
		session.WaitWithDefaultTimeout()
1657
		Expect(session).Should(ExitCleanly())
1658
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1659
		cid := session.OutputToString()
1660

1661
		session = podmanTest.Podman([]string{
1662
			"inspect",
1663
			"--format",
1664
			"{{.OCIRuntime}}",
1665
			cid,
1666
		})
1667
		session.WaitWithDefaultTimeout()
1668
		Expect(session).Should(ExitCleanly())
1669
		runtime := session.OutputToString()
1670

1671
		fileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
1672

1673
		result := podmanTest.Podman([]string{
1674
			"container",
1675
			"checkpoint",
1676
			cid, "-e",
1677
			fileName,
1678
		})
1679
		result.WaitWithDefaultTimeout()
1680

1681
		// As the container has been started with '--rm' it will be completely
1682
		// cleaned up after checkpointing.
1683
		Expect(result).Should(ExitCleanly())
1684
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1685
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
1686

1687
		// This should fail as the container was checkpointed with runc
1688
		result = podmanTest.Podman([]string{
1689
			"--runtime",
1690
			"crun",
1691
			"container",
1692
			"restore",
1693
			"-i",
1694
			fileName,
1695
		})
1696
		result.WaitWithDefaultTimeout()
1697

1698
		Expect(result).Should(ExitWithError(125, "and cannot be restored with runtime"))
1699

1700
		result = podmanTest.Podman([]string{
1701
			"--runtime",
1702
			"runc",
1703
			"container",
1704
			"restore",
1705
			"-i",
1706
			fileName,
1707
		})
1708
		result.WaitWithDefaultTimeout()
1709
		Expect(result).Should(ExitCleanly())
1710

1711
		result = podmanTest.Podman([]string{
1712
			"inspect",
1713
			"--format",
1714
			"{{.OCIRuntime}}",
1715
			cid,
1716
		})
1717
		result.WaitWithDefaultTimeout()
1718
		Expect(result).Should(ExitCleanly())
1719
		Expect(result.OutputToString()).To(Equal(runtime))
1720

1721
		result = podmanTest.Podman([]string{
1722
			"--runtime",
1723
			"runc",
1724
			"rm",
1725
			"-fa",
1726
		})
1727
		result.WaitWithDefaultTimeout()
1728
		Expect(result).Should(ExitCleanly())
1729
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1730
		// Remove exported checkpoint
1731
		os.Remove(fileName)
1732
	})
1733

1734
	It("podman checkpoint and restore dev/shm content with --export and --import", func() {
1735
		localRunString := getRunString([]string{"--rm", ALPINE, "top"})
1736
		session := podmanTest.Podman(localRunString)
1737
		session.WaitWithDefaultTimeout()
1738
		Expect(session).Should(ExitCleanly())
1739
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1740
		cid := session.OutputToString()
1741

1742
		// Add test file in dev/shm
1743
		result := podmanTest.Podman([]string{"exec", cid, "/bin/sh", "-c", "echo test" + cid + "test > /dev/shm/test.output"})
1744
		result.WaitWithDefaultTimeout()
1745
		Expect(result).Should(ExitCleanly())
1746

1747
		session = podmanTest.Podman([]string{"inspect", "--format", "{{.OCIRuntime}}", cid})
1748
		session.WaitWithDefaultTimeout()
1749
		Expect(session).Should(ExitCleanly())
1750
		runtime := session.OutputToString()
1751

1752
		checkpointFileName := filepath.Join(podmanTest.TempDir, "/checkpoint-"+cid+".tar.gz")
1753
		result = podmanTest.Podman([]string{"container", "checkpoint", cid, "-e", checkpointFileName})
1754
		result.WaitWithDefaultTimeout()
1755

1756
		// As the container has been started with '--rm' it will be completely
1757
		// cleaned up after checkpointing.
1758
		Expect(result).Should(ExitCleanly())
1759
		Expect(result.OutputToString()).To(ContainSubstring(cid))
1760
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1761
		Expect(podmanTest.NumberOfContainers()).To(Equal(0))
1762

1763
		result = podmanTest.Podman([]string{"container", "restore", "-i", checkpointFileName})
1764
		result.WaitWithDefaultTimeout()
1765
		Expect(result).Should(ExitCleanly())
1766
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1767
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
1768

1769
		// The restored container should have the same runtime as the original container
1770
		result = podmanTest.Podman([]string{"inspect", "--format", "{{.OCIRuntime}}", cid})
1771
		result.WaitWithDefaultTimeout()
1772
		Expect(result).Should(ExitCleanly())
1773
		Expect(session.OutputToString()).To(Equal(runtime))
1774

1775
		// Verify the test file content in dev/shm
1776
		result = podmanTest.Podman([]string{"exec", cid, "cat", "/dev/shm/test.output"})
1777
		result.WaitWithDefaultTimeout()
1778
		Expect(result).Should(ExitCleanly())
1779
		Expect(result.OutputToString()).To(ContainSubstring("test" + cid + "test"))
1780

1781
		// Remove exported checkpoint
1782
		os.Remove(checkpointFileName)
1783
	})
1784

1785
	It("podman checkpoint and restore dev/shm content", func() {
1786
		localRunString := getRunString([]string{ALPINE, "top"})
1787
		session := podmanTest.Podman(localRunString)
1788
		session.WaitWithDefaultTimeout()
1789
		Expect(session).Should(ExitCleanly())
1790
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1791
		cid := session.OutputToString()
1792

1793
		// Add test file in dev/shm
1794
		result := podmanTest.Podman([]string{"exec", cid, "/bin/sh", "-c", "echo test" + cid + "test > /dev/shm/test.output"})
1795
		result.WaitWithDefaultTimeout()
1796
		Expect(result).Should(ExitCleanly())
1797

1798
		result = podmanTest.Podman([]string{"container", "checkpoint", cid})
1799
		result.WaitWithDefaultTimeout()
1800
		Expect(result).Should(ExitCleanly())
1801
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1802
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
1803

1804
		result = podmanTest.Podman([]string{"container", "restore", cid})
1805
		result.WaitWithDefaultTimeout()
1806
		Expect(result).Should(ExitCleanly())
1807
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
1808
		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
1809

1810
		// Verify the test file content in dev/shm
1811
		result = podmanTest.Podman([]string{"exec", cid, "cat", "/dev/shm/test.output"})
1812
		result.WaitWithDefaultTimeout()
1813
		Expect(result).Should(ExitCleanly())
1814
		Expect(result.OutputToString()).To(ContainSubstring("test" + cid + "test"))
1815

1816
		result = podmanTest.Podman([]string{"rm", "-t", "0", "-fa"})
1817
		result.WaitWithDefaultTimeout()
1818
		Expect(result).Should(ExitCleanly())
1819
		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
1820
	})
1821
})
1822