podman

1
//go:build !remote && (linux || freebsd)
2

3
package libpod
4

5
import (
6
	"context"
7
	"errors"
8
	"fmt"
9
	"io"
10
	"io/fs"
11
	"math"
12
	"net"
13
	"os"
14
	"os/user"
15
	"path"
16
	"path/filepath"
17
	"runtime"
18
	"strconv"
19
	"strings"
20
	"syscall"
21
	"time"
22

23
	metadata "github.com/checkpoint-restore/checkpointctl/lib"
24
	"github.com/checkpoint-restore/go-criu/v7/stats"
25
	"github.com/containers/buildah"
26
	"github.com/containers/buildah/pkg/chrootuser"
27
	"github.com/containers/buildah/pkg/overlay"
28
	butil "github.com/containers/buildah/util"
29
	"github.com/containers/common/libnetwork/etchosts"
30
	"github.com/containers/common/libnetwork/resolvconf"
31
	"github.com/containers/common/libnetwork/types"
32
	"github.com/containers/common/pkg/apparmor"
33
	"github.com/containers/common/pkg/chown"
34
	"github.com/containers/common/pkg/config"
35
	"github.com/containers/common/pkg/subscriptions"
36
	"github.com/containers/common/pkg/umask"
37
	is "github.com/containers/image/v5/storage"
38
	"github.com/containers/podman/v5/libpod/define"
39
	"github.com/containers/podman/v5/libpod/events"
40
	"github.com/containers/podman/v5/pkg/annotations"
41
	"github.com/containers/podman/v5/pkg/checkpoint/crutils"
42
	"github.com/containers/podman/v5/pkg/criu"
43
	"github.com/containers/podman/v5/pkg/lookup"
44
	"github.com/containers/podman/v5/pkg/rootless"
45
	"github.com/containers/podman/v5/pkg/util"
46
	"github.com/containers/podman/v5/version"
47
	"github.com/containers/storage/pkg/archive"
48
	"github.com/containers/storage/pkg/fileutils"
49
	"github.com/containers/storage/pkg/idtools"
50
	"github.com/containers/storage/pkg/lockfile"
51
	"github.com/containers/storage/pkg/unshare"
52
	stypes "github.com/containers/storage/types"
53
	securejoin "github.com/cyphar/filepath-securejoin"
54
	runcuser "github.com/moby/sys/user"
55
	spec "github.com/opencontainers/runtime-spec/specs-go"
56
	"github.com/opencontainers/runtime-tools/generate"
57
	"github.com/opencontainers/selinux/go-selinux"
58
	"github.com/opencontainers/selinux/go-selinux/label"
59
	"github.com/sirupsen/logrus"
60
	"golang.org/x/exp/slices"
61
	"golang.org/x/sys/unix"
62
	cdi "tags.cncf.io/container-device-interface/pkg/cdi"
63
)
64

65
func parseOptionIDs(ctrMappings []idtools.IDMap, option string) ([]idtools.IDMap, error) {
66
	ranges := strings.Split(option, "#")
67
	ret := make([]idtools.IDMap, len(ranges))
68
	for i, m := range ranges {
69
		var v idtools.IDMap
70

71
		if m == "" {
72
			return nil, fmt.Errorf("invalid empty range for %q", option)
73
		}
74

75
		relative := false
76
		if m[0] == '@' {
77
			relative = true
78
			m = m[1:]
79
		}
80
		_, err := fmt.Sscanf(m, "%d-%d-%d", &v.ContainerID, &v.HostID, &v.Size)
81
		if err != nil {
82
			return nil, err
83
		}
84
		if v.ContainerID < 0 || v.HostID < 0 || v.Size < 1 {
85
			return nil, fmt.Errorf("invalid value for %q", option)
86
		}
87

88
		if relative {
89
			found := false
90
			for _, m := range ctrMappings {
91
				if v.HostID >= m.ContainerID && v.HostID < m.ContainerID+m.Size {
92
					v.HostID += m.HostID - m.ContainerID
93
					found = true
94
					break
95
				}
96
			}
97
			if !found {
98
				return nil, fmt.Errorf("could not find a user namespace mapping for the relative mapping %q", option)
99
			}
100
		}
101
		ret[i] = v
102
	}
103
	return ret, nil
104
}
105

106
func parseIDMapMountOption(idMappings stypes.IDMappingOptions, option string) ([]spec.LinuxIDMapping, []spec.LinuxIDMapping, error) {
107
	uidMap := idMappings.UIDMap
108
	gidMap := idMappings.GIDMap
109
	if strings.HasPrefix(option, "idmap=") {
110
		var err error
111
		options := strings.Split(strings.SplitN(option, "=", 2)[1], ";")
112
		for _, i := range options {
113
			switch {
114
			case strings.HasPrefix(i, "uids="):
115
				uidMap, err = parseOptionIDs(idMappings.UIDMap, strings.Replace(i, "uids=", "", 1))
116
				if err != nil {
117
					return nil, nil, err
118
				}
119
			case strings.HasPrefix(i, "gids="):
120
				gidMap, err = parseOptionIDs(idMappings.GIDMap, strings.Replace(i, "gids=", "", 1))
121
				if err != nil {
122
					return nil, nil, err
123
				}
124
			default:
125
				return nil, nil, fmt.Errorf("unknown option %q", i)
126
			}
127
		}
128
	}
129

130
	uidMappings := make([]spec.LinuxIDMapping, len(uidMap))
131
	gidMappings := make([]spec.LinuxIDMapping, len(gidMap))
132
	for i, uidmap := range uidMap {
133
		uidMappings[i] = spec.LinuxIDMapping{
134
			HostID:      uint32(uidmap.HostID),
135
			ContainerID: uint32(uidmap.ContainerID),
136
			Size:        uint32(uidmap.Size),
137
		}
138
	}
139
	for i, gidmap := range gidMap {
140
		gidMappings[i] = spec.LinuxIDMapping{
141
			HostID:      uint32(gidmap.HostID),
142
			ContainerID: uint32(gidmap.ContainerID),
143
			Size:        uint32(gidmap.Size),
144
		}
145
	}
146
	return uidMappings, gidMappings, nil
147
}
148

149
// Internal only function which returns upper and work dir from
150
// overlay options.
151
func getOverlayUpperAndWorkDir(options []string) (string, string, error) {
152
	upperDir := ""
153
	workDir := ""
154
	for _, o := range options {
155
		if strings.HasPrefix(o, "upperdir") {
156
			splitOpt := strings.SplitN(o, "=", 2)
157
			if len(splitOpt) > 1 {
158
				upperDir = splitOpt[1]
159
				if upperDir == "" {
160
					return "", "", errors.New("cannot accept empty value for upperdir")
161
				}
162
			}
163
		}
164
		if strings.HasPrefix(o, "workdir") {
165
			splitOpt := strings.SplitN(o, "=", 2)
166
			if len(splitOpt) > 1 {
167
				workDir = splitOpt[1]
168
				if workDir == "" {
169
					return "", "", errors.New("cannot accept empty value for workdir")
170
				}
171
			}
172
		}
173
	}
174
	if (upperDir != "" && workDir == "") || (upperDir == "" && workDir != "") {
175
		return "", "", errors.New("must specify both upperdir and workdir")
176
	}
177
	return upperDir, workDir, nil
178
}
179

180
// Generate spec for a container
181
// Accepts a map of the container's dependencies
182
func (c *Container) generateSpec(ctx context.Context) (s *spec.Spec, cleanupFuncRet func(), err error) {
183
	var safeMounts []*safeMountInfo
184
	// lock the thread so that the current thread will be kept alive until the mounts are used
185
	runtime.LockOSThread()
186
	cleanupFunc := func() {
187
		runtime.UnlockOSThread()
188
		for _, s := range safeMounts {
189
			s.Close()
190
		}
191
	}
192
	defer func() {
193
		if err != nil {
194
			cleanupFunc()
195
		}
196
	}()
197
	overrides := c.getUserOverrides()
198
	execUser, err := lookup.GetUserGroupInfo(c.state.Mountpoint, c.config.User, overrides)
199
	if err != nil {
200
		if slices.Contains(c.config.HostUsers, c.config.User) {
201
			execUser, err = lookupHostUser(c.config.User)
202
		}
203
		if err != nil {
204
			return nil, nil, err
205
		}
206
	}
207

208
	// NewFromSpec() is deprecated according to its comment
209
	// however the recommended replace just causes a nil map panic
210
	g := generate.NewFromSpec(c.config.Spec)
211

212
	// If the flag to mount all devices is set for a privileged container, add
213
	// all the devices from the host's machine into the container
214
	if c.config.MountAllDevices {
215
		systemdMode := false
216
		if c.config.Systemd != nil {
217
			systemdMode = *c.config.Systemd
218
		}
219
		if err := util.AddPrivilegedDevices(&g, systemdMode); err != nil {
220
			return nil, nil, err
221
		}
222
	}
223

224
	// If network namespace was requested, add it now
225
	if err := c.addNetworkNamespace(&g); err != nil {
226
		return nil, nil, err
227
	}
228

229
	// Apply AppArmor checks and load the default profile if needed.
230
	if len(c.config.Spec.Process.ApparmorProfile) > 0 {
231
		updatedProfile, err := apparmor.CheckProfileAndLoadDefault(c.config.Spec.Process.ApparmorProfile)
232
		if err != nil {
233
			return nil, nil, err
234
		}
235
		g.SetProcessApparmorProfile(updatedProfile)
236
	}
237

238
	if err := c.makeBindMounts(); err != nil {
239
		return nil, nil, err
240
	}
241

242
	if err := c.mountNotifySocket(g); err != nil {
243
		return nil, nil, err
244
	}
245

246
	// Get host UID and GID based on the container process UID and GID.
247
	hostUID, hostGID, err := butil.GetHostIDs(util.IDtoolsToRuntimeSpec(c.config.IDMappings.UIDMap), util.IDtoolsToRuntimeSpec(c.config.IDMappings.GIDMap), uint32(execUser.Uid), uint32(execUser.Gid))
248
	if err != nil {
249
		return nil, nil, err
250
	}
251

252
	// Add named volumes
253
	for _, namedVol := range c.config.NamedVolumes {
254
		volume, err := c.runtime.GetVolume(namedVol.Name)
255
		if err != nil {
256
			return nil, nil, fmt.Errorf("retrieving volume %s to add to container %s: %w", namedVol.Name, c.ID(), err)
257
		}
258
		mountPoint, err := volume.MountPoint()
259
		if err != nil {
260
			return nil, nil, err
261
		}
262

263
		if len(namedVol.SubPath) > 0 {
264
			safeMount, err := c.safeMountSubPath(mountPoint, namedVol.SubPath)
265
			if err != nil {
266
				return nil, nil, err
267
			}
268
			safeMounts = append(safeMounts, safeMount)
269

270
			mountPoint = safeMount.mountPoint
271
		}
272

273
		overlayFlag := false
274
		upperDir := ""
275
		workDir := ""
276
		for _, o := range namedVol.Options {
277
			if o == "O" {
278
				overlayFlag = true
279
				upperDir, workDir, err = getOverlayUpperAndWorkDir(namedVol.Options)
280
				if err != nil {
281
					return nil, nil, err
282
				}
283
			}
284
		}
285

286
		if overlayFlag {
287
			var overlayMount spec.Mount
288
			var overlayOpts *overlay.Options
289
			contentDir, err := overlay.TempDir(c.config.StaticDir, c.RootUID(), c.RootGID())
290
			if err != nil {
291
				return nil, nil, err
292
			}
293

294
			overlayOpts = &overlay.Options{RootUID: c.RootUID(),
295
				RootGID:                c.RootGID(),
296
				UpperDirOptionFragment: upperDir,
297
				WorkDirOptionFragment:  workDir,
298
				GraphOpts:              c.runtime.store.GraphOptions(),
299
			}
300

301
			overlayMount, err = overlay.MountWithOptions(contentDir, mountPoint, namedVol.Dest, overlayOpts)
302
			if err != nil {
303
				return nil, nil, fmt.Errorf("mounting overlay failed %q: %w", mountPoint, err)
304
			}
305

306
			for _, o := range namedVol.Options {
307
				if o == "U" {
308
					if err := c.ChangeHostPathOwnership(mountPoint, true, int(hostUID), int(hostGID)); err != nil {
309
						return nil, nil, err
310
					}
311

312
					if err := c.ChangeHostPathOwnership(contentDir, true, int(hostUID), int(hostGID)); err != nil {
313
						return nil, nil, err
314
					}
315
				}
316
			}
317
			g.AddMount(overlayMount)
318
		} else {
319
			volMount := spec.Mount{
320
				Type:        define.TypeBind,
321
				Source:      mountPoint,
322
				Destination: namedVol.Dest,
323
				Options:     namedVol.Options,
324
			}
325
			g.AddMount(volMount)
326
		}
327
	}
328

329
	// Check if the spec file mounts contain the options z, Z, U or idmap.
330
	// If they have z or Z, relabel the source directory and then remove the option.
331
	// If they have U, chown the source directory and then remove the option.
332
	// If they have idmap, then calculate the mappings to use in the OCI config file.
333
	for i := range g.Config.Mounts {
334
		m := &g.Config.Mounts[i]
335
		var options []string
336
		for _, o := range m.Options {
337
			if strings.HasPrefix(o, "subpath=") {
338
				subpath := strings.Split(o, "=")[1]
339
				safeMount, err := c.safeMountSubPath(m.Source, subpath)
340
				if err != nil {
341
					return nil, nil, err
342
				}
343
				safeMounts = append(safeMounts, safeMount)
344
				m.Source = safeMount.mountPoint
345
				continue
346
			}
347
			if o == "idmap" || strings.HasPrefix(o, "idmap=") {
348
				var err error
349
				m.UIDMappings, m.GIDMappings, err = parseIDMapMountOption(c.config.IDMappings, o)
350
				if err != nil {
351
					return nil, nil, err
352
				}
353
				continue
354
			}
355
			switch o {
356
			case "U":
357
				if m.Type == define.TypeTmpfs {
358
					options = append(options, []string{fmt.Sprintf("uid=%d", execUser.Uid), fmt.Sprintf("gid=%d", execUser.Gid)}...)
359
				} else {
360
					// only chown on initial creation of container
361
					if err := c.ChangeHostPathOwnership(m.Source, true, int(hostUID), int(hostGID)); err != nil {
362
						return nil, nil, err
363
					}
364
				}
365
			case "z":
366
				fallthrough
367
			case "Z":
368
				if err := c.relabel(m.Source, c.MountLabel(), label.IsShared(o)); err != nil {
369
					return nil, nil, err
370
				}
371
			case "no-dereference":
372
				// crun calls the option `copy-symlink`.
373
				// Podman decided for --no-dereference as many
374
				// bin-utils tools (e..g, touch, chown, cp) do.
375
				options = append(options, "copy-symlink")
376
			default:
377
				options = append(options, o)
378
			}
379
		}
380
		m.Options = options
381
	}
382

383
	c.setProcessLabel(&g)
384
	c.setMountLabel(&g)
385

386
	// Add bind mounts to container
387
	for dstPath, srcPath := range c.state.BindMounts {
388
		newMount := spec.Mount{
389
			Type:        define.TypeBind,
390
			Source:      srcPath,
391
			Destination: dstPath,
392
			Options:     bindOptions,
393
		}
394
		if c.IsReadOnly() && (dstPath != "/dev/shm" || !c.config.ReadWriteTmpfs) {
395
			newMount.Options = append(newMount.Options, "ro", "nosuid", "noexec", "nodev")
396
		}
397
		if dstPath == "/dev/shm" && c.state.BindMounts["/dev/shm"] == c.config.ShmDir {
398
			newMount.Options = append(newMount.Options, "nosuid", "noexec", "nodev")
399
		}
400
		if !MountExists(g.Mounts(), dstPath) {
401
			g.AddMount(newMount)
402
		} else {
403
			logrus.Infof("User mount overriding libpod mount at %q", dstPath)
404
		}
405
	}
406

407
	// Add overlay volumes
408
	for _, overlayVol := range c.config.OverlayVolumes {
409
		upperDir, workDir, err := getOverlayUpperAndWorkDir(overlayVol.Options)
410
		if err != nil {
411
			return nil, nil, err
412
		}
413
		contentDir, err := overlay.TempDir(c.config.StaticDir, c.RootUID(), c.RootGID())
414
		if err != nil {
415
			return nil, nil, err
416
		}
417
		overlayOpts := &overlay.Options{RootUID: c.RootUID(),
418
			RootGID:                c.RootGID(),
419
			UpperDirOptionFragment: upperDir,
420
			WorkDirOptionFragment:  workDir,
421
			GraphOpts:              c.runtime.store.GraphOptions(),
422
		}
423

424
		overlayMount, err := overlay.MountWithOptions(contentDir, overlayVol.Source, overlayVol.Dest, overlayOpts)
425
		if err != nil {
426
			return nil, nil, fmt.Errorf("mounting overlay failed %q: %w", overlayVol.Source, err)
427
		}
428

429
		// Check overlay volume options
430
		for _, o := range overlayVol.Options {
431
			if o == "U" {
432
				if err := c.ChangeHostPathOwnership(overlayVol.Source, true, int(hostUID), int(hostGID)); err != nil {
433
					return nil, nil, err
434
				}
435

436
				if err := c.ChangeHostPathOwnership(contentDir, true, int(hostUID), int(hostGID)); err != nil {
437
					return nil, nil, err
438
				}
439
			}
440
		}
441

442
		g.AddMount(overlayMount)
443
	}
444

445
	// Add image volumes as overlay mounts
446
	for _, volume := range c.config.ImageVolumes {
447
		// Mount the specified image.
448
		img, _, err := c.runtime.LibimageRuntime().LookupImage(volume.Source, nil)
449
		if err != nil {
450
			return nil, nil, fmt.Errorf("creating image volume %q:%q: %w", volume.Source, volume.Dest, err)
451
		}
452
		mountPoint, err := img.Mount(ctx, nil, "")
453
		if err != nil {
454
			return nil, nil, fmt.Errorf("mounting image volume %q:%q: %w", volume.Source, volume.Dest, err)
455
		}
456

457
		contentDir, err := overlay.TempDir(c.config.StaticDir, c.RootUID(), c.RootGID())
458
		if err != nil {
459
			return nil, nil, fmt.Errorf("failed to create TempDir in the %s directory: %w", c.config.StaticDir, err)
460
		}
461

462
		var overlayMount spec.Mount
463
		if volume.ReadWrite {
464
			overlayMount, err = overlay.Mount(contentDir, mountPoint, volume.Dest, c.RootUID(), c.RootGID(), c.runtime.store.GraphOptions())
465
		} else {
466
			overlayMount, err = overlay.MountReadOnly(contentDir, mountPoint, volume.Dest, c.RootUID(), c.RootGID(), c.runtime.store.GraphOptions())
467
		}
468
		if err != nil {
469
			return nil, nil, fmt.Errorf("creating overlay mount for image %q failed: %w", volume.Source, err)
470
		}
471
		g.AddMount(overlayMount)
472
	}
473

474
	err = c.setHomeEnvIfNeeded()
475
	if err != nil {
476
		return nil, nil, err
477
	}
478

479
	if c.config.User != "" {
480
		// User and Group must go together
481
		g.SetProcessUID(uint32(execUser.Uid))
482
		g.SetProcessGID(uint32(execUser.Gid))
483
		g.AddProcessAdditionalGid(uint32(execUser.Gid))
484
	}
485

486
	if c.config.Umask != "" {
487
		umask, err := c.umask()
488
		if err != nil {
489
			return nil, nil, err
490
		}
491
		g.Config.Process.User.Umask = &umask
492
	}
493

494
	// Add addition groups if c.config.GroupAdd is not empty
495
	if len(c.config.Groups) > 0 {
496
		gids, err := lookup.GetContainerGroups(c.config.Groups, c.state.Mountpoint, overrides)
497
		if err != nil {
498
			return nil, nil, fmt.Errorf("looking up supplemental groups for container %s: %w", c.ID(), err)
499
		}
500
		for _, gid := range gids {
501
			g.AddProcessAdditionalGid(gid)
502
		}
503
	}
504

505
	if err := c.addSystemdMounts(&g); err != nil {
506
		return nil, nil, err
507
	}
508

509
	// Look up and add groups the user belongs to, if a group wasn't directly specified
510
	if !strings.Contains(c.config.User, ":") {
511
		// the gidMappings that are present inside the container user namespace
512
		var gidMappings []idtools.IDMap
513

514
		switch {
515
		case len(c.config.IDMappings.GIDMap) > 0:
516
			gidMappings = c.config.IDMappings.GIDMap
517
		case rootless.IsRootless():
518
			// Check whether the current user namespace has enough gids available.
519
			availableGids, err := rootless.GetAvailableGids()
520
			if err != nil {
521
				return nil, nil, fmt.Errorf("cannot read number of available GIDs: %w", err)
522
			}
523
			gidMappings = []idtools.IDMap{{
524
				ContainerID: 0,
525
				HostID:      0,
526
				Size:        int(availableGids),
527
			}}
528
		default:
529
			gidMappings = []idtools.IDMap{{
530
				ContainerID: 0,
531
				HostID:      0,
532
				Size:        math.MaxInt32,
533
			}}
534
		}
535
		for _, gid := range execUser.Sgids {
536
			isGIDAvailable := false
537
			for _, m := range gidMappings {
538
				if gid >= m.ContainerID && gid < m.ContainerID+m.Size {
539
					isGIDAvailable = true
540
					break
541
				}
542
			}
543
			if isGIDAvailable {
544
				g.AddProcessAdditionalGid(uint32(gid))
545
			} else {
546
				logrus.Warnf("Additional gid=%d is not present in the user namespace, skip setting it", gid)
547
			}
548
		}
549
	}
550

551
	// Add shared namespaces from other containers
552
	if err := c.addSharedNamespaces(&g); err != nil {
553
		return nil, nil, err
554
	}
555

556
	g.SetRootPath(c.state.Mountpoint)
557
	g.AddAnnotation("org.opencontainers.image.stopSignal", strconv.FormatUint(uint64(c.config.StopSignal), 10))
558

559
	if _, exists := g.Config.Annotations[annotations.ContainerManager]; !exists {
560
		g.AddAnnotation(annotations.ContainerManager, annotations.ContainerManagerLibpod)
561
	}
562

563
	if err := c.setCgroupsPath(&g); err != nil {
564
		return nil, nil, err
565
	}
566

567
	// Warning: CDI may alter g.Config in place.
568
	if len(c.config.CDIDevices) > 0 {
569
		registry, err := cdi.NewCache(
570
			cdi.WithAutoRefresh(false),
571
		)
572
		if err != nil {
573
			return nil, nil, fmt.Errorf("creating CDI registry: %w", err)
574
		}
575
		if err := registry.Refresh(); err != nil {
576
			logrus.Debugf("The following error was triggered when refreshing the CDI registry: %v", err)
577
		}
578
		if _, err := registry.InjectDevices(g.Config, c.config.CDIDevices...); err != nil {
579
			return nil, nil, fmt.Errorf("setting up CDI devices: %w", err)
580
		}
581
	}
582

583
	// Mounts need to be sorted so paths will not cover other paths
584
	mounts := sortMounts(g.Mounts())
585
	g.ClearMounts()
586

587
	for _, m := range mounts {
588
		// We need to remove all symlinks from tmpfs mounts.
589
		// Runc and other runtimes may choke on them.
590
		// Easy solution: use securejoin to do a scoped evaluation of
591
		// the links, then trim off the mount prefix.
592
		if m.Type == define.TypeTmpfs {
593
			finalPath, err := securejoin.SecureJoin(c.state.Mountpoint, m.Destination)
594
			if err != nil {
595
				return nil, nil, fmt.Errorf("resolving symlinks for mount destination %s: %w", m.Destination, err)
596
			}
597
			trimmedPath := strings.TrimPrefix(finalPath, strings.TrimSuffix(c.state.Mountpoint, "/"))
598
			m.Destination = trimmedPath
599
		}
600
		g.AddMount(m)
601
	}
602

603
	if err := c.addRootPropagation(&g, mounts); err != nil {
604
		return nil, nil, err
605
	}
606

607
	// Warning: precreate hooks may alter g.Config in place.
608
	if c.state.ExtensionStageHooks, err = c.setupOCIHooks(ctx, g.Config); err != nil {
609
		return nil, nil, fmt.Errorf("setting up OCI Hooks: %w", err)
610
	}
611
	if len(c.config.EnvSecrets) > 0 {
612
		manager, err := c.runtime.SecretsManager()
613
		if err != nil {
614
			return nil, nil, err
615
		}
616
		for name, secr := range c.config.EnvSecrets {
617
			_, data, err := manager.LookupSecretData(secr.Name)
618
			if err != nil {
619
				return nil, nil, err
620
			}
621
			g.AddProcessEnv(name, string(data))
622
		}
623
	}
624

625
	// Pass down the LISTEN_* environment (see #10443).
626
	for _, key := range []string{"LISTEN_PID", "LISTEN_FDS", "LISTEN_FDNAMES"} {
627
		if val, ok := os.LookupEnv(key); ok {
628
			// Force the PID to `1` since we cannot rely on (all
629
			// versions of) all runtimes to do it for us.
630
			if key == "LISTEN_PID" {
631
				val = "1"
632
			}
633
			g.AddProcessEnv(key, val)
634
		}
635
	}
636

637
	// setup rlimits
638
	nofileSet := false
639
	nprocSet := false
640
	isRootless := rootless.IsRootless()
641
	isRunningInUserNs := unshare.IsRootless()
642
	if isRunningInUserNs && g.Config.Process != nil && g.Config.Process.OOMScoreAdj != nil {
643
		var err error
644
		*g.Config.Process.OOMScoreAdj, err = maybeClampOOMScoreAdj(*g.Config.Process.OOMScoreAdj)
645
		if err != nil {
646
			return nil, nil, err
647
		}
648
	}
649
	if isRootless {
650
		for _, rlimit := range c.config.Spec.Process.Rlimits {
651
			if rlimit.Type == "RLIMIT_NOFILE" {
652
				nofileSet = true
653
			}
654
			if rlimit.Type == "RLIMIT_NPROC" {
655
				nprocSet = true
656
			}
657
		}
658
		if !nofileSet {
659
			max := rlimT(define.RLimitDefaultValue)
660
			current := rlimT(define.RLimitDefaultValue)
661
			var rlimit unix.Rlimit
662
			if err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit); err != nil {
663
				logrus.Warnf("Failed to return RLIMIT_NOFILE ulimit %q", err)
664
			}
665
			if rlimT(rlimit.Cur) < current {
666
				current = rlimT(rlimit.Cur)
667
			}
668
			if rlimT(rlimit.Max) < max {
669
				max = rlimT(rlimit.Max)
670
			}
671
			g.AddProcessRlimits("RLIMIT_NOFILE", uint64(max), uint64(current))
672
		}
673
		if !nprocSet {
674
			max := rlimT(define.RLimitDefaultValue)
675
			current := rlimT(define.RLimitDefaultValue)
676
			var rlimit unix.Rlimit
677
			if err := unix.Getrlimit(unix.RLIMIT_NPROC, &rlimit); err != nil {
678
				logrus.Warnf("Failed to return RLIMIT_NPROC ulimit %q", err)
679
			}
680
			if rlimT(rlimit.Cur) < current {
681
				current = rlimT(rlimit.Cur)
682
			}
683
			if rlimT(rlimit.Max) < max {
684
				max = rlimT(rlimit.Max)
685
			}
686
			g.AddProcessRlimits("RLIMIT_NPROC", uint64(max), uint64(current))
687
		}
688
	}
689

690
	c.addMaskedPaths(&g)
691

692
	return g.Config, cleanupFunc, nil
693
}
694

695
// isWorkDirSymlink returns true if resolved workdir is symlink or a chain of symlinks,
696
// and final resolved target is present either on  volume, mount or inside of container
697
// otherwise it returns false. Following function is meant for internal use only and
698
// can change at any point of time.
699
func (c *Container) isWorkDirSymlink(resolvedPath string) bool {
700
	// We cannot create workdir since explicit --workdir is
701
	// set in config but workdir could also be a symlink.
702
	// If it's a symlink, check if the resolved target is present in the container.
703
	// If so, that's a valid use case: return nil.
704

705
	maxSymLinks := 0
706
	for {
707
		// Linux only supports a chain of 40 links.
708
		// Reference: https://github.com/torvalds/linux/blob/master/include/linux/namei.h#L13
709
		if maxSymLinks > 40 {
710
			break
711
		}
712
		resolvedSymlink, err := os.Readlink(resolvedPath)
713
		if err != nil {
714
			// End sym-link resolution loop.
715
			break
716
		}
717
		if resolvedSymlink != "" {
718
			_, resolvedSymlinkWorkdir, err := c.resolvePath(c.state.Mountpoint, resolvedSymlink)
719
			if isPathOnVolume(c, resolvedSymlinkWorkdir) || isPathOnMount(c, resolvedSymlinkWorkdir) {
720
				// Resolved symlink exists on external volume or mount
721
				return true
722
			}
723
			if err != nil {
724
				// Could not resolve path so end sym-link resolution loop.
725
				break
726
			}
727
			if resolvedSymlinkWorkdir != "" {
728
				resolvedPath = resolvedSymlinkWorkdir
729
				err := fileutils.Exists(resolvedSymlinkWorkdir)
730
				if err == nil {
731
					// Symlink resolved successfully and resolved path exists on container,
732
					// this is a valid use-case so return nil.
733
					logrus.Debugf("Workdir is a symlink with target to %q and resolved symlink exists on container", resolvedSymlink)
734
					return true
735
				}
736
			}
737
		}
738
		maxSymLinks++
739
	}
740
	return false
741
}
742

743
// resolveWorkDir resolves the container's workdir and, depending on the
744
// configuration, will create it, or error out if it does not exist.
745
// Note that the container must be mounted before.
746
func (c *Container) resolveWorkDir() error {
747
	workdir := c.WorkingDir()
748

749
	// If the specified workdir is a subdir of a volume or mount,
750
	// we don't need to do anything.  The runtime is taking care of
751
	// that.
752
	if isPathOnVolume(c, workdir) || isPathOnMount(c, workdir) {
753
		logrus.Debugf("Workdir %q resolved to a volume or mount", workdir)
754
		return nil
755
	}
756

757
	_, resolvedWorkdir, err := c.resolvePath(c.state.Mountpoint, workdir)
758
	if err != nil {
759
		return err
760
	}
761
	logrus.Debugf("Workdir %q resolved to host path %q", workdir, resolvedWorkdir)
762

763
	st, err := os.Stat(resolvedWorkdir)
764
	if err == nil {
765
		if !st.IsDir() {
766
			return fmt.Errorf("workdir %q exists on container %s, but is not a directory", workdir, c.ID())
767
		}
768
		return nil
769
	}
770
	if !c.config.CreateWorkingDir {
771
		// No need to create it (e.g., `--workdir=/foo`), so let's make sure
772
		// the path exists on the container.
773
		if err != nil {
774
			if os.IsNotExist(err) {
775
				// If resolved Workdir path gets marked as a valid symlink,
776
				// return nil cause this is valid use-case.
777
				if c.isWorkDirSymlink(resolvedWorkdir) {
778
					return nil
779
				}
780
				return fmt.Errorf("workdir %q does not exist on container %s", workdir, c.ID())
781
			}
782
			// This might be a serious error (e.g., permission), so
783
			// we need to return the full error.
784
			return fmt.Errorf("detecting workdir %q on container %s: %w", workdir, c.ID(), err)
785
		}
786
		return nil
787
	}
788
	if err := os.MkdirAll(resolvedWorkdir, 0755); err != nil {
789
		if os.IsExist(err) {
790
			return nil
791
		}
792
		return fmt.Errorf("creating container %s workdir: %w", c.ID(), err)
793
	}
794

795
	// Ensure container entrypoint is created (if required).
796
	uid, gid, _, err := chrootuser.GetUser(c.state.Mountpoint, c.User())
797
	if err != nil {
798
		return fmt.Errorf("looking up %s inside of the container %s: %w", c.User(), c.ID(), err)
799
	}
800
	if err := idtools.SafeChown(resolvedWorkdir, int(uid), int(gid)); err != nil {
801
		return fmt.Errorf("chowning container %s workdir to container root: %w", c.ID(), err)
802
	}
803

804
	return nil
805
}
806

807
func (c *Container) getUserOverrides() *lookup.Overrides {
808
	var hasPasswdFile, hasGroupFile bool
809
	overrides := lookup.Overrides{}
810
	for _, m := range c.config.Spec.Mounts {
811
		if m.Destination == "/etc/passwd" {
812
			overrides.ContainerEtcPasswdPath = m.Source
813
			hasPasswdFile = true
814
		}
815
		if m.Destination == "/etc/group" {
816
			overrides.ContainerEtcGroupPath = m.Source
817
			hasGroupFile = true
818
		}
819
		if m.Destination == "/etc" {
820
			if !hasPasswdFile {
821
				overrides.ContainerEtcPasswdPath = filepath.Join(m.Source, "passwd")
822
			}
823
			if !hasGroupFile {
824
				overrides.ContainerEtcGroupPath = filepath.Join(m.Source, "group")
825
			}
826
		}
827
	}
828
	if path, ok := c.state.BindMounts["/etc/passwd"]; ok {
829
		overrides.ContainerEtcPasswdPath = path
830
	}
831
	return &overrides
832
}
833

834
func lookupHostUser(name string) (*runcuser.ExecUser, error) {
835
	var execUser runcuser.ExecUser
836
	// Look up User on host
837
	u, err := util.LookupUser(name)
838
	if err != nil {
839
		return &execUser, err
840
	}
841
	uid, err := strconv.ParseUint(u.Uid, 10, 32)
842
	if err != nil {
843
		return &execUser, err
844
	}
845

846
	gid, err := strconv.ParseUint(u.Gid, 10, 32)
847
	if err != nil {
848
		return &execUser, err
849
	}
850
	execUser.Uid = int(uid)
851
	execUser.Gid = int(gid)
852
	execUser.Home = u.HomeDir
853
	return &execUser, nil
854
}
855

856
// mountNotifySocket mounts the NOTIFY_SOCKET into the container if it's set
857
// and if the sdnotify mode is set to container.  It also sets c.notifySocket
858
// to avoid redundantly looking up the env variable.
859
func (c *Container) mountNotifySocket(g generate.Generator) error {
860
	if c.config.SdNotifySocket == "" {
861
		return nil
862
	}
863
	if c.config.SdNotifyMode != define.SdNotifyModeContainer {
864
		return nil
865
	}
866

867
	notifyDir := filepath.Join(c.bundlePath(), "notify")
868
	logrus.Debugf("Checking notify %q dir", notifyDir)
869
	if err := os.MkdirAll(notifyDir, 0755); err != nil {
870
		if !os.IsExist(err) {
871
			return fmt.Errorf("unable to create notify %q dir: %w", notifyDir, err)
872
		}
873
	}
874
	if err := c.relabel(notifyDir, c.MountLabel(), true); err != nil {
875
		return fmt.Errorf("relabel failed %q: %w", notifyDir, err)
876
	}
877
	logrus.Debugf("Add bindmount notify %q dir", notifyDir)
878
	if _, ok := c.state.BindMounts["/run/notify"]; !ok {
879
		c.state.BindMounts["/run/notify"] = notifyDir
880
	}
881

882
	// Set the container's notify socket to the proxy socket created by conmon
883
	g.AddProcessEnv("NOTIFY_SOCKET", "/run/notify/notify.sock")
884

885
	return nil
886
}
887

888
func (c *Container) addCheckpointImageMetadata(importBuilder *buildah.Builder) error {
889
	// Get information about host environment
890
	hostInfo, err := c.Runtime().hostInfo()
891
	if err != nil {
892
		return fmt.Errorf("getting host info: %v", err)
893
	}
894

895
	criuVersion, err := criu.GetCriuVersion()
896
	if err != nil {
897
		return fmt.Errorf("getting criu version: %v", err)
898
	}
899

900
	rootfsImageID, rootfsImageName := c.Image()
901

902
	// Add image annotations with information about the container and the host.
903
	// This information is useful to check compatibility before restoring the checkpoint
904

905
	checkpointImageAnnotations := map[string]string{
906
		define.CheckpointAnnotationName:                c.config.Name,
907
		define.CheckpointAnnotationRawImageName:        c.config.RawImageName,
908
		define.CheckpointAnnotationRootfsImageID:       rootfsImageID,
909
		define.CheckpointAnnotationRootfsImageName:     rootfsImageName,
910
		define.CheckpointAnnotationPodmanVersion:       version.Version.String(),
911
		define.CheckpointAnnotationCriuVersion:         strconv.Itoa(criuVersion),
912
		define.CheckpointAnnotationRuntimeName:         hostInfo.OCIRuntime.Name,
913
		define.CheckpointAnnotationRuntimeVersion:      hostInfo.OCIRuntime.Version,
914
		define.CheckpointAnnotationConmonVersion:       hostInfo.Conmon.Version,
915
		define.CheckpointAnnotationHostArch:            hostInfo.Arch,
916
		define.CheckpointAnnotationHostKernel:          hostInfo.Kernel,
917
		define.CheckpointAnnotationCgroupVersion:       hostInfo.CgroupsVersion,
918
		define.CheckpointAnnotationDistributionVersion: hostInfo.Distribution.Version,
919
		define.CheckpointAnnotationDistributionName:    hostInfo.Distribution.Distribution,
920
	}
921

922
	for key, value := range checkpointImageAnnotations {
923
		importBuilder.SetAnnotation(key, value)
924
	}
925

926
	return nil
927
}
928

929
func (c *Container) resolveCheckpointImageName(options *ContainerCheckpointOptions) error {
930
	if options.CreateImage == "" {
931
		return nil
932
	}
933

934
	// Resolve image name
935
	resolvedImageName, err := c.runtime.LibimageRuntime().ResolveName(options.CreateImage)
936
	if err != nil {
937
		return err
938
	}
939

940
	options.CreateImage = resolvedImageName
941
	return nil
942
}
943

944
func (c *Container) createCheckpointImage(ctx context.Context, options ContainerCheckpointOptions) error {
945
	if options.CreateImage == "" {
946
		return nil
947
	}
948
	logrus.Debugf("Create checkpoint image %s", options.CreateImage)
949

950
	// Create storage reference
951
	imageRef, err := is.Transport.ParseStoreReference(c.runtime.store, options.CreateImage)
952
	if err != nil {
953
		return errors.New("failed to parse image name")
954
	}
955

956
	// Build an image scratch
957
	builderOptions := buildah.BuilderOptions{
958
		FromImage: "scratch",
959
	}
960
	importBuilder, err := buildah.NewBuilder(ctx, c.runtime.store, builderOptions)
961
	if err != nil {
962
		return err
963
	}
964
	// Clean up buildah working container
965
	defer func() {
966
		if err := importBuilder.Delete(); err != nil {
967
			logrus.Errorf("Image builder delete failed: %v", err)
968
		}
969
	}()
970

971
	if err := c.prepareCheckpointExport(); err != nil {
972
		return err
973
	}
974

975
	// Export checkpoint into temporary tar file
976
	tmpDir, err := os.MkdirTemp("", "checkpoint_image_")
977
	if err != nil {
978
		return err
979
	}
980
	defer os.RemoveAll(tmpDir)
981

982
	options.TargetFile = path.Join(tmpDir, "checkpoint.tar")
983

984
	if err := c.exportCheckpoint(options); err != nil {
985
		return err
986
	}
987

988
	// Copy checkpoint from temporary tar file in the image
989
	addAndCopyOptions := buildah.AddAndCopyOptions{}
990
	if err := importBuilder.Add("", true, addAndCopyOptions, options.TargetFile); err != nil {
991
		return err
992
	}
993

994
	if err := c.addCheckpointImageMetadata(importBuilder); err != nil {
995
		return err
996
	}
997

998
	commitOptions := buildah.CommitOptions{
999
		Squash:        true,
1000
		SystemContext: c.runtime.imageContext,
1001
	}
1002

1003
	// Create checkpoint image
1004
	id, _, _, err := importBuilder.Commit(ctx, imageRef, commitOptions)
1005
	if err != nil {
1006
		return err
1007
	}
1008
	logrus.Debugf("Created checkpoint image: %s", id)
1009
	return nil
1010
}
1011

1012
func (c *Container) exportCheckpoint(options ContainerCheckpointOptions) error {
1013
	if len(c.Dependencies()) == 1 {
1014
		// Check if the dependency is an infra container. If it is we can checkpoint
1015
		// the container out of the Pod.
1016
		if c.config.Pod == "" {
1017
			return errors.New("cannot export checkpoints of containers with dependencies")
1018
		}
1019

1020
		pod, err := c.runtime.state.Pod(c.config.Pod)
1021
		if err != nil {
1022
			return fmt.Errorf("container %s is in pod %s, but pod cannot be retrieved: %w", c.ID(), c.config.Pod, err)
1023
		}
1024
		infraID, err := pod.InfraContainerID()
1025
		if err != nil {
1026
			return fmt.Errorf("cannot retrieve infra container ID for pod %s: %w", c.config.Pod, err)
1027
		}
1028
		if c.Dependencies()[0] != infraID {
1029
			return errors.New("cannot export checkpoints of containers with dependencies")
1030
		}
1031
	}
1032
	if len(c.Dependencies()) > 1 {
1033
		return errors.New("cannot export checkpoints of containers with dependencies")
1034
	}
1035
	logrus.Debugf("Exporting checkpoint image of container %q to %q", c.ID(), options.TargetFile)
1036

1037
	includeFiles := []string{
1038
		"artifacts",
1039
		metadata.DevShmCheckpointTar,
1040
		metadata.ConfigDumpFile,
1041
		metadata.SpecDumpFile,
1042
		metadata.NetworkStatusFile,
1043
		stats.StatsDump,
1044
	}
1045

1046
	if c.LogDriver() == define.KubernetesLogging ||
1047
		c.LogDriver() == define.JSONLogging {
1048
		includeFiles = append(includeFiles, "ctr.log")
1049
	}
1050
	if options.PreCheckPoint {
1051
		includeFiles = append(includeFiles, preCheckpointDir)
1052
	} else {
1053
		includeFiles = append(includeFiles, metadata.CheckpointDirectory)
1054
	}
1055
	// Get root file-system changes included in the checkpoint archive
1056
	var addToTarFiles []string
1057
	if !options.IgnoreRootfs {
1058
		// To correctly track deleted files, let's go through the output of 'podman diff'
1059
		rootFsChanges, err := c.runtime.GetDiff("", c.ID(), define.DiffContainer)
1060
		if err != nil {
1061
			return fmt.Errorf("exporting root file-system diff for %q: %w", c.ID(), err)
1062
		}
1063

1064
		addToTarFiles, err := crutils.CRCreateRootFsDiffTar(&rootFsChanges, c.state.Mountpoint, c.bundlePath())
1065
		if err != nil {
1066
			return err
1067
		}
1068

1069
		includeFiles = append(includeFiles, addToTarFiles...)
1070
	}
1071

1072
	// Folder containing archived volumes that will be included in the export
1073
	expVolDir := filepath.Join(c.bundlePath(), metadata.CheckpointVolumesDirectory)
1074

1075
	// Create an archive for each volume associated with the container
1076
	if !options.IgnoreVolumes {
1077
		if err := os.MkdirAll(expVolDir, 0700); err != nil {
1078
			return fmt.Errorf("creating volumes export directory %q: %w", expVolDir, err)
1079
		}
1080

1081
		for _, v := range c.config.NamedVolumes {
1082
			volumeTarFilePath := filepath.Join(metadata.CheckpointVolumesDirectory, v.Name+".tar")
1083
			volumeTarFileFullPath := filepath.Join(c.bundlePath(), volumeTarFilePath)
1084

1085
			volumeTarFile, err := os.Create(volumeTarFileFullPath)
1086
			if err != nil {
1087
				return fmt.Errorf("creating %q: %w", volumeTarFileFullPath, err)
1088
			}
1089

1090
			volume, err := c.runtime.GetVolume(v.Name)
1091
			if err != nil {
1092
				return err
1093
			}
1094

1095
			mp, err := volume.MountPoint()
1096
			if err != nil {
1097
				return err
1098
			}
1099
			if mp == "" {
1100
				return fmt.Errorf("volume %s is not mounted, cannot export: %w", volume.Name(), define.ErrInternal)
1101
			}
1102

1103
			input, err := archive.TarWithOptions(mp, &archive.TarOptions{
1104
				Compression:      archive.Uncompressed,
1105
				IncludeSourceDir: true,
1106
			})
1107
			if err != nil {
1108
				return fmt.Errorf("reading volume directory %q: %w", v.Dest, err)
1109
			}
1110

1111
			_, err = io.Copy(volumeTarFile, input)
1112
			if err != nil {
1113
				return err
1114
			}
1115
			volumeTarFile.Close()
1116

1117
			includeFiles = append(includeFiles, volumeTarFilePath)
1118
		}
1119
	}
1120

1121
	input, err := archive.TarWithOptions(c.bundlePath(), &archive.TarOptions{
1122
		Compression:      options.Compression,
1123
		IncludeSourceDir: true,
1124
		IncludeFiles:     includeFiles,
1125
	})
1126

1127
	if err != nil {
1128
		return fmt.Errorf("reading checkpoint directory %q: %w", c.ID(), err)
1129
	}
1130

1131
	outFile, err := os.Create(options.TargetFile)
1132
	if err != nil {
1133
		return fmt.Errorf("creating checkpoint export file %q: %w", options.TargetFile, err)
1134
	}
1135
	defer outFile.Close()
1136

1137
	if err := os.Chmod(options.TargetFile, 0600); err != nil {
1138
		return err
1139
	}
1140

1141
	_, err = io.Copy(outFile, input)
1142
	if err != nil {
1143
		return err
1144
	}
1145

1146
	for _, file := range addToTarFiles {
1147
		os.Remove(filepath.Join(c.bundlePath(), file))
1148
	}
1149

1150
	if !options.IgnoreVolumes {
1151
		os.RemoveAll(expVolDir)
1152
	}
1153

1154
	return nil
1155
}
1156

1157
func (c *Container) checkpointRestoreSupported(version int) error {
1158
	if err := criu.CheckForCriu(version); err != nil {
1159
		return err
1160
	}
1161
	if !c.ociRuntime.SupportsCheckpoint() {
1162
		return errors.New("configured runtime does not support checkpoint/restore")
1163
	}
1164
	return nil
1165
}
1166

1167
func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointOptions) (*define.CRIUCheckpointRestoreStatistics, int64, error) {
1168
	if err := c.checkpointRestoreSupported(criu.MinCriuVersion); err != nil {
1169
		return nil, 0, err
1170
	}
1171

1172
	if c.state.State != define.ContainerStateRunning {
1173
		return nil, 0, fmt.Errorf("%q is not running, cannot checkpoint: %w", c.state.State, define.ErrCtrStateInvalid)
1174
	}
1175

1176
	if c.AutoRemove() && options.TargetFile == "" {
1177
		return nil, 0, errors.New("cannot checkpoint containers that have been started with '--rm' unless '--export' is used")
1178
	}
1179

1180
	if err := c.resolveCheckpointImageName(&options); err != nil {
1181
		return nil, 0, err
1182
	}
1183

1184
	if err := crutils.CRCreateFileWithLabel(c.bundlePath(), "dump.log", c.MountLabel()); err != nil {
1185
		return nil, 0, err
1186
	}
1187

1188
	// Setting CheckpointLog early in case there is a failure.
1189
	c.state.CheckpointLog = path.Join(c.bundlePath(), "dump.log")
1190
	c.state.CheckpointPath = c.CheckpointPath()
1191

1192
	runtimeCheckpointDuration, err := c.ociRuntime.CheckpointContainer(c, options)
1193
	if err != nil {
1194
		return nil, 0, err
1195
	}
1196

1197
	// Keep the content of /dev/shm directory
1198
	if c.config.ShmDir != "" && c.state.BindMounts["/dev/shm"] == c.config.ShmDir {
1199
		shmDirTarFileFullPath := filepath.Join(c.bundlePath(), metadata.DevShmCheckpointTar)
1200

1201
		shmDirTarFile, err := os.Create(shmDirTarFileFullPath)
1202
		if err != nil {
1203
			return nil, 0, err
1204
		}
1205
		defer shmDirTarFile.Close()
1206

1207
		input, err := archive.TarWithOptions(c.config.ShmDir, &archive.TarOptions{
1208
			Compression:      archive.Uncompressed,
1209
			IncludeSourceDir: true,
1210
		})
1211
		if err != nil {
1212
			return nil, 0, err
1213
		}
1214

1215
		if _, err = io.Copy(shmDirTarFile, input); err != nil {
1216
			return nil, 0, err
1217
		}
1218
	}
1219

1220
	// Save network.status. This is needed to restore the container with
1221
	// the same IP. Currently limited to one IP address in a container
1222
	// with one interface.
1223
	// FIXME: will this break something?
1224
	if _, err := metadata.WriteJSONFile(c.getNetworkStatus(), c.bundlePath(), metadata.NetworkStatusFile); err != nil {
1225
		return nil, 0, err
1226
	}
1227

1228
	defer c.newContainerEvent(events.Checkpoint)
1229

1230
	// There is a bug from criu: https://github.com/checkpoint-restore/criu/issues/116
1231
	// We have to change the symbolic link from absolute path to relative path
1232
	if options.WithPrevious {
1233
		os.Remove(path.Join(c.CheckpointPath(), "parent"))
1234
		if err := os.Symlink("../pre-checkpoint", path.Join(c.CheckpointPath(), "parent")); err != nil {
1235
			return nil, 0, err
1236
		}
1237
	}
1238

1239
	if options.TargetFile != "" {
1240
		if err := c.exportCheckpoint(options); err != nil {
1241
			return nil, 0, err
1242
		}
1243
	} else {
1244
		if err := c.createCheckpointImage(ctx, options); err != nil {
1245
			return nil, 0, err
1246
		}
1247
	}
1248

1249
	logrus.Debugf("Checkpointed container %s", c.ID())
1250

1251
	if !options.KeepRunning && !options.PreCheckPoint {
1252
		c.state.State = define.ContainerStateStopped
1253
		c.state.Checkpointed = true
1254
		c.state.CheckpointedTime = time.Now()
1255
		c.state.Restored = false
1256
		c.state.RestoredTime = time.Time{}
1257

1258
		// Clean up Storage and Network
1259
		if err := c.cleanup(ctx); err != nil {
1260
			return nil, 0, err
1261
		}
1262
	}
1263

1264
	criuStatistics, err := func() (*define.CRIUCheckpointRestoreStatistics, error) {
1265
		if !options.PrintStats {
1266
			return nil, nil
1267
		}
1268
		statsDirectory, err := os.Open(c.bundlePath())
1269
		if err != nil {
1270
			return nil, fmt.Errorf("not able to open %q: %w", c.bundlePath(), err)
1271
		}
1272

1273
		dumpStatistics, err := stats.CriuGetDumpStats(statsDirectory)
1274
		if err != nil {
1275
			return nil, fmt.Errorf("displaying checkpointing statistics not possible: %w", err)
1276
		}
1277

1278
		return &define.CRIUCheckpointRestoreStatistics{
1279
			FreezingTime: dumpStatistics.GetFreezingTime(),
1280
			FrozenTime:   dumpStatistics.GetFrozenTime(),
1281
			MemdumpTime:  dumpStatistics.GetMemdumpTime(),
1282
			MemwriteTime: dumpStatistics.GetMemwriteTime(),
1283
			PagesScanned: dumpStatistics.GetPagesScanned(),
1284
			PagesWritten: dumpStatistics.GetPagesWritten(),
1285
		}, nil
1286
	}()
1287
	if err != nil {
1288
		return nil, 0, err
1289
	}
1290

1291
	if !options.Keep && !options.PreCheckPoint {
1292
		cleanup := []string{
1293
			"dump.log",
1294
			stats.StatsDump,
1295
			metadata.ConfigDumpFile,
1296
			metadata.SpecDumpFile,
1297
		}
1298
		for _, del := range cleanup {
1299
			file := filepath.Join(c.bundlePath(), del)
1300
			if err := os.Remove(file); err != nil {
1301
				logrus.Debugf("Unable to remove file %s", file)
1302
			}
1303
		}
1304
		// The file has been deleted. Do not mention it.
1305
		c.state.CheckpointLog = ""
1306
	}
1307

1308
	c.state.FinishedTime = time.Now()
1309
	return criuStatistics, runtimeCheckpointDuration, c.save()
1310
}
1311

1312
func (c *Container) generateContainerSpec() error {
1313
	// Make sure the newly created config.json exists on disk
1314

1315
	// NewFromSpec() is deprecated according to its comment
1316
	// however the recommended replace just causes a nil map panic
1317
	g := generate.NewFromSpec(c.config.Spec)
1318

1319
	if err := c.saveSpec(g.Config); err != nil {
1320
		return fmt.Errorf("saving imported container specification for restore failed: %w", err)
1321
	}
1322

1323
	return nil
1324
}
1325

1326
func (c *Container) importCheckpointImage(ctx context.Context, imageID string) error {
1327
	img, _, err := c.Runtime().LibimageRuntime().LookupImage(imageID, nil)
1328
	if err != nil {
1329
		return err
1330
	}
1331

1332
	mountPoint, err := img.Mount(ctx, nil, "")
1333
	defer func() {
1334
		if err := c.unmount(true); err != nil {
1335
			logrus.Errorf("Failed to unmount container: %v", err)
1336
		}
1337
	}()
1338
	if err != nil {
1339
		return err
1340
	}
1341

1342
	// Import all checkpoint files except ConfigDumpFile and SpecDumpFile. We
1343
	// generate new container config files to enable to specifying a new
1344
	// container name.
1345
	checkpoint := []string{
1346
		"artifacts",
1347
		metadata.CheckpointDirectory,
1348
		metadata.CheckpointVolumesDirectory,
1349
		metadata.DevShmCheckpointTar,
1350
		metadata.RootFsDiffTar,
1351
		metadata.DeletedFilesFile,
1352
		metadata.PodOptionsFile,
1353
		metadata.PodDumpFile,
1354
	}
1355

1356
	for _, name := range checkpoint {
1357
		src := filepath.Join(mountPoint, name)
1358
		dst := filepath.Join(c.bundlePath(), name)
1359
		if err := archive.NewDefaultArchiver().CopyWithTar(src, dst); err != nil {
1360
			logrus.Debugf("Can't import '%s' from checkpoint image", name)
1361
		}
1362
	}
1363

1364
	return c.generateContainerSpec()
1365
}
1366

1367
func (c *Container) importCheckpointTar(input string) error {
1368
	if err := crutils.CRImportCheckpointWithoutConfig(c.bundlePath(), input); err != nil {
1369
		return err
1370
	}
1371

1372
	return c.generateContainerSpec()
1373
}
1374

1375
func (c *Container) importPreCheckpoint(input string) error {
1376
	archiveFile, err := os.Open(input)
1377
	if err != nil {
1378
		return fmt.Errorf("failed to open pre-checkpoint archive for import: %w", err)
1379
	}
1380

1381
	defer archiveFile.Close()
1382

1383
	err = archive.Untar(archiveFile, c.bundlePath(), nil)
1384
	if err != nil {
1385
		return fmt.Errorf("unpacking of pre-checkpoint archive %s failed: %w", input, err)
1386
	}
1387
	return nil
1388
}
1389

1390
func (c *Container) restore(ctx context.Context, options ContainerCheckpointOptions) (criuStatistics *define.CRIUCheckpointRestoreStatistics, runtimeRestoreDuration int64, retErr error) {
1391
	minCriuVersion := func() int {
1392
		if options.Pod == "" {
1393
			return criu.MinCriuVersion
1394
		}
1395
		return criu.PodCriuVersion
1396
	}()
1397
	if err := c.checkpointRestoreSupported(minCriuVersion); err != nil {
1398
		return nil, 0, err
1399
	}
1400

1401
	if options.Pod != "" && !crutils.CRRuntimeSupportsPodCheckpointRestore(c.ociRuntime.Path()) {
1402
		return nil, 0, fmt.Errorf("runtime %s does not support pod restore", c.ociRuntime.Path())
1403
	}
1404

1405
	if !c.ensureState(define.ContainerStateConfigured, define.ContainerStateExited) {
1406
		return nil, 0, fmt.Errorf("container %s is running or paused, cannot restore: %w", c.ID(), define.ErrCtrStateInvalid)
1407
	}
1408

1409
	if options.ImportPrevious != "" {
1410
		if err := c.importPreCheckpoint(options.ImportPrevious); err != nil {
1411
			return nil, 0, err
1412
		}
1413
	}
1414

1415
	if options.TargetFile != "" {
1416
		if err := c.importCheckpointTar(options.TargetFile); err != nil {
1417
			return nil, 0, err
1418
		}
1419
	} else if options.CheckpointImageID != "" {
1420
		if err := c.importCheckpointImage(ctx, options.CheckpointImageID); err != nil {
1421
			return nil, 0, err
1422
		}
1423
	}
1424

1425
	// Let's try to stat() CRIU's inventory file. If it does not exist, it makes
1426
	// no sense to try a restore. This is a minimal check if a checkpoint exists.
1427
	if err := fileutils.Exists(filepath.Join(c.CheckpointPath(), "inventory.img")); errors.Is(err, fs.ErrNotExist) {
1428
		return nil, 0, fmt.Errorf("a complete checkpoint for this container cannot be found, cannot restore: %w", err)
1429
	}
1430

1431
	if err := crutils.CRCreateFileWithLabel(c.bundlePath(), "restore.log", c.MountLabel()); err != nil {
1432
		return nil, 0, err
1433
	}
1434

1435
	// Setting RestoreLog early in case there is a failure.
1436
	c.state.RestoreLog = path.Join(c.bundlePath(), "restore.log")
1437
	c.state.CheckpointPath = c.CheckpointPath()
1438

1439
	if options.IgnoreStaticIP || options.IgnoreStaticMAC {
1440
		networks, err := c.networks()
1441
		if err != nil {
1442
			return nil, 0, err
1443
		}
1444

1445
		for net, opts := range networks {
1446
			if options.IgnoreStaticIP {
1447
				opts.StaticIPs = nil
1448
			}
1449
			if options.IgnoreStaticMAC {
1450
				opts.StaticMAC = nil
1451
			}
1452
			if err := c.runtime.state.NetworkModify(c, net, opts); err != nil {
1453
				return nil, 0, fmt.Errorf("failed to rewrite network config: %w", err)
1454
			}
1455
		}
1456
	}
1457

1458
	// Read network configuration from checkpoint
1459
	var netStatus map[string]types.StatusBlock
1460
	_, err := metadata.ReadJSONFile(&netStatus, c.bundlePath(), metadata.NetworkStatusFile)
1461
	if err != nil {
1462
		logrus.Infof("Failed to unmarshal network status, cannot restore the same ip/mac: %v", err)
1463
	}
1464
	// If the restored container should get a new name, the IP address of
1465
	// the container will not be restored. This assumes that if a new name is
1466
	// specified, the container is restored multiple times.
1467
	// TODO: This implicit restoring with or without IP depending on an
1468
	//       unrelated restore parameter (--name) does not seem like the
1469
	//       best solution.
1470
	if err == nil && options.Name == "" && (!options.IgnoreStaticIP || !options.IgnoreStaticMAC) {
1471
		// The file with the network.status does exist. Let's restore the
1472
		// container with the same networks settings as during checkpointing.
1473
		networkOpts, err := c.networks()
1474
		if err != nil {
1475
			return nil, 0, err
1476
		}
1477

1478
		netOpts := make(map[string]types.PerNetworkOptions, len(netStatus))
1479
		for network, perNetOpts := range networkOpts {
1480
			// unset mac and ips before we start adding the ones from the status
1481
			perNetOpts.StaticMAC = nil
1482
			perNetOpts.StaticIPs = nil
1483
			for name, netInt := range netStatus[network].Interfaces {
1484
				perNetOpts.InterfaceName = name
1485
				if !options.IgnoreStaticMAC {
1486
					perNetOpts.StaticMAC = netInt.MacAddress
1487
				}
1488
				if !options.IgnoreStaticIP {
1489
					for _, netAddress := range netInt.Subnets {
1490
						perNetOpts.StaticIPs = append(perNetOpts.StaticIPs, netAddress.IPNet.IP)
1491
					}
1492
				}
1493
				// Normally interfaces have a length of 1, only for some special cni configs we could get more.
1494
				// For now just use the first interface to get the ips this should be good enough for most cases.
1495
				break
1496
			}
1497
			netOpts[network] = perNetOpts
1498
		}
1499
		c.perNetworkOpts = netOpts
1500
	}
1501

1502
	defer func() {
1503
		if retErr != nil {
1504
			if err := c.cleanup(ctx); err != nil {
1505
				logrus.Errorf("Cleaning up container %s: %v", c.ID(), err)
1506
			}
1507
		}
1508
	}()
1509

1510
	if err := c.prepare(); err != nil {
1511
		return nil, 0, err
1512
	}
1513

1514
	// Read config
1515
	jsonPath := filepath.Join(c.bundlePath(), "config.json")
1516
	logrus.Debugf("generate.NewFromFile at %v", jsonPath)
1517
	g, err := generate.NewFromFile(jsonPath)
1518
	if err != nil {
1519
		logrus.Debugf("generate.NewFromFile failed with %v", err)
1520
		return nil, 0, err
1521
	}
1522

1523
	// Restoring from an import means that we are doing migration
1524
	if options.TargetFile != "" || options.CheckpointImageID != "" {
1525
		g.SetRootPath(c.state.Mountpoint)
1526
	}
1527

1528
	// We want to have the same network namespace as before.
1529
	if err := c.addNetworkNamespace(&g); err != nil {
1530
		return nil, 0, err
1531
	}
1532

1533
	if options.Pod != "" {
1534
		// Running in a Pod means that we have to change all namespace settings to
1535
		// the ones from the infrastructure container.
1536
		pod, err := c.runtime.LookupPod(options.Pod)
1537
		if err != nil {
1538
			return nil, 0, fmt.Errorf("pod %q cannot be retrieved: %w", options.Pod, err)
1539
		}
1540

1541
		infraContainer, err := pod.InfraContainer()
1542
		if err != nil {
1543
			return nil, 0, fmt.Errorf("cannot retrieved infra container from pod %q: %w", options.Pod, err)
1544
		}
1545

1546
		infraContainer.lock.Lock()
1547
		if err := infraContainer.syncContainer(); err != nil {
1548
			infraContainer.lock.Unlock()
1549
			return nil, 0, fmt.Errorf("syncing infrastructure container %s status: %w", infraContainer.ID(), err)
1550
		}
1551
		if infraContainer.state.State != define.ContainerStateRunning {
1552
			if err := infraContainer.initAndStart(ctx); err != nil {
1553
				infraContainer.lock.Unlock()
1554
				return nil, 0, fmt.Errorf("starting infrastructure container %s status: %w", infraContainer.ID(), err)
1555
			}
1556
		}
1557
		infraContainer.lock.Unlock()
1558

1559
		if c.config.IPCNsCtr != "" {
1560
			nsPath, err := infraContainer.namespacePath(IPCNS)
1561
			if err != nil {
1562
				return nil, 0, fmt.Errorf("cannot retrieve IPC namespace path for Pod %q: %w", options.Pod, err)
1563
			}
1564
			if err := g.AddOrReplaceLinuxNamespace(string(spec.IPCNamespace), nsPath); err != nil {
1565
				return nil, 0, err
1566
			}
1567
		}
1568

1569
		if c.config.NetNsCtr != "" {
1570
			nsPath, err := infraContainer.namespacePath(NetNS)
1571
			if err != nil {
1572
				return nil, 0, fmt.Errorf("cannot retrieve network namespace path for Pod %q: %w", options.Pod, err)
1573
			}
1574
			if err := g.AddOrReplaceLinuxNamespace(string(spec.NetworkNamespace), nsPath); err != nil {
1575
				return nil, 0, err
1576
			}
1577
		}
1578

1579
		if c.config.PIDNsCtr != "" {
1580
			nsPath, err := infraContainer.namespacePath(PIDNS)
1581
			if err != nil {
1582
				return nil, 0, fmt.Errorf("cannot retrieve PID namespace path for Pod %q: %w", options.Pod, err)
1583
			}
1584
			if err := g.AddOrReplaceLinuxNamespace(string(spec.PIDNamespace), nsPath); err != nil {
1585
				return nil, 0, err
1586
			}
1587
		}
1588

1589
		if c.config.UTSNsCtr != "" {
1590
			nsPath, err := infraContainer.namespacePath(UTSNS)
1591
			if err != nil {
1592
				return nil, 0, fmt.Errorf("cannot retrieve UTS namespace path for Pod %q: %w", options.Pod, err)
1593
			}
1594
			if err := g.AddOrReplaceLinuxNamespace(string(spec.UTSNamespace), nsPath); err != nil {
1595
				return nil, 0, err
1596
			}
1597
		}
1598

1599
		if c.config.CgroupNsCtr != "" {
1600
			nsPath, err := infraContainer.namespacePath(CgroupNS)
1601
			if err != nil {
1602
				return nil, 0, fmt.Errorf("cannot retrieve Cgroup namespace path for Pod %q: %w", options.Pod, err)
1603
			}
1604
			if err := g.AddOrReplaceLinuxNamespace(string(spec.CgroupNamespace), nsPath); err != nil {
1605
				return nil, 0, err
1606
			}
1607
		}
1608
	}
1609

1610
	if err := c.makeBindMounts(); err != nil {
1611
		return nil, 0, err
1612
	}
1613

1614
	if options.TargetFile != "" || options.CheckpointImageID != "" {
1615
		for dstPath, srcPath := range c.state.BindMounts {
1616
			newMount := spec.Mount{
1617
				Type:        define.TypeBind,
1618
				Source:      srcPath,
1619
				Destination: dstPath,
1620
				Options:     []string{define.TypeBind, "private"},
1621
			}
1622
			if c.IsReadOnly() && (dstPath != "/dev/shm" || !c.config.ReadWriteTmpfs) {
1623
				newMount.Options = append(newMount.Options, "ro", "nosuid", "noexec", "nodev")
1624
			}
1625
			if dstPath == "/dev/shm" && c.state.BindMounts["/dev/shm"] == c.config.ShmDir {
1626
				newMount.Options = append(newMount.Options, "nosuid", "noexec", "nodev")
1627
			}
1628
			if !MountExists(g.Mounts(), dstPath) {
1629
				g.AddMount(newMount)
1630
			}
1631
		}
1632
	}
1633

1634
	// Restore /dev/shm content
1635
	if c.config.ShmDir != "" && c.state.BindMounts["/dev/shm"] == c.config.ShmDir {
1636
		shmDirTarFileFullPath := filepath.Join(c.bundlePath(), metadata.DevShmCheckpointTar)
1637
		if err := fileutils.Exists(shmDirTarFileFullPath); err != nil {
1638
			logrus.Debug("Container checkpoint doesn't contain dev/shm: ", err.Error())
1639
		} else {
1640
			shmDirTarFile, err := os.Open(shmDirTarFileFullPath)
1641
			if err != nil {
1642
				return nil, 0, err
1643
			}
1644
			defer shmDirTarFile.Close()
1645

1646
			if err := archive.UntarUncompressed(shmDirTarFile, c.config.ShmDir, nil); err != nil {
1647
				return nil, 0, err
1648
			}
1649
		}
1650
	}
1651

1652
	// Cleanup for a working restore.
1653
	if err := c.removeConmonFiles(); err != nil {
1654
		return nil, 0, err
1655
	}
1656

1657
	// Save the OCI spec to disk
1658
	if err := c.saveSpec(g.Config); err != nil {
1659
		return nil, 0, err
1660
	}
1661

1662
	// When restoring from an imported archive, allow restoring the content of volumes.
1663
	// Volumes are created in setupContainer()
1664
	if !options.IgnoreVolumes && (options.TargetFile != "" || options.CheckpointImageID != "") {
1665
		for _, v := range c.config.NamedVolumes {
1666
			volumeFilePath := filepath.Join(c.bundlePath(), metadata.CheckpointVolumesDirectory, v.Name+".tar")
1667

1668
			volumeFile, err := os.Open(volumeFilePath)
1669
			if err != nil {
1670
				return nil, 0, fmt.Errorf("failed to open volume file %s: %w", volumeFilePath, err)
1671
			}
1672
			defer volumeFile.Close()
1673

1674
			volume, err := c.runtime.GetVolume(v.Name)
1675
			if err != nil {
1676
				return nil, 0, fmt.Errorf("failed to retrieve volume %s: %w", v.Name, err)
1677
			}
1678

1679
			mountPoint, err := volume.MountPoint()
1680
			if err != nil {
1681
				return nil, 0, err
1682
			}
1683
			if mountPoint == "" {
1684
				return nil, 0, fmt.Errorf("unable to import volume %s as it is not mounted: %w", volume.Name(), err)
1685
			}
1686
			if err := archive.UntarUncompressed(volumeFile, mountPoint, nil); err != nil {
1687
				return nil, 0, fmt.Errorf("failed to extract volume %s to %s: %w", volumeFilePath, mountPoint, err)
1688
			}
1689
		}
1690
	}
1691

1692
	// Before actually restarting the container, apply the root file-system changes
1693
	if !options.IgnoreRootfs {
1694
		if err := crutils.CRApplyRootFsDiffTar(c.bundlePath(), c.state.Mountpoint); err != nil {
1695
			return nil, 0, err
1696
		}
1697

1698
		if err := crutils.CRRemoveDeletedFiles(c.ID(), c.bundlePath(), c.state.Mountpoint); err != nil {
1699
			return nil, 0, err
1700
		}
1701
	}
1702

1703
	runtimeRestoreDuration, err = c.ociRuntime.CreateContainer(c, &options)
1704
	if err != nil {
1705
		return nil, 0, err
1706
	}
1707

1708
	criuStatistics, err = func() (*define.CRIUCheckpointRestoreStatistics, error) {
1709
		if !options.PrintStats {
1710
			return nil, nil
1711
		}
1712
		statsDirectory, err := os.Open(c.bundlePath())
1713
		if err != nil {
1714
			return nil, fmt.Errorf("not able to open %q: %w", c.bundlePath(), err)
1715
		}
1716

1717
		restoreStatistics, err := stats.CriuGetRestoreStats(statsDirectory)
1718
		if err != nil {
1719
			return nil, fmt.Errorf("displaying restore statistics not possible: %w", err)
1720
		}
1721

1722
		return &define.CRIUCheckpointRestoreStatistics{
1723
			PagesCompared:   restoreStatistics.GetPagesCompared(),
1724
			PagesSkippedCow: restoreStatistics.GetPagesSkippedCow(),
1725
			ForkingTime:     restoreStatistics.GetForkingTime(),
1726
			RestoreTime:     restoreStatistics.GetRestoreTime(),
1727
			PagesRestored:   restoreStatistics.GetPagesRestored(),
1728
		}, nil
1729
	}()
1730
	if err != nil {
1731
		return nil, 0, err
1732
	}
1733

1734
	logrus.Debugf("Restored container %s", c.ID())
1735

1736
	c.state.State = define.ContainerStateRunning
1737
	c.state.Checkpointed = false
1738
	c.state.Restored = true
1739
	c.state.CheckpointedTime = time.Time{}
1740
	c.state.RestoredTime = time.Now()
1741

1742
	if !options.Keep {
1743
		// Delete all checkpoint related files. At this point, in theory, all files
1744
		// should exist. Still ignoring errors for now as the container should be
1745
		// restored and running. Not erroring out just because some cleanup operation
1746
		// failed. Starting with the checkpoint directory
1747
		err = os.RemoveAll(c.CheckpointPath())
1748
		if err != nil {
1749
			logrus.Debugf("Non-fatal: removal of checkpoint directory (%s) failed: %v", c.CheckpointPath(), err)
1750
		}
1751
		c.state.CheckpointPath = ""
1752
		err = os.RemoveAll(c.PreCheckPointPath())
1753
		if err != nil {
1754
			logrus.Debugf("Non-fatal: removal of pre-checkpoint directory (%s) failed: %v", c.PreCheckPointPath(), err)
1755
		}
1756
		err = os.RemoveAll(c.CheckpointVolumesPath())
1757
		if err != nil {
1758
			logrus.Debugf("Non-fatal: removal of checkpoint volumes directory (%s) failed: %v", c.CheckpointVolumesPath(), err)
1759
		}
1760
		cleanup := [...]string{
1761
			"restore.log",
1762
			"dump.log",
1763
			stats.StatsDump,
1764
			stats.StatsRestore,
1765
			metadata.DevShmCheckpointTar,
1766
			metadata.NetworkStatusFile,
1767
			metadata.RootFsDiffTar,
1768
			metadata.DeletedFilesFile,
1769
		}
1770
		for _, del := range cleanup {
1771
			file := filepath.Join(c.bundlePath(), del)
1772
			err = os.Remove(file)
1773
			if err != nil {
1774
				logrus.Debugf("Non-fatal: removal of checkpoint file (%s) failed: %v", file, err)
1775
			}
1776
		}
1777
		c.state.CheckpointLog = ""
1778
		c.state.RestoreLog = ""
1779
	}
1780

1781
	return criuStatistics, runtimeRestoreDuration, c.save()
1782
}
1783

1784
// Retrieves a container's "root" net namespace container dependency.
1785
func (c *Container) getRootNetNsDepCtr() (depCtr *Container, err error) {
1786
	containersVisited := map[string]int{c.config.ID: 1}
1787
	nextCtr := c.config.NetNsCtr
1788
	for nextCtr != "" {
1789
		// Make sure we aren't in a loop
1790
		if _, visited := containersVisited[nextCtr]; visited {
1791
			return nil, errors.New("loop encountered while determining net namespace container")
1792
		}
1793
		containersVisited[nextCtr] = 1
1794

1795
		depCtr, err = c.runtime.state.Container(nextCtr)
1796
		if err != nil {
1797
			return nil, fmt.Errorf("fetching dependency %s of container %s: %w", c.config.NetNsCtr, c.ID(), err)
1798
		}
1799
		// This should never happen without an error
1800
		if depCtr == nil {
1801
			break
1802
		}
1803
		nextCtr = depCtr.config.NetNsCtr
1804
	}
1805

1806
	if depCtr == nil {
1807
		return nil, errors.New("unexpected error depCtr is nil without reported error from runtime state")
1808
	}
1809
	return depCtr, nil
1810
}
1811

1812
// Ensure standard bind mounts are mounted into all root directories (including chroot directories)
1813
func (c *Container) mountIntoRootDirs(mountName string, mountPath string) error {
1814
	c.state.BindMounts[mountName] = mountPath
1815

1816
	for _, chrootDir := range c.config.ChrootDirs {
1817
		c.state.BindMounts[filepath.Join(chrootDir, mountName)] = mountPath
1818
	}
1819

1820
	return nil
1821
}
1822

1823
// Make standard bind mounts to include in the container
1824
func (c *Container) makeBindMounts() error {
1825
	if err := idtools.SafeChown(c.state.RunDir, c.RootUID(), c.RootGID()); err != nil {
1826
		return fmt.Errorf("cannot chown run directory: %w", err)
1827
	}
1828

1829
	if c.state.BindMounts == nil {
1830
		c.state.BindMounts = make(map[string]string)
1831
	}
1832
	netDisabled, err := c.NetworkDisabled()
1833
	if err != nil {
1834
		return err
1835
	}
1836

1837
	if !netDisabled {
1838
		// If /etc/resolv.conf and /etc/hosts exist, delete them so we
1839
		// will recreate. Only do this if we aren't sharing them with
1840
		// another container.
1841
		if c.config.NetNsCtr == "" {
1842
			if resolvePath, ok := c.state.BindMounts[resolvconf.DefaultResolvConf]; ok {
1843
				if err := os.Remove(resolvePath); err != nil && !os.IsNotExist(err) {
1844
					return fmt.Errorf("container %s: %w", c.ID(), err)
1845
				}
1846
				delete(c.state.BindMounts, resolvconf.DefaultResolvConf)
1847
			}
1848
			if hostsPath, ok := c.state.BindMounts[config.DefaultHostsFile]; ok {
1849
				if err := os.Remove(hostsPath); err != nil && !os.IsNotExist(err) {
1850
					return fmt.Errorf("container %s: %w", c.ID(), err)
1851
				}
1852
				delete(c.state.BindMounts, config.DefaultHostsFile)
1853
			}
1854
		}
1855

1856
		if c.config.NetNsCtr != "" && (!c.config.UseImageResolvConf || !c.config.UseImageHosts) {
1857
			// We share a net namespace.
1858
			// We want /etc/resolv.conf and /etc/hosts from the
1859
			// other container. Unless we're not creating both of
1860
			// them.
1861
			depCtr, err := c.getRootNetNsDepCtr()
1862
			if err != nil {
1863
				return fmt.Errorf("fetching network namespace dependency container for container %s: %w", c.ID(), err)
1864
			}
1865

1866
			// We need that container's bind mounts
1867
			bindMounts, err := depCtr.BindMounts()
1868
			if err != nil {
1869
				return fmt.Errorf("fetching bind mounts from dependency %s of container %s: %w", depCtr.ID(), c.ID(), err)
1870
			}
1871

1872
			// The other container may not have a resolv.conf or /etc/hosts
1873
			// If it doesn't, don't copy them
1874
			resolvPath, exists := bindMounts[resolvconf.DefaultResolvConf]
1875
			if !c.config.UseImageResolvConf && exists {
1876
				err := c.mountIntoRootDirs(resolvconf.DefaultResolvConf, resolvPath)
1877

1878
				if err != nil {
1879
					return fmt.Errorf("assigning mounts to container %s: %w", c.ID(), err)
1880
				}
1881
			}
1882

1883
			// check if dependency container has an /etc/hosts file.
1884
			// It may not have one, so only use it if it does.
1885
			hostsPath, exists := bindMounts[config.DefaultHostsFile]
1886
			if !c.config.UseImageHosts && exists {
1887
				// we cannot use the dependency container lock due ABBA deadlocks in cleanup()
1888
				lock, err := lockfile.GetLockFile(hostsPath)
1889
				if err != nil {
1890
					return fmt.Errorf("failed to lock hosts file: %w", err)
1891
				}
1892
				lock.Lock()
1893

1894
				// add the newly added container to the hosts file
1895
				// we always use 127.0.0.1 as ip since they have the same netns
1896
				err = etchosts.Add(hostsPath, getLocalhostHostEntry(c))
1897
				lock.Unlock()
1898
				if err != nil {
1899
					return fmt.Errorf("creating hosts file for container %s which depends on container %s: %w", c.ID(), depCtr.ID(), err)
1900
				}
1901

1902
				// finally, save it in the new container
1903
				err = c.mountIntoRootDirs(config.DefaultHostsFile, hostsPath)
1904
				if err != nil {
1905
					return fmt.Errorf("assigning mounts to container %s: %w", c.ID(), err)
1906
				}
1907
			}
1908

1909
			if !hasCurrentUserMapped(c) {
1910
				if err := makeAccessible(resolvPath, c.RootUID(), c.RootGID()); err != nil {
1911
					return err
1912
				}
1913
				if err := makeAccessible(hostsPath, c.RootUID(), c.RootGID()); err != nil {
1914
					return err
1915
				}
1916
			}
1917
		} else {
1918
			if !c.config.UseImageResolvConf {
1919
				if err := c.createResolvConf(); err != nil {
1920
					return fmt.Errorf("creating resolv.conf for container %s: %w", c.ID(), err)
1921
				}
1922
			}
1923

1924
			if !c.config.UseImageHosts {
1925
				if err := c.createHostsFile(); err != nil {
1926
					return fmt.Errorf("creating hosts file for container %s: %w", c.ID(), err)
1927
				}
1928
			}
1929
		}
1930

1931
		if c.state.BindMounts[config.DefaultHostsFile] != "" {
1932
			if err := c.relabel(c.state.BindMounts[config.DefaultHostsFile], c.config.MountLabel, true); err != nil {
1933
				return err
1934
			}
1935
		}
1936

1937
		if c.state.BindMounts[resolvconf.DefaultResolvConf] != "" {
1938
			if err := c.relabel(c.state.BindMounts[resolvconf.DefaultResolvConf], c.config.MountLabel, true); err != nil {
1939
				return err
1940
			}
1941
		}
1942
	} else if !c.config.UseImageHosts && c.state.BindMounts[config.DefaultHostsFile] == "" {
1943
		if err := c.createHostsFile(); err != nil {
1944
			return fmt.Errorf("creating hosts file for container %s: %w", c.ID(), err)
1945
		}
1946
	}
1947

1948
	if c.config.ShmDir != "" {
1949
		// If ShmDir has a value SHM is always added when we mount the container
1950
		c.state.BindMounts["/dev/shm"] = c.config.ShmDir
1951
	}
1952

1953
	if c.config.Passwd == nil || *c.config.Passwd {
1954
		newPasswd, newGroup, err := c.generatePasswdAndGroup()
1955
		if err != nil {
1956
			return fmt.Errorf("creating temporary passwd file for container %s: %w", c.ID(), err)
1957
		}
1958
		if newPasswd != "" {
1959
			// Make /etc/passwd
1960
			// If it already exists, delete so we can recreate
1961
			delete(c.state.BindMounts, "/etc/passwd")
1962
			c.state.BindMounts["/etc/passwd"] = newPasswd
1963
		}
1964
		if newGroup != "" {
1965
			// Make /etc/group
1966
			// If it already exists, delete so we can recreate
1967
			delete(c.state.BindMounts, "/etc/group")
1968
			c.state.BindMounts["/etc/group"] = newGroup
1969
		}
1970
	}
1971

1972
	runPath, err := c.getPlatformRunPath()
1973
	if err != nil {
1974
		return fmt.Errorf("cannot determine run directory for container: %w", err)
1975
	}
1976
	containerenvPath := filepath.Join(runPath, ".containerenv")
1977

1978
	_, hasRunContainerenv := c.state.BindMounts[containerenvPath]
1979
	if !hasRunContainerenv {
1980
	Loop:
1981
		// check in the spec mounts
1982
		for _, m := range c.config.Spec.Mounts {
1983
			switch {
1984
			case m.Destination == containerenvPath:
1985
				hasRunContainerenv = true
1986
				break Loop
1987
			case m.Destination == runPath && m.Type != define.TypeTmpfs:
1988
				hasRunContainerenv = true
1989
				break Loop
1990
			}
1991
		}
1992
	}
1993

1994
	// Make .containerenv if it does not exist
1995
	if !hasRunContainerenv {
1996
		containerenv := c.runtime.graphRootMountedFlag(c.config.Spec.Mounts)
1997
		isRootless := 0
1998
		if rootless.IsRootless() {
1999
			isRootless = 1
2000
		}
2001
		imageID, imageName := c.Image()
2002

2003
		if c.Privileged() {
2004
			// Populate the .containerenv with container information
2005
			containerenv = fmt.Sprintf(`engine="podman-%s"
2006
name=%q
2007
id=%q
2008
image=%q
2009
imageid=%q
2010
rootless=%d
2011
%s`, version.Version.String(), c.Name(), c.ID(), imageName, imageID, isRootless, containerenv)
2012
		}
2013
		containerenvHostPath, err := c.writeStringToRundir(".containerenv", containerenv)
2014
		if err != nil {
2015
			return fmt.Errorf("creating containerenv file for container %s: %w", c.ID(), err)
2016
		}
2017
		c.state.BindMounts[containerenvPath] = containerenvHostPath
2018
	}
2019

2020
	// Add Subscription Mounts
2021
	subscriptionMounts := subscriptions.MountsWithUIDGID(c.config.MountLabel, c.state.RunDir, c.runtime.config.Containers.DefaultMountsFile, c.state.Mountpoint, c.RootUID(), c.RootGID(), rootless.IsRootless(), false)
2022
	for _, mount := range subscriptionMounts {
2023
		if _, ok := c.state.BindMounts[mount.Destination]; !ok {
2024
			c.state.BindMounts[mount.Destination] = mount.Source
2025
		}
2026
	}
2027

2028
	// Secrets are mounted by getting the secret data from the secrets manager,
2029
	// copying the data into the container's static dir,
2030
	// then mounting the copied dir into /run/secrets.
2031
	// The secrets mounting must come after subscription mounts, since subscription mounts
2032
	// creates the /run/secrets dir in the container where we mount as well.
2033
	if len(c.Secrets()) > 0 {
2034
		// create /run/secrets if subscriptions did not create
2035
		if err := c.createSecretMountDir(runPath); err != nil {
2036
			return fmt.Errorf("creating secrets mount: %w", err)
2037
		}
2038
		for _, secret := range c.Secrets() {
2039
			secretFileName := secret.Name
2040
			base := filepath.Join(runPath, "secrets")
2041
			if secret.Target != "" {
2042
				secretFileName = secret.Target
2043
				// If absolute path for target given remove base.
2044
				if filepath.IsAbs(secretFileName) {
2045
					base = ""
2046
				}
2047
			}
2048
			src := filepath.Join(c.config.SecretsPath, secret.Name)
2049
			dest := filepath.Join(base, secretFileName)
2050
			c.state.BindMounts[dest] = src
2051
		}
2052
	}
2053

2054
	return c.makePlatformBindMounts()
2055
}
2056

2057
// createResolvConf create the resolv.conf file and bind mount it
2058
func (c *Container) createResolvConf() error {
2059
	destPath := filepath.Join(c.state.RunDir, "resolv.conf")
2060
	f, err := os.Create(destPath)
2061
	if err != nil {
2062
		return err
2063
	}
2064
	f.Close()
2065
	return c.bindMountRootFile(destPath, resolvconf.DefaultResolvConf)
2066
}
2067

2068
// addResolvConf add resolv.conf entries
2069
func (c *Container) addResolvConf() error {
2070
	destPath, ok := c.state.BindMounts[resolvconf.DefaultResolvConf]
2071
	if !ok {
2072
		// no resolv.conf mount, do nothing
2073
		return nil
2074
	}
2075

2076
	var (
2077
		networkNameServers   []string
2078
		networkSearchDomains []string
2079
	)
2080

2081
	netStatus := c.getNetworkStatus()
2082
	for _, status := range netStatus {
2083
		if status.DNSServerIPs != nil {
2084
			for _, nsIP := range status.DNSServerIPs {
2085
				networkNameServers = append(networkNameServers, nsIP.String())
2086
			}
2087
			logrus.Debugf("Adding nameserver(s) from network status of '%q'", status.DNSServerIPs)
2088
		}
2089
		if status.DNSSearchDomains != nil {
2090
			networkSearchDomains = append(networkSearchDomains, status.DNSSearchDomains...)
2091
			logrus.Debugf("Adding search domain(s) from network status of '%q'", status.DNSSearchDomains)
2092
		}
2093
	}
2094

2095
	ipv6 := c.checkForIPv6(netStatus)
2096

2097
	networkBackend := c.runtime.config.Network.NetworkBackend
2098
	nameservers := make([]string, 0, len(c.runtime.config.Containers.DNSServers.Get())+len(c.config.DNSServer))
2099

2100
	// If NetworkBackend is `netavark` do not populate `/etc/resolv.conf`
2101
	// with custom dns server since after https://github.com/containers/netavark/pull/452
2102
	// netavark will always set required `nameservers` in StatusBlock and libpod
2103
	// will correctly populate `networkNameServers`. Also see https://github.com/containers/podman/issues/16172
2104

2105
	// Exception: Populate `/etc/resolv.conf` if container is not connected to any network
2106
	// with dns enabled then we do not get any nameservers back.
2107
	if networkBackend != string(types.Netavark) || len(networkNameServers) == 0 {
2108
		nameservers = append(nameservers, c.runtime.config.Containers.DNSServers.Get()...)
2109
		for _, ip := range c.config.DNSServer {
2110
			nameservers = append(nameservers, ip.String())
2111
		}
2112
	}
2113
	// If the user provided dns, it trumps all; then dns masq; then resolv.conf
2114
	keepHostServers := false
2115
	if len(nameservers) == 0 {
2116
		// when no network name servers or not netavark use host servers
2117
		// for aardvark dns we only want our single server in there
2118
		if len(networkNameServers) == 0 || networkBackend != string(types.Netavark) {
2119
			keepHostServers = true
2120
		}
2121
		// first add the nameservers from the networks status
2122
		nameservers = networkNameServers
2123

2124
		// pasta and slirp4netns have a built in DNS forwarder.
2125
		nameservers = c.addSpecialDNS(nameservers)
2126
	}
2127

2128
	// Set DNS search domains
2129
	search := networkSearchDomains
2130

2131
	if len(c.config.DNSSearch) > 0 || len(c.runtime.config.Containers.DNSSearches.Get()) > 0 {
2132
		customSearch := make([]string, 0, len(c.config.DNSSearch)+len(c.runtime.config.Containers.DNSSearches.Get()))
2133
		customSearch = append(customSearch, c.runtime.config.Containers.DNSSearches.Get()...)
2134
		customSearch = append(customSearch, c.config.DNSSearch...)
2135
		search = customSearch
2136
	}
2137

2138
	options := make([]string, 0, len(c.config.DNSOption)+len(c.runtime.config.Containers.DNSOptions.Get()))
2139
	options = append(options, c.runtime.config.Containers.DNSOptions.Get()...)
2140
	options = append(options, c.config.DNSOption...)
2141

2142
	var namespaces []spec.LinuxNamespace
2143
	if c.config.Spec.Linux != nil {
2144
		namespaces = c.config.Spec.Linux.Namespaces
2145
	}
2146

2147
	if err := resolvconf.New(&resolvconf.Params{
2148
		IPv6Enabled:     ipv6,
2149
		KeepHostServers: keepHostServers,
2150
		Nameservers:     nameservers,
2151
		Namespaces:      namespaces,
2152
		Options:         options,
2153
		Path:            destPath,
2154
		Searches:        search,
2155
	}); err != nil {
2156
		return fmt.Errorf("building resolv.conf for container %s: %w", c.ID(), err)
2157
	}
2158

2159
	return nil
2160
}
2161

2162
// Check if a container uses IPv6.
2163
func (c *Container) checkForIPv6(netStatus map[string]types.StatusBlock) bool {
2164
	for _, status := range netStatus {
2165
		for _, netInt := range status.Interfaces {
2166
			for _, netAddress := range netInt.Subnets {
2167
				// Note: only using To16() does not work since it also returns a valid ip for ipv4
2168
				if netAddress.IPNet.IP.To4() == nil && netAddress.IPNet.IP.To16() != nil {
2169
					return true
2170
				}
2171
			}
2172
		}
2173
	}
2174

2175
	if c.pastaResult != nil {
2176
		return c.pastaResult.IPv6
2177
	}
2178

2179
	return c.isSlirp4netnsIPv6()
2180
}
2181

2182
// Add a new nameserver to the container's resolv.conf, ensuring that it is the
2183
// first nameserver present.
2184
// Usable only with running containers.
2185
func (c *Container) addNameserver(ips []string) error {
2186
	// Take no action if container is not running.
2187
	if !c.ensureState(define.ContainerStateRunning, define.ContainerStateCreated) {
2188
		return nil
2189
	}
2190

2191
	// Do we have a resolv.conf at all?
2192
	path, ok := c.state.BindMounts[resolvconf.DefaultResolvConf]
2193
	if !ok {
2194
		return nil
2195
	}
2196

2197
	if err := resolvconf.Add(path, ips); err != nil {
2198
		return fmt.Errorf("adding new nameserver to container %s resolv.conf: %w", c.ID(), err)
2199
	}
2200

2201
	return nil
2202
}
2203

2204
// Remove an entry from the existing resolv.conf of the container.
2205
// Usable only with running containers.
2206
func (c *Container) removeNameserver(ips []string) error {
2207
	// Take no action if container is not running.
2208
	if !c.ensureState(define.ContainerStateRunning, define.ContainerStateCreated) {
2209
		return nil
2210
	}
2211

2212
	// Do we have a resolv.conf at all?
2213
	path, ok := c.state.BindMounts[resolvconf.DefaultResolvConf]
2214
	if !ok {
2215
		return nil
2216
	}
2217

2218
	if err := resolvconf.Remove(path, ips); err != nil {
2219
		return fmt.Errorf("removing nameservers from container %s resolv.conf: %w", c.ID(), err)
2220
	}
2221

2222
	return nil
2223
}
2224

2225
func getLocalhostHostEntry(c *Container) etchosts.HostEntries {
2226
	return etchosts.HostEntries{{IP: "127.0.0.1", Names: []string{c.Hostname(), c.config.Name}}}
2227
}
2228

2229
// getHostsEntries returns the container ip host entries for the correct netmode
2230
func (c *Container) getHostsEntries() (etchosts.HostEntries, error) {
2231
	var entries etchosts.HostEntries
2232
	names := []string{c.Hostname(), c.config.Name}
2233
	switch {
2234
	case c.config.NetMode.IsBridge():
2235
		entries = etchosts.GetNetworkHostEntries(c.state.NetworkStatus, names...)
2236
	case c.config.NetMode.IsPasta():
2237
		// this should never be the case but check just to be sure and not panic
2238
		if len(c.pastaResult.IPAddresses) > 0 {
2239
			entries = etchosts.HostEntries{{IP: c.pastaResult.IPAddresses[0].String(), Names: names}}
2240
		}
2241
	case c.config.NetMode.IsSlirp4netns():
2242
		ip, err := getSlirp4netnsIP(c.slirp4netnsSubnet)
2243
		if err != nil {
2244
			return nil, err
2245
		}
2246
		entries = etchosts.HostEntries{{IP: ip.String(), Names: names}}
2247
	default:
2248
		if c.hasNetNone() {
2249
			entries = etchosts.HostEntries{{IP: "127.0.0.1", Names: names}}
2250
		}
2251
	}
2252
	return entries, nil
2253
}
2254

2255
func (c *Container) createHostsFile() error {
2256
	targetFile := filepath.Join(c.state.RunDir, "hosts")
2257
	f, err := os.Create(targetFile)
2258
	if err != nil {
2259
		return err
2260
	}
2261
	f.Close()
2262
	return c.bindMountRootFile(targetFile, config.DefaultHostsFile)
2263
}
2264

2265
func (c *Container) addHosts() error {
2266
	targetFile, ok := c.state.BindMounts[config.DefaultHostsFile]
2267
	if !ok {
2268
		// no host file nothing to do
2269
		return nil
2270
	}
2271
	containerIPsEntries, err := c.getHostsEntries()
2272
	if err != nil {
2273
		return fmt.Errorf("failed to get container ip host entries: %w", err)
2274
	}
2275

2276
	// Consider container level BaseHostsFile configuration first.
2277
	// If it is empty, fallback to containers.conf level configuration.
2278
	baseHostsFileConf := c.config.BaseHostsFile
2279
	if baseHostsFileConf == "" {
2280
		baseHostsFileConf = c.runtime.config.Containers.BaseHostsFile
2281
	}
2282
	baseHostFile, err := etchosts.GetBaseHostFile(baseHostsFileConf, c.state.Mountpoint)
2283
	if err != nil {
2284
		return err
2285
	}
2286

2287
	var exclude []net.IP
2288
	if c.pastaResult != nil {
2289
		exclude = c.pastaResult.IPAddresses
2290
	}
2291

2292
	return etchosts.New(&etchosts.Params{
2293
		BaseFile:     baseHostFile,
2294
		ExtraHosts:   c.config.HostAdd,
2295
		ContainerIPs: containerIPsEntries,
2296
		HostContainersInternalIP: etchosts.GetHostContainersInternalIPExcluding(
2297
			c.runtime.config, c.state.NetworkStatus, c.runtime.network, exclude),
2298
		TargetFile: targetFile,
2299
	})
2300
}
2301

2302
// bindMountRootFile will chown and relabel the source file to make it usable in the container.
2303
// It will also add the path to the container bind mount map.
2304
// source is the path on the host, dest is the path in the container.
2305
func (c *Container) bindMountRootFile(source, dest string) error {
2306
	if err := idtools.SafeChown(source, c.RootUID(), c.RootGID()); err != nil {
2307
		return err
2308
	}
2309
	if err := c.relabel(source, c.MountLabel(), false); err != nil {
2310
		return err
2311
	}
2312

2313
	return c.mountIntoRootDirs(dest, source)
2314
}
2315

2316
// generateGroupEntry generates an entry or entries into /etc/group as
2317
// required by container configuration.
2318
// Generally speaking, we will make an entry under two circumstances:
2319
//  1. The container is started as a specific user:group, and that group is both
2320
//     numeric, and does not already exist in /etc/group.
2321
//  2. It is requested that Libpod add the group that launched Podman to
2322
//     /etc/group via AddCurrentUserPasswdEntry (though this does not trigger if
2323
//     the group in question already exists in /etc/passwd).
2324
//
2325
// Returns group entry (as a string that can be appended to /etc/group) and any
2326
// error that occurred.
2327
func (c *Container) generateGroupEntry() (string, error) {
2328
	groupString := ""
2329

2330
	// Things we *can't* handle: adding the user we added in
2331
	// generatePasswdEntry to any *existing* groups.
2332
	addedGID := 0
2333
	if c.config.AddCurrentUserPasswdEntry {
2334
		entry, gid, err := c.generateCurrentUserGroupEntry()
2335
		if err != nil {
2336
			return "", err
2337
		}
2338
		groupString += entry
2339
		addedGID = gid
2340
	}
2341
	if c.config.User != "" || c.config.GroupEntry != "" {
2342
		entry, err := c.generateUserGroupEntry(addedGID)
2343
		if err != nil {
2344
			return "", err
2345
		}
2346
		groupString += entry
2347
	}
2348

2349
	return groupString, nil
2350
}
2351

2352
// Make an entry in /etc/group for the group of the user running podman iff we
2353
// are rootless.
2354
func (c *Container) generateCurrentUserGroupEntry() (string, int, error) {
2355
	gid := rootless.GetRootlessGID()
2356
	if gid == 0 {
2357
		return "", 0, nil
2358
	}
2359

2360
	g, err := user.LookupGroupId(strconv.Itoa(gid))
2361
	if err != nil {
2362
		return "", 0, fmt.Errorf("failed to get current group: %w", err)
2363
	}
2364

2365
	// Look up group name to see if it exists in the image.
2366
	_, err = lookup.GetGroup(c.state.Mountpoint, g.Name)
2367
	if err != runcuser.ErrNoGroupEntries {
2368
		return "", 0, err
2369
	}
2370

2371
	// Look up GID to see if it exists in the image.
2372
	_, err = lookup.GetGroup(c.state.Mountpoint, g.Gid)
2373
	if err != runcuser.ErrNoGroupEntries {
2374
		return "", 0, err
2375
	}
2376

2377
	// We need to get the username of the rootless user so we can add it to
2378
	// the group.
2379
	username := ""
2380
	uid := rootless.GetRootlessUID()
2381
	if uid != 0 {
2382
		u, err := user.LookupId(strconv.Itoa(uid))
2383
		if err != nil {
2384
			return "", 0, fmt.Errorf("failed to get current user to make group entry: %w", err)
2385
		}
2386
		username = u.Username
2387
	}
2388

2389
	// Make the entry.
2390
	return fmt.Sprintf("%s:x:%s:%s\n", g.Name, g.Gid, username), gid, nil
2391
}
2392

2393
// Make an entry in /etc/group for the group the container was specified to run
2394
// as.
2395
func (c *Container) generateUserGroupEntry(addedGID int) (string, error) {
2396
	if c.config.User == "" && c.config.GroupEntry == "" {
2397
		return "", nil
2398
	}
2399

2400
	splitUser := strings.SplitN(c.config.User, ":", 2)
2401
	group := splitUser[0]
2402
	if len(splitUser) > 1 {
2403
		group = splitUser[1]
2404
	}
2405

2406
	gid, err := strconv.ParseUint(group, 10, 32)
2407
	if err != nil {
2408
		return "", nil //nolint: nilerr
2409
	}
2410

2411
	if addedGID != 0 && addedGID == int(gid) {
2412
		return "", nil
2413
	}
2414

2415
	// Check if the group already exists
2416
	g, err := lookup.GetGroup(c.state.Mountpoint, group)
2417
	if err != runcuser.ErrNoGroupEntries {
2418
		return "", err
2419
	}
2420

2421
	if c.config.GroupEntry != "" {
2422
		return c.groupEntry(g.Name, strconv.Itoa(g.Gid), g.List), nil
2423
	}
2424

2425
	return fmt.Sprintf("%d:x:%d:%s\n", gid, gid, splitUser[0]), nil
2426
}
2427

2428
func (c *Container) groupEntry(groupname, gid string, list []string) string {
2429
	s := c.config.GroupEntry
2430
	s = strings.ReplaceAll(s, "$GROUPNAME", groupname)
2431
	s = strings.ReplaceAll(s, "$GID", gid)
2432
	s = strings.ReplaceAll(s, "$USERLIST", strings.Join(list, ","))
2433
	return s + "\n"
2434
}
2435

2436
// generatePasswdEntry generates an entry or entries into /etc/passwd as
2437
// required by container configuration.
2438
// Generally speaking, we will make an entry under two circumstances:
2439
//  1. The container is started as a specific user who is not in /etc/passwd.
2440
//     This only triggers if the user is given as a *numeric* ID.
2441
//  2. It is requested that Libpod add the user that launched Podman to
2442
//     /etc/passwd via AddCurrentUserPasswdEntry (though this does not trigger if
2443
//     the user in question already exists in /etc/passwd) or the UID to be added
2444
//     is 0).
2445
//  3. The user specified additional host user accounts to add to the /etc/passwd file
2446
//
2447
// Returns password entry (as a string that can be appended to /etc/passwd) and
2448
// any error that occurred.
2449
func (c *Container) generatePasswdEntry() (string, error) {
2450
	passwdString := ""
2451

2452
	addedUID := 0
2453
	for _, userid := range c.config.HostUsers {
2454
		// Look up User on host
2455
		u, err := util.LookupUser(userid)
2456
		if err != nil {
2457
			return "", err
2458
		}
2459
		entry, err := c.userPasswdEntry(u)
2460
		if err != nil {
2461
			return "", err
2462
		}
2463
		passwdString += entry
2464
	}
2465
	if c.config.AddCurrentUserPasswdEntry {
2466
		entry, uid, _, err := c.generateCurrentUserPasswdEntry()
2467
		if err != nil {
2468
			return "", err
2469
		}
2470
		passwdString += entry
2471
		addedUID = uid
2472
	}
2473
	if c.config.User != "" {
2474
		entry, err := c.generateUserPasswdEntry(addedUID)
2475
		if err != nil {
2476
			return "", err
2477
		}
2478
		passwdString += entry
2479
	}
2480

2481
	return passwdString, nil
2482
}
2483

2484
// generateCurrentUserPasswdEntry generates an /etc/passwd entry for the user
2485
// running the container engine.
2486
// Returns a passwd entry for the user, and the UID and GID of the added entry.
2487
func (c *Container) generateCurrentUserPasswdEntry() (string, int, int, error) {
2488
	uid := rootless.GetRootlessUID()
2489
	if uid == 0 {
2490
		return "", 0, 0, nil
2491
	}
2492

2493
	u, err := user.LookupId(strconv.Itoa(uid))
2494
	if err != nil {
2495
		return "", 0, 0, fmt.Errorf("failed to get current user: %w", err)
2496
	}
2497
	pwd, err := c.userPasswdEntry(u)
2498
	if err != nil {
2499
		return "", 0, 0, err
2500
	}
2501

2502
	return pwd, uid, rootless.GetRootlessGID(), nil
2503
}
2504

2505
// Sets the HOME env. variable with precedence: existing home env. variable, execUser home
2506
func (c *Container) setHomeEnvIfNeeded() error {
2507
	getExecUserHome := func() (string, error) {
2508
		overrides := c.getUserOverrides()
2509
		execUser, err := lookup.GetUserGroupInfo(c.state.Mountpoint, c.config.User, overrides)
2510
		if err != nil {
2511
			if slices.Contains(c.config.HostUsers, c.config.User) {
2512
				execUser, err = lookupHostUser(c.config.User)
2513
			}
2514

2515
			if err != nil {
2516
				return "", err
2517
			}
2518
		}
2519

2520
		return execUser.Home, nil
2521
	}
2522

2523
	// Ensure HOME is not already set in Env
2524
	for _, s := range c.config.Spec.Process.Env {
2525
		if strings.HasPrefix(s, "HOME=") {
2526
			return nil
2527
		}
2528
	}
2529

2530
	home, err := getExecUserHome()
2531
	if err != nil {
2532
		return err
2533
	}
2534

2535
	c.config.Spec.Process.Env = append(c.config.Spec.Process.Env, fmt.Sprintf("HOME=%s", home))
2536
	return nil
2537
}
2538

2539
func (c *Container) userPasswdEntry(u *user.User) (string, error) {
2540
	// Look up the user to see if it exists in the container image.
2541
	_, err := lookup.GetUser(c.state.Mountpoint, u.Username)
2542
	if err != runcuser.ErrNoPasswdEntries {
2543
		return "", err
2544
	}
2545

2546
	// Look up the UID to see if it exists in the container image.
2547
	_, err = lookup.GetUser(c.state.Mountpoint, u.Uid)
2548
	if err != runcuser.ErrNoPasswdEntries {
2549
		return "", err
2550
	}
2551

2552
	// If the user's actual home directory exists, or was mounted in - use
2553
	// that.
2554
	homeDir := c.WorkingDir()
2555
	hDir := u.HomeDir
2556
	for hDir != "/" {
2557
		if MountExists(c.config.Spec.Mounts, hDir) {
2558
			homeDir = u.HomeDir
2559
			break
2560
		}
2561
		hDir = filepath.Dir(hDir)
2562
	}
2563
	if homeDir != u.HomeDir {
2564
		for _, hDir := range c.UserVolumes() {
2565
			if hDir == u.HomeDir {
2566
				homeDir = u.HomeDir
2567
				break
2568
			}
2569
		}
2570
	}
2571

2572
	if c.config.PasswdEntry != "" {
2573
		return c.passwdEntry(u.Username, u.Uid, u.Gid, u.Name, homeDir), nil
2574
	}
2575

2576
	return fmt.Sprintf("%s:*:%s:%s:%s:%s:/bin/sh\n", u.Username, u.Uid, u.Gid, u.Name, homeDir), nil
2577
}
2578

2579
// generateUserPasswdEntry generates an /etc/passwd entry for the container user
2580
// to run in the container.
2581
// The UID and GID of the added entry will also be returned.
2582
// Accepts one argument, that being any UID that has already been added to the
2583
// passwd file by other functions; if it matches the UID we were given, we don't
2584
// need to do anything.
2585
func (c *Container) generateUserPasswdEntry(addedUID int) (string, error) {
2586
	var (
2587
		groupspec string
2588
		gid       int
2589
	)
2590
	if c.config.User == "" {
2591
		return "", nil
2592
	}
2593
	splitSpec := strings.SplitN(c.config.User, ":", 2)
2594
	userspec := splitSpec[0]
2595
	if len(splitSpec) > 1 {
2596
		groupspec = splitSpec[1]
2597
	}
2598
	// If a non numeric User, then don't generate passwd
2599
	uid, err := strconv.ParseUint(userspec, 10, 32)
2600
	if err != nil {
2601
		return "", nil //nolint: nilerr
2602
	}
2603

2604
	if addedUID != 0 && int(uid) == addedUID {
2605
		return "", nil
2606
	}
2607

2608
	// Look up the user to see if it exists in the container image
2609
	_, err = lookup.GetUser(c.state.Mountpoint, userspec)
2610
	if err != runcuser.ErrNoPasswdEntries {
2611
		return "", err
2612
	}
2613

2614
	if groupspec != "" {
2615
		ugid, err := strconv.ParseUint(groupspec, 10, 32)
2616
		if err == nil {
2617
			gid = int(ugid)
2618
		} else {
2619
			group, err := lookup.GetGroup(c.state.Mountpoint, groupspec)
2620
			if err != nil {
2621
				return "", fmt.Errorf("unable to get gid %s from group file: %w", groupspec, err)
2622
			}
2623
			gid = group.Gid
2624
		}
2625
	}
2626

2627
	if c.config.PasswdEntry != "" {
2628
		entry := c.passwdEntry(strconv.FormatUint(uid, 10), strconv.FormatUint(uid, 10), strconv.FormatInt(int64(gid), 10), "container user", c.WorkingDir())
2629
		return entry, nil
2630
	}
2631

2632
	u, err := user.LookupId(strconv.FormatUint(uid, 10))
2633
	if err == nil {
2634
		return fmt.Sprintf("%s:*:%d:%d:%s:%s:/bin/sh\n", u.Username, uid, gid, u.Name, c.WorkingDir()), nil
2635
	}
2636
	return fmt.Sprintf("%d:*:%d:%d:container user:%s:/bin/sh\n", uid, uid, gid, c.WorkingDir()), nil
2637
}
2638

2639
func (c *Container) passwdEntry(username, uid, gid, name, homeDir string) string {
2640
	s := c.config.PasswdEntry
2641
	s = strings.ReplaceAll(s, "$USERNAME", username)
2642
	s = strings.ReplaceAll(s, "$UID", uid)
2643
	s = strings.ReplaceAll(s, "$GID", gid)
2644
	s = strings.ReplaceAll(s, "$NAME", name)
2645
	s = strings.ReplaceAll(s, "$HOME", homeDir)
2646
	return s + "\n"
2647
}
2648

2649
// generatePasswdAndGroup generates container-specific passwd and group files
2650
// iff g.config.User is a number or we are configured to make a passwd entry for
2651
// the current user or the user specified HostsUsers
2652
// Returns path to file to mount at /etc/passwd, path to file to mount at
2653
// /etc/group, and any error that occurred. If no passwd/group file were
2654
// required, the empty string will be returned for those path (this may occur
2655
// even if no error happened).
2656
// This may modify the mounted container's /etc/passwd and /etc/group instead of
2657
// making copies to bind-mount in, so we don't break useradd (it wants to make a
2658
// copy of /etc/passwd and rename the copy to /etc/passwd, which is impossible
2659
// with a bind mount). This is done in cases where the container is *not*
2660
// read-only. In this case, the function will return nothing ("", "", nil).
2661
func (c *Container) generatePasswdAndGroup() (string, string, error) {
2662
	if !c.config.AddCurrentUserPasswdEntry && c.config.User == "" &&
2663
		len(c.config.HostUsers) == 0 && c.config.GroupEntry == "" {
2664
		return "", "", nil
2665
	}
2666

2667
	needPasswd := true
2668
	needGroup := true
2669

2670
	// First, check if there's a mount at /etc/passwd or group, we don't
2671
	// want to interfere with user mounts.
2672
	if MountExists(c.config.Spec.Mounts, "/etc/passwd") {
2673
		needPasswd = false
2674
	}
2675
	if MountExists(c.config.Spec.Mounts, "/etc/group") {
2676
		needGroup = false
2677
	}
2678

2679
	// Next, check if we already made the files. If we didn't, don't need to
2680
	// do anything more.
2681
	if needPasswd {
2682
		passwdPath := filepath.Join(c.config.StaticDir, "passwd")
2683
		if err := fileutils.Exists(passwdPath); err == nil {
2684
			needPasswd = false
2685
		}
2686
	}
2687
	if needGroup {
2688
		groupPath := filepath.Join(c.config.StaticDir, "group")
2689
		if err := fileutils.Exists(groupPath); err == nil {
2690
			needGroup = false
2691
		}
2692
	}
2693

2694
	// If we don't need a /etc/passwd or /etc/group at this point we can
2695
	// just return.
2696
	if !needPasswd && !needGroup {
2697
		return "", "", nil
2698
	}
2699

2700
	passwdPath := ""
2701
	groupPath := ""
2702

2703
	ro := c.IsReadOnly()
2704

2705
	if needPasswd {
2706
		passwdEntry, err := c.generatePasswdEntry()
2707
		if err != nil {
2708
			return "", "", err
2709
		}
2710

2711
		needsWrite := passwdEntry != ""
2712
		switch {
2713
		case ro && needsWrite:
2714
			logrus.Debugf("Making /etc/passwd for container %s", c.ID())
2715
			originPasswdFile, err := securejoin.SecureJoin(c.state.Mountpoint, "/etc/passwd")
2716
			if err != nil {
2717
				return "", "", fmt.Errorf("creating path to container %s /etc/passwd: %w", c.ID(), err)
2718
			}
2719
			orig, err := os.ReadFile(originPasswdFile)
2720
			if err != nil && !os.IsNotExist(err) {
2721
				return "", "", err
2722
			}
2723
			passwdFile, err := c.writeStringToStaticDir("passwd", string(orig)+passwdEntry)
2724
			if err != nil {
2725
				return "", "", fmt.Errorf("failed to create temporary passwd file: %w", err)
2726
			}
2727
			if err := os.Chmod(passwdFile, 0644); err != nil {
2728
				return "", "", err
2729
			}
2730
			passwdPath = passwdFile
2731
		case !ro && needsWrite:
2732
			logrus.Debugf("Modifying container %s /etc/passwd", c.ID())
2733
			containerPasswd, err := securejoin.SecureJoin(c.state.Mountpoint, "/etc/passwd")
2734
			if err != nil {
2735
				return "", "", fmt.Errorf("looking up location of container %s /etc/passwd: %w", c.ID(), err)
2736
			}
2737

2738
			f, err := os.OpenFile(containerPasswd, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
2739
			if err != nil {
2740
				return "", "", fmt.Errorf("container %s: %w", c.ID(), err)
2741
			}
2742
			defer f.Close()
2743

2744
			if _, err := f.WriteString(passwdEntry); err != nil {
2745
				return "", "", fmt.Errorf("unable to append to container %s /etc/passwd: %w", c.ID(), err)
2746
			}
2747
		default:
2748
			logrus.Debugf("Not modifying container %s /etc/passwd", c.ID())
2749
		}
2750
	}
2751
	if needGroup {
2752
		groupEntry, err := c.generateGroupEntry()
2753
		if err != nil {
2754
			return "", "", err
2755
		}
2756

2757
		needsWrite := groupEntry != ""
2758
		switch {
2759
		case ro && needsWrite:
2760
			logrus.Debugf("Making /etc/group for container %s", c.ID())
2761
			originGroupFile, err := securejoin.SecureJoin(c.state.Mountpoint, "/etc/group")
2762
			if err != nil {
2763
				return "", "", fmt.Errorf("creating path to container %s /etc/group: %w", c.ID(), err)
2764
			}
2765
			orig, err := os.ReadFile(originGroupFile)
2766
			if err != nil && !os.IsNotExist(err) {
2767
				return "", "", err
2768
			}
2769
			groupFile, err := c.writeStringToStaticDir("group", string(orig)+groupEntry)
2770
			if err != nil {
2771
				return "", "", fmt.Errorf("failed to create temporary group file: %w", err)
2772
			}
2773
			if err := os.Chmod(groupFile, 0644); err != nil {
2774
				return "", "", err
2775
			}
2776
			groupPath = groupFile
2777
		case !ro && needsWrite:
2778
			logrus.Debugf("Modifying container %s /etc/group", c.ID())
2779
			containerGroup, err := securejoin.SecureJoin(c.state.Mountpoint, "/etc/group")
2780
			if err != nil {
2781
				return "", "", fmt.Errorf("looking up location of container %s /etc/group: %w", c.ID(), err)
2782
			}
2783

2784
			f, err := os.OpenFile(containerGroup, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
2785
			if err != nil {
2786
				return "", "", fmt.Errorf("container %s: %w", c.ID(), err)
2787
			}
2788
			defer f.Close()
2789

2790
			if _, err := f.WriteString(groupEntry); err != nil {
2791
				return "", "", fmt.Errorf("unable to append to container %s /etc/group: %w", c.ID(), err)
2792
			}
2793
		default:
2794
			logrus.Debugf("Not modifying container %s /etc/group", c.ID())
2795
		}
2796
	}
2797

2798
	return passwdPath, groupPath, nil
2799
}
2800

2801
func (c *Container) cleanupOverlayMounts() error {
2802
	return overlay.CleanupContent(c.config.StaticDir)
2803
}
2804

2805
// Creates and mounts an empty dir to mount secrets into, if it does not already exist
2806
func (c *Container) createSecretMountDir(runPath string) error {
2807
	src := filepath.Join(c.state.RunDir, "/run/secrets")
2808
	err := fileutils.Exists(src)
2809
	if os.IsNotExist(err) {
2810
		if err := umask.MkdirAllIgnoreUmask(src, os.FileMode(0o755)); err != nil {
2811
			return err
2812
		}
2813
		if err := c.relabel(src, c.config.MountLabel, false); err != nil {
2814
			return err
2815
		}
2816
		if err := idtools.SafeChown(src, c.RootUID(), c.RootGID()); err != nil {
2817
			return err
2818
		}
2819
		c.state.BindMounts[filepath.Join(runPath, "secrets")] = src
2820
		return nil
2821
	}
2822

2823
	return err
2824
}
2825

2826
func hasIdmapOption(options []string) bool {
2827
	for _, o := range options {
2828
		if o == "idmap" || strings.HasPrefix(o, "idmap=") {
2829
			return true
2830
		}
2831
	}
2832
	return false
2833
}
2834

2835
// Fix ownership and permissions of the specified volume if necessary.
2836
func (c *Container) fixVolumePermissions(v *ContainerNamedVolume) error {
2837
	vol, err := c.runtime.state.Volume(v.Name)
2838
	if err != nil {
2839
		return fmt.Errorf("retrieving named volume %s for container %s: %w", v.Name, c.ID(), err)
2840
	}
2841

2842
	vol.lock.Lock()
2843
	defer vol.lock.Unlock()
2844

2845
	// The volume may need a copy-up. Check the state.
2846
	if err := vol.update(); err != nil {
2847
		return err
2848
	}
2849

2850
	// Volumes owned by a volume driver are not chowned - we don't want to
2851
	// mess with a mount not managed by us.
2852
	if vol.state.NeedsChown && (!vol.UsesVolumeDriver() && vol.config.Driver != "image") {
2853
		vol.state.NeedsChown = false
2854

2855
		uid := int(c.config.Spec.Process.User.UID)
2856
		gid := int(c.config.Spec.Process.User.GID)
2857

2858
		// if the volume is mounted with "idmap", leave the IDs in from the current environment.
2859
		if c.config.IDMappings.UIDMap != nil && !hasIdmapOption(v.Options) {
2860
			p := idtools.IDPair{
2861
				UID: uid,
2862
				GID: gid,
2863
			}
2864
			mappings := idtools.NewIDMappingsFromMaps(c.config.IDMappings.UIDMap, c.config.IDMappings.GIDMap)
2865
			newPair, err := mappings.ToHost(p)
2866
			if err != nil {
2867
				return fmt.Errorf("mapping user %d:%d: %w", uid, gid, err)
2868
			}
2869
			uid = newPair.UID
2870
			gid = newPair.GID
2871
		}
2872

2873
		vol.state.UIDChowned = uid
2874
		vol.state.GIDChowned = gid
2875

2876
		if err := vol.save(); err != nil {
2877
			return err
2878
		}
2879

2880
		mountPoint, err := vol.MountPoint()
2881
		if err != nil {
2882
			return err
2883
		}
2884

2885
		if err := idtools.SafeLchown(mountPoint, uid, gid); err != nil {
2886
			return err
2887
		}
2888

2889
		// Make sure the new volume matches the permissions of the target directory unless 'U' is
2890
		// provided (since the volume was already chowned in this case).
2891
		// https://github.com/containers/podman/issues/10188
2892
		if slices.Contains(v.Options, "U") {
2893
			return nil
2894
		}
2895

2896
		st, err := os.Lstat(filepath.Join(c.state.Mountpoint, v.Dest))
2897
		if err == nil {
2898
			if stat, ok := st.Sys().(*syscall.Stat_t); ok {
2899
				uid, gid := int(stat.Uid), int(stat.Gid)
2900

2901
				if c.config.IDMappings.UIDMap != nil {
2902
					p := idtools.IDPair{
2903
						UID: uid,
2904
						GID: gid,
2905
					}
2906
					mappings := idtools.NewIDMappingsFromMaps(c.config.IDMappings.UIDMap, c.config.IDMappings.GIDMap)
2907
					newUID, newGID, err := mappings.ToContainer(p)
2908
					if err != nil {
2909
						return fmt.Errorf("mapping user %d:%d: %w", uid, gid, err)
2910
					}
2911
					uid, gid = newUID, newGID
2912
				}
2913

2914
				if err := idtools.SafeLchown(mountPoint, uid, gid); err != nil {
2915
					return err
2916
				}
2917
			}
2918
			if err := os.Chmod(mountPoint, st.Mode()); err != nil {
2919
				return err
2920
			}
2921
			if err := setVolumeAtime(mountPoint, st); err != nil {
2922
				return err
2923
			}
2924
		} else if !os.IsNotExist(err) {
2925
			return err
2926
		}
2927
	}
2928
	return nil
2929
}
2930

2931
func (c *Container) relabel(src, mountLabel string, shared bool) error {
2932
	if !selinux.GetEnabled() || mountLabel == "" {
2933
		return nil
2934
	}
2935
	// only relabel on initial creation of container
2936
	if !c.ensureState(define.ContainerStateConfigured, define.ContainerStateUnknown) {
2937
		label, err := label.FileLabel(src)
2938
		if err != nil {
2939
			return err
2940
		}
2941
		// If labels are different, might be on a tmpfs
2942
		if label == mountLabel {
2943
			return nil
2944
		}
2945
	}
2946
	err := label.Relabel(src, mountLabel, shared)
2947
	if errors.Is(err, unix.ENOTSUP) {
2948
		logrus.Debugf("Labeling not supported on %q", src)
2949
		return nil
2950
	}
2951
	return err
2952
}
2953

2954
func (c *Container) ChangeHostPathOwnership(src string, recurse bool, uid, gid int) error {
2955
	// only chown on initial creation of container
2956
	if !c.ensureState(define.ContainerStateConfigured, define.ContainerStateUnknown) {
2957
		st, err := os.Stat(src)
2958
		if err != nil {
2959
			return err
2960
		}
2961

2962
		// If labels are different, might be on a tmpfs
2963
		if int(st.Sys().(*syscall.Stat_t).Uid) == uid && int(st.Sys().(*syscall.Stat_t).Gid) == gid {
2964
			return nil
2965
		}
2966
	}
2967
	return chown.ChangeHostPathOwnership(src, recurse, uid, gid)
2968
}
2969

2970
func (c *Container) umask() (uint32, error) {
2971
	decVal, err := strconv.ParseUint(c.config.Umask, 8, 32)
2972
	if err != nil {
2973
		return 0, fmt.Errorf("invalid Umask Value: %w", err)
2974
	}
2975
	return uint32(decVal), nil
2976
}
2977

2978
func maybeClampOOMScoreAdj(oomScoreValue int) (int, error) {
2979
	v, err := os.ReadFile("/proc/self/oom_score_adj")
2980
	if err != nil {
2981
		return oomScoreValue, err
2982
	}
2983
	currentValue, err := strconv.Atoi(strings.TrimRight(string(v), "\n"))
2984
	if err != nil {
2985
		return oomScoreValue, err
2986
	}
2987
	if currentValue > oomScoreValue {
2988
		logrus.Warnf("Requested oom_score_adj=%d is lower than the current one, changing to %d", oomScoreValue, currentValue)
2989
		return currentValue, nil
2990
	}
2991
	return oomScoreValue, nil
2992
}
2993