podman

1
package mkcw
2

3
import (
4
	"archive/tar"
5
	"bytes"
6
	"compress/gzip"
7
	"encoding/binary"
8
	"encoding/json"
9
	"errors"
10
	"fmt"
11
	"io"
12
	"io/fs"
13
	"os"
14
	"os/exec"
15
	"path/filepath"
16
	"strconv"
17
	"strings"
18
	"time"
19

20
	"github.com/containers/buildah/internal/tmpdir"
21
	"github.com/containers/buildah/pkg/overlay"
22
	"github.com/containers/luksy"
23
	"github.com/containers/storage/pkg/idtools"
24
	"github.com/containers/storage/pkg/mount"
25
	"github.com/containers/storage/pkg/system"
26
	"github.com/docker/docker/pkg/ioutils"
27
	"github.com/docker/go-units"
28
	digest "github.com/opencontainers/go-digest"
29
	v1 "github.com/opencontainers/image-spec/specs-go/v1"
30
	"github.com/sirupsen/logrus"
31
)
32

33
const minimumImageSize = 10 * 1024 * 1024
34

35
// ArchiveOptions includes optional settings for generating an archive.
36
type ArchiveOptions struct {
37
	// If supplied, we'll register the workload with this server.
38
	// Practically necessary if DiskEncryptionPassphrase is not set, in
39
	// which case we'll generate one and throw it away after.
40
	AttestationURL string
41

42
	// Used to measure the environment.  If left unset (0, ""), defaults will be applied.
43
	CPUs   int
44
	Memory int
45

46
	// Can be manually set.  If left unset ("", false, nil), reasonable values will be used.
47
	TempDir                  string
48
	TeeType                  TeeType
49
	IgnoreAttestationErrors  bool
50
	ImageSize                int64
51
	WorkloadID               string
52
	Slop                     string
53
	DiskEncryptionPassphrase string
54
	FirmwareLibrary          string
55
	Logger                   *logrus.Logger
56
	GraphOptions             []string // passed in from a storage Store, probably
57
	ExtraImageContent        map[string]string
58
}
59

60
type chainRetrievalError struct {
61
	stderr string
62
	err    error
63
}
64

65
func (c chainRetrievalError) Error() string {
66
	if trimmed := strings.TrimSpace(c.stderr); trimmed != "" {
67
		return fmt.Sprintf("retrieving SEV certificate chain: sevctl: %v: %v", strings.TrimSpace(c.stderr), c.err)
68
	}
69
	return fmt.Sprintf("retrieving SEV certificate chain: sevctl: %v", c.err)
70
}
71

72
// Archive generates a WorkloadConfig for a specified directory and produces a
73
// tar archive of a container image's rootfs with the expected contents.
74
func Archive(rootfsPath string, ociConfig *v1.Image, options ArchiveOptions) (io.ReadCloser, WorkloadConfig, error) {
75
	const (
76
		teeDefaultCPUs       = 2
77
		teeDefaultMemory     = 512
78
		teeDefaultFilesystem = "ext4"
79
		teeDefaultTeeType    = SNP
80
	)
81

82
	if rootfsPath == "" {
83
		return nil, WorkloadConfig{}, fmt.Errorf("required path not specified")
84
	}
85
	logger := options.Logger
86
	if logger == nil {
87
		logger = logrus.StandardLogger()
88
	}
89

90
	teeType := options.TeeType
91
	if teeType == "" {
92
		teeType = teeDefaultTeeType
93
	}
94
	cpus := options.CPUs
95
	if cpus == 0 {
96
		cpus = teeDefaultCPUs
97
	}
98
	memory := options.Memory
99
	if memory == 0 {
100
		memory = teeDefaultMemory
101
	}
102
	filesystem := teeDefaultFilesystem
103
	workloadID := options.WorkloadID
104
	if workloadID == "" {
105
		digestInput := rootfsPath + filesystem + time.Now().String()
106
		workloadID = digest.Canonical.FromString(digestInput).Encoded()
107
	}
108
	workloadConfig := WorkloadConfig{
109
		Type:           teeType,
110
		WorkloadID:     workloadID,
111
		CPUs:           cpus,
112
		Memory:         memory,
113
		AttestationURL: options.AttestationURL,
114
	}
115
	if options.TempDir == "" {
116
		options.TempDir = tmpdir.GetTempDir()
117
	}
118

119
	// Do things which are specific to the type of TEE we're building for.
120
	var chainBytes []byte
121
	var chainBytesFile string
122
	var chainInfo fs.FileInfo
123
	switch teeType {
124
	default:
125
		return nil, WorkloadConfig{}, fmt.Errorf("don't know how to generate TeeData for TEE type %q", teeType)
126
	case SEV, SEV_NO_ES:
127
		// If we need a certificate chain, get it.
128
		chain, err := os.CreateTemp(options.TempDir, "chain")
129
		if err != nil {
130
			return nil, WorkloadConfig{}, err
131
		}
132
		chain.Close()
133
		defer func() {
134
			if err := os.Remove(chain.Name()); err != nil {
135
				logger.Warnf("error removing temporary file %q: %v", chain.Name(), err)
136
			}
137
		}()
138
		logrus.Debugf("sevctl export -f %s", chain.Name())
139
		cmd := exec.Command("sevctl", "export", "-f", chain.Name())
140
		var stdout, stderr bytes.Buffer
141
		cmd.Stdout, cmd.Stderr = &stdout, &stderr
142
		if err := cmd.Run(); err != nil {
143
			if !options.IgnoreAttestationErrors {
144
				return nil, WorkloadConfig{}, chainRetrievalError{stderr.String(), err}
145
			}
146
			logger.Warn(chainRetrievalError{stderr.String(), err}.Error())
147
		}
148
		if chainBytes, err = os.ReadFile(chain.Name()); err != nil {
149
			chainBytes = []byte{}
150
		}
151
		var teeData SevWorkloadData
152
		if len(chainBytes) > 0 {
153
			chainBytesFile = "sev.chain"
154
			chainInfo, err = os.Stat(chain.Name())
155
			if err != nil {
156
				return nil, WorkloadConfig{}, err
157
			}
158
			teeData.VendorChain = "/" + chainBytesFile
159
		}
160
		encodedTeeData, err := json.Marshal(teeData)
161
		if err != nil {
162
			return nil, WorkloadConfig{}, fmt.Errorf("encoding tee data: %w", err)
163
		}
164
		workloadConfig.TeeData = string(encodedTeeData)
165
	case SNP:
166
		teeData := SnpWorkloadData{
167
			Generation: "milan",
168
		}
169
		encodedTeeData, err := json.Marshal(teeData)
170
		if err != nil {
171
			return nil, WorkloadConfig{}, fmt.Errorf("encoding tee data: %w", err)
172
		}
173
		workloadConfig.TeeData = string(encodedTeeData)
174
	}
175

176
	// We're going to want to add some content to the rootfs, so set up an
177
	// overlay that uses it as a lower layer so that we can write to it.
178
	st, err := system.Stat(rootfsPath)
179
	if err != nil {
180
		return nil, WorkloadConfig{}, fmt.Errorf("reading information about the container root filesystem: %w", err)
181
	}
182
	// Create a temporary directory to hold all of this.  Use tmpdir.GetTempDir()
183
	// instead of the passed-in location, which a crafty caller might have put in an
184
	// overlay filesystem in storage because there tends to be more room there than
185
	// in, say, /var/tmp, and the plaintext disk image, which we put in the passed-in
186
	// location, can get quite large.
187
	rootfsParentDir, err := os.MkdirTemp(tmpdir.GetTempDir(), "buildah-rootfs")
188
	if err != nil {
189
		return nil, WorkloadConfig{}, fmt.Errorf("setting up parent for container root filesystem: %w", err)
190
	}
191
	defer func() {
192
		if err := os.RemoveAll(rootfsParentDir); err != nil {
193
			logger.Warnf("cleaning up parent for container root filesystem: %v", err)
194
		}
195
	}()
196
	// Create a mountpoint for the new overlay, which we'll use as the rootfs.
197
	rootfsDir := filepath.Join(rootfsParentDir, "rootfs")
198
	if err := idtools.MkdirAndChown(rootfsDir, fs.FileMode(st.Mode()), idtools.IDPair{UID: int(st.UID()), GID: int(st.GID())}); err != nil {
199
		return nil, WorkloadConfig{}, fmt.Errorf("creating mount target for container root filesystem: %w", err)
200
	}
201
	defer func() {
202
		if err := os.Remove(rootfsDir); err != nil {
203
			logger.Warnf("removing mount target for container root filesystem: %v", err)
204
		}
205
	}()
206
	// Create a directory to hold all of the overlay package's working state.
207
	tempDir := filepath.Join(rootfsParentDir, "tmp")
208
	if err = os.Mkdir(tempDir, 0o700); err != nil {
209
		return nil, WorkloadConfig{}, err
210
	}
211
	// Create some working state in there.
212
	overlayTempDir, err := overlay.TempDir(tempDir, int(st.UID()), int(st.GID()))
213
	if err != nil {
214
		return nil, WorkloadConfig{}, fmt.Errorf("setting up mount of container root filesystem: %w", err)
215
	}
216
	defer func() {
217
		if err := overlay.RemoveTemp(overlayTempDir); err != nil {
218
			logger.Warnf("cleaning up mount of container root filesystem: %v", err)
219
		}
220
	}()
221
	// Create a mount point using that working state.
222
	rootfsMount, err := overlay.Mount(overlayTempDir, rootfsPath, rootfsDir, 0, 0, options.GraphOptions)
223
	if err != nil {
224
		return nil, WorkloadConfig{}, fmt.Errorf("setting up support for overlay of container root filesystem: %w", err)
225
	}
226
	defer func() {
227
		if err := overlay.Unmount(overlayTempDir); err != nil {
228
			logger.Warnf("unmounting support for overlay of container root filesystem: %v", err)
229
		}
230
	}()
231
	// Follow through on the overlay or bind mount, whatever the overlay package decided
232
	// to leave to us to do.
233
	rootfsMountOptions := strings.Join(rootfsMount.Options, ",")
234
	logrus.Debugf("mounting %q to %q as %q with options %v", rootfsMount.Source, rootfsMount.Destination, rootfsMount.Type, rootfsMountOptions)
235
	if err := mount.Mount(rootfsMount.Source, rootfsMount.Destination, rootfsMount.Type, rootfsMountOptions); err != nil {
236
		return nil, WorkloadConfig{}, fmt.Errorf("mounting overlay of container root filesystem: %w", err)
237
	}
238
	defer func() {
239
		logrus.Debugf("unmounting %q", rootfsMount.Destination)
240
		if err := mount.Unmount(rootfsMount.Destination); err != nil {
241
			logger.Warnf("unmounting overlay of container root filesystem: %v", err)
242
		}
243
	}()
244
	// Pretend that we didn't have to do any of the preceding.
245
	rootfsPath = rootfsDir
246

247
	// Write extra content to the rootfs, creating intermediate directories if necessary.
248
	for location, content := range options.ExtraImageContent {
249
		err := func() error {
250
			if err := idtools.MkdirAllAndChownNew(filepath.Dir(filepath.Join(rootfsPath, location)), 0o755, idtools.IDPair{UID: int(st.UID()), GID: int(st.GID())}); err != nil {
251
				return fmt.Errorf("ensuring %q is present in container root filesystem: %w", filepath.Dir(location), err)
252
			}
253
			output, err := os.OpenFile(filepath.Join(rootfsPath, location), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
254
			if err != nil {
255
				return fmt.Errorf("preparing to write %q to container root filesystem: %w", location, err)
256
			}
257
			defer output.Close()
258
			input, err := os.Open(content)
259
			if err != nil {
260
				return err
261
			}
262
			defer input.Close()
263
			if _, err := io.Copy(output, input); err != nil {
264
				return fmt.Errorf("copying contents of %q to %q in container root filesystem: %w", content, location, err)
265
			}
266
			if err := output.Chown(int(st.UID()), int(st.GID())); err != nil {
267
				return fmt.Errorf("setting owner of %q in the container root filesystem: %w", location, err)
268
			}
269
			if err := output.Chmod(0o644); err != nil {
270
				return fmt.Errorf("setting permissions on %q in the container root filesystem: %w", location, err)
271
			}
272
			return nil
273
		}()
274
		if err != nil {
275
			return nil, WorkloadConfig{}, err
276
		}
277
	}
278

279
	// Write part of the config blob where the krun init process will be
280
	// looking for it.  The oci2cw tool used `buildah inspect` output, but
281
	// init is just looking for fields that have the right names in any
282
	// object, and the image's config will have that, so let's try encoding
283
	// it directly.
284
	krunConfigPath := filepath.Join(rootfsPath, ".krun_config.json")
285
	krunConfigBytes, err := json.Marshal(ociConfig)
286
	if err != nil {
287
		return nil, WorkloadConfig{}, fmt.Errorf("creating .krun_config from image configuration: %w", err)
288
	}
289
	if err := ioutils.AtomicWriteFile(krunConfigPath, krunConfigBytes, 0o600); err != nil {
290
		return nil, WorkloadConfig{}, fmt.Errorf("saving krun config: %w", err)
291
	}
292

293
	// Encode the workload config, in case it fails for any reason.
294
	cleanedUpWorkloadConfig := workloadConfig
295
	switch cleanedUpWorkloadConfig.Type {
296
	default:
297
		return nil, WorkloadConfig{}, fmt.Errorf("don't know how to canonicalize TEE type %q", cleanedUpWorkloadConfig.Type)
298
	case SEV, SEV_NO_ES:
299
		cleanedUpWorkloadConfig.Type = SEV
300
	case SNP:
301
		cleanedUpWorkloadConfig.Type = SNP
302
	}
303
	workloadConfigBytes, err := json.Marshal(cleanedUpWorkloadConfig)
304
	if err != nil {
305
		return nil, WorkloadConfig{}, err
306
	}
307

308
	// Make sure we have the passphrase to use for encrypting the disk image.
309
	diskEncryptionPassphrase := options.DiskEncryptionPassphrase
310
	if diskEncryptionPassphrase == "" {
311
		diskEncryptionPassphrase, err = GenerateDiskEncryptionPassphrase()
312
		if err != nil {
313
			return nil, WorkloadConfig{}, err
314
		}
315
	}
316

317
	// If we weren't told how big the image should be, get a rough estimate
318
	// of the input data size, then add a hedge to it.
319
	imageSize := slop(options.ImageSize, options.Slop)
320
	if imageSize == 0 {
321
		var sourceSize int64
322
		if err := filepath.WalkDir(rootfsPath, func(path string, d fs.DirEntry, err error) error {
323
			if err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, os.ErrPermission) {
324
				return err
325
			}
326
			info, err := d.Info()
327
			if err != nil && !errors.Is(err, os.ErrNotExist) && !errors.Is(err, os.ErrPermission) {
328
				return err
329
			}
330
			sourceSize += info.Size()
331
			return nil
332
		}); err != nil {
333
			return nil, WorkloadConfig{}, err
334
		}
335
		imageSize = slop(sourceSize, options.Slop)
336
	}
337
	if imageSize%4096 != 0 {
338
		imageSize += (4096 - (imageSize % 4096))
339
	}
340
	if imageSize < minimumImageSize {
341
		imageSize = minimumImageSize
342
	}
343

344
	// Create a file to use as the unencrypted version of the disk image.
345
	plain, err := os.CreateTemp(options.TempDir, "plain.img")
346
	if err != nil {
347
		return nil, WorkloadConfig{}, err
348
	}
349
	removePlain := true
350
	defer func() {
351
		if removePlain {
352
			if err := os.Remove(plain.Name()); err != nil {
353
				logger.Warnf("removing temporary file %q: %v", plain.Name(), err)
354
			}
355
		}
356
	}()
357

358
	// Lengthen the plaintext disk image file.
359
	if err := plain.Truncate(imageSize); err != nil {
360
		plain.Close()
361
		return nil, WorkloadConfig{}, err
362
	}
363
	plainInfo, err := plain.Stat()
364
	plain.Close()
365
	if err != nil {
366
		return nil, WorkloadConfig{}, err
367
	}
368

369
	// Format the disk image with the filesystem contents.
370
	if _, stderr, err := MakeFS(rootfsPath, plain.Name(), filesystem); err != nil {
371
		if strings.TrimSpace(stderr) != "" {
372
			return nil, WorkloadConfig{}, fmt.Errorf("%s: %w", strings.TrimSpace(stderr), err)
373
		}
374
		return nil, WorkloadConfig{}, err
375
	}
376

377
	// If we're registering the workload, we can do that now.
378
	if workloadConfig.AttestationURL != "" {
379
		if err := SendRegistrationRequest(workloadConfig, diskEncryptionPassphrase, options.FirmwareLibrary, options.IgnoreAttestationErrors, logger); err != nil {
380
			return nil, WorkloadConfig{}, err
381
		}
382
	}
383

384
	// Try to encrypt on the fly.
385
	pipeReader, pipeWriter := io.Pipe()
386
	removePlain = false
387
	go func() {
388
		var err error
389
		defer func() {
390
			if err := os.Remove(plain.Name()); err != nil {
391
				logger.Warnf("removing temporary file %q: %v", plain.Name(), err)
392
			}
393
			if err != nil {
394
				pipeWriter.CloseWithError(err)
395
			} else {
396
				pipeWriter.Close()
397
			}
398
		}()
399
		plain, err := os.Open(plain.Name())
400
		if err != nil {
401
			logrus.Errorf("opening unencrypted disk image %q: %v", plain.Name(), err)
402
			return
403
		}
404
		defer plain.Close()
405
		tw := tar.NewWriter(pipeWriter)
406
		defer tw.Flush()
407

408
		// Write /entrypoint
409
		var decompressedEntrypoint bytes.Buffer
410
		decompressor, err := gzip.NewReader(bytes.NewReader(entrypointCompressedBytes))
411
		if err != nil {
412
			logrus.Errorf("decompressing copy of entrypoint: %v", err)
413
			return
414
		}
415
		defer decompressor.Close()
416
		if _, err = io.Copy(&decompressedEntrypoint, decompressor); err != nil {
417
			logrus.Errorf("decompressing copy of entrypoint: %v", err)
418
			return
419
		}
420
		entrypointHeader, err := tar.FileInfoHeader(plainInfo, "")
421
		if err != nil {
422
			logrus.Errorf("building header for entrypoint: %v", err)
423
			return
424
		}
425
		entrypointHeader.Name = "entrypoint"
426
		entrypointHeader.Mode = 0o755
427
		entrypointHeader.Uname, entrypointHeader.Gname = "", ""
428
		entrypointHeader.Uid, entrypointHeader.Gid = 0, 0
429
		entrypointHeader.Size = int64(decompressedEntrypoint.Len())
430
		if err = tw.WriteHeader(entrypointHeader); err != nil {
431
			logrus.Errorf("writing header for %q: %v", entrypointHeader.Name, err)
432
			return
433
		}
434
		if _, err = io.Copy(tw, &decompressedEntrypoint); err != nil {
435
			logrus.Errorf("writing %q: %v", entrypointHeader.Name, err)
436
			return
437
		}
438

439
		// Write /sev.chain
440
		if chainInfo != nil {
441
			chainHeader, err := tar.FileInfoHeader(chainInfo, "")
442
			if err != nil {
443
				logrus.Errorf("building header for %q: %v", chainInfo.Name(), err)
444
				return
445
			}
446
			chainHeader.Name = chainBytesFile
447
			chainHeader.Mode = 0o600
448
			chainHeader.Uname, chainHeader.Gname = "", ""
449
			chainHeader.Uid, chainHeader.Gid = 0, 0
450
			chainHeader.Size = int64(len(chainBytes))
451
			if err = tw.WriteHeader(chainHeader); err != nil {
452
				logrus.Errorf("writing header for %q: %v", chainHeader.Name, err)
453
				return
454
			}
455
			if _, err = tw.Write(chainBytes); err != nil {
456
				logrus.Errorf("writing %q: %v", chainHeader.Name, err)
457
				return
458
			}
459
		}
460

461
		// Write /krun-sev.json.
462
		workloadConfigHeader, err := tar.FileInfoHeader(plainInfo, "")
463
		if err != nil {
464
			logrus.Errorf("building header for %q: %v", plainInfo.Name(), err)
465
			return
466
		}
467
		workloadConfigHeader.Name = "krun-sev.json"
468
		workloadConfigHeader.Mode = 0o600
469
		workloadConfigHeader.Uname, workloadConfigHeader.Gname = "", ""
470
		workloadConfigHeader.Uid, workloadConfigHeader.Gid = 0, 0
471
		workloadConfigHeader.Size = int64(len(workloadConfigBytes))
472
		if err = tw.WriteHeader(workloadConfigHeader); err != nil {
473
			logrus.Errorf("writing header for %q: %v", workloadConfigHeader.Name, err)
474
			return
475
		}
476
		if _, err = tw.Write(workloadConfigBytes); err != nil {
477
			logrus.Errorf("writing %q: %v", workloadConfigHeader.Name, err)
478
			return
479
		}
480

481
		// Write /tmp.
482
		tmpHeader, err := tar.FileInfoHeader(plainInfo, "")
483
		if err != nil {
484
			logrus.Errorf("building header for %q: %v", plainInfo.Name(), err)
485
			return
486
		}
487
		tmpHeader.Name = "tmp/"
488
		tmpHeader.Typeflag = tar.TypeDir
489
		tmpHeader.Mode = 0o1777
490
		tmpHeader.Uname, tmpHeader.Gname = "", ""
491
		tmpHeader.Uid, tmpHeader.Gid = 0, 0
492
		tmpHeader.Size = 0
493
		if err = tw.WriteHeader(tmpHeader); err != nil {
494
			logrus.Errorf("writing header for %q: %v", tmpHeader.Name, err)
495
			return
496
		}
497

498
		// Now figure out the footer that we'll append to the encrypted disk.
499
		var footer bytes.Buffer
500
		lengthBuffer := make([]byte, 8)
501
		footer.Write(workloadConfigBytes)
502
		footer.WriteString("KRUN")
503
		binary.LittleEndian.PutUint64(lengthBuffer, uint64(len(workloadConfigBytes)))
504
		footer.Write(lengthBuffer)
505

506
		// Start encrypting and write /disk.img.
507
		header, encrypt, blockSize, err := luksy.EncryptV1([]string{diskEncryptionPassphrase}, "")
508
		paddingBoundary := int64(4096)
509
		paddingNeeded := (paddingBoundary - ((int64(len(header)) + imageSize + int64(footer.Len())) % paddingBoundary)) % paddingBoundary
510
		diskHeader := workloadConfigHeader
511
		diskHeader.Name = "disk.img"
512
		diskHeader.Mode = 0o600
513
		diskHeader.Size = int64(len(header)) + imageSize + paddingNeeded + int64(footer.Len())
514
		if err = tw.WriteHeader(diskHeader); err != nil {
515
			logrus.Errorf("writing archive header for disk.img: %v", err)
516
			return
517
		}
518
		if _, err = io.Copy(tw, bytes.NewReader(header)); err != nil {
519
			logrus.Errorf("writing encryption header for disk.img: %v", err)
520
			return
521
		}
522
		encryptWrapper := luksy.EncryptWriter(encrypt, tw, blockSize)
523
		if _, err = io.Copy(encryptWrapper, plain); err != nil {
524
			logrus.Errorf("encrypting disk.img: %v", err)
525
			return
526
		}
527
		encryptWrapper.Close()
528
		if _, err = tw.Write(make([]byte, paddingNeeded)); err != nil {
529
			logrus.Errorf("writing padding for disk.img: %v", err)
530
			return
531
		}
532
		if _, err = io.Copy(tw, &footer); err != nil {
533
			logrus.Errorf("writing footer for disk.img: %v", err)
534
			return
535
		}
536
		tw.Close()
537
	}()
538

539
	return pipeReader, workloadConfig, nil
540
}
541

542
func slop(size int64, slop string) int64 {
543
	if slop == "" {
544
		return size * 5 / 4
545
	}
546
	for _, factor := range strings.Split(slop, "+") {
547
		factor = strings.TrimSpace(factor)
548
		if factor == "" {
549
			continue
550
		}
551
		if strings.HasSuffix(factor, "%") {
552
			percentage := strings.TrimSuffix(factor, "%")
553
			percent, err := strconv.ParseInt(percentage, 10, 8)
554
			if err != nil {
555
				logrus.Warnf("parsing percentage %q: %v", factor, err)
556
			} else {
557
				size *= (percent + 100)
558
				size /= 100
559
			}
560
		} else {
561
			more, err := units.RAMInBytes(factor)
562
			if err != nil {
563
				logrus.Warnf("parsing %q as a size: %v", factor, err)
564
			} else {
565
				size += more
566
			}
567
		}
568
	}
569
	return size
570
}
571