podman

1
#!/usr/bin/env bats   -*- bats -*-
2
#
3
# Tests for systemd sdnotify
4
#
5

6
load helpers
7
load helpers.network
8
load helpers.registry
9

10
# Shared throughout this module: PID of socat process, and path to its log
11
_SOCAT_PID=
12
_SOCAT_LOG=
13

14
function setup() {
15
    skip_if_remote "systemd tests are meaningless over remote"
16

17
    # Skip if systemd is not running
18
    systemctl list-units &>/dev/null || skip "systemd not available"
19

20
    # sdnotify fails with runc 1.0.0-3-dev2 on Ubuntu. Let's just
21
    # assume that we work only with crun, nothing else.
22
    runtime=$(podman_runtime)
23
    if [[ "$runtime" != "crun" ]]; then
24
        skip "this test only works with crun, not $runtime"
25
    fi
26

27
    basic_setup
28
}
29

30
function teardown() {
31
    unset NOTIFY_SOCKET
32

33
    _stop_socat
34

35
    basic_teardown
36
}
37

38
###############################################################################
39
# BEGIN helpers
40

41
# Run socat process on a socket, logging to well-known path. Each received
42
# packet is logged with a newline appended, for ease of parsing the log file.
43
function _start_socat() {
44
    _SOCAT_LOG="$PODMAN_TMPDIR/socat.log"
45

46
    # Reset socat logfile to empty
47
    rm -f $_SOCAT_LOG
48
    touch $_SOCAT_LOG
49
    # Execute in subshell so we can close fd3 (which BATS uses).
50
    # This is a superstitious ritual to try to avoid leaving processes behind,
51
    # and thus prevent CI hangs.
52
    (exec socat unix-recvfrom:"$NOTIFY_SOCKET",fork \
53
          system:"(cat;echo) >> $_SOCAT_LOG" 3>&-) &
54
    _SOCAT_PID=$!
55
}
56

57
# Stop the socat background process and clean up logs
58
function _stop_socat() {
59
    if [[ -n "$_SOCAT_PID" ]]; then
60
        # Kill all child processes, then the process itself.
61
        # This is a superstitious incantation to avoid leaving processes behind.
62
        # The '|| true' is because only f35 leaves behind socat processes;
63
        # f33 (and perhaps others?) behave nicely. ARGH!
64
        pkill -P $_SOCAT_PID || true
65
        kill $_SOCAT_PID
66
    fi
67
    _SOCAT_PID=
68

69
    if [[ -n "$_SOCAT_LOG" ]]; then
70
        rm -f $_SOCAT_LOG
71
    fi
72
    _SOCAT_LOG=
73
}
74

75
# Check that MAINPID=xxxxx points to a running conmon process
76
function _assert_mainpid_is_conmon() {
77
    local mainpid=$(expr "$1" : ".*MAINPID=\([0-9]\+\)")
78
    test -n "$mainpid" || die "Could not parse '$1' as 'MAINPID=nnnn'"
79

80
    test -d /proc/$mainpid || die "sdnotify MAINPID=$mainpid - but /proc/$mainpid does not exist"
81

82
    # e.g. /proc/12345/exe -> /usr/bin/conmon
83
    local mainpid_bin=$(readlink /proc/$mainpid/exe)
84
    is "$mainpid_bin" ".*/conmon" "sdnotify MAINPID=$mainpid is conmon process"
85
}
86

87
# END   helpers
88
###############################################################################
89
# BEGIN tests themselves
90

91
@test "sdnotify : ignore" {
92
    export NOTIFY_SOCKET=$PODMAN_TMPDIR/ignore.sock
93
    _start_socat
94

95
    run_podman create --rm --sdnotify=ignore $IMAGE printenv NOTIFY_SOCKET
96
    cid="$output"
97

98
    run_podman container inspect $cid --format "{{.Config.SdNotifyMode}} {{.Config.SdNotifySocket}}"
99
    is "$output" "ignore " "NOTIFY_SOCKET is not set with 'ignore' mode"
100

101
    run_podman 1 start --attach $cid
102
    is "$output" "" "\$NOTIFY_SOCKET in container"
103

104
    is "$(< $_SOCAT_LOG)" "" "nothing received on socket"
105
    _stop_socat
106
}
107

108
# bats test_tags=distro-integration
109
@test "sdnotify : conmon" {
110
    export NOTIFY_SOCKET=$PODMAN_TMPDIR/conmon.sock
111
    _start_socat
112

113
    run_podman run -d --name sdnotify_conmon_c \
114
               --sdnotify=conmon \
115
               $IMAGE \
116
               sh -c 'printenv NOTIFY_SOCKET;echo READY;sleep 999'
117
    cid="$output"
118
    wait_for_ready $cid
119

120
    run_podman container inspect $cid --format "{{.Config.SdNotifyMode}} {{.Config.SdNotifySocket}}"
121
    is "$output" "conmon $NOTIFY_SOCKET"
122

123
    run_podman container inspect sdnotify_conmon_c --format "{{.State.ConmonPid}}"
124
    mainPID="$output"
125

126
    run_podman logs sdnotify_conmon_c
127
    is "$output" "READY" "\$NOTIFY_SOCKET in container"
128

129
    # loop-wait for the final READY line
130
    wait_for_file_content $_SOCAT_LOG "READY=1"
131

132
    # ...and confirm the entire file contents
133
    logcontents="$(< $_SOCAT_LOG)"
134
    assert "$logcontents" = "MAINPID=$mainPID
135
READY=1" "sdnotify sent MAINPID and READY"
136

137
    _assert_mainpid_is_conmon "$logcontents"
138

139
    # Done. Stop container, clean up.
140
    run_podman rm -f -t0 $cid
141
    _stop_socat
142
}
143

144
# These tests can fail in dev. environment because of SELinux.
145
# quick fix: chcon -t container_runtime_exec_t ./bin/podman
146
# bats test_tags=distro-integration
147
@test "sdnotify : container" {
148
    _prefetch $SYSTEMD_IMAGE
149

150
    export NOTIFY_SOCKET=$PODMAN_TMPDIR/container.sock
151
    _start_socat
152

153
    run_podman run -d --sdnotify=container $SYSTEMD_IMAGE \
154
               sh -c 'trap "touch /stop" SIGUSR1;printenv NOTIFY_SOCKET; echo READY; while ! test -f /stop;do sleep 0.1;done;systemd-notify --ready'
155
    cid="$output"
156
    wait_for_ready $cid
157

158
    run_podman container inspect $cid --format "{{.Config.SdNotifyMode}} {{.Config.SdNotifySocket}}"
159
    is "$output" "container $NOTIFY_SOCKET"
160

161
    run_podman logs $cid
162
    is "${lines[0]}" "/run/notify/notify.sock" "NOTIFY_SOCKET is passed to container"
163

164
    run_podman container inspect $cid --format "{{.State.ConmonPid}}"
165
    mainPID="$output"
166

167
    # Container does not send READY=1 until our signal. Until then, there must
168
    # be exactly one line in the log
169
    wait_for_file_content $_SOCAT_LOG "MAINPID=$mainPID"
170
    # ...and that line must contain the expected PID, nothing more
171
    assert "$(< $_SOCAT_LOG)" = "MAINPID=$mainPID" "Container has started, but must not indicate READY yet"
172

173
    # Done. Tell container to stop itself, and clean up
174
    run_podman kill -s USR1 $cid
175
    run_podman wait $cid
176

177
    wait_for_file_content $_SOCAT_LOG "READY=1"
178
    assert "$(< $_SOCAT_LOG)" = "MAINPID=$mainPID
179
READY=1" "Container log after ready signal"
180

181
    run_podman rm $cid
182
    _stop_socat
183
}
184

185
# These tests can fail in dev. environment because of SELinux.
186
# quick fix: chcon -t container_runtime_exec_t ./bin/podman
187
@test "sdnotify : healthy" {
188
    export NOTIFY_SOCKET=$PODMAN_TMPDIR/container.sock
189
    _start_socat
190

191
    wait_file="$PODMAN_TMPDIR/$(random_string).wait_for_me"
192
    run_podman 125 create --sdnotify=healthy $IMAGE
193
    is "$output" "Error: invalid argument: sdnotify policy \"healthy\" requires a healthcheck to be set"
194

195
    # Create a container with a simple `/bin/true` healthcheck that we need to
196
    # run manually.
197
    ctr=$(random_string)
198
    run_podman create --name $ctr     \
199
            --health-cmd=/bin/true    \
200
            --health-retries=1        \
201
            --health-interval=disable \
202
            --sdnotify=healthy        \
203
            $IMAGE sleep infinity
204

205
    # Start the container in the background which will block until the
206
    # container turned healthy.  After that, create the wait_file which
207
    # indicates that start has returned.
208
    (timeout --foreground -v --kill=5 20 $PODMAN start $ctr && touch $wait_file) &
209

210
    run_podman wait --condition=running $ctr
211

212
    # Make sure that the MAINPID is set but without the READY message.
213
    run_podman container inspect $ctr --format "{{.State.ConmonPid}}"
214
    mainPID="$output"
215

216
    # Container does not send READY=1 until it runs a successful health check.
217
    # Until then, there must be exactly one line in the log
218
    wait_for_file_content $_SOCAT_LOG "MAINPID="
219
    # ...and that line must contain the expected PID, nothing more
220
    assert "$(< $_SOCAT_LOG)" = "MAINPID=$mainPID" "Container logs after start, prior to healthcheck run"
221

222
    # Now run the healthcheck and look for the READY message.
223
    run_podman healthcheck run $ctr
224
    is "$output" "" "output from 'podman healthcheck run'"
225

226
    # Wait for start to return.  At that point the READY message must have been
227
    # sent.
228
    wait_for_file_content $_SOCAT_LOG "READY=1"
229
    assert "$(< $_SOCAT_LOG)" = "MAINPID=$mainPID
230
READY=1" "Container log after healthcheck run"
231

232
    run_podman container inspect  --format "{{.State.Status}}" $ctr
233
    is "$output" "running" "make sure container is still running"
234

235
    run_podman rm -f -t0 $ctr
236
    _stop_socat
237
}
238

239
@test "sdnotify : play kube - no policies" {
240
    # Create the YAMl file
241
    yaml_source="$PODMAN_TMPDIR/test.yaml"
242
    cat >$yaml_source <<EOF
243
apiVersion: v1
244
kind: Pod
245
metadata:
246
  labels:
247
    app: test
248
  name: test_pod
249
spec:
250
  restartPolicy: "Never"
251
  containers:
252
  - command:
253
    - /bin/sh
254
    - -c
255
    - 'while :; do if test -e /rain/tears; then exit 0; fi; sleep 1; done'
256
    image: $IMAGE
257
    name: test
258
    resources: {}
259
    volumeMounts:
260
    - mountPath: /rain:z
261
      name: test-mountdir
262
  volumes:
263
  - hostPath:
264
      path: $PODMAN_TMPDIR
265
      type: Directory
266
    name: test-mountdir
267
EOF
268

269
    # The name of the service container is predictable: the first 12 characters
270
    # of the hash of the YAML file followed by the "-service" suffix
271
    yaml_sha=$(sha256sum $yaml_source)
272
    service_container="${yaml_sha:0:12}-service"
273

274
    export NOTIFY_SOCKET=$PODMAN_TMPDIR/conmon.sock
275
    _start_socat
276
    wait_for_file $_SOCAT_LOG
277

278
    run_podman play kube --service-container=true --log-driver journald $yaml_source
279

280
    # The service container is the main PID since no container has a custom
281
    # sdnotify policy.
282
    run_podman container inspect $service_container --format "{{.State.ConmonPid}}"
283
    main_pid="$output"
284

285
    # Tell pod to finish, then wait for all containers to stop
286
    touch $PODMAN_TMPDIR/tears
287
    run_podman container wait $service_container test_pod-test
288

289
    # Make sure the containers have the correct policy.
290
    run_podman container inspect test_pod-test $service_container --format "{{.Config.SdNotifyMode}}"
291
    is "$output" "ignore
292
ignore"
293

294
    wait_for_file_content $_SOCAT_LOG "READY=1"
295
    assert "$(< $_SOCAT_LOG)" = "MAINPID=$main_pid
296
READY=1" "sdnotify sent MAINPID and READY"
297

298
    _stop_socat
299

300
    # Clean up pod and pause image
301
    run_podman play kube --down $PODMAN_TMPDIR/test.yaml
302
    run_podman rmi $(pause_image)
303
}
304

305
@test "sdnotify : play kube - with policies" {
306
    skip_if_journald_unavailable
307

308
    _prefetch $SYSTEMD_IMAGE
309

310
    # Create the YAMl file
311
    yaml_source="$PODMAN_TMPDIR/test.yaml"
312
    cat >$yaml_source <<EOF
313
apiVersion: v1
314
kind: Pod
315
metadata:
316
  labels:
317
    app: test
318
  name: test_pod
319
  annotations:
320
    io.containers.sdnotify:   "container"
321
    io.containers.sdnotify/b: "conmon"
322
spec:
323
  restartPolicy: "Never"
324
  containers:
325
  - command:
326
    - /bin/sh
327
    - -c
328
    - 'printenv NOTIFY_SOCKET; echo READY; while ! test -f /stop;do sleep 0.1;done'
329
    image: $SYSTEMD_IMAGE
330
    name: a
331
  - command:
332
    - /bin/sh
333
    - -c
334
    - 'echo READY; top'
335
    image: $IMAGE
336
    name: b
337
EOF
338
    container_a="test_pod-a"
339
    container_b="test_pod-b"
340

341
    # The name of the service container is predictable: the first 12 characters
342
    # of the hash of the YAML file followed by the "-service" suffix
343
    yaml_sha=$(sha256sum $yaml_source)
344
    service_container="${yaml_sha:0:12}-service"
345

346
    export NOTIFY_SOCKET=$PODMAN_TMPDIR/conmon.sock
347
    _start_socat
348

349
    # Run `play kube` in the background as it will wait for all containers to
350
    # send the READY=1 message.
351
    timeout --foreground -v --kill=10 60 \
352
        $PODMAN play kube --service-container=true --log-driver journald $yaml_source &>/dev/null &
353

354
    # Wait for both containers to be running
355
    containers_running=
356
    for i in $(seq 1 20); do
357
        run_podman "?" container wait $container_a $container_b --condition="running"
358
        if [[ $status == 0 ]]; then
359
            containers_running=1
360
            break
361
        fi
362
        sleep 0.5
363
        # Just for debugging
364
        run_podman ps -a
365
    done
366
    if [[ -z "$containers_running" ]]; then
367
        die "container $container_a and/or $container_b did not start"
368
    fi
369

370
    wait_for_ready $container_a
371
    # Make sure the containers have the correct policy
372
    run_podman container inspect $container_a $container_b $service_container --format "{{.Config.SdNotifyMode}}"
373
    is "$output" "container
374
conmon
375
ignore"
376

377
    is "$(< $_SOCAT_LOG)" "" "nothing received on socket"
378

379
    # Make sure the container received a "proxy" socket and is not using the
380
    # one of `kube play`
381
    run_podman container inspect $container_a --format "{{.Config.SdNotifySocket}}"
382
    assert "$output" != $NOTIFY_SOCKET
383

384
    run_podman logs $container_a
385
    is "${lines[0]}" "/run/notify/notify.sock" "NOTIFY_SOCKET is passed to container"
386

387
    # Send the READY message.  Doing it in an exec session helps debug
388
    # potential issues.
389
    run_podman exec --env NOTIFY_SOCKET="/run/notify/notify.sock" $container_a /usr/bin/systemd-notify --ready
390

391
    # Instruct the container to stop.
392
    # Run detached as the `exec` session races with the cleanup process
393
    # of the exiting container (see #10825).
394
    run_podman exec -d $container_a /bin/touch /stop
395

396
    run_podman container wait $container_a
397
    run_podman container inspect $container_a --format "{{.State.ExitCode}}"
398
    is "$output" "0" "container exited cleanly after sending READY message"
399

400
    wait_for_file_content $_SOCAT_LOG "READY=1"
401
    assert "$(< $_SOCAT_LOG)" =~ "MAINPID=.*
402
READY=1" "sdnotify sent MAINPID and READY"
403

404
    # Make sure that Podman is the service's MainPID
405
    main_pid=$(head -n1 $_SOCAT_LOG | awk -F= '{print $2}')
406
    is "$(</proc/$main_pid/comm)" "podman" "podman is the service mainPID ($main_pid)"
407
    _stop_socat
408

409
    # Clean up pod and pause image
410
    run_podman play kube --down $yaml_source
411
    run_podman rmi $(pause_image)
412
}
413

414
function generate_exit_code_yaml {
415
    local fname=$1
416
    local cmd1=$2
417
    local cmd2=$3
418
    local sdnotify_policy=$4
419
    echo "
420
apiVersion: v1
421
kind: Pod
422
metadata:
423
  labels:
424
    app: test
425
  name: test_pod
426
  annotations:
427
    io.containers.sdnotify: "$sdnotify_policy"
428
spec:
429
  restartPolicy: Never
430
  containers:
431
    - name: ctr1
432
      image: $IMAGE
433
      command:
434
      - $cmd1
435
    - name: ctr2
436
      image: $IMAGE
437
      command:
438
      - $cmd2
439
" > $fname
440
}
441

442
# bats test_tags=distro-integration
443
@test "podman kube play - exit-code propagation" {
444
    fname=$PODMAN_TMPDIR/$(random_string).yaml
445

446
    # Create a test matrix with the following arguments:
447
    # exit-code propagation | ctr1 command | ctr2 command | service-container exit code
448
    exit_tests="
449
all  | true  | true  | 0
450
all  | true  | false | 0
451
all  | false | false | 137
452
any  | true  | true  | 0
453
any  | false | true  | 137
454
any  | false | false | 137
455
none | true  | true  | 0
456
none | true  | false | 0
457
none | false | false | 0
458
"
459

460
    # I am sorry, this is a long test as we need to test the upper matrix
461
    # twice. The first run is using the default sdnotify policy of "ignore".
462
    # In this case, the service container serves as the main PID of the service
463
    # to have a minimal resource footprint.  The second run is using the
464
    # "conmon" sdnotify policy in which case Podman needs to serve as the main
465
    # PID to act as an sdnotify proxy; there Podman will wait for the service
466
    # container to exit and reflects its exit code.
467
    while read exit_code_prop cmd1 cmd2 exit_code; do
468
        for sdnotify_policy in ignore conmon; do
469
            generate_exit_code_yaml $fname $cmd1 $cmd2 $sdnotify_policy
470
            yaml_sha=$(sha256sum $fname)
471
            service_container="${yaml_sha:0:12}-service"
472
            podman_exit=$exit_code
473
            if [[ $sdnotify_policy == "ignore" ]];then
474
                 podman_exit=0
475
            fi
476
            run_podman $podman_exit kube play --service-exit-code-propagation="$exit_code_prop" --service-container $fname
477
            # Make sure that there are no error logs (e.g., #19715)
478
            assert "$output" !~ "error msg="
479
            run_podman container inspect --format '{{.KubeExitCodePropagation}}' $service_container
480
            is "$output" "$exit_code_prop" "service container has the expected policy set in its annotations"
481
            run_podman wait $service_container
482
            is "$output" "$exit_code" "service container exit code (propagation: $exit_code_prop, policy: $service_policy, cmds: $cmd1 + $cmd2)"
483
            run_podman kube down $fname
484
        done
485
    done < <(parse_table "$exit_tests")
486

487
    # A final smoke test to make sure bogus policies lead to an error
488
    run_podman 125 kube play --service-exit-code-propagation=bogus --service-container $fname
489
    is "$output" "Error: unsupported exit-code propagation \"bogus\"" "error on unsupported exit-code propagation"
490

491
    run_podman rmi $(pause_image)
492
}
493

494
@test "podman pull - EXTEND_TIMEOUT_USEC" {
495
    # Make sure that Podman extends the start timeout via DBUS when running
496
    # inside a systemd unit (i.e., with NOTIFY_SOCKET set).  Extending the
497
    # timeout works by continuously sending EXTEND_TIMEOUT_USEC; Podman does
498
    # this at most 10 times, adding up to ~5min.
499

500
    image_on_local_registry=localhost:${PODMAN_LOGIN_REGISTRY_PORT}/name:tag
501
    registry_flags="--tls-verify=false --creds ${PODMAN_LOGIN_USER}:${PODMAN_LOGIN_PASS}"
502
    start_registry
503

504
    export NOTIFY_SOCKET=$PODMAN_TMPDIR/notify.sock
505
    _start_socat
506

507
    run_podman push $registry_flags $IMAGE $image_on_local_registry
508
    run_podman pull $registry_flags $image_on_local_registry
509
    is "${lines[1]}" "Pulling image //$image_on_local_registry inside systemd: setting pull timeout to 5m0s" "NOTIFY_SOCKET is passed to container"
510

511
    run cat $_SOCAT_LOG
512
    # The 'echo's help us debug failed runs
513
    echo "socat log:"
514
    echo "$output"
515
    is "$output" "EXTEND_TIMEOUT_USEC=30000000"
516

517
    run_podman rmi $image_on_local_registry
518
    _stop_socat
519
}
520

521
@test "podman system service" {
522
    # This test makes sure that podman-system-service uses the NOTIFY_SOCKET
523
    # correctly and that it unsets it after sending the expected MAINPID and
524
    # READY message by making sure no EXTEND_TIMEOUT_USEC is sent on pull.
525

526
    # Start a local registry and pre-populate it with an image we'll pull later on.
527
    image_on_local_registry=localhost:${PODMAN_LOGIN_REGISTRY_PORT}/name:tag
528
    registry_flags="--tls-verify=false --creds ${PODMAN_LOGIN_USER}:${PODMAN_LOGIN_PASS}"
529
    start_registry
530
    run_podman push $registry_flags $IMAGE $image_on_local_registry
531

532
    export NOTIFY_SOCKET=$PODMAN_TMPDIR/notify.sock
533
    podman_socket="unix://$PODMAN_TMPDIR/podman.sock"
534
    envfile=$PODMAN_TMPDIR/envfile
535
    _start_socat
536

537
    (timeout --foreground -v --kill=10 30 $PODMAN system service -t0 $podman_socket &)
538

539
    wait_for_file $_SOCAT_LOG
540
    local timeout=10
541
    while [[ $timeout -gt 0 ]]; do
542
        run cat $_SOCAT_LOG
543
        # The 'echo's help us debug failed runs
544
        echo "socat log:"
545
        echo "$output"
546

547
        if [[ "$output" =~ "READY=1" ]]; then
548
            break
549
        fi
550
        timeout=$((timeout - 1))
551
        assert $timeout -gt 0 "Timed out waiting for podman-system-service to send expected data over NOTIFY_SOCKET"
552
        sleep 0.5
553
    done
554

555
    assert "$output" =~ "MAINPID=.*
556
READY=1" "podman-system-service sends expected data over NOTIFY_SOCKET"
557
    mainpid=${lines[0]:8}
558

559
    # Now pull remotely and make sure that the service does _not_ extend the
560
    # timeout; the NOTIFY_SOCKET should be unset at that point.
561
    run_podman --url $podman_socket pull $registry_flags $image_on_local_registry
562

563
    run cat $_SOCAT_LOG
564
    # The 'echo's help us debug failed runs
565
    echo "socat log:"
566
    echo "$output"
567
    assert "$output" !~ "EXTEND_TIMEOUT_USEC="
568

569
    # Give the system-service 5sec to terminate before killing it.
570
    /bin/kill --timeout 5000 KILL --signal TERM $mainpid
571
    run_podman rmi $image_on_local_registry
572
    _stop_socat
573
}
574
# vim: filetype=sh
575