glusterfs

1
# clang-format off
2
checkpoint_time="$(date +%s%N)"
3

4
M0=${M0:=/mnt/glusterfs/0};   # 0th mount point for FUSE
5
M1=${M1:=/mnt/glusterfs/1};   # 1st mount point for FUSE
6
M2=${M2:=/mnt/glusterfs/2};   # 2nd mount point for FUSE
7
M3=${M3:=/mnt/glusterfs/3};   # 3rd mount point for FUSE
8
N0=${N0:=/mnt/nfs/0};         # 0th mount point for NFS
9
N1=${N1:=/mnt/nfs/1};         # 1st mount point for NFS
10
V0=${V0:=patchy};             # volume name to use in tests
11
V1=${V1:=patchy1};            # volume name to use in tests
12
GMV0=${GMV0:=primary};	      # primary volume name to use in geo-rep tests
13
GSV0=${GSV0:=secondary};      # secondary volume name to use in geo-rep tests
14
GSV1=${GSV1:=secondary1}      # secondary volume name to use in geo-rep tests
15
B0=${B0:=/d/backends};        # top level of brick directories
16
DEVDIR=${DEVDIR:=/d/dev} # directory for loop device management
17
WORKDIRS="$B0 $M0 $M1 $M2 $M3 $N0 $N1 $DEVDIR"
18

19
ROOT_GFID="00000000-0000-0000-0000-000000000001"
20
DOT_SHARD_GFID="be318638-e8a0-4c6d-977d-7a937aa84806"
21

22
META_VOL=${META_VOL:=gluster_shared_storage}; # shared gluster storage volume used by snapshot scheduler, nfs ganesha and geo-rep.
23
META_MNT=${META_MNT:=/var/run/gluster/shared_storage}; # Mount point of shared gluster volume.
24

25
CC=cc
26
OSTYPE=$(uname -s)
27
GFREG_ID="$(hostname -s)"
28

29
env_dir=$(dirname $0)
30
while true; do
31
        ENV_RC=${env_dir}/env.rc
32
        if [ -f ${ENV_RC} ]; then
33
                break
34
        fi
35
        new_dir=$(dirname $env_dir)
36
        if [ x"$new_dir" = x"$old_dir" ]; then
37
                ENV_RC="/not/found"
38
                break
39
        fi
40
        old_dir=$env_dir
41
        env_dir=$new_dir
42
done
43

44
if [ ! -f $ENV_RC ]; then
45
   echo "Aborting." | tee /dev/stderr
46
   echo | tee /dev/stderr
47
   echo "env.rc not found" | tee /dev/stderr
48
   echo | tee /dev/stderr
49
   echo "Please correct the problem and try again." | tee /dev/stderr
50
   echo | tee /dev/stderr
51
   exit 1
52
fi
53
. $ENV_RC
54

55
H0=${H0:=`ip -o -4 addr | grep -v "\<lo\>" | awk '{print $4}' | cut -d/ -f1 | head -n 1`}; # hostname
56
MOUNT_TYPE_FUSE="fuse.glusterfs"
57
GREP_MOUNT_OPT_RO="grep (ro"
58
GREP_MOUNT_OPT_RW="grep (rw"
59
UMOUNT_F="umount -f"
60

61
PATH=$PATH:${PWD}/tests/utils
62

63
case $OSTYPE in
64
Linux)
65
  H0=${H0:=`hostname --fqdn`}; # hostname
66
  ;;
67
NetBSD)
68
  MOUNT_TYPE_FUSE="puffs|perfuse|fuse.glusterfs"
69
  GREP_MOUNT_OPT_RO="grep (read-only"
70
  GREP_MOUNT_OPT_RW="grep -v (read-only"
71
  UMOUNT_F="umount -f -R"
72
  ;;
73
*)
74
  ;;
75
esac
76

77
DEBUG=${DEBUG:=0}             # turn on debugging?
78

79
MIGRATION_START_TIMEOUT=5
80
PROCESS_DOWN_TIMEOUT=5
81
PROCESS_UP_TIMEOUT=45
82
NFS_EXPORT_TIMEOUT=20
83
CHILD_UP_TIMEOUT=20
84
PROBE_TIMEOUT=60
85
PEER_SYNC_TIMEOUT=20
86
REBALANCE_TIMEOUT=600
87
REOPEN_TIMEOUT=20
88
HEAL_TIMEOUT=80
89
IO_HEAL_TIMEOUT=120
90
MARKER_UPDATE_TIMEOUT=20
91
JANITOR_TIMEOUT=60
92
UMOUNT_TIMEOUT=5
93
CONFIG_UPDATE_TIMEOUT=5
94
AUTH_REFRESH_INTERVAL=10
95
GRAPH_SWITCH_TIMEOUT=10
96
UNLINK_TIMEOUT=5
97
MDC_TIMEOUT=5
98
IO_WAIT_TIMEOUT=5
99
DISK_FAIL_TIMEOUT=80
100

101
LOGDIR=$(gluster --print-logdir)
102

103
statedumpdir=`gluster --print-statedumpdir`; # Default directory for statedump
104

105
CLI="gluster --mode=script --wignore";
106
CLI_NO_FORCE="gluster --mode=script";
107

108
# CLI_IGNORE_PARTITION makes sure that the warning related to bricks being on
109
# root partition is ignored while running the command in a "no force" mode
110
CLI_IGNORE_PARTITION="gluster --mode=script --wignore-partition"
111

112
function wait_delay() {
113
        local delay="$1"
114
        local interval="$2"
115
        shift 2
116
        local deadline="$(($(date +%s%N) + ${delay}000000000))"
117

118
        $*
119
        while [[ $? -ne 0 ]]; do
120
                if [[ $(date +%s%N) -ge ${deadline} ]]; then
121
                        return 1
122
                fi
123
                sleep ${interval}
124
                $*
125
        done
126

127
        return 0
128
}
129

130
_GFS () {
131
	glusterfs "$@"
132
	local mount_ret=$?
133
	if [ $mount_ret != 0 ]; then
134
		return $mount_ret
135
	fi
136
	local mount_point=${!#}
137
	local i=0
138
	while true; do
139
		touch $mount_point/xy_zzy 2> /dev/null && break
140
		i=$((i+1))
141
		[ $i -lt 100 ] || break
142
		sleep 0.1
143
	done
144
	rm -f $mount_point/xy_zzy
145
	return $mount_ret
146
}
147
GFS="_GFS --attribute-timeout=0 --entry-timeout=0";
148

149
mkdir -p $WORKDIRS
150

151
case $OSTYPE in
152
FreeBSD | Darwin)
153
wc () {
154
   if test "x$1" = "x-l"; then
155
      awk '{ lines++ } END {print lines}'
156
   fi
157
   if test "x$1" = "x-w"; then
158
      awk '{ words += NF } END {print words}' }
159
   fi
160
   if test "x$1" = "x-c"; then
161
      awk '{ chars += length($0) + 1 } END {print chars}'
162
   fi
163
   if test "x$1" = "x-m"; then
164
      awk '{ chars += length($0) + 1 } END {print chars}'
165
   fi
166
}
167
;;
168
NetBSD)
169
wc() {
170
    /usr/bin/wc $@ | sed 's/^ *\([0-9]*\).*$/\1/g'
171
}
172
;;
173
esac
174

175
testcnt=`egrep '^[[:space:]]*(EXPECT|EXPECT_NOT|TEST|TEST_WITHIN|EXPECT_WITHIN|EXPECT_KEYWORD)[[:space:]]' $0 | wc -l`
176
expect_tests=`egrep '^[[:space:]]*TESTS_EXPECTED_IN_LOOP[[:space:]]*' $0`
177

178
x_ifs=$IFS
179
IFS=$'\n'
180
for line in $expect_tests; do
181
        expect_tests=`echo $line | cut -f 2 -d =`
182
        testcnt=`expr $testcnt + $expect_tests`
183
done
184
IFS=$x_ifs
185

186
echo "1..`echo $testcnt`"
187

188
t=1
189

190
function dbg()
191
{
192
        [ "x$DEBUG" = "x0" ] || echo "$*" >&2;
193
}
194

195
function G_LOG()
196
{
197
     local g_log_logdir;
198
     g_log_logdir=`$CLI --print-logdir`
199
     test -d $g_log_logdir
200
     if [ $? != 0 ]; then
201
        return
202
     fi
203
     local g_log_string;
204
     g_log_string="++++++++++ G_LOG:$0: TEST: $@ ++++++++++"
205
     g_log_string="`date -u +["%F %T.%6N"]`:$g_log_string"
206
     local g_log_filename
207
     for  g_log_filename in `find $g_log_logdir/ -type f -name \*.log`;
208
        do
209
                echo "$g_log_string" >> "$g_log_filename"
210
        done
211
}
212

213
function test_header()
214
{
215
        dbg "=========================";
216
        dbg "TEST $t (line $TESTLINE): $*";
217
        saved_cmd="$*"
218
        start_time="$(date +%s%N)"
219
}
220

221

222
function test_footer()
223
{
224
        RET=$?
225
        local lineno=$1
226
        local err=$2
227
        local end_time
228
        local elapsed1
229
        local elapsed2
230

231
        end_time="$(date +%s%N)"
232
        elapsed1="$(((start_time - checkpoint_time) / 1000000))"
233
        elapsed2="$(((end_time - start_time) / 1000000))"
234
        checkpoint_time="$end_time"
235
        if [ $RET -eq 0 ]; then
236
                printf "ok %3d [%7d/%7d] <%4d> '%s'\n" "$t" "$elapsed1" "$elapsed2" "$lineno" "$saved_cmd";
237
        else
238
                printf "not ok %3d [%7d/%7d] <%4d> '%s' -> '%s'\n" "$t" "$elapsed1" "$elapsed2" "$lineno" "$saved_cmd" "$err"
239
                if [ "$EXIT_EARLY" = "1" ]; then
240
			cleanup
241
                        exit $RET
242
                fi
243
        fi
244

245
        dbg "RESULT $t: $RET";
246

247
        t=`expr $t + 1`;
248
}
249

250
function test_expect_footer()
251
{
252
        local lineno=$1
253
        local e=$2
254
        local a=$3
255
        local err=""
256

257
        if ! [[ "$a" =~ $e ]]; then
258
                err="Got \"$a\" instead of \"$e\""
259
        fi
260
        [[ "$a" =~ $e ]];
261

262
        test_footer "$lineno" "$err";
263
}
264

265
function _EXPECT()
266
{
267
        TESTLINE=$1;
268
        shift;
269
        local a=""
270

271
        G_LOG $TESTLINE "$@";
272
        test_header "$@";
273

274
        e="$1";
275
        shift;
276
        a=$("$@" | tail -1)
277

278
        if [ "x$e" = "x" ] ; then
279
                test_expect_footer "$TESTLINE" "x$e" "x$a";
280
        else
281
                test_expect_footer "$TESTLINE" "$e" "$a";
282
        fi
283
}
284

285
function test_expect_not_footer()
286
{
287
        local lineno=$1
288
        local e=$2
289
        local a=$3
290
        local err=""
291

292
        if [[ "$a" =~ $e ]]; then
293
                err="Got \"$a\" when not expecting it"
294
        fi
295

296
        ! [[ "$a" =~ $e ]];
297
        test_footer "$lineno" "$err";
298
}
299

300
function _EXPECT_NOT()
301
{
302
        TESTLINE=$1;
303
        shift;
304
        local a=""
305

306
        G_LOG $TESTLINE "$@";
307
        test_header "$@";
308

309
        e="$1";
310
        shift;
311
        a=$("$@" | tail -1)
312

313
        if [ "x$e" = "x" ] ; then
314
               test_expect_not_footer "$TESTLINE" "x$e" "x$a";
315
        else
316
               test_expect_not_footer "$TESTLINE" "$e" "$a";
317
        fi
318
}
319

320
function _EXPECT_KEYWORD()
321
{
322
        TESTLINE=$1;
323
        shift;
324
        G_LOG $TESTLINE "$@";
325
        test_header "$@";
326

327
        e="$1";
328
        shift;
329
        "$@" | tail -1 | grep -q "$e"
330

331
        test_footer "$TESTLINE";
332
}
333

334
function _TEST()
335
{
336
        TESTLINE=$1;
337
        shift;
338
        local redirect=""
339

340
        G_LOG $TESTLINE "$@";
341
        test_header "$@";
342

343
        if [ "$1" = "!" ]; then
344
                redirect="2>&1"
345
        fi
346

347
        eval "$@" >/dev/null $redirect
348

349
        test_footer "$TESTLINE";
350
}
351

352

353
function _TEST_WITHIN()
354
{
355
        local res output
356

357
        TESTLINE=$1
358
        shift;
359

360
        local timeout=$1
361
        shift;
362

363
        G_LOG $TESTLINE "$@";
364
        test_header "$@"
365

366
        local endtime="$(( ${timeout}000000000 + $(date +%s%N) ))"
367

368
        # We *want* this to be globally visible.
369
        EW_RETRIES=0
370

371
        output=""
372
        res="1"
373
        while [[ "$(date +%s%N)" < "$endtime" ]]; do
374
                output="$("${@}" 2>/dev/null)"
375
                res="${?}"
376
                if [[ "${res}" == "0" ]]; then
377
                        break
378
                fi
379
                sleep 0.25;
380
                EW_RETRIES=$((EW_RETRIES+1))
381
        done
382

383
        ! [[ "${res}" != "0" ]]
384

385
        test_footer "$TESTLINE"
386

387
        TEST_OUTPUT="${output}"
388
}
389

390
#This function should be used carefully.
391
#The expected regex, given to this function, should be
392
#used within ^ and $ to match exactly with the output of
393
#command.
394
function _EXPECT_WITHIN()
395
{
396
        TESTLINE=$1
397
        shift;
398

399
        local timeout=$1
400
        shift;
401

402
        G_LOG $TESTLINE "$@";
403
        test_header "$@"
404

405
        e=$1;
406
        a="";
407
        shift;
408

409
        local endtime="$(( ${timeout}000000000 + $(date +%s%N) ))"
410

411
        # We *want* this to be globally visible.
412
        EW_RETRIES=0
413

414
        while [[ "$(date +%s%N)" < "$endtime" ]]; do
415
                a=$("$@" | tail -1 ; exit ${PIPESTATUS[0]})
416
                ## Check command success
417
                if [ $? -ne 0 ]; then
418
                        break;
419
                fi
420
                ## Check match success
421
                if [[ "$a" =~ $e ]]; then
422
                        break;
423
                fi
424
                sleep 0.25;
425
                EW_RETRIES=$((EW_RETRIES+1))
426
        done
427

428
        if [ "x$e" = "x" ] ; then
429
               test_expect_footer "$TESTLINE" "x$e" "x$a";
430
        else
431
               test_expect_footer "$TESTLINE" "$e" "$a";
432
        fi
433
}
434

435

436
function SKIP_TESTS()
437
{
438
        dbg "Skipping tests $t-$testcnt";
439
        while [ $t -le $testcnt ]; do
440
                true ; test_footer;
441
        done
442
}
443

444

445
function _TEST_IN_LOOP()
446
{
447
        testcnt=`expr $testcnt + 1`;
448
        _TEST $@
449
}
450

451
function _EXPECT_WITHIN_TEST_IN_LOOP()
452
{
453
        testcnt=`expr $testcnt + 1`;
454
        _EXPECT_WITHIN $@
455
}
456

457
which killall > /dev/null || {
458
  killall() {
459
    pkill $@
460
  }
461
}
462

463
which pidof > /dev/null || {
464
  pidof() {
465
    $PYTHON pidof.py $@
466
  }
467
}
468

469
stat -c %s /dev/null > /dev/null 2>&1 || {
470
  stat() {
471
    local format=""
472
    local f=""
473

474
    if [ "x$1" = "x-c" ] ; then
475
      oformat=$2
476
      shift
477
      shift
478
      files=$@
479
    else
480
      files=$@
481
    fi
482

483
    for f in $files ; do
484
        format=$oformat
485

486
      # %t/%T should return 0 for non devices.
487
      case "${format}" in
488
      *%t*|*%T*)
489
        `which stat` -f '%HT' $f | grep -q 'Device$' || \
490
           format=`echo "${format}" | sed 's/%t/0/g; s/%T/0/g;'`
491
        ;;
492
      *)
493
        ;;
494
      esac
495

496
      if [ "x${format}" = "x" ] ; then
497
        `which stat` $f
498
      else
499
         cmd=""
500
         case $format in
501
         *%u*) cmd="${cmd} s/%u/`$( which stat ) -f %u $f`/g;" ;&
502
         *%g*) cmd="${cmd} s/%g/`$( which stat ) -f %g $f`/g;" ;&
503
         *%a*) cmd="${cmd} s/%a/`$( which stat ) -f %p $f |
504
                    sed 's/^..//; s/^0//'`/g;" ;&
505
         *%A*) cmd="${cmd} s/%A/`ls -ld $f|awk '{print $1}'`/g;" ;&
506
         *%s*) cmd="${cmd} s/%s/`$( which stat ) -f %z $f`/g;" ;&
507
         *%h*) cmd="${cmd} s/%h/`$( which stat ) -f %l $f`/g;" ;&
508
         *%F*) cmd="${cmd} s/%F/`$( which stat ) -f %HT $f | sed '
509
            s/Directory/directory/;
510
            s/Fifo File/fifo/;
511
            s/Symbolic Link/symbolic link/;
512
            s/Regular File/regular file/;
513
            s/Block Device/block special file/;
514
            s/Character Device/character special file/;
515
           ' | sed \"$(
516
             test -s $f || echo 's/regular file/regular empty file/g'
517
            )\"`/g;" ;&
518
         *%n*) cmd="${cmd} s|%n|`$( which stat ) -f %N $f`|g;" ;&
519
         *%Y*) cmd="${cmd} s/%Y/`$( which stat ) -f %m $f`/g;" ;&
520
         *%X*) cmd="${cmd} s/%X/`$( which stat ) -f %a $f`/g;" ;&
521
         *%Z*) cmd="${cmd} s/%Z/`$( which stat ) -f %c $f`/g;" ;&
522
         *%.Z*) cmd="${cmd} s/%.Z/`$( which stat ) -f %.9Fc $f`/g;" ;&
523
         *%b*) cmd="${cmd} s/%b/`$( which stat ) -f %b $f`/g;" ;&
524
         *%B*) cmd="${cmd} s/%B/512/g;" ;&
525
         *%t*) cmd="${cmd} s/%t/`$( which stat ) -f %XHr $f`/g;" ;&
526
         *%T*) cmd="${cmd} s/%T/`$( which stat ) -f %XLr $f`/g;" ;&
527
         esac
528

529
         `which stat` -f "`echo $format|sed \"$cmd\"`" $f
530
      fi
531
    done
532
  }
533
}
534

535
function signal_pids() {
536
        local sig="$1"
537
        shift
538
        local pids=($*)
539

540
        if [[ ${#pids[@]} -gt 0 ]]; then
541
                kill -${sig} ${pids[@]} 2>/dev/null || true
542
        fi
543
}
544

545
function check_pids() {
546
        local pids=($*)
547
        local tmp=()
548
        local pid
549

550
        for pid in "${pids[@]}"; do
551
                kill -0 "${pid}" 2>/dev/null && tmp+=(${pid})
552
        done
553

554
        echo "${tmp[@]}"
555
}
556

557
function pids_alive() {
558
        local pids=($*)
559

560
        if [[ "$(check_pids ${pids[@]})" != "" ]]; then
561
                return 1;
562
        fi
563

564
        return 0
565
}
566

567
function terminate_pids() {
568
        local pids=($*)
569

570
        signal_pids TERM ${pids[@]}
571
        wait_delay ${PROCESS_DOWN_TIMEOUT} 0.1 pids_alive ${pids[@]}
572
        if [[ $? -ne 0 ]]; then
573
                pids=($(check_pids ${pids[@]}))
574
                signal_pids KILL ${pids[@]}
575
                wait_delay 1 0.1 pids_alive ${pids[@]}
576
                if [[ $? -ne 0 ]]; then
577
                        return 2
578
                fi
579

580
                return 1
581
        fi
582

583
        return 0
584
}
585

586
function process_pids() {
587
        local proc
588
        local pids=()
589

590
        for proc in $*; do
591
                pids+=($(pgrep ${proc}))
592
        done
593

594
        echo "${pids[@]}"
595
}
596

597
## Lock files should get automatically removed once "usradd" or "groupadd"
598
## command finishes. But sometimes we encounter situations (bugs) where
599
## some of these files may not get properly unlocked after the execution of
600
## the command. In that case, when we execute useradd next time, it may show
601
## the error “cannot lock /etc/password” or “unable to lock group file”.
602
## So, to avoid any such errors, check for any lock files under /etc.
603
## and remove those.
604

605
function remove_lock_files()
606
{
607
        if [ ! -f /etc/passwd.lock ];
608
        then
609
                rm -rf /etc/passwd.lock;
610
        fi
611

612
        if [ ! -f /etc/group.lock ];
613
        then
614
                rm -rf /etc/group.lock;
615
        fi
616

617
        if [ ! -f /etc/shadow.lock ];
618
        then
619
                rm -rf /etc/shadow.lock;
620
        fi
621

622
        if [ ! -f /etc/gshadow.lock ];
623
        then
624
                rm -rf /etc/gshadow.lock;
625
        fi
626
}
627

628

629
function cleanup()
630
{
631
        local end_time
632

633
        # Prepare flags for umount
634
        case `uname -s` in
635
        Linux)
636
                flag="-l"
637
                ;;
638
        NetBSD)
639
                flag="-f -R"
640
                ;;
641
        FreeBSD|Darwin)
642
                flag="-f"
643
                ;;
644
        *)
645
                flag=""
646
                ;;
647
        esac
648

649
        # Clean up lock files.
650
        remove_lock_files
651

652
        # Clean up all client mounts
653
        for m in `mount | grep fuse.glusterfs | awk '{print $3}'`; do
654
                umount $flag $m
655
        done
656

657
        # Unmount all well known mount points
658
        umount $flag $M0 2>/dev/null || umount -f $M0 2>/dev/null || true;
659
        umount $flag $M1 2>/dev/null || umount -f $M1 2>/dev/null || true;
660
        umount $flag $M2 2>/dev/null || umount -f $M2 2>/dev/null || true;
661
        umount $flag $N0 2>/dev/null || umount -f $N0 2>/dev/null || true;
662
        umount $flag $N1 2>/dev/null || umount -f $N1 2>/dev/null || true;
663

664

665
        # unmount all stale mounts from /tmp, This is a temporary work around
666
        # till the stale mount in /tmp is found.
667
        umount $flag /tmp/mnt* 2>/dev/null
668

669

670
        # Send SIGTERM to all gluster processes and rpc.statd that are still running
671
        terminate_pids $(process_pids glusterfs glusterfsd glusterd rpc.statd)
672

673
        test x"$OSTYPE" = x"NetBSD" && pkill -9 perfused || true
674

675
        # unregister nfs and related services from portmapper/rpcbind
676
        ## nfs
677
        rpcinfo -d 100003 3 2>/dev/null || true;
678
        ## mountd
679
        rpcinfo -d 100005 1 2>/dev/null || true;
680
        rpcinfo -d 100005 3 2>/dev/null || true;
681
        ## nlockmgr
682
        rpcinfo -d 100021 1 2>/dev/null || true;
683
        rpcinfo -d 100021 4 2>/dev/null || true;
684
        ## nfs_acl
685
        rpcinfo -d 100227 3 2>/dev/null || true;
686

687
        # unmount brick filesystems after killing daemons
688
        MOUNTPOINTS=`findmnt -nRlT "${B0}" -o TARGET,SOURCE | grep "$B0/" | awk '{print $2}'`
689
        for m in $MOUNTPOINTS;
690
        do
691
                umount $flag $m
692
        done
693

694
        # Cleanup lvm
695
        type cleanup_lvm &>/dev/null && cleanup_lvm || true;
696

697
        # Destroy loop devices
698
        # TODO: This should be a function DESTROY_LOOP
699
        case `uname -s` in
700
        Linux)
701
                for l in $(ls ${DEVDIR}/loop* 2>/dev/null); do
702
                        losetup -d "${l}"
703
                        rm -f "${l}"
704
                done
705
                ;;
706
        NetBSD)
707
                # cleanup loopback device with unmounted backing store
708
                for vnd in /dev/vnd* ; do
709
                        vnconfig -l ${vnd} 2>&1 | \
710
                            grep -q 'Bad file descriptor' && vnconfig -u ${vnd}
711
                done
712

713
                vnd=`vnconfig -l | \
714
                     awk '!/not in use/{printf("%s%s:%d ", $1, $2, $5);}'`
715
                for l in ${vnd} ; do
716
                        dev=${l%%:*}
717
                        tmp=${l#*:}
718
                        fs=${tmp%%:*}
719
                        inode=${tmp#*:}
720
                        file=`find -x ${fs} -inum ${inode} -print -exit`
721
                        echo ${file} | grep "$B0/" && \
722
                            LOOPDEVICES="${LOOPDEVICES} $dev"
723
                done
724
                for l in $LOOPDEVICES;
725
                do
726
                        vnconfig -u $l
727
                done
728
                ;;
729
        *)
730
                echo "`uname -s` loopback device supportmissing"
731
                ;;
732
        esac
733

734
        # remove contents of "GLUSTERD_WORKDIR" except hooks and groups
735
        # directories.
736
        if [ -n $GLUSTERD_WORKDIR ]
737
        then
738
                find  $GLUSTERD_WORKDIR/* -maxdepth 0 -name 'hooks' -prune \
739
                -o -name 'groups' -prune -o -exec rm -rf '{}' ';'
740
        else
741
                echo "GLUSTERD_WORKDIR is not set"
742
        fi
743

744
        # Complete cleanup time
745
        rm -rf "$B0/*" "/etc/glusterd/*";
746
        rm -rf $WORKDIRS
747
        find $GLUSTERD_PIDFILEDIR -name "*.pid" | xargs rm -rf
748
        leftover=""
749
        for d in $WORKDIRS ; do
750
                if test -d $d ; then
751
                       leftover="$leftover $d"
752
                fi
753
        done
754
        if [ "x$leftover" != "x" ] ; then
755
                echo "Aborting."
756
                echo
757
                echo "$d could not be deleted, here are the left over items"
758
                for d in $leftover; do
759
                        find $d -exec ls -ld {} \;
760
                done
761
                echo
762
                echo "Please correct the problem and try again."
763
                echo
764
                return 1;
765
        fi >&2
766

767
        # Physically release unused space
768
        fstrim /d
769

770
        mkdir -p $WORKDIRS
771
	# This is usually the last thing a test script calls, so our return
772
	# value becomes their exit value.  While it's not great for the mkdir
773
	# above to fail, promoting that into a failure of the whole test (and
774
	# thus of an entire regression-test run) seems a bit excessive.  Make
775
	# sure we return good status anyway.
776

777
        return 0
778
}
779

780
function force_terminate () {
781
        local ret=$?;
782
        >&2 echo -e "\nreceived external"\
783
                        "signal --`kill -l $ret`--, calling 'cleanup' ...\n";
784
        cleanup;
785
        exit $ret;
786
}
787

788
trap force_terminate INT TERM HUP
789

790
function cleanup_tester ()
791
{
792
    local exe=$1
793
    rm -f $exe
794
}
795

796
function build_tester ()
797
{
798
    local cfile=$1
799
    local fname=$(basename "$cfile")
800
    local ext="${fname##*.}"
801
    local execname="${fname%.*}"
802
    shift
803
    local cflags=$*
804
    if [ `echo $cflags | grep -c "lgfapi" ` -gt 0 ]
805
    then
806
       cflags="$cflags $(pkg-config glusterfs-api --cflags-only-I --libs-only-L)"
807
    fi
808
    $CC -g -o $(dirname $cfile)/$execname $cfile $cflags
809
}
810

811
function process_leak_count ()
812
{
813
    local pid=$1;
814
    return $(ls -lh /proc/$pid/fd | grep "(deleted)"| wc -l)
815
}
816

817
which truncate > /dev/null || {
818
  truncate() {
819
    local nocreate=0
820
    local ioblocks=0
821
    local fileref=""
822
    local newsize=""
823

824
    args=`getopt xor:s: $*`
825
    if [ $? -ne 0 ]; then
826
      echo 'Usage: truncate [-co](-r file | -s size) file ...'
827
      exit 2
828
    fi
829
    set -- $args
830
    while [ $# -gt 0 ]; do
831
      case "$1" in
832
      -c)
833
        nocreate=1;
834
        ;;
835
      -o)
836
        ioblocks=1;
837
        echo "Unimplemented -o option"
838
        exit 2
839
        ;;
840
      -r)
841
        fileref=$2;
842
        shift;
843
        ;;
844
      -s)
845
        newsize=$2;
846
        shift;
847
        ;;
848
      --)
849
        shift;
850
        break;
851
        ;;
852
      *)
853
        echo 'Usage: truncate [-co](-r file | -s size) file ...'
854
        exit 2;
855
        ;;
856
      esac
857
      shift
858
    done
859

860
    if [ "x$newsize" = "x" -a "x$fileref" = "x" ] ; then
861
      echo 'Usage: truncate [-co](-r file | -s size) file ...'
862
      exit 2;
863
    fi
864

865
    if [ "x$newsize" != "x" -a "x$fileref" != "x" ] ; then
866
      echo 'Usage: truncate [-co](-r file | -s size) file ...'
867
      exit 2;
868
    fi
869

870
    if [ "x$newsize" != "x" ] ; then
871
      echo $newsize | grep -q '^[-_<>%/]' && {
872
        echo "Unimplemented prefix in ${newsize}"
873
        exit 2;
874
      }
875

876
      echo $newsize | egrep -q '[TPEZY]B?$' && {
877
        echo "Unit not implemented for ${newsize}"
878
        exit 2;
879
      }
880

881
      case $newsize in
882
      *KB)
883
        newsize=$(( ${newsize/KB/} * 1000 ))
884
        ;;
885
      *K)
886
        newsize=$(( ${newsize/K/} * 1024 ))
887
        ;;
888
      *MB)
889
        newsize=$(( ${newsize/MB/} * 1000 * 1000 ))
890
        ;;
891
      *M)
892
        newsize=$(( ${newsize/M/} * 1024 * 1024 ))
893
        ;;
894
      *GB)
895
        newsize=$(( ${newsize/GB/} * 1000 * 1000 * 1000 ))
896
        ;;
897
      *G)
898
        newsize=$(( ${newsize/G/} * 1024 * 1024 * 1024 ))
899
        ;;
900
      esac
901

902
    fi
903

904
    if [ "x$fileref" != "x" ] ; then
905
       if [ ! -f $fileref ] ; then
906
         echo "File does not exists: ${fileref}"
907
         exit 2;
908
       fi
909
       newsize=`ls -l ${fileref}|awk '{print $5}'`
910
    fi
911

912
    if [ $# -eq 0 ]; then
913
      echo 'Usage: truncate [-co](-r file | -s size) file ...'
914
      exit 2;
915
    fi
916

917
    for f in $* ; do
918
      if [ "x$nocreate" = "x1" -a ! -f $f ] ; then
919
        continue;
920
      fi
921

922
      dd bs=1 seek=$newsize if=/dev/null of=$f msgfmt=quiet
923
    done
924
  }
925
}
926

927
which md5sum > /dev/null || {
928
  md5sum() {
929
    for f in $* ; do
930
      md5 $f | awk -F'[() ]' '{printf("%s  %s\n", $6, $3)}'
931
    done
932
  }
933
}
934

935
which setfattr > /dev/null || {
936
  setfattr() {
937
    $PYTHON setfattr.py $@
938
  }
939
}
940

941
which getfattr > /dev/null || {
942
  getfattr() {
943
    $PYTHON getfattr.py $@
944
  }
945
}
946

947
which sha1sum > /dev/null || {
948
  sha1sum() {
949
  case $OSTYPE in
950
  Darwin)
951
    for f in $* ; do
952
      openssl sha1 $f | awk -F'[() ]' '{printf("%s %s\n", $4, $2)}'
953
    done
954
    ;;
955
  NetBSD | FreeBSD)
956
    for f in $* ; do
957
      sha1 $f | awk -F'[() ]' '{printf("%s  %s\n", $6, $3)}'
958
    done
959
    ;;
960
  esac
961
  }
962
}
963

964
userdel --help 2>/dev/null | grep -q -- '--force' || {
965
  userdel() {
966
    if [ "x$1" = "x--force" ]; then
967
      user=$2
968
    else
969
      user=$1
970
    fi
971
    eval "$( which userdel ) $user"
972
  }
973
}
974

975
useradd --help 2>/dev/null | grep -q -- '--no-create-home' || {
976
  useradd() {
977
    # Just remove -M (do not create home) which is the default
978
    # other options are identical
979
    args=`echo $*|sed 's/-M//'`
980
    eval "$( which useradd ) $args"
981
  }
982
}
983

984
userdel --help 2>/dev/null | grep -q -- '--force' || {
985
  userdel() {
986
    if [ "x$1" = "x--force" ]; then
987
      user=$2
988
    else
989
      user=$1
990
    fi
991
    eval "$( which userdel ) $user"
992
  }
993
}
994

995
useradd --help 2>/dev/null | grep -q -- '--no-create-home' || {
996
  useradd() {
997
    # Just remove -M (do not create home) which is the default
998
    # other options are identical
999
    args=`echo $*|sed 's/-M//'`
1000
    eval "$( which useradd ) $args"
1001
  }
1002
}
1003

1004
DBG_TEST () {
1005
        read -p "execute \"$*\"? " x;
1006
        case $x in
1007
        'y')
1008
                _TEST "$@"
1009
                ;;
1010
        'q')
1011
                exit 0
1012
                ;;
1013
        *)
1014
                echo "skipping"
1015
                ;;
1016
        esac
1017
}
1018

1019
alias EXPECT='_EXPECT $LINENO'
1020
alias EXPECT_NOT='_EXPECT_NOT $LINENO'
1021
if [ -n "$GF_INTERACTIVE" ]; then
1022
	alias TEST='DBG_TEST $LINENO'
1023
else
1024
	alias TEST='_TEST $LINENO'
1025
fi
1026
alias TEST_WITHIN='_TEST_WITHIN $LINENO'
1027
alias EXPECT_WITHIN='_EXPECT_WITHIN $LINENO'
1028
alias EXPECT_KEYWORD='_EXPECT_KEYWORD $LINENO'
1029
alias TEST_IN_LOOP='_TEST_IN_LOOP $LINENO'
1030
alias EXPECT_WITHIN_TEST_IN_LOOP='_EXPECT_WITHIN_TEST_IN_LOOP $LINENO'
1031
shopt -s expand_aliases
1032

1033
if [ x"$OSTYPE" = x"Linux" ]; then
1034
        alias dd="dd status=none"
1035
elif [ x"$OSTYPE" = x"NetBSD" ]; then
1036
        alias dd="dd msgfmt=quiet"
1037
fi
1038
# MacOS doesn't seem to support either option.  Doing nothing at all is
1039
# probably the safest option there and on anything we don't recognize, but
1040
# if you want to reduce the noise level and know the correct option for
1041
# your favorite platform please feel free to add it here.
1042

1043
function SETUP_LOOP ()
1044
{
1045
  if [ $# != 1 ] ; then
1046
    echo "SETUP_LOOP usage" >&2
1047
    return 1;
1048
  fi
1049

1050
  backend=$1
1051

1052
  case ${OSTYPE} in
1053
  Linux)
1054
    dev="$(losetup --find --show ${backend})"
1055
    ln -sf "${dev}" "${DEVDIR}/$(basename "${dev}")"
1056
    echo "${dev}"
1057
    ;;
1058
  NetBSD)
1059
    vnd=`vnconfig -l|awk -F: '/not in use/{print $1; exit}'`
1060
    if [ "x${vnd}" = "x" ] ; then
1061
      echo "no more vnd" >&2
1062
      return 1;
1063
    fi
1064
    vnconfig ${vnd} ${backend}
1065
    echo ${vnd}
1066
    ;;
1067
  *)
1068
    echo "Please define SETUP_LOOP for ${OSTYPE} in include.rc" >&2
1069
    return 1;
1070
    ;;
1071
  esac
1072
}
1073

1074
function MKFS_LOOP ()
1075
{
1076
  args=`getopt i: $*`
1077
  if [ $? -ne 0 ] ; then
1078
    echo "MKFS_LOOP usage" >&2
1079
    return 1;
1080
  fi
1081
  set -- ${args}
1082

1083
  isize=""
1084
  while test $# -gt 0; do
1085
    case "$1" in
1086
    -i)         isize=$2; shift ;;
1087
    --)         shift; break ;;
1088
    esac
1089
    shift
1090
  done
1091

1092
  dev=$1
1093

1094
  case ${OSTYPE} in
1095
  Linux)
1096
    test "x${isize}" != "x" && isize="-i size=${isize}"
1097
    mkfs.xfs  -f ${isize} ${dev}
1098
    ;;
1099
  NetBSD)
1100
    test "x${isize}" != "x" && isize="-i ${isize}"
1101

1102
    echo ${dev} | grep -q '^vnd'
1103
    if [ $? -ne 0 ] ; then
1104
      vnd=`vnconfig -l|awk -F: '/not in use/{print $1; exit}'`
1105
      if [ "x${vnd}" = "x" ] ; then
1106
        echo "no more vnd" >&2
1107
        return 1;
1108
      fi
1109
      vnconfig ${vnd} ${dev}
1110
    else
1111
      vnd=${dev}
1112
    fi
1113
    newfs ${isize} /dev/r${vnd}a
1114
    ;;
1115
  *)
1116
    echo "Please define MKFS_LOOP for ${OSTYPE} in include.rc" >&2
1117
    return 1;
1118
    ;;
1119
  esac
1120
}
1121

1122
# usage: log_newer timestamp "string"
1123
# search in glusterfs logs for "string" logged after timestamp seconds
1124
# since the Epoch (usually obtained by date +%s)
1125
log_newer()
1126
{
1127
        ts=$1
1128
        msg=$2
1129
        logdir=`$CLI --print-logdir`
1130

1131
        local x_ifs=$IFS
1132
        IFS="["
1133
        for date in `grep -hr "$msg" $logdir | grep -v "G_LOG" | awk -F '[\]]' '{print $1}'` ; do
1134
                if [ `date -d "$date" +%s` -gt $ts ] ; then
1135
                        IFS=$x_ifs
1136
                        return 0;
1137
                fi
1138
        done 2>/dev/null
1139
        IFS=$x_ifs
1140
        return 1
1141
}
1142

1143
function MOUNT_LOOP ()
1144
{
1145
  if [ $# != 2 ] ; then
1146
    echo "MOUNT_LOOP usage" >&2
1147
    return 1;
1148
  fi
1149

1150
  dev=$1
1151
  target=$2
1152

1153
  case ${OSTYPE} in
1154
  Linux)
1155
    echo ${dev} | grep -q '^/dev/loop'
1156
    if [ $? -eq 0 ] ; then
1157
      mount -t xfs  ${dev} ${target}
1158
    else
1159
      mount -o loop  ${dev} ${target}
1160
    fi
1161
    ;;
1162
  NetBSD)
1163
    echo ${dev} | grep -q '^vnd'
1164
    if [ $? -ne 0 ] ; then
1165
      ino=`/usr/bin/stat -f %i ${dev}`
1166
      dev=`vnconfig -l | awk -v ino=${ino} -F'[: ]*' '($5 == ino) {print $1}'`
1167
    fi
1168

1169
    mount /dev/${dev}a ${target} >&2
1170
    if [ $? -ne 0 ] ; then
1171
      echo "failed to mount  /dev/${dev}a on  ${target}" >&2
1172
      return 1;
1173
    fi
1174

1175
    mkdir -p ${target}/.attribute/system  ${target}/.attribute/user
1176
    mount -u -o extattr ${target} >&2
1177

1178
    ;;
1179
  *)
1180
    echo "Please define MOUNT_LOOP for ${OSTYPE} in include.rc" >&2
1181
    return 1;
1182
    ;;
1183
  esac
1184
}
1185

1186
function UMOUNT_LOOP ()
1187
{
1188
  case ${OSTYPE} in
1189
  Linux)
1190
    force_umount $*
1191
    ;;
1192
  NetBSD)
1193
    for target in $* ; do
1194
      dev=`mount | awk -v target=${target} '($3 == target) {print $1}'`
1195
      force_umount ${target}
1196
      echo ${dev} | grep -q '^/dev/vnd'
1197
      if [ $? -eq 0 ] ; then
1198
        dev=`echo ${dev} | sed 's|^/dev/||; s|a$||'`
1199
        vnconfig -u ${dev}
1200
      else
1201
        ino=`/usr/bin/stat -f %i ${dev}`
1202
        dev=`vnconfig -l | awk -v ino=${ino} -F'[: ]*' '($5 == ino) {print $1}'`
1203
        if [ "x${dev}" != "x" ] ; then
1204
          vnconfig -u ${dev}
1205
        fi
1206
      fi
1207
    done
1208
    ;;
1209
  *)
1210
    echo "Please define UMOUNT_LOOP for ${OSTYPE} in include.rc" >&2
1211
    return 1;
1212
    ;;
1213
  esac
1214
}
1215

1216
function STAT()
1217
{
1218
        stat $1
1219
        echo $?
1220
}
1221

1222
function STAT_INO()
1223
{
1224
        local ino=$(stat -c '%i' $1)
1225
        if [ $? -eq 0 ]; then
1226
                echo $ino
1227
        else
1228
                echo 0
1229
        fi
1230
}
1231

1232
function get_md5_sum()
1233
{
1234
    local file=$1;
1235
    md5_sum=$(md5sum $file | awk '{print $1}');
1236
    echo $md5_sum
1237
}
1238