glusterfs

1
declare -A PORTMAP
2
PORTCURR=49152
3
function ta_create_ta_and_volfile()
4
{
5
        local b=$B0/$1
6
        mkdir -p $b/.glusterfs/indices
7
cat > $B0/ta.vol <<EOF
8
volume ta-posix
9
    type storage/posix
10
    option directory $b
11
end-volume
12

13
volume ta-thin-arbiter
14
    type features/thin-arbiter
15
    subvolumes ta-posix
16
end-volume
17

18
volume ta-locks
19
    type features/locks
20
    option notify-contention yes
21
    subvolumes ta-thin-arbiter
22
end-volume
23

24
volume ta-upcall
25
    type features/upcall
26
    option cache-invalidation off
27
    subvolumes ta-locks
28
end-volume
29

30
volume ta-io-threads
31
    type performance/io-threads
32
    subvolumes ta-upcall
33
end-volume
34

35
volume ta-index
36
    type features/index
37
    option xattrop-pending-watchlist trusted.afr.ta-
38
    option xattrop-dirty-watchlist trusted.afr.dirty
39
    option index-base $b/.glusterfs/indices
40
    subvolumes ta-io-threads
41
end-volume
42

43
volume ta-io-stats
44
    type debug/io-stats
45
    option count-fop-hits off
46
    option latency-measurement off
47
    option log-level WARNING
48
    option unique-id $b
49
    subvolumes ta-index
50
end-volume
51

52
volume ta-server
53
    type protocol/server
54
    option transport.listen-backlog 10
55
    option transport.socket.keepalive-count 9
56
    option transport.socket.keepalive-interval 2
57
    option transport.socket.keepalive-time 20
58
    option transport.tcp-user-timeout 0
59
    option transport.socket.keepalive 1
60
    option auth.addr.$b.allow *
61
    option auth-path $b
62
    option transport.address-family inet
63
    option transport-type tcp
64
    subvolumes ta-io-stats
65
end-volume
66
EOF
67
}
68

69
function ta_create_brick_and_volfile()
70
{
71
        local b=$B0/$1
72
        mkdir -p $b/.glusterfs/indices
73
cat > $B0/${1}.vol <<EOF
74
volume ${V0}-posix
75
    type storage/posix
76
    option directory $b
77
end-volume
78

79
volume ${V0}-locks
80
    type features/locks
81
    subvolumes ${V0}-posix
82
end-volume
83

84
volume ${V0}-leases
85
    type features/leases
86
    option leases off
87
    subvolumes ${V0}-locks
88
end-volume
89

90
volume ${V0}-upcall
91
    type features/upcall
92
    option cache-invalidation off
93
    subvolumes ${V0}-leases
94
end-volume
95

96
volume ${V0}-io-threads
97
    type performance/io-threads
98
    subvolumes ${V0}-upcall
99
end-volume
100

101
volume ${V0}-index
102
    type features/index
103
    option xattrop-pending-watchlist trusted.afr.${V0}-
104
    option xattrop-dirty-watchlist trusted.afr.dirty
105
    option index-base $b/.glusterfs/indices
106
    subvolumes ${V0}-io-threads
107
end-volume
108

109
volume $b
110
    type debug/io-stats
111
    option count-fop-hits off
112
    option latency-measurement off
113
    option log-level INFO
114
    option unique-id $b
115
    subvolumes ${V0}-index
116
end-volume
117

118
volume ${V0}-server
119
    type protocol/server
120
    option transport.listen-backlog 1024
121
    option transport.socket.keepalive-count 9
122
    option transport.socket.keepalive-interval 2
123
    option transport.socket.keepalive-time 20
124
    option transport.tcp-user-timeout 0
125
    option transport.socket.keepalive 1
126
    option auth.addr.$b.allow *
127
    option auth-path $b
128
    option auth.login.459d48e8-2a92-4f11-89f2-077b29f6f86d.password a0ad63dd-8314-4f97-9160-1b93e3cb1f0b
129
    option auth.login.$b.allow 459d48e8-2a92-4f11-89f2-077b29f6f86d
130
    option transport.address-family inet
131
    option transport-type tcp
132
    subvolumes $b
133
end-volume
134
EOF
135
}
136

137
function ta_set_port_by_name()
138
{
139
        if [ -z ${PORTMAP[$1]} ]
140
        then
141
                PORTMAP[$1]=$PORTCURR
142
                PORTCURR=$((PORTCURR+1))
143
        fi
144
}
145

146
function ta_start_brick_process()
147
{
148
        ta_set_port_by_name $1
149
        local p=${PORTMAP[$1]}
150
        if glusterfs -p $B0/${1}.pid --volfile=$B0/${1}.vol -l $(gluster --print-logdir)/${1}.log --xlator-option ${V0}-server.listen-port=$p
151
        then
152
                cat $B0/${1}.pid
153
        else
154
                echo ""
155
                return 1
156
        fi
157
}
158

159
function ta_start_ta_process()
160
{
161
        ta_set_port_by_name $1
162
        local p=${PORTMAP[$1]}
163
        if glusterfs -p $B0/${1}.pid --volfile=$B0/${1}.vol -l $(gluster --print-logdir)/${1}.log --xlator-option ta-server.listen-port=$p
164
        then
165
                cat $B0/${1}.pid
166
        else
167
                echo ""
168
                return 1
169
        fi
170
}
171

172
function ta_start_mount_process()
173
{
174
        mkdir -p $1
175
        identifier=$(echo $1 | tr / .)
176
        if glusterfs --entry-timeout=0 --attribute-timeout=0 -p $B0/${identifier}.pid --volfile=$B0/mount.vol $1
177
        then
178
                cat $B0/$identifier.pid
179
        else
180
                echo ""
181
                return 1
182
        fi
183
}
184

185
function ta_get_mount_pid()
186
{
187
    local mount_path=$1
188
    identifier=$(echo $mount_path | tr / .)
189
    cat $B0/${identifier}.pid
190
}
191

192
function ta_create_mount_volfile()
193
{
194
        local b0=$B0/$1
195
        local b1=$B0/$2
196
        local ta=$B0/$3
197
        local b0_port=${PORTMAP[$1]}
198
        local b1_port=${PORTMAP[$2]}
199
        local ta_port=${PORTMAP[$3]}
200
cat > $B0/mount.vol <<EOF
201
volume ${V0}-client-0
202
     type protocol/client
203
     option remote-host $H0
204
     option client-bind-insecure off
205
     option transport.socket.keepalive-interval 2
206
     option transport.socket.keepalive-time 20
207
     option transport.socket.ssl-enabled off
208
     option remote-subvolume $b0
209
     option transport.tcp-user-timeout 0
210
     option transport.socket.keepalive-count 9
211
     option transport-type tcp
212
     option ping-timeout 42
213
     option send-gids on
214
     option remote-port $b0_port
215
     option password a0ad63dd-8314-4f97-9160-1b93e3cb1f0b
216
     option username 459d48e8-2a92-4f11-89f2-077b29f6f86d
217
end-volume
218

219
volume ${V0}-client-1
220
     type protocol/client
221
     option remote-host $H0
222
     option client-bind-insecure off
223
     option transport.socket.keepalive-interval 2
224
     option transport.socket.keepalive-time 20
225
     option transport.socket.ssl-enabled off
226
     option remote-subvolume $b1
227
     option transport.tcp-user-timeout 0
228
     option transport.socket.keepalive-count 9
229
     option transport-type tcp
230
     option ping-timeout 42
231
     option send-gids on
232
     option remote-port $b1_port
233
     option password a0ad63dd-8314-4f97-9160-1b93e3cb1f0b
234
     option username 459d48e8-2a92-4f11-89f2-077b29f6f86d
235
end-volume
236

237
volume ${V0}-thin-arbiter-client
238
     type protocol/client
239
     option client-bind-insecure off
240
     option transport.socket.ssl-enabled off
241
     option remote-subvolume $ta
242
     option ping-timeout 42
243
     option remote-host $H0
244
     option send-gids on
245
     option transport.socket.keepalive-interval 2
246
     option remote-port $ta_port
247
     option transport-type tcp
248
     option transport.tcp-user-timeout 0
249
     option transport.socket.keepalive-time 20
250
     option transport.socket.keepalive-count 9
251
end-volume
252

253
volume ${V0}-replicate-0
254
     type cluster/replicate
255
     option afr-dirty-xattr trusted.afr.dirty
256
     option iam-self-heal-daemon off
257
     option afr-pending-xattr ${V0}-client-0,${V0}-client-1,${V0}-ta-2
258
     option thin-arbiter $H0:$ta
259
     subvolumes ${V0}-client-0 ${V0}-client-1 ${V0}-thin-arbiter-client
260
end-volume
261

262
volume ${V0}-distribute
263
     type cluster/distribute
264
     option rebal-throttle normal
265
     option force-migration off
266
     option lookup-optimize on
267
     option weighted-rebalance on
268
     option write-freq-threshold 0
269
     option assert-no-child-down off
270
     option lock-migration off
271
     option lookup-unhashed on
272
     option randomize-hash-range-by-gfid off
273
     option unhashed-sticky-bit off
274
     option use-readdirp on
275
     option readdir-optimize off
276
     option xattr-name trusted.glusterfs.dht
277
     option read-freq-threshold 0
278
     option min-free-disk 10%
279
     option min-free-inodes 5%
280
     option rebalance-stats off
281
     subvolumes ${V0}-replicate-0
282
end-volume
283

284
volume ${V0}-write-behind
285
     type performance/write-behind
286
     option strict-O_DIRECT off
287
     option strict-write-ordering off
288
     option resync-failed-syncs-after-fsync off
289
     option aggregate-size 128KB
290
     option flush-behind on
291
     option cache-size 1MB
292
     option trickling-writes on
293
     subvolumes ${V0}-distribute
294
end-volume
295

296
volume ${V0}-read-ahead
297
     type performance/read-ahead
298
     option force-atime-update false
299
     option page-count 4
300
     option page-size 131072
301
     option pass-through false
302
     subvolumes ${V0}-write-behind
303
end-volume
304

305
volume ${V0}-readdir-ahead
306
     type performance/readdir-ahead
307
     option rda-low-wmark 4096
308
     option rda-high-wmark 128KB
309
     option rda-cache-limit 10MB
310
     option parallel-readdir off
311
     option pass-through false
312
     option rda-request-size 131072
313
     subvolumes ${V0}-read-ahead
314
end-volume
315

316
volume ${V0}-io-cache
317
     type performance/io-cache
318
     option cache-timeout 1
319
     option cache-size 32MB
320
     option min-file-size 0
321
     option max-file-size 0
322
     option pass-through false
323
     subvolumes ${V0}-readdir-ahead
324
end-volume
325

326
volume ${V0}-quick-read
327
     type performance/quick-read
328
     option cache-invalidation false
329
     option ctime-invalidation false
330
     option cache-size 128MB
331
     option cache-timeout 1
332
     option max-file-size 64KB
333
     subvolumes ${V0}-io-cache
334
end-volume
335

336
volume ${V0}-open-behind
337
     type performance/open-behind
338
     option use-anonymous-fd yes
339
     option lazy-open yes
340
     option read-after-open no
341
     option pass-through false
342
     subvolumes ${V0}-quick-read
343
end-volume
344

345
volume ${V0}-md-cache
346
     type performance/md-cache
347
     option pass-through false
348
     option cache-capability-xattrs true
349
     option cache-posix-acl false
350
     option cache-swift-metadata true
351
     option cache-samba-metadata false
352
     option md-cache-timeout 1
353
     option force-readdirp true
354
     option cache-invalidation false
355
     option md-cache-statfs off
356
     option cache-selinux false
357
     option cache-ima-xattrs true
358
     subvolumes ${V0}-open-behind
359
end-volume
360

361
volume ${V0}-io-threads
362
     type performance/io-threads
363
     option normal-prio-threads 16
364
     option enable-least-priority on
365
     option idle-time 120
366
     option cleanup-disconnected-reqs off
367
     option pass-through false
368
     option thread-count 16
369
     option high-prio-threads 16
370
     option low-prio-threads 16
371
     option least-prio-threads 1
372
     subvolumes ${V0}-md-cache
373
end-volume
374

375
volume ${V0}
376
     type debug/io-stats
377
     option client-logger gluster-log
378
     option client-log-buf-size 5
379
     option latency-measurement off
380
     option client-log-level INFO
381
     option brick-log-level INFO
382
     option count-fop-hits off
383
     option sys-log-level CRITICAL
384
     option brick-log-format with-msg-id
385
     option brick-log-buf-size 5
386
     option dump-fd-stats off
387
     option ios-dump-interval 0
388
     option ios-dump-format json
389
     option client-log-format with-msg-id
390
     option log-buf-size 5
391
     option log-flush-timeout 120
392
     option client-log-flush-timeout 120
393
     option ios-sample-interval 0
394
     option ios-sample-buf-size 65535
395
     option brick-logger gluster-log
396
     option ios-dnscache-ttl-sec 86400
397
     option brick-log-flush-timeout 120
398
     option unique-id /no/such/path
399
     subvolumes ${V0}-io-threads
400
end-volume
401
EOF
402
}
403

404
function ta_kill_brick()
405
{
406
        local p=$(cat $B0/${1}.pid)
407
        echo > $B0/${1}.pid
408
        kill -9 $p
409
}
410

411
function ta_get_pid_by_brick_name()
412
{
413
        cat $B0/${1}.pid
414
}
415

416
function ta_up_status()
417
{
418
        local v=$1
419
        local m=$2
420
        local replica_id=$3
421
        grep -E "^up = " $m/.meta/graphs/active/${v}-replicate-${replica_id}/private | cut -f2 -d'='
422
}
423

424
function ta_create_shd_volfile()
425
{
426
        local b0=$B0/$1
427
        local b1=$B0/$2
428
        local ta=$B0/$3
429
        local b0_port=${PORTMAP[$1]}
430
        local b1_port=${PORTMAP[$2]}
431
        local ta_port=${PORTMAP[$3]}
432
cat > $B0/glustershd.vol <<EOF
433
volume ${V0}-replicate-0-client-0
434
    type protocol/client
435
    option send-gids on
436
    option transport.socket.keepalive-interval 2
437
    option remote-host $H0
438
    option remote-subvolume $b0
439
    option ping-timeout 42
440
    option client-bind-insecure off
441
    option transport.socket.own-thread off
442
    option frame-timeout 1800
443
    option non-blocking-io off
444
    option transport.socket.keepalive 1
445
    option transport.socket.keepalive-count 9
446
    option transport.tcp-user-timeout 0
447
    option transport.socket.nodelay 1
448
    option transport.socket.keepalive-time 20
449
    option transport.socket.read-fail-log off
450
    option transport-type tcp
451
    option filter-O_DIRECT disable
452
    option event-threads 2
453
    option transport.listen-backlog 1024
454
    option transport.socket.ssl-enabled off
455
    option password a0ad63dd-8314-4f97-9160-1b93e3cb1f0b
456
    option username 459d48e8-2a92-4f11-89f2-077b29f6f86d
457
    option remote-port $b0_port
458
end-volume
459

460
volume ${V0}-replicate-0-client-1
461
    type protocol/client
462
    option remote-host $H0
463
    option transport.socket.keepalive-time 20
464
    option transport.socket.keepalive-count 9
465
    option transport.socket.own-thread off
466
    option transport.socket.ssl-enabled off
467
    option transport-type tcp
468
    option remote-subvolume $b1
469
    option event-threads 2
470
    option transport.tcp-user-timeout 0
471
    option transport.socket.keepalive 1
472
    option transport.socket.nodelay 1
473
    option transport.socket.read-fail-log off
474
    option frame-timeout 1800
475
    option ping-timeout 42
476
    option client-bind-insecure off
477
    option filter-O_DIRECT disable
478
    option send-gids on
479
    option non-blocking-io off
480
    option transport.listen-backlog 1024
481
    option transport.socket.keepalive-interval 2
482
    option password a0ad63dd-8314-4f97-9160-1b93e3cb1f0b
483
    option username 459d48e8-2a92-4f11-89f2-077b29f6f86d
484
    option remote-port $b1_port
485
end-volume
486

487
volume ${V0}-replicate-0-thin-arbiter-client
488
    type protocol/client
489
    option frame-timeout 1800
490
    option event-threads 2
491
    option transport.listen-backlog 1024
492
    option transport.socket.nodelay 1
493
    option transport.socket.keepalive-count 9
494
    option transport.socket.ssl-enabled off
495
    option transport-type tcp
496
    option remote-subvolume $ta
497
    option filter-O_DIRECT disable
498
    option non-blocking-io off
499
    option transport.socket.keepalive-interval 2
500
    option transport.socket.read-fail-log off
501
    option remote-host $H0
502
    option send-gids on
503
    option transport.tcp-user-timeout 0
504
    option transport.socket.keepalive-time 20
505
    option ping-timeout 42
506
    option client-bind-insecure off
507
    option transport.socket.keepalive 1
508
    option transport.socket.own-thread off
509
    option remote-port $ta_port
510
end-volume
511

512
volume ${V0}-replicate-0
513
    type cluster/replicate
514
    option background-self-heal-count 8
515
    option metadata-self-heal on
516
    option data-change-log on
517
    option entrylk-trace off
518
    option iam-self-heal-daemon yes
519
    option afr-dirty-xattr trusted.afr.dirty
520
    option heal-timeout 10
521
    option read-hash-mode 1
522
    option metadata-splitbrain-forced-heal off
523
    option thin-arbiter $H0:$ta
524
    option shd-max-threads 1
525
    option afr-pending-xattr ${V0}-client-0,${V0}-client-1,${V0}-ta-2
526
    option halo-max-latency 5
527
    option halo-max-replicas 99999
528
    option entry-change-log on
529
    option halo-nfsd-max-latency 5
530
    option inodelk-trace off
531
    option pre-op-compat on
532
    option eager-lock on
533
    option self-heal-readdir-size 1KB
534
    option ensure-durability on
535
    option locking-scheme full
536
    option halo-enabled False
537
    option heal-wait-queue-length 128
538
    option entry-self-heal on
539
    option self-heal-daemon on
540
    option quorum-reads no
541
    option shd-wait-qlength 1024
542
    option choose-local true
543
    option halo-min-replicas 2
544
    option data-self-heal on
545
    option metadata-change-log on
546
    option consistent-metadata no
547
    option full-lock yes
548
    option use-compound-fops no
549
    option halo-shd-max-latency 99999
550
    option quorum-type none
551
    option favorite-child-policy none
552
    option read-subvolume-index -1
553
    option optimistic-change-log on
554
    option iam-nfs-daemon off
555
    option post-op-delay-secs 1
556
    option granular-entry-heal no
557
    option consistent-io no
558
    option data-self-heal-window-size 1
559
    subvolumes ${V0}-replicate-0-client-0 ${V0}-replicate-0-client-1 ${V0}-replicate-0-thin-arbiter-client
560
end-volume
561

562
volume glustershd
563
    type debug/io-stats
564
    option log-buf-size 5
565
    option ios-dump-format json
566
    option latency-measurement off
567
    option sys-log-level CRITICAL
568
    option brick-log-level INFO
569
    option client-logger gluster-log
570
    option client-log-format with-msg-id
571
    option brick-log-format with-msg-id
572
    option client-log-buf-size 5
573
    option log-flush-timeout 120
574
    option ios-dump-interval 0
575
    option ios-sample-interval 0
576
    option ios-dnscache-ttl-sec 86400
577
    option count-fop-hits off
578
    option client-log-level INFO
579
    option brick-logger gluster-log
580
    option brick-log-buf-size 5
581
    option ios-sample-buf-size 65535
582
    option client-log-flush-timeout 120
583
    option brick-log-flush-timeout 120
584
    option unique-id /no/such/path
585
    option dump-fd-stats off
586
    subvolumes ${V0}-replicate-0
587
end-volume
588
EOF
589
}
590

591
function ta_start_shd_process()
592
{
593
        if glusterfs -p $B0/${1}.pid --volfile=$B0/${1}.vol -l $(gluster --print-logdir)/${1}.log --process-name=glustershd
594
        then
595
                cat $B0/${1}.pid
596
        else
597
                echo ""
598
                return 1
599
        fi
600
}
601

602
function ta_mount_child_up_status()
603
{
604
        local mount_path=$1
605
        #brick_id is (brick-num in volume info - 1)
606
        local vol=$2
607
        local brick_id=$3
608
        local pid=$(ta_get_mount_pid $mount_path)
609
        local fpath=$(generate_statedump $pid)
610
        up=$(grep -a -B1 trusted.afr.$vol-client-$brick_id $fpath | head -1 | cut -f2 -d'=')
611
        rm -f $fpath
612
        echo "$up"
613
}
614