glusterfs

1
/*
2
   Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
3
   This file is part of GlusterFS.
4

5
   This file is licensed to you under your choice of the GNU Lesser
6
   General Public License, version 3 or any later version (LGPLv3 or
7
   later), or the GNU General Public License, version 2 (GPLv2), in all
8
   cases as published by the Free Software Foundation.
9
*/
10

11
#include <time.h>
12
#include <sys/uio.h>
13

14
#include <libgen.h>
15
#include <glusterfs/compat-uuid.h>
16

17
#include "fnmatch.h"
18
#include <glusterfs/compat.h>
19
#include <glusterfs/list.h>
20
#include "glusterd-messages.h"
21
#include <glusterfs/dict.h>
22
#include <glusterfs/compat.h>
23
#include <glusterfs/compat-errno.h>
24
#include "glusterd-sm.h"
25
#include "glusterd-op-sm.h"
26
#include "glusterd-utils.h"
27
#include "glusterd-store.h"
28
#include "glusterd-svc-helper.h"
29
#include "glusterd-snapshot-utils.h"
30
#include "glusterd-server-quorum.h"
31
#include "glusterd-gfproxyd-svc-helper.h"
32

33
char local_node_hostname[PATH_MAX] = {
34
    0,
35
};
36

37
static struct cds_list_head gd_friend_sm_queue;
38

39
static char *glusterd_friend_sm_state_names[] = {
40
    "Establishing Connection",
41
    "Probe Sent to Peer",
42
    "Probe Received from Peer",
43
    "Peer in Cluster",
44
    "Accepted peer request",
45
    "Sent and Received peer request",
46
    "Peer Rejected",
47
    "Peer detach in progress",
48
    "Probe Received from peer",
49
    "Connected to Peer",
50
    "Peer is connected and Accepted",
51
    "Invalid State"};
52

53
static char *glusterd_friend_sm_event_names[] = {
54
    "GD_FRIEND_EVENT_NONE",
55
    "GD_FRIEND_EVENT_PROBE",
56
    "GD_FRIEND_EVENT_INIT_FRIEND_REQ",
57
    "GD_FRIEND_EVENT_RCVD_ACC",
58
    "GD_FRIEND_EVENT_LOCAL_ACC",
59
    "GD_FRIEND_EVENT_RCVD_RJT",
60
    "GD_FRIEND_EVENT_LOCAL_RJT",
61
    "GD_FRIEND_EVENT_RCVD_FRIEND_REQ",
62
    "GD_FRIEND_EVENT_INIT_REMOVE_FRIEND",
63
    "GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND",
64
    "GD_FRIEND_EVENT_REMOVE_FRIEND",
65
    "GD_FRIEND_EVENT_CONNECTED",
66
    "GD_FRIEND_EVENT_NEW_NAME",
67
    "GD_FRIEND_EVENT_MAX"};
68

69
char *
70
glusterd_friend_sm_state_name_get(int state)
71
{
72
    if (state < 0 || state >= GD_FRIEND_STATE_MAX)
73
        return glusterd_friend_sm_state_names[GD_FRIEND_STATE_MAX];
74
    return glusterd_friend_sm_state_names[state];
75
}
76

77
char *
78
glusterd_friend_sm_event_name_get(int event)
79
{
80
    if (event < 0 || event >= GD_FRIEND_EVENT_MAX)
81
        return glusterd_friend_sm_event_names[GD_FRIEND_EVENT_MAX];
82
    return glusterd_friend_sm_event_names[event];
83
}
84

85
void
86
glusterd_destroy_probe_ctx(glusterd_probe_ctx_t *ctx)
87
{
88
    if (!ctx)
89
        return;
90

91
    GF_FREE(ctx->hostname);
92
    GF_FREE(ctx);
93
}
94

95
void
96
glusterd_destroy_friend_req_ctx(glusterd_friend_req_ctx_t *ctx)
97
{
98
    if (!ctx)
99
        return;
100

101
    if (ctx->vols)
102
        dict_unref(ctx->vols);
103
    if (ctx->peer_ver)
104
        dict_unref(ctx->peer_ver);
105
    GF_FREE(ctx->hostname);
106
    GF_FREE(ctx);
107
}
108

109
void
110
glusterd_destroy_friend_update_ctx(glusterd_friend_update_ctx_t *ctx)
111
{
112
    if (!ctx)
113
        return;
114
    GF_FREE(ctx->hostname);
115
    GF_FREE(ctx);
116
}
117

118
int
119
glusterd_broadcast_friend_delete(char *hostname, uuid_t uuid)
120
{
121
    int ret = 0;
122
    rpc_clnt_procedure_t *proc = NULL;
123
    xlator_t *this = THIS;
124
    glusterd_friend_update_ctx_t ctx = {
125
        {0},
126
    };
127
    glusterd_peerinfo_t *peerinfo = NULL;
128
    glusterd_conf_t *priv = NULL;
129
    dict_t *friends = NULL;
130
    char key[64] = {
131
        0,
132
    };
133
    int keylen;
134
    int32_t count = 0;
135

136
    priv = this->private;
137

138
    GF_ASSERT(priv);
139

140
    ctx.hostname = hostname;
141
    ctx.op = GD_FRIEND_UPDATE_DEL;
142

143
    friends = dict_new();
144
    if (!friends) {
145
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
146
        goto out;
147
    }
148

149
    keylen = snprintf(key, sizeof(key), "op");
150
    ret = dict_set_int32n(friends, key, keylen, ctx.op);
151
    if (ret) {
152
        gf_smsg(this->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_SET_FAILED,
153
                "Key=%s", key, NULL);
154
        goto out;
155
    }
156

157
    keylen = snprintf(key, sizeof(key), "hostname");
158
    ret = dict_set_strn(friends, key, keylen, hostname);
159
    if (ret) {
160
        gf_smsg(this->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_SET_FAILED,
161
                "Key=%s", key, NULL);
162
        goto out;
163
    }
164

165
    ret = dict_set_int32_sizen(friends, "count", count);
166
    if (ret) {
167
        gf_smsg(this->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_SET_FAILED,
168
                "Key=%s", key, NULL);
169
        goto out;
170
    }
171

172
    RCU_READ_LOCK;
173
    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
174
    {
175
        if (!peerinfo->connected || !peerinfo->peer)
176
            continue;
177

178
        /* Setting a direct reference to peerinfo in the dict is okay as
179
         * it is only going to be used within this read critical section
180
         * (in glusterd_rpc_friend_update)
181
         */
182
        ret = dict_set_static_ptr(friends, "peerinfo", peerinfo);
183
        if (ret) {
184
            RCU_READ_UNLOCK;
185
            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
186
                   "failed to set peerinfo");
187
            goto out;
188
        }
189

190
        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
191
        if (proc->fn) {
192
            ret = proc->fn(NULL, this, friends);
193
        }
194
    }
195

196
    RCU_READ_UNLOCK;
197
out:
198
    if (friends)
199
        dict_unref(friends);
200

201
    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
202
    return ret;
203
}
204

205
static int
206
glusterd_ac_none(glusterd_friend_sm_event_t *event, void *ctx)
207
{
208
    int ret = 0;
209

210
    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
211

212
    return ret;
213
}
214

215
static int
216
glusterd_ac_error(glusterd_friend_sm_event_t *event, void *ctx)
217
{
218
    int ret = 0;
219

220
    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_AC_ERROR, "Received event %d ",
221
           event->event);
222

223
    return ret;
224
}
225

226
static int
227
glusterd_ac_reverse_probe_begin(glusterd_friend_sm_event_t *event, void *ctx)
228
{
229
    int ret = 0;
230
    glusterd_peerinfo_t *peerinfo = NULL;
231
    glusterd_friend_sm_event_t *new_event = NULL;
232
    glusterd_probe_ctx_t *new_ev_ctx = NULL;
233

234
    GF_ASSERT(event);
235
    GF_ASSERT(ctx);
236

237
    new_ev_ctx = GF_CALLOC(1, sizeof(*new_ev_ctx), gf_gld_mt_probe_ctx_t);
238

239
    RCU_READ_LOCK;
240

241
    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
242
    if (!peerinfo) {
243
        RCU_READ_UNLOCK;
244
        ret = -1;
245
        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
246
               "Could not find peer %s(%s)", event->peername,
247
               uuid_utoa(event->peerid));
248
        goto out;
249
    }
250

251
    ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_PROBE, &new_event);
252

253
    if (ret) {
254
        RCU_READ_UNLOCK;
255
        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
256
               "Unable to get new new_event");
257
        ret = -1;
258
        goto out;
259
    }
260

261
    if (!new_ev_ctx) {
262
        RCU_READ_UNLOCK;
263
        ret = -1;
264
        goto out;
265
    }
266

267
    new_ev_ctx->hostname = gf_strdup(peerinfo->hostname);
268
    new_ev_ctx->port = peerinfo->port;
269
    new_ev_ctx->req = NULL;
270

271
    new_event->peername = gf_strdup(peerinfo->hostname);
272
    gf_uuid_copy(new_event->peerid, peerinfo->uuid);
273
    new_event->ctx = new_ev_ctx;
274

275
    ret = glusterd_friend_sm_inject_event(new_event);
276

277
    RCU_READ_UNLOCK;
278

279
    if (ret) {
280
        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_INJECT_FAIL,
281
               "Unable to inject new_event %d, "
282
               "ret = %d",
283
               new_event->event, ret);
284
    }
285

286
out:
287
    if (ret) {
288
        if (new_event)
289
            GF_FREE(new_event->peername);
290
        GF_FREE(new_event);
291
        if (new_ev_ctx)
292
            GF_FREE(new_ev_ctx->hostname);
293
        GF_FREE(new_ev_ctx);
294
    }
295
    gf_msg_debug("glusterd", 0, "returning with %d", ret);
296
    return ret;
297
}
298

299
static int
300
glusterd_ac_friend_add(glusterd_friend_sm_event_t *event, void *ctx)
301
{
302
    int ret = 0;
303
    glusterd_peerinfo_t *peerinfo = NULL;
304
    rpc_clnt_procedure_t *proc = NULL;
305
    call_frame_t *frame = NULL;
306
    glusterd_conf_t *conf = NULL;
307
    xlator_t *this = THIS;
308

309
    GF_ASSERT(event);
310

311
    conf = this->private;
312

313
    GF_ASSERT(conf);
314

315
    RCU_READ_LOCK;
316

317
    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
318
    if (!peerinfo) {
319
        RCU_READ_UNLOCK;
320
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
321
               "Could not find peer %s(%s)", event->peername,
322
               uuid_utoa(event->peerid));
323
        goto out;
324
    }
325

326
    if (!peerinfo->peer) {
327
        RCU_READ_UNLOCK;
328
        goto out;
329
    }
330
    proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_ADD];
331
    if (proc->fn) {
332
        frame = create_frame(this, this->ctx->pool);
333
        if (!frame) {
334
            RCU_READ_UNLOCK;
335
            goto out;
336
        }
337
        frame->local = ctx;
338
        ret = proc->fn(frame, this, event);
339
    }
340

341
    RCU_READ_UNLOCK;
342
out:
343
    if (ret && frame)
344
        STACK_DESTROY(frame->root);
345

346
    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
347
    return ret;
348
}
349

350
static int
351
glusterd_ac_friend_probe(glusterd_friend_sm_event_t *event, void *ctx)
352
{
353
    int ret = -1;
354
    rpc_clnt_procedure_t *proc = NULL;
355
    call_frame_t *frame = NULL;
356
    glusterd_conf_t *conf = NULL;
357
    xlator_t *this = THIS;
358
    glusterd_probe_ctx_t *probe_ctx = NULL;
359
    glusterd_peerinfo_t *peerinfo = NULL;
360
    dict_t *dict = NULL;
361

362
    GF_ASSERT(ctx);
363

364
    probe_ctx = ctx;
365

366
    conf = this->private;
367

368
    GF_ASSERT(conf);
369

370
    RCU_READ_LOCK;
371
    peerinfo = glusterd_peerinfo_find(NULL, probe_ctx->hostname);
372
    if (peerinfo == NULL) {
373
        // We should not reach this state ideally
374
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, NULL);
375
        ret = -1;
376
        goto unlock;
377
    }
378

379
    if (!peerinfo->peer) {
380
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_ADDRESS_GET_FAIL,
381
                NULL);
382
        goto unlock;
383
    }
384
    proc = &peerinfo->peer->proctable[GLUSTERD_PROBE_QUERY];
385
    if (proc->fn) {
386
        frame = create_frame(this, this->ctx->pool);
387
        if (!frame) {
388
            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL,
389
                    NULL);
390
            goto unlock;
391
        }
392
        frame->local = ctx;
393
        dict = dict_new();
394
        if (!dict) {
395
            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL,
396
                    NULL);
397
            goto unlock;
398
        }
399
        ret = dict_set_str_sizen(dict, "hostname", probe_ctx->hostname);
400
        if (ret) {
401
            gf_smsg(this->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_SET_FAILED,
402
                    "Key=hostname", NULL);
403
            goto unlock;
404
        }
405

406
        ret = dict_set_int32_sizen(dict, "port", probe_ctx->port);
407
        if (ret) {
408
            gf_smsg(this->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_SET_FAILED,
409
                    "Key=port", NULL);
410
            goto unlock;
411
        }
412

413
        /* The peerinfo reference being set here is going to be used
414
         * only within this critical section, in glusterd_rpc_probe
415
         * (ie. proc->fn).
416
         */
417
        ret = dict_set_static_ptr(dict, "peerinfo", peerinfo);
418
        if (ret) {
419
            RCU_READ_UNLOCK;
420
            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
421
                   "failed to set peerinfo");
422
            goto out;
423
        }
424

425
        ret = proc->fn(frame, this, dict);
426
        if (ret)
427
            goto unlock;
428
    }
429
unlock:
430
    RCU_READ_UNLOCK;
431
out:
432

433
    if (dict)
434
        dict_unref(dict);
435
    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
436

437
    if (ret && frame)
438
        STACK_DESTROY(frame->root);
439

440
    return ret;
441
}
442

443
static int
444
glusterd_ac_send_friend_remove_req(glusterd_friend_sm_event_t *event,
445
                                   void *data)
446
{
447
    int ret = 0;
448
    glusterd_peerinfo_t *peerinfo = NULL;
449
    rpc_clnt_procedure_t *proc = NULL;
450
    call_frame_t *frame = NULL;
451
    glusterd_conf_t *conf = NULL;
452
    xlator_t *this = THIS;
453
    glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
454
    glusterd_probe_ctx_t *ctx = NULL;
455
    glusterd_friend_sm_event_t *new_event = NULL;
456

457
    GF_ASSERT(event);
458

459
    conf = this->private;
460

461
    GF_ASSERT(conf);
462

463
    RCU_READ_LOCK;
464

465
    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
466
    if (!peerinfo) {
467
        RCU_READ_UNLOCK;
468
        ret = -1;
469
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
470
               "Could not find peer %s(%s)", event->peername,
471
               uuid_utoa(event->peerid));
472
        goto out;
473
    }
474
    ctx = event->ctx;
475

476
    if (!peerinfo->connected) {
477
        event_type = GD_FRIEND_EVENT_REMOVE_FRIEND;
478

479
        ret = glusterd_friend_sm_new_event(event_type, &new_event);
480

481
        if (!ret) {
482
            new_event->peername = peerinfo->hostname;
483
            gf_uuid_copy(new_event->peerid, peerinfo->uuid);
484
            ret = glusterd_friend_sm_inject_event(new_event);
485
        } else {
486
            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_EVENT_NEW_GET_FAIL,
487
                   "Unable to get event");
488
        }
489

490
        if (ctx) {
491
            ret = glusterd_xfer_cli_deprobe_resp(ctx->req, ret, 0, NULL,
492
                                                 ctx->hostname, ctx->dict);
493
            glusterd_broadcast_friend_delete(ctx->hostname, NULL);
494
            glusterd_destroy_probe_ctx(ctx);
495
        }
496
        goto unlock;
497
    }
498

499
    if (!peerinfo->peer) {
500
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_PEER_ADDRESS_GET_FAIL,
501
                NULL);
502
        goto unlock;
503
    }
504
    proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_REMOVE];
505
    if (proc->fn) {
506
        frame = create_frame(this, this->ctx->pool);
507
        if (!frame) {
508
            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_FRAME_CREATE_FAIL,
509
                    NULL);
510
            goto unlock;
511
        }
512
        frame->local = data;
513
        ret = proc->fn(frame, this, event);
514
    }
515

516
unlock:
517
    RCU_READ_UNLOCK;
518
out:
519

520
    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
521

522
    if (ret && frame)
523
        STACK_DESTROY(frame->root);
524

525
    return ret;
526
}
527

528
static gf_boolean_t
529
glusterd_should_update_peer(glusterd_peerinfo_t *peerinfo,
530
                            glusterd_peerinfo_t *cur_peerinfo)
531
{
532
    if ((peerinfo == cur_peerinfo) ||
533
        (peerinfo->state == GD_FRIEND_STATE_BEFRIENDED))
534
        return _gf_true;
535

536
    return _gf_false;
537
}
538

539
static int
540
glusterd_ac_send_friend_update(glusterd_friend_sm_event_t *event, void *ctx)
541
{
542
    int ret = 0;
543
    glusterd_peerinfo_t *cur_peerinfo = NULL;
544
    glusterd_peerinfo_t *peerinfo = NULL;
545
    rpc_clnt_procedure_t *proc = NULL;
546
    xlator_t *this = THIS;
547
    glusterd_friend_update_ctx_t ev_ctx = {{0}};
548
    glusterd_conf_t *priv = NULL;
549
    dict_t *friends = NULL;
550
    char key[64] = {
551
        0,
552
    };
553
    int keylen;
554
    int32_t count = 0;
555

556
    GF_ASSERT(event);
557

558
    priv = this->private;
559

560
    GF_ASSERT(priv);
561

562
    keylen = snprintf(key, sizeof(key), "op");
563
    friends = dict_new();
564

565
    RCU_READ_LOCK;
566

567
    cur_peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
568
    if (!cur_peerinfo) {
569
        RCU_READ_UNLOCK;
570
        ret = -1;
571
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
572
               "Could not find peer %s(%s)", event->peername,
573
               uuid_utoa(event->peerid));
574
        goto out;
575
    }
576

577
    if (!friends) {
578
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
579
        goto unlock;
580
    }
581

582
    ev_ctx.op = GD_FRIEND_UPDATE_ADD;
583
    ret = dict_set_int32n(friends, key, keylen, ev_ctx.op);
584
    if (ret) {
585
        gf_smsg(this->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_SET_FAILED,
586
                "Key=%s", key, NULL);
587
        goto unlock;
588
    }
589

590
    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
591
    {
592
        if (!glusterd_should_update_peer(peerinfo, cur_peerinfo))
593
            continue;
594

595
        count++;
596

597
        snprintf(key, sizeof(key), "friend%d", count);
598
        ret = gd_add_friend_to_dict(peerinfo, friends, key);
599
        if (ret)
600
            goto unlock;
601
    }
602

603
    ret = dict_set_int32_sizen(friends, "count", count);
604
    if (ret) {
605
        gf_smsg(this->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_SET_FAILED,
606
                "Key=count", NULL);
607
        goto unlock;
608
    }
609

610
    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
611
    {
612
        if (!peerinfo->connected || !peerinfo->peer)
613
            continue;
614

615
        if (!glusterd_should_update_peer(peerinfo, cur_peerinfo))
616
            continue;
617

618
        ret = dict_set_static_ptr(friends, "peerinfo", peerinfo);
619
        if (ret) {
620
            RCU_READ_UNLOCK;
621
            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
622
                   "failed to set peerinfo");
623
            goto out;
624
        }
625

626
        proc = &peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
627
        if (proc->fn) {
628
            ret = proc->fn(NULL, this, friends);
629
        }
630
    }
631

632
unlock:
633
    RCU_READ_UNLOCK;
634
out:
635

636
    if (friends)
637
        dict_unref(friends);
638

639
    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
640
    return ret;
641
}
642

643
/* ac_update_friend only sends friend update to the friend that caused this
644
 * event to happen
645
 */
646
static int
647
glusterd_ac_update_friend(glusterd_friend_sm_event_t *event, void *ctx)
648
{
649
    int ret = 0;
650
    glusterd_peerinfo_t *cur_peerinfo = NULL;
651
    glusterd_peerinfo_t *peerinfo = NULL;
652
    rpc_clnt_procedure_t *proc = NULL;
653
    xlator_t *this = THIS;
654
    glusterd_friend_update_ctx_t ev_ctx = {{0}};
655
    glusterd_conf_t *priv = NULL;
656
    dict_t *friends = NULL;
657
    char key[64] = {
658
        0,
659
    };
660
    int keylen;
661
    int32_t count = 0;
662

663
    GF_ASSERT(event);
664

665
    priv = this->private;
666

667
    GF_ASSERT(priv);
668

669
    friends = dict_new();
670
    keylen = snprintf(key, sizeof(key), "op");
671

672
    RCU_READ_LOCK;
673

674
    cur_peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
675
    if (!cur_peerinfo) {
676
        RCU_READ_UNLOCK;
677
        ret = -1;
678
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
679
               "Could not find peer %s(%s)", event->peername,
680
               uuid_utoa(event->peerid));
681
        goto out;
682
    }
683

684
    /* Bail out early if peer is not connected.
685
     * We cannot send requests to the peer until we have established our
686
     * client connection to it.
687
     */
688
    if (!cur_peerinfo->connected || !cur_peerinfo->peer) {
689
        ret = 0;
690
        goto unlock;
691
    }
692

693
    if (!friends) {
694
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
695
        goto out;
696
    }
697

698
    ev_ctx.op = GD_FRIEND_UPDATE_ADD;
699
    ret = dict_set_int32n(friends, key, keylen, ev_ctx.op);
700
    if (ret) {
701
        gf_smsg(this->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_SET_FAILED,
702
                "Key=%s", key, NULL);
703
        goto unlock;
704
    }
705

706
    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
707
    {
708
        if (!glusterd_should_update_peer(peerinfo, cur_peerinfo))
709
            continue;
710

711
        count++;
712

713
        snprintf(key, sizeof(key), "friend%d", count);
714
        ret = gd_add_friend_to_dict(peerinfo, friends, key);
715
        if (ret)
716
            goto unlock;
717
    }
718

719
    ret = dict_set_int32_sizen(friends, "count", count);
720
    if (ret) {
721
        gf_smsg(this->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_SET_FAILED,
722
                "Key=count", NULL);
723
        goto unlock;
724
    }
725

726
    ret = dict_set_static_ptr(friends, "peerinfo", cur_peerinfo);
727
    if (ret) {
728
        RCU_READ_UNLOCK;
729
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
730
               "failed to set peerinfo");
731
        goto out;
732
    }
733

734
    proc = &cur_peerinfo->peer->proctable[GLUSTERD_FRIEND_UPDATE];
735
    if (proc->fn)
736
        ret = proc->fn(NULL, this, friends);
737

738
    gf_msg_debug(this->name, 0, "Returning with %d", ret);
739

740
unlock:
741
    RCU_READ_UNLOCK;
742
out:
743

744
    if (friends)
745
        dict_unref(friends);
746

747
    return ret;
748
}
749

750
/* Clean up stale volumes on the peer being detached. The volumes which have
751
 * bricks on other peers are stale with respect to the detached peer.
752
 */
753
static void
754
glusterd_peer_detach_cleanup(glusterd_conf_t *priv)
755
{
756
    int ret = -1;
757
    glusterd_volinfo_t *volinfo = NULL;
758
    glusterd_volinfo_t *tmp_volinfo = NULL;
759
    glusterd_svc_t *svc = NULL;
760

761
    GF_ASSERT(priv);
762

763
    cds_list_for_each_entry_safe(volinfo, tmp_volinfo, &priv->volumes, vol_list)
764
    {
765
        /* The peer detach checks make sure that, at this point in the
766
         * detach process, there are only volumes contained completely
767
         * within or completely outside the detached peer.
768
         * The only stale volumes at this point are the ones
769
         * completely outside the peer and can be safely deleted.
770
         */
771
        if (!glusterd_friend_contains_vol_bricks(volinfo, MY_UUID)) {
772
            gf_msg(THIS->name, GF_LOG_INFO, 0, GD_MSG_STALE_VOL_DELETE_INFO,
773
                   "Deleting stale volume %s", volinfo->volname);
774

775
            /*Stop snapd daemon service if snapd daemon is running*/
776
            if (!volinfo->is_snap_volume) {
777
                svc = &(volinfo->snapd.svc);
778
                ret = svc->stop(svc, SIGTERM);
779
                if (ret) {
780
                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
781
                           "Failed "
782
                           "to stop snapd daemon service");
783
                }
784
            }
785

786
            if (glusterd_is_shd_compatible_volume(volinfo)) {
787
                svc = &(volinfo->shd.svc);
788
                ret = svc->stop(svc, SIGTERM);
789
                if (ret) {
790
                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
791
                           "Failed "
792
                           "to stop shd daemon service");
793
                }
794
            }
795

796
            if (glusterd_is_gfproxyd_enabled(volinfo)) {
797
                svc = &(volinfo->gfproxyd.svc);
798
                ret = svc->stop(svc, SIGTERM);
799
                if (ret) {
800
                    gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
801
                           "Failed "
802
                           "to stop gfproxyd daemon service");
803
                }
804
            }
805

806
            ret = glusterd_cleanup_snaps_for_volume(volinfo);
807
            if (ret) {
808
                gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_VOL_DELETE_FAIL,
809
                       "Error deleting snapshots for volume %s",
810
                       volinfo->volname);
811
            }
812

813
            ret = glusterd_delete_volume(volinfo);
814
            if (ret) {
815
                gf_msg(THIS->name, GF_LOG_ERROR, 0,
816
                       GD_MSG_STALE_VOL_REMOVE_FAIL,
817
                       "Error deleting stale volume");
818
            }
819
        }
820
    }
821

822
    /*Reconfigure all daemon services upon peer detach*/
823
    ret = glusterd_svcs_reconfigure(NULL);
824
    if (ret) {
825
        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SVC_STOP_FAIL,
826
               "Failed to reconfigure all daemon services.");
827
    }
828
}
829

830
static int
831
glusterd_ac_handle_friend_remove_req(glusterd_friend_sm_event_t *event,
832
                                     void *ctx)
833
{
834
    int ret = 0;
835
    glusterd_peerinfo_t *peerinfo = NULL;
836
    glusterd_friend_req_ctx_t *ev_ctx = NULL;
837
    glusterd_friend_sm_event_t *new_event = NULL;
838
    glusterd_conf_t *priv = NULL;
839

840
    GF_ASSERT(ctx);
841
    ev_ctx = ctx;
842

843
    priv = THIS->private;
844
    GF_ASSERT(priv);
845

846
    ret = glusterd_xfer_friend_remove_resp(ev_ctx->req, ev_ctx->hostname,
847
                                           ev_ctx->port);
848

849
    RCU_READ_LOCK;
850
    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
851
    {
852
        ret = glusterd_friend_sm_new_event(GD_FRIEND_EVENT_REMOVE_FRIEND,
853
                                           &new_event);
854
        if (ret) {
855
            RCU_READ_UNLOCK;
856
            goto out;
857
        }
858

859
        new_event->peername = gf_strdup(peerinfo->hostname);
860
        gf_uuid_copy(new_event->peerid, peerinfo->uuid);
861

862
        ret = glusterd_friend_sm_inject_event(new_event);
863
        if (ret) {
864
            RCU_READ_UNLOCK;
865
            goto out;
866
        }
867

868
        new_event = NULL;
869
    }
870
    RCU_READ_UNLOCK;
871

872
    glusterd_peer_detach_cleanup(priv);
873
out:
874
    if (new_event)
875
        GF_FREE(new_event->peername);
876
    GF_FREE(new_event);
877

878
    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
879
    return ret;
880
}
881

882
static int
883
glusterd_ac_friend_remove(glusterd_friend_sm_event_t *event, void *ctx)
884
{
885
    int ret = -1;
886
    glusterd_peerinfo_t *peerinfo = NULL;
887

888
    GF_ASSERT(event);
889

890
    RCU_READ_LOCK;
891

892
    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
893
    if (!peerinfo) {
894
        RCU_READ_UNLOCK;
895
        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
896
               "Could not find peer %s(%s)", event->peername,
897
               uuid_utoa(event->peerid));
898
        goto out;
899
    }
900
    ret = glusterd_friend_remove_cleanup_vols(peerinfo->uuid);
901
    RCU_READ_UNLOCK;
902
    if (ret)
903
        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOL_CLEANUP_FAIL,
904
               "Volumes cleanup failed");
905

906
    /* Exiting read critical section as glusterd_peerinfo_cleanup calls
907
     * synchronize_rcu before freeing the peerinfo
908
     */
909

910
    ret = glusterd_peerinfo_cleanup(peerinfo);
911
    if (ret) {
912
        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_PEER_DETACH_CLEANUP_FAIL,
913
               "Cleanup returned: %d", ret);
914
    }
915
out:
916
    return 0;
917
}
918

919
/*static int
920
glusterd_ac_none (void *ctx)
921
{
922
        int ret = 0;
923

924
        gf_log ("", GF_LOG_DEBUG, "Returning with %d", ret);
925

926
        return ret;
927
}*/
928

929
static int
930
glusterd_ac_handle_friend_add_req(glusterd_friend_sm_event_t *event, void *ctx)
931
{
932
    int ret = 0;
933
    uuid_t uuid;
934
    glusterd_peerinfo_t *peerinfo = NULL;
935
    glusterd_friend_req_ctx_t *ev_ctx = NULL;
936
    glusterd_friend_update_ctx_t *new_ev_ctx = NULL;
937
    glusterd_friend_sm_event_t *new_event = NULL;
938
    glusterd_friend_sm_event_type_t event_type = GD_FRIEND_EVENT_NONE;
939
    glusterd_conf_t *conf = NULL;
940
    int status = 0;
941
    int32_t op_ret = -1;
942
    int32_t op_errno = 0;
943
    xlator_t *this = THIS;
944
    char *hostname = NULL;
945

946
    GF_ASSERT(ctx);
947
    ev_ctx = ctx;
948
    gf_uuid_copy(uuid, ev_ctx->uuid);
949

950
    RCU_READ_LOCK;
951
    peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
952
    if (!peerinfo) {
953
        RCU_READ_UNLOCK;
954
        ret = -1;
955
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
956
               "Could not find peer %s(%s)", event->peername,
957
               uuid_utoa(event->peerid));
958
        goto out;
959
    }
960

961
    /* TODO: How do you do an atomic copy of uuid_t */
962
    /* TODO: Updating within a read-critical section is also invalid
963
     *       Update properly with updater synchronization
964
     */
965
    gf_uuid_copy(peerinfo->uuid, ev_ctx->uuid);
966

967
    RCU_READ_UNLOCK;
968

969
    conf = this->private;
970
    GF_ASSERT(conf);
971

972
    /* Passing the peername from the event. glusterd_compare_friend_data
973
     * updates volumes and will use synchronize_rcu. If we were to pass
974
     * peerinfo->hostname, we would have to do it under a read critical
975
     * section which would lead to a deadlock
976
     */
977

978
    // Build comparison logic here.
979
    pthread_mutex_lock(&conf->import_volumes);
980
    {
981
        ret = glusterd_compare_friend_data(ev_ctx->vols, ev_ctx->peer_ver,
982
                                           &status, event->peername);
983
        if (ret) {
984
            pthread_mutex_unlock(&conf->import_volumes);
985
            goto out;
986
        }
987

988
        if (GLUSTERD_VOL_COMP_RJT != status) {
989
            event_type = GD_FRIEND_EVENT_LOCAL_ACC;
990
            op_ret = 0;
991
        } else {
992
            event_type = GD_FRIEND_EVENT_LOCAL_RJT;
993
            op_errno = GF_PROBE_VOLUME_CONFLICT;
994
            op_ret = -1;
995
        }
996

997
        /* Compare missed_snapshot list with the peer *
998
         * if volume comparison is successful */
999
        if (op_ret == 0) {
1000
            ret = glusterd_import_friend_missed_snap_list(ev_ctx->vols);
1001
            if (ret) {
1002
                gf_msg(this->name, GF_LOG_ERROR, 0,
1003
                       GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
1004
                       "Failed to import peer's "
1005
                       "missed_snaps_list.");
1006
                event_type = GD_FRIEND_EVENT_LOCAL_RJT;
1007
                op_errno = GF_PROBE_MISSED_SNAP_CONFLICT;
1008
                op_ret = -1;
1009
            }
1010

1011
            /* glusterd_compare_friend_snapshots and functions only require
1012
             * a peers hostname and uuid. It also does updates, which
1013
             * require use of synchronize_rcu. So we pass the hostname and
1014
             * id from the event instead of the peerinfo object to prevent
1015
             * deadlocks as above.
1016
             */
1017
            ret = glusterd_compare_friend_snapshots(
1018
                ev_ctx->vols, event->peername, event->peerid);
1019
            if (ret) {
1020
                gf_msg(this->name, GF_LOG_ERROR, 0,
1021
                       GD_MSG_SNAP_COMPARE_CONFLICT,
1022
                       "Conflict in comparing peer's snapshots");
1023
                event_type = GD_FRIEND_EVENT_LOCAL_RJT;
1024
                op_errno = GF_PROBE_SNAP_CONFLICT;
1025
                op_ret = -1;
1026
            }
1027
        }
1028
    }
1029
    pthread_mutex_unlock(&conf->import_volumes);
1030
    ret = glusterd_friend_sm_new_event(event_type, &new_event);
1031

1032
    if (ret) {
1033
        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
1034
               "Out of Memory");
1035
        goto out;
1036
    }
1037

1038
    new_event->peername = gf_strdup(event->peername);
1039
    gf_uuid_copy(new_event->peerid, event->peerid);
1040

1041
    new_ev_ctx = GF_CALLOC(1, sizeof(*new_ev_ctx),
1042
                           gf_gld_mt_friend_update_ctx_t);
1043
    if (!new_ev_ctx) {
1044
        ret = -1;
1045
        goto out;
1046
    }
1047

1048
    gf_uuid_copy(new_ev_ctx->uuid, ev_ctx->uuid);
1049
    new_ev_ctx->hostname = gf_strdup(ev_ctx->hostname);
1050
    new_ev_ctx->op = GD_FRIEND_UPDATE_ADD;
1051

1052
    new_event->ctx = new_ev_ctx;
1053

1054
    ret = dict_get_str(ev_ctx->vols, "hostname_in_cluster", &hostname);
1055
    if (ret || !hostname) {
1056
        gf_msg_debug(this->name, 0, "Unable to fetch local hostname from peer");
1057
    } else if (snprintf(local_node_hostname, sizeof(local_node_hostname), "%s",
1058
                        hostname) >= sizeof(local_node_hostname)) {
1059
        gf_msg_debug(this->name, 0, "local_node_hostname truncated");
1060
        ret = -1;
1061
        goto out;
1062
    }
1063

1064
    glusterd_friend_sm_inject_event(new_event);
1065
    new_event = NULL;
1066

1067
    ret = glusterd_xfer_friend_add_resp(ev_ctx->req, ev_ctx->hostname,
1068
                                        event->peername, ev_ctx->port, op_ret,
1069
                                        op_errno);
1070

1071
out:
1072
    if (new_event)
1073
        GF_FREE(new_event->peername);
1074
    GF_FREE(new_event);
1075

1076
    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
1077
    return ret;
1078
}
1079

1080
static int
1081
glusterd_friend_sm_transition_state(uuid_t peerid, char *peername,
1082
                                    glusterd_sm_t *state,
1083
                                    glusterd_friend_sm_event_type_t event_type)
1084
{
1085
    int ret = -1;
1086
    glusterd_peerinfo_t *peerinfo = NULL;
1087

1088
    GF_ASSERT(state);
1089
    GF_ASSERT(peername);
1090

1091
    RCU_READ_LOCK;
1092
    peerinfo = glusterd_peerinfo_find(peerid, peername);
1093
    if (!peerinfo) {
1094
        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_PEER_NOT_FOUND, NULL);
1095
        goto out;
1096
    }
1097

1098
    (void)glusterd_sm_tr_log_transition_add(&peerinfo->sm_log, peerinfo->state,
1099
                                            state[event_type].next_state,
1100
                                            event_type);
1101

1102
    uatomic_set(&peerinfo->state, state[event_type].next_state);
1103

1104
    ret = 0;
1105
out:
1106
    RCU_READ_UNLOCK;
1107
    return ret;
1108
}
1109

1110
glusterd_sm_t glusterd_state_default[] = {
1111
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},
1112
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe},  // EV_PROBE
1113
    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_friend_add},   // EV_INIT_FRIEND_REQ
1114
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},          // EVENT_RCVD_ACC
1115
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
1116
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_RJT
1117
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
1118
    {GD_FRIEND_STATE_REQ_RCVD,
1119
     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
1120
    {GD_FRIEND_STATE_DEFAULT,
1121
     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
1122
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
1123
    {GD_FRIEND_STATE_DEFAULT,
1124
     glusterd_ac_friend_remove},                          // EVENT_REMOVE_FRIEND
1125
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_friend_probe},  // EVENT_CONNECTED
1126
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},          // EVENT_NEW_NAME
1127
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},          // EVENT_MAX
1128
};
1129

1130
glusterd_sm_t glusterd_state_probe_rcvd[] = {
1131
    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},
1132
    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EV_PROBE
1133
    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EV_INIT_FRIEND_REQ
1134
    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_ACC
1135
    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
1136
    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_RJT
1137
    {GD_FRIEND_STATE_PROBE_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
1138
    {GD_FRIEND_STATE_REQ_RCVD,
1139
     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
1140
    {GD_FRIEND_STATE_DEFAULT,
1141
     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
1142
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
1143
    {GD_FRIEND_STATE_DEFAULT,
1144
     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
1145
    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
1146
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},         // EVENT_NEW_NAME
1147
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},         // EVENT_MAX
1148
};
1149

1150
glusterd_sm_t glusterd_state_connected_rcvd[] = {
1151
    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},
1152
    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EV_PROBE
1153
    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EV_INIT_FRIEND_REQ
1154
    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_RCVD_ACC
1155
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
1156
     glusterd_ac_reverse_probe_begin},                   // EVENT_RCVD_LOCAL_ACC
1157
    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_RCVD_RJT
1158
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},        // EVENT_RCVD_LOCAL_RJT
1159
    {GD_FRIEND_STATE_CONNECTED_RCVD,
1160
     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
1161
    {GD_FRIEND_STATE_DEFAULT,
1162
     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
1163
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
1164
    {GD_FRIEND_STATE_DEFAULT,
1165
     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
1166
    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
1167
    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_NEW_NAME
1168
    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_MAX
1169
};
1170

1171
glusterd_sm_t glusterd_state_connected_accepted[] = {
1172
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},
1173
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_friend_probe},  // EV_PROBE
1174
    {GD_FRIEND_STATE_REQ_SENT_RCVD,
1175
     glusterd_ac_friend_add},  // EV_INIT_FRIEND_REQ
1176
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_ACC
1177
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
1178
     glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
1179
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_RJT
1180
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
1181
     glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
1182
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
1183
     glusterd_ac_none},  // EVENT_RCV_FRIEND_REQ
1184
    {GD_FRIEND_STATE_DEFAULT,
1185
     glusterd_ac_send_friend_remove_req},         // EV_INIT_REMOVE_FRIEND
1186
    {GD_FRIEND_STATE_DEFAULT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
1187
    {GD_FRIEND_STATE_DEFAULT,
1188
     glusterd_ac_friend_remove},  // EVENT_REMOVE_FRIEND
1189
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_CONNECTED
1190
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_NEW_NAME
1191
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED, glusterd_ac_none},  // EVENT_MAX
1192
};
1193

1194
glusterd_sm_t glusterd_state_req_sent[] = {
1195
    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_NONE,
1196
    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_PROBE,
1197
    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_INIT_FRIEND_REQ,
1198
    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_ACC
1199
    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_RCVD_LOCAL_ACC
1200
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},      // EVENT_RCVD_RJT
1201
    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_RCVD_LOCAL_RJT
1202
    {GD_FRIEND_STATE_REQ_SENT_RCVD,
1203
     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
1204
    {GD_FRIEND_STATE_UNFRIEND_SENT,
1205
     glusterd_ac_send_friend_remove_req},          // EVENT_INIT_REMOVE_FRIEND,
1206
    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
1207
    {GD_FRIEND_STATE_DEFAULT,
1208
     glusterd_ac_friend_remove},                   // EVENT_REMOVE_FRIEND
1209
    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_CONNECTED
1210
    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_NEW_NAME
1211
    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},  // EVENT_MAX
1212
};
1213

1214
glusterd_sm_t glusterd_state_req_rcvd[] = {
1215
    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},  // EVENT_NONE,
1216
    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},  // EVENT_PROBE,
1217
    {GD_FRIEND_STATE_REQ_SENT_RCVD,
1218
     glusterd_ac_none},                                // EVENT_INIT_FRIEND_REQ,
1219
    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},      // EVENT_RCVD_ACC
1220
    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
1221
    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},      // EVENT_RCVD_RJT
1222
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},      // EVENT_RCVD_LOCAL_RJT
1223
    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},      // EVENT_RCV_FRIEND_REQ
1224
    {GD_FRIEND_STATE_DEFAULT,
1225
     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND,
1226
    {GD_FRIEND_STATE_DEFAULT,
1227
     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
1228
    {GD_FRIEND_STATE_DEFAULT,
1229
     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
1230
    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
1231
    {GD_FRIEND_STATE_CONNECTED_RCVD, glusterd_ac_none},  // EVENT_NEW_NAME
1232
    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},        // EVENT_MAX
1233
};
1234

1235
glusterd_sm_t glusterd_state_befriended[] = {
1236
    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_NONE,
1237
    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_PROBE,
1238
    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
1239
    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_update_friend},  // EVENT_RCVD_ACC
1240
    {GD_FRIEND_STATE_BEFRIENDED,
1241
     glusterd_ac_update_friend},                   // EVENT_RCVD_LOCAL_ACC
1242
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_RJT
1243
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
1244
    {GD_FRIEND_STATE_BEFRIENDED,
1245
     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
1246
    {GD_FRIEND_STATE_UNFRIEND_SENT,
1247
     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND,
1248
    {GD_FRIEND_STATE_DEFAULT,
1249
     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
1250
    {GD_FRIEND_STATE_DEFAULT,
1251
     glusterd_ac_friend_remove},  // EVENT_REMOVE_FRIEND
1252
    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_friend_add},  // EVENT_CONNECTED
1253
    {GD_FRIEND_STATE_BEFRIENDED,
1254
     glusterd_ac_send_friend_update},                // EVENT_NEW_NAME
1255
    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_MAX
1256
};
1257

1258
glusterd_sm_t glusterd_state_req_sent_rcvd[] = {
1259
    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_NONE,
1260
    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_PROBE,
1261
    {GD_FRIEND_STATE_REQ_SENT_RCVD,
1262
     glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
1263
    {GD_FRIEND_STATE_BEFRIENDED,
1264
     glusterd_ac_send_friend_update},                   // EVENT_RCVD_ACC
1265
    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
1266
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},       // EVENT_RCVD_RJT
1267
    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
1268
    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_RCV_FRIEND_REQ
1269
    {GD_FRIEND_STATE_UNFRIEND_SENT,
1270
     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND,
1271
    {GD_FRIEND_STATE_DEFAULT,
1272
     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
1273
    {GD_FRIEND_STATE_DEFAULT,
1274
     glusterd_ac_friend_remove},                        // EVENT_REMOVE_FRIEND
1275
    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_CONNECTED
1276
    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_NEW_NAME
1277
    {GD_FRIEND_STATE_REQ_SENT_RCVD, glusterd_ac_none},  // EVENT_MAX
1278
};
1279

1280
glusterd_sm_t glusterd_state_rejected[] = {
1281
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},          // EVENT_NONE,
1282
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_probe},  // EVENT_PROBE,
1283
    {GD_FRIEND_STATE_REQ_SENT,
1284
     glusterd_ac_friend_add},                        // EVENT_INIT_FRIEND_REQ,
1285
    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_RCVD_ACC
1286
    {GD_FRIEND_STATE_BEFRIENDED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_ACC
1287
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},    // EVENT_RCVD_RJT
1288
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},    // EVENT_RCVD_LOCAL_RJT
1289
    {GD_FRIEND_STATE_REQ_RCVD,
1290
     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
1291
    {GD_FRIEND_STATE_DEFAULT,
1292
     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND
1293
    {GD_FRIEND_STATE_DEFAULT,
1294
     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
1295
    {GD_FRIEND_STATE_DEFAULT,
1296
     glusterd_ac_friend_remove},                         // EVENT_REMOVE_FRIEND
1297
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_friend_add},  // EVENT_CONNECTED
1298
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},        // EVENT_NEW_NAME
1299
    {GD_FRIEND_STATE_REQ_RCVD, glusterd_ac_none},        // EVENT_MAX
1300
};
1301

1302
glusterd_sm_t glusterd_state_req_accepted[] = {
1303
    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_NONE,
1304
    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_PROBE,
1305
    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
1306
    {GD_FRIEND_STATE_BEFRIENDED,
1307
     glusterd_ac_send_friend_update},  // EVENT_RCVD_ACC
1308
    {GD_FRIEND_STATE_BEFRIENDED,
1309
     glusterd_ac_send_friend_update},              // EVENT_RCVD_LOCAL_ACC
1310
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_RJT
1311
    {GD_FRIEND_STATE_REJECTED, glusterd_ac_none},  // EVENT_RCVD_LOCAL_RJT
1312
    {GD_FRIEND_STATE_REQ_ACCEPTED,
1313
     glusterd_ac_handle_friend_add_req},  // EVENT_RCV_FRIEND_REQ
1314
    {GD_FRIEND_STATE_REQ_ACCEPTED,
1315
     glusterd_ac_send_friend_remove_req},  // EVENT_INIT_REMOVE_FRIEND
1316
    {GD_FRIEND_STATE_DEFAULT,
1317
     glusterd_ac_handle_friend_remove_req},  // EVENT_RCVD_REMOVE_FRIEND
1318
    {GD_FRIEND_STATE_DEFAULT,
1319
     glusterd_ac_friend_remove},  // EVENT_REMOVE_FRIEND
1320
    {GD_FRIEND_STATE_CONNECTED_ACCEPTED,
1321
     glusterd_ac_reverse_probe_begin},                 // EVENT_CONNECTED
1322
    {GD_FRIEND_STATE_REQ_ACCEPTED, glusterd_ac_none},  // EVENT_NEW_NAME
1323
    {GD_FRIEND_STATE_REQ_SENT, glusterd_ac_none},      // EVENT_MAX
1324
};
1325

1326
glusterd_sm_t glusterd_state_unfriend_sent[] = {
1327
    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},   // EVENT_NONE,
1328
    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_PROBE,
1329
    {GD_FRIEND_STATE_UNFRIEND_SENT,
1330
     glusterd_ac_none},  // EVENT_INIT_FRIEND_REQ,
1331
    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},   // EVENT_RCVD_ACC
1332
    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},   // EVENT_RCVD_LOCAL_ACC
1333
    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_RCVD_RJT
1334
    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_RCVD_LOCAL_RJT
1335
    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_error},  // EVENT_RCV_FRIEND_REQ
1336
    {GD_FRIEND_STATE_UNFRIEND_SENT,
1337
     glusterd_ac_none},  // EVENT_INIT_REMOVE_FRIEND
1338
    {GD_FRIEND_STATE_UNFRIEND_SENT,
1339
     glusterd_ac_none},  // EVENT_RCVD_REMOVE_FRIEND
1340
    {GD_FRIEND_STATE_DEFAULT,
1341
     glusterd_ac_friend_remove},                        // EVENT_REMOVE_FRIEND
1342
    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},  // EVENT_CONNECTED
1343
    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},  // EVENT_NEW_NAME
1344
    {GD_FRIEND_STATE_UNFRIEND_SENT, glusterd_ac_none},  // EVENT_MAX
1345
};
1346

1347
glusterd_sm_t *glusterd_friend_state_table[] = {
1348
    glusterd_state_default,           glusterd_state_req_sent,
1349
    glusterd_state_req_rcvd,          glusterd_state_befriended,
1350
    glusterd_state_req_accepted,      glusterd_state_req_sent_rcvd,
1351
    glusterd_state_rejected,          glusterd_state_unfriend_sent,
1352
    glusterd_state_probe_rcvd,        glusterd_state_connected_rcvd,
1353
    glusterd_state_connected_accepted};
1354

1355
int
1356
glusterd_friend_sm_new_event(glusterd_friend_sm_event_type_t event_type,
1357
                             glusterd_friend_sm_event_t **new_event)
1358
{
1359
    glusterd_friend_sm_event_t *event = NULL;
1360

1361
    GF_ASSERT(new_event);
1362
    GF_ASSERT(GD_FRIEND_EVENT_NONE <= event_type &&
1363
              GD_FRIEND_EVENT_MAX > event_type);
1364

1365
    event = GF_CALLOC(1, sizeof(*event), gf_gld_mt_friend_sm_event_t);
1366

1367
    if (!event)
1368
        return -1;
1369

1370
    *new_event = event;
1371
    event->event = event_type;
1372
    CDS_INIT_LIST_HEAD(&event->list);
1373

1374
    return 0;
1375
}
1376

1377
int
1378
glusterd_friend_sm_inject_event(glusterd_friend_sm_event_t *event)
1379
{
1380
    GF_ASSERT(event);
1381
    gf_msg_debug("glusterd", 0, "Enqueue event: '%s'",
1382
                 glusterd_friend_sm_event_name_get(event->event));
1383
    cds_list_add_tail(&event->list, &gd_friend_sm_queue);
1384

1385
    return 0;
1386
}
1387

1388
void
1389
glusterd_destroy_friend_event_context(glusterd_friend_sm_event_t *event)
1390
{
1391
    if (!event)
1392
        return;
1393

1394
    switch (event->event) {
1395
        case GD_FRIEND_EVENT_RCVD_FRIEND_REQ:
1396
        case GD_FRIEND_EVENT_RCVD_REMOVE_FRIEND:
1397
            glusterd_destroy_friend_req_ctx(event->ctx);
1398
            break;
1399
        case GD_FRIEND_EVENT_LOCAL_ACC:
1400
        case GD_FRIEND_EVENT_LOCAL_RJT:
1401
        case GD_FRIEND_EVENT_RCVD_ACC:
1402
        case GD_FRIEND_EVENT_RCVD_RJT:
1403
            glusterd_destroy_friend_update_ctx(event->ctx);
1404
            break;
1405
        default:
1406
            break;
1407
    }
1408
}
1409

1410
gf_boolean_t
1411
gd_does_peer_affect_quorum(glusterd_friend_sm_state_t old_state,
1412
                           glusterd_friend_sm_event_type_t event_type,
1413
                           glusterd_peerinfo_t *peerinfo)
1414
{
1415
    gf_boolean_t affects = _gf_false;
1416

1417
    // When glusterd comes up with friends in BEFRIENDED state in store,
1418
    // wait until compare-data happens.
1419
    if ((old_state == GD_FRIEND_STATE_BEFRIENDED) &&
1420
        (event_type != GD_FRIEND_EVENT_RCVD_ACC) &&
1421
        (event_type != GD_FRIEND_EVENT_LOCAL_ACC))
1422
        goto out;
1423
    if ((peerinfo->state == GD_FRIEND_STATE_BEFRIENDED) &&
1424
        peerinfo->connected) {
1425
        affects = _gf_true;
1426
    }
1427
out:
1428
    return affects;
1429
}
1430

1431
int
1432
glusterd_friend_sm(void)
1433
{
1434
    glusterd_friend_sm_event_t *event = NULL;
1435
    glusterd_friend_sm_event_t *tmp = NULL;
1436
    int ret = -1;
1437
    glusterd_friend_sm_ac_fn handler = NULL;
1438
    glusterd_sm_t *state = NULL;
1439
    glusterd_peerinfo_t *peerinfo = NULL;
1440
    glusterd_friend_sm_event_type_t event_type = 0;
1441
    gf_boolean_t is_await_conn = _gf_false;
1442
    gf_boolean_t quorum_action = _gf_false;
1443
    glusterd_friend_sm_state_t old_state = GD_FRIEND_STATE_DEFAULT;
1444
    xlator_t *this = THIS;
1445
    glusterd_conf_t *priv = NULL;
1446

1447
    priv = this->private;
1448
    GF_ASSERT(priv);
1449

1450
    while (!cds_list_empty(&gd_friend_sm_queue)) {
1451
        cds_list_for_each_entry_safe(event, tmp, &gd_friend_sm_queue, list)
1452
        {
1453
            cds_list_del_init(&event->list);
1454
            event_type = event->event;
1455

1456
            RCU_READ_LOCK;
1457

1458
            peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
1459
            if (!peerinfo) {
1460
                RCU_READ_UNLOCK;
1461
                gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_PEER_NOT_FOUND,
1462
                       "Received"
1463
                       " event %s with empty peer info",
1464
                       glusterd_friend_sm_event_name_get(event_type));
1465

1466
                GF_FREE(event);
1467
                continue;
1468
            }
1469
            old_state = peerinfo->state;
1470
            RCU_READ_UNLOCK;
1471
            gf_msg_debug("glusterd", 0, "Dequeued event of type: '%s'",
1472
                         glusterd_friend_sm_event_name_get(event_type));
1473

1474
            /* Giving up read-critical section here as we only need
1475
             * the current state to call the handler.
1476
             *
1477
             * We cannot continue into the handler in a read
1478
             * critical section as there are handlers who do
1479
             * updates, and could cause deadlocks.
1480
             */
1481

1482
            state = glusterd_friend_state_table[old_state];
1483

1484
            GF_ASSERT(state);
1485

1486
            handler = state[event_type].handler;
1487
            GF_ASSERT(handler);
1488

1489
            ret = handler(event, event->ctx);
1490
            if (ret == GLUSTERD_CONNECTION_AWAITED) {
1491
                is_await_conn = _gf_true;
1492
                ret = 0;
1493
            }
1494

1495
            if (ret) {
1496
                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_HANDLER_RETURNED,
1497
                       "handler returned: "
1498
                       "%d",
1499
                       ret);
1500
                glusterd_destroy_friend_event_context(event);
1501
                GF_FREE(event);
1502
                continue;
1503
            }
1504

1505
            if ((GD_FRIEND_EVENT_REMOVE_FRIEND == event_type) ||
1506
                (GD_FRIEND_EVENT_INIT_REMOVE_FRIEND == event_type)) {
1507
                glusterd_destroy_friend_event_context(event);
1508
                GF_FREE(event);
1509
                continue;
1510
            }
1511

1512
            ret = glusterd_friend_sm_transition_state(
1513
                event->peerid, event->peername, state, event_type);
1514

1515
            if (ret) {
1516
                gf_msg("glusterd", GF_LOG_ERROR, 0,
1517
                       GD_MSG_EVENT_STATE_TRANSITION_FAIL,
1518
                       "Unable to transition"
1519
                       " state from '%s' to '%s' for event '%s'",
1520
                       glusterd_friend_sm_state_name_get(old_state),
1521
                       glusterd_friend_sm_state_name_get(
1522
                           state[event_type].next_state),
1523
                       glusterd_friend_sm_event_name_get(event_type));
1524
                goto out;
1525
            }
1526

1527
            peerinfo = NULL;
1528
            /* We need to obtain peerinfo reference once again as we
1529
             * had exited the read critical section above.
1530
             */
1531
            RCU_READ_LOCK;
1532
            peerinfo = glusterd_peerinfo_find(event->peerid, event->peername);
1533
            if (!peerinfo) {
1534
                RCU_READ_UNLOCK;
1535
                /* A peer can only be deleted as a effect of
1536
                 * this state machine, and two such state
1537
                 * machines can never run at the same time.
1538
                 * So if we cannot find the peerinfo here,
1539
                 * something has gone terribly wrong.
1540
                 */
1541
                ret = -1;
1542
                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_PEER_NOT_FOUND,
1543
                       "Cannot find peer %s(%s)", event->peername,
1544
                       uuid_utoa(event->peerid));
1545
                goto out;
1546
            }
1547
            if (gd_does_peer_affect_quorum(old_state, event_type, peerinfo)) {
1548
                peerinfo->quorum_contrib = QUORUM_UP;
1549
                if (peerinfo->quorum_action) {
1550
                    peerinfo->quorum_action = _gf_false;
1551
                    quorum_action = _gf_true;
1552
                }
1553
            }
1554

1555
            ret = glusterd_store_peerinfo(peerinfo);
1556
            RCU_READ_UNLOCK;
1557
            if (ret) {
1558
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PEERINFO_CREATE_FAIL,
1559
                       "Failed to store peerinfo");
1560
            }
1561

1562
            glusterd_destroy_friend_event_context(event);
1563
            GF_FREE(event);
1564
            if (is_await_conn)
1565
                break;
1566
        }
1567
        if (is_await_conn)
1568
            break;
1569
    }
1570

1571
    ret = 0;
1572
out:
1573
    if (quorum_action) {
1574
        /* When glusterd is restarted, it needs to wait until the 'friends' view
1575
         * of the volumes settle, before it starts any of the internal daemons.
1576
         *
1577
         * Every friend that was part of the cluster, would send its
1578
         * cluster-view, 'our' way. For every friend, who belongs to
1579
         * a partition which has a different cluster-view from our
1580
         * partition, we may update our cluster-view. For subsequent
1581
         * friends from that partition would agree with us, if the first
1582
         * friend wasn't rejected. For every first friend, whom we agreed with,
1583
         * we would need to start internal daemons/bricks belonging to the
1584
         * new volumes.
1585
         * glusterd_spawn_daemons calls functions that are idempotent. ie,
1586
         * the functions spawn process(es) only if they are not started yet.
1587
         *
1588
         * */
1589
        synclock_unlock(&priv->big_lock);
1590
        glusterd_launch_synctask(glusterd_spawn_daemons, NULL);
1591
        synclock_lock(&priv->big_lock);
1592
        glusterd_do_quorum_action();
1593
    }
1594
    return ret;
1595
}
1596

1597
int
1598
glusterd_friend_sm_init(void)
1599
{
1600
    CDS_INIT_LIST_HEAD(&gd_friend_sm_queue);
1601
    return 0;
1602
}
1603