glusterfs

1
/*
2
   Copyright (c) 2011-2012 Red Hat, Inc. <http://www.redhat.com>
3
   This file is part of GlusterFS.
4

5
   This file is licensed to you under your choice of the GNU Lesser
6
   General Public License, version 3 or any later version (LGPLv3 or
7
   later), or the GNU General Public License, version 2 (GPLv2), in all
8
   cases as published by the Free Software Foundation.
9
*/
10
#include <glusterfs/common-utils.h>
11
#include <glusterfs/glusterfs.h>
12
#include "glusterd-op-sm.h"
13
#include "glusterd-geo-rep.h"
14
#include "glusterd-store.h"
15
#include "glusterd-utils.h"
16
#include "glusterd-svc-mgmt.h"
17
#include "glusterd-svc-helper.h"
18
#include "glusterd-volgen.h"
19
#include "glusterd-messages.h"
20
#include "glusterd-server-quorum.h"
21
#include "glusterd-mgmt.h"
22
#include <glusterfs/run.h>
23
#include <glusterfs/syscall.h>
24

25
#include <signal.h>
26

27
int
28
glusterd_mgmt_v3_initiate_replace_brick_cmd_phases(rpcsvc_request_t *req,
29
                                                   glusterd_op_t op,
30
                                                   dict_t *dict);
31
int
32
__glusterd_handle_replace_brick(rpcsvc_request_t *req)
33
{
34
    int32_t ret = -1;
35
    gf_cli_req cli_req = {{
36
        0,
37
    }};
38
    dict_t *dict = NULL;
39
    char *src_brick = NULL;
40
    char *dst_brick = NULL;
41
    char *cli_op = NULL;
42
    glusterd_op_t op = -1;
43
    char *volname = NULL;
44
    char msg[256] = {
45
        0,
46
    };
47
    xlator_t *this = THIS;
48
    glusterd_conf_t *conf = NULL;
49

50
    GF_ASSERT(req);
51
    conf = this->private;
52

53
    ret = xdr_to_generic(req->msg[0], &cli_req, (xdrproc_t)xdr_gf_cli_req);
54
    if (ret < 0) {
55
        // failed to decode msg;
56
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_REQ_DECODE_FAIL,
57
               "Failed to decode "
58
               "request received from cli");
59
        req->rpc_err = GARBAGE_ARGS;
60
        goto out;
61
    }
62

63
    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_REPLACE_BRK_REQ_RCVD,
64
           "Received replace brick req");
65

66
    if (cli_req.dict.dict_len) {
67
        /* Unserialize the dictionary */
68
        dict = dict_new();
69

70
        ret = dict_unserialize(cli_req.dict.dict_val, cli_req.dict.dict_len,
71
                               &dict);
72
        if (ret < 0) {
73
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_UNSERIALIZE_FAIL,
74
                   "failed to "
75
                   "unserialize req-buffer to dictionary");
76
            snprintf(msg, sizeof(msg),
77
                     "Unable to decode the "
78
                     "command");
79
            goto out;
80
        }
81
    }
82

83
    ret = dict_get_str(dict, "volname", &volname);
84
    if (ret) {
85
        snprintf(msg, sizeof(msg), "Could not get volume name");
86
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
87
        goto out;
88
    }
89

90
    ret = dict_get_str(dict, "operation", &cli_op);
91
    if (ret) {
92
        gf_msg_debug(this->name, 0, "dict_get on operation failed");
93
        snprintf(msg, sizeof(msg), "Could not get operation");
94
        goto out;
95
    }
96

97
    op = gd_cli_to_gd_op(cli_op);
98

99
    if (conf->op_version < GD_OP_VERSION_3_9_0 &&
100
        strcmp(cli_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
101
        snprintf(msg, sizeof(msg),
102
                 "Cannot execute command. The "
103
                 "cluster is operating at version %d. reset-brick "
104
                 "command %s is unavailable in this version.",
105
                 conf->op_version, gd_rb_op_to_str(cli_op));
106
        ret = -1;
107
        goto out;
108
    }
109

110
    ret = dict_get_str(dict, "src-brick", &src_brick);
111

112
    if (ret) {
113
        snprintf(msg, sizeof(msg), "Failed to get src brick");
114
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s", msg);
115
        goto out;
116
    }
117
    gf_msg_debug(this->name, 0, "src brick=%s", src_brick);
118

119
    if (!strcmp(cli_op, "GF_RESET_OP_COMMIT") ||
120
        !strcmp(cli_op, "GF_RESET_OP_COMMIT_FORCE") ||
121
        !strcmp(cli_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
122
        ret = dict_get_str(dict, "dst-brick", &dst_brick);
123

124
        if (ret) {
125
            snprintf(msg, sizeof(msg),
126
                     "Failed to get"
127
                     "dest brick");
128
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED, "%s",
129
                   msg);
130
            goto out;
131
        }
132

133
        gf_msg_debug(this->name, 0, "dst brick=%s", dst_brick);
134
    }
135

136
    gf_msg(this->name, GF_LOG_INFO, 0,
137
           (op == GD_OP_REPLACE_BRICK)
138
               ? GD_MSG_REPLACE_BRK_COMMIT_FORCE_REQ_RCVD
139
               : GD_MSG_RESET_BRICK_COMMIT_FORCE_REQ_RCVD,
140
           "Received %s request.", gd_rb_op_to_str(cli_op));
141

142
    ret = glusterd_mgmt_v3_initiate_replace_brick_cmd_phases(req, op, dict);
143

144
out:
145
    if (ret) {
146
        glusterd_op_send_cli_response(op, ret, 0, req, dict, msg);
147
    }
148
    ret = 0;
149
    free(cli_req.dict.dict_val);  // malloced by xdr
150

151
    return ret;
152
}
153

154
int
155
glusterd_handle_reset_brick(rpcsvc_request_t *req)
156
{
157
    return glusterd_big_locked_handler(req, __glusterd_handle_replace_brick);
158
}
159

160
int
161
glusterd_handle_replace_brick(rpcsvc_request_t *req)
162
{
163
    return glusterd_big_locked_handler(req, __glusterd_handle_replace_brick);
164
}
165

166
static int
167
glusterd_rb_check_bricks(glusterd_volinfo_t *volinfo, glusterd_brickinfo_t *src,
168
                         glusterd_brickinfo_t *dst)
169
{
170
    glusterd_replace_brick_t *rb = NULL;
171

172
    GF_ASSERT(volinfo);
173

174
    rb = &volinfo->rep_brick;
175

176
    if (!rb->src_brick || !rb->dst_brick) {
177
        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
178
        return -1;
179
    }
180

181
    if (strcmp(rb->src_brick->hostname, src->hostname) ||
182
        strcmp(rb->src_brick->path, src->path)) {
183
        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RB_SRC_BRICKS_MISMATCH,
184
               "Replace brick src bricks differ");
185
        return -1;
186
    }
187

188
    if (strcmp(rb->dst_brick->hostname, dst->hostname) ||
189
        strcmp(rb->dst_brick->path, dst->path)) {
190
        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RB_DST_BRICKS_MISMATCH,
191
               "Replace brick dst bricks differ");
192
        return -1;
193
    }
194

195
    return 0;
196
}
197

198
int
199
glusterd_op_stage_replace_brick(dict_t *dict, char **op_errstr,
200
                                dict_t *rsp_dict)
201
{
202
    int ret = 0;
203
    char *src_brick = NULL;
204
    char *dst_brick = NULL;
205
    char *volname = NULL;
206
    char *op = NULL;
207
    glusterd_op_t gd_op = -1;
208
    glusterd_volinfo_t *volinfo = NULL;
209
    glusterd_brickinfo_t *src_brickinfo = NULL;
210
    char *host = NULL;
211
    char msg[2048] = {0};
212
    glusterd_peerinfo_t *peerinfo = NULL;
213
    glusterd_brickinfo_t *dst_brickinfo = NULL;
214
    glusterd_conf_t *priv = NULL;
215
    char pidfile[PATH_MAX] = {0};
216
    xlator_t *this = THIS;
217
    gf_boolean_t is_force = _gf_false;
218
    char *dup_dstbrick = NULL;
219

220
    priv = this->private;
221
    GF_ASSERT(priv);
222

223
    ret = glusterd_brick_op_prerequisites(dict, &op, &gd_op, &volname, &volinfo,
224
                                          &src_brick, &src_brickinfo, pidfile,
225
                                          op_errstr, rsp_dict);
226
    if (ret)
227
        goto out;
228

229
    if (volinfo->type == GF_CLUSTER_TYPE_NONE) {
230
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_NOT_PERMITTED,
231
               "replace-brick is not permitted on distribute only "
232
               "volumes");
233
        gf_asprintf(op_errstr,
234
                    "replace-brick is not permitted on "
235
                    "distribute only volumes. Please use add-brick "
236
                    "and remove-brick operations instead.");
237
        ret = -1;
238
        goto out;
239
    }
240
    ret = glusterd_validate_quorum(this, gd_op, dict, op_errstr);
241
    if (ret) {
242
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SERVER_QUORUM_NOT_MET,
243
               "Server quorum not met. Rejecting operation.");
244
        goto out;
245
    }
246

247
    if (strcmp(op, "GF_REPLACE_OP_COMMIT_FORCE")) {
248
        ret = -1;
249
        goto out;
250
    } else {
251
        is_force = _gf_true;
252
    }
253

254
    if (volinfo->snap_count > 0 || !cds_list_empty(&volinfo->snap_volumes)) {
255
        snprintf(msg, sizeof(msg),
256
                 "Volume %s  has %" PRIu64
257
                 " snapshots. "
258
                 "Changing the volume configuration will not effect snapshots."
259
                 "But the snapshot brick mount should be intact to "
260
                 "make them function.",
261
                 volname, volinfo->snap_count);
262
        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_SNAP_WARN, "%s", msg);
263
        msg[0] = '\0';
264
    }
265

266
    glusterd_add_peers_to_auth_list(volname);
267

268
    ret = glusterd_get_dst_brick_info(&dst_brick, volname, op_errstr,
269
                                      &dst_brickinfo, &host, dict,
270
                                      &dup_dstbrick);
271
    if (ret)
272
        goto out;
273

274
    ret = glusterd_new_brick_validate(dst_brick, dst_brickinfo, msg,
275
                                      sizeof(msg), op);
276
    /* fail if brick being replaced with itself */
277
    if (ret) {
278
        *op_errstr = gf_strdup(msg);
279
        ret = -1;
280
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_VALIDATE_FAIL, "%s",
281
               *op_errstr);
282
        goto out;
283
    }
284

285
    volinfo->rep_brick.src_brick = src_brickinfo;
286
    volinfo->rep_brick.dst_brick = dst_brickinfo;
287

288
    if (glusterd_rb_check_bricks(volinfo, src_brickinfo, dst_brickinfo)) {
289
        ret = -1;
290
        *op_errstr = gf_strdup(
291
            "Incorrect source or "
292
            "destination brick");
293
        if (*op_errstr)
294
            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_BRICK_NOT_FOUND,
295
                   "%s", *op_errstr);
296
        goto out;
297
    }
298

299
    if (glusterd_gf_is_local_addr(host)) {
300
        ret = glusterd_validate_and_create_brickpath(
301
            dst_brickinfo, volinfo->volume_id, volinfo->volname, op_errstr,
302
            is_force, _gf_false);
303
        if (ret)
304
            goto out;
305
    }
306

307
    if (!glusterd_gf_is_local_addr(host)) {
308
        RCU_READ_LOCK;
309

310
        peerinfo = glusterd_peerinfo_find(NULL, host);
311
        if (peerinfo == NULL) {
312
            RCU_READ_UNLOCK;
313
            ret = -1;
314
            snprintf(msg, sizeof(msg), "%s, is not a friend", host);
315
            *op_errstr = gf_strdup(msg);
316
            goto out;
317

318
        } else if (!peerinfo->connected) {
319
            RCU_READ_UNLOCK;
320
            ret = -1;
321
            snprintf(msg, sizeof(msg),
322
                     "%s, is not connected at "
323
                     "the moment",
324
                     host);
325
            *op_errstr = gf_strdup(msg);
326
            goto out;
327

328
        } else if (GD_FRIEND_STATE_BEFRIENDED != peerinfo->state) {
329
            RCU_READ_UNLOCK;
330
            ret = -1;
331
            snprintf(msg, sizeof(msg),
332
                     "%s, is not befriended "
333
                     "at the moment",
334
                     host);
335
            *op_errstr = gf_strdup(msg);
336
            goto out;
337
        }
338
        RCU_READ_UNLOCK;
339

340
    } else {
341
        if (!(gf_uuid_compare(dst_brickinfo->uuid, MY_UUID))) {
342
            ret = glusterd_get_brick_mount_dir(dst_brickinfo->path,
343
                                               dst_brickinfo->hostname,
344
                                               dst_brickinfo->mount_dir);
345
            if (ret) {
346
                gf_msg(this->name, GF_LOG_ERROR, 0,
347
                       GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
348
                       "Failed to get brick mount_dir");
349
                goto out;
350
            }
351
            ret = dict_set_dynstr_with_alloc(rsp_dict, "brick1.mount_dir",
352
                                             dst_brickinfo->mount_dir);
353
            if (ret) {
354
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
355
                       "Failed to set brick.mount_dir");
356
                goto out;
357
            }
358
        }
359

360
        ret = dict_set_int32_sizen(rsp_dict, "brick_count", 1);
361
        if (ret) {
362
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
363
                   "Failed to set local_brick_count");
364
            goto out;
365
        }
366
    }
367

368
    ret = 0;
369

370
out:
371
    GF_FREE(dup_dstbrick);
372
    gf_msg_debug(this->name, 0, "Returning %d", ret);
373

374
    return ret;
375
}
376

377
int
378
glusterd_op_perform_replace_brick(glusterd_volinfo_t *volinfo, char *old_brick,
379
                                  char *new_brick, dict_t *dict)
380
{
381
    char *brick_mount_dir = NULL;
382
    glusterd_brickinfo_t *old_brickinfo = NULL;
383
    glusterd_brickinfo_t *new_brickinfo = NULL;
384
    int32_t ret = -1;
385
    xlator_t *this = THIS;
386
    struct statvfs brickstat = {
387
        0,
388
    };
389

390
    GF_ASSERT(dict);
391
    GF_ASSERT(volinfo);
392

393
    ret = glusterd_brickinfo_new_from_brick(new_brick, &new_brickinfo, _gf_true,
394
                                            NULL);
395
    if (ret)
396
        goto out;
397

398
    ret = glusterd_resolve_brick(new_brickinfo);
399
    if (ret)
400
        goto out;
401

402
    if (!gf_uuid_compare(new_brickinfo->uuid, MY_UUID)) {
403
        ret = sys_statvfs(new_brickinfo->path, &brickstat);
404
        if (ret) {
405
            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_STATVFS_FAILED,
406
                   "Failed to fetch disk utilization "
407
                   "from the brick (%s:%s). Please check the health of "
408
                   "the brick. Error code was %s",
409
                   new_brickinfo->hostname, new_brickinfo->path,
410
                   strerror(errno));
411

412
            goto out;
413
        }
414
        new_brickinfo->statfs_fsid = brickstat.f_fsid;
415
    }
416

417
    ret = glusterd_volume_brickinfo_get_by_brick(old_brick, volinfo,
418
                                                 &old_brickinfo, _gf_false);
419
    if (ret)
420
        goto out;
421

422
    (void)snprintf(new_brickinfo->brick_id, sizeof(new_brickinfo->brick_id),
423
                   "%s", old_brickinfo->brick_id);
424
    new_brickinfo->port = old_brickinfo->port;
425

426
    ret = dict_get_str(dict, "brick1.mount_dir", &brick_mount_dir);
427
    if (ret) {
428
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_BRICK_MOUNTDIR_GET_FAIL,
429
               "brick1.mount_dir not present");
430
        goto out;
431
    }
432
    (void)snprintf(new_brickinfo->mount_dir, sizeof(new_brickinfo->mount_dir),
433
                   "%s", brick_mount_dir);
434

435
    cds_list_add(&new_brickinfo->brick_list, &old_brickinfo->brick_list);
436

437
    volinfo->brick_count++;
438

439
    ret = glusterd_op_perform_remove_brick(volinfo, old_brick, 1, NULL);
440
    if (ret)
441
        goto out;
442

443
    /* if the volume is a replicate volume, do: */
444
    if (glusterd_is_volume_replicate(volinfo)) {
445
        if (!gf_uuid_compare(new_brickinfo->uuid, MY_UUID)) {
446
            ret = glusterd_handle_replicate_brick_ops(volinfo, new_brickinfo,
447
                                                      GD_OP_REPLACE_BRICK);
448
            if (ret < 0)
449
                goto out;
450
        }
451
    }
452

453
    ret = glusterd_create_volfiles_and_notify_services(volinfo);
454
    if (ret)
455
        goto out;
456

457
    if (GLUSTERD_STATUS_STARTED == volinfo->status) {
458
        ret = glusterd_brick_start(volinfo, new_brickinfo, _gf_false,
459
                                   _gf_false);
460
        if (ret)
461
            goto out;
462
    }
463

464
out:
465

466
    gf_msg_debug("glusterd", 0, "Returning %d", ret);
467
    return ret;
468
}
469

470
int
471
glusterd_op_replace_brick(dict_t *dict, dict_t *rsp_dict)
472
{
473
    int ret = 0;
474
    char *replace_op = NULL;
475
    glusterd_volinfo_t *volinfo = NULL;
476
    char *volname = NULL;
477
    xlator_t *this = THIS;
478
    glusterd_conf_t *priv = NULL;
479
    char *src_brick = NULL;
480
    char *dst_brick = NULL;
481
    glusterd_brickinfo_t *src_brickinfo = NULL;
482
    glusterd_brickinfo_t *dst_brickinfo = NULL;
483

484
    priv = this->private;
485
    GF_ASSERT(priv);
486

487
    ret = dict_get_str(dict, "src-brick", &src_brick);
488
    if (ret) {
489
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
490
               "Unable to get src brick");
491
        goto out;
492
    }
493

494
    gf_msg_debug(this->name, 0, "src brick=%s", src_brick);
495

496
    ret = dict_get_str(dict, "dst-brick", &dst_brick);
497
    if (ret) {
498
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
499
               "Unable to get dst brick");
500
        goto out;
501
    }
502

503
    gf_msg_debug(this->name, 0, "dst brick=%s", dst_brick);
504

505
    ret = dict_get_str(dict, "volname", &volname);
506
    if (ret) {
507
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
508
               "Unable to get volume name");
509
        goto out;
510
    }
511

512
    ret = dict_get_str(dict, "operation", &replace_op);
513
    if (ret) {
514
        gf_msg_debug(this->name, 0, "dict_get on operation failed");
515
        goto out;
516
    }
517

518
    ret = glusterd_volinfo_find(volname, &volinfo);
519
    if (ret) {
520
        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
521
               "Unable to allocate memory");
522
        goto out;
523
    }
524

525
    ret = glusterd_volume_brickinfo_get_by_brick(src_brick, volinfo,
526
                                                 &src_brickinfo, _gf_false);
527
    if (ret) {
528
        gf_msg_debug(this->name, 0, "Unable to get src-brickinfo");
529
        goto out;
530
    }
531

532
    ret = glusterd_get_rb_dst_brickinfo(volinfo, &dst_brickinfo);
533
    if (ret) {
534
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RB_BRICKINFO_GET_FAIL,
535
               "Unable to get "
536
               "replace brick destination brickinfo");
537
        goto out;
538
    }
539

540
    ret = glusterd_resolve_brick(dst_brickinfo);
541
    if (ret) {
542
        gf_msg_debug(this->name, 0, "Unable to resolve dst-brickinfo");
543
        goto out;
544
    }
545

546
    ret = rb_update_dstbrick_port(dst_brickinfo, rsp_dict, dict);
547
    if (ret)
548
        goto out;
549

550
    if (strcmp(replace_op, "GF_REPLACE_OP_COMMIT_FORCE")) {
551
        ret = -1;
552
        goto out;
553
    }
554

555
    ret = glusterd_svcs_stop(volinfo);
556
    if (ret) {
557
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_GLUSTER_SERVICES_STOP_FAIL,
558
               "Unable to stop gluster services, ret: %d", ret);
559
    }
560

561
    ret = glusterd_op_perform_replace_brick(volinfo, src_brick, dst_brick,
562
                                            dict);
563
    if (ret) {
564
        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_BRICK_ADD_FAIL,
565
               "Unable to add dst-brick: "
566
               "%s to volume: %s",
567
               dst_brick, volinfo->volname);
568
        (void)glusterd_svcs_manager(volinfo);
569
        goto out;
570
    }
571
    if (volinfo->rebal.defrag_status != GF_DEFRAG_STATUS_NOT_STARTED)
572
        volinfo->rebal.defrag_status = GF_DEFRAG_STATUS_RESET_DUE_REPLACE_BRC;
573

574
    ret = glusterd_svcs_manager(volinfo);
575
    if (ret) {
576
        gf_msg(this->name, GF_LOG_CRITICAL, 0,
577
               GD_MSG_GLUSTER_SERVICE_START_FAIL,
578
               "Failed to start one or more gluster services.");
579
    }
580

581
    ret = glusterd_fetchspec_notify(THIS);
582
    glusterd_brickinfo_delete(volinfo->rep_brick.dst_brick);
583
    volinfo->rep_brick.src_brick = NULL;
584
    volinfo->rep_brick.dst_brick = NULL;
585

586
    if (!ret)
587
        ret = glusterd_store_volinfo(volinfo,
588
                                     GLUSTERD_VOLINFO_VER_AC_INCREMENT);
589
    if (ret)
590
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RBOP_STATE_STORE_FAIL,
591
               "Couldn't store"
592
               " replace brick operation's state");
593

594
out:
595
    return ret;
596
}
597

598
int
599
glusterd_mgmt_v3_initiate_replace_brick_cmd_phases(rpcsvc_request_t *req,
600
                                                   glusterd_op_t op,
601
                                                   dict_t *dict)
602
{
603
    int32_t ret = -1;
604
    int32_t op_ret = -1;
605
    uint32_t txn_generation = 0;
606
    uint32_t op_errno = 0;
607
    char *op_errstr = NULL;
608
    dict_t *req_dict = NULL;
609
    dict_t *tmp_dict = NULL;
610
    uuid_t *originator_uuid = NULL;
611
    xlator_t *this = THIS;
612
    glusterd_conf_t *conf = NULL;
613
    gf_boolean_t is_acquired = _gf_false;
614

615
    GF_ASSERT(req);
616
    GF_ASSERT(dict);
617
    conf = this->private;
618
    GF_ASSERT(conf);
619

620
    txn_generation = conf->generation;
621
    originator_uuid = GF_MALLOC(sizeof(uuid_t), gf_common_mt_uuid_t);
622
    if (!originator_uuid) {
623
        ret = -1;
624
        goto out;
625
    }
626

627
    gf_uuid_copy(*originator_uuid, MY_UUID);
628
    ret = dict_set_bin(dict, "originator_uuid", originator_uuid,
629
                       sizeof(uuid_t));
630
    if (ret) {
631
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
632
               "Failed to set originator_uuid.");
633
        GF_FREE(originator_uuid);
634
        goto out;
635
    }
636

637
    ret = dict_set_int32_sizen(dict, "is_synctasked", _gf_true);
638
    if (ret) {
639
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
640
               "Failed to set synctasked flag to true.");
641
        goto out;
642
    }
643

644
    tmp_dict = dict_new();
645
    if (!tmp_dict) {
646
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
647
               "Unable to create dict");
648
        goto out;
649
    }
650
    dict_copy(dict, tmp_dict);
651

652
    ret = glusterd_mgmt_v3_initiate_lockdown(op, dict, &op_errstr, &op_errno,
653
                                             &is_acquired, txn_generation);
654
    if (ret) {
655
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_LOCKDOWN_FAIL,
656
               "mgmt_v3 lockdown failed.");
657
        goto out;
658
    }
659

660
    ret = glusterd_mgmt_v3_build_payload(&req_dict, &op_errstr, dict, op);
661
    if (ret) {
662
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_PAYLOAD_BUILD_FAIL,
663
               LOGSTR_BUILD_PAYLOAD, gd_op_list[op]);
664
        if (op_errstr == NULL)
665
            gf_asprintf(&op_errstr, OPERRSTR_BUILD_PAYLOAD);
666
        goto out;
667
    }
668

669
    ret = glusterd_mgmt_v3_pre_validate(op, req_dict, &op_errstr, &op_errno,
670
                                        txn_generation);
671
    if (ret) {
672
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PRE_VALIDATION_FAIL,
673
               "Pre Validation Failed");
674
        goto out;
675
    }
676

677
    ret = glusterd_mgmt_v3_commit(op, dict, req_dict, &op_errstr, &op_errno,
678
                                  txn_generation);
679
    if (ret) {
680
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_COMMIT_OP_FAIL,
681
               "Commit Op Failed");
682
        goto out;
683
    }
684

685
    ret = 0;
686

687
out:
688
    op_ret = ret;
689

690
    (void)glusterd_mgmt_v3_release_peer_locks(op, dict, op_ret, &op_errstr,
691
                                              is_acquired, txn_generation);
692

693
    if (is_acquired) {
694
        ret = glusterd_multiple_mgmt_v3_unlock(tmp_dict, MY_UUID);
695
        if (ret) {
696
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MGMTV3_UNLOCK_FAIL,
697
                   "Failed to release mgmt_v3 locks on "
698
                   "localhost.");
699
            op_ret = ret;
700
        }
701
    }
702
    /* SEND CLI RESPONSE */
703
    glusterd_op_send_cli_response(op, op_ret, op_errno, req, dict, op_errstr);
704

705
    if (req_dict)
706
        dict_unref(req_dict);
707

708
    if (tmp_dict)
709
        dict_unref(tmp_dict);
710

711
    if (op_errstr) {
712
        GF_FREE(op_errstr);
713
        op_errstr = NULL;
714
    }
715

716
    return 0;
717
}
718