glusterfs

1
/*
2
   Copyright (c) 2006-2013 Red Hat, Inc. <http://www.redhat.com>
3
   This file is part of GlusterFS.
4

5
   This file is licensed to you under your choice of the GNU Lesser
6
   General Public License, version 3 or any later version (LGPLv3 or
7
   later), or the GNU General Public License, version 2 (GPLv2), in all
8
   cases as published by the Free Software Foundation.
9
*/
10

11
#include <time.h>
12
#include <grp.h>
13
#include <sys/uio.h>
14

15
#include <libgen.h>
16
#include <glusterfs/compat-uuid.h>
17

18
#include "rpc-common-xdr.h"
19

20
#include "glusterd.h"
21
#include "rpcsvc.h"
22
#include "fnmatch.h"
23
#include <glusterfs/call-stub.h>
24
#include <glusterfs/list.h>
25
#include <glusterfs/dict.h>
26
#include <glusterfs/options.h>
27
#include <glusterfs/compat.h>
28
#include <glusterfs/compat-errno.h>
29
#include <glusterfs/syscall.h>
30
#include "glusterd-statedump.h"
31
#include "glusterd-sm.h"
32
#include "glusterd-op-sm.h"
33
#include "glusterd-store.h"
34
#include "glusterd-hooks.h"
35
#include "glusterd-utils.h"
36
#include "glusterd-locks.h"
37
#include "glusterd-svc-mgmt.h"
38
#include "glusterd-shd-svc.h"
39
#ifdef BUILD_GNFS
40
#include "glusterd-nfs-svc.h"
41
#endif
42
#include "glusterd-bitd-svc.h"
43
#include "glusterd-scrub-svc.h"
44
#include "glusterd-quotad-svc.h"
45
#include "glusterd-snapd-svc.h"
46
#include "glusterd-messages.h"
47
#include <glusterfs/common-utils.h>
48
#include "glusterd-geo-rep.h"
49
#include <glusterfs/run.h>
50
#include "rpc-clnt-ping.h"
51

52
#include <glusterfs/syncop.h>
53

54
#include "glusterd-mountbroker.h"
55

56
extern struct rpcsvc_program gluster_handshake_prog;
57
extern struct rpcsvc_program gluster_cli_getspec_prog;
58
extern struct rpcsvc_program gluster_pmap_prog;
59
extern glusterd_op_info_t opinfo;
60
extern struct rpcsvc_program gd_svc_mgmt_prog;
61
extern struct rpcsvc_program gd_svc_mgmt_v3_prog;
62
extern struct rpcsvc_program gd_svc_peer_prog;
63
extern struct rpcsvc_program gd_svc_cli_prog;
64
extern struct rpcsvc_program gd_svc_cli_trusted_progs;
65
extern struct rpc_clnt_program gd_brick_prog;
66
extern struct rpcsvc_program glusterd_mgmt_hndsk_prog;
67

68
extern char snap_mount_dir[VALID_GLUSTERD_PATHMAX];
69

70
rpcsvc_cbk_program_t glusterd_cbk_prog = {
71
    .progname = "Gluster Callback",
72
    .prognum = GLUSTER_CBK_PROGRAM,
73
    .progver = GLUSTER_CBK_VERSION,
74
};
75

76
struct rpcsvc_program *gd_inet_programs[] = {
77
    &gd_svc_peer_prog,       &gd_svc_cli_trusted_progs, /* Must be index 1 for
78
                                                           secure_mgmt! */
79
    &gd_svc_mgmt_prog,       &gd_svc_mgmt_v3_prog,      &gluster_pmap_prog,
80
    &gluster_handshake_prog, &glusterd_mgmt_hndsk_prog,
81
};
82
int gd_inet_programs_count = (sizeof(gd_inet_programs) /
83
                              sizeof(gd_inet_programs[0]));
84

85
struct rpcsvc_program *gd_uds_programs[] = {
86
    &gd_svc_cli_prog,
87
    &gluster_cli_getspec_prog,
88
};
89
int gd_uds_programs_count = (sizeof(gd_uds_programs) /
90
                             sizeof(gd_uds_programs[0]));
91

92
const char *gd_op_list[GD_OP_MAX + 1] = {
93
    [GD_OP_NONE] = "Invalid op",
94
    [GD_OP_CREATE_VOLUME] = "Create",
95
    [GD_OP_START_BRICK] = "Start Brick",
96
    [GD_OP_STOP_BRICK] = "Stop Brick",
97
    [GD_OP_DELETE_VOLUME] = "Delete",
98
    [GD_OP_START_VOLUME] = "Start",
99
    [GD_OP_STOP_VOLUME] = "Stop",
100
    [GD_OP_DEFRAG_VOLUME] = "Rebalance",
101
    [GD_OP_ADD_BRICK] = "Add brick",
102
    [GD_OP_DETACH_TIER] = "Detach tier",
103
    [GD_OP_TIER_MIGRATE] = "Tier migration",
104
    [GD_OP_REMOVE_BRICK] = "Remove brick",
105
    [GD_OP_REPLACE_BRICK] = "Replace brick",
106
    [GD_OP_SET_VOLUME] = "Set",
107
    [GD_OP_RESET_VOLUME] = "Reset",
108
    [GD_OP_SYNC_VOLUME] = "Sync",
109
    [GD_OP_LOG_ROTATE] = "Log rotate",
110
    [GD_OP_GSYNC_SET] = "Geo-replication",
111
    [GD_OP_PROFILE_VOLUME] = "Profile",
112
    [GD_OP_QUOTA] = "Quota",
113
    [GD_OP_STATUS_VOLUME] = "Status",
114
    [GD_OP_REBALANCE] = "Rebalance",
115
    [GD_OP_HEAL_VOLUME] = "Heal",
116
    [GD_OP_STATEDUMP_VOLUME] = "Statedump",
117
    [GD_OP_LIST_VOLUME] = "Lists",
118
    [GD_OP_CLEARLOCKS_VOLUME] = "Clear locks",
119
    [GD_OP_DEFRAG_BRICK_VOLUME] = "Rebalance",
120
    [GD_OP_COPY_FILE] = "Copy File",
121
    [GD_OP_SYS_EXEC] = "Execute system commands",
122
    [GD_OP_GSYNC_CREATE] = "Geo-replication Create",
123
    [GD_OP_SNAP] = "Snapshot",
124
    [GD_OP_RESET_BRICK] = "Reset Brick",
125
    [GD_OP_MAX_OPVERSION] = "Maximum supported op-version",
126
    [GD_OP_MAX] = "Invalid op"};
127

128
#define GLUSTERD_DEFAULT_SNAPS_BRICK_DIR "/gluster/snaps"
129
#define GLUSTERD_BITD_RUN_DIR "/bitd"
130
#define GLUSTERD_SCRUB_RUN_DIR "/scrub"
131
#define GLUSTERD_NFS_RUN_DIR "/nfs"
132
#define GLUSTERD_QUOTAD_RUN_DIR "/quotad"
133
#define GLUSTERD_VAR_RUN_DIR "/var/run"
134
#define GLUSTERD_RUN_DIR "/run"
135

136
static int
137
glusterd_opinfo_init(void)
138
{
139
    int32_t ret = -1;
140

141
    opinfo.op = GD_OP_NONE;
142

143
    return ret;
144
}
145

146
int
147
glusterd_uuid_init(void)
148
{
149
    int ret = -1;
150
    xlator_t *this = THIS;
151
    glusterd_conf_t *priv = NULL;
152

153
    priv = this->private;
154
    GF_ASSERT(priv);
155

156
    ret = glusterd_retrieve_uuid();
157
    if (ret == 0) {
158
        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_RETRIEVED_UUID,
159
               "retrieved UUID: %s", uuid_utoa(priv->uuid));
160
        return 0;
161
    }
162

163
    ret = glusterd_uuid_generate_save();
164

165
    if (ret) {
166
        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_UUID_GEN_STORE_FAIL,
167
               "Unable to generate and save new UUID");
168
        return ret;
169
    }
170

171
    return 0;
172
}
173

174
int
175
glusterd_uuid_generate_save(void)
176
{
177
    int ret = -1;
178
    glusterd_conf_t *priv = NULL;
179
    xlator_t *this = THIS;
180

181
    priv = this->private;
182
    GF_ASSERT(priv);
183

184
    gf_uuid_generate(priv->uuid);
185

186
    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_GENERATED_UUID,
187
           "generated UUID: %s", uuid_utoa(priv->uuid));
188

189
    ret = glusterd_store_global_info(this);
190

191
    if (ret)
192
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UUID_STORE_FAIL,
193
               "Unable to store the generated uuid %s", uuid_utoa(priv->uuid));
194

195
    return ret;
196
}
197

198
int
199
glusterd_options_init(xlator_t *this)
200
{
201
    int ret = -1;
202
    glusterd_conf_t *priv = NULL;
203
    char *initial_version = "0";
204

205
    priv = this->private;
206

207
    priv->opts = dict_new();
208
    if (!priv->opts) {
209
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
210
        goto out;
211
    }
212

213
    ret = glusterd_store_retrieve_options(this);
214
    if (ret == 0) {
215
        goto out;
216
    }
217

218
    ret = dict_set_str(priv->opts, GLUSTERD_GLOBAL_OPT_VERSION,
219
                       initial_version);
220
    if (ret)
221
        goto out;
222

223
    ret = glusterd_store_options(this, priv->opts);
224
    if (ret) {
225
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VERS_STORE_FAIL,
226
               "Unable to store version");
227
        return ret;
228
    }
229
out:
230
    return 0;
231
}
232

233
int
234
glusterd_client_statedump_submit_req(char *volname, char *target_ip, char *pid)
235
{
236
    gf_statedump statedump_req = {
237
        0,
238
    };
239
    glusterd_conf_t *conf = NULL;
240
    int ret = 0;
241
    char *end_ptr = NULL;
242
    rpc_transport_t *trans = NULL;
243
    char *ip_addr = NULL;
244
    xlator_t *this = THIS;
245
    char tmp[UNIX_PATH_MAX] = {
246
        0,
247
    };
248

249
    conf = this->private;
250
    GF_ASSERT(conf);
251

252
    if (target_ip == NULL || pid == NULL) {
253
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_INVALID_ARGUMENT, NULL);
254
        ret = -1;
255
        goto out;
256
    }
257

258
    statedump_req.pid = strtol(pid, &end_ptr, 10);
259

260
    gf_msg_debug(this->name, 0,
261
                 "Performing statedump on volume %s "
262
                 "client with pid:%d host:%s",
263
                 volname, statedump_req.pid, target_ip);
264

265
    pthread_mutex_lock(&conf->xprt_lock);
266
    {
267
        list_for_each_entry(trans, &conf->xprt_list, list)
268
        {
269
            /* check if this connection matches "all" or the
270
             * volname */
271
            if (strncmp(volname, "all", NAME_MAX) &&
272
                strncmp(trans->peerinfo.volname, volname, NAME_MAX)) {
273
                /* no match, try next trans */
274
                continue;
275
            }
276

277
            strcpy(tmp, trans->peerinfo.identifier);
278
            ip_addr = strtok(tmp, ":");
279
            if (gf_is_same_address(ip_addr, target_ip)) {
280
                /* Every gluster client would have
281
                 * connected to glusterd(volfile server). This
282
                 * connection is used to send the statedump
283
                 * request rpc to the application.
284
                 */
285
                gf_msg_trace(this->name, 0,
286
                             "Submitting "
287
                             "statedump rpc request for %s",
288
                             trans->peerinfo.identifier);
289
                rpcsvc_request_submit(conf->rpc, trans, &glusterd_cbk_prog,
290
                                      GF_CBK_STATEDUMP, &statedump_req,
291
                                      this->ctx, (xdrproc_t)xdr_gf_statedump);
292
            }
293
        }
294
    }
295
    pthread_mutex_unlock(&conf->xprt_lock);
296
out:
297
    return ret;
298
}
299

300
int
301
glusterd_fetchspec_notify(xlator_t *this)
302
{
303
    int ret = -1;
304
    glusterd_conf_t *priv = NULL;
305
    rpc_transport_t *trans = NULL;
306

307
    priv = this->private;
308

309
    pthread_mutex_lock(&priv->xprt_lock);
310
    {
311
        list_for_each_entry(trans, &priv->xprt_list, list)
312
        {
313
            rpcsvc_callback_submit(priv->rpc, trans, &glusterd_cbk_prog,
314
                                   GF_CBK_FETCHSPEC, NULL, 0, NULL);
315
        }
316
    }
317
    pthread_mutex_unlock(&priv->xprt_lock);
318

319
    ret = 0;
320

321
    return ret;
322
}
323

324
int
325
glusterd_fetchsnap_notify(xlator_t *this)
326
{
327
    int ret = -1;
328
    glusterd_conf_t *priv = NULL;
329
    rpc_transport_t *trans = NULL;
330

331
    priv = this->private;
332

333
    /*
334
     * TODO: As of now, the identification of the rpc clients in the
335
     * handshake protocol is not there. So among so many glusterfs processes
336
     * registered with glusterd, it is hard to identify one particular
337
     * process (in this particular case, the snap daemon). So the callback
338
     * notification is sent to all the transports from the transport list.
339
     * Only those processes which have a rpc client registered for this
340
     * callback will respond to the notification. Once the identification
341
     * of the rpc clients becomes possible, the below section can be changed
342
     * to send callback notification to only those rpc clients, which have
343
     * registered.
344
     */
345
    pthread_mutex_lock(&priv->xprt_lock);
346
    {
347
        list_for_each_entry(trans, &priv->xprt_list, list)
348
        {
349
            rpcsvc_callback_submit(priv->rpc, trans, &glusterd_cbk_prog,
350
                                   GF_CBK_GET_SNAPS, NULL, 0, NULL);
351
        }
352
    }
353
    pthread_mutex_unlock(&priv->xprt_lock);
354

355
    ret = 0;
356

357
    return ret;
358
}
359

360
int32_t
361
mem_acct_init(xlator_t *this)
362
{
363
    int ret = -1;
364

365
    if (!this)
366
        return ret;
367

368
    ret = xlator_mem_acct_init(this, gf_gld_mt_end);
369

370
    if (ret != 0) {
371
        gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
372
               "Memory accounting init"
373
               " failed");
374
        return ret;
375
    }
376

377
    return ret;
378
}
379

380
int
381
glusterd_rpcsvc_notify(rpcsvc_t *rpc, void *xl, rpcsvc_event_t event,
382
                       void *data)
383
{
384
    xlator_t *this = NULL;
385
    rpc_transport_t *xprt = NULL;
386
    glusterd_conf_t *priv = NULL;
387

388
    if (!xl || !data) {
389
        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_NO_INIT,
390
               "Calling rpc_notify without initializing");
391
        goto out;
392
    }
393

394
    this = xl;
395
    xprt = data;
396

397
    priv = this->private;
398

399
    switch (event) {
400
        case RPCSVC_EVENT_ACCEPT: {
401
            pthread_mutex_lock(&priv->xprt_lock);
402
            list_add_tail(&xprt->list, &priv->xprt_list);
403
            pthread_mutex_unlock(&priv->xprt_lock);
404
            break;
405
        }
406
        case RPCSVC_EVENT_DISCONNECT: {
407
            /* A DISCONNECT event could come without an ACCEPT event
408
             * happening for this transport. This happens when the server is
409
             * expecting encrypted connections by the client tries to
410
             * connect unecnrypted
411
             */
412
            if (list_empty(&xprt->list))
413
                break;
414

415
            pthread_mutex_lock(&priv->xprt_lock);
416
            list_del(&xprt->list);
417
            pthread_mutex_unlock(&priv->xprt_lock);
418
            pmap_port_remove(this, 0, NULL, xprt, _gf_false);
419
            break;
420
        }
421

422
        default:
423
            break;
424
    }
425

426
out:
427
    return 0;
428
}
429

430
static int32_t
431
glusterd_program_register(xlator_t *this, rpcsvc_t *svc, rpcsvc_program_t *prog)
432
{
433
    int32_t ret = -1;
434

435
    ret = rpcsvc_program_register(svc, prog, _gf_false);
436
    if (ret) {
437
        gf_msg_debug(this->name, 0,
438
                     "cannot register program (name: %s, prognum:%d, "
439
                     "progver:%d)",
440
                     prog->progname, prog->prognum, prog->progver);
441
        goto out;
442
    }
443

444
out:
445
    return ret;
446
}
447

448
int
449
glusterd_rpcsvc_options_build(dict_t *options)
450
{
451
    int ret = 0;
452
    uint32_t backlog = 0;
453

454
    ret = dict_get_uint32(options, "transport.listen-backlog", &backlog);
455

456
    if (ret) {
457
        backlog = GLUSTERFS_SOCKET_LISTEN_BACKLOG;
458
        ret = dict_set_uint32(options, "transport.listen-backlog", backlog);
459
        if (ret) {
460
            gf_smsg(THIS->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_SET_FAILED,
461
                    "Key=transport.listen-backlog", NULL);
462
            goto out;
463
        }
464
    }
465

466
    gf_msg_debug("glusterd", 0, "listen-backlog value: %d", backlog);
467

468
out:
469
    return ret;
470
}
471

472
#if SYNCDAEMON_COMPILE
473

474
static int
475
glusterd_check_gsync_present(int *valid_state)
476
{
477
    char buff[PATH_MAX] = {
478
        0,
479
    };
480
    runner_t runner = {
481
        0,
482
    };
483
    char *ptr = NULL;
484
    int ret = 0;
485

486
    runinit(&runner);
487
    runner_add_args(&runner, GSYNCD_PREFIX "/gsyncd", "--version", NULL);
488
    runner_redir(&runner, STDOUT_FILENO, RUN_PIPE);
489
    ret = runner_start(&runner);
490
    if (ret == -1) {
491
        if (errno == ENOENT) {
492
            gf_msg("glusterd", GF_LOG_INFO, errno, GD_MSG_MODULE_NOT_INSTALLED,
493
                   GEOREP " module not installed in the system");
494
            *valid_state = 0;
495
        } else {
496
            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_MODULE_NOT_WORKING,
497
                   GEOREP " module not working as desired");
498
            *valid_state = -1;
499
        }
500
        goto out;
501
    }
502

503
    ptr = fgets(buff, sizeof(buff), runner_chio(&runner, STDOUT_FILENO));
504
    if (ptr) {
505
        if (!strstr(buff, "gsyncd")) {
506
            ret = -1;
507
            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_NOT_WORKING,
508
                   GEOREP
509
                   " module not "
510
                   "working as desired");
511
            *valid_state = -1;
512
            goto out;
513
        }
514
    } else {
515
        ret = -1;
516
        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MODULE_NOT_WORKING,
517
               GEOREP
518
               " module not "
519
               "working as desired");
520
        *valid_state = -1;
521
        goto out;
522
    }
523

524
    ret = 0;
525
out:
526

527
    runner_end(&runner);
528

529
    gf_msg_debug("glusterd", 0, "Returning %d", ret);
530
    return ret;
531
}
532

533
static int
534
group_write_allow(char *path, gid_t gid)
535
{
536
    struct stat st = {
537
        0,
538
    };
539
    int ret = 0;
540

541
    ret = sys_stat(path, &st);
542
    if (ret == -1)
543
        goto out;
544
    GF_ASSERT(S_ISDIR(st.st_mode));
545

546
    ret = sys_chown(path, -1, gid);
547
    if (ret == -1)
548
        goto out;
549

550
    ret = sys_chmod(path, (st.st_mode & ~S_IFMT) | S_IWGRP | S_IXGRP | S_ISVTX);
551

552
out:
553
    if (ret == -1)
554
        gf_msg("glusterd", GF_LOG_CRITICAL, errno,
555
               GD_MSG_WRITE_ACCESS_GRANT_FAIL,
556
               "failed to set up write access to %s for group %d (%s)", path,
557
               gid, strerror(errno));
558
    return ret;
559
}
560

561
static int
562
glusterd_crt_georep_folders(char *georepdir, glusterd_conf_t *conf)
563
{
564
    char *greplg_s = NULL;
565
    struct group *gr = NULL;
566
    int ret = 0;
567
    int gr_ret = 0;
568
    int32_t len = 0;
569
    char logdir[PATH_MAX] = {0};
570

571
    GF_ASSERT(georepdir);
572
    GF_ASSERT(conf);
573

574
    if (strlen(conf->workdir) + 2 > PATH_MAX - SLEN(GEOREP)) {
575
        ret = -1;
576
        gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_DIRPATH_TOO_LONG,
577
               "directory path %s/" GEOREP " is longer than PATH_MAX",
578
               conf->workdir);
579
        goto out;
580
    }
581

582
    len = snprintf(georepdir, PATH_MAX, "%s/" GEOREP, conf->workdir);
583
    if ((len < 0) || (len >= PATH_MAX)) {
584
        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
585
        ret = -1;
586
        goto out;
587
    }
588
    ret = mkdir_p(georepdir, 0755, _gf_true);
589
    if (-1 == ret) {
590
        gf_msg("glusterd", GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
591
               "Unable to create " GEOREP " directory %s", georepdir);
592
        goto out;
593
    }
594

595
    ret = dict_get_str(THIS->options, GEOREP "-log-group", &greplg_s);
596
    if (ret) {
597
        gf_smsg("glusterd", GF_LOG_ERROR, -ret, GD_MSG_DICT_GET_FAILED,
598
                "Key=log-group", NULL);
599
        ret = 0;
600
    } else {
601
        gr = getgrnam(greplg_s);
602
        if (!gr) {
603
            gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_LOGGROUP_INVALID,
604
                   "group " GEOREP "-log-group %s does not exist", greplg_s);
605
            gr_ret = -1;
606
        }
607
    }
608
    if ((strlen(conf->logdir) + 2 + SLEN(GEOREP)) >= PATH_MAX) {
609
        ret = -1;
610
        gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_DIRPATH_TOO_LONG,
611
               "directory path %s/" GEOREP " is longer than PATH_MAX",
612
               conf->logdir);
613
        goto out;
614
    }
615
    len = snprintf(logdir, PATH_MAX, "%s/" GEOREP, conf->logdir);
616
    if ((len < 0) || (len >= PATH_MAX)) {
617
        ret = -1;
618
        goto out;
619
    }
620
    ret = mkdir_p(logdir, 0755, _gf_true);
621
    if (-1 == ret) {
622
        gf_msg("glusterd", GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
623
               "Unable to create " GEOREP " log directory");
624
        goto out;
625
    }
626
    if (gr) {
627
        gr_ret = group_write_allow(logdir, gr->gr_gid);
628
    }
629

630
    if ((strlen(conf->logdir) + 2 + SLEN(GEOREP "-secondaries")) >= PATH_MAX) {
631
        ret = -1;
632
        gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_DIRPATH_TOO_LONG,
633
               "directory path  %s/" GEOREP
634
               "-secondaries"
635
               " is longer than PATH_MAX",
636
               conf->logdir);
637
        goto out;
638
    }
639
    len = snprintf(logdir, PATH_MAX, "%s/" GEOREP "-secondaries", conf->logdir);
640
    if ((len < 0) || (len >= PATH_MAX)) {
641
        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
642
        ret = -1;
643
        goto out;
644
    }
645
    ret = mkdir_p(logdir, 0755, _gf_true);
646
    if (-1 == ret) {
647
        gf_msg("glusterd", GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
648
               "Unable to create " GEOREP " secondary log directory");
649
        goto out;
650
    }
651
    if (gr && !gr_ret) {
652
        gr_ret = group_write_allow(logdir, gr->gr_gid);
653
    }
654

655
    /* MountBroker log file directory */
656
    if ((strlen(conf->logdir) + 2 + SLEN(GEOREP "-secondaries/mbr")) >=
657
        PATH_MAX) {
658
        ret = -1;
659
        gf_msg("glusterd", GF_LOG_CRITICAL, 0, GD_MSG_DIRPATH_TOO_LONG,
660
               "directory path  %s/" GEOREP
661
               "-secondaries/mbr"
662
               " is longer than PATH_MAX",
663
               conf->logdir);
664
        goto out;
665
    }
666

667
    len = snprintf(logdir, PATH_MAX, "%s/" GEOREP "-secondaries/mbr",
668
                   conf->logdir);
669
    if ((len < 0) || (len >= PATH_MAX)) {
670
        gf_smsg("glusterd", GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
671
        ret = -1;
672
        goto out;
673
    }
674

675
    ret = mkdir_p(logdir, 0755, _gf_true);
676
    if (-1 == ret) {
677
        gf_msg("glusterd", GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
678
               "Unable to create " GEOREP
679
               " mountbroker secondary log directory");
680
        goto out;
681
    }
682
    if (gr && !gr_ret) {
683
        gr_ret = group_write_allow(logdir, gr->gr_gid);
684
    }
685
    if (gr_ret)
686
        ret = gr_ret;
687
out:
688
    gf_msg_debug("glusterd", 0, "Returning %d", ret);
689
    return ret;
690
}
691

692
static void
693
runinit_gsyncd_setrx(runner_t *runner, glusterd_conf_t *conf)
694
{
695
    runinit(runner);
696
    runner_add_args(runner, GSYNCD_PREFIX "/gsyncd", "-c", NULL);
697
    runner_argprintf(runner, "%s/" GSYNC_CONF_TEMPLATE, conf->workdir);
698
    runner_add_arg(runner, "--config-set-rx");
699
}
700

701
static int
702
run_gsyncd_cmd(runner_t *runner, xlator_t *this)
703
{
704
    int ret = 0;
705

706
    ret = runner_run_reuse(runner);
707
    if (ret < 0) {
708
        runner_log(runner, this->name, GF_LOG_ERROR, "command failed");
709
    }
710
    runner_end(runner);
711
    return ret;
712
}
713

714
static int
715
configure_syncdaemon(glusterd_conf_t *conf)
716
{
717
    int ret = 0;
718
    runner_t runner = {
719
        0,
720
    };
721
    char georepdir[PATH_MAX] = {
722
        0,
723
    };
724
    int valid_state = 0;
725
    xlator_t *this = NULL;
726

727
    GF_ASSERT(conf);
728
    GF_ASSERT(conf->xl);
729
    this = conf->xl;
730

731
    ret = setenv("_GLUSTERD_CALLED_", "1", 1);
732
    if (ret < 0) {
733
        ret = 0;
734
        goto out;
735
    }
736
    valid_state = -1;
737
    ret = glusterd_check_gsync_present(&valid_state);
738
    if (-1 == ret) {
739
        ret = valid_state;
740
        goto out;
741
    }
742

743
    glusterd_crt_georep_folders(georepdir, conf);
744
    if (ret) {
745
        ret = 0;
746
        goto out;
747
    }
748

749
    /************
750
     * primary pre-configuration
751
     ************/
752

753
    /* remote-gsyncd */
754
    runinit_gsyncd_setrx(&runner, conf);
755
    runner_add_args(&runner, "remote-gsyncd", GSYNCD_PREFIX "/gsyncd", ".", ".",
756
                    NULL);
757
    ret = run_gsyncd_cmd(&runner, this);
758
    if (ret)
759
        goto out;
760

761
    runinit_gsyncd_setrx(&runner, conf);
762
    runner_add_args(&runner, "remote-gsyncd", "/nonexistent/gsyncd", ".",
763
                    "^ssh:", NULL);
764
    ret = run_gsyncd_cmd(&runner, this);
765
    if (ret)
766
        goto out;
767

768
    /* gluster-command-dir */
769
    runinit_gsyncd_setrx(&runner, conf);
770
    runner_add_args(&runner, "gluster-command-dir", SBIN_DIR "/", ".", ".",
771
                    NULL);
772
    ret = run_gsyncd_cmd(&runner, this);
773
    if (ret)
774
        goto out;
775

776
    /* gluster-params */
777
    runinit_gsyncd_setrx(&runner, conf);
778
    runner_add_args(&runner, "gluster-params", "aux-gfid-mount acl", ".", ".",
779
                    NULL);
780
    ret = run_gsyncd_cmd(&runner, this);
781
    if (ret)
782
        goto out;
783

784
    /* ssh-command */
785
    runinit_gsyncd_setrx(&runner, conf);
786
    runner_add_arg(&runner, "ssh-command");
787
    runner_argprintf(&runner,
788
                     "ssh -oPasswordAuthentication=no "
789
                     "-oStrictHostKeyChecking=no "
790
                     "-i %s/secret.pem",
791
                     georepdir);
792
    runner_add_args(&runner, ".", ".", NULL);
793
    ret = run_gsyncd_cmd(&runner, this);
794
    if (ret)
795
        goto out;
796

797
    /* ssh-command tar */
798
    runinit_gsyncd_setrx(&runner, conf);
799
    runner_add_arg(&runner, "ssh-command-tar");
800
    runner_argprintf(&runner,
801
                     "ssh -oPasswordAuthentication=no "
802
                     "-oStrictHostKeyChecking=no "
803
                     "-i %s/tar_ssh.pem",
804
                     georepdir);
805
    runner_add_args(&runner, ".", ".", NULL);
806
    ret = run_gsyncd_cmd(&runner, this);
807
    if (ret)
808
        goto out;
809

810
    /* pid-file */
811
    runinit_gsyncd_setrx(&runner, conf);
812
    runner_add_arg(&runner, "pid-file");
813
    runner_argprintf(
814
        &runner, "%s/${primaryvol}_${remotehost}_${secondaryvol}/monitor.pid",
815
        georepdir);
816
    runner_add_args(&runner, ".", ".", NULL);
817
    ret = run_gsyncd_cmd(&runner, this);
818
    if (ret)
819
        goto out;
820

821
    /* geo-rep working dir */
822
    runinit_gsyncd_setrx(&runner, conf);
823
    runner_add_arg(&runner, "georep-session-working-dir");
824
    runner_argprintf(&runner, "%s/${primaryvol}_${remotehost}_${secondaryvol}/",
825
                     georepdir);
826
    runner_add_args(&runner, ".", ".", NULL);
827
    ret = run_gsyncd_cmd(&runner, this);
828
    if (ret)
829
        goto out;
830

831
    /* state-file */
832
    runinit_gsyncd_setrx(&runner, conf);
833
    runner_add_arg(&runner, "state-file");
834
    runner_argprintf(
835
        &runner,
836
        "%s/${primaryvol}_${remotehost}_${secondaryvol}/monitor.status",
837
        georepdir);
838
    runner_add_args(&runner, ".", ".", NULL);
839
    ret = run_gsyncd_cmd(&runner, this);
840
    if (ret)
841
        goto out;
842

843
    /* state-detail-file */
844
    runinit_gsyncd_setrx(&runner, conf);
845
    runner_add_arg(&runner, "state-detail-file");
846
    runner_argprintf(&runner,
847
                     "%s/${primaryvol}_${remotehost}_${secondaryvol}/"
848
                     "${eSecondary}-detail.status",
849
                     georepdir);
850
    runner_add_args(&runner, ".", ".", NULL);
851
    ret = run_gsyncd_cmd(&runner, this);
852
    if (ret)
853
        goto out;
854

855
    /* state-detail-file */
856
    runinit_gsyncd_setrx(&runner, conf);
857
    runner_add_arg(&runner, "state-detail-file");
858
    runner_argprintf(&runner,
859
                     "%s/${primaryvol}_${remotehost}_${secondaryvol}/"
860
                     "${eSecondary}-detail.status",
861
                     georepdir);
862
    runner_add_args(&runner, ".", ".", NULL);
863
    ret = run_gsyncd_cmd(&runner, this);
864
    if (ret)
865
        goto out;
866

867
    /* state-socket */
868
    runinit_gsyncd_setrx(&runner, conf);
869
    runner_add_arg(&runner, "state-socket-unencoded");
870
    runner_argprintf(&runner, "%s/${primaryvol}/${eSecondary}.socket",
871
                     georepdir);
872
    runner_add_args(&runner, ".", ".", NULL);
873
    ret = run_gsyncd_cmd(&runner, this);
874
    if (ret)
875
        goto out;
876

877
    /* socketdir */
878
    runinit_gsyncd_setrx(&runner, conf);
879
    runner_add_args(&runner, "socketdir", GLUSTERD_SOCK_DIR, ".", ".", NULL);
880
    ret = run_gsyncd_cmd(&runner, this);
881
    if (ret)
882
        goto out;
883

884
    /* log-file */
885
    runinit_gsyncd_setrx(&runner, conf);
886
    runner_add_arg(&runner, "log-file");
887
    runner_argprintf(&runner, "%s/" GEOREP "/${primaryvol}/${eSecondary}.log",
888
                     conf->logdir);
889
    runner_add_args(&runner, ".", ".", NULL);
890
    ret = run_gsyncd_cmd(&runner, this);
891
    if (ret)
892
        goto out;
893

894
    /* gluster-log-file */
895
    runinit_gsyncd_setrx(&runner, conf);
896
    runner_add_arg(&runner, "gluster-log-file");
897
    runner_argprintf(&runner,
898
                     "%s/" GEOREP
899
                     "/${primaryvol}/${eSecondary}${local_id}.gluster.log",
900
                     conf->logdir);
901
    runner_add_args(&runner, ".", ".", NULL);
902
    ret = run_gsyncd_cmd(&runner, this);
903
    if (ret)
904
        goto out;
905

906
    /* ignore-deletes */
907
    runinit_gsyncd_setrx(&runner, conf);
908
    runner_add_args(&runner, "ignore-deletes", "true", ".", ".", NULL);
909
    ret = run_gsyncd_cmd(&runner, this);
910
    if (ret)
911
        goto out;
912

913
    /* special-sync-mode */
914
    runinit_gsyncd_setrx(&runner, conf);
915
    runner_add_args(&runner, "special-sync-mode", "partial", ".", ".", NULL);
916
    ret = run_gsyncd_cmd(&runner, this);
917
    if (ret)
918
        goto out;
919

920
    /* change-detector == changelog */
921
    runinit_gsyncd_setrx(&runner, conf);
922
    runner_add_args(&runner, "change-detector", "changelog", ".", ".", NULL);
923
    ret = run_gsyncd_cmd(&runner, this);
924
    if (ret)
925
        goto out;
926

927
    runinit_gsyncd_setrx(&runner, conf);
928
    runner_add_arg(&runner, "working-dir");
929
    runner_argprintf(&runner, "%s/${primaryvol}/${eSecondary}",
930
                     DEFAULT_VAR_RUN_DIRECTORY);
931
    runner_add_args(&runner, ".", ".", NULL);
932
    ret = run_gsyncd_cmd(&runner, this);
933
    if (ret)
934
        goto out;
935

936
    /************
937
     * secondary pre-configuration
938
     ************/
939

940
    /* secondary-gluster-command-dir */
941
    runinit_gsyncd_setrx(&runner, conf);
942
    runner_add_args(&runner, "secondary-gluster-command-dir", SBIN_DIR "/", ".",
943
                    NULL);
944
    ret = run_gsyncd_cmd(&runner, this);
945
    if (ret)
946
        goto out;
947

948
    /* gluster-params */
949
    runinit_gsyncd_setrx(&runner, conf);
950
    runner_add_args(&runner, "gluster-params", "aux-gfid-mount acl", ".", NULL);
951
    ret = run_gsyncd_cmd(&runner, this);
952
    if (ret)
953
        goto out;
954

955
    /* log-file */
956
    runinit_gsyncd_setrx(&runner, conf);
957
    runner_add_arg(&runner, "log-file");
958
    runner_argprintf(
959
        &runner,
960
        "%s/" GEOREP
961
        "-secondaries/"
962
        "${session_owner}:${local_node}${local_id}.${secondaryvol}.log",
963
        conf->logdir);
964
    runner_add_args(&runner, ".", ".", NULL);
965
    ret = run_gsyncd_cmd(&runner, this);
966
    if (ret)
967
        goto out;
968

969
    /* MountBroker log-file */
970
    runinit_gsyncd_setrx(&runner, conf);
971
    runner_add_arg(&runner, "log-file-mbr");
972
    runner_argprintf(
973
        &runner,
974
        "%s/" GEOREP
975
        "-secondaries/mbr/"
976
        "${session_owner}:${local_node}${local_id}.${secondaryvol}.log",
977
        conf->logdir);
978
    runner_add_args(&runner, ".", ".", NULL);
979
    ret = run_gsyncd_cmd(&runner, this);
980
    if (ret)
981
        goto out;
982

983
    /* gluster-log-file */
984
    runinit_gsyncd_setrx(&runner, conf);
985
    runner_add_arg(&runner, "gluster-log-file");
986
    runner_argprintf(
987
        &runner,
988
        "%s/" GEOREP
989
        "-secondaries/"
990
        "${session_owner}:${local_node}${local_id}.${secondaryvol}.gluster.log",
991
        conf->logdir);
992
    runner_add_args(&runner, ".", ".", NULL);
993
    ret = run_gsyncd_cmd(&runner, this);
994
out:
995
    return ret ? -1 : 0;
996
}
997

998
#else /* SYNCDAEMON_COMPILE */
999

1000
static int
1001
configure_syncdaemon(glusterd_conf_t *conf)
1002
{
1003
    return 0;
1004
}
1005

1006
#endif /* !SYNCDAEMON_COMPILE */
1007

1008
static int
1009
check_prepare_mountbroker_root(char *mountbroker_root)
1010
{
1011
    int dfd0 = -1;
1012
    int dfd = -1;
1013
    int dfd2 = -1;
1014
    struct stat st = {
1015
        0,
1016
    };
1017
    struct stat st2 = {
1018
        0,
1019
    };
1020
    int ret = 0;
1021

1022
    ret = open(mountbroker_root, O_RDONLY);
1023
    if (ret != -1) {
1024
        dfd = ret;
1025
        ret = sys_fstat(dfd, &st);
1026
    }
1027
    if (ret == -1 || !S_ISDIR(st.st_mode)) {
1028
        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
1029
               "cannot access mountbroker-root directory %s", mountbroker_root);
1030
        ret = -1;
1031
        goto out;
1032
    }
1033
    if (st.st_uid != 0 || (st.st_mode & (S_IWGRP | S_IWOTH))) {
1034
        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DIR_PERM_LIBERAL,
1035
               "permissions on mountbroker-root directory %s are "
1036
               "too liberal",
1037
               mountbroker_root);
1038
        ret = -1;
1039
        goto out;
1040
    }
1041
    if (!(st.st_mode & (S_IXGRP | S_IXOTH))) {
1042
        gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_DIR_PERM_STRICT,
1043
               "permissions on mountbroker-root directory %s are "
1044
               "probably too strict",
1045
               mountbroker_root);
1046
    }
1047

1048
    dfd0 = dup(dfd);
1049

1050
    for (;;) {
1051
        ret = sys_openat(dfd, "..", O_RDONLY, 0);
1052
        if (ret != -1) {
1053
            dfd2 = ret;
1054
            ret = sys_fstat(dfd2, &st2);
1055
        }
1056
        if (ret == -1) {
1057
            gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
1058
                   "error while checking mountbroker-root ancestors "
1059
                   "%d (%s)",
1060
                   errno, strerror(errno));
1061
            goto out;
1062
        }
1063

1064
        if (st2.st_ino == st.st_ino)
1065
            break; /* arrived to root */
1066

1067
        if (st2.st_uid != 0 ||
1068
            ((st2.st_mode & (S_IWGRP | S_IWOTH)) && !(st2.st_mode & S_ISVTX))) {
1069
            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_DIR_PERM_LIBERAL,
1070
                   "permissions on ancestors of mountbroker-root "
1071
                   "directory are too liberal");
1072
            ret = -1;
1073
            goto out;
1074
        }
1075
        if (!(st.st_mode & (S_IXGRP | S_IXOTH))) {
1076
            gf_msg("glusterd", GF_LOG_WARNING, 0, GD_MSG_DIR_PERM_STRICT,
1077
                   "permissions on ancestors of mountbroker-root "
1078
                   "directory are probably too strict");
1079
        }
1080

1081
        sys_close(dfd);
1082
        dfd = dfd2;
1083
        st = st2;
1084
    }
1085

1086
    ret = sys_mkdirat(dfd0, MB_HIVE, 0711);
1087
    if (ret == -1 && errno == EEXIST)
1088
        ret = 0;
1089
    if (ret != -1)
1090
        ret = sys_fstatat(dfd0, MB_HIVE, &st, AT_SYMLINK_NOFOLLOW);
1091
    if (ret == -1 || st.st_mode != (S_IFDIR | 0711)) {
1092
        gf_msg("glusterd", GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
1093
               "failed to set up mountbroker-root directory %s",
1094
               mountbroker_root);
1095
        ret = -1;
1096
        goto out;
1097
    }
1098

1099
    ret = 0;
1100

1101
out:
1102
    if (dfd0 != -1)
1103
        sys_close(dfd0);
1104
    if (dfd != -1)
1105
        sys_close(dfd);
1106
    if (dfd2 != -1 && dfd != dfd2)
1107
        sys_close(dfd2);
1108

1109
    return ret;
1110
}
1111

1112
static int
1113
_install_mount_spec(dict_t *opts, char *key, data_t *value, void *data)
1114
{
1115
    glusterd_conf_t *priv = THIS->private;
1116
    char *label = NULL;
1117
    gf_boolean_t georep = _gf_false;
1118
    char *pdesc = value->data;
1119
    char *volname = NULL;
1120
    int rv = 0;
1121
    gf_mount_spec_t *mspec = NULL;
1122
    char *user = NULL;
1123

1124
    label = strtail(key, "mountbroker.");
1125

1126
    /* check for presence of geo-rep label */
1127
    if (!label) {
1128
        label = strtail(key, "mountbroker-" GEOREP ".");
1129
        if (label)
1130
            georep = _gf_true;
1131
    }
1132

1133
    if (!label)
1134
        return 0;
1135

1136
    mspec = GF_CALLOC(1, sizeof(*mspec), gf_gld_mt_mount_spec);
1137
    if (!mspec) {
1138
        gf_smsg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_NO_MEMORY, NULL);
1139
        goto err;
1140
    }
1141
    mspec->label = label;
1142

1143
    if (georep) {
1144
        volname = gf_strdup(pdesc);
1145
        if (!volname)
1146
            goto err;
1147
        user = strchr(volname, ':');
1148
        if (user) {
1149
            *user = '\0';
1150
            user++;
1151
        } else
1152
            user = label;
1153

1154
        rv = make_georep_mountspec(mspec, volname, user, priv->logdir);
1155

1156
        GF_FREE(volname);
1157
        if (rv != 0)
1158
            goto err;
1159
    } else if (parse_mount_pattern_desc(mspec, pdesc) != 0)
1160
        goto err;
1161

1162
    cds_list_add_tail(&mspec->speclist, &priv->mount_specs);
1163

1164
    return 0;
1165
err:
1166

1167
    gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_MOUNT_SPEC_INSTALL_FAIL,
1168
           "adding %smount spec failed: label: %s desc: %s",
1169
           georep ? GEOREP " " : "", label, pdesc ? pdesc : "");
1170

1171
    if (mspec) {
1172
        if (mspec->patterns) {
1173
            GF_FREE(mspec->patterns->components);
1174
            GF_FREE(mspec->patterns);
1175
        }
1176
        GF_FREE(mspec);
1177
    }
1178

1179
    return -1;
1180
}
1181

1182
/* The glusterd unix domain socket listener only listens for cli */
1183
rpcsvc_t *
1184
glusterd_init_uds_listener(xlator_t *this)
1185
{
1186
    int ret = -1;
1187
    dict_t *options = NULL;
1188
    rpcsvc_t *rpc = NULL;
1189
    data_t *sock_data = NULL;
1190
    char sockfile[UNIX_PATH_MAX] = {0};
1191
    int i = 0;
1192

1193
    GF_ASSERT(this);
1194

1195
    options = dict_new();
1196
    if (!options) {
1197
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_DICT_CREATE_FAIL, NULL);
1198
        goto out;
1199
    }
1200

1201
    sock_data = dict_get(this->options, "glusterd-sockfile");
1202
    (void)snprintf(sockfile, sizeof(sockfile), "%s",
1203
                   sock_data ? sock_data->data : DEFAULT_GLUSTERD_SOCKFILE);
1204

1205
    ret = rpcsvc_transport_unix_options_build(options, sockfile);
1206
    if (ret)
1207
        goto out;
1208

1209
    rpc = rpcsvc_init(this, this->ctx, options, 8);
1210
    if (rpc == NULL) {
1211
        ret = -1;
1212
        goto out;
1213
    }
1214

1215
    ret = rpcsvc_register_notify(rpc, glusterd_rpcsvc_notify, this);
1216
    if (ret) {
1217
        gf_msg_debug(this->name, 0, "Failed to register notify function");
1218
        goto out;
1219
    }
1220

1221
    ret = rpcsvc_create_listeners(rpc, options, this->name);
1222
    if (ret != 1) {
1223
        gf_msg_debug(this->name, 0, "Failed to create listener");
1224
        goto out;
1225
    }
1226
    ret = 0;
1227

1228
    for (i = 0; i < gd_uds_programs_count; i++) {
1229
        ret = glusterd_program_register(this, rpc, gd_uds_programs[i]);
1230
        if (ret) {
1231
            i--;
1232
            for (; i >= 0; i--)
1233
                rpcsvc_program_unregister(rpc, gd_uds_programs[i]);
1234

1235
            goto out;
1236
        }
1237
    }
1238

1239
out:
1240
    if (options)
1241
        dict_unref(options);
1242

1243
    if (ret) {
1244
        gf_msg(this->name, GF_LOG_ERROR, 0,
1245
               GD_MSG_GLUSTERD_SOCK_LISTENER_START_FAIL,
1246
               "Failed to start glusterd "
1247
               "unix domain socket listener.");
1248
        if (rpc) {
1249
            GF_FREE(rpc);
1250
            rpc = NULL;
1251
        }
1252
    }
1253
    return rpc;
1254
}
1255

1256
void
1257
glusterd_stop_uds_listener(xlator_t *this)
1258
{
1259
    glusterd_conf_t *conf = NULL;
1260
    rpcsvc_listener_t *listener = NULL;
1261
    rpcsvc_listener_t *next = NULL;
1262
    data_t *sock_data = NULL;
1263
    char sockfile[UNIX_PATH_MAX] = {0};
1264

1265
    GF_ASSERT(this);
1266
    conf = this->private;
1267

1268
    (void)rpcsvc_program_unregister(conf->uds_rpc, &gd_svc_cli_prog);
1269
    (void)rpcsvc_program_unregister(conf->uds_rpc, &gluster_handshake_prog);
1270

1271
    list_for_each_entry_safe(listener, next, &conf->uds_rpc->listeners, list)
1272
    {
1273
        rpcsvc_listener_destroy(listener);
1274
    }
1275

1276
    (void)rpcsvc_unregister_notify(conf->uds_rpc, glusterd_rpcsvc_notify, this);
1277

1278
    sock_data = dict_get(this->options, "glusterd-sockfile");
1279
    (void)snprintf(sockfile, sizeof(sockfile), "%s",
1280
                   sock_data ? sock_data->data : DEFAULT_GLUSTERD_SOCKFILE);
1281
    sys_unlink(sockfile);
1282

1283
    return;
1284
}
1285

1286
void
1287
glusterd_stop_listener(xlator_t *this)
1288
{
1289
    glusterd_conf_t *conf = NULL;
1290
    rpcsvc_listener_t *listener = NULL;
1291
    rpcsvc_listener_t *next = NULL;
1292
    int i = 0;
1293

1294
    GF_VALIDATE_OR_GOTO("glusterd", this, out);
1295
    conf = this->private;
1296
    GF_VALIDATE_OR_GOTO(this->name, conf, out);
1297

1298
    gf_msg_debug(this->name, 0, "%s function called ", __func__);
1299

1300
    for (i = 0; i < gd_inet_programs_count; i++) {
1301
        rpcsvc_program_unregister(conf->rpc, gd_inet_programs[i]);
1302
    }
1303

1304
    list_for_each_entry_safe(listener, next, &conf->rpc->listeners, list)
1305
    {
1306
        rpcsvc_listener_destroy(listener);
1307
    }
1308

1309
    (void)rpcsvc_unregister_notify(conf->rpc, glusterd_rpcsvc_notify, this);
1310

1311
out:
1312

1313
    return;
1314
}
1315

1316
static int
1317
glusterd_find_correct_var_run_dir(xlator_t *this, char *var_run_dir)
1318
{
1319
    int ret = -1;
1320
    struct stat buf = {
1321
        0,
1322
    };
1323

1324
    GF_VALIDATE_OR_GOTO("glusterd", this, out);
1325
    GF_VALIDATE_OR_GOTO(this->name, var_run_dir, out);
1326

1327
    /* /var/run is normally a symbolic link to /run dir, which
1328
     * creates problems as the entry point in the mtab for the mount point
1329
     * and glusterd maintained entry point will be different. Therefore
1330
     * identify the correct run dir and use it
1331
     */
1332
    ret = sys_lstat(GLUSTERD_VAR_RUN_DIR, &buf);
1333
    if (ret != 0) {
1334
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
1335
               "stat fails on %s, exiting. (errno = %d)", GLUSTERD_VAR_RUN_DIR,
1336
               errno);
1337
        goto out;
1338
    }
1339

1340
    /* If /var/run is symlink then use /run dir */
1341
    if (S_ISLNK(buf.st_mode)) {
1342
        strcpy(var_run_dir, GLUSTERD_RUN_DIR);
1343
    } else {
1344
        strcpy(var_run_dir, GLUSTERD_VAR_RUN_DIR);
1345
    }
1346

1347
    ret = 0;
1348
out:
1349
    return ret;
1350
}
1351

1352
static int
1353
glusterd_init_var_run_dirs(xlator_t *this, char *var_run_dir,
1354
                           char *dir_to_be_created)
1355
{
1356
    int ret = -1;
1357
    struct stat buf = {
1358
        0,
1359
    };
1360
    char abs_path[PATH_MAX] = {
1361
        0,
1362
    };
1363

1364
    GF_VALIDATE_OR_GOTO("glusterd", this, out);
1365
    GF_VALIDATE_OR_GOTO(this->name, var_run_dir, out);
1366
    GF_VALIDATE_OR_GOTO(this->name, dir_to_be_created, out);
1367

1368
    snprintf(abs_path, sizeof(abs_path), "%s%s", var_run_dir,
1369
             dir_to_be_created);
1370

1371
    ret = sys_stat(abs_path, &buf);
1372
    if ((ret != 0) && (ENOENT != errno)) {
1373
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
1374
               "stat fails on %s, exiting. (errno = %d)", abs_path, errno);
1375
        ret = -1;
1376
        goto out;
1377
    }
1378

1379
    if ((!ret) && (!S_ISDIR(buf.st_mode))) {
1380
        gf_msg(this->name, GF_LOG_CRITICAL, ENOENT, GD_MSG_DIR_NOT_FOUND,
1381
               "Provided snap path %s is not a directory,"
1382
               "exiting",
1383
               abs_path);
1384
        ret = -1;
1385
        goto out;
1386
    }
1387

1388
    if ((-1 == ret) && (ENOENT == errno)) {
1389
        /* Create missing dirs */
1390
        ret = mkdir_p(abs_path, 0755, _gf_true);
1391

1392
        if (-1 == ret) {
1393
            gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
1394
                   "Unable to create directory %s"
1395
                   " ,errno = %d",
1396
                   abs_path, errno);
1397
            goto out;
1398
        }
1399
    }
1400

1401
out:
1402
    return ret;
1403
}
1404

1405
static int
1406
is_upgrade(dict_t *options, gf_boolean_t *upgrade)
1407
{
1408
    int ret = 0;
1409
    char *type = NULL;
1410

1411
    ret = dict_get_str(options, "upgrade", &type);
1412
    if (!ret) {
1413
        ret = gf_string2boolean(type, upgrade);
1414
        if (ret) {
1415
            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_STR_TO_BOOL_FAIL,
1416
                   "upgrade option "
1417
                   "%s is not a valid boolean type",
1418
                   type);
1419
            ret = -1;
1420
            goto out;
1421
        }
1422
    }
1423
    ret = 0;
1424
out:
1425
    return ret;
1426
}
1427

1428
static int
1429
is_downgrade(dict_t *options, gf_boolean_t *downgrade)
1430
{
1431
    int ret = 0;
1432
    char *type = NULL;
1433

1434
    ret = dict_get_str(options, "downgrade", &type);
1435
    if (!ret) {
1436
        ret = gf_string2boolean(type, downgrade);
1437
        if (ret) {
1438
            gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_STR_TO_BOOL_FAIL,
1439
                   "downgrade option "
1440
                   "%s is not a valid boolean type",
1441
                   type);
1442
            ret = -1;
1443
            goto out;
1444
        }
1445
    }
1446
    ret = 0;
1447
out:
1448
    return ret;
1449
}
1450

1451
void
1452
glusterd_destroy_hostname_list(struct list_head *hostname_list_head)
1453
{
1454
    glusterd_hostname_t *hostname_obj = NULL;
1455
    glusterd_hostname_t *tmp = NULL;
1456

1457
    list_for_each_entry_safe(hostname_obj, tmp, hostname_list_head,
1458
                             hostname_list)
1459
    {
1460
        list_del_init(&hostname_obj->hostname_list);
1461
        GF_FREE(hostname_obj->hostname);
1462
        GF_FREE(hostname_obj);
1463
    }
1464
}
1465

1466
static int32_t
1467
glusterd_handle_upgrade_downgrade(dict_t *options, glusterd_conf_t *conf,
1468
                                  gf_boolean_t upgrade, gf_boolean_t downgrade)
1469
{
1470
    int ret = 0;
1471
    gf_boolean_t regenerate_volfiles = _gf_false;
1472
    gf_boolean_t terminate = _gf_false;
1473

1474
    if (_gf_true == upgrade)
1475
        regenerate_volfiles = _gf_true;
1476

1477
    if (upgrade && downgrade) {
1478
        gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_WRONG_OPTS_SETTING,
1479
               "Both upgrade and downgrade"
1480
               " options are set. Only one should be on");
1481
        ret = -1;
1482
        goto out;
1483
    }
1484

1485
    if (!upgrade && !downgrade)
1486
        ret = 0;
1487
    else
1488
        terminate = _gf_true;
1489
    if (regenerate_volfiles) {
1490
        ret = glusterd_recreate_volfiles(conf);
1491
    }
1492
out:
1493
    if (terminate && (ret == 0))
1494
        kill(getpid(), SIGTERM);
1495
    return ret;
1496
}
1497

1498
/*
1499
 * init - called during glusterd initialization
1500
 *
1501
 * @this:
1502
 *
1503
 */
1504
int
1505
init(xlator_t *this)
1506
{
1507
    int32_t ret = -1;
1508
    rpcsvc_t *rpc = NULL;
1509
    rpcsvc_t *uds_rpc = NULL;
1510
    glusterd_conf_t *conf = NULL;
1511
    data_t *dir_data = NULL;
1512
    struct stat buf = {
1513
        0,
1514
    };
1515
    char storedir[PATH_MAX] = {
1516
        0,
1517
    };
1518
    char workdir[PATH_MAX] = {
1519
        0,
1520
    };
1521
    char rundir[PATH_MAX] = {
1522
        0,
1523
    };
1524
    char logdir[VALID_GLUSTERD_PATHMAX] = {
1525
        0,
1526
    };
1527
    char cmd_log_filename[PATH_MAX] = {
1528
        0,
1529
    };
1530
    char *mountbroker_root = NULL;
1531
    int i = 0;
1532
    int total_transport = 0;
1533
    gf_valgrind_tool vgtool;
1534
    char *valgrind_str = NULL;
1535
    char *transport_type = NULL;
1536
    char var_run_dir[PATH_MAX] = {
1537
        0,
1538
    };
1539
    int32_t workers = 0;
1540
    gf_boolean_t upgrade = _gf_false;
1541
    gf_boolean_t downgrade = _gf_false;
1542
    char *localtime_logging = NULL;
1543
    int32_t len = 0;
1544
    int op_version = 0;
1545

1546
    /* Extra directories needed for the regular operation. */
1547
    struct {
1548
        char *base;
1549
        const char *name;
1550
    } *p, dirs[] = {
1551
        { workdir, "vols" },
1552
        { rundir, "vols" },
1553
        { workdir, "snaps" },
1554
        { workdir, "peers" },
1555
        { logdir, "bricks" },
1556
#ifdef BUILD_GNFS
1557
        { workdir, "nfs" },
1558
#endif /* BUILD_GNFS */
1559
        { workdir, "bitd" },
1560
        { workdir, "scrub" },
1561
        { workdir, "glustershd" },
1562
        { workdir, "quotad" },
1563
        { workdir, "groups" },
1564
        { NULL, NULL }
1565
    };
1566

1567
#if defined(RUN_WITH_MEMCHECK)
1568
    vgtool = _gf_memcheck;
1569
#elif defined(RUN_WITH_DRD)
1570
    vgtool = _gf_drd;
1571
#else
1572
    vgtool = _gf_none;
1573
#endif
1574

1575
    gf_set_nofile(0, 65536);
1576

1577
    dir_data = dict_get(this->options, "run-directory");
1578

1579
    if (!dir_data) {
1580
        /* Use default working dir */
1581
        len = snprintf(rundir, PATH_MAX, "%s", DEFAULT_VAR_RUN_DIRECTORY);
1582
    } else {
1583
        len = snprintf(rundir, PATH_MAX, "%s", dir_data->data);
1584
    }
1585
    if (len < 0 || len >= PATH_MAX)
1586
        exit(2);
1587

1588
    dir_data = dict_get(this->options, "logging-directory");
1589
    if (!dir_data) {
1590
        // Check for deprecated 'cluster-test-mode' option
1591
        dir_data = dict_get(this->options, "cluster-test-mode");
1592
        if (dir_data) {
1593
            len = snprintf(logdir, VALID_GLUSTERD_PATHMAX, "%s",
1594
                           dir_data->data);
1595
            gf_msg(
1596
                this->name, GF_LOG_WARNING, 0, GD_MSG_CLUSTER_RC_ENABLE,
1597
                "gluster log directory is set to %s. The option "
1598
                "'cluster-test-mode' is deprecated and will be removed soon. "
1599
                "Please use the new option 'logging-directory' instead.",
1600
                dir_data->data);
1601
        } else {
1602
            /* Use default working dir */
1603
            len = snprintf(logdir, VALID_GLUSTERD_PATHMAX, "%s",
1604
                           DEFAULT_LOG_FILE_DIRECTORY);
1605
        }
1606
    } else {
1607
        len = snprintf(logdir, VALID_GLUSTERD_PATHMAX, "%s", dir_data->data);
1608
        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CLUSTER_RC_ENABLE,
1609
               "gluster log directory is set to %s", dir_data->data);
1610
    }
1611
    if (len < 0 || len >= PATH_MAX)
1612
        exit(2);
1613

1614
    ret = mkdir_p(logdir, 0777, _gf_true);
1615
    if ((ret == -1) && (EEXIST != errno)) {
1616
        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
1617
               "Unable to create log dir %s", logdir);
1618
        exit(1);
1619
    }
1620

1621
    dir_data = dict_get(this->options, "working-directory");
1622

1623
    if (!dir_data) {
1624
        // Use default working dir
1625
        len = snprintf(workdir, PATH_MAX, "%s", GLUSTERD_DEFAULT_WORKDIR);
1626
    } else {
1627
        len = snprintf(workdir, PATH_MAX, "%s", dir_data->data);
1628
    }
1629
    if (len < 0 || len >= PATH_MAX)
1630
        exit(2);
1631

1632
    ret = sys_stat(workdir, &buf);
1633
    if ((ret != 0) && (ENOENT != errno)) {
1634
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
1635
               "stat fails on %s, exiting. (errno = %d)", workdir, errno);
1636
        exit(1);
1637
    }
1638

1639
    if ((!ret) && (!S_ISDIR(buf.st_mode))) {
1640
        gf_msg(this->name, GF_LOG_CRITICAL, ENOENT, GD_MSG_DIR_NOT_FOUND,
1641
               "Provided working area %s is not a directory,"
1642
               "exiting",
1643
               workdir);
1644
        exit(1);
1645
    }
1646

1647
    if ((-1 == ret) && (ENOENT == errno)) {
1648
        ret = mkdir_p(workdir, 0755, _gf_true);
1649

1650
        if (-1 == ret) {
1651
            gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
1652
                   "Unable to create directory %s"
1653
                   " ,errno = %d",
1654
                   workdir, errno);
1655
            exit(1);
1656
        }
1657
    }
1658

1659
    setenv("GLUSTERD_WORKDIR", workdir, 1);
1660
    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CURR_WORK_DIR_INFO,
1661
           "Using %s as working directory", workdir);
1662

1663
    setenv("DEFAULT_VAR_RUN_DIRECTORY", rundir, 1);
1664
    gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_CURR_WORK_DIR_INFO,
1665
           "Using %s as pid file working "
1666
           "directory",
1667
           rundir);
1668

1669
    ret = glusterd_find_correct_var_run_dir(this, var_run_dir);
1670
    if (ret) {
1671
        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_VAR_RUN_DIR_FIND_FAIL,
1672
               "Unable to find "
1673
               "the correct var run dir");
1674
        exit(1);
1675
    }
1676

1677
    ret = glusterd_init_var_run_dirs(this, var_run_dir,
1678
                                     GLUSTERD_DEFAULT_SNAPS_BRICK_DIR);
1679

1680
    if (ret) {
1681
        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED,
1682
               "Unable to create "
1683
               "snap backend folder");
1684
        exit(1);
1685
    }
1686

1687
    len = snprintf(snap_mount_dir, sizeof(snap_mount_dir), "%s%s", var_run_dir,
1688
                   GLUSTERD_DEFAULT_SNAPS_BRICK_DIR);
1689
    if ((len < 0) || (len >= sizeof(snap_mount_dir))) {
1690
        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_DIR_OP_FAILED,
1691
               "Snap mount dir too long");
1692
        exit(1);
1693
    }
1694

1695
    ret = mkdir_p(GLUSTER_SHARED_STORAGE_BRICK_DIR, 0755, _gf_true);
1696
    if (ret) {
1697
        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_DIR_OP_FAILED,
1698
               "Unable to create "
1699
               "shared storage brick");
1700
        exit(1);
1701
    }
1702

1703
    ret = glusterd_init_var_run_dirs(this, rundir, GLUSTERD_BITD_RUN_DIR);
1704
    if (ret) {
1705
        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED,
1706
               "Unable to create "
1707
               "bitd running directory");
1708
        exit(1);
1709
    }
1710

1711
    ret = glusterd_init_var_run_dirs(this, rundir, GLUSTERD_SCRUB_RUN_DIR);
1712
    if (ret) {
1713
        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED,
1714
               "Unable to create "
1715
               "scrub running directory");
1716
        exit(1);
1717
    }
1718

1719
#ifdef BUILD_GNFS
1720
    ret = glusterd_init_var_run_dirs(this, rundir, GLUSTERD_NFS_RUN_DIR);
1721
    if (ret) {
1722
        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED,
1723
               "Unable to create "
1724
               "nfs running directory");
1725
        exit(1);
1726
    }
1727
#endif
1728

1729
    ret = glusterd_init_var_run_dirs(this, rundir, GLUSTERD_QUOTAD_RUN_DIR);
1730
    if (ret) {
1731
        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_CREATE_DIR_FAILED,
1732
               "Unable to create "
1733
               "quota running directory");
1734
        exit(1);
1735
    }
1736

1737
    snprintf(cmd_log_filename, PATH_MAX, "%s/cmd_history.log", logdir);
1738
    ret = gf_cmd_log_init(cmd_log_filename);
1739

1740
    if (ret == -1) {
1741
        gf_msg("this->name", GF_LOG_CRITICAL, errno, GD_MSG_FILE_OP_FAILED,
1742
               "Unable to create cmd log file %s", cmd_log_filename);
1743
        exit(1);
1744
    }
1745

1746
    /* Ensure we have all of the extra directories. */
1747
    for (p = dirs; p->base != NULL; p++) {
1748
        len = snprintf(storedir, sizeof(storedir), "%s/%s", p->base, p->name);
1749

1750
        if (len < 0 || len >= sizeof(storedir)) {
1751
            gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_DIR_OP_FAILED,
1752
                   "'%s' directory name is too long", p->name);
1753
            exit(1);
1754
        }
1755

1756
        ret = sys_mkdir(storedir, 0755);
1757

1758
        if ((-1 == ret) && (errno != EEXIST)) {
1759
            gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_CREATE_DIR_FAILED,
1760
                   "unable to create directory %s", storedir);
1761
            exit(1);
1762
        }
1763
    }
1764

1765
    ret = glusterd_rpcsvc_options_build(this->options);
1766
    if (ret)
1767
        goto out;
1768
    rpc = rpcsvc_init(this, this->ctx, this->options, 64);
1769
    if (rpc == NULL) {
1770
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_INIT_FAIL,
1771
               "failed to init rpc");
1772
        goto out;
1773
    }
1774

1775
    ret = rpcsvc_register_notify(rpc, glusterd_rpcsvc_notify, this);
1776
    if (ret) {
1777
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPCSVC_REG_NOTIFY_RETURNED,
1778
               "rpcsvc_register_notify returned %d", ret);
1779
        goto out;
1780
    }
1781

1782
    /* Enable encryption for the TCP listener is management encryption is
1783
     * enabled
1784
     */
1785
    if (this->ctx->secure_mgmt) {
1786
        ret = dict_set_str(this->options, "transport.socket.ssl-enabled", "on");
1787
        if (ret != 0) {
1788
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
1789
                   "failed to set ssl-enabled in dict");
1790
            goto out;
1791
        }
1792
        /*
1793
         * This is the only place where we want secure_srvr to reflect
1794
         * the management-plane setting.
1795
         */
1796
        this->ctx->secure_srvr = MGMT_SSL_ALWAYS;
1797
    }
1798

1799
    /*
1800
     * only one (at most a pair - rdma and socket) listener for
1801
     * glusterd1_mop_prog, gluster_pmap_prog and gluster_handshake_prog.
1802
     */
1803

1804
    ret = dict_get_str(this->options, "transport-type", &transport_type);
1805
    if (ret) {
1806
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_GET_FAILED,
1807
               "Failed to get transport type");
1808
        ret = -1;
1809
        goto out;
1810
    }
1811

1812
    total_transport = rpc_transport_count(transport_type);
1813
    if (total_transport <= 0) {
1814
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_TRANSPORT_COUNT_GET_FAIL,
1815
               "failed to get total number of available tranpsorts");
1816
        ret = -1;
1817
        goto out;
1818
    }
1819

1820
    ret = rpcsvc_create_listeners(rpc, this->options, this->name);
1821
    if (ret < 1) {
1822
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_LISTENER_CREATE_FAIL,
1823
               "creation of listener failed");
1824
        ret = -1;
1825
        goto out;
1826
    } else if (ret < total_transport) {
1827
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RPC_LISTENER_CREATE_FAIL,
1828
               "creation of %d listeners failed, continuing with "
1829
               "succeeded transport",
1830
               (total_transport - ret));
1831
    }
1832

1833
    for (i = 0; i < gd_inet_programs_count; i++) {
1834
        ret = glusterd_program_register(this, rpc, gd_inet_programs[i]);
1835
        if (ret) {
1836
            i--;
1837
            for (; i >= 0; i--)
1838
                rpcsvc_program_unregister(rpc, gd_inet_programs[i]);
1839

1840
            goto out;
1841
        }
1842
    }
1843

1844
    /*
1845
     * Start a unix domain socket listener just for cli commands This
1846
     * should prevent ports from being wasted by being in TIMED_WAIT when
1847
     * cli commands are done continuously
1848
     */
1849
    uds_rpc = glusterd_init_uds_listener(this);
1850
    if (uds_rpc == NULL) {
1851
        ret = -1;
1852
        goto out;
1853
    }
1854

1855
    conf = GF_CALLOC(1, sizeof(glusterd_conf_t), gf_gld_mt_glusterd_conf_t);
1856
    GF_VALIDATE_OR_GOTO(this->name, conf, out);
1857

1858
    CDS_INIT_LIST_HEAD(&conf->peers);
1859
    CDS_INIT_LIST_HEAD(&conf->volumes);
1860
    CDS_INIT_LIST_HEAD(&conf->snapshots);
1861
    CDS_INIT_LIST_HEAD(&conf->missed_snaps_list);
1862
    CDS_INIT_LIST_HEAD(&conf->brick_procs);
1863
    CDS_INIT_LIST_HEAD(&conf->shd_procs);
1864
    CDS_INIT_LIST_HEAD(&conf->hostnames);
1865
    INIT_LIST_HEAD(&conf->remote_hostnames);
1866
    pthread_mutex_init(&conf->attach_lock, NULL);
1867
    pthread_mutex_init(&conf->volume_lock, NULL);
1868

1869
    pthread_mutex_init(&conf->mutex, NULL);
1870
    conf->rpc = rpc;
1871
    conf->uds_rpc = uds_rpc;
1872
    conf->gfs_mgmt = &gd_brick_prog;
1873
    conf->restart_shd = _gf_false;
1874
    this->private = conf;
1875
    conf->xl = this;
1876

1877
    /* conf->workdir and conf->rundir are smaller than PATH_MAX; gcc's
1878
     * snprintf checking will throw an error here if sprintf is used.
1879
     * Dueling gcc-8 and coverity, now coverity isn't smart enough to
1880
     * detect that these strncpy calls are safe. And for extra fun,
1881
     * the annotations don't do anything. */
1882
    if (strlen(workdir) >= sizeof(conf->workdir)) {
1883
        ret = -1;
1884
        goto out;
1885
    }
1886
    /* coverity[BUFFER_SIZE_WARNING] */
1887
    (void)strncpy(conf->workdir, workdir, sizeof(conf->workdir));
1888
    /* separate tests because combined tests confuses gcc */
1889
    if (strlen(rundir) >= sizeof(conf->rundir)) {
1890
        ret = -1;
1891
        goto out;
1892
    }
1893
    /* coverity[BUFFER_SIZE_WARNING] */
1894
    (void)strncpy(conf->rundir, rundir, sizeof(conf->rundir));
1895

1896
    /* coverity[BUFFER_SIZE_WARNING] */
1897
    (void)strncpy(conf->logdir, logdir, sizeof(conf->logdir));
1898

1899
    synclock_init(&conf->big_lock, SYNC_LOCK_RECURSIVE);
1900
    synccond_init(&conf->cond_restart_bricks);
1901
    synccond_init(&conf->cond_restart_shd);
1902
    synccond_init(&conf->cond_blockers);
1903
    pthread_mutex_init(&conf->xprt_lock, NULL);
1904
    INIT_LIST_HEAD(&conf->xprt_list);
1905
    pthread_mutex_init(&conf->import_volumes, NULL);
1906

1907
    glusterd_friend_sm_init();
1908
    glusterd_op_sm_init();
1909
    glusterd_opinfo_init();
1910
    ret = glusterd_sm_tr_log_init(
1911
        &conf->op_sm_log, glusterd_op_sm_state_name_get,
1912
        glusterd_op_sm_event_name_get, GLUSTERD_TR_LOG_SIZE);
1913
    if (ret)
1914
        goto out;
1915

1916
    conf->base_port = GF_IANA_PRIV_PORTS_START;
1917
    if (dict_get_uint32(this->options, "base-port", &conf->base_port) == 0) {
1918
        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_SET_FAILED,
1919
               "base-port override: %d", conf->base_port);
1920
    }
1921
    conf->max_port = GF_PORT_MAX;
1922
    if (dict_get_uint32(this->options, "max-port", &conf->max_port) == 0) {
1923
        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_SET_FAILED,
1924
               "max-port override: %d", conf->max_port);
1925
    }
1926

1927
    conf->mgmt_v3_lock_timeout = GF_LOCK_TIMER;
1928
    if (dict_get_time(this->options, "lock-timer",
1929
                      &conf->mgmt_v3_lock_timeout) == 0) {
1930
        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_DICT_SET_FAILED,
1931
               "lock-timer override: %ld", conf->mgmt_v3_lock_timeout);
1932
    }
1933

1934
    /* Set option to run bricks on valgrind if enabled in glusterd.vol */
1935
    this->ctx->cmd_args.vgtool = vgtool;
1936
    ret = dict_get_str(this->options, "run-with-valgrind", &valgrind_str);
1937
    if (ret < 0) {
1938
        gf_msg_debug(this->name, 0, "cannot get run-with-valgrind value");
1939
    }
1940
    if (valgrind_str) {
1941
        gf_boolean_t vg = _gf_false;
1942

1943
        if (!strcmp(valgrind_str, "memcheck"))
1944
            this->ctx->cmd_args.vgtool = _gf_memcheck;
1945
        else if (!strcmp(valgrind_str, "drd"))
1946
            this->ctx->cmd_args.vgtool = _gf_drd;
1947
        else if (!gf_string2boolean(valgrind_str, &vg))
1948
            this->ctx->cmd_args.vgtool = (vg ? _gf_memcheck : _gf_none);
1949
        else
1950
            gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_INVALID_ENTRY,
1951
                   "run-with-valgrind is neither boolean"
1952
                   " nor one of 'memcheck' or 'drd'");
1953
    }
1954

1955
    /* Store ping-timeout in conf */
1956
    ret = dict_get_time(this->options, "ping-timeout", &conf->ping_timeout);
1957
    /* Not failing here since ping-timeout can be optional as well */
1958

1959
    glusterd_mgmt_v3_lock_init();
1960
    glusterd_mgmt_v3_lock_timer_init();
1961
    glusterd_txn_opinfo_dict_init();
1962

1963
#ifdef BUILD_GNFS
1964
    glusterd_nfssvc_build(&conf->nfs_svc);
1965
#endif
1966
    glusterd_quotadsvc_build(&conf->quotad_svc);
1967
    glusterd_bitdsvc_build(&conf->bitd_svc);
1968
    glusterd_scrubsvc_build(&conf->scrub_svc);
1969

1970
    /* Make install copies few of the hook-scripts by creating hooks
1971
     * directory. Hence purposefully not doing the check for the presence of
1972
     * hooks directory. Doing so avoids creation of complete hooks directory
1973
     * tree.
1974
     */
1975
    ret = glusterd_hooks_create_hooks_directory(conf->workdir);
1976
    if (-1 == ret) {
1977
        gf_msg(this->name, GF_LOG_CRITICAL, errno, GD_MSG_DIR_OP_FAILED,
1978
               "Unable to create hooks directory ");
1979
        exit(1);
1980
    }
1981

1982
    CDS_INIT_LIST_HEAD(&conf->mount_specs);
1983

1984
    ret = dict_foreach(this->options, _install_mount_spec, NULL);
1985
    if (ret)
1986
        goto out;
1987
    ret = dict_get_str(this->options, "mountbroker-root", &mountbroker_root);
1988
    if (ret)
1989
        ret = 0;
1990
    else
1991
        ret = check_prepare_mountbroker_root(mountbroker_root);
1992
    if (ret)
1993
        goto out;
1994

1995
    ret = is_upgrade(this->options, &upgrade);
1996
    if (ret)
1997
        goto out;
1998

1999
    ret = is_downgrade(this->options, &downgrade);
2000
    if (ret)
2001
        goto out;
2002

2003
    if (!upgrade && !downgrade) {
2004
        ret = configure_syncdaemon(conf);
2005
        if (ret)
2006
            goto out;
2007
    }
2008

2009
    /* Restoring op-version needs to be done before initializing the
2010
     * services as glusterd_svc_init_common () invokes
2011
     * glusterd_conn_build_socket_filepath () which uses MY_UUID macro.
2012
     * MY_UUID generates a new uuid if its not been generated and writes it
2013
     * in the info file, Since the op-version is not read yet
2014
     * the default value i.e. 0 will be written for op-version and restore
2015
     * will fail. This is why restoring op-version needs to happen before
2016
     * service initialization
2017
     * */
2018
    ret = glusterd_restore_op_version(this);
2019
    if (ret) {
2020
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERS_RESTORE_FAIL,
2021
               "Failed to restore op_version");
2022
        goto out;
2023
    }
2024

2025
    ret = glusterd_restore();
2026
    if (ret < 0)
2027
        goto out;
2028

2029
    if (dict_get_str(conf->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
2030
                     &localtime_logging) == 0) {
2031
        int already_enabled = gf_log_get_localtime();
2032

2033
        if (strcmp(localtime_logging, "enable") == 0) {
2034
            gf_log_set_localtime(1);
2035
            if (!already_enabled)
2036
                gf_msg(this->name, GF_LOG_INFO, 0,
2037
                       GD_MSG_LOCALTIME_LOGGING_ENABLE,
2038
                       "localtime logging enable");
2039
        } else if (strcmp(localtime_logging, "disable") == 0) {
2040
            gf_log_set_localtime(0);
2041
            if (already_enabled)
2042
                gf_msg(this->name, GF_LOG_INFO, 0,
2043
                       GD_MSG_LOCALTIME_LOGGING_DISABLE,
2044
                       "localtime logging disable");
2045
        }
2046
    }
2047

2048
    GF_ATOMIC_INIT(conf->blockers, 0);
2049
    ret = glusterd_handle_upgrade_downgrade(this->options, conf, upgrade,
2050
                                            downgrade);
2051
    if (ret)
2052
        goto out;
2053

2054
    ret = glusterd_retrieve_max_op_version(this, &op_version);
2055
    /* first condition indicates file isn't present which means this code
2056
     * change is hitting for the first time or someone has deleted it from the
2057
     * backend.second condition is when max op_version differs, in both cases
2058
     * volfiles should be regenerated
2059
     */
2060
    if (op_version == 0 || op_version != GD_OP_VERSION_MAX) {
2061
        gf_log(this->name, GF_LOG_INFO,
2062
               "Regenerating volfiles due to a max op-version mismatch or "
2063
               "glusterd.upgrade file not being present, op_version retrieved:"
2064
               "%d, max op_version: %d",
2065
               op_version, GD_OP_VERSION_MAX);
2066
        glusterd_recreate_volfiles(conf);
2067
        ret = glusterd_store_max_op_version(this);
2068
        if (ret)
2069
            gf_log(this->name, GF_LOG_ERROR, "Failed to store max op-version");
2070
    }
2071

2072
    /* If the peer count is less than 2 then this would be the best time to
2073
     * spawn process/bricks that may need (re)starting since last time
2074
     * (this) glusterd was up. */
2075
    if (glusterd_get_peers_count() < 2)
2076
        glusterd_launch_synctask(glusterd_spawn_daemons, NULL);
2077

2078
    ret = glusterd_hooks_spawn_worker(this);
2079
    if (ret)
2080
        goto out;
2081

2082
    GF_OPTION_INIT("event-threads", workers, int32, out);
2083
    if (workers > 0 && workers != conf->workers) {
2084
        conf->workers = workers;
2085
        ret = gf_event_reconfigure_threads(this->ctx->event_pool, workers);
2086
        if (ret)
2087
            goto out;
2088
    }
2089

2090
    ret = 0;
2091
out:
2092
    if (ret < 0) {
2093
        if (this->private != NULL) {
2094
            GF_FREE(this->private);
2095
            this->private = NULL;
2096
        }
2097
    }
2098

2099
    return ret;
2100
}
2101

2102
/*
2103
 * fini - finish function for glusterd, called before
2104
 *        unloading gluster.
2105
 *
2106
 * @this:
2107
 *
2108
 */
2109
void
2110
fini(xlator_t *this)
2111
{
2112
    if (!this || !this->private)
2113
        goto out;
2114

2115
    glusterd_conf_t *priv = NULL;
2116
    priv = this->private;
2117

2118
    glusterd_stop_uds_listener(this);                 /*stop unix socket rpc*/
2119
    glusterd_stop_listener(this);                     /*stop tcp/ip socket rpc*/
2120
    glusterd_destroy_hostname_list(&priv->hostnames); /*Destroy hostname list */
2121
    glusterd_destroy_hostname_list(
2122
        &priv->remote_hostnames); /*Destroy remote hostname list*/
2123

2124
#if 0
2125
       /* Running threads might be using these resourses, we have to cancel/stop
2126
        * running threads before deallocating the memory, but we don't have
2127
        * control over the running threads to do pthread_cancel().
2128
        * So memory freeing handover to kernel.
2129
        */
2130
        /*TODO: cancel/stop the running threads*/
2131

2132
        GF_FREE (conf->uds_rpc);
2133
        GF_FREE (conf->rpc);
2134
        FREE (conf->pmap);
2135
        if (conf->handle)
2136
                gf_store_handle_destroy (conf->handle);
2137
        glusterd_sm_tr_log_delete (&conf->op_sm_log);
2138
        glusterd_mgmt_v3_lock_fini ();
2139
        glusterd_mgmt_v3_lock_timer_fini ();
2140
        glusterd_txn_opinfo_dict_fini ();
2141
        GF_FREE (conf);
2142

2143
        this->private = NULL;
2144
#endif
2145
out:
2146
    return;
2147
}
2148

2149
/*
2150
 * notify - notify function for glusterd
2151
 * @this:
2152
 * @trans:
2153
 * @event:
2154
 *
2155
 */
2156
int
2157
notify(xlator_t *this, int32_t event, void *data, ...)
2158
{
2159
    int ret = 0;
2160

2161
    switch (event) {
2162
        case GF_EVENT_POLLIN:
2163
            break;
2164

2165
        case GF_EVENT_POLLERR:
2166
            break;
2167

2168
        case GF_EVENT_CLEANUP:
2169
            break;
2170

2171
        default:
2172
            default_notify(this, event, data);
2173
            break;
2174
    }
2175

2176
    return ret;
2177
}
2178

2179
struct xlator_fops fops;
2180

2181
struct xlator_cbks cbks;
2182

2183
struct xlator_dumpops dumpops = {
2184
    .priv = glusterd_dump_priv,
2185
};
2186

2187
struct volume_options options[] = {
2188
    {
2189
        .key = {"working-directory"},
2190
        .type = GF_OPTION_TYPE_PATH,
2191
    },
2192
    {
2193
        .key = {"transport-type"},
2194
        .type = GF_OPTION_TYPE_ANY,
2195
    },
2196
    {
2197
        .key = {"transport.*"},
2198
        .type = GF_OPTION_TYPE_ANY,
2199
    },
2200
    {
2201
        .key = {"rpc-auth.*"},
2202
        .type = GF_OPTION_TYPE_ANY,
2203
    },
2204
    {
2205
        .key = {"rpc-auth-allow-insecure"},
2206
        .type = GF_OPTION_TYPE_BOOL,
2207
    },
2208
    {
2209
        .key = {"upgrade"},
2210
        .type = GF_OPTION_TYPE_BOOL,
2211
    },
2212
    {
2213
        .key = {"downgrade"},
2214
        .type = GF_OPTION_TYPE_BOOL,
2215
    },
2216
    {
2217
        .key = {"bind-insecure"},
2218
        .type = GF_OPTION_TYPE_BOOL,
2219
    },
2220
    {
2221
        .key = {"mountbroker-root"},
2222
        .type = GF_OPTION_TYPE_PATH,
2223
    },
2224
    {
2225
        .key = {"mountbroker.*"},
2226
        .type = GF_OPTION_TYPE_ANY,
2227
    },
2228
    {
2229
        .key = {"mountbroker-" GEOREP ".*"},
2230
        .type = GF_OPTION_TYPE_ANY,
2231
    },
2232
    {
2233
        .key = {GEOREP "-log-group"},
2234
        .type = GF_OPTION_TYPE_ANY,
2235
    },
2236
    {
2237
        .key = {"run-with-valgrind"},
2238
        .type = GF_OPTION_TYPE_BOOL,
2239
    },
2240
    {.key = {"server-quorum-type"},
2241
     .type = GF_OPTION_TYPE_STR,
2242
     .value = {"none", "server"},
2243
     .default_value = "none",
2244
     .description = "It can be set to none or server. When set to server, "
2245
                    "this option enables the specified volume to "
2246
                    "participate in the server-side quorum. "
2247
                    "This feature is on the server-side i.e. in glusterd. "
2248
                    "Whenever the glusterd on a machine observes that "
2249
                    "the quorum is not met, it brings down the bricks to "
2250
                    "prevent data split-brains. When the network "
2251
                    "connections are brought back up and the quorum is "
2252
                    "restored the bricks in   "
2253
                    "the volume are brought back up."},
2254
    {.key = {"server-quorum-ratio"},
2255
     .type = GF_OPTION_TYPE_PERCENT,
2256
     .description = "Sets the quorum percentage for the trusted "
2257
                    "storage pool."},
2258
    {.key = {"glusterd-sockfile"},
2259
     .type = GF_OPTION_TYPE_PATH,
2260
     .description = "The socket file on which glusterd should listen for "
2261
                    "cli requests. Default is " DEFAULT_GLUSTERD_SOCKFILE "."},
2262
    {.key = {"base-port"},
2263
     .type = GF_OPTION_TYPE_INT,
2264
     .description = "Sets the base port for portmap query"},
2265
    {.key = {"max-port"},
2266
     .type = GF_OPTION_TYPE_INT,
2267
     .max = GF_PORT_MAX,
2268
     .description = "Sets the max port for portmap query"},
2269
    {.key = {"mgmt-v3-lock-timeout"},
2270
     .type = GF_OPTION_TYPE_INT,
2271
     .max = 600,
2272
     .description = "Sets the mgmt-v3-lock-timeout for transactions."
2273
                    "Specifes the default timeout value after which "
2274
                    "lock acquired while performing transaction will "
2275
                    "be released."},
2276
    {.key = {"snap-brick-path"},
2277
     .type = GF_OPTION_TYPE_STR,
2278
     .description =
2279
         "directory where the bricks for the snapshots will be created"},
2280
    {
2281
        .key = {"ping-timeout"},
2282
        .type = GF_OPTION_TYPE_TIME,
2283
        .min = 0,
2284
        .max = 300,
2285
        .default_value = TOSTRING(RPC_DEFAULT_PING_TIMEOUT),
2286
    },
2287
    {.key = {"event-threads"},
2288
     .type = GF_OPTION_TYPE_INT,
2289
     .min = GLUSTERD_MIN_EVENT_THREADS,
2290
     .max = GLUSTERD_MAX_EVENT_THREADS,
2291
     .default_value = TOSTRING(STARTING_EVENT_THREADS),
2292
     .description = "Specifies the number of event threads to execute "
2293
                    "in parallel. Larger values would help process"
2294
                    " responses faster, depending on available processing"
2295
                    " power. Range 1-32 threads."},
2296
    {.key = {NULL}},
2297
};
2298

2299
xlator_api_t xlator_api = {
2300
    .init = init,
2301
    .fini = fini,
2302
    .mem_acct_init = mem_acct_init,
2303
    .op_version = {1}, /* Present from the initial version */
2304
    .dumpops = &dumpops,
2305
    .fops = &fops,
2306
    .cbks = &cbks,
2307
    .options = options,
2308
    .identifier = "glusterd",
2309
    .category = GF_MAINTAINED,
2310
};
2311