glusterfs

1
/*
2
   Copyright (c) 2007-2013 Red Hat, Inc. <http://www.redhat.com>
3
   This file is part of GlusterFS.
4

5
   This file is licensed to you under your choice of the GNU Lesser
6
   General Public License, version 3 or any later version (LGPLv3 or
7
   later), or the GNU General Public License, version 2 (GPLv2), in all
8
   cases as published by the Free Software Foundation.
9
*/
10

11
#include <inttypes.h>
12

13
#include <glusterfs/glusterfs.h>
14
#include <glusterfs/compat.h>
15
#include <glusterfs/dict.h>
16
#include <glusterfs/logging.h>
17
#include <glusterfs/timer.h>
18
#include <glusterfs/syscall.h>
19
#include <glusterfs/compat.h>
20
#include <glusterfs/compat-errno.h>
21
#include <glusterfs/statedump.h>
22
#include "glusterd-mem-types.h"
23
#include "glusterd.h"
24
#include "glusterd-sm.h"
25
#include "glusterd-op-sm.h"
26
#include "glusterd-utils.h"
27
#include "glusterd-hooks.h"
28
#include "glusterd-store.h"
29
#include "glusterd-snapshot-utils.h"
30
#include "glusterd-messages.h"
31

32
#include "rpc-clnt.h"
33
#include <glusterfs/common-utils.h>
34
#include <glusterfs/quota-common-utils.h>
35

36
#include <inttypes.h>
37
#include <dirent.h>
38

39
#if defined(GF_LINUX_HOST_OS)
40
#include <mntent.h>
41
#else
42
#include "mntent_compat.h"
43
#endif
44

45
#define GLUSTERD_GET_BRICK_DIR(path, volinfo, priv)                            \
46
    do {                                                                       \
47
        int32_t _brick_len;                                                    \
48
        if (volinfo->is_snap_volume) {                                         \
49
            _brick_len = snprintf(path, PATH_MAX, "%s/snaps/%s/%s/%s",         \
50
                                  priv->workdir, volinfo->snapshot->snapname,  \
51
                                  volinfo->volname, GLUSTERD_BRICK_INFO_DIR);  \
52
        } else {                                                               \
53
            _brick_len = snprintf(path, PATH_MAX, "%s/%s/%s/%s",               \
54
                                  priv->workdir, GLUSTERD_VOLUME_DIR_PREFIX,   \
55
                                  volinfo->volname, GLUSTERD_BRICK_INFO_DIR);  \
56
        }                                                                      \
57
        if ((_brick_len < 0) || (_brick_len >= PATH_MAX)) {                    \
58
            path[0] = 0;                                                       \
59
        }                                                                      \
60
    } while (0)
61

62
void
63
glusterd_replace_slash_with_hyphen(char *str)
64
{
65
    char *ptr = NULL;
66

67
    ptr = strchr(str, '/');
68

69
    while (ptr) {
70
        *ptr = '-';
71
        ptr = strchr(ptr, '/');
72
    }
73
}
74

75
int32_t
76
glusterd_store_create_brick_dir(glusterd_volinfo_t *volinfo)
77
{
78
    int32_t ret = -1;
79
    char brickdirpath[PATH_MAX] = {
80
        0,
81
    };
82
    glusterd_conf_t *priv = NULL;
83

84
    GF_ASSERT(volinfo);
85

86
    priv = THIS->private;
87
    GF_ASSERT(priv);
88

89
    GLUSTERD_GET_BRICK_DIR(brickdirpath, volinfo, priv);
90
    ret = gf_store_mkdir(brickdirpath);
91

92
    return ret;
93
}
94

95
static void
96
glusterd_store_key_vol_brick_set(glusterd_brickinfo_t *brickinfo,
97
                                 char *key_vol_brick, size_t len)
98
{
99
    GF_ASSERT(brickinfo);
100
    GF_ASSERT(key_vol_brick);
101
    GF_ASSERT(len >= PATH_MAX);
102

103
    snprintf(key_vol_brick, len, "%s", brickinfo->path);
104
    glusterd_replace_slash_with_hyphen(key_vol_brick);
105
}
106

107
static int
108
glusterd_store_brickinfofname_set(glusterd_brickinfo_t *brickinfo,
109
                                  char *brickfname, size_t len)
110
{
111
    char key_vol_brick[PATH_MAX] = {0};
112
    int32_t ret = 0;
113
    GF_ASSERT(brickfname);
114
    GF_ASSERT(brickinfo);
115
    GF_ASSERT(len >= PATH_MAX);
116

117
    glusterd_store_key_vol_brick_set(brickinfo, key_vol_brick,
118
                                     sizeof(key_vol_brick));
119
    ret = snprintf(brickfname, len, "%s:%s", brickinfo->hostname,
120
                   key_vol_brick);
121
    if (ret < 0 || ret >= len)
122
        return -1;
123
    return 0;
124
}
125

126
static int
127
glusterd_store_brickinfopath_set(glusterd_volinfo_t *volinfo,
128
                                 glusterd_brickinfo_t *brickinfo,
129
                                 char *brickpath, size_t len)
130
{
131
    char brickfname[PATH_MAX] = {0};
132
    char brickdirpath[PATH_MAX] = {
133
        0,
134
    };
135
    glusterd_conf_t *priv = NULL;
136
    int32_t ret = 0;
137
    GF_ASSERT(brickpath);
138
    GF_ASSERT(brickinfo);
139
    GF_ASSERT(len >= PATH_MAX);
140

141
    priv = THIS->private;
142
    GF_ASSERT(priv);
143

144
    GLUSTERD_GET_BRICK_DIR(brickdirpath, volinfo, priv);
145
    ret = glusterd_store_brickinfofname_set(brickinfo, brickfname,
146
                                            sizeof(brickfname));
147
    if (ret)
148
        return ret;
149
    ret = snprintf(brickpath, len, "%s/%s", brickdirpath, brickfname);
150
    if (ret < 0 || ret >= len)
151
        return -1;
152
    return 0;
153
}
154

155
static int
156
glusterd_store_snapd_path_set(glusterd_volinfo_t *volinfo, char *snapd_path,
157
                              size_t len)
158
{
159
    char volpath[PATH_MAX] = {
160
        0,
161
    };
162
    glusterd_conf_t *priv = NULL;
163
    int32_t ret = 0;
164
    GF_ASSERT(volinfo);
165
    GF_ASSERT(len >= PATH_MAX);
166

167
    priv = THIS->private;
168
    GF_ASSERT(priv);
169

170
    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
171

172
    ret = snprintf(snapd_path, len, "%s/snapd.info", volpath);
173
    if (ret < 0 || ret >= len)
174
        return -1;
175
    return 0;
176
}
177

178
gf_boolean_t
179
glusterd_store_is_valid_brickpath(char *volname, char *brick)
180
{
181
    glusterd_brickinfo_t *brickinfo = NULL;
182
    glusterd_volinfo_t *volinfo = NULL;
183
    int32_t ret = 0;
184
    size_t volname_len = strlen(volname);
185
    int bpath_len = 0;
186
    const char delim[2] = "/";
187
    char *sub_dir = NULL;
188
    char *saveptr = NULL;
189
    char *brickpath_ptr = NULL;
190

191
    ret = glusterd_brickinfo_new_from_brick(brick, &brickinfo, _gf_false, NULL);
192
    if (ret) {
193
        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_BRICK_CREATION_FAIL,
194
               "Failed to create brick "
195
               "info for brick %s",
196
               brick);
197
        ret = 0;
198
        goto out;
199
    }
200
    ret = glusterd_volinfo_new(&volinfo);
201
    if (ret) {
202
        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_VOLFILE_CREATE_FAIL,
203
               "Failed to create volinfo");
204
        ret = 0;
205
        goto out;
206
    }
207
    if (volname_len >= sizeof(volinfo->volname)) {
208
        gf_msg(THIS->name, GF_LOG_WARNING, 0, GD_MSG_NAME_TOO_LONG,
209
               "volume name too long");
210
        ret = 0;
211
        goto out;
212
    }
213
    memcpy(volinfo->volname, volname, volname_len + 1);
214

215
    /* Check whether brickpath is less than PATH_MAX */
216
    ret = 1;
217
    bpath_len = strlen(brickinfo->path);
218

219
    if (brickinfo->path[bpath_len - 1] != '/') {
220
        if (bpath_len >= PATH_MAX) {
221
            ret = 0;
222
            goto out;
223
        }
224
    } else {
225
        /* Path has a trailing "/" which should not be considered in
226
         * length check validation
227
         */
228
        if (bpath_len >= PATH_MAX + 1) {
229
            ret = 0;
230
            goto out;
231
        }
232
    }
233

234
    /* The following validation checks whether each sub directories in the
235
     * brick path meets the POSIX max length validation
236
     */
237

238
    brickpath_ptr = brickinfo->path;
239
    sub_dir = strtok_r(brickpath_ptr, delim, &saveptr);
240

241
    while (sub_dir != NULL) {
242
        if (strlen(sub_dir) >= _POSIX_PATH_MAX) {
243
            ret = 0;
244
            goto out;
245
        }
246
        sub_dir = strtok_r(NULL, delim, &saveptr);
247
    }
248

249
out:
250
    if (brickinfo)
251
        glusterd_brickinfo_delete(brickinfo);
252
    if (volinfo)
253
        glusterd_volinfo_unref(volinfo);
254

255
    return ret;
256
}
257

258
int32_t
259
glusterd_store_volinfo_brick_fname_write(int vol_fd,
260
                                         glusterd_brickinfo_t *brickinfo,
261
                                         int32_t brick_count,
262
                                         int is_thin_arbiter)
263
{
264
    char key[64] = {
265
        0,
266
    };
267
    char brickfname[PATH_MAX] = {
268
        0,
269
    };
270
    int32_t ret = -1;
271

272
    if (!is_thin_arbiter) {
273
        snprintf(key, sizeof(key), "%s-%d", GLUSTERD_STORE_KEY_VOL_BRICK,
274
                 brick_count);
275
    } else {
276
        snprintf(key, sizeof(key), "%s-%d", GLUSTERD_STORE_KEY_VOL_TA_BRICK,
277
                 brick_count);
278
    }
279
    ret = glusterd_store_brickinfofname_set(brickinfo, brickfname,
280
                                            sizeof(brickfname));
281
    if (ret)
282
        return ret;
283
    ret = gf_store_save_value(vol_fd, key, brickfname);
284
    return ret;
285
}
286

287
int32_t
288
glusterd_store_create_brick_shandle_on_absence(glusterd_volinfo_t *volinfo,
289
                                               glusterd_brickinfo_t *brickinfo)
290
{
291
    char brickpath[PATH_MAX] = {
292
        0,
293
    };
294
    int32_t ret = 0;
295

296
    GF_ASSERT(volinfo);
297
    GF_ASSERT(brickinfo);
298

299
    ret = glusterd_store_brickinfopath_set(volinfo, brickinfo, brickpath,
300
                                           sizeof(brickpath));
301
    if (ret)
302
        return ret;
303
    ret = gf_store_handle_create_on_absence(&brickinfo->shandle, brickpath);
304
    return ret;
305
}
306

307
int32_t
308
glusterd_store_create_snapd_shandle_on_absence(glusterd_volinfo_t *volinfo)
309
{
310
    char snapd_path[PATH_MAX] = {
311
        0,
312
    };
313
    int32_t ret = 0;
314

315
    GF_ASSERT(volinfo);
316

317
    ret = glusterd_store_snapd_path_set(volinfo, snapd_path,
318
                                        sizeof(snapd_path));
319
    if (ret)
320
        return ret;
321
    ret = gf_store_handle_create_on_absence(&volinfo->snapd.handle, snapd_path);
322
    return ret;
323
}
324

325
/* Store the bricks snapshot details only if required
326
 *
327
 * The snapshot details will be stored only if the cluster op-version is
328
 * greater than or equal to 4
329
 */
330
static int
331
gd_store_brick_snap_details_write(int fd, glusterd_brickinfo_t *brickinfo)
332
{
333
    int ret = -1;
334
    xlator_t *this = THIS;
335
    char value[5 * PATH_MAX];
336
    uint total_len = 0;
337

338
    GF_VALIDATE_OR_GOTO(this->name, (fd > 0), out);
339
    GF_VALIDATE_OR_GOTO(this->name, (brickinfo != NULL), out);
340

341
    if (brickinfo->origin_path[0] != '\0') {
342
        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
343
                       GLUSTERD_STORE_KEY_BRICK_ORIGIN_PATH,
344
                       brickinfo->origin_path);
345
        if (ret < 0 || ret >= sizeof(value) - total_len) {
346
            ret = -1;
347
            goto err;
348
        }
349
        total_len += ret;
350
    }
351

352
    if (brickinfo->mount_dir[0] != '\0') {
353
        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
354
                       GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR,
355
                       brickinfo->mount_dir);
356
        if (ret < 0 || ret >= sizeof(value) - total_len) {
357
            ret = -1;
358
            goto err;
359
        }
360
        total_len += ret;
361
    }
362

363
    if (brickinfo->fstype[0] != '\0') {
364
        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
365
                       GLUSTERD_STORE_KEY_BRICK_FSTYPE, brickinfo->fstype);
366
        if (ret < 0 || ret >= sizeof(value) - total_len) {
367
            ret = -1;
368
            goto err;
369
        }
370
        total_len += ret;
371
    }
372

373
    if (strlen(brickinfo->snap_type) > 0) {
374
        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
375
                       GLUSTERD_STORE_KEY_BRICK_SNAPTYPE, brickinfo->snap_type);
376
        if (ret < 0 || ret >= sizeof(value) - total_len) {
377
            ret = -1;
378
            goto err;
379
        }
380
        total_len += ret;
381
    }
382

383
    if (brickinfo->mnt_opts[0] != '\0') {
384
        ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%s\n",
385
                       GLUSTERD_STORE_KEY_BRICK_MNTOPTS, brickinfo->mnt_opts);
386
        if (ret < 0 || ret >= sizeof(value) - total_len) {
387
            ret = -1;
388
            goto err;
389
        }
390
        total_len += ret;
391
    }
392

393
    ret = snprintf(value + total_len, sizeof(value) - total_len, "%s=%d\n",
394
                   GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
395
                   brickinfo->snap_status);
396
    if (ret < 0 || ret >= sizeof(value) - total_len) {
397
        ret = -1;
398
        goto err;
399
    }
400
    total_len += ret;
401

402
    ret = snprintf(value + total_len, sizeof(value) - total_len,
403
                   "%s=%" PRIu64 "\n", GLUSTERD_STORE_KEY_BRICK_FSID,
404
                   brickinfo->statfs_fsid);
405
    if (ret < 0 || ret >= sizeof(value) - total_len) {
406
        ret = -1;
407
        goto err;
408
    }
409

410
    ret = gf_store_save_items(fd, value);
411
err:
412
    if (ret) {
413
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_FS_LABEL_UPDATE_FAIL,
414
               "Failed to save "
415
               "snap detils of brick %s",
416
               brickinfo->path);
417
    }
418
out:
419
    return ret;
420
}
421

422
static int32_t
423
glusterd_store_brickinfo_write(int fd, glusterd_brickinfo_t *brickinfo)
424
{
425
    char value[5 * PATH_MAX];
426
    int32_t ret = -1;
427

428
    GF_ASSERT(brickinfo);
429
    GF_ASSERT(fd > 0);
430

431
    ret = snprintf(value, sizeof(value),
432
                   "%s=%s\n%s=%s\n%s=%s\n%s=%s\n%s=%d\n%s=%d\n%s=%d\n%s=%s\n",
433
                   GLUSTERD_STORE_KEY_BRICK_UUID, uuid_utoa(brickinfo->uuid),
434
                   GLUSTERD_STORE_KEY_BRICK_HOSTNAME, brickinfo->hostname,
435
                   GLUSTERD_STORE_KEY_BRICK_PATH, brickinfo->path,
436
                   GLUSTERD_STORE_KEY_BRICK_REAL_PATH, brickinfo->path,
437
                   GLUSTERD_STORE_KEY_BRICK_PORT, brickinfo->port,
438
                   GLUSTERD_STORE_KEY_BRICK_RDMA_PORT, brickinfo->rdma_port,
439
                   GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
440
                   brickinfo->decommissioned, GLUSTERD_STORE_KEY_BRICK_ID,
441
                   brickinfo->brick_id);
442

443
    if (ret < 0 || ret >= sizeof(value)) {
444
        ret = -1;
445
        goto out;
446
    }
447

448
    ret = gf_store_save_items(fd, value);
449
    if (ret)
450
        goto out;
451

452
    ret = gd_store_brick_snap_details_write(fd, brickinfo);
453
    if (ret)
454
        goto out;
455

456
    if (!brickinfo->vg[0])
457
        goto out;
458

459
    ret = gf_store_save_value(fd, GLUSTERD_STORE_KEY_BRICK_VGNAME,
460
                              brickinfo->vg);
461
out:
462
    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
463
    return ret;
464
}
465

466
int32_t
467
glusterd_store_snapd_write(int fd, glusterd_volinfo_t *volinfo)
468
{
469
    char value[64] = {
470
        0,
471
    };
472
    int32_t ret = 0;
473
    xlator_t *this = THIS;
474

475
    GF_ASSERT(volinfo);
476
    GF_ASSERT(fd > 0);
477

478
    snprintf(value, sizeof(value), "%d", volinfo->snapd.port);
479
    ret = gf_store_save_value(fd, GLUSTERD_STORE_KEY_SNAPD_PORT, value);
480
    if (ret)
481
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_PORT_STORE_FAIL,
482
               "failed to store the snapd "
483
               "port of volume %s",
484
               volinfo->volname);
485

486
    gf_msg_debug(this->name, 0, "Returning %d", ret);
487
    return ret;
488
}
489

490
static int32_t
491
glusterd_store_perform_brick_store(glusterd_brickinfo_t *brickinfo)
492
{
493
    int fd = -1;
494
    int32_t ret = -1;
495
    GF_ASSERT(brickinfo);
496

497
    fd = gf_store_mkstemp(brickinfo->shandle);
498
    if (fd <= 0) {
499
        ret = -1;
500
        goto out;
501
    }
502
    ret = glusterd_store_brickinfo_write(fd, brickinfo);
503
    if (ret)
504
        goto out;
505

506
out:
507
    if (ret && (fd > 0)) {
508
        gf_store_unlink_tmppath(brickinfo->shandle);
509
    }
510
    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
511
    return ret;
512
}
513

514
int32_t
515
glusterd_store_perform_snapd_store(glusterd_volinfo_t *volinfo)
516
{
517
    int fd = -1;
518
    int32_t ret = -1;
519
    xlator_t *this = THIS;
520

521
    GF_ASSERT(volinfo);
522

523
    fd = gf_store_mkstemp(volinfo->snapd.handle);
524
    if (fd <= 0) {
525
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
526
               "failed to create the "
527
               "temporary file for the snapd store handle of volume "
528
               "%s",
529
               volinfo->volname);
530
        goto out;
531
    }
532

533
    ret = glusterd_store_snapd_write(fd, volinfo);
534
    if (ret) {
535
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_PORT_STORE_FAIL,
536
               "failed to write snapd port "
537
               "info to store handle (volume: %s",
538
               volinfo->volname);
539
        goto out;
540
    }
541

542
    ret = gf_store_rename_tmppath(volinfo->snapd.handle);
543

544
out:
545
    if (ret && (fd > 0))
546
        gf_store_unlink_tmppath(volinfo->snapd.handle);
547
    gf_msg_debug(this->name, 0, "Returning %d", ret);
548
    return ret;
549
}
550

551
static int32_t
552
glusterd_store_brickinfo(glusterd_volinfo_t *volinfo,
553
                         glusterd_brickinfo_t *brickinfo, int32_t brick_count,
554
                         int vol_fd, int is_thin_arbiter)
555
{
556
    int32_t ret = -1;
557

558
    GF_ASSERT(volinfo);
559
    GF_ASSERT(brickinfo);
560

561
    ret = glusterd_store_volinfo_brick_fname_write(
562
        vol_fd, brickinfo, brick_count, is_thin_arbiter);
563
    if (ret)
564
        goto out;
565

566
    ret = glusterd_store_create_brick_shandle_on_absence(volinfo, brickinfo);
567
    if (ret)
568
        goto out;
569

570
    ret = glusterd_store_perform_brick_store(brickinfo);
571
out:
572
    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
573
    return ret;
574
}
575

576
int32_t
577
glusterd_store_snapd_info(glusterd_volinfo_t *volinfo)
578
{
579
    int32_t ret = -1;
580
    xlator_t *this = THIS;
581

582
    GF_ASSERT(volinfo);
583

584
    ret = glusterd_store_create_snapd_shandle_on_absence(volinfo);
585
    if (ret) {
586
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_CREATE_FAIL,
587
               "failed to create store "
588
               "handle for snapd (volume: %s)",
589
               volinfo->volname);
590
        goto out;
591
    }
592

593
    ret = glusterd_store_perform_snapd_store(volinfo);
594
    if (ret)
595
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_INFO_STORE_FAIL,
596
               "failed to store snapd info "
597
               "of the volume %s",
598
               volinfo->volname);
599

600
out:
601
    if (ret)
602
        gf_store_unlink_tmppath(volinfo->snapd.handle);
603

604
    gf_msg_debug(this->name, 0, "Returning with %d", ret);
605
    return ret;
606
}
607

608
int32_t
609
glusterd_store_delete_brick(glusterd_brickinfo_t *brickinfo, char *delete_path)
610
{
611
    int32_t ret = -1;
612
    glusterd_conf_t *priv = NULL;
613
    char brickpath[PATH_MAX] = {
614
        0,
615
    };
616
    char *ptr = NULL;
617
    char *tmppath = NULL;
618
    xlator_t *this = THIS;
619

620
    GF_ASSERT(brickinfo);
621

622
    priv = this->private;
623
    GF_ASSERT(priv);
624

625
    tmppath = gf_strdup(brickinfo->path);
626

627
    ptr = strchr(tmppath, '/');
628

629
    while (ptr) {
630
        *ptr = '-';
631
        ptr = strchr(tmppath, '/');
632
    }
633

634
    snprintf(brickpath, sizeof(brickpath),
635
             "%s/" GLUSTERD_BRICK_INFO_DIR "/%s:%s", delete_path,
636
             brickinfo->hostname, tmppath);
637

638
    GF_FREE(tmppath);
639

640
    ret = gf_unlink(brickpath) ? 0 : -1;
641

642
    if (brickinfo->shandle) {
643
        gf_store_handle_destroy(brickinfo->shandle);
644
        brickinfo->shandle = NULL;
645
    }
646
    gf_msg_debug(this->name, 0, "Returning with %d", ret);
647
    return ret;
648
}
649

650
static int
651
_storeopts(dict_t *dict_value, char *key, data_t *value, void *data)
652
{
653
    int32_t ret = 0;
654
    int32_t exists = 0;
655
    int32_t option_len = 0;
656
    gf_store_handle_t *shandle = NULL;
657
    glusterd_volinfo_data_store_t *dict_data = NULL;
658
    xlator_t *this = THIS;
659

660
    dict_data = (glusterd_volinfo_data_store_t *)data;
661
    shandle = dict_data->shandle;
662

663
    GF_ASSERT(shandle);
664
    GF_ASSERT(shandle->fd > 0);
665
    GF_ASSERT(key);
666
    GF_ASSERT(value);
667
    GF_ASSERT(value->data);
668

669
    if (dict_data->key_check == 1) {
670
        if (is_key_glusterd_hooks_friendly(key)) {
671
            exists = 1;
672

673
        } else {
674
            exists = glusterd_check_option_exists(key, NULL);
675
        }
676
    }
677
    if (exists == 1 || dict_data->key_check == 0) {
678
        gf_msg_debug(this->name, 0,
679
                     "Storing in buffer for volinfo:key= %s, "
680
                     "val=%s",
681
                     key, value->data);
682
    } else {
683
        gf_msg_debug(this->name, 0, "Discarding:key= %s, val=%s", key,
684
                     value->data);
685
        return 0;
686
    }
687

688
    /*
689
     * The option_len considers the length of the key value
690
     * pair and along with that '=' and '\n', but as value->len
691
     * already considers a NULL at the end of the data, adding
692
     * just 1.
693
     */
694
    option_len = strlen(key) + value->len + 1;
695

696
    if ((VOLINFO_BUFFER_SIZE - dict_data->buffer_len - 1) < option_len) {
697
        ret = gf_store_save_items(shandle->fd, dict_data->buffer);
698
        if (ret) {
699
            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL);
700
            return -1;
701
        }
702
        dict_data->buffer_len = 0;
703
        dict_data->buffer[0] = '\0';
704
    }
705
    ret = snprintf(dict_data->buffer + dict_data->buffer_len, option_len + 1,
706
                   "%s=%s\n", key, value->data);
707
    if (ret < 0 || ret > option_len + 1) {
708
        gf_smsg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_COPY_FAIL, NULL);
709
        return -1;
710
    }
711

712
    dict_data->buffer_len += ret;
713

714
    return 0;
715
}
716

717
/* Store the volumes snapshot details only if required
718
 *
719
 * The snapshot details will be stored only if the cluster op-version is
720
 * greater than or equal to 4
721
 */
722
static int
723
glusterd_volume_write_snap_details(int fd, glusterd_volinfo_t *volinfo)
724
{
725
    int ret = -1;
726
    xlator_t *this = THIS;
727
    char buf[PATH_MAX] = {
728
        0,
729
    };
730
    uint total_len = 0;
731

732
    GF_VALIDATE_OR_GOTO(this->name, (fd > 0), out);
733
    GF_VALIDATE_OR_GOTO(this->name, (volinfo != NULL), out);
734

735
    ret = snprintf(buf, sizeof(buf), "%s=%s\n%s=%s\n%s=%" PRIu64 "\n",
736
                   GLUSTERD_STORE_KEY_PARENT_VOLNAME, volinfo->parent_volname,
737
                   GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
738
                   uuid_utoa(volinfo->restored_from_snap),
739
                   GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
740
                   volinfo->snap_max_hard_limit);
741
    if (ret < 0 || ret >= sizeof(buf)) {
742
        ret = -1;
743
        goto err;
744
    }
745
    total_len += ret;
746

747
    if (volinfo->restored_from_snapname_id[0] != '\0') {
748
        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
749
                       GLUSTERD_STORE_KEY_VOL_RESTORED_SNAPNAME_ID,
750
                       volinfo->restored_from_snapname_id);
751
        if (ret < 0 || ret >= sizeof(buf) - total_len) {
752
            ret = -1;
753
            goto out;
754
        }
755
        total_len += ret;
756
    }
757

758
    if (volinfo->snap_plugin[0] != '\0') {
759
        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
760
                       GLUSTERD_STORE_KEY_VOL_SNAP_PLUGIN,
761
                       volinfo->snap_plugin);
762
        if (ret < 0 || ret >= sizeof(buf) - total_len) {
763
            ret = -1;
764
            goto out;
765
        }
766
        total_len += ret;
767
    }
768

769
    if (volinfo->restored_from_snapname[0] != '\0') {
770
        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
771
                       GLUSTERD_STORE_KEY_VOL_RESTORED_SNAPNAME,
772
                       volinfo->restored_from_snapname);
773
        if (ret < 0 || ret >= sizeof(buf) - total_len) {
774
            ret = -1;
775
            goto out;
776
        }
777
        total_len += ret;
778
    }
779

780
    ret = gf_store_save_items(fd, buf);
781
    if (ret) {
782
        goto err;
783
    }
784
    ret = glusterd_store_snapd_info(volinfo);
785
err:
786
    if (ret)
787
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPINFO_WRITE_FAIL,
788
               "Failed to write snap details"
789
               " for volume %s",
790
               volinfo->volname);
791
out:
792
    return ret;
793
}
794

795
static int32_t
796
glusterd_volume_exclude_options_write(int fd, glusterd_volinfo_t *volinfo)
797
{
798
    char *str = NULL;
799
    char buf[PATH_MAX];
800
    uint total_len = 0;
801
    int32_t ret = -1;
802
    xlator_t *this = THIS;
803
    glusterd_conf_t *conf = NULL;
804

805
    GF_ASSERT(fd > 0);
806
    GF_ASSERT(volinfo);
807
    conf = this->private;
808
    GF_VALIDATE_OR_GOTO(this->name, (conf != NULL), out);
809

810
    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
811
                   "%s=%d\n%s=%d\n%s=%d\n%s=%d\n%s=%d\n",
812
                   GLUSTERD_STORE_KEY_VOL_TYPE, volinfo->type,
813
                   GLUSTERD_STORE_KEY_VOL_COUNT, volinfo->brick_count,
814
                   GLUSTERD_STORE_KEY_VOL_STATUS, volinfo->status,
815
                   GLUSTERD_STORE_KEY_VOL_SUB_COUNT, volinfo->sub_count,
816
                   GLUSTERD_STORE_KEY_VOL_REPLICA_CNT, volinfo->replica_count);
817
    if (ret < 0 || ret >= sizeof(buf) - total_len) {
818
        ret = -1;
819
        goto out;
820
    }
821
    total_len += ret;
822

823
    if (conf->op_version < GD_OP_VERSION_10_0) {
824
        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
825
                       GLUSTERD_STORE_KEY_VOL_STRIPE_CNT, STRIPE_COUNT);
826
        if (ret < 0 || ret >= sizeof(buf) - total_len) {
827
            ret = -1;
828
            goto out;
829
        }
830
        total_len += ret;
831
    }
832

833
    if (volinfo->arbiter_count) {
834
        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
835
                       GLUSTERD_STORE_KEY_VOL_ARBITER_CNT,
836
                       volinfo->arbiter_count);
837
        if (ret < 0 || ret >= sizeof(buf) - total_len) {
838
            ret = -1;
839
            goto out;
840
        }
841
        total_len += ret;
842
    }
843

844
    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n%s=%d\n",
845
                   GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT, volinfo->disperse_count,
846
                   GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT,
847
                   volinfo->redundancy_count);
848
    if (ret < 0 || ret >= sizeof(buf) - total_len) {
849
        ret = -1;
850
        goto out;
851
    }
852
    total_len += ret;
853

854
    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
855
                   "%s=%d\n%s=%d\n%s=%s\n", GLUSTERD_STORE_KEY_VOL_VERSION,
856
                   volinfo->version, GLUSTERD_STORE_KEY_VOL_TRANSPORT,
857
                   volinfo->transport_type, GLUSTERD_STORE_KEY_VOL_ID,
858
                   uuid_utoa(volinfo->volume_id));
859
    if (ret < 0 || ret >= sizeof(buf) - total_len) {
860
        ret = -1;
861
        goto out;
862
    }
863
    total_len += ret;
864

865
    str = glusterd_auth_get_username(volinfo);
866
    if (str) {
867
        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
868
                       GLUSTERD_STORE_KEY_USERNAME, str);
869
        if (ret < 0 || ret >= sizeof(buf) - total_len) {
870
            ret = -1;
871
            goto out;
872
        }
873
        total_len += ret;
874
    }
875

876
    str = glusterd_auth_get_password(volinfo);
877
    if (str) {
878
        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
879
                       GLUSTERD_STORE_KEY_PASSWORD, str);
880
        if (ret < 0 || ret >= sizeof(buf) - total_len) {
881
            ret = -1;
882
            goto out;
883
        }
884
        total_len += ret;
885
    }
886

887
    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n%s=%d\n",
888
                   GLUSTERD_STORE_KEY_VOL_OP_VERSION, volinfo->op_version,
889
                   GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
890
                   volinfo->client_op_version);
891
    if (ret < 0 || ret >= sizeof(buf) - total_len) {
892
        ret = -1;
893
        goto out;
894
    }
895
    total_len += ret;
896

897
    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
898
                   GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION,
899
                   volinfo->quota_xattr_version);
900
    if (ret < 0 || ret >= sizeof(buf) - total_len) {
901
        ret = -1;
902
        goto out;
903
    }
904
    total_len += ret;
905

906
    if (conf->op_version >= GD_OP_VERSION_3_10_0) {
907
        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=0\n",
908
                       GF_TIER_ENABLED);
909
        if (ret < 0 || ret >= sizeof(buf) - total_len) {
910
            ret = -1;
911
            goto out;
912
        }
913
        total_len += ret;
914
    }
915

916
    if ((conf->op_version >= GD_OP_VERSION_7_0) &&
917
        volinfo->thin_arbiter_count) {
918
        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
919
                       GLUSTERD_STORE_KEY_VOL_THIN_ARBITER_CNT,
920
                       volinfo->thin_arbiter_count);
921
        if (ret < 0 || ret >= sizeof(buf) - total_len) {
922
            ret = -1;
923
            goto out;
924
        }
925
        total_len += ret;
926
    }
927

928
    ret = gf_store_save_items(fd, buf);
929
    if (ret)
930
        goto out;
931

932
    ret = glusterd_volume_write_snap_details(fd, volinfo);
933

934
out:
935
    if (ret)
936
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_VALS_WRITE_FAIL,
937
               "Unable to write volume "
938
               "values for %s",
939
               volinfo->volname);
940
    return ret;
941
}
942

943
static void
944
glusterd_store_voldirpath_set(glusterd_volinfo_t *volinfo, char *voldirpath)
945
{
946
    glusterd_conf_t *priv = NULL;
947

948
    GF_ASSERT(volinfo);
949
    priv = THIS->private;
950
    GF_ASSERT(priv);
951

952
    GLUSTERD_GET_VOLUME_DIR(voldirpath, volinfo, priv);
953
}
954

955
static void
956
glusterd_store_piddirpath_set(glusterd_volinfo_t *volinfo, char *piddirpath)
957
{
958
    glusterd_conf_t *priv = NULL;
959

960
    GF_ASSERT(volinfo);
961
    priv = THIS->private;
962
    GF_ASSERT(priv);
963

964
    GLUSTERD_GET_VOLUME_PID_DIR(piddirpath, volinfo, priv);
965
}
966

967
static int32_t
968
glusterd_store_create_volume_dirs(glusterd_volinfo_t *volinfo)
969
{
970
    int32_t ret = -1;
971
    char dirpath[PATH_MAX] = {
972
        0,
973
    };
974

975
    GF_ASSERT(volinfo);
976

977
    glusterd_store_voldirpath_set(volinfo, dirpath);
978
    ret = gf_store_mkdir(dirpath);
979
    if (ret)
980
        goto out;
981

982
    glusterd_store_piddirpath_set(volinfo, dirpath);
983
    ret = gf_store_mkdir(dirpath);
984
    if (ret)
985
        goto out;
986

987
out:
988
    gf_msg_debug(THIS->name, 0, "Returning with %d", ret);
989
    return ret;
990
}
991

992
int32_t
993
glusterd_store_create_snap_dir(glusterd_snap_t *snap)
994
{
995
    int32_t ret = -1;
996
    char snapdirpath[PATH_MAX] = {
997
        0,
998
    };
999
    glusterd_conf_t *priv = NULL;
1000

1001
    priv = THIS->private;
1002
    GF_ASSERT(priv);
1003
    GF_ASSERT(snap);
1004

1005
    GLUSTERD_GET_SNAP_DIR(snapdirpath, snap, priv);
1006

1007
    ret = mkdir_p(snapdirpath, 0755, _gf_true);
1008
    if (ret) {
1009
        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
1010
               "Failed to create snaps dir "
1011
               "%s",
1012
               snapdirpath);
1013
    }
1014
    return ret;
1015
}
1016

1017
static int32_t
1018
glusterd_store_volinfo_write(int fd, glusterd_volinfo_t *volinfo)
1019
{
1020
    int32_t ret = -1;
1021
    gf_store_handle_t *shandle = NULL;
1022
    GF_ASSERT(fd > 0);
1023
    GF_ASSERT(volinfo);
1024
    GF_ASSERT(volinfo->shandle);
1025
    xlator_t *this = THIS;
1026
    glusterd_volinfo_data_store_t *dict_data = NULL;
1027

1028
    shandle = volinfo->shandle;
1029

1030
    dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t),
1031
                          gf_gld_mt_volinfo_dict_data_t);
1032
    if (dict_data == NULL) {
1033
        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL);
1034
        return -1;
1035
    }
1036

1037
    ret = glusterd_volume_exclude_options_write(fd, volinfo);
1038
    if (ret) {
1039
        goto out;
1040
    }
1041

1042
    dict_data->shandle = shandle;
1043
    dict_data->key_check = 1;
1044

1045
    shandle->fd = fd;
1046
    dict_foreach(volinfo->dict, _storeopts, (void *)dict_data);
1047

1048
    dict_data->key_check = 0;
1049
    dict_foreach(volinfo->gsync_secondaries, _storeopts, (void *)dict_data);
1050

1051
    if (dict_data->buffer_len > 0) {
1052
        ret = gf_store_save_items(fd, dict_data->buffer);
1053
        if (ret) {
1054
            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL);
1055
            goto out;
1056
        }
1057
    }
1058

1059
    shandle->fd = 0;
1060
out:
1061
    GF_FREE(dict_data);
1062
    gf_msg_debug(this->name, 0, "Returning %d", ret);
1063
    return ret;
1064
}
1065

1066
static int32_t
1067
glusterd_store_snapinfo_write(glusterd_snap_t *snap)
1068
{
1069
    int32_t ret = -1;
1070
    int fd = 0;
1071
    char buf[PATH_MAX];
1072
    uint total_len = 0;
1073

1074
    GF_ASSERT(snap);
1075

1076
    fd = gf_store_mkstemp(snap->shandle);
1077
    if (fd <= 0)
1078
        goto out;
1079

1080
    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
1081
                   "%s=%s\n%s=%d\n%s=%d\n", GLUSTERD_STORE_KEY_SNAP_ID,
1082
                   uuid_utoa(snap->snap_id), GLUSTERD_STORE_KEY_SNAP_STATUS,
1083
                   snap->snap_status, GLUSTERD_STORE_KEY_SNAP_RESTORED,
1084
                   snap->snap_restored);
1085
    if (ret < 0 || ret >= sizeof(buf) - total_len) {
1086
        ret = -1;
1087
        goto out;
1088
    }
1089
    total_len += ret;
1090

1091
    if (snap->description) {
1092
        ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n",
1093
                       GLUSTERD_STORE_KEY_SNAP_DESC, snap->description);
1094
        if (ret < 0 || ret >= sizeof(buf) - total_len) {
1095
            ret = -1;
1096
            goto out;
1097
        }
1098
        total_len += ret;
1099
    }
1100

1101
    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%ld\n",
1102
                   GLUSTERD_STORE_KEY_SNAP_TIMESTAMP, snap->time_stamp);
1103
    if (ret < 0 || ret >= sizeof(buf) - total_len) {
1104
        ret = -1;
1105
        goto out;
1106
    }
1107
    ret = gf_store_save_items(fd, buf);
1108

1109
out:
1110
    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
1111
    return ret;
1112
}
1113

1114
static int
1115
glusterd_store_volfpath_set(glusterd_volinfo_t *volinfo, char *volfpath,
1116
                            size_t len)
1117
{
1118
    char voldirpath[PATH_MAX] = {
1119
        0,
1120
    };
1121
    GF_ASSERT(volinfo);
1122
    GF_ASSERT(volfpath);
1123
    GF_ASSERT(len <= PATH_MAX);
1124
    int32_t ret = 0;
1125
    glusterd_store_voldirpath_set(volinfo, voldirpath);
1126
    ret = snprintf(volfpath, len, "%s/%s", voldirpath,
1127
                   GLUSTERD_VOLUME_INFO_FILE);
1128
    if (ret < 0 || ret >= len)
1129
        return -1;
1130
    return 0;
1131
}
1132

1133
static int
1134
glusterd_store_node_state_path_set(glusterd_volinfo_t *volinfo,
1135
                                   char *node_statepath, size_t len)
1136
{
1137
    char voldirpath[PATH_MAX] = {
1138
        0,
1139
    };
1140
    GF_ASSERT(volinfo);
1141
    GF_ASSERT(node_statepath);
1142
    GF_ASSERT(len <= PATH_MAX);
1143
    int32_t ret = 0;
1144
    glusterd_store_voldirpath_set(volinfo, voldirpath);
1145
    ret = snprintf(node_statepath, len, "%s/%s", voldirpath,
1146
                   GLUSTERD_NODE_STATE_FILE);
1147
    if (ret < 0 || ret >= len)
1148
        return -1;
1149
    return 0;
1150
}
1151

1152
static int
1153
glusterd_store_quota_conf_path_set(glusterd_volinfo_t *volinfo,
1154
                                   char *quota_conf_path, size_t len)
1155
{
1156
    char voldirpath[PATH_MAX] = {
1157
        0,
1158
    };
1159
    GF_ASSERT(volinfo);
1160
    GF_ASSERT(quota_conf_path);
1161
    GF_ASSERT(len <= PATH_MAX);
1162
    int32_t ret = 0;
1163
    glusterd_store_voldirpath_set(volinfo, voldirpath);
1164
    ret = snprintf(quota_conf_path, len, "%s/%s", voldirpath,
1165
                   GLUSTERD_VOLUME_QUOTA_CONFIG);
1166
    if (ret < 0 || ret >= len)
1167
        return -1;
1168
    return 0;
1169
}
1170

1171
static void
1172
glusterd_store_missed_snaps_list_path_set(char *missed_snaps_list, size_t len)
1173
{
1174
    glusterd_conf_t *priv = NULL;
1175

1176
    priv = THIS->private;
1177
    GF_ASSERT(priv);
1178
    GF_ASSERT(missed_snaps_list);
1179
    GF_ASSERT(len <= PATH_MAX);
1180

1181
    snprintf(missed_snaps_list, len,
1182
             "%s/snaps/" GLUSTERD_MISSED_SNAPS_LIST_FILE, priv->workdir);
1183
}
1184

1185
static void
1186
glusterd_store_snapfpath_set(glusterd_snap_t *snap, char *snap_fpath,
1187
                             size_t len)
1188
{
1189
    glusterd_conf_t *priv = NULL;
1190
    priv = THIS->private;
1191
    GF_ASSERT(priv);
1192
    GF_ASSERT(snap);
1193
    GF_ASSERT(snap_fpath);
1194
    GF_ASSERT(len <= PATH_MAX);
1195

1196
    snprintf(snap_fpath, len, "%s/snaps/%s/%s", priv->workdir, snap->snapname,
1197
             GLUSTERD_SNAP_INFO_FILE);
1198
}
1199

1200
int32_t
1201
glusterd_store_create_vol_shandle_on_absence(glusterd_volinfo_t *volinfo)
1202
{
1203
    char volfpath[PATH_MAX] = {0};
1204
    int32_t ret = 0;
1205

1206
    GF_ASSERT(volinfo);
1207

1208
    ret = glusterd_store_volfpath_set(volinfo, volfpath, sizeof(volfpath));
1209
    if (ret)
1210
        return ret;
1211
    ret = gf_store_handle_create_on_absence(&volinfo->shandle, volfpath);
1212
    return ret;
1213
}
1214

1215
int32_t
1216
glusterd_store_create_nodestate_sh_on_absence(glusterd_volinfo_t *volinfo)
1217
{
1218
    char node_state_path[PATH_MAX] = {0};
1219
    int32_t ret = 0;
1220

1221
    GF_ASSERT(volinfo);
1222

1223
    ret = glusterd_store_node_state_path_set(volinfo, node_state_path,
1224
                                             sizeof(node_state_path));
1225
    if (ret)
1226
        return ret;
1227
    ret = gf_store_handle_create_on_absence(&volinfo->node_state_shandle,
1228
                                            node_state_path);
1229

1230
    return ret;
1231
}
1232

1233
int32_t
1234
glusterd_store_create_quota_conf_sh_on_absence(glusterd_volinfo_t *volinfo)
1235
{
1236
    char quota_conf_path[PATH_MAX] = {0};
1237
    int32_t ret = 0;
1238

1239
    GF_ASSERT(volinfo);
1240

1241
    ret = glusterd_store_quota_conf_path_set(volinfo, quota_conf_path,
1242
                                             sizeof(quota_conf_path));
1243
    if (ret)
1244
        return ret;
1245
    ret = gf_store_handle_create_on_absence(&volinfo->quota_conf_shandle,
1246
                                            quota_conf_path);
1247

1248
    return ret;
1249
}
1250

1251
static int32_t
1252
glusterd_store_create_missed_snaps_list_shandle_on_absence(void)
1253
{
1254
    char missed_snaps_list[PATH_MAX] = "";
1255
    int32_t ret = -1;
1256
    glusterd_conf_t *priv = NULL;
1257
    xlator_t *this = THIS;
1258

1259
    priv = this->private;
1260
    GF_ASSERT(priv);
1261

1262
    glusterd_store_missed_snaps_list_path_set(missed_snaps_list,
1263
                                              sizeof(missed_snaps_list));
1264

1265
    ret = gf_store_handle_create_on_absence(&priv->missed_snaps_list_shandle,
1266
                                            missed_snaps_list);
1267
    return ret;
1268
}
1269

1270
int32_t
1271
glusterd_store_create_snap_shandle_on_absence(glusterd_snap_t *snap)
1272
{
1273
    char snapfpath[PATH_MAX] = {0};
1274
    int32_t ret = 0;
1275

1276
    GF_ASSERT(snap);
1277

1278
    glusterd_store_snapfpath_set(snap, snapfpath, sizeof(snapfpath));
1279
    ret = gf_store_handle_create_on_absence(&snap->shandle, snapfpath);
1280
    return ret;
1281
}
1282

1283
static int32_t
1284
glusterd_store_brickinfos(glusterd_volinfo_t *volinfo, int vol_fd)
1285
{
1286
    int32_t ret = 0;
1287
    glusterd_brickinfo_t *brickinfo = NULL;
1288
    glusterd_brickinfo_t *ta_brickinfo = NULL;
1289
    int32_t brick_count = 0;
1290
    int32_t ta_brick_count = 0;
1291

1292
    GF_ASSERT(volinfo);
1293

1294
    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
1295
    {
1296
        ret = glusterd_store_brickinfo(volinfo, brickinfo, brick_count, vol_fd,
1297
                                       0);
1298
        if (ret)
1299
            goto out;
1300
        brick_count++;
1301
    }
1302
    if (volinfo->thin_arbiter_count == 1) {
1303
        ta_brickinfo = list_first_entry(&volinfo->ta_bricks,
1304
                                        glusterd_brickinfo_t, brick_list);
1305
        ret = glusterd_store_brickinfo(volinfo, ta_brickinfo, ta_brick_count,
1306
                                       vol_fd, 1);
1307
        if (ret)
1308
            goto out;
1309
    }
1310

1311
out:
1312
    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
1313
    return ret;
1314
}
1315

1316
int32_t
1317
glusterd_store_node_state_write(int fd, glusterd_volinfo_t *volinfo)
1318
{
1319
    int ret = -1;
1320
    char buf[PATH_MAX];
1321
    char uuid[GF_UUID_BUF_SIZE];
1322
    uint total_len = 0;
1323
    glusterd_volinfo_data_store_t *dict_data = NULL;
1324
    gf_store_handle_t shandle;
1325
    xlator_t *this = THIS;
1326

1327
    GF_ASSERT(fd > 0);
1328
    GF_ASSERT(volinfo);
1329

1330
    if (volinfo->rebal.defrag_cmd == GF_DEFRAG_CMD_STATUS) {
1331
        ret = 0;
1332
        goto out;
1333
    }
1334

1335
    gf_uuid_unparse(volinfo->rebal.rebalance_id, uuid);
1336
    ret = snprintf(buf + total_len, sizeof(buf) - total_len,
1337
                   "%s=%d\n%s=%d\n%s=%d\n%s=%s\n",
1338
                   GLUSTERD_STORE_KEY_VOL_DEFRAG, volinfo->rebal.defrag_cmd,
1339
                   GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS,
1340
                   volinfo->rebal.defrag_status, GLUSTERD_STORE_KEY_DEFRAG_OP,
1341
                   volinfo->rebal.op, GF_REBALANCE_TID_KEY, uuid);
1342
    if (ret < 0 || ret >= sizeof(buf) - total_len) {
1343
        ret = -1;
1344
        goto out;
1345
    }
1346
    total_len += ret;
1347

1348
    ret = snprintf(
1349
        buf + total_len, sizeof(buf) - total_len,
1350
        "%s=%" PRIu64 "\n%s=%" PRIu64 "\n%s=%" PRIu64 "\n%s=%" PRIu64
1351
        "\n%s=%" PRIu64 "\n%s=%lf\n",
1352
        GLUSTERD_STORE_KEY_VOL_DEFRAG_REB_FILES, volinfo->rebal.rebalance_files,
1353
        GLUSTERD_STORE_KEY_VOL_DEFRAG_SIZE, volinfo->rebal.rebalance_data,
1354
        GLUSTERD_STORE_KEY_VOL_DEFRAG_SCANNED, volinfo->rebal.lookedup_files,
1355
        GLUSTERD_STORE_KEY_VOL_DEFRAG_FAILURES,
1356
        volinfo->rebal.rebalance_failures,
1357
        GLUSTERD_STORE_KEY_VOL_DEFRAG_SKIPPED, volinfo->rebal.skipped_files,
1358
        GLUSTERD_STORE_KEY_VOL_DEFRAG_RUN_TIME, volinfo->rebal.rebalance_time);
1359
    if (ret < 0 || ret >= sizeof(buf) - total_len) {
1360
        ret = -1;
1361
        goto out;
1362
    }
1363

1364
    ret = gf_store_save_items(fd, buf);
1365
    if (ret) {
1366
        goto out;
1367
    }
1368

1369
    if (volinfo->rebal.dict) {
1370
        dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t),
1371
                              gf_gld_mt_volinfo_dict_data_t);
1372
        if (dict_data == NULL) {
1373
            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL);
1374
            return -1;
1375
        }
1376
        dict_data->shandle = &shandle;
1377
        shandle.fd = fd;
1378
        dict_foreach(volinfo->rebal.dict, _storeopts, (void *)dict_data);
1379
        if (dict_data->buffer_len > 0) {
1380
            ret = gf_store_save_items(fd, dict_data->buffer);
1381
            if (ret) {
1382
                gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED,
1383
                        NULL);
1384
                goto out;
1385
                ;
1386
            }
1387
        }
1388
    }
1389
out:
1390
    GF_FREE(dict_data);
1391
    gf_msg_debug(this->name, 0, "Returning %d", ret);
1392
    return ret;
1393
}
1394

1395
int32_t
1396
glusterd_store_perform_node_state_store(glusterd_volinfo_t *volinfo)
1397
{
1398
    int fd = -1;
1399
    int32_t ret = -1;
1400
    GF_ASSERT(volinfo);
1401

1402
    fd = gf_store_mkstemp(volinfo->node_state_shandle);
1403
    if (fd <= 0) {
1404
        ret = -1;
1405
        goto out;
1406
    }
1407

1408
    ret = glusterd_store_node_state_write(fd, volinfo);
1409
    if (ret)
1410
        goto out;
1411

1412
    ret = gf_store_rename_tmppath(volinfo->node_state_shandle);
1413
    if (ret)
1414
        goto out;
1415

1416
out:
1417
    if (ret && (fd > 0))
1418
        gf_store_unlink_tmppath(volinfo->node_state_shandle);
1419
    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
1420
    return ret;
1421
}
1422

1423
static int32_t
1424
glusterd_store_perform_volume_store(glusterd_volinfo_t *volinfo)
1425
{
1426
    int fd = -1;
1427
    int32_t ret = -1;
1428
    GF_ASSERT(volinfo);
1429

1430
    fd = gf_store_mkstemp(volinfo->shandle);
1431
    if (fd <= 0) {
1432
        ret = -1;
1433
        goto out;
1434
    }
1435

1436
    ret = glusterd_store_volinfo_write(fd, volinfo);
1437
    if (ret)
1438
        goto out;
1439

1440
    ret = glusterd_store_create_brick_dir(volinfo);
1441
    if (ret)
1442
        goto out;
1443

1444
    ret = glusterd_store_brickinfos(volinfo, fd);
1445
    if (ret)
1446
        goto out;
1447

1448
out:
1449
    if (ret && (fd > 0))
1450
        gf_store_unlink_tmppath(volinfo->shandle);
1451
    gf_msg_debug(THIS->name, 0, "Returning %d", ret);
1452
    return ret;
1453
}
1454

1455
void
1456
glusterd_perform_volinfo_version_action(glusterd_volinfo_t *volinfo,
1457
                                        glusterd_volinfo_ver_ac_t ac)
1458
{
1459
    GF_ASSERT(volinfo);
1460

1461
    switch (ac) {
1462
        case GLUSTERD_VOLINFO_VER_AC_NONE:
1463
            break;
1464
        case GLUSTERD_VOLINFO_VER_AC_INCREMENT:
1465
            volinfo->version++;
1466
            break;
1467
        case GLUSTERD_VOLINFO_VER_AC_DECREMENT:
1468
            volinfo->version--;
1469
            break;
1470
    }
1471
}
1472

1473
void
1474
glusterd_store_bricks_cleanup_tmp(glusterd_volinfo_t *volinfo)
1475
{
1476
    glusterd_brickinfo_t *brickinfo = NULL;
1477

1478
    GF_ASSERT(volinfo);
1479

1480
    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
1481
    {
1482
        gf_store_unlink_tmppath(brickinfo->shandle);
1483
    }
1484
}
1485

1486
void
1487
glusterd_store_volume_cleanup_tmp(glusterd_volinfo_t *volinfo)
1488
{
1489
    GF_ASSERT(volinfo);
1490

1491
    glusterd_store_bricks_cleanup_tmp(volinfo);
1492

1493
    gf_store_unlink_tmppath(volinfo->shandle);
1494

1495
    gf_store_unlink_tmppath(volinfo->node_state_shandle);
1496

1497
    gf_store_unlink_tmppath(volinfo->snapd.handle);
1498
}
1499

1500
int32_t
1501
glusterd_store_brickinfos_atomic_update(glusterd_volinfo_t *volinfo)
1502
{
1503
    int ret = -1;
1504
    glusterd_brickinfo_t *brickinfo = NULL;
1505
    glusterd_brickinfo_t *ta_brickinfo = NULL;
1506

1507
    GF_ASSERT(volinfo);
1508

1509
    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
1510
    {
1511
        ret = gf_store_rename_tmppath(brickinfo->shandle);
1512
        if (ret)
1513
            goto out;
1514
    }
1515

1516
    if (volinfo->thin_arbiter_count == 1) {
1517
        ta_brickinfo = list_first_entry(&volinfo->ta_bricks,
1518
                                        glusterd_brickinfo_t, brick_list);
1519
        ret = gf_store_rename_tmppath(ta_brickinfo->shandle);
1520
        if (ret)
1521
            goto out;
1522
    }
1523

1524
out:
1525
    return ret;
1526
}
1527

1528
int32_t
1529
glusterd_store_volinfo_atomic_update(glusterd_volinfo_t *volinfo)
1530
{
1531
    int ret = -1;
1532
    GF_ASSERT(volinfo);
1533

1534
    ret = gf_store_rename_tmppath(volinfo->shandle);
1535
    if (ret)
1536
        goto out;
1537

1538
out:
1539
    if (ret)
1540
        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
1541
               "Couldn't rename "
1542
               "temporary file(s)");
1543
    return ret;
1544
}
1545

1546
int32_t
1547
glusterd_store_volume_atomic_update(glusterd_volinfo_t *volinfo)
1548
{
1549
    int ret = -1;
1550
    GF_ASSERT(volinfo);
1551

1552
    ret = glusterd_store_brickinfos_atomic_update(volinfo);
1553
    if (ret)
1554
        goto out;
1555

1556
    ret = glusterd_store_volinfo_atomic_update(volinfo);
1557

1558
out:
1559
    return ret;
1560
}
1561

1562
int32_t
1563
glusterd_store_snap_atomic_update(glusterd_snap_t *snap)
1564
{
1565
    int ret = -1;
1566
    GF_ASSERT(snap);
1567

1568
    ret = gf_store_rename_tmppath(snap->shandle);
1569
    if (ret)
1570
        gf_msg(THIS->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
1571
               "Couldn't rename "
1572
               "temporary file(s)");
1573

1574
    return ret;
1575
}
1576

1577
int32_t
1578
glusterd_store_snap(glusterd_snap_t *snap)
1579
{
1580
    int32_t ret = -1;
1581

1582
    GF_ASSERT(snap);
1583

1584
    ret = glusterd_store_create_snap_dir(snap);
1585
    if (ret) {
1586
        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPDIR_CREATE_FAIL,
1587
               "Failed to create snap dir");
1588
        goto out;
1589
    }
1590

1591
    ret = glusterd_store_create_snap_shandle_on_absence(snap);
1592
    if (ret) {
1593
        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPINFO_CREATE_FAIL,
1594
               "Failed to create snap info "
1595
               "file");
1596
        goto out;
1597
    }
1598

1599
    ret = glusterd_store_snapinfo_write(snap);
1600
    if (ret) {
1601
        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAPINFO_WRITE_FAIL,
1602
               "Failed to write snap info");
1603
        goto out;
1604
    }
1605

1606
    ret = glusterd_store_snap_atomic_update(snap);
1607
    if (ret) {
1608
        gf_msg(THIS->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_AUTOMIC_UPDATE_FAIL,
1609
               "Failed to do automic update");
1610
        goto out;
1611
    }
1612

1613
out:
1614
    if (ret && snap->shandle)
1615
        gf_store_unlink_tmppath(snap->shandle);
1616

1617
    gf_msg_trace(THIS->name, 0, "Returning %d", ret);
1618
    return ret;
1619
}
1620

1621
int32_t
1622
glusterd_store_volinfo(glusterd_volinfo_t *volinfo,
1623
                       glusterd_volinfo_ver_ac_t ac)
1624
{
1625
    int32_t ret = -1;
1626
    glusterfs_ctx_t *ctx = NULL;
1627
    xlator_t *this = THIS;
1628

1629
    ctx = this->ctx;
1630
    GF_ASSERT(ctx);
1631
    GF_ASSERT(volinfo);
1632

1633
    pthread_mutex_lock(&ctx->cleanup_lock);
1634
    pthread_mutex_lock(&volinfo->store_volinfo_lock);
1635
    {
1636
        glusterd_perform_volinfo_version_action(volinfo, ac);
1637

1638
        ret = glusterd_store_create_volume_dirs(volinfo);
1639
        if (ret)
1640
            goto unlock;
1641

1642
        ret = glusterd_store_create_vol_shandle_on_absence(volinfo);
1643
        if (ret)
1644
            goto unlock;
1645

1646
        ret = glusterd_store_create_nodestate_sh_on_absence(volinfo);
1647
        if (ret)
1648
            goto unlock;
1649

1650
        ret = glusterd_store_perform_volume_store(volinfo);
1651
        if (ret)
1652
            goto unlock;
1653

1654
        ret = glusterd_store_volume_atomic_update(volinfo);
1655
        if (ret) {
1656
            glusterd_perform_volinfo_version_action(
1657
                volinfo, GLUSTERD_VOLINFO_VER_AC_DECREMENT);
1658
            goto unlock;
1659
        }
1660

1661
        ret = glusterd_store_perform_node_state_store(volinfo);
1662
        if (ret)
1663
            goto unlock;
1664

1665
        /* checksum should be computed at the end */
1666
        ret = glusterd_compute_cksum(volinfo, _gf_false);
1667
        if (ret)
1668
            goto unlock;
1669
    }
1670
unlock:
1671
    pthread_mutex_unlock(&volinfo->store_volinfo_lock);
1672
    pthread_mutex_unlock(&ctx->cleanup_lock);
1673

1674
    if (ret)
1675
        glusterd_store_volume_cleanup_tmp(volinfo);
1676

1677
    gf_msg_debug(this->name, 0, "Returning %d", ret);
1678

1679
    return ret;
1680
}
1681

1682
int32_t
1683
glusterd_store_delete_volume(glusterd_volinfo_t *volinfo)
1684
{
1685
    char pathname[PATH_MAX] = {
1686
        0,
1687
    };
1688
    int32_t ret = 0;
1689
    glusterd_conf_t *priv = NULL;
1690
    char delete_path[PATH_MAX] = {
1691
        0,
1692
    };
1693
    char trashdir[PATH_MAX] = {
1694
        0,
1695
    };
1696
    xlator_t *this = THIS;
1697
    gf_boolean_t rename_fail = _gf_false;
1698
    int32_t len = 0;
1699

1700
    GF_ASSERT(volinfo);
1701
    priv = this->private;
1702

1703
    GF_ASSERT(priv);
1704

1705
    GLUSTERD_GET_VOLUME_DIR(pathname, volinfo, priv);
1706

1707
    len = snprintf(delete_path, sizeof(delete_path),
1708
                   "%s/" GLUSTERD_TRASH "/%s.deleted", priv->workdir,
1709
                   uuid_utoa(volinfo->volume_id));
1710
    if ((len < 0) || (len >= sizeof(delete_path))) {
1711
        goto out;
1712
    }
1713

1714
    len = snprintf(trashdir, sizeof(trashdir), "%s/" GLUSTERD_TRASH,
1715
                   priv->workdir);
1716
    if ((len < 0) || (len >= sizeof(trashdir))) {
1717
        goto out;
1718
    }
1719

1720
    ret = sys_mkdir(trashdir, 0755);
1721
    if (ret && errno != EEXIST) {
1722
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
1723
               "Failed to create trash "
1724
               "directory");
1725
        goto out;
1726
    }
1727

1728
    ret = sys_rename(pathname, delete_path);
1729
    if (ret) {
1730
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
1731
               "Failed to rename volume "
1732
               "directory for volume %s",
1733
               volinfo->volname);
1734
        rename_fail = _gf_true;
1735
        goto out;
1736
    }
1737

1738
    ret = recursive_rmdir(trashdir);
1739
    if (ret) {
1740
        gf_msg_debug(this->name, 0, "Failed to rmdir: %s", trashdir);
1741
    }
1742

1743
out:
1744
    if (volinfo->shandle) {
1745
        gf_store_handle_destroy(volinfo->shandle);
1746
        volinfo->shandle = NULL;
1747
    }
1748
    ret = (rename_fail == _gf_true) ? -1 : 0;
1749

1750
    gf_msg_debug(this->name, 0, "Returning %d", ret);
1751
    return ret;
1752
}
1753

1754
/*TODO: cleanup the duplicate code and implement a generic function for
1755
 * deleting snap/volume depending on the parameter flag */
1756
int32_t
1757
glusterd_store_delete_snap(glusterd_snap_t *snap)
1758
{
1759
    char pathname[PATH_MAX] = {
1760
        0,
1761
    };
1762
    int32_t ret = 0;
1763
    glusterd_conf_t *priv = NULL;
1764
    DIR *dir = NULL;
1765
    struct dirent *entry = NULL;
1766
    struct dirent scratch[2] = {
1767
        {
1768
            0,
1769
        },
1770
    };
1771
    char path[PATH_MAX] = {
1772
        0,
1773
    };
1774
    char delete_path[PATH_MAX] = {
1775
        0,
1776
    };
1777
    char trashdir[PATH_MAX] = {
1778
        0,
1779
    };
1780
    struct stat st = {
1781
        0,
1782
    };
1783
    xlator_t *this = THIS;
1784
    gf_boolean_t rename_fail = _gf_false;
1785
    int32_t len = 0;
1786

1787
    priv = this->private;
1788
    GF_ASSERT(priv);
1789

1790
    GF_ASSERT(snap);
1791
    GLUSTERD_GET_SNAP_DIR(pathname, snap, priv);
1792

1793
    len = snprintf(delete_path, sizeof(delete_path),
1794
                   "%s/" GLUSTERD_TRASH "/snap-%s.deleted", priv->workdir,
1795
                   uuid_utoa(snap->snap_id));
1796
    if ((len < 0) || (len >= sizeof(delete_path))) {
1797
        goto out;
1798
    }
1799

1800
    len = snprintf(trashdir, sizeof(trashdir), "%s/" GLUSTERD_TRASH,
1801
                   priv->workdir);
1802
    if ((len < 0) || (len >= sizeof(trashdir))) {
1803
        goto out;
1804
    }
1805

1806
    ret = sys_mkdir(trashdir, 0755);
1807
    if (ret && errno != EEXIST) {
1808
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
1809
               "Failed to create trash "
1810
               "directory");
1811
        goto out;
1812
    }
1813

1814
    ret = sys_rename(pathname, delete_path);
1815
    if (ret) {
1816
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
1817
               "Failed to rename snap "
1818
               "directory %s to %s",
1819
               pathname, delete_path);
1820
        rename_fail = _gf_true;
1821
        goto out;
1822
    }
1823

1824
    dir = sys_opendir(delete_path);
1825
    if (!dir) {
1826
        gf_msg_debug(this->name, 0, "Failed to open directory %s.",
1827
                     delete_path);
1828
        goto out;
1829
    }
1830

1831
    while ((entry = sys_readdir(dir, scratch))) {
1832
        if (gf_irrelevant_entry(entry))
1833
            continue;
1834
        len = snprintf(path, PATH_MAX, "%s/%s", delete_path, entry->d_name);
1835
        if ((len < 0) || (len >= PATH_MAX)) {
1836
            goto stat_failed;
1837
        }
1838
        ret = sys_stat(path, &st);
1839
        if (ret == -1) {
1840
            gf_msg_debug(this->name, 0,
1841
                         "Failed to stat "
1842
                         "entry %s",
1843
                         path);
1844
            goto stat_failed;
1845
        }
1846

1847
        if (S_ISDIR(st.st_mode))
1848
            ret = sys_rmdir(path);
1849
        else
1850
            ret = sys_unlink(path);
1851

1852
        if (ret) {
1853
            gf_msg_debug(this->name, 0,
1854
                         " Failed to remove "
1855
                         "%s",
1856
                         path);
1857
        }
1858

1859
        gf_msg_debug(this->name, 0, "%s %s",
1860
                     ret ? "Failed to remove" : "Removed", entry->d_name);
1861
    stat_failed:
1862
        memset(path, 0, sizeof(path));
1863
    }
1864

1865
    ret = sys_closedir(dir);
1866
    if (ret) {
1867
        gf_msg_debug(this->name, 0, "Failed to close dir %s.", delete_path);
1868
    }
1869

1870
    ret = sys_rmdir(delete_path);
1871
    if (ret) {
1872
        gf_msg_debug(this->name, 0, "Failed to rmdir: %s", delete_path);
1873
    }
1874
    ret = sys_rmdir(trashdir);
1875
    if (ret) {
1876
        gf_msg_debug(this->name, 0, "Failed to rmdir: %s", trashdir);
1877
    }
1878

1879
out:
1880
    if (snap->shandle) {
1881
        gf_store_handle_destroy(snap->shandle);
1882
        snap->shandle = NULL;
1883
    }
1884
    ret = (rename_fail == _gf_true) ? -1 : 0;
1885

1886
    gf_msg_debug(this->name, 0, "Returning %d", ret);
1887
    return ret;
1888
}
1889

1890
int
1891
glusterd_store_global_info(xlator_t *this)
1892
{
1893
    int ret = -1;
1894
    glusterd_conf_t *conf = NULL;
1895
    char buf[PATH_MAX];
1896
    uint total_len = 0;
1897
    gf_store_handle_t *handle = NULL;
1898
    char *uuid_str = NULL;
1899

1900
    conf = this->private;
1901

1902
    uuid_str = gf_strdup(uuid_utoa(MY_UUID));
1903
    if (!uuid_str)
1904
        goto out;
1905

1906
    if (!conf->handle) {
1907
        ret = snprintf(buf, sizeof(buf), "%s/%s", conf->workdir,
1908
                       GLUSTERD_INFO_FILE);
1909
        if ((ret < 0) || (ret >= sizeof(buf))) {
1910
            ret = -1;
1911
            goto out;
1912
        }
1913
        ret = gf_store_handle_new(buf, &handle);
1914
        if (ret) {
1915
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_GET_FAIL,
1916
                   "Unable to get store handle");
1917
            goto out;
1918
        }
1919

1920
        conf->handle = handle;
1921
    } else
1922
        handle = conf->handle;
1923

1924
    /* These options need to be available for all users */
1925
    ret = sys_chmod(handle->path, 0644);
1926
    if (ret) {
1927
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
1928
               "chmod error for %s", GLUSTERD_INFO_FILE);
1929
        goto out;
1930
    }
1931

1932
    handle->fd = gf_store_mkstemp(handle);
1933
    if (handle->fd < 0) {
1934
        ret = -1;
1935
        goto out;
1936
    }
1937

1938
    ret = snprintf(buf, sizeof(buf), "%s=%s\n", GLUSTERD_STORE_UUID_KEY,
1939
                   uuid_str);
1940
    if (ret < 0 || ret >= sizeof(buf)) {
1941
        ret = -1;
1942
        goto out;
1943
    }
1944
    total_len += ret;
1945

1946
    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%d\n",
1947
                   GD_OP_VERSION_KEY, conf->op_version);
1948
    if (ret < 0 || ret >= sizeof(buf) - total_len) {
1949
        ret = -1;
1950
        goto out;
1951
    }
1952

1953
    ret = gf_store_save_items(handle->fd, buf);
1954
    if (ret) {
1955
        gf_msg(this->name, GF_LOG_CRITICAL, 0, GD_MSG_OP_VERS_STORE_FAIL,
1956
               "Storing glusterd global-info failed ret = %d", ret);
1957
        goto out;
1958
    }
1959

1960
    ret = gf_store_rename_tmppath(handle);
1961
out:
1962
    if (handle) {
1963
        if (ret && (handle->fd >= 0))
1964
            gf_store_unlink_tmppath(handle);
1965
    }
1966

1967
    if (uuid_str)
1968
        GF_FREE(uuid_str);
1969

1970
    if (ret)
1971
        gf_msg(this->name, GF_LOG_ERROR, 0,
1972
               GD_MSG_GLUSTERD_GLOBAL_INFO_STORE_FAIL,
1973
               "Failed to store glusterd global-info");
1974

1975
    return ret;
1976
}
1977

1978
int
1979
glusterd_store_max_op_version(xlator_t *this)
1980
{
1981
    int ret = -1;
1982
    glusterd_conf_t *conf = NULL;
1983
    char op_version_str[15] = {
1984
        0,
1985
    };
1986
    char path[PATH_MAX] = {
1987
        0,
1988
    };
1989
    gf_store_handle_t *handle = NULL;
1990
    int32_t len = 0;
1991

1992
    conf = this->private;
1993

1994
    len = snprintf(path, PATH_MAX, "%s/%s", conf->workdir,
1995
                   GLUSTERD_UPGRADE_FILE);
1996
    if ((len < 0) || (len >= PATH_MAX)) {
1997
        goto out;
1998
    }
1999
    ret = gf_store_handle_new(path, &handle);
2000
    if (ret) {
2001
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_GET_FAIL,
2002
               "Unable to get store handle");
2003
        goto out;
2004
    }
2005

2006
    /* These options need to be available for all users */
2007
    ret = sys_chmod(handle->path, 0644);
2008
    if (ret) {
2009
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
2010
               "chmod error for %s", GLUSTERD_UPGRADE_FILE);
2011
        goto out;
2012
    }
2013

2014
    handle->fd = gf_store_mkstemp(handle);
2015
    if (handle->fd < 0) {
2016
        ret = -1;
2017
        goto out;
2018
    }
2019

2020
    snprintf(op_version_str, sizeof(op_version_str), "%d", GD_OP_VERSION_MAX);
2021
    ret = gf_store_save_value(handle->fd, GD_MAX_OP_VERSION_KEY,
2022
                              op_version_str);
2023
    if (ret) {
2024
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_OP_VERS_STORE_FAIL,
2025
               "Storing op-version failed ret = %d", ret);
2026
        goto out;
2027
    }
2028

2029
    ret = gf_store_rename_tmppath(handle);
2030
out:
2031
    if (handle) {
2032
        if (ret && (handle->fd >= 0))
2033
            gf_store_unlink_tmppath(handle);
2034
    }
2035

2036
    if (ret)
2037
        gf_msg(this->name, GF_LOG_ERROR, 0,
2038
               GD_MSG_GLUSTERD_GLOBAL_INFO_STORE_FAIL,
2039
               "Failed to store max op-version");
2040
    if (handle)
2041
        gf_store_handle_destroy(handle);
2042
    return ret;
2043
}
2044

2045
int
2046
glusterd_retrieve_max_op_version(xlator_t *this, int *op_version)
2047
{
2048
    char *op_version_str = NULL;
2049
    glusterd_conf_t *priv = NULL;
2050
    int ret = -1;
2051
    int tmp_version = 0;
2052
    char *tmp = NULL;
2053
    char path[PATH_MAX] = {
2054
        0,
2055
    };
2056
    gf_store_handle_t *handle = NULL;
2057
    int32_t len = 0;
2058

2059
    priv = this->private;
2060

2061
    len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
2062
                   GLUSTERD_UPGRADE_FILE);
2063
    if ((len < 0) || (len >= PATH_MAX)) {
2064
        goto out;
2065
    }
2066
    ret = gf_store_handle_retrieve(path, &handle);
2067

2068
    if (ret) {
2069
        gf_msg_debug(this->name, 0,
2070
                     "Unable to get store "
2071
                     "handle!");
2072
        goto out;
2073
    }
2074

2075
    ret = gf_store_retrieve_value(handle, GD_MAX_OP_VERSION_KEY,
2076
                                  &op_version_str);
2077
    if (ret) {
2078
        gf_msg_debug(this->name, 0, "No previous op_version present");
2079
        goto out;
2080
    }
2081

2082
    tmp_version = strtol(op_version_str, &tmp, 10);
2083
    if ((tmp_version <= 0) || (tmp && strlen(tmp) > 1)) {
2084
        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_UNSUPPORTED_VERSION,
2085
               "invalid version number");
2086
        goto out;
2087
    }
2088

2089
    *op_version = tmp_version;
2090

2091
    ret = 0;
2092
out:
2093
    if (op_version_str)
2094
        GF_FREE(op_version_str);
2095
    if (handle)
2096
        gf_store_handle_destroy(handle);
2097
    return ret;
2098
}
2099

2100
int
2101
glusterd_retrieve_op_version(xlator_t *this, int *op_version)
2102
{
2103
    char *op_version_str = NULL;
2104
    glusterd_conf_t *priv = NULL;
2105
    int ret = -1;
2106
    int tmp_version = 0;
2107
    char *tmp = NULL;
2108
    char path[PATH_MAX] = {
2109
        0,
2110
    };
2111
    gf_store_handle_t *handle = NULL;
2112
    int32_t len = 0;
2113

2114
    priv = this->private;
2115

2116
    if (!priv->handle) {
2117
        len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
2118
                       GLUSTERD_INFO_FILE);
2119
        if ((len < 0) || (len >= PATH_MAX)) {
2120
            goto out;
2121
        }
2122
        ret = gf_store_handle_retrieve(path, &handle);
2123

2124
        if (ret) {
2125
            gf_msg_debug(this->name, 0,
2126
                         "Unable to get store "
2127
                         "handle!");
2128
            goto out;
2129
        }
2130

2131
        priv->handle = handle;
2132
    }
2133

2134
    ret = gf_store_retrieve_value(priv->handle, GD_OP_VERSION_KEY,
2135
                                  &op_version_str);
2136
    if (ret) {
2137
        gf_msg_debug(this->name, 0, "No previous op_version present");
2138
        goto out;
2139
    }
2140

2141
    tmp_version = strtol(op_version_str, &tmp, 10);
2142
    if ((tmp_version <= 0) || (tmp && strlen(tmp) > 1)) {
2143
        gf_msg(this->name, GF_LOG_WARNING, EINVAL, GD_MSG_UNSUPPORTED_VERSION,
2144
               "invalid version number");
2145
        goto out;
2146
    }
2147

2148
    *op_version = tmp_version;
2149

2150
    ret = 0;
2151
out:
2152
    if (op_version_str)
2153
        GF_FREE(op_version_str);
2154

2155
    return ret;
2156
}
2157

2158
int
2159
glusterd_restore_op_version(xlator_t *this)
2160
{
2161
    glusterd_conf_t *conf = NULL;
2162
    int ret = 0;
2163
    int op_version = 0;
2164

2165
    conf = this->private;
2166

2167
    ret = glusterd_retrieve_op_version(this, &op_version);
2168
    if (!ret) {
2169
        if ((op_version < GD_OP_VERSION_MIN) ||
2170
            (op_version > GD_OP_VERSION_MAX)) {
2171
            gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_UNSUPPORTED_VERSION,
2172
                   "wrong op-version (%d) retrieved", op_version);
2173
            ret = -1;
2174
            goto out;
2175
        }
2176
        conf->op_version = op_version;
2177
        gf_msg("glusterd", GF_LOG_INFO, 0, GD_MSG_OP_VERS_INFO,
2178
               "retrieved op-version: %d", conf->op_version);
2179
        goto out;
2180
    }
2181

2182
    /* op-version can be missing from the store file in 2 cases,
2183
     * 1. This is a new install of glusterfs
2184
     * 2. This is an upgrade of glusterfs from a version without op-version
2185
     *    to a version with op-version (eg. 3.3 -> 3.4)
2186
     *
2187
     * Detection of a new install or an upgrade from an older install can be
2188
     * done by checking for the presence of the its peer-id in the store
2189
     * file.  If peer-id is present, the installation is an upgrade else, it
2190
     * is a new install.
2191
     *
2192
     * For case 1, set op-version to GD_OP_VERSION_MAX.
2193
     * For case 2, set op-version to GD_OP_VERSION_MIN.
2194
     */
2195
    ret = glusterd_retrieve_uuid();
2196
    if (ret) {
2197
        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_OP_VERS_SET_INFO,
2198
               "Detected new install. Setting"
2199
               " op-version to maximum : %d",
2200
               GD_OP_VERSION_MAX);
2201
        conf->op_version = GD_OP_VERSION_MAX;
2202
    } else {
2203
        gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_OP_VERS_SET_INFO,
2204
               "Upgrade detected. Setting"
2205
               " op-version to minimum : %d",
2206
               GD_OP_VERSION_MIN);
2207
        conf->op_version = GD_OP_VERSION_MIN;
2208
    }
2209
    ret = 0;
2210
out:
2211
    return ret;
2212
}
2213

2214
int32_t
2215
glusterd_retrieve_uuid(void)
2216
{
2217
    char *uuid_str = NULL;
2218
    int32_t ret = -1;
2219
    gf_store_handle_t *handle = NULL;
2220
    glusterd_conf_t *priv = NULL;
2221
    xlator_t *this = THIS;
2222
    char path[PATH_MAX] = {
2223
        0,
2224
    };
2225
    int32_t len = 0;
2226

2227
    priv = this->private;
2228

2229
    if (!priv->handle) {
2230
        len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
2231
                       GLUSTERD_INFO_FILE);
2232
        if ((len < 0) || (len >= PATH_MAX)) {
2233
            goto out;
2234
        }
2235
        ret = gf_store_handle_retrieve(path, &handle);
2236

2237
        if (ret) {
2238
            gf_msg_debug(this->name, 0,
2239
                         "Unable to get store"
2240
                         "handle!");
2241
            goto out;
2242
        }
2243

2244
        priv->handle = handle;
2245
    }
2246
    pthread_mutex_lock(&priv->mutex);
2247
    {
2248
        ret = gf_store_retrieve_value(priv->handle, GLUSTERD_STORE_UUID_KEY,
2249
                                      &uuid_str);
2250
    }
2251
    pthread_mutex_unlock(&priv->mutex);
2252
    if (ret) {
2253
        gf_msg_debug(this->name, 0, "No previous uuid is present");
2254
        goto out;
2255
    }
2256

2257
    gf_uuid_parse(uuid_str, priv->uuid);
2258

2259
out:
2260
    GF_FREE(uuid_str);
2261
    gf_msg_debug(this->name, 0, "Returning %d", ret);
2262
    return ret;
2263
}
2264

2265
int
2266
glusterd_store_retrieve_snapd(glusterd_volinfo_t *volinfo)
2267
{
2268
    int ret = -1;
2269
    char *key = NULL;
2270
    char *value = NULL;
2271
    char volpath[PATH_MAX] = {
2272
        0,
2273
    };
2274
    char path[PATH_MAX] = {
2275
        0,
2276
    };
2277
    xlator_t *this = THIS;
2278
    glusterd_conf_t *conf = NULL;
2279
    gf_store_iter_t *iter = NULL;
2280
    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
2281
    int32_t len = 0;
2282

2283
    conf = this->private;
2284
    GF_ASSERT(volinfo);
2285

2286
    /*
2287
     * This is needed for upgrade situations. Say a volume is created with
2288
     * older version of glusterfs and upgraded to a glusterfs version equal
2289
     * to or greater than GD_OP_VERSION_3_6_0. The older glusterd would not
2290
     * have created the snapd.info file related to snapshot daemon for user
2291
     * serviceable snapshots. So as part of upgrade when the new glusterd
2292
     * starts, as part of restore (restoring the volume to be precise), it
2293
     * tries to snapd related info from snapd.info file. But since there was
2294
     * no such file till now, the restore operation fails. Thus, to prevent
2295
     * it from happening check whether user serviceable snapshots features
2296
     * is enabled before restoring snapd. If its disabled, then simply
2297
     * exit by returning success (without even checking for the snapd.info).
2298
     */
2299

2300
    if (!dict_get_str_boolean(volinfo->dict, "features.uss", _gf_false)) {
2301
        ret = 0;
2302
        goto out;
2303
    }
2304

2305
    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
2306

2307
    len = snprintf(path, sizeof(path), "%s/%s", volpath,
2308
                   GLUSTERD_VOLUME_SNAPD_INFO_FILE);
2309
    if ((len < 0) || (len >= sizeof(path))) {
2310
        goto out;
2311
    }
2312

2313
    ret = gf_store_handle_retrieve(path, &volinfo->snapd.handle);
2314
    if (ret) {
2315
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDLE_NULL,
2316
               "volinfo handle is NULL");
2317
        goto out;
2318
    }
2319

2320
    ret = gf_store_iter_new(volinfo->snapd.handle, &iter);
2321
    if (ret) {
2322
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
2323
               "Failed to get new store "
2324
               "iter");
2325
        goto out;
2326
    }
2327

2328
    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
2329
    if (ret) {
2330
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
2331
               "Failed to get next store "
2332
               "iter");
2333
        goto out;
2334
    }
2335

2336
    while (!ret) {
2337
        if (!strncmp(key, GLUSTERD_STORE_KEY_SNAPD_PORT,
2338
                     SLEN(GLUSTERD_STORE_KEY_SNAPD_PORT))) {
2339
            volinfo->snapd.port = atoi(value);
2340
        }
2341
        GF_FREE(key);
2342
        GF_FREE(value);
2343
        key = NULL;
2344
        value = NULL;
2345

2346
        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
2347
    }
2348

2349
    if (op_errno != GD_STORE_EOF)
2350
        goto out;
2351

2352
    ret = 0;
2353

2354
out:
2355
    if (gf_store_iter_destroy(&iter)) {
2356
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
2357
               "Failed to destroy store iter");
2358
        ret = -1;
2359
    }
2360

2361
    return ret;
2362
}
2363

2364
int32_t
2365
glusterd_store_retrieve_bricks(glusterd_volinfo_t *volinfo)
2366
{
2367
    int32_t ret = 0;
2368
    glusterd_brickinfo_t *brickinfo = NULL;
2369
    glusterd_brickinfo_t *ta_brickinfo = NULL;
2370
    gf_store_iter_t *iter = NULL;
2371
    char *key = NULL;
2372
    char *value = NULL;
2373
    char brickdir[PATH_MAX] = {
2374
        0,
2375
    };
2376
    char path[PATH_MAX] = {
2377
        0,
2378
    };
2379
    glusterd_conf_t *priv = NULL;
2380
    int32_t brick_count = 0;
2381
    int32_t ta_brick_count = 0;
2382
    char tmpkey[32] = {
2383
        0,
2384
    };
2385
    gf_store_iter_t *tmpiter = NULL;
2386
    char *tmpvalue = NULL;
2387
    char abspath[PATH_MAX] = {0};
2388
    xlator_t *this = THIS;
2389
    int brickid = 0;
2390
    /* ta_brick_id initialization with 2 since ta-brick id starts with
2391
     * volname-ta-2
2392
     */
2393
    int ta_brick_id = 2;
2394
    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
2395
    int32_t len = 0;
2396

2397
    GF_ASSERT(volinfo);
2398
    GF_ASSERT(volinfo->volname);
2399

2400
    priv = this->private;
2401

2402
    GLUSTERD_GET_BRICK_DIR(brickdir, volinfo, priv);
2403

2404
    ret = gf_store_iter_new(volinfo->shandle, &tmpiter);
2405

2406
    if (ret)
2407
        goto out;
2408

2409
    while (brick_count < volinfo->brick_count) {
2410
        ret = glusterd_brickinfo_new(&brickinfo);
2411

2412
        if (ret)
2413
            goto out;
2414
        snprintf(tmpkey, sizeof(tmpkey), "%s-%d", GLUSTERD_STORE_KEY_VOL_BRICK,
2415
                 brick_count);
2416
        ret = gf_store_iter_get_matching(tmpiter, tmpkey, &tmpvalue);
2417
        len = snprintf(path, sizeof(path), "%s/%s", brickdir, tmpvalue);
2418
        GF_FREE(tmpvalue);
2419
        tmpvalue = NULL;
2420
        if ((len < 0) || (len >= sizeof(path))) {
2421
            ret = -1;
2422
            goto out;
2423
        }
2424

2425
        ret = gf_store_handle_retrieve(path, &brickinfo->shandle);
2426

2427
        if (ret)
2428
            goto out;
2429

2430
        ret = gf_store_iter_new(brickinfo->shandle, &iter);
2431

2432
        if (ret)
2433
            goto out;
2434

2435
        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
2436
        if (ret) {
2437
            gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2438
                   GD_MSG_STORE_ITER_GET_FAIL,
2439
                   "Unable to iterate "
2440
                   "the store for brick: %s",
2441
                   path);
2442
            goto out;
2443
        }
2444
        while (!ret) {
2445
            if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
2446
                         SLEN(GLUSTERD_STORE_KEY_BRICK_HOSTNAME))) {
2447
                if (snprintf(brickinfo->hostname, sizeof(brickinfo->hostname),
2448
                             "%s", value) >= sizeof(brickinfo->hostname)) {
2449
                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2450
                           GD_MSG_PARSE_BRICKINFO_FAIL,
2451
                           "brick hostname truncated: %s", brickinfo->hostname);
2452
                    goto out;
2453
                }
2454
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PATH,
2455
                                SLEN(GLUSTERD_STORE_KEY_BRICK_PATH))) {
2456
                if (snprintf(brickinfo->path, sizeof(brickinfo->path), "%s",
2457
                             value) >= sizeof(brickinfo->path)) {
2458
                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2459
                           GD_MSG_PARSE_BRICKINFO_FAIL,
2460
                           "brick path truncated: %s", brickinfo->path);
2461
                    goto out;
2462
                }
2463
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_REAL_PATH,
2464
                                SLEN(GLUSTERD_STORE_KEY_BRICK_REAL_PATH))) {
2465
                if (snprintf(brickinfo->real_path, sizeof(brickinfo->real_path),
2466
                             "%s", value) >= sizeof(brickinfo->real_path)) {
2467
                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2468
                           GD_MSG_PARSE_BRICKINFO_FAIL,
2469
                           "real_path truncated: %s", brickinfo->real_path);
2470
                    goto out;
2471
                }
2472
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PORT,
2473
                                SLEN(GLUSTERD_STORE_KEY_BRICK_PORT))) {
2474
                ret = gf_string2int(value, &brickinfo->port);
2475
                if (ret == -1) {
2476
                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
2477
                           GD_MSG_INCOMPATIBLE_VALUE,
2478
                           "Failed to convert "
2479
                           "string to integer");
2480
                }
2481

2482
                if (brickinfo->port < priv->base_port) {
2483
                    /* This is required to adhere to the
2484
                       IANA standards */
2485
                    brickinfo->port = 0;
2486
                }
2487
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
2488
                                SLEN(GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
2489
                ret = gf_string2int(value, &brickinfo->rdma_port);
2490
                if (ret == -1) {
2491
                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
2492
                           GD_MSG_INCOMPATIBLE_VALUE,
2493
                           "Failed to convert "
2494
                           "string to integer");
2495
                }
2496

2497
                if (brickinfo->rdma_port < priv->base_port) {
2498
                    /* This is required to adhere to the
2499
                       IANA standards */
2500
                    brickinfo->rdma_port = 0;
2501
                }
2502

2503
            } else if (!strncmp(
2504
                           key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
2505
                           SLEN(GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
2506
                ret = gf_string2int(value, &brickinfo->decommissioned);
2507
                if (ret == -1) {
2508
                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
2509
                           GD_MSG_INCOMPATIBLE_VALUE,
2510
                           "Failed to convert "
2511
                           "string to integer");
2512
                }
2513

2514
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_ORIGIN_PATH,
2515
                                SLEN(GLUSTERD_STORE_KEY_BRICK_ORIGIN_PATH))) {
2516
                if (snprintf(brickinfo->origin_path,
2517
                             sizeof(brickinfo->origin_path), "%s",
2518
                             value) >= sizeof(brickinfo->origin_path)) {
2519
                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2520
                           GD_MSG_PARSE_BRICKINFO_FAIL,
2521
                           "origin_path truncated: %s", brickinfo->origin_path);
2522
                    goto out;
2523
                }
2524
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR,
2525
                                SLEN(GLUSTERD_STORE_KEY_BRICK_MOUNT_DIR))) {
2526
                if (snprintf(brickinfo->mount_dir, sizeof(brickinfo->mount_dir),
2527
                             "%s", value) >= sizeof(brickinfo->mount_dir)) {
2528
                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2529
                           GD_MSG_PARSE_BRICKINFO_FAIL,
2530
                           "mount_dir truncated: %s", brickinfo->mount_dir);
2531
                    goto out;
2532
                }
2533
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
2534
                                SLEN(GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS))) {
2535
                ret = gf_string2int(value, &brickinfo->snap_status);
2536
                if (ret == -1) {
2537
                    gf_msg(this->name, GF_LOG_ERROR, EINVAL,
2538
                           GD_MSG_INCOMPATIBLE_VALUE,
2539
                           "Failed to convert "
2540
                           "string to integer");
2541
                }
2542

2543
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_FSTYPE,
2544
                                SLEN(GLUSTERD_STORE_KEY_BRICK_FSTYPE))) {
2545
                if (snprintf(brickinfo->fstype, sizeof(brickinfo->fstype), "%s",
2546
                             value) >= sizeof(brickinfo->fstype)) {
2547
                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2548
                           GD_MSG_PARSE_BRICKINFO_FAIL, "fstype truncated: %s",
2549
                           brickinfo->fstype);
2550
                    goto out;
2551
                }
2552
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_SNAPTYPE,
2553
                                SLEN(GLUSTERD_STORE_KEY_BRICK_SNAPTYPE))) {
2554
                if (snprintf(brickinfo->snap_type, sizeof(brickinfo->snap_type),
2555
                             "%s", value) >= sizeof(brickinfo->snap_type)) {
2556
                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2557
                           GD_MSG_PARSE_BRICKINFO_FAIL,
2558
                           "snap_type truncated: %s", brickinfo->snap_type);
2559
                    goto out;
2560
                }
2561
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_MNTOPTS,
2562
                                SLEN(GLUSTERD_STORE_KEY_BRICK_MNTOPTS))) {
2563
                if (snprintf(brickinfo->mnt_opts, sizeof(brickinfo->mnt_opts),
2564
                             "%s", value) >= sizeof(brickinfo->mnt_opts)) {
2565
                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2566
                           GD_MSG_PARSE_BRICKINFO_FAIL,
2567
                           "mnt_opts truncated: %s", brickinfo->mnt_opts);
2568
                    goto out;
2569
                }
2570
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_VGNAME,
2571
                                SLEN(GLUSTERD_STORE_KEY_BRICK_VGNAME))) {
2572
                if (snprintf(brickinfo->vg, sizeof(brickinfo->vg), "%s",
2573
                             value) >= sizeof(brickinfo->vg)) {
2574
                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2575
                           GD_MSG_PARSE_BRICKINFO_FAIL,
2576
                           "brickinfo->vg truncated: %s", brickinfo->vg);
2577
                    goto out;
2578
                }
2579
            } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_ID)) {
2580
                if (snprintf(brickinfo->brick_id, sizeof(brickinfo->brick_id),
2581
                             "%s", value) >= sizeof(brickinfo->brick_id)) {
2582
                    gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2583
                           GD_MSG_PARSE_BRICKINFO_FAIL,
2584
                           "brick_id truncated: %s", brickinfo->brick_id);
2585
                    goto out;
2586
                }
2587
            } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_FSID,
2588
                                SLEN(GLUSTERD_STORE_KEY_BRICK_FSID))) {
2589
                ret = gf_string2uint64(value, &brickinfo->statfs_fsid);
2590
                if (ret) {
2591
                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
2592
                           "%s "
2593
                           "is not a valid uint64_t value",
2594
                           value);
2595
                }
2596

2597
            } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_UUID)) {
2598
                gf_uuid_parse(value, brickinfo->uuid);
2599
            } else {
2600
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_KEY,
2601
                       "Unknown key: %s", key);
2602
            }
2603

2604
            GF_FREE(key);
2605
            GF_FREE(value);
2606
            key = NULL;
2607
            value = NULL;
2608

2609
            ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
2610
        }
2611

2612
        if (op_errno != GD_STORE_EOF) {
2613
            gf_msg(this->name, GF_LOG_ERROR, op_errno,
2614
                   GD_MSG_PARSE_BRICKINFO_FAIL,
2615
                   "Error parsing brickinfo: "
2616
                   "op_errno=%d",
2617
                   op_errno);
2618
            goto out;
2619
        }
2620

2621
        if (brickinfo->brick_id[0] == '\0') {
2622
            /* This is an old volume upgraded to op_version 4 */
2623
            GLUSTERD_ASSIGN_BRICKID_TO_BRICKINFO(brickinfo, volinfo, brickid++);
2624
        }
2625
        /* Populate brickinfo->real_path for normal volumes, for
2626
         * snapshot or snapshot restored volume this would be done post
2627
         * creating the brick mounts
2628
         */
2629
        if (gf_uuid_is_null(brickinfo->uuid))
2630
            (void)glusterd_resolve_brick(brickinfo);
2631
        if (brickinfo->real_path[0] == '\0' && !volinfo->is_snap_volume &&
2632
            gf_uuid_is_null(volinfo->restored_from_snap)) {
2633
            /* By now if the brick is a local brick then it will be
2634
             * able to resolve which is the only thing we want now
2635
             * for checking  whether the brickinfo->uuid matches
2636
             * with MY_UUID for realpath check. Hence do not handle
2637
             * error
2638
             */
2639
            if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
2640
                if (!realpath(brickinfo->path, abspath)) {
2641
                    gf_msg(this->name, GF_LOG_CRITICAL, errno,
2642
                           GD_MSG_BRICKINFO_CREATE_FAIL,
2643
                           "realpath() failed for brick %s"
2644
                           ". The underlying file system "
2645
                           "may be in bad state",
2646
                           brickinfo->path);
2647
                    ret = -1;
2648
                    goto out;
2649
                }
2650
                if (strlen(abspath) >= sizeof(brickinfo->real_path)) {
2651
                    ret = -1;
2652
                    goto out;
2653
                }
2654
                (void)strncpy(brickinfo->real_path, abspath,
2655
                              sizeof(brickinfo->real_path));
2656
            }
2657
        }
2658

2659
        /* Handle upgrade case of shared_brick_count 'fsid' */
2660
        /* Ideally statfs_fsid should never be 0 if done right */
2661
        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID) &&
2662
            brickinfo->statfs_fsid == 0) {
2663
            struct statvfs brickstat = {
2664
                0,
2665
            };
2666
            ret = sys_statvfs(brickinfo->path, &brickstat);
2667
            if (ret) {
2668
                gf_msg(this->name, GF_LOG_WARNING, errno,
2669
                       GD_MSG_BRICKINFO_CREATE_FAIL,
2670
                       "failed to get statfs() call on brick %s",
2671
                       brickinfo->path);
2672
                /* No need for treating it as an error, lets continue
2673
                   with just a message */
2674
            } else {
2675
                brickinfo->statfs_fsid = brickstat.f_fsid;
2676
            }
2677
        }
2678

2679
        cds_list_add_tail(&brickinfo->brick_list, &volinfo->bricks);
2680
        brick_count++;
2681
        if (gf_store_iter_destroy(&iter)) {
2682
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
2683
                   "Failed to destroy store iter");
2684
            ret = -1;
2685
            goto out;
2686
        }
2687
    }
2688

2689
    if (gf_store_iter_destroy(&tmpiter)) {
2690
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
2691
               "Failed to destroy store iter");
2692
        ret = -1;
2693
        goto out;
2694
    }
2695

2696
    ret = gf_store_iter_new(volinfo->shandle, &tmpiter);
2697

2698
    if (ret)
2699
        goto out;
2700

2701
    if (volinfo->thin_arbiter_count == 1) {
2702
        snprintf(tmpkey, sizeof(tmpkey), "%s-%d",
2703
                 GLUSTERD_STORE_KEY_VOL_TA_BRICK, 0);
2704
        while (ta_brick_count < volinfo->subvol_count) {
2705
            ret = glusterd_brickinfo_new(&ta_brickinfo);
2706
            if (ret)
2707
                goto out;
2708

2709
            ret = gf_store_iter_get_matching(tmpiter, tmpkey, &tmpvalue);
2710

2711
            len = snprintf(path, sizeof(path), "%s/%s", brickdir, tmpvalue);
2712
            GF_FREE(tmpvalue);
2713
            tmpvalue = NULL;
2714
            if ((len < 0) || (len >= sizeof(path))) {
2715
                ret = -1;
2716
                goto out;
2717
            }
2718

2719
            ret = gf_store_handle_retrieve(path, &ta_brickinfo->shandle);
2720

2721
            if (ret)
2722
                goto out;
2723

2724
            ret = gf_store_iter_new(ta_brickinfo->shandle, &iter);
2725

2726
            if (ret)
2727
                goto out;
2728

2729
            ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
2730
            if (ret) {
2731
                gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2732
                       GD_MSG_STORE_ITER_GET_FAIL,
2733
                       "Unable to iterate "
2734
                       "the store for brick: %s",
2735
                       path);
2736
                goto out;
2737
            }
2738

2739
            while (!ret) {
2740
                if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_HOSTNAME,
2741
                             SLEN(GLUSTERD_STORE_KEY_BRICK_HOSTNAME))) {
2742
                    if (snprintf(ta_brickinfo->hostname,
2743
                                 sizeof(ta_brickinfo->hostname), "%s",
2744
                                 value) >= sizeof(ta_brickinfo->hostname)) {
2745
                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2746
                               GD_MSG_PARSE_BRICKINFO_FAIL,
2747
                               "brick hostname truncated: %s",
2748
                               ta_brickinfo->hostname);
2749
                        goto out;
2750
                    }
2751
                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PATH,
2752
                                    SLEN(GLUSTERD_STORE_KEY_BRICK_PATH))) {
2753
                    if (snprintf(ta_brickinfo->path, sizeof(ta_brickinfo->path),
2754
                                 "%s", value) >= sizeof(ta_brickinfo->path)) {
2755
                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2756
                               GD_MSG_PARSE_BRICKINFO_FAIL,
2757
                               "brick path truncated: %s", ta_brickinfo->path);
2758
                        goto out;
2759
                    }
2760
                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_REAL_PATH,
2761
                                    SLEN(GLUSTERD_STORE_KEY_BRICK_REAL_PATH))) {
2762
                    if (snprintf(ta_brickinfo->real_path,
2763
                                 sizeof(ta_brickinfo->real_path), "%s",
2764
                                 value) >= sizeof(ta_brickinfo->real_path)) {
2765
                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2766
                               GD_MSG_PARSE_BRICKINFO_FAIL,
2767
                               "real_path truncated: %s",
2768
                               ta_brickinfo->real_path);
2769
                        goto out;
2770
                    }
2771
                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_PORT,
2772
                                    SLEN(GLUSTERD_STORE_KEY_BRICK_PORT))) {
2773
                    ret = gf_string2int(value, &ta_brickinfo->port);
2774
                    if (ret == -1) {
2775
                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
2776
                               GD_MSG_INCOMPATIBLE_VALUE,
2777
                               "Failed to convert "
2778
                               "string to integer");
2779
                    }
2780

2781
                    if (ta_brickinfo->port < priv->base_port) {
2782
                        /* This is required to adhere to the
2783
                        IANA standards */
2784
                        ta_brickinfo->port = 0;
2785
                    }
2786
                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_RDMA_PORT,
2787
                                    SLEN(GLUSTERD_STORE_KEY_BRICK_RDMA_PORT))) {
2788
                    ret = gf_string2int(value, &ta_brickinfo->rdma_port);
2789
                    if (ret == -1) {
2790
                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
2791
                               GD_MSG_INCOMPATIBLE_VALUE,
2792
                               "Failed to convert "
2793
                               "string to integer");
2794
                    }
2795

2796
                    if (ta_brickinfo->rdma_port < priv->base_port) {
2797
                        /* This is required to adhere to the
2798
                        IANA standards */
2799
                        ta_brickinfo->rdma_port = 0;
2800
                    }
2801
                } else if (!strncmp(
2802
                               key, GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED,
2803
                               SLEN(GLUSTERD_STORE_KEY_BRICK_DECOMMISSIONED))) {
2804
                    ret = gf_string2int(value, &ta_brickinfo->decommissioned);
2805
                    if (ret == -1) {
2806
                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
2807
                               GD_MSG_INCOMPATIBLE_VALUE,
2808
                               "Failed to convert "
2809
                               "string to integer");
2810
                    }
2811

2812
                } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_ID)) {
2813
                    if (snprintf(ta_brickinfo->brick_id,
2814
                                 sizeof(ta_brickinfo->brick_id), "%s",
2815
                                 value) >= sizeof(ta_brickinfo->brick_id)) {
2816
                        gf_msg("glusterd", GF_LOG_ERROR, op_errno,
2817
                               GD_MSG_PARSE_BRICKINFO_FAIL,
2818
                               "brick_id truncated: %s",
2819
                               ta_brickinfo->brick_id);
2820
                        goto out;
2821
                    }
2822
                } else if (!strncmp(key, GLUSTERD_STORE_KEY_BRICK_FSID,
2823
                                    SLEN(GLUSTERD_STORE_KEY_BRICK_FSID))) {
2824
                    ret = gf_string2uint64(value, &ta_brickinfo->statfs_fsid);
2825
                    if (ret) {
2826
                        gf_msg(this->name, GF_LOG_ERROR, 0,
2827
                               GD_MSG_INVALID_ENTRY,
2828
                               "%s "
2829
                               "is not a valid uint64_t value",
2830
                               value);
2831
                    }
2832
                } else if (!strcmp(key, GLUSTERD_STORE_KEY_BRICK_UUID)) {
2833
                    gf_uuid_parse(value, brickinfo->uuid);
2834
                } else if (!strncmp(
2835
                               key, GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS,
2836
                               SLEN(GLUSTERD_STORE_KEY_BRICK_SNAP_STATUS))) {
2837
                    ret = gf_string2int(value, &ta_brickinfo->snap_status);
2838
                    if (ret == -1) {
2839
                        gf_msg(this->name, GF_LOG_ERROR, EINVAL,
2840
                               GD_MSG_INCOMPATIBLE_VALUE,
2841
                               "Failed to convert "
2842
                               "string to integer");
2843
                    }
2844

2845
                } else {
2846
                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_KEY,
2847
                           "Unknown key: %s", key);
2848
                }
2849

2850
                GF_FREE(key);
2851
                GF_FREE(value);
2852
                key = NULL;
2853
                value = NULL;
2854
                ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
2855
            }
2856

2857
            GLUSTERD_ASSIGN_BRICKID_TO_TA_BRICKINFO(ta_brickinfo, volinfo,
2858
                                                    ta_brick_id);
2859
            ta_brick_id += 3;
2860

2861
            cds_list_add_tail(&ta_brickinfo->brick_list, &volinfo->ta_bricks);
2862
            ta_brick_count++;
2863
            if (gf_store_iter_destroy(&iter)) {
2864
                gf_msg(this->name, GF_LOG_ERROR, 0,
2865
                       GD_MSG_STORE_ITER_DESTROY_FAIL,
2866
                       "Failed to destroy store iter");
2867
                ret = -1;
2868
                goto out;
2869
            }
2870
        }
2871
    }
2872

2873
    assign_brick_groups(volinfo);
2874
    ret = 0;
2875

2876
out:
2877
    if (key)
2878
        GF_FREE(key);
2879
    if (value)
2880
        GF_FREE(value);
2881
    if (gf_store_iter_destroy(&tmpiter)) {
2882
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
2883
               "Failed to destroy store iter");
2884
        ret = -1;
2885
    }
2886

2887
    if (gf_store_iter_destroy(&iter)) {
2888
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
2889
               "Failed to destroy store iter");
2890
        ret = -1;
2891
    }
2892

2893
    gf_msg_debug(this->name, 0, "Returning with %d", ret);
2894

2895
    return ret;
2896
}
2897

2898
int32_t
2899
glusterd_store_retrieve_node_state(glusterd_volinfo_t *volinfo)
2900
{
2901
    int32_t ret = -1;
2902
    gf_store_iter_t *iter = NULL;
2903
    char *key = NULL;
2904
    char *value = NULL;
2905
    char *dup_value = NULL;
2906
    char volpath[PATH_MAX] = {
2907
        0,
2908
    };
2909
    glusterd_conf_t *priv = NULL;
2910
    char path[PATH_MAX] = {
2911
        0,
2912
    };
2913
    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
2914
    dict_t *tmp_dict = NULL;
2915
    xlator_t *this = THIS;
2916
    int32_t len = 0;
2917

2918
    priv = this->private;
2919
    GF_ASSERT(priv);
2920
    GF_ASSERT(volinfo);
2921

2922
    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, priv);
2923
    len = snprintf(path, sizeof(path), "%s/%s", volpath,
2924
                   GLUSTERD_NODE_STATE_FILE);
2925
    if ((len < 0) || (len >= PATH_MAX)) {
2926
        goto out;
2927
    }
2928

2929
    ret = gf_store_handle_retrieve(path, &volinfo->node_state_shandle);
2930
    if (ret)
2931
        goto out;
2932

2933
    ret = gf_store_iter_new(volinfo->node_state_shandle, &iter);
2934

2935
    if (ret)
2936
        goto out;
2937

2938
    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
2939

2940
    if (ret)
2941
        goto out;
2942

2943
    while (ret == 0) {
2944
        if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG,
2945
                     SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG))) {
2946
            volinfo->rebal.defrag_cmd = atoi(value);
2947
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS,
2948
                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_STATUS))) {
2949
            volinfo->rebal.defrag_status = atoi(value);
2950
        } else if (!strncmp(key, GF_REBALANCE_TID_KEY,
2951
                            SLEN(GF_REBALANCE_TID_KEY))) {
2952
            gf_uuid_parse(value, volinfo->rebal.rebalance_id);
2953
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_DEFRAG_OP,
2954
                            SLEN(GLUSTERD_STORE_KEY_DEFRAG_OP))) {
2955
            volinfo->rebal.op = atoi(value);
2956
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_REB_FILES,
2957
                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_REB_FILES))) {
2958
            sscanf(value, "%" PRIu64, &volinfo->rebal.rebalance_files);
2959
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_SIZE,
2960
                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_SIZE))) {
2961
            sscanf(value, "%" PRIu64, &volinfo->rebal.rebalance_data);
2962
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_SCANNED,
2963
                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_SCANNED))) {
2964
            sscanf(value, "%" PRIu64, &volinfo->rebal.lookedup_files);
2965
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_FAILURES,
2966
                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_FAILURES))) {
2967
            sscanf(value, "%" PRIu64, &volinfo->rebal.rebalance_failures);
2968
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_SKIPPED,
2969
                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_SKIPPED))) {
2970
            sscanf(value, "%" PRIu64, &volinfo->rebal.skipped_files);
2971
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DEFRAG_RUN_TIME,
2972
                            SLEN(GLUSTERD_STORE_KEY_VOL_DEFRAG_RUN_TIME))) {
2973
            volinfo->rebal.rebalance_time = atoi(value);
2974
        } else {
2975
            if (!tmp_dict) {
2976
                tmp_dict = dict_new();
2977
                if (!tmp_dict) {
2978
                    ret = -1;
2979
                    goto out;
2980
                }
2981
            }
2982
            dup_value = gf_strdup(value);
2983
            if (!dup_value) {
2984
                ret = -1;
2985
                gf_msg(this->name, GF_LOG_ERROR, ENOMEM, GD_MSG_NO_MEMORY,
2986
                       "Failed to strdup value string");
2987
                goto out;
2988
            }
2989
            ret = dict_set_str(tmp_dict, key, dup_value);
2990
            if (ret) {
2991
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
2992
                       "Error setting data in rebal "
2993
                       "dict.");
2994
                goto out;
2995
            }
2996
            dup_value = NULL;
2997
        }
2998

2999
        GF_FREE(key);
3000
        GF_FREE(value);
3001
        key = NULL;
3002
        value = NULL;
3003

3004
        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
3005
    }
3006
    if (tmp_dict) {
3007
        volinfo->rebal.dict = dict_ref(tmp_dict);
3008
    }
3009

3010
    if (op_errno != GD_STORE_EOF) {
3011
        ret = -1;
3012
        goto out;
3013
    }
3014

3015
    ret = 0;
3016

3017
out:
3018
    if (gf_store_iter_destroy(&iter)) {
3019
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
3020
               "Failed to destroy store iter");
3021
        ret = -1;
3022
    }
3023

3024
    if (dup_value)
3025
        GF_FREE(dup_value);
3026
    if (key)
3027
        GF_FREE(key);
3028
    if (value)
3029
        GF_FREE(value);
3030
    if (ret) {
3031
        if (volinfo->rebal.dict)
3032
            dict_unref(volinfo->rebal.dict);
3033
    }
3034
    if (tmp_dict)
3035
        dict_unref(tmp_dict);
3036

3037
    gf_msg_trace(this->name, 0, "Returning with %d", ret);
3038

3039
    return ret;
3040
}
3041

3042
/* change slave$i to secondary$i, where i = 1,2,3,...*/
3043
static void
3044
glusterd_chk_update_geo_rep_key_name(char **key)
3045
{
3046
    char *new_key = NULL;
3047
    int slen_slave = 5;      // strlen("slave")
3048
    int slen_secondary = 9;  // strlen("secondary")
3049

3050
    if (strncmp(*key, "slave", slen_slave) != 0)
3051
        return;
3052

3053
    int new_key_len = slen_secondary + (strlen(*key) - slen_slave);
3054

3055
    new_key = GF_MALLOC(new_key_len + 1, gf_common_mt_char);
3056
    if (!new_key)
3057
        return;
3058

3059
    strcpy(new_key, "secondary");
3060
    strcat(new_key, *key + slen_slave);
3061
    GF_FREE(*key);
3062
    *key = new_key;
3063
}
3064

3065
int
3066
glusterd_store_update_volinfo(glusterd_volinfo_t *volinfo)
3067
{
3068
    int ret = -1;
3069
    int exists = 0;
3070
    char *key = NULL;
3071
    char *value = NULL;
3072
    char volpath[PATH_MAX] = {
3073
        0,
3074
    };
3075
    char path[PATH_MAX] = {
3076
        0,
3077
    };
3078
    xlator_t *this = THIS;
3079
    glusterd_conf_t *conf = NULL;
3080
    gf_store_iter_t *iter = NULL;
3081
    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
3082
    int32_t len = 0;
3083

3084
    conf = this->private;
3085
    GF_ASSERT(volinfo);
3086

3087
    GLUSTERD_GET_VOLUME_DIR(volpath, volinfo, conf);
3088

3089
    len = snprintf(path, sizeof(path), "%s/%s", volpath,
3090
                   GLUSTERD_VOLUME_INFO_FILE);
3091
    if ((len < 0) || (len >= sizeof(path))) {
3092
        goto out;
3093
    }
3094

3095
    ret = gf_store_handle_retrieve(path, &volinfo->shandle);
3096
    if (ret) {
3097
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDLE_NULL,
3098
               "volinfo handle is NULL");
3099
        goto out;
3100
    }
3101

3102
    ret = gf_store_iter_new(volinfo->shandle, &iter);
3103
    if (ret) {
3104
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
3105
               "Failed to get new store "
3106
               "iter");
3107
        goto out;
3108
    }
3109

3110
    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
3111
    if (ret) {
3112
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
3113
               "Failed to get next store "
3114
               "iter");
3115
        goto out;
3116
    }
3117

3118
    while (!ret) {
3119
        gf_msg_debug(this->name, 0, "key = %s value = %s", key, value);
3120
        if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_TYPE,
3121
                     SLEN(GLUSTERD_STORE_KEY_VOL_TYPE))) {
3122
            volinfo->type = atoi(value);
3123
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_COUNT,
3124
                            SLEN(GLUSTERD_STORE_KEY_VOL_COUNT))) {
3125
            volinfo->brick_count = atoi(value);
3126
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_STATUS,
3127
                            SLEN(GLUSTERD_STORE_KEY_VOL_STATUS))) {
3128
            volinfo->status = atoi(value);
3129
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_VERSION,
3130
                            SLEN(GLUSTERD_STORE_KEY_VOL_VERSION))) {
3131
            volinfo->version = atoi(value);
3132
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_PORT,
3133
                            SLEN(GLUSTERD_STORE_KEY_VOL_PORT))) {
3134
            volinfo->port = atoi(value);
3135
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_SUB_COUNT,
3136
                            SLEN(GLUSTERD_STORE_KEY_VOL_SUB_COUNT))) {
3137
            volinfo->sub_count = atoi(value);
3138
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_REPLICA_CNT,
3139
                            SLEN(GLUSTERD_STORE_KEY_VOL_REPLICA_CNT))) {
3140
            volinfo->replica_count = atoi(value);
3141
        } else if (!strcmp(key, GLUSTERD_STORE_KEY_VOL_ARBITER_CNT)) {
3142
            volinfo->arbiter_count = atoi(value);
3143
        } else if (!strcmp(key, GLUSTERD_STORE_KEY_VOL_THIN_ARBITER_CNT)) {
3144
            volinfo->thin_arbiter_count = atoi(value);
3145
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT,
3146
                            SLEN(GLUSTERD_STORE_KEY_VOL_DISPERSE_CNT))) {
3147
            volinfo->disperse_count = atoi(value);
3148
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT,
3149
                            SLEN(GLUSTERD_STORE_KEY_VOL_REDUNDANCY_CNT))) {
3150
            volinfo->redundancy_count = atoi(value);
3151
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_TRANSPORT,
3152
                            SLEN(GLUSTERD_STORE_KEY_VOL_TRANSPORT))) {
3153
            volinfo->transport_type = atoi(value);
3154
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_ID,
3155
                            SLEN(GLUSTERD_STORE_KEY_VOL_ID))) {
3156
            ret = gf_uuid_parse(value, volinfo->volume_id);
3157
            if (ret)
3158
                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_PARSE_FAIL,
3159
                       "failed to parse uuid");
3160

3161
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_USERNAME,
3162
                            SLEN(GLUSTERD_STORE_KEY_USERNAME))) {
3163
            glusterd_auth_set_username(volinfo, value);
3164

3165
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_PASSWORD,
3166
                            SLEN(GLUSTERD_STORE_KEY_PASSWORD))) {
3167
            glusterd_auth_set_password(volinfo, value);
3168

3169
        } else if (strstr(key, "secondary") || strstr(key, "slave")) {
3170
            glusterd_chk_update_geo_rep_key_name(&key);  // Old node upgrade.
3171
            ret = dict_set_dynstr(volinfo->gsync_secondaries, key,
3172
                                  gf_strdup(value));
3173
            if (ret) {
3174
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_SET_FAILED,
3175
                       "Error in "
3176
                       "dict_set_str");
3177
                goto out;
3178
            }
3179
            gf_msg_debug(this->name, 0,
3180
                         "Parsed as " GEOREP
3181
                         " "
3182
                         " secondary:key=%s,value:%s",
3183
                         key, value);
3184

3185
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_OP_VERSION,
3186
                            SLEN(GLUSTERD_STORE_KEY_VOL_OP_VERSION))) {
3187
            volinfo->op_version = atoi(value);
3188
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION,
3189
                            SLEN(GLUSTERD_STORE_KEY_VOL_CLIENT_OP_VERSION))) {
3190
            volinfo->client_op_version = atoi(value);
3191
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT,
3192
                            SLEN(GLUSTERD_STORE_KEY_SNAP_MAX_HARD_LIMIT))) {
3193
            volinfo->snap_max_hard_limit = (uint64_t)atoll(value);
3194
        } else if (!strncmp(
3195
                       key, GLUSTERD_STORE_KEY_VOL_RESTORED_SNAPNAME_ID,
3196
                       SLEN(GLUSTERD_STORE_KEY_VOL_RESTORED_SNAPNAME_ID))) {
3197
            if (snprintf(volinfo->restored_from_snapname_id,
3198
                         sizeof(volinfo->restored_from_snapname_id), "%s",
3199
                         value) >= sizeof(volinfo->restored_from_snapname_id)) {
3200
                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_PARSE_FAIL,
3201
                       "failed to parse restored_from_snapname_id");
3202
            }
3203
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_RESTORED_SNAPNAME,
3204
                            SLEN(GLUSTERD_STORE_KEY_VOL_RESTORED_SNAPNAME))) {
3205
            if (snprintf(volinfo->restored_from_snapname,
3206
                         sizeof(volinfo->restored_from_snapname), "%s",
3207
                         value) >= sizeof(volinfo->restored_from_snapname)) {
3208
                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_DICT_SET_FAILED,
3209
                       "failed to parse restored_from_snapname");
3210
            }
3211
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP,
3212
                            SLEN(GLUSTERD_STORE_KEY_VOL_RESTORED_SNAP))) {
3213
            ret = gf_uuid_parse(value, volinfo->restored_from_snap);
3214
            if (ret)
3215
                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_PARSE_FAIL,
3216
                       "failed to parse restored snap's uuid");
3217
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_PARENT_VOLNAME,
3218
                            SLEN(GLUSTERD_STORE_KEY_PARENT_VOLNAME))) {
3219
            if (snprintf(volinfo->parent_volname,
3220
                         sizeof(volinfo->parent_volname), "%s",
3221
                         value) >= sizeof(volinfo->parent_volname)) {
3222
                gf_msg("glusterd", GF_LOG_ERROR, op_errno,
3223
                       GD_MSG_PARSE_BRICKINFO_FAIL,
3224
                       "parent_volname truncated: %s", volinfo->parent_volname);
3225
                goto out;
3226
            }
3227
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_SNAP_PLUGIN,
3228
                            SLEN(GLUSTERD_STORE_KEY_VOL_SNAP_PLUGIN))) {
3229
            if (snprintf(volinfo->snap_plugin, sizeof(volinfo->snap_plugin),
3230
                         "%s", value) >= sizeof(volinfo->snap_plugin)) {
3231
                gf_msg("glusterd", GF_LOG_ERROR, op_errno,
3232
                       GD_MSG_PARSE_BRICKINFO_FAIL, "snap_plugin truncated: %s",
3233
                       volinfo->snap_plugin);
3234
                goto out;
3235
            }
3236
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION,
3237
                            SLEN(GLUSTERD_STORE_KEY_VOL_QUOTA_VERSION))) {
3238
            volinfo->quota_xattr_version = atoi(value);
3239
        } else {
3240
            if (is_key_glusterd_hooks_friendly(key)) {
3241
                exists = 1;
3242

3243
            } else {
3244
                exists = glusterd_check_option_exists(key, NULL);
3245
            }
3246

3247
            switch (exists) {
3248
                case -1:
3249
                    ret = -1;
3250
                    goto out;
3251

3252
                case 0:
3253
                    /*Ignore GLUSTERD_STORE_KEY_VOL_BRICK since
3254
                     glusterd_store_retrieve_bricks gets it later.
3255
                     also, ignore tier-enabled key as we deprecated
3256
                     tier xlator*/
3257
                    if (!strstr(key, GLUSTERD_STORE_KEY_VOL_BRICK) ||
3258
                        !strstr(key, GF_TIER_ENABLED))
3259
                        gf_msg(this->name, GF_LOG_WARNING, 0,
3260
                               GD_MSG_UNKNOWN_KEY, "Unknown key: %s", key);
3261
                    break;
3262

3263
                case 1:
3264
                    /*The following strcmp check is to ensure that
3265
                     * glusterd does not restore the quota limits
3266
                     * into volinfo->dict post upgradation from 3.3
3267
                     * to 3.4 as the same limits will now be stored
3268
                     * in xattrs on the respective directories.
3269
                     */
3270
                    if (!strcmp(key, "features.limit-usage"))
3271
                        break;
3272
                    ret = dict_set_dynstr(volinfo->dict, key, gf_strdup(value));
3273
                    if (ret) {
3274
                        gf_msg(this->name, GF_LOG_ERROR, 0,
3275
                               GD_MSG_DICT_SET_FAILED,
3276
                               "Error in "
3277
                               "dict_set_str");
3278
                        goto out;
3279
                    }
3280
                    gf_msg_debug(this->name, 0,
3281
                                 "Parsed as Volume-"
3282
                                 "set:key=%s,value:%s",
3283
                                 key, value);
3284
                    break;
3285
            }
3286
        }
3287

3288
        GF_FREE(key);
3289
        GF_FREE(value);
3290
        key = NULL;
3291
        value = NULL;
3292

3293
        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
3294
    }
3295

3296
    /* backward compatibility */
3297
    {
3298
        switch (volinfo->type) {
3299
            case GF_CLUSTER_TYPE_NONE:
3300
                volinfo->replica_count = 1;
3301
                break;
3302

3303
            case GF_CLUSTER_TYPE_REPLICATE:
3304
                volinfo->replica_count = volinfo->sub_count;
3305
                break;
3306

3307
            case GF_CLUSTER_TYPE_DISPERSE:
3308
                GF_ASSERT(volinfo->disperse_count > 0);
3309
                GF_ASSERT(volinfo->redundancy_count > 0);
3310
                break;
3311

3312
            default:
3313
                GF_ASSERT(0);
3314
                break;
3315
        }
3316

3317
        volinfo->dist_leaf_count = glusterd_get_dist_leaf_count(volinfo);
3318

3319
        volinfo->subvol_count = (volinfo->brick_count /
3320
                                 volinfo->dist_leaf_count);
3321

3322
        /* Only calculate volume op-versions if they are not found */
3323
        if (!volinfo->op_version && !volinfo->client_op_version)
3324
            gd_update_volume_op_versions(volinfo);
3325
    }
3326

3327
    if (op_errno != GD_STORE_EOF)
3328
        goto out;
3329

3330
    ret = 0;
3331

3332
out:
3333
    if (gf_store_iter_destroy(&iter)) {
3334
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
3335
               "Failed to destroy store iter");
3336
        ret = -1;
3337
    }
3338

3339
    if (key)
3340
        GF_FREE(key);
3341
    if (value)
3342
        GF_FREE(value);
3343
    return ret;
3344
}
3345

3346
glusterd_volinfo_t *
3347
glusterd_store_retrieve_volume(char *volname, glusterd_snap_t *snap)
3348
{
3349
    int32_t ret = -1;
3350
    glusterd_volinfo_t *volinfo = NULL;
3351
    glusterd_volinfo_t *origin_volinfo = NULL;
3352
    glusterd_conf_t *priv = NULL;
3353
    xlator_t *this = THIS;
3354

3355
    priv = this->private;
3356
    GF_ASSERT(priv);
3357
    GF_ASSERT(volname);
3358

3359
    ret = glusterd_volinfo_new(&volinfo);
3360
    if (ret)
3361
        goto out;
3362

3363
    if (snprintf(volinfo->volname, NAME_MAX + 1, "%s", volname) >= NAME_MAX + 1)
3364
        goto out;
3365
    volinfo->snapshot = snap;
3366
    if (snap)
3367
        volinfo->is_snap_volume = _gf_true;
3368

3369
    ret = glusterd_store_update_volinfo(volinfo);
3370
    if (ret) {
3371
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_UPDATE_FAIL,
3372
               "Failed to update volinfo "
3373
               "for %s volume",
3374
               volname);
3375
        goto out;
3376
    }
3377

3378
    ret = glusterd_store_retrieve_bricks(volinfo);
3379
    if (ret)
3380
        goto out;
3381

3382
    ret = glusterd_store_retrieve_snapd(volinfo);
3383
    if (ret)
3384
        goto out;
3385

3386
    ret = glusterd_compute_cksum(volinfo, _gf_false);
3387
    if (ret)
3388
        goto out;
3389

3390
    ret = glusterd_store_retrieve_quota_version(volinfo);
3391
    if (ret)
3392
        goto out;
3393

3394
    ret = glusterd_store_create_quota_conf_sh_on_absence(volinfo);
3395
    if (ret)
3396
        goto out;
3397

3398
    ret = glusterd_compute_cksum(volinfo, _gf_true);
3399
    if (ret)
3400
        goto out;
3401

3402
    ret = glusterd_store_save_quota_version_and_cksum(volinfo);
3403
    if (ret)
3404
        goto out;
3405

3406
    if (!snap) {
3407
        glusterd_list_add_order(&volinfo->vol_list, &priv->volumes,
3408
                                glusterd_compare_volume_name);
3409

3410
    } else {
3411
        ret = glusterd_volinfo_find(volinfo->parent_volname, &origin_volinfo);
3412
        if (ret) {
3413
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
3414
                   "Parent volinfo "
3415
                   "not found for %s volume",
3416
                   volname);
3417
            goto out;
3418
        }
3419
        glusterd_list_add_snapvol(origin_volinfo, volinfo);
3420
    }
3421

3422
out:
3423
    if (ret) {
3424
        if (volinfo)
3425
            glusterd_volinfo_unref(volinfo);
3426
        volinfo = NULL;
3427
    }
3428

3429
    gf_msg_trace(this->name, 0, "Returning with %d", ret);
3430

3431
    return volinfo;
3432
}
3433

3434
static void
3435
glusterd_store_set_options_path(glusterd_conf_t *conf, char *path, size_t len)
3436
{
3437
    snprintf(path, len, "%s/options", conf->workdir);
3438
}
3439

3440
int32_t
3441
glusterd_store_options(xlator_t *this, dict_t *opts)
3442
{
3443
    gf_store_handle_t *shandle = NULL;
3444
    glusterd_conf_t *conf = NULL;
3445
    char path[PATH_MAX] = {0};
3446
    int fd = -1;
3447
    int32_t ret = -1;
3448
    glusterd_volinfo_data_store_t *dict_data = NULL;
3449

3450
    conf = this->private;
3451
    glusterd_store_set_options_path(conf, path, sizeof(path));
3452

3453
    ret = gf_store_handle_new(path, &shandle);
3454
    if (ret) {
3455
        goto out;
3456
    }
3457

3458
    fd = gf_store_mkstemp(shandle);
3459
    if (fd <= 0) {
3460
        ret = -1;
3461
        goto out;
3462
    }
3463

3464
    dict_data = GF_CALLOC(1, sizeof(glusterd_volinfo_data_store_t),
3465
                          gf_gld_mt_volinfo_dict_data_t);
3466
    if (dict_data == NULL) {
3467
        gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_NO_MEMORY, NULL);
3468
        return -1;
3469
    }
3470
    dict_data->shandle = shandle;
3471
    shandle->fd = fd;
3472
    dict_foreach(opts, _storeopts, (void *)dict_data);
3473
    if (dict_data->buffer_len > 0) {
3474
        ret = gf_store_save_items(fd, dict_data->buffer);
3475
        if (ret) {
3476
            gf_smsg(this->name, GF_LOG_ERROR, 0, GD_MSG_FILE_OP_FAILED, NULL);
3477
            goto out;
3478
        }
3479
    }
3480

3481
    ret = gf_store_rename_tmppath(shandle);
3482
out:
3483
    shandle->fd = 0;
3484
    GF_FREE(dict_data);
3485
    if ((ret < 0) && (fd > 0)) {
3486
        gf_store_unlink_tmppath(shandle);
3487
    }
3488
    gf_store_handle_destroy(shandle);
3489
    return ret;
3490
}
3491

3492
int32_t
3493
glusterd_store_retrieve_options(xlator_t *this)
3494
{
3495
    char path[PATH_MAX] = {0};
3496
    glusterd_conf_t *conf = NULL;
3497
    gf_store_handle_t *shandle = NULL;
3498
    gf_store_iter_t *iter = NULL;
3499
    char *key = NULL;
3500
    char *value = NULL;
3501
    gf_store_op_errno_t op_errno = 0;
3502
    int ret = -1;
3503

3504
    conf = this->private;
3505
    glusterd_store_set_options_path(conf, path, sizeof(path));
3506

3507
    ret = gf_store_handle_retrieve(path, &shandle);
3508
    if (ret)
3509
        goto out;
3510

3511
    ret = gf_store_iter_new(shandle, &iter);
3512
    if (ret)
3513
        goto out;
3514

3515
    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
3516
    while (!ret) {
3517
        ret = dict_set_dynstr(conf->opts, key, value);
3518
        if (ret) {
3519
            GF_FREE(key);
3520
            GF_FREE(value);
3521
            goto out;
3522
        }
3523
        GF_FREE(key);
3524
        key = NULL;
3525
        value = NULL;
3526

3527
        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
3528
    }
3529
    if (op_errno != GD_STORE_EOF)
3530
        goto out;
3531
    ret = 0;
3532
out:
3533
    (void)gf_store_iter_destroy(&iter);
3534
    gf_store_handle_destroy(shandle);
3535
    return ret;
3536
}
3537

3538
int32_t
3539
glusterd_store_retrieve_volumes(xlator_t *this, glusterd_snap_t *snap)
3540
{
3541
    int32_t ret = -1;
3542
    char path[PATH_MAX] = {
3543
        0,
3544
    };
3545
    glusterd_conf_t *priv = NULL;
3546
    DIR *dir = NULL;
3547
    struct dirent *entry = NULL;
3548
    struct dirent scratch[2] = {
3549
        {
3550
            0,
3551
        },
3552
    };
3553
    glusterd_volinfo_t *volinfo = NULL;
3554
    struct stat st = {
3555
        0,
3556
    };
3557
    char entry_path[PATH_MAX] = {
3558
        0,
3559
    };
3560
    int32_t len = 0;
3561

3562
    priv = this->private;
3563

3564
    GF_ASSERT(priv);
3565

3566
    if (snap)
3567
        len = snprintf(path, PATH_MAX, "%s/snaps/%s", priv->workdir,
3568
                       snap->snapname);
3569
    else
3570
        len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
3571
                       GLUSTERD_VOLUME_DIR_PREFIX);
3572
    if ((len < 0) || (len >= PATH_MAX)) {
3573
        goto out;
3574
    }
3575

3576
    dir = sys_opendir(path);
3577

3578
    if (!dir) {
3579
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
3580
               "Unable to open dir %s", path);
3581
        goto out;
3582
    }
3583

3584
    while ((entry = sys_readdir(dir, scratch))) {
3585
        if (gf_irrelevant_entry(entry))
3586
            continue;
3587
        if (snap && ((!strcmp(entry->d_name, "geo-replication")) ||
3588
                     (!strcmp(entry->d_name, "info"))))
3589
            continue;
3590

3591
        len = snprintf(entry_path, PATH_MAX, "%s/%s", path, entry->d_name);
3592
        if ((len < 0) || (len >= PATH_MAX))
3593
            continue;
3594

3595
        ret = sys_lstat(entry_path, &st);
3596
        if (ret == -1) {
3597
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_INVALID_ENTRY,
3598
                   "Failed to stat entry %s : %s", path, strerror(errno));
3599
            continue;
3600
        }
3601

3602
        if (!S_ISDIR(st.st_mode)) {
3603
            gf_msg_debug(this->name, 0, "%s is not a valid volume",
3604
                         entry->d_name);
3605
            continue;
3606
        }
3607

3608
        volinfo = glusterd_store_retrieve_volume(entry->d_name, snap);
3609
        if (!volinfo) {
3610
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOL_RESTORE_FAIL,
3611
                   "Unable to restore "
3612
                   "volume: %s",
3613
                   entry->d_name);
3614
            ret = -1;
3615
            goto out;
3616
        }
3617

3618
        ret = glusterd_store_retrieve_node_state(volinfo);
3619
        if (ret) {
3620
            /* Backward compatibility */
3621
            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NEW_NODE_STATE_CREATION,
3622
                   "Creating a new node_state "
3623
                   "for volume: %s.",
3624
                   entry->d_name);
3625
            glusterd_store_create_nodestate_sh_on_absence(volinfo);
3626
            glusterd_store_perform_node_state_store(volinfo);
3627
        }
3628
    }
3629

3630
    ret = 0;
3631

3632
out:
3633
    if (dir)
3634
        sys_closedir(dir);
3635
    gf_msg_debug(this->name, 0, "Returning with %d", ret);
3636

3637
    return ret;
3638
}
3639

3640
/* Figure out the brick mount path, from the brick path */
3641
int32_t
3642
glusterd_find_brick_mount_path(char *brick_path, char **brick_mount_path)
3643
{
3644
    char *ptr = NULL;
3645
    int32_t ret = -1;
3646
    xlator_t *this = THIS;
3647

3648
    GF_ASSERT(brick_path);
3649
    GF_ASSERT(brick_mount_path);
3650

3651
    *brick_mount_path = gf_strdup(brick_path);
3652
    if (!*brick_mount_path) {
3653
        ret = -1;
3654
        goto out;
3655
    }
3656

3657
    /* Finding the pointer to the end of
3658
     * /var/run/gluster/snaps/<snap-uuid>
3659
     */
3660
    ptr = strstr(*brick_mount_path, "brick");
3661
    if (!ptr) {
3662
        /* Snapshot bricks must have brick num as part
3663
         * of the brickpath
3664
         */
3665
        gf_msg(this->name, GF_LOG_ERROR, EINVAL, GD_MSG_INVALID_ENTRY,
3666
               "Invalid brick path(%s)", brick_path);
3667
        ret = -1;
3668
        goto out;
3669
    }
3670

3671
    /* Moving the pointer to the end of
3672
     * /var/run/gluster/snaps/<snap-uuid>/<brick_num>
3673
     * and assigning '\0' to it.
3674
     */
3675
    while ((*ptr != '\0') && (*ptr != '/'))
3676
        ptr++;
3677

3678
    if (*ptr == '/') {
3679
        *ptr = '\0';
3680
    }
3681

3682
    ret = 0;
3683
out:
3684
    if (ret && *brick_mount_path) {
3685
        GF_FREE(*brick_mount_path);
3686
        *brick_mount_path = NULL;
3687
    }
3688
    gf_msg_trace(this->name, 0, "Returning with %d", ret);
3689
    return ret;
3690
}
3691

3692
/* Check if brick_mount_path is already mounted. If not, mount the device
3693
 * at the brick_mount_path
3694
 */
3695
int32_t
3696
glusterd_mount_brick_paths(glusterd_volinfo_t *volinfo,
3697
                           glusterd_brickinfo_t *brickinfo, int32_t brick_num)
3698
{
3699
    int32_t ret = -1;
3700
    xlator_t *this = THIS;
3701
    glusterd_conf_t *priv = NULL;
3702
    struct glusterd_snap_ops *snap_ops = NULL;
3703
    char snap_volume_id[GD_VOLUME_NAME_MAX] = "";
3704
    char snapname[NAME_MAX] = "";
3705

3706
    GF_ASSERT(brickinfo);
3707

3708
    priv = this->private;
3709
    GF_ASSERT(priv);
3710

3711
    glusterd_snapshot_plugin_by_name(volinfo->snap_plugin, &snap_ops);
3712

3713
    /* Mount the snapshot */
3714
    if (volinfo->snapshot) {
3715
        strcpy(snap_volume_id, volinfo->volname);
3716
        strcpy(snapname, volinfo->snapshot->snapname);
3717
    } else if (strlen(volinfo->restored_from_snapname) == 0) {
3718
        /* This is clone brick, so Volume ID is the backend snap name
3719
           and volname is the Clone name */
3720
        GLUSTERD_GET_UUID_NOHYPHEN(snap_volume_id, volinfo->volume_id);
3721
        strcpy(snapname, volinfo->volname);
3722
    } else {
3723
        strcpy(snap_volume_id, volinfo->restored_from_snapname_id);
3724
        strcpy(snapname, volinfo->restored_from_snapname);
3725
    }
3726
    ret = snap_ops->activate(brickinfo, snapname, snap_volume_id, brick_num);
3727
    if (ret) {
3728
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_MOUNT_FAIL,
3729
               "Failed to mount snapshot.");
3730
        goto out;
3731
    }
3732

3733
out:
3734
    gf_msg_trace(this->name, 0, "Returning with %d", ret);
3735
    return ret;
3736
}
3737

3738
int32_t
3739
glusterd_recreate_vol_brick_mounts(xlator_t *this, glusterd_volinfo_t *volinfo)
3740
{
3741
    glusterd_brickinfo_t *brickinfo = NULL;
3742
    int32_t ret = -1;
3743
    struct stat st_buf = {
3744
        0,
3745
    };
3746
    char abspath[PATH_MAX] = {0};
3747
    int brick_count = -1;
3748

3749
    GF_ASSERT(volinfo);
3750

3751
    cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
3752
    {
3753
        brick_count++;
3754

3755
        /* If the brick is not of this node, or its
3756
         * snapshot is pending, or the brick is not
3757
         * a snapshotted brick, we continue
3758
         */
3759
        if ((gf_uuid_compare(brickinfo->uuid, MY_UUID)) ||
3760
            (brickinfo->snap_status == -1))
3761
            continue;
3762

3763
        /* Check if the brickinfo path is present.
3764
         * If not create the brick_mount_path */
3765
        ret = sys_lstat(brickinfo->path, &st_buf);
3766
        if (ret) {
3767
            if (errno == ENOENT) {
3768
                /* Check if brick_mount_path is already mounted.
3769
                 * If not, mount the device_path at the brick_mount_path */
3770
                ret = glusterd_mount_brick_paths(volinfo, brickinfo,
3771
                                                 brick_count);
3772
                if (ret) {
3773
                    gf_msg(this->name, GF_LOG_ERROR, 0,
3774
                           GD_MSG_BRK_MNTPATH_MOUNT_FAIL,
3775
                           "Failed to mount brick_mount_path");
3776
                    goto out;
3777
                }
3778
            } else {
3779
                gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
3780
                       "Brick Path(%s) not valid. ", brickinfo->path);
3781
                goto out;
3782
            }
3783
        }
3784

3785
        if (!gf_uuid_compare(brickinfo->uuid, MY_UUID)) {
3786
            if (brickinfo->real_path[0] == '\0') {
3787
                if (!realpath(brickinfo->path, abspath)) {
3788
                    gf_msg(this->name, GF_LOG_CRITICAL, errno,
3789
                           GD_MSG_BRICKINFO_CREATE_FAIL,
3790
                           "realpath() failed for brick %s"
3791
                           ". The underlying file system "
3792
                           "may be in bad state",
3793
                           brickinfo->path);
3794
                    ret = -1;
3795
                    goto out;
3796
                }
3797
                if (strlen(abspath) >= sizeof(brickinfo->real_path)) {
3798
                    ret = -1;
3799
                    goto out;
3800
                }
3801
                (void)strncpy(brickinfo->real_path, abspath,
3802
                              sizeof(brickinfo->real_path));
3803
            }
3804
        }
3805
    }
3806

3807
    ret = 0;
3808
out:
3809
    gf_msg_trace(this->name, 0, "Returning with %d", ret);
3810
    return ret;
3811
}
3812

3813
int32_t
3814
glusterd_resolve_snap_bricks(xlator_t *this, glusterd_snap_t *snap)
3815
{
3816
    int32_t ret = -1;
3817
    glusterd_volinfo_t *volinfo = NULL;
3818
    glusterd_brickinfo_t *brickinfo = NULL;
3819

3820
    GF_VALIDATE_OR_GOTO(this->name, snap, out);
3821

3822
    cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
3823
    {
3824
        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
3825
        {
3826
            ret = glusterd_resolve_brick(brickinfo);
3827
            if (ret) {
3828
                gf_event(EVENT_BRICKPATH_RESOLVE_FAILED,
3829
                         "peer=%s;volume=%s;brick=%s", brickinfo->hostname,
3830
                         volinfo->volname, brickinfo->path);
3831
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
3832
                       "resolve brick failed in restore");
3833
                goto out;
3834
            }
3835
        }
3836
    }
3837

3838
    ret = 0;
3839

3840
out:
3841
    gf_msg_trace(this->name, 0, "Returning with %d", ret);
3842

3843
    return ret;
3844
}
3845

3846
int
3847
glusterd_store_update_snap(glusterd_snap_t *snap)
3848
{
3849
    int ret = -1;
3850
    char *key = NULL;
3851
    char *value = NULL;
3852
    char snappath[PATH_MAX] = {
3853
        0,
3854
    };
3855
    char path[PATH_MAX] = {
3856
        0,
3857
    };
3858
    xlator_t *this = THIS;
3859
    glusterd_conf_t *conf = NULL;
3860
    gf_store_iter_t *iter = NULL;
3861
    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
3862
    int32_t len = 0;
3863

3864
    conf = this->private;
3865
    GF_ASSERT(snap);
3866

3867
    GLUSTERD_GET_SNAP_DIR(snappath, snap, conf);
3868

3869
    len = snprintf(path, sizeof(path), "%s/%s", snappath,
3870
                   GLUSTERD_SNAP_INFO_FILE);
3871
    if ((len < 0) || (len >= sizeof(path))) {
3872
        goto out;
3873
    }
3874

3875
    ret = gf_store_handle_retrieve(path, &snap->shandle);
3876
    if (ret) {
3877
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_HANDLE_NULL,
3878
               "snap handle is NULL");
3879
        goto out;
3880
    }
3881

3882
    ret = gf_store_iter_new(snap->shandle, &iter);
3883
    if (ret) {
3884
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
3885
               "Failed to get new store "
3886
               "iter");
3887
        goto out;
3888
    }
3889

3890
    ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
3891
    if (ret) {
3892
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_GET_FAIL,
3893
               "Failed to get next store "
3894
               "iter");
3895
        goto out;
3896
    }
3897

3898
    while (!ret) {
3899
        gf_msg_debug(this->name, 0, "key = %s value = %s", key, value);
3900

3901
        if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_ID,
3902
                     SLEN(GLUSTERD_STORE_KEY_SNAP_ID))) {
3903
            ret = gf_uuid_parse(value, snap->snap_id);
3904
            if (ret)
3905
                gf_msg(this->name, GF_LOG_WARNING, 0, GD_MSG_UUID_PARSE_FAIL,
3906
                       "Failed to parse uuid");
3907
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_RESTORED,
3908
                            SLEN(GLUSTERD_STORE_KEY_SNAP_RESTORED))) {
3909
            snap->snap_restored = atoi(value);
3910
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_STATUS,
3911
                            SLEN(GLUSTERD_STORE_KEY_SNAP_STATUS))) {
3912
            snap->snap_status = atoi(value);
3913
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_DESC,
3914
                            SLEN(GLUSTERD_STORE_KEY_SNAP_DESC))) {
3915
            snap->description = gf_strdup(value);
3916
        } else if (!strncmp(key, GLUSTERD_STORE_KEY_SNAP_TIMESTAMP,
3917
                            SLEN(GLUSTERD_STORE_KEY_SNAP_TIMESTAMP))) {
3918
            snap->time_stamp = atoi(value);
3919
        }
3920

3921
        GF_FREE(key);
3922
        GF_FREE(value);
3923
        key = NULL;
3924
        value = NULL;
3925

3926
        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
3927
    }
3928

3929
    if (op_errno != GD_STORE_EOF)
3930
        goto out;
3931

3932
    ret = 0;
3933

3934
out:
3935
    if (gf_store_iter_destroy(&iter)) {
3936
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_ITER_DESTROY_FAIL,
3937
               "Failed to destroy store iter");
3938
        ret = -1;
3939
    }
3940

3941
    return ret;
3942
}
3943

3944
int32_t
3945
glusterd_store_retrieve_snap(char *snapname)
3946
{
3947
    int32_t ret = -1;
3948
    glusterd_snap_t *snap = NULL;
3949
    glusterd_conf_t *priv = NULL;
3950
    xlator_t *this = THIS;
3951

3952
    priv = this->private;
3953
    GF_ASSERT(priv);
3954
    GF_ASSERT(snapname);
3955

3956
    snap = glusterd_new_snap_object();
3957
    if (!snap) {
3958
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_OBJECT_STORE_FAIL,
3959
               "Failed to create "
3960
               " snap object");
3961
        goto out;
3962
    }
3963

3964
    if (snprintf(snap->snapname, sizeof(snap->snapname), "%s", snapname) >=
3965
        sizeof(snap->snapname))
3966
        goto out;
3967
    ret = glusterd_store_update_snap(snap);
3968
    if (ret) {
3969
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPSHOT_UPDATE_FAIL,
3970
               "Failed to update snapshot "
3971
               "for %s snap",
3972
               snapname);
3973
        goto out;
3974
    }
3975

3976
    ret = glusterd_store_retrieve_volumes(this, snap);
3977
    if (ret) {
3978
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_VOL_RETRIEVE_FAIL,
3979
               "Failed to retrieve "
3980
               "snap volumes for snap %s",
3981
               snapname);
3982
        goto out;
3983
    }
3984

3985
    /* TODO: list_add_order can do 'N-square' comparisons and
3986
       is not efficient. Find a better solution to store the snap
3987
       in order */
3988
    glusterd_list_add_order(&snap->snap_list, &priv->snapshots,
3989
                            glusterd_compare_snap_time);
3990

3991
out:
3992
    gf_msg_trace(this->name, 0, "Returning with %d", ret);
3993
    return ret;
3994
}
3995

3996
/* Read the missed_snap_list and update the in-memory structs */
3997
int32_t
3998
glusterd_store_retrieve_missed_snaps_list(xlator_t *this)
3999
{
4000
    char path[PATH_MAX] = "";
4001
    char *snap_vol_id = NULL;
4002
    char *missed_node_info = NULL;
4003
    char *brick_path = NULL;
4004
    char *value = NULL;
4005
    char *save_ptr = NULL;
4006
    FILE *fp = NULL;
4007
    int32_t brick_num = -1;
4008
    int32_t snap_op = -1;
4009
    int32_t snap_status = -1;
4010
    int32_t ret = -1;
4011
    glusterd_conf_t *priv = NULL;
4012
    gf_store_op_errno_t store_errno = GD_STORE_SUCCESS;
4013

4014
    priv = this->private;
4015
    GF_ASSERT(priv);
4016

4017
    /* Get the path of the missed_snap_list */
4018
    glusterd_store_missed_snaps_list_path_set(path, sizeof(path));
4019

4020
    fp = fopen(path, "r");
4021
    if (!fp) {
4022
        /* If errno is ENOENT then there are no missed snaps yet */
4023
        if (errno != ENOENT) {
4024
            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
4025
                   "Failed to open %s. ", path);
4026
        } else {
4027
            gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_MISSED_SNAP_LIST_EMPTY,
4028
                   "No missed snaps list.");
4029
            ret = 0;
4030
        }
4031
        goto out;
4032
    }
4033

4034
    char buf[8192];
4035
    do {
4036
        ret = gf_store_read_and_tokenize(fp, &missed_node_info, &value,
4037
                                         &store_errno, buf, 8192);
4038
        if (ret) {
4039
            if (store_errno == GD_STORE_EOF) {
4040
                gf_msg_debug(this->name, 0, "EOF for missed_snap_list");
4041
                ret = 0;
4042
                break;
4043
            }
4044
            gf_msg(this->name, GF_LOG_ERROR, store_errno,
4045
                   GD_MSG_MISSED_SNAP_GET_FAIL,
4046
                   "Failed to fetch data from "
4047
                   "missed_snaps_list.");
4048
            goto out;
4049
        }
4050

4051
        /* Fetch the brick_num, brick_path, snap_op and snap status */
4052
        snap_vol_id = strtok_r(value, ":", &save_ptr);
4053
        brick_num = atoi(strtok_r(NULL, ":", &save_ptr));
4054
        brick_path = strtok_r(NULL, ":", &save_ptr);
4055
        snap_op = atoi(strtok_r(NULL, ":", &save_ptr));
4056
        snap_status = atoi(strtok_r(NULL, ":", &save_ptr));
4057

4058
        if (!missed_node_info || !brick_path || !snap_vol_id || brick_num < 1 ||
4059
            snap_op < 1 || snap_status < 1) {
4060
            gf_msg(this->name, GF_LOG_ERROR, EINVAL,
4061
                   GD_MSG_INVALID_MISSED_SNAP_ENTRY,
4062
                   "Invalid missed_snap_entry");
4063
            ret = -1;
4064
            goto out;
4065
        }
4066

4067
        ret = glusterd_add_new_entry_to_list(missed_node_info, snap_vol_id,
4068
                                             brick_num, brick_path, snap_op,
4069
                                             snap_status);
4070
        if (ret) {
4071
            gf_msg(this->name, GF_LOG_ERROR, 0,
4072
                   GD_MSG_MISSED_SNAP_LIST_STORE_FAIL,
4073
                   "Failed to store missed snaps_list");
4074
            goto out;
4075
        }
4076

4077
    } while (store_errno == GD_STORE_SUCCESS);
4078

4079
    ret = 0;
4080
out:
4081
    if (fp)
4082
        fclose(fp);
4083

4084
    gf_msg_trace(this->name, 0, "Returning with %d", ret);
4085
    return ret;
4086
}
4087

4088
int32_t
4089
glusterd_store_retrieve_snaps(xlator_t *this)
4090
{
4091
    int32_t ret = 0;
4092
    char path[PATH_MAX] = {
4093
        0,
4094
    };
4095
    glusterd_conf_t *priv = NULL;
4096
    DIR *dir = NULL;
4097
    struct dirent *entry = NULL;
4098
    struct dirent scratch[2] = {
4099
        {
4100
            0,
4101
        },
4102
    };
4103
    int32_t len = 0;
4104

4105
    priv = this->private;
4106

4107
    GF_ASSERT(priv);
4108

4109
    len = snprintf(path, PATH_MAX, "%s/snaps", priv->workdir);
4110
    if ((len < 0) || (len >= PATH_MAX)) {
4111
        ret = -1;
4112
        goto out;
4113
    }
4114

4115
    dir = sys_opendir(path);
4116

4117
    if (!dir) {
4118
        /* If snaps dir doesn't exists ignore the error for
4119
           backward compatibility */
4120
        if (errno != ENOENT) {
4121
            ret = -1;
4122
            gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
4123
                   "Unable to open dir %s", path);
4124
        }
4125
        goto out;
4126
    }
4127

4128
    while ((entry = sys_readdir(dir, scratch))) {
4129
        if (gf_irrelevant_entry(entry))
4130
            continue;
4131
        if (strcmp(entry->d_name, GLUSTERD_MISSED_SNAPS_LIST_FILE)) {
4132
            ret = glusterd_store_retrieve_snap(entry->d_name);
4133
            if (ret) {
4134
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESTORE_FAIL,
4135
                       "Unable to restore snapshot: %s", entry->d_name);
4136
                goto out;
4137
            }
4138
        }
4139
    }
4140

4141
    /* Retrieve missed_snaps_list */
4142
    ret = glusterd_store_retrieve_missed_snaps_list(this);
4143
    if (ret) {
4144
        gf_msg_debug(this->name, 0, "Failed to retrieve missed_snaps_list");
4145
        goto out;
4146
    }
4147

4148
out:
4149
    if (dir)
4150
        sys_closedir(dir);
4151
    gf_msg_debug(this->name, 0, "Returning with %d", ret);
4152

4153
    return ret;
4154
}
4155

4156
/* Writes all the contents of conf->missed_snap_list */
4157
int32_t
4158
glusterd_store_write_missed_snapinfo(int32_t fd)
4159
{
4160
    char key[GF_UUID_BUF_SIZE * 2];
4161
    char value[PATH_MAX];
4162
    int32_t ret = -1;
4163
    glusterd_conf_t *priv = NULL;
4164
    glusterd_missed_snap_info *missed_snapinfo = NULL;
4165
    glusterd_snap_op_t *snap_opinfo = NULL;
4166
    xlator_t *this = THIS;
4167

4168
    priv = this->private;
4169
    GF_ASSERT(priv);
4170

4171
    /* Write the missed_snap_entry */
4172
    cds_list_for_each_entry(missed_snapinfo, &priv->missed_snaps_list,
4173
                            missed_snaps)
4174
    {
4175
        cds_list_for_each_entry(snap_opinfo, &missed_snapinfo->snap_ops,
4176
                                snap_ops_list)
4177
        {
4178
            snprintf(key, sizeof(key), "%s:%s", missed_snapinfo->node_uuid,
4179
                     missed_snapinfo->snap_uuid);
4180
            snprintf(value, sizeof(value), "%s:%d:%s:%d:%d",
4181
                     snap_opinfo->snap_vol_id, snap_opinfo->brick_num,
4182
                     snap_opinfo->brick_path, snap_opinfo->op,
4183
                     snap_opinfo->status);
4184
            ret = gf_store_save_value(fd, key, value);
4185
            if (ret) {
4186
                gf_msg(this->name, GF_LOG_ERROR, 0,
4187
                       GD_MSG_MISSEDSNAP_INFO_SET_FAIL,
4188
                       "Failed to write missed snapinfo");
4189
                goto out;
4190
            }
4191
        }
4192
    }
4193

4194
    ret = 0;
4195
out:
4196
    gf_msg_trace(this->name, 0, "Returning %d", ret);
4197
    return ret;
4198
}
4199

4200
/* Adds the missed snap entries to the in-memory conf->missed_snap_list *
4201
 * and writes them to disk */
4202
int32_t
4203
glusterd_store_update_missed_snaps(void)
4204
{
4205
    int32_t fd = -1;
4206
    int32_t ret = -1;
4207
    glusterd_conf_t *priv = NULL;
4208
    xlator_t *this = THIS;
4209

4210
    priv = this->private;
4211
    GF_ASSERT(priv);
4212

4213
    ret = glusterd_store_create_missed_snaps_list_shandle_on_absence();
4214
    if (ret) {
4215
        gf_msg(this->name, GF_LOG_ERROR, 0,
4216
               GD_MSG_MISSED_SNAP_LIST_STORE_HANDLE_GET_FAIL,
4217
               "Unable to obtain "
4218
               "missed_snaps_list store handle.");
4219
        goto out;
4220
    }
4221

4222
    fd = gf_store_mkstemp(priv->missed_snaps_list_shandle);
4223
    if (fd <= 0) {
4224
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
4225
               "Failed to create tmp file");
4226
        ret = -1;
4227
        goto out;
4228
    }
4229

4230
    ret = glusterd_store_write_missed_snapinfo(fd);
4231
    if (ret) {
4232
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_MISSED_SNAP_CREATE_FAIL,
4233
               "Failed to write missed snaps to disk");
4234
        goto out;
4235
    }
4236

4237
    ret = gf_store_rename_tmppath(priv->missed_snaps_list_shandle);
4238
    if (ret) {
4239
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_FILE_OP_FAILED,
4240
               "Failed to rename the tmp file");
4241
        goto out;
4242
    }
4243
out:
4244
    if (ret && (fd > 0)) {
4245
        ret = gf_store_unlink_tmppath(priv->missed_snaps_list_shandle);
4246
        if (ret) {
4247
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_TMP_FILE_UNLINK_FAIL,
4248
                   "Failed to unlink the tmp file");
4249
        }
4250
        ret = -1;
4251
    }
4252

4253
    gf_msg_trace(this->name, 0, "Returning %d", ret);
4254
    return ret;
4255
}
4256

4257
int32_t
4258
glusterd_store_delete_peerinfo(glusterd_peerinfo_t *peerinfo)
4259
{
4260
    int32_t ret = -1;
4261
    glusterd_conf_t *priv = NULL;
4262
    xlator_t *this = THIS;
4263
    char peerdir[PATH_MAX] = {
4264
        0,
4265
    };
4266
    char filepath[PATH_MAX] = {
4267
        0,
4268
    };
4269
    char hostname_path[PATH_MAX] = {
4270
        0,
4271
    };
4272
    int32_t len = 0;
4273

4274
    if (!peerinfo) {
4275
        ret = 0;
4276
        goto out;
4277
    }
4278

4279
    priv = this->private;
4280

4281
    len = snprintf(peerdir, PATH_MAX, "%s/peers", priv->workdir);
4282
    if ((len < 0) || (len >= PATH_MAX)) {
4283
        goto out;
4284
    }
4285

4286
    if (gf_uuid_is_null(peerinfo->uuid)) {
4287
        if (peerinfo->hostname) {
4288
            len = snprintf(filepath, PATH_MAX, "%s/%s", peerdir,
4289
                           peerinfo->hostname);
4290
            if ((len < 0) || (len >= PATH_MAX)) {
4291
                goto out;
4292
            }
4293
        } else {
4294
            ret = 0;
4295
            goto out;
4296
        }
4297
    } else {
4298
        len = snprintf(filepath, PATH_MAX, "%s/%s", peerdir,
4299
                       uuid_utoa(peerinfo->uuid));
4300
        if ((len < 0) || (len >= PATH_MAX)) {
4301
            goto out;
4302
        }
4303
        len = snprintf(hostname_path, PATH_MAX, "%s/%s", peerdir,
4304
                       peerinfo->hostname);
4305
        if ((len < 0) || (len >= PATH_MAX)) {
4306
            goto out;
4307
        }
4308

4309
        ret = sys_unlink(hostname_path);
4310

4311
        if (!ret)
4312
            goto out;
4313
    }
4314

4315
    ret = gf_unlink(filepath) ? 0 : -1;
4316

4317
out:
4318
    if (peerinfo && peerinfo->shandle) {
4319
        gf_store_handle_destroy(peerinfo->shandle);
4320
        peerinfo->shandle = NULL;
4321
    }
4322
    gf_msg_debug(this->name, 0, "Returning with %d", ret);
4323

4324
    return ret;
4325
}
4326

4327
void
4328
glusterd_store_peerinfo_dirpath_set(char *path, size_t len)
4329
{
4330
    glusterd_conf_t *priv = NULL;
4331
    GF_ASSERT(path);
4332
    GF_ASSERT(len >= PATH_MAX);
4333

4334
    priv = THIS->private;
4335
    snprintf(path, len, "%s/peers", priv->workdir);
4336
}
4337

4338
int32_t
4339
glusterd_store_create_peer_dir(void)
4340
{
4341
    int32_t ret = 0;
4342
    char path[PATH_MAX];
4343

4344
    glusterd_store_peerinfo_dirpath_set(path, sizeof(path));
4345
    ret = gf_store_mkdir(path);
4346

4347
    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
4348
    return ret;
4349
}
4350

4351
static int
4352
glusterd_store_uuid_peerpath_set(glusterd_peerinfo_t *peerinfo, char *peerfpath,
4353
                                 size_t len)
4354
{
4355
    char peerdir[PATH_MAX];
4356
    char str[50] = {0};
4357
    int32_t ret = -1;
4358

4359
    GF_ASSERT(peerinfo);
4360
    GF_ASSERT(peerfpath);
4361
    GF_ASSERT(len >= PATH_MAX);
4362

4363
    glusterd_store_peerinfo_dirpath_set(peerdir, sizeof(peerdir));
4364
    gf_uuid_unparse(peerinfo->uuid, str);
4365
    ret = snprintf(peerfpath, len, "%s/%s", peerdir, str);
4366
    if (ret < 0 || ret >= len)
4367
        return -1;
4368
    return 0;
4369
}
4370

4371
static int
4372
glusterd_store_hostname_peerpath_set(glusterd_peerinfo_t *peerinfo,
4373
                                     char *peerfpath, size_t len)
4374
{
4375
    char peerdir[PATH_MAX];
4376
    int32_t ret = 0;
4377
    GF_ASSERT(peerinfo);
4378
    GF_ASSERT(peerfpath);
4379
    GF_ASSERT(len >= PATH_MAX);
4380

4381
    glusterd_store_peerinfo_dirpath_set(peerdir, sizeof(peerdir));
4382
    ret = snprintf(peerfpath, len, "%s/%s", peerdir, peerinfo->hostname);
4383
    if (ret < 0 || ret >= len)
4384
        return -1;
4385
    return 0;
4386
}
4387

4388
int32_t
4389
glusterd_store_peerinfo_hostname_shandle_create(glusterd_peerinfo_t *peerinfo)
4390
{
4391
    char peerfpath[PATH_MAX];
4392
    int32_t ret = -1;
4393

4394
    ret = glusterd_store_hostname_peerpath_set(peerinfo, peerfpath,
4395
                                               sizeof(peerfpath));
4396
    if (ret)
4397
        return ret;
4398
    ret = gf_store_handle_create_on_absence(&peerinfo->shandle, peerfpath);
4399
    return ret;
4400
}
4401

4402
int32_t
4403
glusterd_store_peerinfo_uuid_shandle_create(glusterd_peerinfo_t *peerinfo)
4404
{
4405
    char peerfpath[PATH_MAX];
4406
    int32_t ret = -1;
4407

4408
    ret = glusterd_store_uuid_peerpath_set(peerinfo, peerfpath,
4409
                                           sizeof(peerfpath));
4410
    if (ret)
4411
        return ret;
4412

4413
    ret = gf_store_handle_create_on_absence(&peerinfo->shandle, peerfpath);
4414
    return ret;
4415
}
4416

4417
int32_t
4418
glusterd_peerinfo_hostname_shandle_check_destroy(glusterd_peerinfo_t *peerinfo)
4419
{
4420
    char peerfpath[PATH_MAX];
4421
    int32_t ret = -1;
4422
    struct stat stbuf = {
4423
        0,
4424
    };
4425

4426
    ret = glusterd_store_hostname_peerpath_set(peerinfo, peerfpath,
4427
                                               sizeof(peerfpath));
4428
    if (ret)
4429
        return ret;
4430
    ret = sys_stat(peerfpath, &stbuf);
4431
    if (!ret) {
4432
        if (peerinfo->shandle)
4433
            gf_store_handle_destroy(peerinfo->shandle);
4434
        peerinfo->shandle = NULL;
4435
        ret = sys_unlink(peerfpath);
4436
    }
4437
    return ret;
4438
}
4439

4440
int32_t
4441
glusterd_store_create_peer_shandle(glusterd_peerinfo_t *peerinfo)
4442
{
4443
    int32_t ret = 0;
4444

4445
    GF_ASSERT(peerinfo);
4446

4447
    if (gf_uuid_is_null(peerinfo->uuid)) {
4448
        ret = glusterd_store_peerinfo_hostname_shandle_create(peerinfo);
4449
    } else {
4450
        ret = glusterd_peerinfo_hostname_shandle_check_destroy(peerinfo);
4451
        ret = glusterd_store_peerinfo_uuid_shandle_create(peerinfo);
4452
    }
4453
    return ret;
4454
}
4455

4456
static int32_t
4457
glusterd_store_peer_write(int fd, glusterd_peerinfo_t *peerinfo)
4458
{
4459
    char buf[PATH_MAX];
4460
    uint total_len = 0;
4461
    int32_t ret = 0;
4462
    int32_t i = 1;
4463
    glusterd_peer_hostname_t *hostname = NULL;
4464

4465
    ret = snprintf(buf + total_len, sizeof(buf) - total_len, "%s=%s\n%s=%d\n",
4466
                   GLUSTERD_STORE_KEY_PEER_UUID, uuid_utoa(peerinfo->uuid),
4467
                   GLUSTERD_STORE_KEY_PEER_STATE, peerinfo->state);
4468
    if (ret < 0 || ret >= sizeof(buf) - total_len) {
4469
        ret = -1;
4470
        goto out;
4471
    }
4472
    total_len += ret;
4473

4474
    cds_list_for_each_entry(hostname, &peerinfo->hostnames, hostname_list)
4475
    {
4476
        ret = snprintf(buf + total_len, sizeof(buf) - total_len,
4477
                       GLUSTERD_STORE_KEY_PEER_HOSTNAME "%d=%s\n", i,
4478
                       hostname->hostname);
4479
        if (ret < 0 || ret >= sizeof(buf) - total_len) {
4480
            ret = -1;
4481
            goto out;
4482
        }
4483
        total_len += ret;
4484
        i++;
4485
    }
4486

4487
    ret = gf_store_save_items(fd, buf);
4488
out:
4489
    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
4490
    return ret;
4491
}
4492

4493
int32_t
4494
glusterd_store_perform_peer_store(glusterd_peerinfo_t *peerinfo)
4495
{
4496
    int fd = -1;
4497
    int32_t ret = -1;
4498

4499
    GF_ASSERT(peerinfo);
4500

4501
    fd = gf_store_mkstemp(peerinfo->shandle);
4502
    if (fd <= 0) {
4503
        ret = -1;
4504
        goto out;
4505
    }
4506

4507
    ret = glusterd_store_peer_write(fd, peerinfo);
4508
    if (ret)
4509
        goto out;
4510

4511
    ret = gf_store_rename_tmppath(peerinfo->shandle);
4512
out:
4513
    if (ret && (fd > 0))
4514
        gf_store_unlink_tmppath(peerinfo->shandle);
4515
    gf_msg_debug("glusterd", 0, "Returning %d", ret);
4516
    return ret;
4517
}
4518

4519
int32_t
4520
glusterd_store_peerinfo(glusterd_peerinfo_t *peerinfo)
4521
{
4522
    int32_t ret = -1;
4523

4524
    GF_ASSERT(peerinfo);
4525

4526
    ret = glusterd_store_create_peer_dir();
4527
    if (ret)
4528
        goto out;
4529

4530
    ret = glusterd_store_create_peer_shandle(peerinfo);
4531
    if (ret)
4532
        goto out;
4533

4534
    ret = glusterd_store_perform_peer_store(peerinfo);
4535
out:
4536
    gf_msg_debug("glusterd", 0, "Returning with %d", ret);
4537
    return ret;
4538
}
4539

4540
int32_t
4541
glusterd_store_retrieve_peers(xlator_t *this)
4542
{
4543
    int32_t ret = 0;
4544
    glusterd_conf_t *priv = NULL;
4545
    DIR *dir = NULL;
4546
    struct dirent *entry = NULL;
4547
    struct dirent scratch[2] = {
4548
        {
4549
            0,
4550
        },
4551
    };
4552
    char path[PATH_MAX] = {
4553
        0,
4554
    };
4555
    glusterd_peerinfo_t *peerinfo = NULL;
4556
    gf_store_handle_t *shandle = NULL;
4557
    char filepath[PATH_MAX] = {
4558
        0,
4559
    };
4560
    gf_store_iter_t *iter = NULL;
4561
    char *key = NULL;
4562
    char *value = NULL;
4563
    glusterd_peerctx_args_t args = {0};
4564
    gf_store_op_errno_t op_errno = GD_STORE_SUCCESS;
4565
    glusterd_peer_hostname_t *address = NULL;
4566
    uuid_t tmp_uuid;
4567
    gf_boolean_t is_ok;
4568
    int32_t len;
4569

4570
    priv = this->private;
4571

4572
    GF_ASSERT(priv);
4573

4574
    len = snprintf(path, PATH_MAX, "%s/%s", priv->workdir,
4575
                   GLUSTERD_PEER_DIR_PREFIX);
4576
    if ((len < 0) || (len >= PATH_MAX)) {
4577
        ret = -1;
4578
        goto out;
4579
    }
4580

4581
    dir = sys_opendir(path);
4582

4583
    if (!dir) {
4584
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_DIR_OP_FAILED,
4585
               "Unable to open dir %s", path);
4586
        ret = -1;
4587
        goto out;
4588
    }
4589

4590
    while ((entry = sys_readdir(dir, scratch))) {
4591
        if (gf_irrelevant_entry(entry))
4592
            continue;
4593
        if (gf_uuid_parse(entry->d_name, tmp_uuid) != 0) {
4594
            gf_log(this->name, GF_LOG_WARNING, "skipping non-peer file %s",
4595
                   entry->d_name);
4596
            continue;
4597
        }
4598
        is_ok = _gf_false;
4599
        len = snprintf(filepath, PATH_MAX, "%s/%s", path, entry->d_name);
4600
        if ((len < 0) || (len >= PATH_MAX)) {
4601
            goto next;
4602
        }
4603
        ret = gf_store_handle_retrieve(filepath, &shandle);
4604
        if (ret)
4605
            goto next;
4606

4607
        ret = gf_store_iter_new(shandle, &iter);
4608
        if (ret)
4609
            goto next;
4610

4611
        ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
4612
        if (ret) {
4613
            goto next;
4614
        }
4615

4616
        /* Create an empty peerinfo object before reading in the
4617
         * details
4618
         */
4619
        peerinfo = glusterd_peerinfo_new(GD_FRIEND_STATE_DEFAULT, NULL, NULL,
4620
                                         0);
4621
        if (peerinfo == NULL) {
4622
            GF_FREE(key);
4623
            GF_FREE(value);
4624
            key = NULL;
4625
            value = NULL;
4626
            ret = -1;
4627
            goto next;
4628
        }
4629

4630
        while (!ret) {
4631
            if (!strncmp(GLUSTERD_STORE_KEY_PEER_UUID, key,
4632
                         SLEN(GLUSTERD_STORE_KEY_PEER_UUID))) {
4633
                if (value)
4634
                    gf_uuid_parse(value, peerinfo->uuid);
4635
            } else if (!strncmp(GLUSTERD_STORE_KEY_PEER_STATE, key,
4636
                                SLEN(GLUSTERD_STORE_KEY_PEER_STATE))) {
4637
                peerinfo->state = atoi(value);
4638
            } else if (!strncmp(GLUSTERD_STORE_KEY_PEER_HOSTNAME, key,
4639
                                SLEN(GLUSTERD_STORE_KEY_PEER_HOSTNAME))) {
4640
                ret = gd_add_address_to_peer(peerinfo, value, _gf_false);
4641
                if (ret) {
4642
                    gf_msg(this->name, GF_LOG_ERROR, 0,
4643
                           GD_MSG_ADD_ADDRESS_TO_PEER_FAIL,
4644
                           "Could not add address to peer");
4645
                }
4646
            } else {
4647
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_UNKNOWN_KEY,
4648
                       "Unknown key: %s", key);
4649
            }
4650

4651
            GF_FREE(key);
4652
            GF_FREE(value);
4653
            key = NULL;
4654
            value = NULL;
4655

4656
            ret = gf_store_iter_get_next(iter, &key, &value, &op_errno);
4657
        }
4658
        if (op_errno != GD_STORE_EOF) {
4659
            goto next;
4660
        }
4661

4662
        if (gf_uuid_is_null(peerinfo->uuid)) {
4663
            gf_log("", GF_LOG_ERROR,
4664
                   "Null UUID while attempting to read peer from '%s'",
4665
                   filepath);
4666
            goto next;
4667
        }
4668

4669
        /* Set first hostname from peerinfo->hostnames to
4670
         * peerinfo->hostname
4671
         */
4672
        address = cds_list_entry(peerinfo->hostnames.next,
4673
                                 glusterd_peer_hostname_t, hostname_list);
4674
        peerinfo->hostname = gf_strdup(address->hostname);
4675

4676
        ret = glusterd_friend_add_from_peerinfo(peerinfo, 1, NULL);
4677
        if (ret)
4678
            goto next;
4679

4680
        peerinfo->shandle = shandle;
4681
        is_ok = _gf_true;
4682

4683
    next:
4684
        (void)gf_store_iter_destroy(&iter);
4685

4686
        if (!is_ok) {
4687
            gf_log(this->name, GF_LOG_WARNING,
4688
                   "skipping malformed peer file %s", entry->d_name);
4689
            if (peerinfo) {
4690
                glusterd_peerinfo_cleanup(peerinfo);
4691
            }
4692
        }
4693
        peerinfo = NULL;
4694
    }
4695

4696
    args.mode = GD_MODE_ON;
4697

4698
    RCU_READ_LOCK;
4699
    cds_list_for_each_entry_rcu(peerinfo, &priv->peers, uuid_list)
4700
    {
4701
        ret = glusterd_friend_rpc_create(this, peerinfo, &args);
4702
        if (ret)
4703
            break;
4704
    }
4705
    RCU_READ_UNLOCK;
4706
    peerinfo = NULL;
4707

4708
out:
4709

4710
    if (dir)
4711
        sys_closedir(dir);
4712
    gf_msg_debug(this->name, 0, "Returning with %d", ret);
4713

4714
    return ret;
4715
}
4716

4717
/* Bricks for snap volumes are hosted at /var/run/gluster/snaps
4718
 * When a volume is restored, it points to the bricks of the snap
4719
 * volume it was restored from. Hence on a node restart these
4720
 * paths need to be recreated and re-mounted
4721
 */
4722
int32_t
4723
glusterd_recreate_all_snap_brick_mounts(xlator_t *this)
4724
{
4725
    int32_t ret = 0;
4726
    glusterd_conf_t *priv = NULL;
4727
    glusterd_volinfo_t *volinfo = NULL;
4728
    glusterd_snap_t *snap = NULL;
4729

4730
    priv = this->private;
4731
    GF_ASSERT(priv);
4732

4733
    /* Recreate bricks of volumes restored from snaps */
4734
    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
4735
    {
4736
        /* If the volume is not a restored volume then continue */
4737
        if (gf_uuid_is_null(volinfo->restored_from_snap))
4738
            continue;
4739

4740
        ret = glusterd_recreate_vol_brick_mounts(this, volinfo);
4741
        if (ret) {
4742
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRK_MNT_RECREATE_FAIL,
4743
                   "Failed to recreate brick mounts "
4744
                   "for %s",
4745
                   volinfo->volname);
4746
            goto out;
4747
        }
4748
    }
4749

4750
    /* Recreate bricks of snapshot volumes
4751
     * We are not creating brick mounts for stopped snaps.
4752
     */
4753
    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
4754
    {
4755
        cds_list_for_each_entry(volinfo, &snap->volumes, vol_list)
4756
        {
4757
            if (volinfo->status != GLUSTERD_STATUS_STOPPED) {
4758
                ret = glusterd_recreate_vol_brick_mounts(this, volinfo);
4759
                if (ret) {
4760
                    gf_msg(this->name, GF_LOG_ERROR, 0,
4761
                           GD_MSG_BRK_MNT_RECREATE_FAIL,
4762
                           "Failed to recreate brick "
4763
                           "mounts for %s",
4764
                           snap->snapname);
4765
                    goto out;
4766
                }
4767
            }
4768
        }
4769
    }
4770

4771
out:
4772
    gf_msg_trace(this->name, 0, "Returning with %d", ret);
4773
    return ret;
4774
}
4775

4776
/* When the snapshot command from cli is received, the on disk and
4777
 * in memory structures for the snapshot are created (with the status)
4778
 * being marked as GD_SNAP_STATUS_INIT. Once the backend snapshot is
4779
 * taken, the status is changed to GD_SNAP_STATUS_IN_USE. If glusterd
4780
 * dies after taking the backend snapshot, but before updating the
4781
 * status, then when glusterd comes up, it should treat that snapshot
4782
 * as a failed snapshot and clean it up.
4783
 *
4784
 * Restore operation starts by setting the status to
4785
 * GD_SNAP_STATUS_RESTORED. If the server goes down before changing
4786
 * the status the status back we need to revert the partial snapshot
4787
 * taken.
4788
 */
4789
int32_t
4790
glusterd_snap_cleanup(xlator_t *this)
4791
{
4792
    dict_t *dict = NULL;
4793
    int32_t ret = 0;
4794
    glusterd_conf_t *priv = NULL;
4795
    glusterd_snap_t *snap = NULL;
4796
    glusterd_snap_t *tmp_snap = NULL;
4797

4798
    priv = this->private;
4799
    GF_ASSERT(priv);
4800

4801
    dict = dict_new();
4802
    if (!dict) {
4803
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_DICT_CREATE_FAIL,
4804
               "Failed to create dict");
4805
        ret = -1;
4806
        goto out;
4807
    }
4808

4809
    cds_list_for_each_entry_safe(snap, tmp_snap, &priv->snapshots, snap_list)
4810
    {
4811
        if (snap->snap_status == GD_SNAP_STATUS_RESTORED) {
4812
            ret = glusterd_snapshot_revert_restore_from_snap(snap);
4813
            if (ret) {
4814
                gf_msg(this->name, GF_LOG_WARNING, 0,
4815
                       GD_MSG_SNAP_RESTORE_REVERT_FAIL,
4816
                       "Failed to "
4817
                       "revert partially restored snapshot "
4818
                       "(%s)",
4819
                       snap->snapname);
4820
                goto out;
4821
            }
4822
        } else if (snap->snap_status != GD_SNAP_STATUS_IN_USE) {
4823
            ret = glusterd_snap_remove(dict, snap, _gf_true, _gf_true,
4824
                                       _gf_false);
4825
            if (ret) {
4826
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_REMOVE_FAIL,
4827
                       "Failed to remove the snapshot %s", snap->snapname);
4828
                goto out;
4829
            }
4830
        }
4831
    }
4832
out:
4833
    if (dict)
4834
        dict_unref(dict);
4835

4836
    gf_msg_trace(this->name, 0, "Returning with %d", ret);
4837
    return ret;
4838
}
4839

4840
int32_t
4841
glusterd_resolve_all_bricks(xlator_t *this)
4842
{
4843
    int32_t ret = 0;
4844
    glusterd_conf_t *priv = NULL;
4845
    glusterd_volinfo_t *volinfo = NULL;
4846
    glusterd_brickinfo_t *brickinfo = NULL;
4847
    glusterd_snap_t *snap = NULL;
4848

4849
    priv = this->private;
4850

4851
    GF_ASSERT(priv);
4852

4853
    /* Resolve bricks of volumes */
4854
    cds_list_for_each_entry(volinfo, &priv->volumes, vol_list)
4855
    {
4856
        cds_list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
4857
        {
4858
            ret = glusterd_resolve_brick(brickinfo);
4859
            if (ret) {
4860
                gf_event(EVENT_BRICKPATH_RESOLVE_FAILED,
4861
                         "peer=%s;volume=%s;brick=%s", brickinfo->hostname,
4862
                         volinfo->volname, brickinfo->path);
4863
                gf_msg("glusterd", GF_LOG_ERROR, 0, GD_MSG_RESOLVE_BRICK_FAIL,
4864
                       "Failed to resolve brick %s with host %s of volume %s"
4865
                       " in restore",
4866
                       brickinfo->path, brickinfo->hostname, volinfo->volname);
4867
                goto out;
4868
            }
4869
        }
4870
    }
4871

4872
    /* Resolve bricks of snapshot volumes */
4873
    cds_list_for_each_entry(snap, &priv->snapshots, snap_list)
4874
    {
4875
        ret = glusterd_resolve_snap_bricks(this, snap);
4876
        if (ret) {
4877
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_RESOLVE_BRICK_FAIL,
4878
                   "resolving the snap bricks"
4879
                   " failed for snap: %s",
4880
                   snap->snapname);
4881
            goto out;
4882
        }
4883
    }
4884

4885
out:
4886
    gf_msg_trace(this->name, 0, "Returning with %d", ret);
4887
    return ret;
4888
}
4889

4890
int32_t
4891
glusterd_restore(void)
4892
{
4893
    int32_t ret = -1;
4894
    xlator_t *this = THIS;
4895

4896
    ret = glusterd_options_init(this);
4897
    if (ret < 0)
4898
        goto out;
4899

4900
    ret = glusterd_store_retrieve_volumes(this, NULL);
4901
    if (ret)
4902
        goto out;
4903

4904
    ret = glusterd_store_retrieve_peers(this);
4905
    if (ret)
4906
        goto out;
4907

4908
    /* While retrieving snapshots, if the snapshot status
4909
       is not GD_SNAP_STATUS_IN_USE, then the snapshot is
4910
       cleaned up. To do that, the snap volume has to be
4911
       stopped by stopping snapshot volume's bricks. And for
4912
       that the snapshot bricks should be resolved. But without
4913
       retrieving the peers, resolving bricks will fail. So
4914
       do retrieving of snapshots after retrieving peers.
4915
    */
4916
    ret = glusterd_store_retrieve_snaps(this);
4917
    if (ret)
4918
        goto out;
4919

4920
    ret = glusterd_resolve_all_bricks(this);
4921
    if (ret)
4922
        goto out;
4923

4924
    ret = glusterd_snap_cleanup(this);
4925
    if (ret) {
4926
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_CLEANUP_FAIL,
4927
               "Failed to perform "
4928
               "a cleanup of the snapshots");
4929
        goto out;
4930
    }
4931

4932
    ret = glusterd_recreate_all_snap_brick_mounts(this);
4933
    if (ret) {
4934
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAP_BRK_MNT_RECREATE_FAIL,
4935
               "Failed to recreate "
4936
               "all snap brick mounts");
4937
        goto out;
4938
    }
4939

4940
out:
4941
    gf_msg_debug(this->name, 0, "Returning %d", ret);
4942
    return ret;
4943
}
4944

4945
int
4946
glusterd_store_retrieve_quota_version(glusterd_volinfo_t *volinfo)
4947
{
4948
    int ret = -1;
4949
    uint32_t version = 0;
4950
    char cksum_path[PATH_MAX] = {
4951
        0,
4952
    };
4953
    char path[PATH_MAX] = {
4954
        0,
4955
    };
4956
    char *version_str = NULL;
4957
    char *tmp = NULL;
4958
    xlator_t *this = THIS;
4959
    glusterd_conf_t *conf = NULL;
4960
    gf_store_handle_t *handle = NULL;
4961
    int32_t len = 0;
4962

4963
    conf = this->private;
4964
    GF_ASSERT(conf);
4965

4966
    GLUSTERD_GET_VOLUME_DIR(path, volinfo, conf);
4967
    len = snprintf(cksum_path, sizeof(cksum_path), "%s/%s", path,
4968
                   GLUSTERD_VOL_QUOTA_CKSUM_FILE);
4969
    if ((len < 0) || (len >= sizeof(cksum_path))) {
4970
        goto out;
4971
    }
4972

4973
    ret = gf_store_handle_new(cksum_path, &handle);
4974
    if (ret) {
4975
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_STORE_HANDLE_GET_FAIL,
4976
               "Unable to get store handle "
4977
               "for %s",
4978
               cksum_path);
4979
        goto out;
4980
    }
4981

4982
    ret = gf_store_retrieve_value(handle, "version", &version_str);
4983
    if (ret) {
4984
        gf_msg_debug(this->name, 0, "Version absent");
4985
        ret = 0;
4986
        goto out;
4987
    }
4988

4989
    version = strtoul(version_str, &tmp, 10);
4990
    if ((errno == ERANGE) || (errno == EINVAL)) {
4991
        gf_msg_debug(this->name, 0, "Invalid version number");
4992
        goto out;
4993
    }
4994
    volinfo->quota_conf_version = version;
4995
    ret = 0;
4996

4997
out:
4998
    if (version_str)
4999
        GF_FREE(version_str);
5000
    gf_store_handle_destroy(handle);
5001
    return ret;
5002
}
5003

5004
int
5005
glusterd_store_save_quota_version_and_cksum(glusterd_volinfo_t *volinfo)
5006
{
5007
    gf_store_handle_t *shandle = NULL;
5008
    glusterd_conf_t *conf = NULL;
5009
    xlator_t *this = THIS;
5010
    char path[PATH_MAX] = {0};
5011
    char cksum_path[PATH_MAX + 32] = {
5012
        0,
5013
    };
5014
    char buf[64] = {0};
5015
    int fd = -1;
5016
    int32_t ret = -1;
5017
    int32_t len = 0;
5018

5019
    conf = this->private;
5020

5021
    GLUSTERD_GET_VOLUME_DIR(path, volinfo, conf);
5022
    len = snprintf(cksum_path, sizeof(cksum_path), "%s/%s", path,
5023
                   GLUSTERD_VOL_QUOTA_CKSUM_FILE);
5024
    if ((len < 0) || (len >= sizeof(cksum_path))) {
5025
        goto out;
5026
    }
5027

5028
    ret = gf_store_handle_new(cksum_path, &shandle);
5029
    if (ret)
5030
        goto out;
5031

5032
    fd = gf_store_mkstemp(shandle);
5033
    if (fd <= 0) {
5034
        ret = -1;
5035
        goto out;
5036
    }
5037

5038
    ret = snprintf(buf, sizeof(buf), "cksum=%u\nversion=%u\n",
5039
                   volinfo->quota_conf_cksum, volinfo->quota_conf_version);
5040
    if (ret < 0 || ret >= sizeof(buf)) {
5041
        ret = -1;
5042
        goto out;
5043
    }
5044

5045
    ret = gf_store_save_items(fd, buf);
5046
    if (ret) {
5047
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_CKSUM_STORE_FAIL,
5048
               "Failed to store quota cksum and version");
5049
        goto out;
5050
    }
5051

5052
    ret = gf_store_rename_tmppath(shandle);
5053
    if (ret)
5054
        goto out;
5055

5056
out:
5057
    if ((ret < 0) && (fd > 0))
5058
        gf_store_unlink_tmppath(shandle);
5059
    gf_store_handle_destroy(shandle);
5060
    return ret;
5061
}
5062

5063
int32_t
5064
glusterd_quota_conf_write_header(int fd)
5065
{
5066
    int header_len = SLEN(QUOTA_CONF_HEADER);
5067
    int ret;
5068

5069
    ret = gf_nwrite(fd, QUOTA_CONF_HEADER, header_len);
5070

5071
    if (ret != header_len) {
5072
        gf_msg_callingfn("quota", GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONF_WRITE_FAIL,
5073
                         "failed to write "
5074
                         "header to a quota conf");
5075
        return -1;
5076
    }
5077

5078
    return 0;
5079
}
5080

5081
int32_t
5082
glusterd_quota_conf_write_gfid(int fd, void *buf, char type)
5083
{
5084
    int ret;
5085

5086
    ret = gf_nwrite(fd, buf, 16);
5087
    if (ret != 16) {
5088
        ret = -1;
5089
        goto out;
5090
    }
5091

5092
    ret = gf_nwrite(fd, &type, 1);
5093
    if (ret != 1) {
5094
        ret = -1;
5095
        goto out;
5096
    }
5097

5098
    ret = 0;
5099

5100
out:
5101
    if (ret < 0)
5102
        gf_msg_callingfn("quota", GF_LOG_ERROR, 0, GD_MSG_QUOTA_CONF_WRITE_FAIL,
5103
                         "failed to write "
5104
                         "gfid %s to a quota conf",
5105
                         uuid_utoa(buf));
5106

5107
    return ret;
5108
}
5109