glusterfs

1
/*
2
   Copyright (c) 2015 Red Hat, Inc. <http://www.redhat.com>
3
   This file is part of GlusterFS.
4

5
   This file is licensed to you under your choice of the GNU Lesser
6
   General Public License, version 3 or any later version (LGPLv3 or
7
   later), or the GNU General Public License, version 2 (GPLv2), in all
8
   cases as published by the Free Software Foundation.
9
*/
10
#include <glusterfs/common-utils.h>
11
#include "glusterd-utils.h"
12
#include "glusterd-messages.h"
13
#include "glusterd-server-quorum.h"
14
#include "glusterd-store.h"
15
#include "glusterd-syncop.h"
16
#include "glusterd-op-sm.h"
17

18
#define CEILING_POS(X) (((X) - (int)(X)) > 0 ? (int)((X) + 1) : (int)(X))
19

20
static gf_boolean_t
21
glusterd_is_get_op(xlator_t *this, glusterd_op_t op, dict_t *dict)
22
{
23
    char *key = NULL;
24
    char *volname = NULL;
25
    int ret = 0;
26

27
    if (op == GD_OP_STATUS_VOLUME)
28
        return _gf_true;
29

30
    if (op == GD_OP_SET_VOLUME) {
31
        /*check for set volume help*/
32
        ret = dict_get_str(dict, "volname", &volname);
33
        if (volname && ((strcmp(volname, "help") == 0) ||
34
                        (strcmp(volname, "help-xml") == 0))) {
35
            ret = dict_get_str(dict, "key1", &key);
36
            if (ret < 0)
37
                return _gf_true;
38
        }
39
    }
40
    return _gf_false;
41
}
42

43
gf_boolean_t
44
glusterd_is_quorum_validation_required(xlator_t *this, glusterd_op_t op,
45
                                       dict_t *dict)
46
{
47
    gf_boolean_t required = _gf_true;
48
    char *key = NULL;
49
    char *key_fixed = NULL;
50
    int ret = -1;
51

52
    if (glusterd_is_get_op(this, op, dict)) {
53
        required = _gf_false;
54
        goto out;
55
    }
56
    if ((op != GD_OP_SET_VOLUME) && (op != GD_OP_RESET_VOLUME))
57
        goto out;
58
    if (op == GD_OP_SET_VOLUME)
59
        ret = dict_get_str(dict, "key1", &key);
60
    else if (op == GD_OP_RESET_VOLUME)
61
        ret = dict_get_str(dict, "key", &key);
62
    if (ret)
63
        goto out;
64
    ret = glusterd_check_option_exists(key, &key_fixed);
65
    if (ret <= 0)
66
        goto out;
67
    if (key_fixed)
68
        key = key_fixed;
69
    if (glusterd_is_quorum_option(key))
70
        required = _gf_false;
71
out:
72
    GF_FREE(key_fixed);
73
    return required;
74
}
75

76
int
77
glusterd_validate_quorum(xlator_t *this, glusterd_op_t op, dict_t *dict,
78
                         char **op_errstr)
79
{
80
    int ret = 0;
81
    char *volname = NULL;
82
    glusterd_volinfo_t *volinfo = NULL;
83
    char *errstr = NULL;
84

85
    errstr = "Quorum not met. Volume operation not allowed.";
86
    if (!glusterd_is_quorum_validation_required(this, op, dict))
87
        goto out;
88

89
    ret = dict_get_str(dict, "volname", &volname);
90
    if (ret) {
91
        gf_smsg(this->name, GF_LOG_ERROR, -ret, GD_MSG_DICT_GET_FAILED,
92
                "Key=volname", NULL);
93
        ret = 0;
94
        goto out;
95
    }
96

97
    ret = glusterd_volinfo_find(volname, &volinfo);
98
    if (ret) {
99
        ret = 0;
100
        goto out;
101
    }
102

103
    if (!glusterd_is_volume_in_server_quorum(volinfo)) {
104
        ret = 0;
105
        goto out;
106
    }
107

108
    if (does_gd_meet_server_quorum(this)) {
109
        ret = 0;
110
        goto out;
111
    }
112

113
    ret = -1;
114
    *op_errstr = gf_strdup(errstr);
115

116
out:
117
    return ret;
118
}
119

120
gf_boolean_t
121
glusterd_is_quorum_option(char *option)
122
{
123
    gf_boolean_t res = _gf_false;
124
    int i = 0;
125
    static const char *const keys[] = {GLUSTERD_QUORUM_TYPE_KEY,
126
                                       GLUSTERD_QUORUM_RATIO_KEY, NULL};
127

128
    for (i = 0; keys[i]; i++) {
129
        if (strcmp(option, keys[i]) == 0) {
130
            res = _gf_true;
131
            break;
132
        }
133
    }
134
    return res;
135
}
136

137
gf_boolean_t
138
glusterd_is_quorum_changed(dict_t *options, char *option, char *value)
139
{
140
    int ret = 0;
141
    gf_boolean_t reconfigured = _gf_false;
142
    gf_boolean_t all = _gf_false;
143
    char *oldquorum = NULL;
144
    char *newquorum = NULL;
145
    char *oldratio = NULL;
146
    char *newratio = NULL;
147
    xlator_t *this = THIS;
148

149
    if ((strcmp("all", option) != 0) && !glusterd_is_quorum_option(option))
150
        goto out;
151

152
    if (strcmp("all", option) == 0)
153
        all = _gf_true;
154

155
    if (all || (strcmp(GLUSTERD_QUORUM_TYPE_KEY, option) == 0)) {
156
        newquorum = value;
157
        ret = dict_get_str(options, GLUSTERD_QUORUM_TYPE_KEY, &oldquorum);
158
        if (ret)
159
            gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_DICT_GET_FAILED,
160
                   "dict_get_str failed on %s", GLUSTERD_QUORUM_TYPE_KEY);
161
    }
162

163
    if (all || (strcmp(GLUSTERD_QUORUM_RATIO_KEY, option) == 0)) {
164
        newratio = value;
165
        ret = dict_get_str(options, GLUSTERD_QUORUM_RATIO_KEY, &oldratio);
166
        if (ret)
167
            gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_DICT_GET_FAILED,
168
                   "dict_get_str failed on %s", GLUSTERD_QUORUM_RATIO_KEY);
169
    }
170

171
    reconfigured = _gf_true;
172

173
    if (oldquorum && newquorum && (strcmp(oldquorum, newquorum) == 0))
174
        reconfigured = _gf_false;
175
    if (oldratio && newratio && (strcmp(oldratio, newratio) == 0))
176
        reconfigured = _gf_false;
177

178
    if ((oldratio == NULL) && (newratio == NULL) && (oldquorum == NULL) &&
179
        (newquorum == NULL))
180
        reconfigured = _gf_false;
181
out:
182
    return reconfigured;
183
}
184

185
static gf_boolean_t
186
_is_contributing_to_quorum(gd_quorum_contrib_t contrib)
187
{
188
    if ((contrib == QUORUM_UP) || (contrib == QUORUM_DOWN))
189
        return _gf_true;
190
    return _gf_false;
191
}
192

193
gf_boolean_t
194
does_quorum_meet(int active_count, int quorum_count)
195
{
196
    return (active_count >= quorum_count);
197
}
198

199
int
200
glusterd_get_quorum_cluster_counts(xlator_t *this, int *active_count,
201
                                   int *quorum_count)
202
{
203
    glusterd_peerinfo_t *peerinfo = NULL;
204
    glusterd_conf_t *conf = NULL;
205
    int ret = -1;
206
    int inquorum_count = 0;
207
    char *val = NULL;
208
    double quorum_percentage = 0.0;
209
    gf_boolean_t ratio = _gf_false;
210
    int count = 0;
211

212
    conf = this->private;
213

214
    /* Start with counting self */
215
    inquorum_count = 1;
216
    if (active_count)
217
        *active_count = 1;
218

219
    RCU_READ_LOCK;
220
    cds_list_for_each_entry_rcu(peerinfo, &conf->peers, uuid_list)
221
    {
222
        if (_is_contributing_to_quorum(peerinfo->quorum_contrib))
223
            inquorum_count = inquorum_count + 1;
224
        if (active_count && (peerinfo->quorum_contrib == QUORUM_UP))
225
            *active_count = *active_count + 1;
226
    }
227
    RCU_READ_UNLOCK;
228

229
    ret = dict_get_str(conf->opts, GLUSTERD_QUORUM_RATIO_KEY, &val);
230
    if (ret == 0) {
231
        ret = gf_string2percent(val, &quorum_percentage);
232
        if (ret == 0)
233
            ratio = _gf_true;
234
    }
235
    if (ratio)
236
        count = CEILING_POS(inquorum_count * quorum_percentage / 100.0);
237
    else
238
        count = (inquorum_count * 50 / 100) + 1;
239

240
    *quorum_count = count;
241
    ret = 0;
242

243
    return ret;
244
}
245

246
gf_boolean_t
247
glusterd_is_volume_in_server_quorum(glusterd_volinfo_t *volinfo)
248
{
249
    gf_boolean_t res = _gf_false;
250
    char *quorum_type = NULL;
251
    int ret = 0;
252

253
    ret = dict_get_str(volinfo->dict, GLUSTERD_QUORUM_TYPE_KEY, &quorum_type);
254
    if (ret) {
255
        gf_smsg(THIS->name, GF_LOG_DEBUG, -ret, GD_MSG_DICT_GET_FAILED,
256
                "Key=%s", GLUSTERD_QUORUM_TYPE_KEY, NULL);
257
        goto out;
258
    }
259

260
    if (strcmp(quorum_type, GLUSTERD_SERVER_QUORUM) == 0)
261
        res = _gf_true;
262
out:
263
    return res;
264
}
265

266
gf_boolean_t
267
glusterd_is_any_volume_in_server_quorum(xlator_t *this)
268
{
269
    glusterd_conf_t *conf = NULL;
270
    glusterd_volinfo_t *volinfo = NULL;
271

272
    conf = this->private;
273
    list_for_each_entry(volinfo, &conf->volumes, vol_list)
274
    {
275
        if (glusterd_is_volume_in_server_quorum(volinfo)) {
276
            return _gf_true;
277
        }
278
    }
279
    return _gf_false;
280
}
281

282
gf_boolean_t
283
does_gd_meet_server_quorum(xlator_t *this)
284
{
285
    int quorum_count = 0;
286
    int active_count = 0;
287
    gf_boolean_t in = _gf_false;
288
    int ret = -1;
289

290
    ret = glusterd_get_quorum_cluster_counts(this, &active_count,
291
                                             &quorum_count);
292
    if (ret) {
293
        gf_smsg(this->name, GF_LOG_ERROR, errno,
294
                GD_MSG_QUORUM_CLUSTER_COUNT_GET_FAIL, NULL);
295
        goto out;
296
    }
297

298
    if (!does_quorum_meet(active_count, quorum_count)) {
299
        goto out;
300
    }
301

302
    in = _gf_true;
303
out:
304
    return in;
305
}
306

307
void
308
glusterd_do_volume_quorum_action(xlator_t *this, glusterd_volinfo_t *volinfo,
309
                                 gf_boolean_t meets_quorum)
310
{
311
    int ret = -1;
312
    glusterd_brickinfo_t *brickinfo = NULL;
313
    gd_quorum_status_t quorum_status = NOT_APPLICABLE_QUORUM;
314
    gf_boolean_t follows_quorum = _gf_false;
315
    gf_boolean_t quorum_status_unchanged = _gf_false;
316

317
    if (volinfo->status != GLUSTERD_STATUS_STARTED) {
318
        volinfo->quorum_status = NOT_APPLICABLE_QUORUM;
319
        goto out;
320
    }
321

322
    follows_quorum = glusterd_is_volume_in_server_quorum(volinfo);
323
    if (follows_quorum) {
324
        if (meets_quorum)
325
            quorum_status = MEETS_QUORUM;
326
        else
327
            quorum_status = DOESNT_MEET_QUORUM;
328
    } else {
329
        quorum_status = NOT_APPLICABLE_QUORUM;
330
    }
331

332
    /*
333
     * The following check is added to prevent spurious brick starts when
334
     * events occur that affect quorum.
335
     * Example:
336
     * There is a cluster of 10 peers. Volume is in quorum. User
337
     * takes down one brick from the volume to perform maintenance.
338
     * Suddenly one of the peers go down. Cluster is still in quorum. But
339
     * because of this 'peer going down' event, quorum is calculated and
340
     * the bricks that are down are brought up again. In this process it
341
     * also brings up the brick that is purposefully taken down.
342
     */
343
    if (volinfo->quorum_status == quorum_status) {
344
        quorum_status_unchanged = _gf_true;
345
        goto out;
346
    }
347

348
    if (quorum_status == MEETS_QUORUM) {
349
        gf_msg(this->name, GF_LOG_CRITICAL, 0,
350
               GD_MSG_SERVER_QUORUM_MET_STARTING_BRICKS,
351
               "Server quorum regained for volume %s. Starting local "
352
               "bricks.",
353
               volinfo->volname);
354
        gf_event(EVENT_QUORUM_REGAINED, "volume=%s", volinfo->volname);
355
    } else if (quorum_status == DOESNT_MEET_QUORUM) {
356
        gf_msg(this->name, GF_LOG_CRITICAL, 0,
357
               GD_MSG_SERVER_QUORUM_LOST_STOPPING_BRICKS,
358
               "Server quorum lost for volume %s. Stopping local "
359
               "bricks.",
360
               volinfo->volname);
361
        gf_event(EVENT_QUORUM_LOST, "volume=%s", volinfo->volname);
362
    }
363

364
    list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
365
    {
366
        if (!glusterd_is_local_brick(volinfo, brickinfo))
367
            continue;
368
        if (quorum_status == DOESNT_MEET_QUORUM) {
369
            ret = glusterd_brick_stop(volinfo, brickinfo, _gf_false);
370
            if (ret) {
371
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_STOP_FAIL,
372
                       "Failed to "
373
                       "stop brick %s:%s",
374
                       brickinfo->hostname, brickinfo->path);
375
            }
376
        } else {
377
            if (!brickinfo->start_triggered) {
378
                pthread_mutex_lock(&brickinfo->restart_mutex);
379
                {
380
                    /* coverity[SLEEP] */
381
                    ret = glusterd_brick_start(volinfo, brickinfo, _gf_false,
382
                                               _gf_false);
383
                }
384
                pthread_mutex_unlock(&brickinfo->restart_mutex);
385
                if (ret) {
386
                    gf_msg(this->name, GF_LOG_ERROR, 0,
387
                           GD_MSG_BRICK_DISCONNECTED, "Failed to start %s:%s",
388
                           brickinfo->hostname, brickinfo->path);
389
                }
390
            }
391
        }
392
    }
393
    volinfo->quorum_status = quorum_status;
394
    if (quorum_status == MEETS_QUORUM) {
395
        /* bricks might have been restarted and so as the port change
396
         * might have happened
397
         */
398
        ret = glusterd_store_volinfo(volinfo, GLUSTERD_VOLINFO_VER_AC_NONE);
399
        if (ret) {
400
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_STORE_FAIL,
401
                   "Failed to write volinfo for volume %s", volinfo->volname);
402
            goto out;
403
        }
404
    }
405
out:
406
    if (quorum_status_unchanged) {
407
        list_for_each_entry(brickinfo, &volinfo->bricks, brick_list)
408
        {
409
            if (!glusterd_is_local_brick(volinfo, brickinfo))
410
                continue;
411
            ret = glusterd_brick_start(volinfo, brickinfo, _gf_false, _gf_true);
412
            if (ret) {
413
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_BRICK_DISCONNECTED,
414
                       "Failed to "
415
                       "connect to %s:%s",
416
                       brickinfo->hostname, brickinfo->path);
417
            }
418
        }
419
    }
420
    return;
421
}
422

423
int
424
glusterd_do_quorum_action(void)
425
{
426
    xlator_t *this = THIS;
427
    glusterd_conf_t *conf = NULL;
428
    glusterd_volinfo_t *volinfo = NULL;
429
    int ret = 0;
430
    int active_count = 0;
431
    int quorum_count = 0;
432
    gf_boolean_t meets = _gf_false;
433

434
    conf = this->private;
435

436
    conf->pending_quorum_action = _gf_true;
437
    ret = glusterd_lock(conf->uuid);
438
    if (ret)
439
        goto out;
440

441
    {
442
        ret = glusterd_get_quorum_cluster_counts(this, &active_count,
443
                                                 &quorum_count);
444
        if (ret)
445
            goto unlock;
446

447
        if (does_quorum_meet(active_count, quorum_count))
448
            meets = _gf_true;
449
        list_for_each_entry(volinfo, &conf->volumes, vol_list)
450
        {
451
            glusterd_do_volume_quorum_action(this, volinfo, meets);
452
        }
453
    }
454
unlock:
455
    (void)glusterd_unlock(conf->uuid);
456
    conf->pending_quorum_action = _gf_false;
457
out:
458
    return ret;
459
}
460

461
/* ret = 0 represents quorum is not met
462
 * ret = 1 represents quorum is met
463
 * ret = 2 represents quorum not applicable
464
 */
465

466
int
467
check_quorum_for_brick_start(glusterd_volinfo_t *volinfo,
468
                             gf_boolean_t node_quorum)
469
{
470
    gf_boolean_t volume_quorum = _gf_false;
471
    int ret = 0;
472

473
    volume_quorum = glusterd_is_volume_in_server_quorum(volinfo);
474
    if (volume_quorum) {
475
        if (node_quorum)
476
            ret = 1;
477
    } else {
478
        ret = 2;
479
    }
480
    return ret;
481
}
482