glusterfs

1
/*
2
   Copyright (c) 2014 Red Hat, Inc. <http://www.redhat.com>
3
   This file is part of GlusterFS.
4

5
   This file is licensed to you under your choice of the GNU Lesser
6
   General Public License, version 3 or any later version (LGPLv3 or
7
   later), or the GNU General Public License, version 2 (GPLv2), in all
8
   cases as published by the Free Software Foundation.
9
*/
10

11
#include <glusterfs/globals.h>
12
#include <glusterfs/run.h>
13
#include "glusterd-utils.h"
14
#include "glusterd-volgen.h"
15
#include "glusterd-messages.h"
16
#include "glusterd-svc-mgmt.h"
17
#include "glusterd-svc-helper.h"
18
#include "glusterd-conn-mgmt.h"
19
#include "glusterd-proc-mgmt.h"
20
#include "glusterd-snapd-svc.h"
21
#include "glusterd-snapd-svc-helper.h"
22
#include "glusterd-snapshot-utils.h"
23
#include <glusterfs/syscall.h>
24

25
char *snapd_svc_name = "snapd";
26

27
static void
28
glusterd_svc_build_snapd_logdir(char *logdir, char *volname, size_t len)
29
{
30
    glusterd_conf_t *priv = THIS->private;
31
    snprintf(logdir, len, "%s/snaps/%s", priv->logdir, volname);
32
}
33

34

35
void
36
glusterd_snapdsvc_build(glusterd_svc_t *svc)
37
{
38
    svc->manager = glusterd_snapdsvc_manager;
39
    svc->start = glusterd_snapdsvc_start;
40
    svc->stop = glusterd_svc_stop;
41
}
42

43
int
44
glusterd_snapdsvc_init(void *data)
45
{
46
    int ret = -1;
47
    char rundir[PATH_MAX] = {
48
        0,
49
    };
50
    char sockpath[PATH_MAX] = {
51
        0,
52
    };
53
    char pidfile[PATH_MAX] = {
54
        0,
55
    };
56
    char volfile[PATH_MAX] = {
57
        0,
58
    };
59
    char logdir[PATH_MAX] = {
60
        0,
61
    };
62
    char logfile[PATH_MAX] = {
63
        0,
64
    };
65
    char volfileid[256] = {0};
66
    glusterd_svc_t *svc = NULL;
67
    glusterd_volinfo_t *volinfo = NULL;
68
    glusterd_conf_t *priv = NULL;
69
    glusterd_conn_notify_t notify = NULL;
70
    xlator_t *this = THIS;
71
    char *volfileserver = NULL;
72
    int32_t len = 0;
73

74
    priv = this->private;
75
    GF_ASSERT(priv);
76

77
    volinfo = data;
78

79
    svc = &(volinfo->snapd.svc);
80

81
    ret = snprintf(svc->name, sizeof(svc->name), "%s", snapd_svc_name);
82
    if (ret < 0) {
83
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
84
        goto out;
85
    }
86

87
    notify = glusterd_snapdsvc_rpc_notify;
88

89
    glusterd_svc_build_snapd_rundir(volinfo, rundir, sizeof(rundir));
90
    glusterd_svc_create_rundir(rundir);
91

92
    /* Initialize the connection mgmt */
93
    glusterd_svc_build_snapd_socket_filepath(volinfo, sockpath,
94
                                             sizeof(sockpath));
95
    ret = glusterd_conn_init(&(svc->conn), sockpath, 600, notify);
96
    if (ret)
97
        goto out;
98

99
    /* Initialize the process mgmt */
100
    glusterd_svc_build_snapd_pidfile(volinfo, pidfile, sizeof(pidfile));
101
    glusterd_svc_build_snapd_volfile(volinfo, volfile, sizeof(volfile));
102
    glusterd_svc_build_snapd_logdir(logdir, volinfo->volname, sizeof(logdir));
103
    ret = mkdir_p(logdir, 0755, _gf_true);
104
    if ((ret == -1) && (EEXIST != errno)) {
105
        gf_msg(this->name, GF_LOG_ERROR, errno, GD_MSG_CREATE_DIR_FAILED,
106
               "Unable to create logdir %s", logdir);
107
        goto out;
108
    }
109
    len = snprintf(logfile, sizeof(logfile), "%s/snapd.log", logdir);
110
    if ((len < 0) || (len >= sizeof(logfile))) {
111
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
112
        ret = -1;
113
        goto out;
114
    }
115
    len = snprintf(volfileid, sizeof(volfileid), "snapd/%s", volinfo->volname);
116
    if ((len < 0) || (len >= sizeof(volfileid))) {
117
        gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
118
        ret = -1;
119
        goto out;
120
    }
121

122
    if (dict_get_str(this->options, "transport.socket.bind-address",
123
                     &volfileserver) != 0) {
124
        volfileserver = "localhost";
125
    }
126
    ret = glusterd_proc_init(&(svc->proc), snapd_svc_name, pidfile, logdir,
127
                             logfile, volfile, volfileid, volfileserver);
128
    if (ret)
129
        goto out;
130

131
out:
132
    gf_msg_debug(this->name, 0, "Returning %d", ret);
133
    return ret;
134
}
135

136
int
137
glusterd_snapdsvc_manager(glusterd_svc_t *svc, void *data, int flags)
138
{
139
    int ret = 0;
140
    xlator_t *this = THIS;
141
    glusterd_volinfo_t *volinfo = NULL;
142

143
    volinfo = data;
144

145
    if (!svc->inited) {
146
        ret = glusterd_snapdsvc_init(volinfo);
147
        if (ret) {
148
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_INIT_FAIL,
149
                   "Failed to initialize "
150
                   "snapd service for volume %s",
151
                   volinfo->volname);
152
            goto out;
153
        } else {
154
            svc->inited = _gf_true;
155
            gf_msg_debug(this->name, 0,
156
                         "snapd service "
157
                         "initialized");
158
        }
159
    }
160

161
    ret = glusterd_is_snapd_enabled(volinfo);
162
    if (ret == -1) {
163
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
164
               "Failed to read volume "
165
               "options");
166
        goto out;
167
    }
168

169
    if (ret) {
170
        if (!glusterd_is_volume_started(volinfo)) {
171
            if (glusterd_proc_is_running(&svc->proc)) {
172
                ret = svc->stop(svc, SIGTERM);
173
                if (ret)
174
                    gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_STOP_FAIL,
175
                           "Couldn't stop snapd for "
176
                           "volume: %s",
177
                           volinfo->volname);
178
            } else {
179
                /* Since snapd is not running set ret to 0 */
180
                ret = 0;
181
            }
182
            goto out;
183
        }
184

185
        ret = glusterd_snapdsvc_create_volfile(volinfo);
186
        if (ret) {
187
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_CREATE_FAIL,
188
                   "Couldn't create "
189
                   "snapd volfile for volume: %s",
190
                   volinfo->volname);
191
            goto out;
192
        }
193

194
        ret = svc->start(svc, flags);
195
        if (ret) {
196
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
197
                   "Couldn't start "
198
                   "snapd for volume: %s",
199
                   volinfo->volname);
200
            goto out;
201
        }
202

203
        glusterd_volinfo_ref(volinfo);
204
        ret = glusterd_conn_connect(&(svc->conn));
205
        if (ret) {
206
            glusterd_volinfo_unref(volinfo);
207
            goto out;
208
        }
209

210
    } else if (glusterd_proc_is_running(&svc->proc)) {
211
        ret = svc->stop(svc, SIGTERM);
212
        if (ret) {
213
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_STOP_FAIL,
214
                   "Couldn't stop snapd for volume: %s", volinfo->volname);
215
            goto out;
216
        }
217
        volinfo->snapd.port = 0;
218
    }
219

220
out:
221
    if (ret) {
222
        gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
223
                 volinfo->volname, svc->name);
224
    }
225
    gf_msg_debug(this->name, 0, "Returning %d", ret);
226

227
    return ret;
228
}
229

230
int32_t
231
glusterd_snapdsvc_start(glusterd_svc_t *svc, int flags)
232
{
233
    int ret = -1;
234
    runner_t runner = {
235
        0,
236
    };
237
    glusterd_conf_t *priv = NULL;
238
    xlator_t *this = THIS;
239
    char valgrind_logfile[PATH_MAX] = {0};
240
    int snapd_port = 0;
241
    char msg[1024] = {
242
        0,
243
    };
244
    char snapd_id[PATH_MAX] = {
245
        0,
246
    };
247
    glusterd_volinfo_t *volinfo = NULL;
248
    glusterd_snapdsvc_t *snapd = NULL;
249
    char *localtime_logging = NULL;
250
    int32_t len = 0;
251

252
    priv = this->private;
253
    GF_ASSERT(priv);
254

255
    if (glusterd_proc_is_running(&svc->proc)) {
256
        ret = 0;
257
        goto out;
258
    }
259

260
    /* Get volinfo->snapd from svc object */
261
    snapd = cds_list_entry(svc, glusterd_snapdsvc_t, svc);
262
    if (!snapd) {
263
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_OBJ_GET_FAIL,
264
               "Failed to get snapd object "
265
               "from snapd service");
266
        goto out;
267
    }
268

269
    /* Get volinfo from snapd */
270
    volinfo = cds_list_entry(snapd, glusterd_volinfo_t, snapd);
271
    if (!volinfo) {
272
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
273
               "Failed to get volinfo from "
274
               "from snapd");
275
        goto out;
276
    }
277

278
    ret = sys_access(svc->proc.volfile, F_OK);
279
    if (ret) {
280
        gf_msg(this->name, GF_LOG_DEBUG, 0, GD_MSG_VOLINFO_GET_FAIL,
281
               "snapd Volfile %s is not present", svc->proc.volfile);
282
        /* If glusterd is down on one of the nodes and during
283
         * that time "USS is enabled" for the first time. After some
284
         * time when the glusterd which was down comes back it tries
285
         * to look for the snapd volfile and it does not find snapd
286
         * volfile and because of this starting of snapd fails.
287
         * Therefore, if volfile is not present then create a fresh
288
         * volfile.
289
         */
290
        ret = glusterd_snapdsvc_create_volfile(volinfo);
291
        if (ret) {
292
            gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLFILE_CREATE_FAIL,
293
                   "Couldn't create "
294
                   "snapd volfile for volume: %s",
295
                   volinfo->volname);
296
            goto out;
297
        }
298
    }
299
    runinit(&runner);
300

301
    if (this->ctx->cmd_args.vgtool != _gf_none) {
302
        len = snprintf(valgrind_logfile, PATH_MAX, "%s/valgrind-snapd.log",
303
                       svc->proc.logdir);
304
        if ((len < 0) || (len >= PATH_MAX)) {
305
            gf_smsg(this->name, GF_LOG_ERROR, errno, GD_MSG_COPY_FAIL, NULL);
306
            ret = -1;
307
            goto out;
308
        }
309

310
        if (this->ctx->cmd_args.vgtool == _gf_memcheck)
311
            runner_add_args(&runner, "valgrind", "--leak-check=full",
312
                            "--trace-children=yes", "--track-origins=yes",
313
                            NULL);
314
        else
315
            runner_add_args(&runner, "valgrind", "--tool=drd", NULL);
316

317
        runner_argprintf(&runner, "--log-file=%s", valgrind_logfile);
318
    }
319

320
    snprintf(snapd_id, sizeof(snapd_id), "snapd-%s", volinfo->volname);
321
    runner_add_args(&runner, SBIN_DIR "/glusterfsd", "-s",
322
                    svc->proc.volfileserver, "--volfile-id",
323
                    svc->proc.volfileid, "-p", svc->proc.pidfile, "-l",
324
                    svc->proc.logfile, "--brick-name", snapd_id, "-S",
325
                    svc->conn.sockpath, "--process-name", svc->name, NULL);
326
    if (dict_get_str(priv->opts, GLUSTERD_LOCALTIME_LOGGING_KEY,
327
                     &localtime_logging) == 0) {
328
        if (strcmp(localtime_logging, "enable") == 0)
329
            runner_add_arg(&runner, "--localtime-logging");
330
    }
331

332
    snapd_port = pmap_assign_port(this, volinfo->snapd.port, snapd_id);
333
    if (!snapd_port) {
334
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_PORTS_EXHAUSTED,
335
               "All the ports in the range are exhausted, can't start "
336
               "snapd for volume %s",
337
               volinfo->volname);
338
        ret = -1;
339
        goto out;
340
    }
341

342
    volinfo->snapd.port = snapd_port;
343

344
    runner_add_arg(&runner, "--brick-port");
345
    runner_argprintf(&runner, "%d", snapd_port);
346
    runner_add_arg(&runner, "--xlator-option");
347
    runner_argprintf(&runner, "%s-server.listen-port=%d", volinfo->volname,
348
                     snapd_port);
349
    runner_add_arg(&runner, "--no-mem-accounting");
350

351
    snprintf(msg, sizeof(msg), "Starting the snapd service for volume %s",
352
             volinfo->volname);
353
    runner_log(&runner, this->name, GF_LOG_DEBUG, msg);
354

355
    if (flags == PROC_START_NO_WAIT) {
356
        ret = runner_run_nowait(&runner);
357
    } else {
358
        synclock_unlock(&priv->big_lock);
359
        {
360
            ret = runner_run(&runner);
361
        }
362
        synclock_lock(&priv->big_lock);
363
    }
364

365
out:
366
    return ret;
367
}
368

369
int
370
glusterd_snapdsvc_restart(void)
371
{
372
    glusterd_volinfo_t *volinfo = NULL;
373
    glusterd_volinfo_t *tmp = NULL;
374
    int ret = 0;
375
    xlator_t *this = THIS;
376
    glusterd_conf_t *conf = NULL;
377
    glusterd_svc_t *svc = NULL;
378

379
    conf = this->private;
380
    GF_ASSERT(conf);
381

382
    cds_list_for_each_entry_safe(volinfo, tmp, &conf->volumes, vol_list)
383
    {
384
        /* Start per volume snapd svc */
385
        if (volinfo->status == GLUSTERD_STATUS_STARTED) {
386
            svc = &(volinfo->snapd.svc);
387
            ret = svc->manager(svc, volinfo, PROC_START_NO_WAIT);
388
            if (ret) {
389
                gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_START_FAIL,
390
                       "Couldn't resolve snapd for "
391
                       "vol: %s on restart",
392
                       volinfo->volname);
393
                gf_event(EVENT_SVC_MANAGER_FAILED, "volume=%s;svc_name=%s",
394
                         volinfo->volname, svc->name);
395
                goto out;
396
            }
397
        }
398
    }
399
out:
400
    return ret;
401
}
402

403
int
404
glusterd_snapdsvc_rpc_notify(glusterd_conn_t *conn, rpc_clnt_event_t event)
405
{
406
    int ret = 0;
407
    glusterd_svc_t *svc = NULL;
408
    xlator_t *this = THIS;
409
    glusterd_volinfo_t *volinfo = NULL;
410
    glusterd_snapdsvc_t *snapd = NULL;
411

412
    svc = cds_list_entry(conn, glusterd_svc_t, conn);
413
    if (!svc) {
414
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SVC_GET_FAIL,
415
               "Failed to get the service");
416
        return -1;
417
    }
418
    snapd = cds_list_entry(svc, glusterd_snapdsvc_t, svc);
419
    if (!snapd) {
420
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_SNAPD_OBJ_GET_FAIL,
421
               "Failed to get the "
422
               "snapd object");
423
        return -1;
424
    }
425

426
    volinfo = cds_list_entry(snapd, glusterd_volinfo_t, snapd);
427
    if (!volinfo) {
428
        gf_msg(this->name, GF_LOG_ERROR, 0, GD_MSG_VOLINFO_GET_FAIL,
429
               "Failed to get the "
430
               "volinfo object");
431
        return -1;
432
    }
433

434
    switch (event) {
435
        case RPC_CLNT_CONNECT:
436
            gf_msg_debug(this->name, 0,
437
                         "%s has connected with "
438
                         "glusterd.",
439
                         svc->name);
440
            gf_event(EVENT_SVC_CONNECTED, "volume=%s;svc_name=%s",
441
                     volinfo->volname, svc->name);
442
            svc->online = _gf_true;
443
            break;
444

445
        case RPC_CLNT_DISCONNECT:
446
            if (svc->online) {
447
                gf_msg(this->name, GF_LOG_INFO, 0, GD_MSG_NODE_DISCONNECTED,
448
                       "%s has disconnected "
449
                       "from glusterd.",
450
                       svc->name);
451
                gf_event(EVENT_SVC_DISCONNECTED, "volume=%s;svc_name=%s",
452
                         volinfo->volname, svc->name);
453
                svc->online = _gf_false;
454
            }
455
            break;
456

457
        case RPC_CLNT_DESTROY:
458
            glusterd_volinfo_unref(volinfo);
459
            break;
460

461
        default:
462
            gf_msg_trace(this->name, 0, "got some other RPC event %d", event);
463
            break;
464
    }
465

466
    return ret;
467
}
468