glusterfs

1
/*
2
  Copyright (c) 2017 Red Hat, Inc. <http://www.redhat.com>
3
  This file is part of GlusterFS.
4

5
  This file is licensed to you under your choice of the GNU Lesser
6
  General Public License, version 3 or any later version (LGPLv3 or
7
  later), or the GNU General Public License, version 2 (GPLv2), in all
8
  cases as published by the Free Software Foundation.
9
*/
10

11
#include "glusterfs/monitoring.h"
12
#include "glusterfs/xlator.h"
13
#include "glusterfs/syscall.h"
14

15
#include <stdlib.h>
16

17
static void
18
dump_mem_acct_details(xlator_t *xl, int fd)
19
{
20
    struct mem_acct_rec *mem_rec;
21
    int i = 0;
22

23
    if (!xl || !xl->mem_acct || (xl->ctx->active != xl->graph))
24
        return;
25

26
    dprintf(fd, "# %s.%s.total.num_types %d\n", xl->type, xl->name,
27
            xl->mem_acct->num_types);
28

29
    dprintf(fd,
30
            "# type, in-use-size, in-use-units, max-size, "
31
            "max-units, total-allocs\n");
32

33
    for (i = 0; i < xl->mem_acct->num_types; i++) {
34
        mem_rec = &xl->mem_acct->rec[i];
35
        if (!GF_ATOMIC_GET(mem_rec->num_allocs))
36
            continue;
37
#ifdef DEBUG
38
        dprintf(fd, "# %s, %" PRIu64 ", %" PRIu64 ", %u, %" PRIu64 "\n",
39
                mem_rec->typestr, mem_rec->size, mem_rec->max_size,
40
                mem_rec->max_num_allocs, GF_ATOMIC_GET(mem_rec->num_allocs));
41
#else
42
        dprintf(fd, "# %s, %" PRIu64 "\n", mem_rec->typestr,
43
                GF_ATOMIC_GET(mem_rec->num_allocs));
44
#endif
45
    }
46
}
47

48
static void
49
dump_latency_and_count(xlator_t *xl, int fd)
50
{
51
    int32_t index = 0;
52
    uint64_t fop = 0;
53
    uint64_t cbk = 0;
54
    uint64_t total_fop_count = 0;
55
    uint64_t interval_fop_count = 0;
56

57
    if (xl->winds) {
58
        dprintf(fd, "%s.total.pending-winds.count %" PRIu64 "\n", xl->name,
59
                xl->winds);
60
    }
61

62
    /* Need 'fuse' data, and don't need all the old graph info */
63
    if ((xl != xl->ctx->root) && (xl->ctx->active != xl->graph))
64
        return;
65

66
    for (index = 0; index < GF_FOP_MAXVALUE; index++) {
67
        fop = GF_ATOMIC_GET(xl->stats[index].total_fop);
68
        if (fop) {
69
            dprintf(fd, "%s.total.%s.count %" PRIu64 "\n", xl->name,
70
                    gf_fop_list[index], fop);
71
            total_fop_count += fop;
72
        }
73
        fop = GF_ATOMIC_SWAP(xl->stats[index].interval_fop, 0);
74
        if (fop) {
75
            dprintf(fd, "%s.interval.%s.count %" PRIu64 "\n", xl->name,
76
                    gf_fop_list[index], fop);
77
            interval_fop_count += fop;
78
        }
79
        cbk = GF_ATOMIC_SWAP(xl->stats[index].interval_fop_cbk, 0);
80
        if (cbk) {
81
            dprintf(fd, "%s.interval.%s.fail_count %" PRIu64 "\n", xl->name,
82
                    gf_fop_list[index], cbk);
83
        }
84
        if (xl->stats[index].latencies.count != 0) {
85
            dprintf(fd, "%s.interval.%s.latency %lf\n", xl->name,
86
                    gf_fop_list[index],
87
                    (((double)xl->stats[index].latencies.total) /
88
                     xl->stats[index].latencies.count));
89
            dprintf(fd, "%s.interval.%s.max %" PRIu64 "\n", xl->name,
90
                    gf_fop_list[index], xl->stats[index].latencies.max);
91
            dprintf(fd, "%s.interval.%s.min %" PRIu64 "\n", xl->name,
92
                    gf_fop_list[index], xl->stats[index].latencies.min);
93
        }
94
        memset(&xl->stats[index].latencies, 0,
95
               sizeof(xl->stats[index].latencies));
96
    }
97

98
    dprintf(fd, "%s.total.fop-count %" PRIu64 "\n", xl->name, total_fop_count);
99
    dprintf(fd, "%s.interval.fop-count %" PRIu64 "\n", xl->name,
100
            interval_fop_count);
101
}
102

103
static inline void
104
dump_call_stack_details(glusterfs_ctx_t *ctx, int fd)
105
{
106
    dprintf(fd, "total.stack.count %" PRIu64 "\n",
107
            GF_ATOMIC_GET(ctx->pool->total_count));
108
    dprintf(fd, "total.stack.in-flight %" PRIu64 "\n", ctx->pool->cnt);
109
}
110

111
static inline void
112
dump_dict_details(glusterfs_ctx_t *ctx, int fd)
113
{
114
    uint64_t total_dicts = 0;
115
    uint64_t total_pairs = 0;
116

117
    total_dicts = GF_ATOMIC_GET(ctx->stats.total_dicts_used);
118
    total_pairs = GF_ATOMIC_GET(ctx->stats.total_pairs_used);
119

120
    dprintf(fd, "total.dict.max-pairs-per %" PRIu64 "\n",
121
            GF_ATOMIC_GET(ctx->stats.max_dict_pairs));
122
    dprintf(fd, "total.dict.pairs-used %" PRIu64 "\n", total_pairs);
123
    dprintf(fd, "total.dict.used %" PRIu64 "\n", total_dicts);
124
    dprintf(fd, "total.dict.average-pairs %" PRIu64 "\n",
125
            (total_pairs / total_dicts));
126
}
127

128
static void
129
dump_inode_stats(glusterfs_ctx_t *ctx, int fd)
130
{
131
}
132

133
static void
134
dump_global_metrics(glusterfs_ctx_t *ctx, int fd)
135
{
136
    time_t nowtime;
137
    struct tm *nowtm;
138
    char tmbuf[64] = {
139
        0,
140
    };
141

142
    nowtime = gf_time();
143
    nowtm = localtime(&nowtime);
144
    strftime(tmbuf, sizeof tmbuf, "%Y-%m-%d %H:%M:%S", nowtm);
145

146
    /* Let every file have information on which process dumped info */
147
    dprintf(fd, "## %s\n", ctx->cmdlinestr);
148
    dprintf(fd, "### %s\n", tmbuf);
149
    dprintf(fd, "### BrickName: %s\n", ctx->cmd_args.brick_name);
150
    dprintf(fd, "### MountName: %s\n", ctx->cmd_args.mount_point);
151
    dprintf(fd, "### VolumeName: %s\n", ctx->cmd_args.volume_name);
152

153
    dump_call_stack_details(ctx, fd);
154
    dump_dict_details(ctx, fd);
155
    dprintf(fd, "# -----\n");
156

157
    dump_inode_stats(ctx, fd);
158
    dprintf(fd, "# -----\n");
159
}
160

161
static void
162
dump_xl_metrics(glusterfs_ctx_t *ctx, int fd)
163
{
164
    xlator_t *xl;
165

166
    xl = ctx->active->top;
167

168
    while (xl) {
169
        dump_latency_and_count(xl, fd);
170
        dump_mem_acct_details(xl, fd);
171
        if (xl->dump_metrics)
172
            xl->dump_metrics(xl, fd);
173
        xl = xl->next;
174
    }
175

176
    if (ctx->root) {
177
        xl = ctx->root;
178

179
        dump_latency_and_count(xl, fd);
180
        dump_mem_acct_details(xl, fd);
181
        if (xl->dump_metrics)
182
            xl->dump_metrics(xl, fd);
183
    }
184

185
    return;
186
}
187

188
char *
189
gf_monitor_metrics(glusterfs_ctx_t *ctx)
190
{
191
    int ret = -1;
192
    int fd = 0;
193
    char *filepath = NULL, *dumppath = NULL;
194

195
    gf_msg_trace("monitoring", 0, "received monitoring request (sig:USR2)");
196

197
    dumppath = ctx->config.metrics_dumppath;
198
    if (dumppath == NULL) {
199
        dumppath = GLUSTER_METRICS_DIR;
200
    }
201
    ret = mkdir_p(dumppath, 0755, true);
202
    if (ret) {
203
        /* EEXIST is handled in mkdir_p() itself */
204
        gf_msg("monitoring", GF_LOG_ERROR, 0, LG_MSG_STRDUP_ERROR,
205
               "failed to create metrics dir %s (%s)", dumppath,
206
               strerror(errno));
207
        return NULL;
208
    }
209

210
    ret = gf_asprintf(&filepath, "%s/gmetrics.XXXXXX", dumppath);
211
    if (ret < 0) {
212
        return NULL;
213
    }
214

215
    /* coverity[secure_temp] mkstemp uses 0600 as the mode and is safe */
216
    fd = mkstemp(filepath);
217
    if (fd < 0) {
218
        gf_msg("monitoring", GF_LOG_ERROR, 0, LG_MSG_STRDUP_ERROR,
219
               "failed to open tmp file %s (%s)", filepath, strerror(errno));
220
        GF_FREE(filepath);
221
        return NULL;
222
    }
223

224
    dump_global_metrics(ctx, fd);
225

226
    dump_xl_metrics(ctx, fd);
227

228
    /* This below line is used just to capture any errors with dprintf() */
229
    ret = dprintf(fd, "\n# End of metrics\n");
230
    if (ret < 0) {
231
        gf_msg("monitoring", GF_LOG_WARNING, 0, LG_MSG_STRDUP_ERROR,
232
               "dprintf() failed: %s", strerror(errno));
233
    }
234

235
    ret = sys_fsync(fd);
236
    if (ret < 0) {
237
        gf_msg("monitoring", GF_LOG_WARNING, 0, LG_MSG_STRDUP_ERROR,
238
               "fsync() failed: %s", strerror(errno));
239
    }
240
    sys_close(fd);
241

242
    /* Figure this out, not happy with returning this string */
243
    return filepath;
244
}
245