glusterfs

1
/*
2
  Copyright (c) 2012 Red Hat, Inc. <http://www.redhat.com>
3
  This file is part of GlusterFS.
4

5
  This file is licensed to you under your choice of the GNU Lesser
6
  General Public License, version 3 or any later version (LGPLv3 or
7
  later), or the GNU General Public License, version 2 (GPLv2), in all
8
  cases as published by the Free Software Foundation.
9
*/
10

11
#ifndef _GNU_SOURCE
12
#define _GNU_SOURCE
13
#endif
14

15
#define _XOPEN_SOURCE 500
16

17
#include <ftw.h>
18
#include <stdio.h>
19
#include <stdlib.h>
20
#include <sys/types.h>
21
#include <sys/stat.h>
22
#include <unistd.h>
23
#include <errno.h>
24
#include <string.h>
25
#include <dirent.h>
26
#include <stdlib.h>
27
#include <libgen.h>
28
#include <stdint.h>
29
#include <dirent.h>
30
#include <argp.h>
31

32
/*
33
 * FTW_ACTIONRETVAL is a GNU libc extension. It is used here to skip
34
 * hierarchies. On other systems we will still walk the tree, ignoring
35
 * entries.
36
 */
37
#ifndef FTW_ACTIONRETVAL
38
#define FTW_ACTIONRETVAL 0
39
#endif
40

41
int debug = 0;
42

43
typedef struct {
44
    char test_directory[4096];
45
    char **ignored_directory;
46
    unsigned int directories_ignored;
47
} arequal_config_t;
48

49
static arequal_config_t arequal_config;
50

51
static error_t
52
arequal_parse_opts(int key, char *arg, struct argp_state *_state);
53

54
static struct argp_option arequal_options[] = {
55
    {"ignore", 'i', "IGNORED", 0, "entry in the given path to be ignored"},
56
    {"path", 'p', "PATH", 0, "path where arequal has to be run"},
57
    {0, 0, 0, 0, 0}};
58

59
#define DBG(fmt...)                                                            \
60
    do {                                                                       \
61
        if (debug) {                                                           \
62
            fprintf(stderr, "D ");                                             \
63
            fprintf(stderr, fmt);                                              \
64
        }                                                                      \
65
    } while (0)
66

67
void
68
add_to_list(char *arg);
69
void
70
get_absolute_path(char directory[], char *arg);
71

72
static int
73
roof(int a, int b)
74
{
75
    return ((((a) + (b)-1) / ((b) ? (b) : 1)) * (b));
76
}
77

78
void
79
add_to_list(char *arg)
80
{
81
    char *string = NULL;
82
    int index = 0;
83

84
    index = arequal_config.directories_ignored - 1;
85
    string = strdup(arg);
86

87
    if (!arequal_config.ignored_directory) {
88
        arequal_config.ignored_directory = calloc(1, sizeof(char *));
89
    } else
90
        arequal_config.ignored_directory = realloc(
91
            arequal_config.ignored_directory, sizeof(char *) * (index + 1));
92

93
    arequal_config.ignored_directory[index] = string;
94
}
95

96
static error_t
97
arequal_parse_opts(int key, char *arg, struct argp_state *_state)
98
{
99
    switch (key) {
100
        case 'i': {
101
            arequal_config.directories_ignored++;
102
            add_to_list(arg);
103
        } break;
104
        case 'p': {
105
            if (arg[0] == '/')
106
                strcpy(arequal_config.test_directory, arg);
107
            else
108
                get_absolute_path(arequal_config.test_directory, arg);
109

110
            if (arequal_config
111
                    .test_directory[strlen(arequal_config.test_directory) -
112
                                    1] == '/')
113
                arequal_config
114
                    .test_directory[strlen(arequal_config.test_directory) - 1] =
115
                    '\0';
116
        } break;
117

118
        case ARGP_KEY_NO_ARGS:
119
            break;
120
        case ARGP_KEY_ARG:
121
            break;
122
        case ARGP_KEY_END:
123
            if (_state->argc == 1) {
124
                argp_usage(_state);
125
            }
126
    }
127

128
    return 0;
129
}
130

131
void
132
get_absolute_path(char directory[], char *arg)
133
{
134
    char cwd[4096] = {
135
        0,
136
    };
137

138
    if (getcwd(cwd, sizeof(cwd)) == NULL)
139
        printf("some error in getting cwd\n");
140

141
    if (strcmp(arg, ".") != 0) {
142
        if (cwd[strlen(cwd)] != '/')
143
            cwd[strlen(cwd)] = '/';
144
        strcat(cwd, arg);
145
    }
146
    strcpy(directory, cwd);
147
}
148

149
static struct argp argp = {
150
    arequal_options, arequal_parse_opts, "",
151
    "arequal - Tool which calculates the checksum of all the entries"
152
    "present in a given directory"};
153

154
/* All this runs in single thread, hence using 'global' variables */
155

156
unsigned long long avg_uid_file = 0;
157
unsigned long long avg_uid_dir = 0;
158
unsigned long long avg_uid_symlink = 0;
159
unsigned long long avg_uid_other = 0;
160

161
unsigned long long avg_gid_file = 0;
162
unsigned long long avg_gid_dir = 0;
163
unsigned long long avg_gid_symlink = 0;
164
unsigned long long avg_gid_other = 0;
165

166
unsigned long long avg_mode_file = 0;
167
unsigned long long avg_mode_dir = 0;
168
unsigned long long avg_mode_symlink = 0;
169
unsigned long long avg_mode_other = 0;
170

171
unsigned long long global_ctime_checksum = 0;
172

173
unsigned long long count_dir = 0;
174
unsigned long long count_file = 0;
175
unsigned long long count_symlink = 0;
176
unsigned long long count_other = 0;
177

178
unsigned long long checksum_file1 = 0;
179
unsigned long long checksum_file2 = 0;
180
unsigned long long checksum_dir = 0;
181
unsigned long long checksum_symlink = 0;
182
unsigned long long checksum_other = 0;
183

184
unsigned long long
185
checksum_path(const char *path)
186
{
187
    unsigned long long csum = 0;
188
    unsigned long long *nums = 0;
189
    int len = 0;
190
    int cnt = 0;
191

192
    len = roof(strlen(path), sizeof(csum));
193
    cnt = len / sizeof(csum);
194

195
    nums = __builtin_alloca(len);
196
    memset(nums, 0, len);
197
    strcpy((char *)nums, path);
198

199
    while (cnt) {
200
        csum ^= *nums;
201
        nums++;
202
        cnt--;
203
    }
204

205
    return csum;
206
}
207

208
int
209
checksum_md5(const char *path, const struct stat *sb)
210
{
211
    uint64_t this_data_checksum = 0;
212
    FILE *filep = NULL;
213
    char *cmd = NULL;
214
    char strvalue[17] = {
215
        0,
216
    };
217
    int ret = -1;
218
    int len = 0;
219
    const char *pos = NULL;
220
    char *cpos = NULL;
221

222
    /* Have to escape single-quotes in filename.
223
     * First, calculate the size of the buffer I'll need.
224
     */
225
    for (pos = path; *pos; pos++) {
226
        if (*pos == '\'')
227
            len += 4;
228
        else
229
            len += 1;
230
    }
231

232
    cmd = malloc(sizeof(char) * (len + 20));
233
    cmd[0] = '\0';
234

235
    /* Now, build the command with single quotes escaped. */
236

237
    cpos = cmd;
238
#if defined(linux)
239
    strcpy(cpos, "md5sum '");
240
    cpos += 8;
241
#elif defined(__NetBSD__)
242
    strcpy(cpos, "md5 -n '");
243
    cpos += 8;
244
#elif defined(__FreeBSD__) || defined(__APPLE__)
245
    strcpy(cpos, "md5 -q '");
246
    cpos += 8;
247
#else
248
#error "Please add system-specific md5 command"
249
#endif
250

251
    /* Add the file path, with every single quotes replaced with this sequence:
252
     * '\''
253
     */
254

255
    for (pos = path; *pos; pos++) {
256
        if (*pos == '\'') {
257
            strcpy(cpos, "'\\''");
258
            cpos += 4;
259
        } else {
260
            *cpos = *pos;
261
            cpos++;
262
        }
263
    }
264

265
    /* Add on the trailing single-quote and null-terminate. */
266
    strcpy(cpos, "'");
267

268
    filep = popen(cmd, "r");
269
    if (!filep) {
270
        perror(path);
271
        goto out;
272
    }
273

274
    if (fread(strvalue, sizeof(char), 16, filep) != 16) {
275
        fprintf(stderr, "%s: short read\n", path);
276
        goto out;
277
    }
278

279
    this_data_checksum = strtoull(strvalue, NULL, 16);
280
    if (-1 == this_data_checksum) {
281
        fprintf(stderr, "%s: %s\n", strvalue, strerror(errno));
282
        goto out;
283
    }
284
    checksum_file1 ^= this_data_checksum;
285

286
    if (fread(strvalue, sizeof(char), 16, filep) != 16) {
287
        fprintf(stderr, "%s: short read\n", path);
288
        goto out;
289
    }
290

291
    this_data_checksum = strtoull(strvalue, NULL, 16);
292
    if (-1 == this_data_checksum) {
293
        fprintf(stderr, "%s: %s\n", strvalue, strerror(errno));
294
        goto out;
295
    }
296
    checksum_file2 ^= this_data_checksum;
297

298
    ret = 0;
299
out:
300
    if (filep)
301
        pclose(filep);
302

303
    if (cmd)
304
        free(cmd);
305

306
    return ret;
307
}
308

309
int
310
checksum_filenames(const char *path, const struct stat *sb)
311
{
312
    DIR *dirp = NULL;
313
    struct dirent *entry = NULL;
314
    unsigned long long csum = 0;
315
    int i = 0;
316
    int found = 0;
317

318
    dirp = opendir(path);
319
    if (!dirp) {
320
        perror(path);
321
        goto out;
322
    }
323

324
    errno = 0;
325
    while ((entry = readdir(dirp))) {
326
        /* do not calculate the checksum of the entries which user has
327
           told to ignore and proceed to other siblings.*/
328
        if (arequal_config.ignored_directory) {
329
            for (i = 0; i < arequal_config.directories_ignored; i++) {
330
                if ((strcmp(entry->d_name,
331
                            arequal_config.ignored_directory[i]) == 0)) {
332
                    found = 1;
333
                    DBG("ignoring the entry %s\n", entry->d_name);
334
                    break;
335
                }
336
            }
337
            if (found == 1) {
338
                found = 0;
339
                continue;
340
            }
341
        }
342
        csum = checksum_path(entry->d_name);
343
        checksum_dir ^= csum;
344
    }
345

346
    if (errno) {
347
        perror(path);
348
        goto out;
349
    }
350

351
out:
352
    if (dirp)
353
        closedir(dirp);
354

355
    return 0;
356
}
357

358
int
359
process_file(const char *path, const struct stat *sb)
360
{
361
    int ret = 0;
362

363
    count_file++;
364

365
    avg_uid_file ^= sb->st_uid;
366
    avg_gid_file ^= sb->st_gid;
367
    avg_mode_file ^= sb->st_mode;
368

369
    ret = checksum_md5(path, sb);
370

371
    return ret;
372
}
373

374
int
375
process_dir(const char *path, const struct stat *sb)
376
{
377
    unsigned long long csum = 0;
378

379
    count_dir++;
380

381
    avg_uid_dir ^= sb->st_uid;
382
    avg_gid_dir ^= sb->st_gid;
383
    avg_mode_dir ^= sb->st_mode;
384

385
    csum = checksum_filenames(path, sb);
386

387
    checksum_dir ^= csum;
388

389
    return 0;
390
}
391

392
int
393
process_symlink(const char *path, const struct stat *sb)
394
{
395
    int ret = 0;
396
    char buf[4096] = {
397
        0,
398
    };
399
    unsigned long long csum = 0;
400

401
    count_symlink++;
402

403
    avg_uid_symlink ^= sb->st_uid;
404
    avg_gid_symlink ^= sb->st_gid;
405
    avg_mode_symlink ^= sb->st_mode;
406

407
    ret = readlink(path, buf, 4096);
408
    if (ret < 0) {
409
        perror(path);
410
        goto out;
411
    }
412

413
    DBG("readlink (%s) => %s\n", path, buf);
414

415
    csum = checksum_path(buf);
416

417
    DBG("checksum_path (%s) => %llx\n", buf, csum);
418

419
    checksum_symlink ^= csum;
420

421
    ret = 0;
422
out:
423
    return ret;
424
}
425

426
int
427
process_other(const char *path, const struct stat *sb)
428
{
429
    count_other++;
430

431
    avg_uid_other ^= sb->st_uid;
432
    avg_gid_other ^= sb->st_gid;
433
    avg_mode_other ^= sb->st_mode;
434

435
    checksum_other ^= sb->st_rdev;
436

437
    return 0;
438
}
439

440
static int
441
ignore_entry(const char *bname, const char *dname)
442
{
443
    int i;
444

445
    for (i = 0; i < arequal_config.directories_ignored; i++) {
446
        if (strcmp(bname, arequal_config.ignored_directory[i]) == 0 &&
447
            strncmp(arequal_config.test_directory, dname,
448
                    strlen(arequal_config.test_directory)) == 0)
449
            return 1;
450
    }
451

452
    return 0;
453
}
454

455
int
456
process_entry(const char *path, const struct stat *sb, int typeflag,
457
              struct FTW *ftwbuf)
458
{
459
    int ret = 0;
460
    char *name = NULL;
461
    char *bname = NULL;
462
    char *dname = NULL;
463
    int i = 0;
464

465
    /* The if condition below helps in ignoring some directories in
466
       the given path. If the name of the entry is one of the directory
467
       names that the user told to ignore, then that directory will not
468
       be processed and will return FTW_SKIP_SUBTREE to nftw which will
469
       not crawl this directory and move on to other siblings.
470
       Note that for nftw to recognize FTW_SKIP_TREE, FTW_ACTIONRETVAL
471
       should be passed as an argument to nftw.
472

473
       This mainly helps in calculating the checksum of network filesystems
474
       (client-server), where the server might have some hidden directories
475
       for managing the filesystem. So to calculate the sanity of filesystem
476
       one has to get the checksum of the client and then the export directory
477
       of server by telling arequal to ignore some of the directories which
478
       are not part of the namespace.
479
    */
480

481
    if (arequal_config.ignored_directory) {
482
#ifndef FTW_SKIP_SUBTREE
483
        char *cp;
484

485
        name = strdup(path);
486
        dname = dirname(name);
487

488
        for (cp = strtok(name, "/"); cp; cp = strtok(NULL, "/")) {
489
            if (ignore_entry(cp, dname)) {
490
                DBG("ignoring %s\n", path);
491
                if (name)
492
                    free(name);
493
                return 0;
494
            }
495
        }
496
#else  /* FTW_SKIP_SUBTREE */
497
        name = strdup(path);
498

499
        name[strlen(name)] = '\0';
500

501
        bname = strrchr(name, '/');
502
        if (bname)
503
            bname++;
504

505
        dname = dirname(name);
506
        if (ignore_entry(bname, dname)) {
507
            DBG("ignoring %s\n", bname);
508
            ret = FTW_SKIP_SUBTREE;
509
            if (name)
510
                free(name);
511
            return ret;
512
        }
513
#endif /* FTW_SKIP_SUBTREE */
514
    }
515

516
    DBG("processing entry %s\n", path);
517

518
    switch ((S_IFMT & sb->st_mode)) {
519
        case S_IFDIR:
520
            ret = process_dir(path, sb);
521
            break;
522
        case S_IFREG:
523
            ret = process_file(path, sb);
524
            break;
525
        case S_IFLNK:
526
            ret = process_symlink(path, sb);
527
            break;
528
        default:
529
            ret = process_other(path, sb);
530
            break;
531
    }
532

533
    if (name)
534
        free(name);
535
    return ret;
536
}
537

538
int
539
display_counts(FILE *fp)
540
{
541
    fprintf(fp, "\n");
542
    fprintf(fp, "Entry counts\n");
543
    fprintf(fp, "Regular files   : %lld\n", count_file);
544
    fprintf(fp, "Directories     : %lld\n", count_dir);
545
    fprintf(fp, "Symbolic links  : %lld\n", count_symlink);
546
    fprintf(fp, "Other           : %lld\n", count_other);
547
    fprintf(fp, "Total           : %lld\n",
548
            (count_file + count_dir + count_symlink + count_other));
549

550
    return 0;
551
}
552

553
int
554
display_checksums(FILE *fp)
555
{
556
    fprintf(fp, "\n");
557
    fprintf(fp, "Checksums\n");
558
    fprintf(fp, "Regular files   : %llx%llx\n", checksum_file1, checksum_file2);
559
    fprintf(fp, "Directories     : %llx\n", checksum_dir);
560
    fprintf(fp, "Symbolic links  : %llx\n", checksum_symlink);
561
    fprintf(fp, "Other           : %llx\n", checksum_other);
562
    fprintf(fp, "Total           : %llx\n",
563
            (checksum_file1 ^ checksum_file2 ^ checksum_dir ^ checksum_symlink ^
564
             checksum_other));
565

566
    return 0;
567
}
568

569
int
570
display_metadata(FILE *fp)
571
{
572
    fprintf(fp, "\n");
573
    fprintf(fp, "Metadata checksums\n");
574
    fprintf(fp, "Regular files   : %llx\n",
575
            (avg_uid_file + 13) * (avg_gid_file + 11) * (avg_mode_file + 7));
576
    fprintf(fp, "Directories     : %llx\n",
577
            (avg_uid_dir + 13) * (avg_gid_dir + 11) * (avg_mode_dir + 7));
578
    fprintf(fp, "Symbolic links  : %llx\n",
579
            (avg_uid_symlink + 13) * (avg_gid_symlink + 11) *
580
                (avg_mode_symlink + 7));
581
    fprintf(fp, "Other           : %llx\n",
582
            (avg_uid_other + 13) * (avg_gid_other + 11) * (avg_mode_other + 7));
583

584
    return 0;
585
}
586

587
int
588
display_stats(FILE *fp)
589
{
590
    display_counts(fp);
591

592
    display_metadata(fp);
593

594
    display_checksums(fp);
595

596
    return 0;
597
}
598

599
int
600
main(int argc, char *argv[])
601
{
602
    int ret = 0;
603
    int i = 0;
604

605
    ret = argp_parse(&argp, argc, argv, 0, 0, NULL);
606
    if (ret != 0) {
607
        fprintf(stderr, "parsing arguments failed\n");
608
        return -2;
609
    }
610

611
    /* Use FTW_ACTIONRETVAL to take decision on what to do depending upon */
612
    /* the return value of the callback function */
613
    /* (process_entry in this case) */
614
    ret = nftw(arequal_config.test_directory, process_entry, 30,
615
               FTW_ACTIONRETVAL | FTW_PHYS | FTW_MOUNT);
616
    if (ret != 0) {
617
        fprintf(stderr, "ftw (%s) returned %d (%s), terminating\n", argv[1],
618
                ret, strerror(errno));
619
        return 1;
620
    }
621

622
    display_stats(stdout);
623

624
    if (arequal_config.ignored_directory) {
625
        for (i = 0; i < arequal_config.directories_ignored; i++) {
626
            if (arequal_config.ignored_directory[i])
627
                free(arequal_config.ignored_directory[i]);
628
        }
629
        free(arequal_config.ignored_directory);
630
    }
631

632
    return 0;
633
}
634