glusterfs

1
/*
2
   Copyright (c) 2015-2016 Red Hat, Inc. <http://www.redhat.com>
3
   This file is part of GlusterFS.
4

5
   This file is licensed to you under your choice of the GNU Lesser
6
   General Public License, version 3 or any later version (LGPLv3 or
7
   later), or the GNU General Public License, version 2 (GPLv2), in all
8
   cases as published by the Free Software Foundation.
9
*/
10

11
#ifndef _CONFIG_H
12
#define _CONFIG_H
13
#include "config.h"
14
#endif
15

16
#include <glusterfs/upcall-utils.h>
17
#include "leases.h"
18

19
/* Mutex locks used in this xlator and their order of acquisition:
20
 * Check lease conflict:
21
 *         lease_ctx lock
22
 *                 add_timer => internal timer locks
23
 *         lease_ctx unlock
24
 *
25
 * Add/remove lease:
26
 *         lease_ctx lock
27
 *                 add_timer => internal timer locks
28
 *                 OR
29
 *                 priv lock => Adding/removing to/from the cleanup client list
30
 *                 priv unlock
31
 *         lease_ctx unlock
32
 *
33
 * Timer thread:
34
 *         Timer internal lock
35
 *                 priv lock => By timer handler
36
 *                 priv unlock
37
 *         Timer internal unlock
38
 *
39
 * Expired recall cleanup thread:
40
 *         priv lock
41
 *                 priv condwait
42
 *         priv unlock
43
 *         lease_ctx lock
44
 *                 priv lock
45
 *                 priv unlock
46
 *         lease_ctx unlock
47
 */
48

49
/*
50
 * Check if lease_lk is enabled
51
 * Return Value:
52
 * _gf_true  - lease lock option enabled
53
 * _gf_false - lease lock option disabled
54
 */
55
gf_boolean_t
56
is_leases_enabled(xlator_t *this)
57
{
58
    leases_private_t *priv = NULL;
59
    gf_boolean_t is_enabled = _gf_false;
60

61
    GF_VALIDATE_OR_GOTO("leases", this, out);
62

63
    if (this->private) {
64
        priv = (leases_private_t *)this->private;
65
        is_enabled = priv->leases_enabled;
66
    }
67
out:
68
    return is_enabled;
69
}
70

71
/*
72
 * Get the recall_leaselk_timeout
73
 * Return Value:
74
 * timeout value(in seconds) set as an option to this xlator.
75
 * -1 error case
76
 */
77
static time_t
78
get_recall_lease_timeout(xlator_t *this)
79
{
80
    leases_private_t *priv = NULL;
81
    time_t timeout = (time_t)-1;
82

83
    GF_VALIDATE_OR_GOTO("leases", this, out);
84

85
    if (this->private) {
86
        priv = (leases_private_t *)this->private;
87
        timeout = priv->recall_lease_timeout;
88
    }
89
out:
90
    return timeout;
91
}
92

93
static void
94
__dump_leases_info(xlator_t *this, lease_inode_ctx_t *lease_ctx)
95
{
96
    lease_id_entry_t *lease_entry = NULL;
97
    lease_id_entry_t *tmp = NULL;
98

99
    GF_VALIDATE_OR_GOTO("leases", this, out);
100
    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
101

102
    gf_msg_debug(this->name, 0,
103
                 "Lease held on this inode, lease_type: %d,"
104
                 " lease_cnt:%" PRIu64
105
                 ", RD lease:%d, RW lease:%d, "
106
                 "openfd cnt:%" PRIu64,
107
                 lease_ctx->lease_type, lease_ctx->lease_cnt,
108
                 lease_ctx->lease_type_cnt[GF_RD_LEASE],
109
                 lease_ctx->lease_type_cnt[GF_RW_LEASE], lease_ctx->openfd_cnt);
110

111
    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
112
                             lease_id_list)
113
    {
114
        gf_msg_debug(this->name, 0,
115
                     "Leases held by client: %s, lease "
116
                     "ID:%s, RD lease:%d, RW lease:%d, lease_type: %d, "
117
                     "lease_cnt:%" PRIu64,
118
                     lease_entry->client_uid, lease_entry->lease_id,
119
                     lease_entry->lease_type_cnt[GF_RD_LEASE],
120
                     lease_entry->lease_type_cnt[GF_RW_LEASE],
121
                     lease_entry->lease_type, lease_entry->lease_cnt);
122
    }
123
out:
124
    return;
125
}
126

127
static int
128
__lease_ctx_set(inode_t *inode, xlator_t *this)
129
{
130
    lease_inode_ctx_t *inode_ctx = NULL;
131
    int ret;
132

133
    inode_ctx = GF_CALLOC(1, sizeof(*inode_ctx),
134
                          gf_leases_mt_lease_inode_ctx_t);
135
    GF_CHECK_ALLOC(inode_ctx, ret, out);
136

137
    pthread_mutex_init(&inode_ctx->lock, NULL);
138
    INIT_LIST_HEAD(&inode_ctx->lease_id_list);
139
    INIT_LIST_HEAD(&inode_ctx->blocked_list);
140

141
    inode_ctx->lease_cnt = 0;
142

143
    ret = __inode_ctx_set(inode, this, (uint64_t *)inode_ctx);
144
    if (ret) {
145
        GF_FREE(inode_ctx);
146
        gf_msg(this->name, GF_LOG_INFO, 0, LEASE_MSG_INVAL_INODE_CTX,
147
               "failed to set inode ctx (%p)", inode);
148
    }
149
out:
150
    return ret;
151
}
152

153
static lease_inode_ctx_t *
154
__lease_ctx_get(inode_t *inode, xlator_t *this)
155
{
156
    lease_inode_ctx_t *inode_ctx = NULL;
157
    uint64_t ctx = 0;
158
    int ret = 0;
159

160
    ret = __inode_ctx_get(inode, this, &ctx);
161
    if (ret < 0) {
162
        ret = __lease_ctx_set(inode, this);
163
        if (ret < 0)
164
            goto out;
165

166
        ret = __inode_ctx_get(inode, this, &ctx);
167
        if (ret < 0) {
168
            gf_msg(this->name, GF_LOG_WARNING, 0, LEASE_MSG_INVAL_INODE_CTX,
169
                   "failed to get inode ctx (%p)", inode);
170
            goto out;
171
        }
172
    }
173

174
    inode_ctx = (lease_inode_ctx_t *)(long)ctx;
175
out:
176
    return inode_ctx;
177
}
178

179
lease_inode_ctx_t *
180
lease_ctx_get(inode_t *inode, xlator_t *this)
181
{
182
    lease_inode_ctx_t *inode_ctx = NULL;
183

184
    GF_VALIDATE_OR_GOTO("leases", inode, out);
185
    GF_VALIDATE_OR_GOTO("leases", this, out);
186

187
    LOCK(&inode->lock);
188
    {
189
        inode_ctx = __lease_ctx_get(inode, this);
190
    }
191
    UNLOCK(&inode->lock);
192
out:
193
    return inode_ctx;
194
}
195

196
static lease_id_entry_t *
197
new_lease_id_entry(call_frame_t *frame, const char *lease_id)
198
{
199
    lease_id_entry_t *lease_entry = NULL;
200

201
    GF_VALIDATE_OR_GOTO("leases", frame, out);
202
    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
203

204
    lease_entry = GF_CALLOC(1, sizeof(*lease_entry),
205
                            gf_leases_mt_lease_id_entry_t);
206
    if (!lease_entry) {
207
        gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, LEASE_MSG_NO_MEM,
208
               "Memory allocation for lease_entry failed");
209
        return NULL;
210
    }
211

212
    INIT_LIST_HEAD(&lease_entry->lease_id_list);
213
    lease_entry->lease_type = NONE;
214
    lease_entry->lease_cnt = 0;
215
    lease_entry->recall_time = get_recall_lease_timeout(frame->this);
216
    lease_entry->client_uid = gf_strdup(frame->root->client->client_uid);
217
    if (!lease_entry->client_uid) {
218
        gf_msg(frame->this->name, GF_LOG_ERROR, ENOMEM, LEASE_MSG_NO_MEM,
219
               "Memory allocation for client_uid failed");
220
        GF_FREE(lease_entry);
221
        lease_entry = NULL;
222
        goto out;
223
    }
224

225
    memcpy(lease_entry->lease_id, lease_id, LEASE_ID_SIZE);
226
out:
227
    return lease_entry;
228
}
229

230
static void
231
__destroy_lease_id_entry(lease_id_entry_t *lease_entry)
232
{
233
    GF_VALIDATE_OR_GOTO("leases", lease_entry, out);
234

235
    list_del_init(&lease_entry->lease_id_list);
236
    GF_FREE(lease_entry->client_uid);
237
    GF_FREE(lease_entry);
238
out:
239
    return;
240
}
241

242
static inline gf_boolean_t
243
__is_same_lease_id(const char *k1, const char *k2)
244
{
245
    if (memcmp(k1, k2, strlen(k1)) == 0)
246
        return _gf_true;
247

248
    return _gf_false;
249
}
250

251
/* Checks if there are any leases, other than the leases taken
252
 * by the given lease_id
253
 */
254
static gf_boolean_t
255
__another_lease_found(lease_inode_ctx_t *lease_ctx, const char *lease_id)
256
{
257
    lease_id_entry_t *lease_entry = NULL;
258
    lease_id_entry_t *tmp = NULL;
259
    gf_boolean_t found_lease = _gf_false;
260

261
    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
262
    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
263

264
    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
265
                             lease_id_list)
266
    {
267
        if (!__is_same_lease_id(lease_id, lease_entry->lease_id)) {
268
            if (lease_entry->lease_cnt > 0) {
269
                found_lease = _gf_true;
270
                break;
271
            }
272
        }
273
    }
274
out:
275
    return found_lease;
276
}
277

278
/* Returns the lease_id_entry for a given lease_id and a given inode.
279
 * Return values:
280
 * NULL - If no client entry found
281
 * lease_id_entry_t* - a pointer to the client entry if found
282
 */
283
static lease_id_entry_t *
284
__get_lease_id_entry(lease_inode_ctx_t *lease_ctx, const char *lease_id)
285
{
286
    lease_id_entry_t *lease_entry = NULL;
287
    lease_id_entry_t *tmp = NULL;
288
    lease_id_entry_t *found = NULL;
289

290
    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
291
    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
292

293
    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
294
                             lease_id_list)
295
    {
296
        if (__is_same_lease_id(lease_id, lease_entry->lease_id)) {
297
            found = lease_entry;
298
            gf_msg_debug("leases", 0,
299
                         "lease ID entry found "
300
                         "Client UID:%s, lease id:%s",
301
                         lease_entry->client_uid,
302
                         leaseid_utoa(lease_entry->lease_id));
303
            break;
304
        }
305
    }
306
out:
307
    return found;
308
}
309

310
/* Returns the lease_id_entry for a given lease_id and a given inode,
311
 * if none found creates one.
312
 * Return values:
313
 * lease_id_entry_t* - a pointer to the client entry
314
 */
315
static lease_id_entry_t *
316
__get_or_new_lease_entry(call_frame_t *frame, const char *lease_id,
317
                         lease_inode_ctx_t *lease_ctx)
318
{
319
    lease_id_entry_t *lease_entry = NULL;
320

321
    GF_VALIDATE_OR_GOTO("leases", frame, out);
322
    GF_VALIDATE_OR_GOTO("leases", lease_id, out);
323
    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
324

325
    lease_entry = __get_lease_id_entry(lease_ctx, lease_id);
326
    if (!lease_entry) { /* create one */
327
        lease_entry = new_lease_id_entry(frame, lease_id);
328
        if (!lease_entry)
329
            goto out;
330

331
        list_add_tail(&lease_entry->lease_id_list, &lease_ctx->lease_id_list);
332

333
        gf_msg_debug(frame->this->name, 0,
334
                     "lease ID entry added,"
335
                     " Client UID:%s, lease id:%s",
336
                     lease_entry->client_uid,
337
                     leaseid_utoa(lease_entry->lease_id));
338
    }
339
out:
340
    return lease_entry;
341
}
342

343
static lease_inode_t *
344
new_lease_inode(inode_t *inode)
345
{
346
    lease_inode_t *l_inode = GF_MALLOC(sizeof(*l_inode),
347
                                       gf_leases_mt_lease_inode_t);
348
    if (!l_inode)
349
        goto out;
350

351
    INIT_LIST_HEAD(&l_inode->list);
352
    l_inode->inode = inode_ref(inode);
353
out:
354
    return l_inode;
355
}
356

357
static void
358
__destroy_lease_inode(lease_inode_t *l_inode)
359
{
360
    list_del_init(&l_inode->list);
361
    inode_unref(l_inode->inode);
362
    GF_FREE(l_inode);
363
}
364

365
static lease_client_t *
366
new_lease_client(const char *client_uid)
367
{
368
    lease_client_t *clnt = GF_MALLOC(sizeof(*clnt),
369
                                     gf_leases_mt_lease_client_t);
370
    if (!clnt)
371
        goto out;
372

373
    INIT_LIST_HEAD(&clnt->client_list);
374
    INIT_LIST_HEAD(&clnt->inode_list);
375
    clnt->client_uid = gf_strdup(client_uid);
376
out:
377
    return clnt;
378
}
379

380
static void
381
__destroy_lease_client(lease_client_t *clnt)
382
{
383
    list_del_init(&clnt->inode_list);
384
    list_del_init(&clnt->client_list);
385
    GF_FREE(clnt);
386

387
    return;
388
}
389

390
static lease_client_t *
391
__get_lease_client(xlator_t *this, leases_private_t *priv,
392
                   const char *client_uid)
393
{
394
    lease_client_t *clnt = NULL;
395
    lease_client_t *tmp = NULL;
396
    lease_client_t *found = NULL;
397

398
    list_for_each_entry_safe(clnt, tmp, &priv->client_list, client_list)
399
    {
400
        if ((strcmp(clnt->client_uid, client_uid) == 0)) {
401
            found = clnt;
402
            gf_msg_debug(this->name, 0,
403
                         "Client:%s already found "
404
                         "in the cleanup list",
405
                         client_uid);
406
            break;
407
        }
408
    }
409
    return found;
410
}
411

412
static lease_client_t *
413
__get_or_new_lease_client(xlator_t *this, leases_private_t *priv,
414
                          const char *client_uid)
415
{
416
    lease_client_t *found = NULL;
417

418
    found = __get_lease_client(this, priv, client_uid);
419
    if (!found) {
420
        found = new_lease_client(client_uid);
421
        if (!found)
422
            goto out;
423
        list_add_tail(&found->client_list, &priv->client_list);
424
        gf_msg_debug(this->name, 0,
425
                     "Adding a new client:%s entry "
426
                     "to the cleanup list",
427
                     client_uid);
428
    }
429
out:
430
    return found;
431
}
432

433
static int
434
add_inode_to_client_list(xlator_t *this, inode_t *inode, const char *client_uid)
435
{
436
    leases_private_t *priv = this->private;
437
    lease_client_t *clnt = NULL;
438

439
    lease_inode_t *lease_inode = new_lease_inode(inode);
440
    if (!lease_inode)
441
        return -ENOMEM;
442

443
    pthread_mutex_lock(&priv->mutex);
444
    {
445
        clnt = __get_or_new_lease_client(this, priv, client_uid);
446
        if (!clnt) {
447
            pthread_mutex_unlock(&priv->mutex);
448
            __destroy_lease_inode(lease_inode);
449
            return -ENOMEM;
450
        }
451
        list_add_tail(&clnt->inode_list, &lease_inode->list);
452
    }
453
    pthread_mutex_unlock(&priv->mutex);
454
    gf_msg_debug(this->name, 0,
455
                 "Added a new inode:%p to the client(%s) "
456
                 "cleanup list, gfid(%s)",
457
                 inode, client_uid, uuid_utoa(inode->gfid));
458
    return 0;
459
}
460

461
/* Add lease entry to the corresponding client entry.
462
 * Return values:
463
 * 0 Success
464
 * -1 Failure
465
 */
466
static int
467
__add_lease(call_frame_t *frame, inode_t *inode, lease_inode_ctx_t *lease_ctx,
468
            const char *client_uid, struct gf_lease *lease)
469
{
470
    lease_id_entry_t *lease_entry = NULL;
471
    int ret = -1;
472

473
    GF_VALIDATE_OR_GOTO("leases", frame, out);
474
    GF_VALIDATE_OR_GOTO("leases", client_uid, out);
475
    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
476
    GF_VALIDATE_OR_GOTO("leases", inode, out);
477
    GF_VALIDATE_OR_GOTO("leases", lease, out);
478

479
    gf_msg_trace(frame->this->name, 0,
480
                 "Granting lease lock to client %s with lease id %s"
481
                 " on gfid(%s)",
482
                 client_uid, leaseid_utoa(lease->lease_id),
483
                 uuid_utoa(inode->gfid));
484

485
    lease_entry = __get_or_new_lease_entry(frame, lease->lease_id, lease_ctx);
486
    if (!lease_entry) {
487
        errno = ENOMEM;
488
        goto out;
489
    }
490

491
    lease_entry->lease_type_cnt[lease->lease_type]++;
492
    lease_entry->lease_cnt++;
493
    lease_entry->lease_type |= lease->lease_type;
494
    /* If this is the first lease taken by the client on the file, then
495
     * add this inode/file to the client disconnect cleanup list
496
     */
497
    if (lease_entry->lease_cnt == 1) {
498
        add_inode_to_client_list(frame->this, inode, client_uid);
499
    }
500

501
    lease_ctx->lease_cnt++;
502
    lease_ctx->lease_type_cnt[lease->lease_type]++;
503
    lease_ctx->lease_type |= lease->lease_type;
504

505
    /* Take a ref for the first lock taken on this inode. Corresponding
506
     * unref when all the leases are unlocked or during DISCONNECT
507
     * Ref is required because the inode on which lease is acquired should
508
     * not be deleted when lru cleanup kicks in*/
509
    if (lease_ctx->lease_cnt == 1) {
510
        lease_ctx->inode = inode_ref(inode);
511
    }
512

513
    ret = 0;
514
out:
515
    return ret;
516
}
517

518
static gf_boolean_t
519
__is_clnt_lease_none(const char *client_uid, lease_inode_ctx_t *lease_ctx)
520
{
521
    gf_boolean_t lease_none = _gf_true;
522
    lease_id_entry_t *lease_entry = NULL;
523
    lease_id_entry_t *tmp = NULL;
524

525
    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
526
                             lease_id_list)
527
    {
528
        if ((strcmp(client_uid, lease_entry->client_uid) == 0) &&
529
            (lease_entry->lease_cnt != 0)) {
530
            lease_none = _gf_false;
531
            break;
532
        }
533
    }
534

535
    return lease_none;
536
}
537

538
static int
539
__remove_inode_from_clnt_list(xlator_t *this, lease_client_t *clnt,
540
                              inode_t *inode)
541
{
542
    int ret = -1;
543
    lease_inode_t *l_inode = NULL;
544
    lease_inode_t *tmp1 = NULL;
545

546
    list_for_each_entry_safe(l_inode, tmp1, &clnt->inode_list, list)
547
    {
548
        if (l_inode->inode == inode) {
549
            __destroy_lease_inode(l_inode);
550
            gf_msg_debug(this->name, 0,
551
                         "Removed the inode from the client cleanup list");
552
            ret = 0;
553
        }
554
    }
555
    /* TODO: Remove the client entry from the cleanup list */
556

557
    return ret;
558
}
559

560
static int
561
remove_from_clnt_list(xlator_t *this, const char *client_uid, inode_t *inode)
562
{
563
    leases_private_t *priv = NULL;
564
    int ret = -1;
565
    lease_client_t *clnt = NULL;
566

567
    priv = this->private;
568
    if (!priv)
569
        goto out;
570

571
    pthread_mutex_lock(&priv->mutex);
572
    {
573
        clnt = __get_lease_client(this, priv, client_uid);
574
        if (!clnt) {
575
            pthread_mutex_unlock(&priv->mutex);
576
            gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_CLNT_NOTFOUND,
577
                   "There is no client entry found in the cleanup list");
578
            goto out;
579
        }
580
        ret = __remove_inode_from_clnt_list(this, clnt, inode);
581
        if (ret) {
582
            pthread_mutex_unlock(&priv->mutex);
583
            gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_INODE_NOTFOUND,
584
                   "There is no inode entry found in the cleanup list");
585
            goto out;
586
        }
587
    }
588
    pthread_mutex_unlock(&priv->mutex);
589
out:
590
    return ret;
591
}
592

593
/* Remove lease entry in the corresponding client entry.
594
 */
595
static int
596
__remove_lease(xlator_t *this, inode_t *inode, lease_inode_ctx_t *lease_ctx,
597
               const char *client_uid, struct gf_lease *lease)
598
{
599
    lease_id_entry_t *lease_entry = NULL;
600
    int ret = 0;
601
    int32_t lease_type = 0;
602
    leases_private_t *priv = NULL;
603

604
    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
605
    GF_VALIDATE_OR_GOTO("leases", lease, out);
606

607
    priv = this->private;
608

609
    gf_msg_trace(this->name, 0,
610
                 "Removing lease entry for client: %s, "
611
                 "lease type:%d, lease id:%s",
612
                 client_uid, lease->lease_type, leaseid_utoa(lease->lease_id));
613

614
    /* There could be a race where in server recalled the lease and by the time
615
     * client sends lease_unlock request, server may have revoked it. To handle
616
     * such cases, if lease doesnt exist treat it as noop and return success.
617
     */
618
    lease_entry = __get_lease_id_entry(lease_ctx, lease->lease_id);
619
    if (!lease_entry) {
620
        gf_msg(this->name, GF_LOG_INFO, 0, LEASE_MSG_INVAL_UNLK_LEASE,
621
               "Got unlock lease request from client:%s, but has no "
622
               "corresponding lock",
623
               client_uid);
624
        ret = 0;
625
        goto out;
626
    }
627

628
    if (!(lease_entry->lease_type & lease->lease_type)) {
629
        gf_msg(this->name, GF_LOG_INFO, 0, LEASE_MSG_INVAL_UNLK_LEASE,
630
               "Got unlock lease request from client:%s for an invalid "
631
               "lease_type",
632
               client_uid);
633
        ret = -EINVAL;
634
        errno = EINVAL;
635
        goto out;
636
    }
637
    lease_type = lease->lease_type;
638
    lease_entry->lease_type_cnt[lease_type]--;
639
    lease_entry->lease_cnt--;
640

641
    lease_ctx->lease_type_cnt[lease_type]--;
642
    lease_ctx->lease_cnt--;
643

644
    if (lease_entry->lease_type_cnt[lease_type] == 0)
645
        lease_entry->lease_type = lease_entry->lease_type & (~lease_type);
646

647
    if (lease_ctx->lease_type_cnt[lease_type] == 0)
648
        lease_ctx->lease_type = lease_ctx->lease_type & (~lease_type);
649

650
    if (lease_entry->lease_cnt == 0) {
651
        if (__is_clnt_lease_none(client_uid, lease_ctx)) {
652
            gf_msg_trace(this->name, 0,
653
                         "Client(%s) has no leases"
654
                         " on gfid (%s), hence removing the inode"
655
                         " from the client cleanup list",
656
                         client_uid, uuid_utoa(inode->gfid));
657
            remove_from_clnt_list(this, client_uid, lease_ctx->inode);
658
        }
659
        __destroy_lease_id_entry(lease_entry);
660
        lease_ctx->blocked_fops_resuming = _gf_true;
661
    }
662

663
    if (lease_ctx->lease_cnt == 0 && lease_ctx->timer) {
664
        ret = gf_tw_del_timer(priv->timer_wheel, lease_ctx->timer);
665
        lease_ctx->recall_in_progress = _gf_false;
666
        lease_ctx->timer = NULL;
667
    }
668
out:
669
    return ret;
670
}
671

672
static gf_boolean_t
673
__is_lease_grantable(xlator_t *this, lease_inode_ctx_t *lease_ctx,
674
                     struct gf_lease *lease, inode_t *inode)
675
{
676
    uint32_t fd_count = 0;
677
    int32_t flags = 0;
678
    fd_t *iter_fd = NULL;
679
    gf_boolean_t grant = _gf_false;
680
    lease_fd_ctx_t *fd_ctx = NULL;
681

682
    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
683
    GF_VALIDATE_OR_GOTO("leases", lease, out);
684
    GF_VALIDATE_OR_GOTO("leases", inode, out);
685

686
    if (lease_ctx->recall_in_progress) {
687
        gf_msg_debug(this->name, 0,
688
                     "Recall in progress, hence "
689
                     "failing the lease request");
690
        grant = _gf_false;
691
        goto out;
692
    }
693

694
    if (lease_ctx->blocked_fops_resuming) {
695
        gf_msg_debug(this->name, 0,
696
                     "Previously blocked fops resuming, hence "
697
                     "failing the lease request");
698
        grant = _gf_false;
699
        goto out;
700
    }
701

702
    LOCK(&inode->lock);
703
    {
704
        list_for_each_entry(iter_fd, &inode->fd_list, inode_list)
705
        {
706
            fd_ctx = fd_ctx_get_ptr(iter_fd, this);
707
            if (!fd_ctx) {
708
                grant = _gf_false;
709
                UNLOCK(&inode->lock);
710
                gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_INVAL_FD_CTX,
711
                       "Unable to get fd ctx");
712
                goto out;
713
            }
714

715
            /* Check for open fd conflict, note that open fds from
716
             * the same lease id is not checked for conflict, as it is
717
             * lease id based lease.
718
             */
719
            if (fd_ctx->client_uid != NULL &&
720
                !__is_same_lease_id(fd_ctx->lease_id, lease->lease_id)) {
721
                fd_count++;
722
                flags |= iter_fd->flags;
723
            }
724
        }
725
    }
726
    UNLOCK(&inode->lock);
727

728
    gf_msg_debug(this->name, 0, "open fd count:%d flags:%d", fd_count, flags);
729

730
    __dump_leases_info(this, lease_ctx);
731

732
    switch (lease->lease_type) {
733
        case GF_RD_LEASE:
734
            /* check open fd conflict */
735
            if ((fd_count > 0) && ((flags & O_WRONLY) || (flags & O_RDWR))) {
736
                grant = _gf_false;
737
                break;
738
            }
739

740
            /* check for conflict with existing leases */
741
            if (lease_ctx->lease_type == NONE ||
742
                lease_ctx->lease_type == GF_RD_LEASE ||
743
                !(__another_lease_found(lease_ctx, lease->lease_id)))
744
                grant = _gf_true;
745
            else
746
                grant = _gf_false;
747
            break;
748

749
        case GF_RW_LEASE:
750
            /* check open fd conflict; conflict if there are any fds open
751
             * other than the client on which the lease is requested. */
752
            if (fd_count > 0) {
753
                grant = _gf_false;
754
                break;
755
            }
756

757
            /* check existing lease conflict */
758
            if (lease_ctx->lease_type == NONE ||
759
                !(__another_lease_found(lease_ctx, lease->lease_id)))
760
                grant = _gf_true;
761
            else
762
                grant = _gf_false;
763
            break;
764

765
        default:
766
            gf_msg(this->name, GF_LOG_ERROR, EINVAL, LEASE_MSG_INVAL_LEASE_TYPE,
767
                   "Invalid lease type specified");
768
            break;
769
    }
770
out:
771
    return grant;
772
}
773

774
static void
775
do_blocked_fops(xlator_t *this, lease_inode_ctx_t *lease_ctx)
776
{
777
    struct list_head wind_list;
778
    fop_stub_t *blk_fop = NULL;
779
    fop_stub_t *tmp = NULL;
780

781
    INIT_LIST_HEAD(&wind_list);
782

783
    pthread_mutex_lock(&lease_ctx->lock);
784
    {
785
        if (!lease_ctx->blocked_fops_resuming) {
786
            /* lease_ctx->blocked_fops_resuming will be set
787
             * only when the last lease is released. That
788
             * is when we need to resume blocked fops and unref
789
             * the inode taken in __add_lease (when lease_cnt == 1).
790
             * Return otherwise.
791
             */
792
            pthread_mutex_unlock(&lease_ctx->lock);
793
            return;
794
        }
795

796
        list_for_each_entry_safe(blk_fop, tmp, &lease_ctx->blocked_list, list)
797
        {
798
            list_del_init(&blk_fop->list);
799
            list_add_tail(&blk_fop->list, &wind_list);
800
        }
801
    }
802
    pthread_mutex_unlock(&lease_ctx->lock);
803

804
    gf_msg_trace(this->name, 0, "Executing the blocked stubs on gfid(%s)",
805
                 uuid_utoa(lease_ctx->inode->gfid));
806
    list_for_each_entry_safe(blk_fop, tmp, &wind_list, list)
807
    {
808
        list_del_init(&blk_fop->list);
809
        gf_msg_trace(this->name, 0, "Executing fop:%d", blk_fop->stub->fop);
810
        call_resume(blk_fop->stub);
811
        GF_FREE(blk_fop);
812
    }
813

814
    pthread_mutex_lock(&lease_ctx->lock);
815
    {
816
        lease_ctx->lease_type = NONE;
817
        /* unref the inode taken in __add_lease
818
         * (when lease_cnt == 1) */
819
        lease_ctx->blocked_fops_resuming = _gf_false;
820
        inode_unref(lease_ctx->inode);
821
        lease_ctx->inode = NULL;
822
    }
823
    pthread_mutex_unlock(&lease_ctx->lock);
824

825
    return;
826
}
827

828
void
829
recall_lease_timer_handler(struct gf_tw_timer_list *timer, void *data,
830
                           unsigned long calltime)
831
{
832
    inode_t *inode = NULL;
833
    lease_inode_t *lease_inode = NULL;
834
    leases_private_t *priv = NULL;
835
    lease_timer_data_t *timer_data = NULL;
836

837
    timer_data = data;
838

839
    priv = timer_data->this->private;
840
    inode = timer_data->inode;
841
    lease_inode = new_lease_inode(inode);
842
    if (!lease_inode) {
843
        errno = ENOMEM;
844
        goto out;
845
    }
846
    pthread_mutex_lock(&priv->mutex);
847
    {
848
        list_add_tail(&lease_inode->list, &priv->recall_list);
849
        pthread_cond_broadcast(&priv->cond);
850
    }
851
    pthread_mutex_unlock(&priv->mutex);
852
out:
853
    /* unref the inode_ref taken by timer_data in __recall_lease */
854
    inode_unref(timer_data->inode);
855

856
    GF_FREE(timer);
857
}
858

859
static void
860
__recall_lease(xlator_t *this, lease_inode_ctx_t *lease_ctx)
861
{
862
    lease_id_entry_t *lease_entry = NULL;
863
    lease_id_entry_t *tmp = NULL;
864
    struct gf_upcall up_req = {
865
        0,
866
    };
867
    struct gf_upcall_recall_lease recall_req = {
868
        0,
869
    };
870
    int notify_ret = -1;
871
    struct gf_tw_timer_list *timer = NULL;
872
    leases_private_t *priv = NULL;
873
    lease_timer_data_t *timer_data = NULL;
874
    time_t recall_time;
875

876
    if (lease_ctx->recall_in_progress) {
877
        gf_msg_debug(this->name, 0,
878
                     "Lease recall is already in "
879
                     "progress, hence not sending another recall");
880
        goto out;
881
    }
882

883
    priv = this->private;
884
    recall_time = gf_time();
885
    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
886
                             lease_id_list)
887
    {
888
        gf_uuid_copy(up_req.gfid, lease_ctx->inode->gfid);
889
        up_req.client_uid = lease_entry->client_uid;
890
        up_req.event_type = GF_UPCALL_RECALL_LEASE;
891
        up_req.data = &recall_req;
892

893
        notify_ret = this->notify(this, GF_EVENT_UPCALL, &up_req);
894
        if (notify_ret < 0) {
895
            gf_msg(this->name, GF_LOG_ERROR, 0, LEASE_MSG_RECALL_FAIL,
896
                   "Recall notification to client: %s failed",
897
                   lease_entry->client_uid);
898
            /* Do not return from here, continue registering the timer,
899
               this is required mostly o keep replicas in sync*/
900
        } else {
901
            gf_msg_debug(this->name, 0,
902
                         "Recall lease (all)"
903
                         "notification sent to client %s",
904
                         lease_entry->client_uid);
905
        }
906

907
        lease_ctx->recall_in_progress = _gf_true;
908
        lease_entry->recall_time = recall_time;
909
    }
910
    timer = GF_MALLOC(sizeof(*timer), gf_common_mt_tw_timer_list);
911
    if (!timer) {
912
        goto out;
913
    }
914
    timer_data = GF_MALLOC(sizeof(lease_timer_data_t),
915
                           gf_leases_mt_timer_data_t);
916
    if (!timer_data) {
917
        GF_FREE(timer);
918
        goto out;
919
    }
920

921
    timer_data->inode = inode_ref(lease_ctx->inode);
922
    timer_data->this = this;
923
    timer->data = timer_data;
924

925
    INIT_LIST_HEAD(&timer->entry);
926
    timer->expires = get_recall_lease_timeout(this);
927
    timer->function = recall_lease_timer_handler;
928
    lease_ctx->timer = timer;
929
    gf_tw_add_timer(priv->timer_wheel, timer);
930
    gf_msg_trace(this->name, 0,
931
                 "Registering timer "
932
                 "%p, after "
933
                 "sending recall",
934
                 timer);
935
out:
936
    return;
937
}
938

939
/* ret = 0; STACK_UNWIND Success
940
 * ret = -1; STACK_UNWIND failure
941
 */
942
int
943
process_lease_req(call_frame_t *frame, xlator_t *this, inode_t *inode,
944
                  struct gf_lease *lease)
945
{
946
    int ret = 0;
947
    char *client_uid = NULL;
948
    lease_inode_ctx_t *lease_ctx = NULL;
949

950
    GF_VALIDATE_OR_GOTO("leases", frame, out);
951
    GF_VALIDATE_OR_GOTO("leases", this, out);
952
    GF_VALIDATE_OR_GOTO("leases", inode, out);
953
    GF_VALIDATE_OR_GOTO("leases", lease, out);
954

955
    client_uid = frame->root->client->client_uid;
956

957
    if (!is_valid_lease_id(lease->lease_id)) {
958
        gf_msg(this->name, GF_LOG_ERROR, EINVAL, LEASE_MSG_INVAL_LEASE_ID,
959
               "Invalid lease id, from"
960
               "client:%s",
961
               client_uid);
962
        ret = -EINVAL;
963
        errno = EINVAL;
964
        goto out;
965
    }
966

967
    lease_ctx = lease_ctx_get(inode, this);
968
    if (!lease_ctx) {
969
        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
970
               "Unable to create/get inode ctx, "
971
               "inode:%p",
972
               inode);
973
        ret = -ENOMEM;
974
        errno = ENOMEM;
975
        goto out;
976
    }
977

978
    gf_msg_debug(this->name, 0,
979
                 "Lease request from client: %s, "
980
                 "lease type:%d, lease cmd:%d, lease ID:%s, gfid:%s",
981
                 client_uid, lease->lease_type, lease->cmd,
982
                 leaseid_utoa(lease->lease_id), uuid_utoa(inode->gfid));
983

984
    pthread_mutex_lock(&lease_ctx->lock);
985
    {
986
        switch (lease->cmd) {
987
            case GF_GET_LEASE:
988
                lease->lease_type = lease_ctx->lease_type;
989
                gf_msg_debug(this->name, 0,
990
                             "Get lease, existing lease"
991
                             "type: %d",
992
                             lease_ctx->lease_type);
993
                /*TODO:Should it consider lease id or client_uid?*/
994
                break;
995

996
            case GF_SET_LEASE:
997
                if (__is_lease_grantable(this, lease_ctx, lease, inode)) {
998
                    __add_lease(frame, inode, lease_ctx, client_uid, lease);
999
                    ret = 0;
1000
                } else {
1001
                    gf_msg_debug(this->name, GF_LOG_DEBUG,
1002
                                 "Not granting the conflicting lease"
1003
                                 " request from %s on gfid(%s)",
1004
                                 client_uid, uuid_utoa(inode->gfid));
1005
                    __recall_lease(this, lease_ctx);
1006
                    ret = -1;
1007
                }
1008
                break;
1009
            case GF_UNLK_LEASE:
1010
                ret = __remove_lease(this, inode, lease_ctx, client_uid, lease);
1011
                if ((ret >= 0) && (lease_ctx->lease_cnt == 0)) {
1012
                    pthread_mutex_unlock(&lease_ctx->lock);
1013
                    goto unblock;
1014
                }
1015
                break;
1016
            default:
1017
                ret = -EINVAL;
1018
                break;
1019
        }
1020
    }
1021
    pthread_mutex_unlock(&lease_ctx->lock);
1022

1023
    return ret;
1024

1025
unblock:
1026
    do_blocked_fops(this, lease_ctx);
1027
out:
1028
    return ret;
1029
}
1030

1031
/* ret = 1 conflict
1032
 * ret = 0 no conflict
1033
 */
1034
gf_boolean_t
1035
__check_lease_conflict(call_frame_t *frame, lease_inode_ctx_t *lease_ctx,
1036
                       const char *lease_id, gf_boolean_t is_write)
1037
{
1038
    gf_lease_types_t lease_type = {
1039
        0,
1040
    };
1041
    gf_boolean_t conflicts = _gf_false;
1042
    lease_id_entry_t *lease_entry = NULL;
1043

1044
    GF_VALIDATE_OR_GOTO("leases", frame, out);
1045
    GF_VALIDATE_OR_GOTO("leases", lease_ctx, out);
1046

1047
    lease_type = lease_ctx->lease_type;
1048

1049
    /* If the fop is rename or unlink conflict the lease even if its
1050
     * from the same client??
1051
     */
1052
    if ((frame->root->op == GF_FOP_RENAME) ||
1053
        (frame->root->op == GF_FOP_UNLINK)) {
1054
        conflicts = _gf_true;
1055
        goto recall;
1056
    }
1057

1058
    /* As internal fops are used to maintain data integrity but do not
1059
     * make modififications to the client data, no need to conflict with
1060
     * them.
1061
     *
1062
     * @todo: like for locks, even lease state has to be handled by
1063
     * rebalance or self-heal daemon process. */
1064
    if (frame->root->pid < 0) {
1065
        conflicts = _gf_false;
1066
        goto recall;
1067
    }
1068

1069
    /* If lease_id is not sent, set conflicts = true if there is
1070
     * an existing lease */
1071
    if (!lease_id && (lease_ctx->lease_cnt > 0)) {
1072
        conflicts = _gf_true;
1073
        goto recall;
1074
    }
1075

1076
    switch (lease_type) {
1077
        case (GF_RW_LEASE | GF_RD_LEASE):
1078
        case GF_RW_LEASE:
1079
            lease_entry = __get_lease_id_entry(lease_ctx, lease_id);
1080
            if (lease_entry && (lease_entry->lease_type & GF_RW_LEASE))
1081
                conflicts = _gf_false;
1082
            else
1083
                conflicts = _gf_true;
1084
            break;
1085
        case GF_RD_LEASE:
1086
            if (is_write && __another_lease_found(lease_ctx, lease_id))
1087
                conflicts = _gf_true;
1088
            else
1089
                conflicts = _gf_false;
1090
            break;
1091
        default:
1092
            break;
1093
    }
1094

1095
recall:
1096
    /* If there is a conflict found and recall is not already sent to all
1097
     * the clients, then send recall to each of the client holding lease.
1098
     */
1099
    if (conflicts)
1100
        __recall_lease(frame->this, lease_ctx);
1101
out:
1102
    return conflicts;
1103
}
1104

1105
/* Return values:
1106
 * -1 : error, unwind the fop
1107
 * WIND_FOP: No conflict, wind the fop
1108
 * BLOCK_FOP: Found a conflicting lease, block the fop
1109
 */
1110
int
1111
check_lease_conflict(call_frame_t *frame, inode_t *inode, const char *lease_id,
1112
                     uint32_t fop_flags)
1113
{
1114
    lease_inode_ctx_t *lease_ctx = NULL;
1115
    gf_boolean_t is_blocking_fop = _gf_false;
1116
    gf_boolean_t is_write_fop = _gf_false;
1117
    gf_boolean_t conflicts = _gf_false;
1118
    int ret = WIND_FOP;
1119

1120
    lease_ctx = lease_ctx_get(inode, frame->this);
1121
    if (!lease_ctx) {
1122
        gf_msg(frame->this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_NO_MEM,
1123
               "Unable to create/get inode ctx");
1124
        ret = -1;
1125
        errno = ENOMEM;
1126
        goto out;
1127
    }
1128

1129
    is_blocking_fop = ((fop_flags & BLOCKING_FOP) != 0);
1130
    is_write_fop = ((fop_flags & DATA_MODIFY_FOP) != 0);
1131

1132
    pthread_mutex_lock(&lease_ctx->lock);
1133
    {
1134
        if (lease_ctx->lease_type == NONE) {
1135
            pthread_mutex_unlock(&lease_ctx->lock);
1136
            gf_msg_debug(frame->this->name, 0,
1137
                         "No leases found continuing with the"
1138
                         " fop:%s",
1139
                         gf_fop_list[frame->root->op]);
1140
            ret = WIND_FOP;
1141
            goto out;
1142
        }
1143
        conflicts = __check_lease_conflict(frame, lease_ctx, lease_id,
1144
                                           is_write_fop);
1145
        if (conflicts) {
1146
            if (is_blocking_fop) {
1147
                gf_msg_debug(frame->this->name, 0,
1148
                             "Fop: %s "
1149
                             "conflicting existing "
1150
                             "lease: %d, blocking the"
1151
                             "fop",
1152
                             gf_fop_list[frame->root->op],
1153
                             lease_ctx->lease_type);
1154
                ret = BLOCK_FOP;
1155
            } else {
1156
                gf_msg_debug(frame->this->name, 0,
1157
                             "Fop: %s "
1158
                             "conflicting existing "
1159
                             "lease: %d, sending "
1160
                             "EAGAIN",
1161
                             gf_fop_list[frame->root->op],
1162
                             lease_ctx->lease_type);
1163
                errno = EAGAIN;
1164
                ret = -1;
1165
            }
1166
        }
1167
    }
1168
    pthread_mutex_unlock(&lease_ctx->lock);
1169
out:
1170
    return ret;
1171
}
1172

1173
static int
1174
remove_clnt_leases(const char *client_uid, inode_t *inode, xlator_t *this)
1175
{
1176
    lease_inode_ctx_t *lease_ctx = NULL;
1177
    lease_id_entry_t *lease_entry = NULL;
1178
    lease_id_entry_t *tmp = NULL;
1179
    int ret = 0;
1180
    int i = 0;
1181

1182
    lease_ctx = lease_ctx_get(inode, this);
1183
    if (!lease_ctx) {
1184
        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_INVAL_INODE_CTX,
1185
               "Unable to create/get inode ctx");
1186
        ret = -1;
1187
        errno = ENOMEM;
1188
        goto out;
1189
    }
1190

1191
    pthread_mutex_lock(&lease_ctx->lock);
1192
    {
1193
        list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
1194
                                 lease_id_list)
1195
        {
1196
            if (strcmp(client_uid, lease_entry->client_uid) == 0) {
1197
                for (i = 0; i < GF_LEASE_MAX_TYPE; i++) {
1198
                    lease_ctx->lease_type_cnt[i] -= lease_entry
1199
                                                        ->lease_type_cnt[i];
1200
                }
1201
                lease_ctx->lease_cnt -= lease_entry->lease_cnt;
1202
                __destroy_lease_id_entry(lease_entry);
1203
                if (lease_ctx->lease_cnt == 0) {
1204
                    lease_ctx->blocked_fops_resuming = _gf_true;
1205
                    pthread_mutex_unlock(&lease_ctx->lock);
1206
                    goto unblock;
1207
                }
1208
            }
1209
        }
1210
    }
1211
    pthread_mutex_unlock(&lease_ctx->lock);
1212
out:
1213
    return ret;
1214

1215
unblock:
1216
    do_blocked_fops(this, lease_ctx);
1217
    return ret;
1218
}
1219

1220
int
1221
cleanup_client_leases(xlator_t *this, const char *client_uid)
1222
{
1223
    lease_client_t *clnt = NULL;
1224
    lease_client_t *tmp = NULL;
1225
    struct list_head cleanup_list = {
1226
        0,
1227
    };
1228
    lease_inode_t *l_inode = NULL;
1229
    lease_inode_t *tmp1 = NULL;
1230
    leases_private_t *priv = NULL;
1231
    int ret = 0;
1232

1233
    priv = this->private;
1234
    if (!priv) {
1235
        ret = -1;
1236
        errno = EINVAL;
1237
        goto out;
1238
    }
1239

1240
    INIT_LIST_HEAD(&cleanup_list);
1241
    pthread_mutex_lock(&priv->mutex);
1242
    {
1243
        list_for_each_entry_safe(clnt, tmp, &priv->client_list, client_list)
1244
        {
1245
            if ((strcmp(clnt->client_uid, client_uid) == 0)) {
1246
                list_for_each_entry_safe(l_inode, tmp1, &clnt->inode_list, list)
1247
                {
1248
                    list_del_init(&l_inode->list);
1249
                    list_add_tail(&l_inode->list, &cleanup_list);
1250
                }
1251
                __destroy_lease_client(clnt);
1252
                break;
1253
            }
1254
        }
1255
    }
1256
    pthread_mutex_unlock(&priv->mutex);
1257

1258
    l_inode = tmp1 = NULL;
1259
    list_for_each_entry_safe(l_inode, tmp1, &cleanup_list, list)
1260
    {
1261
        remove_clnt_leases(client_uid, l_inode->inode, this);
1262
        __destroy_lease_inode(l_inode);
1263
    }
1264
out:
1265
    return ret;
1266
}
1267

1268
static void
1269
__remove_all_leases(xlator_t *this, lease_inode_ctx_t *lease_ctx)
1270
{
1271
    int i = 0;
1272
    lease_id_entry_t *lease_entry = NULL;
1273
    lease_id_entry_t *tmp = NULL;
1274

1275
    if (lease_ctx->lease_cnt == 0) {
1276
        /* No leases to remove. Return */
1277
        return;
1278
    }
1279
    __dump_leases_info(this, lease_ctx);
1280

1281
    list_for_each_entry_safe(lease_entry, tmp, &lease_ctx->lease_id_list,
1282
                             lease_id_list)
1283
    {
1284
        lease_entry->lease_cnt = 0;
1285
        remove_from_clnt_list(this, lease_entry->client_uid, lease_ctx->inode);
1286
        __destroy_lease_id_entry(lease_entry);
1287
    }
1288
    INIT_LIST_HEAD(&lease_ctx->lease_id_list);
1289
    for (i = 0; i <= GF_LEASE_MAX_TYPE; i++)
1290
        lease_ctx->lease_type_cnt[i] = 0;
1291
    lease_ctx->lease_type = 0;
1292
    lease_ctx->lease_cnt = 0;
1293
    lease_ctx->recall_in_progress = _gf_false;
1294
    lease_ctx->timer = NULL;
1295
    lease_ctx->blocked_fops_resuming = _gf_true;
1296

1297
    /* TODO:
1298
     * - Mark the corresponding fd bad. Could be done on client side
1299
     * as a result of recall
1300
     * - Free the lease_ctx
1301
     */
1302
    return;
1303
}
1304

1305
static int
1306
remove_all_leases(xlator_t *this, inode_t *inode)
1307
{
1308
    lease_inode_ctx_t *lease_ctx = NULL;
1309
    int ret = 0;
1310

1311
    GF_VALIDATE_OR_GOTO("leases", inode, out);
1312

1313
    lease_ctx = lease_ctx_get(inode, this);
1314
    if (!lease_ctx) {
1315
        gf_msg(this->name, GF_LOG_WARNING, ENOMEM, LEASE_MSG_INVAL_INODE_CTX,
1316
               "Unable to create/get inode ctx");
1317
        ret = -1;
1318
        errno = ENOMEM;
1319
        goto out;
1320
    }
1321

1322
    pthread_mutex_lock(&lease_ctx->lock);
1323
    {
1324
        __remove_all_leases(this, lease_ctx);
1325
    }
1326
    pthread_mutex_unlock(&lease_ctx->lock);
1327

1328
    do_blocked_fops(this, lease_ctx);
1329
out:
1330
    return ret;
1331
}
1332

1333
void *
1334
expired_recall_cleanup(void *data)
1335
{
1336
    struct timespec sleep_till = {
1337
        0,
1338
    };
1339
    struct list_head recall_cleanup_list;
1340
    lease_inode_t *recall_entry = NULL;
1341
    lease_inode_t *tmp = NULL;
1342
    leases_private_t *priv = NULL;
1343
    xlator_t *this = NULL;
1344
    time_t time_now;
1345

1346
    GF_VALIDATE_OR_GOTO("leases", data, out);
1347

1348
    this = data;
1349
    priv = this->private;
1350

1351
    gf_msg_debug(this->name, 0, "Started the expired_recall_cleanup thread");
1352

1353
    while (1) {
1354
        time_now = gf_time();
1355
        pthread_mutex_lock(&priv->mutex);
1356
        {
1357
            if (priv->fini) {
1358
                pthread_mutex_unlock(&priv->mutex);
1359
                goto out;
1360
            }
1361
            INIT_LIST_HEAD(&recall_cleanup_list);
1362
            if (list_empty(&priv->recall_list)) {
1363
                sleep_till.tv_sec = time_now + 600;
1364
                pthread_cond_timedwait(&priv->cond, &priv->mutex, &sleep_till);
1365
            }
1366
            if (!list_empty(&priv->recall_list)) {
1367
                gf_msg_debug(this->name, 0, "Found expired recalls");
1368
                list_for_each_entry_safe(recall_entry, tmp, &priv->recall_list,
1369
                                         list)
1370
                {
1371
                    list_del_init(&recall_entry->list);
1372
                    list_add_tail(&recall_entry->list, &recall_cleanup_list);
1373
                }
1374
            }
1375
        }
1376
        pthread_mutex_unlock(&priv->mutex);
1377

1378
        recall_entry = tmp = NULL;
1379
        list_for_each_entry_safe(recall_entry, tmp, &recall_cleanup_list, list)
1380
        {
1381
            gf_msg_debug(this->name, 0,
1382
                         "Recall lease was sent on"
1383
                         " inode:%p, recall timer has expired"
1384
                         " and clients haven't unlocked the lease"
1385
                         " hence cleaning up leases on the inode",
1386
                         recall_entry->inode);
1387
            remove_all_leases(this, recall_entry->inode);
1388
            /* no need to take priv->mutex lock as this entry
1389
             * reference is removed from global recall list. */
1390
            __destroy_lease_inode(recall_entry);
1391
        }
1392
    }
1393

1394
out:
1395
    return NULL;
1396
}
1397