qemu

1
/* SPDX-License-Identifier: GPL-2.0-or-later */
2

3
#include "qemu/osdep.h"
4
#include "qemu/interval-tree.h"
5
#include "qemu/atomic.h"
6

7
/*
8
 * Red Black Trees.
9
 *
10
 * For now, don't expose Linux Red-Black Trees separately, but retain the
11
 * separate type definitions to keep the implementation sane, and allow
12
 * the possibility of separating them later.
13
 *
14
 * Derived from include/linux/rbtree_augmented.h and its dependencies.
15
 */
16

17
/*
18
 * red-black trees properties:  https://en.wikipedia.org/wiki/Rbtree
19
 *
20
 *  1) A node is either red or black
21
 *  2) The root is black
22
 *  3) All leaves (NULL) are black
23
 *  4) Both children of every red node are black
24
 *  5) Every simple path from root to leaves contains the same number
25
 *     of black nodes.
26
 *
27
 *  4 and 5 give the O(log n) guarantee, since 4 implies you cannot have two
28
 *  consecutive red nodes in a path and every red node is therefore followed by
29
 *  a black. So if B is the number of black nodes on every simple path (as per
30
 *  5), then the longest possible path due to 4 is 2B.
31
 *
32
 *  We shall indicate color with case, where black nodes are uppercase and red
33
 *  nodes will be lowercase. Unknown color nodes shall be drawn as red within
34
 *  parentheses and have some accompanying text comment.
35
 *
36
 * Notes on lockless lookups:
37
 *
38
 * All stores to the tree structure (rb_left and rb_right) must be done using
39
 * WRITE_ONCE [qatomic_set for QEMU]. And we must not inadvertently cause
40
 * (temporary) loops in the tree structure as seen in program order.
41
 *
42
 * These two requirements will allow lockless iteration of the tree -- not
43
 * correct iteration mind you, tree rotations are not atomic so a lookup might
44
 * miss entire subtrees.
45
 *
46
 * But they do guarantee that any such traversal will only see valid elements
47
 * and that it will indeed complete -- does not get stuck in a loop.
48
 *
49
 * It also guarantees that if the lookup returns an element it is the 'correct'
50
 * one. But not returning an element does _NOT_ mean it's not present.
51
 */
52

53
typedef enum RBColor
54
{
55
    RB_RED,
56
    RB_BLACK,
57
} RBColor;
58

59
typedef struct RBAugmentCallbacks {
60
    void (*propagate)(RBNode *node, RBNode *stop);
61
    void (*copy)(RBNode *old, RBNode *new);
62
    void (*rotate)(RBNode *old, RBNode *new);
63
} RBAugmentCallbacks;
64

65
static inline uintptr_t rb_pc(const RBNode *n)
66
{
67
    return qatomic_read(&n->rb_parent_color);
68
}
69

70
static inline void rb_set_pc(RBNode *n, uintptr_t pc)
71
{
72
    qatomic_set(&n->rb_parent_color, pc);
73
}
74

75
static inline RBNode *pc_parent(uintptr_t pc)
76
{
77
    return (RBNode *)(pc & ~1);
78
}
79

80
static inline RBNode *rb_parent(const RBNode *n)
81
{
82
    return pc_parent(rb_pc(n));
83
}
84

85
static inline RBNode *rb_red_parent(const RBNode *n)
86
{
87
    return (RBNode *)rb_pc(n);
88
}
89

90
static inline RBColor pc_color(uintptr_t pc)
91
{
92
    return (RBColor)(pc & 1);
93
}
94

95
static inline bool pc_is_red(uintptr_t pc)
96
{
97
    return pc_color(pc) == RB_RED;
98
}
99

100
static inline bool pc_is_black(uintptr_t pc)
101
{
102
    return !pc_is_red(pc);
103
}
104

105
static inline RBColor rb_color(const RBNode *n)
106
{
107
    return pc_color(rb_pc(n));
108
}
109

110
static inline bool rb_is_red(const RBNode *n)
111
{
112
    return pc_is_red(rb_pc(n));
113
}
114

115
static inline bool rb_is_black(const RBNode *n)
116
{
117
    return pc_is_black(rb_pc(n));
118
}
119

120
static inline void rb_set_black(RBNode *n)
121
{
122
    rb_set_pc(n, rb_pc(n) | RB_BLACK);
123
}
124

125
static inline void rb_set_parent_color(RBNode *n, RBNode *p, RBColor color)
126
{
127
    rb_set_pc(n, (uintptr_t)p | color);
128
}
129

130
static inline void rb_set_parent(RBNode *n, RBNode *p)
131
{
132
    rb_set_parent_color(n, p, rb_color(n));
133
}
134

135
static inline void rb_link_node(RBNode *node, RBNode *parent, RBNode **rb_link)
136
{
137
    node->rb_parent_color = (uintptr_t)parent;
138
    node->rb_left = node->rb_right = NULL;
139

140
    /*
141
     * Ensure that node is initialized before insertion,
142
     * as viewed by a concurrent search.
143
     */
144
    qatomic_set_mb(rb_link, node);
145
}
146

147
static RBNode *rb_next(RBNode *node)
148
{
149
    RBNode *parent;
150

151
    /* OMIT: if empty node, return null. */
152

153
    /*
154
     * If we have a right-hand child, go down and then left as far as we can.
155
     */
156
    if (node->rb_right) {
157
        node = node->rb_right;
158
        while (node->rb_left) {
159
            node = node->rb_left;
160
        }
161
        return node;
162
    }
163

164
    /*
165
     * No right-hand children. Everything down and left is smaller than us,
166
     * so any 'next' node must be in the general direction of our parent.
167
     * Go up the tree; any time the ancestor is a right-hand child of its
168
     * parent, keep going up. First time it's a left-hand child of its
169
     * parent, said parent is our 'next' node.
170
     */
171
    while ((parent = rb_parent(node)) && node == parent->rb_right) {
172
        node = parent;
173
    }
174

175
    return parent;
176
}
177

178
static inline void rb_change_child(RBNode *old, RBNode *new,
179
                                   RBNode *parent, RBRoot *root)
180
{
181
    if (!parent) {
182
        qatomic_set(&root->rb_node, new);
183
    } else if (parent->rb_left == old) {
184
        qatomic_set(&parent->rb_left, new);
185
    } else {
186
        qatomic_set(&parent->rb_right, new);
187
    }
188
}
189

190
static inline void rb_rotate_set_parents(RBNode *old, RBNode *new,
191
                                         RBRoot *root, RBColor color)
192
{
193
    uintptr_t pc = rb_pc(old);
194
    RBNode *parent = pc_parent(pc);
195

196
    rb_set_pc(new, pc);
197
    rb_set_parent_color(old, new, color);
198
    rb_change_child(old, new, parent, root);
199
}
200

201
static void rb_insert_augmented(RBNode *node, RBRoot *root,
202
                                const RBAugmentCallbacks *augment)
203
{
204
    RBNode *parent = rb_red_parent(node), *gparent, *tmp;
205

206
    while (true) {
207
        /*
208
         * Loop invariant: node is red.
209
         */
210
        if (unlikely(!parent)) {
211
            /*
212
             * The inserted node is root. Either this is the first node, or
213
             * we recursed at Case 1 below and are no longer violating 4).
214
             */
215
            rb_set_parent_color(node, NULL, RB_BLACK);
216
            break;
217
        }
218

219
        /*
220
         * If there is a black parent, we are done.  Otherwise, take some
221
         * corrective action as, per 4), we don't want a red root or two
222
         * consecutive red nodes.
223
         */
224
        if (rb_is_black(parent)) {
225
            break;
226
        }
227

228
        gparent = rb_red_parent(parent);
229

230
        tmp = gparent->rb_right;
231
        if (parent != tmp) {    /* parent == gparent->rb_left */
232
            if (tmp && rb_is_red(tmp)) {
233
                /*
234
                 * Case 1 - node's uncle is red (color flips).
235
                 *
236
                 *       G            g
237
                 *      / \          / \
238
                 *     p   u  -->   P   U
239
                 *    /            /
240
                 *   n            n
241
                 *
242
                 * However, since g's parent might be red, and 4) does not
243
                 * allow this, we need to recurse at g.
244
                 */
245
                rb_set_parent_color(tmp, gparent, RB_BLACK);
246
                rb_set_parent_color(parent, gparent, RB_BLACK);
247
                node = gparent;
248
                parent = rb_parent(node);
249
                rb_set_parent_color(node, parent, RB_RED);
250
                continue;
251
            }
252

253
            tmp = parent->rb_right;
254
            if (node == tmp) {
255
                /*
256
                 * Case 2 - node's uncle is black and node is
257
                 * the parent's right child (left rotate at parent).
258
                 *
259
                 *      G             G
260
                 *     / \           / \
261
                 *    p   U  -->    n   U
262
                 *     \           /
263
                 *      n         p
264
                 *
265
                 * This still leaves us in violation of 4), the
266
                 * continuation into Case 3 will fix that.
267
                 */
268
                tmp = node->rb_left;
269
                qatomic_set(&parent->rb_right, tmp);
270
                qatomic_set(&node->rb_left, parent);
271
                if (tmp) {
272
                    rb_set_parent_color(tmp, parent, RB_BLACK);
273
                }
274
                rb_set_parent_color(parent, node, RB_RED);
275
                augment->rotate(parent, node);
276
                parent = node;
277
                tmp = node->rb_right;
278
            }
279

280
            /*
281
             * Case 3 - node's uncle is black and node is
282
             * the parent's left child (right rotate at gparent).
283
             *
284
             *        G           P
285
             *       / \         / \
286
             *      p   U  -->  n   g
287
             *     /                 \
288
             *    n                   U
289
             */
290
            qatomic_set(&gparent->rb_left, tmp); /* == parent->rb_right */
291
            qatomic_set(&parent->rb_right, gparent);
292
            if (tmp) {
293
                rb_set_parent_color(tmp, gparent, RB_BLACK);
294
            }
295
            rb_rotate_set_parents(gparent, parent, root, RB_RED);
296
            augment->rotate(gparent, parent);
297
            break;
298
        } else {
299
            tmp = gparent->rb_left;
300
            if (tmp && rb_is_red(tmp)) {
301
                /* Case 1 - color flips */
302
                rb_set_parent_color(tmp, gparent, RB_BLACK);
303
                rb_set_parent_color(parent, gparent, RB_BLACK);
304
                node = gparent;
305
                parent = rb_parent(node);
306
                rb_set_parent_color(node, parent, RB_RED);
307
                continue;
308
            }
309

310
            tmp = parent->rb_left;
311
            if (node == tmp) {
312
                /* Case 2 - right rotate at parent */
313
                tmp = node->rb_right;
314
                qatomic_set(&parent->rb_left, tmp);
315
                qatomic_set(&node->rb_right, parent);
316
                if (tmp) {
317
                    rb_set_parent_color(tmp, parent, RB_BLACK);
318
                }
319
                rb_set_parent_color(parent, node, RB_RED);
320
                augment->rotate(parent, node);
321
                parent = node;
322
                tmp = node->rb_left;
323
            }
324

325
            /* Case 3 - left rotate at gparent */
326
            qatomic_set(&gparent->rb_right, tmp); /* == parent->rb_left */
327
            qatomic_set(&parent->rb_left, gparent);
328
            if (tmp) {
329
                rb_set_parent_color(tmp, gparent, RB_BLACK);
330
            }
331
            rb_rotate_set_parents(gparent, parent, root, RB_RED);
332
            augment->rotate(gparent, parent);
333
            break;
334
        }
335
    }
336
}
337

338
static void rb_insert_augmented_cached(RBNode *node,
339
                                       RBRootLeftCached *root, bool newleft,
340
                                       const RBAugmentCallbacks *augment)
341
{
342
    if (newleft) {
343
        root->rb_leftmost = node;
344
    }
345
    rb_insert_augmented(node, &root->rb_root, augment);
346
}
347

348
static void rb_erase_color(RBNode *parent, RBRoot *root,
349
                           const RBAugmentCallbacks *augment)
350
{
351
    RBNode *node = NULL, *sibling, *tmp1, *tmp2;
352

353
    while (true) {
354
        /*
355
         * Loop invariants:
356
         * - node is black (or NULL on first iteration)
357
         * - node is not the root (parent is not NULL)
358
         * - All leaf paths going through parent and node have a
359
         *   black node count that is 1 lower than other leaf paths.
360
         */
361
        sibling = parent->rb_right;
362
        if (node != sibling) {  /* node == parent->rb_left */
363
            if (rb_is_red(sibling)) {
364
                /*
365
                 * Case 1 - left rotate at parent
366
                 *
367
                 *     P               S
368
                 *    / \             / \ 
369
                 *   N   s    -->    p   Sr
370
                 *      / \         / \ 
371
                 *     Sl  Sr      N   Sl
372
                 */
373
                tmp1 = sibling->rb_left;
374
                qatomic_set(&parent->rb_right, tmp1);
375
                qatomic_set(&sibling->rb_left, parent);
376
                rb_set_parent_color(tmp1, parent, RB_BLACK);
377
                rb_rotate_set_parents(parent, sibling, root, RB_RED);
378
                augment->rotate(parent, sibling);
379
                sibling = tmp1;
380
            }
381
            tmp1 = sibling->rb_right;
382
            if (!tmp1 || rb_is_black(tmp1)) {
383
                tmp2 = sibling->rb_left;
384
                if (!tmp2 || rb_is_black(tmp2)) {
385
                    /*
386
                     * Case 2 - sibling color flip
387
                     * (p could be either color here)
388
                     *
389
                     *    (p)           (p)
390
                     *    / \           / \ 
391
                     *   N   S    -->  N   s
392
                     *      / \           / \ 
393
                     *     Sl  Sr        Sl  Sr
394
                     *
395
                     * This leaves us violating 5) which
396
                     * can be fixed by flipping p to black
397
                     * if it was red, or by recursing at p.
398
                     * p is red when coming from Case 1.
399
                     */
400
                    rb_set_parent_color(sibling, parent, RB_RED);
401
                    if (rb_is_red(parent)) {
402
                        rb_set_black(parent);
403
                    } else {
404
                        node = parent;
405
                        parent = rb_parent(node);
406
                        if (parent) {
407
                            continue;
408
                        }
409
                    }
410
                    break;
411
                }
412
                /*
413
                 * Case 3 - right rotate at sibling
414
                 * (p could be either color here)
415
                 *
416
                 *   (p)           (p)
417
                 *   / \           / \
418
                 *  N   S    -->  N   sl
419
                 *     / \             \
420
                 *    sl  Sr            S
421
                 *                       \
422
                 *                        Sr
423
                 *
424
                 * Note: p might be red, and then bot
425
                 * p and sl are red after rotation (which
426
                 * breaks property 4). This is fixed in
427
                 * Case 4 (in rb_rotate_set_parents()
428
                 *         which set sl the color of p
429
                 *         and set p RB_BLACK)
430
                 *
431
                 *   (p)            (sl)
432
                 *   / \            /  \
433
                 *  N   sl   -->   P    S
434
                 *       \        /      \
435
                 *        S      N        Sr
436
                 *         \
437
                 *          Sr
438
                 */
439
                tmp1 = tmp2->rb_right;
440
                qatomic_set(&sibling->rb_left, tmp1);
441
                qatomic_set(&tmp2->rb_right, sibling);
442
                qatomic_set(&parent->rb_right, tmp2);
443
                if (tmp1) {
444
                    rb_set_parent_color(tmp1, sibling, RB_BLACK);
445
                }
446
                augment->rotate(sibling, tmp2);
447
                tmp1 = sibling;
448
                sibling = tmp2;
449
            }
450
            /*
451
             * Case 4 - left rotate at parent + color flips
452
             * (p and sl could be either color here.
453
             *  After rotation, p becomes black, s acquires
454
             *  p's color, and sl keeps its color)
455
             *
456
             *      (p)             (s)
457
             *      / \             / \
458
             *     N   S     -->   P   Sr
459
             *        / \         / \
460
             *      (sl) sr      N  (sl)
461
             */
462
            tmp2 = sibling->rb_left;
463
            qatomic_set(&parent->rb_right, tmp2);
464
            qatomic_set(&sibling->rb_left, parent);
465
            rb_set_parent_color(tmp1, sibling, RB_BLACK);
466
            if (tmp2) {
467
                rb_set_parent(tmp2, parent);
468
            }
469
            rb_rotate_set_parents(parent, sibling, root, RB_BLACK);
470
            augment->rotate(parent, sibling);
471
            break;
472
        } else {
473
            sibling = parent->rb_left;
474
            if (rb_is_red(sibling)) {
475
                /* Case 1 - right rotate at parent */
476
                tmp1 = sibling->rb_right;
477
                qatomic_set(&parent->rb_left, tmp1);
478
                qatomic_set(&sibling->rb_right, parent);
479
                rb_set_parent_color(tmp1, parent, RB_BLACK);
480
                rb_rotate_set_parents(parent, sibling, root, RB_RED);
481
                augment->rotate(parent, sibling);
482
                sibling = tmp1;
483
            }
484
            tmp1 = sibling->rb_left;
485
            if (!tmp1 || rb_is_black(tmp1)) {
486
                tmp2 = sibling->rb_right;
487
                if (!tmp2 || rb_is_black(tmp2)) {
488
                    /* Case 2 - sibling color flip */
489
                    rb_set_parent_color(sibling, parent, RB_RED);
490
                    if (rb_is_red(parent)) {
491
                        rb_set_black(parent);
492
                    } else {
493
                        node = parent;
494
                        parent = rb_parent(node);
495
                        if (parent) {
496
                            continue;
497
                        }
498
                    }
499
                    break;
500
                }
501
                /* Case 3 - left rotate at sibling */
502
                tmp1 = tmp2->rb_left;
503
                qatomic_set(&sibling->rb_right, tmp1);
504
                qatomic_set(&tmp2->rb_left, sibling);
505
                qatomic_set(&parent->rb_left, tmp2);
506
                if (tmp1) {
507
                    rb_set_parent_color(tmp1, sibling, RB_BLACK);
508
                }
509
                augment->rotate(sibling, tmp2);
510
                tmp1 = sibling;
511
                sibling = tmp2;
512
            }
513
            /* Case 4 - right rotate at parent + color flips */
514
            tmp2 = sibling->rb_right;
515
            qatomic_set(&parent->rb_left, tmp2);
516
            qatomic_set(&sibling->rb_right, parent);
517
            rb_set_parent_color(tmp1, sibling, RB_BLACK);
518
            if (tmp2) {
519
                rb_set_parent(tmp2, parent);
520
            }
521
            rb_rotate_set_parents(parent, sibling, root, RB_BLACK);
522
            augment->rotate(parent, sibling);
523
            break;
524
        }
525
    }
526
}
527

528
static void rb_erase_augmented(RBNode *node, RBRoot *root,
529
                               const RBAugmentCallbacks *augment)
530
{
531
    RBNode *child = node->rb_right;
532
    RBNode *tmp = node->rb_left;
533
    RBNode *parent, *rebalance;
534
    uintptr_t pc;
535

536
    if (!tmp) {
537
        /*
538
         * Case 1: node to erase has no more than 1 child (easy!)
539
         *
540
         * Note that if there is one child it must be red due to 5)
541
         * and node must be black due to 4). We adjust colors locally
542
         * so as to bypass rb_erase_color() later on.
543
         */
544
        pc = rb_pc(node);
545
        parent = pc_parent(pc);
546
        rb_change_child(node, child, parent, root);
547
        if (child) {
548
            rb_set_pc(child, pc);
549
            rebalance = NULL;
550
        } else {
551
            rebalance = pc_is_black(pc) ? parent : NULL;
552
        }
553
        tmp = parent;
554
    } else if (!child) {
555
        /* Still case 1, but this time the child is node->rb_left */
556
        pc = rb_pc(node);
557
        parent = pc_parent(pc);
558
        rb_set_pc(tmp, pc);
559
        rb_change_child(node, tmp, parent, root);
560
        rebalance = NULL;
561
        tmp = parent;
562
    } else {
563
        RBNode *successor = child, *child2;
564
        tmp = child->rb_left;
565
        if (!tmp) {
566
            /*
567
             * Case 2: node's successor is its right child
568
             *
569
             *    (n)          (s)
570
             *    / \          / \
571
             *  (x) (s)  ->  (x) (c)
572
             *        \
573
             *        (c)
574
             */
575
            parent = successor;
576
            child2 = successor->rb_right;
577

578
            augment->copy(node, successor);
579
        } else {
580
            /*
581
             * Case 3: node's successor is leftmost under
582
             * node's right child subtree
583
             *
584
             *    (n)          (s)
585
             *    / \          / \
586
             *  (x) (y)  ->  (x) (y)
587
             *      /            /
588
             *    (p)          (p)
589
             *    /            /
590
             *  (s)          (c)
591
             *    \
592
             *    (c)
593
             */
594
            do {
595
                parent = successor;
596
                successor = tmp;
597
                tmp = tmp->rb_left;
598
            } while (tmp);
599
            child2 = successor->rb_right;
600
            qatomic_set(&parent->rb_left, child2);
601
            qatomic_set(&successor->rb_right, child);
602
            rb_set_parent(child, successor);
603

604
            augment->copy(node, successor);
605
            augment->propagate(parent, successor);
606
        }
607

608
        tmp = node->rb_left;
609
        qatomic_set(&successor->rb_left, tmp);
610
        rb_set_parent(tmp, successor);
611

612
        pc = rb_pc(node);
613
        tmp = pc_parent(pc);
614
        rb_change_child(node, successor, tmp, root);
615

616
        if (child2) {
617
            rb_set_parent_color(child2, parent, RB_BLACK);
618
            rebalance = NULL;
619
        } else {
620
            rebalance = rb_is_black(successor) ? parent : NULL;
621
        }
622
        rb_set_pc(successor, pc);
623
        tmp = successor;
624
    }
625

626
    augment->propagate(tmp, NULL);
627

628
    if (rebalance) {
629
        rb_erase_color(rebalance, root, augment);
630
    }
631
}
632

633
static void rb_erase_augmented_cached(RBNode *node, RBRootLeftCached *root,
634
                                      const RBAugmentCallbacks *augment)
635
{
636
    if (root->rb_leftmost == node) {
637
        root->rb_leftmost = rb_next(node);
638
    }
639
    rb_erase_augmented(node, &root->rb_root, augment);
640
}
641

642

643
/*
644
 * Interval trees.
645
 *
646
 * Derived from lib/interval_tree.c and its dependencies,
647
 * especially include/linux/interval_tree_generic.h.
648
 */
649

650
#define rb_to_itree(N)  container_of(N, IntervalTreeNode, rb)
651

652
static bool interval_tree_compute_max(IntervalTreeNode *node, bool exit)
653
{
654
    IntervalTreeNode *child;
655
    uint64_t max = node->last;
656

657
    if (node->rb.rb_left) {
658
        child = rb_to_itree(node->rb.rb_left);
659
        if (child->subtree_last > max) {
660
            max = child->subtree_last;
661
        }
662
    }
663
    if (node->rb.rb_right) {
664
        child = rb_to_itree(node->rb.rb_right);
665
        if (child->subtree_last > max) {
666
            max = child->subtree_last;
667
        }
668
    }
669
    if (exit && node->subtree_last == max) {
670
        return true;
671
    }
672
    node->subtree_last = max;
673
    return false;
674
}
675

676
static void interval_tree_propagate(RBNode *rb, RBNode *stop)
677
{
678
    while (rb != stop) {
679
        IntervalTreeNode *node = rb_to_itree(rb);
680
        if (interval_tree_compute_max(node, true)) {
681
            break;
682
        }
683
        rb = rb_parent(&node->rb);
684
    }
685
}
686

687
static void interval_tree_copy(RBNode *rb_old, RBNode *rb_new)
688
{
689
    IntervalTreeNode *old = rb_to_itree(rb_old);
690
    IntervalTreeNode *new = rb_to_itree(rb_new);
691

692
    new->subtree_last = old->subtree_last;
693
}
694

695
static void interval_tree_rotate(RBNode *rb_old, RBNode *rb_new)
696
{
697
    IntervalTreeNode *old = rb_to_itree(rb_old);
698
    IntervalTreeNode *new = rb_to_itree(rb_new);
699

700
    new->subtree_last = old->subtree_last;
701
    interval_tree_compute_max(old, false);
702
}
703

704
static const RBAugmentCallbacks interval_tree_augment = {
705
    .propagate = interval_tree_propagate,
706
    .copy = interval_tree_copy,
707
    .rotate = interval_tree_rotate,
708
};
709

710
/* Insert / remove interval nodes from the tree */
711
void interval_tree_insert(IntervalTreeNode *node, IntervalTreeRoot *root)
712
{
713
    RBNode **link = &root->rb_root.rb_node, *rb_parent = NULL;
714
    uint64_t start = node->start, last = node->last;
715
    IntervalTreeNode *parent;
716
    bool leftmost = true;
717

718
    while (*link) {
719
        rb_parent = *link;
720
        parent = rb_to_itree(rb_parent);
721

722
        if (parent->subtree_last < last) {
723
            parent->subtree_last = last;
724
        }
725
        if (start < parent->start) {
726
            link = &parent->rb.rb_left;
727
        } else {
728
            link = &parent->rb.rb_right;
729
            leftmost = false;
730
        }
731
    }
732

733
    node->subtree_last = last;
734
    rb_link_node(&node->rb, rb_parent, link);
735
    rb_insert_augmented_cached(&node->rb, root, leftmost,
736
                               &interval_tree_augment);
737
}
738

739
void interval_tree_remove(IntervalTreeNode *node, IntervalTreeRoot *root)
740
{
741
    rb_erase_augmented_cached(&node->rb, root, &interval_tree_augment);
742
}
743

744
/*
745
 * Iterate over intervals intersecting [start;last]
746
 *
747
 * Note that a node's interval intersects [start;last] iff:
748
 *   Cond1: node->start <= last
749
 * and
750
 *   Cond2: start <= node->last
751
 */
752

753
static IntervalTreeNode *interval_tree_subtree_search(IntervalTreeNode *node,
754
                                                      uint64_t start,
755
                                                      uint64_t last)
756
{
757
    while (true) {
758
        /*
759
         * Loop invariant: start <= node->subtree_last
760
         * (Cond2 is satisfied by one of the subtree nodes)
761
         */
762
        RBNode *tmp = qatomic_read(&node->rb.rb_left);
763
        if (tmp) {
764
            IntervalTreeNode *left = rb_to_itree(tmp);
765

766
            if (start <= left->subtree_last) {
767
                /*
768
                 * Some nodes in left subtree satisfy Cond2.
769
                 * Iterate to find the leftmost such node N.
770
                 * If it also satisfies Cond1, that's the
771
                 * match we are looking for. Otherwise, there
772
                 * is no matching interval as nodes to the
773
                 * right of N can't satisfy Cond1 either.
774
                 */
775
                node = left;
776
                continue;
777
            }
778
        }
779
        if (node->start <= last) {         /* Cond1 */
780
            if (start <= node->last) {     /* Cond2 */
781
                return node; /* node is leftmost match */
782
            }
783
            tmp = qatomic_read(&node->rb.rb_right);
784
            if (tmp) {
785
                node = rb_to_itree(tmp);
786
                if (start <= node->subtree_last) {
787
                    continue;
788
                }
789
            }
790
        }
791
        return NULL; /* no match */
792
    }
793
}
794

795
IntervalTreeNode *interval_tree_iter_first(IntervalTreeRoot *root,
796
                                           uint64_t start, uint64_t last)
797
{
798
    IntervalTreeNode *node, *leftmost;
799

800
    if (!root || !root->rb_root.rb_node) {
801
        return NULL;
802
    }
803

804
    /*
805
     * Fastpath range intersection/overlap between A: [a0, a1] and
806
     * B: [b0, b1] is given by:
807
     *
808
     *         a0 <= b1 && b0 <= a1
809
     *
810
     *  ... where A holds the lock range and B holds the smallest
811
     * 'start' and largest 'last' in the tree. For the later, we
812
     * rely on the root node, which by augmented interval tree
813
     * property, holds the largest value in its last-in-subtree.
814
     * This allows mitigating some of the tree walk overhead for
815
     * for non-intersecting ranges, maintained and consulted in O(1).
816
     */
817
    node = rb_to_itree(root->rb_root.rb_node);
818
    if (node->subtree_last < start) {
819
        return NULL;
820
    }
821

822
    leftmost = rb_to_itree(root->rb_leftmost);
823
    if (leftmost->start > last) {
824
        return NULL;
825
    }
826

827
    return interval_tree_subtree_search(node, start, last);
828
}
829

830
IntervalTreeNode *interval_tree_iter_next(IntervalTreeNode *node,
831
                                          uint64_t start, uint64_t last)
832
{
833
    RBNode *rb, *prev;
834

835
    rb = qatomic_read(&node->rb.rb_right);
836
    while (true) {
837
        /*
838
         * Loop invariants:
839
         *   Cond1: node->start <= last
840
         *   rb == node->rb.rb_right
841
         *
842
         * First, search right subtree if suitable
843
         */
844
        if (rb) {
845
            IntervalTreeNode *right = rb_to_itree(rb);
846

847
            if (start <= right->subtree_last) {
848
                return interval_tree_subtree_search(right, start, last);
849
            }
850
        }
851

852
        /* Move up the tree until we come from a node's left child */
853
        do {
854
            rb = rb_parent(&node->rb);
855
            if (!rb) {
856
                return NULL;
857
            }
858
            prev = &node->rb;
859
            node = rb_to_itree(rb);
860
            rb = qatomic_read(&node->rb.rb_right);
861
        } while (prev == rb);
862

863
        /* Check if the node intersects [start;last] */
864
        if (last < node->start) {  /* !Cond1 */
865
            return NULL;
866
        }
867
        if (start <= node->last) { /* Cond2 */
868
            return node;
869
        }
870
    }
871
}
872

873
/* Occasionally useful for calling from within the debugger. */
874
#if 0
875
static void debug_interval_tree_int(IntervalTreeNode *node,
876
                                    const char *dir, int level)
877
{
878
    printf("%4d %*s %s [%" PRIu64 ",%" PRIu64 "] subtree_last:%" PRIu64 "\n",
879
           level, level + 1, dir, rb_is_red(&node->rb) ? "r" : "b",
880
           node->start, node->last, node->subtree_last);
881

882
    if (node->rb.rb_left) {
883
        debug_interval_tree_int(rb_to_itree(node->rb.rb_left), "<", level + 1);
884
    }
885
    if (node->rb.rb_right) {
886
        debug_interval_tree_int(rb_to_itree(node->rb.rb_right), ">", level + 1);
887
    }
888
}
889

890
void debug_interval_tree(IntervalTreeNode *node);
891
void debug_interval_tree(IntervalTreeNode *node)
892
{
893
    if (node) {
894
        debug_interval_tree_int(node, "*", 0);
895
    } else {
896
        printf("null\n");
897
    }
898
}
899
#endif
900