qemu

1
/*
2
 * Atomic operations on 64-bit quantities.
3
 *
4
 * Copyright (C) 2017 Red Hat, Inc.
5
 *
6
 * Author: Paolo Bonzini <pbonzini@redhat.com>
7
 *
8
 * This work is licensed under the terms of the GNU GPL, version 2 or later.
9
 * See the COPYING file in the top-level directory.
10
 */
11

12
#include "qemu/osdep.h"
13
#include "qemu/atomic.h"
14
#include "qemu/stats64.h"
15
#include "qemu/processor.h"
16

17
#ifndef CONFIG_ATOMIC64
18
static inline void stat64_rdlock(Stat64 *s)
19
{
20
    /* Keep out incoming writers to avoid them starving us. */
21
    qatomic_add(&s->lock, 2);
22

23
    /* If there is a concurrent writer, wait for it.  */
24
    while (qatomic_read(&s->lock) & 1) {
25
        cpu_relax();
26
    }
27
}
28

29
static inline void stat64_rdunlock(Stat64 *s)
30
{
31
    qatomic_sub(&s->lock, 2);
32
}
33

34
static inline bool stat64_wrtrylock(Stat64 *s)
35
{
36
    return qatomic_cmpxchg(&s->lock, 0, 1) == 0;
37
}
38

39
static inline void stat64_wrunlock(Stat64 *s)
40
{
41
    qatomic_dec(&s->lock);
42
}
43

44
uint64_t stat64_get(const Stat64 *s)
45
{
46
    uint32_t high, low;
47

48
    stat64_rdlock((Stat64 *)s);
49

50
    /* 64-bit writes always take the lock, so we can read in
51
     * any order.
52
     */
53
    high = qatomic_read(&s->high);
54
    low = qatomic_read(&s->low);
55
    stat64_rdunlock((Stat64 *)s);
56

57
    return ((uint64_t)high << 32) | low;
58
}
59

60
void stat64_set(Stat64 *s, uint64_t val)
61
{
62
    while (!stat64_wrtrylock(s)) {
63
        cpu_relax();
64
    }
65

66
    qatomic_set(&s->high, val >> 32);
67
    qatomic_set(&s->low, val);
68
    stat64_wrunlock(s);
69
}
70

71
bool stat64_add32_carry(Stat64 *s, uint32_t low, uint32_t high)
72
{
73
    uint32_t old;
74

75
    if (!stat64_wrtrylock(s)) {
76
        cpu_relax();
77
        return false;
78
    }
79

80
    /* 64-bit reads always take the lock, so they don't care about the
81
     * order of our update.  By updating s->low first, we can check
82
     * whether we have to carry into s->high.
83
     */
84
    old = qatomic_fetch_add(&s->low, low);
85
    high += (old + low) < old;
86
    qatomic_add(&s->high, high);
87
    stat64_wrunlock(s);
88
    return true;
89
}
90

91
bool stat64_min_slow(Stat64 *s, uint64_t value)
92
{
93
    uint32_t high, low;
94
    uint64_t orig;
95

96
    if (!stat64_wrtrylock(s)) {
97
        cpu_relax();
98
        return false;
99
    }
100

101
    high = qatomic_read(&s->high);
102
    low = qatomic_read(&s->low);
103

104
    orig = ((uint64_t)high << 32) | low;
105
    if (value < orig) {
106
        /* We have to set low before high, just like stat64_min reads
107
         * high before low.  The value may become higher temporarily, but
108
         * stat64_get does not notice (it takes the lock) and the only ill
109
         * effect on stat64_min is that the slow path may be triggered
110
         * unnecessarily.
111
         */
112
        qatomic_set(&s->low, (uint32_t)value);
113
        smp_wmb();
114
        qatomic_set(&s->high, value >> 32);
115
    }
116
    stat64_wrunlock(s);
117
    return true;
118
}
119

120
bool stat64_max_slow(Stat64 *s, uint64_t value)
121
{
122
    uint32_t high, low;
123
    uint64_t orig;
124

125
    if (!stat64_wrtrylock(s)) {
126
        cpu_relax();
127
        return false;
128
    }
129

130
    high = qatomic_read(&s->high);
131
    low = qatomic_read(&s->low);
132

133
    orig = ((uint64_t)high << 32) | low;
134
    if (value > orig) {
135
        /* We have to set low before high, just like stat64_max reads
136
         * high before low.  The value may become lower temporarily, but
137
         * stat64_get does not notice (it takes the lock) and the only ill
138
         * effect on stat64_max is that the slow path may be triggered
139
         * unnecessarily.
140
         */
141
        qatomic_set(&s->low, (uint32_t)value);
142
        smp_wmb();
143
        qatomic_set(&s->high, value >> 32);
144
    }
145
    stat64_wrunlock(s);
146
    return true;
147
}
148
#endif
149