qemu

1
/*
2
 * ucontext coroutine initialization code
3
 *
4
 * Copyright (C) 2006  Anthony Liguori <anthony@codemonkey.ws>
5
 * Copyright (C) 2011  Kevin Wolf <kwolf@redhat.com>
6
 *
7
 * This library is free software; you can redistribute it and/or
8
 * modify it under the terms of the GNU Lesser General Public
9
 * License as published by the Free Software Foundation; either
10
 * version 2.0 of the License, or (at your option) any later version.
11
 *
12
 * This library is distributed in the hope that it will be useful,
13
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
14
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
15
 * Lesser General Public License for more details.
16
 *
17
 * You should have received a copy of the GNU Lesser General Public
18
 * License along with this library; if not, see <http://www.gnu.org/licenses/>.
19
 */
20

21
/* XXX Is there a nicer way to disable glibc's stack check for longjmp? */
22
#undef _FORTIFY_SOURCE
23
#define _FORTIFY_SOURCE 0
24

25
#include "qemu/osdep.h"
26
#include <ucontext.h>
27
#include "qemu/coroutine_int.h"
28
#include "qemu/coroutine-tls.h"
29

30
#ifdef CONFIG_VALGRIND_H
31
#include <valgrind/valgrind.h>
32
#endif
33

34
#ifdef QEMU_SANITIZE_ADDRESS
35
#ifdef CONFIG_ASAN_IFACE_FIBER
36
#define CONFIG_ASAN 1
37
#include <sanitizer/asan_interface.h>
38
#endif
39
#endif
40

41
#ifdef CONFIG_TSAN
42
#include <sanitizer/tsan_interface.h>
43
#endif
44

45
typedef struct {
46
    Coroutine base;
47
    void *stack;
48
    size_t stack_size;
49
#ifdef CONFIG_SAFESTACK
50
    /* Need an unsafe stack for each coroutine */
51
    void *unsafe_stack;
52
    size_t unsafe_stack_size;
53
#endif
54
    sigjmp_buf env;
55

56
#ifdef CONFIG_TSAN
57
    void *tsan_co_fiber;
58
    void *tsan_caller_fiber;
59
#endif
60

61
#ifdef CONFIG_VALGRIND_H
62
    unsigned int valgrind_stack_id;
63
#endif
64

65
} CoroutineUContext;
66

67
/**
68
 * Per-thread coroutine bookkeeping
69
 */
70
QEMU_DEFINE_STATIC_CO_TLS(Coroutine *, current);
71
QEMU_DEFINE_STATIC_CO_TLS(CoroutineUContext, leader);
72

73
/*
74
 * va_args to makecontext() must be type 'int', so passing
75
 * the pointer we need may require several int args. This
76
 * union is a quick hack to let us do that
77
 */
78
union cc_arg {
79
    void *p;
80
    int i[2];
81
};
82

83
/*
84
 * QEMU_ALWAYS_INLINE only does so if __OPTIMIZE__, so we cannot use it.
85
 * always_inline is required to avoid TSan runtime fatal errors.
86
 */
87
static inline __attribute__((always_inline))
88
void on_new_fiber(CoroutineUContext *co)
89
{
90
#ifdef CONFIG_TSAN
91
    co->tsan_co_fiber = __tsan_create_fiber(0); /* flags: sync on switch */
92
    co->tsan_caller_fiber = __tsan_get_current_fiber();
93
#endif
94
}
95

96
/* always_inline is required to avoid TSan runtime fatal errors. */
97
static inline __attribute__((always_inline))
98
void finish_switch_fiber(void *fake_stack_save)
99
{
100
#ifdef CONFIG_ASAN
101
    CoroutineUContext *leaderp = get_ptr_leader();
102
    const void *bottom_old;
103
    size_t size_old;
104

105
    __sanitizer_finish_switch_fiber(fake_stack_save, &bottom_old, &size_old);
106

107
    if (!leaderp->stack) {
108
        leaderp->stack = (void *)bottom_old;
109
        leaderp->stack_size = size_old;
110
    }
111
#endif
112
#ifdef CONFIG_TSAN
113
    if (fake_stack_save) {
114
        __tsan_release(fake_stack_save);
115
        __tsan_switch_to_fiber(fake_stack_save, 0);  /* 0=synchronize */
116
    }
117
#endif
118
}
119

120
/* always_inline is required to avoid TSan runtime fatal errors. */
121
static inline __attribute__((always_inline))
122
void start_switch_fiber_asan(void **fake_stack_save,
123
                             const void *bottom, size_t size)
124
{
125
#ifdef CONFIG_ASAN
126
    __sanitizer_start_switch_fiber(fake_stack_save, bottom, size);
127
#endif
128
}
129

130
/* always_inline is required to avoid TSan runtime fatal errors. */
131
static inline __attribute__((always_inline))
132
void start_switch_fiber_tsan(void **fake_stack_save,
133
                             CoroutineUContext *co,
134
                             bool caller)
135
{
136
#ifdef CONFIG_TSAN
137
    void *new_fiber = caller ?
138
                      co->tsan_caller_fiber :
139
                      co->tsan_co_fiber;
140
    void *curr_fiber = __tsan_get_current_fiber();
141
    __tsan_acquire(curr_fiber);
142

143
    *fake_stack_save = curr_fiber;
144
    __tsan_switch_to_fiber(new_fiber, 0);  /* 0=synchronize */
145
#endif
146
}
147

148
static void coroutine_trampoline(int i0, int i1)
149
{
150
    union cc_arg arg;
151
    CoroutineUContext *self;
152
    Coroutine *co;
153
    void *fake_stack_save = NULL;
154

155
    finish_switch_fiber(NULL);
156

157
    arg.i[0] = i0;
158
    arg.i[1] = i1;
159
    self = arg.p;
160
    co = &self->base;
161

162
    /* Initialize longjmp environment and switch back the caller */
163
    if (!sigsetjmp(self->env, 0)) {
164
        CoroutineUContext *leaderp = get_ptr_leader();
165

166
        start_switch_fiber_asan(&fake_stack_save,
167
                                leaderp->stack, leaderp->stack_size);
168
        start_switch_fiber_tsan(&fake_stack_save, self, true); /* true=caller */
169
        siglongjmp(*(sigjmp_buf *)co->entry_arg, 1);
170
    }
171

172
    finish_switch_fiber(fake_stack_save);
173

174
    while (true) {
175
        co->entry(co->entry_arg);
176
        qemu_coroutine_switch(co, co->caller, COROUTINE_TERMINATE);
177
    }
178
}
179

180
Coroutine *qemu_coroutine_new(void)
181
{
182
    CoroutineUContext *co;
183
    ucontext_t old_uc, uc;
184
    sigjmp_buf old_env;
185
    union cc_arg arg = {0};
186
    void *fake_stack_save = NULL;
187

188
    /* The ucontext functions preserve signal masks which incurs a
189
     * system call overhead.  sigsetjmp(buf, 0)/siglongjmp() does not
190
     * preserve signal masks but only works on the current stack.
191
     * Since we need a way to create and switch to a new stack, use
192
     * the ucontext functions for that but sigsetjmp()/siglongjmp() for
193
     * everything else.
194
     */
195

196
    if (getcontext(&uc) == -1) {
197
        abort();
198
    }
199

200
    co = g_malloc0(sizeof(*co));
201
    co->stack_size = COROUTINE_STACK_SIZE;
202
    co->stack = qemu_alloc_stack(&co->stack_size);
203
#ifdef CONFIG_SAFESTACK
204
    co->unsafe_stack_size = COROUTINE_STACK_SIZE;
205
    co->unsafe_stack = qemu_alloc_stack(&co->unsafe_stack_size);
206
#endif
207
    co->base.entry_arg = &old_env; /* stash away our jmp_buf */
208

209
    uc.uc_link = &old_uc;
210
    uc.uc_stack.ss_sp = co->stack;
211
    uc.uc_stack.ss_size = co->stack_size;
212
    uc.uc_stack.ss_flags = 0;
213

214
#ifdef CONFIG_VALGRIND_H
215
    co->valgrind_stack_id =
216
        VALGRIND_STACK_REGISTER(co->stack, co->stack + co->stack_size);
217
#endif
218

219
    arg.p = co;
220

221
    on_new_fiber(co);
222
    makecontext(&uc, (void (*)(void))coroutine_trampoline,
223
                2, arg.i[0], arg.i[1]);
224

225
    /* swapcontext() in, siglongjmp() back out */
226
    if (!sigsetjmp(old_env, 0)) {
227
        start_switch_fiber_asan(&fake_stack_save, co->stack, co->stack_size);
228
        start_switch_fiber_tsan(&fake_stack_save,
229
                                co, false); /* false=not caller */
230

231
#ifdef CONFIG_SAFESTACK
232
        /*
233
         * Before we swap the context, set the new unsafe stack
234
         * The unsafe stack grows just like the normal stack, so start from
235
         * the last usable location of the memory area.
236
         * NOTE: we don't have to re-set the usp afterwards because we are
237
         * coming back to this context through a siglongjmp.
238
         * The compiler already wrapped the corresponding sigsetjmp call with
239
         * code that saves the usp on the (safe) stack before the call, and
240
         * restores it right after (which is where we return with siglongjmp).
241
         */
242
        void *usp = co->unsafe_stack + co->unsafe_stack_size;
243
        __safestack_unsafe_stack_ptr = usp;
244
#endif
245

246
        swapcontext(&old_uc, &uc);
247
    }
248

249
    finish_switch_fiber(fake_stack_save);
250

251
    return &co->base;
252
}
253

254
#ifdef CONFIG_VALGRIND_H
255
/* Work around an unused variable in the valgrind.h macro... */
256
#if !defined(__clang__)
257
#pragma GCC diagnostic push
258
#pragma GCC diagnostic ignored "-Wunused-but-set-variable"
259
#endif
260
static inline void valgrind_stack_deregister(CoroutineUContext *co)
261
{
262
    VALGRIND_STACK_DEREGISTER(co->valgrind_stack_id);
263
}
264
#if !defined(__clang__)
265
#pragma GCC diagnostic pop
266
#endif
267
#endif
268

269
#if defined(CONFIG_ASAN) && defined(CONFIG_COROUTINE_POOL)
270
static void coroutine_fn terminate_asan(void *opaque)
271
{
272
    CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, opaque);
273

274
    set_current(opaque);
275
    start_switch_fiber_asan(NULL, to->stack, to->stack_size);
276
    G_STATIC_ASSERT(!IS_ENABLED(CONFIG_TSAN));
277
    siglongjmp(to->env, COROUTINE_ENTER);
278
}
279
#endif
280

281
void qemu_coroutine_delete(Coroutine *co_)
282
{
283
    CoroutineUContext *co = DO_UPCAST(CoroutineUContext, base, co_);
284

285
#if defined(CONFIG_ASAN) && defined(CONFIG_COROUTINE_POOL)
286
    co_->entry_arg = qemu_coroutine_self();
287
    co_->entry = terminate_asan;
288
    qemu_coroutine_switch(co_->entry_arg, co_, COROUTINE_ENTER);
289
#endif
290

291
#ifdef CONFIG_VALGRIND_H
292
    valgrind_stack_deregister(co);
293
#endif
294

295
    qemu_free_stack(co->stack, co->stack_size);
296
#ifdef CONFIG_SAFESTACK
297
    qemu_free_stack(co->unsafe_stack, co->unsafe_stack_size);
298
#endif
299
    g_free(co);
300
}
301

302
/* This function is marked noinline to prevent GCC from inlining it
303
 * into coroutine_trampoline(). If we allow it to do that then it
304
 * hoists the code to get the address of the TLS variable "current"
305
 * out of the while() loop. This is an invalid transformation because
306
 * the sigsetjmp() call may be called when running thread A but
307
 * return in thread B, and so we might be in a different thread
308
 * context each time round the loop.
309
 */
310
CoroutineAction __attribute__((noinline))
311
qemu_coroutine_switch(Coroutine *from_, Coroutine *to_,
312
                      CoroutineAction action)
313
{
314
    CoroutineUContext *from = DO_UPCAST(CoroutineUContext, base, from_);
315
    CoroutineUContext *to = DO_UPCAST(CoroutineUContext, base, to_);
316
    int ret;
317
    void *fake_stack_save = NULL;
318

319
    set_current(to_);
320

321
    ret = sigsetjmp(from->env, 0);
322
    if (ret == 0) {
323
        start_switch_fiber_asan(IS_ENABLED(CONFIG_COROUTINE_POOL) ||
324
                                action != COROUTINE_TERMINATE ?
325
                                    &fake_stack_save : NULL,
326
                                to->stack, to->stack_size);
327
        start_switch_fiber_tsan(&fake_stack_save,
328
                                to, false); /* false=not caller */
329
        siglongjmp(to->env, action);
330
    }
331

332
    finish_switch_fiber(fake_stack_save);
333

334
    return ret;
335
}
336

337
Coroutine *qemu_coroutine_self(void)
338
{
339
    Coroutine *self = get_current();
340
    CoroutineUContext *leaderp = get_ptr_leader();
341

342
    if (!self) {
343
        self = &leaderp->base;
344
        set_current(self);
345
    }
346
#ifdef CONFIG_TSAN
347
    if (!leaderp->tsan_co_fiber) {
348
        leaderp->tsan_co_fiber = __tsan_get_current_fiber();
349
    }
350
#endif
351
    return self;
352
}
353

354
bool qemu_in_coroutine(void)
355
{
356
    Coroutine *self = get_current();
357

358
    return self && self->caller;
359
}
360