qemu

1
/*
2
 * Inter-VM Shared Memory PCI device.
3
 *
4
 * Author:
5
 *      Cam Macdonell <cam@cs.ualberta.ca>
6
 *
7
 * Based On: cirrus_vga.c
8
 *          Copyright (c) 2004 Fabrice Bellard
9
 *          Copyright (c) 2004 Makoto Suzuki (suzu)
10
 *
11
 *      and rtl8139.c
12
 *          Copyright (c) 2006 Igor Kovalenko
13
 *
14
 * This code is licensed under the GNU GPL v2.
15
 *
16
 * Contributions after 2012-01-13 are licensed under the terms of the
17
 * GNU GPL, version 2 or (at your option) any later version.
18
 */
19

20
#include "qemu/osdep.h"
21
#include "qemu/units.h"
22
#include "qapi/error.h"
23
#include "qemu/cutils.h"
24
#include "hw/pci/pci.h"
25
#include "hw/qdev-properties.h"
26
#include "hw/qdev-properties-system.h"
27
#include "hw/pci/msi.h"
28
#include "hw/pci/msix.h"
29
#include "sysemu/kvm.h"
30
#include "migration/blocker.h"
31
#include "migration/vmstate.h"
32
#include "qemu/error-report.h"
33
#include "qemu/event_notifier.h"
34
#include "qemu/module.h"
35
#include "qom/object_interfaces.h"
36
#include "chardev/char-fe.h"
37
#include "sysemu/hostmem.h"
38
#include "qapi/visitor.h"
39

40
#include "hw/misc/ivshmem.h"
41
#include "qom/object.h"
42

43
#define PCI_VENDOR_ID_IVSHMEM   PCI_VENDOR_ID_REDHAT_QUMRANET
44
#define PCI_DEVICE_ID_IVSHMEM   0x1110
45

46
#define IVSHMEM_MAX_PEERS UINT16_MAX
47
#define IVSHMEM_IOEVENTFD   0
48
#define IVSHMEM_MSI     1
49

50
#define IVSHMEM_REG_BAR_SIZE 0x100
51

52
#define IVSHMEM_DEBUG 0
53
#define IVSHMEM_DPRINTF(fmt, ...)                       \
54
    do {                                                \
55
        if (IVSHMEM_DEBUG) {                            \
56
            printf("IVSHMEM: " fmt, ## __VA_ARGS__);    \
57
        }                                               \
58
    } while (0)
59

60
#define TYPE_IVSHMEM_COMMON "ivshmem-common"
61
typedef struct IVShmemState IVShmemState;
62
DECLARE_INSTANCE_CHECKER(IVShmemState, IVSHMEM_COMMON,
63
                         TYPE_IVSHMEM_COMMON)
64

65
#define TYPE_IVSHMEM_PLAIN "ivshmem-plain"
66
DECLARE_INSTANCE_CHECKER(IVShmemState, IVSHMEM_PLAIN,
67
                         TYPE_IVSHMEM_PLAIN)
68

69
#define TYPE_IVSHMEM_DOORBELL "ivshmem-doorbell"
70
DECLARE_INSTANCE_CHECKER(IVShmemState, IVSHMEM_DOORBELL,
71
                         TYPE_IVSHMEM_DOORBELL)
72

73
#define TYPE_IVSHMEM "ivshmem"
74
DECLARE_INSTANCE_CHECKER(IVShmemState, IVSHMEM,
75
                         TYPE_IVSHMEM)
76

77
typedef struct Peer {
78
    int nb_eventfds;
79
    EventNotifier *eventfds;
80
} Peer;
81

82
typedef struct MSIVector {
83
    PCIDevice *pdev;
84
    int virq;
85
    bool unmasked;
86
} MSIVector;
87

88
struct IVShmemState {
89
    /*< private >*/
90
    PCIDevice parent_obj;
91
    /*< public >*/
92

93
    uint32_t features;
94

95
    /* exactly one of these two may be set */
96
    HostMemoryBackend *hostmem; /* with interrupts */
97
    CharBackend server_chr; /* without interrupts */
98

99
    /* registers */
100
    uint32_t intrmask;
101
    uint32_t intrstatus;
102
    int vm_id;
103

104
    /* BARs */
105
    MemoryRegion ivshmem_mmio;  /* BAR 0 (registers) */
106
    MemoryRegion *ivshmem_bar2; /* BAR 2 (shared memory) */
107
    MemoryRegion server_bar2;   /* used with server_chr */
108

109
    /* interrupt support */
110
    Peer *peers;
111
    int nb_peers;               /* space in @peers[] */
112
    uint32_t vectors;
113
    MSIVector *msi_vectors;
114
    uint64_t msg_buf;           /* buffer for receiving server messages */
115
    int msg_buffered_bytes;     /* #bytes in @msg_buf */
116

117
    /* migration stuff */
118
    OnOffAuto master;
119
    Error *migration_blocker;
120
};
121

122
/* registers for the Inter-VM shared memory device */
123
enum ivshmem_registers {
124
    INTRMASK = 0,
125
    INTRSTATUS = 4,
126
    IVPOSITION = 8,
127
    DOORBELL = 12,
128
};
129

130
static inline uint32_t ivshmem_has_feature(IVShmemState *ivs,
131
                                                    unsigned int feature) {
132
    return (ivs->features & (1 << feature));
133
}
134

135
static inline bool ivshmem_is_master(IVShmemState *s)
136
{
137
    assert(s->master != ON_OFF_AUTO_AUTO);
138
    return s->master == ON_OFF_AUTO_ON;
139
}
140

141
static void ivshmem_IntrMask_write(IVShmemState *s, uint32_t val)
142
{
143
    IVSHMEM_DPRINTF("IntrMask write(w) val = 0x%04x\n", val);
144

145
    s->intrmask = val;
146
}
147

148
static uint32_t ivshmem_IntrMask_read(IVShmemState *s)
149
{
150
    uint32_t ret = s->intrmask;
151

152
    IVSHMEM_DPRINTF("intrmask read(w) val = 0x%04x\n", ret);
153
    return ret;
154
}
155

156
static void ivshmem_IntrStatus_write(IVShmemState *s, uint32_t val)
157
{
158
    IVSHMEM_DPRINTF("IntrStatus write(w) val = 0x%04x\n", val);
159

160
    s->intrstatus = val;
161
}
162

163
static uint32_t ivshmem_IntrStatus_read(IVShmemState *s)
164
{
165
    uint32_t ret = s->intrstatus;
166

167
    /* reading ISR clears all interrupts */
168
    s->intrstatus = 0;
169
    return ret;
170
}
171

172
static void ivshmem_io_write(void *opaque, hwaddr addr,
173
                             uint64_t val, unsigned size)
174
{
175
    IVShmemState *s = opaque;
176

177
    uint16_t dest = val >> 16;
178
    uint16_t vector = val & 0xff;
179

180
    addr &= 0xfc;
181

182
    IVSHMEM_DPRINTF("writing to addr " HWADDR_FMT_plx "\n", addr);
183
    switch (addr)
184
    {
185
        case INTRMASK:
186
            ivshmem_IntrMask_write(s, val);
187
            break;
188

189
        case INTRSTATUS:
190
            ivshmem_IntrStatus_write(s, val);
191
            break;
192

193
        case DOORBELL:
194
            /* check that dest VM ID is reasonable */
195
            if (dest >= s->nb_peers) {
196
                IVSHMEM_DPRINTF("Invalid destination VM ID (%d)\n", dest);
197
                break;
198
            }
199

200
            /* check doorbell range */
201
            if (vector < s->peers[dest].nb_eventfds) {
202
                IVSHMEM_DPRINTF("Notifying VM %d on vector %d\n", dest, vector);
203
                event_notifier_set(&s->peers[dest].eventfds[vector]);
204
            } else {
205
                IVSHMEM_DPRINTF("Invalid destination vector %d on VM %d\n",
206
                                vector, dest);
207
            }
208
            break;
209
        default:
210
            IVSHMEM_DPRINTF("Unhandled write " HWADDR_FMT_plx "\n", addr);
211
    }
212
}
213

214
static uint64_t ivshmem_io_read(void *opaque, hwaddr addr,
215
                                unsigned size)
216
{
217

218
    IVShmemState *s = opaque;
219
    uint32_t ret;
220

221
    switch (addr)
222
    {
223
        case INTRMASK:
224
            ret = ivshmem_IntrMask_read(s);
225
            break;
226

227
        case INTRSTATUS:
228
            ret = ivshmem_IntrStatus_read(s);
229
            break;
230

231
        case IVPOSITION:
232
            ret = s->vm_id;
233
            break;
234

235
        default:
236
            IVSHMEM_DPRINTF("why are we reading " HWADDR_FMT_plx "\n", addr);
237
            ret = 0;
238
    }
239

240
    return ret;
241
}
242

243
static const MemoryRegionOps ivshmem_mmio_ops = {
244
    .read = ivshmem_io_read,
245
    .write = ivshmem_io_write,
246
    .endianness = DEVICE_LITTLE_ENDIAN,
247
    .impl = {
248
        .min_access_size = 4,
249
        .max_access_size = 4,
250
    },
251
};
252

253
static void ivshmem_vector_notify(void *opaque)
254
{
255
    MSIVector *entry = opaque;
256
    PCIDevice *pdev = entry->pdev;
257
    IVShmemState *s = IVSHMEM_COMMON(pdev);
258
    int vector = entry - s->msi_vectors;
259
    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
260

261
    if (!event_notifier_test_and_clear(n)) {
262
        return;
263
    }
264

265
    IVSHMEM_DPRINTF("interrupt on vector %p %d\n", pdev, vector);
266
    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
267
        if (msix_enabled(pdev)) {
268
            msix_notify(pdev, vector);
269
        }
270
    } else {
271
        ivshmem_IntrStatus_write(s, 1);
272
    }
273
}
274

275
static int ivshmem_vector_unmask(PCIDevice *dev, unsigned vector,
276
                                 MSIMessage msg)
277
{
278
    IVShmemState *s = IVSHMEM_COMMON(dev);
279
    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
280
    MSIVector *v = &s->msi_vectors[vector];
281
    int ret;
282

283
    IVSHMEM_DPRINTF("vector unmask %p %d\n", dev, vector);
284
    if (!v->pdev) {
285
        error_report("ivshmem: vector %d route does not exist", vector);
286
        return -EINVAL;
287
    }
288
    assert(!v->unmasked);
289

290
    ret = kvm_irqchip_update_msi_route(kvm_state, v->virq, msg, dev);
291
    if (ret < 0) {
292
        return ret;
293
    }
294
    kvm_irqchip_commit_routes(kvm_state);
295

296
    ret = kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL, v->virq);
297
    if (ret < 0) {
298
        return ret;
299
    }
300
    v->unmasked = true;
301

302
    return 0;
303
}
304

305
static void ivshmem_vector_mask(PCIDevice *dev, unsigned vector)
306
{
307
    IVShmemState *s = IVSHMEM_COMMON(dev);
308
    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
309
    MSIVector *v = &s->msi_vectors[vector];
310
    int ret;
311

312
    IVSHMEM_DPRINTF("vector mask %p %d\n", dev, vector);
313
    if (!v->pdev) {
314
        error_report("ivshmem: vector %d route does not exist", vector);
315
        return;
316
    }
317
    assert(v->unmasked);
318

319
    ret = kvm_irqchip_remove_irqfd_notifier_gsi(kvm_state, n, v->virq);
320
    if (ret < 0) {
321
        error_report("remove_irqfd_notifier_gsi failed");
322
        return;
323
    }
324
    v->unmasked = false;
325
}
326

327
static void ivshmem_vector_poll(PCIDevice *dev,
328
                                unsigned int vector_start,
329
                                unsigned int vector_end)
330
{
331
    IVShmemState *s = IVSHMEM_COMMON(dev);
332
    unsigned int vector;
333

334
    IVSHMEM_DPRINTF("vector poll %p %d-%d\n", dev, vector_start, vector_end);
335

336
    vector_end = MIN(vector_end, s->vectors);
337

338
    for (vector = vector_start; vector < vector_end; vector++) {
339
        EventNotifier *notifier = &s->peers[s->vm_id].eventfds[vector];
340

341
        if (!msix_is_masked(dev, vector)) {
342
            continue;
343
        }
344

345
        if (event_notifier_test_and_clear(notifier)) {
346
            msix_set_pending(dev, vector);
347
        }
348
    }
349
}
350

351
static void watch_vector_notifier(IVShmemState *s, EventNotifier *n,
352
                                 int vector)
353
{
354
    int eventfd = event_notifier_get_fd(n);
355

356
    assert(!s->msi_vectors[vector].pdev);
357
    s->msi_vectors[vector].pdev = PCI_DEVICE(s);
358

359
    qemu_set_fd_handler(eventfd, ivshmem_vector_notify,
360
                        NULL, &s->msi_vectors[vector]);
361
}
362

363
static void ivshmem_add_eventfd(IVShmemState *s, int posn, int i)
364
{
365
    memory_region_add_eventfd(&s->ivshmem_mmio,
366
                              DOORBELL,
367
                              4,
368
                              true,
369
                              (posn << 16) | i,
370
                              &s->peers[posn].eventfds[i]);
371
}
372

373
static void ivshmem_del_eventfd(IVShmemState *s, int posn, int i)
374
{
375
    memory_region_del_eventfd(&s->ivshmem_mmio,
376
                              DOORBELL,
377
                              4,
378
                              true,
379
                              (posn << 16) | i,
380
                              &s->peers[posn].eventfds[i]);
381
}
382

383
static void close_peer_eventfds(IVShmemState *s, int posn)
384
{
385
    int i, n;
386

387
    assert(posn >= 0 && posn < s->nb_peers);
388
    n = s->peers[posn].nb_eventfds;
389

390
    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
391
        memory_region_transaction_begin();
392
        for (i = 0; i < n; i++) {
393
            ivshmem_del_eventfd(s, posn, i);
394
        }
395
        memory_region_transaction_commit();
396
    }
397

398
    for (i = 0; i < n; i++) {
399
        event_notifier_cleanup(&s->peers[posn].eventfds[i]);
400
    }
401

402
    g_free(s->peers[posn].eventfds);
403
    s->peers[posn].nb_eventfds = 0;
404
}
405

406
static void resize_peers(IVShmemState *s, int nb_peers)
407
{
408
    int old_nb_peers = s->nb_peers;
409
    int i;
410

411
    assert(nb_peers > old_nb_peers);
412
    IVSHMEM_DPRINTF("bumping storage to %d peers\n", nb_peers);
413

414
    s->peers = g_renew(Peer, s->peers, nb_peers);
415
    s->nb_peers = nb_peers;
416

417
    for (i = old_nb_peers; i < nb_peers; i++) {
418
        s->peers[i].eventfds = g_new0(EventNotifier, s->vectors);
419
        s->peers[i].nb_eventfds = 0;
420
    }
421
}
422

423
static void ivshmem_add_kvm_msi_virq(IVShmemState *s, int vector,
424
                                     Error **errp)
425
{
426
    PCIDevice *pdev = PCI_DEVICE(s);
427
    KVMRouteChange c;
428
    int ret;
429

430
    IVSHMEM_DPRINTF("ivshmem_add_kvm_msi_virq vector:%d\n", vector);
431
    assert(!s->msi_vectors[vector].pdev);
432

433
    c = kvm_irqchip_begin_route_changes(kvm_state);
434
    ret = kvm_irqchip_add_msi_route(&c, vector, pdev);
435
    if (ret < 0) {
436
        error_setg(errp, "kvm_irqchip_add_msi_route failed");
437
        return;
438
    }
439
    kvm_irqchip_commit_route_changes(&c);
440

441
    s->msi_vectors[vector].virq = ret;
442
    s->msi_vectors[vector].pdev = pdev;
443
}
444

445
static void setup_interrupt(IVShmemState *s, int vector, Error **errp)
446
{
447
    EventNotifier *n = &s->peers[s->vm_id].eventfds[vector];
448
    bool with_irqfd = kvm_msi_via_irqfd_enabled() &&
449
        ivshmem_has_feature(s, IVSHMEM_MSI);
450
    PCIDevice *pdev = PCI_DEVICE(s);
451
    Error *err = NULL;
452

453
    IVSHMEM_DPRINTF("setting up interrupt for vector: %d\n", vector);
454

455
    if (!with_irqfd) {
456
        IVSHMEM_DPRINTF("with eventfd\n");
457
        watch_vector_notifier(s, n, vector);
458
    } else if (msix_enabled(pdev)) {
459
        IVSHMEM_DPRINTF("with irqfd\n");
460
        ivshmem_add_kvm_msi_virq(s, vector, &err);
461
        if (err) {
462
            error_propagate(errp, err);
463
            return;
464
        }
465

466
        if (!msix_is_masked(pdev, vector)) {
467
            kvm_irqchip_add_irqfd_notifier_gsi(kvm_state, n, NULL,
468
                                               s->msi_vectors[vector].virq);
469
            /* TODO handle error */
470
        }
471
    } else {
472
        /* it will be delayed until msix is enabled, in write_config */
473
        IVSHMEM_DPRINTF("with irqfd, delayed until msix enabled\n");
474
    }
475
}
476

477
static void process_msg_shmem(IVShmemState *s, int fd, Error **errp)
478
{
479
    struct stat buf;
480
    size_t size;
481

482
    if (s->ivshmem_bar2) {
483
        error_setg(errp, "server sent unexpected shared memory message");
484
        close(fd);
485
        return;
486
    }
487

488
    if (fstat(fd, &buf) < 0) {
489
        error_setg_errno(errp, errno,
490
            "can't determine size of shared memory sent by server");
491
        close(fd);
492
        return;
493
    }
494

495
    size = buf.st_size;
496

497
    /* mmap the region and map into the BAR2 */
498
    if (!memory_region_init_ram_from_fd(&s->server_bar2, OBJECT(s),
499
                                        "ivshmem.bar2", size, RAM_SHARED,
500
                                        fd, 0, errp)) {
501
        return;
502
    }
503

504
    s->ivshmem_bar2 = &s->server_bar2;
505
}
506

507
static void process_msg_disconnect(IVShmemState *s, uint16_t posn,
508
                                   Error **errp)
509
{
510
    IVSHMEM_DPRINTF("posn %d has gone away\n", posn);
511
    if (posn >= s->nb_peers || posn == s->vm_id) {
512
        error_setg(errp, "invalid peer %d", posn);
513
        return;
514
    }
515
    close_peer_eventfds(s, posn);
516
}
517

518
static void process_msg_connect(IVShmemState *s, uint16_t posn, int fd,
519
                                Error **errp)
520
{
521
    Peer *peer = &s->peers[posn];
522
    int vector;
523

524
    /*
525
     * The N-th connect message for this peer comes with the file
526
     * descriptor for vector N-1.  Count messages to find the vector.
527
     */
528
    if (peer->nb_eventfds >= s->vectors) {
529
        error_setg(errp, "Too many eventfd received, device has %d vectors",
530
                   s->vectors);
531
        close(fd);
532
        return;
533
    }
534
    vector = peer->nb_eventfds++;
535

536
    IVSHMEM_DPRINTF("eventfds[%d][%d] = %d\n", posn, vector, fd);
537
    event_notifier_init_fd(&peer->eventfds[vector], fd);
538
    g_unix_set_fd_nonblocking(fd, true, NULL); /* msix/irqfd poll non block */
539

540
    if (posn == s->vm_id) {
541
        setup_interrupt(s, vector, errp);
542
        /* TODO do we need to handle the error? */
543
    }
544

545
    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD)) {
546
        ivshmem_add_eventfd(s, posn, vector);
547
    }
548
}
549

550
static void process_msg(IVShmemState *s, int64_t msg, int fd, Error **errp)
551
{
552
    IVSHMEM_DPRINTF("posn is %" PRId64 ", fd is %d\n", msg, fd);
553

554
    if (msg < -1 || msg > IVSHMEM_MAX_PEERS) {
555
        error_setg(errp, "server sent invalid message %" PRId64, msg);
556
        close(fd);
557
        return;
558
    }
559

560
    if (msg == -1) {
561
        process_msg_shmem(s, fd, errp);
562
        return;
563
    }
564

565
    if (msg >= s->nb_peers) {
566
        resize_peers(s, msg + 1);
567
    }
568

569
    if (fd >= 0) {
570
        process_msg_connect(s, msg, fd, errp);
571
    } else {
572
        process_msg_disconnect(s, msg, errp);
573
    }
574
}
575

576
static int ivshmem_can_receive(void *opaque)
577
{
578
    IVShmemState *s = opaque;
579

580
    assert(s->msg_buffered_bytes < sizeof(s->msg_buf));
581
    return sizeof(s->msg_buf) - s->msg_buffered_bytes;
582
}
583

584
static void ivshmem_read(void *opaque, const uint8_t *buf, int size)
585
{
586
    IVShmemState *s = opaque;
587
    Error *err = NULL;
588
    int fd;
589
    int64_t msg;
590

591
    assert(size >= 0 && s->msg_buffered_bytes + size <= sizeof(s->msg_buf));
592
    memcpy((unsigned char *)&s->msg_buf + s->msg_buffered_bytes, buf, size);
593
    s->msg_buffered_bytes += size;
594
    if (s->msg_buffered_bytes < sizeof(s->msg_buf)) {
595
        return;
596
    }
597
    msg = le64_to_cpu(s->msg_buf);
598
    s->msg_buffered_bytes = 0;
599

600
    fd = qemu_chr_fe_get_msgfd(&s->server_chr);
601

602
    process_msg(s, msg, fd, &err);
603
    if (err) {
604
        error_report_err(err);
605
    }
606
}
607

608
static int64_t ivshmem_recv_msg(IVShmemState *s, int *pfd, Error **errp)
609
{
610
    int64_t msg;
611
    int n, ret;
612

613
    n = 0;
614
    do {
615
        ret = qemu_chr_fe_read_all(&s->server_chr, (uint8_t *)&msg + n,
616
                                   sizeof(msg) - n);
617
        if (ret < 0) {
618
            if (ret == -EINTR) {
619
                continue;
620
            }
621
            error_setg_errno(errp, -ret, "read from server failed");
622
            return INT64_MIN;
623
        }
624
        n += ret;
625
    } while (n < sizeof(msg));
626

627
    *pfd = qemu_chr_fe_get_msgfd(&s->server_chr);
628
    return le64_to_cpu(msg);
629
}
630

631
static void ivshmem_recv_setup(IVShmemState *s, Error **errp)
632
{
633
    Error *err = NULL;
634
    int64_t msg;
635
    int fd;
636

637
    msg = ivshmem_recv_msg(s, &fd, &err);
638
    if (err) {
639
        error_propagate(errp, err);
640
        return;
641
    }
642
    if (msg != IVSHMEM_PROTOCOL_VERSION) {
643
        error_setg(errp, "server sent version %" PRId64 ", expecting %d",
644
                   msg, IVSHMEM_PROTOCOL_VERSION);
645
        return;
646
    }
647
    if (fd != -1) {
648
        error_setg(errp, "server sent invalid version message");
649
        return;
650
    }
651

652
    /*
653
     * ivshmem-server sends the remaining initial messages in a fixed
654
     * order, but the device has always accepted them in any order.
655
     * Stay as compatible as practical, just in case people use
656
     * servers that behave differently.
657
     */
658

659
    /*
660
     * ivshmem_device_spec.txt has always required the ID message
661
     * right here, and ivshmem-server has always complied.  However,
662
     * older versions of the device accepted it out of order, but
663
     * broke when an interrupt setup message arrived before it.
664
     */
665
    msg = ivshmem_recv_msg(s, &fd, &err);
666
    if (err) {
667
        error_propagate(errp, err);
668
        return;
669
    }
670
    if (fd != -1 || msg < 0 || msg > IVSHMEM_MAX_PEERS) {
671
        error_setg(errp, "server sent invalid ID message");
672
        return;
673
    }
674
    s->vm_id = msg;
675

676
    /*
677
     * Receive more messages until we got shared memory.
678
     */
679
    do {
680
        msg = ivshmem_recv_msg(s, &fd, &err);
681
        if (err) {
682
            error_propagate(errp, err);
683
            return;
684
        }
685
        process_msg(s, msg, fd, &err);
686
        if (err) {
687
            error_propagate(errp, err);
688
            return;
689
        }
690
    } while (msg != -1);
691

692
    /*
693
     * This function must either map the shared memory or fail.  The
694
     * loop above ensures that: it terminates normally only after it
695
     * successfully processed the server's shared memory message.
696
     * Assert that actually mapped the shared memory:
697
     */
698
    assert(s->ivshmem_bar2);
699
}
700

701
/* Select the MSI-X vectors used by device.
702
 * ivshmem maps events to vectors statically, so
703
 * we just enable all vectors on init and after reset. */
704
static void ivshmem_msix_vector_use(IVShmemState *s)
705
{
706
    PCIDevice *d = PCI_DEVICE(s);
707
    int i;
708

709
    for (i = 0; i < s->vectors; i++) {
710
        msix_vector_use(d, i);
711
    }
712
}
713

714
static void ivshmem_disable_irqfd(IVShmemState *s);
715

716
static void ivshmem_reset(DeviceState *d)
717
{
718
    IVShmemState *s = IVSHMEM_COMMON(d);
719

720
    ivshmem_disable_irqfd(s);
721

722
    s->intrstatus = 0;
723
    s->intrmask = 0;
724
    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
725
        ivshmem_msix_vector_use(s);
726
    }
727
}
728

729
static int ivshmem_setup_interrupts(IVShmemState *s, Error **errp)
730
{
731
    /* allocate QEMU callback data for receiving interrupts */
732
    s->msi_vectors = g_new0(MSIVector, s->vectors);
733

734
    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
735
        if (msix_init_exclusive_bar(PCI_DEVICE(s), s->vectors, 1, errp)) {
736
            return -1;
737
        }
738

739
        IVSHMEM_DPRINTF("msix initialized (%d vectors)\n", s->vectors);
740
        ivshmem_msix_vector_use(s);
741
    }
742

743
    return 0;
744
}
745

746
static void ivshmem_remove_kvm_msi_virq(IVShmemState *s, int vector)
747
{
748
    IVSHMEM_DPRINTF("ivshmem_remove_kvm_msi_virq vector:%d\n", vector);
749

750
    if (s->msi_vectors[vector].pdev == NULL) {
751
        return;
752
    }
753

754
    /* it was cleaned when masked in the frontend. */
755
    kvm_irqchip_release_virq(kvm_state, s->msi_vectors[vector].virq);
756

757
    s->msi_vectors[vector].pdev = NULL;
758
}
759

760
static void ivshmem_enable_irqfd(IVShmemState *s)
761
{
762
    PCIDevice *pdev = PCI_DEVICE(s);
763
    int i;
764

765
    for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
766
        Error *err = NULL;
767

768
        ivshmem_add_kvm_msi_virq(s, i, &err);
769
        if (err) {
770
            error_report_err(err);
771
            goto undo;
772
        }
773
    }
774

775
    if (msix_set_vector_notifiers(pdev,
776
                                  ivshmem_vector_unmask,
777
                                  ivshmem_vector_mask,
778
                                  ivshmem_vector_poll)) {
779
        error_report("ivshmem: msix_set_vector_notifiers failed");
780
        goto undo;
781
    }
782
    return;
783

784
undo:
785
    while (--i >= 0) {
786
        ivshmem_remove_kvm_msi_virq(s, i);
787
    }
788
}
789

790
static void ivshmem_disable_irqfd(IVShmemState *s)
791
{
792
    PCIDevice *pdev = PCI_DEVICE(s);
793
    int i;
794

795
    if (!pdev->msix_vector_use_notifier) {
796
        return;
797
    }
798

799
    msix_unset_vector_notifiers(pdev);
800

801
    for (i = 0; i < s->peers[s->vm_id].nb_eventfds; i++) {
802
        /*
803
         * MSI-X is already disabled here so msix_unset_vector_notifiers()
804
         * didn't call our release notifier.  Do it now to keep our masks and
805
         * unmasks balanced.
806
         */
807
        if (s->msi_vectors[i].unmasked) {
808
            ivshmem_vector_mask(pdev, i);
809
        }
810
        ivshmem_remove_kvm_msi_virq(s, i);
811
    }
812

813
}
814

815
static void ivshmem_write_config(PCIDevice *pdev, uint32_t address,
816
                                 uint32_t val, int len)
817
{
818
    IVShmemState *s = IVSHMEM_COMMON(pdev);
819
    int is_enabled, was_enabled = msix_enabled(pdev);
820

821
    pci_default_write_config(pdev, address, val, len);
822
    is_enabled = msix_enabled(pdev);
823

824
    if (kvm_msi_via_irqfd_enabled()) {
825
        if (!was_enabled && is_enabled) {
826
            ivshmem_enable_irqfd(s);
827
        } else if (was_enabled && !is_enabled) {
828
            ivshmem_disable_irqfd(s);
829
        }
830
    }
831
}
832

833
static void ivshmem_common_realize(PCIDevice *dev, Error **errp)
834
{
835
    ERRP_GUARD();
836
    IVShmemState *s = IVSHMEM_COMMON(dev);
837
    Error *err = NULL;
838
    uint8_t *pci_conf;
839

840
    /* IRQFD requires MSI */
841
    if (ivshmem_has_feature(s, IVSHMEM_IOEVENTFD) &&
842
        !ivshmem_has_feature(s, IVSHMEM_MSI)) {
843
        error_setg(errp, "ioeventfd/irqfd requires MSI");
844
        return;
845
    }
846

847
    pci_conf = dev->config;
848
    pci_conf[PCI_COMMAND] = PCI_COMMAND_IO | PCI_COMMAND_MEMORY;
849

850
    memory_region_init_io(&s->ivshmem_mmio, OBJECT(s), &ivshmem_mmio_ops, s,
851
                          "ivshmem-mmio", IVSHMEM_REG_BAR_SIZE);
852

853
    /* region for registers*/
854
    pci_register_bar(dev, 0, PCI_BASE_ADDRESS_SPACE_MEMORY,
855
                     &s->ivshmem_mmio);
856

857
    if (s->hostmem != NULL) {
858
        IVSHMEM_DPRINTF("using hostmem\n");
859

860
        s->ivshmem_bar2 = host_memory_backend_get_memory(s->hostmem);
861
        host_memory_backend_set_mapped(s->hostmem, true);
862
    } else {
863
        Chardev *chr = qemu_chr_fe_get_driver(&s->server_chr);
864
        assert(chr);
865

866
        IVSHMEM_DPRINTF("using shared memory server (socket = %s)\n",
867
                        chr->filename);
868

869
        /* we allocate enough space for 16 peers and grow as needed */
870
        resize_peers(s, 16);
871

872
        /*
873
         * Receive setup messages from server synchronously.
874
         * Older versions did it asynchronously, but that creates a
875
         * number of entertaining race conditions.
876
         */
877
        ivshmem_recv_setup(s, &err);
878
        if (err) {
879
            error_propagate(errp, err);
880
            return;
881
        }
882

883
        if (s->master == ON_OFF_AUTO_ON && s->vm_id != 0) {
884
            error_setg(errp,
885
                       "master must connect to the server before any peers");
886
            return;
887
        }
888

889
        qemu_chr_fe_set_handlers(&s->server_chr, ivshmem_can_receive,
890
                                 ivshmem_read, NULL, NULL, s, NULL, true);
891

892
        if (ivshmem_setup_interrupts(s, errp) < 0) {
893
            error_prepend(errp, "Failed to initialize interrupts: ");
894
            return;
895
        }
896
    }
897

898
    if (s->master == ON_OFF_AUTO_AUTO) {
899
        s->master = s->vm_id == 0 ? ON_OFF_AUTO_ON : ON_OFF_AUTO_OFF;
900
    }
901

902
    if (!ivshmem_is_master(s)) {
903
        error_setg(&s->migration_blocker,
904
                   "Migration is disabled when using feature 'peer mode' in device 'ivshmem'");
905
        if (migrate_add_blocker(&s->migration_blocker, errp) < 0) {
906
            return;
907
        }
908
    }
909

910
    vmstate_register_ram(s->ivshmem_bar2, DEVICE(s));
911
    pci_register_bar(PCI_DEVICE(s), 2,
912
                     PCI_BASE_ADDRESS_SPACE_MEMORY |
913
                     PCI_BASE_ADDRESS_MEM_PREFETCH |
914
                     PCI_BASE_ADDRESS_MEM_TYPE_64,
915
                     s->ivshmem_bar2);
916
}
917

918
static void ivshmem_exit(PCIDevice *dev)
919
{
920
    IVShmemState *s = IVSHMEM_COMMON(dev);
921
    int i;
922

923
    migrate_del_blocker(&s->migration_blocker);
924

925
    if (memory_region_is_mapped(s->ivshmem_bar2)) {
926
        if (!s->hostmem) {
927
            void *addr = memory_region_get_ram_ptr(s->ivshmem_bar2);
928
            int fd;
929

930
            if (munmap(addr, memory_region_size(s->ivshmem_bar2) == -1)) {
931
                error_report("Failed to munmap shared memory %s",
932
                             strerror(errno));
933
            }
934

935
            fd = memory_region_get_fd(s->ivshmem_bar2);
936
            close(fd);
937
        }
938

939
        vmstate_unregister_ram(s->ivshmem_bar2, DEVICE(dev));
940
    }
941

942
    if (s->hostmem) {
943
        host_memory_backend_set_mapped(s->hostmem, false);
944
    }
945

946
    if (s->peers) {
947
        for (i = 0; i < s->nb_peers; i++) {
948
            close_peer_eventfds(s, i);
949
        }
950
        g_free(s->peers);
951
    }
952

953
    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
954
        msix_uninit_exclusive_bar(dev);
955
    }
956

957
    g_free(s->msi_vectors);
958
}
959

960
static int ivshmem_pre_load(void *opaque)
961
{
962
    IVShmemState *s = opaque;
963

964
    if (!ivshmem_is_master(s)) {
965
        error_report("'peer' devices are not migratable");
966
        return -EINVAL;
967
    }
968

969
    return 0;
970
}
971

972
static int ivshmem_post_load(void *opaque, int version_id)
973
{
974
    IVShmemState *s = opaque;
975

976
    if (ivshmem_has_feature(s, IVSHMEM_MSI)) {
977
        ivshmem_msix_vector_use(s);
978
    }
979
    return 0;
980
}
981

982
static void ivshmem_common_class_init(ObjectClass *klass, void *data)
983
{
984
    DeviceClass *dc = DEVICE_CLASS(klass);
985
    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
986

987
    k->realize = ivshmem_common_realize;
988
    k->exit = ivshmem_exit;
989
    k->config_write = ivshmem_write_config;
990
    k->vendor_id = PCI_VENDOR_ID_IVSHMEM;
991
    k->device_id = PCI_DEVICE_ID_IVSHMEM;
992
    k->class_id = PCI_CLASS_MEMORY_RAM;
993
    k->revision = 1;
994
    dc->reset = ivshmem_reset;
995
    set_bit(DEVICE_CATEGORY_MISC, dc->categories);
996
    dc->desc = "Inter-VM shared memory";
997
}
998

999
static const TypeInfo ivshmem_common_info = {
1000
    .name          = TYPE_IVSHMEM_COMMON,
1001
    .parent        = TYPE_PCI_DEVICE,
1002
    .instance_size = sizeof(IVShmemState),
1003
    .abstract      = true,
1004
    .class_init    = ivshmem_common_class_init,
1005
    .interfaces = (InterfaceInfo[]) {
1006
        { INTERFACE_CONVENTIONAL_PCI_DEVICE },
1007
        { },
1008
    },
1009
};
1010

1011
static const VMStateDescription ivshmem_plain_vmsd = {
1012
    .name = TYPE_IVSHMEM_PLAIN,
1013
    .version_id = 0,
1014
    .minimum_version_id = 0,
1015
    .pre_load = ivshmem_pre_load,
1016
    .post_load = ivshmem_post_load,
1017
    .fields = (const VMStateField[]) {
1018
        VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
1019
        VMSTATE_UINT32(intrstatus, IVShmemState),
1020
        VMSTATE_UINT32(intrmask, IVShmemState),
1021
        VMSTATE_END_OF_LIST()
1022
    },
1023
};
1024

1025
static Property ivshmem_plain_properties[] = {
1026
    DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF),
1027
    DEFINE_PROP_LINK("memdev", IVShmemState, hostmem, TYPE_MEMORY_BACKEND,
1028
                     HostMemoryBackend *),
1029
    DEFINE_PROP_END_OF_LIST(),
1030
};
1031

1032
static void ivshmem_plain_realize(PCIDevice *dev, Error **errp)
1033
{
1034
    IVShmemState *s = IVSHMEM_COMMON(dev);
1035

1036
    if (!s->hostmem) {
1037
        error_setg(errp, "You must specify a 'memdev'");
1038
        return;
1039
    } else if (host_memory_backend_is_mapped(s->hostmem)) {
1040
        error_setg(errp, "can't use already busy memdev: %s",
1041
                   object_get_canonical_path_component(OBJECT(s->hostmem)));
1042
        return;
1043
    }
1044

1045
    ivshmem_common_realize(dev, errp);
1046
}
1047

1048
static void ivshmem_plain_class_init(ObjectClass *klass, void *data)
1049
{
1050
    DeviceClass *dc = DEVICE_CLASS(klass);
1051
    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1052

1053
    k->realize = ivshmem_plain_realize;
1054
    device_class_set_props(dc, ivshmem_plain_properties);
1055
    dc->vmsd = &ivshmem_plain_vmsd;
1056
}
1057

1058
static const TypeInfo ivshmem_plain_info = {
1059
    .name          = TYPE_IVSHMEM_PLAIN,
1060
    .parent        = TYPE_IVSHMEM_COMMON,
1061
    .instance_size = sizeof(IVShmemState),
1062
    .class_init    = ivshmem_plain_class_init,
1063
};
1064

1065
static const VMStateDescription ivshmem_doorbell_vmsd = {
1066
    .name = TYPE_IVSHMEM_DOORBELL,
1067
    .version_id = 0,
1068
    .minimum_version_id = 0,
1069
    .pre_load = ivshmem_pre_load,
1070
    .post_load = ivshmem_post_load,
1071
    .fields = (const VMStateField[]) {
1072
        VMSTATE_PCI_DEVICE(parent_obj, IVShmemState),
1073
        VMSTATE_MSIX(parent_obj, IVShmemState),
1074
        VMSTATE_UINT32(intrstatus, IVShmemState),
1075
        VMSTATE_UINT32(intrmask, IVShmemState),
1076
        VMSTATE_END_OF_LIST()
1077
    },
1078
};
1079

1080
static Property ivshmem_doorbell_properties[] = {
1081
    DEFINE_PROP_CHR("chardev", IVShmemState, server_chr),
1082
    DEFINE_PROP_UINT32("vectors", IVShmemState, vectors, 1),
1083
    DEFINE_PROP_BIT("ioeventfd", IVShmemState, features, IVSHMEM_IOEVENTFD,
1084
                    true),
1085
    DEFINE_PROP_ON_OFF_AUTO("master", IVShmemState, master, ON_OFF_AUTO_OFF),
1086
    DEFINE_PROP_END_OF_LIST(),
1087
};
1088

1089
static void ivshmem_doorbell_init(Object *obj)
1090
{
1091
    IVShmemState *s = IVSHMEM_DOORBELL(obj);
1092

1093
    s->features |= (1 << IVSHMEM_MSI);
1094
}
1095

1096
static void ivshmem_doorbell_realize(PCIDevice *dev, Error **errp)
1097
{
1098
    IVShmemState *s = IVSHMEM_COMMON(dev);
1099

1100
    if (!qemu_chr_fe_backend_connected(&s->server_chr)) {
1101
        error_setg(errp, "You must specify a 'chardev'");
1102
        return;
1103
    }
1104

1105
    ivshmem_common_realize(dev, errp);
1106
}
1107

1108
static void ivshmem_doorbell_class_init(ObjectClass *klass, void *data)
1109
{
1110
    DeviceClass *dc = DEVICE_CLASS(klass);
1111
    PCIDeviceClass *k = PCI_DEVICE_CLASS(klass);
1112

1113
    k->realize = ivshmem_doorbell_realize;
1114
    device_class_set_props(dc, ivshmem_doorbell_properties);
1115
    dc->vmsd = &ivshmem_doorbell_vmsd;
1116
}
1117

1118
static const TypeInfo ivshmem_doorbell_info = {
1119
    .name          = TYPE_IVSHMEM_DOORBELL,
1120
    .parent        = TYPE_IVSHMEM_COMMON,
1121
    .instance_size = sizeof(IVShmemState),
1122
    .instance_init = ivshmem_doorbell_init,
1123
    .class_init    = ivshmem_doorbell_class_init,
1124
};
1125

1126
static void ivshmem_register_types(void)
1127
{
1128
    type_register_static(&ivshmem_common_info);
1129
    type_register_static(&ivshmem_plain_info);
1130
    type_register_static(&ivshmem_doorbell_info);
1131
}
1132

1133
type_init(ivshmem_register_types)
1134