kvm-guest-drivers-windows

1
/*
2
 * Implementation of virtio_system_ops VirtioLib callbacks
3
 *
4
 * Copyright (c) 2016-2017 Red Hat, Inc.
5
 *
6
 * Author(s):
7
 *  Yuri Benditovich <ybendito@redhat.com>
8
 *
9
 * Redistribution and use in source and binary forms, with or without
10
 * modification, are permitted provided that the following conditions
11
 * are met :
12
 * 1. Redistributions of source code must retain the above copyright
13
 *    notice, this list of conditions and the following disclaimer.
14
 * 2. Redistributions in binary form must reproduce the above copyright
15
 *    notice, this list of conditions and the following disclaimer in the
16
 *    documentation and / or other materials provided with the distribution.
17
 * 3. Neither the names of the copyright holders nor the names of their contributors
18
 *    may be used to endorse or promote products derived from this software
19
 *    without specific prior written permission.
20
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
21
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23
 * ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE
24
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30
 * SUCH DAMAGE.
31
 */
32
#include "osdep.h"
33
#include "virtio_pci.h"
34
#include "VirtIOWdf.h"
35
#include "private.h"
36

37
static EVT_WDF_OBJECT_CONTEXT_DESTROY OnDmaTransactionDestroy;
38
static EVT_WDF_PROGRAM_DMA            OnDmaTransactionProgramDma;
39

40
static void *AllocateCommonBuffer(PVIRTIO_WDF_DRIVER pWdfDriver, size_t size, ULONG groupTag)
41
{
42
    NTSTATUS status;
43
    WDFCOMMONBUFFER commonBuffer;
44
    PVIRTIO_WDF_MEMORY_BLOCK_CONTEXT context;
45
    WDF_OBJECT_ATTRIBUTES attr;
46
    WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attr, VIRTIO_WDF_MEMORY_BLOCK_CONTEXT);
47

48
    if (KeGetCurrentIrql() > PASSIVE_LEVEL) {
49
        DPrintf(0, "%s FAILED(irql)\n", __FUNCTION__);
50
        return NULL;
51
    }
52
    status = WdfCommonBufferCreate(pWdfDriver->DmaEnabler, size, &attr, &commonBuffer);
53
    if (!NT_SUCCESS(status)) {
54
        return NULL;
55
    }
56
    WdfSpinLockAcquire(pWdfDriver->DmaSpinlock);
57
    status = WdfCollectionAdd(pWdfDriver->MemoryBlockCollection, commonBuffer);
58
    if (!NT_SUCCESS(status)) {
59
        WdfObjectDelete(commonBuffer);
60
        WdfSpinLockRelease(pWdfDriver->DmaSpinlock);
61
        return NULL;
62
    }
63
    context = GetMemoryBlockContext(commonBuffer);
64
    context->WdfBuffer = commonBuffer;
65
    context->Length = size;
66
    context->PhysicalAddress = WdfCommonBufferGetAlignedLogicalAddress(commonBuffer);
67
    context->pVirtualAddress = WdfCommonBufferGetAlignedVirtualAddress(commonBuffer);
68
    context->groupTag = groupTag;
69
    context->bToBeDeleted = FALSE;
70
    WdfSpinLockRelease(pWdfDriver->DmaSpinlock);
71
    RtlZeroMemory(context->pVirtualAddress, size);
72

73
    DPrintf(1, "%s done %p@%I64x(tag %08X), size 0x%x\n", __FUNCTION__,
74
        context->pVirtualAddress,
75
        context->PhysicalAddress.QuadPart,
76
        context->groupTag,
77
        (ULONG)size);
78

79
    return context->pVirtualAddress;
80
}
81

82
void *VirtIOWdfDeviceAllocDmaMemory(VirtIODevice *vdev, size_t size, ULONG groupTag)
83
{
84
    return AllocateCommonBuffer(vdev->DeviceContext, size, groupTag);
85
}
86

87
static BOOLEAN FindCommonBuffer(
88
    PVIRTIO_WDF_DRIVER pWdfDriver,
89
    void *p,
90
    PHYSICAL_ADDRESS *ppa,
91
    size_t *pOffset,
92
    BOOLEAN bRemoval)
93
{
94
    BOOLEAN b = FALSE;
95
    ULONG_PTR va = (ULONG_PTR)p;
96
    ULONG i, n;
97
    WDFOBJECT obj = NULL;
98
    WdfSpinLockAcquire(pWdfDriver->DmaSpinlock);
99
    n = WdfCollectionGetCount(pWdfDriver->MemoryBlockCollection);
100
    for (i = 0; i < n; ++i) {
101
        obj = WdfCollectionGetItem(pWdfDriver->MemoryBlockCollection, i);
102
        if (!obj) {
103
            break;
104
        }
105
        PVIRTIO_WDF_MEMORY_BLOCK_CONTEXT context = GetMemoryBlockContext(obj);
106
        if (context->bToBeDeleted && !bRemoval) {
107
            continue;
108
        }
109
        ULONG_PTR currentVaStart = (ULONG_PTR)context->pVirtualAddress;
110
        if (va >= currentVaStart && va < (currentVaStart + context->Length)) {
111
            *ppa = context->PhysicalAddress;
112
            *pOffset = va - currentVaStart;
113
            b = TRUE;
114
            if (bRemoval) {
115
                b = *pOffset == 0;
116
                if (b) {
117
                    context->bToBeDeleted = TRUE;
118
                }
119
            }
120
            break;
121
        }
122
    }
123
    WdfSpinLockRelease(pWdfDriver->DmaSpinlock);
124
    if (!b) {
125
        DPrintf(0, "%s(%s) FAILED!\n", __FUNCTION__, bRemoval ? "Remove" : "Locate");
126
    }
127
    else if (bRemoval) {
128
        if (KeGetCurrentIrql() == PASSIVE_LEVEL) {
129

130
            WdfSpinLockAcquire(pWdfDriver->DmaSpinlock);
131
            WdfCollectionRemove(pWdfDriver->MemoryBlockCollection, obj);
132
            WdfSpinLockRelease(pWdfDriver->DmaSpinlock);
133

134
            WdfObjectDelete(obj);
135
            DPrintf(1, "%s %p freed (%d common buffers)\n", __FUNCTION__, va, n - 1);
136
        }
137
        else {
138
            DPrintf(0, "%s %p marked for deletion\n", __FUNCTION__, va);
139
        }
140
    }
141
    return b;
142
}
143

144
static PHYSICAL_ADDRESS GetPhysicalAddress(PVIRTIO_WDF_DRIVER pWdfDriver, PVOID va)
145
{
146
    PHYSICAL_ADDRESS pa;
147
    size_t offset;
148
    pa.QuadPart = 0;
149
    if (FindCommonBuffer(pWdfDriver, va, &pa, &offset, FALSE)) {
150
        pa.QuadPart += offset;
151
    }
152
    return pa;
153
}
154

155
PHYSICAL_ADDRESS VirtIOWdfDeviceGetPhysicalAddress(VirtIODevice *vdev, void *va)
156
{
157
    return GetPhysicalAddress(vdev->DeviceContext, va);
158
}
159

160
void VirtIOWdfDeviceFreeDmaMemory(VirtIODevice *vdev, void *va)
161
{
162
    PHYSICAL_ADDRESS pa;
163
    size_t offset;
164
    FindCommonBuffer(vdev->DeviceContext, va, &pa, &offset, TRUE);
165
}
166

167
static BOOLEAN FindCommonBufferByTag(
168
    PVIRTIO_WDF_DRIVER pWdfDriver,
169
    ULONG tag
170
)
171
{
172
    BOOLEAN b = FALSE;
173
    ULONG i, n;
174
    WDFOBJECT obj = NULL;
175
    PVIRTIO_WDF_MEMORY_BLOCK_CONTEXT context = NULL;
176
    WdfSpinLockAcquire(pWdfDriver->DmaSpinlock);
177
    n = WdfCollectionGetCount(pWdfDriver->MemoryBlockCollection);
178
    for (i = 0; i < n; ++i) {
179
        obj = WdfCollectionGetItem(pWdfDriver->MemoryBlockCollection, i);
180
        if (!obj) {
181
            break;
182
        }
183
        context = GetMemoryBlockContext(obj);
184
        if (context->groupTag == tag) {
185
            b = TRUE;
186
            break;
187
        }
188
    }
189
    WdfSpinLockRelease(pWdfDriver->DmaSpinlock);
190
    if (b) {
191
        DPrintf(1, "%s %p (tag %08X) freed (%d common buffers)\n", __FUNCTION__,
192
            context->pVirtualAddress, tag, n - 1);
193
        WdfSpinLockAcquire(pWdfDriver->DmaSpinlock);
194
        WdfCollectionRemove(pWdfDriver->MemoryBlockCollection, obj);
195
        WdfSpinLockRelease(pWdfDriver->DmaSpinlock);
196
        WdfObjectDelete(obj);
197
    }
198
    return b;
199
}
200

201
void VirtIOWdfDeviceFreeDmaMemoryByTag(VirtIODevice *vdev, ULONG groupTag)
202
{
203
    if (KeGetCurrentIrql() > PASSIVE_LEVEL) {
204
        DPrintf(0, "%s FAILED(irql)\n", __FUNCTION__);
205
        return;
206
    }
207
    if (!groupTag) {
208
        DPrintf(0, "%s FAILED(default tag)\n", __FUNCTION__);
209
        return;
210
    }
211
    if (!vdev->DeviceContext) {
212
        DPrintf(0, "%s was not initialized\n", __FUNCTION__);
213
        return;
214
    }
215
    while (FindCommonBufferByTag(vdev->DeviceContext, groupTag));
216
}
217

218
static void FreeSlicedBlock(PVIRTIO_DMA_MEMORY_SLICED p)
219
{
220
    size_t offset;
221
    FindCommonBuffer(p->drv, p->va, &p->pa, &offset, TRUE);
222
    ExFreePoolWithTag(p, p->drv->MemoryTag);
223
}
224

225
static PVOID AllocateSlice(PVIRTIO_DMA_MEMORY_SLICED p, PHYSICAL_ADDRESS *ppa)
226
{
227
    ULONG offset, index = RtlFindClearBitsAndSet(&p->bitmap, 1, 0);
228
    if (index >= p->bitmap.SizeOfBitMap) {
229
        return NULL;
230
    }
231
    offset = p->slice * index;
232
    ppa->QuadPart = p->pa.QuadPart + offset;
233
    return (PUCHAR)p->va + offset;
234
}
235

236
static void FreeSlice(PVIRTIO_DMA_MEMORY_SLICED p, PVOID va)
237
{
238
    PHYSICAL_ADDRESS pa;
239
    size_t offset;
240
    if (!FindCommonBuffer(p->drv, va, &pa, &offset, FALSE)) {
241
        DPrintf(0, "%s: block with va %p not found\n", __FUNCTION__, va);
242
        return;
243
    }
244
    if (offset % p->slice) {
245
        DPrintf(0, "%s: offset %d is wrong for slice %d\n", __FUNCTION__,
246
            (ULONG)offset, p->slice);
247
        return;
248
    }
249
    ULONG index = (ULONG)(offset / p->slice);
250
    if (!RtlTestBit(&p->bitmap, index)) {
251
        DPrintf(0, "%s: bit %d is NOT set\n", __FUNCTION__, index);
252
        return;
253
    }
254
    RtlClearBit(&p->bitmap, index);
255
}
256

257
PVIRTIO_DMA_MEMORY_SLICED VirtIOWdfDeviceAllocDmaMemorySliced(
258
    VirtIODevice *vdev,
259
    size_t blockSize,
260
    ULONG sliceSize)
261
{
262
    PVIRTIO_WDF_DRIVER pWdfDriver = vdev->DeviceContext;
263
    size_t allocSize = sizeof(VIRTIO_DMA_MEMORY_SLICED) + (blockSize / sliceSize) / 8 + sizeof(ULONG);
264
    PVIRTIO_DMA_MEMORY_SLICED p = ExAllocatePoolUninitialized(NonPagedPool, allocSize, pWdfDriver->MemoryTag);
265
    if (!p) {
266
        return NULL;
267
    }
268
    __analysis_assume(allocSize > sizeof(*p));
269
    RtlZeroMemory(p, sizeof(*p));
270
    p->va = AllocateCommonBuffer(pWdfDriver, blockSize, 0);
271
    p->pa = GetPhysicalAddress(pWdfDriver, p->va);
272
    if (!p->va || !p->pa.QuadPart) {
273
        ExFreePoolWithTag(p, pWdfDriver->MemoryTag);
274
        return NULL;
275
    }
276
    p->slice = sliceSize;
277
    p->drv = pWdfDriver;
278
    RtlInitializeBitMap(&p->bitmap, p->bitmap_buffer, (ULONG)blockSize / sliceSize);
279
    p->return_slice = FreeSlice;
280
    p->get_slice = AllocateSlice;
281
    p->destroy   = FreeSlicedBlock;
282
    return p;
283
}
284

285
VOID OnDmaTransactionDestroy(WDFOBJECT Object)
286
{
287
    PVIRTIO_WDF_DMA_TRANSACTION_CONTEXT ctx = GetDmaTransactionContext(Object);
288
    DPrintf(1, "%s %p\n", __FUNCTION__, Object);
289
    if (ctx->mdl) {
290
        IoFreeMdl(ctx->mdl);
291
    }
292
    if (ctx->buffer) {
293
        ExFreePoolWithTag(ctx->buffer, ctx->parameters.allocationTag);
294
    }
295
}
296

297
static FORCEINLINE void RefTransaction(PVIRTIO_WDF_DMA_TRANSACTION_CONTEXT ctx)
298
{
299
    InterlockedIncrement(&ctx->refCount);
300
}
301

302
static FORCEINLINE void DerefTransaction(PVIRTIO_WDF_DMA_TRANSACTION_CONTEXT ctx)
303
{
304
    if (!InterlockedDecrement(&ctx->refCount)) {
305
        WdfObjectDelete(ctx->parameters.transaction);
306
    }
307
}
308

309
BOOLEAN OnDmaTransactionProgramDma(
310
    WDFDMATRANSACTION Transaction,
311
    WDFDEVICE Device,
312
    WDFCONTEXT Context,
313
    WDF_DMA_DIRECTION Direction,
314
    PSCATTER_GATHER_LIST SgList
315
)
316
{
317
    PVIRTIO_WDF_DMA_TRANSACTION_CONTEXT ctx = GetDmaTransactionContext(Transaction);
318
    RefTransaction(ctx);
319
    ctx->parameters.transaction = Transaction;
320
    ctx->parameters.sgList = SgList;
321
    DPrintf(1, "-->%s %p %d frags\n", __FUNCTION__,
322
        Transaction,
323
        SgList->NumberOfElements);
324
    BOOLEAN bFailed = !ctx->callback(&ctx->parameters);
325
    DPrintf(1, "<--%s %s\n", __FUNCTION__, bFailed ? "Failed" : "OK");
326
    DerefTransaction(ctx);
327
    return TRUE;
328
}
329

330
static BOOLEAN VirtIOWdfDeviceDmaAsync(VirtIODevice *vdev,
331
    PVIRTIO_DMA_TRANSACTION_PARAMS params,
332
    VirtIOWdfDmaTransactionCallback callback,
333
    WDF_DMA_DIRECTION Direction)
334
{
335
    PVIRTIO_WDF_DRIVER pWdfDriver = vdev->DeviceContext;
336
    WDFDMATRANSACTION tr;
337
    WDF_OBJECT_ATTRIBUTES attr;
338
    NTSTATUS status;
339
    WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(&attr, VIRTIO_WDF_DMA_TRANSACTION_CONTEXT);
340
    attr.EvtDestroyCallback = OnDmaTransactionDestroy;
341
    status = WdfDmaTransactionCreate(pWdfDriver->DmaEnabler, &attr, &tr);
342
    if (!NT_SUCCESS(status)) {
343
        DPrintf(0, "%s FAILED(create) %X\n", __FUNCTION__, status);
344
        return FALSE;
345
    }
346
    PVIRTIO_WDF_DMA_TRANSACTION_CONTEXT ctx = GetDmaTransactionContext(tr);
347
    RtlZeroMemory(ctx, sizeof(*ctx));
348
    ctx->parameters = *params;
349
    ctx->callback = callback;
350
    ctx->refCount = 1;
351
    ctx->direction = Direction;
352
    if (params->req) {
353
        status = WdfDmaTransactionInitializeUsingRequest(
354
            tr, params->req, OnDmaTransactionProgramDma, Direction);
355
    } else {
356
        ctx->buffer = ExAllocatePoolUninitialized(NonPagedPool, ctx->parameters.size, ctx->parameters.allocationTag);
357
        if (ctx->buffer) {
358
            if (Direction == WdfDmaDirectionWriteToDevice) {
359
                RtlCopyMemory(ctx->buffer, params->buffer, params->size);
360
            }
361
            ctx->mdl = IoAllocateMdl(ctx->buffer, params->size, FALSE, FALSE, NULL);
362
            if (ctx->mdl) {
363
                MmBuildMdlForNonPagedPool(ctx->mdl);
364
                status = WdfDmaTransactionInitialize(
365
                    tr, OnDmaTransactionProgramDma, Direction,
366
                    ctx->mdl, ctx->buffer, params->size);
367
            } else {
368
                status = STATUS_INSUFFICIENT_RESOURCES;
369
            }
370
        } else {
371
            status = STATUS_INSUFFICIENT_RESOURCES;
372
        }
373
    }
374

375
    if (!NT_SUCCESS(status)) {
376
        DPrintf(0, "%s FAILED(init) %X\n", __FUNCTION__, status);
377
        WdfObjectDelete(tr);
378
        return FALSE;
379
    }
380

381
    status = WdfDmaTransactionExecute(tr, NULL);
382
    if (!NT_SUCCESS(status)) {
383
        DPrintf(0, "%s FAILED(execution) %X\n", __FUNCTION__, status);
384
        WdfObjectDelete(tr);
385
        return FALSE;
386
    }
387

388
    return TRUE;
389
}
390

391
BOOLEAN VirtIOWdfDeviceDmaTxAsync(VirtIODevice* vdev,
392
    PVIRTIO_DMA_TRANSACTION_PARAMS params,
393
    VirtIOWdfDmaTransactionCallback callback)
394
{
395
    return VirtIOWdfDeviceDmaAsync(vdev, params, callback, WdfDmaDirectionWriteToDevice);
396
}
397

398
BOOLEAN VirtIOWdfDeviceDmaRxAsync(VirtIODevice* vdev,
399
    PVIRTIO_DMA_TRANSACTION_PARAMS params,
400
    VirtIOWdfDmaTransactionCallback callback)
401
{
402
    return VirtIOWdfDeviceDmaAsync(vdev, params, callback, WdfDmaDirectionReadFromDevice);
403
}
404

405
void VirtIOWdfDeviceDmaTxComplete(VirtIODevice *vdev, WDFDMATRANSACTION transaction)
406
{
407
    PVIRTIO_WDF_DRIVER pWdfDriver = vdev->DeviceContext;
408
    PVIRTIO_WDF_DMA_TRANSACTION_CONTEXT ctx = GetDmaTransactionContext(transaction);
409
    NTSTATUS status;
410
    DPrintf(1, "%s %p\n", __FUNCTION__, transaction);
411
    WdfDmaTransactionDmaCompletedFinal(transaction, 0, &status);
412
    DerefTransaction(ctx);
413
}
414

415
void VirtIOWdfDeviceDmaRxComplete(VirtIODevice* vdev, WDFDMATRANSACTION transaction, ULONG length)
416
{
417
    PVIRTIO_WDF_DRIVER pWdfDriver = vdev->DeviceContext;
418
    PVIRTIO_WDF_DMA_TRANSACTION_CONTEXT ctx = GetDmaTransactionContext(transaction);
419
    NTSTATUS status;
420
    DPrintf(1, "%s %p, len %d\n", __FUNCTION__, transaction, length);
421
    WdfDmaTransactionDmaCompletedFinal(transaction, length, &status);
422
    if (length && ctx->buffer && ctx->parameters.buffer) {
423
        RtlCopyMemory(ctx->parameters.buffer, ctx->buffer, length);
424
    }
425
    DerefTransaction(ctx);
426
}
427