kvm-guest-drivers-windows

1
/*
2
 * Placeholder for the Recv path functions
3
 *
4
 * Copyright (c) 2020 Virtuozzo International GmbH
5
 *
6
 * Redistribution and use in source and binary forms, with or without
7
 * modification, are permitted provided that the following conditions
8
 * are met :
9
 * 1. Redistributions of source code must retain the above copyright
10
 *    notice, this list of conditions and the following disclaimer.
11
 * 2. Redistributions in binary form must reproduce the above copyright
12
 *    notice, this list of conditions and the following disclaimer in the
13
 *    documentation and / or other materials provided with the distribution.
14
 * 3. Neither the names of the copyright holders nor the names of their contributors
15
 *    may be used to endorse or promote products derived from this software
16
 *    without specific prior written permission.
17
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ``AS IS'' AND
18
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20
 * ARE DISCLAIMED.IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE
21
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27
 * SUCH DAMAGE.
28
 */
29

30
#include "precomp.h"
31
#include "viosock.h"
32

33
#if defined(EVENT_TRACING)
34
#include "Rx.tmh"
35
#endif
36

37
#define VIOSOCK_DMA_RX_PAGES    1           //contiguous buffer
38

39
#define VIOSOCK_BYTES_TO_MERGE  128         //max bytes to merge with prev buffer
40

41
#define VIOSOCK_CB_ENTRIES(n) ((n)+(n>>1))  //default chained buffer queue size
42

43
 //Chained Buffer entry
44
typedef struct _VIOSOCK_RX_CB
45
{
46
    union {
47
        SINGLE_LIST_ENTRY   FreeListEntry;
48
        LIST_ENTRY          ListEntry;      //Request buffer list
49
    };
50

51
    PVOID               BufferVA;   //common buffer of VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE bytes
52
    PHYSICAL_ADDRESS    BufferPA;   //common buffer PA
53

54
    PCHAR               ReadPtr;
55
    ULONG               BytesToRead;
56

57
    ULONG               DataLen;    //Valid data len (pkt.header.len)
58
    WDFREQUEST          Request;    //Write request for loopback
59
    WDFMEMORY           Memory;
60
}VIOSOCK_RX_CB, *PVIOSOCK_RX_CB;
61

62
WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(VIOSOCK_RX_CB, GetRequestRxCb);
63

64
typedef struct _VIOSOCK_RX_PKT
65
{
66
    VIRTIO_VSOCK_HDR    Header;
67
    PVIOSOCK_RX_CB      Buffer;     //Chained buffer
68
    union {
69
        BYTE IndirectDescs[SIZE_OF_SINGLE_INDIRECT_DESC * (1 + VIOSOCK_DMA_RX_PAGES)]; //Header + buffer
70
        SINGLE_LIST_ENTRY   RxPktListEntry;
71
        LIST_ENTRY          CompletionListEntry;
72
    };
73
}VIOSOCK_RX_PKT, *PVIOSOCK_RX_PKT;
74

75
typedef struct _VIOSOCK_RX_CONTEXT {
76
    ULONG BufferLen;
77
    ULONG FreeBytes;
78
    union {
79
        struct
80
        {
81
            LONGLONG    Timeout; //100ns
82
            LONG        Counter;
83
            ULONG       Flags;
84
            PCHAR       FreePtr;
85
        };
86
        struct
87
        {
88
            LIST_ENTRY ListEntry;
89
            WDFREQUEST ThisRequest;
90
        };
91
    };
92
}VIOSOCK_RX_CONTEXT, *PVIOSOCK_RX_CONTEXT;
93

94
WDF_DECLARE_CONTEXT_TYPE_WITH_NAME(VIOSOCK_RX_CONTEXT, GetRequestRxContext);
95

96
BOOLEAN
97
VIOSockRxCbInit(
98
    IN PDEVICE_CONTEXT  pContext
99
);
100

101
BOOLEAN
102
VIOSockRxCbAdd(
103
    IN PDEVICE_CONTEXT pContext
104
);
105

106
VOID
107
VIOSockRxCbCleanup(
108
    IN PDEVICE_CONTEXT pContext
109
);
110

111
EVT_WDF_IO_QUEUE_IO_READ    VIOSockRead;
112
EVT_WDF_IO_QUEUE_IO_CANCELED_ON_QUEUE       VIOSockReadIoCanceledOnQueue;
113
EVT_WDF_IO_QUEUE_STATE      VIOSockReadSocketState;
114
EVT_WDF_REQUEST_CANCEL      VIOSockRxRequestCancelCb;
115
EVT_WDF_TIMER               VIOSockReadTimerFunc;
116

117

118
#ifdef ALLOC_PRAGMA
119
#pragma alloc_text (PAGE, VIOSockRxCbInit)
120
#pragma alloc_text (PAGE, VIOSockRxCbAdd)
121
#pragma alloc_text (PAGE, VIOSockRxCbCleanup)
122
#pragma alloc_text (PAGE, VIOSockReadQueueInit)
123
#pragma alloc_text (PAGE, VIOSockReadSocketQueueInit)
124
#endif
125

126
//////////////////////////////////////////////////////////////////////////
127
_Requires_lock_held_(pContext->RxLock)
128
__inline
129
VOID
130
VIOSockRxCbPush(
131
    PDEVICE_CONTEXT pContext,
132
    PVIOSOCK_RX_CB pCb
133
)
134
{
135
    PushEntryList(&pContext->RxCbBuffers, &pCb->FreeListEntry);
136
}
137

138
_Requires_lock_not_held_(pContext->RxLock)
139
__inline
140
VOID
141
VIOSockRxCbPushLocked(
142
    PDEVICE_CONTEXT pContext,
143
    PVIOSOCK_RX_CB pCb
144
)
145
{
146
    WdfSpinLockAcquire(pContext->RxLock);
147
    VIOSockRxCbPush(pContext, pCb);
148
    WdfSpinLockRelease(pContext->RxLock);
149
}
150

151
_Requires_lock_held_(pContext->RxLock)
152
__inline
153
PVIOSOCK_RX_CB
154
VIOSockRxCbPop(
155
    IN PDEVICE_CONTEXT pContext
156
)
157
{
158
    PSINGLE_LIST_ENTRY pListEntry = PopEntryList(&pContext->RxCbBuffers);
159

160
    if (pListEntry)
161
    {
162
        return CONTAINING_RECORD(pListEntry, VIOSOCK_RX_CB, FreeListEntry);
163
    }
164

165
    return NULL;
166
}
167

168
__inline
169
VOID
170
VIOSockRxCbFree(
171
    IN PDEVICE_CONTEXT pContext,
172
    IN PVIOSOCK_RX_CB pCb
173
)
174
{
175
    ASSERT(pCb && pCb->BufferVA);
176

177
    ASSERT(pCb->BufferPA.QuadPart && pCb->Request == WDF_NO_HANDLE);
178

179
    if (pCb->BufferPA.QuadPart)
180
        VirtIOWdfDeviceFreeDmaMemory(&pContext->VDevice.VIODevice, pCb->BufferVA);
181

182
    WdfObjectDelete(pCb->Memory);
183
}
184

185
static
186
BOOLEAN
187
VIOSockRxCbAdd(
188
    IN PDEVICE_CONTEXT pContext
189
)
190
{
191
    NTSTATUS        status;
192
    PVIOSOCK_RX_CB  pCb;
193
    WDFMEMORY       Memory;
194
    BOOLEAN         bRes = FALSE;
195

196
    PAGED_CODE();
197

198
    status = WdfMemoryCreateFromLookaside(pContext->RxCbBufferMemoryList, &Memory);
199
    if (NT_SUCCESS(status))
200
    {
201
        pCb = WdfMemoryGetBuffer(Memory, NULL);
202

203
        RtlZeroMemory(pCb, sizeof(*pCb));
204

205
        pCb->Memory = Memory;
206
        pCb->BufferVA = VirtIOWdfDeviceAllocDmaMemory(&pContext->VDevice.VIODevice,
207
            VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE, VIOSOCK_DRIVER_MEMORY_TAG);
208

209
        if (!pCb->BufferVA)
210
        {
211
            TraceEvents(TRACE_LEVEL_ERROR, DBG_HW_ACCESS,
212
                "VirtIOWdfDeviceAllocDmaMemory(%u bytes for Rx buffer) failed\n", VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE);
213
            WdfObjectDelete(pCb->Memory);
214
        }
215
        else
216
        {
217
            pCb->BufferPA = VirtIOWdfDeviceGetPhysicalAddress(&pContext->VDevice.VIODevice,
218
                pCb->BufferVA);
219

220
            ASSERT(pCb->BufferPA.QuadPart);
221

222
            if (!pCb->BufferPA.QuadPart)
223
            {
224
                TraceEvents(TRACE_LEVEL_ERROR, DBG_HW_ACCESS, "VirtIOWdfDeviceGetPhysicalAddress failed\n");
225
                VIOSockRxCbFree(pContext, pCb);
226
            }
227
            else
228
            {
229
                //no need to lock, init call
230
                VIOSockRxCbPush(pContext, pCb);
231
                bRes = TRUE;
232
            }
233
        }
234
    }
235
    else
236
    {
237
        TraceEvents(TRACE_LEVEL_ERROR, DBG_HW_ACCESS, "WdfMemoryCreateFromLookaside failed: 0x%x\n", status);
238
    }
239

240
    return bRes;
241
}
242

243
static
244
PVIOSOCK_RX_CB
245
VIOSockRxCbEntryForRequest(
246
    IN PDEVICE_CONTEXT  pContext,
247
    IN WDFREQUEST       Request,
248
    IN ULONG            Length
249
)
250
{
251
    NTSTATUS                status;
252
    WDF_OBJECT_ATTRIBUTES   attributes;
253
    PVIOSOCK_RX_CB          pCb = NULL;
254

255
    ASSERT(Request != WDF_NO_HANDLE);
256

257
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
258

259
    WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(
260
        &attributes,
261
        VIOSOCK_RX_CB
262
    );
263
    status = WdfObjectAllocateContext(
264
        Request,
265
        &attributes,
266
        &pCb
267
    );
268
    if (!NT_SUCCESS(status))
269
    {
270
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ, "WdfObjectAllocateContext failed: 0x%x\n", status);
271
        return NULL;
272
    }
273

274
    pCb->Memory = WDF_NO_HANDLE;
275
    pCb->BufferPA.QuadPart = 0;
276
    InitializeListHead(&pCb->ListEntry);
277

278
    status = WdfRequestRetrieveInputBuffer(Request, 0, &pCb->BufferVA, NULL);
279
    if (NT_SUCCESS(status))
280
    {
281
        pCb->Request = Request;
282
        pCb->BytesToRead = pCb->DataLen = (ULONG)Length;
283
        pCb->ReadPtr = pCb->BufferVA;
284
    }
285
    else
286
    {
287
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ, "WdfRequestRetrieveOutputBuffer failed: 0x%x\n", status);
288
        pCb = NULL;
289
    }
290

291
    return pCb;
292
}
293

294
static
295
BOOLEAN
296
VIOSockRxCbInit(
297
    IN PDEVICE_CONTEXT  pContext
298
)
299
{
300
    ULONG i;
301
    BOOLEAN bRes = TRUE;
302
    WDF_OBJECT_ATTRIBUTES lockAttributes, memAttributes;
303
    NTSTATUS status;
304

305
    PAGED_CODE();
306

307
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
308

309
    pContext->RxCbBuffersNum = VIOSOCK_CB_ENTRIES(pContext->RxPktNum);
310

311
    if (pContext->RxCbBuffersNum < pContext->RxPktNum)
312
        pContext->RxCbBuffersNum = pContext->RxPktNum;
313

314
    WDF_OBJECT_ATTRIBUTES_INIT(&memAttributes);
315
    memAttributes.ParentObject = pContext->ThisDevice;
316

317
    status = WdfLookasideListCreate(&memAttributes,
318
        sizeof(VIOSOCK_RX_CB), NonPagedPoolNx,
319
        &memAttributes, VIOSOCK_DRIVER_MEMORY_TAG,
320
        &pContext->RxCbBufferMemoryList);
321

322
    if (!NT_SUCCESS(status))
323
    {
324
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ,
325
            "WdfLookasideListCreate failed: 0x%x\n", status);
326
        return FALSE;
327
    }
328

329
    TraceEvents(TRACE_LEVEL_INFORMATION, DBG_READ,
330
        "Initialize chained buffer with %u entries\n", pContext->RxCbBuffersNum);
331

332
    pContext->RxCbBuffers.Next = NULL;
333

334
    for (i = 0; i < pContext->RxCbBuffersNum; ++i)
335
    {
336
        if (!VIOSockRxCbAdd(pContext))
337
        {
338
            TraceEvents(TRACE_LEVEL_ERROR, DBG_READ,
339
                "VIOSockRxCbAdd failed, cleanup chained buffer\n");
340
            bRes = FALSE;
341
            break;
342
        }
343
    }
344

345
    if (!bRes)
346
        VIOSockRxCbCleanup(pContext);
347

348
    return bRes;
349
}
350

351
static
352
VOID
353
VIOSockRxCbCleanup(
354
    IN PDEVICE_CONTEXT pContext
355
)
356
{
357
    PVIOSOCK_RX_CB pCb;
358

359
    PAGED_CODE();
360

361
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
362

363
    //no need to lock
364
    while (pCb = VIOSockRxCbPop(pContext))
365
    {
366
        VIOSockRxCbFree(pContext, pCb);
367
    }
368

369
}
370

371
//////////////////////////////////////////////////////////////////////////
372
__inline
373
VOID
374
VIOSockRxPktCleanup(
375
    IN PDEVICE_CONTEXT pContext,
376
    IN PVIOSOCK_RX_PKT pPkt
377
)
378
{
379
    ASSERT(pPkt);
380

381
    if (pPkt->Buffer)
382
    {
383
        VIOSockRxCbPush(pContext, pPkt->Buffer);
384
        pPkt->Buffer = NULL;
385
    }
386
}
387

388
C_ASSERT((VIOSOCK_DMA_RX_PAGES + 1) == 2);
389

390
#define VIOSOCK_PKT_PA_FROM_VA(ctx,pva) (((ctx)->RxPktPA.QuadPart) + (ULONGLONG)((PCHAR)(pva) - (PCHAR)((ctx)->RxPktVA)))
391

392
_Requires_lock_held_(pContext->RxLock)
393
static
394
BOOLEAN
395
VIOSockRxPktInsert(
396
    IN PDEVICE_CONTEXT pContext,
397
    IN PVIOSOCK_RX_PKT pPkt
398
)
399
{
400
    BOOLEAN bRes = TRUE;
401
    VIOSOCK_SG_DESC sg[VIOSOCK_DMA_RX_PAGES + 1];
402
    PHYSICAL_ADDRESS pPKtPA;
403
    int ret;
404

405
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_HW_ACCESS, "--> %s\n", __FUNCTION__);
406

407
    if (!pPkt->Buffer)
408
    {
409
        pPkt->Buffer = VIOSockRxCbPop(pContext);
410
        if (!pPkt->Buffer)
411
        {
412
            TraceEvents(TRACE_LEVEL_INFORMATION, DBG_HW_ACCESS, "VIOSockRxCbPop returns NULL\n");
413
            return FALSE;
414
        }
415
    }
416

417
    ASSERT(pPkt->Buffer->Request == WDF_NO_HANDLE);
418

419
    pPKtPA.QuadPart = VIOSOCK_PKT_PA_FROM_VA(pContext, pPkt);
420

421
    sg[0].length = sizeof(VIRTIO_VSOCK_HDR);
422
    sg[0].physAddr.QuadPart = pPKtPA.QuadPart + FIELD_OFFSET(VIOSOCK_RX_PKT, Header);
423

424
    sg[1].length = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE;
425
    sg[1].physAddr.QuadPart = pPkt->Buffer->BufferPA.QuadPart;
426

427
    ret = virtqueue_add_buf(pContext->RxVq, sg, 0, 2, pPkt, &pPkt->IndirectDescs,
428
        pPKtPA.QuadPart + FIELD_OFFSET(VIOSOCK_RX_PKT, IndirectDescs));
429

430
    ASSERT(ret >= 0);
431
    if (ret < 0)
432
    {
433
        TraceEvents(TRACE_LEVEL_ERROR, DBG_HW_ACCESS,
434
            "Error adding buffer to Rx queue (ret = %d)\n", ret);
435
        VIOSockRxPktCleanup(pContext, pPkt);
436
        return FALSE;
437
    }
438

439
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_HW_ACCESS, "<-- %s\n", __FUNCTION__);
440
    return TRUE;
441
}
442

443
_Requires_lock_held_(pContext->RxLock)
444
__inline
445
VOID
446
VIOSockRxPktListProcess(
447
    IN PDEVICE_CONTEXT pContext
448
)
449
{
450
    PSINGLE_LIST_ENTRY pListEntry;
451

452
    while (pListEntry = PopEntryList(&pContext->RxPktList))
453
    {
454
        PVIOSOCK_RX_PKT pPkt = CONTAINING_RECORD(pListEntry, VIOSOCK_RX_PKT, RxPktListEntry);
455

456
        if (!VIOSockRxPktInsert(pContext, pPkt))
457
        {
458
            PushEntryList(&pContext->RxPktList, &pPkt->RxPktListEntry);
459
            break;
460
        }
461
    }
462
}
463

464
_Requires_lock_not_held_(pContext->RxLock)
465
__inline
466
VOID
467
VIOSockRxPktListProcessLocked(
468
    IN PDEVICE_CONTEXT pContext
469
)
470
{
471
    WdfSpinLockAcquire(pContext->RxLock);
472
    VIOSockRxPktListProcess(pContext);
473
    bool bNotify = virtqueue_kick_prepare(pContext->RxVq);
474
    WdfSpinLockRelease(pContext->RxLock);
475

476
    if (bNotify)
477
        virtqueue_notify(pContext->RxVq);
478
}
479

480
_Requires_lock_not_held_(pContext->RxLock)
481
__inline
482
VOID
483
VIOSockRxPktInsertOrPostpone(
484
    IN PDEVICE_CONTEXT pContext,
485
    IN PVIOSOCK_RX_PKT pPkt
486
)
487
{
488
    bool bNotify = false;
489
    WdfSpinLockAcquire(pContext->RxLock);
490
    if (!VIOSockRxPktInsert(pContext, pPkt))
491
    {
492
        //postpone packet
493
        TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "Postpone packet\n");
494
        PushEntryList(&pContext->RxPktList, &pPkt->RxPktListEntry);
495
    }
496
    else
497
    {
498
        VIOSockRxPktListProcess(pContext);
499
        bNotify = virtqueue_kick_prepare(pContext->RxVq);
500
    }
501
    WdfSpinLockRelease(pContext->RxLock);
502

503
    if (bNotify)
504
        virtqueue_notify(pContext->RxVq);
505
}
506

507
_Requires_lock_held_(pSocket->RxLock)
508
__inline
509
BOOLEAN
510
VIOSockRxPktInc(
511
    IN PSOCKET_CONTEXT  pSocket,
512
    IN ULONG            uPktLen
513
)
514
{
515
    if (pSocket->RxBytes + uPktLen > pSocket->buf_alloc)
516
        return FALSE;
517

518
    pSocket->RxBytes += uPktLen;
519
    return TRUE;
520

521
}
522

523
_Requires_lock_held_(pSocket->RxLock)
524
__inline
525
VOID
526
VIOSockRxPktDec(
527
    IN PSOCKET_CONTEXT  pSocket,
528
    IN ULONG            uPktLen
529
)
530
{
531
    pSocket->RxBytes -= uPktLen;
532
    pSocket->fwd_cnt += uPktLen;
533
}
534

535
_Requires_lock_not_held_(pContext->RxLock)
536
VOID
537
VIOSockRxVqCleanup(
538
    IN PDEVICE_CONTEXT pContext
539
)
540
{
541
    PVIOSOCK_RX_PKT pPkt;
542
    PSINGLE_LIST_ENTRY pListEntry;
543

544
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_HW_ACCESS, "--> %s\n", __FUNCTION__);
545

546
    ASSERT(pContext->RxVq && pContext->RxPktVA);
547

548
    //drain queue
549
    WdfSpinLockAcquire(pContext->RxLock);
550
    while (pPkt = (PVIOSOCK_RX_PKT)virtqueue_detach_unused_buf(pContext->RxVq))
551
    {
552
        VIOSockRxPktCleanup(pContext, pPkt);
553
    }
554

555
    while (pListEntry = PopEntryList(&pContext->RxPktList))
556
    {
557
        VIOSockRxPktCleanup(pContext, CONTAINING_RECORD(pListEntry, VIOSOCK_RX_PKT, RxPktListEntry));
558
    }
559
    WdfSpinLockRelease(pContext->RxLock);
560

561
    VIOSockRxCbCleanup(pContext);
562

563
    if (pContext->RxPktVA)
564
    {
565
        VirtIOWdfDeviceFreeDmaMemory(&pContext->VDevice.VIODevice, pContext->RxPktVA);
566
        pContext->RxPktVA = NULL;
567
    }
568

569
    pContext->RxVq = NULL;
570
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_HW_ACCESS, "<-- %s\n", __FUNCTION__);
571
}
572

573
NTSTATUS
574
VIOSockRxVqInit(
575
    IN PDEVICE_CONTEXT pContext
576
)
577
{
578
    NTSTATUS status = STATUS_SUCCESS;
579
    USHORT uNumEntries;
580
    ULONG uRingSize, uHeapSize, uBufferSize;
581

582
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_HW_ACCESS, "--> %s\n", __FUNCTION__);
583

584
    pContext->RxPktList.Next = NULL;
585

586
    status = virtio_query_queue_allocation(&pContext->VDevice.VIODevice, VIOSOCK_VQ_RX,
587
        &uNumEntries, &uRingSize, &uHeapSize);
588
    if (!NT_SUCCESS(status))
589
    {
590
        TraceEvents(TRACE_LEVEL_ERROR, DBG_HW_ACCESS, "virtio_query_queue_allocation(VIOSOCK_VQ_RX) failed\n");
591
        pContext->RxVq = NULL;
592
        return status;
593
    }
594

595
    pContext->RxPktNum = uNumEntries;
596

597
    uBufferSize = sizeof(VIOSOCK_RX_PKT) * uNumEntries;
598

599
    TraceEvents(TRACE_LEVEL_INFORMATION, DBG_HW_ACCESS, "Allocating common buffer of %u bytes for %u Rx packets\n",
600
        uBufferSize, uNumEntries);
601

602
    pContext->RxPktVA = (PVIOSOCK_RX_PKT)VirtIOWdfDeviceAllocDmaMemory(&pContext->VDevice.VIODevice,
603
        uBufferSize, VIOSOCK_DRIVER_MEMORY_TAG);
604

605
    ASSERT(pContext->RxPktVA);
606
    if (!pContext->RxPktVA)
607
    {
608
        TraceEvents(TRACE_LEVEL_ERROR, DBG_HW_ACCESS,
609
            "VirtIOWdfDeviceAllocDmaMemory(%u bytes for RxPackets) failed\n", uBufferSize);
610
        status = STATUS_INSUFFICIENT_RESOURCES;
611
    }
612
    else if (VIOSockRxCbInit(pContext))
613
    {
614
        ULONG i;
615
        PVIOSOCK_RX_PKT RxPktVA = (PVIOSOCK_RX_PKT)pContext->RxPktVA;
616

617
        pContext->RxPktPA = VirtIOWdfDeviceGetPhysicalAddress(&pContext->VDevice.VIODevice, pContext->RxPktVA);
618
        ASSERT(pContext->RxPktPA.QuadPart);
619

620
        //fill queue, no lock
621
        for (i = 0; i < uNumEntries; i++)
622
        {
623
            if (!VIOSockRxPktInsert(pContext, &RxPktVA[i]))
624
            {
625
                TraceEvents(TRACE_LEVEL_ERROR, DBG_HW_ACCESS, "VIOSockRxPktInsert[%u] failed\n", i);
626
                status = STATUS_UNSUCCESSFUL;
627
                break;
628
            }
629
        }
630
    }
631
    else
632
    {
633
        TraceEvents(TRACE_LEVEL_ERROR, DBG_HW_ACCESS, "VIOSockRxCbInit failed\n");
634
        status = STATUS_UNSUCCESSFUL;
635
    }
636

637
    if (!NT_SUCCESS(status))
638
    {
639
        VIOSockRxVqCleanup(pContext);
640
    }
641

642
    return status;
643
}
644

645
_Requires_lock_not_held_(pSocket->StateLock)
646
static
647
VOID
648
VIOSockRxPktHandleConnecting(
649
    IN PSOCKET_CONTEXT  pSocket,
650
    IN PVIOSOCK_RX_PKT  pPkt,
651
    IN BOOLEAN          bTxHasSpace
652
)
653
{
654
    WDFREQUEST  PendedRequest;
655
    NTSTATUS    status;
656

657
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_SOCKET, "--> %s\n", __FUNCTION__);
658

659
    status = VIOSockPendedRequestGetLocked(pSocket, &PendedRequest);
660

661
    if (NT_SUCCESS(status))
662
    {
663
        if (PendedRequest == WDF_NO_HANDLE &&
664
            !VIOSockIsNonBlocking(pSocket))
665
        {
666
            TraceEvents(TRACE_LEVEL_ERROR, DBG_SOCKET, "No PendedRequest found\n");
667
            status = STATUS_CANCELLED;
668
        }
669
        else
670
        {
671
            switch (pPkt->Header.op)
672
            {
673
            case VIRTIO_VSOCK_OP_RESPONSE:
674
                WdfSpinLockAcquire(pSocket->StateLock);
675
                VIOSockStateSet(pSocket, VIOSOCK_STATE_CONNECTED);
676
                VIOSockEventSetBit(pSocket, FD_CONNECT_BIT, STATUS_SUCCESS);
677
                if (bTxHasSpace)
678
                    VIOSockEventSetBit(pSocket, FD_WRITE_BIT, STATUS_SUCCESS);
679
                WdfSpinLockRelease(pSocket->StateLock);
680
                status = STATUS_SUCCESS;
681
                break;
682
            case VIRTIO_VSOCK_OP_INVALID:
683
                if (PendedRequest != WDF_NO_HANDLE)
684
                {
685
                    status = VIOSockPendedRequestSetResumeLocked(pSocket, PendedRequest);
686
                    if (NT_SUCCESS(status))
687
                        PendedRequest = WDF_NO_HANDLE;
688
                }
689
                break;
690
            case VIRTIO_VSOCK_OP_RST:
691
                status = STATUS_CONNECTION_REFUSED;
692
                break;
693
            default:
694
                status = STATUS_CONNECTION_INVALID;
695
            }
696
        }
697
    }
698

699
    if (!NT_SUCCESS(status))
700
    {
701
        WdfSpinLockAcquire(pSocket->StateLock);
702
        VIOSockEventSetBit(pSocket, FD_CONNECT_BIT, status);
703
        VIOSockStateSet(pSocket, VIOSOCK_STATE_CLOSE);
704
        WdfSpinLockRelease(pSocket->StateLock);
705
        if (pPkt->Header.op != VIRTIO_VSOCK_OP_RST)
706
            VIOSockSendReset(pSocket, TRUE);
707
    }
708

709
    if (PendedRequest)
710
    {
711
        WdfRequestComplete(PendedRequest, status);
712
    }
713

714
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_SOCKET, "<-- %s\n", __FUNCTION__);
715
}
716

717
_Requires_lock_not_held_(pSocket->RxLock)
718
static
719
BOOLEAN
720
VIOSockRxPktEnqueueCb(
721
    IN PSOCKET_CONTEXT pSocket,
722
    IN PVIOSOCK_RX_PKT pPkt
723
)
724
{
725
    PDEVICE_CONTEXT pContext = GetDeviceContextFromSocket(pSocket);
726
    PVIOSOCK_RX_CB pCurrentCb = NULL;
727
    ULONG BufferFree, PktLen;
728
    BOOLEAN bRes = FALSE;
729

730
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
731

732
    ASSERT(pPkt && pPkt->Buffer && pPkt->Header.len);
733

734
    PktLen = pPkt->Header.len;
735

736
    //Merge buffers
737
    WdfSpinLockAcquire(pSocket->RxLock);
738
    if (!IsListEmpty(&pSocket->RxCbList) && PktLen <= VIOSOCK_BYTES_TO_MERGE)
739
    {
740
        pCurrentCb = CONTAINING_RECORD(pSocket->RxCbList.Blink, VIOSOCK_RX_CB, ListEntry);
741

742
        BufferFree = VIRTIO_VSOCK_DEFAULT_RX_BUF_SIZE - pCurrentCb->DataLen;
743

744
        if (BufferFree >= PktLen)
745
        {
746
            if (VIOSockRxPktInc(pSocket, PktLen))
747
            {
748
                TraceEvents(TRACE_LEVEL_INFORMATION, DBG_READ, "RxCb merged: %d + %d bytes\n", pCurrentCb->DataLen, PktLen);
749
                memcpy((PCHAR)pCurrentCb->BufferVA + pCurrentCb->DataLen, pPkt->Buffer->BufferVA, PktLen);
750
                pCurrentCb->DataLen += PktLen;
751
                pCurrentCb->BytesToRead += PktLen;
752
            }
753
            else
754
            {
755
                TraceEvents(TRACE_LEVEL_WARNING, DBG_READ, "Rx buffer full, drop packet\n");
756
            }
757
            //just leave buffer with pkt
758
            WdfSpinLockRelease(pSocket->RxLock);
759
            return FALSE;
760
        }
761
    }
762
    else
763
    {
764
        bRes = TRUE;    //Scan read queue
765
    }
766

767
    //Enqueue buffer
768
    if (VIOSockRxPktInc(pSocket, PktLen))
769
    {
770
        pPkt->Buffer->DataLen = PktLen;
771
        pPkt->Buffer->BytesToRead = PktLen;
772
        pPkt->Buffer->ReadPtr = pPkt->Buffer->BufferVA;
773
        InsertTailList(&pSocket->RxCbList, &pPkt->Buffer->ListEntry);
774
        pSocket->RxBuffers++;
775
        pPkt->Buffer = NULL; //remove buffer from pkt
776
        TraceEvents(TRACE_LEVEL_INFORMATION, DBG_READ, "RxCb enqueued: %d bytes\n", PktLen);
777
    }
778
    else
779
    {
780
        TraceEvents(TRACE_LEVEL_WARNING, DBG_READ, "Rx buffer full, drop packet\n");
781
        bRes = FALSE;
782
    }
783

784
    WdfSpinLockRelease(pSocket->RxLock);
785

786
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "<-- %s\n", __FUNCTION__);
787
    return bRes;
788
}
789

790
static
791
VOID
792
VIOSockRxRequestCancelCb(
793
    IN WDFREQUEST Request
794
)
795
{
796
    PSOCKET_CONTEXT pSocket = GetSocketContextFromRequest(Request);
797
    PVIOSOCK_RX_CB  pCb = GetRequestRxCb(Request);
798
    NTSTATUS status = STATUS_CANCELLED;;
799

800
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
801

802
    WdfSpinLockAcquire(pSocket->RxLock);
803
    VIOSockRxPktDec(pSocket, pCb->BytesToRead);
804
    RemoveEntryList(&pCb->ListEntry);
805
    WdfSpinLockRelease(pSocket->RxLock);
806

807
    if (pCb->BytesToRead != pCb->DataLen)
808
        status = STATUS_SUCCESS;
809

810
    WdfRequestCompleteWithInformation(Request, status, pCb->DataLen - pCb->BytesToRead);
811
}
812

813
_Requires_lock_not_held_(pSocket->RxLock)
814
NTSTATUS
815
VIOSockRxRequestEnqueueCb(
816
    IN PSOCKET_CONTEXT  pSocket,
817
    IN WDFREQUEST       Request,
818
    IN ULONG            Length
819
)
820
{
821
    PDEVICE_CONTEXT pContext = GetDeviceContextFromSocket(pSocket);
822
    PVIOSOCK_RX_CB  pCurrentCb = NULL;
823
    ULONG           BufferFree;
824
    NTSTATUS        status;
825

826
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
827

828
    pCurrentCb = VIOSockRxCbEntryForRequest(pContext, Request, Length);
829
    if (!pCurrentCb)
830
        return STATUS_INSUFFICIENT_RESOURCES;
831

832
    WdfSpinLockAcquire(pSocket->RxLock);
833

834
    //Enqueue buffer
835
    if (VIOSockRxPktInc(pSocket, pCurrentCb->DataLen))
836
    {
837
        status = WdfRequestMarkCancelableEx(Request, VIOSockRxRequestCancelCb);
838
        if (NT_SUCCESS(status))
839
        {
840
            InsertTailList(&pSocket->RxCbList, &pCurrentCb->ListEntry);
841
            pSocket->RxBuffers++;
842
            status = STATUS_SUCCESS;
843
        }
844
        else
845
        {
846
            ASSERT(status == STATUS_CANCELLED);
847
            TraceEvents(TRACE_LEVEL_WARNING, DBG_READ, "Loopback request canceled: 0x%x\n", status);
848
        }
849
    }
850
    else
851
    {
852
        TraceEvents(TRACE_LEVEL_WARNING, DBG_READ, "Rx buffer full, drop packet\n");
853
        status = STATUS_BUFFER_TOO_SMALL;
854
    }
855

856
    WdfSpinLockRelease(pSocket->RxLock);
857

858
    return status;
859
}
860

861
static
862
VOID
863
VIOSockRxPktHandleConnected(
864
    IN PSOCKET_CONTEXT  pSocket,
865
    IN PVIOSOCK_RX_PKT  pPkt,
866
    IN BOOLEAN          bTxHasSpace
867
)
868
{
869
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_SOCKET, "--> %s\n", __FUNCTION__);
870

871
    switch (pPkt->Header.op)
872
    {
873
    case VIRTIO_VSOCK_OP_RW:
874
        if (VIOSockRxPktEnqueueCb(pSocket, pPkt))
875
        {
876
            VIOSockEventSetBitLocked(pSocket, FD_READ_BIT, STATUS_SUCCESS);
877
            VIOSockReadProcessDequeueCb(pSocket);
878
        }
879
        break;
880
    case VIRTIO_VSOCK_OP_CREDIT_UPDATE:
881
        if (bTxHasSpace)
882
        {
883
            VIOSockEventSetBitLocked(pSocket, FD_WRITE_BIT, STATUS_SUCCESS);
884
        }
885
        break;
886
    case VIRTIO_VSOCK_OP_SHUTDOWN:
887
        if (VIOSockShutdownFromPeer(pSocket,
888
            pPkt->Header.flags & VIRTIO_VSOCK_SHUTDOWN_MASK) &&
889
            !VIOSockRxHasData(pSocket) &&
890
            !VIOSockIsDone(pSocket))
891
        {
892
            VIOSockSendReset(pSocket, FALSE);
893
            VIOSockDoClose(pSocket);
894
        }
895
        break;
896
    case VIRTIO_VSOCK_OP_RST:
897
        VIOSockDoClose(pSocket);
898
        VIOSockEventSetBitLocked(pSocket, FD_CLOSE_BIT, STATUS_CONNECTION_RESET);
899
        break;
900
    }
901

902
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_SOCKET, "<-- %s\n", __FUNCTION__);
903
}
904

905
static
906
VOID
907
VIOSockRxPktHandleDisconnecting(
908
    IN PSOCKET_CONTEXT pSocket,
909
    IN PVIOSOCK_RX_PKT pPkt
910
)
911
{
912
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_SOCKET, "--> %s\n", __FUNCTION__);
913

914
    if (pPkt->Header.op == VIRTIO_VSOCK_OP_RST)
915
    {
916
        VIOSockDoClose(pSocket);
917

918
//         if (pSocket->PeerShutdown & VIRTIO_VSOCK_SHUTDOWN_MASK != VIRTIO_VSOCK_SHUTDOWN_MASK)
919
//         {
920
//             VIOSockEventSetBit(pSocket, FD_CLOSE_BIT, STATUS_CONNECTION_RESET);
921
//         }
922
    }
923
}
924

925
static
926
VOID
927
VIOSockRxPktHandleListen(
928
    IN PSOCKET_CONTEXT pSocket,
929
    IN PVIOSOCK_RX_PKT pPkt
930
)
931
{
932
    PDEVICE_CONTEXT pContext = GetDeviceContextFromSocket(pSocket);
933
    NTSTATUS    status;
934

935
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_SOCKET, "--> %s\n", __FUNCTION__);
936

937
    if (pPkt->Header.op == VIRTIO_VSOCK_OP_RST)
938
    {
939
        //remove pended accept
940
        VIOSockAcceptRemovePkt(pSocket, &pPkt->Header);
941
        return;
942
    }
943
    else if (pPkt->Header.op != VIRTIO_VSOCK_OP_REQUEST)
944
    {
945
        TraceEvents(TRACE_LEVEL_ERROR, DBG_SOCKET, "Invalid packet: %u\n", pPkt->Header.op);
946
        VIOSockSendResetNoSock(pContext, &pPkt->Header);
947
        return;
948
    }
949

950
    status = VIOSockAcceptEnqueuePkt(pSocket, &pPkt->Header);
951
    if (!NT_SUCCESS(status))
952
    {
953
        TraceEvents(TRACE_LEVEL_ERROR, DBG_SOCKET, "VIOSockAcceptEnqueuePkt failed: 0x%x\n", status);
954
        VIOSockSendResetNoSock(pContext, &pPkt->Header);
955
    }
956
}
957

958
_Requires_lock_not_held_(pContext->RxLock)
959
VOID
960
VIOSockRxVqProcess(
961
    IN PDEVICE_CONTEXT pContext
962
)
963
{
964
    PVIOSOCK_RX_PKT     pPkt;
965
    UINT                len;
966
    LIST_ENTRY          CompletionList;
967
    PSINGLE_LIST_ENTRY  pCurrentEntry;
968
    PSOCKET_CONTEXT     pSocket = NULL;
969
    BOOLEAN             bStop = FALSE;
970

971
    NTSTATUS            status;
972

973
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_HW_ACCESS, "--> %s\n", __FUNCTION__);
974

975
    InitializeListHead(&CompletionList);
976

977
    WdfSpinLockAcquire(pContext->RxLock);
978
    do
979
    {
980
        virtqueue_disable_cb(pContext->RxVq);
981

982
        while (TRUE)
983
        {
984
            if (!VIOSockTxMoreReplies(pContext)) {
985
                /* Stop rx until the device processes already
986
                 * pending replies.  Leave rx virtqueue
987
                 * callbacks disabled.
988
                 */
989
                TraceEvents(TRACE_LEVEL_VERBOSE, DBG_HW_ACCESS, "Stop rx\n");
990

991
                bStop = TRUE;
992
                break;
993
            }
994

995
            pPkt = (PVIOSOCK_RX_PKT)virtqueue_get_buf(pContext->RxVq, &len);
996
            if (!pPkt)
997
                break;
998

999
            /* Drop short/long packets */
1000

1001
            if (len < sizeof(pPkt->Header) ||
1002
                len > sizeof(pPkt->Header) + pPkt->Header.len)
1003
            {
1004
                TraceEvents(TRACE_LEVEL_ERROR, DBG_HW_ACCESS, "Short/Long packet (%d)\n", len);
1005
                VIOSockRxPktInsert(pContext, pPkt);
1006
                continue;
1007
            }
1008

1009
            ASSERT(pPkt->Header.len == len - sizeof(pPkt->Header));
1010

1011
            TraceEvents(TRACE_LEVEL_INFORMATION, DBG_READ, "Recv packet %!Op! (%d:%d <-- %d:%d), len: %d, flags: %d, buf_alloc: %d, fwd_cnt: %d\n",
1012
                pPkt->Header.op,
1013
                (ULONG)pPkt->Header.dst_cid, pPkt->Header.dst_port,
1014
                (ULONG)pPkt->Header.src_cid, pPkt->Header.src_port,
1015
                pPkt->Header.len, pPkt->Header.flags, pPkt->Header.buf_alloc, pPkt->Header.fwd_cnt);
1016

1017
            //"complete" buffers later
1018
            InsertTailList(&CompletionList, &pPkt->CompletionListEntry);
1019
        }
1020
    } while (!virtqueue_enable_cb(pContext->RxVq) && !bStop);
1021

1022
    WdfSpinLockRelease(pContext->RxLock);
1023

1024
    //complete buffers
1025
    while (!IsListEmpty(&CompletionList))
1026
    {
1027
        BOOLEAN bTxHasSpace;
1028

1029
        pPkt = CONTAINING_RECORD(RemoveHeadList(&CompletionList), VIOSOCK_RX_PKT, CompletionListEntry);
1030

1031
        //find socket
1032
        pSocket = VIOSockConnectedFindByRxPkt(pContext, &pPkt->Header);
1033
        if (!pSocket)
1034
        {
1035
            //no connected socket for incoming packet
1036
            TraceEvents(TRACE_LEVEL_INFORMATION, DBG_SOCKET, "No connected socket found\n");
1037
            pSocket = VIOSockBoundFindByPort(pContext, pPkt->Header.dst_port);
1038
        }
1039

1040
        if (pSocket && pSocket->type != pPkt->Header.type)
1041
        {
1042
            TraceEvents(TRACE_LEVEL_WARNING, DBG_SOCKET, "Invalid socket %d type\n", pSocket->SocketId);
1043
            pSocket = NULL;
1044
        }
1045
        if (!pSocket)
1046
        {
1047
            TraceEvents(TRACE_LEVEL_WARNING, DBG_SOCKET, "Socket for packet is not exists\n");
1048
            VIOSockSendResetNoSock(pContext, &pPkt->Header);
1049
            VIOSockRxPktInsertOrPostpone(pContext, pPkt);
1050
            continue;
1051
        }
1052

1053

1054
        ASSERT(pContext->Config.guest_cid == (ULONG32)pPkt->Header.dst_cid);
1055

1056
        TraceEvents(TRACE_LEVEL_INFORMATION, DBG_SOCKET, "Socket %d found, state %!State!\n",
1057
            pSocket->SocketId, VIOSockStateGet(pSocket));
1058

1059
        bTxHasSpace = !!VIOSockTxSpaceUpdate(pSocket, &pPkt->Header);
1060

1061
        switch (VIOSockStateGet(pSocket))
1062
        {
1063
        case VIOSOCK_STATE_CONNECTING:
1064
            VIOSockRxPktHandleConnecting(pSocket, pPkt, bTxHasSpace);
1065
            break;
1066
        case VIOSOCK_STATE_CONNECTED:
1067
            VIOSockRxPktHandleConnected(pSocket, pPkt, bTxHasSpace);
1068
            break;
1069
        case VIOSOCK_STATE_CLOSING:
1070
            VIOSockRxPktHandleDisconnecting(pSocket, pPkt);
1071
            break;
1072
        case VIOSOCK_STATE_LISTEN:
1073
            VIOSockRxPktHandleListen(pSocket, pPkt);
1074
            break;
1075
        default:
1076
            TraceEvents(TRACE_LEVEL_WARNING, DBG_SOCKET,
1077
                "Invalid socket %d state for Rx packet %d\n", pSocket->SocketId, pPkt->Header.op);
1078
        }
1079

1080
        //reinsert handled packet
1081
        VIOSockRxPktInsertOrPostpone(pContext, pPkt);
1082
    };
1083

1084
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_HW_ACCESS, "<-- %s\n", __FUNCTION__);
1085
}
1086

1087
//////////////////////////////////////////////////////////////////////////
1088
_Requires_lock_not_held_(pSocket->RxLock)
1089
static
1090
NTSTATUS
1091
VIOSockReadForward(
1092
    IN PSOCKET_CONTEXT pSocket,
1093
    IN WDFREQUEST Request,
1094
    IN ULONG Flags
1095
)
1096
{
1097
    PVIOSOCK_RX_CONTEXT pRequest;
1098
    WDF_OBJECT_ATTRIBUTES attributes;
1099
    NTSTATUS status;
1100
    BOOLEAN bTimer = FALSE;
1101
    SIZE_T Length;
1102

1103
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_HW_ACCESS, "--> %s\n", __FUNCTION__);
1104

1105
    WDF_OBJECT_ATTRIBUTES_INIT_CONTEXT_TYPE(
1106
        &attributes,
1107
        VIOSOCK_RX_CONTEXT
1108
    );
1109
    status = WdfObjectAllocateContext(
1110
        Request,
1111
        &attributes,
1112
        &pRequest
1113
    );
1114
    if (!NT_SUCCESS(status))
1115
    {
1116
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ, "WdfObjectAllocateContext failed: 0x%x\n", status);
1117
        return status;
1118
    }
1119

1120
    status = WdfRequestRetrieveOutputBuffer(Request, 0, &pRequest->FreePtr, &Length);
1121
    if (!NT_SUCCESS(status))
1122
    {
1123
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ, "WdfRequestRetrieveOutputBuffer failed: 0x%x\n", status);
1124
        return status;
1125
    }
1126

1127

1128
    pRequest->BufferLen = pRequest->FreeBytes = (ULONG)Length;
1129
    pRequest->Flags = Flags;
1130

1131
    if (!VIOSockIsNonBlocking(pSocket) &&
1132
        pSocket->RecvTimeout != LONG_MAX)
1133
    {
1134
        pRequest->Timeout = WDF_ABS_TIMEOUT_IN_MS(pSocket->RecvTimeout);
1135
        pRequest->Counter = 0;
1136

1137
        WdfSpinLockAcquire(pSocket->RxLock);
1138
        VIOSockTimerStart(&pSocket->ReadTimer, pRequest->Timeout);
1139
        WdfSpinLockRelease(pSocket->RxLock);
1140

1141
        bTimer = TRUE;
1142
    }
1143

1144
    VIOSockEventClearBit(pSocket, FD_READ_BIT);
1145

1146
    status = WdfRequestForwardToIoQueue(Request, pSocket->ReadQueue);
1147
    if (!NT_SUCCESS(status))
1148
    {
1149

1150
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ, "WdfRequestForwardToIoQueue failed: 0x%x\n", status);
1151
        if (bTimer)
1152
        {
1153
            WdfSpinLockAcquire(pSocket->RxLock);
1154
            VIOSockTimerDeref(&pSocket->ReadTimer, TRUE);
1155
            WdfSpinLockRelease(pSocket->RxLock);
1156
        }
1157
    }
1158

1159
    return status;
1160
}
1161

1162
static
1163
VOID
1164
VIOSockRead(
1165
    IN WDFQUEUE Queue,
1166
    IN WDFREQUEST Request,
1167
    IN size_t Length
1168
)
1169
{
1170
    PSOCKET_CONTEXT pSocket = GetSocketContextFromRequest(Request);
1171
    NTSTATUS status;
1172

1173
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
1174
    //check Request
1175
    if (VIOSockIsFlag(pSocket, SOCK_CONTROL))
1176
    {
1177
        TraceEvents(TRACE_LEVEL_WARNING, DBG_READ, "Invalid socket %d for read\n", pSocket->SocketId);
1178
        WdfRequestComplete(Request, STATUS_NOT_SOCKET);
1179
        return;
1180
    }
1181

1182
    status = VIOSockReadForward(pSocket, Request, 0);
1183

1184
    if (!NT_SUCCESS(status))
1185
        WdfRequestComplete(Request, status);
1186
}
1187

1188
NTSTATUS
1189
VIOSockReadWithFlags(
1190
    IN WDFREQUEST Request
1191
)
1192
{
1193
    PSOCKET_CONTEXT pSocket = GetSocketContextFromRequest(Request);
1194
    NTSTATUS status;
1195
    PVIRTIO_VSOCK_READ_PARAMS pReadParams;
1196
    ULONG Flags = 0;
1197

1198
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
1199

1200
    //validate request
1201
    status = WdfRequestRetrieveInputBuffer(Request, sizeof(*pReadParams), &pReadParams, NULL);
1202
    if (!NT_SUCCESS(status))
1203
    {
1204
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ,
1205
            "WdfRequestRetrieveInputBuffer failed: 0x%x\n", status);
1206
    }
1207
    else if (pReadParams->Flags & ~(MSG_PEEK | MSG_WAITALL))
1208
    {
1209

1210
        TraceEvents(TRACE_LEVEL_WARNING, DBG_READ,
1211
            "Unsupported flags: 0x%x\n", pReadParams->Flags & ~(MSG_PEEK | MSG_WAITALL));
1212
        status = STATUS_NOT_SUPPORTED;
1213
    }
1214
    else if ((pReadParams->Flags & (MSG_PEEK | MSG_WAITALL)) == (MSG_PEEK | MSG_WAITALL))
1215
    {
1216
        TraceEvents(TRACE_LEVEL_WARNING, DBG_READ,
1217
            "Incompatible flags: 0x%x\n", MSG_PEEK | MSG_WAITALL);
1218
        status = STATUS_NOT_SUPPORTED;
1219
    }
1220
    else
1221
    {
1222
        Flags = pReadParams->Flags;
1223
    }
1224

1225
    if (NT_SUCCESS(status))
1226
    {
1227
        if (NT_SUCCESS(VIOSockReadForward(pSocket, Request, Flags)))
1228
            status = STATUS_PENDING;
1229
    }
1230

1231
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "<-- %s, status: 0x%08x\n", __FUNCTION__, status);
1232
    return status;
1233
}
1234

1235
NTSTATUS
1236
VIOSockReadQueueInit(
1237
    IN WDFDEVICE hDevice
1238
)
1239
{
1240
    PDEVICE_CONTEXT         pContext = GetDeviceContext(hDevice);
1241
    WDF_IO_QUEUE_CONFIG     queueConfig;
1242
    NTSTATUS                status;
1243
    WDF_OBJECT_ATTRIBUTES   attributes;
1244

1245
    PAGED_CODE();
1246

1247
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
1248

1249
    WDF_OBJECT_ATTRIBUTES_INIT(&attributes);
1250
    attributes.ParentObject = pContext->ThisDevice;
1251

1252
    status = WdfSpinLockCreate(
1253
        &attributes,
1254
        &pContext->RxLock
1255
    );
1256

1257
    if (!NT_SUCCESS(status))
1258
    {
1259
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ, "WdfSpinLockCreate failed: 0x%x\n", status);
1260
        return status;
1261
    }
1262

1263
    WDF_IO_QUEUE_CONFIG_INIT(&queueConfig,
1264
        WdfIoQueueDispatchParallel
1265
    );
1266

1267
    queueConfig.EvtIoRead = VIOSockRead;
1268
    queueConfig.AllowZeroLengthRequests = WdfFalse;
1269

1270
    //
1271
    // By default, Static Driver Verifier (SDV) displays a warning if it
1272
    // doesn't find the EvtIoStop callback on a power-managed queue.
1273
    // The 'assume' below causes SDV to suppress this warning.
1274
    //
1275
    // No need to handle EvtIoStop:
1276

1277
    __analysis_assume(queueConfig.EvtIoStop != 0);
1278
    status = WdfIoQueueCreate(hDevice,
1279
        &queueConfig,
1280
        WDF_NO_OBJECT_ATTRIBUTES,
1281
        &pContext->ReadQueue
1282
    );
1283
    __analysis_assume(queueConfig.EvtIoStop == 0);
1284

1285
    if (!NT_SUCCESS(status))
1286
    {
1287
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ,
1288
            "WdfIoQueueCreate failed (Read Queue): 0x%x\n", status);
1289
        return status;
1290
    }
1291

1292
    status = WdfDeviceConfigureRequestDispatching(hDevice,
1293
        pContext->ReadQueue,
1294
        WdfRequestTypeRead);
1295

1296
    if (!NT_SUCCESS(status))
1297
    {
1298
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ,
1299
            "WdfDeviceConfigureRequestDispatching failed (Read Queue): 0x%x\n", status);
1300
        return status;
1301
    }
1302

1303
    return STATUS_SUCCESS;
1304
}
1305

1306

1307
//only loopback buffers have request assigned
1308
#define VIOSockIsLoopbackCb(cb) ((cb)->Request != WDF_NO_HANDLE)
1309

1310
_Requires_lock_not_held_(pSocket->RxLock)
1311
BOOLEAN
1312
VIOSockReadDequeueCb(
1313
    IN PSOCKET_CONTEXT  pSocket
1314
)
1315
{
1316
    PDEVICE_CONTEXT pContext = GetDeviceContextFromSocket(pSocket);
1317
    WDFREQUEST      ReadRequest = WDF_NO_HANDLE;
1318
    NTSTATUS        status = STATUS_SUCCESS;
1319
    PVIOSOCK_RX_CB  pCurrentCb;
1320
    LIST_ENTRY      LoopbackList, *pCurrentItem;
1321
    ULONG           FreeSpace;
1322
    BOOLEAN         bSetBit, bStop = FALSE, bRequeue = FALSE, bRestart = TRUE, bAlwaysTrue = TRUE, bProcessRxPktList = FALSE;
1323
    PVIOSOCK_RX_CONTEXT pRequest = NULL;
1324
    LONGLONG llTimeout = 0;
1325

1326
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
1327

1328
    if (pSocket->RxProcessingThreadId == (LONG64)PsGetCurrentThreadId())
1329
    {
1330
        TraceEvents(TRACE_LEVEL_INFORMATION, DBG_READ, "Another instance of VIOSockReadDequeueCb already running, stop Cb dequeue\n");
1331
        return FALSE; //one running instance allowed
1332
    }
1333

1334
    InitializeListHead(&LoopbackList);
1335

1336
    status = WdfIoQueueRetrieveNextRequest(pSocket->ReadQueue, &ReadRequest);
1337
    if (!NT_SUCCESS(status))
1338
    {
1339
        TraceEvents(TRACE_LEVEL_INFORMATION, DBG_READ, "The read queue is empty, exiting from VIOSockReadDequeueCb\n");
1340
        return FALSE;
1341
    }
1342

1343
    pRequest = GetRequestRxContext(ReadRequest);
1344

1345
    WdfSpinLockAcquire(pSocket->RxLock);
1346

1347
    if (pRequest->Timeout)
1348
    {
1349
        llTimeout = VIOSockTimerPassed(&pSocket->ReadTimer);
1350
        VIOSockTimerDeref(&pSocket->ReadTimer, TRUE);
1351
        if (llTimeout < pRequest->Timeout)
1352
            llTimeout = pRequest->Timeout - llTimeout;
1353
        else
1354
            llTimeout = VIOSOCK_TIMER_TOLERANCE;
1355
    }
1356

1357
    if (!VIOSockRxHasData(pSocket))
1358
    {
1359
        if (VIOSockIsNonBlocking(pSocket) ||
1360
            VIOSockStateGet(pSocket) != VIOSOCK_STATE_CONNECTED)
1361
        {
1362
//            VIOSockEventClearBit(pSocket, FD_READ_BIT);
1363

1364
            //complete request
1365
            bStop = TRUE;
1366

1367
            status = VIOSockStateValidate(pSocket, FALSE);
1368

1369
            if (NT_SUCCESS(status))
1370
            {
1371
                if (VIOSockIsNonBlocking(pSocket))
1372
                {
1373
                    status = STATUS_CANT_WAIT;
1374
                }
1375
                else
1376
                {
1377
                    ASSERT(FALSE);
1378
                    bRequeue = TRUE;
1379
                }
1380
            }
1381
            else if (status == STATUS_REMOTE_DISCONNECT)
1382
                status = STATUS_SUCCESS; //return zero bytes on peer shutdown
1383
        }
1384
        else
1385
        {
1386
            // The socket is connected, however, it might
1387
            // receive a shutdown notification from the other end.
1388
            // In such a case, we need to complete the recv request with success,
1389
            // possibly reporting zero data received. If the shutdown is here,
1390
            // the request must not be requeued since that may cause it being
1391
            // in the queue forever.
1392
            status = VIOSockStateValidate(pSocket, FALSE);
1393
            if (!NT_SUCCESS(status))
1394
            {
1395
                bStop = TRUE;
1396
                if (status == STATUS_LOCAL_DISCONNECT ||
1397
                    status == STATUS_REMOTE_DISCONNECT)
1398
                    status = STATUS_SUCCESS;
1399
            }
1400
            else bRequeue = TRUE;
1401
        }
1402
    }
1403

1404
    if (bRequeue)
1405
    {
1406
        bStop = TRUE;
1407
        if (llTimeout && llTimeout <= VIOSOCK_TIMER_TOLERANCE)
1408
        {
1409
            status = STATUS_IO_TIMEOUT;
1410
        }
1411
        else
1412
        {
1413
            pSocket->RxProcessingThreadId = (LONG64)PsGetCurrentThreadId();
1414
            status = WdfRequestRequeue(ReadRequest);
1415
            pSocket->RxProcessingThreadId = INVALID_THREAD_ID;
1416
            if (NT_SUCCESS(status))
1417
            {
1418
                //continue timer
1419
                if (llTimeout)
1420
                    VIOSockTimerStart(&pSocket->ReadTimer, llTimeout);
1421
                ReadRequest = WDF_NO_HANDLE;
1422
                status = STATUS_SUCCESS;
1423
            }
1424
        }
1425
    }
1426

1427
    if (bStop)
1428
    {
1429
        WdfSpinLockRelease(pSocket->RxLock);
1430

1431
        if (ReadRequest != WDF_NO_HANDLE)
1432
        {
1433
            TraceEvents(TRACE_LEVEL_INFORMATION, DBG_READ, "Complete request without CB dequeue: 0x%08x\n", status);
1434
            WdfRequestCompleteWithInformation(ReadRequest, status, pRequest->BufferLen - pRequest->FreeBytes);
1435
        }
1436
        else
1437
        {
1438
            TraceEvents(TRACE_LEVEL_INFORMATION, DBG_READ, "No read request to complete\n");
1439
        }
1440
        return FALSE;
1441
    }
1442

1443
    //process chained buffer
1444
    for (pCurrentItem = pSocket->RxCbList.Flink;
1445
        pCurrentItem != &pSocket->RxCbList;
1446
        pCurrentItem = pCurrentItem->Flink)
1447
    {
1448
        //peek the first buffer
1449
        pCurrentCb = CONTAINING_RECORD(pCurrentItem, VIOSOCK_RX_CB, ListEntry);
1450

1451
        //can we copy the whole CB?
1452
        if (pRequest->FreeBytes >= pCurrentCb->BytesToRead)
1453
        {
1454
            memcpy(pRequest->FreePtr, pCurrentCb->ReadPtr, pCurrentCb->BytesToRead);
1455

1456
            //update request buffer data ptr
1457
            pRequest->FreePtr += pCurrentCb->BytesToRead;
1458
            pRequest->FreeBytes -= pCurrentCb->BytesToRead;
1459

1460
            if (!(pRequest->Flags & MSG_PEEK))
1461
            {
1462
                ULONG uBytesToRead = pCurrentCb->BytesToRead;
1463
                pCurrentCb->BytesToRead = 0;
1464
                pCurrentItem = pCurrentItem->Blink;
1465
                RemoveEntryList(&pCurrentCb->ListEntry);
1466

1467
                if (VIOSockIsLoopbackCb(pCurrentCb))
1468
                {
1469
                    status = WdfRequestUnmarkCancelable(pCurrentCb->Request);
1470

1471
                    if (NT_SUCCESS(status))
1472
                    {
1473
                        InsertTailList(&LoopbackList, &pCurrentCb->ListEntry); //complete loopback requests later
1474
                        VIOSockRxPktDec(pSocket, uBytesToRead);
1475
                    }
1476
                    else
1477
                    {
1478
                        ASSERT(status == STATUS_CANCELLED);
1479
                        TraceEvents(TRACE_LEVEL_WARNING, DBG_READ, "Loopback request is canceling: 0x%x\n", status);
1480
                        status = STATUS_SUCCESS;
1481
                        InitializeListHead(&pCurrentCb->ListEntry);//cancellation routine removes element from the list
1482
                    }
1483
                }
1484
                else
1485
                {
1486
                    VIOSockRxCbPushLocked(pContext, pCurrentCb);
1487
                    VIOSockRxPktDec(pSocket, uBytesToRead);
1488
                    bProcessRxPktList = TRUE;
1489
                }
1490
            }
1491
        }
1492
        else //request buffer is not big enough
1493
        {
1494
            memcpy(pRequest->FreePtr, pCurrentCb->ReadPtr, pRequest->FreeBytes);
1495

1496
            if (!(pRequest->Flags & MSG_PEEK))
1497
            {
1498
                //update current CB data ptr
1499
                pCurrentCb->ReadPtr += pRequest->FreeBytes;
1500
                pCurrentCb->BytesToRead -= pRequest->FreeBytes;
1501
                VIOSockRxPktDec(pSocket, pRequest->FreeBytes);
1502
            }
1503
            pRequest->FreeBytes = 0;
1504

1505
            break;
1506
        }
1507
    }
1508

1509
    if (bProcessRxPktList)
1510
    {
1511
        VIOSockRxPktListProcessLocked(pContext);
1512
    }
1513

1514
    if (!pRequest->FreeBytes || pRequest->Flags & MSG_PEEK)
1515
    {
1516
        TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "Should complete read request\n");
1517
    }
1518
    else if (pRequest->BufferLen == pRequest->FreeBytes || pRequest->Flags & MSG_WAITALL)
1519
    {
1520
        if (llTimeout && llTimeout <= VIOSOCK_TIMER_TOLERANCE)
1521
        {
1522
            status = STATUS_IO_TIMEOUT;
1523
        }
1524
        else
1525
        {
1526
            //requeue request
1527
            pSocket->RxProcessingThreadId = (LONG64)PsGetCurrentThreadId();
1528
            status = WdfRequestRequeue(ReadRequest);
1529
            pSocket->RxProcessingThreadId = INVALID_THREAD_ID;
1530
            if (NT_SUCCESS(status))
1531
            {
1532
                //continue timer
1533
                if (llTimeout)
1534
                    VIOSockTimerStart(&pSocket->ReadTimer, llTimeout);
1535
                ReadRequest = WDF_NO_HANDLE;
1536
                status = STATUS_SUCCESS;
1537
                bRestart = FALSE;
1538
            }
1539
        }
1540
    }
1541

1542
    bSetBit = (ReadRequest != WDF_NO_HANDLE && VIOSockRxHasData(pSocket));
1543
    FreeSpace = pSocket->buf_alloc - (pSocket->fwd_cnt - pSocket->last_fwd_cnt);
1544

1545
    WdfSpinLockRelease(pSocket->RxLock);
1546

1547
    if (bSetBit)
1548
        VIOSockEventSetBit(pSocket, FD_READ_BIT, STATUS_SUCCESS);
1549
    else
1550
        VIOSockEventClearBit(pSocket, FD_READ_BIT);
1551

1552
    if (FreeSpace < VIRTIO_VSOCK_MAX_PKT_BUF_SIZE)
1553
        VIOSockSendCreditUpdate(pSocket);
1554

1555
    if (ReadRequest != WDF_NO_HANDLE)
1556
    {
1557
        ASSERT(pSocket->PendedRequest == WDF_NO_HANDLE);
1558

1559
        WdfRequestCompleteWithInformation(ReadRequest, status, pRequest->BufferLen - pRequest->FreeBytes);
1560
    }
1561

1562
    //Static Driver Verifier(SDV) tracks only one request, and when SDV unwinds the loop,
1563
    //it treats completion of the second request as another completion of the first request.
1564
    //The 'assume' below causes SDV to skip loop analysis.
1565

1566
    __analysis_assume(bAlwaysTrue == FALSE);
1567

1568
    //complete loopback requests (succeed and canceled)
1569
    if (bAlwaysTrue)
1570
    {
1571
        while (!IsListEmpty(&LoopbackList))
1572
        {
1573
            pCurrentCb = CONTAINING_RECORD(RemoveHeadList(&LoopbackList), VIOSOCK_RX_CB, ListEntry);
1574

1575
            //NOTE! SDV thinks we are completing INVALID request marked as cancelable,
1576
            //but request is appeared in this list only if WdfRequestMarkCancelableEx failed
1577
            WdfRequestCompleteWithInformation(pCurrentCb->Request,
1578
                pCurrentCb->DataLen ? STATUS_SUCCESS : STATUS_CANCELLED,
1579
                pCurrentCb->DataLen);
1580
        }
1581
    }
1582

1583
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "<-- %s\n", __FUNCTION__);
1584

1585
    return bRestart;
1586
}
1587

1588
_Requires_lock_not_held_(pSocket->RxLock)
1589
VOID
1590
VIOSockReadCleanupCb(
1591
    IN PSOCKET_CONTEXT pSocket
1592
)
1593
{
1594
    PDEVICE_CONTEXT pContext = GetDeviceContextFromSocket(pSocket);
1595
    PVIOSOCK_RX_CB  pCurrentCb;
1596
    LIST_ENTRY      LoopbackList;
1597
    BOOLEAN         bAlwaysTrue = TRUE;
1598

1599
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
1600

1601
    InitializeListHead(&LoopbackList);
1602

1603
    WdfSpinLockAcquire(pSocket->RxLock);
1604

1605
    //process chained buffer
1606
    while (!IsListEmpty(&pSocket->RxCbList))
1607
    {
1608
        //peek the first buffer
1609
        pCurrentCb = CONTAINING_RECORD(RemoveHeadList(&pSocket->RxCbList), VIOSOCK_RX_CB, ListEntry);
1610

1611
        if (VIOSockIsLoopbackCb(pCurrentCb))
1612
        {
1613
            NTSTATUS status = WdfRequestUnmarkCancelable(pCurrentCb->Request);
1614
            if (!NT_SUCCESS(status))
1615
            {
1616
                ASSERT(status == STATUS_CANCELLED);
1617
                TraceEvents(TRACE_LEVEL_WARNING, DBG_READ, "Loopback request canceled\n");
1618
                InitializeListHead(&pCurrentCb->ListEntry);//cancellation routine removes element from the list
1619
            }
1620
            else
1621
                InsertTailList(&LoopbackList, &pCurrentCb->ListEntry); //complete loopback requests later
1622
        }
1623
        else
1624
            VIOSockRxCbPushLocked(pContext, pCurrentCb);
1625
    }
1626

1627
    WdfSpinLockRelease(pSocket->RxLock);
1628

1629
    VIOSockRxPktListProcessLocked(pContext);
1630

1631
    //Static Driver Verifier(SDV) tracks only one request, and when SDV unwinds the loop,
1632
    //it treats completion of the second request as another completion of the first request.
1633
    //The 'assume' below causes SDV to skip loop analysis.
1634
    __analysis_assume(bAlwaysTrue == FALSE);
1635

1636
    //complete loopback
1637
    if (bAlwaysTrue)
1638
    {
1639
        while (!IsListEmpty(&LoopbackList))
1640
        {
1641
            pCurrentCb = CONTAINING_RECORD(RemoveHeadList(&LoopbackList), VIOSOCK_RX_CB, ListEntry);
1642
            WdfRequestComplete(pCurrentCb->Request, STATUS_CANCELLED);
1643
        }
1644
    }
1645

1646
}
1647

1648
static
1649
VOID
1650
VIOSockReadSocketState(
1651
    WDFQUEUE Queue,
1652
    WDFCONTEXT Context
1653
)
1654
{
1655
    VIOSockReadProcessDequeueCb((PSOCKET_CONTEXT)Context);
1656
}
1657

1658
VOID
1659
VIOSockReadIoCanceledOnQueue(
1660
    IN WDFQUEUE Queue,
1661
    IN WDFREQUEST Request
1662
)
1663
{
1664
    PVIOSOCK_RX_CONTEXT pRequest = GetRequestRxContext(Request);
1665

1666
    ASSERT(pRequest);
1667
    if (pRequest->BufferLen != pRequest->FreeBytes)
1668
        WdfRequestCompleteWithInformation(Request, STATUS_SUCCESS, pRequest->BufferLen - pRequest->FreeBytes);
1669
    else
1670
        WdfRequestComplete(Request, STATUS_CANCELLED);
1671
}
1672

1673
NTSTATUS
1674
VIOSockReadSocketQueueInit(
1675
    IN PSOCKET_CONTEXT pSocket
1676
)
1677
{
1678
    WDFDEVICE               hDevice = WdfFileObjectGetDevice(pSocket->ThisSocket);
1679
    PDEVICE_CONTEXT         pContext = GetDeviceContext(hDevice);
1680
    WDF_OBJECT_ATTRIBUTES   queueAttributes;
1681
    WDF_IO_QUEUE_CONFIG     queueConfig;
1682
    NTSTATUS status;
1683

1684
    PAGED_CODE();
1685

1686
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
1687

1688
    WDF_IO_QUEUE_CONFIG_INIT(&queueConfig,
1689
        WdfIoQueueDispatchManual
1690
    );
1691

1692
    queueConfig.EvtIoCanceledOnQueue = VIOSockReadIoCanceledOnQueue;
1693
    queueConfig.AllowZeroLengthRequests = WdfFalse;
1694

1695
    WDF_OBJECT_ATTRIBUTES_INIT(&queueAttributes);
1696
    queueAttributes.ParentObject = pSocket->ThisSocket;
1697

1698
    //
1699
    // By default, Static Driver Verifier (SDV) displays a warning if it
1700
    // doesn't find the EvtIoStop callback on a power-managed queue.
1701
    // The 'assume' below causes SDV to suppress this warning.
1702
    //
1703
    // No need to handle EvtIoStop:
1704

1705
    __analysis_assume(queueConfig.EvtIoStop != 0);
1706
    status = WdfIoQueueCreate(hDevice,
1707
        &queueConfig,
1708
        &queueAttributes,
1709
        &pSocket->ReadQueue
1710
    );
1711
    __analysis_assume(queueConfig.EvtIoStop == 0);
1712

1713
    if (!NT_SUCCESS(status))
1714
    {
1715
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ,
1716
            "WdfIoQueueCreate failed (Socket Read Queue): 0x%x\n", status);
1717
        return status;
1718
    }
1719

1720
    status = WdfIoQueueReadyNotify(pSocket->ReadQueue, VIOSockReadSocketState, pSocket);
1721
    if (!NT_SUCCESS(status))
1722
    {
1723
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ,
1724
            "WdfIoQueueReadyNotify failed (Socket Read Queue): 0x%x\n", status);
1725
        return status;
1726
    }
1727

1728
    status = VIOSockTimerCreate(&pSocket->ReadTimer, pSocket->ThisSocket, VIOSockReadTimerFunc);
1729
    if (!NT_SUCCESS(status))
1730
    {
1731
        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ,
1732
            "VIOSockTimerCreate failed (Socket Read Queue): 0x%x\n", status);
1733
        return status;
1734
    }
1735

1736
    return STATUS_SUCCESS;
1737
}
1738

1739
//////////////////////////////////////////////////////////////////////////
1740
VOID
1741
VIOSockReadTimerFunc(
1742
    WDFTIMER Timer
1743
)
1744
{
1745
    static LONG lCounter;
1746
    PSOCKET_CONTEXT pSocket = GetSocketContext(WdfTimerGetParentObject(Timer));
1747
    LONGLONG Timeout = LONGLONG_MAX;
1748
    WDFREQUEST PrevTagRequest = WDF_NO_HANDLE, TagRequest = WDF_NO_HANDLE, Request;
1749
    NTSTATUS status;
1750
    LIST_ENTRY CompletionList;
1751
    PVIOSOCK_RX_CONTEXT pRequest;
1752

1753
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "--> %s\n", __FUNCTION__);
1754

1755
    InterlockedIncrement(&lCounter);
1756
    InitializeListHead(&CompletionList);
1757

1758
    do
1759
    {
1760
        status = WdfIoQueueFindRequest(pSocket->ReadQueue, PrevTagRequest, WDF_NO_HANDLE, NULL, &TagRequest);
1761

1762
        if (PrevTagRequest != WDF_NO_HANDLE)
1763
        {
1764
            WdfObjectDereference(PrevTagRequest);
1765
        }
1766

1767
        if (NT_SUCCESS(status))
1768
        {
1769
            pRequest = GetRequestRxContext(TagRequest);
1770

1771
            WdfSpinLockAcquire(pSocket->RxLock);
1772

1773
            if (pRequest->Timeout && pRequest->Counter < lCounter)
1774
            {
1775
                if (pRequest->Timeout <= pSocket->ReadTimer.Timeout + VIOSOCK_TIMER_TOLERANCE)
1776
                {
1777
                    status = WdfIoQueueRetrieveFoundRequest(pSocket->ReadQueue, TagRequest, &Request);
1778

1779
                    WdfObjectDereference(TagRequest);
1780

1781
                    if (status == STATUS_NOT_FOUND)
1782
                    {
1783
                        TagRequest = PrevTagRequest = WDF_NO_HANDLE;
1784
                        status = STATUS_SUCCESS;
1785
                    }
1786
                    else if (!NT_SUCCESS(status))
1787
                    {
1788
                        TraceEvents(TRACE_LEVEL_ERROR, DBG_READ, "WdfIoQueueRetrieveFoundRequest failed: 0x%08x\n", status);
1789
                        WdfSpinLockRelease(pSocket->RxLock);
1790
                        break;
1791
                    }
1792
                    else
1793
                    {
1794
                        TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "Complete expired queued Rx request %p\n", Request);
1795

1796
                        VIOSockTimerDeref(&pSocket->ReadTimer, FALSE);
1797
                        InsertTailList(&CompletionList, &pRequest->ListEntry);
1798
                        pRequest->ThisRequest = Request;
1799
                    }
1800
                }
1801
                else
1802
                {
1803
                    pRequest->Counter = lCounter;
1804
                    pRequest->Timeout -= pSocket->ReadTimer.Timeout;
1805

1806
                    if (pRequest->Timeout < Timeout)
1807
                        Timeout = pRequest->Timeout;
1808

1809
                    PrevTagRequest = TagRequest;
1810
                }
1811
            }
1812

1813
            WdfSpinLockRelease(pSocket->RxLock);
1814
        }
1815
        else if (status == STATUS_NO_MORE_ENTRIES)
1816
        {
1817
            break;
1818
        }
1819
        else if (status == STATUS_NOT_FOUND)
1820
        {
1821
            TagRequest = PrevTagRequest = WDF_NO_HANDLE;
1822
            status = STATUS_SUCCESS;
1823
        }
1824
    } while (NT_SUCCESS(status));
1825

1826
    if (Timeout != LONGLONG_MAX)
1827
    {
1828
        WdfSpinLockAcquire(pSocket->RxLock);
1829
        VIOSockTimerSet(&pSocket->ReadTimer, Timeout);
1830
        WdfSpinLockRelease(pSocket->RxLock);
1831
    }
1832

1833
    while (!IsListEmpty(&CompletionList))
1834
    {
1835
        pRequest = CONTAINING_RECORD(RemoveHeadList(&CompletionList), VIOSOCK_RX_CONTEXT, ListEntry);
1836
        WdfRequestCompleteWithInformation(pRequest->ThisRequest, STATUS_IO_TIMEOUT, pRequest->BufferLen - pRequest->FreeBytes);
1837
    }
1838

1839
    TraceEvents(TRACE_LEVEL_VERBOSE, DBG_READ, "<-- %s, status: 0x%08x, counter: %u\n", __FUNCTION__, status, lCounter);
1840
}
1841