onnxruntime

1
#include "testPch.h"
2

3
#include "imageTestHelper.h"
4
#include "robuffer.h"
5
#include "winrt/Windows.Storage.h"
6
#include "winrt/Windows.Storage.Streams.h"
7

8
#include <d3dx12.h>
9
#include <MemoryBuffer.h>
10
#include <wil\Resource.h>
11

12
#define FENCE_SIGNAL_VALUE 1
13

14
using namespace winrt;
15
using namespace winml;
16
using namespace wfc;
17
using namespace wm;
18
using namespace wgi;
19

20
namespace ImageTestHelper {
21
BitmapPixelFormat GetPixelFormat(const std::wstring& inputPixelFormat) {
22
  // Return corresponding BitmapPixelFormat according to input string
23
  if (L"Bgra8" == inputPixelFormat || L"Bgr8" == inputPixelFormat) {
24
    return BitmapPixelFormat::Bgra8;
25
  } else if (L"Rgba8" == inputPixelFormat || L"Rgb8" == inputPixelFormat) {
26
    return BitmapPixelFormat::Rgba8;
27
  } else if (L"Gray8" == inputPixelFormat) {
28
    return BitmapPixelFormat::Gray8;
29
  } else {
30
    throw std::invalid_argument("Unsupported pixelFormat");
31
  }
32
}
33

34
TensorFloat LoadInputImageFromCPU(SoftwareBitmap softwareBitmap, const std::wstring& modelPixelFormat) {
35
  softwareBitmap = SoftwareBitmap::Convert(softwareBitmap, BitmapPixelFormat::Bgra8);
36
  BYTE* pData = nullptr;
37
  UINT32 size = 0;
38
  wgi::BitmapBuffer spBitmapBuffer(softwareBitmap.LockBuffer(wgi::BitmapBufferAccessMode::Read));
39
  wf::IMemoryBufferReference reference = spBitmapBuffer.CreateReference();
40
  auto spByteAccess = reference.as<::Windows::Foundation::IMemoryBufferByteAccess>();
41
  spByteAccess->GetBuffer(&pData, &size);
42
  uint32_t height = softwareBitmap.PixelHeight();
43
  uint32_t width = softwareBitmap.PixelWidth();
44

45
  // TODO: Need modification for Gray8
46
  std::vector<int64_t> shape = {1, 3, height, width};
47
  float* pCPUTensor;
48
  uint32_t uCapacity;
49
  TensorFloat tf = TensorFloat::Create(shape);
50
  com_ptr<ITensorNative> itn = tf.as<ITensorNative>();
51
  itn->GetBuffer(reinterpret_cast<BYTE**>(&pCPUTensor), &uCapacity);
52
  if (BitmapPixelFormat::Bgra8 == GetPixelFormat(modelPixelFormat)) {
53
    // loop condition is i < size - 2 to avoid potential for extending past the memory buffer
54
    for (UINT32 i = 0; i < size - 2; i += 4) {
55
      UINT32 pixelInd = i / 4;
56
      pCPUTensor[pixelInd] = (float)pData[i];
57
      pCPUTensor[(height * width) + pixelInd] = (float)pData[i + 1];
58
      pCPUTensor[(height * width * 2) + pixelInd] = (float)pData[i + 2];
59
    }
60
  } else if (BitmapPixelFormat::Rgba8 == GetPixelFormat(modelPixelFormat)) {
61
    for (UINT32 i = 0; i < size - 2; i += 4) {
62
      UINT32 pixelInd = i / 4;
63
      pCPUTensor[pixelInd] = (float)pData[i + 2];
64
      pCPUTensor[(height * width) + pixelInd] = (float)pData[i + 1];
65
      pCPUTensor[(height * width * 2) + pixelInd] = (float)pData[i];
66
    }
67
  }
68
  // else if()
69
  // TODO: for Gray8
70
  else {
71
    std::cerr << "Unsupported pixelFormat";
72
  }
73
  return tf;
74
}
75

76
TensorFloat LoadInputImageFromGPU(SoftwareBitmap softwareBitmap, const std::wstring& modelPixelFormat) {
77
  softwareBitmap = SoftwareBitmap::Convert(softwareBitmap, BitmapPixelFormat::Bgra8);
78
  BYTE* pData = nullptr;
79
  UINT32 size = 0;
80
  BitmapBuffer spBitmapBuffer(softwareBitmap.LockBuffer(wgi::BitmapBufferAccessMode::Read));
81
  wf::IMemoryBufferReference reference = spBitmapBuffer.CreateReference();
82
  com_ptr<::Windows::Foundation::IMemoryBufferByteAccess> spByteAccess =
83
    reference.as<::Windows::Foundation::IMemoryBufferByteAccess>();
84
  spByteAccess->GetBuffer(&pData, &size);
85

86
  std::vector<int64_t> shape = {1, 3, softwareBitmap.PixelHeight(), softwareBitmap.PixelWidth()};
87
  float* pCPUTensor;
88
  uint32_t uCapacity;
89

90
  // CPU tensor initialization
91
  TensorFloat tf = TensorFloat::Create(shape);
92
  com_ptr<ITensorNative> itn = tf.as<ITensorNative>();
93
  itn->GetBuffer(reinterpret_cast<BYTE**>(&pCPUTensor), &uCapacity);
94

95
  uint32_t height = softwareBitmap.PixelHeight();
96
  uint32_t width = softwareBitmap.PixelWidth();
97
  if (BitmapPixelFormat::Bgra8 == GetPixelFormat(modelPixelFormat)) {
98
    // loop condition is i < size - 2 to avoid potential for extending past the memory buffer
99
    for (UINT32 i = 0; i < size - 2; i += 4) {
100
      UINT32 pixelInd = i / 4;
101
      pCPUTensor[pixelInd] = (float)pData[i];
102
      pCPUTensor[(height * width) + pixelInd] = (float)pData[i + 1];
103
      pCPUTensor[(height * width * 2) + pixelInd] = (float)pData[i + 2];
104
    }
105
  } else if (BitmapPixelFormat::Rgba8 == GetPixelFormat(modelPixelFormat)) {
106
    for (UINT32 i = 0; i < size - 2; i += 4) {
107
      UINT32 pixelInd = i / 4;
108
      pCPUTensor[pixelInd] = (float)pData[i + 2];
109
      pCPUTensor[(height * width) + pixelInd] = (float)pData[i + 1];
110
      pCPUTensor[(height * width * 2) + pixelInd] = (float)pData[i];
111
    }
112
  }
113
  // else if()
114
  // TODO: for Gray8
115
  else {
116
    std::cerr << "unsupported pixelFormat";
117
  }
118

119
  // create the d3d device.
120
  com_ptr<ID3D12Device> pD3D12Device = nullptr;
121
  WINML_EXPECT_NO_THROW(D3D12CreateDevice(
122
    nullptr, D3D_FEATURE_LEVEL::D3D_FEATURE_LEVEL_11_0, __uuidof(ID3D12Device), reinterpret_cast<void**>(&pD3D12Device)
123
  ));
124

125
  // create the command queue.
126
  com_ptr<ID3D12CommandQueue> dxQueue = nullptr;
127
  D3D12_COMMAND_QUEUE_DESC commandQueueDesc = {};
128
  commandQueueDesc.Type = D3D12_COMMAND_LIST_TYPE_DIRECT;
129
  pD3D12Device->CreateCommandQueue(&commandQueueDesc, __uuidof(ID3D12CommandQueue), reinterpret_cast<void**>(&dxQueue));
130
  com_ptr<ILearningModelDeviceFactoryNative> devicefactory =
131
    get_activation_factory<LearningModelDevice, ILearningModelDeviceFactoryNative>();
132
  com_ptr<ITensorStaticsNative> tensorfactory = get_activation_factory<TensorFloat, ITensorStaticsNative>();
133
  com_ptr<::IUnknown> spUnk;
134
  devicefactory->CreateFromD3D12CommandQueue(dxQueue.get(), spUnk.put());
135

136
  // Create ID3D12GraphicsCommandList and Allocator
137
  D3D12_COMMAND_LIST_TYPE queuetype = dxQueue->GetDesc().Type;
138
  com_ptr<ID3D12CommandAllocator> alloctor;
139
  com_ptr<ID3D12GraphicsCommandList> cmdList;
140

141
  pD3D12Device->CreateCommandAllocator(queuetype, winrt::guid_of<ID3D12CommandAllocator>(), alloctor.put_void());
142

143
  pD3D12Device->CreateCommandList(
144
    0, queuetype, alloctor.get(), nullptr, winrt::guid_of<ID3D12CommandList>(), cmdList.put_void()
145
  );
146

147
  // Create Committed Resource
148
  // 3 is number of channels we use. R G B without alpha.
149
  UINT64 bufferbytesize = 3 * sizeof(float) * softwareBitmap.PixelWidth() * softwareBitmap.PixelHeight();
150
  D3D12_HEAP_PROPERTIES heapProperties = {
151
    D3D12_HEAP_TYPE_DEFAULT, D3D12_CPU_PAGE_PROPERTY_UNKNOWN, D3D12_MEMORY_POOL_UNKNOWN, 0, 0
152
  };
153
  D3D12_RESOURCE_DESC resourceDesc = {
154
    D3D12_RESOURCE_DIMENSION_BUFFER,
155
    0,
156
    bufferbytesize,
157
    1,
158
    1,
159
    1,
160
    DXGI_FORMAT_UNKNOWN,
161
    {1, 0},
162
    D3D12_TEXTURE_LAYOUT_ROW_MAJOR,
163
    D3D12_RESOURCE_FLAG_ALLOW_UNORDERED_ACCESS
164
  };
165

166
  com_ptr<ID3D12Resource> pGPUResource = nullptr;
167
  com_ptr<ID3D12Resource> imageUploadHeap;
168
  pD3D12Device->CreateCommittedResource(
169
    &heapProperties,
170
    D3D12_HEAP_FLAG_NONE,
171
    &resourceDesc,
172
    D3D12_RESOURCE_STATE_COMMON,
173
    nullptr,
174
    __uuidof(ID3D12Resource),
175
    pGPUResource.put_void()
176
  );
177

178
  // Create the GPU upload buffer.
179
  auto heap_properties = CD3DX12_HEAP_PROPERTIES(D3D12_HEAP_TYPE_UPLOAD);
180
  auto buffer_desc = CD3DX12_RESOURCE_DESC::Buffer(bufferbytesize);
181
  WINML_EXPECT_NO_THROW(pD3D12Device->CreateCommittedResource(
182
    &heap_properties,
183
    D3D12_HEAP_FLAG_NONE,
184
    &buffer_desc,
185
    D3D12_RESOURCE_STATE_GENERIC_READ,
186
    nullptr,
187
    __uuidof(ID3D12Resource),
188
    imageUploadHeap.put_void()
189
  ));
190

191
  // Copy from Cpu to GPU
192
  D3D12_SUBRESOURCE_DATA CPUData = {};
193
  CPUData.pData = reinterpret_cast<BYTE*>(pCPUTensor);
194
  CPUData.RowPitch = static_cast<LONG_PTR>(bufferbytesize);
195
  CPUData.SlicePitch = static_cast<LONG_PTR>(bufferbytesize);
196
  UpdateSubresources(cmdList.get(), pGPUResource.get(), imageUploadHeap.get(), 0, 0, 1, &CPUData);
197

198
  // Close the command list and execute it to begin the initial GPU setup.
199
  WINML_EXPECT_NO_THROW(cmdList->Close());
200
  ID3D12CommandList* ppCommandLists[] = {cmdList.get()};
201
  dxQueue->ExecuteCommandLists(_countof(ppCommandLists), ppCommandLists);
202

203
  //Create Event
204
  HANDLE directEvent = CreateEvent(nullptr, FALSE, FALSE, nullptr);
205
  wil::unique_event hDirectEvent(directEvent);
206

207
  //Create Fence
208
  ::Microsoft::WRL::ComPtr<ID3D12Fence> spDirectFence = nullptr;
209
  WINML_EXPECT_HRESULT_SUCCEEDED(
210
    pD3D12Device->CreateFence(0, D3D12_FENCE_FLAG_NONE, IID_PPV_ARGS(spDirectFence.ReleaseAndGetAddressOf()))
211
  );
212
  //Adds fence to queue
213
  WINML_EXPECT_HRESULT_SUCCEEDED(dxQueue->Signal(spDirectFence.Get(), FENCE_SIGNAL_VALUE));
214
  WINML_EXPECT_HRESULT_SUCCEEDED(spDirectFence->SetEventOnCompletion(FENCE_SIGNAL_VALUE, hDirectEvent.get()));
215

216
  //Wait for signal
217
  DWORD retVal = WaitForSingleObject(hDirectEvent.get(), INFINITE);
218
  if (retVal != WAIT_OBJECT_0) {
219
    WINML_EXPECT_HRESULT_SUCCEEDED(E_UNEXPECTED);
220
  }
221

222
  // GPU tensorize
223
  com_ptr<::IUnknown> spUnkTensor;
224
  TensorFloat input1imagetensor(nullptr);
225
  int64_t shapes[4] = {1, 3, softwareBitmap.PixelWidth(), softwareBitmap.PixelHeight()};
226
  tensorfactory->CreateFromD3D12Resource(pGPUResource.get(), shapes, 4, spUnkTensor.put());
227
  spUnkTensor.try_as(input1imagetensor);
228

229
  return input1imagetensor;
230
}
231

232
bool VerifyHelper(VideoFrame actual, VideoFrame expected) {
233
  // Verify two input ImageFeatureValues are identified.
234
  auto softwareBitmapActual = actual.SoftwareBitmap();
235
  auto softwareBitmapExpected = expected.SoftwareBitmap();
236
  WINML_EXPECT_TRUE(softwareBitmapActual.PixelHeight() == softwareBitmapExpected.PixelHeight());
237
  WINML_EXPECT_TRUE(softwareBitmapActual.PixelWidth() == softwareBitmapExpected.PixelWidth());
238
  WINML_EXPECT_TRUE(softwareBitmapActual.BitmapPixelFormat() == softwareBitmapExpected.BitmapPixelFormat());
239

240
  uint32_t size = 4 * softwareBitmapActual.PixelHeight() * softwareBitmapActual.PixelWidth();
241

242
  ws::Streams::Buffer actualOutputBuffer(size);
243
  ws::Streams::Buffer expectedOutputBuffer(size);
244

245
  softwareBitmapActual.CopyToBuffer(actualOutputBuffer);
246
  softwareBitmapExpected.CopyToBuffer(expectedOutputBuffer);
247

248
  byte* actualBytes;
249
  actualOutputBuffer.try_as<::Windows::Storage::Streams::IBufferByteAccess>()->Buffer(&actualBytes);
250
  byte* expectedBytes;
251
  expectedOutputBuffer.try_as<::Windows::Storage::Streams::IBufferByteAccess>()->Buffer(&expectedBytes);
252

253
  byte* pActualByte = actualBytes;
254
  byte* pExpectedByte = expectedBytes;
255

256
  // hard code, might need to be modified later.
257
  const float cMaxErrorRate = 0.4f;
258
  int8_t epsilon = 20;
259

260
  // Even given two same ImageFeatureValues, the comparison cannot exactly match.
261
  // So we use error rate.
262
  UINT errors = 0;
263
  for (uint32_t i = 0; i < size; i++, pActualByte++, pExpectedByte++) {
264
    // Only the check the first three channels, which are (B, G, R)
265
    if ((i + 1) % 4 == 0)
266
      continue;
267
    auto diff = (*pActualByte - *pExpectedByte);
268
    if (diff > epsilon) {
269
      errors++;
270
    }
271
  }
272
  std::cerr << "total errors is " << errors << "/" << size << ", errors rate is " << (float)errors / size << std::endl;
273
  return (float)errors / size < cMaxErrorRate;
274
}
275
}  // namespace ImageTestHelper
276