onnxruntime

1
// Copyright (c) Microsoft Corporation. All rights reserved.
2
// Licensed under the MIT License.
3

4
#include "lib/Api.Image/pch.h"
5
#include "inc/ImageConversionHelpers.h"
6

7
#include <winrt/Windows.Graphics.DirectX.Direct3D11.h>
8

9
using namespace Microsoft::WRL;
10
using namespace Windows::Graphics::DirectX::Direct3D11;
11

12
static LUID GetLUIDFromDirect3DSurface(const wgdx::Direct3D11::IDirect3DSurface& surface) {
13
  ComPtr<ID3D11Device> spDx11Device;
14
  ComPtr<IDirect3DDxgiInterfaceAccess> spDxgiInterfaceAccess;
15
  ComPtr<ID3D11Texture2D> spDx11Texture2D;
16
  ComPtr<IDXGIDevice> spDXGIDevice;
17
  ComPtr<IDXGIAdapter> spDXGIAdapter;
18
  DXGI_ADAPTER_DESC adapterDesc = {0};
19

20
  spDxgiInterfaceAccess = surface.as<IDirect3DDxgiInterfaceAccess>().get();
21
  WINML_THROW_IF_FAILED(spDxgiInterfaceAccess->GetInterface(IID_PPV_ARGS(&spDx11Texture2D)));
22
  spDx11Texture2D->GetDevice(&spDx11Device);
23
  WINML_THROW_IF_FAILED(spDx11Device->QueryInterface(IID_PPV_ARGS(&spDXGIDevice)));
24
  WINML_THROW_IF_FAILED(spDXGIDevice->GetAdapter(&spDXGIAdapter));
25
  WINML_THROW_IF_FAILED(spDXGIAdapter->GetDesc(&adapterDesc));
26

27
  return adapterDesc.AdapterLuid;
28
}
29

30
static HRESULT GetVideoFrameInfo(
31
  _In_ const wm::IVideoFrame& inputVideoFrame,
32
  _Out_ DWORD& format,
33
  _Out_ int& width,
34
  _Out_ int& height,
35
  _Out_ LUID& luid
36
) {
37
  wgdx::Direct3D11::IDirect3DSurface spInputSurface = inputVideoFrame.Direct3DSurface();
38
  if (spInputSurface != nullptr) {
39
    wgdx::Direct3D11::Direct3DSurfaceDescription description;
40
    description = spInputSurface.Description();
41
    format = (DWORD)description.Format;
42
    width = description.Width;
43
    height = description.Height;
44
    luid = GetLUIDFromDirect3DSurface(spInputSurface);
45
  } else {
46
    wgi::SoftwareBitmap spInputSoftwareBitmap = inputVideoFrame.SoftwareBitmap();
47
    if (spInputSoftwareBitmap != nullptr) {
48
      format = (DWORD)spInputSoftwareBitmap.BitmapPixelFormat();
49
      height = spInputSoftwareBitmap.PixelHeight();
50
      width = spInputSoftwareBitmap.PixelWidth();
51
      luid.HighPart = luid.LowPart = 0;
52
    } else {
53
      return E_INVALIDARG;
54
    }
55
  }
56
  return S_OK;
57
}
58

59
void _winmli::ConvertVideoFrameToVideoFrame(
60
  _In_ const wm::IVideoFrame& inputVideoFrame,
61
  _In_ const wgi::BitmapBounds& inputBounds,
62
  _In_ UINT32 outputWidth,
63
  _In_ UINT32 outputHeight,
64
  _Inout_ wm::VideoFrame& pOutputVideoFrame
65
) {
66
  wgi::BitmapBounds outputBounds = {0, 0, outputWidth, outputHeight};
67

68
  wgi::SoftwareBitmap spInputSoftwareBitmap = inputVideoFrame.SoftwareBitmap();
69
  wgdx::Direct3D11::IDirect3DSurface spInputDirect3DSurface = inputVideoFrame.Direct3DSurface();
70

71
  // only one of softwarebitmap or direct3Dsurface should be non-null
72
  if ((spInputSoftwareBitmap == nullptr && spInputDirect3DSurface == nullptr) ||
73
      (spInputSoftwareBitmap != nullptr && spInputDirect3DSurface != nullptr)) {
74
    WINML_THROW_HR(E_INVALIDARG);
75
  }
76

77
  auto pInputVideoFrame2 = inputVideoFrame.as<wm::IVideoFrame2>();
78
  pInputVideoFrame2.CopyToAsync(pOutputVideoFrame, inputBounds, outputBounds).get();
79
}
80

81
bool _winmli::SoftwareBitmapFormatSupported(const wgi::SoftwareBitmap& softwareBitmap) {
82
  assert(softwareBitmap != nullptr);
83

84
  switch (softwareBitmap.BitmapPixelFormat()) {
85
    case wgi::BitmapPixelFormat::Bgra8:
86
    case wgi::BitmapPixelFormat::Rgba8:
87
    case wgi::BitmapPixelFormat::Gray8:
88
      return true;
89
  }
90

91
  return false;
92
}
93

94
bool _winmli::DirectXPixelFormatSupported(wgdx::DirectXPixelFormat format) {
95
  switch (format) {
96
    case wgdx::DirectXPixelFormat::B8G8R8X8UIntNormalized:
97
    case wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized:
98
    case wgdx::DirectXPixelFormat::R8G8B8A8UIntNormalized:
99
    case wgdx::DirectXPixelFormat::R8UIntNormalized:
100
      return true;
101
  }
102

103
  return false;
104
}
105

106
bool _winmli::FormatSupportedForUAV(_In_ ID3D12Device1* device, _In_ DXGI_FORMAT format) {
107
  assert(device != nullptr);
108

109
  D3D12_FEATURE_DATA_FORMAT_SUPPORT formatSupport = {format};
110
  HRESULT hr = device->CheckFeatureSupport(D3D12_FEATURE_FORMAT_SUPPORT, &formatSupport, sizeof(formatSupport));
111

112
  return SUCCEEDED(hr) && (formatSupport.Support1 & D3D12_FORMAT_SUPPORT1_TYPED_UNORDERED_ACCESS_VIEW);
113
}
114

115
// This helper method uses the input parameters do determine if a conversion is necessary
116
// A conversion is not necessary if
117
// 1. input bounds cover the entire input bitmap/surface (else we are cropping)
118
// 2. desired output size is equal to input size (else we are resizing)
119
// 3. (mapping softwarebitmap to softwarebitmap) OR (mapping from d3dsurface to d3dsurface AND the two surfaces are on the same device)
120
// 4. the input is already in the desired format (BGRA8/B8G8R8X8UIntNormalized)
121
bool _winmli::NeedsVideoFrameConversion(
122
  _In_ const wm::IVideoFrame& inputVideoFrame,
123
  _In_ LUID outputLuid,
124
  _In_ const wgi::BitmapBounds& inputBounds,
125
  _In_ UINT32 outputWidth,
126
  _In_ UINT32 outputHeight
127
) {
128
  bool bNeedConversion = false;
129
  HRESULT hr = S_OK;
130

131
  DWORD format = 0;
132
  int width = 0, height = 0;
133
  LUID luid;
134

135
  if (FAILED((hr = GetVideoFrameInfo(inputVideoFrame, format, width, height, luid)))) {
136
    bNeedConversion = true;
137
  } else if (((int)inputBounds.Width != outputWidth) || (inputBounds.X != 0) ||
138
             ((int)inputBounds.Height != outputHeight) || (inputBounds.Y != 0) ||
139
             (inputVideoFrame == nullptr))  // Check crop
140
  {
141
    bNeedConversion = true;
142
  } else if (luid.HighPart != outputLuid.HighPart || luid.LowPart != outputLuid.LowPart) {
143
    bNeedConversion = true;
144
  } else if (static_cast<uint32_t>(width) != outputWidth || static_cast<uint32_t>(height) != outputHeight) {
145
    bNeedConversion = true;
146
  } else if (outputLuid.HighPart != 0 || outputLuid.LowPart != 0) {
147
    if (format != (DWORD)wgdx::DirectXPixelFormat::B8G8R8X8UIntNormalized) {
148
      bNeedConversion = true;
149
    }
150
  } else {
151
    if (format != (DWORD)wgi::BitmapPixelFormat::Bgra8) {
152
      bNeedConversion = true;
153
    }
154
  }
155

156
  TraceLoggingWrite(
157
    winml_trace_logging_provider,
158
    "InputVideoFrame",
159
    TraceLoggingKeyword(WINML_PROVIDER_KEYWORD_DEFAULT),
160
    TraceLoggingBool(bNeedConversion, "Convert"),
161
    TraceLoggingHexInt32(hr, "HRESULT"),
162
    TraceLoggingInt32(width, "iWidth"),
163
    TraceLoggingInt32(outputWidth, "oWidth"),
164
    TraceLoggingInt32(height, "iHeight"),
165
    TraceLoggingInt32(outputWidth, "oHeight"),
166
    TraceLoggingHexInt64(*((ULONGLONG*)&luid), "iLuid"),
167
    TraceLoggingHexInt64(*((ULONGLONG*)&outputLuid), "oLuid"),
168
    TraceLoggingHexInt32(format, "iFormat"),
169
    TraceLoggingInt32(inputBounds.X, "rX"),
170
    TraceLoggingInt32(inputBounds.Y, "rY"),
171
    TraceLoggingInt32(inputBounds.Width, "rW"),
172
    TraceLoggingInt32(inputBounds.Height, "rH")
173
  );
174

175
  return bNeedConversion;
176
}
177

178
_winml::ImageTensorChannelType _winmli::GetChannelTypeFromSoftwareBitmap(const wgi::SoftwareBitmap& softwareBitmap) {
179
  assert(softwareBitmap != nullptr);
180

181
  switch (softwareBitmap.BitmapPixelFormat()) {
182
    case wgi::BitmapPixelFormat::Bgra8:
183
      return _winml::kImageTensorChannelTypeBGR8;
184
    case wgi::BitmapPixelFormat::Rgba8:
185
      return _winml::kImageTensorChannelTypeRGB8;
186
    case wgi::BitmapPixelFormat::Gray8:
187
      return _winml::kImageTensorChannelTypeGRAY8;
188
  }
189

190
  WINML_THROW_HR(E_INVALIDARG);
191
}
192

193
wgi::BitmapPixelFormat _winmli::GetBitmapPixelFormatFromChannelType(_winml::ImageTensorChannelType channelType) {
194
  switch (channelType) {
195
    case _winml::kImageTensorChannelTypeBGR8:
196
      return wgi::BitmapPixelFormat::Bgra8;
197
    case _winml::kImageTensorChannelTypeRGB8:
198
      return wgi::BitmapPixelFormat::Rgba8;
199
    case _winml::kImageTensorChannelTypeGRAY8:
200
      return wgi::BitmapPixelFormat::Gray8;
201
  }
202

203
  WINML_THROW_HR(E_INVALIDARG);
204
}
205

206
_winml::ImageTensorChannelType _winmli::GetChannelTypeFromDirect3DSurface(
207
  const wgdx::Direct3D11::IDirect3DSurface& direct3DSurface
208
) {
209
  assert(direct3DSurface != nullptr);
210

211
  switch (direct3DSurface.Description().Format) {
212
    case wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized:
213
    case wgdx::DirectXPixelFormat::B8G8R8X8UIntNormalized:
214
      return _winml::kImageTensorChannelTypeBGR8;
215

216
    case wgdx::DirectXPixelFormat::R8G8B8A8UIntNormalized:
217
      return _winml::kImageTensorChannelTypeRGB8;
218

219
    case wgdx::DirectXPixelFormat::R8UIntNormalized:
220
      return _winml::kImageTensorChannelTypeGRAY8;
221
  }
222

223
  WINML_THROW_HR(E_INVALIDARG);
224
}
225

226
wgdx::DirectXPixelFormat _winmli::GetDirectXPixelFormatFromDXGIFormat(DXGI_FORMAT dxgiFormat) {
227
  switch (dxgiFormat) {
228
    case DXGI_FORMAT_B8G8R8A8_UNORM:
229
      return wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized;
230
    case DXGI_FORMAT_B8G8R8X8_UNORM:
231
      return wgdx::DirectXPixelFormat::B8G8R8X8UIntNormalized;
232
    case DXGI_FORMAT_R8G8B8A8_UNORM:
233
      return wgdx::DirectXPixelFormat::R8G8B8A8UIntNormalized;
234
    case DXGI_FORMAT_R8_UNORM:
235
      return wgdx::DirectXPixelFormat::R8UIntNormalized;
236
  }
237

238
  WINML_THROW_HR(E_INVALIDARG);
239
}
240

241
DXGI_FORMAT _winmli::GetDXGIFormatFromDirectXPixelFormat(_In_ wgdx::DirectXPixelFormat directXPixelFormat) {
242
  switch (directXPixelFormat) {
243
    case wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized:
244
      return DXGI_FORMAT_B8G8R8A8_UNORM;
245
    case wgdx::DirectXPixelFormat::B8G8R8X8UIntNormalized:
246
      return DXGI_FORMAT_B8G8R8X8_UNORM;
247
    case wgdx::DirectXPixelFormat::R8G8B8A8UIntNormalized:
248
      return DXGI_FORMAT_R8G8B8A8_UNORM;
249
    case wgdx::DirectXPixelFormat::R8UIntNormalized:
250
      return DXGI_FORMAT_R8_UNORM;
251
  }
252

253
  WINML_THROW_HR(E_INVALIDARG);
254
}
255

256
wgdx::DirectXPixelFormat _winmli::GetDirectXPixelFormatFromChannelType(_In_ _winml::ImageTensorChannelType channelType
257
) {
258
  switch (channelType) {
259
    case _winml::kImageTensorChannelTypeBGR8:
260
      return wgdx::DirectXPixelFormat::B8G8R8A8UIntNormalized;
261
    case _winml::kImageTensorChannelTypeRGB8:
262
      return wgdx::DirectXPixelFormat::R8G8B8A8UIntNormalized;
263
    case _winml::kImageTensorChannelTypeGRAY8:
264
      return wgdx::DirectXPixelFormat::R8UIntNormalized;
265
  }
266

267
  WINML_THROW_HR(E_INVALIDARG);
268
}
269

270
wgdx::Direct3D11::IDirect3DDevice _winmli::GetDeviceFromDirect3DSurface(
271
  const wgdx::Direct3D11::IDirect3DSurface& d3dSurface
272
) {
273
  assert(d3dSurface != nullptr);
274

275
  ComPtr<ID3D11Texture2D> spDx11Texture2D;
276
  ComPtr<IDirect3DDxgiInterfaceAccess> spDxgiInterfaceAccess = d3dSurface.as<IDirect3DDxgiInterfaceAccess>().get();
277
  WINML_THROW_IF_FAILED(spDxgiInterfaceAccess->GetInterface(IID_PPV_ARGS(&spDx11Texture2D)));
278

279
  ComPtr<ID3D11Device> spDx11Device;
280
  spDx11Texture2D->GetDevice(&spDx11Device);
281

282
  ComPtr<IDXGIDevice> spDXGIDevice;
283
  WINML_THROW_IF_FAILED(spDx11Device->QueryInterface(IID_PPV_ARGS(&spDXGIDevice)));
284

285
  ComPtr<::IInspectable> spInspectable;
286
  WINML_THROW_IF_FAILED(CreateDirect3D11DeviceFromDXGIDevice(spDXGIDevice.Get(), &spInspectable));
287

288
  wgdx::Direct3D11::IDirect3DDevice d3dDevice;
289
  WINML_THROW_IF_FAILED(spInspectable->QueryInterface(
290
    winrt::guid_of<wgdx::Direct3D11::IDirect3DDevice>(), reinterpret_cast<void**>(winrt::put_abi(d3dDevice))
291
  ));
292

293
  return d3dDevice;
294
}
295

296
bool _winmli::TexturesHaveSameDevice(_In_ ID3D11Texture2D* pTexture1, _In_ ID3D11Texture2D* pTexture2) {
297
  if (pTexture1 && pTexture2) {
298
    ComPtr<ID3D11Device> spDevice1;
299
    pTexture1->GetDevice(&spDevice1);
300

301
    ComPtr<ID3D11Device> spDevice2;
302
    pTexture2->GetDevice(&spDevice2);
303

304
    return spDevice1.Get() == spDevice2.Get();
305
  }
306

307
  return false;
308
}
309

310
bool _winmli::TextureIsOnDevice(_In_ ID3D11Texture2D* pTexture, _In_ ID3D11Device* pDevice) {
311
  if (pTexture && pDevice) {
312
    ComPtr<ID3D11Device> spDevice1;
313
    pTexture->GetDevice(&spDevice1);
314

315
    return spDevice1.Get() == pDevice;
316
  }
317

318
  return false;
319
}
320

321
ComPtr<ID3D11Texture2D> _winmli::GetTextureFromDirect3DSurface(const wgdx::Direct3D11::IDirect3DSurface& d3dSurface) {
322
  auto spDxgiInterfaceAccess = d3dSurface.as<IDirect3DDxgiInterfaceAccess>();
323
  ComPtr<ID3D11Texture2D> d3d11Texture;
324
  WINML_THROW_IF_FAILED(spDxgiInterfaceAccess->GetInterface(IID_PPV_ARGS(&d3d11Texture)));
325

326
  return d3d11Texture;
327
}
328

329
bool _winmli::VideoFramesHaveSameDimensions(const wm::IVideoFrame& videoFrame1, const wm::IVideoFrame& videoFrame2) {
330
  if (videoFrame1 && videoFrame2) {
331
    auto desc1 = videoFrame1.Direct3DSurface().Description();
332
    auto desc2 = videoFrame2.Direct3DSurface().Description();
333

334
    return desc1.Width == desc2.Width && desc1.Height == desc2.Height;
335
  }
336

337
  return false;
338
}
339

340
bool _winmli::VideoFramesHaveSameDevice(const wm::IVideoFrame& videoFrame1, const wm::IVideoFrame& videoFrame2) {
341
  if (videoFrame1 && videoFrame2) {
342
    ComPtr<ID3D11Texture2D> spTexture1 = _winmli::GetTextureFromDirect3DSurface(videoFrame1.Direct3DSurface());
343
    ComPtr<ID3D11Texture2D> spTexture2 = _winmli::GetTextureFromDirect3DSurface(videoFrame2.Direct3DSurface());
344

345
    ComPtr<ID3D11Device> spDevice1, spDevice2;
346
    spTexture1->GetDevice(&spDevice1);
347
    spTexture2->GetDevice(&spDevice2);
348

349
    return spDevice1.Get() == spDevice2.Get();
350
  }
351

352
  return false;
353
}
354