onnxruntime

1
// Copyright (c) Microsoft Corporation. All rights reserved.
2
// Licensed under the MIT License.
3

4
#include "testPch.h"
5

6
#include "APITest.h"
7
#include "LearningModelBindingAPITest.h"
8
#include "SqueezeNetValidator.h"
9

10
#include <sstream>
11

12
using namespace winrt;
13
using namespace winml;
14
using namespace wfc;
15
using namespace wgi;
16
using namespace wm;
17
using namespace ws;
18

19
static void LearningModelBindingAPITestsClassSetup() {
20
  init_apartment();
21
#ifdef BUILD_INBOX
22
  winrt_activation_handler = WINRT_RoGetActivationFactory;
23
#endif
24
}
25

26
static void CpuSqueezeNet() {
27
  std::string cpuInstance("CPU");
28
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
29
    cpuInstance,
30
    LearningModelDeviceKind::Cpu,
31
    /*dataTolerance*/ 0.00001f,
32
    false
33
  ));
34
}
35

36
static void CpuSqueezeNetEmptyOutputs() {
37
  std::string cpuInstance("CPU");
38
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
39
                          cpuInstance,
40
                          LearningModelDeviceKind::Cpu,
41
                          /*dataTolerance*/ 0.00001f,
42
                          false,
43
                          OutputBindingStrategy::Empty
44
  ););
45
}
46

47
static void CpuSqueezeNetUnboundOutputs() {
48
  std::string cpuInstance("CPU");
49
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
50
                          cpuInstance,
51
                          LearningModelDeviceKind::Cpu,
52
                          /*dataTolerance*/ 0.00001f,
53
                          false,
54
                          OutputBindingStrategy::Unbound
55
  ););
56
}
57

58
static void CpuSqueezeNetBindInputTensorAsInspectable() {
59
  std::string cpuInstance("CPU");
60
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
61
                          cpuInstance,
62
                          LearningModelDeviceKind::Cpu,
63
                          /*dataTolerance*/ 0.00001f,
64
                          false,
65
                          OutputBindingStrategy::Bound /* empty outputs */,
66
                          true /* bind inputs as inspectables */
67
  ););
68
}
69

70
static void CastMapInt64() {
71
  WINML_EXPECT_NO_THROW(LearningModel::LoadFromFilePath(FileHelpers::GetModulePath() + L"castmap-int64.onnx"));
72
  // TODO: Check Descriptor
73
}
74

75
static void DictionaryVectorizerMapInt64() {
76
  LearningModel learningModel = nullptr;
77
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"dictvectorizer-int64.onnx", learningModel));
78

79
  auto inputDescriptor = learningModel.InputFeatures().First().Current();
80
  WINML_EXPECT_TRUE(inputDescriptor.Kind() == LearningModelFeatureKind::Map);
81
  auto mapDescriptor = inputDescriptor.as<MapFeatureDescriptor>();
82
  WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::Int64);
83
  WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor);
84
  auto tensorDescriptor = mapDescriptor.ValueDescriptor().as<TensorFeatureDescriptor>();
85
  // empty size means tensor of scalar value
86
  WINML_EXPECT_TRUE(tensorDescriptor.Shape().Size() == 0);
87
  WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float);
88

89
  LearningModelSession modelSession(learningModel);
90
  LearningModelBinding binding(modelSession);
91
  std::unordered_map<int64_t, float> map;
92
  map[1] = 1.f;
93
  map[10] = 10.f;
94
  map[3] = 3.f;
95

96
  auto mapInputName = inputDescriptor.Name();
97

98
  // Bind as IMap
99
  auto abiMap = winrt::single_threaded_map(std::move(map));
100
  binding.Bind(mapInputName, abiMap);
101
  auto mapInputInspectable = abiMap.as<wf::IInspectable>();
102
  auto first = binding.First();
103
  WINML_EXPECT_TRUE(first.Current().Key() == mapInputName);
104
  WINML_EXPECT_TRUE(first.Current().Value() == mapInputInspectable);
105
  WINML_EXPECT_TRUE(binding.Lookup(mapInputName) == mapInputInspectable);
106

107
  // Bind as IMapView
108
  auto mapView = abiMap.GetView();
109
  binding.Bind(mapInputName, mapView);
110
  mapInputInspectable = mapView.as<wf::IInspectable>();
111
  first = binding.First();
112
  WINML_EXPECT_TRUE(first.Current().Key() == mapInputName);
113
  WINML_EXPECT_TRUE(first.Current().Value() == mapView);
114
  WINML_EXPECT_TRUE(binding.Lookup(mapInputName) == mapView);
115
}
116

117
static void DictionaryVectorizerMapString() {
118
  LearningModel learningModel = nullptr;
119
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"dictvectorizer-string.onnx", learningModel));
120

121
  auto inputDescriptor = learningModel.InputFeatures().First().Current();
122
  WINML_EXPECT_TRUE(inputDescriptor.Kind() == LearningModelFeatureKind::Map);
123

124
  auto mapDescriptor = inputDescriptor.as<MapFeatureDescriptor>();
125
  WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::String);
126
  WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor);
127

128
  auto tensorDescriptor = mapDescriptor.ValueDescriptor().as<TensorFeatureDescriptor>();
129
  // empty size means tensor of scalar value
130
  WINML_EXPECT_TRUE(tensorDescriptor.Shape().Size() == 0);
131
  WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float);
132

133
  LearningModelSession modelSession(learningModel);
134
  LearningModelBinding binding(modelSession);
135
  std::unordered_map<winrt::hstring, float> map;
136
  map[L"1"] = 1.f;
137
  map[L"10"] = 10.f;
138
  map[L"2"] = 2.f;
139

140
  auto mapInputName = inputDescriptor.Name();
141
  auto abiMap = winrt::single_threaded_map(std::move(map));
142
  binding.Bind(mapInputName, abiMap);
143

144
  auto mapInputInspectable = abiMap.as<wf::IInspectable>();
145
  auto first = binding.First();
146
  WINML_EXPECT_TRUE(first.Current().Key() == mapInputName);
147
  WINML_EXPECT_TRUE(first.Current().Value() == mapInputInspectable);
148
  WINML_EXPECT_TRUE(binding.Lookup(mapInputName) == mapInputInspectable);
149

150
  modelSession.Evaluate(binding, L"");
151
}
152

153
static void RunZipMapInt64(winml::LearningModel model, OutputBindingStrategy bindingStrategy) {
154
  auto outputFeatures = model.OutputFeatures();
155
  auto outputDescriptor = outputFeatures.First().Current();
156
  WINML_EXPECT_TRUE(outputDescriptor.Kind() == LearningModelFeatureKind::Sequence);
157

158
  auto seqDescriptor = outputDescriptor.as<SequenceFeatureDescriptor>();
159
  auto mapDescriptor = seqDescriptor.ElementDescriptor().as<MapFeatureDescriptor>();
160
  WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::Int64);
161

162
  WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor);
163
  auto tensorDescriptor = mapDescriptor.ValueDescriptor().as<TensorFeatureDescriptor>();
164
  WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float);
165

166
  LearningModelSession session(model);
167
  LearningModelBinding binding(session);
168

169
  std::vector<float> inputs = {0.5f, 0.25f, 0.125f};
170
  std::vector<int64_t> shape = {1, 3};
171

172
  // Bind inputs
173
  auto inputTensor = TensorFloat::CreateFromArray(shape, winrt::array_view<const float>(std::move(inputs)));
174
  binding.Bind(winrt::hstring(L"X"), inputTensor);
175

176
  typedef IMap<int64_t, float> ABIMap;
177
  typedef IVector<ABIMap> ABISequeneceOfMap;
178

179
  ABISequeneceOfMap abiOutput = nullptr;
180
  // Bind outputs
181
  if (bindingStrategy == OutputBindingStrategy::Bound) {
182
    abiOutput = winrt::single_threaded_vector<ABIMap>();
183
    binding.Bind(winrt::hstring(L"Y"), abiOutput);
184
  }
185

186
  // Evaluate
187
  auto result = session.Evaluate(binding, L"0").Outputs();
188

189
  if (bindingStrategy == OutputBindingStrategy::Bound) {
190
    // from output binding
191
    const auto& out1 = abiOutput.GetAt(0);
192
    const auto& out2 = result.Lookup(L"Y").as<IVectorView<ABIMap>>().GetAt(0);
193
    WINML_LOG_COMMENT((std::ostringstream() << "size: " << out1.Size()).str());
194
    // check outputs
195
    auto iter1 = out1.First();
196
    auto iter2 = out2.First();
197
    for (uint32_t i = 0, size = (uint32_t)inputs.size(); i < size; ++i) {
198
      WINML_EXPECT_TRUE(iter1.HasCurrent());
199
      WINML_EXPECT_TRUE(iter2.HasCurrent());
200
      const auto& pair1 = iter1.Current();
201
      const auto& pair2 = iter2.Current();
202
      WINML_LOG_COMMENT((std::ostringstream() << "key: " << pair1.Key() << ", value: " << pair2.Value()).str());
203
      WINML_EXPECT_TRUE(pair1.Key() == i && pair2.Key() == i);
204
      WINML_EXPECT_TRUE(pair1.Value() == inputs[i] && pair2.Value() == inputs[i]);
205
      iter1.MoveNext();
206
      iter2.MoveNext();
207
    }
208
    WINML_EXPECT_TRUE(!iter1.HasCurrent());
209
    WINML_EXPECT_TRUE(!iter2.HasCurrent());
210
  } else {
211
    abiOutput = result.Lookup(L"Y").as<ABISequeneceOfMap>();
212
    WINML_EXPECT_TRUE(abiOutput.Size() == 1);
213
    ABIMap map = abiOutput.GetAt(0);
214
    WINML_EXPECT_TRUE(map.Size() == 3);
215
    WINML_EXPECT_TRUE(map.Lookup(0) == 0.5);
216
    WINML_EXPECT_TRUE(map.Lookup(1) == .25);
217
    WINML_EXPECT_TRUE(map.Lookup(2) == .125);
218
  }
219
}
220

221
static void ZipMapInt64() {
222
  LearningModel learningModel = nullptr;
223
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-int64.onnx", learningModel));
224
  RunZipMapInt64(learningModel, OutputBindingStrategy::Bound);
225
}
226

227
static void ZipMapInt64Unbound() {
228
  LearningModel learningModel = nullptr;
229
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-int64.onnx", learningModel));
230
  RunZipMapInt64(learningModel, OutputBindingStrategy::Unbound);
231
}
232

233
static void ZipMapString() {
234
  // output constraint: "seq(map(string, float))" or "seq(map(int64, float))"
235
  LearningModel learningModel = nullptr;
236
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-string.onnx", learningModel));
237
  auto outputs = learningModel.OutputFeatures();
238
  auto outputDescriptor = outputs.First().Current();
239
  WINML_EXPECT_TRUE(outputDescriptor.Kind() == LearningModelFeatureKind::Sequence);
240
  auto mapDescriptor = outputDescriptor.as<SequenceFeatureDescriptor>().ElementDescriptor().as<MapFeatureDescriptor>();
241
  WINML_EXPECT_TRUE(mapDescriptor.KeyKind() == TensorKind::String);
242
  WINML_EXPECT_TRUE(mapDescriptor.ValueDescriptor().Kind() == LearningModelFeatureKind::Tensor);
243
  auto tensorDescriptor = mapDescriptor.ValueDescriptor().as<TensorFeatureDescriptor>();
244
  WINML_EXPECT_TRUE(tensorDescriptor.TensorKind() == TensorKind::Float);
245

246
  LearningModelSession session(learningModel);
247
  LearningModelBinding binding(session);
248

249
  std::vector<float> inputs = {0.5f, 0.25f, 0.125f};
250
  std::vector<int64_t> shape = {1, 3};
251
  std::vector<winrt::hstring> labels = {L"cat", L"dog", L"lion"};
252
  std::map<winrt::hstring, float> mapData = {
253
    { L"cat", 0.0f},
254
    { L"dog", 0.0f},
255
    {L"lion", 0.0f}
256
  };
257
  typedef IMap<winrt::hstring, float> ABIMap;
258
  ABIMap abiMap = winrt::single_threaded_map<winrt::hstring, float>(std::move(mapData));
259
  std::vector<ABIMap> seqOutput = {abiMap};
260
  IVector<ABIMap> ABIOutput = winrt::single_threaded_vector<ABIMap>(std::move(seqOutput));
261

262
  TensorFloat inputTensor = TensorFloat::CreateFromArray(shape, winrt::array_view<const float>(std::move(inputs)));
263
  binding.Bind(winrt::hstring(L"X"), inputTensor);
264
  binding.Bind(winrt::hstring(L"Y"), ABIOutput);
265
  auto result = session.Evaluate(binding, L"0").Outputs();
266
  // from output binding
267
  const auto& out1 = ABIOutput.GetAt(0);
268
  const auto& out2 = result.Lookup(L"Y").as<IVectorView<ABIMap>>().GetAt(0);
269
  WINML_LOG_COMMENT((std::ostringstream() << "size: " << out1.Size()).str());
270
  // single key,value pair for each map
271
  auto iter1 = out1.First();
272
  auto iter2 = out2.First();
273
  for (uint32_t i = 0, size = (uint32_t)inputs.size(); i < size; ++i) {
274
    WINML_EXPECT_TRUE(iter2.HasCurrent());
275
    const auto& pair1 = iter1.Current();
276
    const auto& pair2 = iter2.Current();
277
    WINML_LOG_COMMENT((std::ostringstream() << "key: " << pair1.Key().c_str() << ", value " << pair2.Value()).str());
278
    WINML_EXPECT_TRUE(std::wstring(pair1.Key().c_str()).compare(labels[i]) == 0);
279
    WINML_EXPECT_TRUE(std::wstring(pair2.Key().c_str()).compare(labels[i]) == 0);
280
    WINML_EXPECT_TRUE(pair1.Value() == inputs[i] && pair2.Value() == inputs[i]);
281
    iter1.MoveNext();
282
    iter2.MoveNext();
283
  }
284
  WINML_EXPECT_TRUE(!iter1.HasCurrent());
285
  WINML_EXPECT_TRUE(!iter2.HasCurrent());
286
}
287

288
static void GpuSqueezeNet() {
289
  std::string gpuInstance("GPU");
290
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
291
                          gpuInstance,
292
                          LearningModelDeviceKind::DirectX,
293
                          /*dataTolerance*/ 0.00001f
294
  ););
295
}
296

297
static void GpuSqueezeNetEmptyOutputs() {
298
  std::string gpuInstance("GPU");
299
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
300
                          gpuInstance,
301
                          LearningModelDeviceKind::DirectX,
302
                          /*dataTolerance*/ 0.00001f,
303
                          false,
304
                          OutputBindingStrategy::Empty
305
  ););
306
}
307

308
static void GpuSqueezeNetUnboundOutputs() {
309
  std::string gpuInstance("GPU");
310
  WINML_EXPECT_NO_THROW(WinML::Engine::Test::ModelValidator::SqueezeNet(
311
                          gpuInstance,
312
                          LearningModelDeviceKind::DirectX,
313
                          /*dataTolerance*/ 0.00001f,
314
                          false,
315
                          OutputBindingStrategy::Unbound
316
  ););
317
}
318

319
// Validates that when the input image is the same as the model expects, the binding step is executed correctly.
320
static void ImageBindingDimensions() {
321
  LearningModelBinding learningModelBinding = nullptr;
322
  LearningModel learningModel = nullptr;
323
  LearningModelSession learningModelSession = nullptr;
324
  LearningModelDevice leraningModelDevice = nullptr;
325
  std::wstring filePath = FileHelpers::GetModulePath() + L"model.onnx";
326
  // load a model with expected input size: 224 x 224
327
  WINML_EXPECT_NO_THROW(leraningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default));
328
  WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(filePath));
329
  WINML_EXPECT_TRUE(learningModel != nullptr);
330
  WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, leraningModelDevice));
331
  WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession));
332

333
  // Create input images and execute bind
334
  // Test Case 1: both width and height are larger than model expects
335
  VideoFrame inputImage1(BitmapPixelFormat::Rgba8, 1000, 1000);
336
  ImageFeatureValue inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage1);
337
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor));
338

339
  // Test Case 2: only height is larger, while width is smaller
340
  VideoFrame inputImage2(BitmapPixelFormat::Rgba8, 20, 1000);
341
  inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage2);
342
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor));
343

344
  // Test Case 3: only width is larger, while height is smaller
345
  VideoFrame inputImage3(BitmapPixelFormat::Rgba8, 1000, 20);
346
  inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage3);
347
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor));
348

349
  // Test Case 4: both width and height are smaller than model expects
350
  VideoFrame inputImage4(BitmapPixelFormat::Rgba8, 20, 20);
351
  inputTensor = ImageFeatureValue::CreateFromVideoFrame(inputImage4);
352
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"data_0", inputTensor));
353
}
354

355
static void VerifyInvalidBindExceptions() {
356
  LearningModel learningModel = nullptr;
357
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"zipmap-int64.onnx", learningModel));
358

359
  LearningModelSession session(learningModel);
360
  LearningModelBinding binding(session);
361

362
  std::vector<float> inputs = {0.5f, 0.25f, 0.125f};
363
  std::vector<int64_t> shape = {1, 3};
364

365
  auto matchException = [](const winrt::hresult_error& e, HRESULT hr) -> bool { return e.code() == hr; };
366

367
  auto ensureWinmlSizeMismatch = std::bind(matchException, std::placeholders::_1, WINML_ERR_SIZE_MISMATCH);
368
  auto ensureWinmlInvalidBinding = std::bind(matchException, std::placeholders::_1, WINML_ERR_INVALID_BINDING);
369

370
  /*
371
    Verify tensor bindings throw correct bind exceptions
372
  */
373

374
  // Bind invalid image as tensorfloat input
375
  auto image = FileHelpers::LoadImageFeatureValue(L"227x227.png");
376
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", image), winrt::hresult_error, ensureWinmlSizeMismatch);
377

378
  // Bind invalid map as tensorfloat input
379
  std::unordered_map<float, float> map;
380
  auto abiMap = winrt::single_threaded_map(std::move(map));
381
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", abiMap), winrt::hresult_error, ensureWinmlInvalidBinding);
382

383
  // Bind invalid sequence as tensorfloat input
384
  std::vector<uint32_t> sequence;
385
  auto abiSequence = winrt::single_threaded_vector(std::move(sequence));
386
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding);
387

388
  // Bind invalid tensor size as tensorfloat input
389
  auto tensorBoolean = TensorBoolean::Create();
390
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding);
391

392
  // Bind invalid tensor shape as tensorfloat input
393
  auto tensorInvalidShape = TensorFloat::Create(std::vector<int64_t>{2, 3, 4});
394
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"X", tensorInvalidShape), winrt::hresult_error, ensureWinmlInvalidBinding);
395

396
  /*
397
    Verify sequence bindings throw correct bind exceptions
398
  */
399

400
  // Bind invalid image as sequence<map<int, float> output
401
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", image), winrt::hresult_error, ensureWinmlInvalidBinding);
402

403
  // Bind invalid map as sequence<map<int, float> output
404
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", abiMap), winrt::hresult_error, ensureWinmlInvalidBinding);
405

406
  // Bind invalid sequence<int> as sequence<map<int, float> output
407
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding);
408

409
  // Bind invalid tensor as sequence<map<int, float> output
410
  WINML_EXPECT_THROW_SPECIFIC(binding.Bind(L"Y", tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding);
411

412
  /*
413
    Verify map bindings throw correct bind exceptions
414
  */
415
  WINML_EXPECT_NO_THROW(APITest::LoadModel(L"dictvectorizer-int64.onnx", learningModel));
416

417
  LearningModelSession mapSession(learningModel);
418
  LearningModelBinding mapBinding(mapSession);
419

420
  auto inputName = learningModel.InputFeatures().First().Current().Name();
421

422
  // Bind invalid image as image input
423
  auto smallImage = FileHelpers::LoadImageFeatureValue(L"100x100.png");
424
  WINML_EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, smallImage), winrt::hresult_error, ensureWinmlInvalidBinding);
425

426
  // Bind invalid map as image input
427
  WINML_EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, abiMap), winrt::hresult_error, ensureWinmlInvalidBinding);
428

429
  // Bind invalid sequence as image input
430
  WINML_EXPECT_THROW_SPECIFIC(mapBinding.Bind(inputName, abiSequence), winrt::hresult_error, ensureWinmlInvalidBinding);
431

432
  // Bind invalid tensor type as image input
433
  WINML_EXPECT_THROW_SPECIFIC(
434
    mapBinding.Bind(inputName, tensorBoolean), winrt::hresult_error, ensureWinmlInvalidBinding
435
  );
436
}
437

438
// Verify that it throws an error when binding an invalid name.
439
static void BindInvalidInputName() {
440
  LearningModel learningModel = nullptr;
441
  LearningModelBinding learningModelBinding = nullptr;
442
  LearningModelDevice learningModelDevice = nullptr;
443
  LearningModelSession learningModelSession = nullptr;
444
  std::wstring modelPath = FileHelpers::GetModulePath() + L"Add_ImageNet1920.onnx";
445
  WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(modelPath));
446
  WINML_EXPECT_TRUE(learningModel != nullptr);
447
  WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default));
448
  WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, learningModelDevice));
449
  WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession));
450

451
  VideoFrame iuputImage(BitmapPixelFormat::Rgba8, 1920, 1080);
452
  ImageFeatureValue inputTensor = ImageFeatureValue::CreateFromVideoFrame(iuputImage);
453

454
  auto first = learningModel.InputFeatures().First();
455
  std::wstring testInvalidName = L"0";
456

457
  // Verify that testInvalidName is not in model's InputFeatures
458
  while (first.HasCurrent()) {
459
    WINML_EXPECT_NOT_EQUAL(testInvalidName, first.Current().Name());
460
    first.MoveNext();
461
  }
462

463
  // Bind inputTensor to a valid input name
464
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"input_39:0", inputTensor));
465

466
  // Bind inputTensor to an invalid input name
467
  WINML_EXPECT_THROW_SPECIFIC(
468
    learningModelBinding.Bind(testInvalidName, inputTensor),
469
    winrt::hresult_error,
470
    [](const winrt::hresult_error& e) -> bool { return e.code() == WINML_ERR_INVALID_BINDING; }
471
  );
472
}
473

474
static void VerifyOutputAfterEvaluateAsyncCalledTwice() {
475
  LearningModel learningModel = nullptr;
476
  LearningModelBinding learningModelBinding = nullptr;
477
  LearningModelDevice learningModelDevice = nullptr;
478
  LearningModelSession learningModelSession = nullptr;
479
  std::wstring filePath = FileHelpers::GetModulePath() + L"relu.onnx";
480
  WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default));
481
  WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(filePath));
482
  WINML_EXPECT_TRUE(learningModel != nullptr);
483
  WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, learningModelDevice));
484
  WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession));
485

486
  auto inputShape = std::vector<int64_t>{5};
487
  auto inputData1 = std::vector<float>{-50.f, -25.f, 0.f, 25.f, 50.f};
488
  auto inputValue1 =
489
    TensorFloat::CreateFromIterable(inputShape, single_threaded_vector<float>(std::move(inputData1)).GetView());
490

491
  auto inputData2 = std::vector<float>{50.f, 25.f, 0.f, -25.f, -50.f};
492
  auto inputValue2 =
493
    TensorFloat::CreateFromIterable(inputShape, single_threaded_vector<float>(std::move(inputData2)).GetView());
494

495
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"X", inputValue1));
496

497
  auto outputValue = TensorFloat::Create();
498
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"Y", outputValue));
499

500
  WINML_EXPECT_NO_THROW(learningModelSession.Evaluate(learningModelBinding, L""));
501

502
  auto buffer1 = outputValue.GetAsVectorView();
503
  WINML_EXPECT_TRUE(buffer1 != nullptr);
504

505
  // The second evaluation
506
  // If we don't bind output again, the output value will not change
507
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"X", inputValue2));
508
  WINML_EXPECT_NO_THROW(learningModelSession.Evaluate(learningModelBinding, L""));
509
  auto buffer2 = outputValue.GetAsVectorView();
510
  WINML_EXPECT_EQUAL(buffer1.Size(), buffer2.Size());
511
  bool isSame = true;
512
  for (uint32_t i = 0; i < buffer1.Size(); ++i) {
513
    if (buffer1.GetAt(i) != buffer2.GetAt(i)) {
514
      isSame = false;
515
      break;
516
    }
517
  }
518
  WINML_EXPECT_FALSE(isSame);
519
}
520

521
static VideoFrame CreateVideoFrame(const wchar_t* path) {
522
  auto imagefile = StorageFile::GetFileFromPathAsync(path).get();
523
  auto stream = imagefile.OpenAsync(FileAccessMode::Read).get();
524
  auto decoder = BitmapDecoder::CreateAsync(stream).get();
525
  auto softwareBitmap = decoder.GetSoftwareBitmapAsync().get();
526
  return VideoFrame::CreateWithSoftwareBitmap(softwareBitmap);
527
}
528

529
static void VerifyOutputAfterImageBindCalledTwice() {
530
  std::wstring fullModelPath = FileHelpers::GetModulePath() + L"model.onnx";
531
  std::wstring fullImagePath1 = FileHelpers::GetModulePath() + L"kitten_224.png";
532
  std::wstring fullImagePath2 = FileHelpers::GetModulePath() + L"fish.png";
533

534
  // winml model creation
535
  LearningModel model = nullptr;
536
  WINML_EXPECT_NO_THROW(model = LearningModel::LoadFromFilePath(fullModelPath));
537
  LearningModelSession modelSession = nullptr;
538
  WINML_EXPECT_NO_THROW(
539
    modelSession = LearningModelSession(model, LearningModelDevice(LearningModelDeviceKind::Default))
540
  );
541
  LearningModelBinding modelBinding(modelSession);
542

543
  // create the tensor for the actual output
544
  auto output = TensorFloat::Create();
545
  modelBinding.Bind(L"softmaxout_1", output);
546

547
  // Bind image 1 and evaluate
548
  auto frame = CreateVideoFrame(fullImagePath1.c_str());
549
  auto imageTensor = ImageFeatureValue::CreateFromVideoFrame(frame);
550
  WINML_EXPECT_NO_THROW(modelBinding.Bind(L"data_0", imageTensor));
551
  WINML_EXPECT_NO_THROW(modelSession.Evaluate(modelBinding, L""));
552

553
  // Store 1st result
554
  auto outputVectorView1 = output.GetAsVectorView();
555

556
  // Bind image 2 and evaluate
557
  // In this scenario, the backing videoframe is updated, and the imagefeaturevalue is rebound.
558
  // The expected result is that the videoframe will be re-tensorized at bind
559
  auto frame2 = CreateVideoFrame(fullImagePath2.c_str());
560
  frame2.CopyToAsync(frame).get();
561
  WINML_EXPECT_NO_THROW(modelBinding.Bind(L"data_0", imageTensor));
562
  WINML_EXPECT_NO_THROW(modelSession.Evaluate(modelBinding, L""));
563

564
  // Store 2nd result
565
  auto outputVectorView2 = output.GetAsVectorView();
566

567
  WINML_EXPECT_EQUAL(outputVectorView1.Size(), outputVectorView2.Size());
568
  bool isSame = true;
569
  for (uint32_t i = 0; i < outputVectorView1.Size(); ++i) {
570
    if (outputVectorView1.GetAt(i) != outputVectorView2.GetAt(i)) {
571
      isSame = false;
572
      break;
573
    }
574
  }
575
  WINML_EXPECT_FALSE(isSame);
576
}
577

578
static void SequenceLengthTensorFloat() {
579
  // Tests sequence of tensor float as an input
580
  LearningModel learningModel = nullptr;
581
  LearningModelBinding learningModelBinding = nullptr;
582
  LearningModelDevice learningModelDevice = nullptr;
583
  LearningModelSession learningModelSession = nullptr;
584
  std::wstring filePath = FileHelpers::GetModulePath() + L"sequence_length.onnx";
585
  WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default));
586
  WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(filePath));
587
  WINML_EXPECT_TRUE(learningModel != nullptr);
588
  WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, learningModelDevice));
589
  WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession));
590

591
  auto input = winrt::single_threaded_vector<TensorFloat>();
592
  for (int i = 0; i < 3; i++) {
593
    std::vector<int64_t> shape = {5, 3 * i + 1};
594
    std::vector<float> data(
595
      static_cast<size_t>(std::accumulate(shape.begin(), shape.end(), static_cast<int64_t>(1), std::multiplies()))
596
    );
597
    input.Append(TensorFloat::CreateFromShapeArrayAndDataArray(shape, data));
598
  }
599

600
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"X", input));
601
  auto results = learningModelSession.Evaluate(learningModelBinding, L"");
602

603
  WINML_EXPECT_EQUAL(3, results.Outputs().Lookup(L"Y").as<TensorInt64Bit>().GetAsVectorView().GetAt(0));
604
}
605

606
static void SequenceConstructTensorString() {
607
  LearningModel learningModel = nullptr;
608
  LearningModelBinding learningModelBinding = nullptr;
609
  LearningModelDevice learningModelDevice = nullptr;
610
  LearningModelSession learningModelSession = nullptr;
611
  std::wstring filePath = FileHelpers::GetModulePath() + L"sequence_construct.onnx";
612
  WINML_EXPECT_NO_THROW(learningModelDevice = LearningModelDevice(LearningModelDeviceKind::Default));
613
  WINML_EXPECT_NO_THROW(learningModel = LearningModel::LoadFromFilePath(filePath));
614
  WINML_EXPECT_TRUE(learningModel != nullptr);
615
  WINML_EXPECT_NO_THROW(learningModelSession = LearningModelSession(learningModel, learningModelDevice));
616
  WINML_EXPECT_NO_THROW(learningModelBinding = LearningModelBinding(learningModelSession));
617

618
  std::vector<int64_t> shape1 = {2, 3};
619
  std::vector<int64_t> data1(
620
    static_cast<size_t>(std::accumulate(shape1.begin(), shape1.end(), static_cast<int64_t>(1), std::multiplies()))
621
  );
622
  auto input1 = TensorInt64Bit::CreateFromShapeArrayAndDataArray(shape1, data1);
623
  std::vector<int64_t> shape2 = {2, 3};
624
  std::vector<int64_t> data2(
625
    static_cast<size_t>(std::accumulate(shape2.begin(), shape2.end(), static_cast<int64_t>(1), std::multiplies()))
626
  );
627
  auto input2 = TensorInt64Bit::CreateFromShapeArrayAndDataArray(shape2, data2);
628

629
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"tensor1", input1));
630
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"tensor2", input2));
631
  auto results = learningModelSession.Evaluate(learningModelBinding, L"");
632

633
  auto output_sequence = results.Outputs().Lookup(L"output_sequence").as<wfc::IVectorView<TensorInt64Bit>>();
634
  WINML_EXPECT_EQUAL(static_cast<uint32_t>(2), output_sequence.Size());
635
  WINML_EXPECT_EQUAL(2, output_sequence.GetAt(0).Shape().GetAt(0));
636
  WINML_EXPECT_EQUAL(3, output_sequence.GetAt(0).Shape().GetAt(1));
637
  WINML_EXPECT_EQUAL(2, output_sequence.GetAt(1).Shape().GetAt(0));
638
  WINML_EXPECT_EQUAL(3, output_sequence.GetAt(1).Shape().GetAt(1));
639

640
  auto bound_output_sequence = winrt::single_threaded_vector<TensorInt64Bit>();
641
  WINML_EXPECT_NO_THROW(learningModelBinding.Bind(L"output_sequence", bound_output_sequence));
642
  WINML_EXPECT_NO_THROW(learningModelSession.Evaluate(learningModelBinding, L""));
643
  WINML_EXPECT_EQUAL(static_cast<uint32_t>(2), bound_output_sequence.Size());
644
  WINML_EXPECT_EQUAL(2, bound_output_sequence.GetAt(0).Shape().GetAt(0));
645
  WINML_EXPECT_EQUAL(3, bound_output_sequence.GetAt(0).Shape().GetAt(1));
646
  WINML_EXPECT_EQUAL(2, bound_output_sequence.GetAt(1).Shape().GetAt(0));
647
  WINML_EXPECT_EQUAL(3, bound_output_sequence.GetAt(1).Shape().GetAt(1));
648
}
649

650
const LearningModelBindingAPITestsApi& getapi() {
651
  static LearningModelBindingAPITestsApi api = {
652
    LearningModelBindingAPITestsClassSetup,
653
    CpuSqueezeNet,
654
    CpuSqueezeNetEmptyOutputs,
655
    CpuSqueezeNetUnboundOutputs,
656
    CpuSqueezeNetBindInputTensorAsInspectable,
657
    CastMapInt64,
658
    DictionaryVectorizerMapInt64,
659
    DictionaryVectorizerMapString,
660
    ZipMapInt64,
661
    ZipMapInt64Unbound,
662
    ZipMapString,
663
    GpuSqueezeNet,
664
    GpuSqueezeNetEmptyOutputs,
665
    GpuSqueezeNetUnboundOutputs,
666
    ImageBindingDimensions,
667
    VerifyInvalidBindExceptions,
668
    BindInvalidInputName,
669
    VerifyOutputAfterEvaluateAsyncCalledTwice,
670
    VerifyOutputAfterImageBindCalledTwice,
671
    SequenceLengthTensorFloat,
672
    SequenceConstructTensorString
673
  };
674

675
  if (SkipGpuTests()) {
676
    api.GpuSqueezeNet = SkipTest;
677
    api.GpuSqueezeNetEmptyOutputs = SkipTest;
678
    api.GpuSqueezeNetUnboundOutputs = SkipTest;
679
  }
680
  if (RuntimeParameterExists(L"noVideoFrameTests")) {
681
    api.ImageBindingDimensions = SkipTest;
682
    api.BindInvalidInputName = SkipTest;
683
    api.VerifyOutputAfterImageBindCalledTwice = SkipTest;
684
    api.VerifyInvalidBindExceptions = SkipTest;
685
  }
686
  return api;
687
}
688