6
import hypothesis.strategies as st
8
import numpy.testing as npt
9
from hypothesis import given, settings
11
import caffe2.python.hypothesis_test_util as hu
13
from caffe2.python import (
14
layer_model_instantiator,
19
from caffe2.python.layers.layers import (
26
is_request_only_scalar,
29
from caffe2.python.layers.tags import Tags
30
from caffe2.python.layer_test_util import (
35
logger = logging.getLogger(__name__)
38
class TestLayers(LayersTestCase):
39
def testSparseDropoutWithReplacement(self):
40
input_record = schema.NewRecord(self.model.net, IdList)
41
self.model.output_schema = schema.Struct()
43
lengths_blob = input_record.field_blobs()[0]
44
values_blob = input_record.field_blobs()[1]
45
lengths = np.array([1] * 10).astype(np.int32)
46
values = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]).astype(np.int64)
47
workspace.FeedBlob(lengths_blob, lengths)
48
workspace.FeedBlob(values_blob, values)
50
out = self.model.SparseDropoutWithReplacement(
51
input_record, 0.0, 0.5, 1.0, -1, output_names_or_num=1)
52
self.assertEqual(schema.List(schema.Scalar(np.int64,)), out)
54
train_init_net, train_net = self.get_training_nets()
55
eval_net = self.get_eval_net()
56
predict_net = self.get_predict_net()
58
workspace.RunNetOnce(train_init_net)
59
workspace.RunNetOnce(train_net)
60
out_values = workspace.FetchBlob(out.items())
61
out_lengths = workspace.FetchBlob(out.lengths())
62
self.assertBlobsEqual(out_values, values)
63
self.assertBlobsEqual(out_lengths, lengths)
65
workspace.RunNetOnce(eval_net)
67
workspace.RunNetOnce(predict_net)
68
predict_values = workspace.FetchBlob("values_auto_0")69
predict_lengths = workspace.FetchBlob("lengths_auto_0")70
self.assertBlobsEqual(predict_values, np.array([-1] * 10).astype(np.int64))
71
self.assertBlobsEqual(predict_lengths, lengths)
73
def testAddLoss(self):
74
input_record_LR = self.new_record(
76
('label', schema.Scalar((np.float64, (1, )))),77
('logit', schema.Scalar((np.float32, (2, )))),78
('weight', schema.Scalar((np.float64, (1, ))))81
loss_LR = self.model.BatchLRLoss(input_record_LR)
83
self.model.add_loss(loss_LR)
84
assert 'unnamed' in self.model.loss
86
schema.Scalar((np.float32, tuple())), self.model.loss.unnamed
88
self.assertEqual(loss_LR, self.model.loss.unnamed)
90
self.model.add_loss(loss_LR, 'addLoss')
91
assert 'addLoss' in self.model.loss
93
schema.Scalar((np.float32, tuple())), self.model.loss.addLoss
95
self.assertEqual(loss_LR, self.model.loss.addLoss)
99
dtype=np.float32, blob=core.BlobReference('loss_blob_1')102
assert 'addLoss_auto_0' in self.model.loss
104
schema.Scalar((np.float32, tuple())), self.model.loss.addLoss_auto_0
106
assert core.BlobReference('loss_blob_1') in self.model.loss.field_blobs()111
'structName', schema.Scalar(
113
blob=core.BlobReference('loss_blob_2')118
assert 'addLoss_auto_1' in self.model.loss
120
schema.Struct(('structName', schema.Scalar((np.float32, tuple())))),121
self.model.loss.addLoss_auto_1
123
assert core.BlobReference('loss_blob_2') in self.model.loss.field_blobs()125
loss_in_tuple_0 = schema.Scalar(
126
dtype=np.float32, blob=core.BlobReference('loss_blob_in_tuple_0')129
loss_in_tuple_1 = schema.Scalar(
130
dtype=np.float32, blob=core.BlobReference('loss_blob_in_tuple_1')133
loss_tuple = schema.NamedTuple(
134
'loss_in_tuple', * [loss_in_tuple_0, loss_in_tuple_1]
136
self.model.add_loss(loss_tuple, 'addLoss')
137
assert 'addLoss_auto_2' in self.model.loss
140
('loss_in_tuple_0', schema.Scalar((np.float32, tuple()))),141
('loss_in_tuple_1', schema.Scalar((np.float32, tuple())))142
), self.model.loss.addLoss_auto_2
144
assert core.BlobReference('loss_blob_in_tuple_0')\145
in self.model.loss.field_blobs()
146
assert core.BlobReference('loss_blob_in_tuple_1')\147
in self.model.loss.field_blobs()
149
def testFilterMetricSchema(self):
150
self.model.add_metric_field("a:b", schema.Scalar())151
self.model.add_metric_field("a:c", schema.Scalar())152
self.model.add_metric_field("d", schema.Scalar())155
self.model.metrics_schema,
158
("b", schema.Scalar()),159
("c", schema.Scalar()),161
("d", schema.Scalar()),164
self.model.filter_metrics_schema({"a:b", "d"})166
self.model.metrics_schema,
169
("b", schema.Scalar()),171
("d", schema.Scalar()),174
def testAddOutputSchema(self):
175
# add the first field
176
self.model.add_output_schema('struct', schema.Struct())177
expected_output_schema = schema.Struct(('struct', schema.Struct()))179
self.model.output_schema,
180
expected_output_schema,
183
# add the second field
184
self.model.add_output_schema('scalar', schema.Scalar(np.float64))185
expected_output_schema = schema.Struct(
186
('struct', schema.Struct()),187
('scalar', schema.Scalar(np.float64)),190
self.model.output_schema,
191
expected_output_schema,
194
# overwrite a field should raise
195
with self.assertRaises(AssertionError):
196
self.model.add_output_schema('scalar', schema.Struct())198
def _test_net(self, net, ops_list):
200
Helper function to assert the net contains some set of operations and
204
net -- the network to test and run
205
ops_list -- the list of operation specifications to check for
208
ops_output = self.assertNetContainOps(net, ops_list)
209
workspace.RunNetOnce(net)
212
def testFCWithoutBias(self):
214
fc_without_bias = self.model.FCWithoutBias(
215
self.model.input_feature_schema.float_features, output_dims)
216
self.model.output_schema = fc_without_bias
219
schema.Scalar((np.float32, (output_dims, ))),
223
train_init_net, train_net = self.get_training_nets()
225
init_ops = self.assertNetContainOps(
228
OpSpec("UniformFill", None, None),232
mat_mul_spec = OpSpec(
235
self.model.input_feature_schema.float_features(),
236
init_ops[0].output[0],
238
fc_without_bias.field_blobs()
241
self.assertNetContainOps(train_net, [mat_mul_spec])
243
predict_net = self.get_predict_net()
244
self.assertNetContainOps(predict_net, [mat_mul_spec])
246
def testFCWithBootstrap(self):
248
fc_with_bootstrap = self.model.FCWithBootstrap(
249
self.model.input_feature_schema.float_features,
250
output_dims=output_dims,
254
self.model.output_schema = fc_with_bootstrap
257
self.assertEqual(len(fc_with_bootstrap), 4)
259
# must be in this order
261
core.BlobReference("fc_with_bootstrap/bootstrap_iteration_0/indices") == fc_with_bootstrap[0].field_blobs()[0]264
core.BlobReference("fc_with_bootstrap/bootstrap_iteration_0/preds") == fc_with_bootstrap[1].field_blobs()[0]267
core.BlobReference("fc_with_bootstrap/bootstrap_iteration_1/indices") == fc_with_bootstrap[2].field_blobs()[0]270
core.BlobReference("fc_with_bootstrap/bootstrap_iteration_1/preds") == fc_with_bootstrap[3].field_blobs()[0]273
train_init_net, train_net = self.get_training_nets()
274
predict_net = layer_model_instantiator.generate_predict_net(self.model)
276
train_proto = train_net.Proto()
277
eval_proto = predict_net.Proto()
279
train_ops = train_proto.op
280
eval_ops = eval_proto.op
284
"GivenTensorInt64Fill",
286
"GivenTensorIntFill",
287
"GivenTensorIntFill",
300
"GivenTensorInt64Fill",
302
"GivenTensorIntFill",
303
"GivenTensorIntFill",
312
assert len(train_ops) == len(master_train_ops)
313
assert len(eval_ops) == len(master_eval_ops)
315
assert train_proto.external_input == eval_proto.external_input
316
assert train_proto.external_output == list()
318
# make sure all the ops are present and unchanged for train_net and eval_net
319
for idx, op in enumerate(master_train_ops):
320
assert train_ops[idx].type == op
322
for idx, op in enumerate(master_eval_ops):
323
assert eval_ops[idx].type == op
326
def testFCwithAxis2(self):
330
input_record = self.new_record(
332
('history_sequence', schema.Scalar((np.float32, (max_length,336
fc_out = self.model.FC(
337
input_record.history_sequence, output_dim,
339
self.model.output_schema = fc_out
341
schema.Scalar((np.float32, (max_length, output_dim))),
345
train_init_net, train_net = self.get_training_nets()
347
def testFCTransposed(self):
351
input_record = self.new_record(
353
('history_sequence', schema.Scalar((np.float32, (max_length,357
fc_transposed_out = self.model.FC(
358
input_record.history_sequence, output_dim,
359
axis=2, transposed=True)
360
self.model.output_schema = fc_transposed_out
362
schema.Scalar((np.float32, (max_length, output_dim))),
366
train_init_net, train_net = self.get_training_nets()
368
def testFCTransposedWithMaxFCSize(self):
372
input_record = self.new_record(
374
('history_sequence', schema.Scalar((np.float32, (max_length,378
fc_transposed_out = self.model.FC(
379
input_record.history_sequence, output_dim,
380
max_fc_size=input_dim * output_dim // 2,
381
axis=2, transposed=True)
382
self.model.output_schema = fc_transposed_out
384
schema.Scalar((np.float32, (max_length, output_dim))),
388
train_init_net, train_net = self.get_training_nets()
390
def testSparseLookupSumPoolingWithEviction(self):
391
# Create test embedding table of 1 row
392
record = schema.NewRecord(self.model.net, schema.Struct(
393
('sparse', schema.Struct(394
('sparse_feature_0', schema.ListWithEvicted(395
schema.Scalar(np.int64,
396
metadata=schema.Metadata(categorical_limit=1)),)),)),
399
lengths_blob = record.sparse.sparse_feature_0.lengths.get()
400
values_blob = record.sparse.sparse_feature_0.items.get()
401
evicted_values_blob = record.sparse.sparse_feature_0._evicted_values.get()
402
lengths = np.array([1]).astype(np.int32)
403
values = np.array([0]).astype(np.int64)
404
# Need to reset row 0
405
evicted_values = np.array([0]).astype(np.int64)
406
workspace.FeedBlob(lengths_blob, lengths)
407
workspace.FeedBlob(values_blob, values)
408
workspace.FeedBlob(evicted_values_blob, evicted_values)
410
embedding_after_pooling = self.model.SparseLookup(
411
record.sparse.sparse_feature_0, [embedding_dim], 'Sum', weight_init=("ConstantFill", {"value": 1.0}))413
self.model.output_schema = schema.Struct()
415
schema.Scalar((np.float32, (embedding_dim, ))),
416
embedding_after_pooling
418
train_init_net, train_net = self.get_training_nets()
419
workspace.RunNetOnce(train_init_net)
420
embedding_after_init = workspace.FetchBlob("sparse_lookup/w")421
# Change row 0's value before reset
422
new_values = np.array([[2, 2, 2, 2, 2, 2, 2, 2]]).astype(np.float32)
423
workspace.FeedBlob("sparse_lookup/w", new_values)424
workspace.RunNetOnce(train_net.Proto())
425
embedding_after_training = workspace.FetchBlob("sparse_lookup/w")426
# Verify row 0's value does not change after reset
427
self.assertEqual(embedding_after_training.all(), embedding_after_init.all())
430
def testSparseLookupSumPooling(self):
431
record = schema.NewRecord(self.model.net, schema.Struct(
432
('sparse', schema.Struct(433
('sparse_feature_0', schema.List(434
schema.Scalar(np.int64,
435
metadata=schema.Metadata(categorical_limit=1000)))),
439
embedding_after_pooling = self.model.SparseLookup(
440
record.sparse.sparse_feature_0, [embedding_dim], 'Sum')
441
self.model.output_schema = schema.Struct()
443
schema.Scalar((np.float32, (embedding_dim, ))),
444
embedding_after_pooling
447
train_init_net, train_net = self.get_training_nets()
449
init_ops = self.assertNetContainOps(
452
OpSpec("UniformFill", None, None),453
OpSpec("ConstantFill", None, None),456
sparse_lookup_op_spec = OpSpec(
459
init_ops[0].output[0],
460
record.sparse.sparse_feature_0.items(),
461
record.sparse.sparse_feature_0.lengths(),
463
[embedding_after_pooling()]
465
self.assertNetContainOps(train_net, [sparse_lookup_op_spec])
467
predict_net = self.get_predict_net()
468
self.assertNetContainOps(predict_net, [sparse_lookup_op_spec])
471
use_hashing=st.booleans(),
472
modulo=st.integers(min_value=100, max_value=200),
473
use_divide_mod=st.booleans(),
474
divisor=st.integers(min_value=10, max_value=20),
476
def testSparseFeatureHashIdList(self, use_hashing, modulo, use_divide_mod, divisor):
477
record = schema.NewRecord(
479
schema.List(schema.Scalar(
481
metadata=schema.Metadata(categorical_limit=60000)
484
use_divide_mod = use_divide_mod if use_hashing is False else False
485
output_schema = self.model.SparseFeatureHash(
488
use_hashing=use_hashing,
489
use_divide_mod=use_divide_mod,
493
self.model.output_schema = output_schema
495
self.assertEqual(len(self.model.layers), 1)
496
self.assertEqual(output_schema._items.metadata.categorical_limit,
498
train_init_net, train_net = self.get_training_nets()
500
self.assertEqual(len(train_net.Proto().op), 3)
502
self.assertEqual(len(train_net.Proto().op), 2)
505
use_hashing=st.booleans(),
506
modulo=st.integers(min_value=100, max_value=200),
508
def testSparseFeatureHashIdScoreList(self, use_hashing, modulo):
509
record = schema.NewRecord(self.model.net,
510
schema.Map(schema.Scalar(np.int64,
511
metadata=schema.Metadata(
512
categorical_limit=60000)),
515
output_schema = self.model.SparseFeatureHash(
518
use_hashing=use_hashing)
520
self.model.output_schema = output_schema
522
self.assertEqual(len(self.model.layers), 1)
523
self.assertEqual(output_schema._items.keys.metadata.categorical_limit,
525
train_init_net, train_net = self.get_training_nets()
527
def testSparseLookupIncorrectPositionWeightedOnIdList(self):
529
Currently the implementation of SparseLookup assumed input is id_score_list
530
when use PositionWeighted.
532
record = schema.NewRecord(self.model.net, schema.Struct(
533
('sparse', schema.Struct(534
('sparse_feature_0', schema.List(535
schema.Scalar(np.int64,
536
metadata=schema.Metadata(categorical_limit=1000)))),
541
with self.assertRaises(AssertionError):
542
self.model.SparseLookup(
543
record.sparse.sparse_feature_0, [embedding_dim], 'PositionWeighted')
545
def testSparseLookupPositionWeightedOnIdList(self):
546
record = schema.NewRecord(self.model.net, schema.Struct(
547
('sparse', schema.Struct(548
('sparse_feature_0', schema.List(549
schema.Scalar(np.int64,
550
metadata=schema.Metadata(categorical_limit=1000)))),
554
# convert id_list to id_score_list with PositionWeighted layer
555
sparse_segment = record.sparse.sparse_feature_0
556
pos_w_layer = self.model.PositionWeighted(sparse_segment)
558
sparse_segment = schema.Map(
559
keys=get_key(sparse_segment),
560
values=pos_w_layer.position_weights,
561
lengths_blob=sparse_segment.lengths
565
embedding_after_pooling = self.model.SparseLookup(
566
sparse_segment, [embedding_dim], 'PositionWeighted')
567
self.model.output_schema = schema.Struct()
569
schema.Scalar((np.float32, (embedding_dim, ))),
570
embedding_after_pooling
573
train_init_net, train_net = self.get_training_nets()
575
self.assertNetContainOps(
578
OpSpec("ConstantFill", None, None), # position_weights/pos_w579
OpSpec("UniformFill", None, None),580
OpSpec("ConstantFill", None, None),583
self.assertNetContainOps(train_net, [
584
OpSpec("LengthsRangeFill", None, None),585
OpSpec("Gather", None, None),586
OpSpec("SparseLengthsWeightedSum", None, None),589
predict_net = self.get_predict_net()
590
self.assertNetContainOps(predict_net, [
591
OpSpec("LengthsRangeFill", None, None),592
OpSpec("Gather", None, None),593
OpSpec("SparseLengthsWeightedSum", None, None),596
def testSparseLookupPositionWeightedOnIdScoreList(self):
597
record = schema.NewRecord(self.model.net, schema.Struct(
598
('sparse', schema.Struct(599
('id_score_list_0', schema.Map(602
metadata=schema.Metadata(
603
categorical_limit=1000
612
embedding_after_pooling = self.model.SparseLookup(
613
record.sparse.id_score_list_0, [embedding_dim], 'PositionWeighted')
614
self.model.output_schema = schema.Struct()
616
schema.Scalar((np.float32, (embedding_dim, ))),
617
embedding_after_pooling
620
train_init_net, train_net = self.get_training_nets()
622
init_ops = self.assertNetContainOps(
625
OpSpec("UniformFill", None, None),626
OpSpec("ConstantFill", None, None),629
sparse_lookup_op_spec = OpSpec(
630
'SparseLengthsWeightedSum',
632
init_ops[0].output[0],
633
record.sparse.id_score_list_0.values(),
634
record.sparse.id_score_list_0.keys(),
635
record.sparse.id_score_list_0.lengths(),
637
[embedding_after_pooling()]
639
self.assertNetContainOps(train_net, [sparse_lookup_op_spec])
641
predict_net = self.get_predict_net()
642
self.assertNetContainOps(predict_net, [sparse_lookup_op_spec])
644
def testSparseLookupIncorrectRecencyWeightedOnIdList(self):
646
Currently the implementation of SparseLookup assumed input is id_score_list
647
when use RecencyWeighted.
649
record = schema.NewRecord(self.model.net, schema.Struct(
650
('sparse', schema.Struct(651
('sparse_feature_0', schema.List(652
schema.Scalar(np.int64,
653
metadata=schema.Metadata(categorical_limit=1000)))),
658
with self.assertRaises(AssertionError):
659
self.model.SparseLookup(
660
record.sparse.sparse_feature_0, [embedding_dim], 'RecencyWeighted')
662
def testSparseLookupRecencyWeightedOnIdScoreList(self):
663
record = schema.NewRecord(self.model.net, schema.Struct(
664
('sparse', schema.Struct(665
('id_score_list_0', schema.Map(668
metadata=schema.Metadata(
669
categorical_limit=1000
678
embedding_after_pooling = self.model.SparseLookup(
679
record.sparse.id_score_list_0, [embedding_dim], 'RecencyWeighted')
680
self.model.output_schema = schema.Struct()
682
schema.Scalar((np.float32, (embedding_dim, ))),
683
embedding_after_pooling
686
train_init_net, train_net = self.get_training_nets()
688
init_ops = self.assertNetContainOps(
691
OpSpec("UniformFill", None, None),692
OpSpec("ConstantFill", None, None),695
sparse_lookup_op_spec = OpSpec(
696
'SparseLengthsWeightedSum',
698
init_ops[0].output[0],
699
record.sparse.id_score_list_0.values(),
700
record.sparse.id_score_list_0.keys(),
701
record.sparse.id_score_list_0.lengths(),
703
[embedding_after_pooling()]
705
self.assertNetContainOps(train_net, [sparse_lookup_op_spec])
707
predict_net = self.get_predict_net()
708
self.assertNetContainOps(predict_net, [sparse_lookup_op_spec])
710
def testPairwiseSimilarityWithAllEmbeddings(self):
713
record = schema.NewRecord(self.model.net, schema.Struct(
714
('all_embeddings', schema.Scalar(715
((np.float32, (N, embedding_dim)))
718
current = self.model.PairwiseSimilarity(
722
schema.Scalar((np.float32, (N * N, ))),
726
train_init_net, train_net = self.get_training_nets()
727
self.assertNetContainOps(train_init_net, [])
728
self.assertNetContainOps(train_net, [
729
OpSpec("BatchMatMul", None, None),730
OpSpec("Flatten", None, None),733
def testPairwiseSimilarityWithXandYEmbeddings(self):
735
record = schema.NewRecord(self.model.net, schema.Struct(
736
('x_embeddings', schema.Scalar(737
((np.float32, (5, embedding_dim)))
739
('y_embeddings', schema.Scalar(740
((np.float32, (6, embedding_dim)))
743
current = self.model.PairwiseSimilarity(
747
schema.Scalar((np.float32, (5 * 6, ))),
751
train_init_net, train_net = self.get_training_nets()
752
self.assertNetContainOps(train_init_net, [])
753
self.assertNetContainOps(train_net, [
754
OpSpec("BatchMatMul", None, None),755
OpSpec("Flatten", None, None),758
def testPairwiseSimilarityWithXandYEmbeddingsAndGather(self):
761
output_idx = [1, 3, 5]
762
output_idx_blob = self.model.add_global_constant(
763
str(self.model.net.NextScopedBlob('pairwise_dot_product_gather')),767
indices_to_gather = schema.Scalar(
768
(np.int32, len(output_idx)),
772
record = schema.NewRecord(self.model.net, schema.Struct(
773
('x_embeddings', schema.Scalar(774
((np.float32, (5, embedding_dim)))
776
('y_embeddings', schema.Scalar(777
((np.float32, (6, embedding_dim)))
779
('indices_to_gather', indices_to_gather),781
current = self.model.PairwiseSimilarity(
782
record, len(output_idx))
784
# This assert is not necessary,
785
# output size is passed into PairwiseSimilarity
787
schema.Scalar((np.float32, (len(output_idx), ))),
791
train_init_net, train_net = self.get_training_nets()
792
self.assertNetContainOps(train_init_net, [])
793
self.assertNetContainOps(train_net, [
794
OpSpec("BatchMatMul", None, None),795
OpSpec("Flatten", None, None),796
OpSpec("BatchGather", None, None),799
def testPairwiseSimilarityIncorrectInput(self):
801
record = schema.NewRecord(self.model.net, schema.Struct(
802
('x_embeddings', schema.Scalar(803
((np.float32, (5, embedding_dim)))
806
with self.assertRaises(AssertionError):
807
self.model.PairwiseSimilarity(
810
record = schema.NewRecord(self.model.net, schema.Struct(
811
('all_embeddings', schema.List(np.float32))813
with self.assertRaises(AssertionError):
814
self.model.PairwiseSimilarity(
817
def testConcat(self):
819
input_record = self.new_record(schema.Struct(
820
('input1', schema.Scalar((np.float32, (embedding_dim, )))),821
('input2', schema.Scalar((np.float32, (embedding_dim, )))),822
('input3', schema.Scalar((np.float32, (embedding_dim, )))),825
output = self.model.Concat(input_record)
827
schema.Scalar((np.float32, ((len(input_record.fields) * embedding_dim, )))),
831
# Note that in Concat layer we assume first dimension is batch.
832
# so input is B * embedding_dim
833
# add_axis=1 make it B * 1 * embedding_dim
834
# concat on axis=1 make it B * N * embedding_dim
835
output = self.model.Concat(input_record, axis=1, add_axis=1)
837
schema.Scalar((np.float32, ((len(input_record.fields), embedding_dim)))),
841
def testSamplingTrain(self):
844
indices = self.new_record(schema.Scalar((np.int32, (10,))))
845
sampling_prob = self.new_record(schema.Scalar((np.float32, (10, ))))
847
sampled_fc = self.model.SamplingTrain(
849
('input', self.model.input_feature_schema.float_features),850
('indices', indices),851
('sampling_prob', sampling_prob),856
self.model.output_schema = sampled_fc
858
# Check that we don't add prediction layer into the model
859
self.assertEqual(1, len(self.model.layers))
862
schema.Scalar((np.float32, (output_dims, ))),
866
train_init_net, train_net = self.get_training_nets()
868
init_ops = self.assertNetContainOps(
871
OpSpec("UniformFill", None, None),872
OpSpec("UniformFill", None, None),876
sampled_fc_layer = self.model.layers[0]
878
gather_w_spec = OpSpec(
881
init_ops[0].output[0],
885
sampled_fc_layer._prediction_layer.train_param_blobs[0]
888
gather_b_spec = OpSpec(
891
init_ops[1].output[0],
895
sampled_fc_layer._prediction_layer.train_param_blobs[1]
898
train_fc_spec = OpSpec(
901
self.model.input_feature_schema.float_features(),
902
] + sampled_fc_layer._prediction_layer.train_param_blobs,
903
sampled_fc.field_blobs()
905
log_spec = OpSpec("Log", [sampling_prob()], [None])908
[sampled_fc.field_blobs()[0], None],
909
sampled_fc.field_blobs()
912
train_ops = self.assertNetContainOps(
914
[gather_w_spec, gather_b_spec, train_fc_spec, log_spec, sub_spec])
916
self.assertEqual(train_ops[3].output[0], train_ops[4].input[1])
918
predict_net = self.get_predict_net()
919
self.assertNetContainOps(
925
self.model.input_feature_schema.float_features(),
926
init_ops[0].output[0],
927
init_ops[1].output[0],
929
sampled_fc.field_blobs()
934
def testBatchLRLoss(self):
935
input_record = self.new_record(schema.Struct(
936
('label', schema.Scalar((np.float64, (1,)))),937
('logit', schema.Scalar((np.float32, (2,)))),938
('weight', schema.Scalar((np.float64, (1,))))940
loss = self.model.BatchLRLoss(input_record)
941
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
943
def testBatchLRLossWithUncertainty(self):
944
input_record = self.new_record(schema.Struct(
945
('label', schema.Scalar((np.float64, (1,)))),946
('logit', schema.Scalar((np.float32, (2,)))),947
('weight', schema.Scalar((np.float64, (1,)))),948
('log_variance', schema.Scalar((np.float64, (1,)))),950
loss = self.model.BatchLRLoss(input_record)
951
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
953
def testMarginRankLoss(self):
954
input_record = self.new_record(schema.Struct(
955
('pos_prediction', schema.Scalar((np.float32, (1,)))),956
('neg_prediction', schema.List(np.float32)),958
pos_items = np.array([0.1, 0.2, 0.3], dtype=np.float32)
959
neg_lengths = np.array([1, 2, 3], dtype=np.int32)
960
neg_items = np.array([0.1, 0.2, 0.3, 0.4, 0.5, 0.6], dtype=np.float32)
963
[pos_items, neg_lengths, neg_items]
965
loss = self.model.MarginRankLoss(input_record)
966
self.run_train_net_forward_only()
967
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
969
def testBPRLoss(self):
970
input_record = self.new_record(schema.Struct(
971
('pos_prediction', schema.Scalar((np.float32, (1,)))),972
('neg_prediction', schema.List(np.float32)),974
pos_items = np.array([0.8, 0.9], dtype=np.float32)
975
neg_lengths = np.array([1, 2], dtype=np.int32)
976
neg_items = np.array([0.1, 0.2, 0.3], dtype=np.float32)
979
[pos_items, neg_lengths, neg_items]
981
loss = self.model.BPRLoss(input_record)
982
self.run_train_net_forward_only()
983
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
984
result = workspace.FetchBlob('bpr_loss/output')985
np.testing.assert_array_almost_equal(np.array(1.24386, dtype=np.float32), result)
987
def testBatchMSELoss(self):
988
input_record = self.new_record(schema.Struct(
989
('label', schema.Scalar((np.float64, (1,)))),990
('prediction', schema.Scalar((np.float32, (2,)))),992
loss = self.model.BatchMSELoss(input_record)
993
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
995
def testBatchHuberLoss(self):
996
input_record = self.new_record(schema.Struct(
997
('label', schema.Scalar((np.float32, (1,)))),998
('prediction', schema.Scalar((np.float32, (2,)))),1000
loss = self.model.BatchHuberLoss(input_record)
1001
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
1003
def testBatchSigmoidCrossEntropyLoss(self):
1004
input_record = self.new_record(schema.Struct(
1005
('label', schema.Scalar((np.float32, (32,)))),1006
('prediction', schema.Scalar((np.float32, (32,))))1008
loss = self.model.BatchSigmoidCrossEntropyLoss(input_record)
1009
self.assertEqual(schema.Scalar((np.float32, tuple())), loss)
1011
def testBatchSoftmaxLoss(self):
1012
input_record = self.new_record(schema.Struct(
1013
('label', schema.Scalar((np.float32, tuple()))),1014
('prediction', schema.Scalar((np.float32, (32,))))1016
loss = self.model.BatchSoftmaxLoss(input_record)
1017
self.assertEqual(schema.Struct(
1018
('softmax', schema.Scalar((np.float32, (32,)))),1019
('loss', schema.Scalar(np.float32)),1022
def testBatchSoftmaxLossWeight(self):
1023
input_record = self.new_record(schema.Struct(
1024
('label', schema.Scalar((np.float32, tuple()))),1025
('prediction', schema.Scalar((np.float32, (32,)))),1026
('weight', schema.Scalar((np.float64, (1,))))1028
loss = self.model.BatchSoftmaxLoss(input_record)
1029
self.assertEqual(schema.Struct(
1030
('softmax', schema.Scalar((np.float32, (32,)))),1031
('loss', schema.Scalar(np.float32)),1035
X=hu.arrays(dims=[2, 5]),
1037
def testBatchNormalization(self, X):
1038
input_record = self.new_record(schema.Scalar((np.float32, (5,))))
1039
schema.FeedRecord(input_record, [X])
1040
bn_output = self.model.BatchNormalization(input_record)
1041
self.assertEqual(schema.Scalar((np.float32, (5,))), bn_output)
1042
self.model.output_schema = schema.Struct()
1044
train_init_net, train_net = self.get_training_nets()
1046
init_ops = self.assertNetContainOps(
1049
OpSpec("ConstantFill", None, None),1050
OpSpec("ConstantFill", None, None),1051
OpSpec("ConstantFill", None, None),1052
OpSpec("ConstantFill", None, None),1056
input_blob = input_record.field_blobs()[0]
1057
output_blob = bn_output.field_blobs()[0]
1059
expand_dims_spec = OpSpec(
1065
train_bn_spec = OpSpec(
1067
[None, init_ops[0].output[0], init_ops[1].output[0],
1068
init_ops[2].output[0], init_ops[3].output[0]],
1069
[output_blob, init_ops[2].output[0], init_ops[3].output[0], None, None],
1070
{'is_test': 0, 'order': 'NCHW', 'momentum': 0.9},1073
test_bn_spec = OpSpec(
1075
[None, init_ops[0].output[0], init_ops[1].output[0],
1076
init_ops[2].output[0], init_ops[3].output[0]],
1078
{'is_test': 1, 'order': 'NCHW', 'momentum': 0.9},1081
squeeze_spec = OpSpec(
1087
self.assertNetContainOps(
1089
[expand_dims_spec, train_bn_spec, squeeze_spec]
1092
eval_net = self.get_eval_net()
1094
self.assertNetContainOps(
1096
[expand_dims_spec, test_bn_spec, squeeze_spec]
1099
predict_net = self.get_predict_net()
1101
self.assertNetContainOps(
1103
[expand_dims_spec, test_bn_spec, squeeze_spec]
1106
workspace.RunNetOnce(train_init_net)
1107
workspace.RunNetOnce(train_net)
1109
schema.FeedRecord(input_record, [X])
1110
workspace.RunNetOnce(eval_net)
1112
schema.FeedRecord(input_record, [X])
1113
workspace.RunNetOnce(predict_net)
1116
X=hu.arrays(dims=[2, 5, 6]),
1117
use_layer_norm_op=st.booleans(),
1119
def testLayerNormalization(self, X, use_layer_norm_op):
1121
if not use_layer_norm_op:
1122
X = X.reshape(10, 6)
1124
input_record = self.new_record(schema.Scalar((np.float32, expect)))
1125
schema.FeedRecord(input_record, [X])
1126
ln_output = self.model.LayerNormalization(
1127
input_record, use_layer_norm_op=use_layer_norm_op
1129
self.assertEqual(schema.Scalar((np.float32, expect)), ln_output)
1130
self.model.output_schema = schema.Struct()
1132
train_init_net, train_net = self.get_training_nets(add_constants=True)
1133
workspace.RunNetOnce(train_init_net)
1134
workspace.RunNetOnce(train_net)
1137
X=hu.arrays(dims=[5, 2]),
1138
num_to_collect=st.integers(min_value=1, max_value=10),
1140
def testLastNWindowCollector(self, X, num_to_collect):
1141
input_record = self.new_record(schema.Scalar(np.float32))
1142
schema.FeedRecord(input_record, [X])
1143
last_n = self.model.LastNWindowCollector(input_record, num_to_collect)
1144
self.run_train_net_forward_only()
1145
output_record = schema.FetchRecord(last_n.last_n)
1146
start = max(0, 5 - num_to_collect)
1147
npt.assert_array_equal(X[start:], output_record())
1148
num_visited = schema.FetchRecord(last_n.num_visited)
1149
npt.assert_array_equal([5], num_visited())
1152
X=hu.arrays(dims=[5, 2]),
1153
num_to_collect=st.integers(min_value=3, max_value=3),
1155
@settings(deadline=1000)
1156
def testReservoirSamplingWithID(self, X, num_to_collect):
1157
ID = np.array([1, 2, 3, 1, 2], dtype=np.int64)
1158
input_record = self.new_record(
1160
('record', schema.Struct(1161
('dense', schema.Scalar()),1163
('object_id', schema.Scalar(np.int64)),1166
schema.FeedRecord(input_record, [X, ID])
1167
packed_record = self.model.PackRecords(
1168
input_record.record, 1, fields=input_record.record.field_names())
1169
reservoir_input = schema.Struct(
1170
('data', packed_record),1171
('object_id', input_record.object_id),1173
reservoir = self.model.ReservoirSampling(
1174
reservoir_input, num_to_collect)
1175
self.model.output_schema = schema.Struct()
1176
train_init_net, train_net = \
1177
layer_model_instantiator.generate_training_nets_forward_only(
1179
workspace.RunNetOnce(train_init_net)
1180
workspace.CreateNet(train_net)
1181
workspace.RunNet(train_net.Proto().name, num_iter=2)
1182
num_visited = schema.FetchRecord(reservoir.num_visited)
1183
npt.assert_array_equal([3], num_visited())
1184
for param in self.model.params:
1185
serialized = workspace.SerializeBlob(str(param))
1186
workspace.DeserializeBlob(str(param), serialized)
1187
ID = np.array([3, 5, 3, 3, 5], dtype=np.int64)
1188
schema.FeedRecord(input_record.object_id, [ID])
1189
workspace.RunNet(train_net.Proto().name, num_iter=2)
1190
num_visited = schema.FetchRecord(reservoir.num_visited)
1191
npt.assert_array_equal([2], num_visited())
1193
def testUniformSampling(self):
1194
input_record = self.new_record(schema.Scalar(np.int32))
1195
input_array = np.array([3, 10, 11, 15, 20, 99], dtype=np.int32)
1196
schema.FeedRecord(input_record, [input_array])
1199
uniform_sampling_output = self.model.UniformSampling(
1200
input_record, num_samples, num_elements)
1201
self.model.loss = uniform_sampling_output
1202
self.run_train_net()
1203
samples = workspace.FetchBlob(uniform_sampling_output.samples())
1204
sampling_prob = workspace.FetchBlob(
1205
uniform_sampling_output.sampling_prob())
1206
self.assertEqual(num_samples, len(samples))
1207
np.testing.assert_array_equal(input_array, samples[:len(input_array)])
1208
np.testing.assert_almost_equal(
1209
np.array([float(num_samples) / num_elements] * num_samples,
1214
def testUniformSamplingWithIncorrectSampleSize(self):
1215
input_record = self.new_record(schema.Scalar(np.int32))
1218
with self.assertRaises(AssertionError):
1219
self.model.UniformSampling(input_record, num_samples, num_elements)
1221
def testGatherRecord(self):
1222
indices = np.array([1, 3, 4], dtype=np.int32)
1223
dense = np.array(list(range(20)), dtype=np.float32).reshape(10, 2)
1224
lengths = np.array(list(range(10)), dtype=np.int32)
1225
items = np.array(list(range(lengths.sum())), dtype=np.int64)
1226
items_lengths = np.array(list(range(lengths.sum())), dtype=np.int32)
1227
items_items = np.array(list(range(items_lengths.sum())), dtype=np.int64)
1228
record = self.new_record(schema.Struct(
1229
('dense', schema.Scalar(np.float32)),1230
('sparse', schema.Struct(1231
('list', schema.List(np.int64)),1232
('list_of_list', schema.List(schema.List(np.int64))),1234
('empty_struct', schema.Struct())1236
indices_record = self.new_record(schema.Scalar(np.int32))
1237
input_record = schema.Struct(
1238
('indices', indices_record),1243
[indices, dense, lengths, items, lengths, items_lengths,
1245
gathered_record = self.model.GatherRecord(input_record)
1246
self.assertTrue(schema.equal_schemas(gathered_record, record))
1248
self.run_train_net_forward_only()
1249
gathered_dense = workspace.FetchBlob(gathered_record.dense())
1250
np.testing.assert_array_equal(
1251
np.concatenate([dense[i:i + 1] for i in indices]), gathered_dense)
1252
gathered_lengths = workspace.FetchBlob(
1253
gathered_record.sparse.list.lengths())
1254
np.testing.assert_array_equal(
1255
np.concatenate([lengths[i:i + 1] for i in indices]),
1257
gathered_items = workspace.FetchBlob(
1258
gathered_record.sparse.list.items())
1259
offsets = lengths.cumsum() - lengths
1260
np.testing.assert_array_equal(
1262
items[offsets[i]: offsets[i] + lengths[i]]
1266
gathered_items_lengths = workspace.FetchBlob(
1267
gathered_record.sparse.list_of_list.items.lengths())
1268
np.testing.assert_array_equal(
1270
items_lengths[offsets[i]: offsets[i] + lengths[i]]
1273
gathered_items_lengths
1281
nested_offsets.append(nested_offset)
1284
nested_offset += items_lengths[j]
1285
nested_length += items_lengths[j]
1287
nested_lengths.append(nested_length)
1289
gathered_items_items = workspace.FetchBlob(
1290
gathered_record.sparse.list_of_list.items.items())
1291
np.testing.assert_array_equal(
1293
items_items[nested_offsets[i]:
1294
nested_offsets[i] + nested_lengths[i]]
1297
gathered_items_items
1300
def testMapToRange(self):
1301
input_record = self.new_record(schema.Scalar(np.int32))
1302
indices_blob = self.model.MapToRange(input_record,
1303
max_index=100).indices
1304
self.model.output_schema = schema.Struct()
1306
train_init_net, train_net = self.get_training_nets()
1310
[np.array([10, 3, 20, 99, 15, 11, 3, 11], dtype=np.int32)]
1312
workspace.RunNetOnce(train_init_net)
1313
workspace.RunNetOnce(train_net)
1314
indices = workspace.FetchBlob(indices_blob())
1315
np.testing.assert_array_equal(
1316
np.array([1, 2, 3, 4, 5, 6, 2, 6], dtype=np.int32),
1322
[np.array([10, 3, 23, 35, 60, 15, 10, 15], dtype=np.int32)]
1324
workspace.RunNetOnce(train_net)
1325
indices = workspace.FetchBlob(indices_blob())
1326
np.testing.assert_array_equal(
1327
np.array([1, 2, 7, 8, 9, 5, 1, 5], dtype=np.int32),
1331
eval_net = self.get_eval_net()
1335
[np.array([10, 3, 23, 35, 60, 15, 200], dtype=np.int32)]
1337
workspace.RunNetOnce(eval_net)
1338
indices = workspace.FetchBlob(indices_blob())
1339
np.testing.assert_array_equal(
1340
np.array([1, 2, 7, 8, 9, 5, 0], dtype=np.int32),
1346
[np.array([10, 3, 23, 15, 101, 115], dtype=np.int32)]
1348
workspace.RunNetOnce(eval_net)
1349
indices = workspace.FetchBlob(indices_blob())
1350
np.testing.assert_array_equal(
1351
np.array([1, 2, 7, 5, 0, 0], dtype=np.int32),
1355
predict_net = self.get_predict_net()
1359
[np.array([3, 3, 20, 23, 151, 35, 60, 15, 200], dtype=np.int32)]
1361
workspace.RunNetOnce(predict_net)
1362
indices = workspace.FetchBlob(indices_blob())
1363
np.testing.assert_array_equal(
1364
np.array([2, 2, 3, 7, 0, 8, 9, 5, 0], dtype=np.int32),
1368
def testSelectRecordByContext(self):
1369
float_features = self.model.input_feature_schema.float_features
1371
float_array = np.array([1.0, 2.0], dtype=np.float32)
1373
schema.FeedRecord(float_features, [float_array])
1375
with Tags(Tags.EXCLUDE_FROM_PREDICTION):
1376
log_float_features = self.model.Log(float_features, 1)
1377
joined = self.model.SelectRecordByContext(
1379
(InstantiationContext.PREDICTION, float_features),
1380
(InstantiationContext.TRAINING, log_float_features),
1381
# TODO: TRAIN_ONLY layers are also generated in eval
1382
(InstantiationContext.EVAL, log_float_features),
1386
# model.output_schema has to a struct
1387
self.model.output_schema = schema.Struct((
1390
predict_net = layer_model_instantiator.generate_predict_net(self.model)
1391
workspace.RunNetOnce(predict_net)
1392
predict_output = schema.FetchRecord(predict_net.output_record())
1393
npt.assert_array_equal(float_array,
1394
predict_output['joined']())
1395
eval_net = layer_model_instantiator.generate_eval_net(self.model)
1396
workspace.RunNetOnce(eval_net)
1397
eval_output = schema.FetchRecord(eval_net.output_record())
1398
npt.assert_array_equal(np.log(float_array),
1399
eval_output['joined']())
1401
layer_model_instantiator.generate_training_nets_forward_only(
1405
workspace.RunNetOnce(train_net)
1406
train_output = schema.FetchRecord(train_net.output_record())
1407
npt.assert_array_equal(np.log(float_array),
1408
train_output['joined']())
1410
def testFunctionalLayer(self):
1411
def normalize(net, in_record, out_record):
1412
mean = net.ReduceFrontMean(in_record(), 1)
1414
[in_record(), mean],
1417
normalized = self.model.Functional(
1418
self.model.input_feature_schema.float_features, 1,
1419
normalize, name="normalizer")
1421
# Attach metadata to one of the outputs and use it in FC
1422
normalized.set_type((np.float32, 32))
1423
self.model.output_schema = self.model.FC(normalized, 2)
1425
predict_net = layer_model_instantiator.generate_predict_net(
1427
ops = predict_net.Proto().op
1428
assert len(ops) == 3
1429
assert ops[0].type == "ReduceFrontMean"
1430
assert ops[1].type == "Sub"
1431
assert ops[2].type == "FC"
1432
assert len(ops[0].input) == 1
1433
assert ops[0].input[0] ==\
1434
self.model.input_feature_schema.float_features()
1435
assert len(ops[1].output) == 1
1436
assert ops[1].output[0] in ops[2].input
1438
def testFunctionalLayerHelper(self):
1439
mean = self.model.ReduceFrontMean(
1440
self.model.input_feature_schema.float_features, 1)
1441
normalized = self.model.Sub(
1443
self.model.input_feature_schema.float_features, mean),
1445
# Attach metadata to one of the outputs and use it in FC
1446
normalized.set_type((np.float32, (32,)))
1447
self.model.output_schema = self.model.FC(normalized, 2)
1449
predict_net = layer_model_instantiator.generate_predict_net(
1451
ops = predict_net.Proto().op
1452
assert len(ops) == 3
1453
assert ops[0].type == "ReduceFrontMean"
1454
assert ops[1].type == "Sub"
1455
assert ops[2].type == "FC"
1456
assert len(ops[0].input) == 1
1457
assert ops[0].input[0] ==\
1458
self.model.input_feature_schema.float_features()
1459
assert len(ops[1].output) == 1
1460
assert ops[1].output[0] in ops[2].input
1462
def testFunctionalLayerHelperAutoInference(self):
1463
softsign = self.model.Softsign(
1464
schema.Tuple(self.model.input_feature_schema.float_features),
1466
assert softsign.field_type().base == np.float32
1467
assert softsign.field_type().shape == (32,)
1468
self.model.output_schema = self.model.FC(softsign, 2)
1470
predict_net = layer_model_instantiator.generate_predict_net(
1472
ops = predict_net.Proto().op
1473
assert len(ops) == 2
1474
assert ops[0].type == "Softsign"
1475
assert ops[1].type == "FC"
1476
assert len(ops[0].input) == 1
1477
assert ops[0].input[0] ==\
1478
self.model.input_feature_schema.float_features()
1479
assert len(ops[0].output) == 1
1480
assert ops[0].output[0] in ops[1].input
1482
def testHalfToFloatTypeInference(self):
1483
input = self.new_record(schema.Scalar((np.float32, (32,))))
1485
output = self.model.FloatToHalf(input, 1)
1486
assert output.field_type().base == np.float16
1487
assert output.field_type().shape == (32, )
1489
output = self.model.HalfToFloat(output, 1)
1490
assert output.field_type().base == np.float32
1491
assert output.field_type().shape == (32, )
1493
def testFunctionalLayerHelperAutoInferenceScalar(self):
1494
loss = self.model.AveragedLoss(self.model.input_feature_schema, 1)
1495
self.assertEqual(1, len(loss.field_types()))
1496
self.assertEqual(np.float32, loss.field_types()[0].base)
1497
self.assertEqual(tuple(), loss.field_types()[0].shape)
1499
def testFunctionalLayerInputCoercion(self):
1500
one = self.model.global_constants['ONE']
1501
two = self.model.Add([one, one], 1)
1502
self.model.loss = two
1503
self.run_train_net()
1504
data = workspace.FetchBlob(two.field_blobs()[0])
1505
np.testing.assert_array_equal([2.0], data)
1507
def testFunctionalLayerWithOutputNames(self):
1509
topk = self.model.TopK(
1510
self.model.input_feature_schema,
1511
output_names_or_num=['values', 'indices'],
1514
self.assertEqual(2, len(topk.field_types()))
1515
self.assertEqual(np.float32, topk.field_types()[0].base)
1516
self.assertEqual((k,), topk.field_types()[0].shape)
1517
self.assertEqual(np.int32, topk.field_types()[1].base)
1518
self.assertEqual((k,), topk.field_types()[1].shape)
1519
self.assertEqual(['TopK/values', 'TopK/indices'], topk.field_blobs())
1521
def testFunctionalLayerSameOperatorOutputNames(self):
1522
Con1 = self.model.ConstantFill([], 1, value=1)
1523
Con2 = self.model.ConstantFill([], 1, value=2)
1524
self.assertNotEqual(str(Con1), str(Con2))
1526
def testFunctionalLayerWithOutputDtypes(self):
1527
loss = self.model.AveragedLoss(
1528
self.model.input_feature_schema,
1530
output_dtypes=(np.float32, (1,)),
1532
self.assertEqual(1, len(loss.field_types()))
1533
self.assertEqual(np.float32, loss.field_types()[0].base)
1534
self.assertEqual((1,), loss.field_types()[0].shape)
1536
def testPropagateRequestOnly(self):
1537
# test case when output is request only
1538
input_record = self.new_record(schema.Struct(
1539
('input1', schema.Scalar((np.float32, (32, )))),1540
('input2', schema.Scalar((np.float32, (64, )))),1541
('input3', schema.Scalar((np.float32, (16, )))),1544
set_request_only(input_record)
1545
concat_output = self.model.Concat(input_record)
1546
self.assertEqual(is_request_only_scalar(concat_output), True)
1548
# test case when output is not request only
1549
input_record2 = self.new_record(schema.Struct(
1550
('input4', schema.Scalar((np.float32, (100, ))))1553
concat_output2 = self.model.Concat(input_record2)
1554
self.assertEqual(is_request_only_scalar(concat_output2), False)
1556
def testSetRequestOnly(self):
1557
input_record = schema.Scalar(np.int64)
1558
schema.attach_metadata_to_scalars(
1561
categorical_limit=100000000,
1563
feature_specs=schema.FeatureSpec(
1564
feature_ids=[1, 100, 1001]
1569
set_request_only(input_record)
1570
self.assertEqual(input_record.metadata.categorical_limit, 100000000)
1571
self.assertEqual(input_record.metadata.expected_value, 99)
1573
input_record.metadata.feature_specs.feature_ids,
1578
X=hu.arrays(dims=[5, 5]), # Shape of X is irrelevant
1579
dropout_for_eval=st.booleans(),
1581
def testDropout(self, X, dropout_for_eval):
1582
input_record = self.new_record(schema.Scalar((np.float32, (1,))))
1583
schema.FeedRecord(input_record, [X])
1584
d_output = self.model.Dropout(
1586
dropout_for_eval=dropout_for_eval
1588
self.assertEqual(schema.Scalar((np.float32, (1,))), d_output)
1589
self.model.output_schema = schema.Struct()
1591
train_init_net, train_net = self.get_training_nets()
1593
input_blob = input_record.field_blobs()[0]
1594
output_blob = d_output.field_blobs()[0]
1596
with_d_spec = OpSpec(
1599
[output_blob, None],
1600
{'is_test': 0, 'ratio': 0.5}1603
without_d_spec = OpSpec(
1606
[output_blob, None],
1607
{'is_test': 1, 'ratio': 0.5}1610
self.assertNetContainOps(
1615
eval_net = self.get_eval_net()
1616
predict_net = self.get_predict_net()
1618
if dropout_for_eval:
1619
self.assertNetContainOps(
1623
self.assertNetContainOps(
1628
self.assertNetContainOps(
1632
self.assertNetContainOps(
1637
workspace.RunNetOnce(train_init_net)
1638
workspace.RunNetOnce(train_net)
1640
schema.FeedRecord(input_record, [X])
1641
workspace.RunNetOnce(eval_net)
1643
schema.FeedRecord(input_record, [X])
1644
workspace.RunNetOnce(predict_net)
1647
num_inputs=st.integers(1, 3),
1648
batch_size=st.integers(5, 10)
1650
def testMergeIdListsLayer(self, num_inputs, batch_size):
1652
for _ in range(num_inputs):
1653
lengths = np.random.randint(5, size=batch_size).astype(np.int32)
1654
size = lengths.sum()
1655
values = np.random.randint(1, 10, size=size).astype(np.int64)
1656
inputs.append(lengths)
1657
inputs.append(values)
1658
input_schema = schema.Tuple(
1660
schema.Scalar(dtype=np.int64, metadata=schema.Metadata(
1661
categorical_limit=20
1662
))) for _ in range(num_inputs)]
1665
input_record = schema.NewRecord(self.model.net, input_schema)
1666
schema.FeedRecord(input_record, inputs)
1667
output_schema = self.model.MergeIdLists(input_record)
1668
assert schema.equal_schemas(
1669
output_schema, IdList,
1670
check_field_names=False)
1673
batch_size=st.integers(min_value=2, max_value=10),
1674
input_dims=st.integers(min_value=5, max_value=10),
1675
output_dims=st.integers(min_value=5, max_value=10),
1676
bandwidth=st.floats(min_value=0.1, max_value=5),
1678
def testRandomFourierFeatures(self, batch_size, input_dims, output_dims, bandwidth):
1680
def _rff_hypothesis_test(rff_output, X, W, b, scale):
1682
Runs hypothesis test for Semi Random Features layer.
1685
rff_output -- output of net after running random fourier features layer
1687
W -- weight parameter from train_init_net
1688
b -- bias parameter from train_init_net
1689
scale -- value by which to scale the output vector
1691
output = workspace.FetchBlob(rff_output)
1692
output_ref = scale * np.cos(np.dot(X, np.transpose(W)) + b)
1693
npt.assert_allclose(output, output_ref, rtol=1e-3, atol=1e-3)
1695
X = np.random.random((batch_size, input_dims)).astype(np.float32)
1696
scale = np.sqrt(2.0 / output_dims)
1697
input_record = self.new_record(schema.Scalar((np.float32, (input_dims,))))
1698
schema.FeedRecord(input_record, [X])
1699
input_blob = input_record.field_blobs()[0]
1700
rff_output = self.model.RandomFourierFeatures(input_record,
1703
self.model.output_schema = schema.Struct()
1706
schema.Scalar((np.float32, (output_dims, ))),
1710
train_init_net, train_net = self.get_training_nets()
1712
# Init net assertions
1714
OpSpec("GaussianFill", None, None),1715
OpSpec("UniformFill", None, None),1717
init_ops = self._test_net(train_init_net, init_ops_list)
1718
W = workspace.FetchBlob(self.model.layers[0].w)
1719
b = workspace.FetchBlob(self.model.layers[0].b)
1721
# Operation specifications
1722
fc_spec = OpSpec("FC", [input_blob, init_ops[0].output[0],1723
init_ops[1].output[0]], None)
1724
cosine_spec = OpSpec("Cos", None, None)1725
scale_spec = OpSpec("Scale", None, rff_output.field_blobs(),1733
# Train net assertions
1734
self._test_net(train_net, ops_list)
1735
_rff_hypothesis_test(rff_output(), X, W, b, scale)
1737
# Eval net assertions
1738
eval_net = self.get_eval_net()
1739
self._test_net(eval_net, ops_list)
1740
_rff_hypothesis_test(rff_output(), X, W, b, scale)
1742
# Predict net assertions
1743
predict_net = self.get_predict_net()
1744
self._test_net(predict_net, ops_list)
1745
_rff_hypothesis_test(rff_output(), X, W, b, scale)
1748
batch_size=st.integers(min_value=2, max_value=10),
1749
input_dims=st.integers(min_value=5, max_value=10),
1750
output_dims=st.integers(min_value=5, max_value=10),
1751
s=st.integers(min_value=0, max_value=3),
1752
scale=st.floats(min_value=0.1, max_value=5),
1753
set_weight_as_global_constant=st.booleans()
1755
def testArcCosineFeatureMap(self, batch_size, input_dims, output_dims, s, scale,
1756
set_weight_as_global_constant):
1758
def _arc_cosine_hypothesis_test(ac_output, X, W, b, s):
1760
Runs hypothesis test for Arc Cosine layer.
1763
ac_output -- output of net after running arc cosine layer
1765
W -- weight parameter from train_init_net
1766
b -- bias parameter from train_init_net
1767
s -- degree parameter
1769
# Get output from net
1770
net_output = workspace.FetchBlob(ac_output)
1772
# Computing output directly
1773
x_rand = np.matmul(X, np.transpose(W)) + b
1774
x_pow = np.power(x_rand, s)
1776
h_rand_features = np.piecewise(x_rand,
1777
[x_rand <= 0, x_rand > 0],
1780
h_rand_features = np.piecewise(x_rand,
1781
[x_rand <= 0, x_rand > 0],
1782
[0, lambda x: x / (1 + x)])
1783
output_ref = np.multiply(x_pow, h_rand_features)
1785
# Comparing net output and computed output
1786
npt.assert_allclose(net_output, output_ref, rtol=1e-3, atol=1e-3)
1788
X = np.random.normal(size=(batch_size, input_dims)).astype(np.float32)
1789
input_record = self.new_record(schema.Scalar((np.float32, (input_dims,))))
1790
schema.FeedRecord(input_record, [X])
1791
input_blob = input_record.field_blobs()[0]
1793
ac_output = self.model.ArcCosineFeatureMap(
1798
set_weight_as_global_constant=set_weight_as_global_constant
1800
self.model.output_schema = schema.Struct()
1802
schema.Scalar((np.float32, (output_dims, ))),
1806
train_init_net, train_net = self.get_training_nets()
1808
# Run create_init_net to initialize the global constants, and W and b
1809
workspace.RunNetOnce(train_init_net)
1810
workspace.RunNetOnce(self.model.create_init_net(name='init_net'))
1812
if set_weight_as_global_constant:
1813
W = workspace.FetchBlob(
1814
self.model.global_constants['arc_cosine_feature_map_fixed_rand_W']
1816
b = workspace.FetchBlob(
1817
self.model.global_constants['arc_cosine_feature_map_fixed_rand_b']
1820
W = workspace.FetchBlob(self.model.layers[0].random_w)
1821
b = workspace.FetchBlob(self.model.layers[0].random_b)
1823
# Operation specifications
1824
fc_spec = OpSpec("FC", [input_blob, None, None], None)1825
softsign_spec = OpSpec("Softsign", None, None)1826
relu_spec = OpSpec("Relu", None, None)1827
relu_spec_output = OpSpec("Relu", None, ac_output.field_blobs())1828
pow_spec = OpSpec("Pow", None, None, {'exponent': float(s - 1)})1829
mul_spec = OpSpec("Mul", None, ac_output.field_blobs())1850
# Train net assertions
1851
self._test_net(train_net, ops_list)
1852
_arc_cosine_hypothesis_test(ac_output(), X, W, b, s)
1854
# Eval net assertions
1855
eval_net = self.get_eval_net()
1856
self._test_net(eval_net, ops_list)
1857
_arc_cosine_hypothesis_test(ac_output(), X, W, b, s)
1859
# Predict net assertions
1860
predict_net = self.get_predict_net()
1861
self._test_net(predict_net, ops_list)
1862
_arc_cosine_hypothesis_test(ac_output(), X, W, b, s)
1865
batch_size=st.integers(min_value=2, max_value=10),
1866
input_dims=st.integers(min_value=5, max_value=10),
1867
output_dims=st.integers(min_value=5, max_value=10),
1868
s=st.integers(min_value=0, max_value=3),
1869
scale=st.floats(min_value=0.1, max_value=5),
1870
set_weight_as_global_constant=st.booleans(),
1871
use_struct_input=st.booleans(),
1873
def testSemiRandomFeatures(self, batch_size, input_dims, output_dims, s, scale,
1874
set_weight_as_global_constant, use_struct_input):
1876
def _semi_random_hypothesis_test(srf_output, X_full, X_random, rand_w,
1879
Runs hypothesis test for Semi Random Features layer.
1882
srf_output -- output of net after running semi random features layer
1883
X_full -- full input data
1884
X_random -- random-output input data
1885
rand_w -- random-initialized weight parameter from train_init_net
1886
rand_b -- random-initialized bias parameter from train_init_net
1887
s -- degree parameter
1890
# Get output from net
1891
net_output = workspace.FetchBlob(srf_output)
1893
# Fetch learned parameter blobs
1894
learned_w = workspace.FetchBlob(self.model.layers[0].learned_w)
1895
learned_b = workspace.FetchBlob(self.model.layers[0].learned_b)
1897
# Computing output directly
1898
x_rand = np.matmul(X_random, np.transpose(rand_w)) + rand_b
1899
x_learn = np.matmul(X_full, np.transpose(learned_w)) + learned_b
1900
x_pow = np.power(x_rand, s)
1902
h_rand_features = np.piecewise(x_rand,
1903
[x_rand <= 0, x_rand > 0],
1906
h_rand_features = np.piecewise(x_rand,
1907
[x_rand <= 0, x_rand > 0],
1908
[0, lambda x: x / (1 + x)])
1909
output_ref = np.multiply(np.multiply(x_pow, h_rand_features), x_learn)
1911
# Comparing net output and computed output
1912
npt.assert_allclose(net_output, output_ref, rtol=1e-3, atol=1e-3)
1914
X_full = np.random.normal(size=(batch_size, input_dims)).astype(np.float32)
1915
if use_struct_input:
1916
X_random = np.random.normal(size=(batch_size, input_dims)).\
1918
input_data = [X_full, X_random]
1919
input_record = self.new_record(schema.Struct(
1920
('full', schema.Scalar(1921
(np.float32, (input_dims,))
1923
('random', schema.Scalar(1924
(np.float32, (input_dims,))
1929
input_data = [X_full]
1930
input_record = self.new_record(schema.Scalar(
1931
(np.float32, (input_dims,))
1934
schema.FeedRecord(input_record, input_data)
1935
srf_output = self.model.SemiRandomFeatures(
1940
scale_learned=scale,
1941
set_weight_as_global_constant=set_weight_as_global_constant
1944
self.model.output_schema = schema.Struct()
1948
('full', schema.Scalar(1949
(np.float32, (output_dims,))
1951
('random', schema.Scalar(1952
(np.float32, (output_dims,))
1959
OpSpec("GaussianFill", None, None),1960
OpSpec("UniformFill", None, None),1961
OpSpec("GaussianFill", None, None),1962
OpSpec("UniformFill", None, None),1964
train_init_net, train_net = self.get_training_nets()
1966
# Need to run to initialize the global constants for layer
1967
workspace.RunNetOnce(self.model.create_init_net(name='init_net'))
1969
if set_weight_as_global_constant:
1970
# If weight params are global constants, they won't be in train_init_net
1971
init_ops = self._test_net(train_init_net, init_ops_list[:2])
1972
rand_w = workspace.FetchBlob(
1973
self.model.global_constants['semi_random_features_fixed_rand_W']
1975
rand_b = workspace.FetchBlob(
1976
self.model.global_constants['semi_random_features_fixed_rand_b']
1979
# Operation specifications
1980
fc_random_spec = OpSpec("FC", [None, None, None], None)1981
fc_learned_spec = OpSpec("FC", [None, init_ops[0].output[0],1982
init_ops[1].output[0]], None)
1984
init_ops = self._test_net(train_init_net, init_ops_list)
1985
rand_w = workspace.FetchBlob(self.model.layers[0].random_w)
1986
rand_b = workspace.FetchBlob(self.model.layers[0].random_b)
1988
# Operation specifications
1989
fc_random_spec = OpSpec("FC", [None, init_ops[0].output[0],1990
init_ops[1].output[0]], None)
1991
fc_learned_spec = OpSpec("FC", [None, init_ops[2].output[0],1992
init_ops[3].output[0]], None)
1994
softsign_spec = OpSpec("Softsign", None, None)1995
relu_spec = OpSpec("Relu", None, None)1996
relu_output_spec = OpSpec("Relu", None, srf_output.random.field_blobs())1997
pow_spec = OpSpec("Pow", None, None, {'exponent': float(s - 1)})1998
mul_interim_spec = OpSpec("Mul", None, srf_output.random.field_blobs())1999
mul_spec = OpSpec("Mul", None, srf_output.full.field_blobs())2026
# Train net assertions
2027
self._test_net(train_net, ops_list)
2028
_semi_random_hypothesis_test(srf_output.full(), X_full, X_random,
2031
# Eval net assertions
2032
eval_net = self.get_eval_net()
2033
self._test_net(eval_net, ops_list)
2034
_semi_random_hypothesis_test(srf_output.full(), X_full, X_random,
2037
# Predict net assertions
2038
predict_net = self.get_predict_net()
2039
self._test_net(predict_net, ops_list)
2040
_semi_random_hypothesis_test(srf_output.full(), X_full, X_random,
2058
input_record = self.new_record(schema.Scalar((np.float32, (H, W, C))))
2059
X = np.random.random((batch_size, H, W, C)).astype(np.float32)
2060
schema.FeedRecord(input_record, [X])
2061
conv = self.model.Conv(
2076
schema.Scalar((np.float32, (output_dims,))),
2080
self.run_train_net_forward_only()
2081
output_record = schema.FetchRecord(conv)
2082
# check the number of output channels is the same as input in this example
2083
assert output_record.field_types()[0].shape == (H, W, output_dims)
2084
assert output_record().shape == (batch_size, H, W, output_dims)
2086
train_init_net, train_net = self.get_training_nets()
2087
# Init net assertions
2088
init_ops = self.assertNetContainOps(
2091
OpSpec("XavierFill", None, None),2092
OpSpec("ConstantFill", None, None),2098
input_record.field_blobs()[0],
2099
init_ops[0].output[0],
2100
init_ops[1].output[0],
2105
# Train net assertions
2106
self.assertNetContainOps(train_net, [conv_spec])
2108
# Predict net assertions
2109
predict_net = self.get_predict_net()
2110
self.assertNetContainOps(predict_net, [conv_spec])
2112
# Eval net assertions
2113
eval_net = self.get_eval_net()
2114
self.assertNetContainOps(eval_net, [conv_spec])
2117
num=st.integers(min_value=10, max_value=100),
2118
feed_weight=st.booleans(),
2119
use_inv_var_parameterization=st.booleans(),
2120
use_log_barrier=st.booleans(),
2121
enable_diagnose=st.booleans(),
2124
@settings(deadline=1000)
2125
def testAdaptiveWeight(
2126
self, num, feed_weight, use_inv_var_parameterization, use_log_barrier,
2127
enable_diagnose, gc, dc
2129
input_record = self.new_record(schema.RawTuple(num))
2130
data = np.random.random(num)
2132
input_record, [np.array(x).astype(np.float32) for x in data]
2134
weights = np.random.random(num) if feed_weight else None
2135
result = self.model.AdaptiveWeight(
2139
'inv_var' if use_inv_var_parameterization else 'log_std'
2142
'log_barrier' if use_log_barrier else 'pos_grad_proj'
2144
enable_diagnose=enable_diagnose
2146
train_init_net, train_net = self.get_training_nets(True)
2147
workspace.RunNetOnce(train_init_net)
2148
workspace.RunNetOnce(train_net)
2149
result = workspace.FetchBlob(result())
2151
weights = np.array([1. / num for _ in range(num)])
2152
expected = np.sum(weights * data + 0.5 * np.log(1. / 2. / weights))
2153
npt.assert_allclose(expected, result, atol=1e-4, rtol=1e-4)
2155
assert len(self.model.ad_hoc_plot_blobs) == num
2156
reconst_weights_from_ad_hoc = np.array(
2157
[workspace.FetchBlob(b) for b in self.model.ad_hoc_plot_blobs]
2159
npt.assert_allclose(
2160
reconst_weights_from_ad_hoc, weights, atol=1e-4, rtol=1e-4
2163
assert len(self.model.ad_hoc_plot_blobs) == 0
2165
@given(num=st.integers(min_value=10, max_value=100), **hu.gcs)
2166
def testConstantWeight(self, num, gc, dc):
2167
input_record = self.new_record(schema.RawTuple(num))
2168
data = np.random.random(num)
2170
input_record, [np.array(x).astype(np.float32) for x in data]
2172
weights = np.random.random(num)
2173
result = self.model.ConstantWeight(input_record, weights=weights)
2174
train_init_net, train_net = self.get_training_nets(True)
2175
workspace.RunNetOnce(train_init_net)
2176
workspace.RunNetOnce(train_net)
2177
result = workspace.FetchBlob(result())
2178
expected = np.sum(weights * data)
2179
npt.assert_allclose(expected, result, atol=1e-4, rtol=1e-4)
2182
@settings(deadline=10000)
2183
def testHomotopyWeight(self, gc, dc):
2184
input_record = self.new_record(schema.RawTuple(2))
2185
data = np.random.random(2)
2187
input_record, [np.array(x).astype(np.float32) for x in data]
2189
# ensure: quad_life > 2 * half_life
2190
half_life = int(np.random.random() * 1e2 + 1)
2191
quad_life = int(np.random.random() * 1e3 + 2 * half_life + 1)
2192
min_weight = np.random.random()
2193
max_weight = np.random.random() + min_weight + 1e-5
2194
result = self.model.HomotopyWeight(
2196
min_weight=min_weight,
2197
max_weight=max_weight,
2198
half_life=half_life,
2199
quad_life=quad_life,
2201
train_init_net, train_net = self.get_training_nets(True)
2202
workspace.RunNetOnce(train_init_net)
2203
workspace.CreateNet(train_net)
2204
workspace.RunNet(train_net.Name(), num_iter=half_life)
2205
half_life_result = workspace.FetchBlob(result())
2206
workspace.RunNet(train_net.Name(), num_iter=quad_life - half_life)
2207
quad_life_result = workspace.FetchBlob(result())
2209
alpha = (min_weight + max_weight) / 2.
2210
beta = (min_weight + max_weight) / 2.
2211
expected_half_life_result = alpha * data[0] + beta * data[1]
2212
alpha = (3 * min_weight + max_weight) / 4.
2213
beta = (min_weight + 3 * max_weight) / 4.
2214
expected_quad_life_result = alpha * data[0] + beta * data[1]
2215
npt.assert_allclose(
2216
expected_half_life_result, half_life_result, atol=1e-2, rtol=1e-2
2218
npt.assert_allclose(
2219
expected_quad_life_result, quad_life_result, atol=1e-2, rtol=1e-2
2222
def _testLabelSmooth(self, categories, binary_prob_label, bsz):
2223
label = self.new_record(schema.Scalar((np.float32, (1, ))))
2224
label_np = np.random.randint(categories, size=bsz).astype(np.float32)
2225
schema.FeedRecord(label, [label_np])
2226
smooth_matrix_shape = (
2227
2 if binary_prob_label else (categories, categories)
2229
smooth_matrix = np.random.random(smooth_matrix_shape)
2230
smoothed_label = self.model.LabelSmooth(label, smooth_matrix)
2231
train_init_net, train_net = self.get_training_nets(True)
2232
workspace.RunNetOnce(train_init_net)
2233
workspace.RunNetOnce(train_net)
2234
smoothed_label_np = workspace.FetchBlob(smoothed_label())
2235
if binary_prob_label:
2236
expected = np.array(
2238
smooth_matrix[0] if x == 0.0 else smooth_matrix[1]
2243
expected = np.array([smooth_matrix[int(x)] for x in label_np])
2244
npt.assert_allclose(expected, smoothed_label_np, atol=1e-4, rtol=1e-4)
2247
categories=st.integers(min_value=2, max_value=10),
2248
bsz=st.integers(min_value=10, max_value=100),
2251
def testLabelSmoothForCategoricalLabel(self, categories, bsz, gc, dc):
2252
self._testLabelSmooth(categories, False, bsz)
2255
bsz=st.integers(min_value=10, max_value=100),
2258
def testLabelSmoothForBinaryProbLabel(self, bsz, gc, dc):
2259
self._testLabelSmooth(2, True, bsz)
2262
num_inputs=st.integers(min_value=2, max_value=10),
2263
batch_size=st.integers(min_value=2, max_value=10),
2264
input_dim=st.integers(min_value=5, max_value=10),
2265
seed=st.integers(1, 10),
2267
def testBlobWeightedSum(self, num_inputs, batch_size, input_dim, seed):
2269
def get_blob_weighted_sum():
2271
for i in range(num_inputs):
2272
w_blob_name = 'blob_weighted_sum/w_{0}'.format(i)2273
assert workspace.HasBlob(w_blob_name), (
2274
"cannot fine blob {}".format(w_blob_name)2276
w = workspace.FetchBlob(w_blob_name)
2280
input_data[idx] * weights[idx] for idx in range(num_inputs)
2284
np.random.seed(seed)
2285
expected_output_schema = schema.Scalar((np.float32, (input_dim,)))
2286
input_schema = schema.Tuple(
2287
*[expected_output_schema for _ in range(num_inputs)]
2290
np.random.random((batch_size, input_dim)).astype(np.float32)
2291
for _ in range(num_inputs)
2293
input_record = self.new_record(input_schema)
2294
schema.FeedRecord(input_record, input_data)
2296
# test output schema
2297
ws_output = self.model.BlobWeightedSum(input_record)
2298
self.assertEqual(len(self.model.layers), 1)
2299
assert schema.equal_schemas(ws_output, expected_output_schema)
2302
train_init_net, train_net = self.get_training_nets()
2303
workspace.RunNetOnce(train_init_net)
2304
workspace.RunNetOnce(train_net)
2305
output = workspace.FetchBlob(ws_output())
2306
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
2308
self.run_train_net_forward_only()
2309
output = workspace.FetchBlob(ws_output())
2310
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
2313
eval_net = self.get_eval_net()
2314
workspace.RunNetOnce(eval_net)
2315
output = workspace.FetchBlob(ws_output())
2316
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
2319
pred_net = self.get_predict_net()
2320
workspace.RunNetOnce(pred_net)
2321
output = workspace.FetchBlob(ws_output())
2322
npt.assert_almost_equal(get_blob_weighted_sum(), output, decimal=5)
2324
def testFeatureSparseToDenseGetAccessedFeatures(self):
2325
float_features_column = "float_features"
2326
float_features_type = "FLOAT"
2327
float_features_ids = [1, 2, 3]
2329
id_list_features_column = "id_list_features"
2330
id_list_features_type = "ID_LIST"
2331
id_list_features_ids = [4, 5, 6]
2333
id_score_list_features_column = "id_score_list_features"
2334
id_score_list_features_type = "ID_SCORE_LIST"
2335
id_score_list_features_ids = [7, 8 , 9]
2337
feature_names = ["a", "b", "c"]
2339
input_record = self.new_record(schema.Struct(
2340
(float_features_column, schema.Map(np.int32, np.float32)),
2341
(id_list_features_column,
2342
schema.Map(np.int32, schema.List(np.int64))),
2343
(id_score_list_features_column,
2344
schema.Map(np.int32, schema.Map(np.int64, np.float32))),
2349
float_features_column,
2351
feature_type=float_features_type,
2352
feature_ids=float_features_ids,
2353
feature_names=feature_names,
2357
id_list_features_column,
2359
feature_type=id_list_features_type,
2360
feature_ids=id_list_features_ids,
2361
feature_names=feature_names,
2365
id_score_list_features_column,
2367
feature_type=id_score_list_features_type,
2368
feature_ids=id_score_list_features_ids,
2369
feature_names=feature_names,
2374
self.model.FeatureSparseToDense(input_record, input_specs)
2376
expected_accessed_features = {2377
float_features_column: [
2378
AccessedFeatures(float_features_type, set(float_features_ids))],
2379
id_list_features_column: [
2380
AccessedFeatures(id_list_features_type, set(id_list_features_ids))],
2381
id_score_list_features_column: [
2382
AccessedFeatures(id_score_list_features_type, set(id_score_list_features_ids))],
2385
self.assertEqual(len(self.model.layers), 1)
2387
self.model.layers[0].get_accessed_features(),
2388
expected_accessed_features
2391
def test_get_key(self):
2392
def _is_id_list(input_record):
2393
return almost_equal_schemas(input_record, IdList)
2396
def _is_id_score_list(input_record):
2397
return almost_equal_schemas(input_record,
2399
check_field_types=False)
2401
def old_get_sparse_key_logic(input_record):
2402
if _is_id_list(input_record):
2403
sparse_key = input_record.items()
2404
elif _is_id_score_list(input_record):
2405
sparse_key = input_record.keys()
2407
raise NotImplementedError()
2410
id_score_list_record = schema.NewRecord(
2415
metadata=schema.Metadata(
2416
categorical_limit=1000
2424
get_key(id_score_list_record)(),
2425
old_get_sparse_key_logic(id_score_list_record)
2428
id_list_record = schema.NewRecord(
2433
metadata=schema.Metadata(categorical_limit=1000)
2439
get_key(id_list_record)(),
2440
old_get_sparse_key_logic(id_list_record)
2443
def testSparseLookupWithAttentionWeightOnIdScoreList(self):
2444
record = schema.NewRecord(
2449
metadata=schema.Metadata(categorical_limit=1000),
2455
embedding_after_pooling = self.model.SparseLookup(
2456
record, [embedding_dim], "Sum", use_external_weights=True
2458
self.model.output_schema = schema.Struct()
2460
schema.Scalar((np.float32, (embedding_dim,))), embedding_after_pooling
2463
train_init_net, train_net = self.get_training_nets()
2465
init_ops = self.assertNetContainOps(
2467
[OpSpec("UniformFill", None, None), OpSpec("ConstantFill", None, None)],2469
sparse_lookup_op_spec = OpSpec(
2470
"SparseLengthsWeightedSum",
2472
init_ops[0].output[0],
2477
[embedding_after_pooling()],
2479
self.assertNetContainOps(train_net, [sparse_lookup_op_spec])
2481
predict_net = self.get_predict_net()
2482
self.assertNetContainOps(predict_net, [sparse_lookup_op_spec])
2484
def testSparseItemwiseDropoutWithReplacement(self):
2485
input_record = schema.NewRecord(self.model.net, IdList)
2486
self.model.output_schema = schema.Struct()
2488
lengths_blob = input_record.field_blobs()[0]
2489
values_blob = input_record.field_blobs()[1]
2490
lengths = np.array([1] * 10).astype(np.int32)
2491
values = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9, 10]).astype(np.int64)
2492
workspace.FeedBlob(lengths_blob, lengths)
2493
workspace.FeedBlob(values_blob, values)
2495
out = self.model.SparseItemwiseDropoutWithReplacement(
2496
input_record, 0.0, 0.5, 1.0, -1, output_names_or_num=1)
2497
self.assertEqual(schema.List(schema.Scalar(np.int64,)), out)
2499
train_init_net, train_net = self.get_training_nets()
2500
eval_net = self.get_eval_net()
2501
predict_net = self.get_predict_net()
2503
workspace.RunNetOnce(train_init_net)
2504
workspace.RunNetOnce(train_net)
2505
out_values = workspace.FetchBlob(out.items())
2506
out_lengths = workspace.FetchBlob(out.lengths())
2507
self.assertBlobsEqual(out_values, values)
2508
self.assertBlobsEqual(out_lengths, lengths)
2510
workspace.RunNetOnce(eval_net)
2512
workspace.RunNetOnce(predict_net)
2513
predict_values = workspace.FetchBlob("values_auto_0")2514
predict_lengths = workspace.FetchBlob("lengths_auto_0")2515
self.assertBlobsEqual(predict_values, np.array([-1] * 10).astype(np.int64))
2516
self.assertBlobsEqual(predict_lengths, lengths)