pytorch

1

2

3

4

5

6
from caffe2.proto import caffe2_pb2
7
from caffe2.python import core
8
from hypothesis import assume, given, settings, HealthCheck
9
import caffe2.python.hypothesis_test_util as hu
10
import caffe2.python.serialized_test.serialized_test_util as serial
11
import hypothesis.strategies as st
12
import numpy as np
13
import unittest
14

15

16
class TestFcOperator(serial.SerializedTestCase):
17
    def _run_test(self, n, m, k, transposed, multi_dim, dtype, engine, gc, dc):
18
        if dtype == np.float16:
19
            # fp16 only supported with CUDA/HIP
20
            assume(core.IsGPUDeviceType(gc.device_type))
21
            dc = [d for d in dc if core.IsGPUDeviceType(d.device_type)]
22

23
        if engine == 'TENSORCORE':
24
            # TensorCore only makes sense with CUDA
25
            assume(gc.device_type == caffe2_pb2.CUDA)
26
            # ensures TensorCore kernels can be called
27
            m *= 8
28
            k *= 8
29
            n *= 8
30

31
        X = np.random.rand(m, k).astype(dtype) - 0.5
32
        if multi_dim:
33
            if transposed:
34
                W = np.random.rand(k, n, 1, 1).astype(dtype) - 0.5
35
            else:
36
                W = np.random.rand(n, k, 1, 1).astype(dtype) - 0.5
37
        else:
38
            if transposed:
39
                W = np.random.rand(k, n).astype(dtype) - 0.5
40
            else:
41
                W = np.random.rand(n, k).astype(dtype) - 0.5
42
        b = np.random.rand(n).astype(dtype) - 0.5
43

44
        def fc_op(X, W, b):
45
            return [np.dot(X, W.reshape(n, k).transpose()) + b.reshape(n)]
46

47
        def fc_transposed_op(X, W, b):
48
            return [np.dot(X, W.reshape(k, n)) + b.reshape(n)]
49

50
        op = core.CreateOperator(
51
            'FCTransposed' if transposed else 'FC',
52
            ['X', 'W', 'b'],
53
            'out',
54
            engine=engine,
55
        )
56

57
        if dtype == np.float16 and core.IsGPUDeviceType(gc.device_type):
58
            a = caffe2_pb2.Argument()
59
            a.i = 1
60
            a.name = "float16_compute"
61
            op.arg.extend([a])
62

63
        # Check against numpy reference
64
        # ReferenceChecks is flaky, Relaxing to 1e-3.
65
        threshold = 1e-3
66
        self.assertReferenceChecks(
67
            device_option=gc,
68
            op=op,
69
            inputs=[X, W, b],
70
            reference=fc_transposed_op if transposed else fc_op,
71
            threshold=threshold
72
        )
73
        # Check over multiple devices
74
        self.assertDeviceChecks(dc, op, [X, W, b], [0])
75

76
        # Gradient checks
77
        threshold = 0.5 if dtype == np.float16 else 0.005
78
        stepsize = 0.5 if dtype == np.float16 else 0.05
79
        for i in range(3):
80
            self.assertGradientChecks(gc, op, [X, W, b], i, [0],
81
                                      threshold=threshold, stepsize=stepsize)
82

83
    @settings(max_examples=50, suppress_health_check=[HealthCheck.filter_too_much])
84
    @serial.given(n=st.integers(1, 5),
85
           m=st.integers(0, 5),
86
           k=st.integers(1, 5),
87
           multi_dim=st.sampled_from([True, False]),
88
           dtype=st.sampled_from([np.float32, np.float16]),
89
           engine=st.sampled_from(['', 'TENSORCORE']),
90
           **hu.gcs)
91
    def test_fc(self, **kwargs):
92
        self._run_test(transposed=False, **kwargs)
93

94
    @settings(max_examples=50, suppress_health_check=[HealthCheck.filter_too_much])
95
    @given(n=st.integers(1, 5),
96
           m=st.integers(0, 5),
97
           k=st.integers(1, 5),
98
           multi_dim=st.sampled_from([True, False]),
99
           dtype=st.sampled_from([np.float32, np.float16]),
100
           engine=st.sampled_from(['', 'TENSORCORE']),
101
           **hu.gcs)
102
    def test_fc_transposed(self, **kwargs):
103
        self._run_test(transposed=True, **kwargs)
104

105

106
if __name__ == "__main__":
107
    import unittest
108
    unittest.main()
109