/
githubmirror
/
incubator-mxnet
Обзор
Документация
Войти
/
githubmirror
/
incubator-mxnet
Код
Запросы
0
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
tests/python/unittest/test_sparse_operator.py
2 083 строки
99 KB
Joe Evans
Port top-level-project updates from v1.x branch (#21162)
04 янв 2023, 15:09
Не верифицирован
04 янв 2023, 15:09
48d7f4a
Код
Авторство
О чём код?
# Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. from mxnet.test_utils import * from mxnet.base import MXNetError import pytest from common import assertRaises import random import warnings def is_scalar(var): return False if hasattr(var, "__len__") else True def get_result_type(call, dflt_stype): """Try to infer result storage type for a sparse matrix and a given unary operation""" if call is not None and dflt_stype != 'default': zero = np.zeros(([1])) result = do_normalize(call(zero)) if not almost_equal(result, zero, equal_nan=True): expected_result_type = 'default' else: if dflt_stype is not None: expected_result_type = dflt_stype; else: expected_result_type = 'default' else: expected_result_type = 'default' return expected_result_type def get_result_type_with_scalar(call, dflt_stype): """Try to infer result storage type when operating a sparse matrices and a scalar""" if call is not None and dflt_stype != 'default': zero = np.zeros(([1])) result = call(zero, 5) if not almost_equal(result, zero, equal_nan=True): expected_result_type = 'default' else: if dflt_stype is not None: expected_result_type = dflt_stype; else: expected_result_type = 'default' else: expected_result_type = 'default' return expected_result_type def get_result_type_2(call, dflt_stype): """Try to infer result storage type when operating on two sparse matrices""" if call is not None and dflt_stype != 'default': zero = np.zeros(([1])) need_default = False for outer in [zero, np.ones(zero.shape)]: for inner in [zero, np.ones(zero.shape)]: result = do_normalize(call(outer, inner)) if not almost_equal(result, zero, equal_nan=True): need_default = True break if need_default is True: break if not need_default and dflt_stype is not None: expected_result_type = dflt_stype else: expected_result_type = 'default' else: expected_result_type = 'default' return expected_result_type def get_result_type_3(call, dflt_stype): """Try to infer result storage type when operating on three sparse matrices""" if call is not None and dflt_stype != 'default': zero = np.zeros(([1])) need_default = False for moon in [zero]: for outer in [zero]: for inner in [zero]: res_1, res_2 = call(moon, outer, inner) result = do_normalize(res_1) if not almost_equal(result, zero, equal_nan=True): need_default = True break result = do_normalize(res_2) if not almost_equal(result, zero, equal_nan=True): need_default = True break if need_default is True: break if need_default is True: break if not need_default and dflt_stype is not None: expected_result_type = dflt_stype else: expected_result_type = 'default' else: expected_result_type = 'default' return expected_result_type def get_fw_bw_result_types(forward_numpy_call, fwd_res_dflt, backward_numpy_call, bwd_res_dflt): return (get_result_type(forward_numpy_call, fwd_res_dflt), get_result_type(backward_numpy_call, bwd_res_dflt)) def get_fw_bw_result_types_2(forward_numpy_call, fwd_res_dflt, backward_numpy_call, bwd_res_dflt): return (get_result_type(forward_numpy_call, fwd_res_dflt), get_result_type_2(backward_numpy_call, bwd_res_dflt)) def get_fw_bw_result_types_with_scalar(forward_numpy_call, fwd_res_dflt, backward_numpy_call, bwd_res_dflt): return (get_result_type_with_scalar(forward_numpy_call, fwd_res_dflt), get_result_type_with_scalar(backward_numpy_call, bwd_res_dflt)) def gen_rsp_random_indices(shape, density=.5, force_indices=None): assert density >= 0 and density <= 1 indices = set() if force_indices is not None: for val in force_indices: indices.add(int(val)) if not np.isclose(density, .0, rtol=1.e-3, atol=1.e-3, equal_nan=True) and len(shape) > 0: row_count = shape[0] for i in range(row_count): r = random.uniform(0, 1) if r <= density and len(indices) < shape[0]: indices.add(i) assert len(indices) <= shape[0] return list(indices) def all_zero(var): return 0 @pytest.mark.skip(reason="https://github.com/apache/incubator-mxnet/issues/18740") def test_elemwise_binary_ops(): # skip testing on GPU because only CPU ops are implemented if default_device().device_type is 'gpu': return def test_elemwise_binary_op(name, lhs_stype, rhs_stype, shape, forward_mxnet_call, forward_numpy_call, backward_numpy_call, lhs_grad_stype, rhs_grad_stype, expected_result_storage_type=None, modifier_func=None, lhs_density=.5, rhs_density=.5, force_lr_overlap=False, force_grad_overlap=False, ograd_density=0.0, skip_gradient_check=False, shuffle_csr_indices=True, verbose=False): if lhs_grad_stype is None: lhs_grad_stype = lhs_stype if rhs_grad_stype is None: rhs_grad_stype = rhs_stype lhs_grad_stype = get_result_type_3(backward_numpy_call, lhs_grad_stype) rhs_grad_stype = get_result_type_3(backward_numpy_call, rhs_grad_stype) if verbose is True: print("testing: {} lhs={}, rhs={}, lhs_grad_stype={}, rhs_grad_stype={}" .format(name, lhs_stype, rhs_stype, lhs_grad_stype, rhs_grad_stype)) # Output type should be same as lvalue type, unless otherwise specified if expected_result_storage_type is None: if lhs_stype == 'default' or rhs_stype == 'default': expected_result_storage_type = 'default' else: expected_result_storage_type = lhs_stype lhs = mx.symbol.Variable('lhs', stype=lhs_stype) rhs = mx.symbol.Variable('rhs', stype=rhs_stype) grad_stypes = dict() grad_stypes['lhs'] = lhs_grad_stype grad_stypes['rhs'] = rhs_grad_stype if lhs_stype == 'default': lhs_nd = rand_ndarray(shape, 'default') if abs(lhs_density) < 1e-4: func = all_zero else: func = modifier_func lhs_nd = mx.nd.array(assign_each(lhs_nd.asnumpy(), func)) else: lhs_nd = create_sparse_array_zd( shape, lhs_stype, density=lhs_density, modifier_func=modifier_func, shuffle_csr_indices=shuffle_csr_indices, rsp_indices=gen_rsp_random_indices( shape, density=lhs_density, force_indices=[(shape[0]/2)] if force_lr_overlap is True else None )) if rhs_stype == 'default': rhs_nd = rand_ndarray(shape, 'default') if abs(rhs_density) < 1e-4: func = all_zero else: func = modifier_func rhs_nd = mx.nd.array(assign_each(rhs_nd.asnumpy(), func)) else: rhs_nd = create_sparse_array_zd( shape, rhs_stype, density=rhs_density, modifier_func=modifier_func, shuffle_csr_indices=shuffle_csr_indices, rsp_indices=gen_rsp_random_indices( shape, density=rhs_density, force_indices=[(shape[0]/2)] if force_lr_overlap is True else None )) lhs_np = lhs_nd.asnumpy() rhs_np = rhs_nd.asnumpy() if verbose is True: print("lhs input: {}".format(lhs_np)) print("rhs input: {}".format(rhs_np)) out_np = forward_numpy_call(lhs_np, rhs_np) if verbose is True: print("out_np: {}".format(out_np)) test = forward_mxnet_call(lhs, rhs) location = {'lhs': lhs_nd, 'rhs': rhs_nd} outputs = check_symbolic_forward(test, location, [out_np], equal_nan=True) assert len(outputs) == 1 assert outputs[0].stype == expected_result_storage_type if verbose is True: print ("mx forward output: ", outputs[0].asnumpy()) print ("lhs_nd: ", lhs_nd.stype) print ("rhs_nd: ", rhs_nd.stype) print ("forward output: ", outputs[0].stype) if outputs[0].stype != 'default': out_grad = create_sparse_array_zd( shape, outputs[0].stype, density=ograd_density, data_init=1, modifier_func=lambda x: 2, shuffle_csr_indices=shuffle_csr_indices, rsp_indices=gen_rsp_random_indices( shape, density=ograd_density, force_indices=[(shape[0]/2)] if force_grad_overlap is True else None )) else: if abs(ograd_density) < 1e-4: out_grad = mx.nd.array(np.zeros(shape)) else: out_grad = mx.nd.array(np.ones(shape)) out_grad_np = out_grad.asnumpy() if verbose is True: print("out_grad_np", out_grad_np) ingrad_lhs_np, ingrad_rhs_np = backward_numpy_call(out_grad_np, lhs_np, rhs_np) if verbose is True: print("out_grad", out_grad.asnumpy()) print("ingrad_lhs_np", ingrad_lhs_np) print("ingrad_rhs_np", ingrad_rhs_np) igrads_result = check_symbolic_backward(test, location, [out_grad], [ingrad_lhs_np, ingrad_rhs_np], grad_stypes=grad_stypes, equal_nan=True) if verbose is True: print("ingrad_lhs", igrads_result['lhs'].asnumpy()) print("ingrad_rhs", igrads_result['rhs'].asnumpy()) assert len(igrads_result) == 2 if lhs_grad_stype is not None: assert igrads_result['lhs'].stype == lhs_grad_stype if rhs_grad_stype is not None: assert igrads_result['rhs'].stype == rhs_grad_stype if not skip_gradient_check: check_numeric_gradient(test, location, grad_stype_dict=grad_stypes) def check_all(l, r, check_function): assert l.shape == r.shape return check_function(l, r) def gt(l, r): return check_all(l, r, lambda a, b: a > b) def ge(l, r): return check_all(l, r, lambda a, b: a >= b) def lt(l, r): return check_all(l, r, lambda a, b: a < b) def le(l, r): return check_all(l, r, lambda a, b: a <= b) def elemwise_mul_stype(lstype, rstype): if lstype == rstype: return lstype elif lstype == 'default' and rstype == 'row_sparse': return 'row_sparse' elif lstype == 'row_sparse' and rstype == 'default': return 'row_sparse' else: return 'default' def elemwise_mul_lhs_grad_stype(lstype, rstype): return elemwise_mul_stype(elemwise_mul_stype(lstype, rstype), rstype) def elemwise_mul_rhs_grad_stype(lstype, rstype): return elemwise_mul_stype(elemwise_mul_stype(lstype, rstype), lstype) def check_elemwise_binary_ops(lhs_stype, rhs_stype, shape, lhs_grad_stype=None, rhs_grad_stype=None, lhs_density=.5, rhs_density=.5, force_lr_overlap=False, force_grad_overlap=False, ograd_density=0.0): test_elemwise_binary_op("elemwise_add", lhs_stype, rhs_stype, shape, lambda l, r: mx.sym.sparse.elemwise_add(l, r), lambda l, r: l + r, lambda outg, l, r: (outg, outg), lhs_grad_stype, rhs_grad_stype, ograd_density=ograd_density, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, lhs_density=lhs_density, rhs_density=rhs_density, verbose=False) if ((lhs_stype is 'default' and rhs_stype is 'row_sparse') or (lhs_stype is 'row_sparse' and rhs_stype is 'row_sparse') and (rhs_density == 0.0)): test_elemwise_binary_op("elemwise_add", lhs_stype, rhs_stype, shape, lambda l, r: mx.sym.sparse.elemwise_add(l, r, out=l), lambda l, r: l + r, lambda outg, l, r: (outg, outg), lhs_grad_stype, rhs_grad_stype, ograd_density=ograd_density, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, lhs_density=lhs_density, rhs_density=rhs_density, verbose=False) test_elemwise_binary_op("elemwise_sub", lhs_stype, rhs_stype, shape, lambda l, r: mx.sym.sparse.elemwise_sub(l, r, out=l), lambda l, r: l - r, lambda outg, l, r: (outg, -outg), lhs_grad_stype, rhs_grad_stype, ograd_density=ograd_density, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, lhs_density=lhs_density, rhs_density=rhs_density, verbose=False) if ((lhs_stype is 'row_sparse' and rhs_stype is 'row_sparse') and (lhs_density == 0.0)): test_elemwise_binary_op("elemwise_add", lhs_stype, rhs_stype, shape, lambda l, r: mx.sym.sparse.elemwise_add(l, r, out=r), lambda l, r: l + r, lambda outg, l, r: (outg, outg), lhs_grad_stype, rhs_grad_stype, ograd_density=ograd_density, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, lhs_density=lhs_density, rhs_density=rhs_density, verbose=False) test_elemwise_binary_op("elemwise_sub", lhs_stype, rhs_stype, shape, lambda l, r: mx.sym.sparse.elemwise_sub(l, r, out=l), lambda l, r: l - r, lambda outg, l, r: (outg, -outg), lhs_grad_stype, rhs_grad_stype, ograd_density=ograd_density, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, lhs_density=lhs_density, rhs_density=rhs_density, verbose=False) test_elemwise_binary_op("elemwise_sub", lhs_stype, rhs_stype, shape, lambda l, r: mx.sym.sparse.elemwise_sub(l, r), lambda l, r: l - r, lambda outg, l, r: (outg, -outg), lhs_grad_stype, rhs_grad_stype, ograd_density=ograd_density, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, lhs_density=lhs_density, rhs_density=rhs_density, verbose=False) test_elemwise_binary_op("elemwise_mul", lhs_stype, rhs_stype, shape, lambda l, r: mx.sym.sparse.elemwise_mul(l, r), lambda l, r: l * r, lambda outg, l, r: (outg * r, outg * l), elemwise_mul_lhs_grad_stype(lhs_stype, rhs_stype), elemwise_mul_rhs_grad_stype(lhs_stype, rhs_stype), expected_result_storage_type=elemwise_mul_stype(lhs_stype, rhs_stype), ograd_density=ograd_density, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, lhs_density=lhs_density, rhs_density=rhs_density, verbose=False) test_elemwise_binary_op("elemwise_div", lhs_stype, rhs_stype, shape, lambda l, r: mx.sym.sparse.elemwise_div(l, r), lambda l, r: l / r, lambda outg, l, r: (outg * (1/r), outg * (-l/(r*r))), lhs_grad_stype, rhs_grad_stype, modifier_func=lambda a: a if abs(a) > 0.25 else abs(a) + 1, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, lhs_density=lhs_density, rhs_density=rhs_density, ograd_density=ograd_density, expected_result_storage_type='default', skip_gradient_check=True, verbose=False) test_elemwise_binary_op("maximum", lhs_stype, rhs_stype, shape, lambda l, r: mx.sym._internal._maximum(l, r), lambda l, r: np.maximum(l, r), lambda outg, l, r: (outg * ge(l, r), outg * lt(l, r)), lhs_grad_stype, rhs_grad_stype, modifier_func=lambda a: a if abs(a) > 0.25 else abs(a) + 1, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, lhs_density=lhs_density, rhs_density=rhs_density, skip_gradient_check=True, ograd_density=ograd_density, verbose=False) test_elemwise_binary_op("minimum", lhs_stype, rhs_stype, shape, lambda l, r: mx.sym._internal._minimum(l, r), lambda l, r: np.minimum(l, r), lambda outg, l, r: (outg * le(l, r), outg * gt(l, r)), lhs_grad_stype, rhs_grad_stype, modifier_func=lambda a: a if abs(a) > 0.25 else abs(a) + 1, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, lhs_density=lhs_density, rhs_density=rhs_density, ograd_density=ograd_density, skip_gradient_check=True, verbose=False) test_elemwise_binary_op("hypot", lhs_stype, rhs_stype, shape, lambda l, r: mx.sym._internal._hypot(l, r), lambda l, r: np.hypot(l, r), lambda outg, l, r: ( outg * assign_each2( l, r, lambda a, b: a/np.sqrt(a * a + b * b)), outg * assign_each2( l, r, lambda a, b: b/np.sqrt(a * a + b * b)) ), lhs_grad_stype, rhs_grad_stype, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, lhs_density=lhs_density, rhs_density=rhs_density, ograd_density=ograd_density, skip_gradient_check=True, verbose=False) # Run basic tests with warnings.catch_warnings(): warnings.simplefilter("ignore") for _ in range(1): # Run defaults check_elemwise_binary_ops('default', 'default', rand_shape_2d(5, 5)) # Try different densities shape = rand_shape_2d(5, 5) for lhs_density in [0.0, random.uniform(0, 1), 1.0]: for rhs_density in [0.0, random.uniform(0, 1), 1.0]: for ograd_density in [0.0, random.uniform(0, 1), 1.0]: print("lhs_density={}, rhs_density={}, ograd_density={}, shape: {}".format( lhs_density, rhs_density, ograd_density, shape)) # Try row_sparse overlaps for force_lr_overlap in [False, True]: for force_grad_overlap in [False, True]: print(" force_lr_overlap={}, force_grad_overlap={}, shape={}". format(force_lr_overlap, force_grad_overlap, shape)) # Back to left-right overlap possiblities check_elemwise_binary_ops('row_sparse', 'row_sparse', shape, lhs_grad_stype='row_sparse', rhs_grad_stype='row_sparse', lhs_density=lhs_density, rhs_density=rhs_density, force_lr_overlap=force_lr_overlap, force_grad_overlap=force_grad_overlap, ograd_density=ograd_density) def test_elemwise_csr_same_zeros(): # Zeroes a = mx.nd.sparse.zeros('csr', (1,1)) b = mx.nd.elemwise_add(a,a) res = a.asnumpy() + a.asnumpy() assert_almost_equal(b.asnumpy(), res) def as_dense(arr): if arr.stype != 'default': return mx.nd.cast_storage(arr, stype='default') else: return arr; # Make sure that 0's look like 0's when we do a comparison def do_normalize(arr): ret = arr.copy() idx = np.isclose(arr, -0, rtol=1.e-3, atol=1.e-3, equal_nan=True) ret[idx] = 0 return ret def check_sparse_mathematical_core(name, stype, forward_mxnet_call, forward_numpy_call, backward_numpy_call=None, rhs_arg=None, data_init=9., grad_init=2., output_grad_stype=None, input_grad_stype=None, force_overlap=False, density=.5, ograd_density=.5, verbose=False, shuffle_csr_indices=True): if verbose is True: print("TESTING: " + name) data = mx.symbol.Variable('data', stype=stype) temp_input_grad_stype = input_grad_stype if temp_input_grad_stype is None: temp_input_grad_stype = stype if rhs_arg is not None: if is_scalar(rhs_arg): expected_result_type, expected_grad_result_type = \ get_fw_bw_result_types_with_scalar(forward_numpy_call, stype, backward_numpy_call, temp_input_grad_stype) else: expected_result_type, expected_grad_result_type = \ get_fw_bw_result_types_2(forward_numpy_call, stype, backward_numpy_call, temp_input_grad_stype) else: expected_result_type, expected_grad_result_type = \ get_fw_bw_result_types(forward_numpy_call, stype, backward_numpy_call, temp_input_grad_stype) if input_grad_stype is not None and input_grad_stype != expected_grad_result_type: print("{}: explicit override of deduced input grad type '{}' with '{}'".format( name, expected_grad_result_type, input_grad_stype)) expected_grad_result_type = input_grad_stype shape = rand_shape_2d() if verbose is True: print("Shape: ", shape, "density: ", density, "force_overlap", force_overlap) if stype == 'default': data_tmp = np.zeros(shape) if abs(density) >= 1e-4: data_tmp[:] = data_init arr_data = mx.nd.array(data_tmp) else: arr_data = create_sparse_array_zd( shape, stype, density=density, data_init=data_init, shuffle_csr_indices=shuffle_csr_indices, rsp_indices=gen_rsp_random_indices( shape, density=density, force_indices=[(shape[0]/2)] if force_overlap is True else None ) ) data_tmp = arr_data.asnumpy() if verbose is True: print("arr_data indices", arr_data.indices.asnumpy()) if verbose is True: print("input", data_tmp) if backward_numpy_call is None: arr_grad = None elif expected_grad_result_type == 'default': if abs(density) < 1e-4: arr_grad = mx.nd.zeros(shape) else: arr_grad = mx.nd.ones(shape) else: arr_grad = create_sparse_array_zd( shape, expected_grad_result_type, density=density, data_init=1, shuffle_csr_indices=shuffle_csr_indices, rsp_indices=gen_rsp_random_indices( shape, density=density, force_indices=[(shape[0]/2)] if force_overlap is True else None ) ) if rhs_arg is not None: test = forward_mxnet_call(data, rhs_arg) else: test = forward_mxnet_call(data) args = list() args.append(arr_data) if arr_grad is not None: exe_test = test._bind(default_device(), args=args, args_grad=[arr_grad]) else: exe_test = test._bind(default_device(), args=args) exe_test.forward(is_train=True) assert exe_test.outputs[0].stype == expected_result_type out = exe_test.outputs[0].asnumpy() if rhs_arg is not None: npout = forward_numpy_call(data_tmp, rhs_arg) else: npout = forward_numpy_call(data_tmp) if verbose is True: print("out", out) print("npout", npout) assert_almost_equal(out, npout, equal_nan=True) if backward_numpy_call is not None: if output_grad_stype == 'default' or output_grad_stype is None: out_grad = mx.nd.empty(shape) out_grad[:] = grad_init else: out_grad = create_sparse_array_zd( shape, output_grad_stype, density=density, data_init=grad_init, shuffle_csr_indices=shuffle_csr_indices, rsp_indices=gen_rsp_random_indices( shape, density=ograd_density, force_indices=[(shape[0]/2)] if force_overlap is True else None)) npout_grad = out_grad.asnumpy() if verbose is True: print("npout_grad", npout_grad) if rhs_arg is not None: temp = backward_numpy_call(data_tmp, rhs_arg) else: temp = backward_numpy_call(data_tmp) input_grad = npout_grad * temp if verbose is True: print(arr_grad.asnumpy()) exe_test.backward(out_grad) if verbose is True: print(arr_grad.asnumpy()) assert arr_grad.stype == expected_grad_result_type if verbose is True: print(name) print("arr_grad", arr_grad.asnumpy()) print("input_grad", input_grad) assert_almost_equal(arr_grad, input_grad, equal_nan=True) @pytest.mark.serial @pytest.mark.skip(reason='https://github.com/apache/incubator-mxnet/issues/18829') def test_sparse_mathematical_core(): def util_sign(a): if np.isclose(a, -0, rtol=1.e-3, atol=1.e-3, equal_nan=True): return 0 elif np.isclose(a, 0, rtol=1.e-3, atol=1.e-3, equal_nan=True): return 0 elif a < 0.0: return -1 else: # a > 0.0: return 1 # Check scalar binary operators def check_binary_op_with_scalar(stype, output_grad_stype=None, input_grad_stype=None, density=.5, ograd_density=.5, force_overlap=False,): # mul_scalar check_sparse_mathematical_core("mul_scalar", stype, lambda x, y: x * y, lambda x, y: x * y, lambda input, rhs: rhs, rhs_arg=5.0, data_init=2, grad_init=3, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, density=density, ograd_density=ograd_density, force_overlap=force_overlap, verbose=False) # plus_scalar check_sparse_mathematical_core("plus_scalar", stype, lambda x, y: x + y, lambda x, y: x + y, lambda input, rhs: 1, rhs_arg=5.0, data_init=2, grad_init=3, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, density=density, ograd_density=ograd_density, force_overlap=force_overlap, verbose=False) # minus_scalar check_sparse_mathematical_core("minus_scalar", stype, lambda x, y: x - y, lambda x, y: x - y, lambda input, rhs: 1, rhs_arg=5.0, data_init=2, grad_init=3, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, density=density, ograd_density=ograd_density, force_overlap=force_overlap, verbose=False) # Check many basic unary operators def check_mathematical_core(stype, output_grad_stype=None, input_grad_stype=None, force_overlap=False, density=.5, ograd_density=.5): # negative check_sparse_mathematical_core("negative", stype, lambda x: mx.sym.sparse.negative(x), lambda x: np.negative(x), force_overlap=force_overlap, density=density, input_grad_stype=input_grad_stype, ograd_density=ograd_density) # square check_sparse_mathematical_core("square", stype, lambda x: mx.sym.sparse.square(x), lambda x: np.square(x), lambda x: 2 * x, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density, verbose=False) # sqrt check_sparse_mathematical_core("sqrt", stype, lambda x: mx.sym.sparse.sqrt(x), lambda x: np.sqrt(x), lambda x: 1.0/(2.0 * np.sqrt(x)), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density, verbose=False) # cbrt check_sparse_mathematical_core("cbrt", stype, lambda x: mx.sym.sparse.cbrt(x), lambda x: np.cbrt(x), lambda x: 1.0/(3.0 * np.cbrt(x) * np.cbrt(x)), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density, verbose=False) # rint check_sparse_mathematical_core("rint", stype, lambda x: mx.sym.sparse.rint(x), lambda x: np.rint(x), force_overlap=force_overlap, density=density, input_grad_stype=input_grad_stype, ograd_density=ograd_density) # fix check_sparse_mathematical_core("fix", stype, lambda x: mx.sym.sparse.fix(x), lambda x: np.fix(x), force_overlap=force_overlap, density=density, input_grad_stype=input_grad_stype, ograd_density=ograd_density) # floor check_sparse_mathematical_core("floor", stype, lambda x: mx.sym.sparse.floor(x), lambda x: np.floor(x), force_overlap=force_overlap, input_grad_stype=input_grad_stype, density=density, ograd_density=ograd_density) # ceil check_sparse_mathematical_core("ceil", stype, lambda x: mx.sym.sparse.ceil(x), lambda x: np.ceil(x), force_overlap=force_overlap, input_grad_stype=input_grad_stype, density=density, ograd_density=ograd_density) # round check_sparse_mathematical_core("round", stype, lambda x: mx.sym.sparse.round(x), lambda x: np.round(x), force_overlap=force_overlap, input_grad_stype=input_grad_stype, density=density, ograd_density=ograd_density) # trunc check_sparse_mathematical_core("trunc", stype, lambda x: mx.sym.sparse.trunc(x), lambda x: np.trunc(x), force_overlap=force_overlap, input_grad_stype=input_grad_stype, density=density, ograd_density=ograd_density) # sign check_sparse_mathematical_core("sign", stype, lambda x: mx.sym.sparse.sign(x), lambda x: np.sign(x), lambda x: np.zeros(x.shape), output_grad_stype=output_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # log1p check_sparse_mathematical_core("log1p", stype, lambda x: mx.sym.sparse.log1p(x), lambda x: np.log1p(x), lambda x: 1. / (1.0 + x), data_init=0.5, grad_init=0.5, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # expm1 check_sparse_mathematical_core("expm1", stype, lambda x: mx.sym.sparse.expm1(x), lambda x: np.expm1(x), lambda x: np.exp(x), data_init=0.5, grad_init=0.5, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # sin check_sparse_mathematical_core("sin", stype, lambda x: mx.sym.sparse.sin(x), lambda x: np.sin(x), lambda x: np.cos(x), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # tan check_sparse_mathematical_core("tan", stype, lambda x: mx.sym.sparse.tan(x), lambda x: np.tan(x), lambda x: np.tan(x) ** 2 + 1, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, density=density, ograd_density=ograd_density) # arcsin check_sparse_mathematical_core("arcsin", stype, lambda x: mx.sym.sparse.arcsin(x), lambda x: np.arcsin(x), lambda x: 1. / (1. - x ** 2) ** (1. / 2.), data_init=0.5, grad_init=0.5, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # arctan check_sparse_mathematical_core("arctan", stype, lambda x: mx.sym.sparse.arctan(x), lambda x: np.arctan(x), lambda x: 1. / (x ** 2. + 1.), data_init=0.5, grad_init=0.5, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # degrees check_sparse_mathematical_core("degrees", stype, lambda x: mx.sym.sparse.degrees(x), lambda x: np.degrees(x), lambda x: assign_each(x, lambda a: 180./np.pi), data_init=0.5, grad_init=0.5, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # radians check_sparse_mathematical_core("radians", stype, lambda x: mx.sym.sparse.radians(x), lambda x: np.radians(x), lambda x: assign_each(x, lambda a: np.pi / 180.), data_init=0.6, grad_init=1, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # sinh check_sparse_mathematical_core("sinh", stype, lambda x: mx.sym.sparse.sinh(x), lambda x: np.sinh(x), lambda x: np.cosh(x), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # tanh check_sparse_mathematical_core("tanh", stype, lambda x: mx.sym.sparse.tanh(x), lambda x: np.tanh(x), lambda x: 1. - np.tanh(x) ** 2, data_init=0.5, grad_init=1, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # arcsinh check_sparse_mathematical_core("arcsinh", stype, lambda x: mx.sym.sparse.arcsinh(x), lambda x: np.arcsinh(x), lambda x: 1./(x**2 + 1.)**(1./2.), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # arctanh check_sparse_mathematical_core("arctanh", stype, lambda x: mx.sym.sparse.arctanh(x), lambda x: np.arctanh(x), lambda x: -1./(x**2 - 1.), data_init=0.5, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # abs check_sparse_mathematical_core("abs", stype, lambda x: mx.sym.sparse.abs(x), lambda x: np.abs(x), lambda x: assign_each(x, function=util_sign), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) if stype != "csr": # rsqrt check_sparse_mathematical_core("rsqrt", stype, lambda x: mx.sym.sparse.rsqrt(x), lambda x: 1 / np.sqrt(x), lambda x: -(1.0 / (2.0 * x * np.sqrt(x))), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # cos check_sparse_mathematical_core("cos", stype, lambda x: mx.sym.sparse.cos(x), lambda x: np.cos(x), lambda x: -np.sin(x), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # arccos check_sparse_mathematical_core("arccos", stype, lambda x: mx.sym.sparse.arccos(x), lambda x: np.arccos(x), lambda x: -1. / (1. - x ** 2.) ** (1. / 2.), data_init=0.5, grad_init=0.5, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # cosh check_sparse_mathematical_core("cosh", stype, lambda x: mx.sym.sparse.cosh(x), lambda x: np.cosh(x), lambda x: np.sinh(x), data_init=5, grad_init=5, output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # arccosh check_sparse_mathematical_core("arccosh", stype, lambda x: mx.sym.sparse.arccosh(x), lambda x: np.arccosh(x), lambda x: 1./(x**2 - 1.)**(1./2.), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # log10 check_sparse_mathematical_core("log10", stype, lambda x: mx.sym.sparse.log10(x), lambda x: np.log10(x), lambda x: 1. / (x * np.log(10.)), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # log2 check_sparse_mathematical_core("log2", stype, lambda x: mx.sym.sparse.log2(x), lambda x: np.log2(x), lambda x: 1. / (x * np.log(2.)), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) try: from scipy import special as scipy_special # On scipy v1.0, psi([0, -1, -2, -3, ...]) = [ inf, inf, inf, inf, ...] # On scipy v1.1, psi([0, -1, -2, -3, ...]) = [-inf, nan, nan, nan, ...] # Map the behavior of v1.1 psi() to that of v1.0 for ints <= 0 for consistency scipy_psi = np.vectorize(lambda x: np.inf if float(x).is_integer() and x <= 0 else scipy_special.psi(x)) # gamma check_sparse_mathematical_core("gamma", stype, lambda x: mx.sym.sparse.gamma(x), lambda x: scipy_special.gamma(x), lambda x: scipy_special.gamma(x) * scipy_psi(x), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) # gammaln check_sparse_mathematical_core("gammaln", stype, lambda x: mx.sym.sparse.gammaln(x), lambda x: scipy_special.gammaln(x), lambda x: scipy_psi(x), output_grad_stype=output_grad_stype, input_grad_stype=input_grad_stype, force_overlap=force_overlap, density=density, ograd_density=ograd_density) except ImportError: print("Could not import scipy. Skipping unit tests for special functions") for i in range(1): print("pass", i) for density in [0.0, random.uniform(0, 1), 1.0]: for ograd_density in [0.0, random.uniform(0, 1), 1.0]: for force_overlap in [False, True]: print("{}, {}, {}".format(density, ograd_density, force_overlap)) with warnings.catch_warnings(): warnings.simplefilter("ignore") # Check unary ops (unary fwd, binary bwd) check_mathematical_core('default', force_overlap=force_overlap, density=density, ograd_density=ograd_density) check_mathematical_core('row_sparse', force_overlap=force_overlap, density=density, ograd_density=ograd_density) check_mathematical_core('row_sparse', output_grad_stype='default', force_overlap=force_overlap, density=density, ograd_density=ograd_density) check_mathematical_core('row_sparse', output_grad_stype='row_sparse', force_overlap=force_overlap, density=density, ograd_density=ograd_density) check_mathematical_core('csr', output_grad_stype='default', force_overlap=force_overlap, density=density, ograd_density=ograd_density) check_mathematical_core('csr', output_grad_stype='csr', force_overlap=force_overlap, density=density, ograd_density=ograd_density) # Check binary with scalar ops check_binary_op_with_scalar('default', density=density, ograd_density=ograd_density, force_overlap=force_overlap) check_binary_op_with_scalar('row_sparse', density=density, ograd_density=ograd_density, force_overlap=force_overlap) check_binary_op_with_scalar('row_sparse', output_grad_stype='default', density=density, ograd_density=ograd_density, force_overlap=force_overlap) check_binary_op_with_scalar('row_sparse', output_grad_stype='row_sparse', density=density, ograd_density=ograd_density, force_overlap=force_overlap) check_binary_op_with_scalar('csr', output_grad_stype='csr', input_grad_stype='default', density=density, ograd_density=ograd_density, force_overlap=force_overlap) check_binary_op_with_scalar('csr', output_grad_stype='csr', input_grad_stype='csr', density=density, ograd_density=ograd_density, force_overlap=force_overlap) check_binary_op_with_scalar('csr', output_grad_stype='default', density=density, ograd_density=ograd_density, force_overlap=force_overlap) @pytest.mark.serial def test_elemwise_add_ex(): def check_elemwise_add_ex(lhs_stype, rhs_stype, shape, lhs_grad_stype=None, rhs_grad_stype=None): lhs = mx.symbol.Variable('lhs', stype=lhs_stype) rhs = mx.symbol.Variable('rhs', stype=rhs_stype) lhs_nd = rand_ndarray(shape, lhs_stype) rhs_nd = rand_ndarray(shape, rhs_stype) lhs_np = lhs_nd.asnumpy() rhs_np = rhs_nd.asnumpy() out_np = lhs_np + rhs_np test = mx.symbol.sparse.elemwise_add(lhs, rhs) location = {'lhs': lhs_nd, 'rhs': rhs_nd} check_symbolic_forward(test, location, [out_np]) check_numeric_gradient(test, location) grad_stypes = {} if lhs_grad_stype is not None and lhs_grad_stype != 'default': grad_stypes['lhs'] = lhs_grad_stype if rhs_grad_stype is not None and rhs_grad_stype != 'default': grad_stypes['rhs'] = rhs_grad_stype check_symbolic_backward(test, location, [out_np], [out_np, out_np], grad_stypes=grad_stypes) shapes = [rand_shape_2d(), rand_shape_3d()] for shape in shapes: check_elemwise_add_ex('default', 'default', shape) check_elemwise_add_ex('row_sparse', 'row_sparse', shape, lhs_grad_stype='row_sparse', rhs_grad_stype='row_sparse') @pytest.mark.serial def test_cast_storage_ex(): def check_cast_storage(shape, density, from_stype, to_stype, check_numeric_grad=True): x = mx.symbol.Variable('x', stype=from_stype) x_nd = rand_ndarray(shape, from_stype, density=density) x_np = x_nd.asnumpy() out_np = x_np test = mx.symbol.cast_storage(x, stype=to_stype) location = {'x': x_nd} check_symbolic_forward(test, location, [out_np]) # consider disable the numeric grad check for gpu block kernel since the input is large if check_numeric_grad: check_numeric_gradient(test, location) grad_stypes = {'x': to_stype} check_symbolic_backward(test, location, [out_np], [out_np], grad_stypes=grad_stypes) density = [1.00, 0.50, 0.01] for d in density: shape_2d = rand_shape_2d() shape_3d = rand_shape_3d() check_cast_storage(shape_2d, d, 'csr', 'default') check_cast_storage(shape_2d, d, 'default', 'csr') check_cast_storage(shape_2d, d, 'csr', 'csr') check_cast_storage(shape_2d, d, 'row_sparse', 'default') check_cast_storage(shape_2d, d, 'default', 'row_sparse') check_cast_storage(shape_2d, d, 'row_sparse', 'row_sparse') check_cast_storage(shape_3d, d, 'row_sparse', 'default') check_cast_storage(shape_3d, d, 'default', 'row_sparse') check_cast_storage(shape_3d, d, 'row_sparse', 'row_sparse') for i in range(4, 6): shape = rand_shape_nd(i, 5) check_cast_storage(shape, d, 'default', 'row_sparse') check_cast_storage(shape, d, 'row_sparse', 'default') # Test specific gpu kernels if default_device().device_type is 'gpu': dim0 = rnd.randint(1, 10) # test gpu thread kernel check_cast_storage((dim0, rnd.randint( 1, 32)), d, 'default', 'csr') # test gpu warp kernel check_cast_storage((dim0, rnd.randint( 32, 512)), d, 'default', 'csr') # test gpu block kernel check_cast_storage((dim0, rnd.randint(512, 1024)), d, 'default', 'csr', check_numeric_grad=False) # check race condition in block kernel check_cast_storage((200, 128 * 2 + 1), d, 'default', 'csr', check_numeric_grad=False) # test gpu thread kernel check_cast_storage((dim0, rnd.randint( 1, 32)), d, 'default', 'row_sparse') # test gpu warp kernel check_cast_storage((dim0, rnd.randint( 32, 512)), d, 'default', 'row_sparse') # test gpu block kernel check_cast_storage((dim0, rnd.randint(512, 1024)), d, 'default', 'row_sparse', check_numeric_grad=False) @pytest.mark.serial def test_sparse_dot(): def test_infer_forward_stype(lhs_shape, rhs_shape, lhs_density, rhs_density, trans_a, trans_b): all_stypes = ["default", "csr", "row_sparse"] lhs_nd = rand_ndarray(lhs_shape, 'default', density=lhs_density) rhs_nd = rand_ndarray(rhs_shape, 'default', density=rhs_density) out_nd = mx.nd.dot(lhs_nd, rhs_nd, transpose_a=trans_a, transpose_b=trans_b) out_np = out_nd.asnumpy() for lhs_stype in all_stypes: for rhs_stype in all_stypes: for forward_stype in all_stypes: lhs = lhs_nd.tostype(lhs_stype) rhs = rhs_nd.tostype(rhs_stype) out = mx.nd.dot(lhs, rhs, forward_stype=forward_stype, transpose_a=trans_a, transpose_b=trans_b) assert_almost_equal(out.tostype('default').asnumpy(), out_np, rtol=1e-3, atol=1e-4) lhs_var = mx.symbol.Variable('lhs', stype=lhs_stype) rhs_var = mx.symbol.Variable('rhs', stype=rhs_stype) out = mx.symbol.sparse.dot(lhs_var, rhs_var, forward_stype=forward_stype, transpose_a=trans_a, transpose_b=trans_b) location = {'lhs': lhs, 'rhs': rhs} check_symbolic_forward(out, location, [out_np], rtol=1e-3, atol=1e-4) def test_dot_csr(lhs_shape, rhs_shape, rhs_stype, trans_lhs, lhs_density, rhs_density): lhs_nd = rand_ndarray(lhs_shape, 'csr', density=lhs_density, shuffle_csr_indices=False) lhs_dns = lhs_nd.tostype('default') rhs_nd = rand_ndarray(rhs_shape, rhs_stype, density=rhs_density) rhs_dns = rhs_nd if rhs_stype == 'default' else rhs_nd.tostype('default') out = mx.nd.dot(lhs_nd, rhs_nd, transpose_a=trans_lhs) out_dns = mx.nd.dot(lhs_dns, rhs_dns, transpose_a=trans_lhs) out_np = out_dns.asnumpy() assert_almost_equal(out.asnumpy(), out_np, rtol=1e-3, atol=1e-5) # test symbolic forward lhs = mx.symbol.Variable('lhs', stype='csr') rhs = mx.symbol.Variable('rhs', stype=rhs_stype) out = mx.symbol.sparse.dot(lhs, rhs, transpose_a=trans_lhs) location = {'lhs': lhs_nd, 'rhs': rhs_nd} check_symbolic_forward(out, location, [out_np], rtol=1e-3, atol=1e-4) # test symbolic backward backward_trans = not trans_lhs rhs_backward_grad = mx.nd.dot(lhs_dns, out_dns, transpose_a=backward_trans).asnumpy() expected = {'rhs': rhs_backward_grad} check_symbolic_backward(out, location, [out_np], expected, grad_req={'lhs': 'null', 'rhs': 'write'}, rtol=1e-3, atol=1e-4) def test_dot_dns_csr(lhs_shape, rhs_shape, lhs_density, rhs_density, trans_lhs=False, trans_rhs=False): lhs_nd = rand_ndarray(lhs_shape, stype='default', density=lhs_density) rhs_nd = rand_ndarray(rhs_shape, stype='csr', density=rhs_density) rhs_dns = rhs_nd.tostype('default') if default_device() == mx.cpu(): forward_stype = 'csr' else: forward_stype = 'default' out = mx.nd.sparse.dot(lhs_nd, rhs_nd, transpose_a=trans_lhs, transpose_b=trans_rhs, forward_stype=forward_stype) out_dns = mx.nd.dot(lhs_nd, rhs_dns, transpose_a=trans_lhs, transpose_b=trans_rhs, forward_stype=forward_stype) out_np = out_dns.asnumpy() assert_almost_equal(out.asnumpy(), out_np, rtol=1e-3, atol=1e-5) # test symbolic forward lhs = mx.symbol.Variable('lhs', stype='default') rhs = mx.symbol.Variable('rhs', stype='csr') out = mx.symbol.sparse.dot(lhs, rhs, transpose_a=trans_lhs, transpose_b=trans_rhs, forward_stype=forward_stype) location = {'lhs': lhs_nd, 'rhs': rhs_nd} check_symbolic_forward(out, location, [out_np], rtol=1e-3, atol=1e-4) if default_device() == mx.cpu(): # test symbolic backward backward_trans = not trans_lhs rhs_backward_grad = mx.nd.dot(lhs_nd, out_dns, transpose_a=backward_trans).asnumpy() if trans_rhs is True: rhs_backward_grad = rhs_backward_grad.T expected = {'rhs': rhs_backward_grad} check_symbolic_backward(out, location, [out_np], expected, grad_req={'lhs': 'null', 'rhs': 'write'}, rtol=1e-3, atol=1e-4) else: transpose_b = not trans_rhs lhs_backward_grad = mx.nd.dot(out_dns, rhs_dns, transpose_b=transpose_b) expected = {'lhs': lhs_backward_grad.asnumpy()} check_symbolic_backward(out, location, [out_np], expected, grad_req={'lhs': 'write', 'rhs': 'null'}, rtol=1e-3, atol=1e-4) def test_sparse_dot_zero_output(lhs_shape, trans_lhs, rhs_num_cols): """Test for nnr_out = 0. Before the fix, the test would fail.""" lhs = mx.nd.zeros(lhs_shape) irow = np.random.randint(0, lhs_shape[0]) icol = np.random.randint(0, lhs_shape[1]) lhs[irow, icol] = 1.0 if trans_lhs: rhs = rand_ndarray(shape=(lhs_shape[0], rhs_num_cols), stype='default') rhs[irow, :] = 0 else: rhs = rand_ndarray(shape=(lhs_shape[1], rhs_num_cols), stype='default') rhs[icol, :] = 0 dns_out = mx.nd.dot(lhs, rhs, transpose_a=trans_lhs) assert mx.nd.sum(mx.nd.abs(dns_out)).asscalar() == 0 sps_out = mx.nd.sparse.dot(lhs.tostype('csr'), rhs.tostype('row_sparse'), transpose_a=trans_lhs) assert same(dns_out.asnumpy(), sps_out.asnumpy()) density = [1.00, 0.5, 0.01] for lhs_d in density: lhs_shape = rand_shape_2d(50, 200) rhs_d = 1 test_dot_csr(lhs_shape, (lhs_shape[1], 1), 'default', False, lhs_d, rhs_d) # test gpu SpMV test_dot_csr(lhs_shape, (lhs_shape[0], 1), 'default', True, lhs_d, rhs_d) # (vector kernel) test_dot_csr(lhs_shape, (lhs_shape[1], rnd.randint(5, 10)), 'default', False, lhs_d, rhs_d) # test gpu SpMM test_dot_csr(lhs_shape, (lhs_shape[0], rnd.randint(5, 10)), 'default', True, lhs_d, rhs_d) # (scalar kernel) test_dot_dns_csr(lhs_shape, (lhs_shape[1], rnd.randint(50, 200)), lhs_d, lhs_d) test_dot_dns_csr(lhs_shape, (rnd.randint(50, 200), lhs_shape[1]), lhs_d, lhs_d, trans_rhs=True) for rhs_d in density: test_dot_csr(lhs_shape, (lhs_shape[1], rnd.randint(1, 10)), 'row_sparse', False, lhs_d, rhs_d) test_dot_csr(lhs_shape, (lhs_shape[0], rnd.randint(1, 10)), 'row_sparse', True, lhs_d, rhs_d) test_infer_forward_stype(lhs_shape, (lhs_shape[1], rnd.randint(10, 20)), lhs_d, rhs_d, False, False) test_infer_forward_stype(lhs_shape, (rnd.randint(10, 20), lhs_shape[1]), lhs_d, rhs_d, False, True) test_infer_forward_stype(lhs_shape, (lhs_shape[0], rnd.randint(10, 20)), lhs_d, rhs_d, True, False) test_infer_forward_stype(lhs_shape, (rnd.randint(10, 20), lhs_shape[0]), lhs_d, rhs_d, True, True) test_sparse_dot_zero_output(rand_shape_2d(50, 200), False, 40) test_sparse_dot_zero_output(rand_shape_2d(50, 200), True, 40) @pytest.mark.serial def test_sparse_dot_determinism(): def check_dot_determinism(lhs_stype, rhs_stype, lhs_density, rhs_density, transpose_a, transpose_b, forward_stype): lhs_row = rnd.randint(50, 100) lhs_col = rnd.randint(50, 100) if transpose_a: if transpose_b: rhs_shape = (rnd.randint(50, 100), lhs_row) else: rhs_shape = (lhs_row, rnd.randint(50, 100)) else: if transpose_b: rhs_shape = (rnd.randint(50, 100), lhs_col) else: rhs_shape = (lhs_col, rnd.randint(50, 100)) lhs_shape = (lhs_row, lhs_col) lhs = rand_ndarray(lhs_shape, lhs_stype, density=lhs_density) rhs = rand_ndarray(rhs_shape, rhs_stype, density=rhs_density) res1 = mx.nd.sparse.dot(lhs, rhs, transpose_a=transpose_a, transpose_b=transpose_b, forward_stype=forward_stype) res2 = mx.nd.sparse.dot(lhs, rhs, transpose_a=transpose_a, transpose_b=transpose_b, forward_stype=forward_stype) assert_almost_equal(res1.asnumpy(), res2.asnumpy(), rtol=0.0, atol=0.0) check_dot_determinism('csr', 'default', 0.1, 1.0, True, False, 'row_sparse') forward_stype = 'csr' if default_device() == mx.cpu() else 'default' check_dot_determinism('default', 'csr', 1.0, 0.1, False, False, forward_stype) check_dot_determinism('default', 'csr', 1.0, 0.1, False, True, forward_stype) check_dot_determinism('csr', 'default', 0.1, 1.0, True, False, 'default') def test_sparse_slice(): def check_csr_slice(shape, slice_input): storage_type = 'csr' B, _ = rand_sparse_ndarray(shape, storage_type) np = B.asnumpy() begin = rnd.randint(0, B.shape[0] - 1) end = rnd.randint(begin + 1, B.shape[0]) nd_slice = mx.nd.crop(B, begin=begin, end=end) assert same(nd_slice.asnumpy(), np[begin:end]), (nd_slice.asnumpy(), np[begin:end]) shape = (rnd.randint(7, 15), rnd.randint(1, 10)) check_csr_slice(shape, True) check_csr_slice(shape, False) @pytest.mark.serial def test_sparse_retain(): def check_sparse_retain(shape, density, index_type=np.int64): num_rows = shape[0] rsp, _ = rand_sparse_ndarray(shape=shape, stype='row_sparse', density=density) length = np.random.randint(1, num_rows + 1) idx = random_sample(list(range(0, num_rows)), length) idx.sort() dns = rsp.asnumpy() tensor_retained_expected = np.zeros(shape) for i in idx: tensor_retained_expected[i][:] = dns[i] indices = mx.nd.array(idx, dtype=index_type) rsp_retained = mx.nd.sparse.retain(rsp, indices=indices) assert same(tensor_retained_expected, rsp_retained.asnumpy()) # check numeric gradient data = mx.symbol.Variable('data') idx = mx.symbol.Variable('indices') sym = mx.sym.sparse.retain(data=data, indices=idx) check_numeric_gradient(sym, [rsp, indices], grad_nodes=['data'], grad_stype_dict={'data': 'row_sparse'}) shape = rand_shape_2d() shape_3d = rand_shape_3d() densities = [0.01, 0.5, 1.0] index_types = [np.float32, np.int32, np.int64] for density in densities: for itype in index_types: check_sparse_retain(shape, density, itype) check_sparse_retain(shape_3d, density, itype) def test_sparse_unary_with_numerics(): def check_sparse_simple(name, stype, mxnet_func, forward_numpy_call, backward_numpy_call, output_grad_stype=None, backward_is_use_output=False): if output_grad_stype is None: output_grad_stype = stype expected_result_type, expected_grad_result_type = \ get_fw_bw_result_types_2(forward_numpy_call, stype, backward_numpy_call, output_grad_stype) if backward_is_use_output is True: expected_grad_result_type = expected_result_type shape = (3, 4) data = mx.symbol.Variable("data") grad_stypes = {'data' : expected_grad_result_type} y = mxnet_func(data) if stype == 'default': xa = np.random.uniform(low=-1.0, high=1.0, size=shape) xa_np = xa else: xa = create_sparse_array(shape, stype, data_init=None, rsp_indices=[1], modifier_func=lambda a: a - 0.5, shuffle_csr_indices=True) xa_np = xa.asnumpy() if output_grad_stype != 'default': out_grad = create_sparse_array(shape, output_grad_stype, data_init=None, rsp_indices=[1, 2], modifier_func=lambda a: a - 0.5, shuffle_csr_indices=True) out_grad_np = out_grad.asnumpy() else: out_grad_np = np.ones(xa.shape) out_grad = mx.nd.array(out_grad_np) output_np = forward_numpy_call(xa_np) input_grad_np = backward_numpy_call(output_np, out_grad_np) outputs = check_symbolic_forward(y, [xa], [output_np]) output = outputs[0] assert output.stype == expected_result_type input_grad_dict = check_symbolic_backward(y, location=[xa], out_grads=[out_grad], expected=[input_grad_np], grad_stypes=grad_stypes) inp_grad = input_grad_dict["data"] assert inp_grad.stype == expected_grad_result_type def check_sparse_function(name, mxnet_func, forward_numpy_call, backward_numpy_call, backward_is_use_output=False): check_sparse_simple(name, 'default', mxnet_func, forward_numpy_call, backward_numpy_call) for output_grad_stype in [None, "row_sparse", "default"]: check_sparse_simple(name, 'row_sparse', mxnet_func, forward_numpy_call, backward_numpy_call, output_grad_stype=output_grad_stype, backward_is_use_output=backward_is_use_output) for output_grad_stype in [None, "csr", "default"]: check_sparse_simple(name, 'csr', mxnet_func, forward_numpy_call, backward_numpy_call, output_grad_stype=output_grad_stype, backward_is_use_output=backward_is_use_output) check_sparse_function('relu', lambda x: mx.sym.relu(x), lambda x: np.maximum(x, 0.0), lambda output, outg: outg * assign_each(output, lambda x: x > 0.0), backward_is_use_output=True) check_sparse_function('sigmoid', lambda x: mx.sym.sigmoid(x), lambda x: np.divide(1.0, (1.0 + np.exp(-x))), lambda output, outg: outg * assign_each(output, lambda x: x * (1.0 - x)), backward_is_use_output=True) @pytest.mark.serial def test_sparse_nd_zeros(): def check_sparse_nd_zeros(stype, shape): zero = mx.nd.zeros(shape) sparse_zero = mx.nd.zeros(shape=shape, stype=stype) assert_almost_equal(sparse_zero.asnumpy(), zero.asnumpy()) shape = rand_shape_2d() check_sparse_nd_zeros('row_sparse', shape) check_sparse_nd_zeros('csr', shape) check_sparse_nd_zeros('default', shape) @pytest.mark.serial def test_sparse_nd_zeros_like(): def check_sparse_nd_zeros_like(stype, shape): zero = mx.nd.zeros(shape, stype=stype) zero_like = mx.nd.sparse.zeros_like(zero) assert_almost_equal(zero.asnumpy(), zero_like.asnumpy()) shape = rand_shape_2d() check_sparse_nd_zeros_like('row_sparse', shape) check_sparse_nd_zeros_like('csr', shape) @pytest.mark.serial def test_sparse_axis_operations(): def test_variations(func_name): dim0 = 30 dim1 = 100 axes = [0, 1] densities = [0, 0.5, 1] for density in densities: shape = rand_shape_2d(dim0, dim1) csr_array = rand_ndarray(shape=shape, stype='csr', density=density) dns = csr_array.tostype('default') for axis in axes: ret = func_name(csr_array, axis=axis) assert ret.stype == 'default' ret_expected = func_name(dns, axis=axis) assert_almost_equal(ret.asnumpy(), ret_expected.asnumpy()) def test_fallback(func_name, axis=0, keepdims=True, exclude=True): dim0 = 30 dim1 = 100 shape = rand_shape_2d(dim0, dim1) csr_array = rand_ndarray(shape=shape, stype='csr', density=0.01) ret= func_name(csr_array, axis=axis, keepdims=keepdims, exclude=exclude) test_variations(mx.nd.sum) test_fallback(mx.nd.sum, axis=0, keepdims=True, exclude=True) test_variations(mx.nd.mean) test_fallback(mx.nd.mean, axis=0, keepdims=True, exclude=True) @pytest.mark.serial def test_sparse_square_sum(): dim0 = 30 dim1 = 30 axes = [0, 1] keepdims = [False, True] densities = [0, 0.01, 0.2, 0.5, 1.0] for density in densities: shape = rand_shape_2d(dim0, dim1) rsp = rand_ndarray(shape, 'row_sparse', density) dns = rsp.tostype('default') for axis in axes: for keepdim in keepdims: ret = mx.nd._internal._square_sum(rsp, axis=axis, keepdims=keepdim) if axis == 1 and keepdim: assert ret.stype == 'row_sparse' else: assert ret.stype == 'default' ret_expected = mx.nd.sum(dns*dns, axis=axis, keepdims=keepdim) # check forward result assert_almost_equal(ret.asnumpy(), ret_expected.asnumpy()) rsp_data = mx.sym.Variable('data', stype='row_sparse') test = mx.symbol._internal._square_sum(rsp_data, axis=axis, keepdims=keepdim) # check symbolic backward since ograd can be an rsp # and cannot be checked through check_numeric_gradient # because it will add a loss layer as the output layer # which makes ograd of the square_sum dense if axis == 1 and keepdim: dns_data = mx.sym.Variable('data') baseline = mx.sym.sum(mx.sym.square(dns_data), axis=axis, keepdims=keepdim) igrad_expected = mx.nd.empty(dns.shape) baseline_exec = baseline._bind(default_device(), args=[dns], args_grad=[igrad_expected]) baseline_exec.forward(is_train=True) baseline_exec.backward([ret_expected]) # check backward when ograd is row sparse check_symbolic_backward(test, [rsp], [ret_expected.tostype('row_sparse')], [igrad_expected.asnumpy()], grad_stypes={'data': 'row_sparse'}) # check backward when ograd is dense # the stype of output of the square_sum is deteremined in symbol binding stage. # The ograd stype of the last layer is the same as the output stype of the last layer. # Need to add one more layer after square_sum to trigger the kernel for ograd # with default stype in square_sum op. baseline1 = baseline + 1 baseline_exec1 = baseline1._bind(default_device(), args=[dns], args_grad=[igrad_expected]) baseline_exec1.forward(is_train=True) baseline_exec1.backward([ret_expected]) test1 = test + 1 check_symbolic_backward(test1, [rsp], [ret_expected], [igrad_expected.asnumpy()], grad_stypes={'data': 'row_sparse'}) # check numeric gradient check_numeric_gradient(test, [rsp], grad_stype_dict={'data': 'row_sparse'}, atol=1e-2, rtol=0.1) @pytest.mark.serial @pytest.mark.flaky def test_sparse_storage_fallback(): """ test operators which don't implement FComputeEx or FStatefulComputeEx """ def check_broadcast_add(shape, lhs_stype, rhs_stype): lhs = mx.symbol.Variable('lhs', stype=lhs_stype) rhs = mx.symbol.Variable('rhs', stype=rhs_stype) lhs_nd = rand_ndarray(shape, lhs_stype) rhs_nd = rand_ndarray(shape, rhs_stype) lhs_dns = mx.nd.cast_storage(lhs_nd, stype='default') rhs_dns = mx.nd.cast_storage(rhs_nd, stype='default') out_dns = (lhs_dns + rhs_dns).asnumpy() test = mx.symbol.broadcast_add(lhs, rhs) location = {'lhs': lhs_nd, 'rhs': rhs_nd} check_symbolic_forward(test, location, [out_dns]) check_numeric_gradient(test, location) check_symbolic_backward(test, location, [out_dns], [out_dns, out_dns]) def np_softmax(x, axis=-1): # fix for old numpy on Travis not supporting keepdims x = x - np.max(x, axis=axis, keepdims=True) x = np.exp(x) x /= np.sum(x, axis=axis, keepdims=True) return x def check_concat(shape, lhs_stype, rhs_stype): x = mx.symbol.Variable('x', stype=lhs_stype) w = mx.symbol.Variable('w', stype=rhs_stype) test = mx.sym.Concat(x, w) x_nd = rand_ndarray(shape, lhs_stype) w_nd = rand_ndarray(shape, rhs_stype) location = {'x': x_nd, 'w': w_nd} check_numeric_gradient(test, location) def check_operator_with_temp_resource(shape, stype): x = mx.symbol.Variable('x', stype=stype) test = mx.sym.sum(x) x_nd = rand_ndarray(shape, stype) location = {'x': x_nd} check_numeric_gradient(test, location) shape = rand_shape_2d() stypes = ['default', 'csr', 'row_sparse'] for lhs in stypes: check_operator_with_temp_resource(shape, lhs) for rhs in stypes: check_broadcast_add(shape, lhs, rhs) check_concat(shape, lhs, rhs) @pytest.mark.serial def test_sparse_elementwise_sum(): def check_sparse_elementwise_sum_with_shape(stypes, shape, n): # forward inputs = [mx.symbol.Variable(f'arg{i}') for i in range(n)] out = mx.symbol.sparse.add_n(*inputs, name='esum') arr = [] arr_grad = [mx.nd.empty(shape, stype=stype) for stype in stypes] densities = [0, 0.01, 0.5, 1.0] for stype in stypes: arr.append(rand_ndarray(shape, stype, densities[np.random.randint(0, len(densities))])) exec1 = out._bind(default_device(), args=arr, args_grad=arr_grad) exec1.forward(is_train=True) out1 = exec1.outputs[0].asnumpy() out = sum(a.asnumpy() for a in arr) assert_almost_equal(out, out1, atol=1e-5) out_grad = mx.nd.empty(shape) out_grad[:] = np.random.uniform(-10, 10, shape) # backward exec1.backward([out_grad]) for a in arr_grad: assert_almost_equal(a.asnumpy(), out_grad.asnumpy(), atol=1e-5) all_stypes = ['default', 'csr', 'row_sparse'] for dim in range(2, 4): shape = tuple(np.random.randint(5, 10, size=dim)) rsp_test_cnt = np.random.randint(1, 9) check_sparse_elementwise_sum_with_shape(['row_sparse' for i in range(rsp_test_cnt)], shape, rsp_test_cnt) if dim is 2: check_sparse_elementwise_sum_with_shape(['default', 'csr', 'default'], shape, 3) test_len = np.random.randint(5, 10) # at least one default type stypes = ['default'] for _ in range(test_len): pick_side = np.random.randint(2) pick_type = np.random.randint(3) stypes = ([all_stypes[pick_type]] if pick_side is 0 else []) + stypes + ([all_stypes[pick_type]] if pick_side is 1 else []) check_sparse_elementwise_sum_with_shape(stypes, shape, test_len+1) @pytest.mark.serial def test_sparse_embedding(): ''' test sparse embedding operator ''' def check_sparse_embedding(in_dim, out_dim, batch, densities, sparse_grad): target_stype = 'row_sparse' if sparse_grad else 'default' # init executor data = mx.sym.Variable("data") weight = mx.sym.Variable("embed_weight") embed = mx.sym.sparse.Embedding(data=data, weight=weight, input_dim=in_dim, sparse_grad=sparse_grad, output_dim=out_dim, name='embed') grad_req = {'data': 'null', 'embed_weight': 'write'} args = {'embed_weight': mx.nd.zeros((in_dim, out_dim)), 'data': mx.nd.ones((batch,))} weight_grad = mx.nd.zeros((in_dim, out_dim)) if sparse_grad: weight_grad = weight_grad.tostype('row_sparse') args_grad = {'embed_weight': weight_grad} exe_test = embed._bind(default_device(), args=args, args_grad=args_grad, grad_req=grad_req) arg_map = dict(zip(embed.list_arguments(), exe_test.arg_arrays)) grad_map = dict(zip(embed.list_arguments(), exe_test.grad_arrays)) # init data np_data = np.random.randint(low=0, high=in_dim, size=batch) np_onehot = np.zeros((batch, in_dim)).astype(np.float32) np_onehot[np.arange(batch), np_data] = 1.0 arg_map["data"][:] = np_data # weight weight = arg_map["embed_weight"] for density in densities: # update weight based on density weight[:] = rand_ndarray(weight.shape, 'default', density=density) # check forward exe_test.forward(is_train=True) # init grad np_grad = np.random.uniform(-1, 1, exe_test.outputs[0].shape) grad = mx.nd.zeros(np_grad.shape) grad[:] = np_grad assert_almost_equal(exe_test.outputs[0].asnumpy(), np.dot(np_onehot, weight.asnumpy()), atol=1e-4) # check backward exe_test.backward([grad]) assert_almost_equal(grad_map["embed_weight"].asnumpy(), np.dot(np_onehot.T, grad.asnumpy()), atol=1e-4) # check grad stype assert(grad_map["embed_weight"].stype == target_stype) densities = [0, 0.5, 1] in_dim = 50 out_dim = 3 batch = 8 sparse_grads = [True, False] for sparse_grad in sparse_grads: check_sparse_embedding(in_dim, out_dim, batch, densities, sparse_grad) def test_sparse_broadcast_add_sub(): def check_broadcast_add(mx_lhs, mx_rhs, np_lhs, np_rhs, dtype): assert_almost_equal(mx.nd.sparse.add(mx_lhs, mx_rhs).asnumpy(), np.add(np_lhs, np_rhs), atol=1e-4) def check_broadcast_sub(mx_lhs, mx_rhs, np_lhs, np_rhs, dtype): assert_almost_equal(mx.nd.sparse.subtract(mx_lhs, mx_rhs).asnumpy(), np.subtract(np_lhs, np_rhs), atol=1e-4) stype = 'csr' shape = rand_shape_2d() num_rows = shape[0] num_cols = shape[1] for density in [0.1 * i for i in range(10)]: mx_lhs = rand_ndarray(shape, stype, density) np_lhs = mx_lhs.asnumpy() mx_rhs_row_2D = rand_ndarray((1, num_cols), 'default') mx_rhs_row_1D = mx_rhs_row_2D.reshape((num_cols)) mx_rhs_col = rand_ndarray((num_rows, 1), 'default') mx_rhs_scalar_2D = rand_ndarray((1, 1), 'default') mx_rhs_scalar_1D = mx_rhs_scalar_2D.reshape((1, )) for mx_rhs in [mx_rhs_row_2D, mx_rhs_row_1D, mx_rhs_col, mx_rhs_scalar_2D, mx_rhs_scalar_1D]: np_rhs = mx_rhs.asnumpy() check_broadcast_add(mx_lhs, mx_rhs, np_lhs, np_rhs, np.float32) check_broadcast_sub(mx_lhs, mx_rhs, np_lhs, np_rhs, np.float32) check_broadcast_add(mx_rhs, mx_lhs, np_rhs, np_lhs, np.float32) check_broadcast_sub(mx_rhs, mx_lhs, np_rhs, np_lhs, np.float32) def test_sparse_broadcast_mul_div(): def check_broadcast_mul(mx_lhs, mx_rhs, np_lhs, np_rhs, dtype): assert_almost_equal(mx.nd.sparse.multiply(mx_lhs, mx_rhs).asnumpy(), np.multiply(np_lhs, np_rhs), atol=1e-4) def check_broadcast_div(mx_lhs, mx_rhs, np_lhs, np_rhs, dtype): assert_almost_equal(mx.nd.sparse.divide(mx_lhs, mx_rhs).asnumpy(), np.divide(np_lhs, np_rhs), atol=1e-4) stype = 'csr' shape = rand_shape_2d() num_rows = shape[0] num_cols = shape[1] for density in [0.1 * i for i in range(10)]: mx_lhs = rand_ndarray(shape, stype, density) np_lhs = mx_lhs.asnumpy() mx_rhs_row_2D = rand_ndarray((1, num_cols), 'default') mx_rhs_row_1D = mx_rhs_row_2D.reshape((num_cols)) mx_rhs_col = rand_ndarray((num_rows, 1), 'default') mx_rhs_scalar_2D = rand_ndarray((1, 1), 'default') mx_rhs_scalar_1D = mx_rhs_scalar_2D.reshape((1, )) for mx_rhs in [mx_rhs_row_2D, mx_rhs_row_1D, mx_rhs_col, mx_rhs_scalar_2D, mx_rhs_scalar_1D]: np_rhs = mx_rhs.asnumpy() check_broadcast_mul(mx_lhs, mx_rhs, np_lhs, np_rhs, np.float32) check_broadcast_div(mx_lhs, mx_rhs, np_lhs, np_rhs, np.float32) def test_batchnorm_fallback(): # same test as test_operator.test_batchnorm_training, but tests fallback logic of batchnorm stype = 'row_sparse' for shape in [(2, 3), (2, 3, 2, 2)]: data_tmp = np.random.normal(-0.1, 0.1, size=shape) s = shape[1], gamma = np.ones(s) beta = np.ones(s) gamma[1] = 3 beta[0] = 3 rolling_mean = np.random.uniform(size=s) rolling_std = np.random.uniform(size=s) data = mx.symbol.Variable('data', stype=stype) in_location = [mx.nd.array(data_tmp).tostype(stype), mx.nd.array(gamma).tostype(stype), mx.nd.array(beta).tostype(stype)] mean_std = [mx.nd.array(rolling_mean).tostype(stype), mx.nd.array(rolling_std).tostype(stype)] test = mx.symbol.BatchNorm(data, fix_gamma=True) assertRaises(MXNetError, check_numeric_gradient, test, in_location, mean_std, numeric_eps=1e-3, rtol=0.16, atol=1e-2) test = mx.symbol.BatchNorm(data, fix_gamma=True, use_global_stats=True) assertRaises(MXNetError, check_numeric_gradient, test, in_location, mean_std, numeric_eps=1e-3, rtol=0.16, atol=1e-2) test = mx.symbol.BatchNorm(data, fix_gamma=False) check_numeric_gradient(test, in_location, mean_std, numeric_eps=1e-3, rtol=0.16, atol=1e-2) test = mx.symbol.BatchNorm(data, fix_gamma=False, use_global_stats=True) check_numeric_gradient(test, in_location, mean_std, numeric_eps=1e-3, rtol=0.16, atol=1e-2) # Test varying channel axis dim = len(shape) for chaxis in range(-dim, dim): chaxis_true = chaxis if chaxis < 0: chaxis_true = dim + chaxis shapex = shape channel_count = shapex[chaxis_true] data_tmp = np.random.normal(-0.1, 0.1, size=shapex) gamma = np.ones(channel_count) beta = np.ones(channel_count) if channel_count > 1: gamma[1] = 3 beta[0] = 3 in_location = [mx.nd.array(data_tmp).tostype(stype), mx.nd.array(gamma).tostype(stype), mx.nd.array(beta).tostype(stype)] xrolling_mean = np.random.uniform(size=channel_count) xrolling_std = np.random.uniform(size=channel_count) xmean_std = [mx.nd.array(xrolling_mean).tostype(stype), mx.nd.array(xrolling_std).tostype(stype)] test = mx.symbol.BatchNorm(data, fix_gamma=True, axis=chaxis) assertRaises(MXNetError, check_numeric_gradient, test, in_location, xmean_std, numeric_eps=1e-3, rtol=0.2, atol=0.01) test = mx.symbol.BatchNorm(data, fix_gamma=True, use_global_stats=True, axis=chaxis) assertRaises(MXNetError, check_numeric_gradient, test, in_location, xmean_std, numeric_eps=1e-3, rtol=0.2, atol=0.01) test = mx.symbol.BatchNorm(data, fix_gamma=False, axis=chaxis) check_numeric_gradient(test, in_location, xmean_std, numeric_eps=1e-3, rtol=0.2, atol=0.01) test = mx.symbol.BatchNorm(data, fix_gamma=False, use_global_stats=True, axis=chaxis) check_numeric_gradient(test, in_location, xmean_std, numeric_eps=1e-3, rtol=0.2, atol=0.01) @pytest.mark.serial def test_dnnl_sparse(): # This test is trying to create a race condition describedd in # https://github.com/apache/mxnet/issues/10189 arr = mx.nd.random.uniform(shape=(10, 10, 32, 32)) weight1 = mx.nd.random.uniform(shape=(10, 10, 3, 3)) arr = mx.nd.Convolution(data=arr, weight=weight1, no_bias=True, kernel=(3, 3), num_filter=10) rs_arr = mx.nd.sparse.row_sparse_array((mx.nd.zeros_like(arr), np.arange(arr.shape[0]))) weight2 = mx.nd.random.uniform(shape=(10, np.prod(arr.shape[1:4]))) fc_res = mx.nd.FullyConnected(data=arr, weight=weight2, no_bias=True, num_hidden=10) sum_res = mx.nd.elemwise_sub(arr, rs_arr) res1 = np.dot(mx.nd.flatten(sum_res).asnumpy(), weight2.asnumpy().T) print(res1 - fc_res.asnumpy()) almost_equal(res1, fc_res.asnumpy()) @pytest.mark.serial def test_sparse_nd_where(): def get_forward_expected_output(condition, x, y): original_shape = x.shape out = np.zeros(original_shape) if condition.shape == x.shape: for index, c in np.ndenumerate(condition): if c != 0: out[index] = x[index] else: out[index] = y[index] else: raise RuntimeError("Invalid condition shape for where op") out = out.reshape(original_shape) return out def get_forward_inputs_same_shape(shape): condition_np = np.random.randint(0, 2, np.prod(shape)).reshape(shape) x_np = np.random.randint(1, 6, np.prod(shape)).reshape(shape) y_np = np.random.randint(7, 11, np.prod(shape)).reshape(shape) return condition_np, x_np, y_np def get_backward_input(shape): return np.random.randint(20, 30, np.prod(shape)).reshape(shape) def get_backward_expected_outputs(grad_in, condition): shape = grad_in.shape grad_cond = np.zeros(condition.shape) grad_x = np.empty(shape) grad_y = np.empty(shape) for index, c in np.ndenumerate(condition): if 0 != c: grad_x[index] = grad_in[index] grad_y[index] = 0 else: grad_x[index] = 0 grad_y[index] = grad_in[index] return grad_cond, grad_x, grad_y def test_where_helper(shape): condition_np, x_np, y_np = get_forward_inputs_same_shape(shape) out_expected = get_forward_expected_output(condition_np, x_np, y_np) grad_in_np = get_backward_input(shape) grad_expected_cond, grad_expected_x, grad_expected_y \ = get_backward_expected_outputs(grad_in_np, condition_np) condition = mx.sym.Variable('condition', stype='csr') x = mx.sym.Variable('x') y = mx.sym.Variable('y') grad_in_mx = mx.nd.array(grad_in_np, dtype=np.int32) where_sym = mx.sym.where(condition, x, y) cond_nd = mx.nd.array(condition_np) args = {'condition': cond_nd.tostype('csr'), 'x': mx.nd.array(x_np), 'y' : mx.nd.array(y_np)} args_grad = {'condition': mx.nd.zeros_like(cond_nd), 'x': mx.nd.array(x_np).tostype('csr'), 'y' : mx.nd.array(y_np)} # test req='write' where_exe_write = where_sym._bind(ctx=default_device(), args=args, args_grad=args_grad, grad_req='write') # test forward req='write' outputs = where_exe_write.forward(is_train=True) assert same(outputs[0].asnumpy(), out_expected) # test backward req='write' where_exe_write.backward(grad_in_mx.astype('float32')) assert same(where_exe_write.grad_dict['x'].asnumpy(), grad_expected_x) assert same(where_exe_write.grad_dict['y'].asnumpy(), grad_expected_y) assert same(where_exe_write.grad_dict['condition'].asnumpy(), grad_expected_cond) # test req='add' x_grad_init = np.random.randint(30, 40, np.prod(shape)).reshape(shape) y_grad_init = np.random.randint(40, 50, np.prod(shape)).reshape(shape) where_exe_add = where_sym._bind(ctx=default_device(), args=args, args_grad=args_grad, grad_req='add') where_exe_add.grad_dict['x'][:] = x_grad_init where_exe_add.grad_dict['y'][:] = y_grad_init # test forward req='add' outputs = where_exe_add.forward(is_train=True) assert same(outputs[0].asnumpy(), out_expected) def test_where_numeric_gradient(shape): condition = mx.sym.Variable('condition', stype='csr') x = mx.sym.Variable('x') y = mx.sym.Variable('y') where_sym = mx.sym.where(condition, x, y) condition_np, x_np, y_np = get_forward_inputs_same_shape(shape) check_numeric_gradient(where_sym, [condition_np, x_np, y_np], grad_nodes=['x', 'y']) test_where_helper((5, 9)) test_where_numeric_gradient((5, 9)) @pytest.mark.serial def test_sparse_quadratic_function(): def f(x, a, b, c): return a * x**2 + b * x + c def check_sparse_quadratic_function(a, b, c, expected_stype): # check forward and compare the result with dense op ndim = 2 shape = rand_shape_nd(ndim, 5) data = rand_ndarray(shape=shape, stype='csr') data_np = data.asnumpy() expected = f(data_np, a, b, c) output = mx.nd.contrib.quadratic(data, a=a, b=b, c=c) assert(output.stype == expected_stype) assert_almost_equal(output.asnumpy(), expected) a = np.random.random_sample() b = np.random.random_sample() check_sparse_quadratic_function(a, b, 0.0, 'csr') check_sparse_quadratic_function(a, b, 1.0, 'default') def test_reshape_backward_fallback(): """ out | \ w_x x / w in which x is a sparse tensor. Due to sparse gradient optimization in sym.dot, grad(w_x) is sparse. Though sym.reshape itself does not have sparse version, if we somehow make grad(w) sparse as well, e.g., - by setting args_grad in symbol.bind - or, we can have out_y = sym.dot(sparse_y, w), then grad(w) will be inferred as sparse reshape backward (from w_x to w) needs to understand how to handle sparse inputs. """ ctx = default_device() w_shape = (12, 4) w_x_shape = (1, 48) x_nd = rand_ndarray((4, 1), 'csr') w_nd = rand_ndarray(w_shape) w_x_nd = w_nd.reshape(w_x_shape) out_x_nd = mx.nd.dot(x_nd, w_x_nd) w_x_backward_grad = mx.nd.dot(x_nd, out_x_nd, transpose_a=True).asnumpy() expected_grad_nd = w_x_backward_grad.reshape(w_shape) x = mx.sym.Variable('x', stype='csr') w = mx.sym.Variable('w') w_x = mx.sym.reshape(w, w_x_shape, name="w_x") out = mx.sym.sparse.dot(x, w_x, name='out_x') grad_w_nd = rand_ndarray(w_shape, 'row_sparse') executor = out._bind(ctx=ctx, args={"x": x_nd, "w": w_nd}, args_grad={"w": grad_w_nd}) executor.forward(is_train=True) executor.backward(out_x_nd) assert_almost_equal(grad_w_nd.asnumpy(), expected_grad_nd)