/
vuron
/
adept
Обзор
Документация
Войти
/
vuron
/
adept
Код
Запросы
0
Задачи
Вики
Пакеты
0
Релизы
0
Аналитика
Безопасность
master
src/nn/linear.cpp
54 строки
2 KB
kolkir
Add tests for Linear layer and update matmul usage
28 мар 2025, 23:53
28 мар 2025, 23:53
bc788d8
Код
Авторство
О чём код?
#include <adept/nn/linear.hpp> namespace adept { LinearImpl::LinearImpl(index_t in, index_t out, device_t device, dtype_t dtype) : weight_(Tensor::empty({.shape = {out, in}, .device = device, .dtype = dtype})), bias_(Tensor::empty({.shape = {out}, .device = device, .dtype = dtype})) { init_weights(); set_name("Linear"); register_parameter("weight", weight_); register_parameter("bias", bias_); } Variable LinearImpl::forward(const Variable& input) { // TODO: matmul+bias can be fused for 2d case, consider use reshape instead of transpose auto result = input.data().matmul(weight_.data().transpose2d()) + bias_.data(); Variable var(result, {input}, "linear"); var.set_backward_fn([this, input = input](const auto& out_grad) mutable { // weight in fwd was transposed, so here we use unchanged(transposed by desing) if (input.requires_grad()) input.add_grad(out_grad.matmul(weight_.data())); // transpose results because weight and bias are transposed by desing weight_.add_grad( out_grad.unsqueeze(1).transpose2d().matmul(input.data().unsqueeze(1)).sum_dim0()); bias_.add_grad(out_grad.sum_dim0()); }); return var; } void LinearImpl::set_weights(const Tensor& weight) { weight_.data() = weight; } void LinearImpl::set_bias(const Tensor& bias) { bias_.data() = bias; } Variable LinearImpl::weights() const { return weight_; } Variable LinearImpl::bias() const { return bias_; } void LinearImpl::init_weights() { auto stdv = static_cast<float32_t>(1. / std::sqrt(weight_.data().properties().shape.dim(1))); fill_uniform(-stdv, stdv, weight_.data()); fill_uniform(-stdv, stdv, bias_.data()); } } // namespace adept