vuron/adept: Платформа для обучения и вывода нейронных сетей, с API приближенным к PyTorch.

src/nn/activations.cpp

45 строк1 KB

kolkir

Activation functions implementaions

12 фев 2025, 00:20

12 фев 2025, 00:20a5158cd

#include "../dispatch/activations.hpp"
#include <adept/nn/activations.hpp>

namespace adept {

Variable relu(const Variable& a) {
  auto result = relu_fwd(a.data());
  Variable var(result, {a}, "ReLU");
  var.set_backward_fn([a = a](const auto& out_grad) mutable {
    if (a.requires_grad())
      a.add_grad(relu_bwd(a.data(), out_grad));
  });
  return var;
}

Variable leaky_relu(const Variable& a, float32_t negative_slope) {
  auto result = leaky_relu_fwd(a.data(), negative_slope);
  Variable var(result, {a}, "LeakyReLU");
  var.set_backward_fn([a = a, negative_slope](const auto& out_grad) mutable {
    if (a.requires_grad())
      a.add_grad(leaky_relu_bwd(a.data(), out_grad, negative_slope));
  });
  return var;
}

Variable sigmoid(const Variable& a) {
  auto result = sigmoid_fwd(a.data());
  Variable var(result, {a}, "Sigmoid");
  var.set_backward_fn([a = a, result](const auto& out_grad) mutable {
    if (a.requires_grad())
      a.add_grad(sigmoid_bwd(result, out_grad));
  });
  return var;
}

Variable silu(const Variable& a) {
  auto result = silu_fwd(a.data());
  Variable var(result, {a}, "SiLU");
  var.set_backward_fn([a = a](const auto& out_grad) mutable {
    if (a.requires_grad())
      a.add_grad(silu_bwd(a.data(), out_grad));
  });
  return var;
}
}  // namespace adept