vuron/adept: Платформа для обучения и вывода нейронных сетей, с API приближенным к PyTorch.

tests/activation_tests.cpp
133 строки8 KB
kolkir
Activation functions implementaions
12 фев 2025, 00:20
12 фев 2025, 00:20a5158cd
#include <adept/autograd/autograd.hpp>
#include <adept/nn/activations.hpp>
#include "catch.hpp"

using namespace adept;

TEST_CASE("ReLU", "[activations]") {
  std::vector<float32_t> x_data = {0, 2, -3, 4, -5, 0, -7, 0, 9};

  auto x = Variable(Tensor::from_blob(
      x_data.data(),
      {.shape = {1, 1, 3, 3}, .device = device_t::CPU, .dtype = to_dtype<float32_t>()}));

  auto y = relu(x);

  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 0}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 1}), Catch::WithinRel(2.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 2}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 0}), Catch::WithinRel(4.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 1}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 2}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 0}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 1}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 2}), Catch::WithinRel(9.0f, 0.001f));

  mean(y).backward();

  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 0}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 1}), Catch::WithinRel(0.1111f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 2}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 0}), Catch::WithinRel(0.1111f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 1}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 2}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 0}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 1}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 2}), Catch::WithinRel(0.1111f, 0.001f));
}

TEST_CASE("LeakyReLU", "[activations]") {
  std::vector<float32_t> x_data = {0, 2, -3, 4, -5, 0, -7, 0, 9};

  auto x = Variable(Tensor::from_blob(
      x_data.data(),
      {.shape = {1, 1, 3, 3}, .device = device_t::CPU, .dtype = to_dtype<float32_t>()}));

  auto y = leaky_relu(x);

  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 0}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 1}), Catch::WithinRel(2.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 2}), Catch::WithinRel(-0.03f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 0}), Catch::WithinRel(4.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 1}), Catch::WithinRel(-0.05f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 2}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 0}), Catch::WithinRel(-0.07f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 1}), Catch::WithinRel(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 2}), Catch::WithinRel(9.0f, 0.001f));

  mean(y).backward();

  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 0}), Catch::WithinAbs(0.0011f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 1}), Catch::WithinAbs(0.1111f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 2}), Catch::WithinAbs(0.0011f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 0}), Catch::WithinAbs(0.1111f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 1}), Catch::WithinAbs(0.0011f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 2}), Catch::WithinAbs(0.0011f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 0}), Catch::WithinAbs(0.0011f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 1}), Catch::WithinAbs(0.0011f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 2}), Catch::WithinAbs(0.1111f, 0.001f));
}

TEST_CASE("Sigmoid", "[activations]") {
  std::vector<float32_t> x_data = {0, 2, -3, 4, -5, 0, -7, 0, 9};

  auto x = Variable(Tensor::from_blob(
      x_data.data(),
      {.shape = {1, 1, 3, 3}, .device = device_t::CPU, .dtype = to_dtype<float32_t>()}));

  auto y = sigmoid(x);

  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 0}), Catch::WithinAbs(5.0000e-01f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 1}), Catch::WithinAbs(8.8080e-01f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 2}), Catch::WithinAbs(4.7426e-02f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 0}), Catch::WithinAbs(9.8201e-01f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 1}), Catch::WithinAbs(6.6929e-03f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 2}), Catch::WithinAbs(5.0000e-01f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 0}), Catch::WithinAbs(9.1105e-04f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 1}), Catch::WithinAbs(5.0000e-01f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 2}), Catch::WithinAbs(9.9988e-01f, 0.001f));

  mean(y).backward();

  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 0}), Catch::WithinAbs(2.7778e-02f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 1}), Catch::WithinAbs(1.1666e-02f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 2}), Catch::WithinAbs(5.0196e-03f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 0}), Catch::WithinAbs(1.9625e-03f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 1}), Catch::WithinAbs(7.3867e-04f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 2}), Catch::WithinAbs(2.7778e-02f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 0}), Catch::WithinAbs(1.0114e-04f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 1}), Catch::WithinAbs(2.7778e-02f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 2}), Catch::WithinAbs(1.3707e-05f, 0.001f));
}

TEST_CASE("SiLU", "[activations]") {
  std::vector<float32_t> x_data = {0, 2, -3, 4, -5, 0, -7, 0, 9};

  auto x = Variable(Tensor::from_blob(
      x_data.data(),
      {.shape = {1, 1, 3, 3}, .device = device_t::CPU, .dtype = to_dtype<float32_t>()}));

  auto y = silu(x);

  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 0}), Catch::WithinAbs(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 1}), Catch::WithinAbs(1.7616e+00f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 0, 2}), Catch::WithinAbs(-1.4228e-01f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 0}), Catch::WithinAbs(3.9281e+00f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 1}), Catch::WithinAbs(-3.3464e-02f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 1, 2}), Catch::WithinAbs(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 0}), Catch::WithinAbs(-6.3774e-03f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 1}), Catch::WithinAbs(0.0f, 0.001f));
  REQUIRE_THAT(y.data().at<float32_t>({0, 0, 2, 2}), Catch::WithinAbs(8.9989e+00f, 0.001f));

  mean(y).backward();

  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 0}), Catch::WithinAbs(0.0556f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 1}), Catch::WithinAbs(0.1212f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 0, 2}), Catch::WithinAbs(-0.0098f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 0}), Catch::WithinAbs(0.1170f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 1}), Catch::WithinAbs(-0.0029f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 1, 2}), Catch::WithinAbs(0.0556f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 0}), Catch::WithinAbs(-0.0006f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 1}), Catch::WithinAbs(0.0556f, 0.001f));
  REQUIRE_THAT(x.grad().at<float32_t>({0, 0, 2, 2}), Catch::WithinAbs(0.1112f, 0.001f));
}