vuron/adept: Платформа для обучения и вывода нейронных сетей, с API приближенным к PyTorch.

tests/batchnorm2d_tests.cpp
155 строк6 KB
kolkir
Add batchnorm train test
23 фев 2025, 12:15
23 фев 2025, 12:15dfd7947
#include <adept/nn/activations.hpp>
#include <adept/nn/batchnorm2d.hpp>

#include "catch.hpp"

using namespace adept;

TEST_CASE("Batchnorm2d fwd eval 1x1x2x2", "[batchnorm2d]") {
  std::vector<float32_t> im_data = {1, 2, 3, 4};
  std::vector<float32_t> res_data = {1, 2, 3, 4};

  float32_t var = 1.0f;
  float32_t mean = 3.25f;
  float32_t alpha = 1.5f;
  float32_t beta = 2.7f;
  float32_t eps = 1e-05f;
  for (auto& v : res_data) {
    v = (v - mean) / std::sqrt(var + eps) * 1.5f + 2.7f;
  }

  index_t channels = 1;

  auto im = Tensor::from_blob(
      im_data.data(),
      {.shape = {1, channels, 2, 2}, .device = device_t::CPU, .dtype = to_dtype<float32_t>()});

  auto weight = Tensor::from_values({alpha}, Shape({channels}), device_t::CPU);
  auto bias = Tensor::from_values({beta}, Shape({channels}), device_t::CPU);
  auto run_mean = Tensor::from_values({mean}, Shape({channels}), device_t::CPU);
  auto run_var = Tensor::from_values({var}, Shape({channels}), device_t::CPU);

  BarchNorm2d norm(channels);
  norm->eval();
  norm->set_weights(weight);
  norm->set_bias(bias);
  norm->set_running_mean(run_mean);
  norm->set_running_var(run_var);
  auto res = norm(im);

  REQUIRE_THAT(res.data().at<float32_t>({0, 0, 0, 0}), Catch::WithinRel(res_data[0], 0.001f));
  REQUIRE_THAT(res.data().at<float32_t>({0, 0, 0, 1}), Catch::WithinRel(res_data[1], 0.001f));
  REQUIRE_THAT(res.data().at<float32_t>({0, 0, 1, 0}), Catch::WithinRel(res_data[2], 0.001f));
  REQUIRE_THAT(res.data().at<float32_t>({0, 0, 1, 1}), Catch::WithinRel(res_data[3], 0.001f));
}

TEST_CASE("Batchnorm2d fwd eval 1x2x2x2", "[batchnorm2d]") {
  std::vector<float32_t> im_data = {1, 2, 3, 4, 1, 2, 3, 4};
  std::vector<float32_t> res_data = {1, 2, 3, 4, 1, 2, 3, 4};

  float32_t var = 1.0f;
  float32_t mean = 3.25f;
  float32_t alpha = 1.5f;
  float32_t beta = 2.7f;
  float32_t eps = 1e-05f;
  for (auto& v : res_data) {
    v = (v - mean) / std::sqrt(var + eps) * 1.5f + 2.7f;
  }

  index_t channels = 2;

  auto im = Tensor::from_blob(
      im_data.data(),
      {.shape = {1, channels, 2, 2}, .device = device_t::CPU, .dtype = to_dtype<float32_t>()});

  auto weight = Tensor::from_values({alpha, alpha}, Shape({channels}), device_t::CPU);
  auto bias = Tensor::from_values({beta, beta}, Shape({channels}), device_t::CPU);
  auto run_mean = Tensor::from_values({mean, mean}, Shape({channels}), device_t::CPU);
  auto run_var = Tensor::from_values({var, var}, Shape({channels}), device_t::CPU);

  BarchNorm2d norm(channels);
  norm->eval();
  norm->set_weights(weight);
  norm->set_bias(bias);
  norm->set_running_mean(run_mean);
  norm->set_running_var(run_var);
  auto res = norm(im);

  for (auto c : irange(channels)) {
    REQUIRE_THAT(res.data().at<float32_t>({0, c, 0, 0}), Catch::WithinRel(res_data[0], 0.001f));
    REQUIRE_THAT(res.data().at<float32_t>({0, c, 0, 1}), Catch::WithinRel(res_data[1], 0.001f));
    REQUIRE_THAT(res.data().at<float32_t>({0, c, 1, 0}), Catch::WithinRel(res_data[2], 0.001f));
    REQUIRE_THAT(res.data().at<float32_t>({0, c, 1, 1}), Catch::WithinRel(res_data[3], 0.001f));
  }
}

TEST_CASE("Batchnorm2d fwd train 1x2x2x2 stats", "[batchnorm2d]") {
  std::vector<float32_t> im_data = {1, 2, 3, 4, 1, 2, 3, 4};

  float32_t var = 0.0f;
  float32_t mean = 2.5f;
  for (const auto& v : im_data) {
    auto d = (v - mean);
    var += d * d;
  }
  var /= im_data.size() - 2;

  index_t channels = 2;

  auto im = Tensor::from_blob(
      im_data.data(),
      {.shape = {1, channels, 2, 2}, .device = device_t::CPU, .dtype = to_dtype<float32_t>()});

  BarchNorm2d norm(channels, /*eps*/ 1e-05f, /*momentum*/ 1.f);
  norm->train();
  auto res = norm(im);

  auto weigth = norm->weights().data();
  REQUIRE_THAT(weigth.at<float32_t>({0}), Catch::WithinRel(1.f, 0.001f));
  REQUIRE_THAT(weigth.at<float32_t>({1}), Catch::WithinRel(1.f, 0.001f));

  auto bias = norm->bias().data();
  REQUIRE_THAT(bias.at<float32_t>({0}), Catch::WithinRel(0.f, 0.001f));
  REQUIRE_THAT(bias.at<float32_t>({1}), Catch::WithinRel(0.f, 0.001f));

  auto run_mean = norm->running_mean().data();
  REQUIRE_THAT(run_mean.at<float32_t>({0}), Catch::WithinRel(2.5f, 0.001f));
  REQUIRE_THAT(run_mean.at<float32_t>({1}), Catch::WithinRel(2.5f, 0.001f));

  auto run_var = norm->running_var().data();
  REQUIRE_THAT(run_var.at<float32_t>({0}), Catch::WithinRel(var, 0.001f));
  REQUIRE_THAT(run_var.at<float32_t>({1}), Catch::WithinRel(var, 0.001f));
}

TEST_CASE("Batchnorm2d train 1x2x2x2", "[batchnorm2d]") {
  std::vector<float32_t> im_data = {1, 2, 3, 4, 1, 2, 3, 4};
  std::vector<float32_t> res_data = {-1.3416, -0.4472, 0.4472, 1.3416};

  index_t channels = 2;

  auto im = Tensor::from_blob(
      im_data.data(),
      {.shape = {1, channels, 2, 2}, .device = device_t::CPU, .dtype = to_dtype<float32_t>()});

  BarchNorm2d norm(channels);
  norm->train();

  auto res = norm(im);

  for (auto c : irange(channels)) {
    REQUIRE_THAT(res.data().at<float32_t>({0, c, 0, 0}), Catch::WithinRel(res_data[0], 0.001f));
    REQUIRE_THAT(res.data().at<float32_t>({0, c, 0, 1}), Catch::WithinRel(res_data[1], 0.001f));
    REQUIRE_THAT(res.data().at<float32_t>({0, c, 1, 0}), Catch::WithinRel(res_data[2], 0.001f));
    REQUIRE_THAT(res.data().at<float32_t>({0, c, 1, 1}), Catch::WithinRel(res_data[3], 0.001f));
  }

  mean(silu(res)).backward();

  auto wgrad = norm->weights().grad();
  REQUIRE_THAT(wgrad.at<float32_t>({0}), Catch::WithinRel(0.1963f, 0.001f));
  REQUIRE_THAT(wgrad.at<float32_t>({1}), Catch::WithinRel(0.1963f, 0.001f));

  auto bgrad = norm->bias().grad();
  REQUIRE_THAT(bgrad.at<float32_t>({0}), Catch::WithinRel(0.25f, 0.001f));
  REQUIRE_THAT(bgrad.at<float32_t>({1}), Catch::WithinRel(0.25f, 0.001f));
}