githubmirror/omim

coding/transliteration.cpp
165 строк4 KB
tatiana-yan
Review fixes.
10 июл 2020, 13:48
10 июл 2020, 13:4890a9bf3
#include "coding/transliteration.hpp"

#include "coding/string_utf8_multilang.hpp"

#include "base/logging.hpp"
#include "base/string_utils.hpp"

#include "3party/icu/common/unicode/uclean.h"
#include "3party/icu/common/unicode/unistr.h"
#include "3party/icu/common/unicode/utypes.h"
#include "3party/icu/i18n/unicode/translit.h"
#include "3party/icu/i18n/unicode/utrans.h"

#include <cstring>
#include <mutex>

struct Transliteration::TransliteratorInfo
{
  TransliteratorInfo()
    : m_initialized(false)
  {}

  std::atomic<bool> m_initialized;
  std::mutex m_mutex;
  std::unique_ptr<Transliterator> m_transliterator;
};

Transliteration::Transliteration()
  : m_mode(Mode::Enabled)
{}

Transliteration::~Transliteration()
{
  // The use of u_cleanup() just before an application terminates is optional,
  // but it should be called only once for performance reasons.
  // The primary benefit is to eliminate reports of memory or resource leaks originating
  // in ICU code from the results generated by heap analysis tools.
  m_transliterators.clear();
  u_cleanup();
}

Transliteration & Transliteration::Instance()
{
  static Transliteration instance;
  return instance;
}

void Transliteration::Init(std::string const & icuDataDir)
{
  // Fast atomic check before mutex lock.
  if (m_inited)
    return;

  std::lock_guard<std::mutex> lock(m_initializationMutex);
  if (m_inited)
    return;

  // This function should be called before the first ICU operation that will require the loading of
  // an ICU data file.
  u_setDataDirectory(icuDataDir.c_str());

  for (auto const & lang : StringUtf8Multilang::GetSupportedLanguages())
  {
    for (auto const & t : lang.m_transliteratorsIds)
    {
      if (m_transliterators.count(t) == 0)
        m_transliterators.emplace(t, std::make_unique<TransliteratorInfo>());
    }
  }

  // We need "Hiragana-Katakana" for strings normalization, not for latin transliteration.
  // That's why it is not mentioned in StringUtf8Multilang transliterators list.
  m_transliterators.emplace("Hiragana-Katakana", std::make_unique<TransliteratorInfo>());
  m_inited = true;
}

void Transliteration::SetMode(Transliteration::Mode mode)
{
  m_mode = mode;
}

bool Transliteration::Transliterate(std::string transliteratorId, UnicodeString & ustr) const
{
  CHECK(m_inited, ());
  CHECK(!transliteratorId.empty(), (transliteratorId));

  auto it = m_transliterators.find(transliteratorId);
  if (it == m_transliterators.end())
  {
    LOG(LWARNING, ("Transliteration failed, unknown transliterator \"", transliteratorId, "\""));
    return false;
  }

  if (!it->second->m_initialized)
  {
    std::lock_guard<std::mutex> lock(it->second->m_mutex);
    if (!it->second->m_initialized)
    {
      UErrorCode status = U_ZERO_ERROR;

      std::string const removeDiacriticRule =
          ";NFD;[\u02B9-\u02D3\u0301-\u0358\u00B7\u0027]Remove;NFC";
      transliteratorId.append(removeDiacriticRule);

      UnicodeString translitId(transliteratorId.c_str());

      it->second->m_transliterator.reset(
          Transliterator::createInstance(translitId, UTRANS_FORWARD, status));

      if (it->second->m_transliterator == nullptr)
      {
        LOG(LWARNING,
            ("Cannot create transliterator \"", transliteratorId, "\", icu error =", status));
      }

      it->second->m_initialized = true;
    }
  }

  if (it->second->m_transliterator == nullptr)
    return false;

  it->second->m_transliterator->transliterate(ustr);

  if (ustr.isEmpty())
    return false;

  return true;
}

bool Transliteration::TransliterateForce(std::string const & str, std::string const & transliteratorId,
                                         std::string & out) const
{
  CHECK(m_inited, ());
  UnicodeString ustr(str.c_str());
  auto const res = Transliterate(transliteratorId, ustr);
  if (res)
    ustr.toUTF8String(out);
  return res;
}

bool Transliteration::Transliterate(std::string const & str, int8_t langCode,
                                    std::string & out) const
{
  CHECK(m_inited, ());
  if (m_mode != Mode::Enabled)
    return false;

  if (str.empty() || strings::IsASCIIString(str))
    return false;

  auto const & transliteratorsIds = StringUtf8Multilang::GetTransliteratorsIdsByCode(langCode);
  if (transliteratorsIds.empty())
    return false;

  UnicodeString ustr(str.c_str());
  for (auto transliteratorId : transliteratorsIds)
    Transliterate(transliteratorId, ustr);

  if (ustr.isEmpty())
    return false;

  ustr.toUTF8String(out);
  return true;
}