githubmirror/omim

search/ranking_utils.cpp
135 строк4 KB
tatiana-yan
[search] Query token synonyms minor refactoring.
13 фев 2019, 19:53
13 фев 2019, 19:539a510f4
#include "search/ranking_utils.hpp"
#include "search/token_slice.hpp"
#include "search/utils.hpp"

#include "base/dfa_helpers.hpp"

#include "indexer/categories_holder.hpp"
#include "indexer/feature_data.hpp"

#include "std/transform_iterator.hpp"

#include <algorithm>
#include <sstream>

using namespace std;
using namespace strings;

namespace search
{
namespace
{
struct TokenInfo
{
  bool m_isCategoryToken = false;
  bool m_inFeatureTypes = false;
};
}  // namespace

// CategoriesInfo ----------------------------------------------------------------------------------
CategoriesInfo::CategoriesInfo(feature::TypesHolder const & holder, TokenSlice const & tokens,
                               Locales const & locales, CategoriesHolder const & categories)
{
  QuerySlice slice(tokens);
  vector<TokenInfo> infos(slice.Size());
  ForEachCategoryType(slice, locales, categories, [&holder, &infos](size_t i, uint32_t t) {
    ASSERT_LESS(i, infos.size(), ());
    auto & info = infos[i];

    info.m_isCategoryToken = true;
    if (holder.Has(t))
      info.m_inFeatureTypes = true;
  });

  // Note that m_inFeatureTypes implies m_isCategoryToken.

  m_pureCategories = all_of(infos.begin(), infos.end(), [](TokenInfo const & info) {
    ASSERT(!info.m_inFeatureTypes || info.m_isCategoryToken, ());
    return info.m_inFeatureTypes != 0;
  });

  m_falseCategories = all_of(infos.begin(), infos.end(), [](TokenInfo const & info) {
    return !info.m_inFeatureTypes && info.m_isCategoryToken;
  });
}

// ErrorsMade --------------------------------------------------------------------------------------
string DebugPrint(ErrorsMade const & errorsMade)
{
  ostringstream os;
  os << "ErrorsMade [ ";
  if (errorsMade.IsValid())
    os << errorsMade.m_errorsMade;
  else
    os << "invalid";
  os << " ]";
  return os.str();
}

namespace impl
{
bool FullMatch(QueryParams::Token const & token, UniString const & text)
{
  return token.AnyOf([&text](UniString const & s) { return s == text; });
}

bool PrefixMatch(QueryParams::Token const & token, UniString const & text)
{
  return token.AnyOf([&text](UniString const & s) { return StartsWith(text, s); });
}

ErrorsMade GetMinErrorsMade(vector<strings::UniString> const & tokens,
                            strings::UniString const & text)
{
  auto const dfa = BuildLevenshteinDFA(text);

  ErrorsMade errorsMade;

  for (auto const & token : tokens)
  {
    auto it = dfa.Begin();
    strings::DFAMove(it, token.begin(), token.end());
    if (it.Accepts())
      errorsMade = ErrorsMade::Min(errorsMade, ErrorsMade(it.ErrorsMade()));
  }

  return errorsMade;
}
}  // namespace impl

bool IsStopWord(UniString const & s)
{
  /// @todo Get all common used stop words and factor out this array into
  /// search_string_utils.cpp module for example.
  static char const * arr[] = {"a", "de", "di", "da", "la", "le", "де", "ди", "да", "ла", "ля", "ле"};

  static set<UniString> const kStopWords(
      make_transform_iterator(arr, &MakeUniString),
      make_transform_iterator(arr + ARRAY_SIZE(arr), &MakeUniString));

  return kStopWords.count(s) > 0;
}

void PrepareStringForMatching(string const & name, vector<strings::UniString> & tokens)
{
  auto filter = [&tokens](strings::UniString const & token)
  {
    if (!IsStopWord(token))
      tokens.push_back(token);
  };
  SplitUniString(NormalizeAndSimplifyString(name), filter, Delimiters());
}

string DebugPrint(NameScore score)
{
  switch (score)
  {
  case NAME_SCORE_ZERO: return "Zero";
  case NAME_SCORE_SUBSTRING: return "Substring";
  case NAME_SCORE_PREFIX: return "Prefix";
  case NAME_SCORE_FULL_MATCH: return "Full Match";
  case NAME_SCORE_COUNT: return "Count";
  }
  return "Unknown";
}
}  // namespace search