indexer/metadata_serdes.cpp - master - githubmirror/omim

indexer/metadata_serdes.cpp
186 строк6 KB
tatiana-yan
fixup
06 окт 2020, 10:05
06 окт 2020, 10:051df14fd
#include "indexer/metadata_serdes.hpp"

#include "indexer/feature_meta.hpp"

#include "coding/varint.hpp"

#include "base/assert.hpp"
#include "base/checked_cast.hpp"

#include <type_traits>

using namespace std;

namespace indexer
{
void MetadataDeserializer::Header::Read(Reader & reader)
{
  static_assert(is_same<underlying_type_t<Version>, uint8_t>::value, "");
  NonOwningReaderSource source(reader);
  m_version = static_cast<Version>(ReadPrimitiveFromSource<uint8_t>(source));
  CHECK_EQUAL(base::Underlying(m_version), base::Underlying(Version::V0), ());
  m_stringsOffset = ReadPrimitiveFromSource<uint32_t>(source);
  m_stringsSize = ReadPrimitiveFromSource<uint32_t>(source);
  m_metadataMapOffset = ReadPrimitiveFromSource<uint32_t>(source);
  m_metadataMapSize = ReadPrimitiveFromSource<uint32_t>(source);
}

bool MetadataDeserializer::Get(uint32_t featureId, feature::MetadataBase & meta)
{
  MetaIds metaIds;
  if (!m_map->GetThreadsafe(featureId, metaIds))
    return false;

  lock_guard<mutex> guard(m_stringsMutex);
  for (auto const & id : metaIds)
  {
    CHECK_LESS_OR_EQUAL(id.second, m_strings.GetNumStrings(), ());
    meta.Set(id.first, m_strings.ExtractString(*m_stringsSubreader, id.second));
  }
  return true;
}

bool MetadataDeserializer::GetIds(uint32_t featureId, MetaIds & metaIds) const
{
  return m_map->GetThreadsafe(featureId, metaIds);
}

std::string MetadataDeserializer::GetMetaById(uint32_t id)
{
  lock_guard<mutex> guard(m_stringsMutex);
  return m_strings.ExtractString(*m_stringsSubreader, id);
}

// static
unique_ptr<MetadataDeserializer> MetadataDeserializer::Load(Reader & reader)
{
  auto deserializer = make_unique<MetadataDeserializer>();
  deserializer->m_version = Version::V0;

  Header header;
  header.Read(reader);

  deserializer->m_stringsSubreader =
      reader.CreateSubReader(header.m_stringsOffset, header.m_stringsSize);
  if (!deserializer->m_stringsSubreader)
    return {};
  deserializer->m_strings.InitializeIfNeeded(*deserializer->m_stringsSubreader);

  deserializer->m_mapSubreader =
      reader.CreateSubReader(header.m_metadataMapOffset, header.m_metadataMapSize);
  if (!deserializer->m_mapSubreader)
    return {};

  // Decodes block encoded by writeBlockCallback from MetadataBuilder::Freeze.
  auto const readBlockCallback = [&](NonOwningReaderSource & source, uint32_t blockSize,
                                     vector<MetaIds> & values) {
    // We may have some unused values it the tail of the last block but it's ok because
    // default block size is 64.
    values.resize(blockSize);
    for (size_t i = 0; i < blockSize && source.Size() > 0; ++i)
    {
      auto const size = ReadVarUint<uint32_t>(source);
      values[i].resize(size);
      CHECK_GREATER(size, 0, ());
      for (auto & value : values[i])
        value.first = ReadPrimitiveFromSource<uint8_t>(source);
      values[i][0].second = ReadVarUint<uint32_t>(source);
      for (size_t j = 1; j < values[i].size(); ++j)
      {
        int32_t delta = ReadVarInt<int32_t>(source);
        values[i][j].second = values[i][j - 1].second + delta;
      }
    }
  };

  deserializer->m_map = Map::Load(*deserializer->m_mapSubreader, readBlockCallback);
  if (!deserializer->m_map)
    return {};

  return deserializer;
}

// MetadataBuilder -----------------------------------------------------------------------------
void MetadataBuilder::Put(uint32_t featureId, feature::MetadataBase const & meta)
{
  MetadataDeserializer::MetaIds metaIds;
  for (auto const & type : meta.GetPresentTypes())
  {
    uint32_t id = 0;
    auto const value = meta.Get(type);
    auto const it = m_stringToId.find(value);
    if (it != m_stringToId.end())
    {
      id = it->second;
      CHECK_LESS_OR_EQUAL(id, m_stringToId.size(), ());
      CHECK_EQUAL(m_idToString.count(id), 1, ());
    }
    else
    {
      id = base::asserted_cast<uint32_t>(m_stringToId.size());
      m_stringToId[value] = id;
      CHECK(m_idToString.emplace(id, value).second, ());
      CHECK_EQUAL(m_idToString.size(), m_stringToId.size(), ());
    }
    metaIds.emplace_back(type, id);
  }
  m_builder.Put(featureId, metaIds);
}

void MetadataBuilder::Freeze(Writer & writer) const
{
  size_t startOffset = writer.Pos();
  CHECK(coding::IsAlign8(startOffset), ());

  MetadataDeserializer::Header header;
  header.Serialize(writer);

  uint64_t bytesWritten = writer.Pos();
  coding::WritePadding(writer, bytesWritten);

  header.m_stringsOffset = base::asserted_cast<uint32_t>(writer.Pos() - startOffset);
  {
    coding::BlockedTextStorageWriter<decltype(writer)> stringsWriter(writer, 1000 /* blockSize */);
    for (size_t i = 0; i < m_idToString.size(); ++i)
    {
      auto const it = m_idToString.find(base::asserted_cast<uint32_t>(i));
      CHECK(it != m_idToString.end(), ());
      stringsWriter.Append(it->second);
    }
    // stringsWriter destructor writes strings section index right after strings.
  }

  header.m_stringsSize =
      base::asserted_cast<uint32_t>(writer.Pos() - header.m_stringsOffset - startOffset);
  bytesWritten = writer.Pos();
  coding::WritePadding(writer, bytesWritten);

  header.m_metadataMapOffset = base::asserted_cast<uint32_t>(writer.Pos() - startOffset);
  auto const writeBlockCallback = [](auto & w, auto begin, auto end) {
    for (auto it = begin; it != end; ++it)
    {
      MetadataDeserializer::MetaIds const & metaIds = *it;
      CHECK_GREATER(metaIds.size(), 0, ());
      WriteVarUint(w, metaIds.size());
      for (auto const & kv : metaIds)
        WriteToSink(w, kv.first);

      WriteVarUint(w, metaIds[0].second);
      for (size_t i = 1; i < it->size(); ++i)
      {
        WriteVarInt(w, static_cast<int32_t>(metaIds[i].second) -
                           static_cast<int32_t>(metaIds[i - 1].second));
      }
    }
  };
  m_builder.Freeze(writer, writeBlockCallback);

  header.m_metadataMapSize =
      base::asserted_cast<uint32_t>(writer.Pos() - header.m_metadataMapOffset - startOffset);

  auto const endOffset = writer.Pos();
  writer.Seek(startOffset);
  header.Serialize(writer);
  writer.Seek(endOffset);
}
}  // namespace indexer