Ton

BlobView.cpp
340 строк · 10.6 Кб
Перенос по словам
1
/*
2
    This file is part of TON Blockchain Library.
3

4
    TON Blockchain Library is free software: you can redistribute it and/or modify
5
    it under the terms of the GNU Lesser General Public License as published by
6
    the Free Software Foundation, either version 2 of the License, or
7
    (at your option) any later version.
8

9
    TON Blockchain Library is distributed in the hope that it will be useful,
10
    but WITHOUT ANY WARRANTY; without even the implied warranty of
11
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
12
    GNU Lesser General Public License for more details.
13

14
    You should have received a copy of the GNU Lesser General Public License
15
    along with TON Blockchain Library.  If not, see <http://www.gnu.org/licenses/>.
16

17
    Copyright 2017-2020 Telegram Systems LLP
18
*/
19
#include "td/db/utils/BlobView.h"
20

21
#include "td/utils/port/FileFd.h"
22
#include "td/utils/HashMap.h"
23

24
#include "td/utils/format.h"
25
#include "td/utils/port/RwMutex.h"
26
#include "td/utils/port/MemoryMapping.h"
27

28
#include <limits>
29
#include <mutex>
30

31
namespace td {
32

33
class BlobViewImpl {
34
 public:
35
  virtual ~BlobViewImpl() = default;
36
  td::Result<td::Slice> view(td::MutableSlice slice, td::uint64 offset);
37
  td::Result<size_t> view_copy(td::MutableSlice slice, td::uint64 offset);
38
  td::Result<td::BufferSlice> to_buffer_slice();
39
  td::Result<size_t> write(td::Slice data, td::uint64 offset);
40
  virtual td::Status sync() {
41
    return td::Status::OK();
42
  }
43
  virtual td::uint64 size() = 0;
44

45
 private:
46
  virtual td::Result<td::Slice> view_impl(td::MutableSlice slice, td::uint64 offset) = 0;
47
  virtual td::Result<size_t> write_impl(td::Slice data, td::uint64 offset) {
48
    return td::Status::Error("Read only blob");
49
  }
50
};
51

52
BlobView::BlobView() = default;
53

54
BlobView::BlobView(std::unique_ptr<BlobViewImpl> impl) : impl_(std::move(impl)) {
55
}
56

57
BlobView::BlobView(BlobView &&) = default;
58

59
BlobView &BlobView::operator=(BlobView &&) = default;
60

61
BlobView::~BlobView() {
62
}
63

64
td::Result<td::Slice> BlobView::view(td::MutableSlice slice, td::uint64 offset) {
65
  CHECK(impl_);
66
  return impl_->view(slice, offset);
67
}
68
td::Result<size_t> BlobView::write(td::Slice data, td::uint64 offset) {
69
  CHECK(impl_);
70
  return impl_->write(data, offset);
71
}
72

73
td::Result<td::BufferSlice> BlobView::to_buffer_slice() {
74
  td::BufferSlice res(size());
75
  TRY_RESULT(read_size, view_copy(res.as_slice(), 0));
76
  if (read_size != res.size()) {
77
    return td::Status::Error("Can't view the whole blob");
78
  }
79
  return std::move(res);
80
}
81

82
td::Result<size_t> BlobView::view_copy(td::MutableSlice slice, td::uint64 offset) {
83
  TRY_RESULT(res, view(slice, offset));
84
  if (res.begin() != slice.begin()) {
85
    slice.copy_from(res);
86
  }
87
  return res.size();
88
}
89

90
td::uint64 BlobView::size() {
91
  CHECK(impl_);
92
  return impl_->size();
93
}
94

95
td::Result<td::Slice> BlobViewImpl::view(td::MutableSlice slice, td::uint64 offset) {
96
  if (offset > size() || slice.size() > size() - offset) {
97
    return td::Status::Error(PSLICE() << "BlobView: invalid range requested " << td::tag("slice offset", offset)
98
                                      << td::tag("slice size", slice.size()) << td::tag("blob size", size()));
99
  }
100
  return view_impl(slice, offset);
101
}
102

103
td::Result<size_t> BlobViewImpl::write(td::Slice slice, td::uint64 offset) {
104
  if (offset > size() || slice.size() > size() - offset) {
105
    return td::Status::Error(PSLICE() << "BlobView: invalid range requested " << td::tag("slice offset", offset)
106
                                      << td::tag("slice size", slice.size()) << td::tag("blob size", size()));
107
  }
108
  return write_impl(slice, offset);
109
}
110

111
namespace {
112
class BufferSliceBlobViewImpl : public BlobViewImpl {
113
 public:
114
  BufferSliceBlobViewImpl(td::BufferSlice slice) : slice_(std::move(slice)) {
115
  }
116
  td::Result<td::Slice> view_impl(td::MutableSlice slice, td::uint64 offset) override {
117
    // optimize anyway
118
    if (offset > std::numeric_limits<std::size_t>::max()) {
119
      return td::Slice();
120
    }
121
    return slice_.as_slice().substr(static_cast<std::size_t>(offset), slice.size());
122
  }
123

124
  td::Result<size_t> write_impl(td::Slice data, td::uint64 offset) override {
125
    slice_.as_slice().substr(offset).copy_from(data);
126
    return data.size();
127
  }
128

129
  td::uint64 size() override {
130
    return slice_.size();
131
  }
132

133
 private:
134
  td::BufferSlice slice_;
135
};
136
}  // namespace
137

138
BlobView BufferSliceBlobView::create(td::BufferSlice slice) {
139
  return BlobView(std::make_unique<BufferSliceBlobViewImpl>(std::move(slice)));
140
}
141

142
class FileBlobViewImpl : public BlobViewImpl {
143
 public:
144
  FileBlobViewImpl(td::FileFd fd, td::uint64 file_size) : fd_(std::move(fd)), file_size_(file_size) {
145
  }
146

147
  td::uint64 size() override {
148
    return file_size_;
149
  }
150
  td::Result<td::Slice> view_impl(td::MutableSlice slice, td::uint64 offset) override {
151
    CHECK(offset < size());
152
    CHECK(size() - offset >= slice.size());
153
    slice.truncate(file_size_ - offset);
154
    auto first_page = offset / page_size;
155
    auto last_page = (offset + slice.size() - 1) / page_size;
156
    td::uint64 res_offset = 0;
157
    for (auto page_i = first_page; page_i <= last_page; page_i++) {
158
      auto page_offset = page_i * page_size;
159
      auto from = td::max(page_offset, offset);
160
      auto till = td::min(page_offset + page_size, offset + slice.size());
161
      CHECK(from < till);
162
      TRY_RESULT(page, load_page(page_i));
163
      auto len = till - from;
164
      slice.substr(res_offset, len).copy_from(page.substr(from - page_offset, len));
165
      res_offset += len;
166
    }
167
    CHECK(slice.size() == res_offset);
168
    total_view_size_ += slice.size();
169
    return slice;
170
  }
171
  ~FileBlobViewImpl() {
172
    //LOG(ERROR) << "LOADED " << pages_.size() << " " << total_view_size_;
173
  }
174

175
 private:
176
  td::FileFd fd_;
177
  td::uint64 file_size_;
178
  const td::uint64 page_size = 4096;
179
  td::uint64 total_view_size_{0};
180

181
  td::RwMutex pages_rw_mutex_;
182
  td::HashMap<td::uint64, td::BufferSlice> pages_;
183

184
  std::mutex fd_mutex_;
185

186
  td::Result<td::Slice> load_page(td::uint64 page_i) {
187
    {
188
      auto pages_guard = pages_rw_mutex_.lock_read();
189
      auto it = pages_.find(page_i);
190
      if (it != pages_.end()) {
191
        return it->second.as_slice();
192
      }
193
    }
194

195
    std::lock_guard<std::mutex> fd_guard(fd_mutex_);
196
    {
197
      auto pages_guard = pages_rw_mutex_.lock_read();
198
      auto it = pages_.find(page_i);
199
      if (it != pages_.end()) {
200
        return it->second.as_slice();
201
      }
202
    }
203
    auto offset = page_i * page_size;
204

205
    auto size = td::min(file_size_ - offset, page_size);
206
    auto buffer_slice = td::BufferSlice(size);
207
    TRY_RESULT(read_size, fd_.pread(buffer_slice.as_slice(), offset));
208
    if (read_size != buffer_slice.size()) {
209
      return td::Status::Error("not enough data in file");
210
    }
211

212
    auto pages_guard = pages_rw_mutex_.lock_write();
213
    auto &res = pages_[page_i];
214
    res = std::move(buffer_slice);
215
    return res.as_slice();
216
  }
217
};
218

219
td::Result<BlobView> FileBlobView::create(td::CSlice file_path, td::uint64 file_size) {
220
  TRY_RESULT(fd, td::FileFd::open(file_path, td::FileFd::Flags::Read));
221
  TRY_RESULT(stat, fd.stat());
222
  if (file_size == 0) {
223
    file_size = stat.size_;
224
  } else if (file_size != (td::uint64)stat.size_) {
225
    return td::Status::Error(PSLICE() << "Wrong file size (1) expected:" << file_size << " got:" << stat.size_);
226
  }
227
  return BlobView(std::make_unique<FileBlobViewImpl>(std::move(fd), file_size));
228
}
229

230
class FileNoCacheBlobViewImpl : public BlobViewImpl {
231
 public:
232
  FileNoCacheBlobViewImpl(td::FileFd fd, td::uint64 file_size) : fd_(std::move(fd)), file_size_(file_size) {
233
  }
234

235
  td::uint64 size() override {
236
    return file_size_;
237
  }
238
  td::Result<td::Slice> view_impl(td::MutableSlice slice, td::uint64 offset) override {
239
    CHECK(offset < size());
240
    CHECK(size() - offset >= slice.size());
241
    slice.truncate(file_size_ - offset);
242
    TRY_RESULT(read_size, fd_.pread(slice, offset));
243
    slice.truncate(read_size);
244
    return slice;
245
  }
246

247
  td::Result<size_t> write_impl(td::Slice data, td::uint64 offset) override {
248
    return fd_.pwrite(data, offset);
249
  }
250

251
  ~FileNoCacheBlobViewImpl() {
252
  }
253

254
 private:
255
  td::FileFd fd_;
256
  td::uint64 file_size_;
257
};
258

259
td::Result<BlobView> FileNoCacheBlobView::create(td::CSlice file_path, td::uint64 file_size, bool may_write) {
260
  td::int32 flags = td::FileFd::Flags::Read;
261
  if (may_write) {
262
    flags |= td::FileFd::Flags::Create | td::FileFd::Flags::Write;
263
  }
264
  TRY_RESULT(fd, td::FileFd::open(file_path, flags));
265
  TRY_RESULT(stat, fd.stat());
266
  if (file_size == 0) {
267
    file_size = stat.size_;
268
  } else if (file_size != (td::uint64)stat.size_) {
269
    if (stat.size_ == 0) {
270
      TRY_STATUS(fd.seek(file_size));
271
      TRY_STATUS(fd.truncate_to_current_position(file_size));
272
      TRY_STATUS(fd.seek(0));
273
    } else {
274
      LOG(ERROR) << file_path;
275
      return td::Status::Error(PSLICE() << "Wrong file size (2) expected:" << file_size << " got:" << stat.size_);
276
    }
277
  }
278
  return BlobView(std::make_unique<FileNoCacheBlobViewImpl>(std::move(fd), file_size));
279
}
280

281
class FileMemoryMappingBlobViewImpl : public BlobViewImpl {
282
 public:
283
  FileMemoryMappingBlobViewImpl(td::MemoryMapping mapping) : mapping_(std::move(mapping)) {
284
  }
285
  td::Result<td::Slice> view_impl(td::MutableSlice slice, td::uint64 offset) override {
286
    // optimize anyway
287
    return mapping_.as_slice().substr(offset, slice.size());
288
  }
289
  td::uint64 size() override {
290
    return mapping_.as_slice().size();
291
  }
292

293
 private:
294
  td::MemoryMapping mapping_;
295
};
296

297
td::Result<BlobView> FileMemoryMappingBlobView::create(td::CSlice file_path, td::uint64 file_size) {
298
  TRY_RESULT(fd, td::FileFd::open(file_path, td::FileFd::Flags::Read));
299
  TRY_RESULT(stat, fd.stat());
300
  if (file_size == 0) {
301
    file_size = stat.size_;
302
  } else if (file_size != (td::uint64)stat.size_) {
303
    return td::Status::Error(PSLICE() << "Wrong file size (3) expected:" << file_size << " got:" << stat.size_);
304
  }
305

306
  TRY_RESULT(mapping, td::MemoryMapping::create_from_file(fd));
307

308
  return BlobView(std::make_unique<FileMemoryMappingBlobViewImpl>(std::move(mapping)));
309
}
310

311
class CyclicBlobViewImpl : public BlobViewImpl {
312
 public:
313
  CyclicBlobViewImpl(td::BufferSlice data, td::uint64 total_size) : data_(std::move(data)), total_size_(total_size) {
314
    CHECK(!data_.empty());
315
  }
316
  td::Result<td::Slice> view_impl(td::MutableSlice slice, td::uint64 offset) override {
317
    auto res = slice;
318
    offset %= data_.size();
319
    while (!slice.empty()) {
320
      auto from = data_.as_slice().substr(offset).truncate(slice.size());
321
      slice.copy_from(from);
322
      slice.remove_prefix(from.size());
323
      offset = 0;
324
    }
325
    return res;
326
  }
327
  td::uint64 size() override {
328
    return total_size_;
329
  }
330

331
 private:
332
  td::BufferSlice data_;
333
  td::uint64 total_size_;
334
};
335

336
td::Result<td::BlobView> CycicBlobView::create(td::BufferSlice data, td::uint64 total_size) {
337
  return BlobView(std::make_unique<CyclicBlobViewImpl>(std::move(data), total_size));
338
}
339

340
}  // namespace td
341
Ton

Использование cookies
