githubmirror/aria2

src/RequestGroup.cc
1 279 строк40 KB
Tatsuhiro Tsujikawa
Merge branch 'master' into random-webseeding
26 мар 2016, 11:20
26 мар 2016, 11:20d640896
/* <!-- copyright */
/*
 * aria2 - The high speed download utility
 *
 * Copyright (C) 2006 Tatsuhiro Tsujikawa
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
 *
 * In addition, as a special exception, the copyright holders give
 * permission to link the code of portions of this program with the
 * OpenSSL library under certain conditions as described in each
 * individual source file, and distribute linked combinations
 * including the two.
 * You must obey the GNU General Public License in all respects
 * for all of the code used other than OpenSSL.  If you modify
 * file(s) with this exception, you may extend this exception to your
 * version of the file(s), but you are not obligated to do so.  If you
 * do not wish to do so, delete this exception statement from your
 * version.  If you delete this exception statement from all source
 * files in the program, then also delete it here.
 */
/* copyright --> */
#include "RequestGroup.h"

#include <cassert>
#include <algorithm>

#include "PostDownloadHandler.h"
#include "DownloadEngine.h"
#include "SegmentMan.h"
#include "NullProgressInfoFile.h"
#include "Dependency.h"
#include "prefs.h"
#include "CreateRequestCommand.h"
#include "File.h"
#include "message.h"
#include "util.h"
#include "LogFactory.h"
#include "Logger.h"
#include "DiskAdaptor.h"
#include "DiskWriterFactory.h"
#include "RecoverableException.h"
#include "StreamCheckIntegrityEntry.h"
#include "CheckIntegrityCommand.h"
#include "UnknownLengthPieceStorage.h"
#include "DownloadContext.h"
#include "DlAbortEx.h"
#include "DownloadFailureException.h"
#include "RequestGroupMan.h"
#include "DefaultBtProgressInfoFile.h"
#include "DefaultPieceStorage.h"
#include "download_handlers.h"
#include "MemoryBufferPreDownloadHandler.h"
#include "DownloadHandlerConstants.h"
#include "Option.h"
#include "FileEntry.h"
#include "Request.h"
#include "FileAllocationIterator.h"
#include "fmt.h"
#include "A2STR.h"
#include "URISelector.h"
#include "InorderURISelector.h"
#include "PieceSelector.h"
#include "a2functional.h"
#include "SocketCore.h"
#include "SimpleRandomizer.h"
#include "Segment.h"
#include "SocketRecvBuffer.h"
#include "RequestGroupCriteria.h"
#include "CheckIntegrityCommand.h"
#include "ChecksumCheckIntegrityEntry.h"
#ifdef ENABLE_BITTORRENT
#include "bittorrent_helper.h"
#include "BtRegistry.h"
#include "BtCheckIntegrityEntry.h"
#include "DefaultPeerStorage.h"
#include "DefaultBtAnnounce.h"
#include "BtRuntime.h"
#include "BtSetup.h"
#include "BtPostDownloadHandler.h"
#include "DHTSetup.h"
#include "DHTRegistry.h"
#include "DHTNode.h"
#include "DHTRoutingTable.h"
#include "DHTTaskQueue.h"
#include "DHTTaskFactory.h"
#include "DHTTokenTracker.h"
#include "DHTMessageDispatcher.h"
#include "DHTMessageReceiver.h"
#include "DHTMessageFactory.h"
#include "DHTMessageCallback.h"
#include "BtMessageFactory.h"
#include "BtRequestFactory.h"
#include "BtMessageDispatcher.h"
#include "BtMessageReceiver.h"
#include "PeerConnection.h"
#include "ExtensionMessageFactory.h"
#include "DHTPeerAnnounceStorage.h"
#include "DHTEntryPointNameResolveCommand.h"
#include "LongestSequencePieceSelector.h"
#include "PriorityPieceSelector.h"
#include "bittorrent_helper.h"
#endif // ENABLE_BITTORRENT
#ifdef ENABLE_METALINK
#include "MetalinkPostDownloadHandler.h"
#endif // ENABLE_METALINK

namespace aria2 {

RequestGroup::RequestGroup(const std::shared_ptr<GroupId>& gid,
                           const std::shared_ptr<Option>& option)
    : belongsToGID_(0),
      gid_(gid),
      option_(option),
      progressInfoFile_(std::make_shared<NullProgressInfoFile>()),
      uriSelector_(make_unique<InorderURISelector>()),
      requestGroupMan_(nullptr),
#ifdef ENABLE_BITTORRENT
      btRuntime_(nullptr),
      peerStorage_(nullptr),
#endif // ENABLE_BITTORRENT
      followingGID_(0),
      lastModifiedTime_(Time::null()),
      timeout_(option->getAsInt(PREF_TIMEOUT)),
      state_(STATE_WAITING),
      numConcurrentCommand_(option->getAsInt(PREF_SPLIT)),
      numStreamConnection_(0),
      numStreamCommand_(0),
      numCommand_(0),
      fileNotFoundCount_(0),
      maxDownloadSpeedLimit_(option->getAsInt(PREF_MAX_DOWNLOAD_LIMIT)),
      maxUploadSpeedLimit_(option->getAsInt(PREF_MAX_UPLOAD_LIMIT)),
      resumeFailureCount_(0),
      haltReason_(RequestGroup::NONE),
      lastErrorCode_(error_code::UNDEFINED),
      saveControlFile_(true),
      preLocalFileCheckEnabled_(true),
      haltRequested_(false),
      forceHaltRequested_(false),
      pauseRequested_(false),
      inMemoryDownload_(false),
      seedOnly_(false)
{
  fileAllocationEnabled_ = option_->get(PREF_FILE_ALLOCATION) != V_NONE;
  if (!option_->getAsBool(PREF_DRY_RUN)) {
    initializePreDownloadHandler();
    initializePostDownloadHandler();
  }
}

RequestGroup::~RequestGroup() {}

bool RequestGroup::isCheckIntegrityReady()
{
  return option_->getAsBool(PREF_CHECK_INTEGRITY) &&
         ((downloadContext_->isChecksumVerificationAvailable() &&
           downloadFinishedByFileLength()) ||
          downloadContext_->isPieceHashVerificationAvailable());
}

bool RequestGroup::downloadFinished() const
{
  if (!pieceStorage_) {
    return false;
  }
  return pieceStorage_->downloadFinished();
}

bool RequestGroup::allDownloadFinished() const
{
  if (!pieceStorage_) {
    return false;
  }
  return pieceStorage_->allDownloadFinished();
}

std::pair<error_code::Value, std::string> RequestGroup::downloadResult() const
{
  if (downloadFinished() && !downloadContext_->isChecksumVerificationNeeded()) {
    return std::make_pair(error_code::FINISHED, "");
  }

  if (haltReason_ == RequestGroup::USER_REQUEST) {
    return std::make_pair(error_code::REMOVED, "");
  }

  if (lastErrorCode_ == error_code::UNDEFINED) {
    if (haltReason_ == RequestGroup::SHUTDOWN_SIGNAL) {
      return std::make_pair(error_code::IN_PROGRESS, "");
    }
    return std::make_pair(error_code::UNKNOWN_ERROR, "");
  }

  return std::make_pair(lastErrorCode_, lastErrorMessage_);
}

void RequestGroup::closeFile()
{
  if (pieceStorage_) {
    pieceStorage_->flushWrDiskCacheEntry();
    pieceStorage_->getDiskAdaptor()->closeFile();
  }
}

// TODO The function name is not intuitive at all.. it does not convey
// that this function open file.
std::unique_ptr<CheckIntegrityEntry> RequestGroup::createCheckIntegrityEntry()
{
  auto infoFile = std::make_shared<DefaultBtProgressInfoFile>(
      downloadContext_, pieceStorage_, option_.get());

  if (option_->getAsBool(PREF_CHECK_INTEGRITY) &&
      downloadContext_->isPieceHashVerificationAvailable()) {
    // When checking piece hash, we don't care file is downloaded and
    // infoFile exists.
    loadAndOpenFile(infoFile);
    return make_unique<StreamCheckIntegrityEntry>(this);
  }

  if (isPreLocalFileCheckEnabled() &&
      (infoFile->exists() || (File(getFirstFilePath()).exists() &&
                              option_->getAsBool(PREF_CONTINUE)))) {
    // If infoFile exists or -c option is given, we need to check
    // download has been completed (which is determined after
    // loadAndOpenFile()). If so, use ChecksumCheckIntegrityEntry when
    // verification is enabled, because CreateRequestCommand does not
    // issue checksum verification and download fails without it.
    loadAndOpenFile(infoFile);
    if (downloadFinished()) {
      if (downloadContext_->isChecksumVerificationNeeded()) {
        A2_LOG_INFO(MSG_HASH_CHECK_NOT_DONE);
        auto tempEntry = make_unique<ChecksumCheckIntegrityEntry>(this);
        tempEntry->setRedownload(true);
        return std::move(tempEntry);
      }
      downloadContext_->setChecksumVerified(true);
      A2_LOG_NOTICE(fmt(MSG_DOWNLOAD_ALREADY_COMPLETED, gid_->toHex().c_str(),
                        downloadContext_->getBasePath().c_str()));
      return nullptr;
    }
    return make_unique<StreamCheckIntegrityEntry>(this);
  }

  if (downloadFinishedByFileLength() &&
      downloadContext_->isChecksumVerificationAvailable()) {
    pieceStorage_->markAllPiecesDone();
    loadAndOpenFile(infoFile);
    auto tempEntry = make_unique<ChecksumCheckIntegrityEntry>(this);
    tempEntry->setRedownload(true);
    return std::move(tempEntry);
  }

  loadAndOpenFile(infoFile);
  return make_unique<StreamCheckIntegrityEntry>(this);
}

void RequestGroup::createInitialCommand(
    std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e)
{
  // Start session timer here.  When file size becomes known, it will
  // be reset again in *FileAllocationEntry, because hash check and
  // file allocation takes a time.  For downloads in which file size
  // is unknown, session timer will not be reset.
  downloadContext_->resetDownloadStartTime();
#ifdef ENABLE_BITTORRENT
  if (downloadContext_->hasAttribute(CTX_ATTR_BT)) {
    auto torrentAttrs = bittorrent::getTorrentAttrs(downloadContext_);
    bool metadataGetMode = torrentAttrs->metadata.empty();
    if (option_->getAsBool(PREF_DRY_RUN)) {
      throw DOWNLOAD_FAILURE_EXCEPTION(
          "Cancel BitTorrent download in dry-run context.");
    }
    auto& btRegistry = e->getBtRegistry();
    if (btRegistry->getDownloadContext(torrentAttrs->infoHash)) {
      // TODO If metadataGetMode == false and each FileEntry has
      // URI, then go without BT.
      throw DOWNLOAD_FAILURE_EXCEPTION2(
          fmt("InfoHash %s is already registered.",
              bittorrent::getInfoHashString(downloadContext_).c_str()),
          error_code::DUPLICATE_INFO_HASH);
    }
    if (metadataGetMode) {
      // Use UnknownLengthPieceStorage.
      initPieceStorage();
    }
    else if (e->getRequestGroupMan()->isSameFileBeingDownloaded(this)) {
      throw DOWNLOAD_FAILURE_EXCEPTION2(
          fmt(EX_DUPLICATE_FILE_DOWNLOAD,
              downloadContext_->getBasePath().c_str()),
          error_code::DUPLICATE_DOWNLOAD);
    }
    else {
      initPieceStorage();
      if (downloadContext_->getFileEntries().size() > 1) {
        pieceStorage_->setupFileFilter();
      }
    }

    std::shared_ptr<DefaultBtProgressInfoFile> progressInfoFile;
    if (!metadataGetMode) {
      progressInfoFile = std::make_shared<DefaultBtProgressInfoFile>(
          downloadContext_, pieceStorage_, option_.get());
    }

    auto btRuntime = std::make_shared<BtRuntime>();
    btRuntime->setMaxPeers(option_->getAsInt(PREF_BT_MAX_PEERS));
    btRuntime_ = btRuntime.get();
    if (progressInfoFile) {
      progressInfoFile->setBtRuntime(btRuntime);
    }

    auto peerStorage = std::make_shared<DefaultPeerStorage>();
    peerStorage->setBtRuntime(btRuntime);
    peerStorage->setPieceStorage(pieceStorage_);
    peerStorage_ = peerStorage.get();
    if (progressInfoFile) {
      progressInfoFile->setPeerStorage(peerStorage);
    }

    auto btAnnounce = std::make_shared<DefaultBtAnnounce>(
        downloadContext_.get(), option_.get());
    btAnnounce->setBtRuntime(btRuntime);
    btAnnounce->setPieceStorage(pieceStorage_);
    btAnnounce->setPeerStorage(peerStorage);
    btAnnounce->setUserDefinedInterval(
        std::chrono::seconds(option_->getAsInt(PREF_BT_TRACKER_INTERVAL)));
    btAnnounce->shuffleAnnounce();

    assert(!btRegistry->get(gid_->getNumericId()));
    btRegistry->put(
        gid_->getNumericId(),
        make_unique<BtObject>(
            downloadContext_, pieceStorage_, peerStorage, btAnnounce, btRuntime,
            (progressInfoFile ? progressInfoFile : progressInfoFile_)));

    if (option_->getAsBool(PREF_ENABLE_DHT) ||
        (!e->getOption()->getAsBool(PREF_DISABLE_IPV6) &&
         option_->getAsBool(PREF_ENABLE_DHT6))) {

      if (option_->getAsBool(PREF_ENABLE_DHT)) {
        std::vector<std::unique_ptr<Command>> c, rc;
        std::tie(c, rc) = DHTSetup().setup(e, AF_INET);

        e->addCommand(std::move(c));
        for (auto& a : rc) {
          e->addRoutineCommand(std::move(a));
        }
      }

      if (!e->getOption()->getAsBool(PREF_DISABLE_IPV6) &&
          option_->getAsBool(PREF_ENABLE_DHT6)) {
        std::vector<std::unique_ptr<Command>> c, rc;
        std::tie(c, rc) = DHTSetup().setup(e, AF_INET6);

        e->addCommand(std::move(c));
        for (auto& a : rc) {
          e->addRoutineCommand(std::move(a));
        }
      }
      const auto& nodes = torrentAttrs->nodes;
      // TODO Are nodes in torrent IPv4 only?
      if (!torrentAttrs->privateTorrent && !nodes.empty() &&
          DHTRegistry::isInitialized()) {
        auto command = make_unique<DHTEntryPointNameResolveCommand>(
            e->newCUID(), e, nodes);
        command->setTaskQueue(DHTRegistry::getData().taskQueue.get());
        command->setTaskFactory(DHTRegistry::getData().taskFactory.get());
        command->setRoutingTable(DHTRegistry::getData().routingTable.get());
        command->setLocalNode(DHTRegistry::getData().localNode);
        e->addCommand(std::move(command));
      }
    }
    else if (metadataGetMode) {
      A2_LOG_NOTICE(_("For BitTorrent Magnet URI, enabling DHT is strongly"
                      " recommended. See --enable-dht option."));
    }

    if (metadataGetMode) {
      BtCheckIntegrityEntry{this}.onDownloadIncomplete(commands, e);
      return;
    }

    removeDefunctControlFile(progressInfoFile);
    {
      int64_t actualFileSize = pieceStorage_->getDiskAdaptor()->size();
      if (actualFileSize == downloadContext_->getTotalLength()) {
        // First, make DiskAdaptor read-only mode to allow the
        // program to seed file in read-only media.
        pieceStorage_->getDiskAdaptor()->enableReadOnly();
      }
      else {
        // Open file in writable mode to allow the program
        // truncate the file to downloadContext_->getTotalLength()
        A2_LOG_DEBUG(fmt("File size not match. File is opened in writable"
                         " mode. Expected:%" PRId64 " Actual:%" PRId64 "",
                         downloadContext_->getTotalLength(), actualFileSize));
      }
    }
    // Call Load, Save and file allocation command here
    if (progressInfoFile->exists()) {
      // load .aria2 file if it exists.
      progressInfoFile->load();
      pieceStorage_->getDiskAdaptor()->openFile();
    }
    else if (pieceStorage_->getDiskAdaptor()->fileExists()) {
      if (!option_->getAsBool(PREF_CHECK_INTEGRITY) &&
          !option_->getAsBool(PREF_ALLOW_OVERWRITE) &&
          !option_->getAsBool(PREF_BT_SEED_UNVERIFIED)) {
        // TODO we need this->haltRequested = true?
        throw DOWNLOAD_FAILURE_EXCEPTION2(
            fmt(MSG_FILE_ALREADY_EXISTS,
                downloadContext_->getBasePath().c_str()),
            error_code::FILE_ALREADY_EXISTS);
      }
      pieceStorage_->getDiskAdaptor()->openFile();
      if (option_->getAsBool(PREF_BT_SEED_UNVERIFIED)) {
        pieceStorage_->markAllPiecesDone();
      }
    }
    else {
      pieceStorage_->getDiskAdaptor()->openFile();
    }
    progressInfoFile_ = progressInfoFile;

    auto entry = make_unique<BtCheckIntegrityEntry>(this);
    // --bt-seed-unverified=true is given and download has completed, skip
    // validation for piece hashes.
    if (option_->getAsBool(PREF_BT_SEED_UNVERIFIED) &&
        pieceStorage_->downloadFinished()) {
      entry->onDownloadFinished(commands, e);
    }
    else {
      processCheckIntegrityEntry(commands, std::move(entry), e);
    }
    return;
  }
#endif // ENABLE_BITTORRENT

  if (downloadContext_->getFileEntries().size() == 1) {
    // TODO I assume here when totallength is set to DownloadContext and it is
    // not 0, then filepath is also set DownloadContext correctly....
    if (option_->getAsBool(PREF_DRY_RUN) ||
        downloadContext_->getTotalLength() == 0) {
      createNextCommand(commands, e, 1);
      return;
    }
    auto progressInfoFile = std::make_shared<DefaultBtProgressInfoFile>(
        downloadContext_, nullptr, option_.get());
    adjustFilename(progressInfoFile);
    initPieceStorage();
    auto checkEntry = createCheckIntegrityEntry();
    if (checkEntry) {
      processCheckIntegrityEntry(commands, std::move(checkEntry), e);
    }
    return;
  }

  // TODO --dry-run is not supported for multifile download for now.
  if (option_->getAsBool(PREF_DRY_RUN)) {
    throw DOWNLOAD_FAILURE_EXCEPTION(
        "--dry-run in multi-file download is not supported yet.");
  }
  // TODO file size is known in this context?

  // In this context, multiple FileEntry objects are in
  // DownloadContext.
  if (e->getRequestGroupMan()->isSameFileBeingDownloaded(this)) {
    throw DOWNLOAD_FAILURE_EXCEPTION2(
        fmt(EX_DUPLICATE_FILE_DOWNLOAD,
            downloadContext_->getBasePath().c_str()),
        error_code::DUPLICATE_DOWNLOAD);
  }
  initPieceStorage();
  if (downloadContext_->getFileEntries().size() > 1) {
    pieceStorage_->setupFileFilter();
  }
  auto progressInfoFile = std::make_shared<DefaultBtProgressInfoFile>(
      downloadContext_, pieceStorage_, option_.get());
  removeDefunctControlFile(progressInfoFile);
  // Call Load, Save and file allocation command here
  if (progressInfoFile->exists()) {
    // load .aria2 file if it exists.
    progressInfoFile->load();
    pieceStorage_->getDiskAdaptor()->openFile();
  }
  else if (pieceStorage_->getDiskAdaptor()->fileExists()) {
    if (!isCheckIntegrityReady() && !option_->getAsBool(PREF_ALLOW_OVERWRITE)) {
      // TODO we need this->haltRequested = true?
      throw DOWNLOAD_FAILURE_EXCEPTION2(
          fmt(MSG_FILE_ALREADY_EXISTS, downloadContext_->getBasePath().c_str()),
          error_code::FILE_ALREADY_EXISTS);
    }
    pieceStorage_->getDiskAdaptor()->openFile();
  }
  else {
    pieceStorage_->getDiskAdaptor()->openFile();
  }
  progressInfoFile_ = progressInfoFile;
  processCheckIntegrityEntry(commands,
                             make_unique<StreamCheckIntegrityEntry>(this), e);
}

void RequestGroup::processCheckIntegrityEntry(
    std::vector<std::unique_ptr<Command>>& commands,
    std::unique_ptr<CheckIntegrityEntry> entry, DownloadEngine* e)
{
  int64_t actualFileSize = pieceStorage_->getDiskAdaptor()->size();
  if (actualFileSize > downloadContext_->getTotalLength()) {
    entry->cutTrailingGarbage();
  }
  if ((option_->getAsBool(PREF_CHECK_INTEGRITY) ||
       downloadContext_->isChecksumVerificationNeeded()) &&
      entry->isValidationReady()) {
    entry->initValidator();
    // Don't save control file(.aria2 file) when user presses
    // control-c key while aria2 is checking hashes. If control file
    // doesn't exist when aria2 launched, the completed length in
    // saved control file will be 0 byte and this confuses user.
    // enableSaveControlFile() will be called after hash checking is
    // done. See CheckIntegrityCommand.
    disableSaveControlFile();
    e->getCheckIntegrityMan()->pushEntry(std::move(entry));
    return;
  }

  entry->onDownloadIncomplete(commands, e);
}

void RequestGroup::initPieceStorage()
{
  std::shared_ptr<PieceStorage> tempPieceStorage;
  if (downloadContext_->knowsTotalLength() &&
      // Following conditions are needed for chunked encoding with
      // content-length = 0. Google's dl server used this before.
      (downloadContext_->getTotalLength() > 0
#ifdef ENABLE_BITTORRENT
       || downloadContext_->hasAttribute(CTX_ATTR_BT)
#endif // ENABLE_BITTORRENT
           )) {
#ifdef ENABLE_BITTORRENT
    auto ps =
        std::make_shared<DefaultPieceStorage>(downloadContext_, option_.get());
    if (downloadContext_->hasAttribute(CTX_ATTR_BT)) {
      if (isUriSuppliedForRequsetFileEntry(
              downloadContext_->getFileEntries().begin(),
              downloadContext_->getFileEntries().end())) {
        // Use LongestSequencePieceSelector when HTTP/FTP/BitTorrent
        // integrated downloads.
        A2_LOG_DEBUG("Using LongestSequencePieceSelector");
        ps->setPieceSelector(make_unique<LongestSequencePieceSelector>());
      }
      if (option_->defined(PREF_BT_PRIORITIZE_PIECE)) {
        std::vector<size_t> result;
        util::parsePrioritizePieceRange(result,
                                        option_->get(PREF_BT_PRIORITIZE_PIECE),
                                        downloadContext_->getFileEntries(),
                                        downloadContext_->getPieceLength());
        if (!result.empty()) {
          std::shuffle(std::begin(result), std::end(result),
                       *SimpleRandomizer::getInstance());
          auto priSelector =
              make_unique<PriorityPieceSelector>(ps->popPieceSelector());
          priSelector->setPriorityPiece(std::begin(result), std::end(result));
          ps->setPieceSelector(std::move(priSelector));
        }
      }
    }
#else  // !ENABLE_BITTORRENT
    auto ps =
        std::make_shared<DefaultPieceStorage>(downloadContext_, option_.get());
#endif // !ENABLE_BITTORRENT
    if (requestGroupMan_) {
      ps->setWrDiskCache(requestGroupMan_->getWrDiskCache());
    }
    if (diskWriterFactory_) {
      ps->setDiskWriterFactory(diskWriterFactory_);
    }
    tempPieceStorage = ps;
  }
  else {
    auto ps = std::make_shared<UnknownLengthPieceStorage>(downloadContext_);
    if (diskWriterFactory_) {
      ps->setDiskWriterFactory(diskWriterFactory_);
    }
    tempPieceStorage = ps;
  }
  tempPieceStorage->initStorage();
  if (requestGroupMan_) {
    tempPieceStorage->getDiskAdaptor()->setOpenedFileCounter(
        requestGroupMan_->getOpenedFileCounter());
  }
  segmentMan_ =
      std::make_shared<SegmentMan>(downloadContext_, tempPieceStorage);
  pieceStorage_ = tempPieceStorage;
}

void RequestGroup::dropPieceStorage()
{
  segmentMan_.reset();
  pieceStorage_.reset();
}

bool RequestGroup::downloadFinishedByFileLength()
{
  // assuming that a control file doesn't exist.
  if (!isPreLocalFileCheckEnabled() ||
      option_->getAsBool(PREF_ALLOW_OVERWRITE)) {
    return false;
  }
  if (!downloadContext_->knowsTotalLength()) {
    return false;
  }
  File outfile(getFirstFilePath());
  if (outfile.exists() &&
      downloadContext_->getTotalLength() == outfile.size()) {
    return true;
  }
  return false;
}

void RequestGroup::adjustFilename(
    const std::shared_ptr<BtProgressInfoFile>& infoFile)
{
  if (!isPreLocalFileCheckEnabled()) {
    // OK, no need to care about filename.
    return;
  }
  // TODO need this?
  if (requestGroupMan_) {
    if (requestGroupMan_->isSameFileBeingDownloaded(this)) {
      // The file name must be renamed
      tryAutoFileRenaming();
      A2_LOG_NOTICE(fmt(MSG_FILE_RENAMED, getFirstFilePath().c_str()));
      return;
    }
  }
  if (!option_->getAsBool(PREF_DRY_RUN) &&
      option_->getAsBool(PREF_REMOVE_CONTROL_FILE) && infoFile->exists()) {
    infoFile->removeFile();
    A2_LOG_NOTICE(fmt(_("Removed control file for %s because it is requested by"
                        " user."),
                      infoFile->getFilename().c_str()));
  }

  if (infoFile->exists()) {
    // Use current filename
    return;
  }

  File outfile(getFirstFilePath());
  if (outfile.exists() && option_->getAsBool(PREF_CONTINUE) &&
      outfile.size() <= downloadContext_->getTotalLength()) {
    // File exists but user decided to resume it.
  }
  else if (outfile.exists() && isCheckIntegrityReady()) {
    // check-integrity existing file
  }
  else {
    shouldCancelDownloadForSafety();
  }
}

void RequestGroup::removeDefunctControlFile(
    const std::shared_ptr<BtProgressInfoFile>& progressInfoFile)
{
  // Remove the control file if download file doesn't exist
  if (progressInfoFile->exists() &&
      !pieceStorage_->getDiskAdaptor()->fileExists()) {
    progressInfoFile->removeFile();
    A2_LOG_NOTICE(fmt(MSG_REMOVED_DEFUNCT_CONTROL_FILE,
                      progressInfoFile->getFilename().c_str(),
                      downloadContext_->getBasePath().c_str()));
  }
}

void RequestGroup::loadAndOpenFile(
    const std::shared_ptr<BtProgressInfoFile>& progressInfoFile)
{
  try {
    if (!isPreLocalFileCheckEnabled()) {
      pieceStorage_->getDiskAdaptor()->initAndOpenFile();
      return;
    }
    removeDefunctControlFile(progressInfoFile);
    if (progressInfoFile->exists()) {
      progressInfoFile->load();
      pieceStorage_->getDiskAdaptor()->openExistingFile();
    }
    else {
      File outfile(getFirstFilePath());
      if (outfile.exists() && option_->getAsBool(PREF_CONTINUE) &&
          outfile.size() <= getTotalLength()) {
        pieceStorage_->getDiskAdaptor()->openExistingFile();
        pieceStorage_->markPiecesDone(outfile.size());
      }
      else if (outfile.exists() && isCheckIntegrityReady()) {
        pieceStorage_->getDiskAdaptor()->openExistingFile();
      }
      else {
        pieceStorage_->getDiskAdaptor()->initAndOpenFile();
      }
    }
    setProgressInfoFile(progressInfoFile);
  }
  catch (RecoverableException& e) {
    throw DOWNLOAD_FAILURE_EXCEPTION2(EX_DOWNLOAD_ABORTED, e);
  }
}

// assuming that a control file does not exist
void RequestGroup::shouldCancelDownloadForSafety()
{
  if (option_->getAsBool(PREF_ALLOW_OVERWRITE)) {
    return;
  }
  File outfile(getFirstFilePath());
  if (!outfile.exists()) {
    return;
  }

  tryAutoFileRenaming();
  A2_LOG_NOTICE(fmt(MSG_FILE_RENAMED, getFirstFilePath().c_str()));
}

void RequestGroup::tryAutoFileRenaming()
{
  if (!option_->getAsBool(PREF_AUTO_FILE_RENAMING)) {
    throw DOWNLOAD_FAILURE_EXCEPTION2(
        fmt(MSG_FILE_ALREADY_EXISTS, getFirstFilePath().c_str()),
        error_code::FILE_ALREADY_EXISTS);
  }

  std::string filepath = getFirstFilePath();
  if (filepath.empty()) {
    throw DOWNLOAD_FAILURE_EXCEPTION2(
        fmt("File renaming failed: %s", getFirstFilePath().c_str()),
        error_code::FILE_RENAMING_FAILED);
  }
  for (int i = 1; i < 10000; ++i) {
    auto newfilename = fmt("%s.%d", filepath.c_str(), i);
    File newfile(newfilename);
    File ctrlfile(newfile.getPath() + DefaultBtProgressInfoFile::getSuffix());
    if (!newfile.exists() || (newfile.exists() && ctrlfile.exists())) {
      downloadContext_->getFirstFileEntry()->setPath(newfile.getPath());
      return;
    }
  }
  throw DOWNLOAD_FAILURE_EXCEPTION2(
      fmt("File renaming failed: %s", getFirstFilePath().c_str()),
      error_code::FILE_RENAMING_FAILED);
}

void RequestGroup::createNextCommandWithAdj(
    std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e,
    int numAdj)
{
  int numCommand;
  if (getTotalLength() == 0) {
    numCommand = 1 + numAdj;
  }
  else {
    numCommand = std::min(downloadContext_->getNumPieces(),
                          static_cast<size_t>(numConcurrentCommand_));
    numCommand += numAdj;
  }

  if (numCommand > 0) {
    createNextCommand(commands, e, numCommand);
  }
}

void RequestGroup::createNextCommand(
    std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e)
{
  int numCommand;
  if (getTotalLength() == 0) {
    if (numStreamCommand_ > 0) {
      numCommand = 0;
    }
    else {
      numCommand = 1;
    }
  }
  else if (numStreamCommand_ >= numConcurrentCommand_) {
    numCommand = 0;
  }
  else {
    numCommand = std::min(
        downloadContext_->getNumPieces(),
        static_cast<size_t>(numConcurrentCommand_ - numStreamCommand_));
  }

  if (numCommand > 0) {
    createNextCommand(commands, e, numCommand);
  }
}

void RequestGroup::createNextCommand(
    std::vector<std::unique_ptr<Command>>& commands, DownloadEngine* e,
    int numCommand)
{
  for (; numCommand > 0; --numCommand) {
    commands.push_back(
        make_unique<CreateRequestCommand>(e->newCUID(), this, e));
  }
  if (!commands.empty()) {
    e->setNoWait(true);
  }
}

std::string RequestGroup::getFirstFilePath() const
{
  assert(downloadContext_);
  if (inMemoryDownload()) {
    return "[MEMORY]" +
           File(downloadContext_->getFirstFileEntry()->getPath()).getBasename();
  }
  return downloadContext_->getFirstFileEntry()->getPath();
}

int64_t RequestGroup::getTotalLength() const
{
  if (!pieceStorage_) {
    return 0;
  }

  if (pieceStorage_->isSelectiveDownloadingMode()) {
    return pieceStorage_->getFilteredTotalLength();
  }

  return pieceStorage_->getTotalLength();
}

int64_t RequestGroup::getCompletedLength() const
{
  if (!pieceStorage_) {
    return 0;
  }

  if (pieceStorage_->isSelectiveDownloadingMode()) {
    return pieceStorage_->getFilteredCompletedLength();
  }

  return pieceStorage_->getCompletedLength();
}

void RequestGroup::validateFilename(const std::string& expectedFilename,
                                    const std::string& actualFilename) const
{
  if (expectedFilename.empty()) {
    return;
  }

  if (expectedFilename != actualFilename) {
    throw DL_ABORT_EX(fmt(EX_FILENAME_MISMATCH, expectedFilename.c_str(),
                          actualFilename.c_str()));
  }
}

void RequestGroup::validateTotalLength(int64_t expectedTotalLength,
                                       int64_t actualTotalLength) const
{
  if (expectedTotalLength <= 0) {
    return;
  }

  if (expectedTotalLength != actualTotalLength) {
    throw DL_ABORT_EX(
        fmt(EX_SIZE_MISMATCH, expectedTotalLength, actualTotalLength));
  }
}

void RequestGroup::validateFilename(const std::string& actualFilename) const
{
  validateFilename(downloadContext_->getFileEntries().front()->getBasename(),
                   actualFilename);
}

void RequestGroup::validateTotalLength(int64_t actualTotalLength) const
{
  validateTotalLength(getTotalLength(), actualTotalLength);
}

void RequestGroup::increaseStreamCommand() { ++numStreamCommand_; }

void RequestGroup::decreaseStreamCommand() { --numStreamCommand_; }

void RequestGroup::increaseStreamConnection() { ++numStreamConnection_; }

void RequestGroup::decreaseStreamConnection() { --numStreamConnection_; }

int RequestGroup::getNumConnection() const
{
  int numConnection = numStreamConnection_;
#ifdef ENABLE_BITTORRENT
  if (btRuntime_) {
    numConnection += btRuntime_->getConnections();
  }
#endif // ENABLE_BITTORRENT
  return numConnection;
}

void RequestGroup::increaseNumCommand() { ++numCommand_; }

void RequestGroup::decreaseNumCommand()
{
  --numCommand_;
  if (!numCommand_ && requestGroupMan_) {
    A2_LOG_DEBUG(fmt("GID#%s - Request queue check", gid_->toHex().c_str()));
    requestGroupMan_->requestQueueCheck();
  }
}

TransferStat RequestGroup::calculateStat() const
{
  TransferStat stat = downloadContext_->getNetStat().toTransferStat();
#ifdef ENABLE_BITTORRENT
  if (btRuntime_) {
    stat.allTimeUploadLength =
        btRuntime_->getUploadLengthAtStartup() + stat.sessionUploadLength;
  }
#endif // ENABLE_BITTORRENT
  return stat;
}

void RequestGroup::setHaltRequested(bool f, HaltReason haltReason)
{
  haltRequested_ = f;
  if (haltRequested_) {
    pauseRequested_ = false;
    haltReason_ = haltReason;
  }
#ifdef ENABLE_BITTORRENT
  if (btRuntime_) {
    btRuntime_->setHalt(f);
  }
#endif // ENABLE_BITTORRENT
}

void RequestGroup::setForceHaltRequested(bool f, HaltReason haltReason)
{
  setHaltRequested(f, haltReason);
  forceHaltRequested_ = f;
}

void RequestGroup::setPauseRequested(bool f) { pauseRequested_ = f; }

void RequestGroup::releaseRuntimeResource(DownloadEngine* e)
{
#ifdef ENABLE_BITTORRENT
  e->getBtRegistry()->remove(gid_->getNumericId());
  btRuntime_ = nullptr;
  peerStorage_ = nullptr;
#endif // ENABLE_BITTORRENT
  if (pieceStorage_) {
    pieceStorage_->removeAdvertisedPiece(0_s);
  }
  // Don't reset segmentMan_ and pieceStorage_ here to provide
  // progress information via RPC
  progressInfoFile_ = std::make_shared<NullProgressInfoFile>();
  downloadContext_->releaseRuntimeResource();
  // Reset seedOnly_, so that we can handle pause/unpause-ing seeding
  // torrent with --bt-detach-seed-only.
  seedOnly_ = false;
}

void RequestGroup::preDownloadProcessing()
{
  A2_LOG_DEBUG(fmt("Finding PreDownloadHandler for path %s.",
                   getFirstFilePath().c_str()));
  try {
    for (const auto& pdh : preDownloadHandlers_) {
      if (pdh->canHandle(this)) {
        pdh->execute(this);
        return;
      }
    }
  }
  catch (RecoverableException& ex) {
    A2_LOG_ERROR_EX(EX_EXCEPTION_CAUGHT, ex);
    return;
  }

  A2_LOG_DEBUG("No PreDownloadHandler found.");
  return;
}

void RequestGroup::postDownloadProcessing(
    std::vector<std::shared_ptr<RequestGroup>>& groups)
{
  A2_LOG_DEBUG(fmt("Finding PostDownloadHandler for path %s.",
                   getFirstFilePath().c_str()));
  try {
    for (const auto& pdh : postDownloadHandlers_) {
      if (pdh->canHandle(this)) {
        pdh->getNextRequestGroups(groups, this);
        return;
      }
    }
  }
  catch (RecoverableException& ex) {
    A2_LOG_ERROR_EX(EX_EXCEPTION_CAUGHT, ex);
  }

  A2_LOG_DEBUG("No PostDownloadHandler found.");
}

void RequestGroup::initializePreDownloadHandler()
{
#ifdef ENABLE_BITTORRENT
  if (option_->get(PREF_FOLLOW_TORRENT) == V_MEM) {
    preDownloadHandlers_.push_back(
        download_handlers::getBtPreDownloadHandler());
  }
#endif // ENABLE_BITTORRENT
#ifdef ENABLE_METALINK
  if (option_->get(PREF_FOLLOW_METALINK) == V_MEM) {
    preDownloadHandlers_.push_back(
        download_handlers::getMetalinkPreDownloadHandler());
  }
#endif // ENABLE_METALINK
}

void RequestGroup::initializePostDownloadHandler()
{
#ifdef ENABLE_BITTORRENT
  if (option_->getAsBool(PREF_FOLLOW_TORRENT) ||
      option_->get(PREF_FOLLOW_TORRENT) == V_MEM) {
    postDownloadHandlers_.push_back(
        download_handlers::getBtPostDownloadHandler());
  }
#endif // ENABLE_BITTORRENT
#ifdef ENABLE_METALINK
  if (option_->getAsBool(PREF_FOLLOW_METALINK) ||
      option_->get(PREF_FOLLOW_METALINK) == V_MEM) {
    postDownloadHandlers_.push_back(
        download_handlers::getMetalinkPostDownloadHandler());
  }
#endif // ENABLE_METALINK
}

bool RequestGroup::isDependencyResolved()
{
  if (!dependency_) {
    return true;
  }
  return dependency_->resolve();
}

void RequestGroup::dependsOn(const std::shared_ptr<Dependency>& dep)
{
  dependency_ = dep;
}

void RequestGroup::setDiskWriterFactory(
    const std::shared_ptr<DiskWriterFactory>& diskWriterFactory)
{
  diskWriterFactory_ = diskWriterFactory;
}

void RequestGroup::addPostDownloadHandler(const PostDownloadHandler* handler)
{
  postDownloadHandlers_.push_back(handler);
}

void RequestGroup::addPreDownloadHandler(const PreDownloadHandler* handler)
{
  preDownloadHandlers_.push_back(handler);
}

void RequestGroup::clearPostDownloadHandler() { postDownloadHandlers_.clear(); }

void RequestGroup::clearPreDownloadHandler() { preDownloadHandlers_.clear(); }

void RequestGroup::setPieceStorage(
    const std::shared_ptr<PieceStorage>& pieceStorage)
{
  pieceStorage_ = pieceStorage;
}

void RequestGroup::setProgressInfoFile(
    const std::shared_ptr<BtProgressInfoFile>& progressInfoFile)
{
  progressInfoFile_ = progressInfoFile;
}

bool RequestGroup::needsFileAllocation() const
{
  return isFileAllocationEnabled() &&
         option_->getAsLLInt(PREF_NO_FILE_ALLOCATION_LIMIT) <=
             getTotalLength() &&
         !pieceStorage_->getDiskAdaptor()->fileAllocationIterator()->finished();
}

std::shared_ptr<DownloadResult> RequestGroup::createDownloadResult() const
{
  A2_LOG_DEBUG(fmt("GID#%s - Creating DownloadResult.", gid_->toHex().c_str()));
  TransferStat st = calculateStat();
  auto res = std::make_shared<DownloadResult>();
  res->gid = gid_;
  res->fileEntries = downloadContext_->getFileEntries();
  res->inMemoryDownload = inMemoryDownload_;
  res->sessionDownloadLength = st.sessionDownloadLength;
  res->sessionTime = std::chrono::duration_cast<std::chrono::milliseconds>(
      downloadContext_->calculateSessionTime());

  auto result = downloadResult();
  res->result = result.first;
  res->resultMessage = result.second;
  res->followedBy = followedByGIDs_;
  res->following = followingGID_;
  res->belongsTo = belongsToGID_;
  res->option = option_;
  res->metadataInfo = metadataInfo_;
  res->totalLength = getTotalLength();
  res->completedLength = getCompletedLength();
  res->uploadLength = st.allTimeUploadLength;
  if (pieceStorage_ && pieceStorage_->getBitfieldLength() > 0) {
    res->bitfield.assign(pieceStorage_->getBitfield(),
                         pieceStorage_->getBitfield() +
                             pieceStorage_->getBitfieldLength());
  }
#ifdef ENABLE_BITTORRENT
  if (downloadContext_->hasAttribute(CTX_ATTR_BT)) {
    const unsigned char* p = bittorrent::getInfoHash(downloadContext_);
    res->infoHash.assign(p, p + INFO_HASH_LENGTH);
  }
#endif // ENABLE_BITTORRENT
  res->pieceLength = downloadContext_->getPieceLength();
  res->numPieces = downloadContext_->getNumPieces();
  res->dir = option_->get(PREF_DIR);
  return res;
}

void RequestGroup::reportDownloadFinished()
{
  A2_LOG_NOTICE(fmt(MSG_FILE_DOWNLOAD_COMPLETED,
                    downloadContext_->getBasePath().c_str()));
  uriSelector_->resetCounters();
#ifdef ENABLE_BITTORRENT
  if (downloadContext_->hasAttribute(CTX_ATTR_BT)) {
    TransferStat stat = calculateStat();
    int64_t completedLength = getCompletedLength();
    double shareRatio = completedLength == 0
                            ? 0.0
                            : 1.0 * stat.allTimeUploadLength / completedLength;
    auto attrs = bittorrent::getTorrentAttrs(downloadContext_);
    if (!attrs->metadata.empty()) {
      A2_LOG_NOTICE(fmt(MSG_SHARE_RATIO_REPORT, shareRatio,
                        util::abbrevSize(stat.allTimeUploadLength).c_str(),
                        util::abbrevSize(completedLength).c_str()));
    }
  }
#endif // ENABLE_BITTORRENT
}

void RequestGroup::setURISelector(std::unique_ptr<URISelector> uriSelector)
{
  uriSelector_ = std::move(uriSelector);
}

void RequestGroup::applyLastModifiedTimeToLocalFiles()
{
  if (!pieceStorage_ || !lastModifiedTime_.good()) {
    return;
  }
  A2_LOG_INFO(fmt("Applying Last-Modified time: %s",
                  lastModifiedTime_.toHTTPDate().c_str()));
  size_t n = pieceStorage_->getDiskAdaptor()->utime(Time(), lastModifiedTime_);
  A2_LOG_INFO(fmt("Last-Modified attrs of %lu files were updated.",
                  static_cast<unsigned long>(n)));
}

void RequestGroup::updateLastModifiedTime(const Time& time)
{
  if (time.good() && lastModifiedTime_ < time) {
    lastModifiedTime_ = time;
  }
}

void RequestGroup::increaseAndValidateFileNotFoundCount()
{
  ++fileNotFoundCount_;
  const int maxCount = option_->getAsInt(PREF_MAX_FILE_NOT_FOUND);
  if (maxCount > 0 && fileNotFoundCount_ >= maxCount &&
      downloadContext_->getNetStat().getSessionDownloadLength() == 0) {
    throw DOWNLOAD_FAILURE_EXCEPTION2(
        fmt("Reached max-file-not-found count=%d", maxCount),
        error_code::MAX_FILE_NOT_FOUND);
  }
}

void RequestGroup::markInMemoryDownload() { inMemoryDownload_ = true; }

void RequestGroup::setTimeout(std::chrono::seconds timeout)
{
  timeout_ = std::move(timeout);
}

bool RequestGroup::doesDownloadSpeedExceed()
{
  int spd = downloadContext_->getNetStat().calculateDownloadSpeed();
  return maxDownloadSpeedLimit_ > 0 && maxDownloadSpeedLimit_ < spd;
}

bool RequestGroup::doesUploadSpeedExceed()
{
  int spd = downloadContext_->getNetStat().calculateUploadSpeed();
  return maxUploadSpeedLimit_ > 0 && maxUploadSpeedLimit_ < spd;
}

void RequestGroup::saveControlFile() const
{
  if (saveControlFile_) {
    progressInfoFile_->save();
  }
}

void RequestGroup::removeControlFile() const
{
  progressInfoFile_->removeFile();
}

void RequestGroup::setDownloadContext(
    const std::shared_ptr<DownloadContext>& downloadContext)
{
  downloadContext_ = downloadContext;
  if (downloadContext_) {
    downloadContext_->setOwnerRequestGroup(this);
  }
}

bool RequestGroup::p2pInvolved() const
{
#ifdef ENABLE_BITTORRENT
  return downloadContext_->hasAttribute(CTX_ATTR_BT);
#else  // !ENABLE_BITTORRENT
  return false;
#endif // !ENABLE_BITTORRENT
}

void RequestGroup::enableSeedOnly()
{
  if (seedOnly_ || !option_->getAsBool(PREF_BT_DETACH_SEED_ONLY)) {
    return;
  }

  if (requestGroupMan_) {
    seedOnly_ = true;

    requestGroupMan_->decreaseNumActive();
    requestGroupMan_->requestQueueCheck();
  }
}

bool RequestGroup::isSeeder() const
{
#ifdef ENABLE_BITTORRENT
  return downloadContext_->hasAttribute(CTX_ATTR_BT) &&
         !bittorrent::getTorrentAttrs(downloadContext_)->metadata.empty() &&
         downloadFinished();
#else  // !ENABLE_BITTORRENT
  return false;
#endif // !ENABLE_BITTORRENT
}

} // namespace aria2