/
envoy.istio
/
oceanbase
GigaCode
beta
ОбзорЦентр заботыВойти

oceanbase

Форк
0
Ветки: 12Коммиты: 12104Теги: 40
oceanbase
/src
/observer
/mysql
/
ob_eliminate_task.cpp 
224 строки · 9.4 Кб
1
/**
2
 * Copyright (c) 2021 OceanBase
3
 * OceanBase CE is licensed under Mulan PubL v2.
4
 * You can use this software according to the terms and conditions of the Mulan PubL v2.
5
 * You may obtain a copy of Mulan PubL v2 at:
6
 *          http://license.coscl.org.cn/MulanPubL-2.0
7
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10
 * See the Mulan PubL v2 for more details.
11
 */
12


13
#define USING_LOG_PREFIX SERVER
14


15
#include "ob_eliminate_task.h"
16
#include "ob_mysql_request_manager.h"
17


18
using namespace oceanbase::obmysql;
19


20
ObEliminateTask::ObEliminateTask()
21
    :request_manager_(NULL),
22
     config_mem_limit_(0)
23
{
24


25
}
26


27
ObEliminateTask::~ObEliminateTask()
28
{
29


30
}
31


32
int ObEliminateTask::init(const ObMySQLRequestManager *request_manager)
33
{
34
  int ret = OB_SUCCESS;
35
  if (OB_ISNULL(request_manager)) {
36
    ret = OB_INVALID_ARGUMENT;
37
    LOG_WARN("invalid argument", K(request_manager_), K(ret));
38
  } else {
39
    request_manager_ = const_cast<ObMySQLRequestManager*>(request_manager);
40
    // can't call ObMySQLRequestManager::get_mem_limit for now, tenant not inited
41
    // set config_mem_limit_ to 16M
42
    config_mem_limit_ = 16 * 1024 * 1024; // 16M
43
    common::ObConcurrentFIFOAllocator  *allocator = request_manager_->get_allocator();
44
    if (OB_ISNULL(allocator)) {
45
      ret = OB_NOT_INIT;
46
      LOG_WARN("request manager allocator not init", K(ret));
47
    } else {
48
      allocator->set_total_limit(config_mem_limit_);
49
    }
50
    disable_timeout_check();
51
  }
52
  return ret;
53
}
54


55
// 检查配置内存限时是否更改：mem_limit = tenant_mem_limit * ob_sql_audit_percentage
56
int ObEliminateTask::check_config_mem_limit(bool &is_change)
57
{
58
  const int64_t MINIMUM_LIMIT = 64 * 1024 * 1024;   // at lease 64M
59
  const int64_t MAXIMUM_LIMIT = 1024 * 1024 * 1024; // 1G maximum
60
  int ret = OB_SUCCESS;
61
  is_change = false;
62
  int64_t mem_limit = config_mem_limit_;
63
  int64_t tenant_id = OB_INVALID_TENANT_ID;
64
  if (OB_ISNULL(request_manager_)) {
65
    ret = OB_INVALID_ARGUMENT;
66
    LOG_WARN("invalid argument", K(request_manager_), K(ret));
67
  } else if (FALSE_IT(tenant_id = request_manager_->get_tenant_id())) {
68
  } else if (tenant_id > OB_SYS_TENANT_ID && tenant_id <= OB_MAX_RESERVED_TENANT_ID) {
69
    // 50x租户在没有对应的tenant schema，查询配置一定失败
70
    // do nothing
71
  } else if (OB_FAIL(ObMySQLRequestManager::get_mem_limit(tenant_id, mem_limit))) {
72
    // if memory limit is not retrivable
73
    // overwrite error code, not change the size of config_mem_limit_
74
    LOG_WARN("failed to get mem limit", K(ret), K(tenant_id), K(mem_limit), K(config_mem_limit_));
75
    ret = OB_SUCCESS;
76
  } else {
77
    if (config_mem_limit_ != mem_limit) {
78
      LOG_INFO("change config mem limit", K(config_mem_limit_), K(mem_limit), K(tenant_id));
79
      bool use_mini_mem = lib::is_mini_mode() || MTL_IS_MINI_MODE() || is_meta_tenant(tenant_id);
80
      config_mem_limit_ = mem_limit;
81
      if (mem_limit < MINIMUM_LIMIT && !use_mini_mem) {
82
        config_mem_limit_ = MINIMUM_LIMIT;
83
      }
84
      is_change = true;
85
    }
86
  }
87
  return ret;
88
}
89


90
//剩余内存淘汰曲线图,当mem_limit在[64M, 100M]时, 内存剩余20M时淘汰;
91
//               当mem_limit在[100M, 5G]时, 内存甚于mem_limit*0.2时淘汰;
92
//               当mem_limit在[5G, +∞]时, 内存剩余1G时淘汰;
93
//高低水位线内存差曲线图，当mem_limit在[64M, 100M]时, 内存差为:20M;
94
//                        当mem_limit在[100M, 5G]时，内存差：mem_limit*0.2;
95
//                        当mem_limit在[5G, +∞]时, 内存差是：1G,
96
//        ______
97
//       /
98
// _____/
99
//   100M 5G
100
int ObEliminateTask::calc_evict_mem_level(int64_t &low, int64_t &high)
101
{
102
  int ret = OB_SUCCESS;
103
  const double HIGH_LEVEL_PRECENT = 0.80;
104
  const double LOW_LEVEL_PRECENT = 0.60;
105
  const double HALF_PRECENT = 0.50;
106
  const int64_t BIG_MEMORY_LIMIT = 5368709120; //5G
107
  const int64_t SMALL_MEMORY_LIMIT = 100*1024*1024; //100M
108
  const int64_t LOW_CONFIG = 64*1024*1024; //64M
109
  if (OB_ISNULL(request_manager_) || config_mem_limit_ < 0) {
110
    ret = OB_INVALID_ARGUMENT;
111
    LOG_WARN("invalid argument", K(request_manager_), K(config_mem_limit_), K(ret));
112
  } else {
113
    if (config_mem_limit_ > BIG_MEMORY_LIMIT) { // mem_limit > 5G
114
      high = config_mem_limit_ - static_cast<int64_t>(BIG_MEMORY_LIMIT * (1.0 - HIGH_LEVEL_PRECENT));
115
      low = config_mem_limit_ - static_cast<int64_t>(BIG_MEMORY_LIMIT * (1.0 - LOW_LEVEL_PRECENT)) ;
116
    } else if (config_mem_limit_ >= LOW_CONFIG &&  config_mem_limit_  < SMALL_MEMORY_LIMIT) { // 64M =< mem_limit < 100M
117
      high = config_mem_limit_ - static_cast<int64_t>(SMALL_MEMORY_LIMIT * (1.0 - HIGH_LEVEL_PRECENT));
118
      low = config_mem_limit_ - static_cast<int64_t>(SMALL_MEMORY_LIMIT * (1.0 - LOW_LEVEL_PRECENT));
119
    } else if (config_mem_limit_ < LOW_CONFIG) { //mem_limit < 64M
120
      high = static_cast<int64_t>(static_cast<double>(config_mem_limit_) * HALF_PRECENT);
121
      low = 0;
122
    } else {
123
      high = static_cast<int64_t>(static_cast<double>(config_mem_limit_) * HIGH_LEVEL_PRECENT);
124
      low = static_cast<int64_t>(static_cast<double>(config_mem_limit_) * LOW_LEVEL_PRECENT);
125
    }
126
  }
127
  return ret;
128
}
129


130
void ObEliminateTask::runTimerTask()
131
{
132
  int ret = OB_SUCCESS;
133
  common::ObConcurrentFIFOAllocator *allocator = NULL;
134
  bool is_change = false;
135
  int64_t release_cnt = 0;
136
  int64_t evict_high_mem_level = 0;
137
  int64_t evict_low_mem_level = 0;
138
  int64_t evict_high_size_level = 0;
139
  int64_t evict_low_size_level = 0;
140
  flt_mgr_ = MTL(ObFLTSpanMgr*);
141
  if (flt_mgr_->get_size() > (ObFLTSpanMgr::MAX_QUEUE_SIZE-ObFLTSpanMgr::RELEASE_QUEUE_SIZE)) {
142
    for (int i = 0; i < ObFLTSpanMgr::RELEASE_QUEUE_SIZE/ObFLTSpanMgr::BATCH_RELEASE_COUNT; i++) {
143
      flt_mgr_->release_old(ObFLTSpanMgr::BATCH_RELEASE_COUNT);
144
    }
145
  }
146
  if (OB_ISNULL(request_manager_)) {
147
    ret = OB_INVALID_ARGUMENT;
148
    LOG_WARN("invalid argument", K(request_manager_), K(ret));
149
  } else if (OB_FAIL(check_config_mem_limit(is_change))) {
150
    LOG_WARN("fail to check mem limit stat", K(ret));
151
  } else if (OB_FAIL(calc_evict_mem_level(evict_low_mem_level, evict_high_mem_level))) {
152
    LOG_WARN("fail to get sql audit evict memory level", K(ret));
153
  } else {
154
    int64_t queue_size = request_manager_->get_capacity();
155
    bool use_mini_queue = lib::is_mini_mode() || MTL_IS_MINI_MODE()
156
                          || is_meta_tenant(request_manager_->get_tenant_id());
157
    release_cnt = use_mini_queue
158
                  ? ObMySQLRequestManager::MINI_MODE_BATCH_RELEASE_SIZE
159
                  : ObMySQLRequestManager::BATCH_RELEASE_SIZE;
160
    evict_high_size_level = queue_size * ObMySQLRequestManager::HIGH_LEVEL_EVICT_PERCENTAGE;
161
    evict_low_size_level = queue_size * ObMySQLRequestManager::LOW_LEVEL_EVICT_PERCENTAGE;
162
    allocator = request_manager_->get_allocator();
163
    if (OB_ISNULL(allocator)) {
164
      ret = OB_NOT_INIT;
165
      LOG_WARN("fail to get sql audit evict memory level", K(ret));
166
    }
167
    if (OB_SUCC(ret) && REACH_TIME_INTERVAL(30 * 1000 * 1000)) { // 30s delay
168
      LOG_INFO("Eliminate task evict sql audit",
169
          K(request_manager_->get_tenant_id()), K(queue_size), K(config_mem_limit_),
170
          K(request_manager_->get_size_used()), K(evict_high_size_level), K(evict_low_size_level),
171
          K(allocator->allocated()), K(evict_high_mem_level), K(evict_low_mem_level));
172
    }
173
  }
174


175
  if (OB_SUCC(ret)) {
176
    int64_t start_time = ObTimeUtility::current_time();
177
    int64_t evict_batch_count = 0;
178
    //按内存淘汰
179
    if (evict_high_mem_level < allocator->allocated()) {
180
      LOG_INFO("sql audit evict mem start",
181
               K(request_manager_->get_tenant_id()),
182
               K(evict_low_mem_level),
183
               K(evict_high_mem_level),
184
               "size_used",request_manager_->get_size_used(),
185
               "mem_used", allocator->allocated());
186
      int64_t last_time_allocated = allocator->allocated();
187
      while (evict_low_mem_level < allocator->allocated()) {
188
        request_manager_->release_old(release_cnt);
189
        evict_batch_count++;
190
        if ((evict_low_mem_level < allocator->allocated()) && (last_time_allocated == allocator->allocated())) {
191
          LOG_INFO("release old cannot free more memory");
192
          break;
193
        }
194
        last_time_allocated = allocator->allocated();
195
      }
196
    }
197
    //按记录数淘汰
198
    if (request_manager_->get_size_used() > evict_high_size_level) {
199
      evict_batch_count = (request_manager_->get_size_used() - evict_low_size_level) / release_cnt;
200
      LOG_INFO("sql audit evict record start",
201
               K(request_manager_->get_tenant_id()),
202
               K(evict_high_size_level),
203
               K(evict_low_size_level),
204
               "size_used",request_manager_->get_size_used(),
205
               "mem_used", allocator->allocated());
206
      for (int i = 0; i < evict_batch_count; i++) {
207
        request_manager_->release_old(release_cnt);
208
      }
209
    }
210
    //如果sql_audit_memory_limit改变, 则需要将ObConcurrentFIFOAllocator中total_limit_更新;
211
    if (true == is_change) {
212
      allocator->set_total_limit(config_mem_limit_);
213
    }
214
    int64_t end_time = ObTimeUtility::current_time();
215
    LOG_INFO("sql audit evict task end",
216
             K(request_manager_->get_tenant_id()),
217
             K(evict_high_mem_level),
218
             K(evict_high_size_level),
219
             K(evict_batch_count),
220
             "elapse_time", end_time - start_time,
221
             "size_used",request_manager_->get_size_used(),
222
             "mem_used", allocator->allocated());
223
  }
224
}
225

Использование cookies

Мы используем файлы cookie в соответствии с Политикой конфиденциальности и Политикой использования cookies.

Нажимая кнопку «Принимаю», Вы даете АО «СберТех» согласие на обработку Ваших персональных данных в целях совершенствования нашего веб-сайта и Сервиса GitVerse, а также повышения удобства их использования.

Запретить использование cookies Вы можете самостоятельно в настройках Вашего браузера.