oceanbase

1
/**
2
 * Copyright (c) 2021 OceanBase
3
 * OceanBase CE is licensed under Mulan PubL v2.
4
 * You can use this software according to the terms and conditions of the Mulan PubL v2.
5
 * You may obtain a copy of Mulan PubL v2 at:
6
 *          http://license.coscl.org.cn/MulanPubL-2.0
7
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10
 * See the Mulan PubL v2 for more details.
11
 */
12

13
#define USING_LOG_PREFIX RS_LB
14
#include "ob_server_balancer.h"
15
#include "ob_balance_info.h"
16
#include "lib/container/ob_array_iterator.h"
17
#include "observer/ob_server_struct.h"
18
#include "storage/ob_file_system_router.h"
19
#include "rootserver/ob_zone_manager.h"
20
#include "rootserver/ob_unit_stat_manager.h"
21
#include "rootserver/ob_root_utils.h"
22
#include "rootserver/ob_root_service.h"
23
#include "storage/ob_file_system_router.h"
24
#include "share/ob_all_server_tracer.h"
25
#include "share/ob_server_table_operator.h"
26
#include "rootserver/ob_heartbeat_service.h"
27
#include "share/ob_share_util.h" // ObShareUtil
28
#include "lib/utility/ob_tracepoint.h"
29

30
using namespace oceanbase::common;
31
using namespace oceanbase::common::hash;
32
using namespace oceanbase::rootserver;
33
using namespace oceanbase::share;
34
ObServerBalancer::ObServerBalancer()
35
    :inited_(false),
36
     schema_service_(NULL),
37
     unit_mgr_(NULL),
38
     zone_mgr_(NULL),
39
     server_mgr_(NULL),
40
     unit_stat_mgr_(),
41
     count_balance_strategy_(*this),
42
     inner_ttg_balance_strategy_(count_balance_strategy_),
43
     zone_disk_statistic_()
44
{}
45

46
ObServerBalancer::~ObServerBalancer() {}
47

48
int ObServerBalancer::init(
49
    share::schema::ObMultiVersionSchemaService &schema_service,
50
    ObUnitManager &unit_manager,
51
    ObZoneManager &zone_mgr,
52
    ObServerManager &server_mgr,
53
    common::ObMySQLProxy &sql_proxy)
54
{
55
  int ret = OB_SUCCESS;
56
  if (inited_) {
57
    ret = OB_INIT_TWICE;
58
  } else if (OB_FAIL(unit_stat_mgr_.init(sql_proxy))) {
59
    LOG_WARN("init unit_stat_mgr failed", KR(ret));
60
  } else {
61
    schema_service_ = &schema_service;
62
    unit_mgr_ = &unit_manager;
63
    zone_mgr_ = &zone_mgr;
64
    server_mgr_ = &server_mgr;
65
    inited_ = true;
66
  }
67
  return ret;
68
}
69

70
int ObServerBalancer::get_active_servers_info_and_resource_info_of_zone(
71
      const ObZone &zone,
72
      ObIArray<share::ObServerInfoInTable> &servers_info,
73
      ObIArray<obrpc::ObGetServerResourceInfoResult> &server_resources_info)
74
{
75
  int ret = OB_SUCCESS;
76
  servers_info.reset();
77
  server_resources_info.reset();
78
  ObServerResourceInfo resource_info_in_server_status;
79
  obrpc::ObGetServerResourceInfoResult resource_info_result;
80
  if (OB_FAIL(SVR_TRACER.get_active_servers_info(zone, servers_info))) {
81
    LOG_WARN("fail to execute get_active_servers_info", KR(ret), K(zone));
82
  } else if (OB_ISNULL(server_mgr_)) {
83
    ret = OB_ERR_UNEXPECTED;
84
    LOG_WARN("server_mgr_ is null", KR(ret), KP(server_mgr_));
85
  } else {
86
    for (int64_t i = 0; OB_SUCC(ret) && i < servers_info.count(); i++) {
87
      const ObAddr &server = servers_info.at(i).get_server();
88
      resource_info_result.reset();
89
      resource_info_in_server_status.reset();
90
      if (OB_FAIL(server_mgr_->get_server_resource_info(server, resource_info_in_server_status))) {
91
        LOG_WARN("fail to get resource_info_in_server_status", KR(ret), K(server));
92
      } else if (OB_UNLIKELY(!resource_info_in_server_status.is_valid())) {
93
        ret = OB_INVALID_ARGUMENT;
94
        LOG_WARN("invalid resource_info_in_server_status", KR(ret), K(server), K(resource_info_in_server_status));
95
      } else if (OB_FAIL(resource_info_result.init(server,resource_info_in_server_status))) {
96
        LOG_WARN("fail to init", KR(ret), K(server), K(resource_info_in_server_status));
97
      } else if (OB_FAIL(server_resources_info.push_back(resource_info_result))) {
98
        LOG_WARN("fail to push an element into server_resources_info", KR(ret), K(resource_info_result));
99
      }
100
    }
101
  }
102
  return ret;
103
}
104

105
int ObServerBalancer::tenant_group_balance()
106
{
107
  int ret = OB_SUCCESS;
108
  ObRootBalanceHelp::BalanceController balance_controller;
109
  ObString switch_config_str = GCONF.__balance_controller.str();
110
  {
111
    SpinWLockGuard guard(unit_mgr_->get_lock());  // lock!
112
    ObArray<ObZone> zones;
113
    if (!check_inner_stat()) {
114
      ret = OB_INNER_STAT_ERROR;
115
      LOG_WARN("fail to check inner stat", K(ret));
116
    } else if (OB_FAIL(unit_stat_mgr_.gather_stat())) {
117
      LOG_WARN("gather all tenant unit stat failed, refuse to do server_balance", K(ret));
118
      unit_stat_mgr_.reuse();
119
      ret = OB_SUCCESS;
120
    } else if (OB_FAIL(ObRootBalanceHelp::parse_balance_info(
121
            switch_config_str, balance_controller))) {
122
      LOG_WARN("fail to parse balance switch", K(ret), "balance switch", switch_config_str);
123
    } else if (ObShareUtil::is_tenant_enable_rebalance(OB_SYS_TENANT_ID)
124
            || balance_controller.at(ObRootBalanceHelp::ENABLE_SERVER_BALANCE)) {
125
      if (OB_FAIL(zone_mgr_->get_zone(ObZoneStatus::ACTIVE, zones))) {
126
        LOG_WARN("get_zone failed", "status", ObZoneStatus::ACTIVE, K(ret));
127
      } else {
128
        LOG_INFO("start to do tenant group unit balance");
129
        // The server balance of all zones is completed independently
130
        for (int64_t i = 0; OB_SUCC(ret) && i < zones.count(); ++i) {
131
          bool can_execute_rebalance = false;
132
          if (OB_FAIL(check_can_execute_rebalance(zones.at(i), can_execute_rebalance))) {
133
            LOG_WARN("fail to check can execute rebalance", K(ret));
134
          } else if (!can_execute_rebalance) {
135
            // cannot do rebalance
136
          } else if (OB_FAIL(rebalance_servers_v2(zones.at(i)))) {
137
            LOG_WARN("failed to rebalance servers", "zone", zones.at(i), K(ret));
138
          }
139
        }
140
      }
141
    }
142
  }
143
  return ret;
144
}
145

146
int ObServerBalancer::check_has_unit_in_migration(
147
    const common::ObIArray<ObUnitManager::ObUnitLoad> *unit_load_array,
148
    bool &has_unit_in_migration)
149
{
150
  int ret = OB_SUCCESS;
151
  if (!check_inner_stat()) {
152
    ret = OB_INNER_STAT_ERROR;
153
    LOG_WARN("fail to check inner stat", KR(ret));
154
  } else if (OB_UNLIKELY(nullptr == unit_load_array)) {
155
    ret = OB_INVALID_ARGUMENT;
156
    LOG_WARN("invalid argument", KR(ret), KP(unit_load_array));
157
  } else {
158
    has_unit_in_migration = false;
159
    for (int64_t i = 0;
160
         !has_unit_in_migration && OB_SUCC(ret) && i < unit_load_array->count();
161
         ++i) {
162
      const ObUnitManager::ObUnitLoad &unit_load = unit_load_array->at(i);
163
      if (OB_UNLIKELY(!unit_load.is_valid())) {
164
        ret = OB_INVALID_ARGUMENT;
165
        LOG_WARN("invalid unit load", KR(ret), K(unit_load));
166
      } else if (unit_load.unit_->migrate_from_server_.is_valid()) {
167
        has_unit_in_migration = true;
168
      } else {
169
        // bypass
170
      }
171
    }
172
  }
173
  return ret;
174
}
175

176
int ObServerBalancer::balance_servers()
177
{
178
  int ret = OB_SUCCESS;
179
  LOG_INFO("start do unit balance");
180
  ObRootBalanceHelp::BalanceController balance_controller;
181
  ObString switch_config_str = GCONF.__balance_controller.str();
182
  DEBUG_SYNC(START_UNIT_BALANCE);
183
  {
184
    SpinWLockGuard guard(unit_mgr_->get_lock());  // lock!
185
    const bool enable_sys_unit_standalone = GCONF.enable_sys_unit_standalone;
186
    if (!check_inner_stat()) {
187
      ret = OB_INNER_STAT_ERROR;
188
      LOG_WARN("check_inner_stat failed", K(inited_), K(ret));
189
    } else if (OB_FAIL(ObRootBalanceHelp::parse_balance_info(switch_config_str, balance_controller))) {
190
      LOG_WARN("fail to parse balance switch", K(ret), "balance_swtich", switch_config_str);
191
    } else if (OB_FAIL(unit_stat_mgr_.gather_stat())) {
192
      LOG_WARN("gather all tenant unit stat failed, refuse to do balance for status change", K(ret));
193
      unit_stat_mgr_.reuse();
194
      ret = OB_SUCCESS;
195
    } else {
196
      // When the server status changes,
197
      // try to adjust the unit on the server without configuration item control
198
      if (GCONF.is_rereplication_enabled()
199
          || ObShareUtil::is_tenant_enable_rebalance(OB_SYS_TENANT_ID)
200
          || balance_controller.at(ObRootBalanceHelp::ENABLE_SERVER_BALANCE)) {
201
        if (OB_FAIL(distribute_for_server_status_change())) {
202
          LOG_WARN("fail to distribute for server status change", K(ret));
203
        }
204
      }
205

206
      if (OB_SUCC(ret) && enable_sys_unit_standalone) {
207
        if (OB_FAIL(distribute_for_standalone_sys_unit())) {
208
          LOG_WARN("fail to distribute for standalone sys unit");
209
        }
210
      }
211
    }
212
  }
213
  LOG_INFO("finish do unit balance", K(ret));
214
  return ret;
215
}
216

217
int ObServerBalancer::distribute_pool_for_standalone_sys_unit(
218
    const share::ObResourcePool &pool,
219
    const common::ObIArray<common::ObAddr> &sys_unit_server_array)
220
{
221
  int ret = OB_SUCCESS;
222
  const char *module = "UNIT_BALANCE_FOR_STANDALONE_SYS";
223
  share::ObUnitConfig *unit_config = nullptr;
224
  common::ObArray<share::ObUnit *> *pool_unit_array = nullptr;
225
  if (OB_UNLIKELY(OB_INVALID_ID == pool.resource_pool_id_)) {
226
    ret = OB_INVALID_ARGUMENT;
227
    LOG_WARN("invalid argument", K(ret), "pool_id", pool.resource_pool_id_);
228
  } else if (OB_UNLIKELY(nullptr == unit_mgr_)) {
229
    ret = OB_ERR_UNEXPECTED;
230
    LOG_WARN("unit mgr ptr is nullptr", K(ret));
231
  } else if (OB_FAIL(unit_mgr_->get_units_by_pool(pool.resource_pool_id_, pool_unit_array))) {
232
    LOG_WARN("fail to get units by pool", K(ret), "pool_id", pool.resource_pool_id_);
233
  } else if (OB_UNLIKELY(nullptr == pool_unit_array)) {
234
    ret = OB_ERR_UNEXPECTED;
235
    LOG_WARN("pool unit array", K(ret));
236
  } else if (OB_FAIL(unit_mgr_->get_unit_config_by_id(pool.unit_config_id_, unit_config))) {
237
    LOG_WARN("fail to get unit config by pool name", K(pool));
238
  } else if (OB_UNLIKELY(nullptr == unit_config)) {
239
    ret = OB_ERR_UNEXPECTED;
240
    LOG_WARN("unit config ptr is null", K(ret), K(pool));
241
  } else {
242
    ObServerInfoInTable server_info;
243
    ObUnitStat unit_stat;
244
    ObArray<ObUnitStat> in_migrate_unit_stat;
245
    common::ObArray<common::ObAddr> excluded_servers;
246
    common::ObAddr migrate_server;
247
    std::string resource_not_enough_reason;
248
    ObArray<ObServerInfoInTable> servers_info_of_zone;
249
    ObArray<ObServerInfoInTable> active_servers_info_of_zone;
250
    ObArray<obrpc::ObGetServerResourceInfoResult> active_servers_resource_info_of_zone;
251
    for (int64_t i = 0; OB_SUCC(ret) && i < pool_unit_array->count(); ++i) {
252
      excluded_servers.reset();
253
      server_info.reset();
254
      unit_stat.reset();
255
      migrate_server.reset();
256
      servers_info_of_zone.reset();
257
      active_servers_resource_info_of_zone.reset();
258
      active_servers_info_of_zone.reset();
259
      share::ObUnit *unit = pool_unit_array->at(i);
260
      if (OB_UNLIKELY(nullptr == unit)) {
261
        ret = OB_ERR_UNEXPECTED;
262
        LOG_WARN("unit ptr is null", K(ret));
263
      } else if (unit->migrate_from_server_.is_valid()) {
264
        // unit in migrate, bypass
265
      } else if (OB_FAIL(SVR_TRACER.get_server_info(unit->server_, server_info))) {
266
        LOG_WARN("fail to get server status", K(ret), "server", unit->server_);
267
      } else if (!server_info.is_active()) {
268
        // Only process servers that are active, skip non-active servers
269
        LOG_INFO("unit server status not active", K(ret), K(server_info), K(*unit));
270
      } else if (!has_exist_in_array(sys_unit_server_array, unit->server_)) {
271
        // bypass
272
      } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
273
              unit->unit_id_,
274
              unit->zone_,
275
              unit_stat))) {
276
        LOG_WARN("fail to locate unit", K(ret), "unit", *unit);
277
      } else if (OB_FAIL(SVR_TRACER.get_servers_info(unit->zone_, servers_info_of_zone))) {
278
        LOG_WARN("fail to servers_info_of_zone", KR(ret), K(unit->zone_));
279
      } else if (OB_FAIL(get_active_servers_info_and_resource_info_of_zone(
280
          unit->zone_,
281
          active_servers_info_of_zone,
282
          active_servers_resource_info_of_zone))) {
283
        LOG_WARN("fail to execute get_active_servers_info_and_resource_info_of_zone", KR(ret), K(unit->zone_));
284
      } else if (OB_FAIL(unit_mgr_->get_excluded_servers(
285
          *unit,
286
          unit_stat,
287
          module,
288
          servers_info_of_zone,
289
          active_servers_resource_info_of_zone,
290
          excluded_servers))) {
291
        LOG_WARN("fail to get exclude servers", K(ret), KPC(unit), K(servers_info_of_zone),
292
            K(active_servers_resource_info_of_zone));
293
      } else if (OB_FAIL(append(excluded_servers, sys_unit_server_array))) {
294
        LOG_WARN("fail tp append sys unit server array", K(ret));
295
      } else if (OB_FAIL(unit_mgr_->choose_server_for_unit(
296
          unit_config->unit_resource(), unit->zone_,
297
          excluded_servers,
298
          module,
299
          active_servers_info_of_zone,
300
          active_servers_resource_info_of_zone,
301
          migrate_server,
302
          resource_not_enough_reason))) {
303
        if (OB_ZONE_RESOURCE_NOT_ENOUGH == ret || OB_ZONE_SERVER_NOT_ENOUGH == ret) {
304
          LOG_WARN("has no place to migrate unit", K(module), KR(ret), "unit", *unit,
305
              K(excluded_servers), "resource_not_enough_reason", resource_not_enough_reason.c_str());
306
          ret = OB_SUCCESS;
307
        } else {
308
          LOG_WARN("fail to choose server for unit", KR(ret), K(unit), K(excluded_servers));
309
        }
310
      } else if (OB_FAIL(get_unit_resource_reservation(unit->unit_id_,
311
                                                       migrate_server,
312
                                                       in_migrate_unit_stat))) {
313
       LOG_WARN("fail to get unit resource reservation", K(ret), K(unit), K(migrate_server));
314
      } else if (OB_FAIL(try_migrate_unit(unit->unit_id_, pool.tenant_id_, unit_stat,
315
                                          in_migrate_unit_stat, migrate_server))) {
316
        LOG_WARN("fail to migrate unit", K(ret), K(unit), K(unit_stat), K(in_migrate_unit_stat));
317
      }
318
    }
319
  }
320
  return ret;
321
}
322

323
int ObServerBalancer::distribute_for_standalone_sys_unit()
324
{
325
  int ret = OB_SUCCESS;
326
  const bool enable_sys_unit_standalone = GCONF.enable_sys_unit_standalone;
327
  ObArray<ObAddr> sys_unit_server_array;
328
  const common::ObZone empty_zone; // means all zones
329
  LOG_INFO("start distribute for standalone sys unit");
330
  if (!check_inner_stat()) {
331
    ret = OB_INNER_STAT_ERROR;
332
    LOG_WARN("fail to check inner stat", K(ret), K(inited_));
333
  } else if (!enable_sys_unit_standalone) {
334
    ret = OB_STATE_NOT_MATCH;
335
    LOG_WARN("sys unit standalone deployment is disabled", K(ret));
336
  } else if (OB_FAIL(unit_mgr_->get_tenant_unit_servers(
337
          OB_SYS_TENANT_ID, empty_zone, sys_unit_server_array))) {
338
    LOG_WARN("fail to get tenant unit server array", K(ret));
339
  } else {
340
    ObHashMap<uint64_t, share::ObResourcePool *>::const_iterator iter = unit_mgr_->get_id_pool_map().begin();
341
    ObHashMap<uint64_t, share::ObResourcePool *>::const_iterator end = unit_mgr_->get_id_pool_map().end();
342
    for (; OB_SUCC(ret) && iter != end; ++iter) {
343
      if (OB_UNLIKELY(OB_INVALID_ID == iter->first)) {
344
        ret = OB_ERR_UNEXPECTED;
345
        LOG_WARN("pool id unexpected", K(ret));
346
      } else if (OB_UNLIKELY(nullptr == iter->second)) {
347
        ret = OB_ERR_UNEXPECTED;
348
        LOG_WARN("resource pool is null", K(ret));
349
      } else if (OB_SYS_TENANT_ID == iter->second->tenant_id_) {
350
        // no need to distribute for sys tenant
351
      } else if (OB_FAIL(distribute_pool_for_standalone_sys_unit(
352
              *iter->second, sys_unit_server_array))) {
353
        LOG_WARN("fail to distribute pool for standalone sys unit", K(ret));
354
      }
355
    }
356
  }
357
  return ret;
358
}
359

360
int ObServerBalancer::distribute_for_server_status_change()
361
{
362
  int ret = OB_SUCCESS;
363
  LOG_INFO("start distribute for server status change");
364
  if (!check_inner_stat()) {
365
    ret = OB_INNER_STAT_ERROR;
366
    LOG_WARN("check_inner_stat failed", K_(inited),  K(ret));
367
  }
368

369
  ObHashMap<uint64_t, ObArray<share::ObResourcePool *> *>::const_iterator it =
370
      unit_mgr_->get_tenant_pools_map().begin();
371
  ObHashMap<uint64_t, ObArray<share::ObResourcePool *> *>::const_iterator it_end =
372
      unit_mgr_->get_tenant_pools_map().end();
373
  // Let the system tenant do it first
374
  for (; OB_SUCC(ret) && it != it_end; ++it) {
375
    if (OB_SYS_TENANT_ID != it->first) {
376
      // bypass, non sys tenant, do later
377
    } else if (NULL == it->second) {
378
      ret = OB_ERR_UNEXPECTED;
379
      LOG_WARN("it->second is null", KP(it->second), K(ret));
380
    } else if (OB_FAIL(distribute_by_tenant(it->first, *(it->second)))) {
381
      LOG_WARN("distribute by tenant failed", "tenant_id", it->first, K(ret));
382
    }
383
  }
384
  ObHashMap<uint64_t, ObArray<share::ObResourcePool *> *>::const_iterator it2 =
385
      unit_mgr_->get_tenant_pools_map().begin();
386
  ObHashMap<uint64_t, ObArray<share::ObResourcePool *> *>::const_iterator it2_end =
387
      unit_mgr_->get_tenant_pools_map().end();
388
  // other tenant
389
  for (; OB_SUCC(ret) && it2 != it2_end; ++it2) {
390
    if (OB_SYS_TENANT_ID == it2->first) {
391
      // bypass, sys tenant has been done before
392
    } else if (NULL == it2->second) {
393
      ret = OB_ERR_UNEXPECTED;
394
      LOG_WARN("it->second is null", KP(it2->second), K(ret));
395
    } else if (OB_FAIL(distribute_by_tenant(it2->first, *(it2->second)))) {
396
      LOG_WARN("distribute by tenant failed", "tenant_id", it2->first, K(ret));
397
    }
398
  }
399

400
  ObHashMap<uint64_t, share::ObResourcePool*>::const_iterator pool_it =
401
      unit_mgr_->get_id_pool_map().begin();
402
  ObHashMap<uint64_t, share::ObResourcePool*>::const_iterator pool_it_end =
403
      unit_mgr_->get_id_pool_map().end();
404
  for (; OB_SUCC(ret) && pool_it != pool_it_end; ++pool_it) {
405
    if (NULL == pool_it->second) {
406
      ret = OB_ERR_UNEXPECTED;
407
      LOG_WARN("pool_it->second is null", KP(pool_it->second), K(ret));
408
    } else if (OB_INVALID_ID != pool_it->second->tenant_id_) {
409
      // bypass, already distribute in distribute by tenant
410
    } else if (OB_FAIL(distribute_by_pool(pool_it->second))) {
411
      LOG_WARN("distribute by pool failed", "pool", *(pool_it->second), K(ret));
412
    }
413
  }
414
  LOG_INFO("finish distribute for server status change", K(ret));
415
  return ret;
416
}
417

418
int ObServerBalancer::distribute_by_tenant(const uint64_t tenant_id,
419
                                        const ObArray<share::ObResourcePool *> &pools)
420
{
421
  int ret = OB_SUCCESS;
422
  ObArray<ObUnitManager::ZoneUnit> zone_units;
423

424
  if (!check_inner_stat()) {
425
    ret = OB_INNER_STAT_ERROR;
426
    LOG_WARN("check_inner_stat failed", K_(inited),  K(ret));
427
  } else if (OB_INVALID_ID == tenant_id || pools.count() <= 0) {
428
    ret = OB_INVALID_ARGUMENT;
429
    LOG_WARN("invalid argument", K(tenant_id), K(pools), K(ret));
430
  } else if (OB_FAIL(unit_mgr_->get_zone_units(pools, zone_units))) {
431
    LOG_WARN("get_units_of_pools failed", K(pools), K(ret));
432
  } else {
433
    FOREACH_CNT_X(zone_unit, zone_units, OB_SUCCESS == ret) {
434
      if (OB_FAIL(distribute_zone_unit(*zone_unit))) {
435
        LOG_WARN("distribute zone unit failed", "zone_unit", *zone_unit, K(ret));
436
      }
437
    }
438
  }
439
  return ret;
440
}
441

442
int ObServerBalancer::distribute_by_pool(share::ObResourcePool *pool)
443
{
444
  int ret = OB_SUCCESS;
445
  ObArray<share::ObResourcePool *> pools;
446
  ObArray<ObUnitManager::ZoneUnit> zone_units;
447
  if (!check_inner_stat()) {
448
    ret = OB_INNER_STAT_ERROR;
449
    LOG_WARN("check_inner_stat failed", K_(inited),  K(ret));
450
  } else if (NULL == pool) {
451
    ret = OB_INVALID_ARGUMENT;
452
    LOG_WARN("pool is null", K(ret));
453
  } else if (OB_FAIL(pools.push_back(pool))) {
454
    LOG_WARN("push_back failed", K(ret));
455
  } else if (OB_FAIL(unit_mgr_->get_zone_units(pools, zone_units))) {
456
    LOG_WARN("get_units_of_pools failed", K(pools), K(ret));
457
  } else {
458
    FOREACH_CNT_X(zone_unit, zone_units, OB_SUCCESS == ret) {
459
      if (OB_FAIL(distribute_zone_unit(*zone_unit))) {
460
        LOG_WARN("distribute zone unit failed", "zone_unit", *zone_unit, K(ret));
461
      }
462
    }
463
  }
464
  return ret;
465
}
466

467
int ObServerBalancer::distribute_zone_unit(const ObUnitManager::ZoneUnit &zone_unit)
468
{
469
  int ret = OB_SUCCESS;
470
  if (!check_inner_stat()) {
471
    ret = OB_INNER_STAT_ERROR;
472
    LOG_WARN("check inner stat failed", K_(inited), K(ret));
473
  } else if (!zone_unit.is_valid()) {
474
    ret = OB_INVALID_ARGUMENT;
475
    LOG_WARN("invalid argument", K(zone_unit), K(ret));
476
  } else {
477
    ObServerInfoInTable server_info;
478
    FOREACH_CNT_X(unit_info, zone_unit.unit_infos_, OB_SUCCESS == ret) {
479
      server_info.reset();
480
      if (ObUnit::UNIT_STATUS_ACTIVE != unit_info->unit_.status_) {
481
        // ignore the unit that is in deleting
482
      } else if (OB_FAIL(SVR_TRACER.get_server_info(unit_info->unit_.server_, server_info))) {
483
        LOG_WARN("get_server_info failed", "server", unit_info->unit_.server_, KR(ret));
484
      } else if (server_info.is_active()) {
485
        if (OB_FAIL(distribute_for_active(server_info, *unit_info))) {
486
          LOG_WARN("distribute_for_active failed", K(server_info), "unit_info", *unit_info, K(ret));
487
        }
488
      } else if (server_info.is_permanent_offline() || server_info.is_deleting()) {
489
        if (OB_FAIL(distribute_for_permanent_offline_or_delete(server_info, *unit_info))) {
490
          LOG_WARN("distribute for permanent offline or delete failed",
491
                    K(server_info), "unit_info", *unit_info, KR(ret));
492
        }
493
      }
494
    }
495
  }
496
  return ret;
497
}
498

499
int ObServerBalancer::distribute_for_active(
500
    const ObServerInfoInTable &server_info,
501
    const ObUnitInfo &unit_info)
502
{
503
  int ret = OB_SUCCESS;
504
  if (!check_inner_stat()) {
505
    ret = OB_INNER_STAT_ERROR;
506
    LOG_WARN("check inner stat failed", K_(inited), K(ret));
507
  } else if (!server_info.is_valid()
508
      || !server_info.is_active()
509
      || !unit_info.is_valid()
510
      || unit_info.unit_.server_ != server_info.get_server()) {
511
    ret = OB_INVALID_ARGUMENT;
512
    LOG_WARN("invalid argument", K(server_info), K(unit_info), K(ret));
513
  } else {
514
    //When the destination is blocked, cancel this migration
515
    //Temporary offline does not cancel the task, need to wait for permanent offline
516
    if ((server_info.is_migrate_in_blocked())
517
        && unit_info.unit_.migrate_from_server_.is_valid()) {
518
      LOG_INFO("find unit server active but can't migrate in, "
519
          "migrate_from_server is set", "unit", unit_info.unit_);
520
      if (OB_FAIL(distribute_for_migrate_in_blocked(unit_info))) {
521
        LOG_WARN("distribute for migrate in blocked failed", K(unit_info), K(ret));
522
      }
523
    }
524
  }
525
  return ret;
526
}
527

528
//When the migration destination is permanently offline,
529
//need to change to another destination
530
//Need to make sure that the member has been kicked out after being permanently offline
531
int ObServerBalancer::distribute_for_permanent_offline_or_delete(
532
    const share::ObServerInfoInTable &server_info,
533
    const ObUnitInfo &unit_info)
534
{
535
  int ret = OB_SUCCESS;
536
  const char *module = "UNIT_BALANCE_FOR_SERVER_PERMANENT_OFFLINE_OR_DELETE";
537

538
  LOG_INFO("find unit server permanent offline or delete, need distribute unit",
539
           K(module), "unit", unit_info.unit_, K(server_info));
540
  const bool enable_sys_unit_standalone = GCONF.enable_sys_unit_standalone;
541
  bool need_migrate_unit = false;
542
  if (!check_inner_stat()) {
543
    ret = OB_INNER_STAT_ERROR;
544
    LOG_WARN("check inner stat failed", K_(inited), K(ret));
545
  } else if (!server_info.is_valid()
546
      || !unit_info.is_valid()
547
      || unit_info.unit_.server_ != server_info.get_server()
548
      || (!server_info.is_deleting() && !server_info.is_permanent_offline())) {
549
    ret = OB_INVALID_ARGUMENT;
550
    LOG_WARN("invalid argument", K(server_info), K(unit_info), KR(ret));
551
  } else if (!unit_info.unit_.migrate_from_server_.is_valid()) {
552
    //The current unit is in a stable state, move it out
553
    need_migrate_unit = true;
554
    LOG_INFO("server is permanent offline or in deleting status, need migrate unit",
555
             K(unit_info), K(server_info));
556
  } else {
557
    //Currently moving in, try to cancel
558
    bool is_canceled = false;
559
    if (OB_FAIL(try_cancel_migrate_unit(unit_info.unit_, is_canceled))) {
560
      LOG_WARN("fail to cancel migrate unit", K(ret), K(unit_info));
561
    } else if (!is_canceled) {
562
      //If cancel fails, wait for the result of the check-in process
563
      //If the move-in process cannot be ended,
564
      //the delete server lasts for too long, and manual intervention should be required
565
      // ** FIXME (linqiucen): now we do not do the following commented process due to the deprecated variable with_partition
566
      // ** FIXME (linqiucen): in the future, we can do this process again by directly looking up the related table
567
      // if (!status.is_with_partition()) {
568
      //   //If there is no local replica, cancel this migration directly
569
      //   const ObUnitManager::EndMigrateOp op = ObUnitManager::ABORT;
570
      //   if (OB_FAIL(unit_mgr_->end_migrate_unit(unit_info.unit_.unit_id_, op))) {
571
      //     LOG_WARN("end_migrate_unit failed", "unit_id", unit_info.unit_.unit_id_, K(op), K(ret));
572
      //   } else {
573
      //     need_migrate_unit = true;
574
      //     LOG_INFO("unit has no partition, abort the migration",
575
      //              K(ret), K(unit_info), K(op), K(status));
576
      //   }
577
      // }
578
    } else {
579
      LOG_INFO("revert migrate unit success", K(ret), K(unit_info), K(server_info));
580
    }
581
  }
582
  ObUnitStat unit_stat;
583
  ObArray<ObAddr> excluded_servers;
584
  const ObZone zone = unit_info.unit_.zone_;
585
  ObAddr migrate_server;
586
  std::string resource_not_enough_reason;
587
  ObArray<ObServerInfoInTable> servers_info_of_zone;
588
  ObArray<ObServerInfoInTable> active_servers_info_of_zone;
589
  ObArray<obrpc::ObGetServerResourceInfoResult> active_servers_resource_info_of_zone;
590
  if (OB_FAIL(ret) || !need_migrate_unit) {
591
    //nothing todo
592
  } else if (OB_ISNULL(unit_mgr_)) {
593
    ret = OB_ERR_UNEXPECTED;
594
    LOG_WARN("unit_mgr_ is null", KR(ret), KP(unit_mgr_));
595
  } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
596
          unit_info.unit_.unit_id_,
597
          unit_info.unit_.zone_,
598
          unit_stat))) {
599
    LOG_WARN("fail to locate unit", K(ret), "unit", unit_info.unit_);
600
  } else if (OB_FAIL(SVR_TRACER.get_servers_info(unit_info.unit_.zone_, servers_info_of_zone))) {
601
    LOG_WARN("fail to servers_info_of_zone", KR(ret), K(unit_info.unit_.zone_));
602
  } else if (OB_FAIL(get_active_servers_info_and_resource_info_of_zone(
603
          unit_info.unit_.zone_,
604
          active_servers_info_of_zone,
605
          active_servers_resource_info_of_zone))) {
606
        LOG_WARN("fail to execute get_active_servers_info_and_resource_info_of_zone", KR(ret), K(unit_info.unit_.zone_));
607
  } else if (OB_FAIL(unit_mgr_->get_excluded_servers(
608
      unit_info.unit_,
609
      unit_stat,
610
      module,
611
      servers_info_of_zone,
612
      active_servers_resource_info_of_zone,
613
      excluded_servers))) {
614
    LOG_WARN("get_excluded_servers failed", "unit", unit_info.unit_, KR(ret), K(servers_info_of_zone),
615
        K(active_servers_resource_info_of_zone));
616
  } else if (OB_FAIL(unit_mgr_->choose_server_for_unit(
617
      unit_info.config_.unit_resource(),
618
      zone,
619
      excluded_servers,
620
      module,
621
      active_servers_info_of_zone,
622
      active_servers_resource_info_of_zone,
623
      migrate_server,
624
      resource_not_enough_reason))) {
625
    if (OB_ZONE_RESOURCE_NOT_ENOUGH == ret || OB_ZONE_SERVER_NOT_ENOUGH == ret) {
626
      LOG_WARN("has no place to migrate unit", K(module), KR(ret), K(zone), K(excluded_servers),
627
          K(unit_info), "resource_not_enough_reason", resource_not_enough_reason.c_str());
628
      // return success when resource not enough to go on next balance task
629
      ret = OB_SUCCESS;
630
    } else {
631
      LOG_WARN("choose_unit_server failed", "config", unit_info.config_,
632
          K(zone), K(excluded_servers), K(ret));
633
    }
634
  } else {
635
    ObArray<ObUnitStat> in_migrate_unit_stat;
636
    if (OB_FAIL(get_unit_resource_reservation(unit_info.unit_.unit_id_,
637
        migrate_server,
638
        in_migrate_unit_stat))) {
639
      LOG_WARN("get_unit_resource_reservation failed", K(unit_info), K(ret));
640
    } else if (OB_FAIL(try_migrate_unit(unit_info.unit_.unit_id_,
641
        unit_info.pool_.tenant_id_,
642
        unit_stat,
643
        in_migrate_unit_stat,
644
        migrate_server))) {
645
      LOG_WARN("fail to try migrate unit", "unit", unit_info.unit_, K(migrate_server), K(ret));
646
    } else {
647
      LOG_INFO("migrate unit success", K(module), K(unit_info), K(server_info), "dest_server", migrate_server);
648
    }
649
  }
650
  return ret;
651
}
652

653
// NOTE:
654
//  When the unit migrates from A to B, if B is down or the disk is full,
655
//  two strategies will be tried in turn
656
//  1. A can move in, cancel this migration, and re-migrate the data back to A
657
//  2. A cannot move in, then choose a new machine C, and move unit to C
658
int ObServerBalancer::distribute_for_migrate_in_blocked(const ObUnitInfo &unit_info)
659
{
660
  int ret = OB_SUCCESS;
661
  ObServerInfoInTable server_info;
662
  if (!check_inner_stat()) {
663
    ret = OB_INNER_STAT_ERROR;
664
    LOG_WARN("check inner stat failed", K_(inited), K(ret));
665
  } else if (!unit_info.is_valid() || !unit_info.unit_.migrate_from_server_.is_valid()) {
666
    ret = OB_INVALID_ARGUMENT;
667
    LOG_WARN("invalid argument", K(unit_info), K(ret));
668
  } else if (OB_FAIL(SVR_TRACER.get_server_info(
669
              unit_info.unit_.migrate_from_server_, server_info))) {
670
    LOG_WARN("get_server_status failed",
671
             "server", unit_info.unit_.migrate_from_server_, K(ret));
672
  } else if (ObUnit::UNIT_STATUS_ACTIVE != unit_info.unit_.status_) {
673
    // ignore the unit which is in deleting
674
  } else {
675
    if (server_info.can_migrate_in()) {
676
      LOG_INFO("unit migrate_from_server can migrate in, "
677
          "migrate unit back to migrate_from_server", "unit", unit_info.unit_);
678
      const ObUnitManager::EndMigrateOp op = ObUnitManager::REVERSE;
679
      if (OB_FAIL(unit_mgr_->end_migrate_unit(unit_info.unit_.unit_id_, op))) {
680
        LOG_WARN("end_migrate_unit failed", "unit_id", unit_info.unit_.unit_id_, K(op), K(ret));
681
      }
682
    } else {
683
      //TODO: @wanhong.wwh need support when dest server is blocked and src server can not migrate in
684
      //nothing todo
685
      LOG_WARN("NOTICE: unit migration is hung. dest server is blocked "
686
          "and source server can not migrate in. NEED to be involved manually.",
687
               "unit", unit_info.unit_, "migrate_from_server", server_info);
688
    }
689

690
    /*
691
    {
692
      LOG_INFO("unit migrate_from_server can not migrate in, and unit_server is empty now"
693
          "migrate unit to new server", "unit", unit_info.unit_);
694
      ObAddr new_server;
695
      ObArray<ObAddr> excluded_servers;
696
      ObUnitStat unit_stat;
697
      ObArray<ObUnitStat> in_migrate_unit_stat;
698
      ObUnitManager::EndMigrateOp op = ObUnitManager::ABORT;
699
      if (OB_FAIL(unit_mgr_->end_migrate_unit(unit_info.unit_.unit_id_, op))) {
700
        LOG_WARN("fail to end migrate unit", K(ret), "unit", unit_info.unit_);
701
      } else if (OB_FAIL(unit_stat_mgr_->get_unit_stat(
702
              unit_info.unit_.unit_id_, unit_info.unit_.zone_, unit_stat))) {
703
        LOG_WARN("fail to locate unit", K(ret), "unit", unit_info.unit_);
704
      } else if (OB_FAIL(unit_mgr_->get_excluded_servers(unit_info.unit_, unit_stat, excluded_servers))) {
705
        LOG_WARN("get_excluded_servers failed", "unit", unit_info.unit_, K(ret));
706
      } else if (OB_FAIL(unit_mgr_->choose_server_for_unit(unit_info.config_, unit_info.unit_.zone_,
707
          excluded_servers, new_server))) {
708
        LOG_WARN("choose_unit_server failed", "config", unit_info.config_,
709
            "zone", unit_info.unit_.zone_, K(excluded_servers), K(ret));
710
      } else if (OB_FAIL(get_unit_resource_reservation(unit_info.unit_.unit_id_,
711
                                                       new_server,
712
                                                       in_migrate_unit_stat))) {
713
        LOG_WARN("get_unit_resource_reservation failed", K(unit_info), K(ret));
714
      } else if (OB_FAIL(try_migrate_unit(unit_info.unit_.unit_id_,
715
                                          unit_info.pool_.tenant_id_,
716
                                          unit_stat,
717
                                          in_migrate_unit_stat,
718
                                          new_server))) {
719
        LOG_WARN("fail to try migrate unit ", "unit", unit_info.unit_, K(new_server), K(ret));
720
      }
721
    }
722
    */
723
  }
724
  return ret;
725
}
726

727
int ObServerBalancer::get_unit_resource_reservation(uint64_t unit_id,
728
                                                    const ObAddr &new_server,
729
                                                    ObIArray<ObUnitStat> &in_migrate_unit_stat)
730
{
731
  int ret = OB_SUCCESS;
732
  UNUSED(unit_id);
733
  ObArray<ObUnitManager::ObUnitLoad> *unit_loads = NULL;
734
  if (!inited_) {
735
    ret = OB_NOT_INIT;
736
    LOG_WARN("server balancer not init", K_(inited), K(ret));
737
  } else {
738
    if (OB_FAIL(unit_mgr_->get_loads_by_server(new_server, unit_loads))) {
739
      if (OB_ENTRY_NOT_EXIST != ret) {
740
        LOG_WARN("fail to get loads by server", K(ret));
741
      } else {
742
        ret = OB_SUCCESS;
743
      }
744
    } else if (NULL == unit_loads) {
745
      ret = OB_ERR_UNEXPECTED;
746
      LOG_WARN("unit loads ptr is null", K(ret));
747
    } else {
748
      // Statistics of the units that are being migrated to the new server
749
      for (int64_t i = 0; OB_SUCC(ret) && i < unit_loads->count(); ++i) {
750
        ObUnitStat in_mig_stat;
751
        const ObUnitManager::ObUnitLoad &load = unit_loads->at(i);
752
        if (!load.is_valid()) {
753
          ret = OB_ERR_UNEXPECTED;
754
          LOG_WARN("unit load is invalid", K(ret), K(load));
755
        } else if (new_server == load.unit_->migrate_from_server_) {
756
          // by pass
757
        } else if (!load.unit_->migrate_from_server_.is_valid()) {
758
          // by pass
759
        } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
760
                load.unit_->unit_id_, load.unit_->zone_, in_mig_stat))) {
761
          LOG_WARN("get unit stat fail", "unit_id", load.unit_->unit_id_, K(ret));
762
        } else if (OB_FAIL(in_migrate_unit_stat.push_back(in_mig_stat))) {
763
          LOG_WARN("push back fail", K(in_mig_stat), K(ret));
764
        }
765
      }
766
    }
767
  }
768
  return ret;
769
}
770

771
int ObServerBalancer::try_migrate_unit(const uint64_t unit_id,
772
                                       const uint64_t tenant_id,
773
                                       const ObUnitStat &unit_stat,
774
                                       const ObIArray<ObUnitStat> &migrating_unit_stat,
775
                                       const ObAddr &dst)
776
{
777
  int ret = OB_SUCCESS;
778
  ObServerResourceInfo dst_resource_info;
779
  if (!inited_) {
780
    ret = OB_NOT_INIT;
781
    LOG_WARN("server balancer not init", K_(inited), K(ret));
782
  } else if (OB_ISNULL(server_mgr_)) {
783
    ret = OB_ERR_UNEXPECTED;
784
    LOG_WARN("server_mgr_ is null", KR(ret), KP(server_mgr_));
785
  } else if (OB_FAIL(server_mgr_->get_server_resource_info(dst, dst_resource_info))) {
786
    LOG_WARN("fail to get dst_resource_info", KR(ret), K(dst));
787
  } else {
788
    ret = unit_mgr_->try_migrate_unit(
789
        unit_id,
790
        tenant_id,
791
        unit_stat,
792
        migrating_unit_stat,
793
        dst,
794
        dst_resource_info);
795
  }
796
  return ret;
797
}
798

799
int ObServerBalancer::try_cancel_migrate_unit(const share::ObUnit &unit, bool &is_canceled)
800
{
801
  int ret = OB_SUCCESS;
802
  if (!inited_) {
803
    ret = OB_NOT_INIT;
804
    LOG_WARN("server balancer not init", K_(inited), K(ret));
805
  } else {
806
    ret = unit_mgr_->try_cancel_migrate_unit(unit, is_canceled);
807
  }
808
  return ret;
809
}
810

811
const double ObServerBalancer::CPU_EPSILON = 0.0001;
812
const double ObServerBalancer::EPSILON = 0.0000000001;
813

814
// the new version server balance
815
int ObServerBalancer::UnitMigrateStatCmp::get_ret() const
816
{
817
  return ret_;
818
}
819

820
bool ObServerBalancer::UnitMigrateStatCmp::operator()(
821
     const UnitMigrateStat &left,
822
     const UnitMigrateStat &right)
823
{
824
  bool bool_ret = false;
825
  if (OB_SUCCESS != ret_) {
826
    // do nothing
827
  } else if (left.arranged_pos_ < right.arranged_pos_) {
828
    bool_ret = true;
829
  } else if (right.arranged_pos_ < left.arranged_pos_) {
830
    bool_ret = false;
831
  } else {
832
    ret_ = OB_ERR_UNEXPECTED;
833
  }
834
  return bool_ret;
835
}
836

837
int ObServerBalancer::check_can_execute_rebalance(
838
    const common::ObZone &zone,
839
    bool &can_execute_rebalance)
840
{
841
  int ret = OB_SUCCESS;
842
  HEAP_VAR(share::ObZoneInfo, zone_info) {
843
    common::ObArray<common::ObAddr> server_list;
844
    common::ObArray<uint64_t> unit_ids;
845
    if (OB_UNLIKELY(!inited_)) {
846
      ret = OB_NOT_INIT;
847
      LOG_WARN("not init", K(ret));
848
    } else if (OB_UNLIKELY(zone.is_empty())) {
849
      ret = OB_INVALID_ARGUMENT;
850
      LOG_WARN("invalid argument", K(ret), K(zone));
851
    } else if (OB_ISNULL(unit_mgr_) || OB_ISNULL(zone_mgr_) || OB_ISNULL(server_mgr_)) {
852
      ret = OB_ERR_UNEXPECTED;
853
      LOG_WARN("unit_mgr_, zone_mgr_ or server_mgr_ is null", KR(ret), KP(unit_mgr_), KP(zone_mgr_), KP(server_mgr_));
854
    } else if (OB_FAIL(zone_mgr_->get_zone(zone, zone_info))) {
855
      LOG_WARN("fail to get zone info", K(ret), K(zone));
856
    } else if (ObZoneStatus::ACTIVE != zone_info.status_) {
857
      can_execute_rebalance = false;
858
      LOG_INFO("cannot execute server rebalance: zone inactive", K(zone));
859
    } else if (OB_FAIL(SVR_TRACER.get_servers_of_zone(zone, server_list))) {
860
      LOG_WARN("fail to get servers of zone", K(ret), K(zone));
861
    } else if (OB_FAIL(unit_mgr_->inner_get_unit_ids(unit_ids))) {
862
      LOG_WARN("fail to get unit ids", K(ret));
863
    } else {
864
      can_execute_rebalance = true;
865
      share::ObUnitConfig sum_load;
866
      for (int64_t i = 0; can_execute_rebalance && OB_SUCC(ret) && i < server_list.count(); ++i) {
867
        const common::ObAddr &server = server_list.at(i);
868
        ObServerInfoInTable server_info;
869
        ObServerResourceInfo resource_info;
870
        ObArray<ObUnitManager::ObUnitLoad> *unit_loads = nullptr;
871
        sum_load.reset();
872
        if (OB_FAIL(unit_mgr_->get_loads_by_server(server, unit_loads))) {
873
          if (OB_ENTRY_NOT_EXIST != ret) {
874
            LOG_WARN("fail to get loads by server", K(ret));
875
          } else {
876
            ret = OB_SUCCESS; // unit_loads not exist, no load no this server
877
          }
878
        } else if (OB_UNLIKELY(nullptr == unit_loads)) {
879
          ret = OB_ERR_UNEXPECTED;
880
          LOG_WARN("unit loads ptr is null", K(ret));
881
        } else if (OB_FAIL(unit_mgr_->calc_sum_load(unit_loads, sum_load))) {
882
          LOG_WARN("fail to calc sum load", K(ret));
883
        }
884
        if (OB_FAIL(ret)) {
885
          // failed
886
        } else if (OB_FAIL(SVR_TRACER.get_server_info(server, server_info))) {
887
          LOG_WARN("fail to get server status", K(ret));
888
        } else if (server_info.is_temporary_offline() || server_info.is_stopped()) {
889
          can_execute_rebalance = false;
890
          LOG_INFO("cannot execute server rebalance: Server status is not normal", K(zone), K(server_info));
891
        } else if (OB_FAIL(server_mgr_->get_server_resource_info(server_info.get_server(), resource_info))) {
892
          LOG_WARN("fail to execute get_server_resource_info", KR(ret), K(server_info.get_server()));
893
        } else if (fabs(resource_info.report_cpu_assigned_ - sum_load.min_cpu()) > CPU_EPSILON
894
            || fabs(resource_info.report_cpu_max_assigned_ - sum_load.max_cpu()) > CPU_EPSILON
895
            || resource_info.report_mem_assigned_ != sum_load.memory_size()) {
896
          can_execute_rebalance = false;
897
          LOG_INFO("cannot execute server rebalance: "
898
                   "Server resource_info and unit_manager sum_load not equal", K(zone), K(resource_info), K(sum_load));
899
        } else {} // no more to do
900
      }
901
      for (int64_t j = 0; can_execute_rebalance && OB_SUCC(ret) && j < unit_ids.count(); ++j) {
902
        ObUnit *unit = NULL;
903
        if (OB_FAIL(unit_mgr_->get_unit_by_id(unit_ids.at(j), unit))) {
904
          ret = OB_INVALID_ARGUMENT;
905
          LOG_WARN("unit_load is invalid", K(ret));
906
        } else if (NULL == unit) {
907
          ret = OB_ERR_UNEXPECTED;
908
          LOG_WARN("unit ptr is null", K(ret), KP(unit));
909
        } else if (ObUnit::UNIT_STATUS_DELETING == unit->status_) {
910
          can_execute_rebalance = false;
911
          LOG_INFO("cannot execute server rebalance: unit deleting", K(zone), "unit", *unit);
912
        } else if (unit->migrate_from_server_.is_valid()) {
913
          can_execute_rebalance = false;
914
          LOG_INFO("cannot execute server rebalance: unit migrating", K(zone), "unit", *unit);
915
        } else {} // unit in stable status
916
      }
917
    }
918
  }
919
  return ret;
920
}
921

922
int ObServerBalancer::rebalance_servers_v2(
923
    const common::ObZone &zone)
924
{
925
  int ret = OB_SUCCESS;
926
  common::ObArray<Matrix<uint64_t> > group_tenant_array;
927
  common::ObArray<uint64_t> standalone_tenant_array;
928
  common::ObArray<ObUnitManager::ObUnitLoad> not_grant_units;
929
  common::ObArray<ObUnitManager::ObUnitLoad> standalone_units;
930
  LOG_INFO("start to do tenant group unit balance", K(zone));
931
  if (OB_UNLIKELY(!inited_)) {
932
    ret = OB_NOT_INIT;
933
    LOG_WARN("not init", K(ret));
934
  } else if (OB_FAIL(generate_zone_server_disk_statistic(zone))) {
935
    LOG_WARN("fail to generate zone server disk statistic", K(ret), K(zone));
936
  } else if (OB_FAIL(generate_tenant_array(zone, group_tenant_array, standalone_tenant_array))) {
937
    LOG_WARN("fail to generate tenant array", K(ret));
938
  } else if (OB_FAIL(generate_standalone_units(zone, standalone_tenant_array, standalone_units))) {
939
    LOG_WARN("fail to generate standalone units", K(ret), K(zone));
940
  } else if (OB_FAIL(generate_not_grant_pool_units(zone, not_grant_units))) {
941
    LOG_WARN("fail to generate not grant pool units", K(ret), K(zone));
942
  } else if (OB_FAIL(do_rebalance_servers_v2(
943
          zone, group_tenant_array, standalone_units, not_grant_units))) {
944
    LOG_WARN("fail to rebalance servers", K(ret), K(zone));
945
  } else {} // no more to do
946
  return ret;
947
}
948

949
int ObServerBalancer::generate_single_tenant_group(
950
    Matrix<uint64_t> &tenant_id_matrix,
951
    const ObTenantGroupParser::TenantNameGroup &tenant_name_group)
952
{
953
  int ret = OB_SUCCESS;
954
  share::schema::ObSchemaGetterGuard schema_guard;
955
  if (OB_UNLIKELY(!inited_)) {
956
    ret = OB_NOT_INIT;
957
    LOG_WARN("not init", K(ret));
958
  } else if (OB_UNLIKELY(tenant_name_group.row_ <= 0 || tenant_name_group.column_ <= 0)) {
959
    ret = OB_INVALID_ARGUMENT;
960
    LOG_WARN("invalid argument", K(ret), K(tenant_name_group));
961
  } else if (OB_UNLIKELY(NULL == schema_service_)) {
962
    ret = OB_ERR_UNEXPECTED;
963
    LOG_WARN("schema service ptr is null", K(ret), KP(schema_service_));
964
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
965
    ret = OB_ERR_UNEXPECTED;
966
    LOG_WARN("unit_mgr_ ptr is null", K(ret), KP(unit_mgr_));
967
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
968
    LOG_WARN("fail to get schema guard", K(ret));
969
  } else if (OB_FAIL(tenant_id_matrix.init(tenant_name_group.row_, tenant_name_group.column_))) {
970
    LOG_WARN("fail to init tenant id matrix", K(ret));
971
  } else {
972
    for (int64_t row = 0; OB_SUCC(ret) && row < tenant_name_group.row_; ++row) {
973
      for (int64_t column = 0; OB_SUCC(ret) && column < tenant_name_group.column_; ++column) {
974
        const int64_t idx = row * tenant_name_group.column_ + column;
975
        const share::schema::ObTenantSchema *tenant_schema = NULL;
976
        if (idx >= tenant_name_group.tenants_.count()) {
977
          ret = OB_ERR_UNEXPECTED;
978
          LOG_WARN("idx value unexpected", K(ret), K(idx),
979
                   "tenants_count", tenant_name_group.tenants_.count());
980
        } else if (OB_FAIL(schema_guard.get_tenant_info(
981
                tenant_name_group.tenants_.at(idx), tenant_schema))) {
982
          LOG_WARN("fail to get tenant info", K(ret),
983
                   "tenant_name", tenant_name_group.tenants_.at(idx));
984
        } else if (OB_UNLIKELY(NULL == tenant_schema)) {
985
          ret = OB_TENANT_NOT_EXIST;
986
          LOG_WARN("tenant schema ptr is null", K(ret), KP(tenant_schema));
987
        } else if (OB_UNLIKELY(OB_SYS_TENANT_ID == tenant_schema->get_tenant_id())) {
988
          ret = OB_ERR_UNEXPECTED;
989
          LOG_ERROR("sys tenant cannot be configured in tenant groups", K(ret));
990
        } else if (OB_FAIL(tenant_id_matrix.set(row, column, tenant_schema->get_tenant_id()))) {
991
          LOG_WARN("fail to set tenant id matrix", K(ret), K(row), K(column),
992
                   "tenant_id", tenant_schema->get_tenant_id());
993
        } else {} // next matrix cell
994
      }
995
    }
996
  }
997
  return ret;
998
}
999

1000
int ObServerBalancer::generate_group_tenant_array(
1001
    const common::ObIArray<ObTenantGroupParser::TenantNameGroup> &tenant_groups,
1002
    common::ObIArray<Matrix<uint64_t> > &group_tenant_array)
1003
{
1004
  int ret = OB_SUCCESS;
1005
  if (OB_UNLIKELY(!inited_)) {
1006
    ret = OB_NOT_INIT;
1007
    LOG_WARN("not init", K(ret));
1008
  } else {
1009
    group_tenant_array.reset();
1010
    for (int64_t i = 0; OB_SUCC(ret) && i < tenant_groups.count(); ++i) {
1011
      Matrix<uint64_t> tenant_id_matrix;
1012
      const ObTenantGroupParser::TenantNameGroup &tenant_name_group = tenant_groups.at(i);
1013
      if (OB_FAIL(generate_single_tenant_group(tenant_id_matrix, tenant_name_group))) {
1014
        LOG_WARN("fail to generate single tenant group", K(ret));
1015
      } else if (OB_FAIL(group_tenant_array.push_back(tenant_id_matrix))) {
1016
        LOG_WARN("fail to push back", K(ret));
1017
      } else {} // go on next
1018
    }
1019
  }
1020
  return ret;
1021
}
1022

1023
bool ObServerBalancer::has_exist_in_group_tenant_array(
1024
     const uint64_t tenant_id,
1025
     const common::ObIArray<Matrix<uint64_t> > &group_tenant_array)
1026
{
1027
  bool exist = false;
1028
  for (int64_t i = 0; !exist && i < group_tenant_array.count(); ++i) {
1029
    const Matrix<uint64_t> &tenant_group = group_tenant_array.at(i);
1030
    if (tenant_group.is_contain(tenant_id)) {
1031
      exist = true;
1032
    } else {} // not contain, go on next
1033
  }
1034
  return exist;
1035
}
1036

1037
int ObServerBalancer::generate_standalone_tenant_array(
1038
    const common::ObIArray<Matrix<uint64_t> > &group_tenant_array,
1039
    common::ObIArray<uint64_t> &standalone_tenant_array)
1040
{
1041
  int ret = OB_SUCCESS;
1042
  share::schema::ObSchemaGetterGuard schema_guard;
1043
  common::ObArray<uint64_t> tenant_ids;
1044
  if (OB_UNLIKELY(!inited_)) {
1045
    ret = OB_NOT_INIT;
1046
    LOG_WARN("not init", K(ret));
1047
  } else if (OB_UNLIKELY(NULL == schema_service_)) {
1048
    ret = OB_ERR_UNEXPECTED;
1049
    LOG_WARN("schema_service_ ptr is null", K(ret));
1050
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
1051
    LOG_WARN("fail to get schema guard", K(ret));
1052
  } else if (OB_FAIL(schema_guard.get_tenant_ids(tenant_ids))) {
1053
    LOG_WARN("fail to get tenant ids", K(ret));
1054
  } else {
1055
    standalone_tenant_array.reset();
1056
    for (int64_t i = 0; OB_SUCC(ret) && i < tenant_ids.count(); ++i) {
1057
      if (common::OB_SYS_TENANT_ID == tenant_ids.at(i)) {
1058
        // System tenant unit scheduling special processing,
1059
        // does not belong to any tenant group, nor is it standalone
1060
      } else if (has_exist_in_group_tenant_array(tenant_ids.at(i), group_tenant_array)) {
1061
        // Included in a tenant group, not standalone
1062
      } else if (OB_FAIL(standalone_tenant_array.push_back(tenant_ids.at(i)))) {
1063
        LOG_WARN("fail to push back", K(ret));
1064
      } else {} // next tenant id
1065
    }
1066
  }
1067
  return ret;
1068
}
1069

1070
int ObServerBalancer::generate_tenant_array(
1071
    const common::ObZone &zone,
1072
    common::ObIArray<Matrix<uint64_t> > &group_tenant_array,
1073
    common::ObIArray<uint64_t> &standalone_tenant_array)
1074
{
1075
  int ret = OB_SUCCESS;
1076
  if (OB_UNLIKELY(!inited_)) {
1077
    ret = OB_NOT_INIT;
1078
    LOG_WARN("not init", K(ret));
1079
  } else if (OB_FAIL(generate_standalone_tenant_array(
1080
          group_tenant_array, standalone_tenant_array))) {
1081
    LOG_WARN("fail to generate standalone tenant array", K(ret));
1082
  } else {} // no more to do
1083
  LOG_INFO("generate tenant array finish", K(ret), K(zone),
1084
           K(standalone_tenant_array), K(group_tenant_array));
1085
  return ret;
1086
}
1087

1088
int ObServerBalancer::fill_pool_units_in_zone(
1089
    const common::ObZone &zone,
1090
    share::ObResourcePool &pool,
1091
    common::ObIArray<ObUnitManager::ObUnitLoad> &unit_loads)
1092
{
1093
  int ret = OB_SUCCESS;
1094
  if (OB_UNLIKELY(!inited_)) {
1095
    ret = OB_NOT_INIT;
1096
    LOG_WARN("not init", K(ret));
1097
  } else if (OB_UNLIKELY(zone.is_empty())) {
1098
    ret = OB_INVALID_ARGUMENT;
1099
    LOG_WARN("invalid argument", K(ret), K(zone));
1100
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
1101
    ret = OB_ERR_UNEXPECTED;
1102
    LOG_WARN("unit_mgr_ ptr is null", K(ret));
1103
  } else {
1104
    ObArray<ObUnit *> *units = NULL;
1105
    ObUnitConfig *unit_config = NULL;
1106
    if (!has_exist_in_array(pool.zone_list_, zone)) {
1107
      // do not exist in pool zone list
1108
    } else if (OB_FAIL(unit_mgr_->get_units_by_pool(pool.resource_pool_id_, units))) {
1109
      LOG_WARN("fail to get units by pool", K(ret));
1110
    } else if (OB_UNLIKELY(NULL == units)) {
1111
      ret = OB_ERR_UNEXPECTED;
1112
      LOG_WARN("units ptr is null", K(ret), KP(units));
1113
    } else if (OB_FAIL(unit_mgr_->get_unit_config_by_id(pool.unit_config_id_, unit_config))) {
1114
      LOG_WARN("fail to get unit config by id", K(ret), "config_id", pool.unit_config_id_);
1115
    } else if (OB_UNLIKELY(NULL == unit_config)) {
1116
      ret = OB_ERR_UNEXPECTED;
1117
      LOG_WARN("unit_config ptr is null", K(ret));
1118
    } else {
1119
      for (int64_t j = 0; OB_SUCC(ret) && j < units->count(); ++j) {
1120
        ObUnitManager::ObUnitLoad unit_load;
1121
        unit_load.unit_ = units->at(j);
1122
        unit_load.unit_config_ = unit_config;
1123
        unit_load.pool_ = &pool;
1124
        if (NULL == units->at(j)) {
1125
          ret = OB_ERR_UNEXPECTED;
1126
          LOG_WARN("unit ptr is null", K(ret));
1127
        } else if (zone != units->at(j)->zone_) {
1128
          // do not belong to this zone
1129
        } else if (OB_FAIL(unit_loads.push_back(unit_load))) {
1130
          LOG_WARN("fail to push back", K(ret));
1131
        } else {} // no more to do
1132
      }
1133
    }
1134
  }
1135
  return ret;
1136
}
1137

1138
int ObServerBalancer::generate_not_grant_pool_units(
1139
    const common::ObZone &zone,
1140
    common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units)
1141
{
1142
  int ret = OB_SUCCESS;
1143
  if (OB_UNLIKELY(!inited_)) {
1144
    ret = OB_NOT_INIT;
1145
    LOG_WARN("not init", K(ret));
1146
  } else if (OB_UNLIKELY(zone.is_empty())) {
1147
    ret = OB_INVALID_ARGUMENT;
1148
    LOG_WARN("invalid argument", K(ret), K(zone));
1149
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
1150
    ret = OB_ERR_UNEXPECTED;
1151
    LOG_WARN("unit_mgr_ ptr is null", K(ret));
1152
  } else {
1153
    not_grant_units.reset();
1154
    ObHashMap<uint64_t, share::ObResourcePool *>::const_iterator iter;
1155
    for (iter = unit_mgr_->id_pool_map_.begin();
1156
         OB_SUCC(ret) && iter != unit_mgr_->id_pool_map_.end();
1157
         ++iter) {
1158
      share::ObResourcePool *pool = iter->second;
1159
      if (NULL == pool) {
1160
        ret = OB_ERR_UNEXPECTED;
1161
        LOG_WARN("pool is null", KP(pool), K(ret));
1162
      } else if (OB_INVALID_ID != pool->tenant_id_) {
1163
        // already grant to some tenant
1164
      } else if (OB_FAIL(fill_pool_units_in_zone(zone, *pool, not_grant_units))) {
1165
        LOG_WARN("fail to fill pool units in zone", K(ret));
1166
      } else {} // no more to do
1167
    }
1168
  }
1169
  return ret;
1170
}
1171

1172
int ObServerBalancer::generate_standalone_units(
1173
    const common::ObZone &zone,
1174
    const common::ObIArray<uint64_t> &standalone_tenant_array,
1175
    common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units)
1176
{
1177
   int ret = OB_SUCCESS;
1178
   if (OB_UNLIKELY(!inited_)) {
1179
     ret = OB_NOT_INIT;
1180
     LOG_WARN("not init", K(ret));
1181
   } else if (OB_UNLIKELY(zone.is_empty())) {
1182
     ret = OB_INVALID_ARGUMENT;
1183
     LOG_WARN("invalid argument", K(ret), K(zone));
1184
   } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
1185
     ret = OB_ERR_UNEXPECTED;
1186
     LOG_WARN("unit_mgr_ ptr is null", K(ret));
1187
   } else {
1188
     standalone_units.reset();
1189
     for (int64_t i = 0; OB_SUCC(ret) && i < standalone_tenant_array.count(); ++i) {
1190
       common::ObArray<share::ObResourcePool *> *pools = NULL;
1191
       const uint64_t tenant_id = standalone_tenant_array.at(i);
1192
       if (OB_UNLIKELY(OB_INVALID_ID == tenant_id)) {
1193
         ret = OB_ERR_UNEXPECTED;
1194
         LOG_WARN("tenant id unexpected", K(ret), K(tenant_id));
1195
       } else if (is_meta_tenant(tenant_id)) {
1196
         // meta tenant has no unit, do not handle
1197
       } else if (OB_FAIL(unit_mgr_->get_pools_by_tenant_(tenant_id, pools))) {
1198
         LOG_WARN("fail to get pools by tenant", K(ret), K(tenant_id));
1199
       } else if (OB_UNLIKELY(NULL == pools)) {
1200
         ret = OB_ERR_UNEXPECTED;
1201
         LOG_WARN("pools ptr is null", K(ret));
1202
       } else {
1203
         for (int64_t j = 0; OB_SUCC(ret) && j < pools->count(); ++j) {
1204
           share::ObResourcePool *pool = pools->at(j);
1205
           if (NULL == pool) {
1206
             ret = OB_ERR_UNEXPECTED;
1207
             LOG_WARN("pool is null", K(ret), KP(pool));
1208
           } else if (OB_FAIL(fill_pool_units_in_zone(zone, *pool, standalone_units))) {
1209
             LOG_WARN("fail to fill pool units in zone", K(ret));
1210
           } else {} // no more to do
1211
         }
1212
       }
1213
     }
1214
   }
1215
   return ret;
1216
}
1217

1218
int ObServerBalancer::generate_available_servers(
1219
    const common::ObZone &zone,
1220
    const bool excluded_sys_unit_server,
1221
    common::ObIArray<common::ObAddr> &available_servers)
1222
{
1223
  int ret = OB_SUCCESS;
1224
  HEAP_VAR(share::ObZoneInfo, zone_info) {
1225
    common::ObArray<common::ObAddr> server_list;
1226
    ObArray<ObAddr> sys_unit_server_array;
1227
    if (OB_UNLIKELY(!inited_)) {
1228
      ret = OB_NOT_INIT;
1229
      LOG_WARN("not init", K(ret));
1230
    } else if (OB_UNLIKELY(zone.is_empty())) {
1231
      ret = OB_INVALID_ARGUMENT;
1232
      LOG_WARN("invalid argument", K(ret), K(zone));
1233
    } else if (OB_ISNULL(zone_mgr_) || OB_ISNULL(unit_mgr_)) {
1234
      ret = OB_ERR_UNEXPECTED;
1235
      LOG_WARN("zone_mgr_ or unit_mgr_ is null", K(ret), KP(unit_mgr_), KP(zone_mgr_));
1236
    } else if (OB_FAIL(zone_mgr_->get_zone(zone, zone_info))) {
1237
      LOG_WARN("fail to get zone info", K(ret), K(zone));
1238
    } else if (ObZoneStatus::ACTIVE != zone_info.status_) {
1239
      ret = OB_STATE_NOT_MATCH;
1240
      LOG_WARN("zone is not in active", K(ret), K(zone_info));
1241
    } else if (OB_FAIL(SVR_TRACER.get_servers_of_zone(zone, server_list))) {
1242
      LOG_WARN("fail to get servers of zone", K(ret), K(zone));
1243
    } else if (OB_FAIL(unit_mgr_->get_tenant_unit_servers(
1244
            OB_SYS_TENANT_ID, zone, sys_unit_server_array))) {
1245
      LOG_WARN("fail to get tenant unit server array", K(ret));
1246
    } else {
1247
      available_servers.reset();
1248
      for (int64_t i = 0; OB_SUCC(ret) && i < server_list.count(); ++i) {
1249
        share::ObServerInfoInTable server_info;
1250
        if (OB_FAIL(SVR_TRACER.get_server_info(server_list.at(i), server_info))) {
1251
          LOG_WARN("fail to get server status", K(ret));
1252
        } else if (server_info.is_temporary_offline() || server_info.is_stopped()) {
1253
          ret = OB_STATE_NOT_MATCH;
1254
          LOG_WARN("server in zone is not stable, stop balance servers", K(ret), K(server_info),
1255
                   "is_temporary_offline", server_info.is_temporary_offline(),
1256
                   "is_stopped", server_info.is_stopped());
1257
        } else if (excluded_sys_unit_server
1258
                   && has_exist_in_array(sys_unit_server_array, server_list.at(i))) {
1259
          // bypass
1260
        } else if (server_info.is_active()) {
1261
          if (OB_FAIL(available_servers.push_back(server_list.at(i)))) {
1262
            LOG_WARN("fail to push back", K(ret));
1263
          }
1264
        } else {}
1265
        // Permanently offline and deleting servers are not available servers
1266
      }
1267
    }
1268
    LOG_INFO("generate available servers finish", K(ret), K(zone), K(available_servers));
1269
  }
1270
  return ret;
1271
}
1272

1273
int ObServerBalancer::check_need_balance_sys_tenant_units(
1274
    const common::ObIArray<ObUnitManager::ObUnitLoad> &sys_tenant_units,
1275
    const common::ObIArray<common::ObAddr> &available_servers,
1276
    bool &need_balance)
1277
{
1278
  int ret = OB_SUCCESS;
1279
  need_balance = false;
1280
  for (int64_t i = 0; OB_SUCC(ret) && !need_balance && i < sys_tenant_units.count(); ++i) {
1281
    const ObUnitManager::ObUnitLoad &unit_load = sys_tenant_units.at(i);
1282
    if (!unit_load.is_valid()) {
1283
      ret = OB_ERR_UNEXPECTED;
1284
      LOG_WARN("unit load invalid", K(ret), K(unit_load));
1285
    } else if (!has_exist_in_array(available_servers, unit_load.unit_->server_)) {
1286
      need_balance = true;
1287
      LOG_INFO("need to execute sys tenant units balance", "unit", *unit_load.unit_);
1288
    } else {} // go on to check next
1289
  }
1290
  return ret;
1291
}
1292

1293
int ObServerBalancer::pick_server_load(
1294
    const common::ObAddr &server,
1295
    ObIArray<ServerTotalLoad *> &server_load_ptrs_sorted,
1296
    ServerTotalLoad *&target_ptr)
1297
{
1298
  int ret = OB_SUCCESS;
1299
  if (OB_UNLIKELY(!inited_)) {
1300
    ret = OB_NOT_INIT;
1301
    LOG_WARN("not init", K(ret));
1302
  } else if (OB_UNLIKELY(!server.is_valid())) {
1303
    ret = OB_INVALID_ARGUMENT;
1304
    LOG_WARN("invalid argument", K(ret), K(server));
1305
  } else {
1306
    bool find = false;
1307
    for (int64_t i = 0; !find && i < server_load_ptrs_sorted.count() && OB_SUCC(ret); ++i) {
1308
      ServerTotalLoad *ptr = server_load_ptrs_sorted.at(i);
1309
      if (NULL == ptr) {
1310
        ret = OB_ERR_UNEXPECTED;
1311
        LOG_WARN("ptr is null", K(ret));
1312
      } else if (server != ptr->server_) {
1313
        // go on to check next
1314
      } else {
1315
        target_ptr = ptr;
1316
        find = true;
1317
      }
1318
    }
1319
    if (OB_FAIL(ret)) {
1320
    } else if (!find) {
1321
      ret = OB_ENTRY_NOT_EXIST;
1322
      LOG_WARN("server load not found", K(ret), K(server));
1323
    }
1324
  }
1325
  return ret;
1326
}
1327

1328
int ObServerBalancer::try_balance_single_unit_by_disk(
1329
    const ObUnitManager::ObUnitLoad &unit_load,
1330
    common::ObArray<ServerTotalLoad *> &server_load_ptrs_sorted,
1331
    common::ObIArray<UnitMigrateStat> &task_array,
1332
    const common::ObIArray<common::ObAddr> &excluded_servers,
1333
    common::ObIArray<PoolOccupation> &pool_occupation,
1334
    bool &do_balance)
1335
{
1336
  int ret = OB_SUCCESS;
1337
  ObUnitStat unit_stat;
1338
  if (OB_UNLIKELY(!inited_)) {
1339
    ret = OB_NOT_INIT;
1340
    LOG_WARN("not init", K(ret));
1341
  } else if (OB_UNLIKELY(!unit_load.is_valid())) {
1342
    ret = OB_INVALID_ARGUMENT;
1343
    LOG_WARN("invalid unit load", K(ret), K(unit_load));
1344
  } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
1345
          unit_load.unit_->unit_id_,
1346
          unit_load.unit_->zone_,
1347
          unit_stat))) {
1348
    LOG_WARN("fail to get unit stat", K(ret), "unit", *unit_load.unit_);
1349
  } else {
1350
    do_balance = false;
1351
    for (int64_t i = 0;
1352
         !do_balance
1353
          && OB_SUCC(ret)
1354
          && i < zone_disk_statistic_.server_disk_statistic_array_.count();
1355
         ++i) {
1356
      ServerTotalLoad *server_load = NULL;
1357
      ServerDiskStatistic &server_disk_statistic
1358
          = zone_disk_statistic_.server_disk_statistic_array_.at(i);
1359
      if (server_disk_statistic.wild_server_) {
1360
        // Can't move unit to wild server
1361
      } else if (unit_load.unit_->server_ == server_disk_statistic.server_) {
1362
        // Can't move to oneself
1363
      } else if (has_exist_in_array(excluded_servers, server_disk_statistic.server_)) {
1364
        // In the excluded server, it cannot be used as the target machine
1365
      } else if (OB_FAIL(pick_server_load(
1366
              server_disk_statistic.server_, server_load_ptrs_sorted, server_load))) {
1367
        LOG_WARN("fail to pick server load", K(ret));
1368
      } else if (NULL == server_load) {
1369
        ret = OB_ERR_UNEXPECTED;
1370
        LOG_WARN("server load ptr is null", K(ret), KP(server_load));
1371
      } else {
1372
        LoadSum this_load;
1373
        bool enough = true;
1374
        const bool mind_disk_waterlevel = false;
1375
        UnitMigrateStat unit_migrate_stat;
1376
        unit_migrate_stat.original_pos_ = unit_load.unit_->server_;
1377
        unit_migrate_stat.arranged_pos_ = server_load->server_;
1378
        unit_migrate_stat.unit_load_ = unit_load;
1379
        if (OB_FAIL(this_load.append_load(unit_load))) {
1380
          LOG_WARN("fail to append load", K(ret));
1381
        } else if (OB_FAIL(check_single_server_resource_enough(
1382
                this_load, unit_stat, *server_load, enough, mind_disk_waterlevel))) {
1383
          LOG_WARN("fail to check single server resource enough", K(ret));
1384
        } else if (!enough) {
1385
          // no enough
1386
        } else if (OB_FAIL(do_update_dst_server_load(
1387
                unit_load, *server_load, unit_stat, pool_occupation))) {
1388
          LOG_WARN("fail to append migrate unit task", K(ret));
1389
        } else if (OB_FAIL(task_array.push_back(unit_migrate_stat))) {
1390
          LOG_WARN("fail to push back", K(ret));
1391
        } else {
1392
          ServerTotalLoadCmp cmp;
1393
          std::sort(server_load_ptrs_sorted.begin(), server_load_ptrs_sorted.end(), cmp);
1394
          if (OB_FAIL(cmp.get_ret())) {
1395
            LOG_WARN("fail to sort server loads", K(ret));
1396
          }
1397
          do_balance = true;
1398
        }
1399
      }
1400
    }
1401
  }
1402
  return ret;
1403
}
1404

1405
int ObServerBalancer::do_update_src_server_load(
1406
    const ObUnitManager::ObUnitLoad &unit_load,
1407
    ServerTotalLoad &this_server_load,
1408
    const share::ObUnitStat &unit_stat)
1409
{
1410
  int ret = OB_SUCCESS;
1411
  share::ObUnit *unit = NULL;
1412
  share::ObUnitStat my_unit_stat = unit_stat;
1413
  if (OB_UNLIKELY(!inited_)) {
1414
    ret = OB_NOT_INIT;
1415
    LOG_WARN("not init", K(ret));
1416
  } else if (OB_UNLIKELY(!unit_load.is_valid())) {
1417
    ret = OB_INVALID_ARGUMENT;
1418
    LOG_WARN("invalid unit load", K(ret));
1419
  } else if (OB_FAIL(unit_mgr_->get_unit_by_id(unit_stat.get_unit_id(), unit))) {
1420
    LOG_WARN("fail to get unit by id", K(ret), "unit_id", unit_stat.get_unit_id());
1421
  } else if (OB_UNLIKELY(NULL == unit)) {
1422
    ret = OB_ERR_UNEXPECTED;
1423
    LOG_WARN("unit ptr is null", K(ret));
1424
  } else if (common::REPLICA_TYPE_LOGONLY == unit->replica_type_) {
1425
    ret = OB_ERR_UNEXPECTED;
1426
    LOG_WARN("Replica type LOGONLY is unexpected", KR(ret), KP(unit));
1427
  }
1428
  if (OB_FAIL(ret)) {
1429
  } else if (OB_FAIL(zone_disk_statistic_.reduce_server_disk_use(
1430
          this_server_load.server_, my_unit_stat.get_required_size()))) {
1431
    LOG_WARN("fail to reduce server disk use", K(ret));
1432
  } else if (OB_FAIL(this_server_load.load_sum_.remove_load(unit_load))) {
1433
    LOG_WARN("fail to remove load", K(ret));
1434
  } else if (OB_FAIL(this_server_load.update_load_value())) {
1435
    LOG_WARN("fail to update load value", K(ret));
1436
  } else {} // no more to do
1437
  return ret;
1438
}
1439

1440
int ObServerBalancer::do_update_dst_server_load(
1441
    const ObUnitManager::ObUnitLoad &unit_load,
1442
    ServerTotalLoad &this_server_load,
1443
    const share::ObUnitStat &unit_stat,
1444
    common::ObIArray<PoolOccupation> &pool_occupation)
1445
{
1446
  int ret = OB_SUCCESS;
1447
  share::ObUnit *unit = NULL;
1448
  share::ObUnitStat my_unit_stat = unit_stat;
1449
  if (OB_UNLIKELY(!inited_)) {
1450
    ret = OB_NOT_INIT;
1451
    LOG_WARN("not init", K(ret));
1452
  } else if (OB_UNLIKELY(!unit_load.is_valid())) {
1453
    ret = OB_INVALID_ARGUMENT;
1454
    LOG_WARN("invalid unit load", K(ret), K(unit_load));
1455
  } else if (OB_FAIL(unit_mgr_->get_unit_by_id(unit_stat.get_unit_id(), unit))) {
1456
    LOG_WARN("fail to get unit by id", K(ret), "unit_id", unit_stat.get_unit_id());
1457
  } else if (OB_UNLIKELY(NULL == unit)) {
1458
    ret = OB_ERR_UNEXPECTED;
1459
    LOG_WARN("unit ptr is null", K(ret));
1460
  } else if (common::REPLICA_TYPE_LOGONLY == unit->replica_type_) {
1461
    ret = OB_ERR_UNEXPECTED;
1462
    LOG_WARN("Replica type LOGONLY is unexpected", KR(ret), KP(unit));
1463
  }
1464
  if (OB_FAIL(ret)) {
1465
  } else if (OB_FAIL(pool_occupation.push_back(
1466
          PoolOccupation(unit_load.pool_->tenant_id_,
1467
                         unit_load.pool_->resource_pool_id_,
1468
                         this_server_load.server_)))) {
1469
    LOG_WARN("fail to push back", K(ret));
1470
  } else if (OB_FAIL(zone_disk_statistic_.raise_server_disk_use(
1471
          this_server_load.server_, my_unit_stat.get_required_size()))) {
1472
    LOG_WARN("fail to raise server disk use", K(ret));
1473
  } else if (OB_FAIL(this_server_load.load_sum_.append_load(unit_load))) {
1474
    LOG_WARN("fail to append load", K(ret));
1475
  } else if (OB_FAIL(this_server_load.update_load_value())) {
1476
    LOG_WARN("fail to update load value", K(ret));
1477
  } else {} // no more to do
1478
  return ret;
1479
}
1480

1481
int ObServerBalancer::try_balance_single_unit_by_cm(
1482
    const ObUnitManager::ObUnitLoad &unit_load,
1483
    common::ObArray<ServerTotalLoad *> &server_load_ptrs_sorted,
1484
    common::ObIArray<UnitMigrateStat> &task_array,
1485
    const common::ObIArray<common::ObAddr> &excluded_servers,
1486
    common::ObIArray<PoolOccupation> &pool_occupation,
1487
    bool &do_balance)
1488
{
1489
  int ret = OB_SUCCESS;
1490
  ObUnitStat unit_stat;
1491
  if (OB_UNLIKELY(!inited_)) {
1492
    ret = OB_NOT_INIT;
1493
    LOG_WARN("not init", K(ret));
1494
  } else if (OB_UNLIKELY(!unit_load.is_valid())) {
1495
    ret = OB_INVALID_ARGUMENT;
1496
    LOG_WARN("invalid argument", K(ret), K(unit_load));
1497
  } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
1498
          unit_load.unit_->unit_id_,
1499
          unit_load.unit_->zone_,
1500
          unit_stat))) {
1501
    LOG_WARN("fail to get unit stat", K(ret), "unit", *unit_load.unit_);
1502
  } else {
1503
    do_balance = false;
1504
    for (int64_t j = server_load_ptrs_sorted.count() - 1;
1505
         !do_balance && OB_SUCC(ret) && j >= 0;
1506
         --j) {
1507
      ServerTotalLoad *server_load = server_load_ptrs_sorted.at(j);
1508
      if (OB_UNLIKELY(NULL == server_load)) {
1509
        ret = OB_ERR_UNEXPECTED;
1510
        LOG_WARN("server load ptr is null", K(ret), KP(server_load));
1511
      } else {
1512
        LoadSum this_load;
1513
        bool enough = true;
1514
        UnitMigrateStat unit_migrate_stat;
1515
        unit_migrate_stat.original_pos_ = unit_load.unit_->server_;
1516
        unit_migrate_stat.arranged_pos_ = server_load->server_;
1517
        unit_migrate_stat.unit_load_ = unit_load;
1518
        if (server_load->wild_server_) {
1519
          // Can't move unit to wild server
1520
        } else if (server_load->server_ == unit_load.unit_->server_) {
1521
          // Don't move to oneself
1522
        } else if (has_exist_in_array(excluded_servers, server_load->server_)) {
1523
          // it cannot be used as the target machine in the excluded server
1524
        } else if (OB_FAIL(this_load.append_load(unit_load))) {
1525
          LOG_WARN("fail to append load", K(ret));
1526
        } else if (OB_FAIL(check_single_server_resource_enough(
1527
                this_load, unit_stat, *server_load, enough))) {
1528
          LOG_WARN("fail to check server resource", K(ret));
1529
        } else if (!enough) {
1530
          // The resources of this machine are not enough to support the move of this unit
1531
        } else if (OB_FAIL(do_update_dst_server_load(
1532
                unit_load, *server_load, unit_stat, pool_occupation))) {
1533
          LOG_WARN("fail to append migrate unit task", K(ret));
1534
        } else if (OB_FAIL(task_array.push_back(unit_migrate_stat))) {
1535
          LOG_WARN("fail to push back", K(ret));
1536
        } else {
1537
          ServerTotalLoadCmp cmp;
1538
          std::sort(server_load_ptrs_sorted.begin(), server_load_ptrs_sorted.end(), cmp);
1539
          if (OB_FAIL(cmp.get_ret())) {
1540
            LOG_WARN("fail to sort server loads", K(ret));
1541
          }
1542
          do_balance = true;
1543
        }
1544
      }
1545
    }
1546
  }
1547
  return ret;
1548
}
1549

1550
int ObServerBalancer::do_balance_sys_tenant_single_unit(
1551
    const ObUnitManager::ObUnitLoad &unit_load,
1552
    common::ObArray<ServerTotalLoad *> &server_load_ptrs_sorted,
1553
    common::ObIArray<UnitMigrateStat> &task_array,
1554
    common::ObIArray<PoolOccupation> &pool_occupation)
1555
{
1556
  int ret = OB_SUCCESS;
1557
  bool do_balance = false;
1558
  ObArray<common::ObAddr> excluded_servers;
1559
  if (OB_UNLIKELY(!inited_)) {
1560
    ret = OB_NOT_INIT;
1561
    LOG_WARN("not init", K(ret));
1562
  } else if (OB_UNLIKELY(!unit_load.is_valid())) {
1563
    ret = OB_INVALID_ARGUMENT;
1564
    LOG_WARN("invalid argument", K(ret), K(unit_load));
1565
  } else if (OB_FAIL(get_pool_occupation_excluded_dst_servers(
1566
          unit_load, pool_occupation, excluded_servers))) {
1567
    LOG_WARN("fail to get excluded servers", K(ret));
1568
  } else if (OB_FAIL(try_balance_single_unit_by_cm(
1569
          unit_load, server_load_ptrs_sorted, task_array,
1570
          excluded_servers, pool_occupation, do_balance))) {
1571
    LOG_WARN("fail to try balance sys single unit by cpu and memory", K(ret));
1572
  } else if (do_balance) {
1573
    // do execute balance according to cpu and memory, no more to do
1574
  } else if (OB_FAIL(try_balance_single_unit_by_disk(
1575
          unit_load, server_load_ptrs_sorted, task_array,
1576
          excluded_servers, pool_occupation, do_balance))) {
1577
    LOG_WARN("fail to do balance sys single unit by disk", K(ret));
1578
  } else if (do_balance) {
1579
    // good,
1580
  } else {
1581
    ret = OB_MACHINE_RESOURCE_NOT_ENOUGH;
1582
    LOG_WARN("no available server to hold more unit", K(ret),
1583
             "unit", *unit_load.unit_,
1584
             "unit_config", *unit_load.unit_config_,
1585
             "pool", *unit_load.pool_);
1586
  }
1587
  return ret;
1588
}
1589

1590
int ObServerBalancer::do_balance_sys_tenant_units(
1591
    const common::ObIArray<ObUnitManager::ObUnitLoad> &sys_tenant_units,
1592
    const common::ObIArray<common::ObAddr> &available_servers,
1593
    common::ObArray<ServerTotalLoad> &server_loads)
1594
{
1595
  int ret = OB_SUCCESS;
1596
  ObArray<PoolOccupation> pool_occupation;
1597
  if (OB_UNLIKELY(!inited_)) {
1598
    ret = OB_NOT_INIT;
1599
    LOG_WARN("not init", K(ret));
1600
  } else if (OB_FAIL(generate_pool_occupation_array(sys_tenant_units, pool_occupation))) {
1601
    LOG_WARN("fail to generate pool occupation array", K(ret));
1602
  } else {
1603
    ObArray<ServerTotalLoad *> server_load_ptrs_sorted;
1604
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
1605
      if (OB_FAIL(server_load_ptrs_sorted.push_back(&server_loads.at(i)))) {
1606
        LOG_WARN("fail to push back", K(ret));
1607
      }
1608
    }
1609
    if (OB_SUCC(ret)) {
1610
      ServerTotalLoadCmp cmp;
1611
      std::sort(server_load_ptrs_sorted.begin(), server_load_ptrs_sorted.end(), cmp);
1612
      if (OB_FAIL(cmp.get_ret())) {
1613
        LOG_WARN("fail to sort", K(ret));
1614
      }
1615
    }
1616
    common::ObArray<UnitMigrateStat> task_array;
1617
    for (int64_t i = 0; OB_SUCC(ret) && i < sys_tenant_units.count(); ++i) {
1618
      const ObUnitManager::ObUnitLoad &unit_load = sys_tenant_units.at(i);
1619
      if (OB_UNLIKELY(!unit_load.is_valid())) {
1620
        ret = OB_ERR_UNEXPECTED;
1621
        LOG_WARN("invalid unit load", K(ret), K(unit_load));
1622
      } else if (has_exist_in_array(available_servers, unit_load.unit_->server_)) {
1623
        // skip in available server
1624
      } else if (OB_FAIL(do_balance_sys_tenant_single_unit(
1625
              unit_load, server_load_ptrs_sorted, task_array, pool_occupation))) {
1626
        LOG_WARN("fail to do balance sys tenant single unit", K(ret));
1627
      } else {} // no more to do
1628
    }
1629
    if (OB_FAIL(ret)) {
1630
    } else if (task_array.count() <= 0) {
1631
    } else if (OB_FAIL(check_and_do_migrate_unit_task_(task_array))) {
1632
      LOG_WARN("fail to check and do migrate unit task", KR(ret), K(task_array));
1633
    } else {} // no more to do
1634
  }
1635
  return ret;
1636
}
1637

1638
int ObServerBalancer::try_balance_sys_tenant_units(
1639
    const common::ObZone &zone,
1640
    bool &do_execute)
1641
{
1642
  int ret = OB_SUCCESS;
1643
  common::ObArray<ObUnitManager::ObUnitLoad> sys_tenant_units;
1644
  common::ObArray<common::ObAddr> available_servers;
1645
  do_execute = false;
1646
  bool need_balance = false;
1647
  double g_res_weights[RES_MAX];
1648
  ObArray<ServerTotalLoad> server_loads;
1649
  if (OB_UNLIKELY(!inited_)) {
1650
    ret = OB_NOT_INIT;
1651
    LOG_WARN("not init", K(ret));
1652
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
1653
    ret = OB_ERR_UNEXPECTED;
1654
    LOG_WARN("unit_mgr_ ptr is null", K(ret));
1655
  } else if (OB_UNLIKELY(zone.is_empty())) {
1656
    ret = OB_INVALID_ARGUMENT;
1657
    LOG_WARN("invalid argument", K(ret), K(zone));
1658
  } else if (OB_FAIL(generate_available_servers(
1659
          zone, false /* sys unit not excluded*/ , available_servers))) {
1660
    LOG_WARN("fail to generate available servers", K(ret), K(zone));
1661
  } else if (available_servers.count() <= 0) {
1662
    LOG_INFO("no available servers, bypass", K(zone));
1663
  } else if (OB_FAIL(unit_mgr_->get_tenant_zone_all_unit_loads(
1664
          common::OB_SYS_TENANT_ID, zone, sys_tenant_units))) {
1665
    if (OB_ENTRY_NOT_EXIST == ret) {
1666
      ret = OB_SUCCESS;
1667
    } else {
1668
      LOG_WARN("fail to get tenant zone unit loads", K(ret), K(zone));
1669
    }
1670
  } else if (OB_FAIL(check_need_balance_sys_tenant_units(
1671
          sys_tenant_units, available_servers, need_balance))) {
1672
    LOG_WARN("fail to check need balance sys tenant units", K(ret));
1673
  } else if (!need_balance) {
1674
    do_execute = false;
1675
  } else if (OB_FAIL(calc_global_balance_resource_weights(
1676
         zone, available_servers, g_res_weights, RES_MAX))) {
1677
    LOG_WARN("fail to calc whole balance resource weights", K(ret));
1678
  } else if (OB_FAIL(generate_complete_server_loads(
1679
          zone, available_servers, g_res_weights, RES_MAX, server_loads))) {
1680
    LOG_WARN("fail to generate complete server loads", K(ret));
1681
  } else if (OB_FAIL(do_balance_sys_tenant_units(
1682
          sys_tenant_units, available_servers, server_loads))) {
1683
    LOG_WARN("fail to do balance sys tenant units", K(ret));
1684
  } else {
1685
    do_execute = true;
1686
  }
1687
  return ret;
1688
}
1689

1690
int ObServerBalancer::try_balance_non_sys_tenant_units(
1691
    const common::ObZone &zone,
1692
    const common::ObIArray<Matrix<uint64_t> > &group_tenant_array,
1693
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
1694
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units)
1695
{
1696
  int ret = OB_SUCCESS;
1697
  ObArray<TenantGroupBalanceInfo> balance_info_array;
1698
  ObArray<Matrix<uint64_t> > degraded_tenant_group_array;
1699
  common::ObArray<common::ObAddr> available_servers;
1700
  const bool enable_sys_unit_standalone = GCONF.enable_sys_unit_standalone;
1701
  if (OB_UNLIKELY(!inited_)) {
1702
    ret = OB_NOT_INIT;
1703
    LOG_WARN("not init", K(ret));
1704
  } else if (OB_UNLIKELY(zone.is_empty())) {
1705
    ret = OB_INVALID_ARGUMENT;
1706
    LOG_WARN("invalid argument", K(ret), K(zone), K(available_servers));
1707
  } else if (OB_FAIL(balance_info_array.reserve(2 * group_tenant_array.count()))) {
1708
    LOG_WARN("ObArray fail to reserve", K(ret), "capacity", group_tenant_array.count());
1709
  } else if (OB_FAIL(generate_tenant_balance_info_array(
1710
          zone, group_tenant_array, degraded_tenant_group_array, balance_info_array))) {
1711
    LOG_WARN("fail to generate tenant balance info array", K(ret));
1712
  } else if (OB_FAIL(generate_available_servers(
1713
          zone, enable_sys_unit_standalone, available_servers))) {
1714
    LOG_WARN("fail to generate available servers", K(ret));
1715
  } else if (available_servers.count() <= 0) {
1716
    LOG_INFO("no available servers, bypass", K(zone));
1717
  } else {
1718
    for (int64_t i = 0; OB_SUCC(ret) && i < balance_info_array.count(); ++i) {
1719
      TenantGroupBalanceInfo &balance_info = balance_info_array.at(i);
1720
      if (OB_UNLIKELY(NULL == balance_info.tenant_id_matrix_)) {
1721
        ret = OB_ERR_UNEXPECTED;
1722
        LOG_WARN("tenant id matrix in balance info is null", K(ret));
1723
      } else if (OB_FAIL(generate_original_unit_matrix(
1724
              *balance_info.tenant_id_matrix_, zone, balance_info.column_unit_num_array_,
1725
              balance_info.unit_migrate_stat_matrix_))) {
1726
        LOG_WARN("fail to generate original unit matrix", K(ret));
1727
      } else if (OB_FAIL(single_unit_migrate_stat_matrix_balance(
1728
              balance_info, available_servers))) {
1729
        LOG_WARN("fail to do migrate stat matrix balance", K(ret));
1730
      } else {
1731
        LOG_INFO("finish single ttg balance",
1732
                 "tenant_id_matrix", *balance_info.tenant_id_matrix_);
1733
      }
1734
    }
1735
    if (OB_FAIL(ret)) {
1736
    } else if (OB_FAIL(try_amend_and_execute_unit_balance_task(
1737
            zone, balance_info_array, standalone_units,
1738
            not_grant_units, available_servers))) {
1739
      LOG_WARN("fail to do global unit balance", K(ret));
1740
    } else {} // no more to do
1741
  }
1742
  return ret;
1743
}
1744

1745
int ObServerBalancer::generate_tenant_balance_info_array(
1746
    const common::ObZone &zone,
1747
    const common::ObIArray<Matrix<uint64_t> > &group_tenant_array,
1748
    common::ObIArray<Matrix<uint64_t> > &degraded_tenant_group_array,
1749
    common::ObIArray<TenantGroupBalanceInfo> &balance_info_array)
1750
{
1751
  int ret = OB_SUCCESS;
1752
  ObArray<const Matrix<uint64_t> *> unit_num_not_match_array;
1753
  if (OB_UNLIKELY(!inited_)) {
1754
    ret = OB_NOT_INIT;
1755
    LOG_WARN("not init", K(ret));
1756
  } else if (OB_UNLIKELY(zone.is_empty())) {
1757
    ret = OB_INVALID_ARGUMENT;
1758
    LOG_WARN("invalid argument", K(ret), K(zone));
1759
  } else {
1760
    for (int64_t i = 0; OB_SUCC(ret) && i < group_tenant_array.count(); ++i) {
1761
      bool unit_num_match = false;
1762
      TenantGroupBalanceInfo balance_info;
1763
      balance_info.tenant_id_matrix_ = &group_tenant_array.at(i);
1764
      if (OB_FAIL(check_and_get_tenant_matrix_unit_num(
1765
              group_tenant_array.at(i), zone, unit_num_match,
1766
              balance_info.column_unit_num_array_))) {
1767
        LOG_WARN("fail to check and get tenant matrix unit num", K(ret));
1768
      } else if (!unit_num_match) {
1769
        // Suppose the tenant group has m rows and n columns.
1770
        // When the unit num of the tenant group does not match,
1771
        // the tenant group will degenerate into m tenant groups with 1 row and n columns.
1772
        if (OB_FAIL(unit_num_not_match_array.push_back(&group_tenant_array.at(i)))) {
1773
          LOG_WARN("fail to push back", K(ret));
1774
        }
1775
      } else if (balance_info.column_unit_num_array_.count() > 0) {
1776
        if (OB_FAIL(balance_info_array.push_back(balance_info))) {
1777
          LOG_WARN("fail to push back", K(ret));
1778
        }
1779
      } else {} // no tenant of this group exists in this zone
1780
    }
1781
    if (OB_FAIL(ret)) {
1782
    } else if (OB_FAIL(try_generate_degraded_balance_info_array(
1783
            zone, unit_num_not_match_array, degraded_tenant_group_array, balance_info_array))) {
1784
      LOG_WARN("fail to try generate degraded balance info array", K(ret));
1785
    } else {} // no more to do
1786
  }
1787
  return ret;
1788
}
1789

1790
int ObServerBalancer::do_degrade_tenant_group_matrix(
1791
    const Matrix<uint64_t> &source_tenant_group,
1792
    common::ObIArray<Matrix<uint64_t> > &degraded_tenant_group_array)
1793
{
1794
  int ret = OB_SUCCESS;
1795
  if (OB_UNLIKELY(!inited_)) {
1796
    ret = OB_NOT_INIT;
1797
    LOG_WARN("not init", K(ret));
1798
  } else {
1799
    LOG_INFO("start degrade tenant group matrix", K(source_tenant_group));
1800
    for (int64_t i = 0; OB_SUCC(ret) && i < source_tenant_group.get_row_count(); ++i) {
1801
      Matrix<uint64_t> tenant_id_vector;
1802
      const int64_t VECTOR_ROW_CNT = 1;
1803
      const int64_t VECTOR_ROW_IDX = 0;
1804
      if (OB_FAIL(tenant_id_vector.init(VECTOR_ROW_CNT, source_tenant_group.get_column_count()))) {
1805
        LOG_WARN("fail to init tenant id vector", K(ret));
1806
      } else {
1807
        for (int64_t j = 0; OB_SUCC(ret) && j < source_tenant_group.get_column_count(); ++j) {
1808
          uint64_t tenant_id = OB_INVALID_ID;
1809
          if (OB_FAIL(source_tenant_group.get(i, j, tenant_id))) {
1810
            LOG_WARN("fail to get tenant id", K(ret), K(i), K(j));
1811
          } else if (OB_UNLIKELY(OB_INVALID_ID == tenant_id)) {
1812
            ret = OB_ERR_UNEXPECTED;
1813
            LOG_WARN("invalid tenant id", K(ret), K(tenant_id));
1814
          } else if (OB_FAIL(tenant_id_vector.set(VECTOR_ROW_IDX, j, tenant_id))) {
1815
            LOG_WARN("fail to set tenant id vector", K(ret), K(j), K(tenant_id));
1816
          } else {} // no more to do
1817
        }
1818
        if (OB_FAIL(ret)) {
1819
        } else if (OB_FAIL(degraded_tenant_group_array.push_back(tenant_id_vector))) {
1820
          LOG_WARN("fail to push back", K(ret));
1821
        } else {
1822
          LOG_INFO("output vector tenant group", K(i), "vector_ttg", tenant_id_vector);
1823
        }
1824
      }
1825
    }
1826
  }
1827
  return ret;
1828
}
1829

1830
int ObServerBalancer::try_generate_degraded_balance_info_array(
1831
    const common::ObZone &zone,
1832
    const common::ObIArray<const Matrix<uint64_t> *> &unit_num_not_match_array,
1833
    common::ObIArray<Matrix<uint64_t> > &degraded_tenant_group_array,
1834
    common::ObIArray<TenantGroupBalanceInfo> &balance_info_array)
1835
{
1836
  int ret = OB_SUCCESS;
1837
  if (OB_UNLIKELY(!inited_)) {
1838
    ret = OB_NOT_INIT;
1839
    LOG_WARN("not init", K(ret));
1840
  } else if (OB_UNLIKELY(zone.is_empty())) {
1841
    ret = OB_INVALID_ARGUMENT;
1842
    LOG_WARN("invalid argument", K(ret), K(zone));
1843
  } else {
1844
    for (int64_t i = 0; OB_SUCC(ret) && i < unit_num_not_match_array.count(); ++i) {
1845
      const Matrix<uint64_t> *this_tenant_group = unit_num_not_match_array.at(i);
1846
      if (OB_UNLIKELY(NULL == this_tenant_group)) {
1847
        ret = OB_ERR_UNEXPECTED;
1848
        LOG_WARN("this tenant group ptr is null", K(ret));
1849
      } else if (OB_FAIL(do_degrade_tenant_group_matrix(
1850
              *this_tenant_group, degraded_tenant_group_array))) {
1851
        LOG_WARN("fail to do degraded tenant group matrix", K(ret), "source", *this_tenant_group);
1852
      } else {} // no more to do
1853
    }
1854
    for (int64_t i = 0; OB_SUCC(ret) && i < degraded_tenant_group_array.count(); ++i) {
1855
      bool unit_num_match = false;
1856
      TenantGroupBalanceInfo balance_info;
1857
      balance_info.tenant_id_matrix_ = &degraded_tenant_group_array.at(i);
1858
      if (OB_FAIL(check_and_get_tenant_matrix_unit_num(
1859
              degraded_tenant_group_array.at(i), zone, unit_num_match,
1860
              balance_info.column_unit_num_array_))) {
1861
        LOG_WARN("fail to check and get tenant matrix unit num", K(ret));
1862
      } else if (!unit_num_match) {
1863
        ret = OB_ERR_UNEXPECTED;
1864
        LOG_WARN("unit num not match unexpected, since this tenant group is a vector",
1865
                 K(ret), "vector tenant group", degraded_tenant_group_array.at(i));
1866
      } else if (balance_info.column_unit_num_array_.count() > 0) {
1867
        if (OB_FAIL(balance_info_array.push_back(balance_info))) {
1868
          LOG_WARN("fail to push back", K(ret));
1869
        }
1870
      } else {} // no tenant of this group exists in this zone
1871
    }
1872
  }
1873
  return ret;
1874
}
1875

1876
int ObServerBalancer::do_rebalance_servers_v2(
1877
    const common::ObZone &zone,
1878
    const common::ObIArray<Matrix<uint64_t> > &group_tenant_array,
1879
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
1880
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units)
1881
{
1882
  int ret = OB_SUCCESS;
1883
  bool do_sys_unit_balanced = false;
1884
  if (OB_UNLIKELY(!inited_)) {
1885
    ret = OB_NOT_INIT;
1886
    LOG_WARN("not init", K(ret));
1887
  } else if (OB_UNLIKELY(zone.is_empty())) {
1888
    ret = OB_INVALID_ARGUMENT;
1889
    LOG_WARN("invalid argument", K(ret), K(zone));
1890
  } else if (OB_FAIL(try_balance_sys_tenant_units(zone, do_sys_unit_balanced))) {
1891
    LOG_WARN("fail to try balance sys tenant units", K(ret));
1892
  } else if (do_sys_unit_balanced) {
1893
    // bypass, first balance sys unit
1894
  } else if (OB_FAIL(try_balance_non_sys_tenant_units(
1895
          zone, group_tenant_array, standalone_units, not_grant_units))) {
1896
    LOG_WARN("fail to try balance non sys tenant units", K(ret));
1897
  }
1898
  return ret;
1899
}
1900

1901
int ObServerBalancer::try_amend_and_execute_unit_balance_task(
1902
    const common::ObZone &zone,
1903
    common::ObIArray<TenantGroupBalanceInfo> &balance_info_array,
1904
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
1905
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units,
1906
    const common::ObIArray<common::ObAddr> &available_servers)
1907
{
1908
  int ret = OB_SUCCESS;
1909
  common::ObArray<TenantGroupBalanceInfo *> stable_tenant_group;
1910
  common::ObArray<TenantGroupBalanceInfo *> unstable_tenant_group;
1911
  bool do_amend_unstable_ttg = false;
1912
  bool do_amend_stable_ttg = false;
1913
  if (OB_UNLIKELY(!inited_)) {
1914
    ret = OB_NOT_INIT;
1915
    LOG_WARN("not init", K(ret));
1916
  } else if (OB_UNLIKELY(available_servers.count() <= 0)) {
1917
    ret = OB_INVALID_ARGUMENT;
1918
    LOG_WARN("invalid argument", K(ret));
1919
  } else if (OB_FAIL(divide_tenantgroup_balance_info(
1920
          balance_info_array, stable_tenant_group, unstable_tenant_group))) {
1921
    LOG_WARN("fail to divide tenantgroup balance info", K(ret));
1922
  } else if (OB_FAIL(try_amend_and_execute_stable_tenantgroup(
1923
          zone, unstable_tenant_group, stable_tenant_group, standalone_units,
1924
          not_grant_units, available_servers, do_amend_stable_ttg))) {
1925
    LOG_WARN("fail to amend and execute stable tenantgroup", K(ret));
1926
  } else if (do_amend_stable_ttg) {
1927
    // The stable tenant group has been adjusted between groups and no further operations will be performed
1928
  } else {
1929
    if (unstable_tenant_group.count() > 0) {
1930
      if (OB_FAIL(try_amend_and_execute_unstable_tenantgroup(
1931
              zone, unstable_tenant_group, stable_tenant_group, standalone_units,
1932
              not_grant_units, available_servers, do_amend_unstable_ttg))) {
1933
        LOG_WARN("fail to do amend and execute unit balance info", K(ret));
1934
      }
1935
    } else {} // do as follows
1936

1937
    if (OB_FAIL(ret)) {
1938
    } else if (unstable_tenant_group.count() <= 0 || !do_amend_unstable_ttg) {
1939
      bool do_balance_disk = false;
1940
      if (OB_FAIL(try_balance_disk_by_stable_tenantgroup(
1941
              zone, stable_tenant_group, available_servers, do_balance_disk))) {
1942
        LOG_WARN("fail to do disk balance", K(ret));
1943
      } else if (do_balance_disk) {
1944
        // Disk adjustment is made, and no need to do non-tenantgroup balance
1945
      } else if (OB_FAIL(do_non_tenantgroup_unit_balance_task(
1946
              zone, standalone_units, not_grant_units, available_servers))) {
1947
        LOG_WARN("fail to do non tenantgroup unit balance task", K(ret));
1948
      }
1949
    }
1950
  }
1951
  return ret;
1952
}
1953

1954
int ObServerBalancer::generate_simple_ug_loads(
1955
    common::ObIArray<SimpleUgLoad> &simple_ug_loads,
1956
    TenantGroupBalanceInfo &balance_info)
1957
{
1958
  int ret = OB_SUCCESS;
1959
  if (OB_UNLIKELY(!inited_)) {
1960
    ret = OB_NOT_INIT;
1961
    LOG_WARN("not init", K(ret));
1962
  } else if (balance_info.unitgroup_load_array_.count()
1963
             != balance_info.unit_migrate_stat_matrix_.get_column_count()) {
1964
    ret = OB_ERR_UNEXPECTED;
1965
    LOG_WARN("unitgroup load count and unit migrate stat matrix column not match", K(ret),
1966
             "unitgroup_count", balance_info.unitgroup_load_array_.count(),
1967
             "unit_matrix_column", balance_info.unit_migrate_stat_matrix_.get_column_count());
1968
  } else if (balance_info.unit_migrate_stat_matrix_.get_row_count() <= 0) {
1969
    ret = OB_ERR_UNEXPECTED;
1970
    LOG_WARN("unit matrix row unexpected", K(ret),
1971
             "unit_matrix_row", balance_info.unit_migrate_stat_matrix_.get_row_count());
1972
  } else {
1973
    simple_ug_loads.reset();
1974
    for (int64_t column = 0;
1975
         OB_SUCC(ret) && column < balance_info.unit_migrate_stat_matrix_.get_column_count();
1976
         ++column) {
1977
      UnitGroupLoad &unitgroup_load = balance_info.unitgroup_load_array_.at(column);
1978
      UnitMigrateStat unit_migrate_stat;
1979
      if (OB_FAIL(balance_info.unit_migrate_stat_matrix_.get(0, column, unit_migrate_stat))) {
1980
        LOG_WARN("fail to get from matrix", K(ret));
1981
      } else {
1982
        SimpleUgLoad simple_ug_load;
1983
        simple_ug_load.column_tenant_id_ = unitgroup_load.column_tenant_id_;
1984
        simple_ug_load.original_server_ = unit_migrate_stat.original_pos_;
1985
        simple_ug_load.arranged_server_ = unit_migrate_stat.arranged_pos_;
1986
        simple_ug_load.load_sum_ = unitgroup_load.load_sum_;
1987
        if (OB_FAIL(simple_ug_loads.push_back(simple_ug_load))) {
1988
          LOG_WARN("fail to push back", K(ret));
1989
        }
1990
      }
1991
    }
1992
  }
1993
  return ret;
1994
}
1995

1996
int ObServerBalancer::get_ug_exchange_excluded_dst_servers(
1997
    const int64_t ug_idx,
1998
    const Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
1999
    common::ObIArray<common::ObAddr> &excluded_servers)
2000
{
2001
  int ret = OB_SUCCESS;
2002
  excluded_servers.reset();
2003
  common::ObArray<ObUnitManager::ObUnitLoad> zone_units;
2004
  if (OB_UNLIKELY(!inited_)) {
2005
    ret = OB_NOT_INIT;
2006
    LOG_WARN("not init", K(ret));
2007
  } else if (OB_UNLIKELY(ug_idx >= unit_migrate_stat_matrix.get_column_count())) {
2008
    ret = OB_INVALID_ARGUMENT;
2009
    LOG_WARN("invalid argument", K(ret), K(ug_idx),
2010
             "unit migrate stat matrix column", unit_migrate_stat_matrix.get_column_count());
2011
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
2012
    ret = OB_ERR_UNEXPECTED;
2013
    LOG_WARN("unit_mgr_ ptr is null", K(ret));
2014
  }
2015
  for (int64_t row = 0; OB_SUCC(ret) && row < unit_migrate_stat_matrix.get_row_count(); ++row) {
2016
    zone_units.reset();
2017
    UnitMigrateStat unit_migrate_stat;
2018
    if (OB_FAIL(unit_migrate_stat_matrix.get(row, ug_idx, unit_migrate_stat))) {
2019
      LOG_WARN("fail to get unit migrate stat", K(ret));
2020
    } else if (OB_FAIL(unit_mgr_->get_tenant_zone_all_unit_loads(
2021
            unit_migrate_stat.tenant_id_, zone_disk_statistic_.zone_, zone_units))) {
2022
      LOG_WARN("fail to get tenant zone all unit loads", K(ret), K(unit_migrate_stat));
2023
    } else {
2024
      for (int64_t i = 0; OB_SUCC(ret) && i < zone_units.count(); ++i) {
2025
        ObUnitManager::ObUnitLoad &unit_load = zone_units.at(i);
2026
        if (!unit_load.is_valid()) {
2027
          ret = OB_ERR_UNEXPECTED;
2028
          LOG_WARN("unit load is invalid", K(ret), K(unit_load));
2029
        }
2030

2031
        if (OB_FAIL(ret)) {
2032
        } else if (has_exist_in_array(excluded_servers, unit_load.unit_->server_)) {
2033
          // already exist in excluded servers
2034
        } else if (OB_FAIL(excluded_servers.push_back(unit_load.unit_->server_))) {
2035
          LOG_WARN("fail to push back", K(ret));
2036
        }
2037

2038
        if (OB_FAIL(ret)) {
2039
        } else if (!unit_load.unit_->migrate_from_server_.is_valid()) {
2040
          // migrate from invalid, no need to put into excluded servers
2041
        } else if (has_exist_in_array(excluded_servers, unit_load.unit_->migrate_from_server_)) {
2042
          // already exist in excluded servers
2043
        } else if (OB_FAIL(excluded_servers.push_back(unit_load.unit_->migrate_from_server_))) {
2044
          LOG_WARN("fail to push back", K(ret));
2045
        }
2046
      }
2047
    }
2048
  }
2049
  return ret;
2050
}
2051

2052
int ObServerBalancer::check_cm_resource_enough(
2053
    const LoadSum &this_load,
2054
    const ServerTotalLoad &server_load,
2055
    bool &enough)
2056
{
2057
  int ret = OB_SUCCESS;
2058
  double hard_limit = 0.0;
2059
  if (OB_UNLIKELY(!inited_)) {
2060
    ret = OB_NOT_INIT;
2061
    LOG_WARN("not init", K(ret));
2062
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
2063
    ret = OB_ERR_UNEXPECTED;
2064
    LOG_WARN("unit_mgr_ ptr is null", K(ret));
2065
  } else if (OB_FAIL(unit_mgr_->get_hard_limit(hard_limit))) {
2066
    LOG_WARN("fail to get hard limit", K(ret));
2067
  } else {
2068
    enough = ((this_load.load_sum_.max_cpu() + server_load.load_sum_.load_sum_.max_cpu()
2069
                <= server_load.resource_info_.cpu_ * hard_limit)
2070
              && (this_load.load_sum_.min_cpu() + server_load.load_sum_.load_sum_.min_cpu()
2071
                <= server_load.resource_info_.cpu_)
2072
              && (static_cast<double>(this_load.load_sum_.memory_size())
2073
                  + static_cast<double>(server_load.load_sum_.load_sum_.memory_size())
2074
                <= static_cast<double>(server_load.resource_info_.mem_total_))
2075
              && (static_cast<double>(this_load.load_sum_.log_disk_size())
2076
                  + static_cast<double>(server_load.load_sum_.load_sum_.log_disk_size())
2077
                <= static_cast<double>(server_load.resource_info_.log_disk_total_)));
2078

2079
    LOG_INFO("[SERVER_BALANCE] check_cm_resource_enough", K(enough), K(this_load.load_sum_),
2080
        K(server_load.load_sum_.load_sum_), K(server_load.resource_info_), K(hard_limit));
2081
  }
2082
  return ret;
2083
}
2084

2085
int ObServerBalancer::check_exchange_ug_make_sense(
2086
    common::ObIArray<ServerTotalLoad> &server_loads,
2087
    const int64_t left_idx,
2088
    const int64_t right_idx,
2089
    common::ObIArray<SimpleUgLoad> &simple_ug_loads,
2090
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
2091
    bool &do_make_sense)
2092
{
2093
  int ret = OB_SUCCESS;
2094
  double disk_waterlevel = 0.0;
2095
  double disk_usage_limit = 0.0;
2096
  if (OB_UNLIKELY(!inited_)) {
2097
    ret = OB_NOT_INIT;
2098
    LOG_WARN("not init", K(ret));
2099
  } else if (left_idx >= simple_ug_loads.count()
2100
             || right_idx >= simple_ug_loads.count()
2101
             || simple_ug_loads.count() != unit_migrate_stat_matrix.get_column_count()) {
2102
    ret = OB_INVALID_ARGUMENT;
2103
    LOG_WARN("invalid argument", K(ret), K(left_idx), K(right_idx),
2104
             "ug_loads_count", simple_ug_loads.count(),
2105
             "unit_matrix_column", unit_migrate_stat_matrix.get_column_count());
2106
  } else if (OB_FAIL(get_server_balance_critical_disk_waterlevel(disk_waterlevel))) {
2107
    LOG_WARN("fail to get disk waterlevel", K(ret));
2108
  } else if (OB_FAIL(get_server_data_disk_usage_limit(disk_usage_limit))) {
2109
    LOG_WARN("fail to get data disk usage limit", K(ret));
2110
  } else {
2111
    ObArray<ServerTotalLoad *> server_load_ptrs;
2112
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
2113
      if (OB_FAIL(server_load_ptrs.push_back(&server_loads.at(i)))) {
2114
        LOG_WARN("fail to push back", K(ret));
2115
      }
2116
    }
2117
    int64_t left_unit_use = 0;
2118
    int64_t right_unit_use = 0;
2119
    ServerDiskStatistic left_disk_statistic;
2120
    ServerDiskStatistic right_disk_statistic;
2121
    ServerTotalLoad *left_server_load = NULL;
2122
    ServerTotalLoad *right_server_load = NULL;
2123
    for (int64_t row = 0; OB_SUCC(ret) && row < unit_migrate_stat_matrix.get_row_count(); ++row) {
2124
      UnitMigrateStat left_unit_migrate;
2125
      ObUnitStat left_unit_stat;
2126
      UnitMigrateStat right_unit_migrate;
2127
      ObUnitStat right_unit_stat;
2128
      if (OB_FAIL(unit_migrate_stat_matrix.get(row, left_idx, left_unit_migrate))) {
2129
        LOG_WARN("fail to get from matrix", K(ret), K(row), "column", left_idx);
2130
      } else if (OB_UNLIKELY(!left_unit_migrate.unit_load_.is_valid())) {
2131
        ret = OB_ERR_UNEXPECTED;
2132
        LOG_WARN("unit load is invalid", K(ret));
2133
      } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
2134
              left_unit_migrate.unit_load_.unit_->unit_id_,
2135
              left_unit_migrate.unit_load_.unit_->zone_,
2136
              left_unit_stat))) {
2137
        LOG_WARN("fail to get unit stat", K(ret));
2138
      } else if (OB_FAIL(unit_migrate_stat_matrix.get(row, right_idx, right_unit_migrate))) {
2139
        LOG_WARN("fail to get from matrix", K(ret), K(row), "column", right_idx);
2140
      } else if (OB_UNLIKELY(!right_unit_migrate.unit_load_.is_valid())) {
2141
        ret = OB_ERR_UNEXPECTED;
2142
        LOG_WARN("unit load is invalid", K(ret));
2143
      } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
2144
              right_unit_migrate.unit_load_.unit_->unit_id_,
2145
              right_unit_migrate.unit_load_.unit_->zone_,
2146
              right_unit_stat))) {
2147
        LOG_WARN("fail to get unit stat", K(ret));
2148
      } else {
2149
        left_unit_use += left_unit_stat.get_required_size();
2150
        right_unit_use += right_unit_stat.get_required_size();
2151
      }
2152
    }
2153
    if (OB_SUCC(ret)) {
2154
      bool enough = false;
2155
      if (OB_FAIL(zone_disk_statistic_.get_server_disk_statistic(
2156
              simple_ug_loads.at(left_idx).original_server_, left_disk_statistic))) {
2157
        LOG_WARN("fail to get server disk statistic", K(ret));
2158
      } else if (OB_FAIL(zone_disk_statistic_.get_server_disk_statistic(
2159
              simple_ug_loads.at(right_idx).original_server_, right_disk_statistic))) {
2160
        LOG_WARN("fail to get server disk statistic", K(ret));
2161
      } else if (OB_FAIL(pick_server_load(
2162
              simple_ug_loads.at(left_idx).original_server_,
2163
              server_load_ptrs, left_server_load))) {
2164
        LOG_WARN("fail to get server load", K(ret));
2165
      } else if (OB_FAIL(pick_server_load(
2166
              simple_ug_loads.at(right_idx).original_server_,
2167
              server_load_ptrs, right_server_load))) {
2168
        LOG_WARN("fail to get server load", K(ret));
2169
      } else if (NULL == left_server_load || NULL == right_server_load) {
2170
        ret = OB_ERR_UNEXPECTED;
2171
        LOG_WARN("left server or right server load ptr is null", K(ret));
2172
      } else if (OB_FAIL(check_cm_resource_enough(
2173
              simple_ug_loads.at(left_idx).load_sum_, *right_server_load, enough))) {
2174
        LOG_WARN("fail to check cpu memory resource enough", K(ret));
2175
      } else if (!enough) {
2176
        do_make_sense = false;
2177
      } else if (OB_FAIL(check_cm_resource_enough(
2178
              simple_ug_loads.at(right_idx).load_sum_, *left_server_load, enough))) {
2179
        LOG_WARN("fail to check cpu memory resource enough", K(ret));
2180
      } else if (!enough) {
2181
        do_make_sense = false;
2182
      } else if (left_unit_use < right_unit_use) {
2183
        do_make_sense = false; // Cannot lower the left disk after swap
2184
      } else if (
2185
          static_cast<double>(right_disk_statistic.disk_in_use_ - right_unit_use + left_unit_use)
2186
          > disk_waterlevel * static_cast<double>(right_disk_statistic.disk_total_)) {
2187
        do_make_sense = false; // The right disk exceeds the water mark after swapping
2188
      } else if (
2189
          static_cast<double>(right_disk_statistic.disk_in_use_ + left_unit_use)
2190
          > disk_usage_limit * static_cast<double>(right_disk_statistic.disk_total_)) {
2191
        do_make_sense = false; // Exceeded the upper limit of the right disk during the exchange
2192
      } else if (
2193
          static_cast<double>(left_disk_statistic.disk_in_use_ + right_unit_use)
2194
          > disk_usage_limit * static_cast<double>(left_disk_statistic.disk_total_)) {
2195
        do_make_sense = false;
2196
      } else if (
2197
          static_cast<double>(left_unit_use - right_unit_use)
2198
          < static_cast<double>(left_disk_statistic.disk_total_) / static_cast<double>(200)) {
2199
        do_make_sense = false; // The disk on the left side is not significantly reduced after swapping
2200
      } else {
2201
        do_make_sense = true;
2202
      }
2203
    }
2204
  }
2205
  return ret;
2206
}
2207

2208
int ObServerBalancer::coordinate_unit_migrate_stat_matrix(
2209
    const int64_t left_idx,
2210
    const int64_t right_idx,
2211
    common::ObIArray<SimpleUgLoad> &simple_ug_loads,
2212
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix)
2213
{
2214
  int ret = OB_SUCCESS;
2215
  if (OB_UNLIKELY(!inited_)) {
2216
    ret = OB_NOT_INIT;
2217
    LOG_WARN("not init", K(ret));
2218
  } else if (OB_UNLIKELY(left_idx >= simple_ug_loads.count()
2219
                         || right_idx >= simple_ug_loads.count())) {
2220
    ret = OB_INVALID_ARGUMENT;
2221
    LOG_WARN("invalid argument", K(ret), K(left_idx), K(right_idx),
2222
             "simple ug loads count", simple_ug_loads.count());
2223
  } else {
2224
    SimpleUgLoad &left_ug = simple_ug_loads.at(left_idx);
2225
    SimpleUgLoad &right_ug = simple_ug_loads.at(right_idx);
2226
    for (int64_t row = 0; OB_SUCC(ret) && row < unit_migrate_stat_matrix.get_row_count(); ++row) {
2227
      UnitMigrateStat left_unit_migrate;
2228
      UnitMigrateStat right_unit_migrate;
2229
      if (OB_FAIL(unit_migrate_stat_matrix.get(row, left_idx, left_unit_migrate))) {
2230
        LOG_WARN("fail to get from matrix", K(ret), K(row), "column", left_idx);
2231
      } else if (OB_FAIL(unit_migrate_stat_matrix.get(row, right_idx, right_unit_migrate))) {
2232
        LOG_WARN("fail to get from matrix", K(ret), K(row), "column", right_idx);
2233
      } else {
2234
        left_unit_migrate.arranged_pos_ = right_ug.original_server_;
2235
        right_unit_migrate.arranged_pos_ = left_ug.original_server_;
2236
        if (OB_FAIL(unit_migrate_stat_matrix.set(row, left_idx, left_unit_migrate))) {
2237
          LOG_WARN("fail to set", K(ret), K(row), "column", left_idx);
2238
        } else if (OB_FAIL(unit_migrate_stat_matrix.set(row, right_idx, right_unit_migrate))) {
2239
          LOG_WARN("fail to set", K(ret), K(row), "column", right_idx);
2240
        } else {} // no more to do
2241
      }
2242
    }
2243
    if (OB_SUCC(ret)) {
2244
      left_ug.arranged_server_ = right_ug.original_server_;
2245
      right_ug.arranged_server_ = left_ug.original_server_;
2246
    }
2247
  }
2248
  return ret;
2249
}
2250

2251
int ObServerBalancer::generate_exchange_ug_migrate_task(
2252
    const int64_t left_idx,
2253
    const int64_t right_idx,
2254
    common::ObIArray<SimpleUgLoad> &simple_ug_loads,
2255
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
2256
    common::ObIArray<UnitMigrateStat> &task_array)
2257
{
2258
  int ret = OB_SUCCESS;
2259
  if (OB_UNLIKELY(!inited_)) {
2260
    ret = OB_NOT_INIT;
2261
    LOG_WARN("not init", K(ret));
2262
  } else if (OB_UNLIKELY(left_idx >= unit_migrate_stat_matrix.get_column_count()
2263
                         || right_idx >= unit_migrate_stat_matrix.get_column_count())) {
2264
    ret = OB_INVALID_ARGUMENT;
2265
    LOG_WARN("invalid argument", K(ret), K(left_idx), K(right_idx),
2266
             "unit_matrix_column", unit_migrate_stat_matrix.get_column_count());
2267
  } else {
2268
    int64_t left_unit_use = 0;
2269
    int64_t right_unit_use = 0;
2270
    ServerDiskStatistic left_disk_statistic;
2271
    ServerDiskStatistic right_disk_statistic;
2272
    for (int64_t row = 0; OB_SUCC(ret) && row < unit_migrate_stat_matrix.get_row_count(); ++row) {
2273
      UnitMigrateStat left_unit_migrate;
2274
      ObUnitStat left_unit_stat;
2275
      UnitMigrateStat right_unit_migrate;
2276
      ObUnitStat right_unit_stat;
2277
      if (OB_FAIL(unit_migrate_stat_matrix.get(row, left_idx, left_unit_migrate))) {
2278
        LOG_WARN("fail to get from matrix", K(ret), K(row), "column", left_idx);
2279
      } else if (OB_UNLIKELY(!left_unit_migrate.unit_load_.is_valid())) {
2280
        ret = OB_ERR_UNEXPECTED;
2281
        LOG_WARN("unit load is invalid", K(ret));
2282
      } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
2283
              left_unit_migrate.unit_load_.unit_->unit_id_,
2284
              left_unit_migrate.unit_load_.unit_->zone_,
2285
              left_unit_stat))) {
2286
        LOG_WARN("fail to get unit stat", K(ret));
2287
      } else if (OB_FAIL(unit_migrate_stat_matrix.get(row, right_idx, right_unit_migrate))) {
2288
        LOG_WARN("fail to get from matrix", K(ret), K(row), "column", right_idx);
2289
      } else if (OB_UNLIKELY(!right_unit_migrate.unit_load_.is_valid())) {
2290
        ret = OB_ERR_UNEXPECTED;
2291
        LOG_WARN("unit load is invalid", K(ret));
2292
      } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
2293
              right_unit_migrate.unit_load_.unit_->unit_id_,
2294
              right_unit_migrate.unit_load_.unit_->zone_,
2295
              right_unit_stat))) {
2296
        LOG_WARN("fail to get unit stat", K(ret));
2297
      } else if (OB_FAIL(task_array.push_back(left_unit_migrate))) {
2298
        LOG_WARN("fail to push back", K(ret));
2299
      } else if (OB_FAIL(task_array.push_back(right_unit_migrate))) {
2300
        LOG_WARN("fail to push back", K(ret));
2301
      } else {
2302
        left_unit_use += left_unit_stat.get_required_size();
2303
        right_unit_use += right_unit_stat.get_required_size();
2304
      }
2305
    }
2306
    if (OB_SUCC(ret)) {
2307
      if (OB_FAIL(zone_disk_statistic_.get_server_disk_statistic(
2308
              simple_ug_loads.at(left_idx).original_server_, left_disk_statistic))) {
2309
        LOG_WARN("fail to get server disk statistic", K(ret));
2310
      } else if (OB_FAIL(zone_disk_statistic_.reduce_server_disk_use(
2311
              simple_ug_loads.at(left_idx).original_server_, left_unit_use))) {
2312
        LOG_WARN("fail to reduce server disk", K(ret));
2313
      } else if (OB_FAIL(zone_disk_statistic_.raise_server_disk_use(
2314
              simple_ug_loads.at(left_idx).original_server_, right_unit_use))) {
2315
        LOG_WARN("fail to reduce server disk", K(ret));
2316
      } else if (OB_FAIL(zone_disk_statistic_.get_server_disk_statistic(
2317
              simple_ug_loads.at(right_idx).original_server_, right_disk_statistic))) {
2318
        LOG_WARN("fail to get server disk statistic", K(ret));
2319
      } else if (OB_FAIL(zone_disk_statistic_.reduce_server_disk_use(
2320
              simple_ug_loads.at(right_idx).original_server_, right_unit_use))) {
2321
        LOG_WARN("fail to reduce server disk", K(ret));
2322
      } else if (OB_FAIL(zone_disk_statistic_.raise_server_disk_use(
2323
              simple_ug_loads.at(right_idx).original_server_, left_unit_use))) {
2324
        LOG_WARN("fail to reduce server disk", K(ret));
2325
      } else {} // no more to do
2326
    }
2327
  }
2328
  return ret;
2329
}
2330

2331
int ObServerBalancer::try_balance_server_disk_onebyone(
2332
    common::ObIArray<ServerTotalLoad> &server_loads,
2333
    TenantGroupBalanceInfo &balance_info,
2334
    common::ObIArray<SimpleUgLoad> &simple_ug_loads,
2335
    const int64_t ug_idx,
2336
    const common::ObIArray<common::ObAddr> &available_servers,
2337
    const common::ObIArray<common::ObAddr> &disk_over_servers,
2338
    common::ObIArray<UnitMigrateStat> &task_array,
2339
    bool &do_balance_disk)
2340
{
2341
  int ret = OB_SUCCESS;
2342
  common::ObArray<common::ObAddr> left_excluded_servers;
2343
  common::ObArray<common::ObAddr> right_excluded_servers;
2344
  if (OB_UNLIKELY(!inited_)) {
2345
    ret = OB_NOT_INIT;
2346
    LOG_WARN("not init", K(ret));
2347
  } else if (OB_UNLIKELY(ug_idx >= simple_ug_loads.count())) {
2348
    ret = OB_ERR_UNEXPECTED;
2349
    LOG_WARN("ug_idx unexpected", K(ret), K(ug_idx), "ug_load count", simple_ug_loads.count());
2350
  } else if (OB_FAIL(get_ug_exchange_excluded_dst_servers(
2351
          ug_idx, balance_info.unit_migrate_stat_matrix_, left_excluded_servers))) {
2352
    LOG_WARN("fail to get ug exchange dst servers", K(ret));
2353
  } else {
2354
    SimpleUgLoad &left_ug_load = simple_ug_loads.at(ug_idx);
2355
    do_balance_disk = false;
2356
    for (int64_t i = 0;
2357
         !do_balance_disk
2358
          && OB_SUCC(ret)
2359
          && i < zone_disk_statistic_.server_disk_statistic_array_.count();
2360
         ++i) {
2361
      ServerDiskStatistic &disk_statistic
2362
        = zone_disk_statistic_.server_disk_statistic_array_.at(i);
2363
      if (has_exist_in_array(left_excluded_servers, disk_statistic.server_)) {
2364
        // bypass, since in excluded
2365
      } else if (has_exist_in_array(disk_over_servers, disk_statistic.server_)) {
2366
        // bypass, since in disk over servers
2367
      } else if (!has_exist_in_array(available_servers, disk_statistic.server_)) {
2368
        // bypass, since not in available servers
2369
      } else if (disk_statistic.wild_server_) {
2370
        // bypass, wild server
2371
      } else {
2372
        for (int64_t j = 0; !do_balance_disk && OB_SUCC(ret) && j < simple_ug_loads.count(); ++j) {
2373
          SimpleUgLoad &right_ug_load = simple_ug_loads.at(j);
2374
          bool do_make_sense = false;
2375
          if (left_ug_load.column_tenant_id_ == right_ug_load.column_tenant_id_) {
2376
            // by pass, come from the same tenant column
2377
          } else if (right_ug_load.original_server_ != right_ug_load.arranged_server_) {
2378
            // already in migrating, ignore
2379
          } else if (right_ug_load.original_server_ != disk_statistic.server_) {
2380
            // not in this server
2381
          } else if ((left_ug_load.load_sum_.load_sum_.min_cpu()
2382
                          != right_ug_load.load_sum_.load_sum_.min_cpu())
2383
                     || (left_ug_load.load_sum_.load_sum_.memory_size()
2384
                          != right_ug_load.load_sum_.load_sum_.memory_size())) {
2385
            // cpu and memory not match, exchange will violate the balance result
2386
          } else if (OB_FAIL(get_ug_exchange_excluded_dst_servers(
2387
                  j, balance_info.unit_migrate_stat_matrix_, right_excluded_servers))) {
2388
            LOG_WARN("fail to get ug exchange dst servers", K(ret));
2389
          } else if (has_exist_in_array(right_excluded_servers, left_ug_load.original_server_)) {
2390
            // left ug server in right excluded servers
2391
          } else if (OB_FAIL(check_exchange_ug_make_sense(
2392
                  server_loads, ug_idx, j, simple_ug_loads,
2393
                  balance_info.unit_migrate_stat_matrix_, do_make_sense))) {
2394
            LOG_WARN("fail to check exchange ug make sence", K(ret));
2395
          } else if (!do_make_sense) {
2396
            // do not make sence
2397
          } else if (OB_FAIL(coordinate_unit_migrate_stat_matrix(
2398
                  ug_idx, j, simple_ug_loads, balance_info.unit_migrate_stat_matrix_))) {
2399
            LOG_WARN("fail to coordinate unit migrate stat matrix", K(ret));
2400
          } else if (OB_FAIL(generate_exchange_ug_migrate_task(
2401
                  ug_idx, j, simple_ug_loads,
2402
                  balance_info.unit_migrate_stat_matrix_, task_array))) {
2403
            LOG_WARN("fail to generate exchange ug migrate task", K(ret));
2404
          } else {
2405
            do_balance_disk = true;
2406
          }
2407
        }
2408
      }
2409
    }
2410
  }
2411
  return ret;
2412
}
2413

2414
int ObServerBalancer::try_balance_server_disk_by_ttg(
2415
    common::ObIArray<ServerTotalLoad> &server_loads,
2416
    const common::ObAddr &src_server,
2417
    TenantGroupBalanceInfo &balance_info,
2418
    const common::ObIArray<common::ObAddr> &available_servers,
2419
    const common::ObIArray<common::ObAddr> &disk_over_servers,
2420
    common::ObIArray<UnitMigrateStat> &task_array,
2421
    bool &do_balance_disk)
2422
{
2423
  int ret = OB_SUCCESS;
2424
  ObArray<SimpleUgLoad> simple_ug_loads;
2425
  double disk_waterlevel = 0.0;
2426
  if (OB_UNLIKELY(!inited_)) {
2427
    ret = OB_NOT_INIT;
2428
    LOG_WARN("not init", K(ret));
2429
  } else if (OB_UNLIKELY(!src_server.is_valid())) {
2430
    ret = OB_INVALID_ARGUMENT;
2431
    LOG_WARN("invalid argument", K(ret), K(src_server));
2432
  } else if (OB_FAIL(generate_simple_ug_loads(simple_ug_loads, balance_info))) {
2433
    LOG_WARN("fail to generate simple ug loads", K(ret));
2434
  } else if (OB_FAIL(get_server_balance_critical_disk_waterlevel(disk_waterlevel))) {
2435
    LOG_WARN("fail to get disk waterlevel", K(ret));
2436
  } else {
2437
    do_balance_disk = false;
2438
    for (int64_t i = 0; OB_SUCC(ret) && i < simple_ug_loads.count(); ++i) {
2439
      bool my_do_balance_disk = false;
2440
      SimpleUgLoad &simple_ug_load = simple_ug_loads.at(i);
2441
      if (simple_ug_load.original_server_ != simple_ug_load.arranged_server_) {
2442
        // bypass, since this has already been migrated by others
2443
      } else if (src_server != simple_ug_load.original_server_) {
2444
        // bypass, since server not match
2445
      } else if (OB_FAIL(try_balance_server_disk_onebyone(
2446
              server_loads, balance_info, simple_ug_loads, i, available_servers,
2447
              disk_over_servers, task_array, my_do_balance_disk))) {
2448
        LOG_WARN("fail to try balance server disk onebyone", K(ret));
2449
      } else {
2450
        do_balance_disk |= my_do_balance_disk;
2451
      }
2452
      if (OB_FAIL(ret)) {
2453
      } else if (do_balance_disk) {
2454
        break; // Performed a set of disk swaps
2455
      }
2456
    }
2457
  }
2458
  return ret;
2459
}
2460

2461
int ObServerBalancer::try_balance_disk_by_stable_tenantgroup(
2462
    const common::ObZone &zone,
2463
    common::ObIArray<TenantGroupBalanceInfo *> &stable_ttg,
2464
    const common::ObIArray<common::ObAddr> &available_servers,
2465
    bool &do_balance_disk)
2466
{
2467
  int ret = OB_SUCCESS;
2468
  common::ObArray<common::ObAddr> disk_over_servers;
2469
  common::ObArray<ServerTotalLoad> server_loads;
2470
  double g_res_weights[RES_MAX];
2471
  double disk_waterlevel = 0.0;
2472
  if (OB_UNLIKELY(!inited_)) {
2473
    ret = OB_NOT_INIT;
2474
    LOG_WARN("not init", K(ret));
2475
  } else if (!zone_disk_statistic_.over_disk_waterlevel()) {
2476
    do_balance_disk = false; // The disk is within the water mark, no need to adjust
2477
  } else if (stable_ttg.count() <= 0) {
2478
    do_balance_disk = false; // stable ttg is empty, bypass
2479
  } else if (OB_UNLIKELY(available_servers.count() <= 0)) {
2480
    ret = OB_INVALID_ARGUMENT;
2481
    LOG_WARN("invalid argument", K(ret), K(available_servers));
2482
  } else if (OB_FAIL(get_disk_over_available_servers(disk_over_servers))) {
2483
    LOG_WARN("fail to get disk over servers", K(ret));
2484
  } else if (OB_FAIL(get_server_balance_critical_disk_waterlevel(disk_waterlevel))) {
2485
    LOG_WARN("fail to get disk waterlevel", K(ret));
2486
  } else if (OB_FAIL(calc_global_balance_resource_weights(
2487
          zone, available_servers, g_res_weights, RES_MAX))) {
2488
    LOG_WARN("fail to calc whole balance resource weights", K(ret));
2489
  } else if (OB_FAIL(generate_complete_server_loads(
2490
          zone, available_servers, g_res_weights, RES_MAX, server_loads))) {
2491
    LOG_WARN("fail to generate complete server loads", K(ret));
2492
  } else {
2493
    LOG_INFO("do balance disk by stable tenantgroup", K(ret), K(disk_over_servers));
2494
    common::ObArray<UnitMigrateStat> task_array;
2495
    do_balance_disk = false;
2496
    for (int64_t i = 0; OB_SUCC(ret) && i < disk_over_servers.count(); ++i) {
2497
      common::ObAddr &src_server = disk_over_servers.at(i);
2498
      for (int64_t j = 0; OB_SUCC(ret) && j < stable_ttg.count(); ++j) {
2499
        bool my_do_balance_disk = false;
2500
        TenantGroupBalanceInfo *balance_info = stable_ttg.at(j);
2501
        if (NULL == balance_info) {
2502
          ret = OB_ERR_UNEXPECTED;
2503
          LOG_WARN("balance info ptr is null", K(ret));
2504
        } else if (OB_FAIL(try_balance_server_disk_by_ttg(
2505
                server_loads, src_server, *balance_info, available_servers,
2506
                disk_over_servers, task_array, my_do_balance_disk))) {
2507
          LOG_WARN("fail to try balance server disk onebyone", K(ret));
2508
        } else {
2509
          do_balance_disk |= my_do_balance_disk;
2510
        }
2511
        if (OB_FAIL(ret)) {
2512
        } else if (do_balance_disk) {
2513
          break; // Exchange
2514
        }
2515
      }
2516
    }
2517
    if (OB_SUCC(ret) && task_array.count() > 0) {
2518
      if (OB_FAIL(check_and_do_migrate_unit_task_(task_array))) {
2519
        LOG_WARN("fail to check and do migrate unit task array", KR(ret), K(task_array));
2520
      }
2521
    }
2522
  }
2523
  return ret;
2524
}
2525

2526
int ObServerBalancer::try_amend_and_execute_stable_tenantgroup(
2527
    const common::ObZone &zone,
2528
    const common::ObIArray<TenantGroupBalanceInfo *> &unstable_tenant_group,
2529
    const common::ObIArray<TenantGroupBalanceInfo *> &stable_tenant_group,
2530
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
2531
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units,
2532
    const common::ObIArray<common::ObAddr> &available_servers,
2533
    bool &do_amend)
2534
{
2535
  int ret = OB_SUCCESS;
2536
  TenantGroupBalanceInfo *info_may_amend = NULL;
2537
  ObArray<TenantGroupBalanceInfo *> other_stable_ttg;
2538
  const bool is_stable_ttg = true;
2539
  if (OB_UNLIKELY(!inited_)) {
2540
    ret = OB_NOT_INIT;
2541
    LOG_WARN("not init", K(ret));
2542
  } else if (stable_tenant_group.count() <= 0) {
2543
    do_amend = false;
2544
  } else if (OB_FAIL(choose_balance_info_amend(
2545
          stable_tenant_group, is_stable_ttg, info_may_amend, other_stable_ttg))) {
2546
    LOG_WARN("fail to choose stable balance info amend", K(ret));
2547
  } else if (NULL == info_may_amend) {
2548
    ret = OB_INVALID_ARGUMENT;
2549
    LOG_WARN("invalid argument", K(ret), KP(info_may_amend));
2550
  } else if (OB_FAIL(try_amend_and_execute_tenantgroup(
2551
          zone, info_may_amend, other_stable_ttg, unstable_tenant_group,
2552
          standalone_units, not_grant_units, available_servers, do_amend))) {
2553
    LOG_WARN("fail to amend and execute tenantgroup", K(ret));
2554
  } else {} // no more to do
2555
  return ret;
2556
}
2557

2558
int ObServerBalancer::try_amend_and_execute_unstable_tenantgroup(
2559
    const common::ObZone &zone,
2560
    const common::ObIArray<TenantGroupBalanceInfo *> &unstable_tenant_group,
2561
    const common::ObIArray<TenantGroupBalanceInfo *> &stable_tenant_group,
2562
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
2563
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units,
2564
    const common::ObIArray<common::ObAddr> &available_servers,
2565
    bool &do_amend)
2566
{
2567
  int ret = OB_SUCCESS;
2568
  TenantGroupBalanceInfo *info_need_amend = NULL;
2569
  ObArray<TenantGroupBalanceInfo *> other_unstable_ttg;
2570
  const bool is_stable_ttg = false;
2571
  if (OB_UNLIKELY(!inited_)) {
2572
    ret = OB_NOT_INIT;
2573
    LOG_WARN("not init", K(ret));
2574
  } else if (unstable_tenant_group.count() <= 0) {
2575
    do_amend = false;
2576
  } else if (OB_FAIL(choose_balance_info_amend(
2577
          unstable_tenant_group, is_stable_ttg, info_need_amend, other_unstable_ttg))) {
2578
    LOG_WARN("fail to choose unstable balance info amend", K(ret));
2579
  } else if (OB_UNLIKELY(NULL == info_need_amend)) {
2580
    ret = OB_INVALID_ARGUMENT;
2581
    LOG_WARN("invalid argument", K(ret), KP(info_need_amend));
2582
  } else if (OB_FAIL(try_amend_and_execute_tenantgroup(
2583
          zone, info_need_amend, stable_tenant_group, other_unstable_ttg,
2584
          standalone_units, not_grant_units, available_servers, do_amend))) {
2585
    LOG_WARN("fail to amend and execute tenantgroup", K(ret));
2586
  } else {} // no more to do
2587
  return ret;
2588
}
2589

2590
int ObServerBalancer::get_sys_tenant_unitgroup_loads(
2591
    const common::ObZone &zone,
2592
    common::ObIArray<UnitGroupLoad> &sys_tenant_ug_loads)
2593
{
2594
  int ret = OB_SUCCESS;
2595
  sys_tenant_ug_loads.reset();
2596
  common::ObArray<ObUnitManager::ObUnitLoad> unit_loads;
2597
  common::ObReplicaType replica_type = common::REPLICA_TYPE_FULL;
2598
  if (OB_UNLIKELY(!inited_)) {
2599
    ret = OB_NOT_INIT;
2600
    LOG_WARN("not init", K(ret));
2601
  } else if (OB_UNLIKELY(zone.is_empty())) {
2602
    ret = OB_INVALID_ARGUMENT;
2603
    LOG_WARN("invalid argument", K(ret));
2604
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
2605
    ret = OB_ERR_UNEXPECTED;
2606
    LOG_WARN("unit_mgr_ ptr is null", K(ret));
2607
  } else if (OB_FAIL(unit_mgr_->get_tenant_zone_unit_loads(
2608
          common::OB_SYS_TENANT_ID, zone, replica_type, unit_loads))) {
2609
    if (OB_ENTRY_NOT_EXIST == ret) {
2610
      ret = OB_SUCCESS;
2611
    } else {
2612
      LOG_WARN("fail to get tenant zone unit infos", K(ret));
2613
    }
2614
  } else {
2615
    for (int64_t i = 0; OB_SUCC(ret) && i < unit_loads.count(); ++i) {
2616
      ObUnitManager::ObUnitLoad &unit_load = unit_loads.at(i);
2617
      if (OB_UNLIKELY(!unit_load.is_valid())) {
2618
        ret = OB_ERR_UNEXPECTED;
2619
        LOG_WARN("unit load is valid", K(ret));
2620
      } else {
2621
        UnitGroupLoad ug_load;
2622
        ug_load.column_tenant_id_ = common::OB_SYS_TENANT_ID;
2623
        ug_load.start_column_idx_ = 0; // dummy for sys tenant
2624
        ug_load.column_count_ = 1; // dummy for sys tenant
2625
        ug_load.server_ = unit_load.unit_->server_;
2626
        // assign server_load_ ptr in append\_alien\_ug\_loads()
2627
        ug_load.server_load_ = NULL;
2628
        if (OB_UNLIKELY(ug_load.unit_loads_.push_back(*unit_load.unit_config_))) {
2629
          LOG_WARN("fail to push back", K(ret));
2630
        } else if (OB_FAIL(ug_load.sum_group_load())) {
2631
          LOG_WARN("fail to sum group load", K(ret));
2632
        } else if (OB_FAIL(sys_tenant_ug_loads.push_back(ug_load))) {
2633
          LOG_WARN("fail to push back", K(ret));
2634
        } else {} // no more to do
2635
      }
2636
    }
2637
  }
2638
  return ret;
2639
}
2640

2641
int ObServerBalancer::try_amend_and_execute_tenantgroup(
2642
    const common::ObZone &zone,
2643
    TenantGroupBalanceInfo *info_amend,
2644
    const common::ObIArray<TenantGroupBalanceInfo *> &stable_tenant_group,
2645
    const common::ObIArray<TenantGroupBalanceInfo *> &unstable_tenant_group,
2646
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
2647
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units,
2648
    const common::ObIArray<common::ObAddr> &available_servers,
2649
    bool &do_amend)
2650
{
2651
  int ret = OB_SUCCESS;
2652
  common::ObArray<UnitGroupLoad> sys_tenant_ug_loads;
2653
  if (OB_UNLIKELY(!inited_)) {
2654
    ret = OB_NOT_INIT;
2655
    LOG_WARN("not init", K(ret));
2656
  } else if (OB_UNLIKELY(NULL == info_amend)) {
2657
    ret = OB_INVALID_ARGUMENT;
2658
    LOG_WARN("invalid argument", K(ret));
2659
  } else if (OB_FAIL(get_sys_tenant_unitgroup_loads(zone, sys_tenant_ug_loads))) {
2660
    LOG_WARN("fail to get sys tenant unitgroup_loads", K(ret));
2661
  } else if (OB_FAIL(calc_inter_ttg_weights(
2662
          info_amend, stable_tenant_group, sys_tenant_ug_loads,
2663
          available_servers, info_amend->inter_weights_, RES_MAX))) {
2664
    LOG_WARN("fail to calc inter ttg weights", K(ret));
2665
  } else if (OB_FAIL(generate_inter_ttg_server_loads(
2666
          info_amend, stable_tenant_group, sys_tenant_ug_loads, available_servers))) {
2667
    LOG_WARN("fail to generate inter ttg server loads", K(ret));
2668
  } else if (OB_FAIL(do_amend_inter_ttg_balance(
2669
          info_amend, stable_tenant_group, info_amend->server_load_array_))) {
2670
    LOG_WARN("fail to do amend inter ttg balance", K(ret));
2671
  } else if (OB_FAIL(try_execute_unit_balance_task(
2672
          zone, info_amend, standalone_units, not_grant_units,
2673
          unstable_tenant_group, available_servers, do_amend))) {
2674
    LOG_WARN("fail to do exuecte unit balance task", K(ret));
2675
  } else {} // no more to do
2676
  return ret;
2677
}
2678

2679
int ObServerBalancer::calc_inter_ttg_weights(
2680
    TenantGroupBalanceInfo *info_need_amend,
2681
    const common::ObIArray<TenantGroupBalanceInfo *> &stable_tenant_group,
2682
    const common::ObIArray<UnitGroupLoad> &sys_tenant_ug_loads,
2683
    const common::ObIArray<common::ObAddr> &available_servers,
2684
    double *const resource_weights,
2685
    const int64_t weights_count)
2686
{
2687
  int ret = OB_SUCCESS;
2688
  if (OB_UNLIKELY(!inited_)) {
2689
    ret = OB_NOT_INIT;
2690
    LOG_WARN("not init", K(ret));
2691
  } else if (OB_UNLIKELY(NULL == resource_weights
2692
        || RES_MAX != weights_count
2693
        || NULL == info_need_amend)) {
2694
    ret = OB_INVALID_ARGUMENT;
2695
    LOG_WARN("invalid argument", K(ret), KP(info_need_amend));
2696
  } else if (OB_ISNULL(server_mgr_)) {
2697
    ret = OB_ERR_UNEXPECTED;
2698
    LOG_WARN("server_mgr_ is null", KR(ret), KP(server_mgr_));
2699
  } else {
2700
    LoadSum load_sum;
2701
    for (int64_t i = 0;
2702
         OB_SUCC(ret) && i < info_need_amend->unitgroup_load_array_.count();
2703
         ++i) {
2704
      const UnitGroupLoad &unitgroup_load = info_need_amend->unitgroup_load_array_.at(i);
2705
      if (OB_UNLIKELY(!unitgroup_load.is_valid())) {
2706
        ret = OB_ERR_UNEXPECTED;
2707
        LOG_WARN("unitgroup_load is invalid, unexpected", K(ret), K(unitgroup_load));
2708
      } else if (OB_FAIL(load_sum.append_load(unitgroup_load.load_sum_))) {
2709
        LOG_WARN("append load failed", K(ret));
2710
      } else {} // no more to do
2711
    }
2712
    for (int64_t i = 0; OB_SUCC(ret) && i < sys_tenant_ug_loads.count(); ++i) {
2713
      const UnitGroupLoad &unitgroup_load = sys_tenant_ug_loads.at(i);
2714
      if (OB_UNLIKELY(!unitgroup_load.is_valid())) {
2715
        ret = OB_ERR_UNEXPECTED;
2716
        LOG_WARN("unitgroup_load is invalid, unexpected", K(ret), K(unitgroup_load));
2717
      } else if (OB_FAIL(load_sum.append_load(unitgroup_load.load_sum_))) {
2718
        LOG_WARN("append load failed", K(ret));
2719
      } else {} // no more to do
2720
    }
2721
    for (int64_t i = 0;
2722
         OB_SUCC(ret) && i < stable_tenant_group.count();
2723
         ++i) {
2724
      const TenantGroupBalanceInfo *balance_info = stable_tenant_group.at(i);
2725
      if (OB_UNLIKELY(NULL == balance_info)) {
2726
        ret = OB_ERR_UNEXPECTED;
2727
        LOG_WARN("balance info ptr is null", K(ret));
2728
      } else {
2729
        for (int64_t j = 0;
2730
             OB_SUCC(ret) && j < balance_info->unitgroup_load_array_.count();
2731
             ++j) {
2732
          const UnitGroupLoad &unitgroup_load = balance_info->unitgroup_load_array_.at(j);
2733
          if (OB_UNLIKELY(!unitgroup_load.is_valid())) {
2734
            ret = OB_ERR_UNEXPECTED;
2735
            LOG_WARN("unit_load is invalid, unexpected", K(ret), K(unitgroup_load));
2736
          } else if (OB_FAIL(load_sum.append_load(unitgroup_load.load_sum_))) {
2737
            LOG_WARN("append load failed", K(ret));
2738
          } else {} // no more to do
2739
        }
2740
      }
2741
    }
2742
    ResourceSum resource_sum;
2743
    for (int64_t i = 0; OB_SUCC(ret) && i < available_servers.count(); ++i) {
2744
      const common::ObAddr &server = available_servers.at(i);
2745
      share::ObServerResourceInfo resource_info;
2746
      if (OB_FAIL(server_mgr_->get_server_resource_info(server, resource_info))) {
2747
        LOG_WARN("fail to get server resource_info", KR(ret), K(server));
2748
      } else if (OB_FAIL(resource_sum.append_resource(resource_info))) {
2749
        LOG_WARN("fail to append resource", K(ret));
2750
      } else {} // no more to do
2751
    }
2752
    for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
2753
      ObResourceType resource_type = static_cast<ObResourceType>(i);
2754
      const double required = load_sum.get_required(resource_type);
2755
      const double capacity = resource_sum.get_capacity(resource_type);
2756
      if (required <= 0 || capacity <= 0) {
2757
        resource_weights[resource_type] = 0.0;
2758
      } else if (required >= capacity) {
2759
        resource_weights[resource_type] = 1.0;
2760
      } else {
2761
        resource_weights[resource_type] = required / capacity;
2762
      }
2763
    }
2764
    if (OB_SUCC(ret)) { // Weight normalization
2765
      double sum = 0.0;
2766
      const int64_t N = available_servers.count();
2767
      for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
2768
        resource_weights[i] /= static_cast<double>(N);
2769
        sum += resource_weights[i];
2770
        if (resource_weights[i] < 0 || resource_weights[i] > 1) {
2771
          ret = common::OB_ERR_UNEXPECTED;
2772
          LOG_ERROR("weight shall be in interval [0,1]", K(i), "w", resource_weights[i]);
2773
        }
2774
      }
2775
      if (OB_SUCC(ret) && sum > 0) {
2776
        for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
2777
          resource_weights[i] /= sum;
2778
        }
2779
      }
2780
    }
2781
  }
2782
  return ret;
2783
}
2784

2785
int ObServerBalancer::append_alien_ug_loads(
2786
    ServerLoad &server_load,
2787
    const common::ObIArray<TenantGroupBalanceInfo *> &tenant_groups,
2788
    common::ObIArray<UnitGroupLoad> &sys_tenant_ug_loads)
2789
{
2790
  int ret = OB_SUCCESS;
2791
  if (OB_UNLIKELY(!inited_)) {
2792
    ret = OB_NOT_INIT;
2793
    LOG_WARN("not init", K(ret));
2794
  } else {
2795
    for (int64_t i = 0; OB_SUCC(ret) && i < sys_tenant_ug_loads.count(); ++i) {
2796
      UnitGroupLoad &unitgroup_load = sys_tenant_ug_loads.at(i);
2797
      if (server_load.server_ != unitgroup_load.server_) {
2798
        // not the same server
2799
      } else if (FALSE_IT(unitgroup_load.server_load_ = &server_load)) {
2800
        // will never be here
2801
      } else if (OB_FAIL(server_load.alien_ug_loads_.push_back(&unitgroup_load))) {
2802
        LOG_WARN("fail to push back", K(ret));
2803
      } else {} // no more to do
2804
    }
2805
    for (int64_t i = 0; OB_SUCC(ret) && i < tenant_groups.count(); ++i) {
2806
      TenantGroupBalanceInfo *balance_info = tenant_groups.at(i);
2807
      if (OB_UNLIKELY(NULL == balance_info)) {
2808
        ret = OB_ERR_UNEXPECTED;
2809
        LOG_WARN("balance info ptr is null", K(ret), KP(balance_info));
2810
      } else {
2811
        for (int64_t i = 0;
2812
             OB_SUCC(ret) && i < balance_info->unitgroup_load_array_.count();
2813
             ++i) {
2814
          UnitGroupLoad &unitgroup_load = balance_info->unitgroup_load_array_.at(i);
2815
          if (OB_UNLIKELY(!unitgroup_load.is_valid())) {
2816
            ret = OB_ERR_UNEXPECTED;
2817
            LOG_WARN("unitgroup load is invalid, unexpected", K(ret), K(unitgroup_load));
2818
          } else if (server_load.server_ != unitgroup_load.server_) {
2819
            // not the same server
2820
          } else if (OB_FAIL(server_load.alien_ug_loads_.push_back(&unitgroup_load))) {
2821
            LOG_WARN("fail to push back", K(ret));
2822
          } else {} // no more to do
2823
        }
2824
      }
2825
    }
2826
  }
2827
  return ret;
2828
}
2829

2830
int ObServerBalancer::generate_inter_ttg_server_loads(
2831
    TenantGroupBalanceInfo *info_need_amend,
2832
    const common::ObIArray<TenantGroupBalanceInfo *> &stable_tenant_group,
2833
    common::ObIArray<UnitGroupLoad> &sys_tenant_ug_loads,
2834
    const common::ObIArray<common::ObAddr> &available_servers)
2835
{
2836
  int ret = OB_SUCCESS;
2837
  UNUSED(available_servers);
2838
  if (OB_UNLIKELY(!inited_)) {
2839
    ret = OB_NOT_INIT;
2840
    LOG_WARN("not init", K(ret));
2841
  } else if (OB_UNLIKELY(NULL == info_need_amend || available_servers.count() <= 0)) {
2842
    ret = OB_INVALID_ARGUMENT;
2843
    LOG_WARN("invalid argument", K(ret), KP(info_need_amend),
2844
             "server_count", available_servers.count());
2845
  } else {
2846
    common::ObArray<ServerLoad> &target_server_loads = info_need_amend->server_load_array_;
2847
    for (int64_t i = 0; OB_SUCC(ret) && i < target_server_loads.count(); ++i) {
2848
      ServerLoad &server_load = target_server_loads.at(i);
2849
      if (!server_load.is_valid()) {
2850
        ret = OB_ERR_UNEXPECTED;
2851
        LOG_WARN("should be a valid server load here", K(ret), K(server_load));
2852
      } else {
2853
        for (int32_t j = RES_CPU; j < RES_MAX; ++j) {
2854
          server_load.inter_weights_[j] = info_need_amend->inter_weights_[j];
2855
        }
2856
      }
2857
    }
2858
    for (int64_t i = 0; OB_SUCC(ret) && i < target_server_loads.count(); ++i) {
2859
      target_server_loads.at(i).alien_ug_loads_.reset();
2860
      if (OB_FAIL(append_alien_ug_loads(
2861
              target_server_loads.at(i), stable_tenant_group, sys_tenant_ug_loads))) {
2862
        LOG_WARN("fail to append alien ug loads", K(ret));
2863
      } else if (OB_FAIL(target_server_loads.at(i).update_load_value())) {
2864
        LOG_WARN("fail to update load value", K(ret));
2865
      }
2866
    }
2867
  }
2868
  return ret;
2869
}
2870

2871
int ObServerBalancer::do_amend_inter_ttg_balance(
2872
    TenantGroupBalanceInfo *balance_info,
2873
    const common::ObIArray<TenantGroupBalanceInfo *> &stable_tenant_group,
2874
    common::ObIArray<ServerLoad> &server_loads)
2875
{
2876
  int ret = OB_SUCCESS;
2877
  UNUSED(stable_tenant_group);
2878
  common::ObArray<ServerLoad *> server_load_ptrs_sorted;
2879
  common::ObArray<ServerLoad *> over_server_loads;
2880
  common::ObArray<ServerLoad *> under_server_loads;
2881
  double upper_lmt = 0.0;
2882
  if (OB_UNLIKELY(!inited_)) {
2883
    ret = OB_NOT_INIT;
2884
    LOG_WARN("not init", K(ret));
2885
  } else if (OB_UNLIKELY(NULL == balance_info
2886
        || server_loads.count() <= 0)) {
2887
    ret = OB_INVALID_ARGUMENT;
2888
    LOG_WARN("invalid argument", K(ret), KP(balance_info),
2889
             "server_load_count", server_loads.count());
2890
  } else if (OB_FAIL(server_load_ptrs_sorted.reserve(server_loads.count()))) {
2891
    LOG_WARN("fail to reserve", K(ret));
2892
  } else if (OB_FAIL(over_server_loads.reserve(server_loads.count()))) {
2893
    LOG_WARN("fail to reserve", K(ret));
2894
  } else if (OB_FAIL(under_server_loads.reserve(server_loads.count()))) {
2895
    LOG_WARN("fail to reserve", K(ret));
2896
  } else {
2897
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
2898
      ServerLoad &server_load = server_loads.at(i);
2899
      if (OB_FAIL(server_load_ptrs_sorted.push_back(&server_load))) {
2900
        LOG_WARN("fail to push back", K(ret));
2901
      }
2902
    }
2903
    if (OB_SUCC(ret)) {
2904
      InterServerLoadCmp cmp;
2905
      std::sort(server_load_ptrs_sorted.begin(), server_load_ptrs_sorted.end(), cmp);
2906
      if (OB_FAIL(cmp.get_ret())) {
2907
        LOG_WARN("fail to sort server load ptrs", K(ret));
2908
      } else if (OB_FAIL(sort_server_loads_for_balance(
2909
              server_load_ptrs_sorted, over_server_loads, under_server_loads, upper_lmt))) {
2910
        LOG_WARN("fail to sort server loads for inter ttg balance", K(ret));
2911
      } else if (OB_FAIL(make_inter_ttg_server_under_load(
2912
              balance_info, over_server_loads, under_server_loads, upper_lmt))) {
2913
        LOG_WARN("fail to make inter ttg server under load", K(ret));
2914
      } else if (OB_FAIL(inner_ttg_balance_strategy_.coordinate_all_column_unit_group(
2915
              balance_info->column_unit_num_array_, balance_info->unit_migrate_stat_matrix_,
2916
              balance_info->unitgroup_load_array_))) {
2917
        LOG_WARN("fail to coordinate column units", K(ret));
2918
      }
2919
    }
2920
  }
2921
  return ret;
2922
}
2923

2924
int ObServerBalancer::sort_server_loads_for_balance(
2925
    common::ObIArray<ServerLoad *> &complete_server_loads,
2926
    common::ObIArray<ServerLoad *> &over_server_loads,
2927
    common::ObIArray<ServerLoad *> &under_server_loads,
2928
    double &upper_lmt)
2929
{
2930
  int ret = OB_SUCCESS;
2931
  if (OB_UNLIKELY(!inited_)) {
2932
    ret = OB_NOT_INIT;
2933
    LOG_WARN("not init", K(ret));
2934
  } else if (OB_UNLIKELY(complete_server_loads.count() <= 0)) {
2935
    ret = OB_INVALID_ARGUMENT;
2936
    LOG_WARN("invalid argument", K(ret), "server_load_count", complete_server_loads.count());
2937
  } else {
2938
    over_server_loads.reset();
2939
    under_server_loads.reset();
2940
    double sum_load = 0.0;
2941
    for (int64_t i = 0; OB_SUCC(ret) && i < complete_server_loads.count(); ++i) {
2942
      const ServerLoad *server_load = complete_server_loads.at(i);
2943
      if (OB_UNLIKELY(NULL == server_load)) {
2944
        ret = OB_ERR_UNEXPECTED;
2945
        LOG_WARN("server load ptr is null", K(ret), KP(server_load));
2946
      } else {
2947
        sum_load += server_load->inter_ttg_load_value_;
2948
      }
2949
    }
2950
    const int64_t cpu_mem_tolerance = GCONF.server_balance_cpu_mem_tolerance_percent;
2951
    const double delta_percent = static_cast<double>(cpu_mem_tolerance) / static_cast<double>(100);
2952
    const double average_load = sum_load / static_cast<double>(complete_server_loads.count());
2953
    upper_lmt = average_load + delta_percent * average_load;
2954
    for (int64_t i = 0; OB_SUCC(ret) && i < complete_server_loads.count(); ++i) {
2955
      ServerLoad *server_load = complete_server_loads.at(i);
2956
      if (OB_UNLIKELY(NULL == server_load)) {
2957
        ret = OB_ERR_UNEXPECTED;
2958
        LOG_WARN("server load ptr is null", K(ret), KP(server_load));
2959
      } else if (server_load->inter_ttg_load_value_ >= upper_lmt) {
2960
        if (OB_FAIL(over_server_loads.push_back(server_load))) {
2961
          LOG_WARN("fail to push back", K(ret));
2962
        }
2963
      } else {
2964
        if (OB_FAIL(under_server_loads.push_back(server_load))) {
2965
          LOG_WARN("fail to push back", K(ret));
2966
        }
2967
      }
2968
    }
2969
  }
2970
  return ret;
2971
}
2972

2973
int ObServerBalancer::make_inter_ttg_server_under_load(
2974
    TenantGroupBalanceInfo *balance_info,
2975
    common::ObIArray<ServerLoad *> &over_server_loads,
2976
    common::ObIArray<ServerLoad *> &under_server_loads,
2977
    const double inter_ttg_upper_lmt)
2978
{
2979
  int ret = OB_SUCCESS;
2980
  double hard_limit = 0.0;
2981
  if (OB_UNLIKELY(!inited_)) {
2982
    ret = OB_NOT_INIT;
2983
    LOG_WARN("not init", K(ret));
2984
  } else if (OB_UNLIKELY(NULL == balance_info)) {
2985
    ret = OB_INVALID_ARGUMENT;
2986
    LOG_WARN("invalid balance info", K(ret), KP(balance_info));
2987
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
2988
    ret = OB_ERR_UNEXPECTED;
2989
    LOG_WARN("unit_mgr_ ptr is null", K(ret));
2990
  } else if (OB_FAIL(unit_mgr_->get_hard_limit(hard_limit))) {
2991
    LOG_WARN("fail to get hard limit", K(ret));
2992
  } else {
2993
    double upper_limit = std::min(inter_ttg_upper_lmt, hard_limit);
2994
    for (int64_t i = 0; OB_SUCC(ret) && i < over_server_loads.count(); ++i) {
2995
      ServerLoad *over_server_load = over_server_loads.at(i);
2996
      for (int64_t j = under_server_loads.count() - 1; OB_SUCC(ret) && j >= 0; --j) {
2997
        ServerLoad *under_server_load = under_server_loads.at(j);
2998
        if (OB_FAIL(amend_ug_inter_ttg_server_load(
2999
                over_server_load, under_server_load, balance_info, upper_limit))) {
3000
          LOG_WARN("fail to amend ug inter ttg server load", K(ret));
3001
        }
3002
      }
3003
    }
3004
  }
3005
  return ret;
3006
}
3007

3008
int ObServerBalancer::amend_ug_inter_ttg_server_load(
3009
    ServerLoad *src_inter_load,
3010
    ServerLoad *dst_inter_load,
3011
    TenantGroupBalanceInfo *balance_info,
3012
    const double inter_ttg_upper_lmt)
3013
{
3014
  int ret = OB_SUCCESS;
3015
  if (OB_UNLIKELY(!inited_)) {
3016
    ret = OB_NOT_INIT;
3017
    LOG_WARN("not init", K(ret));
3018
  } else if (OB_UNLIKELY(NULL == src_inter_load
3019
                         || NULL == dst_inter_load
3020
                         || NULL == balance_info)) {
3021
    ret = OB_INVALID_ARGUMENT;
3022
    LOG_WARN("invalid argument", K(ret), KP(src_inter_load),
3023
             KP(dst_inter_load), KP(balance_info));
3024
  } else if (OB_FAIL(inner_ttg_balance_strategy_.amend_ug_inter_ttg_server_load(
3025
          src_inter_load, dst_inter_load, balance_info, inter_ttg_upper_lmt))) {
3026
    LOG_WARN("fail to amend ug inter ttg server load", K(ret));
3027
  }
3028
  return ret;
3029
}
3030

3031
int ObServerBalancer::CountBalanceStrategy::amend_ug_inter_ttg_server_load(
3032
    ServerLoad *src_inter_load,
3033
    ServerLoad *dst_inter_load,
3034
    TenantGroupBalanceInfo *balance_info,
3035
    const double inter_ttg_upper_lmt)
3036
{
3037
  int ret = OB_SUCCESS;
3038
  UNUSED(inter_ttg_upper_lmt);
3039
  if (OB_UNLIKELY(NULL == src_inter_load
3040
                  || NULL == dst_inter_load
3041
                  || NULL == balance_info)) {
3042
    ret = OB_INVALID_ARGUMENT;
3043
    LOG_WARN("invalid argument", K(ret), KP(src_inter_load),
3044
             KP(dst_inter_load), KP(balance_info));
3045
  } else {
3046
    ObIArray<UnitGroupLoad> &unitgroup_loads = balance_info->unitgroup_load_array_;
3047
    if (unitgroup_loads.count() <= 0) {
3048
      ret = OB_ERR_UNEXPECTED;
3049
      LOG_WARN("unit group load or server load empty", K(ret), K(unitgroup_loads));
3050
    } else {
3051
      if (src_inter_load->unitgroup_loads_.count() == dst_inter_load->unitgroup_loads_.count()) {
3052
        if (OB_FAIL(try_exchange_ug_balance_inter_ttg_load(
3053
                *src_inter_load, *dst_inter_load, balance_info,
3054
                unitgroup_loads, inter_ttg_upper_lmt))) {
3055
          LOG_WARN("fail to exchange ug balance inter ttg load", K(ret));
3056
        }
3057
      } else if (OB_FAIL(try_move_single_ug_balance_inter_ttg_load(
3058
              *src_inter_load, *dst_inter_load, balance_info, inter_ttg_upper_lmt))) {
3059
        LOG_WARN("fail to try move ug balance inter ttg load", K(ret));
3060
      } else if (OB_FAIL(try_exchange_ug_balance_inter_ttg_load(
3061
              *src_inter_load, *dst_inter_load, balance_info,
3062
              unitgroup_loads, inter_ttg_upper_lmt))) {
3063
        LOG_WARN("fail to try exchange ug balance inter ttg load", K(ret));
3064
      }
3065
    }
3066
  }
3067
  return ret;
3068
}
3069

3070
int ObServerBalancer::CountBalanceStrategy::try_exchange_ug_balance_inter_ttg_load(
3071
    ServerLoad &src_inter_load,
3072
    ServerLoad &dst_inter_load,
3073
    TenantGroupBalanceInfo *balance_info,
3074
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
3075
    const double inter_ttg_upper_lmt)
3076
{
3077
  int ret = OB_SUCCESS;
3078
  if (OB_UNLIKELY(NULL == balance_info)) {
3079
    ret = OB_INVALID_ARGUMENT;
3080
    LOG_WARN("invalid argument", K(ret), KP(balance_info));
3081
  } else {
3082
    int64_t max_times = src_inter_load.unitgroup_loads_.count()
3083
                        + dst_inter_load.unitgroup_loads_.count();
3084
    if (max_times >= INT32_MAX) {
3085
      ret = OB_ERR_UNEXPECTED;
3086
      LOG_WARN("unitgroup loads cnt is so large", K(ret));
3087
    } else {
3088
      max_times = max_times * max_times;
3089
    }
3090
    int64_t times = 0;
3091
    while (OB_SUCC(ret)
3092
         && times < max_times
3093
         && src_inter_load.inter_ttg_load_value_ - dst_inter_load.inter_ttg_load_value_ > EPSILON) {
3094
      bool do_exchange = false;
3095
      if (OB_FAIL(try_exchange_ug_balance_inter_ttg_load_foreach(
3096
              src_inter_load, dst_inter_load, balance_info,
3097
              unitgroup_loads, inter_ttg_upper_lmt, do_exchange))) {
3098
        LOG_WARN("fail to exchange ug balance inter ttg load foreach", K(ret));
3099
      } else if (!do_exchange) {
3100
        break;
3101
      } else {
3102
        times++;
3103
      }
3104
    }
3105
  }
3106
  return ret;
3107
}
3108

3109
int ObServerBalancer::CountBalanceStrategy::try_exchange_ug_balance_inter_ttg_load_foreach(
3110
    ServerLoad &left_inter_load,
3111
    ServerLoad &right_inter_load,
3112
    TenantGroupBalanceInfo *balance_info,
3113
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
3114
    const double inter_ttg_upper_lmt,
3115
    bool &do_exchange)
3116
{
3117
  int ret = OB_SUCCESS;
3118
  do_exchange = false;
3119
  if (OB_UNLIKELY(NULL == balance_info)) {
3120
    ret = OB_INVALID_ARGUMENT;
3121
    LOG_WARN("invalid argument", K(ret), KP(balance_info));
3122
  } else if (left_inter_load.inter_ttg_load_value_
3123
             - right_inter_load.inter_ttg_load_value_ <= EPSILON) {
3124
    // ignore, since this almost has no effect for load balance
3125
  } else {
3126
    for (int64_t i = 0;
3127
         !do_exchange && OB_SUCC(ret) && i < left_inter_load.unitgroup_loads_.count();
3128
         ++i) {
3129
      common::ObArray<common::ObAddr> excluded_servers;
3130
      UnitGroupLoad *left_ug = left_inter_load.unitgroup_loads_.at(i);
3131
      if (OB_UNLIKELY(NULL == left_ug)) {
3132
        ret = OB_ERR_UNEXPECTED;
3133
        LOG_WARN("src ug ptr is null", K(ret), KP(left_ug));
3134
      } else if (left_inter_load.inter_ttg_load_value_ < inter_ttg_upper_lmt) {
3135
        break;
3136
      } else if (OB_FAIL(get_ug_balance_excluded_dst_servers(
3137
              *left_ug, unitgroup_loads, excluded_servers))) {
3138
        LOG_WARN("fail to get excluded server", K(ret));
3139
      } else if (has_exist_in_array(excluded_servers, right_inter_load.server_)) {
3140
        // ignore this
3141
      } else {
3142
        for (int64_t j = right_inter_load.unitgroup_loads_.count() - 1;
3143
             !do_exchange && OB_SUCC(ret) && j >= 0;
3144
             --j) {
3145
          // When calculating ug load, the denominator used is right server load
3146
          double left_ug_load_value = 0.0;
3147
          double right_ug_load_value = 0.0;
3148
          UnitGroupLoad *right_ug = right_inter_load.unitgroup_loads_.at(j);
3149
          bool can_exchange = false;
3150
          if (NULL == right_ug) {
3151
            ret = OB_ERR_UNEXPECTED;
3152
            LOG_WARN("src or dst ug ptr is null", K(ret), KP(right_ug));
3153
          } else if (left_ug->column_tenant_id_ == right_ug->column_tenant_id_) {
3154
            // ignore this
3155
          } else if (OB_FAIL(get_ug_balance_excluded_dst_servers(
3156
                  *right_ug, unitgroup_loads, excluded_servers))) {
3157
            LOG_WARN("fail to get excluded servers", K(ret));
3158
          } else if (has_exist_in_array(excluded_servers, left_inter_load.server_)) {
3159
            // ignore this
3160
          } else if (left_ug->get_inter_ttg_load_value(right_inter_load, left_ug_load_value)) {
3161
            LOG_WARN("fail to get inter ttg load value", K(ret));
3162
          } else if (right_ug->get_inter_ttg_load_value(right_inter_load, right_ug_load_value)) {
3163
            LOG_WARN("fail to get inter ttg load value", K(ret));
3164
          } else if (left_ug_load_value - right_ug_load_value <= EPSILON) {
3165
            // ignore this, since this almost has no effect for load balance
3166
          } else if (right_inter_load.inter_ttg_load_value_
3167
                     + left_ug_load_value - right_ug_load_value >= inter_ttg_upper_lmt) {
3168
            // ignore this
3169
          } else if (OB_FAIL(check_can_exchange_ug_balance_inter_ttg_load(
3170
                *left_ug, left_inter_load, *right_ug, right_inter_load, can_exchange))) {
3171
            LOG_WARN("fail to check can exchange ug balance inter ttg load", K(ret));
3172
          } else if (!can_exchange) {
3173
            // ignore this
3174
          } else if (OB_FAIL(exchange_ug_between_server_loads(
3175
                *left_ug, i, left_inter_load, *right_ug, j, right_inter_load))) {
3176
            LOG_WARN("fail to exchange ug between server loads", K(ret));
3177
          } else if (OB_FAIL(coordinate_single_column_unit_group(
3178
                  left_ug->start_column_idx_, left_ug->column_count_,
3179
                  balance_info->unit_migrate_stat_matrix_,
3180
                  balance_info->unitgroup_load_array_))) {
3181
            LOG_WARN("fail to coordinate single column unit", K(ret));
3182
          } else if (OB_FAIL(coordinate_single_column_unit_group(
3183
                  right_ug->start_column_idx_, right_ug->column_count_,
3184
                  balance_info->unit_migrate_stat_matrix_,
3185
                  balance_info->unitgroup_load_array_))) {
3186
            LOG_WARN("fail to coordinate single column unit", K(ret));
3187
          } else {
3188
            do_exchange = true;
3189
          }
3190
        }
3191
      }
3192
    }
3193
    if (OB_SUCC(ret)) {
3194
      UnitGroupLoadCmp unitgroup_load_cmp(left_inter_load);
3195
      std::sort(left_inter_load.unitgroup_loads_.begin(),
3196
                left_inter_load.unitgroup_loads_.end(),
3197
                unitgroup_load_cmp);
3198
      if (OB_FAIL(unitgroup_load_cmp.get_ret())) {
3199
        LOG_WARN("fail to sort", K(ret));
3200
      }
3201
    }
3202
    if (OB_SUCC(ret)) {
3203
      UnitGroupLoadCmp unitgroup_load_cmp(right_inter_load);
3204
      std::sort(right_inter_load.unitgroup_loads_.begin(),
3205
                right_inter_load.unitgroup_loads_.end(),
3206
                unitgroup_load_cmp);
3207
      if (OB_FAIL(unitgroup_load_cmp.get_ret())) {
3208
        LOG_WARN("fail to sort", K(ret));
3209
      }
3210
    }
3211
  }
3212
  return ret;
3213
}
3214

3215
int ObServerBalancer::CountBalanceStrategy::try_move_single_ug_balance_inter_ttg_load(
3216
    ServerLoad &src_inter_load,
3217
    ServerLoad &dst_inter_load,
3218
    TenantGroupBalanceInfo *balance_info,
3219
    const double inter_ttg_upper_lmt)
3220
{
3221
  int ret = OB_SUCCESS;
3222
  if (OB_UNLIKELY(NULL == balance_info)) {
3223
    ret = OB_INVALID_ARGUMENT;
3224
    LOG_WARN("invalid argument", K(ret), KP(balance_info));
3225
  } else {
3226
    common::ObIArray<UnitGroupLoad> &unitgroup_loads = balance_info->unitgroup_load_array_;
3227
    if (src_inter_load.unitgroup_loads_.count() <= dst_inter_load.unitgroup_loads_.count()) {
3228
      // ignore since, src intra cnt is not greater than dst
3229
    } else if (src_inter_load.inter_ttg_load_value_ <= dst_inter_load.inter_ttg_load_value_) {
3230
      // ignore since src intra load is not greater than dst
3231
    } else {
3232
      ObArray<common::ObAddr> excluded_dst_servers;
3233
      for (int64_t i = src_inter_load.unitgroup_loads_.count() - 1;
3234
           OB_SUCC(ret) && i >= 0;
3235
           --i) {
3236
        UnitGroupLoad &ug_load = *src_inter_load.unitgroup_loads_.at(i);
3237
        double ug_load_value = 0.0;
3238
        bool can_move = false;
3239
        excluded_dst_servers.reset();
3240
        if (OB_FAIL(get_ug_balance_excluded_dst_servers(
3241
                ug_load, unitgroup_loads, excluded_dst_servers))) {
3242
          LOG_WARN("fail to get excluded server", K(ret));
3243
        } else if (has_exist_in_array(excluded_dst_servers, dst_inter_load.server_)) {
3244
          // in excluded servers, ignore
3245
        } else if (OB_FAIL(ug_load.get_inter_ttg_load_value(dst_inter_load, ug_load_value))) {
3246
          LOG_WARN("fail to get inter ttg load value", K(ret));
3247
        } else if (dst_inter_load.inter_ttg_load_value_ + ug_load_value >= inter_ttg_upper_lmt) {
3248
          // ignore, since this amendment will violate the balance in terms of inter ttg
3249
        } else if (OB_FAIL(check_can_move_ug_balance_inter_ttg_load(
3250
                src_inter_load, ug_load, dst_inter_load, can_move))) {
3251
          LOG_WARN("fail to check can move ug balance inter ttg load", K(ret));
3252
        } else if (!can_move) {
3253
          // ignore
3254
        } else if (OB_FAIL(move_ug_between_server_loads(
3255
                src_inter_load, dst_inter_load, ug_load, i))) {
3256
          LOG_WARN("fail to move ug between server loads", K(ret));
3257
        } else if (OB_FAIL(coordinate_single_column_unit_group(
3258
                ug_load.start_column_idx_, ug_load.column_count_,
3259
                balance_info->unit_migrate_stat_matrix_,
3260
                balance_info->unitgroup_load_array_))) {
3261
          LOG_WARN("fail to coordinate single column unit", K(ret));
3262
        } else {
3263
          break;
3264
        }
3265
      }
3266
      if (OB_SUCC(ret)) {
3267
        UnitGroupLoadCmp unitgroup_load_cmp(src_inter_load);
3268
        std::sort(src_inter_load.unitgroup_loads_.begin(),
3269
                  src_inter_load.unitgroup_loads_.end(),
3270
                  unitgroup_load_cmp);
3271
        if (OB_FAIL(unitgroup_load_cmp.get_ret())) {
3272
          LOG_WARN("fail to sort", K(ret));
3273
        }
3274
      }
3275
      if (OB_SUCC(ret)) {
3276
        UnitGroupLoadCmp unitgroup_load_cmp(dst_inter_load);
3277
        std::sort(dst_inter_load.unitgroup_loads_.begin(),
3278
                  dst_inter_load.unitgroup_loads_.end(),
3279
                  unitgroup_load_cmp);
3280
        if (OB_FAIL(unitgroup_load_cmp.get_ret())) {
3281
          LOG_WARN("fail to sort", K(ret));
3282
        }
3283
      }
3284
    }
3285
  }
3286
  return ret;
3287
}
3288

3289
int ObServerBalancer::try_execute_unit_balance_task(
3290
    const common::ObZone &zone,
3291
    TenantGroupBalanceInfo *info_need_amend,
3292
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
3293
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units,
3294
    const common::ObIArray<TenantGroupBalanceInfo *> &unstable_tenant_group,
3295
    const common::ObIArray<common::ObAddr> &available_servers,
3296
    bool &do_amend)
3297
{
3298
  int ret = OB_SUCCESS;
3299
  common::ObArray<UnitMigrateStat *> task_array;
3300
  if (OB_UNLIKELY(!inited_)) {
3301
    ret = OB_NOT_INIT;
3302
    LOG_WARN("not init", K(ret));
3303
  } else if (OB_UNLIKELY(NULL == info_need_amend)) {
3304
    ret = OB_INVALID_ARGUMENT;
3305
    LOG_WARN("invalid argument", K(ret));
3306
  } else if (info_need_amend->is_stable()) {
3307
    do_amend = false; // balance info is stable, no need to execute unit balance task
3308
  } else if (info_need_amend->unit_migrate_stat_matrix_.get_element_count() <= 0
3309
             || info_need_amend->unit_migrate_stat_matrix_.get_element_count() >= INT32_MAX) {
3310
    ret = OB_INVALID_ARGUMENT;
3311
    LOG_WARN("invalid argument", K(ret), K(*info_need_amend));
3312
  } else if (OB_FAIL(task_array.reserve(
3313
          info_need_amend->unit_migrate_stat_matrix_.get_element_count()))) {
3314
    LOG_WARN("fail to reserve", K(ret));
3315
  } else if (OB_FAIL(generate_unit_balance_task(*info_need_amend, task_array))) {
3316
    LOG_WARN("fail to generate unit balance task", K(ret));
3317
  } else if (task_array.count() <= 0) {
3318
    if (OB_FAIL(vacate_space_for_ttg_balance(
3319
            zone, *info_need_amend, standalone_units, not_grant_units,
3320
            unstable_tenant_group, available_servers, do_amend))) {
3321
      LOG_WARN("fail to vacate space for ttg balance", K(ret));
3322
    }
3323
  } else if (OB_FAIL(do_migrate_unit_task(task_array))) {
3324
    LOG_WARN("fail to do migrate unit task array", K(ret));
3325
  } else {
3326
    do_amend = true; // A migration task is executed
3327
  }
3328
  return ret;
3329
}
3330

3331
int ObServerBalancer::do_migrate_unit_task(
3332
    const common::ObIArray<UnitMigrateStat> &task_array)
3333
{
3334
  int ret = OB_SUCCESS;
3335
  if (OB_UNLIKELY(!inited_)) {
3336
    ret = OB_NOT_INIT;
3337
    LOG_WARN("not init", K(ret));
3338
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
3339
    ret = OB_ERR_UNEXPECTED;
3340
    LOG_WARN("unit_mgr_ ptr is null", K(ret), KP(unit_mgr_));
3341
  } else {
3342
    for (int64_t i = 0; OB_SUCC(ret) && i < task_array.count(); ++i) {
3343
      ObUnitStat unit_stat;
3344
      ObArray<ObUnitStat> in_migrate_unit_stat;
3345
      const UnitMigrateStat &unit_migrate_stat = task_array.at(i);
3346
      bool can_migrate_in = false;
3347
      if (!unit_migrate_stat.unit_load_.is_valid()) {
3348
        ret = OB_ERR_UNEXPECTED;
3349
        LOG_WARN("invalid argument", K(ret), "unit_load", unit_migrate_stat.unit_load_);
3350
      } else if (OB_FAIL(SVR_TRACER.check_server_can_migrate_in(
3351
              unit_migrate_stat.arranged_pos_, can_migrate_in))) {
3352
        LOG_WARN("fail to check can migrate in", K(ret));
3353
      } else if (!can_migrate_in) {
3354
        // bypass
3355
      } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
3356
              unit_migrate_stat.unit_load_.unit_->unit_id_,
3357
              unit_migrate_stat.unit_load_.unit_->zone_,
3358
              unit_stat))) {
3359
        LOG_WARN("fail to get unit stat", K(ret));
3360
      } else if (OB_FAIL(get_unit_resource_reservation(
3361
                  unit_migrate_stat.unit_load_.unit_->unit_id_, unit_migrate_stat.arranged_pos_,
3362
                  in_migrate_unit_stat))) {
3363
        LOG_WARN("fail to get unit resource reservation", K(ret));
3364
      } else if (OB_FAIL(try_migrate_unit(
3365
              unit_migrate_stat.unit_load_.unit_->unit_id_,
3366
              unit_migrate_stat.unit_load_.pool_->tenant_id_, unit_stat,
3367
              in_migrate_unit_stat, unit_migrate_stat.arranged_pos_))) {
3368
        if (OB_OP_NOT_ALLOW == ret) {
3369
          LOG_INFO("cannot migrate unit, server full",
3370
                   "unit", *unit_migrate_stat.unit_load_.unit_,
3371
                   "dst_server", unit_migrate_stat.arranged_pos_);
3372
          ret = OB_SUCCESS; // ingore this unit
3373
        } else {
3374
          LOG_WARN("fail to migrate unit", K(ret),
3375
                   "unit", *unit_migrate_stat.unit_load_.unit_,
3376
                   "dst_server", unit_migrate_stat.arranged_pos_);
3377
        }
3378
      } else {} // no more to do
3379
    }
3380
  }
3381
  return ret;
3382
}
3383

3384
int ObServerBalancer::do_migrate_unit_task(
3385
    const common::ObIArray<UnitMigrateStat *> &task_array)
3386
{
3387
  int ret = OB_SUCCESS;
3388
  if (OB_UNLIKELY(!inited_)) {
3389
    ret = OB_NOT_INIT;
3390
    LOG_WARN("not init", K(ret));
3391
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
3392
    ret = OB_ERR_UNEXPECTED;
3393
    LOG_WARN("unit_mgr_ ptr is null", K(ret), KP(unit_mgr_));
3394
  } else {
3395
    for (int64_t i = 0; OB_SUCC(ret) && i < task_array.count(); ++i) {
3396
      ObUnitStat unit_stat;
3397
      ObArray<ObUnitStat> in_migrate_unit_stat;
3398
      const UnitMigrateStat *unit_migrate_stat = task_array.at(i);
3399
      bool can_migrate_in = false;
3400
      if (OB_UNLIKELY(NULL == unit_migrate_stat)) {
3401
        ret = OB_ERR_UNEXPECTED;
3402
        LOG_WARN("unit migrate stat ptr is null", K(ret), KP(unit_migrate_stat));
3403
      } else if (!unit_migrate_stat->unit_load_.is_valid()) {
3404
        ret = OB_ERR_UNEXPECTED;
3405
        LOG_WARN("invalid argument", K(ret), "unit_load", unit_migrate_stat->unit_load_);
3406
      } else if (OB_FAIL(SVR_TRACER.check_server_can_migrate_in(
3407
          unit_migrate_stat->arranged_pos_,
3408
          can_migrate_in))) {
3409
        LOG_WARN("fail to check can migrate in", K(ret));
3410
      } else if (!can_migrate_in) {
3411
        // bypass
3412
      } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
3413
              unit_migrate_stat->unit_load_.unit_->unit_id_,
3414
              unit_migrate_stat->unit_load_.unit_->zone_,
3415
              unit_stat))) {
3416
        LOG_WARN("fail to get unit stat", K(ret));
3417
      } else if (OB_FAIL(get_unit_resource_reservation(
3418
              unit_migrate_stat->unit_load_.unit_->unit_id_, unit_migrate_stat->arranged_pos_,
3419
              in_migrate_unit_stat))) {
3420
        LOG_WARN("fail to get unit resource reservation", K(ret));
3421
      } else if (OB_FAIL(try_migrate_unit(
3422
              unit_migrate_stat->unit_load_.unit_->unit_id_,
3423
              unit_migrate_stat->unit_load_.pool_->tenant_id_, unit_stat,
3424
              in_migrate_unit_stat, unit_migrate_stat->arranged_pos_))) {
3425
        if (OB_OP_NOT_ALLOW == ret) {
3426
          LOG_INFO("cannot migrate unit, server full",
3427
                   "unit", *unit_migrate_stat->unit_load_.unit_,
3428
                   "dst_server", unit_migrate_stat->arranged_pos_);
3429
          ret = OB_SUCCESS; // ingore this unit
3430
        } else {
3431
          LOG_WARN("fail to migrate unit", K(ret),
3432
                   "unit", *unit_migrate_stat->unit_load_.unit_,
3433
                   "dst_server", unit_migrate_stat->arranged_pos_);
3434
        }
3435
      } else {} // no more to do
3436
    }
3437
  }
3438
  return ret;
3439
}
3440

3441
// 1. Try to perform all tasks in the matrix, if all tasks cannot be performed
3442
// 2. Try to execute the first row of tasks in the matrix,
3443
//    if the first row of tasks cannot be executed
3444
// 3. Try to execute the first task in the matrix,
3445
//    if the first one cannot be executed, give up
3446
int ObServerBalancer::generate_unit_balance_task(
3447
    TenantGroupBalanceInfo &balance_info,
3448
    common::ObIArray<UnitMigrateStat *> &task_array)
3449
{
3450
  int ret = OB_SUCCESS;
3451
  LOG_INFO("tenant group migrate unit stat matrix",
3452
           "unit migrate stat matrix", balance_info.unit_migrate_stat_matrix_);
3453
  if (OB_UNLIKELY(!inited_)) {
3454
    ret = OB_NOT_INIT;
3455
    LOG_WARN("not init", K(ret));
3456
  } else if (OB_FAIL(try_generate_square_task_from_ttg_matrix(
3457
          balance_info.unit_migrate_stat_matrix_, task_array))) {
3458
    LOG_WARN("fail to generate balance task from ttg matrix", K(ret));
3459
  } else if (task_array.count() > 0) {
3460
    // good, has task generated
3461
  } else if (OB_FAIL(try_generate_line_task_from_ttg_matrix(
3462
          balance_info.unit_migrate_stat_matrix_, task_array))) {
3463
    LOG_WARN("fail to generate balance task from ttg matrix", K(ret));
3464
  } else if (task_array.count() > 0) {
3465
    // good, has task generated
3466
  } else if (OB_FAIL(try_generate_dot_task_from_ttg_matrix(
3467
          balance_info.unit_migrate_stat_matrix_, task_array))) {
3468
    LOG_WARN("fail to generate balance task from ttg matrix", K(ret));
3469
  } else if (task_array.count() > 0) {
3470
    // good, has task generated
3471
  } else {
3472
    // no task generated, sad
3473
  }
3474
  return ret;
3475
}
3476

3477
int ObServerBalancer::check_unit_migrate_stat_unit_collide(
3478
    UnitMigrateStat &unit_migrate_stat,
3479
    ObUnitManager::ObUnitLoad &collide_unit,
3480
    bool &tenant_unit_collide)
3481
{
3482
  int ret = OB_SUCCESS;
3483
  common::ObArray<ObUnitManager::ObUnitLoad> zone_units;
3484
  if (OB_UNLIKELY(!inited_)) {
3485
    ret = OB_NOT_INIT;
3486
    LOG_WARN("not init", K(ret));
3487
  } else if (OB_UNLIKELY(common::OB_INVALID_ID == unit_migrate_stat.tenant_id_)) {
3488
    ret = OB_INVALID_ARGUMENT;
3489
    LOG_WARN("invalid argument", K(ret), K(unit_migrate_stat));
3490
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
3491
    ret = OB_ERR_UNEXPECTED;
3492
    LOG_WARN("unit_mgr_ ptr is null", K(ret));
3493
  } else if (OB_FAIL(unit_mgr_->get_tenant_zone_all_unit_loads(
3494
          unit_migrate_stat.tenant_id_, zone_disk_statistic_.zone_, zone_units))) {
3495
    LOG_WARN("fail to get tenant zone all unit loads", K(ret), K(unit_migrate_stat));
3496
  } else {
3497
    tenant_unit_collide = false;
3498
    for (int64_t i = 0; !tenant_unit_collide && OB_SUCC(ret) && i < zone_units.count(); ++i) {
3499
      ObUnitManager::ObUnitLoad &unit_load = zone_units.at(i);
3500
      if (!unit_load.is_valid()) {
3501
        ret = OB_ERR_UNEXPECTED;
3502
        LOG_WARN("unit load is invalid", K(ret), K(unit_load));
3503
      } else if (unit_load.unit_->server_ == unit_migrate_stat.arranged_pos_) {
3504
        tenant_unit_collide = true;
3505
        collide_unit = unit_load;
3506
      } else {} // good, go on to check next
3507
    }
3508
  }
3509
  return ret;
3510
}
3511

3512
int ObServerBalancer::accumulate_balance_task_loadsum(
3513
    const UnitMigrateStat &unit_migrate_stat,
3514
    common::ObIArray<ServerLoadSum> &server_load_sums)
3515
{
3516
  int ret = OB_SUCCESS;
3517
  if (OB_UNLIKELY(!inited_)) {
3518
    ret = OB_NOT_INIT;
3519
    LOG_WARN("not init", K(ret));
3520
  } else if (OB_UNLIKELY(!unit_migrate_stat.unit_load_.is_valid())) {
3521
    ret = OB_INVALID_ARGUMENT;
3522
    LOG_WARN("invalid argument", K(ret));
3523
  } else {
3524
    ObUnitStat unit_stat;
3525
    int64_t idx = 0;
3526
    for (; idx < server_load_sums.count(); ++idx) {
3527
      ServerLoadSum &server_load_sum = server_load_sums.at(idx);
3528
      if (server_load_sum.server_ != unit_migrate_stat.arranged_pos_) {
3529
        // next
3530
      } else {
3531
        break; // find one
3532
      }
3533
    }
3534
    if (idx >= server_load_sums.count()) {
3535
      ServerLoadSum server_load_sum;
3536
      server_load_sum.server_ = unit_migrate_stat.arranged_pos_;
3537
      if (OB_FAIL(server_load_sums.push_back(server_load_sum))) {
3538
        LOG_WARN("fail to push back", K(ret));
3539
      }
3540
    }
3541
    if (OB_FAIL(ret)) {
3542
    } else if (idx >= server_load_sums.count()) {
3543
      ret = OB_ERR_UNEXPECTED;
3544
      LOG_WARN("server load sums count unexpecte", K(ret), K(idx));
3545
    } else if (OB_FAIL(server_load_sums.at(idx).load_sum_.append_load(
3546
            *unit_migrate_stat.unit_load_.unit_config_))) {
3547
      LOG_WARN("fail to append load", K(ret));
3548
    } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
3549
            unit_migrate_stat.unit_load_.unit_->unit_id_,
3550
            unit_migrate_stat.unit_load_.unit_->zone_,
3551
            unit_stat))) {
3552
      LOG_WARN("fail to get unit stat", K(ret));
3553
    } else {
3554
      server_load_sums.at(idx).disk_in_use_ += unit_stat.get_required_size();
3555
    }
3556
  }
3557
  return ret;
3558
}
3559

3560
int ObServerBalancer::check_servers_resource_enough(
3561
    const common::ObIArray<ServerLoadSum> &server_load_sums,
3562
    bool &enough)
3563
{
3564
  int ret = OB_SUCCESS;
3565
  double hard_limit = 0.0;
3566
  double disk_waterlevel = 0.0;
3567
  if (OB_UNLIKELY(!inited_)) {
3568
    ret = OB_NOT_INIT;
3569
    LOG_WARN("not init", K(ret));
3570
  } else if (OB_ISNULL(unit_mgr_) || OB_ISNULL(server_mgr_)) {
3571
    ret = OB_ERR_UNEXPECTED;
3572
    LOG_WARN("unit_mgr_ or server_mgr_is null", K(ret), KP(unit_mgr_), KP(server_mgr_));
3573
  } else if (OB_FAIL(unit_mgr_->get_hard_limit(hard_limit))) {
3574
    LOG_WARN("fail to hard limit", K(ret));
3575
  } else if (OB_FAIL(get_server_balance_critical_disk_waterlevel(disk_waterlevel))) {
3576
    LOG_WARN("fail to get critical disk waterlevel", K(ret));
3577
  } else {
3578
    enough = true;
3579
    for (int64_t i = 0; OB_SUCC(ret) && enough && i < server_load_sums.count(); ++i) {
3580
      ObArray<ObUnitManager::ObUnitLoad> *unit_loads = NULL;
3581
      share::ObServerResourceInfo server_resource_info;
3582
      const common::ObAddr &server = server_load_sums.at(i).server_;
3583
      LoadSum load_sum = server_load_sums.at(i).load_sum_;
3584
      int64_t disk_in_use = server_load_sums.at(i).disk_in_use_;
3585
      ServerDiskStatistic disk_statistic;
3586
      if (OB_FAIL(zone_disk_statistic_.get_server_disk_statistic(server, disk_statistic))) {
3587
        LOG_WARN("fail to get disk statistic", K(ret), K(server));
3588
      } else if (OB_FAIL(server_mgr_->get_server_resource_info(server, server_resource_info))) {
3589
        // **TODO (linqiucen.lqc): temp.solution
3590
        LOG_WARN("fail to get server resource info", KR(ret), K(server));
3591
      } else if (OB_FAIL(unit_mgr_->get_loads_by_server(server, unit_loads))) {
3592
        if (OB_ENTRY_NOT_EXIST != ret) {
3593
          LOG_WARN("fail to get loads by server", K(ret));
3594
        } else {
3595
          ret = OB_SUCCESS;
3596
          // unit_loads does not exist, there is no load on this server
3597
        }
3598
      } else if (NULL == unit_loads) {
3599
        ret = OB_ERR_UNEXPECTED;
3600
        LOG_WARN("unit loads ptr is null", K(ret));
3601
      } else {
3602
        for (int64_t j = 0; OB_SUCC(ret) && j < unit_loads->count(); ++j) {
3603
          const ObUnitManager::ObUnitLoad &load = unit_loads->at(j);
3604
          if (!load.is_valid()) {
3605
            ret = OB_ERR_UNEXPECTED;
3606
            LOG_WARN("load is invalid", K(ret));
3607
          } else if (OB_FAIL(load_sum.append_load(*load.unit_config_))) {
3608
            LOG_WARN("fail to append", K(ret));
3609
          } else {} // no more to do
3610
        }
3611
      }
3612
      if (OB_SUCC(ret)) {
3613
        if (load_sum.load_sum_.max_cpu()
3614
                 > server_resource_info.cpu_ * hard_limit
3615
            || load_sum.load_sum_.min_cpu()
3616
                 > server_resource_info.cpu_
3617
            || static_cast<double>(load_sum.load_sum_.memory_size())
3618
                 > static_cast<double>(server_resource_info.mem_total_)
3619
            || static_cast<double>(load_sum.load_sum_.log_disk_size())
3620
                 > static_cast<double>(server_resource_info.log_disk_total_)
3621
            || static_cast<double>(disk_in_use + disk_statistic.disk_in_use_)
3622
                 > static_cast<double>(disk_statistic.disk_total_) * disk_waterlevel) {
3623
          enough = false;
3624
          LOG_INFO("resource not enough", K(load_sum), K(hard_limit), K(disk_waterlevel),
3625
                   K(disk_in_use), K(disk_statistic),
3626
                   "server_load_sum", server_load_sums.at(i));
3627
        }
3628
      }
3629
    }
3630
  }
3631
  return ret;
3632
}
3633

3634
int ObServerBalancer::try_generate_square_task_from_ttg_matrix(
3635
    Matrix<UnitMigrateStat> &unit_migrate_stat_task,
3636
    common::ObIArray<UnitMigrateStat *> &task_array)
3637
{
3638
  int ret = OB_SUCCESS;
3639
  task_array.reset();
3640
  common::ObArray<ServerLoadSum> server_load_sums;
3641
  if (OB_UNLIKELY(!inited_)) {
3642
    ret = OB_NOT_INIT;
3643
    LOG_WARN("not init", K(ret));
3644
  } else {
3645
    ObUnitManager::ObUnitLoad dummy_collide_unit;
3646
    bool tenant_unit_collide = false;
3647
    for (int64_t i = 0;
3648
         !tenant_unit_collide && OB_SUCC(ret) && i < unit_migrate_stat_task.get_row_count();
3649
         ++i) {
3650
      for (int64_t j = 0;
3651
           !tenant_unit_collide && OB_SUCC(ret) && j < unit_migrate_stat_task.get_column_count();
3652
           ++j) {
3653
        UnitMigrateStat *ptr = unit_migrate_stat_task.get(i, j);
3654
        if (OB_UNLIKELY(NULL == ptr)) {
3655
          ret = OB_ERR_UNEXPECTED;
3656
          LOG_WARN("unit migrate stat ptr is null", K(ret));
3657
        } else if (ptr->original_pos_ == ptr->arranged_pos_) {
3658
          // ignore since this do not need to migrate
3659
        } else if (OB_FAIL(check_unit_migrate_stat_unit_collide(
3660
                *ptr, dummy_collide_unit, tenant_unit_collide))) {
3661
          LOG_WARN("fail to check unit migrate stat unit collide", K(ret));
3662
        } else if (tenant_unit_collide) {
3663
          LOG_INFO("migrate unit collide", "migrate_unit", *ptr,
3664
                   "collide_unit", dummy_collide_unit);
3665
        } else if (OB_FAIL(accumulate_balance_task_loadsum(*ptr, server_load_sums))) {
3666
          LOG_WARN("fail to accumulate balance task loadsum", K(ret));
3667
        } else if (OB_FAIL(task_array.push_back(ptr))) {
3668
          LOG_WARN("fail to push back", K(ret));
3669
        } else {} // no more to do
3670
      }
3671
    }
3672
    bool enough = true;
3673
    if (OB_FAIL(ret)) {
3674
      // previous procedure failed
3675
    } else if (tenant_unit_collide) {
3676
      task_array.reset();
3677
      // Matrix task cannot be executed at the same time, clear task
3678
    } else if (OB_FAIL(check_servers_resource_enough(server_load_sums, enough))) {
3679
      LOG_WARN("fail to check server resource capacity", K(ret));
3680
    } else if (enough) {
3681
      // good, return the task array
3682
    } else {
3683
      LOG_INFO("server resource not enough");
3684
      task_array.reset();
3685
      // Matrix task cannot be executed at the same time, clear task
3686
    }
3687
  }
3688
  return ret;
3689
}
3690

3691
int ObServerBalancer::try_generate_line_task_from_ttg_matrix(
3692
    Matrix<UnitMigrateStat> &unit_migrate_stat_task,
3693
    common::ObIArray<UnitMigrateStat *> &task_array)
3694
{
3695
  int ret = OB_SUCCESS;
3696
  task_array.reset();
3697
  common::ObArray<ServerLoadSum> server_load_sums;
3698
  if (OB_UNLIKELY(!inited_)) {
3699
    ret = OB_NOT_INIT;
3700
    LOG_WARN("not init", K(ret));
3701
  } else {
3702
    ObUnitManager::ObUnitLoad dummy_collide_unit;
3703
    for (int64_t i = 0; OB_SUCC(ret) && i < unit_migrate_stat_task.get_row_count(); ++i) {
3704
      server_load_sums.reset();
3705
      bool enough = true;
3706
      bool tenant_unit_collide = false;
3707
      for (int64_t j = 0;
3708
           !tenant_unit_collide && OB_SUCC(ret) && j < unit_migrate_stat_task.get_column_count();
3709
           ++j) {
3710
        UnitMigrateStat *ptr = unit_migrate_stat_task.get(i, j);
3711
        if (OB_UNLIKELY(NULL == ptr)) {
3712
          ret = OB_ERR_UNEXPECTED;
3713
          LOG_WARN("unit migrate stat ptr is null", K(ret));
3714
        } else if (ptr->original_pos_ == ptr->arranged_pos_) {
3715
          // ignore since this do not need to migrate
3716
        } else if (OB_FAIL(check_unit_migrate_stat_unit_collide(
3717
                *ptr, dummy_collide_unit, tenant_unit_collide))) {
3718
          LOG_WARN("fail to check unit migrate stat unit collide", K(ret));
3719
        } else if (tenant_unit_collide) {
3720
          LOG_INFO("migrate unit collide", "migrate_unit", *ptr,
3721
                   "collide_unit", dummy_collide_unit);
3722
        } else if (OB_FAIL(accumulate_balance_task_loadsum(*ptr, server_load_sums))) {
3723
          LOG_WARN("fail to accumulate balance task loadsum", K(ret));
3724
        } else if (OB_FAIL(task_array.push_back(ptr))) {
3725
          LOG_WARN("fail to push back", K(ret));
3726
        } else {} // no more to do
3727
      }
3728
      if (OB_FAIL(ret)) {
3729
      } else if (tenant_unit_collide) {
3730
        task_array.reset();
3731
        break;
3732
      } else if (task_array.count() <= 0) {
3733
        // do nothing
3734
      } else if (OB_FAIL(check_servers_resource_enough(server_load_sums, enough))) {
3735
        LOG_WARN("fail to check server resource capacity", K(ret));
3736
      } else {
3737
        if (enough) {
3738
          // good, return the task array
3739
        } else {
3740
          LOG_INFO("server resource not enough");
3741
          task_array.reset();
3742
          // row task cannot be executed at the same time, clear task
3743
        }
3744
        break;
3745
      }
3746
    }
3747
  }
3748
  return ret;
3749
}
3750

3751
int ObServerBalancer::try_generate_dot_task_from_migrate_stat(
3752
    UnitMigrateStat &unit_migrate_stat,
3753
    common::ObIArray<UnitMigrateStat *> &task_array)
3754
{
3755
  int ret = OB_SUCCESS;
3756
  common::ObArray<ServerLoadSum> server_load_sums;
3757
  ObUnitManager::ObUnitLoad dummy_collide_unit;
3758
  bool tenant_unit_collide = false;
3759
  bool enough = true;
3760
  if (OB_UNLIKELY(!inited_)) {
3761
    ret = OB_NOT_INIT;
3762
    LOG_WARN("not init", K(ret));
3763
  } else if (unit_migrate_stat.original_pos_ == unit_migrate_stat.arranged_pos_) {
3764
    // ignore since this do not need to migrate
3765
  } else if (OB_FAIL(check_unit_migrate_stat_unit_collide(
3766
          unit_migrate_stat, dummy_collide_unit, tenant_unit_collide))) {
3767
    LOG_WARN("fail to check unit migrate stat unit", K(ret));
3768
  } else if (tenant_unit_collide) {
3769
    LOG_INFO("migrate unit collide", K(unit_migrate_stat), K(dummy_collide_unit));
3770
    task_array.reset();
3771
  } else if (OB_FAIL(accumulate_balance_task_loadsum(unit_migrate_stat, server_load_sums))) {
3772
    LOG_WARN("fail to accumulate balance task loadsum", K(ret));
3773
  } else if (OB_FAIL(task_array.push_back(&unit_migrate_stat))) {
3774
    LOG_WARN("fail to push back", K(ret));
3775
  } else if (OB_FAIL(check_servers_resource_enough(server_load_sums, enough))) {
3776
    LOG_WARN("fail to check server resource capacity", K(ret));
3777
  } else if (enough) {
3778
    // good
3779
  } else {
3780
    LOG_INFO("server resource not enough", K(server_load_sums));
3781
    task_array.reset();
3782
  }
3783
  return ret;
3784
}
3785

3786
int ObServerBalancer::try_generate_dot_task_from_ttg_matrix(
3787
    Matrix<UnitMigrateStat> &unit_migrate_stat_task,
3788
    common::ObIArray<UnitMigrateStat *> &task_array)
3789
{
3790
  int ret = OB_SUCCESS;
3791
  task_array.reset();
3792
  if (OB_UNLIKELY(!inited_)) {
3793
    ret = OB_NOT_INIT;
3794
    LOG_WARN("not init", K(ret));
3795
  } else if (unit_migrate_stat_task.get_row_count() <= 0
3796
             || unit_migrate_stat_task.get_column_count() <= 0) {
3797
    ret = OB_ERR_UNEXPECTED;
3798
    LOG_WARN("unit migrate stat task array unexpected", K(ret), K(unit_migrate_stat_task));
3799
  } else {
3800
    bool first_row_stable = true;
3801
    for (int64_t column = 0;
3802
         first_row_stable && OB_SUCC(ret) && column < unit_migrate_stat_task.get_column_count();
3803
         ++column) {
3804
      UnitMigrateStat *ptr = unit_migrate_stat_task.get(0, column);
3805
      if (OB_UNLIKELY(NULL == ptr)) {
3806
        ret = OB_ERR_UNEXPECTED;
3807
        LOG_WARN("unit migrate stat ptr is null", K(ret));
3808
      } else if (ptr->original_pos_ == ptr->arranged_pos_) {
3809
        // good, and bypass
3810
      } else {
3811
        first_row_stable = false;
3812
      }
3813
    }
3814
    if (OB_FAIL(ret)) {
3815
    } else if (first_row_stable) {
3816
      // The first line is the reference line, no migration is required, starting from the second line
3817
      bool finish = false;
3818
      for (int64_t i = 1;
3819
           !finish && OB_SUCC(ret) && i < unit_migrate_stat_task.get_row_count();
3820
           ++i) {
3821
        for (int64_t j = 0;
3822
             !finish && OB_SUCC(ret) && j < unit_migrate_stat_task.get_column_count();
3823
             ++j) {
3824
          UnitMigrateStat *ptr = unit_migrate_stat_task.get(i, j);
3825
          if (OB_UNLIKELY(NULL == ptr)) {
3826
            ret = OB_ERR_UNEXPECTED;
3827
            LOG_WARN("unit migrate stat ptr is null", K(ret));
3828
          } else if (OB_FAIL(try_generate_dot_task_from_migrate_stat(*ptr, task_array))) {
3829
            LOG_WARN("fail to try generate dot task from migrate stat", K(ret));
3830
          } else if (task_array.count() > 0) {
3831
            finish = true;
3832
          } else {} // not finish, go on next
3833
        }
3834
      }
3835
    } else {
3836
      bool finish = false;
3837
      for (int64_t column = 0;
3838
           !finish && OB_SUCC(ret) && column < unit_migrate_stat_task.get_column_count();
3839
           ++column) {
3840
        UnitMigrateStat *ptr = unit_migrate_stat_task.get(0, column);
3841
        if (OB_UNLIKELY(NULL == ptr)) {
3842
          ret = OB_ERR_UNEXPECTED;
3843
          LOG_WARN("unit migrate stat ptr is null", K(ret));
3844
        } else if (OB_FAIL(try_generate_dot_task_from_migrate_stat(*ptr, task_array))) {
3845
          LOG_WARN("fail to try generate dot task from migrate stat", K(ret));
3846
        } else if (task_array.count() > 0) {
3847
          finish = true;
3848
        } else {} // not finish, go on next
3849
      }
3850
    }
3851
  }
3852
  return ret;
3853
}
3854

3855
int ObServerBalancer::vacate_space_for_ttg_balance(
3856
    const common::ObZone &zone,
3857
    TenantGroupBalanceInfo &balance_info,
3858
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
3859
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units,
3860
    const common::ObIArray<TenantGroupBalanceInfo *> &unstable_tenant_group,
3861
    const common::ObIArray<common::ObAddr> &available_servers,
3862
    bool &do_amend)
3863
{
3864
  int ret = OB_SUCCESS;
3865
  double g_res_weights[RES_MAX];
3866
  ObArray<ServerTotalLoad> server_loads;
3867
  if (OB_UNLIKELY(!inited_)) {
3868
    ret = OB_NOT_INIT;
3869
    LOG_WARN("not init", K(ret));
3870
  } else if (OB_FAIL(calc_global_balance_resource_weights(
3871
         zone, available_servers, g_res_weights, RES_MAX))) {
3872
    LOG_WARN("fail to calc whole balance resource weights", K(ret));
3873
  } else if (OB_FAIL(generate_complete_server_loads(
3874
          zone, available_servers, g_res_weights, RES_MAX, server_loads))) {
3875
    LOG_WARN("fail to generate complete server loads", K(ret));
3876
  } else {
3877
    bool find = false;
3878
    Matrix<UnitMigrateStat> &migrate_matrix = balance_info.unit_migrate_stat_matrix_;
3879
    for (int64_t i = 0;
3880
         !find && OB_SUCC(ret) && i < migrate_matrix.get_row_count();
3881
         ++i) {
3882
      for (int64_t j = 0;
3883
           !find && OB_SUCC(ret) && j < migrate_matrix.get_column_count();
3884
           ++j) {
3885
        UnitMigrateStat *unit_migrate_stat = migrate_matrix.get(i, j);
3886
        bool tenant_unit_collide = false;
3887
        ObUnitManager::ObUnitLoad collide_unit;
3888
        if (OB_UNLIKELY(NULL == unit_migrate_stat)) {
3889
          ret = OB_ERR_UNEXPECTED;
3890
          LOG_WARN("migrate stat ptr is null", K(ret), KP(unit_migrate_stat));
3891
        } else if (unit_migrate_stat->original_pos_ == unit_migrate_stat->arranged_pos_) {
3892
          // ignore this, since this do not need to amend
3893
        } else if (OB_FAIL(check_unit_migrate_stat_unit_collide(
3894
                *unit_migrate_stat, collide_unit, tenant_unit_collide))) {
3895
          LOG_WARN("fail to check unit migrate stat unit collide", K(ret));
3896
        } else if (tenant_unit_collide) {
3897
          if (OB_FAIL(do_migrate_out_collide_unit(
3898
                  *unit_migrate_stat, collide_unit, server_loads, do_amend))) {
3899
            LOG_WARN("fail to do migrate out collide unit", K(ret));
3900
          } else {
3901
            find = true;
3902
          }
3903
        } else {
3904
          if (OB_FAIL(do_vacate_space_for_ttg_balance(
3905
                  *unit_migrate_stat, standalone_units, not_grant_units,
3906
                  unstable_tenant_group, server_loads, do_amend))) {
3907
            LOG_WARN("fail to do vacate space for ttg balance", K(ret));
3908
          } else {
3909
            find = true;
3910
          }
3911
        }
3912
      }
3913
    }
3914
  }
3915
  return ret;
3916
}
3917

3918
int ObServerBalancer::do_migrate_out_collide_unit(
3919
    UnitMigrateStat &unit_migrate_stat,
3920
    ObUnitManager::ObUnitLoad &collide_unit_load,
3921
    common::ObArray<ServerTotalLoad> &server_loads,
3922
    bool &do_amend)
3923
{
3924
  int ret = OB_SUCCESS;
3925
  common::ObArray<ObUnitManager::ObUnitLoad> zone_units;
3926
  common::ObArray<PoolOccupation> pool_occupation;
3927
  common::ObArray<ServerTotalLoad *> server_load_ptrs_sorted;
3928
  ObArray<common::ObAddr> excluded_servers;
3929
  ServerTotalLoad *server_load_to_vacate = NULL;
3930
  ObUnitStat unit_stat;
3931
  if (OB_UNLIKELY(!inited_)) {
3932
    ret = OB_NOT_INIT;
3933
    LOG_WARN("not init", K(ret));
3934
  } else if (OB_UNLIKELY(common::OB_INVALID_ID == unit_migrate_stat.tenant_id_
3935
                         || !collide_unit_load.is_valid())) {
3936
    ret = OB_INVALID_ARGUMENT;
3937
    LOG_WARN("invalid argument", K(ret), K(unit_migrate_stat), K(collide_unit_load));
3938
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
3939
    ret = OB_ERR_UNEXPECTED;
3940
    LOG_WARN("unit_mgr_ ptr is null", K(ret));
3941
  } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
3942
          collide_unit_load.unit_->unit_id_, collide_unit_load.unit_->zone_, unit_stat))) {
3943
    LOG_WARN("fail to get unit stat", K(ret), K(collide_unit_load));
3944
  } else if (OB_FAIL(unit_mgr_->get_tenant_zone_all_unit_loads(
3945
          unit_migrate_stat.tenant_id_, zone_disk_statistic_.zone_, zone_units))) {
3946
    LOG_WARN("fail to get tenant zone all unit loads", K(ret), K(unit_migrate_stat));
3947
  } else if (OB_FAIL(generate_pool_occupation_array(zone_units, pool_occupation))) {
3948
    LOG_WARN("fail to generate pool occupation", K(ret));
3949
  } else if (OB_FAIL(pick_vacate_server_load(
3950
          server_loads, server_load_ptrs_sorted, unit_migrate_stat, server_load_to_vacate))) {
3951
    LOG_WARN("fail to pick vacate server load", K(ret));
3952
  } else if (OB_UNLIKELY(NULL == server_load_to_vacate)) {
3953
    ret = OB_ERR_UNEXPECTED;
3954
    LOG_WARN("vacate server load ptr is null", K(ret));
3955
  } else if (server_load_to_vacate->server_ != collide_unit_load.unit_->server_) {
3956
    ret = OB_ERR_UNEXPECTED;
3957
    LOG_WARN("server not match", K(ret),
3958
             "left_server", server_load_to_vacate->server_,
3959
             "right_server", collide_unit_load.unit_->server_);
3960
  } else if (OB_FAIL(get_pool_occupation_excluded_dst_servers(
3961
          collide_unit_load, pool_occupation, excluded_servers))) {
3962
    LOG_WARN("fail to get excluded servers", K(ret));
3963
  } else {
3964
    do_amend = false;
3965
    ObArray<UnitMigrateStat> task_array;
3966
    for (int64_t j = server_load_ptrs_sorted.count() - 1; OB_SUCC(ret) && j >= 0; --j) {
3967
      LoadSum this_load;
3968
      ServerTotalLoad *server_load = server_load_ptrs_sorted.at(j);
3969
      bool enough = false;
3970
      if (OB_UNLIKELY(NULL == server_load)) {
3971
        ret = OB_ERR_UNEXPECTED;
3972
        LOG_WARN("server load ptr is null", K(ret));
3973
      } else if (server_load->wild_server_) {
3974
      } else if (server_load->server_ == collide_unit_load.unit_->server_) {
3975
      } else if (has_exist_in_array(excluded_servers, server_load->server_)) {
3976
      } else if (OB_FAIL(this_load.append_load(collide_unit_load))) {
3977
        LOG_WARN("fail to append load", K(ret));
3978
      } else if (OB_FAIL(check_single_server_resource_enough(
3979
              this_load, unit_stat, *server_load, enough))) {
3980
        LOG_WARN("fail to check server resource", K(ret));
3981
      } else if (!enough) {
3982
      } else {
3983
        const common::ObAddr &server = server_load->server_;
3984
        UnitMigrateStat unit_migrate_stat;
3985
        unit_migrate_stat.original_pos_ = collide_unit_load.unit_->server_;
3986
        unit_migrate_stat.arranged_pos_ = server;
3987
        unit_migrate_stat.unit_load_ = collide_unit_load;
3988
        if (OB_FAIL(do_update_src_server_load(
3989
                collide_unit_load, *server_load_to_vacate, unit_stat))) {
3990
          LOG_WARN("fail to update src server load", K(ret));
3991
        } else if (OB_FAIL(do_update_dst_server_load(
3992
                collide_unit_load, *server_load, unit_stat, pool_occupation))) {
3993
          LOG_WARN("fail to do update dst server load", K(ret));
3994
        } else if (OB_FAIL(task_array.push_back(unit_migrate_stat))) {
3995
          LOG_WARN("fail to push back", K(ret));
3996
        } else {
3997
          ServerTotalLoadCmp cmp;
3998
          std::sort(server_load_ptrs_sorted.begin(), server_load_ptrs_sorted.end(), cmp);
3999
          if (OB_FAIL(cmp.get_ret())) {
4000
            LOG_WARN("fail to sort server loads", K(ret));
4001
          }
4002
          break;
4003
        }
4004
      }
4005
    }
4006
    if (OB_FAIL(ret)) {
4007
    } else if (task_array.count() <= 0) {
4008
    } else if (OB_FAIL(check_and_do_migrate_unit_task_(task_array))) {
4009
      LOG_WARN("fail to check and do migrate task", KR(ret), K(task_array));
4010
    } else {
4011
      do_amend = true;
4012
    }
4013
  }
4014
  return ret;
4015
}
4016

4017
int ObServerBalancer::pick_vacate_server_load(
4018
    common::ObIArray<ServerTotalLoad> &server_loads,
4019
    common::ObArray<ServerTotalLoad *> &server_load_ptrs_sorted,
4020
    UnitMigrateStat &unit_migrate_stat,
4021
    ServerTotalLoad *&vacate_server_load)
4022
{
4023
  int ret = OB_SUCCESS;
4024
  if (OB_UNLIKELY(!inited_)) {
4025
    ret = OB_NOT_INIT;
4026
    LOG_WARN("not init", K(ret));
4027
  } else {
4028
    vacate_server_load = NULL;
4029
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
4030
      ServerTotalLoad &server_load = server_loads.at(i);
4031
      if (server_load.server_ == unit_migrate_stat.arranged_pos_ && !server_load.wild_server_) {
4032
        vacate_server_load = &server_load;
4033
        break;
4034
      } else {} // go on next
4035
    }
4036
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
4037
      ServerTotalLoad &server_load = server_loads.at(i);
4038
      if (OB_FAIL(server_load_ptrs_sorted.push_back(&server_load))) {
4039
        LOG_WARN("fail to push back", K(ret));
4040
      }
4041
    }
4042
    if (OB_SUCC(ret)) {
4043
      ServerTotalLoadCmp cmp;
4044
      if (FALSE_IT(std::sort(server_load_ptrs_sorted.begin(),
4045
                             server_load_ptrs_sorted.end(),
4046
                             cmp))) {
4047
      } else if (OB_FAIL(cmp.get_ret())) {
4048
        LOG_WARN("fail to sort", K(ret));
4049
      }
4050
    }
4051
  }
4052
  return ret;
4053
}
4054

4055
int ObServerBalancer::do_vacate_space_for_ttg_balance(
4056
    UnitMigrateStat &unit_migrate_stat,
4057
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
4058
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units,
4059
    const common::ObIArray<TenantGroupBalanceInfo *> &unstable_tenant_group,
4060
    common::ObArray<ServerTotalLoad> &server_loads,
4061
    bool &do_amend)
4062
{
4063
  int ret = OB_SUCCESS;
4064
  double hard_limit = 0.0;
4065
  ServerTotalLoad *server_load_to_vacate = NULL;
4066
  common::ObArray<ServerTotalLoad *> server_load_ptrs_sorted;
4067
  ObUnitStat unit_stat;
4068
  ObUnitManager::ObUnitLoad &unit_load_to_vacate = unit_migrate_stat.unit_load_;
4069
  if (OB_UNLIKELY(!inited_)) {
4070
    ret = OB_NOT_INIT;
4071
    LOG_WARN("not init", K(ret));
4072
  } else if (OB_UNLIKELY(!unit_load_to_vacate.is_valid())) {
4073
    ret = OB_INVALID_ARGUMENT;
4074
    LOG_WARN("invalid argument", K(ret), K(unit_load_to_vacate));
4075
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
4076
    ret = OB_ERR_UNEXPECTED;
4077
    LOG_WARN("unit_mgr_ is null", K(ret), KP(unit_mgr_));
4078
  } else if (OB_FAIL(unit_mgr_->get_hard_limit(hard_limit))) {
4079
    LOG_WARN("fail to get hard limit", K(ret));
4080
  } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
4081
          unit_load_to_vacate.unit_->unit_id_, unit_load_to_vacate.unit_->zone_, unit_stat))) {
4082
    LOG_WARN("fail to get unit stat", K(ret), "unit", *unit_load_to_vacate.unit_);
4083
  } else if (OB_FAIL(pick_vacate_server_load(
4084
          server_loads, server_load_ptrs_sorted, unit_migrate_stat, server_load_to_vacate))) {
4085
    LOG_WARN("fail to pick vacate server load", K(ret));
4086
  } else if (OB_UNLIKELY(NULL == server_load_to_vacate)) {
4087
    ret = OB_ERR_UNEXPECTED;
4088
    LOG_WARN("vacate server load ptr is null", K(ret));
4089
  } else {
4090
    ObArray<UnitMigrateStat> task_array;
4091
    bool vacate_enough = false;
4092
    LoadSum load_to_vacate;
4093
    if (OB_FAIL(load_to_vacate.append_load(unit_migrate_stat.unit_load_))) {
4094
      LOG_WARN("fail to append load", K(ret));
4095
    } else if (OB_FAIL(vacate_space_by_unit_array(
4096
            load_to_vacate, unit_load_to_vacate, *server_load_to_vacate,
4097
            task_array, not_grant_units, server_load_ptrs_sorted))) {
4098
      LOG_WARN("fail to vacate space by not grant units", K(ret));
4099
    } else if (OB_FAIL(check_single_server_resource_enough(
4100
            load_to_vacate, unit_stat, *server_load_to_vacate, vacate_enough))) {
4101
      LOG_WARN("fail to check single server resource enough", K(ret));
4102
    } else if (vacate_enough) {
4103
      // good, enough
4104
    } else if (OB_FAIL(vacate_space_by_unit_array(
4105
            load_to_vacate, unit_load_to_vacate, *server_load_to_vacate,
4106
            task_array, standalone_units, server_load_ptrs_sorted))) {
4107
      LOG_WARN("fail to vacate space by not grant units", K(ret));
4108
    } else if (OB_FAIL(check_single_server_resource_enough(
4109
            load_to_vacate, unit_stat, *server_load_to_vacate, vacate_enough))) {
4110
      LOG_WARN("fail to check single server resource enough", K(ret));
4111
    } else if (vacate_enough) {
4112
      // good, enough
4113
    } else if (OB_FAIL(vacate_space_by_tenantgroup(
4114
            load_to_vacate, unit_load_to_vacate, *server_load_to_vacate,
4115
            task_array, unstable_tenant_group, server_load_ptrs_sorted))) {
4116
      LOG_WARN("fail to vacate space by tenantgroup", K(ret));
4117
    } else if (OB_FAIL(check_single_server_resource_enough(
4118
            load_to_vacate, unit_stat, *server_load_to_vacate, vacate_enough))) {
4119
      LOG_WARN("fail to check single server resource enough", K(ret));
4120
    } else if (vacate_enough) {
4121
      // good, enough
4122
    } else {
4123
      do_amend = false; // resource not enough
4124
    }
4125
    if (OB_SUCC(ret) && vacate_enough) {
4126
      if (OB_FAIL(check_and_do_migrate_unit_task_(task_array))) {
4127
        LOG_WARN("fail to check and do migrate task", KR(ret), K(task_array));
4128
      } else {
4129
        do_amend = true;
4130
      }
4131
    }
4132
  }
4133
  return ret;
4134
}
4135

4136
int ObServerBalancer::generate_pool_occupation_array(
4137
    const common::ObIArray<ObUnitManager::ObUnitLoad> &unit_loads,
4138
    common::ObIArray<PoolOccupation> &pool_occupation)
4139
{
4140
  int ret = OB_SUCCESS;
4141
  pool_occupation.reset();
4142
  for (int64_t i = 0; OB_SUCC(ret) && i < unit_loads.count(); ++i) {
4143
    const ObUnitManager::ObUnitLoad &unit_load = unit_loads.at(i);
4144
    if (OB_UNLIKELY(!unit_load.is_valid())) {
4145
      ret = OB_ERR_UNEXPECTED;
4146
      LOG_WARN("unit load unexpected", K(ret), K(unit_load));
4147
    } else if (OB_FAIL(pool_occupation.push_back(
4148
            PoolOccupation(unit_load.pool_->tenant_id_,
4149
                           unit_load.pool_->resource_pool_id_,
4150
                           unit_load.unit_->server_)))) {
4151
      LOG_WARN("fail to push back", K(ret));
4152
    } else {}
4153
  }
4154
  return ret;
4155
}
4156

4157
int ObServerBalancer::vacate_space_by_tenantgroup(
4158
    LoadSum &load_to_vacate,
4159
    const ObUnitManager::ObUnitLoad &unit_load_to_vacate,
4160
    ServerTotalLoad &server_load_to_vacate,
4161
    common::ObIArray<UnitMigrateStat> &task_array,
4162
    const common::ObIArray<TenantGroupBalanceInfo *> &unstable_tenant_group,
4163
    common::ObArray<ServerTotalLoad *> &server_load_ptrs_sorted)
4164
{
4165
  int ret = OB_SUCCESS;
4166
  ObUnitStat unit_stat;
4167
  if (OB_UNLIKELY(!inited_)) {
4168
    ret = OB_NOT_INIT;
4169
    LOG_WARN("not init", K(ret));
4170
  } else if (OB_UNLIKELY(!unit_load_to_vacate.is_valid())) {
4171
    ret = OB_INVALID_ARGUMENT;
4172
    LOG_WARN("invalid argument", K(ret), K(unit_load_to_vacate));
4173
  } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
4174
          unit_load_to_vacate.unit_->unit_id_, unit_load_to_vacate.unit_->zone_, unit_stat))) {
4175
    LOG_WARN("fail to get unit stat", K(ret), "unit", *unit_load_to_vacate.unit_);
4176
  } else {
4177
    for (int64_t i = 0; OB_SUCC(ret) && i < unstable_tenant_group.count(); ++i) {
4178
      bool vacate_enough = false;
4179
      common::ObArray<ObUnitManager::ObUnitLoad> unit_loads;
4180
      TenantGroupBalanceInfo *ptr = unstable_tenant_group.at(i);
4181
      if (OB_UNLIKELY(NULL == ptr)) {
4182
        ret = OB_ERR_UNEXPECTED;
4183
        LOG_WARN("ptr is null", K(ret));
4184
      } else if (OB_FAIL(ptr->get_all_unit_loads(unit_loads))) {
4185
        LOG_WARN("fail to get all unit loads", K(ret));
4186
      } else if (OB_FAIL(vacate_space_by_unit_array(
4187
              load_to_vacate, unit_load_to_vacate, server_load_to_vacate,
4188
              task_array, unit_loads, server_load_ptrs_sorted))) {
4189
        LOG_WARN("fail to vacate space by unit array", K(ret));
4190
      } else if (OB_FAIL(check_single_server_resource_enough(
4191
              load_to_vacate, unit_stat, server_load_to_vacate, vacate_enough))) {
4192
        LOG_WARN("fail to check single server resource enough", K(ret));
4193
      } else if (vacate_enough) {
4194
        break;
4195
      } else {} // go on to the next
4196
    }
4197
  }
4198
  return ret;
4199
}
4200

4201
int ObServerBalancer::vacate_space_by_unit_array(
4202
    LoadSum &load_to_vacate,
4203
    const ObUnitManager::ObUnitLoad &unit_load_to_vacate,
4204
    ServerTotalLoad &server_load_to_vacate,
4205
    common::ObIArray<UnitMigrateStat> &task_array,
4206
    const common::ObIArray<ObUnitManager::ObUnitLoad> &units,
4207
    common::ObArray<ServerTotalLoad *> &server_load_ptrs_sorted)
4208
{
4209
  int ret = OB_SUCCESS;
4210
  ObUnitStat unit_stat;
4211
  ObArray<PoolOccupation> pool_occupation;
4212
  if (OB_UNLIKELY(!inited_)) {
4213
    ret = OB_NOT_INIT;
4214
    LOG_WARN("not init", K(ret));
4215
  } else if (OB_UNLIKELY(!unit_load_to_vacate.is_valid())) {
4216
    ret = OB_INVALID_ARGUMENT;
4217
    LOG_WARN("invalid unit load", K(ret), K(unit_load_to_vacate));
4218
  } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
4219
          unit_load_to_vacate.unit_->unit_id_, unit_load_to_vacate.unit_->zone_, unit_stat))) {
4220
    LOG_WARN("fail to get unit stat", K(ret), "unit", *unit_load_to_vacate.unit_);
4221
  } else if (OB_FAIL(generate_pool_occupation_array(units, pool_occupation))) {
4222
    LOG_WARN("fail to generate pool occupation array", K(ret));
4223
  } else {
4224
    for (int64_t i = 0; OB_SUCC(ret) && i < units.count(); ++i) {
4225
      bool vacate_enough = false;
4226
      const ObUnitManager::ObUnitLoad &unit_load = units.at(i);
4227
      if (OB_UNLIKELY(!unit_load.is_valid())) {
4228
        ret = OB_ERR_UNEXPECTED;
4229
        LOG_WARN("unit load is invalid", K(ret), K(unit_load));
4230
      } else if (unit_load.unit_->server_ != server_load_to_vacate.server_) {
4231
        // unit is not belong to vacate, skip
4232
      } else if (OB_FAIL(vacate_space_by_single_unit(
4233
              server_load_to_vacate, unit_load,
4234
              task_array, pool_occupation, server_load_ptrs_sorted))) {
4235
        LOG_WARN("fail to vacate space by single unit", K(ret));
4236
      } else if (OB_FAIL(check_single_server_resource_enough(
4237
            load_to_vacate, unit_stat, server_load_to_vacate, vacate_enough))) {
4238
        LOG_WARN("fail to check single server resource enough", K(ret));
4239
      } else if (vacate_enough) {
4240
        break;  // good, enough
4241
      } else {} // go on
4242
    }
4243
  }
4244
  return ret;
4245
}
4246

4247
int ObServerBalancer::check_single_server_resource_enough(
4248
    const LoadSum &this_load,
4249
    const ObUnitStat &unit_stat,
4250
    const ServerTotalLoad &server_load,
4251
    bool &enough,
4252
    const bool mind_disk_waterlevel)
4253
{
4254
  int ret = OB_SUCCESS;
4255
  ServerDiskStatistic disk_statistic;
4256
  double hard_limit = 0.0;
4257
  double disk_waterlevel = 0.0;
4258
  ObUnitStat my_unit_stat = unit_stat;
4259
  if (OB_UNLIKELY(!inited_)) {
4260
    ret = OB_NOT_INIT;
4261
    LOG_WARN("not init", K(ret));
4262
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
4263
    ret = OB_ERR_UNEXPECTED;
4264
    LOG_WARN("unit_mgr_ is null", K(ret), KP(unit_mgr_));
4265
  } else if (OB_FAIL(unit_mgr_->get_hard_limit(hard_limit))) {
4266
    LOG_WARN("fail to get hard limit", K(ret));
4267
  } else if (OB_FAIL(zone_disk_statistic_.get_server_disk_statistic(
4268
          server_load.server_, disk_statistic))) {
4269
    LOG_WARN("fail to get disk statistic", K(ret), "server", server_load.server_);
4270
  } else if (mind_disk_waterlevel) {
4271
    if (OB_FAIL(get_server_balance_critical_disk_waterlevel(disk_waterlevel))) {
4272
      LOG_WARN("fail to get server balance critical disk waterlevle", K(ret));
4273
    }
4274
  } else {
4275
    disk_waterlevel = 1.0;
4276
  }
4277
  if (OB_SUCC(ret)) {
4278
    ObUnit *unit = NULL;
4279
    if (OB_FAIL(unit_mgr_->get_unit_by_id(unit_stat.get_unit_id(), unit))) {
4280
      LOG_WARN("fail to get unit by id", K(ret), "unit_id", unit_stat.get_unit_id());
4281
    } else if (OB_UNLIKELY(NULL == unit)) {
4282
      ret = OB_ERR_UNEXPECTED;
4283
      LOG_WARN("unit ptr is null", K(ret));
4284
    } else if (common::REPLICA_TYPE_LOGONLY == unit->replica_type_) {
4285
      ret = OB_ERR_UNEXPECTED;
4286
      LOG_WARN("Replica type LOGONLY is unexpected", KR(ret), KP(unit));
4287
    }
4288
  }
4289
  if (OB_SUCC(ret)) {
4290
    enough = ((this_load.load_sum_.max_cpu() + server_load.load_sum_.load_sum_.max_cpu()
4291
                <= server_load.resource_info_.cpu_ * hard_limit)
4292
              && (this_load.load_sum_.min_cpu() + server_load.load_sum_.load_sum_.min_cpu()
4293
                <= server_load.resource_info_.cpu_)
4294
              && (static_cast<double>(this_load.load_sum_.memory_size())
4295
                  + static_cast<double>(server_load.load_sum_.load_sum_.memory_size())
4296
                <= static_cast<double>(server_load.resource_info_.mem_total_))
4297
              && (static_cast<double>(this_load.load_sum_.log_disk_size())
4298
                  + static_cast<double>(server_load.load_sum_.load_sum_.log_disk_size())
4299
                <= static_cast<double>(server_load.resource_info_.log_disk_total_))
4300
              && (static_cast<double>(my_unit_stat.get_required_size() + disk_statistic.disk_in_use_)
4301
                <= static_cast<double>(disk_statistic.disk_total_) * disk_waterlevel));
4302
  }
4303
  return ret;
4304
}
4305

4306
int ObServerBalancer::vacate_space_by_single_unit(
4307
    ServerTotalLoad &server_load_to_vacate,
4308
    const ObUnitManager::ObUnitLoad &unit_load,
4309
    common::ObIArray<UnitMigrateStat> &task_array,
4310
    common::ObIArray<PoolOccupation> &pool_occupation,
4311
    common::ObArray<ServerTotalLoad *> &server_load_ptrs_sorted)
4312
{
4313
  int ret = OB_SUCCESS;
4314
  ObUnitStat unit_stat;
4315
  ObArray<common::ObAddr> excluded_servers;
4316
  if (OB_UNLIKELY(!inited_)) {
4317
    ret = OB_NOT_INIT;
4318
    LOG_WARN("not init", K(ret));
4319
  } else if (OB_UNLIKELY(!unit_load.is_valid()
4320
        || server_load_to_vacate.server_ != unit_load.unit_->server_)) {
4321
    ret = OB_INVALID_ARGUMENT;
4322
    LOG_WARN("invalid argument", K(ret), "unit_server", unit_load.unit_->server_,
4323
             "server", server_load_to_vacate.server_);
4324
  } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
4325
          unit_load.unit_->unit_id_, unit_load.unit_->zone_, unit_stat))) {
4326
    LOG_WARN("fail to get unit stat", K(ret), "unit", *unit_load.unit_);
4327
  } else if (OB_FAIL(get_pool_occupation_excluded_dst_servers(
4328
          unit_load, pool_occupation, excluded_servers))) {
4329
    LOG_WARN("fail to get excluded servers", K(ret));
4330
  } else {
4331
    for (int64_t j = server_load_ptrs_sorted.count() - 1; OB_SUCC(ret) && j >= 0; --j) {
4332
      LoadSum this_load;
4333
      bool enough = true;
4334
      ServerTotalLoad *server_load = server_load_ptrs_sorted.at(j);
4335
      if (OB_UNLIKELY(NULL == server_load)) {
4336
        ret = OB_ERR_UNEXPECTED;
4337
        LOG_WARN("server load ptr is null", K(ret));
4338
      } else if (server_load->wild_server_) {
4339
      } else if (server_load->server_ == server_load_to_vacate.server_) {
4340
      } else if (has_exist_in_array(excluded_servers, server_load->server_)) {
4341
      } else if (OB_FAIL(this_load.append_load(unit_load))) {
4342
        LOG_WARN("fail to append load", K(ret));
4343
      } else if (OB_FAIL(check_single_server_resource_enough(
4344
              this_load, unit_stat, *server_load_ptrs_sorted.at(j), enough))) {
4345
        LOG_WARN("fail to check server resource", K(ret));
4346
      } else if (!enough) {
4347
      } else {
4348
        const common::ObAddr &server = server_load->server_;
4349
        UnitMigrateStat unit_migrate_stat;
4350
        unit_migrate_stat.original_pos_ = unit_load.unit_->server_;
4351
        unit_migrate_stat.arranged_pos_ = server;
4352
        unit_migrate_stat.unit_load_ = unit_load;
4353
        if (OB_FAIL(do_update_src_server_load(unit_load, server_load_to_vacate, unit_stat))) {
4354
          LOG_WARN("fail to update src server load", K(ret));
4355
        } else if (OB_FAIL(do_update_dst_server_load(
4356
                unit_load, *server_load, unit_stat, pool_occupation))) {
4357
          LOG_WARN("fail to do update dst server load", K(ret));
4358
        } else if (OB_FAIL(task_array.push_back(unit_migrate_stat))) {
4359
          LOG_WARN("fail to push back", K(ret));
4360
        } else {
4361
          ServerTotalLoadCmp cmp;
4362
          std::sort(server_load_ptrs_sorted.begin(), server_load_ptrs_sorted.end(), cmp);
4363
          if (OB_FAIL(cmp.get_ret())) {
4364
            LOG_WARN("fail to sort server loads", K(ret));
4365
          }
4366
          break;
4367
        }
4368
      }
4369
    }
4370
  }
4371
  return ret;
4372
}
4373

4374
int ObServerBalancer::get_pool_occupation_excluded_dst_servers(
4375
    const ObUnitManager::ObUnitLoad &this_load,
4376
    const common::ObIArray<PoolOccupation> &pool_occupation,
4377
    common::ObIArray<common::ObAddr> &excluded_servers)
4378
{
4379
  int ret = OB_SUCCESS;
4380
  excluded_servers.reset();
4381
  if (OB_UNLIKELY(!inited_)) {
4382
    ret = OB_NOT_INIT;
4383
    LOG_WARN("not init", K(ret));
4384
  } else if (OB_UNLIKELY(!this_load.is_valid())) {
4385
    ret = OB_INVALID_ARGUMENT;
4386
    LOG_WARN("invalid argument", K(ret), K(this_load));
4387
  } else {
4388
    // not grant to any tenant
4389
    for (int64_t i = 0; OB_SUCC(ret) && i < pool_occupation.count(); ++i) {
4390
      const PoolOccupation &po = pool_occupation.at(i);
4391
      if (common::OB_INVALID_ID == this_load.pool_->tenant_id_
4392
          && this_load.pool_->resource_pool_id_ != po.resource_pool_id_) {
4393
        // not grant to any tenant, compare resource pool id
4394
      } else if (common::OB_INVALID_ID != this_load.pool_->tenant_id_
4395
          && this_load.pool_->tenant_id_ != po.tenant_id_) {
4396
        // already grant to any tenant, compare tenant id
4397
      } else if (OB_FAIL(excluded_servers.push_back(po.server_))) {
4398
        LOG_WARN("fail to push back", K(ret));
4399
      } else {} // no more to do
4400
    }
4401
  }
4402
  return ret;
4403
}
4404

4405
int ObServerBalancer::generate_complete_server_loads(
4406
    const common::ObZone &zone,
4407
    const common::ObIArray<common::ObAddr> &available_servers,
4408
    double *const resource_weights,
4409
    const int64_t weights_count,
4410
    common::ObArray<ServerTotalLoad> &server_loads)
4411
{
4412
  int ret = OB_SUCCESS;
4413
  common::ObArray<common::ObAddr> zone_servers;
4414
  if (OB_UNLIKELY(!inited_)) {
4415
    ret = OB_NOT_INIT;
4416
    LOG_WARN("not init", K(ret));
4417
  } else if (OB_UNLIKELY(NULL == resource_weights
4418
                         || RES_MAX != weights_count
4419
                         || zone.is_empty())) {
4420
    ret = OB_INVALID_ARGUMENT;
4421
    LOG_WARN("invalid argument", K(ret), KP(resource_weights), K(weights_count));
4422
  } else if (OB_ISNULL(unit_mgr_) || OB_ISNULL(server_mgr_)) {
4423
    ret = OB_ERR_UNEXPECTED;
4424
    LOG_WARN("unit_mgr_ or server_mgr_ is null", K(ret), KP(unit_mgr_), KP(server_mgr_));
4425
  } else if (OB_FAIL(SVR_TRACER.get_servers_of_zone(zone, zone_servers))) {
4426
    LOG_WARN("fail to get servers of zone", K(ret), K(zone));
4427
  } else {
4428
    for (int64_t i = 0; OB_SUCC(ret) && i < zone_servers.count(); ++i) {
4429
      const common::ObAddr &server = zone_servers.at(i);
4430
      ServerTotalLoad server_load;
4431
      server_load.server_ = server;
4432
      share::ObServerResourceInfo server_resource_info;
4433
      ObArray<ObUnitManager::ObUnitLoad> *unit_loads = NULL;
4434
      LoadSum load_sum;
4435
      server_load.wild_server_ = !has_exist_in_array(available_servers, server);
4436
      if (OB_FAIL(server_mgr_->get_server_resource_info(server, server_resource_info))) {
4437
        LOG_WARN("fail to get server status", K(ret), K(server));
4438
      } else if (OB_FAIL(unit_mgr_->get_loads_by_server(server, unit_loads))) {
4439
        if (OB_ENTRY_NOT_EXIST != ret) {
4440
          LOG_WARN("get loads by server failed", K(ret), K(server));
4441
        } else {
4442
          ret = OB_SUCCESS;
4443
        }
4444
      } else if (OB_UNLIKELY(NULL == unit_loads)) {
4445
        ret = OB_ERR_UNEXPECTED;
4446
        LOG_WARN("unit_loads is null", K(ret), KP(unit_loads));
4447
      } else if (OB_FAIL(load_sum.append_load(*unit_loads))) {
4448
        LOG_WARN("fail to append load", K(ret));
4449
      }
4450
      if (OB_SUCC(ret)) {
4451
        for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
4452
          server_load.resource_weights_[i] = resource_weights[i];
4453
        }
4454
        server_load.load_sum_ = load_sum;
4455
        server_load.resource_info_ = server_resource_info;
4456
        if (OB_FAIL(server_load.update_load_value())) {
4457
          LOG_WARN("fail to update load value", K(ret));
4458
        } else if (OB_FAIL(server_loads.push_back(server_load))) {
4459
          LOG_WARN("fail to push back", K(ret));
4460
        } else {} // no more to do
4461
      }
4462
      if (OB_SUCC(ret)) {
4463
        ServerTotalLoadCmp cmp;
4464
        std::sort(server_loads.begin(), server_loads.end(), cmp);
4465
        if (OB_FAIL(cmp.get_ret())) {
4466
          LOG_WARN("fail to sort", K(ret));
4467
        }
4468
      }
4469
    }
4470
    if (OB_SUCC(ret)) { // Defensive inspection
4471
      for (int64_t i = 0; OB_SUCC(ret) && i < available_servers.count(); ++i) {
4472
        if (!has_exist_in_array(zone_servers, available_servers.at(i))) {
4473
          ret = OB_ERR_UNEXPECTED;
4474
          LOG_WARN("available server is not a member of zone servers", K(zone_servers),
4475
                   "available server", available_servers.at(i));
4476
        } else {} // good, in zone servers array
4477
      }
4478
    }
4479
  }
4480
  return ret;
4481
}
4482

4483
int ObServerBalancer::divide_complete_server_loads_for_balance(
4484
    const common::ObIArray<common::ObAddr> &available_servers,
4485
    common::ObIArray<ServerTotalLoad> &server_loads,
4486
    common::ObIArray<ServerTotalLoad *> &wild_server_loads,
4487
    common::ObIArray<ServerTotalLoad *> &over_server_loads,
4488
    common::ObIArray<ServerTotalLoad *> &under_server_loads,
4489
    double &upper_lmt)
4490
{
4491
  int ret = OB_SUCCESS;
4492
  if (OB_UNLIKELY(!inited_)) {
4493
    ret = OB_NOT_INIT;
4494
    LOG_WARN("not init", K(ret));
4495
  } else if (OB_UNLIKELY(server_loads.count() <= 0
4496
                         || available_servers.count() <= 0)) {
4497
    ret = OB_INVALID_ARGUMENT;
4498
    LOG_WARN("invalid argument", K(ret), "server_load_count", server_loads.count(),
4499
             "available_servers count", available_servers.count());
4500
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
4501
    ret = OB_ERR_UNEXPECTED;
4502
    LOG_WARN("unit_mgr_ ptr is null", K(ret));
4503
  } else {
4504
    wild_server_loads.reset();
4505
    over_server_loads.reset();
4506
    under_server_loads.reset();
4507
    double sum_load = 0.0;
4508
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
4509
      const ServerTotalLoad &server_load = server_loads.at(i);
4510
      sum_load += server_load.inter_ttg_load_value_;
4511
    }
4512
    const int64_t cpu_mem_tolerance = GCONF.server_balance_cpu_mem_tolerance_percent;
4513
    const double delta_percent = static_cast<double>(cpu_mem_tolerance) / static_cast<double>(100);
4514
    const double average_load = sum_load / static_cast<double>(available_servers.count());
4515
    upper_lmt = average_load + delta_percent * average_load;
4516
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
4517
      ServerTotalLoad &server_load = server_loads.at(i);
4518
      if (server_load.wild_server_) {
4519
        ObArray<ObUnitManager::ObUnitLoad> *loads = NULL;
4520
        if (OB_FAIL(unit_mgr_->get_loads_by_server(server_load.server_, loads))) {
4521
          if (OB_ENTRY_NOT_EXIST == ret) {
4522
            ret = OB_SUCCESS; // good, has no load, no need to put into array
4523
          } else {
4524
            LOG_WARN("fail to get load by server", K(ret), "server", server_load.server_);
4525
          }
4526
        } else if (OB_UNLIKELY(NULL == loads)) {
4527
          ret = OB_ERR_UNEXPECTED;
4528
          LOG_WARN("loads ptr is null", K(ret));
4529
        } else if (OB_FAIL(wild_server_loads.push_back(&server_load))) {
4530
          LOG_WARN("fail to push back", K(ret));
4531
        }
4532
      } else if (server_load.inter_ttg_load_value_ >= upper_lmt) {
4533
        if (OB_FAIL(over_server_loads.push_back(&server_load))) {
4534
          LOG_WARN("fail to push back", K(ret));
4535
        }
4536
      } else {
4537
        if (OB_FAIL(under_server_loads.push_back(&server_load))) {
4538
          LOG_WARN("fail to push back", K(ret));
4539
        }
4540
      }
4541
    }
4542

4543
    LOG_INFO("divide complete server loads for balance", KR(ret), K(server_loads),
4544
        K(available_servers),
4545
        K(wild_server_loads), K(over_server_loads), K(under_server_loads));
4546
  }
4547
  return ret;
4548
}
4549

4550
int ObServerBalancer::generate_available_server_loads(
4551
    ObIArray<ServerTotalLoad *> &over_server_loads,
4552
    ObIArray<ServerTotalLoad *> &under_server_loads,
4553
    ObIArray<ServerTotalLoad *> &available_server_loads)
4554
{
4555
  int ret = OB_SUCCESS;
4556
  available_server_loads.reset();
4557
  for (int64_t i = 0; OB_SUCC(ret) && i < over_server_loads.count(); ++i) {
4558
    if (OB_FAIL(available_server_loads.push_back(over_server_loads.at(i)))) {
4559
      LOG_WARN("fail to push back", K(ret));
4560
    }
4561
  }
4562
  for (int64_t i = 0; OB_SUCC(ret) && i < under_server_loads.count(); ++i) {
4563
    if (OB_FAIL(available_server_loads.push_back(under_server_loads.at(i)))) {
4564
      LOG_WARN("fail to push back", K(ret));
4565
    }
4566
  }
4567
  return ret;
4568
}
4569

4570
int ObServerBalancer::make_single_wild_server_empty_by_units(
4571
    common::ObIArray<PoolOccupation> &pool_occupation,
4572
    ServerTotalLoad &wild_server_load,
4573
    const common::ObIArray<ObUnitManager::ObUnitLoad> &units,
4574
    common::ObArray<ServerTotalLoad *> &available_server_loads,
4575
    common::ObIArray<UnitMigrateStat> &task_array)
4576
{
4577
  int ret = OB_SUCCESS;
4578
  for (int64_t i = 0; OB_SUCC(ret) && i < units.count(); ++i) {
4579
    bool do_balance = false;
4580
    ObArray<common::ObAddr> excluded_servers;
4581
    LoadSum this_load;
4582
    ServerTotalLoadCmp cmp;
4583
    const ObUnitManager::ObUnitLoad &unit_load = units.at(i);
4584
    if (OB_UNLIKELY(!unit_load.is_valid())) {
4585
      ret = OB_ERR_UNEXPECTED;
4586
      LOG_WARN("unit load is invalid", K(ret), K(unit_load));
4587
    } else if (wild_server_load.server_ != unit_load.unit_->server_) {
4588
      // not in this wild server, ignore
4589
    } else if (OB_FAIL(get_pool_occupation_excluded_dst_servers(
4590
            unit_load, pool_occupation, excluded_servers))) {
4591
      LOG_WARN("fail to get excluded servers", K(ret));
4592
    } else if (FALSE_IT(std::sort(available_server_loads.begin(),
4593
                                  available_server_loads.end(),
4594
                                  cmp))) {
4595
      // sort cannot fail
4596
    } else if (OB_FAIL(cmp.get_ret())) {
4597
      LOG_WARN("fail to sort", K(ret));
4598
    } else if (OB_FAIL(try_balance_single_unit_by_cm(
4599
          unit_load, available_server_loads, task_array,
4600
          excluded_servers, pool_occupation, do_balance))) {
4601
      LOG_WARN("fail to try balance single unit by cmp and memory", K(ret));
4602
    } else if (do_balance) {
4603
      // do execute balance according to cpu and memory, no more to do
4604
    } else if (OB_FAIL(try_balance_single_unit_by_disk(
4605
            unit_load, available_server_loads, task_array,
4606
            excluded_servers, pool_occupation, do_balance))) {
4607
      LOG_WARN("fail to do balance single unit by disk", K(ret));
4608
    } else if (do_balance) {
4609
      // good
4610
    } else {
4611
      ret = OB_MACHINE_RESOURCE_NOT_ENOUGH;
4612
      LOG_WARN("no available server to hold more unit", K(ret),
4613
               "unit", *unit_load.unit_,
4614
               "unit_config", *unit_load.unit_config_,
4615
               "pool", *unit_load.pool_);
4616
    }
4617
  }
4618
  return ret;
4619
}
4620

4621
int ObServerBalancer::make_wild_servers_empty(
4622
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
4623
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units,
4624
    common::ObIArray<ServerTotalLoad *> &wild_server_loads,
4625
    common::ObIArray<ServerTotalLoad *> &over_server_loads,
4626
    common::ObIArray<ServerTotalLoad *> &under_server_loads)
4627
{
4628
  int ret = OB_SUCCESS;
4629
  ObArray<ServerTotalLoad *> available_server_loads;
4630
  common::ObArray<UnitMigrateStat> task_array;
4631
  if (OB_UNLIKELY(!inited_)) {
4632
    ret = OB_NOT_INIT;
4633
    LOG_WARN("not init", K(ret));
4634
  } else if (OB_FAIL(generate_available_server_loads(
4635
          over_server_loads, under_server_loads, available_server_loads))) {
4636
    LOG_WARN("fail to generate available server loads", K(ret));
4637
  } else if (OB_FAIL(make_wild_servers_empty_by_units(
4638
          wild_server_loads, not_grant_units, available_server_loads, task_array))) {
4639
    LOG_WARN("fail to make wild servers empty by units", K(ret));
4640
  } else if (OB_FAIL(make_wild_servers_empty_by_units(
4641
          wild_server_loads, standalone_units, available_server_loads, task_array))) {
4642
    LOG_WARN("fail to make wild servers empty by units", K(ret));
4643
  } else if (task_array.count() <= 0) {
4644
    // no task generate
4645
  } else if (OB_FAIL(check_and_do_migrate_unit_task_(task_array))) {
4646
    LOG_WARN("fail to check and do migrate unit task", KR(ret), K(task_array));
4647
  } else {} // no more to do
4648
  return ret;
4649
}
4650

4651
int ObServerBalancer::make_wild_servers_empty_by_units(
4652
    common::ObIArray<ServerTotalLoad *> &wild_server_loads,
4653
    const common::ObIArray<ObUnitManager::ObUnitLoad> &units,
4654
    common::ObArray<ServerTotalLoad *> &available_server_loads,
4655
    common::ObIArray<UnitMigrateStat> &task_array)
4656
{
4657
  int ret = OB_SUCCESS;
4658
  // pool_occupation is a placeholder for unit on the server
4659
  common::ObArray<PoolOccupation> pool_occupation;
4660
  if (OB_UNLIKELY(!inited_)) {
4661
    ret = OB_NOT_INIT;
4662
    LOG_WARN("not init", K(ret));
4663
  } else if (OB_FAIL(generate_pool_occupation_array(units, pool_occupation))) {
4664
    LOG_WARN("fail to generate pool occupation array", K(ret));
4665
  } else {
4666
    for (int64_t i = 0; OB_SUCC(ret) && i < wild_server_loads.count(); ++i) {
4667
      ServerTotalLoad *wild_server_load = wild_server_loads.at(i);
4668
      if (OB_UNLIKELY(NULL == wild_server_load)) {
4669
        ret = OB_ERR_UNEXPECTED;
4670
        LOG_WARN("wild server load ptr is null", K(ret), KP(wild_server_load));
4671
      } else if (OB_FAIL(make_single_wild_server_empty_by_units(
4672
              pool_occupation, *wild_server_load, units, available_server_loads, task_array))) {
4673
        LOG_WARN("fail to make single wild server empty by units", K(ret));
4674
      } else {} // no more to do
4675
    }
4676
  }
4677
  return ret;
4678
}
4679

4680
int ObServerBalancer::get_disk_over_available_servers(
4681
    common::ObIArray<common::ObAddr> &disk_over_servers)
4682
{
4683
  int ret = OB_SUCCESS;
4684
  double disk_waterlevel = 0.0;
4685
  if (OB_UNLIKELY(!inited_)) {
4686
    ret = OB_NOT_INIT;
4687
    LOG_WARN("not init", K(ret));
4688
  } else if (OB_FAIL(get_server_balance_critical_disk_waterlevel(disk_waterlevel))) {
4689
    LOG_WARN("fail to get critical disk waterlevel", K(ret));
4690
  } else {
4691
    disk_over_servers.reset();
4692
    for (int64_t i = 0;
4693
         OB_SUCC(ret) && i < zone_disk_statistic_.server_disk_statistic_array_.count(); ++i) {
4694
      ServerDiskStatistic &disk_statistic
4695
        = zone_disk_statistic_.server_disk_statistic_array_.at(i);
4696
      if (static_cast<double>(disk_statistic.disk_in_use_)
4697
                 > static_cast<double>(disk_statistic.disk_total_) * disk_waterlevel) {
4698
        if (OB_FAIL(disk_over_servers.push_back(disk_statistic.server_))) {
4699
          LOG_WARN("fail to push back", K(ret));
4700
        }
4701
      } else {} // still ok
4702
    }
4703
  }
4704
  return ret;
4705
}
4706

4707
int ObServerBalancer::make_non_ttg_balance_under_load_by_disk(
4708
    common::ObIArray<PoolOccupation> &pool_occupation,
4709
    common::ObIArray<UnitMigrateStat> &task_array,
4710
    const ObUnitManager::ObUnitLoad &unit_load,
4711
    ServerTotalLoad *src_server_load,
4712
    const common::ObIArray<common::ObAddr> &disk_over_servers,
4713
    common::ObIArray<ServerTotalLoad *> &available_server_loads)
4714
{
4715
  int ret = OB_SUCCESS;
4716
  common::ObArray<common::ObAddr> excluded_servers;
4717
  double disk_waterlevel = 0.0;
4718
  bool is_disk_over_waterlevel = true;
4719
  ObUnitStat unit_stat;
4720
  LoadSum this_load;
4721
  if (OB_UNLIKELY(!inited_)) {
4722
    ret = OB_NOT_INIT;
4723
    LOG_WARN("not init", K(ret));
4724
  } else if (NULL == src_server_load || !unit_load.is_valid()) {
4725
    ret = OB_INVALID_ARGUMENT;
4726
    LOG_WARN("invalid argument", K(ret), KP(src_server_load), K(unit_load));
4727
  } else if (OB_FAIL(get_server_balance_critical_disk_waterlevel(disk_waterlevel))) {
4728
    LOG_WARN("fail to get server balance critical disk waterlevel", K(ret));
4729
  } else if (OB_FAIL(zone_disk_statistic_.check_server_over_disk_waterlevel(
4730
          src_server_load->server_, disk_waterlevel, is_disk_over_waterlevel))) {
4731
    LOG_WARN("fail to check disk over waterlevel", K(ret));
4732
  } else if (!is_disk_over_waterlevel) {
4733
    // do_nothing, it is already below the water mark, no need to migrate out unit
4734
  } else if (OB_FAIL(get_pool_occupation_excluded_dst_servers(
4735
          unit_load, pool_occupation, excluded_servers))) {
4736
    LOG_WARN("fail to get excluded servers", K(ret));
4737
  } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
4738
          unit_load.unit_->unit_id_, unit_load.unit_->zone_, unit_stat))) {
4739
    LOG_WARN("fail to get unit stat", K(ret), "unit", *unit_load.unit_);
4740
  } else if (0 == unit_stat.get_required_size()) {
4741
    // do nothing, skip unit with 0 data disk usage
4742
  } else if (OB_FAIL(this_load.append_load(unit_load))) {
4743
    LOG_WARN("fail to append load", K(ret));
4744
  } else {
4745
    for (int64_t i = 0;
4746
         OB_SUCC(ret) && i < zone_disk_statistic_.server_disk_statistic_array_.count();
4747
         ++i) {
4748
      ServerDiskStatistic &disk_statistic = zone_disk_statistic_.server_disk_statistic_array_.at(i);
4749
      bool enough = true;
4750
      UnitMigrateStat unit_migrate;
4751
      unit_migrate.unit_load_ = unit_load;
4752
      unit_migrate.original_pos_ = src_server_load->server_;
4753
      unit_migrate.arranged_pos_ = disk_statistic.server_;
4754
      ServerTotalLoad *dst_server_load = NULL;
4755
      if (disk_statistic.wild_server_) {
4756
        // skip, wild server
4757
      } else if (has_exist_in_array(excluded_servers, disk_statistic.server_)) {
4758
        // skip, dst in excluded servers
4759
      } else if (has_exist_in_array(disk_over_servers, disk_statistic.server_)) {
4760
        // skip, dst in disk over servers
4761
      } else if (OB_FAIL(pick_server_load(
4762
              disk_statistic.server_, available_server_loads, dst_server_load))) {
4763
        LOG_WARN("fail to pick server load", K(ret));
4764
      } else if (OB_UNLIKELY(NULL == dst_server_load)) {
4765
        ret = OB_ERR_UNEXPECTED;
4766
        LOG_WARN("dst server load ptr is null", K(ret), KP(dst_server_load));
4767
      } else if (OB_FAIL(check_single_server_resource_enough(
4768
              this_load, unit_stat, *dst_server_load, enough))) {
4769
        LOG_WARN("fail to check server resource enough", K(ret));
4770
      } else if (!enough) {
4771
        // resource not enough
4772
      } else if (OB_FAIL(do_update_src_server_load(unit_load, *src_server_load, unit_stat))) {
4773
        LOG_WARN("fail to do update src server load", K(ret));
4774
      } else if (OB_FAIL(do_update_dst_server_load(
4775
              unit_load, *dst_server_load, unit_stat, pool_occupation))) {
4776
        LOG_WARN("fail to update dst server load", K(ret));
4777
      } else if (OB_FAIL(task_array.push_back(unit_migrate))) {
4778
        LOG_WARN("fail to push back", K(ret));
4779
      } else {
4780
        LOG_INFO("unit migration task generated for disk in use over waterlevel", KR(ret), K(unit_stat), K(unit_migrate));
4781
        break;
4782
        // unit migrate success
4783
      }
4784
    }
4785
  }
4786
  return ret;
4787
}
4788

4789
int ObServerBalancer::make_available_servers_balance_by_disk(
4790
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
4791
    common::ObIArray<ServerTotalLoad *> &available_server_loads,
4792
    int64_t &task_count)
4793
{
4794
  int ret = OB_SUCCESS;
4795
  ObArray<PoolOccupation> pool_occupation;
4796
  common::ObArray<common::ObAddr> disk_over_servers;
4797
  task_count = 0;
4798
  if (OB_UNLIKELY(!inited_)) {
4799
    ret = OB_NOT_INIT;
4800
    LOG_WARN("not init", K(ret));
4801
  } else if (OB_FAIL(get_disk_over_available_servers(disk_over_servers))) {
4802
    LOG_WARN("fail to get disk over servers", K(ret));
4803
  } else if (OB_FAIL(generate_pool_occupation_array(standalone_units, pool_occupation))) {
4804
    LOG_WARN("fail to generate pool occupation array", K(ret));
4805
  } else {
4806
    common::ObArray<const ObUnitManager::ObUnitLoad *> standalone_unit_ptrs;
4807
    for (int64_t i = 0; OB_SUCC(ret) && i < standalone_units.count(); ++i) {
4808
      if (OB_FAIL(standalone_unit_ptrs.push_back(&standalone_units.at(i)))) {
4809
        LOG_WARN("fail to push back", K(ret));
4810
      }
4811
    }
4812
    if (OB_SUCC(ret)) {
4813
      UnitLoadDiskCmp cmp(unit_stat_mgr_);
4814
      std::sort(standalone_unit_ptrs.begin(), standalone_unit_ptrs.end(), cmp);
4815
      if (OB_FAIL(cmp.get_ret())) {
4816
        LOG_WARN("fail to get ret", K(ret));
4817
      }
4818
    }
4819
    common::ObArray<UnitMigrateStat> task_array;
4820
    for (int64_t i = 0; OB_SUCC(ret) && i < disk_over_servers.count(); ++i) {
4821
      common::ObAddr &src_server = disk_over_servers.at(i);
4822
      for (int64_t j = 0; OB_SUCC(ret) && j < standalone_unit_ptrs.count(); ++j) {
4823
        const ObUnitManager::ObUnitLoad *unit_load = standalone_unit_ptrs.at(j);
4824
        ServerTotalLoad *src_server_load = NULL;
4825
        if (OB_UNLIKELY(NULL == unit_load)) {
4826
          ret = OB_ERR_UNEXPECTED;
4827
          LOG_WARN("unit load ptr is null", K(ret));
4828
        } else if (OB_UNLIKELY(!unit_load->is_valid())) {
4829
          ret = OB_ERR_UNEXPECTED;
4830
          LOG_WARN("unit load is invalid", K(ret), K(*unit_load));
4831
        } else if (src_server != unit_load->unit_->server_) {
4832
          // no this server
4833
        } else if (common::REPLICA_TYPE_LOGONLY == unit_load->unit_->replica_type_) {
4834
          ret = OB_ERR_UNEXPECTED;
4835
          LOG_WARN("Replica type LOGONLY is unexpected", KR(ret), KP(unit_load));
4836
        } else if (OB_FAIL(pick_server_load(
4837
                src_server, available_server_loads, src_server_load))) {
4838
          LOG_WARN("fail to pick server load", K(ret));
4839
        } else if (NULL == src_server_load) {
4840
          ret = OB_ERR_UNEXPECTED;
4841
          LOG_WARN("dst server load ptr is null", K(ret));
4842
        } else if (OB_FAIL(make_non_ttg_balance_under_load_by_disk(
4843
                pool_occupation, task_array, *unit_load,
4844
                src_server_load, disk_over_servers, available_server_loads))) {
4845
          LOG_WARN("fail to make non ttg balance under load by disk", K(ret));
4846
        } else {} // no more to do
4847
      }
4848
    }
4849

4850
    if (OB_SUCC(ret) && task_array.count() > 0) {
4851
      task_count = task_array.count();
4852
      if (OB_FAIL(check_and_do_migrate_unit_task_(task_array))) {
4853
        LOG_WARN("fail to check and do migrate task", KR(ret), K(task_array));
4854
      }
4855
    }
4856
  }
4857
  return ret;
4858
}
4859

4860
// DO NOT migrate unit for CREATING tenant
4861
int ObServerBalancer::check_and_do_migrate_unit_task_(
4862
    const common::ObIArray<UnitMigrateStat> &task_array)
4863
{
4864
  int ret = OB_SUCCESS;
4865
  share::schema::ObSchemaGetterGuard schema_guard;
4866
  ObArray<UnitMigrateStat> new_task_array;
4867

4868
  if (OB_ISNULL(schema_service_)) {
4869
    ret = OB_ERR_UNEXPECTED;
4870
    LOG_WARN("schema service ptr is null", KR(ret), KP(schema_service_));
4871
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
4872
    LOG_WARN("fail to get sys schema guard", KR(ret));
4873
  } else {
4874
    for (int64_t idx = 0; OB_SUCC(ret) && idx < task_array.count(); idx++) {
4875
      const UnitMigrateStat &task = task_array.at(idx);
4876
      const share::schema::ObSimpleTenantSchema *tenant_schema = NULL;
4877
      bool can_do_unit_migrate = true;
4878
      uint64_t tenant_id = task.unit_load_.get_tenant_id();
4879

4880
      if (OB_INVALID_ID == tenant_id || OB_INVALID_TENANT_ID == tenant_id) {
4881
        LOG_INFO("[SERVER_BALANCE] [CHECK_CAN_DO] not-grant unit can always migrate", K(idx),
4882
            K(task), K(lbt()));
4883
        can_do_unit_migrate = true;
4884
      } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
4885
        LOG_WARN("fail to get tenant info", KR(ret), K(tenant_id));
4886
      } else if (OB_ISNULL(tenant_schema)) {
4887
        // tenant not exist, ignore this unit
4888
        LOG_WARN("[SERVER_BALANCE] [CHECK_CAN_DO] tenant not exist, ignore this unit migrate task",
4889
            K(idx), K(task));
4890
        can_do_unit_migrate = false;
4891
      } else if (tenant_schema->is_creating()) {
4892
        LOG_INFO("[SERVER_BALANCE] [CHECK_CAN_DO] unit of creating tenant can not do migrate",
4893
            K(idx), K(task), K(lbt()));
4894
        can_do_unit_migrate = false;
4895
      } else {
4896
        LOG_INFO("[SERVER_BALANCE] [CHECK_CAN_DO] unit can do migrate", K(idx), K(task), K(lbt()));
4897
        // other tenant unit can do migrate
4898
        can_do_unit_migrate = true;
4899
      }
4900

4901
      if (OB_SUCC(ret) && can_do_unit_migrate && OB_FAIL(new_task_array.push_back(task))) {
4902
        LOG_WARN("push task into array fail", KR(ret), K(task));
4903
      }
4904
    }
4905

4906
    if (OB_FAIL(ret)) {
4907
    } else if (new_task_array.count() > 0 && OB_FAIL(do_migrate_unit_task(new_task_array))) {
4908
      LOG_WARN("do migrate unit task fail", KR(ret), K(new_task_array), K(task_array));
4909
    }
4910
  }
4911
  return ret;
4912
}
4913

4914
int ObServerBalancer::make_available_servers_balance_by_cm(
4915
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
4916
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units,
4917
    common::ObIArray<ServerTotalLoad *> &over_server_loads,
4918
    common::ObIArray<ServerTotalLoad *> &under_server_loads,
4919
    const double upper_lmt,
4920
    double *const g_res_weights,
4921
    const int64_t weights_count,
4922
    int64_t &task_count)
4923
{
4924
  int ret = OB_SUCCESS;
4925
  task_count = 0;
4926
  if (OB_UNLIKELY(!inited_)) {
4927
    ret = OB_NOT_INIT;
4928
    LOG_WARN("not init", K(ret));
4929
  } else {
4930
    common::ObArray<UnitMigrateStat> task_array;
4931
    if (OB_FAIL(do_non_ttg_unit_balance_by_cm(
4932
            task_array, not_grant_units, over_server_loads, under_server_loads,
4933
            upper_lmt, g_res_weights, weights_count))) {
4934
      LOG_WARN("fail to do non ttg unit balance by units", K(ret));
4935
    } else if (OB_FAIL(do_non_ttg_unit_balance_by_cm(
4936
            task_array, standalone_units, over_server_loads, under_server_loads,
4937
            upper_lmt, g_res_weights, weights_count))) {
4938
      LOG_WARN("fail to do non ttg unit balance by units", K(ret));
4939
    } else if (task_array.count() > 0) {
4940
      task_count = task_array.count();
4941

4942
      if (OB_FAIL(check_and_do_migrate_unit_task_(task_array))) {
4943
        LOG_WARN("fail to check and do migrate task", KR(ret), K(task_array));
4944
      }
4945
    } else {} // no more to do
4946
  }
4947
  return ret;
4948
}
4949

4950
int ObServerBalancer::generate_sort_available_servers_disk_statistic(
4951
    common::ObArray<ServerDiskStatistic> &server_disk_statistic_array,
4952
    common::ObArray<ServerDiskStatistic *> &available_servers_disk_statistic)
4953
{
4954
  int ret = OB_SUCCESS;
4955
  if (OB_UNLIKELY(!inited_)) {
4956
    ret = OB_NOT_INIT;
4957
    LOG_WARN("not init", K(ret));
4958
  } else {
4959
    available_servers_disk_statistic.reset();
4960
    for (int64_t i = 0; OB_SUCC(ret) && i < server_disk_statistic_array.count(); ++i) {
4961
      ServerDiskStatistic &disk_statistic = server_disk_statistic_array.at(i);
4962
      if (disk_statistic.wild_server_) {
4963
        // a wild server, bypass
4964
      } else if (OB_FAIL(available_servers_disk_statistic.push_back(&disk_statistic))) {
4965
        LOG_WARN("fail to push back", K(ret));
4966
      } else {} // no more to do
4967
    }
4968
    if (OB_SUCC(ret)) {
4969
      ServerDiskPercentCmp cmp;
4970
      std::sort(available_servers_disk_statistic.begin(),
4971
                available_servers_disk_statistic.end(),
4972
                cmp);
4973
      if (OB_FAIL(cmp.get_ret())) {
4974
        LOG_WARN("fail to sort", K(ret));
4975
      }
4976
    }
4977
  }
4978
  return ret;
4979
}
4980

4981
int ObServerBalancer::divide_available_servers_disk_statistic(
4982
    common::ObIArray<ServerDiskStatistic *> &available_servers_disk_statistic,
4983
    common::ObIArray<ServerDiskStatistic *> &over_disk_statistic,
4984
    common::ObIArray<ServerDiskStatistic *> &under_disk_statistic,
4985
    double &upper_lmt)
4986
{
4987
  int ret = OB_SUCCESS;
4988
  if (OB_UNLIKELY(!inited_)) {
4989
    ret = OB_NOT_INIT;
4990
    LOG_WARN("not init", K(ret));
4991
  } else if (available_servers_disk_statistic.count() <= 0) {
4992
    ret = OB_INVALID_ARGUMENT;
4993
    LOG_WARN("invalid argument", K(ret), K(available_servers_disk_statistic));
4994
  } else {
4995
    double total_value = 0.0;
4996
    for (int64_t i = 0; OB_SUCC(ret) && i < available_servers_disk_statistic.count(); ++i) {
4997
      ServerDiskStatistic *disk_statistic = available_servers_disk_statistic.at(i);
4998
      if (OB_UNLIKELY(NULL == disk_statistic)) {
4999
        ret = OB_ERR_UNEXPECTED;
5000
        LOG_WARN("disk statistic ptr is null", K(ret));
5001
      } else {
5002
        total_value += disk_statistic->get_disk_used_percent();
5003
      }
5004
    }
5005
    if (OB_SUCC(ret)) {
5006
      const int64_t disk_tolerance = GCONF.server_balance_disk_tolerance_percent;
5007
      const double delta = static_cast<double>(disk_tolerance) / static_cast<double>(100);
5008
      const double average_value
5009
          = total_value / static_cast<double>(available_servers_disk_statistic.count());
5010
      upper_lmt = average_value + delta;
5011
      for (int64_t i = 0; OB_SUCC(ret) && i < available_servers_disk_statistic.count(); ++i) {
5012
        ServerDiskStatistic *disk_statistic = available_servers_disk_statistic.at(i);
5013
        if (OB_UNLIKELY(NULL == disk_statistic)) {
5014
          ret = OB_ERR_UNEXPECTED;
5015
          LOG_WARN("disk statistic ptr is null", K(ret));
5016
        } else if (disk_statistic->get_disk_used_percent() >= upper_lmt) {
5017
          if (OB_FAIL(over_disk_statistic.push_back(disk_statistic))) {
5018
            LOG_WARN("fail to push back", K(ret));
5019
          }
5020
        } else {
5021
          if (OB_FAIL(under_disk_statistic.push_back(disk_statistic))) {
5022
            LOG_WARN("fail to push back", K(ret));
5023
          }
5024
        }
5025
      }
5026
    }
5027
  }
5028
  return ret;
5029
}
5030

5031
int ObServerBalancer::make_server_disk_underload_by_unit(
5032
    ServerTotalLoad *src_server_load,
5033
    const ObUnitManager::ObUnitLoad &unit_load,
5034
    const double upper_lmt,
5035
    common::ObIArray<PoolOccupation> &pool_occupation,
5036
    common::ObIArray<ServerTotalLoad *> &available_server_loads,
5037
    common::ObArray<ServerDiskStatistic *> &under_disk_statistic,
5038
    common::ObIArray<UnitMigrateStat> &task_array)
5039
{
5040
  int ret = OB_SUCCESS;
5041
  if (OB_UNLIKELY(!inited_)) {
5042
    ret = OB_NOT_INIT;
5043
    LOG_WARN("not init", K(ret));
5044
  } else if (OB_UNLIKELY(NULL == src_server_load || !unit_load.is_valid())) {
5045
    ret = OB_INVALID_ARGUMENT;
5046
    LOG_WARN("invalid argument", K(ret), KP(src_server_load), K(unit_load));
5047
  } else if (src_server_load->server_ != unit_load.unit_->server_) {
5048
    ret = OB_ERR_UNEXPECTED;
5049
    LOG_WARN("server not match", K(ret),
5050
             "left_server", src_server_load->server_,
5051
             "right_server", unit_load.unit_->server_);
5052
  } else {
5053
    ObArray<common::ObAddr> excluded_servers;
5054
    ObUnitStat unit_stat;
5055
    LoadSum this_load;
5056
    if (OB_FAIL(get_pool_occupation_excluded_dst_servers(
5057
            unit_load, pool_occupation, excluded_servers))) {
5058
      LOG_WARN("fail to get excluded dst servers", K(ret));
5059
    } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
5060
            unit_load.unit_->unit_id_, unit_load.unit_->zone_, unit_stat))) {
5061
      LOG_WARN("fail to get unit stat", K(ret), "unit", *unit_load.unit_);
5062
    } else if (OB_FAIL(this_load.append_load(unit_load))) {
5063
      LOG_WARN("fail to append load", K(ret));
5064
    } else {
5065
      for (int64_t j = under_disk_statistic.count() - 1; OB_SUCC(ret) && j >= 0; --j) {
5066
        ServerTotalLoad *dst_server_load = NULL;
5067
        ServerDiskStatistic *under_server = under_disk_statistic.at(j);
5068
        const bool mind_disk_waterlevel = false;
5069
        bool enough = true;
5070
        if (OB_UNLIKELY(NULL == under_server)) {
5071
          ret = OB_ERR_UNEXPECTED;
5072
          LOG_WARN("under server ptr is null", K(ret));
5073
        } else if (has_exist_in_array(excluded_servers, under_server->server_)) {
5074
          // in excluded server
5075
        } else if (OB_FAIL(pick_server_load(
5076
                under_server->server_, available_server_loads, dst_server_load))) {
5077
          LOG_WARN("fail to pick server load", K(ret));
5078
        } else if (NULL == dst_server_load) {
5079
          ret = OB_ERR_UNEXPECTED;
5080
          LOG_WARN("dst server load ptr is null", K(ret));
5081
        } else if (OB_FAIL(check_single_server_resource_enough(
5082
                this_load, unit_stat, *dst_server_load, enough, mind_disk_waterlevel))) {
5083
          LOG_WARN("fail to check server resource enough", K(ret));
5084
        } else if (!enough) {
5085
          // resource not enough
5086
        } else if (under_server->get_disk_used_percent_if_add(unit_stat.get_required_size())
5087
                   > upper_lmt) {
5088
          // Exceeds the upper_lmt value
5089
        } else {
5090
          UnitMigrateStat unit_migrate;
5091
          unit_migrate.unit_load_ = unit_load;
5092
          unit_migrate.original_pos_ = src_server_load->server_;
5093
          unit_migrate.arranged_pos_ = dst_server_load->server_;
5094
          ServerDiskPercentCmp cmp;
5095
          if (OB_FAIL(do_update_src_server_load(unit_load, *src_server_load, unit_stat))) {
5096
            LOG_WARN("fail to do update srce server load", K(ret));
5097
          } else if (OB_FAIL(do_update_dst_server_load(
5098
                  unit_load, *dst_server_load, unit_stat, pool_occupation))) {
5099
            LOG_WARN("fail to update dst server load", K(ret));
5100
          } else if (OB_FAIL(task_array.push_back(unit_migrate))) {
5101
            LOG_WARN("fail to push back", K(ret));
5102
          } else if (FALSE_IT(std::sort(under_disk_statistic.begin(),
5103
                                        under_disk_statistic.end(),
5104
                                        cmp))) {
5105
          } else if (OB_FAIL(cmp.get_ret())) {
5106
            LOG_WARN("fail to sort", K(ret));
5107
          } else {
5108
            break;
5109
          }
5110
        }
5111
      }
5112
    }
5113
  }
5114
  return ret;
5115
}
5116

5117
int ObServerBalancer::make_server_disk_underload(
5118
    ServerDiskStatistic &src_disk_statistic,
5119
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
5120
    const double upper_lmt,
5121
    common::ObIArray<PoolOccupation> &pool_occupation,
5122
    common::ObIArray<ServerTotalLoad *> &available_server_loads,
5123
    common::ObArray<ServerDiskStatistic *> &under_disk_statistic,
5124
    common::ObIArray<UnitMigrateStat> &task_array)
5125
{
5126
  int ret = OB_SUCCESS;
5127
  if (OB_UNLIKELY(!inited_)) {
5128
    ret = OB_NOT_INIT;
5129
    LOG_WARN("not init", K(ret));
5130
  } else {
5131
    common::ObArray<const ObUnitManager::ObUnitLoad *> standalone_unit_ptrs;
5132
    for (int64_t i = 0; OB_SUCC(ret) && i < standalone_units.count(); ++i) {
5133
      if (OB_FAIL(standalone_unit_ptrs.push_back(&standalone_units.at(i)))) {
5134
        LOG_WARN("fail to push back", K(ret));
5135
      }
5136
    }
5137
    if (OB_SUCC(ret)) {
5138
      UnitLoadDiskCmp cmp(unit_stat_mgr_);
5139
      std::sort(standalone_unit_ptrs.begin(), standalone_unit_ptrs.end(), cmp);
5140
      if (OB_FAIL(cmp.get_ret())) {
5141
        LOG_WARN("fail to get ret", K(ret));
5142
      }
5143
    }
5144
    for (int64_t i = 0; OB_SUCC(ret) && i < standalone_unit_ptrs.count(); ++i) {
5145
      const ObUnitManager::ObUnitLoad *unit_load = standalone_unit_ptrs.at(i);
5146
      ServerTotalLoad *src_server_load = NULL;
5147
      if (src_disk_statistic.get_disk_used_percent() < upper_lmt) {
5148
        break; // src already lower than limit
5149
      } else if (NULL == unit_load) {
5150
        ret = OB_ERR_UNEXPECTED;
5151
        LOG_WARN("unit load ptr is null", K(ret));
5152
      } else if (OB_FAIL(pick_server_load(
5153
              src_disk_statistic.server_, available_server_loads, src_server_load))) {
5154
        LOG_WARN("fail to pick server load", K(ret));
5155
      } else if (OB_UNLIKELY(NULL == src_server_load)) {
5156
        ret = OB_ERR_UNEXPECTED;
5157
        LOG_WARN("src server load ptr is null", K(ret));
5158
      } else if (!unit_load->is_valid()) {
5159
        ret = OB_ERR_UNEXPECTED;
5160
        LOG_WARN("unit load is invalid", K(ret));
5161
      } else if (unit_load->unit_->server_ != src_disk_statistic.server_) {
5162
        // bypass
5163
      } else if (OB_FAIL(make_server_disk_underload_by_unit(
5164
              src_server_load, *unit_load, upper_lmt, pool_occupation,
5165
              available_server_loads, under_disk_statistic, task_array))) {
5166
        LOG_WARN("fail to make server disk underload by disk", K(ret));
5167
      } else {} // no more to do
5168
    }
5169
  }
5170
  return ret;
5171
}
5172

5173
int ObServerBalancer::make_available_servers_disk_balance(
5174
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
5175
    common::ObIArray<ServerTotalLoad *> &available_server_loads,
5176
    int64_t &task_count)
5177
{
5178
  common::ObArray<ServerDiskStatistic *> available_servers_disk_statistic;
5179
  common::ObArray<ServerDiskStatistic *> over_disk_statistic;
5180
  common::ObArray<ServerDiskStatistic *> under_disk_statistic;
5181
  common::ObArray<PoolOccupation> pool_occupation;
5182
  int ret = OB_SUCCESS;
5183
  double upper_limit = 0.0;
5184
  task_count = 0;
5185
  if (OB_UNLIKELY(!inited_)) {
5186
    ret = OB_NOT_INIT;
5187
    LOG_WARN("not init", K(ret));
5188
  } else if (OB_FAIL(generate_pool_occupation_array(standalone_units, pool_occupation))) {
5189
    LOG_WARN("fail to generate pool occupation array", K(ret));
5190
  } else if (OB_FAIL(generate_sort_available_servers_disk_statistic(
5191
          zone_disk_statistic_.server_disk_statistic_array_, available_servers_disk_statistic))) {
5192
    LOG_WARN("fail to generate sort available server disk statistic", K(ret));
5193
  } else if (OB_FAIL(divide_available_servers_disk_statistic(
5194
          available_servers_disk_statistic, over_disk_statistic,
5195
          under_disk_statistic, upper_limit))) {
5196
    LOG_WARN("fail to divide available servers disk statistic", K(ret));
5197
  } else {
5198
    common::ObArray<UnitMigrateStat> task_array;
5199
    for (int64_t i = 0; OB_SUCC(ret) && i < over_disk_statistic.count(); ++i) {
5200
      ServerDiskStatistic *src_disk_statistic = over_disk_statistic.at(i);
5201
      if (OB_UNLIKELY(NULL == src_disk_statistic)) {
5202
        ret = OB_ERR_UNEXPECTED;
5203
        LOG_WARN("disk statistic ptr is null", K(ret));
5204
      } else if (OB_FAIL(make_server_disk_underload(
5205
              *src_disk_statistic, standalone_units, upper_limit, pool_occupation,
5206
              available_server_loads, under_disk_statistic, task_array))) {
5207
        LOG_WARN("fail to make server disk underload", K(ret));
5208
      }
5209
    }
5210
    if (OB_SUCC(ret) && task_array.count() > 0) {
5211
      task_count = task_array.count();
5212
      if (OB_FAIL(check_and_do_migrate_unit_task_(task_array))) {
5213
        LOG_WARN("fail to check and do migrate task", KR(ret), K(task_array));
5214
      }
5215
    }
5216
  }
5217
  return ret;
5218
}
5219

5220
int ObServerBalancer::make_available_servers_balance(
5221
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
5222
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units,
5223
    common::ObIArray<ServerTotalLoad *> &over_server_loads,
5224
    common::ObIArray<ServerTotalLoad *> &under_server_loads,
5225
    const double upper_lmt,
5226
    double *const g_res_weights,
5227
    const int64_t weights_count)
5228
{
5229
  int ret = OB_SUCCESS;
5230
  double disk_waterlevel = 0.0;
5231
  bool all_available_servers_disk_over = false;
5232
  int64_t balance_task_count = 0;
5233
  const char *balance_reason = "NONE";
5234
  if (OB_UNLIKELY(!inited_)) {
5235
    ret = OB_NOT_INIT;
5236
    LOG_WARN("not init", K(ret));
5237
  } else if (OB_FAIL(get_server_balance_critical_disk_waterlevel(disk_waterlevel))) {
5238
    LOG_WARN("fail to get disk waterlevel", K(ret));
5239
  } else if (OB_FAIL(zone_disk_statistic_.check_all_available_servers_over_disk_waterlevel(
5240
          disk_waterlevel, all_available_servers_disk_over))) {
5241
    LOG_WARN("fail to check all available servers over disk waterlevel", K(ret));
5242
  } else if (all_available_servers_disk_over) {
5243
    // The disk usage of all servers exceeds the warning water mark,
5244
    // used a complete disk balancing strategy.
5245
    // balance the disk usage when the cpu and memory can be accommodated.
5246
    ObArray<ServerTotalLoad *> available_server_loads;
5247
    if (OB_FAIL(generate_available_server_loads(
5248
            over_server_loads, under_server_loads, available_server_loads))) {
5249
      LOG_WARN("fail to generate available server loads", K(ret));
5250
    } else if (OB_FAIL(make_available_servers_disk_balance(
5251
            standalone_units, available_server_loads, balance_task_count))) {
5252
      LOG_WARN("fail to make available servers disk balance", K(ret));
5253
    } else {
5254
      balance_reason = "disk_balance_as_all_server_disk_over";
5255
    }
5256
  } else if (zone_disk_statistic_.over_disk_waterlevel()) {
5257
    // No need to deal with not grant, because they do not occupy disk
5258
    ObArray<ServerTotalLoad *> available_server_loads;
5259
    if (OB_FAIL(generate_available_server_loads(
5260
            over_server_loads, under_server_loads, available_server_loads))) {
5261
      LOG_WARN("fail to generate available server loads", K(ret));
5262
    } else if (OB_FAIL(make_available_servers_balance_by_disk(
5263
            standalone_units, available_server_loads, balance_task_count))) {
5264
      LOG_WARN("fail to make available servers balance by disk", K(ret));
5265
    } else {
5266
      balance_reason = "disk_balance_as_over_disk_waterlevel";
5267
    }
5268
  } else {
5269
    if (OB_FAIL(make_available_servers_balance_by_cm(
5270
            standalone_units, not_grant_units, over_server_loads, under_server_loads,
5271
            upper_lmt, g_res_weights, weights_count, balance_task_count))) {
5272
      LOG_WARN("fail to make available servers balance by cpu and memory", K(ret));
5273
    } else {
5274
      balance_reason = "load_balance";
5275
    }
5276
  }
5277

5278
  LOG_INFO("finish zone servers balance", KR(ret), K(disk_waterlevel),
5279
      K(balance_task_count), K(balance_reason),
5280
      K(all_available_servers_disk_over),
5281
      "zone_over_disk_waterlevel", zone_disk_statistic_.over_disk_waterlevel(),
5282
      K(standalone_units.count()), K(not_grant_units.count()));
5283
  return ret;
5284
}
5285

5286
int ObServerBalancer::do_non_tenantgroup_unit_balance_task(
5287
    const common::ObZone &zone,
5288
    const common::ObIArray<ObUnitManager::ObUnitLoad> &standalone_units,
5289
    const common::ObIArray<ObUnitManager::ObUnitLoad> &not_grant_units,
5290
    const common::ObIArray<common::ObAddr> &available_servers)
5291
{
5292
  int ret = OB_SUCCESS;
5293
  double g_res_weights[RES_MAX];
5294
  ObArray<ServerTotalLoad> server_loads;
5295
  common::ObArray<ServerTotalLoad *> wild_server_loads;
5296
  common::ObArray<ServerTotalLoad *> over_server_loads;
5297
  common::ObArray<ServerTotalLoad *> under_server_loads;
5298
  double upper_lmt = 0.0;
5299

5300
  LOG_INFO("start do non-tenantgroup unit balance task", K(zone));
5301

5302
  if (OB_UNLIKELY(!inited_)) {
5303
    ret = OB_NOT_INIT;
5304
    LOG_WARN("not init", K(ret));
5305
  } else if (OB_UNLIKELY(available_servers.count() <= 0)) {
5306
    ret = OB_INVALID_ARGUMENT;
5307
    LOG_WARN("invalid argument", K(ret));
5308
  } else if (OB_FAIL(calc_global_balance_resource_weights(
5309
         zone, available_servers, g_res_weights, RES_MAX))) {
5310
    LOG_WARN("fail to calc whole balance resource weights", K(ret));
5311
  } else if (OB_FAIL(generate_complete_server_loads(
5312
          zone, available_servers, g_res_weights, RES_MAX, server_loads))) {
5313
    LOG_WARN("fail to generate complete server loads", K(ret));
5314
  } else if (OB_FAIL(divide_complete_server_loads_for_balance(
5315
          available_servers, server_loads, wild_server_loads,
5316
          over_server_loads, under_server_loads, upper_lmt))) {
5317
    LOG_WARN("fail to sort server loads for inter ttg balance", K(ret));
5318
  } else if (wild_server_loads.count() > 0) {
5319
    if (OB_FAIL(make_wild_servers_empty(
5320
            standalone_units, not_grant_units, wild_server_loads,
5321
            over_server_loads, under_server_loads))) {
5322
      LOG_WARN("fail to make wild servers empty", K(ret));
5323
    }
5324
  } else {
5325
    if (OB_FAIL(make_available_servers_balance(
5326
            standalone_units, not_grant_units, over_server_loads,
5327
            under_server_loads, upper_lmt, g_res_weights, RES_MAX))) {
5328
      LOG_WARN("fail to make available servers balance", K(ret));
5329
    }
5330
  }
5331

5332
  LOG_INFO("finish do non-tenantgroup unit balance task", KR(ret), K(zone),
5333
      K(upper_lmt),
5334
      K(wild_server_loads.count()),
5335
      K(over_server_loads.count()),
5336
      K(under_server_loads.count()),
5337
      K(standalone_units), K(not_grant_units), K(available_servers));
5338
  return ret;
5339
}
5340

5341
int ObServerBalancer::do_non_ttg_unit_balance_by_cm(
5342
    common::ObIArray<UnitMigrateStat> &task_array,
5343
    const common::ObIArray<ObUnitManager::ObUnitLoad> &units,
5344
    common::ObIArray<ServerTotalLoad *> &over_server_loads,
5345
    common::ObIArray<ServerTotalLoad *> &under_server_loads,
5346
    const double upper_lmt,
5347
    double *const g_res_weights,
5348
    const int64_t weights_count)
5349
{
5350
  int ret = OB_SUCCESS;
5351
  common::ObArray<PoolOccupation> pool_occupation;
5352
  if (OB_UNLIKELY(!inited_)) {
5353
    ret = OB_NOT_INIT;
5354
    LOG_WARN("not init", K(ret));
5355
  } else if (OB_FAIL(generate_pool_occupation_array(units, pool_occupation))) {
5356
    LOG_WARN("fail to generate pool occupation array", K(ret));
5357
  } else {
5358
    for (int64_t i = 0; OB_SUCC(ret) && i < over_server_loads.count(); ++i) {
5359
      ServerTotalLoad *src = over_server_loads.at(i);
5360
      for (int64_t j = under_server_loads.count() - 1;
5361
           OB_SUCC(ret) && j >= 0;
5362
           --j) {
5363
        ServerTotalLoad *dst = under_server_loads.at(j);
5364
        if (OB_FAIL(make_non_ttg_balance_under_load_by_cm(
5365
                pool_occupation, task_array, units, src, dst,
5366
                upper_lmt, g_res_weights, weights_count))) {
5367
          LOG_WARN("fail to make non ttg balance under load", K(ret));
5368
        } else {} // no more to do
5369
      }
5370
    }
5371
  }
5372
  return ret;
5373
}
5374

5375
int ObServerBalancer::make_non_ttg_balance_under_load_by_cm(
5376
    common::ObIArray<PoolOccupation> &pool_occupation,
5377
    common::ObIArray<UnitMigrateStat> &task_array,
5378
    const common::ObIArray<ObUnitManager::ObUnitLoad> &units,
5379
    ServerTotalLoad *src_server_load,
5380
    ServerTotalLoad *dst_server_load,
5381
    const double upper_lmt,
5382
    double *const g_res_weights,
5383
    const int64_t weights_count)
5384
{
5385
  int ret = OB_SUCCESS;
5386
  double hard_limit = 0.0;
5387
  ObArray<common::ObAddr> excluded_servers;
5388
  if (OB_UNLIKELY(!inited_)) {
5389
    ret = OB_NOT_INIT;
5390
    LOG_WARN("not init", K(ret));
5391
  } else if (NULL == src_server_load || NULL == dst_server_load
5392
             || NULL == g_res_weights || RES_MAX != weights_count) {
5393
    ret = OB_INVALID_ARGUMENT;
5394
    LOG_WARN("invalid argument", K(ret), KP(src_server_load), K(dst_server_load));
5395
  } else if (OB_UNLIKELY(src_server_load->wild_server_)) {
5396
    ret = OB_ERR_UNEXPECTED;
5397
    LOG_WARN("should not be here", K(ret), "wild_server", src_server_load->wild_server_);
5398
  } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
5399
    ret = OB_ERR_UNEXPECTED;
5400
    LOG_WARN("unit_mgr_ ptr is null", K(ret), K(unit_mgr_));
5401
  } else if (OB_FAIL(unit_mgr_->get_hard_limit(hard_limit))) {
5402
    LOG_WARN("fail to get hard limit", K(ret));
5403
  } else {
5404
    double upper_limit = std::min(upper_lmt, hard_limit);
5405
    for (int64_t i = 0; OB_SUCC(ret) && i < units.count(); ++i) {
5406
      LoadSum this_load;
5407
      ObUnitStat unit_stat;
5408
      const ObUnitManager::ObUnitLoad &unit_load = units.at(i);
5409
      double dst_load_value = 0.0;
5410
      UnitMigrateStat unit_migrate;
5411
      unit_migrate.unit_load_ = unit_load;
5412
      unit_migrate.original_pos_ = src_server_load->server_;
5413
      unit_migrate.arranged_pos_ = dst_server_load->server_;
5414
      bool enough = true;
5415
      if (src_server_load->inter_ttg_load_value_ < upper_limit) {
5416
        break;
5417
      } else if (!unit_load.is_valid()) {
5418
        ret = OB_ERR_UNEXPECTED;
5419
        LOG_WARN("unit load is invalid", K(ret), K(unit_load));
5420
      } else if (src_server_load->server_ != unit_load.unit_->server_) {
5421
        // not in this src, ignore
5422
      } else if (OB_FAIL(unit_stat_mgr_.get_unit_stat(
5423
              unit_load.unit_->unit_id_, unit_load.unit_->zone_, unit_stat))) {
5424
        LOG_WARN("fail to get unit stat", K(ret), "unit", *unit_load.unit_);
5425
      } else if (OB_FAIL(get_pool_occupation_excluded_dst_servers(
5426
              unit_load, pool_occupation, excluded_servers))) {
5427
        LOG_WARN("fail to get excluded servers", K(ret));
5428
      } else if (has_exist_in_array(excluded_servers, dst_server_load->server_)) {
5429
        // in excluded server
5430
      } else if (OB_FAIL(this_load.append_load(unit_load))) {
5431
        LOG_WARN("fail to append load", K(ret));
5432
      } else if (OB_FAIL(check_single_server_resource_enough(
5433
              this_load, unit_stat, *dst_server_load, enough))) {
5434
        LOG_WARN("fail to check server resource enough", K(ret));
5435
      } else if (!enough) {
5436
      } else if (OB_FAIL(this_load.calc_load_value(
5437
              g_res_weights, weights_count, *dst_server_load, dst_load_value))) {
5438
        LOG_WARN("fail to calc load value", K(ret));
5439
      } else if (dst_server_load->inter_ttg_load_value_ + dst_load_value >= upper_limit) {
5440
        // Unbalanced results
5441
      } else if (OB_FAIL(do_update_src_server_load(unit_load, *src_server_load, unit_stat))) {
5442
        LOG_WARN("fail to do update src server load", K(ret));
5443
      } else if (OB_FAIL(do_update_dst_server_load(
5444
              unit_load, *dst_server_load, unit_stat, pool_occupation))) {
5445
        LOG_WARN("fail to update dst server load", K(ret));
5446
      } else if (OB_FAIL(task_array.push_back(unit_migrate))) {
5447
        LOG_WARN("fail to push back", K(ret));
5448
      } else {} // no more to do
5449
    }
5450
  }
5451
  return ret;
5452
}
5453

5454
int ObServerBalancer::choose_balance_info_amend(
5455
    const common::ObIArray<TenantGroupBalanceInfo *> &tenant_group,
5456
    const bool is_stable_tenantgroup,
5457
    TenantGroupBalanceInfo *&info_need_amend,
5458
    common::ObIArray<TenantGroupBalanceInfo *> &other_ttg)
5459
{
5460
  int ret = OB_SUCCESS;
5461
  if (OB_UNLIKELY(!inited_)) {
5462
    ret = OB_NOT_INIT;
5463
    LOG_WARN("not init", K(ret));
5464
  } else if (tenant_group.count() <= 0) {
5465
    ret = OB_INVALID_ARGUMENT;
5466
    LOG_WARN("tenant group count is zero", K(ret));
5467
  } else {
5468
    int64_t idx = -1;
5469
    info_need_amend = NULL;
5470
    for (int64_t i = 0; OB_SUCC(ret) && i < tenant_group.count(); ++i) {
5471
      uint64_t amend_id = UINT64_MAX;
5472
      uint64_t this_id = UINT64_MAX;
5473
      TenantGroupBalanceInfo *this_info = tenant_group.at(i);
5474
      if (OB_UNLIKELY(NULL == this_info)) {
5475
        ret = OB_ERR_UNEXPECTED;
5476
        LOG_WARN("tenant group balance info ptr is null", K(ret), KP(this_info));
5477
      } else if (NULL == info_need_amend) {
5478
        info_need_amend = this_info;
5479
        idx = i;
5480
      } else if (OB_FAIL(info_need_amend->get_trick_id(amend_id))) {
5481
        LOG_WARN("fail to get trick id", K(ret));
5482
      } else if (OB_FAIL(this_info->get_trick_id(this_id))) {
5483
        LOG_WARN("fail to get trick id", K(ret));
5484
      } else if (UINT64_MAX == amend_id || UINT64_MAX == this_id) {
5485
        ret = OB_ERR_UNEXPECTED;
5486
        LOG_WARN("get invalid info id", K(ret), K(amend_id), K(this_id));
5487
      } else if (!is_stable_tenantgroup) {
5488
        if (amend_id > this_id) {
5489
          info_need_amend = this_info;
5490
          idx = i;
5491
        }
5492
      } else {
5493
        if (amend_id < this_id) {
5494
          info_need_amend = this_info;
5495
          idx = i;
5496
        }
5497
      }
5498
    }
5499
    if (-1 == idx) {
5500
      ret = OB_ERR_UNEXPECTED;
5501
      LOG_WARN("fail to get info", K(ret));
5502
    } else {
5503
      other_ttg.reset();
5504
      for (int64_t i = 0; OB_SUCC(ret) && i < tenant_group.count(); ++i) {
5505
        if (idx == i) {
5506
          // do nothing
5507
        } else if (OB_FAIL(other_ttg.push_back(tenant_group.at(i)))) {
5508
          LOG_WARN("fail to push back", K(ret));
5509
        }
5510
      }
5511
    }
5512
  }
5513
  return ret;
5514
}
5515

5516
int ObServerBalancer::divide_tenantgroup_balance_info(
5517
    common::ObIArray<TenantGroupBalanceInfo> &balance_info_array,
5518
    common::ObIArray<TenantGroupBalanceInfo *> &stable_tenant_group,
5519
    common::ObIArray<TenantGroupBalanceInfo *> &unstable_tenant_group)
5520
{
5521
  int ret = OB_SUCCESS;
5522
  if (OB_UNLIKELY(!inited_)) {
5523
    ret = OB_NOT_INIT;
5524
    LOG_WARN("not init", K(ret));
5525
  } else {
5526
    stable_tenant_group.reset();
5527
    unstable_tenant_group.reset();
5528
    for (int64_t i = 0; OB_SUCC(ret) && i < balance_info_array.count(); ++i) {
5529
      TenantGroupBalanceInfo &balance_info = balance_info_array.at(i);
5530
      if (balance_info.is_stable()) {
5531
        if (OB_FAIL(stable_tenant_group.push_back(&balance_info))) {
5532
          LOG_WARN("fail to push back", K(ret));
5533
        }
5534
      } else {
5535
        if (OB_FAIL(unstable_tenant_group.push_back(&balance_info))) {
5536
          LOG_WARN("fail to push back", K(ret));
5537
        }
5538
      }
5539
    }
5540
  }
5541
  return ret;
5542
}
5543

5544
int ObServerBalancer::calc_global_balance_resource_weights(
5545
    const common::ObZone &zone,
5546
    const common::ObIArray<common::ObAddr> &available_servers,
5547
    double *const resource_weights,
5548
    const int64_t weights_count)
5549
{
5550
  int ret = OB_SUCCESS;
5551
  common::ObArray<ObUnitManager::ObUnitLoad> zone_unit_loads;
5552
  common::ObArray<common::ObAddr> zone_servers;
5553
  if (OB_UNLIKELY(!inited_)) {
5554
    ret = OB_NOT_INIT;
5555
    LOG_WARN("not init", K(ret));
5556
  } else if (OB_UNLIKELY(zone.is_empty()
5557
                         || NULL == resource_weights
5558
                         || RES_MAX != weights_count)) {
5559
    ret = OB_INVALID_ARGUMENT;
5560
    LOG_WARN("invalid argument", K(ret), K(zone));
5561
  } else if (OB_ISNULL(unit_mgr_) || OB_ISNULL(server_mgr_)) {
5562
    ret = OB_ERR_UNEXPECTED;
5563
    LOG_WARN("unit_mgr_ or server_mgr_ is null", KR(ret), KP(unit_mgr_), KP(server_mgr_));
5564
  } else if (OB_FAIL(SVR_TRACER.get_servers_of_zone(zone, zone_servers))) {
5565
    LOG_WARN("fail to get zone servers", K(ret), K(zone));
5566
  } else {
5567
    LoadSum load_sum;
5568
    for (int64_t i = 0; OB_SUCC(ret) && i < zone_servers.count(); ++i) {
5569
      ObArray<ObUnitManager::ObUnitLoad> *unit_loads = NULL;
5570
      if (OB_FAIL(unit_mgr_->get_loads_by_server(zone_servers.at(i), unit_loads))) {
5571
        if (OB_ENTRY_NOT_EXIST != ret) {
5572
          LOG_WARN("fail to get loads by server", K(ret));
5573
        } else {
5574
          ret = OB_SUCCESS;
5575
        }
5576
      } else if (OB_UNLIKELY(NULL == unit_loads)) {
5577
        ret = OB_ERR_UNEXPECTED;
5578
        LOG_WARN("unit_loads ptr is null", K(ret), KP(unit_loads));
5579
      } else if (OB_FAIL(load_sum.append_load(*unit_loads))) {
5580
        LOG_WARN("fail to append load", K(ret));
5581
      } else {} // no more to do
5582
    }
5583
    ResourceSum resource_sum;
5584
    for (int64_t i = 0; OB_SUCC(ret) && i < available_servers.count(); ++i) {
5585
      const common::ObAddr &server = available_servers.at(i);
5586
      share::ObServerResourceInfo resource_info;
5587
      if (OB_FAIL(server_mgr_->get_server_resource_info(server, resource_info))) {
5588
        LOG_WARN("fail to get resource_info", KR(ret), K(server));
5589
      } else if (OB_FAIL(resource_sum.append_resource(resource_info))) {
5590
        LOG_WARN("fail to append resource", K(ret));
5591
      } else {} // no more to do
5592
    }
5593
    for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
5594
      ObResourceType resource_type = static_cast<ObResourceType>(i);
5595
      const double required = load_sum.get_required(resource_type);
5596
      const double capacity = resource_sum.get_capacity(resource_type);
5597
      if (required <= 0 || capacity <= 0) {
5598
        resource_weights[resource_type] = 0.0;
5599
      } else if (required >= capacity) {
5600
        resource_weights[resource_type] = 1.0;
5601
      } else {
5602
        resource_weights[resource_type] = required / capacity;
5603
      }
5604
    }
5605
    if (OB_SUCC(ret)) {
5606
      double sum = 0.0;
5607
      const int64_t N = available_servers.count();
5608
      for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
5609
        resource_weights[i] /= static_cast<double>(N);
5610
        sum += resource_weights[i];
5611
        if (resource_weights[i] < 0 || resource_weights[i] > 1) {
5612
          ret = common::OB_ERR_UNEXPECTED;
5613
          LOG_ERROR("weight shall be in interval [0,1]", K(i), "w", resource_weights[i]);
5614
        }
5615
      }
5616
      if (OB_SUCC(ret) && sum > 0) {
5617
        for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
5618
          resource_weights[i] /= sum;
5619
        }
5620
      }
5621
    }
5622
  }
5623

5624
  LOG_INFO("finish calc global balance resource weights", KR(ret), K(zone),
5625
      "cpu_weights", resource_weights[RES_CPU],
5626
      "mem_weights", resource_weights[RES_MEM],
5627
      "log_disk_weights", resource_weights[RES_LOG_DISK],
5628
      K(available_servers));
5629
  return ret;
5630
}
5631

5632
int ObServerBalancer::check_and_get_tenant_matrix_unit_num(
5633
    const Matrix<uint64_t> &tenant_id_matrix,
5634
    const common::ObZone &zone,
5635
    bool &unit_num_match,
5636
    ObIArray<int64_t> &column_unit_num_array)
5637
{
5638
  int ret = OB_SUCCESS;
5639
  if (OB_UNLIKELY(!inited_)) {
5640
    ret = OB_NOT_INIT;
5641
    LOG_WARN("not init", K(ret));
5642
  } else if (OB_UNLIKELY(!tenant_id_matrix.is_valid() || zone.is_empty())) {
5643
    ret = OB_INVALID_ARGUMENT;
5644
    LOG_WARN("invalid argument", K(ret), K(tenant_id_matrix), K(zone));
5645
  } else {
5646
    column_unit_num_array.reset();
5647
    unit_num_match = true;
5648
    const int64_t row_count = tenant_id_matrix.get_row_count();
5649
    const int64_t column_count = tenant_id_matrix.get_column_count();
5650
    for (int64_t i = 0; OB_SUCC(ret) && unit_num_match && i < column_count; ++i) {
5651
      ObArray<uint64_t> tenant_column;
5652
      for (int64_t j = 0; OB_SUCC(ret) && j < row_count; ++j) {
5653
        uint64_t tenant_id = OB_INVALID_ID;
5654
        if (OB_FAIL(tenant_id_matrix.get(j, i, tenant_id))) {
5655
          LOG_WARN("fail to get element", K(ret));
5656
        } else if (OB_UNLIKELY(OB_INVALID_ID == tenant_id)) {
5657
          ret = OB_ERR_UNEXPECTED;
5658
          LOG_WARN("tenant id value unexpected", K(ret), K(tenant_id));
5659
        } else if (OB_FAIL(tenant_column.push_back(tenant_id))) {
5660
          LOG_WARN("fail to push back", K(ret));
5661
        } else {} // no more to do
5662
      }
5663
      if (OB_SUCC(ret)) {
5664
        int64_t unit_num = 0;
5665
        if (OB_FAIL(check_and_get_tenant_column_unit_num(
5666
                tenant_column, zone, unit_num_match, unit_num))) {
5667
          LOG_WARN("fail to check and get tenant column unit num", K(ret));
5668
        } else if (!unit_num_match || 0 == unit_num) {
5669
          // unit num not match or unit num is zero on this zone
5670
        } else if (OB_FAIL(column_unit_num_array.push_back(unit_num))) {
5671
          LOG_WARN("fail to push back", K(ret));
5672
        } else {} // no more to do
5673
      }
5674
    }
5675
  }
5676
  return ret;
5677
}
5678

5679
int ObServerBalancer::check_and_get_tenant_column_unit_num(
5680
    const ObIArray<uint64_t> &tenant_id_column,
5681
    const common::ObZone &zone,
5682
    bool &unit_num_match,
5683
    int64_t &unit_num)
5684
{
5685
  int ret = OB_SUCCESS;
5686
  if (OB_UNLIKELY(!inited_)) {
5687
    ret = OB_NOT_INIT;
5688
    LOG_WARN("not init", K(ret));
5689
  } else if (OB_UNLIKELY(tenant_id_column.count() <= 0 || zone.is_empty())) {
5690
    ret = OB_INVALID_ARGUMENT;
5691
    LOG_WARN("invalid argument", K(ret), K(zone));
5692
  } else {
5693
    unit_num_match = true;
5694
    bool is_first = true;
5695
    int64_t prev_unit_num = -1;
5696
    for (int64_t i = 0; unit_num_match && OB_SUCC(ret) && i < tenant_id_column.count(); ++i) {
5697
      int64_t this_unit_num = -1;
5698
      const uint64_t tenant_id = tenant_id_column.at(i);
5699
      if (OB_UNLIKELY(NULL == unit_mgr_)) {
5700
        ret = OB_ERR_UNEXPECTED;
5701
        LOG_WARN("unit_mgr_ is null", K(ret), KP(unit_mgr_));
5702
      } else if (OB_FAIL(unit_mgr_->inner_get_tenant_zone_full_unit_num(
5703
              tenant_id, zone, this_unit_num))) {
5704
        LOG_WARN("fail to get tenant zone unit num", K(ret));
5705
      } else if (is_first || this_unit_num == prev_unit_num) {
5706
        is_first = false;
5707
        prev_unit_num = this_unit_num;
5708
      } else {
5709
        unit_num_match = false;
5710
      }
5711
    }
5712
    if (OB_SUCC(ret) && unit_num_match) {
5713
      unit_num = prev_unit_num;
5714
    }
5715
  }
5716
  return ret;
5717
}
5718

5719
int ObServerBalancer::generate_original_unit_matrix(
5720
    const Matrix<uint64_t> &tenant_id_matrix,
5721
    const common::ObZone &zone,
5722
    const ObIArray<int64_t> &column_unit_num_array,
5723
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix)
5724
{
5725
  int ret = OB_SUCCESS;
5726
  if (OB_UNLIKELY(!inited_)) {
5727
    ret = OB_NOT_INIT;
5728
    LOG_WARN("not init", K(ret));
5729
  } else if (OB_UNLIKELY(zone.is_empty())) {
5730
    ret = OB_INVALID_ARGUMENT;
5731
    LOG_WARN("invalid argument", K(ret), K(zone));
5732
  } else {
5733
    const int64_t tenant_matrix_row_count = tenant_id_matrix.get_row_count();
5734
    const int64_t tenant_matrix_column_count = tenant_id_matrix.get_column_count();
5735
    const int64_t unit_matrix_row_count = tenant_id_matrix.get_row_count();
5736
    int64_t unit_matrix_column_count = 0;
5737
    for (int64_t i = 0; i < column_unit_num_array.count(); ++i) {
5738
      unit_matrix_column_count += column_unit_num_array.at(i);
5739
    }
5740
    if (tenant_matrix_row_count <= 0 || tenant_matrix_row_count >= INT32_MAX
5741
        || tenant_matrix_column_count <= 0 || tenant_matrix_column_count >= INT32_MAX) {
5742
      ret = OB_ERR_UNEXPECTED;
5743
      LOG_WARN("tenant matrix row or column count unexpected", K(ret),
5744
               K(tenant_matrix_row_count), K(tenant_matrix_column_count));
5745
    } else if (unit_matrix_column_count <= 0 || unit_matrix_column_count >= INT32_MAX) {
5746
      ret = OB_ERR_UNEXPECTED;
5747
      LOG_WARN("unit matrix column count value unexpected", K(ret));
5748
    } else if (OB_FAIL(unit_migrate_stat_matrix.init(
5749
            unit_matrix_row_count, unit_matrix_column_count))) {
5750
      LOG_WARN("fail to init unit migrate stat", K(ret));
5751
    } else {
5752
      common::ObArray<ObUnitManager::ObUnitLoad> unit_loads;
5753
      const common::ObReplicaType replica_type = common::REPLICA_TYPE_FULL;
5754
      for (int64_t row = 0; OB_SUCC(ret) && row < tenant_matrix_row_count; ++row) {
5755
        int64_t acc_column_idx = 0;
5756
        for (int64_t column = 0; OB_SUCC(ret) && column < tenant_matrix_column_count; ++column) {
5757
          unit_loads.reset();
5758
          uint64_t tenant_id = OB_INVALID_ID;
5759
          if (OB_FAIL(tenant_id_matrix.get(row, column, tenant_id))) {
5760
            LOG_WARN("fail to get tenant id", K(ret));
5761
          } else if (OB_INVALID_ID == tenant_id) {
5762
            ret = OB_ERR_UNEXPECTED;
5763
            LOG_WARN("tenant id unexpected", K(ret), K(tenant_id));
5764
          } else if (OB_UNLIKELY(NULL == unit_mgr_)) {
5765
            ret = OB_ERR_UNEXPECTED;
5766
            LOG_WARN("unit_mgr_ is null", K(ret), KP(unit_mgr_));
5767
          } else if (OB_FAIL(unit_mgr_->get_tenant_zone_unit_loads(
5768
                  tenant_id, zone, replica_type, unit_loads))) {
5769
            if (OB_ENTRY_NOT_EXIST != ret) {
5770
              LOG_WARN("fail to get tenant zone unit infos", K(ret));
5771
            } else {
5772
              ret = OB_SUCCESS;
5773
            }
5774
          } else if (OB_FAIL(fill_unit_migrate_stat_matrix(
5775
                  unit_migrate_stat_matrix, tenant_id, row, acc_column_idx, unit_loads))) {
5776
            LOG_WARN("fail to fill unit migrate stat matrix", K(ret));
5777
          } else {
5778
            acc_column_idx += unit_loads.count();
5779
          }
5780
        }
5781
      }
5782
    }
5783
  }
5784
  return ret;
5785
}
5786

5787
int ObServerBalancer::fill_unit_migrate_stat_matrix(
5788
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
5789
    const uint64_t tenant_id,
5790
    const int64_t row,
5791
    const int64_t start_column_id,
5792
    const common::ObIArray<ObUnitManager::ObUnitLoad> &unit_loads)
5793
{
5794
  int ret = OB_SUCCESS;
5795
  if (OB_UNLIKELY(!inited_)) {
5796
    ret = OB_NOT_INIT;
5797
    LOG_WARN("not init", K(ret));
5798
  } else {
5799
    const int64_t unit_matrix_row_count = unit_migrate_stat_matrix.get_row_count();
5800
    const int64_t unit_matrix_column_count = unit_migrate_stat_matrix.get_column_count();
5801
    if (row >= unit_matrix_row_count
5802
        || start_column_id >= unit_matrix_column_count
5803
        || start_column_id + unit_loads.count() > unit_matrix_column_count) {
5804
      ret = OB_INVALID_ARGUMENT;
5805
      LOG_WARN("invalid argument", K(ret), K(row), K(start_column_id), K(unit_loads),
5806
               K(unit_matrix_row_count), K(unit_matrix_column_count));
5807
    } else {
5808
      for (int64_t i = 0; OB_SUCC(ret) && i < unit_loads.count(); ++i) {
5809
        const ObUnitManager::ObUnitLoad &unit_load = unit_loads.at(i);
5810
        if (!unit_load.is_valid()) {
5811
          ret = OB_ERR_UNEXPECTED;
5812
          LOG_WARN("unit load invalid", K(ret), K(unit_load));
5813
        } else {
5814
          UnitMigrateStat stat;
5815
          stat.tenant_id_ = tenant_id;
5816
          stat.original_pos_ = unit_load.unit_->server_;
5817
          stat.arranged_pos_ = unit_load.unit_->server_;
5818
          stat.unit_load_ = unit_load;
5819
          if (OB_FAIL(unit_migrate_stat_matrix.set(row, start_column_id + i, stat))) {
5820
            LOG_WARN("fail to set", K(ret));
5821
          } else {} // no more to do
5822
        }
5823
      }
5824
    }
5825
  }
5826
  return ret;
5827
}
5828

5829
int ObServerBalancer::single_unit_migrate_stat_matrix_balance(
5830
    TenantGroupBalanceInfo &balance_info,
5831
    const common::ObIArray<common::ObAddr> &available_servers)
5832
{
5833
  int ret = OB_SUCCESS;
5834
  const ObIArray<int64_t> &column_unit_num_array = balance_info.column_unit_num_array_;
5835
  Matrix<UnitMigrateStat> &unit_migrate_stat_matrix = balance_info.unit_migrate_stat_matrix_;
5836
  ObIArray<UnitGroupLoad> &unitgroup_loads = balance_info.unitgroup_load_array_;
5837
  ObIArray<ServerLoad> &server_loads = balance_info.server_load_array_;
5838
  if (OB_UNLIKELY(!inited_)) {
5839
    ret = OB_NOT_INIT;
5840
    LOG_WARN("not init", K(ret));
5841
  } else if (column_unit_num_array.count() <= 0) {
5842
    ret = OB_INVALID_ARGUMENT;
5843
    LOG_WARN("invalid argument", K(ret));
5844
  } else if (OB_FAIL(balance_row_units(
5845
          balance_info, column_unit_num_array, unit_migrate_stat_matrix,
5846
          unitgroup_loads, server_loads, available_servers))) {
5847
    LOG_WARN("fail to balance row units", K(ret));
5848
  } else if (OB_FAIL(inner_ttg_balance_strategy_.coordinate_all_column_unit_group(
5849
          column_unit_num_array, unit_migrate_stat_matrix, unitgroup_loads))) {
5850
    LOG_WARN("fail to coordinate column units", K(ret));
5851
  } else {} // no more to do
5852
  return ret;
5853
}
5854

5855
int ObServerBalancer::InnerTenantGroupBalanceStrategy::coordinate_all_column_unit_group(
5856
    const common::ObIArray<int64_t> &column_unit_num_array,
5857
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
5858
    const common::ObIArray<UnitGroupLoad> &unitgroup_loads)
5859
{
5860
  int ret = OB_SUCCESS;
5861
  if (column_unit_num_array.count() <= 0
5862
             || unit_migrate_stat_matrix.get_row_count() <= 0
5863
             || unitgroup_loads.count() <= 0) {
5864
    ret = OB_INVALID_ARGUMENT;
5865
    LOG_WARN("invalid argument", K(ret), K(column_unit_num_array),
5866
             K(unit_migrate_stat_matrix), K(unitgroup_loads));
5867
  } else {
5868
    int64_t accumulate_column_idx = 0;
5869
    for (int64_t i = 0; OB_SUCC(ret) && i < column_unit_num_array.count(); ++i) {
5870
      if (OB_FAIL(coordinate_single_column_unit_group(
5871
              accumulate_column_idx, column_unit_num_array.at(i),
5872
              unit_migrate_stat_matrix, unitgroup_loads))) {
5873
        LOG_WARN("fail to coordinate single column unit", K(ret));
5874
      } else {
5875
        accumulate_column_idx += column_unit_num_array.at(i);
5876
      }
5877
    }
5878
  }
5879
  return ret;
5880
}
5881

5882
int ObServerBalancer::InnerTenantGroupBalanceStrategy::coordinate_single_column_unit_group(
5883
    const int64_t start_column_idx,
5884
    const int64_t this_column_count,
5885
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
5886
    const common::ObIArray<UnitGroupLoad> &unitgroup_loads)
5887
{
5888
  int ret = OB_SUCCESS;
5889
  if (start_column_idx < 0
5890
      || this_column_count <= 0
5891
      || start_column_idx + this_column_count > unit_migrate_stat_matrix.get_column_count()
5892
      || unit_migrate_stat_matrix.get_row_count() <= 0
5893
      || unit_migrate_stat_matrix.get_column_count() <= 0
5894
      || unitgroup_loads.count() <= 0
5895
      || unitgroup_loads.count() != unit_migrate_stat_matrix.get_column_count()) {
5896
    ret = OB_INVALID_ARGUMENT;
5897
    LOG_WARN("invalid argument", K(ret), K(start_column_idx), K(this_column_count),
5898
             K(unit_migrate_stat_matrix), K(unitgroup_loads));
5899
  } else {
5900
    const int64_t end_column_idx = start_column_idx + this_column_count;
5901
    ObArray<common::ObAddr> dest_server_array;
5902
    for (int64_t i = start_column_idx; OB_SUCC(ret) && i < end_column_idx; ++i) {
5903
      if (OB_FAIL(dest_server_array.push_back(unitgroup_loads.at(i).server_))) {
5904
        LOG_WARN("fail to push back", K(ret));
5905
      }
5906
    }
5907
    if (OB_SUCC(ret)) {
5908
      if (OB_FAIL(do_coordinate_single_column_unit_group(
5909
              start_column_idx, this_column_count,
5910
              unit_migrate_stat_matrix, dest_server_array))) {
5911
        LOG_WARN("fail to coordinate single column unit group", K(ret));
5912
      } else {} // no more to do
5913
    }
5914
  }
5915
  return ret;
5916
}
5917

5918
int ObServerBalancer::InnerTenantGroupBalanceStrategy::do_coordinate_single_column_unit_group(
5919
    const int64_t start_column_idx,
5920
    const int64_t column_count,
5921
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
5922
    const common::ObIArray<common::ObAddr> &dest_servers)
5923
{
5924
  int ret = OB_SUCCESS;
5925
  if (start_column_idx < 0
5926
      || column_count <= 0
5927
      || dest_servers.count() <= 0
5928
      || start_column_idx + column_count > unit_migrate_stat_matrix.get_column_count()
5929
      || unit_migrate_stat_matrix.get_row_count() <= 0
5930
      || unit_migrate_stat_matrix.get_column_count() <= 0
5931
      || dest_servers.count() != column_count) {
5932
    ret = OB_INVALID_ARGUMENT;
5933
    LOG_WARN("invalid argument", K(ret), K(start_column_idx), K(column_count),
5934
             K(unit_migrate_stat_matrix), K(dest_servers));
5935
  } else {
5936
    for (int64_t row = 0;
5937
         OB_SUCC(ret) && row < unit_migrate_stat_matrix.get_row_count();
5938
         ++row) {
5939
      if (OB_FAIL(do_coordinate_single_unit_row(
5940
              row, start_column_idx, column_count,
5941
              dest_servers, unit_migrate_stat_matrix))) {
5942
        LOG_WARN("fail to coordinate single unit row", K(ret));
5943
      }
5944
    }
5945
  }
5946
  // After the above operation is over, the actual unit has been aggregated.
5947
  // In order to be more tidy in the form,
5948
  // the matrix is sorted vertically according to the server to facilitate the observation of information during debugging.
5949
  UnitMigrateStatCmp cmp_operator;
5950
  if (OB_FAIL(ret)) {
5951
    // failed in the previous procedure
5952
  } else if (OB_FAIL(unit_migrate_stat_matrix.sort_column_group(
5953
            start_column_idx, column_count, cmp_operator))) {
5954
    LOG_WARN("fail to sort unit group by arranged pos", K(ret));
5955
  } else if (OB_FAIL(cmp_operator.get_ret())) {
5956
    LOG_WARN("fail to sort column group", K(ret));
5957
  } else {} // no more to do
5958
  return ret;
5959
}
5960

5961
int ObServerBalancer::InnerTenantGroupBalanceStrategy::do_coordinate_single_unit_row(
5962
    const int64_t row,
5963
    const int64_t start_column_idx,
5964
    const int64_t column_count,
5965
    const common::ObIArray<common::ObAddr> &candidate_servers,
5966
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix)
5967
{
5968
  int ret = OB_SUCCESS;
5969
  if (row < 0 || start_column_idx < 0 || column_count <= 0
5970
      || start_column_idx + column_count > unit_migrate_stat_matrix.get_column_count()
5971
      || unit_migrate_stat_matrix.get_row_count() <= 0
5972
      || unit_migrate_stat_matrix.get_column_count() <= 0
5973
      || row >= unit_migrate_stat_matrix.get_row_count()
5974
      || candidate_servers.count() <= 0) {
5975
    ret = OB_INVALID_ARGUMENT;
5976
    LOG_WARN("invalid argument", K(ret), K(row), K(start_column_idx), K(column_count),
5977
             K(unit_migrate_stat_matrix), K(candidate_servers));
5978
  } else {
5979
    const int64_t end_column_idx = start_column_idx + column_count;
5980
    ObArray<common::ObAddr> excluded_array;
5981
    // Need to run twice, in the first pass, throw the ones that meet the requirements into the exclude array
5982
    // The second pass will adjust those that do not meet the requirements
5983
    for (int64_t column = start_column_idx; OB_SUCC(ret) && column < end_column_idx; ++column) {
5984
      UnitMigrateStat unit_migrate_stat;
5985
      if (OB_FAIL(unit_migrate_stat_matrix.get(row, column, unit_migrate_stat))) {
5986
        LOG_WARN("fail to get unit migrate stat", K(ret));
5987
      } else if (!has_exist_in_array(candidate_servers, unit_migrate_stat.original_pos_)) {
5988
        // The current location is not in the candidate servers and needs to be re-allocated,
5989
        // so the first pass will not be processed here
5990
      } else {
5991
        unit_migrate_stat.arranged_pos_ = unit_migrate_stat.original_pos_;
5992
        if (OB_FAIL(unit_migrate_stat_matrix.set(row, column, unit_migrate_stat))) {
5993
          LOG_WARN("fail to set unit migrate stat", K(ret));
5994
        } else if (OB_FAIL(excluded_array.push_back(unit_migrate_stat.original_pos_))) {
5995
          LOG_WARN("fail to push back", K(ret));
5996
        }
5997
      }
5998
    }
5999
    for (int64_t column = start_column_idx; OB_SUCC(ret) && column < end_column_idx; ++column) {
6000
      UnitMigrateStat unit_migrate_stat;
6001
      if (OB_FAIL(unit_migrate_stat_matrix.get(row, column, unit_migrate_stat))) {
6002
        LOG_WARN("fail to get unit migrate stat", K(ret));
6003
      } else if (has_exist_in_array(candidate_servers, unit_migrate_stat.original_pos_)) {
6004
        // The current position is already in the candidate servers,
6005
        // it has been processed in the first pass, skip it
6006
      } else {
6007
        // The current position is not in the candidate servers and needs to be re-allocated
6008
        if (OB_FAIL(arrange_unit_migrate_stat_pos(
6009
                candidate_servers, unit_migrate_stat, excluded_array))) {
6010
          LOG_WARN("fail to arrange unit migrate stat pos", K(ret));
6011
        } else if (OB_FAIL(unit_migrate_stat_matrix.set(row, column, unit_migrate_stat))) {
6012
          LOG_WARN("fail to set unit migrate stat", K(ret));
6013
        } else if (OB_FAIL(excluded_array.push_back(unit_migrate_stat.arranged_pos_))) {
6014
          LOG_WARN("fail to push back", K(ret));
6015
        } else {} // fine, no more to do
6016
      }
6017
    }
6018
  }
6019
  return ret;
6020
}
6021

6022
int ObServerBalancer::InnerTenantGroupBalanceStrategy::arrange_unit_migrate_stat_pos(
6023
    const common::ObIArray<common::ObAddr> &candidate_servers,
6024
    UnitMigrateStat &unit_migrate_stat,
6025
    const common::ObIArray<common::ObAddr> &excluded_array)
6026
{
6027
  int ret = OB_SUCCESS;
6028
  if (OB_UNLIKELY(candidate_servers.count() <= 0)) {
6029
    ret = OB_INVALID_ARGUMENT;
6030
    LOG_WARN("invalid argument", K(ret), K(candidate_servers));
6031
  } else if (has_exist_in_array(candidate_servers, unit_migrate_stat.original_pos_)) {
6032
    ret = OB_INVALID_ARGUMENT;
6033
    LOG_WARN("unit migrate stat already in candidate", K(ret),
6034
             K(unit_migrate_stat), K(candidate_servers));
6035
  } else {
6036
    bool find = false;
6037
    for (int64_t i = 0; !find && OB_SUCC(ret) && i < candidate_servers.count(); ++i) {
6038
      const common::ObAddr &this_candidate = candidate_servers.at(i);
6039
      if (has_exist_in_array(excluded_array, this_candidate)) {
6040
        // Cannot be used as a new pos in the excluded array
6041
      } else {
6042
        unit_migrate_stat.arranged_pos_ = this_candidate;
6043
        find = true;
6044
      }
6045
    }
6046
    if (OB_FAIL(ret)) {
6047
    } else if (!find) {
6048
      ret = OB_ERR_UNEXPECTED;
6049
      LOG_WARN("cannot find a new pos for unit migrate stat", K(ret),
6050
               K(candidate_servers), K(excluded_array));
6051
    } else { /* good */ }
6052
  }
6053
  return ret;
6054
}
6055

6056
int ObServerBalancer::balance_row_units(
6057
    TenantGroupBalanceInfo &balance_info,
6058
    const common::ObIArray<int64_t> &column_unit_num_array,
6059
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
6060
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6061
    common::ObIArray<ServerLoad> &server_loads,
6062
    const common::ObIArray<common::ObAddr> &available_servers)
6063
{
6064
  int ret = OB_SUCCESS;
6065
  if (OB_UNLIKELY(!inited_)) {
6066
    ret = OB_NOT_INIT;
6067
    LOG_WARN("not init", K(ret));
6068
  } else if (column_unit_num_array.count() <= 0
6069
             || unit_migrate_stat_matrix.get_row_count() <= 0
6070
             || unit_migrate_stat_matrix.get_column_count() <= 0
6071
             || available_servers.count() <= 0) {
6072
    ret = OB_INVALID_ARGUMENT;
6073
    LOG_WARN("invalid argument", K(ret), K(column_unit_num_array),
6074
             K(available_servers), K(unit_migrate_stat_matrix));
6075
  } else if (OB_FAIL(generate_unitgroup_and_server_load(
6076
          unit_migrate_stat_matrix, column_unit_num_array, available_servers,
6077
          unitgroup_loads, server_loads))) {
6078
    LOG_WARN("fail to generate server and unitgroup load", K(ret));
6079
  } else if (OB_FAIL(calc_row_balance_resource_weights(
6080
          server_loads, unitgroup_loads, balance_info.intra_weights_, RES_MAX))) {
6081
    LOG_WARN("fail to calc resource weights", K(ret));
6082
  } else if (OB_FAIL(update_server_load_value(
6083
          balance_info.intra_weights_, RES_MAX, server_loads))) {
6084
    LOG_WARN("fail to update unitgroup and server load value", K(ret));
6085
  } else if (OB_FAIL(do_balance_row_units(
6086
          balance_info, unit_migrate_stat_matrix, unitgroup_loads, server_loads))) {
6087
    LOG_WARN("fail to do balance row units", K(ret));
6088
  } else {} // no more to do
6089
  return ret;
6090
}
6091

6092
int ObServerBalancer::generate_unitgroup_and_server_load(
6093
    const Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
6094
    const common::ObIArray<int64_t> &column_unit_num_array,
6095
    const common::ObIArray<common::ObAddr> &available_servers,
6096
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6097
    common::ObIArray<ServerLoad> &server_loads)
6098
{
6099
  int ret = OB_SUCCESS;
6100
  if (OB_UNLIKELY(!inited_)) {
6101
    ret = OB_NOT_INIT;
6102
    LOG_WARN("not init", K(ret));
6103
  } else if (unit_migrate_stat_matrix.get_row_count() <= 0
6104
             || unit_migrate_stat_matrix.get_column_count() <= 0
6105
             || column_unit_num_array.count() <= 0
6106
             || available_servers.count() <= 0) {
6107
    ret = OB_INVALID_ARGUMENT;
6108
    LOG_WARN("invalid argument", K(ret), K(unit_migrate_stat_matrix), K(available_servers));
6109
  } else if (OB_FAIL(generate_unitgroup_load(
6110
          unit_migrate_stat_matrix, column_unit_num_array, unitgroup_loads))) {
6111
    LOG_WARN("fail to generate unitgroup load", K(ret));
6112
  } else if (OB_FAIL(generate_server_load(available_servers, unitgroup_loads, server_loads))) {
6113
    LOG_WARN("fail to generate server load");
6114
  } else {} // finish
6115
  return ret;
6116
}
6117

6118
int ObServerBalancer::generate_unitgroup_load(
6119
    const Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
6120
    const common::ObIArray<int64_t> &column_unit_num_array,
6121
    common::ObIArray<UnitGroupLoad> &unitgroup_loads)
6122
{
6123
  int ret = OB_SUCCESS;
6124
  if (OB_UNLIKELY(!inited_)) {
6125
    ret = OB_NOT_INIT;
6126
    LOG_WARN("not init", K(ret));
6127
  } else {
6128
    unitgroup_loads.reset();
6129
    // A separate unitgroup_loads is generated for each column of the unit matrix
6130
    int64_t accumulate_idx = 0;
6131
    for (int64_t i = 0; OB_SUCC(ret) && i < column_unit_num_array.count(); ++i) {
6132
      for (int64_t j = 0; OB_SUCC(ret) && j < column_unit_num_array.at(i); ++j) {
6133
        int64_t column = accumulate_idx + j;
6134
        if (OB_FAIL(generate_column_unitgroup_load(
6135
                column, unit_migrate_stat_matrix, accumulate_idx,
6136
                column_unit_num_array.at(i), unitgroup_loads))) {
6137
          LOG_WARN("fail to generate unit group load", K(ret));
6138
        }
6139
      }
6140
      if (OB_SUCC(ret)) {
6141
        accumulate_idx += column_unit_num_array.at(i);
6142
      }
6143
    }
6144
  }
6145
  return ret;
6146
}
6147

6148
int ObServerBalancer::generate_column_unitgroup_load(
6149
    const int64_t column,
6150
    const Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
6151
    const int64_t start_column_idx,
6152
    const int64_t column_count,
6153
    common::ObIArray<UnitGroupLoad> &unitgroup_loads)
6154
{
6155
  int ret = OB_SUCCESS;
6156
  if (OB_UNLIKELY(!inited_)) {
6157
    ret = OB_NOT_INIT;
6158
    LOG_WARN("not init", K(ret));
6159
  } else if (column < 0 || start_column_idx < 0 || column_count <= 0
6160
             || start_column_idx + column_count > unit_migrate_stat_matrix.get_column_count()
6161
             || column >= unit_migrate_stat_matrix.get_column_count()) {
6162
    ret = OB_INVALID_ARGUMENT;
6163
    LOG_WARN("invalid argument", K(ret), K(start_column_idx), K(column),
6164
             K(column_count), K(unit_migrate_stat_matrix));
6165
  } else {
6166
    UnitGroupLoad unitgroup_load;
6167
    common::ObAddr server_addr;
6168
    uint64_t column_tenant_id = OB_INVALID_ID;
6169
    bool is_first = true;
6170
    for (int64_t row = 0; row < unit_migrate_stat_matrix.get_row_count(); ++row) {
6171
      UnitMigrateStat unit_migrate_stat;
6172
      if (OB_FAIL(unit_migrate_stat_matrix.get(row, column, unit_migrate_stat))) {
6173
        LOG_WARN("fail to get unit migrate stat matrix", K(ret));
6174
      } else if (OB_FAIL(unitgroup_load.unit_loads_.push_back(
6175
              *unit_migrate_stat.unit_load_.unit_config_))) {
6176
        LOG_WARN("fail to append unit load", K(ret));
6177
      } else {
6178
        if (is_first) {
6179
          column_tenant_id = unit_migrate_stat.tenant_id_;
6180
          server_addr = unit_migrate_stat.arranged_pos_;
6181
        }
6182
        is_first = false;
6183
      }
6184
    }
6185
    if (OB_FAIL(ret)) {
6186
    } else if (!server_addr.is_valid() || OB_INVALID_ID == column_tenant_id) {
6187
      ret = OB_ERR_UNEXPECTED;
6188
      LOG_WARN("server addr unexpected", K(ret), K(column),
6189
               K(server_addr), K(column_tenant_id), K(unit_migrate_stat_matrix));
6190
    } else {
6191
      unitgroup_load.server_ = server_addr;
6192
      unitgroup_load.column_tenant_id_ = column_tenant_id;
6193
      unitgroup_load.start_column_idx_ = start_column_idx;
6194
      unitgroup_load.column_count_ = column_count;
6195
      if (OB_FAIL(unitgroup_load.sum_group_load())) {
6196
        LOG_WARN("fail to sum group load", K(ret));
6197
      } else if (OB_FAIL(unitgroup_loads.push_back(unitgroup_load))) {
6198
        LOG_WARN("fail to push back", K(ret));
6199
      } else {} // good
6200
    }
6201
  }
6202
  return ret;
6203
}
6204

6205
// When balancing within a group,
6206
// it is necessary to virtually set the resources (cpu, memory) of each server in the zone to the same value.
6207
// Here is actually directly taking the resources of the first server
6208
int ObServerBalancer::try_regulate_intra_ttg_resource_info(
6209
    const share::ObServerResourceInfo &this_resource_info,
6210
    share::ObServerResourceInfo &intra_ttg_resource_info)
6211
{
6212
  int ret = OB_SUCCESS;
6213

6214
  if (OB_UNLIKELY(!inited_)) {
6215
    ret = OB_NOT_INIT;
6216
    LOG_WARN("not init", K(ret));
6217
  } else if (OB_UNLIKELY(!this_resource_info.is_valid())) {
6218
    ret = OB_INVALID_ARGUMENT;
6219
    LOG_WARN("invalid argument", K(ret), K(this_resource_info));
6220
  } else if (intra_ttg_resource_info.is_valid()) {
6221
    // bypass
6222
  } else {
6223
    intra_ttg_resource_info = this_resource_info;
6224
  }
6225

6226
  return ret;
6227
}
6228

6229
int ObServerBalancer::generate_server_load(
6230
    const common::ObIArray<common::ObAddr> &available_servers,
6231
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6232
    common::ObIArray<ServerLoad> &server_loads)
6233
{
6234
  int ret = OB_SUCCESS;
6235
  if (OB_UNLIKELY(!inited_)) {
6236
    ret = OB_NOT_INIT;
6237
    LOG_WARN("not init", K(ret));
6238
  } else if (available_servers.count() <= 0) {
6239
    ret = OB_INVALID_ARGUMENT;
6240
    LOG_WARN("invalid argument", K(ret), K(available_servers));
6241
  } else if (OB_ISNULL(server_mgr_)) {
6242
    ret = OB_ERR_UNEXPECTED;
6243
    LOG_WARN("server_mgr_ is null", KR(ret), KP(server_mgr_));
6244
  } else {
6245
    // Place the generated unitgroup load into the corresponding server load
6246
    server_loads.reset();
6247
    ServerLoad server_load;
6248
    share::ObServerResourceInfo resource_info;
6249
    share::ObServerResourceInfo intra_ttg_resource_info;
6250
    // Pre-fill the server first, and fill in the server resource info
6251
    for (int64_t i = 0; OB_SUCC(ret) && i < available_servers.count(); ++i) {
6252
      server_load.reset();
6253
      resource_info.reset();
6254
      server_load.server_ = available_servers.at(i);
6255
      if (OB_FAIL(server_mgr_->get_server_resource_info(server_load.server_, resource_info))) {
6256
        LOG_WARN("fail to get server status", KR(ret), K(server_load.server_));
6257
      } else if (OB_FAIL(try_regulate_intra_ttg_resource_info(
6258
          resource_info,
6259
          intra_ttg_resource_info))) {
6260
        LOG_WARN("fail to try regulate intra resource info", K(ret));
6261
      } else {
6262
        server_load.resource_info_ = resource_info;
6263
        if (OB_FAIL(server_loads.push_back(server_load))) {
6264
          LOG_WARN("fail to push back", K(ret));
6265
        } else {} // no more to do
6266
      }
6267
    }
6268
    // Then fill in the intra ttg resource info of each server
6269
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
6270
      ServerLoad &server_load = server_loads.at(i);
6271
      if (OB_UNLIKELY(!intra_ttg_resource_info.is_valid())) {
6272
        ret = OB_ERR_UNEXPECTED;
6273
        LOG_WARN("invalid intra ttg resource info", K(ret), K(intra_ttg_resource_info));
6274
      } else {
6275
        server_load.intra_ttg_resource_info_ = intra_ttg_resource_info;
6276
      }
6277
    }
6278
    // Then fill the unitgroup load into the corresponding server load
6279
    const int64_t server_load_count = server_loads.count();
6280
    common::ObArray<UnitGroupLoad *> wild_ug_loads; // the ug not in available servers
6281
    for (int64_t i = 0; OB_SUCC(ret) && i < unitgroup_loads.count(); ++i) {
6282
      bool find = false;
6283
      UnitGroupLoad &unitgroup_load = unitgroup_loads.at(i);
6284
      for (int64_t j = 0; !find && OB_SUCC(ret) && j < server_load_count; ++j) {
6285
        ServerLoad &server_load = server_loads.at(j);
6286
        if (server_load.server_ != unitgroup_load.server_) {
6287
          // go on to check next
6288
        } // NOTE: Push in here is the pointer
6289
        else if (OB_FAIL(server_load.unitgroup_loads_.push_back(&unitgroup_load))) {
6290
          LOG_WARN("fail to push back", K(ret));
6291
        } else {
6292
          unitgroup_load.server_load_ = &server_load;
6293
          find = true;
6294
        }
6295
      }
6296
      if (OB_FAIL(ret)) {
6297
      } else if (server_load_count <= 0) {
6298
        ret = OB_ERR_UNEXPECTED;
6299
        LOG_WARN("server load count unexpected", K(ret), K(server_load_count));
6300
      } else if (!find) {
6301
        if (OB_FAIL(wild_ug_loads.push_back(&unitgroup_load))) {
6302
          LOG_WARN("fail to push back", K(ret));
6303
        }
6304
      } else {} // find one, good
6305
    }
6306
    for (int64_t i = 0; OB_SUCC(ret) && i < wild_ug_loads.count(); ++i) {
6307
      UnitGroupLoad *ug_load = wild_ug_loads.at(i);
6308
      if (OB_UNLIKELY(NULL == ug_load)) {
6309
        ret = OB_ERR_UNEXPECTED;
6310
        LOG_WARN("ug ptr is null", K(ret));
6311
      } else if (OB_FAIL(arrange_server_load_for_wild_unitgroup(*ug_load, server_loads))) {
6312
        LOG_WARN("cannot arrange server load for wild unitgroup", K(ret));
6313
      } else {} // no more to do
6314
    }
6315
  }
6316
  return ret;
6317
}
6318

6319
int ObServerBalancer::arrange_server_load_for_wild_unitgroup(
6320
    UnitGroupLoad &unitgroup_load,
6321
    common::ObIArray<ServerLoad> &server_loads)
6322
{
6323
  int ret = OB_SUCCESS;
6324
  bool find = false;
6325
  for (int64_t i = 0; !find && OB_SUCC(ret) && i < server_loads.count(); ++i) {
6326
    ServerLoad &server_load = server_loads.at(i);
6327
    bool same_pool_found = false;
6328
    for (int64_t j = 0;
6329
         !same_pool_found && OB_SUCC(ret) && j < server_load.unitgroup_loads_.count();
6330
         ++j) {
6331
      UnitGroupLoad *this_ug = server_load.unitgroup_loads_.at(j);
6332
      if (OB_UNLIKELY(NULL == this_ug)) {
6333
        ret = OB_ERR_UNEXPECTED;
6334
        LOG_WARN("ug ptr is null", K(ret));
6335
      } else if (this_ug->column_tenant_id_ != unitgroup_load.column_tenant_id_) {
6336
        // good, go on to check next
6337
      } else {
6338
        same_pool_found = true;
6339
      }
6340
    }
6341
    if (OB_FAIL(ret)) {
6342
    } else if (same_pool_found) {
6343
      // A ug from the same resource pool as unitgroup_load was found on the server,
6344
      // and the server is not suitable
6345
    } else if (OB_FAIL(server_load.unitgroup_loads_.push_back(&unitgroup_load))) {
6346
      LOG_WARN("fail to push back", K(ret));
6347
    } else {
6348
      unitgroup_load.server_ = server_load.server_;
6349
      unitgroup_load.server_load_ = &server_load;
6350
      find = true;
6351
    }
6352
  }
6353
  if (OB_FAIL(ret)) {
6354
  } else if (!find) {
6355
    ret = OB_MACHINE_RESOURCE_NOT_ENOUGH;
6356
    LOG_WARN("no available server to hold more unit",
6357
             K(ret), "tenant_id", unitgroup_load.column_tenant_id_);
6358
  }
6359
  return ret;
6360
}
6361

6362
int ObServerBalancer::calc_row_balance_resource_weights(
6363
    const common::ObIArray<ServerLoad> &server_loads,
6364
    const common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6365
    double *const resource_weights,
6366
    const int64_t weights_count)
6367
{
6368
  int ret = OB_SUCCESS;
6369
  if (OB_UNLIKELY(!inited_)) {
6370
    ret = OB_NOT_INIT;
6371
    LOG_WARN("not init", K(ret));
6372
  } else if (OB_UNLIKELY(server_loads.count() <= 0
6373
                         || unitgroup_loads.count() <= 0
6374
                         || NULL == resource_weights
6375
                         || RES_MAX != weights_count)) {
6376
    ret = OB_INVALID_ARGUMENT;
6377
    LOG_WARN("invalid argument", K(ret), K(server_loads), K(unitgroup_loads));
6378
  } else {
6379
    for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
6380
      ObResourceType resource_type = static_cast<ObResourceType>(i);
6381
      double sum_capacity = 0.0;
6382
      for (int64_t j = 0; j < server_loads.count(); ++j) {
6383
        sum_capacity += server_loads.at(j).get_intra_ttg_resource_capacity(resource_type);
6384
      }
6385
      double sum_assigned = 0.0;
6386
      for (int64_t j = 0; j < unitgroup_loads.count(); ++j) {
6387
        sum_assigned += unitgroup_loads.at(j).load_sum_.get_required(resource_type);
6388
      }
6389
      if (sum_assigned <= 0 || sum_capacity <= 0) {
6390
        resource_weights[resource_type] = 0.0;
6391
      } else if (sum_assigned >= sum_capacity) {
6392
        resource_weights[resource_type] = 1.0;
6393
      } else {
6394
        resource_weights[resource_type] = sum_assigned / sum_capacity;
6395
      }
6396
    }
6397
    if (OB_SUCC(ret)) { // Weight normalization
6398
      double sum = 0.0;
6399
      const int64_t N = server_loads.count();
6400
      for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
6401
        resource_weights[i] /= static_cast<double>(N);
6402
        sum += resource_weights[i];
6403
        if (resource_weights[i] < 0 || resource_weights[i] > 1) {
6404
          ret = common::OB_ERR_UNEXPECTED;
6405
          LOG_ERROR("weight shall be in interval [0,1]", K(i), "w", resource_weights[i]);
6406
        }
6407
      }
6408
      if (OB_SUCC(ret) && sum > 0) {
6409
        for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
6410
          resource_weights[i] /= sum;
6411
        }
6412
      }
6413
    }
6414
  }
6415
  return ret;
6416
}
6417

6418
int ObServerBalancer::update_server_load_value(
6419
    const double *const resource_weights,
6420
    const int64_t weights_count,
6421
    common::ObIArray<ServerLoad> &server_loads)
6422
{
6423
  int ret = OB_SUCCESS;
6424
  if (OB_UNLIKELY(!inited_)) {
6425
    ret = OB_NOT_INIT;
6426
    LOG_WARN("not init", K(ret));
6427
  } else if (weights_count != RES_MAX || NULL == resource_weights) {
6428
    ret = OB_INVALID_ARGUMENT;
6429
    LOG_WARN("invalid argument", K(ret), K(weights_count), KP(resource_weights));
6430
  } else {
6431
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
6432
      ServerLoad &server_load = server_loads.at(i);
6433
      if (!server_load.is_valid()) {
6434
        ret = OB_ERR_UNEXPECTED;
6435
        LOG_WARN("should be a valid server load here", K(ret), K(server_load));
6436
      } else {
6437
        for (int32_t j = RES_CPU; j < RES_MAX; ++j) {
6438
          server_load.intra_weights_[j] = resource_weights[j];
6439
        }
6440
      }
6441
    }
6442
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
6443
      ServerLoad &server_load = server_loads.at(i);
6444
      if (!server_load.is_valid()) {
6445
        ret = OB_ERR_UNEXPECTED;
6446
        LOG_WARN("should be a valid server load here", K(ret), K(server_load));
6447
      } else if (OB_FAIL(server_load.update_load_value())) {
6448
        LOG_WARN("server fails to update load value", K(ret));
6449
      } else {} // no more to do
6450
    }
6451
  }
6452
  return ret;
6453
}
6454

6455
int ObServerBalancer::do_balance_row_units(
6456
    TenantGroupBalanceInfo &balance_info,
6457
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
6458
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6459
    common::ObIArray<ServerLoad> &server_loads)
6460
{
6461
  int ret = OB_SUCCESS;
6462
  if (OB_UNLIKELY(!inited_)) {
6463
    ret = OB_NOT_INIT;
6464
    LOG_WARN("not init", K(ret));
6465
  } else if (OB_FAIL(inner_ttg_balance_strategy_.do_balance_row_units(
6466
          balance_info, unit_migrate_stat_matrix, unitgroup_loads, server_loads))) {
6467
    LOG_WARN("fail to do balance row units", K(ret));
6468
  } else {}
6469
  return ret;
6470
}
6471

6472
int ObServerBalancer::CountBalanceStrategy::do_balance_row_units(
6473
    TenantGroupBalanceInfo &balance_info,
6474
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
6475
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6476
    common::ObIArray<ServerLoad> &server_loads)
6477
{
6478
  int ret = OB_SUCCESS;
6479
  if (OB_UNLIKELY(unitgroup_loads.count() <= 0 || server_loads.count() <= 0)) {
6480
    ret = OB_INVALID_ARGUMENT;
6481
    LOG_WARN("invalid argument", K(ret), K(server_loads), K(unitgroup_loads));
6482
  } else if (OB_FAIL(make_count_balanced(
6483
          balance_info, unit_migrate_stat_matrix, unitgroup_loads, server_loads))) {
6484
    LOG_WARN("fail to make count balanced", K(ret));
6485
  } else if (OB_FAIL(move_or_exchange_ug_balance_ttg_load(
6486
          balance_info, unitgroup_loads, server_loads))) {
6487
    LOG_WARN("fail to balance load", K(ret));
6488
  } else {} // no more to do
6489
  return ret;
6490
}
6491

6492
// Continue to exchange the unitgroups on the maximum server load and the minimum server load
6493
// until the exchange cannot reduce the difference between the two or the difference between the two is less than tolerance
6494
int ObServerBalancer::CountBalanceStrategy::move_or_exchange_ug_balance_ttg_load(
6495
    TenantGroupBalanceInfo &balance_info,
6496
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6497
    common::ObIArray<ServerLoad> &server_loads)
6498
{
6499
  int ret = OB_SUCCESS;
6500
  UNUSED(balance_info);
6501
  if (OB_UNLIKELY(unitgroup_loads.count() <= 0 || server_loads.count() <= 0)) {
6502
    ret = OB_INVALID_ARGUMENT;
6503
    LOG_WARN("invalid argument", K(ret), K(unitgroup_loads), K(server_loads));
6504
  } else {
6505
    double sum_load = 0.0;
6506
    ObArray<ServerLoad *> server_load_ptrs_sorted;
6507
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
6508
      if (OB_FAIL(server_load_ptrs_sorted.push_back(&server_loads.at(i)))) {
6509
        LOG_WARN("fail to push back", K(ret));
6510
      } else {
6511
        sum_load += server_loads.at(i).intra_ttg_load_value_;
6512
      }
6513
    }
6514
    if (OB_SUCC(ret)) {
6515
      double tolerance = sum_load / 100;
6516
      // Sort by intra load descending order
6517
      IntraServerLoadCmp cmp;
6518
      std::sort(server_load_ptrs_sorted.begin(),
6519
                server_load_ptrs_sorted.end(),
6520
                cmp);
6521
      if (OB_FAIL(cmp.get_ret())) {
6522
        LOG_WARN("fail to sorted", K(ret));
6523
      } else if (OB_FAIL(do_move_or_exchange_ug_balance_ttg_load(
6524
              server_load_ptrs_sorted, unitgroup_loads, tolerance))) {
6525
        LOG_WARN("fail to exchange ug", K(ret));
6526
      } else {} // no more to do
6527
    }
6528
  }
6529
  return ret;
6530
}
6531

6532
int ObServerBalancer::CountBalanceStrategy::do_move_or_exchange_ug_balance_ttg_load(
6533
    common::ObArray<ServerLoad *> &server_load_ptrs_sorted,
6534
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6535
    const double tolerance)
6536
{
6537
  int ret = OB_SUCCESS;
6538
  if (server_load_ptrs_sorted.count() <= 0
6539
      || unitgroup_loads.count() <= 0
6540
      || unitgroup_loads.count() >= INT32_MAX) {
6541
    ret = OB_INVALID_ARGUMENT;
6542
    LOG_WARN("invalid argument", K(ret), K(server_load_ptrs_sorted), K(unitgroup_loads));
6543
  } else {
6544
    const int64_t max_times = unitgroup_loads.count() * unitgroup_loads.count();
6545
    int64_t times = 0;
6546
    bool do_operate = true;
6547
    while (OB_SUCC(ret) && do_operate && times < max_times) {
6548
      ServerLoad *max_load_server = NULL;
6549
      ServerLoad *min_load_server = NULL;
6550
      const int64_t server_cnt = server_load_ptrs_sorted.count();
6551
      if (server_cnt <= 0) {
6552
        ret = OB_ERR_UNEXPECTED;
6553
        LOG_WARN("server load array is empty", K(ret), K(server_cnt));
6554
      } else if (NULL == (max_load_server = server_load_ptrs_sorted.at(0))) {
6555
        ret = OB_ERR_UNEXPECTED;
6556
        LOG_WARN("load ptr is null", K(ret), KP(max_load_server));
6557
      } else if (NULL == (min_load_server = server_load_ptrs_sorted.at(server_cnt - 1))) {
6558
        ret = OB_ERR_UNEXPECTED;
6559
        LOG_WARN("load ptr is null", K(ret), KP(min_load_server));
6560
      } else if (max_load_server->intra_ttg_load_value_
6561
                 < tolerance + min_load_server->intra_ttg_load_value_) {
6562
        break; // load balanced
6563
      } else if (max_load_server->unitgroup_loads_.count()
6564
                 <= min_load_server->unitgroup_loads_.count()) {
6565
        if (OB_FAIL(try_exchange_ug_balance_ttg_load_foreach(
6566
              *max_load_server, *min_load_server, unitgroup_loads, do_operate))) {
6567
          LOG_WARN("fail to exchange ug", K(ret));
6568
        }
6569
      } else if (max_load_server->unitgroup_loads_.count()
6570
                 > min_load_server->unitgroup_loads_.count() + 1) {
6571
        ret = OB_ERR_UNEXPECTED;
6572
        LOG_WARN("unitgroup load count unexpected", K(ret),
6573
                 K(*max_load_server), K(*min_load_server));
6574
      } else {
6575
        if (OB_FAIL(try_move_ug_balance_ttg_load_foreach(
6576
                *max_load_server, *min_load_server, unitgroup_loads, do_operate))) {
6577
          LOG_WARN("fail to move ug", K(ret));
6578
        } else if (do_operate) {
6579
          // by pass
6580
        } else if (OB_FAIL(try_exchange_ug_balance_ttg_load_foreach(
6581
                *max_load_server, *min_load_server, unitgroup_loads, do_operate))) {
6582
          LOG_WARN("fail to exchange ug", K(ret));
6583
        }
6584
      }
6585
      if (OB_FAIL(ret)) {
6586
      } else if (!do_operate) {
6587
        // will leave the loop
6588
      } else {
6589
        times++;
6590
        IntraServerLoadCmp cmp;
6591
        std::sort(server_load_ptrs_sorted.begin(),
6592
                  server_load_ptrs_sorted.end(),
6593
                  cmp);
6594
        if (OB_FAIL(cmp.get_ret())) {
6595
          LOG_WARN("fail to sort", K(ret));
6596
        }
6597
      }
6598
    }
6599
  }
6600
  return ret;
6601
}
6602

6603
int ObServerBalancer::CountBalanceStrategy::try_move_ug_balance_ttg_load_foreach(
6604
    ServerLoad &max_server_load,
6605
    ServerLoad &min_server_load,
6606
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6607
    bool &do_move)
6608
{
6609
  int ret = OB_SUCCESS;
6610
  do_move = false;
6611
  if (max_server_load.unitgroup_loads_.count() <= min_server_load.unitgroup_loads_.count()) {
6612
    // by pass
6613
  } else {
6614
    ObArray<common::ObAddr> excluded_dst_servers;
6615
    for (int64_t i = max_server_load.unitgroup_loads_.count() - 1;
6616
         !do_move && OB_SUCC(ret) && i >= 0;
6617
         --i) {
6618
      UnitGroupLoad &ug_load = *max_server_load.unitgroup_loads_.at(i);
6619
      bool can_move = false;
6620
      excluded_dst_servers.reset();
6621
      if (OB_FAIL(get_ug_balance_excluded_dst_servers(
6622
              ug_load, unitgroup_loads, excluded_dst_servers))) {
6623
        LOG_WARN("fail to get excluded server", K(ret));
6624
      } else if (has_exist_in_array(excluded_dst_servers, min_server_load.server_)) {
6625
        // in excluded servers, ignore
6626
      } else if (OB_FAIL(check_can_move_ug_balance_intra_ttg_load(
6627
              max_server_load, ug_load, min_server_load, can_move))) {
6628
        LOG_WARN("fail to check can move ug balance intra ttg load", K(ret));
6629
      } else if (!can_move) {
6630
        // ignore
6631
      } else if (OB_FAIL(move_ug_between_server_loads(
6632
              max_server_load, min_server_load, ug_load, i))) {
6633
        LOG_WARN("fail to move ug between server loads", K(ret));
6634
      } else {
6635
        do_move = true;
6636
      }
6637
    }
6638
    if (OB_SUCC(ret)) {
6639
      UnitGroupLoadCmp unitgroup_load_cmp(max_server_load);
6640
      std::sort(max_server_load.unitgroup_loads_.begin(),
6641
                max_server_load.unitgroup_loads_.end(),
6642
                unitgroup_load_cmp);
6643
      if (OB_FAIL(unitgroup_load_cmp.get_ret())) {
6644
        LOG_WARN("fail to sort", K(ret));
6645
      }
6646
    }
6647
    if (OB_SUCC(ret)) {
6648
      UnitGroupLoadCmp unitgroup_load_cmp(min_server_load);
6649
      std::sort(min_server_load.unitgroup_loads_.begin(),
6650
                min_server_load.unitgroup_loads_.end(),
6651
                unitgroup_load_cmp);
6652
      if (OB_FAIL(unitgroup_load_cmp.get_ret())) {
6653
        LOG_WARN("fail to sort", K(ret));
6654
      }
6655
    }
6656
  }
6657
  return ret;
6658
}
6659

6660
int ObServerBalancer::CountBalanceStrategy::try_exchange_ug_balance_ttg_load_foreach(
6661
    ServerLoad &max_server_load,
6662
    ServerLoad &min_server_load,
6663
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6664
    bool &do_exchange)
6665
{
6666
  int ret = OB_SUCCESS;
6667
  do_exchange = false;
6668
  if (max_server_load.intra_ttg_load_value_ - min_server_load.intra_ttg_load_value_ < EPSILON) {
6669
    do_exchange = false;
6670
  } else if (max_server_load.unitgroup_loads_.count() <= 0
6671
             || min_server_load.unitgroup_loads_.count() <= 0) {
6672
    do_exchange = false;
6673
  } else {
6674
    for (int64_t i = 0;
6675
         !do_exchange && OB_SUCC(ret) && i < max_server_load.unitgroup_loads_.count();
6676
         ++i) {
6677
      common::ObArray<common::ObAddr> excluded_servers;
6678
      UnitGroupLoad *left_ug = max_server_load.unitgroup_loads_.at(i);
6679
      double left_load = 0.0;
6680
      if (OB_UNLIKELY(NULL == left_ug)) {
6681
        ret = OB_ERR_UNEXPECTED;
6682
        LOG_WARN("src ug ptr is null", K(ret), KP(left_ug));
6683
      } else if (OB_FAIL(get_ug_balance_excluded_dst_servers(
6684
              *left_ug, unitgroup_loads, excluded_servers))) {
6685
        LOG_WARN("fail to get excluded server", K(ret));
6686
      } else if (has_exist_in_array(excluded_servers, min_server_load.server_)) {
6687
        // ignore this
6688
      } else if (OB_FAIL(left_ug->get_intra_ttg_load_value(max_server_load, left_load))) {
6689
        LOG_WARN("fail to get intra ttg load value", K(ret));
6690
      } else {
6691
        for (int64_t j = min_server_load.unitgroup_loads_.count() - 1;
6692
             !do_exchange && OB_SUCC(ret) && j >= 0;
6693
             --j) {
6694
          UnitGroupLoad *right_ug = min_server_load.unitgroup_loads_.at(j);
6695
          // The calculation denominator of left load takes max_server_load,
6696
          // so the calculation denominator of right load also takes max_server_load,
6697
          // So that the comparison of the two values makes sense
6698
          double right_load = 0.0;
6699
          bool can_exchange = false;
6700
          if (NULL == right_ug) {
6701
            ret = OB_ERR_UNEXPECTED;
6702
            LOG_WARN("src or dst ug ptr is null", K(ret), KP(right_ug));
6703
          } else if (left_ug->column_tenant_id_ == right_ug->column_tenant_id_) {
6704
            // ignore this,
6705
          } else if (OB_FAIL(get_ug_balance_excluded_dst_servers(
6706
                  *right_ug, unitgroup_loads, excluded_servers))) {
6707
            LOG_WARN("fail to get excluded servers", K(ret));
6708
          } else if (has_exist_in_array(excluded_servers, max_server_load.server_)) {
6709
            // ignore this
6710
          } else if (OB_FAIL(right_ug->get_intra_ttg_load_value(max_server_load, right_load))) {
6711
            LOG_WARN("fail to get intra ttg load value", K(ret));
6712
          } else if (left_load - right_load < EPSILON) {
6713
            // When the left is smaller than the right or a little bigger than the right (EPSILON), no swap
6714
          } else if (OB_FAIL(check_can_exchange_ug_balance_intra_ttg_load(
6715
                *left_ug, max_server_load, *right_ug, min_server_load, can_exchange))) {
6716
            LOG_WARN("fail to check can exchange ug balance ttg load", K(ret));
6717
          } else if (!can_exchange) {
6718
            // ignore this
6719
          } else if (OB_FAIL(exchange_ug_between_server_loads(
6720
                *left_ug, i, max_server_load, *right_ug, j, min_server_load))) {
6721
            LOG_WARN("fail to exchange ug between server loads", K(ret));
6722
          } else {
6723
            do_exchange = true;
6724
          }
6725
        }
6726
      }
6727
    }
6728
    if (OB_SUCC(ret)) {
6729
      UnitGroupLoadCmp unitgroup_load_cmp(max_server_load);
6730
      std::sort(max_server_load.unitgroup_loads_.begin(),
6731
                max_server_load.unitgroup_loads_.end(),
6732
                unitgroup_load_cmp);
6733
      if (OB_FAIL(unitgroup_load_cmp.get_ret())) {
6734
        LOG_WARN("fail to sort", K(ret));
6735
      }
6736
    }
6737
    if (OB_SUCC(ret)) {
6738
      UnitGroupLoadCmp unitgroup_load_cmp(min_server_load);
6739
      std::sort(min_server_load.unitgroup_loads_.begin(),
6740
                min_server_load.unitgroup_loads_.end(),
6741
                unitgroup_load_cmp);
6742
      if (OB_FAIL(unitgroup_load_cmp.get_ret())) {
6743
        LOG_WARN("fail to sort", K(ret));
6744
      }
6745
    }
6746
  }
6747
  return ret;
6748
}
6749

6750
// Keep moving the unitgroup from the server load with the largest cnt to the server load with the smallest cnt,
6751
// until the difference between the largest server load and the smallest server load is less than or equal to 1 unitgroup
6752
int ObServerBalancer::CountBalanceStrategy::make_count_balanced(
6753
    TenantGroupBalanceInfo &balance_info,
6754
    Matrix<UnitMigrateStat> &unit_migrate_stat_matrix,
6755
    common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6756
    common::ObIArray<ServerLoad> &server_loads)
6757
{
6758
  int ret = OB_SUCCESS;
6759
  UNUSED(balance_info);
6760
  UNUSED(unit_migrate_stat_matrix);
6761
  if (OB_UNLIKELY(unitgroup_loads.count() <= 0 || server_loads.count() <= 0)) {
6762
    ret = OB_INVALID_ARGUMENT;
6763
    LOG_WARN("invalid argument", K(ret), K(server_loads), K(unitgroup_loads));
6764
  } else {
6765
    ObArray<ServerLoad *> server_cnt_ptrs_sorted;
6766
    for (int64_t i = 0; OB_SUCC(ret) && i < server_loads.count(); ++i) {
6767
      if (OB_FAIL(server_cnt_ptrs_sorted.push_back(&server_loads.at(i)))) {
6768
        LOG_WARN("fail to push back", K(ret));
6769
      }
6770
    }
6771
    if (OB_SUCC(ret)) {
6772
      // Sort by the number of unitgroups on the server in ascending order
6773
      ServerLoadUgCntCmp cmp;
6774
      std::sort(server_cnt_ptrs_sorted.begin(),
6775
                server_cnt_ptrs_sorted.end(),
6776
                cmp);
6777
      if (OB_FAIL(cmp.get_ret())) {
6778
        LOG_WARN("fail to sort", K(ret));
6779
      } else if (OB_FAIL(do_make_count_balanced(server_cnt_ptrs_sorted, unitgroup_loads))) {
6780
        LOG_WARN("fail to make count balanced", K(ret));
6781
      } else {} // no more to do
6782
    }
6783
  }
6784
  return ret;
6785
}
6786

6787
int ObServerBalancer::CountBalanceStrategy::do_make_count_balanced(
6788
    common::ObArray<ServerLoad *> &server_cnt_ptrs_sorted,
6789
    common::ObIArray<UnitGroupLoad> &unitgroup_loads)
6790
{
6791
  int ret = OB_SUCCESS;
6792
  if (server_cnt_ptrs_sorted.count() <= 0 || unitgroup_loads.count() <= 0) {
6793
    ret = OB_INVALID_ARGUMENT;
6794
    LOG_WARN("invalid argument", K(ret), K(server_cnt_ptrs_sorted), K(unitgroup_loads));
6795
  } else {
6796
    int64_t times = 0;
6797
    while (OB_SUCC(ret) && times < unitgroup_loads.count()) {
6798
      ServerLoad *first = NULL;
6799
      ServerLoad *last = NULL;
6800
      const int64_t server_cnt = server_cnt_ptrs_sorted.count();
6801
      if (server_cnt <= 0) {
6802
        ret = OB_ERR_UNEXPECTED;
6803
        LOG_WARN("server load array is empty", K(ret));
6804
      } else if (NULL == (first = server_cnt_ptrs_sorted.at(0))) {
6805
        ret = OB_ERR_UNEXPECTED;
6806
        LOG_WARN("load ptr is null", K(ret), KP(first));
6807
      } else if (NULL == (last = server_cnt_ptrs_sorted.at(server_cnt - 1))) {
6808
        ret = OB_ERR_UNEXPECTED;
6809
        LOG_WARN("last ptr is null", K(ret), KP(last));
6810
      } else if (last->unitgroup_loads_.count() - first->unitgroup_loads_.count() <= 1) {
6811
        break;
6812
      } else if (OB_FAIL(do_make_count_balanced_foreach(*last, *first, unitgroup_loads))) {
6813
        LOG_WARN("fail to make count balanced foreach");
6814
      } else {
6815
        times++;
6816
        ServerLoadUgCntCmp cmp;
6817
        std::sort(server_cnt_ptrs_sorted.begin(),
6818
                  server_cnt_ptrs_sorted.end(),
6819
                  cmp);
6820
        if (OB_FAIL(cmp.get_ret())) {
6821
          LOG_WARN("fail to sort", K(ret));
6822
        }
6823
      }
6824
    }
6825
  }
6826
  return ret;
6827
}
6828

6829
int ObServerBalancer::CountBalanceStrategy::do_make_count_balanced_foreach(
6830
    ServerLoad &src_server_load,
6831
    ServerLoad &dst_server_load,
6832
    common::ObIArray<UnitGroupLoad> &unitgroup_loads)
6833
{
6834
  int ret = OB_SUCCESS;
6835
  if (unitgroup_loads.count() <= 0) {
6836
    ret = OB_INVALID_ARGUMENT;
6837
    LOG_WARN("invalid argument", K(ret), K(unitgroup_loads));
6838
  } else {
6839
    for (int64_t i = src_server_load.unitgroup_loads_.count() - 1;
6840
         OB_SUCC(ret) && i >= 0;
6841
         --i) {
6842
      common::ObArray<common::ObAddr> excluded_servers;
6843
      UnitGroupLoad *ug_load = src_server_load.unitgroup_loads_.at(i);
6844
      if (src_server_load.unitgroup_loads_.count()
6845
                 <= dst_server_load.unitgroup_loads_.count() + 1) {
6846
        break;
6847
      } else if (OB_UNLIKELY(NULL == ug_load)) {
6848
        ret = OB_ERR_UNEXPECTED;
6849
        LOG_WARN("ug_load ptr is null", K(ret), KP(ug_load));
6850
      } else if (OB_FAIL(get_ug_balance_excluded_dst_servers(
6851
              *ug_load, unitgroup_loads, excluded_servers))) {
6852
        LOG_WARN("fail to get excluded servers", K(ret));
6853
      } else if (has_exist_in_array(excluded_servers, dst_server_load.server_)) {
6854
        // ignore this ug load, since it cannot move to dst server
6855
      } else if (OB_FAIL(move_ug_between_server_loads(
6856
              src_server_load, dst_server_load, *ug_load, i))) {
6857
        LOG_WARN("fail to move ug", K(ret));
6858
      } else {} // no more to do
6859
    }
6860
    if (OB_SUCC(ret)) {
6861
      UnitGroupLoadCmp unitgroup_load_cmp(src_server_load);
6862
      std::sort(src_server_load.unitgroup_loads_.begin(),
6863
                src_server_load.unitgroup_loads_.end(),
6864
                unitgroup_load_cmp);
6865
      if (OB_FAIL(unitgroup_load_cmp.get_ret())) {
6866
        LOG_WARN("fail to sort", K(ret));
6867
      }
6868
    }
6869
    if (OB_SUCC(ret)) {
6870
      UnitGroupLoadCmp unitgroup_load_cmp(dst_server_load);
6871
      std::sort(dst_server_load.unitgroup_loads_.begin(),
6872
                dst_server_load.unitgroup_loads_.end(),
6873
                unitgroup_load_cmp);
6874
      if (OB_FAIL(unitgroup_load_cmp.get_ret())) {
6875
        LOG_WARN("fail to sort", K(ret));
6876
      }
6877
    }
6878
  }
6879
  return ret;
6880
}
6881

6882
int ObServerBalancer::InnerTenantGroupBalanceStrategy::get_ug_balance_excluded_dst_servers(
6883
    const UnitGroupLoad &unitgroup_load,
6884
    const common::ObIArray<UnitGroupLoad> &unitgroup_loads,
6885
    common::ObIArray<common::ObAddr> &excluded_servers)
6886
{
6887
  int ret = OB_SUCCESS;
6888
  excluded_servers.reset();
6889
  const uint64_t column_tenant_id = unitgroup_load.column_tenant_id_;
6890
  for (int64_t i = 0; OB_SUCC(ret) && i < unitgroup_loads.count(); ++i) {
6891
    const UnitGroupLoad &this_load = unitgroup_loads.at(i);
6892
    if (column_tenant_id != this_load.column_tenant_id_) {
6893
      // not from the same tenant column
6894
    } else if (!has_exist_in_array(excluded_servers, this_load.server_)) {
6895
      if (OB_FAIL(excluded_servers.push_back(this_load.server_))) {
6896
        LOG_WARN("fail to push back", K(ret));
6897
      }
6898
    }
6899
  }
6900
  return ret;
6901
}
6902

6903
int ObServerBalancer::InnerTenantGroupBalanceStrategy::move_ug_between_server_loads(
6904
    ServerLoad &src_server_load,
6905
    ServerLoad &dst_server_load,
6906
    UnitGroupLoad &ug_load,
6907
    const int64_t ug_idx)
6908
{
6909
  int ret = OB_SUCCESS;
6910
  if (ug_idx < 0 || ug_idx >= src_server_load.unitgroup_loads_.count()) {
6911
    ret = OB_INVALID_ARGUMENT;
6912
    LOG_WARN("invalid argument", K(ug_idx), K(src_server_load));
6913
  } else if (&ug_load != src_server_load.unitgroup_loads_.at(ug_idx)) {
6914
    ret = OB_ERR_UNEXPECTED;
6915
    LOG_WARN("ug load ptr and idx not match", K(ret), K(src_server_load), K(ug_load), K(ug_idx));
6916
  } else {
6917
    if (OB_FAIL(src_server_load.unitgroup_loads_.remove(ug_idx))) {
6918
      LOG_WARN("remove failed", K(ret), K(ug_idx));
6919
    } else if (OB_FAIL(dst_server_load.unitgroup_loads_.push_back(&ug_load))) {
6920
      LOG_WARN("fail to push back", K(ret));
6921
    } else if (OB_FAIL(src_server_load.update_load_value())) {
6922
      LOG_WARN("fail to update load value", K(ret));
6923
    } else if (OB_FAIL(dst_server_load.update_load_value())) {
6924
      LOG_WARN("fail to update load value", K(ret));
6925
    } else {
6926
      ug_load.server_ = dst_server_load.server_;
6927
      ug_load.server_load_ = &dst_server_load;
6928
    }
6929
  }
6930
  return ret;
6931
}
6932

6933
int ObServerBalancer::InnerTenantGroupBalanceStrategy::exchange_ug_between_server_loads(
6934
    UnitGroupLoad &left_ug,
6935
    const int64_t left_idx,
6936
    ServerLoad &left_server,
6937
    UnitGroupLoad &right_ug,
6938
    const int64_t right_idx,
6939
    ServerLoad &right_server)
6940
{
6941
  int ret = OB_SUCCESS;
6942
  if (left_idx < 0 || left_idx >= left_server.unitgroup_loads_.count()
6943
      || right_idx < 0 || right_idx >= right_server.unitgroup_loads_.count()) {
6944
    ret = OB_INVALID_ARGUMENT;
6945
    LOG_WARN("invalid argument", K(ret),
6946
            K(left_idx),
6947
            "left_server_ug_cnt", left_server.unitgroup_loads_.count(),
6948
            K(right_idx),
6949
            "right_server_ug_cnt", right_server.unitgroup_loads_.count());
6950
  } else if (&left_ug != left_server.unitgroup_loads_.at(left_idx)
6951
             || &right_ug != right_server.unitgroup_loads_.at(right_idx)) {
6952
    ret = OB_ERR_UNEXPECTED;
6953
    LOG_WARN("ug ptr and idx not match", K(ret), K(left_idx), K(right_idx));
6954
  } else {
6955
    if (OB_FAIL(left_server.unitgroup_loads_.remove(left_idx))) {
6956
      LOG_WARN("fail to remove", K(ret));
6957
    } else if (OB_FAIL(right_server.unitgroup_loads_.remove(right_idx))) {
6958
      LOG_WARN("fail to remove", K(ret));
6959
    } else if (OB_FAIL(left_server.unitgroup_loads_.push_back(&right_ug))) {
6960
      LOG_WARN("fail to push back", K(ret));
6961
    } else if (OB_FAIL(right_server.unitgroup_loads_.push_back(&left_ug))) {
6962
      LOG_WARN("fail to push back", K(ret));
6963
    } else if (OB_FAIL(left_server.update_load_value())) {
6964
      LOG_WARN("fail to update load value", K(ret));
6965
    } else if (OB_FAIL(right_server.update_load_value())) {
6966
      LOG_WARN("fail to update load value", K(ret));
6967
    } else {
6968
      left_ug.server_ = right_server.server_;
6969
      left_ug.server_load_ = &right_server;
6970
      right_ug.server_ = left_server.server_;
6971
      right_ug.server_load_ = &left_server;
6972
    }
6973
  }
6974
  return ret;
6975
}
6976

6977
int ObServerBalancer::CountBalanceStrategy::check_can_move_ug_balance_intra_ttg_load(
6978
    ServerLoad &left_server,
6979
    UnitGroupLoad &ug_load,
6980
    ServerLoad &right_server,
6981
    bool &can_move)
6982
{
6983
  int ret = OB_SUCCESS;
6984
  can_move = false;
6985
  double left_before = left_server.intra_ttg_load_value_;
6986
  double right_before = right_server.intra_ttg_load_value_;
6987
  double ug_on_left_svr = 0.0; // ug's own load value of ug on the left server
6988
  double ug_on_right_svr = 0.0; // Load value of ug itself after ug is moved to the right server
6989
  double left_after = 0.0;
6990
  double right_after = 0.0;
6991
  if (OB_FAIL(ug_load.get_intra_ttg_load_value(left_server, ug_on_left_svr))) {
6992
    LOG_WARN("fail to get left intra ttg load value on left", K(ret));
6993
  } else if (OB_FAIL(ug_load.get_intra_ttg_load_value(right_server, ug_on_right_svr))) {
6994
    LOG_WARN("fail to get left intra ttg load value on right", K(ret));
6995
  } else {
6996
    left_after = left_before - ug_on_left_svr;
6997
    right_after = right_before + ug_on_right_svr;
6998
    if (left_after > right_after) {
6999
      can_move = (left_before - right_before > EPSILON)
7000
                 && ((left_before - right_before) - (left_after - right_after) > EPSILON);
7001
    } else {
7002
      can_move = (left_before - right_before > EPSILON)
7003
                 && ((left_before - right_before) - (right_after - left_after) > EPSILON);
7004
    }
7005
  }
7006
  return ret;
7007
}
7008

7009
int ObServerBalancer::CountBalanceStrategy::check_can_exchange_ug_balance_intra_ttg_load(
7010
    UnitGroupLoad &left_ug,
7011
    ServerLoad &left_server,
7012
    UnitGroupLoad &right_ug,
7013
    ServerLoad &right_server,
7014
    bool &can_exchange)
7015
{
7016
  int ret = OB_SUCCESS;
7017
  can_exchange = false;
7018
  double left_before = left_server.intra_ttg_load_value_;
7019
  double right_before = right_server.intra_ttg_load_value_;
7020
  double left_ug_on_left_svr = 0.0; // Left ug's own load value of ug on the left server
7021
  double left_ug_on_right_svr = 0.0; // The load value of ug itself after the left ug is moved to the right server
7022
  double right_ug_on_right_svr = 0.0; // Right ug's own load value of ug on the right server
7023
  double right_ug_on_left_svr = 0.0; // The load value of ug itself after the right ug is moved to the left server
7024
  double left_after = 0.0;
7025
  double right_after = 0.0;
7026
  if (OB_FAIL(left_ug.get_intra_ttg_load_value(left_server, left_ug_on_left_svr))) {
7027
    LOG_WARN("fail to get left intra ttg load value on left", K(ret));
7028
  } else if (OB_FAIL(left_ug.get_intra_ttg_load_value(right_server, left_ug_on_right_svr))) {
7029
    LOG_WARN("fail to get left intra ttg load value on right", K(ret));
7030
  } else if (OB_FAIL(right_ug.get_intra_ttg_load_value(right_server, right_ug_on_right_svr))) {
7031
    LOG_WARN("fail to get right intra ttg load value on right", K(ret));
7032
  } else if (OB_FAIL(right_ug.get_intra_ttg_load_value(left_server, right_ug_on_left_svr))) {
7033
    LOG_WARN("fail to get right intra ttg load value on left", K(ret));
7034
  } else {
7035
    left_after = left_before - left_ug_on_left_svr + right_ug_on_left_svr;
7036
    right_after = right_before - right_ug_on_right_svr + left_ug_on_right_svr;
7037
    if (left_after > right_after) {
7038
      can_exchange = (left_before - right_before > EPSILON)
7039
                     && ((left_before - right_before) - (left_after - right_after) > EPSILON);
7040
    } else {
7041
      can_exchange = (left_before - right_before > EPSILON)
7042
                     && ((left_before - right_before) - (right_after - left_after) > EPSILON);
7043
    }
7044
  }
7045
  return ret;
7046
}
7047

7048
int ObServerBalancer::CountBalanceStrategy::check_can_move_ug_balance_inter_ttg_load(
7049
    ServerLoad &left_server,
7050
    UnitGroupLoad &ug_load,
7051
    ServerLoad &right_server,
7052
    bool &can_move)
7053
{
7054
  int ret = OB_SUCCESS;
7055
  can_move = false;
7056
  double left_before = left_server.intra_ttg_load_value_;
7057
  double right_before = right_server.intra_ttg_load_value_;
7058
  double ug_on_left_svr = 0.0; // ug's own load value of ug on the left server
7059
  double ug_on_right_svr = 0.0; // Load value of ug itself after ug is moved to the right server
7060
  double left_after = 0.0;
7061
  double right_after = 0.0;
7062
  if (OB_FAIL(ug_load.get_intra_ttg_load_value(left_server, ug_on_left_svr))) {
7063
    LOG_WARN("fail to get left intra ttg load value on left", K(ret));
7064
  } else if (OB_FAIL(ug_load.get_intra_ttg_load_value(right_server, ug_on_right_svr))) {
7065
    LOG_WARN("fail to get left intra ttg load value on right", K(ret));
7066
  } else {
7067
    left_after = left_before - ug_on_left_svr;
7068
    right_after = right_before + ug_on_right_svr;
7069
    if (left_after > right_after) {
7070
      can_move = (left_before - right_before > EPSILON)
7071
                 && ((left_before - right_before) - (left_after - right_after) > -EPSILON);
7072
    } else {
7073
      can_move = (left_before - right_before > EPSILON)
7074
                 && ((left_before - right_before) - (right_after - left_after) > -EPSILON);
7075
    }
7076
  }
7077
  return ret;
7078
}
7079

7080
int ObServerBalancer::CountBalanceStrategy::check_can_exchange_ug_balance_inter_ttg_load(
7081
    UnitGroupLoad &left_ug,
7082
    ServerLoad &left_server,
7083
    UnitGroupLoad &right_ug,
7084
    ServerLoad &right_server,
7085
    bool &can_exchange)
7086
{
7087
  int ret = OB_SUCCESS;
7088
  can_exchange = false;
7089
  double left_before = left_server.intra_ttg_load_value_;
7090
  double right_before = right_server.intra_ttg_load_value_;
7091
  double left_ug_on_left_svr = 0.0;
7092
  double left_ug_on_right_svr = 0.0;
7093
  double right_ug_on_right_svr = 0.0;
7094
  double right_ug_on_left_svr = 0.0;
7095
  double left_after = 0.0;
7096
  double right_after = 0.0;
7097
  if (OB_FAIL(left_ug.get_intra_ttg_load_value(left_server, left_ug_on_left_svr))) {
7098
    LOG_WARN("fail to get left intra ttg load value on left", K(ret));
7099
  } else if (OB_FAIL(left_ug.get_intra_ttg_load_value(right_server, left_ug_on_right_svr))) {
7100
    LOG_WARN("fail to get left intra ttg load value on right", K(ret));
7101
  } else if (OB_FAIL(right_ug.get_intra_ttg_load_value(right_server, right_ug_on_right_svr))) {
7102
    LOG_WARN("fail to get right intra ttg load value on right", K(ret));
7103
  } else if (OB_FAIL(right_ug.get_intra_ttg_load_value(left_server, right_ug_on_left_svr))) {
7104
    LOG_WARN("fail to get right intra ttg load value on left", K(ret));
7105
  } else {
7106
    left_after = left_before - left_ug_on_left_svr + right_ug_on_left_svr;
7107
    right_after = right_before - right_ug_on_right_svr + left_ug_on_right_svr;
7108
    if (left_after > right_after) {
7109
      can_exchange = (left_before - right_before > EPSILON)
7110
                     && ((left_before - right_before) - (left_after - right_after) > -EPSILON);
7111
    } else {
7112
      can_exchange = (left_before - right_before > EPSILON)
7113
                     && ((left_before - right_before) - (right_after - left_after) > -EPSILON);
7114
    }
7115
  }
7116
  return ret;
7117
}
7118

7119
bool ObServerBalancer::UnitGroupLoad::is_valid() const
7120
{
7121
  bool bool_ret = OB_INVALID_ID != column_tenant_id_
7122
                  && start_column_idx_ >= 0
7123
                  && column_count_ > 0
7124
                  && unit_loads_.count() > 0
7125
                  && server_.is_valid()
7126
                  && load_sum_.is_valid();
7127
  for (int64_t i = 0; bool_ret && i < unit_loads_.count(); ++i) {
7128
    bool_ret = unit_loads_.at(i).is_valid();
7129
  }
7130
  return bool_ret;
7131
}
7132

7133
void ObServerBalancer::UnitGroupLoad::reset()
7134
{
7135
  column_tenant_id_ = OB_INVALID_ID;
7136
  start_column_idx_ = -1;
7137
  column_count_ = 0;
7138
  server_.reset();
7139
  server_load_ = NULL;
7140
  unit_loads_.reset();
7141
  load_sum_.reset();
7142
}
7143

7144
int ObServerBalancer::UnitGroupLoad::sum_group_load()
7145
{
7146
  int ret = OB_SUCCESS;
7147
  load_sum_.reset();
7148
  for (int64_t j = 0; OB_SUCC(ret) && j < unit_loads_.count(); ++j) {
7149
    const ObUnitConfig &unit_load = unit_loads_.at(j);
7150
    if (!unit_load.is_valid()) {
7151
      ret = OB_ERR_UNEXPECTED;
7152
      LOG_WARN("invalid unit load", K(ret), K(unit_load));
7153
    } else if (OB_FAIL(load_sum_.append_load(unit_load))) {
7154
      LOG_WARN("fail to append load", K(ret));
7155
    } else {} // no more to do
7156
  }
7157
  return ret;
7158
}
7159

7160
int ObServerBalancer::UnitGroupLoad::get_intra_ttg_load_value(
7161
    const ServerLoad &target_server_load,
7162
    double &intra_ttg_load_value) const
7163
{
7164
  int ret = OB_SUCCESS;
7165
  intra_ttg_load_value = 0.0;
7166
  for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
7167
    ObResourceType res_type = static_cast<ObResourceType>(i);
7168
    if (target_server_load.get_intra_ttg_resource_capacity(res_type) <= 0
7169
        || load_sum_.get_required(res_type) <= 0) {
7170
      // has no effect on load value
7171
    } else if (load_sum_.get_required(res_type) > target_server_load.get_intra_ttg_resource_capacity(res_type)) {
7172
      intra_ttg_load_value += target_server_load.intra_weights_[i] * 1.0;
7173
    } else {
7174
      double quotient
7175
          = load_sum_.get_required(res_type) / target_server_load.get_intra_ttg_resource_capacity(res_type);
7176
      intra_ttg_load_value += target_server_load.intra_weights_[i] * quotient;
7177
    }
7178
  }
7179
  return ret;
7180
}
7181

7182
int ObServerBalancer::UnitGroupLoad::get_inter_ttg_load_value(
7183
    const ServerLoad &target_server_load,
7184
    double &inter_ttg_load_value) const
7185
{
7186
  int ret = OB_SUCCESS;
7187
  inter_ttg_load_value = 0.0;
7188
  for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
7189
    ObResourceType res_type = static_cast<ObResourceType>(i);
7190
    if (target_server_load.get_true_capacity(res_type) <= 0
7191
        || load_sum_.get_required(res_type) <= 0) {
7192
      // has no effect on load value
7193
    } else if (load_sum_.get_required(res_type) > target_server_load.get_true_capacity(res_type)) {
7194
      inter_ttg_load_value += target_server_load.inter_weights_[i] * 1.0;
7195
    } else {
7196
      double quotient
7197
          = load_sum_.get_required(res_type) / target_server_load.get_true_capacity(res_type);
7198
      inter_ttg_load_value += target_server_load.inter_weights_[i] * quotient;
7199
    }
7200
  }
7201
  return ret;
7202
}
7203

7204
int ObServerBalancer::UnitGroupLoadCmp::get_ret() const
7205
{
7206
  return ret_;
7207
}
7208

7209
bool ObServerBalancer::UnitGroupLoadCmp::operator()(
7210
     const UnitGroupLoad *left,
7211
     const UnitGroupLoad *right)
7212
{
7213
  // Sort by load in ascending order
7214
  bool bool_ret = false;
7215
  double left_value = 0.0;
7216
  double right_value = 0.0;
7217
  int &ret = ret_;
7218
  if (OB_UNLIKELY(OB_SUCCESS != ret_)) {
7219
    // ignore
7220
  } else if (OB_UNLIKELY(NULL == left || NULL == right)) {
7221
    ret_ = OB_INVALID_ARGUMENT;
7222
    LOG_WARN("invalid argument", K(ret_), KP(left), K(right));
7223
  } else if (OB_SUCCESS != (ret_ = left->get_intra_ttg_load_value(server_load_, left_value))) {
7224
    LOG_WARN("fail to get left intra load value", K(ret_));
7225
  } else if (OB_SUCCESS != (ret_ = right->get_intra_ttg_load_value(server_load_, right_value))) {
7226
    LOG_WARN("fail to get right intra load value", K(ret_));
7227
  } else if (left_value < right_value) {
7228
    bool_ret = true;
7229
  } else {
7230
    bool_ret = false;
7231
  }
7232
  return bool_ret;
7233
}
7234

7235
bool ObServerBalancer::ServerLoad::is_valid() const
7236
{
7237
  // unitgroup loads can be empty, indicating that there is no load on the server, but server_ must be valid
7238
  bool bool_ret = unitgroup_loads_.count() >= 0 && ServerResourceLoad::is_valid();
7239
  for (int64_t i = 0; bool_ret && i < unitgroup_loads_.count(); ++i) {
7240
    const UnitGroupLoad *ins = unitgroup_loads_.at(i);
7241
    bool_ret = (NULL != ins && ins->is_valid());
7242
  }
7243
  return bool_ret;
7244
}
7245

7246
double ObServerBalancer::ServerResourceLoad::get_true_capacity(const ObResourceType resource_type) const
7247
{
7248
  double ret = -1;
7249
  switch (resource_type) {
7250
  case RES_CPU:
7251
    ret = resource_info_.cpu_;
7252
    break;
7253
  case RES_MEM:
7254
    ret = static_cast<double>(resource_info_.mem_total_);
7255
    break;
7256
  case RES_LOG_DISK:
7257
    ret = static_cast<double>(resource_info_.log_disk_total_);
7258
    break;
7259
  default:
7260
    ret = -1;
7261
    break;
7262
  }
7263
  return ret;
7264
}
7265

7266
double ObServerBalancer::ServerLoad::get_intra_ttg_resource_capacity(
7267
       const ObResourceType resource_type) const
7268
{
7269
  double ret = -1;
7270
  switch (resource_type) {
7271
  case RES_CPU:
7272
    ret = intra_ttg_resource_info_.cpu_;
7273
    break;
7274
  case RES_MEM:
7275
    ret = static_cast<double>(intra_ttg_resource_info_.mem_total_);
7276
    break;
7277
  case RES_LOG_DISK:
7278
    ret = static_cast<double>(intra_ttg_resource_info_.log_disk_total_);
7279
    break;
7280
  default:
7281
    ret = -1;
7282
    break;
7283
  }
7284
  return ret;
7285
}
7286

7287
int ObServerBalancer::ServerLoad::update_load_value()
7288
{
7289
  int ret = OB_SUCCESS;
7290
  intra_ttg_load_value_ = 0.0;
7291
  inter_ttg_load_value_ = 0.0;
7292
  LoadSum intra_load_sum;
7293
  LoadSum inter_load_sum;
7294
  for (int64_t i = 0; OB_SUCC(ret) && i < unitgroup_loads_.count(); ++i) {
7295
    const UnitGroupLoad *unitgroup_load = unitgroup_loads_.at(i);
7296
    if (NULL == unitgroup_load) {
7297
      ret = OB_ERR_UNEXPECTED;
7298
      LOG_WARN("unitgroup load null", K(ret), KP(unitgroup_load));
7299
    } else if (OB_FAIL(intra_load_sum.append_load(unitgroup_load->load_sum_))) {
7300
      LOG_WARN("fail to append intra load", K(ret));
7301
    } else if (OB_FAIL(inter_load_sum.append_load(unitgroup_load->load_sum_))) {
7302
      LOG_WARN("fail to append inter load", K(ret));
7303
    }
7304
  }
7305
  for (int64_t i = 0; OB_SUCC(ret) && i < alien_ug_loads_.count(); ++i) {
7306
    const UnitGroupLoad *unitgroup_load = alien_ug_loads_.at(i);
7307
    if (NULL == unitgroup_load) {
7308
      ret = OB_ERR_UNEXPECTED;
7309
      LOG_WARN("unitgroup load null", K(ret), KP(unitgroup_load));
7310
    } else if (OB_FAIL(inter_load_sum.append_load(unitgroup_load->load_sum_))) {
7311
      LOG_WARN("fail to append load", K(ret));
7312
    }
7313
  }
7314
  for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
7315
    ObResourceType resource_type = static_cast<ObResourceType>(i);
7316
    const double required = intra_load_sum.get_required(resource_type);
7317
    const double capacity = get_intra_ttg_resource_capacity(resource_type);
7318
    if (required <= 0 || capacity <= 0) {
7319
      // has on effect on load value
7320
    } else if (required > capacity) {
7321
      intra_ttg_load_value_ += intra_weights_[i] * 1.0;
7322
    } else {
7323
      intra_ttg_load_value_ += intra_weights_[i] * (required / capacity);
7324
    }
7325
  }
7326
  for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
7327
    ObResourceType resource_type = static_cast<ObResourceType>(i);
7328
    const double required = inter_load_sum.get_required(resource_type);
7329
    const double capacity = get_true_capacity(resource_type);
7330
    if (required <= 0 || capacity <= 0) {
7331
      // has on effect on load value
7332
    } else if (required > capacity) {
7333
      inter_ttg_load_value_ += inter_weights_[i] * 1.0;
7334
    } else {
7335
      inter_ttg_load_value_ += inter_weights_[i] * (required / capacity);
7336
    }
7337
  }
7338
  return ret;
7339
}
7340

7341
int ObServerBalancer::ServerTotalLoad::update_load_value()
7342
{
7343
  int ret = OB_SUCCESS;
7344
  inter_ttg_load_value_ = 0.0;
7345
  for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
7346
    ObResourceType resource_type = static_cast<ObResourceType>(i);
7347
    const double required = load_sum_.get_required(resource_type);
7348
    const double capacity = get_true_capacity(resource_type);
7349
    if (required <= 0 || capacity <= 0) {
7350
      // has on effect on load value
7351
    } else if (required > capacity) {
7352
      inter_ttg_load_value_ += resource_weights_[i] * 1.0;
7353
    } else {
7354
      inter_ttg_load_value_ += resource_weights_[i] * (required / capacity);
7355
    }
7356
  }
7357
  return ret;
7358
}
7359

7360
bool ObServerBalancer::ServerTotalLoadCmp::operator()(
7361
     const ServerTotalLoad *left,
7362
     const ServerTotalLoad *right)
7363
{
7364
  bool bool_ret = false;
7365
  if (OB_UNLIKELY(OB_SUCCESS != ret_)) {
7366
    // ignore
7367
  } else if (OB_UNLIKELY(NULL == left || NULL == right)) {
7368
    ret_ = OB_INVALID_ARGUMENT;
7369
    LOG_WARN_RET(ret_, "invalid argument", K(ret_), KP(left), KP(right));
7370
  } else if (left->wild_server_ && !left->wild_server_) {
7371
    bool_ret = true;
7372
  } else if (!left->wild_server_ && right->wild_server_) {
7373
    bool_ret = false;
7374
  } else {
7375
    if (left->inter_ttg_load_value_ > right->inter_ttg_load_value_) {
7376
      bool_ret = true;
7377
    } else {
7378
      bool_ret = false;
7379
    }
7380
  }
7381
  return bool_ret;
7382
}
7383

7384
int ObServerBalancer::ServerTotalLoadCmp::get_ret() const
7385
{
7386
  return ret_;
7387
}
7388

7389
bool ObServerBalancer::ServerTotalLoadCmp::operator()(
7390
     const ServerTotalLoad &left,
7391
     const ServerTotalLoad &right)
7392
{
7393
  bool bool_ret = false;
7394
  if (OB_UNLIKELY(OB_SUCCESS != ret_)) {
7395
    // ignore
7396
  } else if (left.wild_server_ && !right.wild_server_) {
7397
    bool_ret = true;
7398
  } else if (!left.wild_server_ && right.wild_server_) {
7399
    bool_ret = false;
7400
  } else {
7401
    if (left.inter_ttg_load_value_ > right.inter_ttg_load_value_) {
7402
      bool_ret = true;
7403
    } else {
7404
      bool_ret = false;
7405
    }
7406
  }
7407
  return bool_ret;
7408
}
7409

7410
bool ObServerBalancer::IntraServerLoadCmp::operator()(
7411
     const ServerLoad *left,
7412
     const ServerLoad *right)
7413
{
7414
  bool bool_ret = false;
7415
  if (OB_UNLIKELY(OB_SUCCESS != ret_)) {
7416
    // ignore
7417
  } else if (OB_UNLIKELY(NULL == left || NULL == right)) {
7418
    ret_ = OB_INVALID_ARGUMENT;
7419
    LOG_WARN_RET(ret_, "invalid argument", K(ret_), KP(left), KP(right));
7420
  } else if (left->intra_ttg_load_value_ > right->intra_ttg_load_value_) {
7421
    bool_ret = true;
7422
  } else {
7423
    bool_ret = false;
7424
  }
7425
  return bool_ret;
7426
}
7427

7428
int ObServerBalancer::IntraServerLoadCmp::get_ret() const
7429
{
7430
  return ret_;
7431
}
7432

7433
bool ObServerBalancer::IntraServerLoadCmp::operator()(
7434
     const ServerLoad &left,
7435
     const ServerLoad &right)
7436
{
7437
  bool bool_ret = false;
7438
  if (OB_UNLIKELY(OB_SUCCESS != ret_)) {
7439
    // ignore
7440
  } else if (left.intra_ttg_load_value_ > right.intra_ttg_load_value_) {
7441
    bool_ret = true;
7442
  } else {
7443
    bool_ret = false;
7444
  }
7445
  return bool_ret;
7446
}
7447

7448
bool ObServerBalancer::InterServerLoadCmp::operator()(
7449
     const ServerLoad *left,
7450
     const ServerLoad *right)
7451
{
7452
  bool bool_ret = false;
7453
  if (OB_UNLIKELY(OB_SUCCESS != ret_)) {
7454
    // ignore
7455
  } else if (OB_UNLIKELY(NULL == left || NULL == right)) {
7456
    ret_ = OB_INVALID_ARGUMENT;
7457
    LOG_WARN_RET(ret_, "invalid argument", K(ret_), KP(left), KP(right));
7458
  } else if (left->inter_ttg_load_value_ > right->inter_ttg_load_value_) {
7459
    bool_ret = true;
7460
  } else {
7461
    bool_ret = false;
7462
  }
7463
  return bool_ret;
7464
}
7465

7466
int ObServerBalancer::InterServerLoadCmp::get_ret() const
7467
{
7468
  return ret_;
7469
}
7470

7471
bool ObServerBalancer::InterServerLoadCmp::operator()(
7472
     const ServerLoad &left,
7473
     const ServerLoad &right)
7474
{
7475
  bool bool_ret = false;
7476
  if (OB_UNLIKELY(OB_SUCCESS != ret_)) {
7477
    // ignore
7478
  } else if (left.inter_ttg_load_value_ > right.inter_ttg_load_value_) {
7479
    bool_ret = true;
7480
  } else {
7481
    bool_ret = false;
7482
  }
7483
  return bool_ret;
7484
}
7485

7486
bool ObServerBalancer::ServerLoadUgCntCmp::operator()(
7487
     const ServerLoad *left,
7488
     const ServerLoad *right)
7489
{
7490
  bool bool_ret = false;
7491
  if (OB_UNLIKELY(OB_SUCCESS != ret_)) {
7492
    // ignore
7493
  } else if (OB_UNLIKELY(NULL == left || NULL == right)) {
7494
    ret_ = OB_INVALID_ARGUMENT;
7495
    LOG_WARN_RET(ret_, "invalid argument", K(ret_), KP(left), KP(right));
7496
  } else if (left->unitgroup_loads_.count() < right->unitgroup_loads_.count()) {
7497
    bool_ret = true;
7498
  } else {
7499
    bool_ret = false;
7500
  }
7501
  return bool_ret;
7502
}
7503

7504
int ObServerBalancer::ServerLoadUgCntCmp::get_ret() const
7505
{
7506
  return ret_;
7507
}
7508

7509
bool ObServerBalancer::LoadSum::is_valid() const
7510
{
7511
  return load_sum_.min_cpu() >= 0
7512
         && load_sum_.max_cpu() >= load_sum_.min_cpu()
7513
         && load_sum_.memory_size() >= 0
7514
         && load_sum_.log_disk_size() >= 0;
7515
}
7516

7517
void ObServerBalancer::LoadSum::reset()
7518
{
7519
  load_sum_.reset();
7520
}
7521

7522
double ObServerBalancer::LoadSum::get_required(const ObResourceType resource_type) const
7523
{
7524
  double ret = -1;
7525
  switch (resource_type) {
7526
  case RES_CPU:
7527
    ret = load_sum_.min_cpu();
7528
    break;
7529
  case RES_MEM:
7530
    ret = static_cast<double>(load_sum_.memory_size());
7531
    break;
7532
  case RES_LOG_DISK:
7533
    ret = static_cast<double>(load_sum_.log_disk_size());
7534
    break;
7535
  default:
7536
    ret = -1;
7537
    break;
7538
  }
7539
  return ret;
7540
}
7541

7542
int ObServerBalancer::LoadSum::calc_load_value(
7543
    double *const resource_weights,
7544
    const int64_t weights_count,
7545
    const ServerResourceLoad &server_resource,
7546
    double &load_value)
7547
{
7548
  int ret = OB_SUCCESS;
7549
  if (NULL == resource_weights || RES_MAX != weights_count) {
7550
    ret = OB_INVALID_ARGUMENT;
7551
    LOG_WARN("invalid argument", K(ret));
7552
  } else {
7553
    load_value = 0.0;
7554
    for (int32_t i = RES_CPU; i < RES_MAX; ++i) {
7555
      ObResourceType res_type = static_cast<ObResourceType>(i);
7556
      if (server_resource.get_true_capacity(res_type) <= 0
7557
          || get_required(res_type) <= 0) {
7558
        // has no effect on load value
7559
      } else if (get_required(res_type) > server_resource.get_true_capacity(res_type)) {
7560
        load_value += resource_weights[i] * 1.0;
7561
      } else {
7562
        double quotient = get_required(res_type) / server_resource.get_true_capacity(res_type);
7563
        load_value += resource_weights[i] * quotient;
7564
      }
7565
    }
7566
  }
7567
  return ret;
7568
}
7569

7570
int ObServerBalancer::LoadSum::remove_load(
7571
    const ObUnitManager::ObUnitLoad &unit_load)
7572
{
7573
  int ret = OB_SUCCESS;
7574
  if (OB_UNLIKELY(!unit_load.is_valid())) {
7575
    ret = OB_INVALID_ARGUMENT;
7576
    LOG_WARN("invalid argument", K(ret), K(unit_load));
7577
  } else {
7578
    load_sum_ -= *unit_load.unit_config_;
7579
  }
7580
  return ret;
7581
}
7582

7583
int ObServerBalancer::LoadSum::append_load(
7584
    const share::ObUnitConfig &load)
7585
{
7586
  int ret = OB_SUCCESS;
7587
  load_sum_ += load;
7588
  return ret;
7589
}
7590

7591
int ObServerBalancer::LoadSum::append_load(
7592
    const ObUnitManager::ObUnitLoad &unit_load)
7593
{
7594
  int ret = OB_SUCCESS;
7595
  if (OB_UNLIKELY(!unit_load.is_valid())) {
7596
    ret = OB_INVALID_ARGUMENT;
7597
    LOG_WARN("invalid argument", K(ret), K(unit_load));
7598
  } else {
7599
    load_sum_ += *unit_load.unit_config_;
7600
  }
7601
  return ret;
7602
}
7603

7604
int ObServerBalancer::LoadSum::append_load(
7605
    const common::ObIArray<ObUnitManager::ObUnitLoad> &unit_loads)
7606
{
7607
  int ret = OB_SUCCESS;
7608
  for (int64_t i = 0; OB_SUCC(ret) && i < unit_loads.count(); ++i) {
7609
    const ObUnitManager::ObUnitLoad &unit_load = unit_loads.at(i);
7610
    if (OB_UNLIKELY(!unit_load.is_valid())) {
7611
      ret = OB_INVALID_ARGUMENT;
7612
      LOG_WARN("invalid argument", K(ret), K(unit_load));
7613
    } else {
7614
      load_sum_ += *unit_load.unit_config_;
7615
    }
7616
  }
7617
  return ret;
7618
}
7619

7620
int ObServerBalancer::LoadSum::append_load(
7621
    const ObServerBalancer::LoadSum &load)
7622
{
7623
  int ret = OB_SUCCESS;
7624
  load_sum_ += load.load_sum_;
7625
  return ret;
7626
}
7627

7628
bool ObServerBalancer::ResourceSum::is_valid() const
7629
{
7630
  return resource_sum_.cpu_ > 0
7631
         && resource_sum_.mem_total_ > 0
7632
         && resource_sum_.disk_total_ > 0;
7633
}
7634

7635
void ObServerBalancer::ResourceSum::reset()
7636
{
7637
  resource_sum_.reset();
7638
}
7639

7640
double ObServerBalancer::ResourceSum::get_capacity(const ObResourceType resource_type) const
7641
{
7642
  double ret = -1;
7643
  switch (resource_type) {
7644
  case RES_CPU:
7645
    ret = resource_sum_.cpu_;
7646
    break;
7647
  case RES_MEM:
7648
    ret = static_cast<double>(resource_sum_.mem_total_);
7649
    break;
7650
  case RES_LOG_DISK:
7651
    ret = static_cast<double>(resource_sum_.log_disk_total_);
7652
    break;
7653
  default:
7654
    ret = -1;
7655
    break;
7656
  }
7657
  return ret;
7658
}
7659

7660
int ObServerBalancer::ResourceSum::append_resource(
7661
    const share::ObServerResourceInfo &resource)
7662
{
7663
  int ret = OB_SUCCESS;
7664
  resource_sum_.cpu_ += resource.cpu_;
7665
  resource_sum_.mem_total_ += resource.mem_total_;
7666
  resource_sum_.disk_total_ += resource.disk_total_;
7667
  resource_sum_.log_disk_total_ += resource.log_disk_total_;
7668
  return ret;
7669
}
7670

7671
int ObServerBalancer::ResourceSum::append_resource(
7672
    const ResourceSum &resource)
7673
{
7674
  int ret = OB_SUCCESS;
7675
  resource_sum_.cpu_ += resource.resource_sum_.cpu_;
7676
  resource_sum_.mem_total_ += resource.resource_sum_.mem_total_;
7677
  resource_sum_.disk_total_ += resource.resource_sum_.disk_total_;
7678
  resource_sum_.log_disk_total_ += resource.resource_sum_.log_disk_total_;
7679
  return ret;
7680
}
7681

7682
int ObServerBalancer::TenantGroupBalanceInfo::get_trick_id(
7683
    uint64_t &trick_id)
7684
{
7685
  int ret = OB_SUCCESS;
7686
  if (OB_UNLIKELY(NULL == tenant_id_matrix_)) {
7687
    ret = OB_ERR_UNEXPECTED;
7688
    LOG_WARN("tenant id matrix ptr is null", K(ret), KP(tenant_id_matrix_));
7689
  } else if (tenant_id_matrix_->get_row_count() <= 0
7690
             || tenant_id_matrix_->get_row_count() >= INT32_MAX
7691
             || tenant_id_matrix_->get_column_count() <= 0
7692
             || tenant_id_matrix_->get_column_count() >= INT32_MAX) {
7693
    ret = OB_ERR_UNEXPECTED;
7694
    LOG_WARN("tenant id matrix size unexpected", K(ret),
7695
             "row_count", tenant_id_matrix_->get_row_count(),
7696
             "column_count", tenant_id_matrix_->get_column_count());
7697
  } else {
7698
    trick_id = UINT64_MAX;
7699
    for (int64_t row = 0;
7700
         OB_SUCC(ret) && row < tenant_id_matrix_->get_row_count();
7701
         ++row) {
7702
      for (int64_t column = 0;
7703
           OB_SUCC(ret) && column < tenant_id_matrix_->get_column_count();
7704
           ++column) {
7705
        uint64_t my_trick_id = UINT64_MAX;
7706
        if (OB_FAIL(tenant_id_matrix_->get(row, column, my_trick_id))) {
7707
          LOG_WARN("fail to get from tenant id matrix", K(ret), K(row), K(column));
7708
        } else if (my_trick_id < trick_id) {
7709
          trick_id = my_trick_id;
7710
        } else {} // next
7711
      }
7712
    }
7713
  }
7714
  return ret;
7715
}
7716

7717
ObServerBalancer::ServerLoad *ObServerBalancer::TenantGroupBalanceInfo::get_server_load(
7718
    const common::ObAddr &server)
7719
{
7720
  ServerLoad *ret_ptr = NULL;
7721
  for (int64_t i = 0; NULL == ret_ptr && i < server_load_array_.count(); ++i) {
7722
    ServerLoad &server_load = server_load_array_.at(i);
7723
    if (server_load.server_ != server) {
7724
      // go on to check next
7725
    } else {
7726
      ret_ptr = &server_load;
7727
    }
7728
  }
7729
  return ret_ptr;
7730
}
7731

7732
int ObServerBalancer::TenantGroupBalanceInfo::get_all_unit_loads(
7733
    common::ObIArray<ObUnitManager::ObUnitLoad> &unit_loads)
7734
{
7735
  int ret = OB_SUCCESS;
7736
  unit_loads.reset();
7737
  for (int64_t i = 0; OB_SUCC(ret) && i < unit_migrate_stat_matrix_.get_row_count(); ++i) {
7738
    for (int64_t j = 0; OB_SUCC(ret) && j < unit_migrate_stat_matrix_.get_column_count(); ++j) {
7739
      UnitMigrateStat *unit_stat = unit_migrate_stat_matrix_.get(i, j);
7740
      if (OB_UNLIKELY(NULL == unit_stat)) {
7741
        ret = OB_ERR_UNEXPECTED;
7742
        LOG_WARN("unit stat ptr is null", K(ret), KP(unit_stat));
7743
      } else if (OB_FAIL(unit_loads.push_back(unit_stat->unit_load_))) {
7744
        LOG_WARN("fail to push back", K(ret));
7745
      } else {} // go on next
7746
    }
7747
  }
7748
  return ret;
7749
}
7750

7751
bool ObServerBalancer::TenantGroupBalanceInfo::is_stable() const
7752
{
7753
  bool stable = true;
7754
  for (int64_t i = 0; stable && i < unit_migrate_stat_matrix_.get_row_count(); ++i) {
7755
    for (int64_t j = 0; stable && j < unit_migrate_stat_matrix_.get_column_count(); ++j) {
7756
      const UnitMigrateStat *unit_stat = unit_migrate_stat_matrix_.get(i, j);
7757
      if (NULL == unit_stat) {
7758
        stable = false;
7759
      } else if (unit_stat->original_pos_ != unit_stat->arranged_pos_) {
7760
        stable = false;
7761
      } else {} // go on next
7762
    }
7763
  }
7764
  return stable;
7765
}
7766

7767
int ObServerBalancer::check_tenant_group_config_legality(
7768
    common::ObIArray<ObTenantGroupParser::TenantNameGroup> &tenant_groups,
7769
    bool &legal)
7770
{
7771
  int ret = OB_SUCCESS;
7772
  SpinRLockGuard guard(unit_mgr_->get_lock());  // lock!
7773
  legal = true;
7774
  share::schema::ObSchemaGetterGuard schema_guard;
7775
  if (OB_UNLIKELY(!inited_)) {
7776
    ret = OB_NOT_INIT;
7777
    LOG_WARN("not init", K(ret));
7778
  } else if (OB_UNLIKELY(NULL == schema_service_)) {
7779
    ret = OB_ERR_UNEXPECTED;
7780
    LOG_WARN("schema service ptr is null", K(ret), KP(schema_service_));
7781
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
7782
    LOG_WARN("fail to get schema guard", K(ret));
7783
  } else {
7784
    for (int64_t i = 0; OB_SUCC(ret) && i < tenant_groups.count() && legal; ++i) {
7785
      const ObTenantGroupParser::TenantNameGroup &tenant_name_group = tenant_groups.at(i);
7786
      for (int64_t j = 0; OB_SUCC(ret) && j < tenant_name_group.tenants_.count(); ++j) {
7787
        const share::schema::ObTenantSchema *tenant_schema = NULL;
7788
        if (OB_FAIL(schema_guard.get_tenant_info(
7789
                tenant_name_group.tenants_.at(j), tenant_schema))) {
7790
          LOG_WARN("fail to get tenant info", K(ret));
7791
        } else if (OB_UNLIKELY(NULL == tenant_schema)) {
7792
          ret = OB_TENANT_NOT_EXIST;
7793
          LOG_WARN("tenant schema ptr is null", K(ret), KP(tenant_schema));
7794
        } else if (OB_UNLIKELY(OB_SYS_TENANT_ID == tenant_schema->get_tenant_id())) {
7795
          ret = OB_ERR_UNEXPECTED;
7796
          LOG_ERROR("sys tenant cannot be configured in tenant groups", K(ret));
7797
        } else {} // good and go on next
7798
      }
7799
    }
7800
  }
7801
  return ret;
7802
}
7803

7804
int ObServerBalancer::get_server_data_disk_usage_limit(
7805
    double &data_disk_usage_limit) const
7806
{
7807
  int ret = OB_SUCCESS;
7808
  if (OB_UNLIKELY(!inited_)) {
7809
    ret = OB_NOT_INIT;
7810
    LOG_WARN("not init", K(ret));
7811
  } else {
7812
    int64_t value = GCONF.data_disk_usage_limit_percentage - 1;
7813
    value = (value <= 0 ? 0 : value);
7814
    data_disk_usage_limit = static_cast<double>(value) / 100;
7815
  }
7816
  return ret;
7817
}
7818

7819
int ObServerBalancer::get_server_balance_critical_disk_waterlevel(
7820
    double &server_balance_critical_disk_waterlevel) const
7821
{
7822
  int ret = OB_SUCCESS;
7823
  if (OB_UNLIKELY(!inited_)) {
7824
    ret = OB_NOT_INIT;
7825
    LOG_WARN("not init", K(ret));
7826
  } else {
7827
    int64_t value = GCONF.server_balance_critical_disk_waterlevel;
7828
    server_balance_critical_disk_waterlevel = static_cast<double>(value) / 100;
7829
  }
7830
  return ret;
7831
}
7832

7833
ERRSIM_POINT_DEF(ERRSIM_SERVER_DISK_ASSIGN);
7834

7835
int ObServerBalancer::generate_zone_server_disk_statistic(
7836
    const common::ObZone &zone)
7837
{
7838
  int ret = OB_SUCCESS;
7839
  common::ObArray<common::ObAddr> server_list;
7840
  double disk_waterlevel = 0.0;
7841
  if (OB_UNLIKELY(!inited_)) {
7842
    ret = OB_NOT_INIT;
7843
    LOG_WARN("not init", K(ret));
7844
  } else if (OB_UNLIKELY(zone.is_empty())) {
7845
    ret = OB_INVALID_ARGUMENT;
7846
    LOG_WARN("invalid argument", K(ret), K(zone));
7847
  } else if (OB_ISNULL(server_mgr_)) {
7848
    ret = OB_ERR_UNEXPECTED;
7849
    LOG_WARN("server_mgr_ ptr is null", KR(ret), KP(server_mgr_));
7850
  } else if (OB_FAIL(SVR_TRACER.get_servers_of_zone(zone, server_list))) {
7851
    LOG_WARN("fail to get servers of zone", KR(ret), K(zone));
7852
  } else if (OB_FAIL(get_server_balance_critical_disk_waterlevel(disk_waterlevel))) {
7853
    LOG_WARN("fail to get server balance disk water level", K(ret));
7854
  } else {
7855
    zone_disk_statistic_.reset();
7856
    zone_disk_statistic_.zone_ = zone;
7857
    for (int64_t i = 0; OB_SUCC(ret) && i < server_list.count(); ++i) {
7858
      const common::ObAddr &server = server_list.at(i);
7859
      share::ObServerResourceInfo server_resource_info;
7860
      share::ObServerInfoInTable server_info;
7861
      ServerDiskStatistic disk_statistic;
7862
      if (OB_FAIL(SVR_TRACER.get_server_info(server, server_info))) {
7863
        LOG_WARN("fail to get server info", KR(ret), K(server));
7864
      } else if (server_info.is_temporary_offline() || server_info.is_stopped()) {
7865
        ret = OB_STATE_NOT_MATCH;
7866
        LOG_WARN("server is not stable, stop balance servers in this zone",
7867
            KR(ret), K(server), K(zone),
7868
            "is_temporary_offline", server_info.is_temporary_offline(),
7869
            "is_stopped", server_info.is_stopped());
7870
      } else if (OB_FAIL(server_mgr_->get_server_resource_info(server, server_resource_info))) {
7871
        LOG_WARN("fail to get server resource info", KR(ret), K(server));
7872
      } else if (server_info.is_active()) {
7873
        disk_statistic.server_ = server;
7874
        disk_statistic.wild_server_ = false;
7875
        if (OB_SUCC(ERRSIM_SERVER_DISK_ASSIGN)) {
7876
          disk_statistic.disk_in_use_ = server_resource_info.disk_in_use_;
7877
        } else {
7878
          // ONLY FOR TEST, errsim triggered, make disk_in_use equal to (1GB * unit_num)
7879
          ObArray<ObUnitManager::ObUnitLoad> *unit_loads_ptr;
7880
          if (OB_FAIL(unit_mgr_->get_loads_by_server(server, unit_loads_ptr))) {
7881
            if(OB_ENTRY_NOT_EXIST == ret) {
7882
              ret = OB_SUCCESS; // disk_in_use is 0
7883
            } else {
7884
              LOG_WARN("fail to get_loads_by_server", KR(ret), K(server));
7885
            }
7886
          } else {
7887
            disk_statistic.disk_in_use_ = unit_loads_ptr->count() * (1024 * 1024 * 1024);
7888
          }
7889
          LOG_ERROR("errsim triggered, assign server disk_in_use as unit count * 1GB", KR(ret), K(disk_statistic));
7890
        }
7891
        disk_statistic.disk_total_ = server_resource_info.disk_total_;
7892
        if (static_cast<double>(disk_statistic.disk_in_use_)
7893
            > disk_waterlevel * static_cast<double>(disk_statistic.disk_total_)) {
7894
          zone_disk_statistic_.over_disk_waterlevel_ = true;
7895
        }
7896
      } else if (server_info.is_deleting() || server_info.is_permanent_offline()) {
7897
        disk_statistic.server_ = server;
7898
        disk_statistic.wild_server_ = true;
7899
        disk_statistic.disk_in_use_ = server_resource_info.disk_in_use_;
7900
        disk_statistic.disk_total_ = server_resource_info.disk_total_;
7901
      } else {
7902
        ret = OB_ERR_UNEXPECTED;
7903
        LOG_WARN("unknow server_info", K(ret), K(server_info));
7904
      }
7905
      if (OB_FAIL(ret)) {
7906
      } else if (OB_FAIL(zone_disk_statistic_.append(disk_statistic))) {
7907
        LOG_WARN("fail to append server statistic", K(ret));
7908
      } else {} // no more to do
7909
    }
7910
    if (OB_SUCC(ret)) {
7911
      LOG_INFO("build zone disk statistic succeed", K(ret), K(zone_disk_statistic_));
7912
    }
7913
  }
7914
  return ret;
7915
}
7916

7917
bool ObServerBalancer::ServerDiskStatisticCmp::operator()(
7918
     const ServerDiskStatistic &left,
7919
     const ServerDiskStatistic &right)
7920
{
7921
  bool bool_ret = false;
7922
  if (!left.wild_server_ && right.wild_server_) {
7923
    bool_ret = true;
7924
  } else if (left.wild_server_ && !right.wild_server_) {
7925
    bool_ret = false;
7926
  } else if (left.wild_server_ && right.wild_server_) {
7927
    // whatever, I don't care the sequence of two wild servers
7928
  } else {
7929
    int64_t left_available = left.disk_total_ - left.disk_in_use_;
7930
    int64_t right_available = right.disk_total_ - right.disk_in_use_;
7931
    if (left_available > right_available) {
7932
      bool_ret = true;
7933
    } else {
7934
      bool_ret = false;
7935
    }
7936
  }
7937
  return bool_ret;
7938
}
7939

7940
int ObServerBalancer::ZoneServerDiskStatistic::append(
7941
    ServerDiskStatistic &server_disk_statistic)
7942
{
7943
  int ret = OB_SUCCESS;
7944
  if (OB_FAIL(server_disk_statistic_array_.push_back(server_disk_statistic))) {
7945
    LOG_WARN("fail to push back", K(ret));
7946
  } else {
7947
    ServerDiskStatisticCmp cmp;
7948
    std::sort(server_disk_statistic_array_.begin(),
7949
              server_disk_statistic_array_.end(),
7950
              cmp);
7951
  }
7952
  return ret;
7953
}
7954

7955
int ObServerBalancer::ZoneServerDiskStatistic::raise_server_disk_use(
7956
    const common::ObAddr &server,
7957
    const int64_t unit_required_size)
7958
{
7959
  int ret = OB_SUCCESS;
7960
  if (OB_UNLIKELY(!server.is_valid())) {
7961
    ret = OB_INVALID_ARGUMENT;
7962
    LOG_WARN("invalid argument", K(ret));
7963
  } else {
7964
    bool find = false;
7965
    for (int64_t i = 0;
7966
         OB_SUCC(ret) && !find && i < server_disk_statistic_array_.count();
7967
         ++i) {
7968
      if (server_disk_statistic_array_.at(i).server_ != server) {
7969
        // go on to check next
7970
      } else {
7971
        find = true;
7972
        ServerDiskStatistic &server_disk_statistic = server_disk_statistic_array_.at(i);
7973
        server_disk_statistic.disk_in_use_ += unit_required_size;
7974
      }
7975
    }
7976
    if (OB_FAIL(ret)) {
7977
    } else if (!find) {
7978
      ret = OB_ENTRY_NOT_EXIST;
7979
      LOG_WARN("server not exist", K(ret), K(server));
7980
    } else {
7981
      ServerDiskStatisticCmp cmp;
7982
      std::sort(server_disk_statistic_array_.begin(),
7983
                server_disk_statistic_array_.end(),
7984
                cmp);
7985
    }
7986
  }
7987
  return ret;
7988
}
7989

7990
int ObServerBalancer::ZoneServerDiskStatistic::reduce_server_disk_use(
7991
    const common::ObAddr &server,
7992
    const int64_t unit_required_size)
7993
{
7994
  int ret = OB_SUCCESS;
7995
  if (OB_UNLIKELY(!server.is_valid())) {
7996
    ret = OB_INVALID_ARGUMENT;
7997
    LOG_WARN("invalid argument", K(ret));
7998
  } else {
7999
    bool find = false;
8000
    for (int64_t i = 0;
8001
         OB_SUCC(ret) && !find && i < server_disk_statistic_array_.count();
8002
         ++i) {
8003
      if (server_disk_statistic_array_.at(i).server_ != server) {
8004
        // go on to check next
8005
      } else {
8006
        find = true;
8007
        ServerDiskStatistic &server_disk_statistic = server_disk_statistic_array_.at(i);
8008
        server_disk_statistic.disk_in_use_ -= unit_required_size;
8009
      }
8010
    }
8011
    if (OB_FAIL(ret)) {
8012
    } else if (!find) {
8013
      ret = OB_ENTRY_NOT_EXIST;
8014
      LOG_WARN("server not exist", K(ret), K(server));
8015
    } else {
8016
      ServerDiskStatisticCmp cmp;
8017
      std::sort(server_disk_statistic_array_.begin(),
8018
                server_disk_statistic_array_.end(),
8019
                cmp);
8020
    }
8021
  }
8022
  return ret;
8023
}
8024

8025
int ObServerBalancer::ZoneServerDiskStatistic::get_server_disk_statistic(
8026
    const common::ObAddr &server,
8027
    ServerDiskStatistic &server_disk_statistic)
8028
{
8029
  int ret = OB_SUCCESS;
8030
  if (OB_UNLIKELY(!server.is_valid())) {
8031
    ret = OB_INVALID_ARGUMENT;
8032
    LOG_WARN("invalid argument", K(ret));
8033
  } else {
8034
    server_disk_statistic.reset();
8035
    bool find = false;
8036
    for (int64_t i = 0;
8037
         OB_SUCC(ret) && !find && i < server_disk_statistic_array_.count();
8038
         ++i) {
8039
      if (server_disk_statistic_array_.at(i).server_ != server) {
8040
        // go on to check next
8041
      } else {
8042
        find = true;
8043
        server_disk_statistic = server_disk_statistic_array_.at(i);
8044
      }
8045
    }
8046
    if (OB_FAIL(ret)) {
8047
    } else if (!find) {
8048
      ret = OB_ENTRY_NOT_EXIST;
8049
    }
8050
  }
8051
  return ret;
8052
}
8053

8054
int ObServerBalancer::ZoneServerDiskStatistic::check_server_over_disk_waterlevel(
8055
    const common::ObAddr &server,
8056
    const double disk_waterlevel,
8057
    bool &disk_over_waterlevel)
8058
{
8059
  int ret = OB_SUCCESS;
8060
  if (OB_UNLIKELY(!server.is_valid())) {
8061
    ret = OB_INVALID_ARGUMENT;
8062
    LOG_WARN("invalid argument", K(ret), K(server));
8063
  } else {
8064
    bool find = false;
8065
    for (int64_t i = 0; !find && i < server_disk_statistic_array_.count(); ++i) {
8066
      ServerDiskStatistic &disk_statistic = server_disk_statistic_array_.at(i);
8067
      if (server != disk_statistic.server_) {
8068
        // go on to check next
8069
      } else {
8070
        find = true;
8071
        disk_over_waterlevel
8072
            = (static_cast<double>(disk_statistic.disk_in_use_)
8073
               > disk_waterlevel * static_cast<double>(disk_statistic.disk_total_));
8074
      }
8075
    }
8076
    if (!find) {
8077
      ret = OB_ENTRY_NOT_EXIST;
8078
      LOG_WARN("server not exist", K(ret));
8079
    }
8080
  }
8081
  return ret;
8082
}
8083

8084
int ObServerBalancer::ZoneServerDiskStatistic::check_all_available_servers_over_disk_waterlevel(
8085
    const double disk_waterlevel,
8086
    bool &all_available_servers_over_disk_waterlevel)
8087
{
8088
  int ret = OB_SUCCESS;
8089
  all_available_servers_over_disk_waterlevel = true;
8090
  for (int64_t i = 0;
8091
       all_available_servers_over_disk_waterlevel
8092
       && OB_SUCC(ret)
8093
       && i < server_disk_statistic_array_.count();
8094
       ++i) {
8095
    ServerDiskStatistic &disk_statistic = server_disk_statistic_array_.at(i);
8096
    all_available_servers_over_disk_waterlevel
8097
        = (static_cast<double>(disk_statistic.disk_in_use_)
8098
           > disk_waterlevel * static_cast<double>(disk_statistic.disk_total_));
8099

8100
    LOG_INFO("check server over disk waterlevel",
8101
        K_(zone), K(disk_statistic),
8102
        "over_disk_waterlevel", all_available_servers_over_disk_waterlevel);
8103
  }
8104
  LOG_INFO("check all available servers over disk waterlevel",
8105
      K_(zone),
8106
      K(all_available_servers_over_disk_waterlevel),
8107
      "zone_server_over_disk_waterlevel", over_disk_waterlevel_,
8108
      K_(server_disk_statistic_array));
8109
  return ret;
8110
}
8111

8112
bool ObServerBalancer::ServerDiskPercentCmp::operator()(
8113
     const ServerDiskStatistic *left,
8114
     const ServerDiskStatistic *right)
8115
{
8116
  // Sort by the percentage of disk usage from highest to bottom
8117
  bool bool_ret = false;
8118
  if (OB_UNLIKELY(OB_SUCCESS != ret_)) {
8119
    // ignore
8120
  } else if (OB_UNLIKELY(NULL == left || NULL == right)) {
8121
    ret_ = OB_INVALID_ARGUMENT;
8122
    LOG_WARN_RET(ret_,"invalid argument", K(ret_), KP(left), KP(right));
8123
  } else {
8124
    double left_percent = left->get_disk_used_percent();
8125
    double right_percent = right->get_disk_used_percent();
8126
    if (left_percent > right_percent) {
8127
      bool_ret = true;
8128
    } else {
8129
      bool_ret = false;
8130
    }
8131
  }
8132
  return bool_ret;
8133
}
8134

8135
bool ObServerBalancer::UnitLoadDiskCmp::operator()(
8136
     const ObUnitManager::ObUnitLoad *left,
8137
     const ObUnitManager::ObUnitLoad *right)
8138
{
8139
  bool bool_ret = false;
8140
  ObUnitStat left_stat;
8141
  ObUnitStat right_stat;
8142
  int &ret = ret_;
8143
  if (common::OB_SUCCESS != ret_) {
8144
    // bypass
8145
  } else if (NULL == left || NULL == right) {
8146
    ret_ = OB_ERR_UNEXPECTED;
8147
    LOG_WARN("unit load ptr is null", K(ret_), KP(left), KP(right));
8148
  } else if (!left->is_valid() || !right->is_valid()) {
8149
    ret_ = OB_ERR_UNEXPECTED;
8150
    LOG_WARN("unit load unexpected", K(ret_), K(*left), K(*right));
8151
  } else if (OB_SUCCESS != (ret_ = unit_stat_mgr_.get_unit_stat(
8152
          left->unit_->unit_id_, left->unit_->zone_, left_stat))) {
8153
    LOG_WARN("fail to get left unit stat", K(ret_));
8154
  } else if (OB_SUCCESS != (ret_ = unit_stat_mgr_.get_unit_stat(
8155
          right->unit_->unit_id_, right->unit_->zone_, right_stat))) {
8156
    LOG_WARN("fail to get right unit stat", K(ret_));
8157
  } else if (left_stat.get_required_size() > right_stat.get_required_size()) {
8158
    bool_ret = true;
8159
  } else {
8160
    bool_ret = false;
8161
  }
8162
  return bool_ret;
8163
}
8164

8165