oceanbase

1
/**
2
 * Copyright (c) 2021 OceanBase
3
 * OceanBase CE is licensed under Mulan PubL v2.
4
 * You can use this software according to the terms and conditions of the Mulan PubL v2.
5
 * You may obtain a copy of Mulan PubL v2 at:
6
 *          http://license.coscl.org.cn/MulanPubL-2.0
7
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10
 * See the Mulan PubL v2 for more details.
11
 */
12

13
#define USING_LOG_PREFIX RS
14

15
#include "ob_unit_manager.h"
16

17
#include <cmath>
18
#include <float.h>
19

20
#include "lib/string/ob_sql_string.h"
21
#include "lib/allocator/ob_mod_define.h"
22
#include "lib/time/ob_time_utility.h"
23
#include "lib/container/ob_array_iterator.h"
24
#include "lib/mysqlclient/ob_mysql_proxy.h"
25
#include "lib/mysqlclient/ob_mysql_transaction.h"
26
#include "lib/utility/ob_tracepoint.h"
27
#include "share/ob_unit_getter.h"
28
#include "share/ob_unit_stat.h"
29
#include "share/ob_debug_sync.h"
30
#include "share/ob_srv_rpc_proxy.h"
31
#include "share/config/ob_server_config.h"
32
#include "share/ob_schema_status_proxy.h"
33
#include "share/ob_dml_sql_splicer.h"
34
#include "share/ob_max_id_fetcher.h"
35
#include "share/inner_table/ob_inner_table_schema.h"
36
#include "share/ob_tenant_memstore_info_operator.h"
37
#include "share/ob_rpc_struct.h"
38
#include "storage/ob_file_system_router.h"
39
#include "observer/ob_server_struct.h"
40
#include "observer/omt/ob_tenant_node_balancer.h"
41
#include "rootserver/ob_balance_info.h"
42
#include "rootserver/ob_zone_manager.h"
43
#include "rootserver/ob_rs_event_history_table_operator.h"
44
#include "rootserver/ob_unit_placement_strategy.h"
45
#include "rootserver/ob_rs_job_table_operator.h"
46
#include "rootserver/ob_root_service.h"
47
#include "rootserver/ob_root_balancer.h"
48
#include "rootserver/ob_server_manager.h"
49
#include "storage/ob_file_system_router.h"
50
#include "share/ob_all_server_tracer.h"
51
#include "rootserver/ob_heartbeat_service.h"
52

53
namespace oceanbase
54
{
55
using namespace common;
56
using namespace common::sqlclient;
57
using namespace common::hash;
58
using namespace share;
59
using namespace share::schema;
60
namespace rootserver
61
{
62

63
int ObUnitManager::ZoneUnitPtr::assign(const ZoneUnitPtr &other)
64
{
65
  int ret = OB_SUCCESS;
66
  zone_ = other.zone_;
67
  if (OB_FAIL(copy_assign(unit_ptrs_, other.unit_ptrs_))) {
68
    LOG_WARN("fail to assign unit_ptrs", K(ret));
69
  }
70
  return ret;
71
}
72

73
int ObUnitManager::ZoneUnitPtr::sort_by_unit_id_desc()
74
{
75
  UnitGroupIdCmp cmp;
76
  std::sort(unit_ptrs_.begin(), unit_ptrs_.end(), cmp);
77
  return cmp.get_ret();
78
}
79

80
int ObUnitManager::ZoneUnit::assign(const ZoneUnit &other)
81
{
82
  int ret = OB_SUCCESS;
83
  zone_ = other.zone_;
84
  if (OB_FAIL(copy_assign(unit_infos_, other.unit_infos_))) {
85
    LOG_WARN("failed to assign unit_infos_", K(ret));
86
  }
87
  return ret;
88
}
89
////////////////////////////////////////////////////////////////
90
double ObUnitManager::ObUnitLoad::get_demand(ObResourceType resource_type) const
91
{
92
  double ret = -1;
93
  switch (resource_type) {
94
    case RES_CPU:
95
      ret = unit_config_->min_cpu();
96
      break;
97
    case RES_MEM:
98
      ret = static_cast<double>(unit_config_->memory_size());
99
      break;
100
    case RES_LOG_DISK:
101
      ret = static_cast<double>(unit_config_->log_disk_size());
102
      break;
103
    default:
104
      ret = -1;
105
      break;
106
  }
107
  return ret;
108
}
109

110
const char *ObUnitManager::end_migrate_op_type_to_str(const ObUnitManager::EndMigrateOp &t)
111
{
112
  const char* str = "UNKNOWN";
113
  if (EndMigrateOp::COMMIT == t) {
114
    str = "COMMIT";
115
  } else if (EndMigrateOp::ABORT == t) {
116
    str = "ABORT";
117
  } else if (EndMigrateOp::REVERSE == t) {
118
    str = "REVERSE";
119
  } else {
120
    str = "NONE";
121
  }
122
  return str;
123
}
124
////////////////////////////////////////////////////////////////
125
ObUnitManager::ObUnitManager(ObServerManager &server_mgr, ObZoneManager &zone_mgr)
126
: inited_(false), loaded_(false), proxy_(NULL), server_config_(NULL),
127
    srv_rpc_proxy_(NULL), server_mgr_(server_mgr),
128
    zone_mgr_(zone_mgr), ut_operator_(), id_config_map_(),
129
    name_config_map_(), config_ref_count_map_(), config_pools_map_(),
130
    config_pools_allocator_(OB_MALLOC_NORMAL_BLOCK_SIZE, ObMalloc(ObModIds::OB_RS_UNIT_MANAGER)),
131
    config_allocator_(OB_MALLOC_NORMAL_BLOCK_SIZE, ObMalloc(ObModIds::OB_RS_UNIT_MANAGER)),
132
    id_pool_map_(), name_pool_map_(),
133
    pool_allocator_(OB_MALLOC_NORMAL_BLOCK_SIZE, ObMalloc(ObModIds::OB_RS_UNIT_MANAGER)),
134
    pool_unit_map_(),
135
    pool_unit_allocator_(OB_MALLOC_NORMAL_BLOCK_SIZE, ObMalloc(ObModIds::OB_RS_UNIT_MANAGER)),
136
    id_unit_map_(),
137
    allocator_(OB_MALLOC_NORMAL_BLOCK_SIZE, ObMalloc(ObModIds::OB_RS_UNIT_MANAGER)),
138
    server_loads_(),
139
    load_allocator_(OB_MALLOC_NORMAL_BLOCK_SIZE, ObMalloc(ObModIds::OB_RS_UNIT_MANAGER)),
140
    tenant_pools_map_(),
141
    tenant_pools_allocator_(OB_MALLOC_NORMAL_BLOCK_SIZE, ObMalloc(ObModIds::OB_RS_UNIT_MANAGER)),
142
    server_migrate_units_map_(),
143
    migrate_units_allocator_(OB_MALLOC_NORMAL_BLOCK_SIZE, ObMalloc(ObModIds::OB_RS_UNIT_MANAGER)),
144
    lock_(ObLatchIds::UNIT_MANAGER_LOCK),
145
    schema_service_(NULL), root_balance_(NULL)
146
{
147
}
148

149
ObUnitManager::~ObUnitManager()
150
{
151
  ObHashMap<uint64_t, ObArray<share::ObResourcePool *> *>::iterator iter1;
152
  for (iter1 = config_pools_map_.begin(); iter1 != config_pools_map_.end(); ++iter1) {
153
    ObArray<share::ObResourcePool *> *ptr = iter1->second;
154
    if (NULL != ptr) {
155
      ptr->reset();
156
      ptr = NULL;
157
    }
158
  }
159
  ObHashMap<uint64_t, ObArray<ObUnit *> *>::iterator iter2;
160
  for (iter2 = pool_unit_map_.begin(); iter2 != pool_unit_map_.end(); ++iter2) {
161
    ObArray<share::ObUnit *> *ptr = iter2->second;
162
    if (NULL != ptr) {
163
      ptr->reset();
164
      ptr = NULL;
165
    }
166
  }
167
  ObHashMap<ObAddr, ObArray<ObUnitLoad> *>::iterator iter3;
168
  for (iter3 = server_loads_.begin(); iter3 != server_loads_.end(); ++iter3) {
169
    ObArray<ObUnitLoad> *ptr = iter3->second;
170
    if (NULL != ptr) {
171
      ptr->reset();
172
      ptr = NULL;
173
    }
174
  }
175
  TenantPoolsMap::iterator iter4;
176
  for (iter4 = tenant_pools_map_.begin(); iter4 != tenant_pools_map_.end(); ++iter4) {
177
    common::ObArray<share::ObResourcePool *> *ptr = iter4->second;
178
    if (NULL != ptr) {
179
      ptr->reset();
180
      ptr = NULL;
181
    }
182
  }
183
  ObHashMap<ObAddr, ObArray<uint64_t> *>::iterator iter5;
184
  for (iter5 = server_migrate_units_map_.begin();
185
       iter5 != server_migrate_units_map_.end();
186
       ++iter5) {
187
    common::ObArray<uint64_t> *ptr = iter5->second;
188
    if (NULL != ptr) {
189
      ptr->reset();
190
      ptr = NULL;
191
    }
192
  }
193
}
194

195
int ObUnitManager::init(ObMySQLProxy &proxy,
196
                        ObServerConfig &server_config,
197
                        obrpc::ObSrvRpcProxy &srv_rpc_proxy,
198
                        share::schema::ObMultiVersionSchemaService &schema_service,
199
                        ObRootBalancer &root_balance,
200
                        ObRootService &root_service)
201
{
202
  int ret = OB_SUCCESS;
203
  if (inited_) {
204
    ret = OB_INIT_TWICE;
205
    LOG_WARN("init twice", K(ret));
206
  } else if (OB_FAIL(ut_operator_.init(proxy))) {
207
    LOG_WARN("init unit table operator failed", K(ret));
208
  } else if (OB_FAIL(pool_unit_map_.create(
209
              POOL_MAP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_POOL_UNIT_MAP))) {
210
    LOG_WARN("pool_unit_map_ create failed", LITERAL_K(POOL_MAP_BUCKET_NUM), K(ret));
211
  } else if (OB_FAIL(id_unit_map_.create(
212
              UNIT_MAP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_ID_UNIT_MAP))) {
213
    LOG_WARN("id_unit_map_ create failed",
214
             LITERAL_K(UNIT_MAP_BUCKET_NUM), K(ret));
215
  } else if (OB_FAIL(id_config_map_.create(
216
              CONFIG_MAP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_ID_CONFIG_MAP))) {
217
    LOG_WARN("id_config_map_ create failed",
218
             LITERAL_K(CONFIG_MAP_BUCKET_NUM), K(ret));
219
  } else if (OB_FAIL(name_config_map_.create(
220
              CONFIG_MAP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_NAME_CONFIG_MAP))) {
221
    LOG_WARN("name_config_map_ create failed",
222
             LITERAL_K(CONFIG_MAP_BUCKET_NUM), K(ret));
223
  } else if (OB_FAIL(config_ref_count_map_.create(
224
              CONFIG_REF_COUNT_MAP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_CONFIG_REF_COUNT_MAP))) {
225
    LOG_WARN("config_ref_count_map_ create failed",
226
             LITERAL_K(CONFIG_REF_COUNT_MAP_BUCKET_NUM), K(ret));
227
  } else if (OB_FAIL(config_pools_map_.create(CONFIG_POOLS_MAP_BUCKET_NUM,
228
                                              ObModIds::OB_HASH_BUCKET_CONFIG_POOLS_MAP))) {
229
    LOG_WARN("create config_pools_map failed",
230
             LITERAL_K(CONFIG_POOLS_MAP_BUCKET_NUM), K(ret));
231
  } else if (OB_FAIL(id_pool_map_.create(
232
              POOL_MAP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_ID_POOL_MAP))) {
233
    LOG_WARN("id_pool_map_ create failed",
234
             LITERAL_K(POOL_MAP_BUCKET_NUM), K(ret));
235
  } else if (OB_FAIL(name_pool_map_.create(
236
              POOL_MAP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_NAME_POOL_MAP))) {
237
    LOG_WARN("name_pool_map_ create failed",
238
             LITERAL_K(POOL_MAP_BUCKET_NUM), K(ret));
239
  } else if (OB_FAIL(server_loads_.create(
240
              UNITLOAD_MAP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_SERVER_UNITLOAD_MAP))) {
241
    LOG_WARN("server_loads_ create failed",
242
             LITERAL_K(UNITLOAD_MAP_BUCKET_NUM), K(ret));
243
  } else if (OB_FAIL(tenant_pools_map_.create(
244
              TENANT_POOLS_MAP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_TENANT_POOLS_MAP))) {
245
    LOG_WARN("tenant_pools_map_ create failed",
246
             LITERAL_K(TENANT_POOLS_MAP_BUCKET_NUM), K(ret));
247
  } else if (OB_FAIL(server_migrate_units_map_.create(
248
              SERVER_MIGRATE_UNITS_MAP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_SERVER_MIGRATE_UNIT_MAP))) {
249
    LOG_WARN("server_migrate_units_map_ create failed",
250
             LITERAL_K(SERVER_MIGRATE_UNITS_MAP_BUCKET_NUM), K(ret));
251
  } else {
252
    proxy_ = &proxy;
253
    server_config_ = &server_config;
254
    srv_rpc_proxy_ = &srv_rpc_proxy;
255
    root_service_ = &root_service;
256
    schema_service_ = &schema_service;
257
    root_balance_ = &root_balance;
258
    loaded_ = false;
259
    inited_ = true;
260
  }
261
  return ret;
262
}
263

264
int ObUnitManager::load()
265
{
266
  DEBUG_SYNC(BEFORE_RELOAD_UNIT);
267
  int ret = OB_SUCCESS;
268
  SpinWLockGuard guard(lock_);
269
  if (!inited_) {
270
    ret = OB_NOT_INIT;
271
    LOG_WARN("not init", K(ret));
272
  } else {
273
    loaded_ = false;
274
    ObHashMap<uint64_t, ObArray<share::ObResourcePool *> *>::iterator iter1;
275
    for (iter1 = config_pools_map_.begin(); iter1 != config_pools_map_.end(); ++iter1) {
276
      ObArray<share::ObResourcePool *> *ptr = iter1->second;
277
      if (NULL != ptr) {
278
        ptr->reset();
279
        ptr = NULL;
280
      }
281
    }
282
    ObHashMap<uint64_t, ObArray<ObUnit *> *>::iterator iter2;
283
    for (iter2 = pool_unit_map_.begin(); iter2 != pool_unit_map_.end(); ++iter2) {
284
      ObArray<share::ObUnit *> *ptr = iter2->second;
285
      if (NULL != ptr) {
286
        ptr->reset();
287
        ptr = NULL;
288
      }
289
    }
290
    ObHashMap<ObAddr, ObArray<ObUnitLoad> *>::iterator iter3;
291
    for (iter3 = server_loads_.begin(); iter3 != server_loads_.end(); ++iter3) {
292
      ObArray<ObUnitLoad> *ptr = iter3->second;
293
      if (NULL != ptr) {
294
        ptr->reset();
295
        ptr = NULL;
296
      }
297
    }
298
    TenantPoolsMap::iterator iter4;
299
    for (iter4 = tenant_pools_map_.begin(); iter4 != tenant_pools_map_.end(); ++iter4) {
300
      common::ObArray<share::ObResourcePool *> *ptr = iter4->second;
301
      if (NULL != ptr) {
302
        ptr->reset();
303
        ptr = NULL;
304
      }
305
    }
306
    ObHashMap<ObAddr, ObArray<uint64_t> *>::iterator iter5;
307
    for (iter5 = server_migrate_units_map_.begin();
308
         iter5 != server_migrate_units_map_.end();
309
         ++iter5) {
310
      common::ObArray<uint64_t> *ptr = iter5->second;
311
      if (NULL != ptr) {
312
        ptr->reset();
313
        ptr = NULL;
314
      }
315
    }
316
    if (OB_FAIL(id_config_map_.clear())) {
317
      LOG_WARN("id_config_map_ clear failed", K(ret));
318
    } else if (OB_FAIL(name_config_map_.clear())) {
319
      LOG_WARN("name_pool_map_  clear failed", K(ret));
320
    } else if (OB_FAIL(config_ref_count_map_.clear())) {
321
      LOG_WARN("config_ref_count_map_ clear failed", K(ret));
322
    } else if (OB_FAIL(config_pools_map_.clear())) {
323
      LOG_WARN("config_pools_map_ clear failed", K(ret));
324
    } else if (OB_FAIL(id_pool_map_.clear())) {
325
      LOG_WARN("id_pool_map_ clear failed", K(ret));
326
    } else if (OB_FAIL(name_pool_map_.clear())) {
327
      LOG_WARN("name_pool_map_ clear failed", K(ret));
328
    } else if (OB_FAIL(pool_unit_map_.clear())) {
329
      LOG_WARN("pool_unit_map_ clear failed", K(ret));
330
    } else if (OB_FAIL(id_unit_map_.clear())) {
331
      LOG_WARN("id_unit_map_ clear failed", K(ret));
332
    } else if (OB_FAIL(server_loads_.clear())) {
333
      LOG_WARN("server_loads_ clear failed", K(ret));
334
    } else if (OB_FAIL(tenant_pools_map_.clear())) {
335
      LOG_WARN("tenant_pools_map_ clear failed", K(ret));
336
    } else if (OB_FAIL(server_migrate_units_map_.clear())) {
337
      LOG_WARN("server_migrate_units_map_ clear failed", K(ret));
338
    }
339

340
    // free all memory
341
    if (OB_SUCC(ret)) {
342
      config_allocator_.reset();
343
      config_pools_allocator_.reset();
344
      pool_allocator_.reset();
345
      pool_unit_allocator_.reset();
346
      allocator_.reset();
347
      load_allocator_.reset();
348
      tenant_pools_allocator_.reset();
349
      migrate_units_allocator_.reset();
350
    }
351

352
    // load unit config
353
    ObArray<ObUnitConfig> configs;
354
    if (OB_SUCC(ret)) {
355
      if (OB_FAIL(ut_operator_.get_unit_configs(configs))) {
356
        LOG_WARN("get_unit_configs failed", K(ret));
357
      } else if (OB_FAIL(build_config_map(configs))) {
358
        LOG_WARN("build_config_map failed", K(ret));
359
      }
360
    }
361

362
    // load resource pool
363
    ObArray<share::ObResourcePool> pools;
364
    if (OB_SUCC(ret)) {
365
      if (OB_FAIL(ut_operator_.get_resource_pools(pools))) {
366
        LOG_WARN("get_resource_pools failed", K(ret));
367
      } else if (OB_FAIL(build_pool_map(pools))) {
368
        LOG_WARN("build_pool_map failed", K(ret));
369
      }
370
    }
371

372
    // load unit
373
    ObArray<ObUnit> units;
374
    if (OB_SUCC(ret)) {
375
      if (OB_FAIL(ut_operator_.get_units(units))) {
376
        LOG_WARN("get_units failed", K(ret));
377
      } else if (OB_FAIL(build_unit_map(units))) {
378
        LOG_WARN("build_unit_map failed", K(ret));
379
      }
380
    }
381

382
    // build tenant pools
383
    if (OB_SUCC(ret)) {
384
      for (ObHashMap<uint64_t, share::ObResourcePool *>::iterator it = id_pool_map_.begin();
385
           OB_SUCCESS == ret && it != id_pool_map_.end(); ++it) {
386
        // pool not grant to tenant don't add to tenant_pools_map
387
        if (NULL == it->second) {
388
          ret = OB_ERR_UNEXPECTED;
389
          LOG_WARN("it->second is null", KP(it->second), K(ret));
390
        } else if (it->second->is_granted_to_tenant()) {
391
          if (OB_FAIL(insert_tenant_pool(it->second->tenant_id_, it->second))) {
392
            LOG_WARN("insert_tenant_pool failed", "tenant_id", it->second->tenant_id_,
393
                     "pool", *(it->second), K(ret));
394
          }
395
        }
396
      }
397
    }
398

399
    // build server migrate units
400
    if (OB_SUCC(ret)) {
401
      FOREACH_CNT_X(unit, units, OB_SUCCESS == ret) {
402
        if (unit->migrate_from_server_.is_valid()) {
403
          if (OB_FAIL(insert_migrate_unit(unit->migrate_from_server_, unit->unit_id_))) {
404
            LOG_WARN("insert_migrate_unit failed", "server", unit->migrate_from_server_,
405
                     "unit_id", unit->unit_id_, K(ret));
406
          }
407
        }
408
      }
409
    }
410

411
    if (OB_SUCC(ret)) {
412
      loaded_ = true;
413
    }
414
  }
415
  LOG_INFO("unit manager load finish", K(ret));
416
  return ret;
417
}
418

419
int ObUnitManager::create_unit_config(const ObUnitConfig &unit_config, const bool if_not_exist)
420
{
421
  int ret = OB_SUCCESS;
422
  SpinWLockGuard guard(lock_);
423
  if (OB_FAIL(inner_create_unit_config_(unit_config, if_not_exist))) {
424
    LOG_WARN("fail to create unit config", KR(ret), K(unit_config), K(if_not_exist));
425
  }
426
  return ret;
427
}
428

429
int ObUnitManager::inner_create_unit_config_(const ObUnitConfig &unit_config, const bool if_not_exist)
430
{
431
  int ret = OB_SUCCESS;
432
  ObUnitConfig *temp_config = NULL;
433
  ObUnitConfig *new_config = NULL;
434
  uint64_t unit_config_id = unit_config.unit_config_id();
435
  const ObUnitConfigName &name = unit_config.name();
436
  const ObUnitResource &rpc_ur = unit_config.unit_resource();
437
  ObUnitResource ur = rpc_ur;
438

439
  LOG_INFO("start create unit config", K(name), K(rpc_ur), K(if_not_exist));
440

441
  if (!check_inner_stat()) {
442
    ret = OB_INNER_STAT_ERROR;
443
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
444
  } else if (name.is_empty()) {
445
    ret = OB_MISS_ARGUMENT;
446
    LOG_WARN("miss 'name' argument", KR(ret), K(name));
447
    LOG_USER_ERROR(OB_MISS_ARGUMENT, "resource unit name");
448
  } else if (OB_FAIL(ur.init_and_check_valid_for_unit(rpc_ur))) {
449
    LOG_WARN("init from user specified unit resource and check valid fail", KR(ret), K(rpc_ur));
450
  } else if (OB_SUCCESS == (ret = get_unit_config_by_name(name, temp_config))) {
451
    if (NULL == temp_config) {
452
      ret = OB_ERR_UNEXPECTED;
453
      LOG_WARN("temp_config is null", KP(temp_config), K(ret));
454
    } else if (if_not_exist) {
455
      LOG_USER_NOTE(OB_RESOURCE_UNIT_EXIST, to_cstring(name));
456
      LOG_INFO("unit config already exist", K(name));
457
    } else {
458
      ret = OB_RESOURCE_UNIT_EXIST;
459
      LOG_USER_ERROR(OB_RESOURCE_UNIT_EXIST, to_cstring(name));
460
      LOG_WARN("unit config already exist", K(name), KR(ret));
461
    }
462
  } else if (OB_ENTRY_NOT_EXIST != ret) {
463
    LOG_WARN("get_unit_config_by_name failed", "config_name", name, KR(ret));
464
  }
465
  // allocate new unit config id
466
  else if (OB_INVALID_ID == unit_config_id && OB_FAIL(fetch_new_unit_config_id(unit_config_id))) {
467
    LOG_WARN("fetch_new_unit_config_id failed", KR(ret), K(unit_config));
468
  } else {
469
    if (OB_ISNULL(new_config = config_allocator_.alloc())) {
470
      ret = OB_ALLOCATE_MEMORY_FAILED;
471
      LOG_ERROR("alloc memory failed", KR(ret));
472
    } else if (OB_FAIL(new_config->init(unit_config_id, name, ur))) {
473
      LOG_WARN("init unit config fail", KR(ret), K(unit_config_id), K(name), K(ur));
474
    } else if (OB_FAIL(ut_operator_.update_unit_config(*proxy_, *new_config))) {
475
      LOG_WARN("update_unit_config failed", "unit config", *new_config, K(ret));
476
    } else if (OB_FAIL(insert_unit_config(new_config))) {
477
      LOG_WARN("insert_unit_config failed", "unit config", *new_config,  K(ret));
478
    }
479

480
    // avoid memory leak
481
    if (OB_FAIL(ret) && NULL != new_config) {
482
      config_allocator_.free(new_config);
483
      new_config = NULL;
484
    }
485
  }
486
  LOG_INFO("finish create unit config", KR(ret), K(name), K(rpc_ur), K(if_not_exist), KPC(new_config));
487
  return ret;
488
}
489

490
int ObUnitManager::alter_unit_config(const ObUnitConfig &unit_config)
491
{
492
  int ret = OB_SUCCESS;
493
  ObUnitConfig *old_config = NULL;
494
  ObArray<share::ObResourcePool *> *pools = NULL;
495
  SpinWLockGuard guard(lock_);
496
  const ObUnitConfigName &name = unit_config.name();
497
  const ObUnitResource &rpc_ur = unit_config.unit_resource();
498

499
  if (!check_inner_stat()) {
500
    ret = OB_INNER_STAT_ERROR;
501
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
502
  } else if (name.is_empty()) {
503
    ret = OB_MISS_ARGUMENT;
504
    LOG_WARN("miss 'name' argument", KR(ret));
505
    LOG_USER_ERROR(OB_MISS_ARGUMENT, "resource unit name");
506
  } else if (OB_FAIL(get_unit_config_by_name(name, old_config))) {
507
    if (OB_ENTRY_NOT_EXIST != ret) {
508
      LOG_WARN("get_unit_config_by_name failed", K(name), K(ret));
509
    } else {
510
      // overwrite ret on purpose
511
      ret = OB_RESOURCE_UNIT_NOT_EXIST;
512
      LOG_WARN("config does not exist", K(name), KR(ret));
513
      LOG_USER_ERROR(OB_RESOURCE_UNIT_NOT_EXIST, to_cstring(name));
514
    }
515
  } else if (OB_ISNULL(old_config)) {
516
    ret = OB_ERR_UNEXPECTED;
517
    LOG_WARN("old_config is null", KP(old_config), KR(ret));
518
  } else {
519
    // copy unit resource
520
    ObUnitResource new_ur = old_config->unit_resource();
521
    const ObUnitResource old_ur = old_config->unit_resource();
522

523
    // update based on user specified
524
    if (OB_FAIL(new_ur.update_and_check_valid_for_unit(rpc_ur))) {
525
      LOG_WARN("update and check valid for unit fail", KR(ret), K(new_ur), K(rpc_ur));
526
    } else {
527
      if (OB_FAIL(get_pools_by_config(old_config->unit_config_id(), pools))) {
528
        if (OB_ENTRY_NOT_EXIST != ret) {
529
          LOG_WARN("get_pools_by_config failed", "config_id", old_config->unit_config_id(), KR(ret));
530
        } else {
531
          ret = OB_SUCCESS;
532
          // this unit config is not used by any resource pools
533
          // update the config directly
534
        }
535
      } else if (NULL == pools) {
536
        ret = OB_ERR_UNEXPECTED;
537
        LOG_WARN("pools is null", KP(pools), K(ret));
538
      } else {
539
        if (OB_FAIL(check_expand_resource_("ALTER_RESOURCE_UNIT", *pools, old_ur, new_ur))) {
540
          LOG_WARN("check expand config failed", K(old_ur), K(new_ur), KR(ret));
541
        } else if (OB_FAIL(check_shrink_resource_(*pools, old_ur, new_ur))) {
542
          LOG_WARN("check shrink config failed", K(old_ur), K(new_ur), KR(ret));
543
        } else if (OB_FAIL(check_full_resource_pool_memory_condition(*pools, new_ur.memory_size()))) {
544
          LOG_WARN("fail to check full resource pool memory condition", K(ret), K(new_ur), KPC(pools));
545
        }
546
      }
547
    }
548

549
    if (OB_SUCC(ret)) {
550
      ObUnitConfig new_config;
551
      if (OB_FAIL(new_config.init(old_config->unit_config_id(), old_config->name(), new_ur))) {
552
        LOG_WARN("init new unit config fail", KR(ret), KPC(old_config), K(new_ur));
553
      } else if (OB_FAIL(ut_operator_.update_unit_config(*proxy_, new_config))) {
554
        LOG_WARN("update_unit_config failed", K(new_config), KR(ret));
555
      } else if (OB_FAIL(old_config->update_unit_resource(new_ur))) {
556
        LOG_WARN("update unit resource of unit config fail", KR(ret), KPC(old_config), K(new_ur));
557
      }
558
    }
559

560
    if (OB_SUCC(ret)) {
561
      ROOTSERVICE_EVENT_ADD("unit", "alter_resource_unit",
562
                            "name", old_config->name(),
563
                            "unit_config_id", old_config->unit_config_id(),
564
                            "old_resource", old_ur,
565
                            "new_resource", new_ur);
566
    }
567
  }
568
  return ret;
569
}
570

571
int ObUnitManager::check_unit_config_exist(const share::ObUnitConfigName &unit_config_name,
572
                                           bool &is_exist)
573
{
574
  int ret = OB_SUCCESS;
575
  SpinRLockGuard guard(lock_);
576
  ObUnitConfig *config = NULL;
577
  is_exist = false;
578

579
  if (!check_inner_stat()) {
580
    ret = OB_INNER_STAT_ERROR;
581
    LOG_WARN("check_inner_stat failed", KR(ret), K(inited_), K(loaded_));
582
  } else if (unit_config_name.is_empty()) {
583
    ret = OB_INVALID_ARGUMENT;
584
    LOG_WARN("invalid argument", KR(ret), K(unit_config_name));
585
  } else if (OB_FAIL(get_unit_config_by_name(unit_config_name, config))) {
586
    if (OB_ENTRY_NOT_EXIST != ret) {
587
      LOG_WARN("get_unit_config_by_name failed", KR(ret), K(unit_config_name));
588
    } else {
589
      ret = OB_SUCCESS;
590
      // is_exist = false
591
    }
592
  } else if (OB_ISNULL(config)) {
593
    ret = OB_ERR_UNEXPECTED;
594
    LOG_WARN("unexpected null", KR(ret), KP(config), K(unit_config_name));
595
  } else {
596
    is_exist = true;
597
  }
598

599
  return ret;
600
}
601

602
int ObUnitManager::drop_unit_config(const ObUnitConfigName &name, const bool if_exist)
603
{
604
  int ret = OB_SUCCESS;
605
  LOG_INFO("start drop unit config", K(name));
606
  SpinWLockGuard guard(lock_);
607
  ObUnitConfig *config = NULL;
608
  int64_t ref_count = 0;
609
  if (!check_inner_stat()) {
610
    ret = OB_INNER_STAT_ERROR;
611
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
612
  } else if (name.is_empty()) {
613
    ret = OB_INVALID_ARGUMENT;
614
    LOG_USER_ERROR(OB_INVALID_ARGUMENT, "resource unit name");
615
    LOG_WARN("invalid argument", K(name), K(ret));
616
  } else if (OB_FAIL(get_unit_config_by_name(name, config))) {
617
    if (OB_ENTRY_NOT_EXIST != ret) {
618
      LOG_WARN("get_unit_config_by_name failed", K(name), K(ret));
619
    } else {
620
      if (if_exist) {
621
        ret = OB_SUCCESS;
622
        LOG_USER_NOTE(OB_RESOURCE_UNIT_NOT_EXIST, to_cstring(name));
623
        LOG_INFO("unit config not exist, no need to delete it", K(name));
624
      } else {
625
        ret = OB_RESOURCE_UNIT_NOT_EXIST;
626
        LOG_USER_ERROR(OB_RESOURCE_UNIT_NOT_EXIST, to_cstring(name));
627
        LOG_WARN("unit config not exist", K(name), K(ret));
628
      }
629
    }
630
  } else if (NULL == config) {
631
    ret = OB_ERR_UNEXPECTED;
632
    LOG_WARN("config is null", KP(config), K(ret));
633
  } else if (OB_FAIL(get_config_ref_count(config->unit_config_id(), ref_count))) {
634
    LOG_WARN("get_config_ref_count failed", "config_id", config->unit_config_id(), K(ret));
635
  } else if (0 != ref_count) {
636
    ret = OB_RESOURCE_UNIT_IS_REFERENCED;
637
    LOG_USER_ERROR(OB_RESOURCE_UNIT_IS_REFERENCED, to_cstring(name));
638
    LOG_WARN("some resource pool is using this unit config, can not delete it",
639
             K(ref_count), K(ret));
640
  } else if (OB_FAIL(ut_operator_.remove_unit_config(*proxy_, config->unit_config_id()))) {
641
    LOG_WARN("remove_unit_config failed", "config_id", config->unit_config_id(), K(ret));
642
  } else if (OB_FAIL(delete_unit_config(config->unit_config_id(), config->name()))) {
643
    LOG_WARN("delete_unit_config failed", "config id", config->unit_config_id(),
644
             "name", config->name(), K(ret));
645
  } else {
646
    ROOTSERVICE_EVENT_ADD("unit", "drop_resource_unit",
647
                          "name", config->name());
648
    // free memory
649
    config_allocator_.free(config);
650
    config = NULL;
651
  }
652
  LOG_INFO("finish drop unit config", K(name), K(ret));
653
  return ret;
654
}
655

656
int ObUnitManager::check_tenant_pools_in_shrinking(
657
    const uint64_t tenant_id,
658
    bool &is_shrinking)
659
{
660
  int ret = OB_SUCCESS;
661
  common::ObArray<share::ObResourcePool *> *pools = NULL;
662
  SpinRLockGuard guard(lock_);
663
  if (!check_inner_stat()) {
664
    ret = OB_INNER_STAT_ERROR;
665
    LOG_WARN("check inner stat failed", K(ret), K(inited_), K(loaded_));
666
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
667
    ret = OB_INVALID_ARGUMENT;
668
    LOG_WARN("invalid argument", K(ret));
669
  } else if (OB_FAIL(get_pools_by_tenant_(tenant_id, pools))) {
670
    LOG_WARN("fail to get pools by tenant", K(ret), K(tenant_id));
671
  } else if (OB_UNLIKELY(NULL == pools)) {
672
    ret = OB_ERR_UNEXPECTED;
673
    LOG_WARN("pools ptr is null", K(ret), KP(pools));
674
  } else {
675
    is_shrinking = false;
676
    for (int64_t i = 0; !is_shrinking && OB_SUCC(ret) && i < pools->count(); ++i) {
677
      common::ObArray<share::ObUnit *> *units = NULL;
678
      const share::ObResourcePool *pool = pools->at(i);
679
      if (NULL == pool) {
680
        ret = OB_ERR_UNEXPECTED;
681
        LOG_WARN("pool ptr is null", K(ret), KP(pool));
682
      } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
683
        LOG_WARN("fail to get units by pool", K(ret), "pool_id", pool->resource_pool_id_);
684
      } else if (NULL == units) {
685
        ret = OB_ERR_UNEXPECTED;
686
        LOG_WARN("units ptrs is null", K(ret), KP(units));
687
      } else {
688
        is_shrinking = false;
689
        for (int64_t j = 0; !is_shrinking && OB_SUCC(ret) && j < units->count(); ++j) {
690
          const ObUnit *unit = units->at(j);
691
          if (OB_UNLIKELY(NULL == unit)) {
692
            ret = OB_ERR_UNEXPECTED;
693
            LOG_WARN("unit ptr si null", K(ret));
694
          } else if (ObUnit::UNIT_STATUS_DELETING == unit->status_) {
695
            is_shrinking = true;
696
          } else if (ObUnit::UNIT_STATUS_ACTIVE == unit->status_) {
697
            // a normal unit, go on and check next
698
          } else {
699
            ret = OB_ERR_UNEXPECTED;
700
            LOG_WARN("unit status unexpected", K(ret), "unit_status", unit->status_);
701
          }
702
        }
703
      }
704
    }
705
  }
706
  return ret;
707
}
708

709
int ObUnitManager::check_pool_in_shrinking(
710
    const uint64_t pool_id,
711
    bool &is_shrinking)
712
{
713
  int ret = OB_SUCCESS;
714
  SpinRLockGuard guard(lock_);
715
  if (OB_FAIL(inner_check_pool_in_shrinking_(pool_id, is_shrinking))) {
716
    LOG_WARN("inner check pool in shrinking failed", KR(ret), K(pool_id), K(is_shrinking));
717
  }
718
  return ret;
719
}
720

721
int ObUnitManager::inner_check_pool_in_shrinking_(
722
    const uint64_t pool_id,
723
    bool &is_shrinking)
724
{
725
  int ret = OB_SUCCESS;
726
  if (!check_inner_stat()) {
727
    ret = OB_INNER_STAT_ERROR;
728
    LOG_WARN("check inner stat failed", K(ret), K(inited_), K(loaded_));
729
  } else if (OB_UNLIKELY(OB_INVALID_ID == pool_id)) {
730
    ret = OB_INVALID_ARGUMENT;
731
    LOG_WARN("invalid argument", K(ret), K(pool_id));
732
  } else {
733
    common::ObArray<share::ObUnit *> *units = NULL;
734
    if (OB_FAIL(get_units_by_pool(pool_id, units))) {
735
      LOG_WARN("fail to get units by pool", K(ret), K(pool_id));
736
    } else if (NULL == units) {
737
      ret = OB_ERR_UNEXPECTED;
738
      LOG_WARN("units ptr is null", K(ret), KP(units));
739
    } else {
740
      is_shrinking = false;
741
      for (int64_t i = 0; !is_shrinking && OB_SUCC(ret) && i < units->count(); ++i) {
742
        const ObUnit *unit = units->at(i);
743
        if (OB_UNLIKELY(NULL == unit)) {
744
          ret = OB_ERR_UNEXPECTED;
745
          LOG_WARN("unit ptr is null", K(ret));
746
        } else if (ObUnit::UNIT_STATUS_DELETING == unit->status_) {
747
          is_shrinking = true;
748
        } else if (ObUnit::UNIT_STATUS_ACTIVE == unit->status_) {
749
          // a normal unit, go on and check next
750
        } else {
751
          ret = OB_ERR_UNEXPECTED;
752
          LOG_WARN("unit status unexpected", K(ret), "unit_status", unit->status_);
753
        }
754
      }
755
    }
756
  }
757
  return ret;
758
}
759

760
int ObUnitManager::construct_resource_pool_to_clone_(
761
    const uint64_t source_tenant_id,
762
    share::ObResourcePool &pool_to_clone)
763
{
764
  int ret = OB_SUCCESS;
765
  int64_t unit_num = 0;
766
  share::schema::ObSchemaGetterGuard schema_guard;
767
  const share::schema::ObTenantSchema *tenant_schema = NULL;
768
  common::ObSEArray<common::ObZone, DEFAULT_ZONE_COUNT> zone_list;
769
  if (OB_UNLIKELY(OB_INVALID_TENANT_ID == source_tenant_id)) {
770
    ret = OB_INVALID_ARGUMENT;
771
    LOG_WARN("invalid argument", KR(ret), K(source_tenant_id));
772
  } else if (OB_FAIL(construct_source_tenant_unit_num_(source_tenant_id, unit_num))) {
773
    LOG_WARN("fail to construct source tenant unit_num", KR(ret), K(source_tenant_id), K(unit_num));
774
  } else if (OB_ISNULL(schema_service_)) {
775
    ret = OB_NOT_INIT;
776
    LOG_WARN("schema service is null", K(schema_service_), KR(ret));
777
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
778
    LOG_WARN("fail to get tenant schema guard", KR(ret));
779
  } else if (OB_FAIL(schema_guard.get_tenant_info(source_tenant_id, tenant_schema))) {
780
    LOG_WARN("fail to get tenant info", KR(ret), K(source_tenant_id));
781
  } else if (OB_ISNULL(tenant_schema)) {
782
    ret = OB_ERR_UNEXPECTED;
783
    LOG_WARN("tenant schema is null", KR(ret), KP(tenant_schema));
784
  // just get units in locality is ok
785
  } else if (OB_FAIL(tenant_schema->get_zone_list(zone_list))) {
786
    LOG_WARN("fail to get zone list", KR(ret));
787
  } else if (0 >= zone_list.count()) {
788
    ret = OB_INVALID_ARGUMENT;
789
    LOG_WARN("source tenant zone list should have at least zone zone", KR(ret), K(source_tenant_id), K(zone_list));
790
  } else if (OB_FAIL(pool_to_clone.zone_list_.assign(zone_list))) {
791
    LOG_WARN("fail to assign zone_list", KR(ret), K(source_tenant_id), K(pool_to_clone), K(zone_list));
792
  } else {
793
    // pool_to_clone.name_ is already setted before
794
    pool_to_clone.unit_count_ = unit_num;
795
    pool_to_clone.replica_type_ = REPLICA_TYPE_FULL;
796
    // parameters below will be setted later:
797
    //    pool_to_clone.resource_pool_id_ will be setted when inner_create_resource_pool
798
    //    pool_to_clone.unit_config_id_ will be setted when inner_create_resource_pool
799
    //    pool_to_clone.tenant_id_ will be setted when granted this pool to tenant
800
  }
801
  return ret;
802
}
803

804
int ObUnitManager::construct_source_tenant_unit_num_(
805
    const uint64_t source_tenant_id,
806
    int64_t &unit_num)
807
{
808
  int ret = OB_SUCCESS;
809
  unit_num = 0;
810
  ObArray<uint64_t> pool_ids;
811
  share::ObResourcePool *pool;
812
  if (OB_UNLIKELY(OB_INVALID_TENANT_ID == source_tenant_id)) {
813
    ret = OB_INVALID_ARGUMENT;
814
    LOG_WARN("invalid argument", KR(ret), K(source_tenant_id));
815
  } else if (OB_FAIL(inner_get_pool_ids_of_tenant(source_tenant_id, pool_ids))) {
816
    LOG_WARN("fail to get resource pool ids of source tenant", KR(ret), K(source_tenant_id));
817
  } else if (OB_UNLIKELY(0 >= pool_ids.count())) {
818
    ret = OB_ERR_UNEXPECTED;
819
    LOG_WARN("source tenant should have at least one pool", KR(ret), K(pool_ids));
820
  } else if (OB_FAIL(get_resource_pool_by_id(pool_ids.at(0), pool))) {
821
    LOG_WARN("fail to get first resource pool", KR(ret), K(pool_ids));
822
  } else if (OB_ISNULL(pool)) {
823
    ret = OB_ERR_UNEXPECTED;
824
    LOG_WARN("pool is null", KR(ret), KP(pool));
825
  } else if (OB_UNLIKELY(0 >= pool->unit_count_)) {
826
    ret = OB_ERR_UNEXPECTED;
827
    LOG_WARN("source tenant should have at least one unit", KR(ret), K(source_tenant_id), KPC(pool));
828
  } else {
829
    // tenant's resource pools should have same unit_num
830
    // so it is ok for us to pick unit_num of first pool
831
    unit_num = pool->unit_count_;
832
  }
833
  return ret;
834
}
835

836
int ObUnitManager::clone_resource_pool(
837
    share::ObResourcePool &resource_pool,
838
    const share::ObUnitConfigName &unit_config_name,
839
    const uint64_t source_tenant_id)
840
{
841
  int ret = OB_SUCCESS;
842
  LOG_INFO("start clone_resource_pool", K(resource_pool), K(unit_config_name), K(source_tenant_id));
843
  SpinWLockGuard guard(lock_);
844
  bool if_not_exist = false;
845
  common::ObArray<share::ObUnitInfo> source_units;
846
  if (OB_UNLIKELY(OB_INVALID_TENANT_ID == source_tenant_id)) {
847
    ret = OB_INVALID_ARGUMENT;
848
    LOG_WARN("invalid argument", KR(ret), K(resource_pool), K(unit_config_name), K(source_tenant_id));
849
  } else if (OB_FAIL(construct_resource_pool_to_clone_(source_tenant_id, resource_pool))) {
850
    LOG_WARN("fail to construct resource pool to clone", KR(ret), K(source_tenant_id), K(resource_pool));
851
  } else if (OB_FAIL(inner_get_all_unit_infos_by_tenant_(source_tenant_id, source_units))) {
852
    LOG_WARN("fail to get units by source tenant", KR(ret), K(source_tenant_id));
853
  } else if (OB_FAIL(inner_create_resource_pool_(
854
                         resource_pool,
855
                         unit_config_name,
856
                         if_not_exist,
857
                         source_tenant_id,
858
                         source_units))) {
859
    LOG_WARN("fail to inner create resource pool for clone tenant", KR(ret), K(resource_pool),
860
             K(unit_config_name), K(if_not_exist), K(source_tenant_id), K(source_units));
861
  }
862
  LOG_INFO("finish clone_resource_pool", KR(ret), K(resource_pool), K(unit_config_name), K(source_tenant_id));
863
  return ret;
864
}
865

866
int ObUnitManager::create_resource_pool(
867
    share::ObResourcePool &resource_pool,
868
    const ObUnitConfigName &config_name,
869
    const bool if_not_exist)
870
{
871
  int ret = OB_SUCCESS;
872
  SpinWLockGuard guard(lock_);
873
  common::ObArray<share::ObUnitInfo> source_units; // not used
874
  if (OB_FAIL(inner_create_resource_pool_(resource_pool, config_name, if_not_exist, OB_INVALID_TENANT_ID/*source_tenant_id*/, source_units))) {
875
    LOG_WARN("fail to inner create resource pool", KR(ret), K(resource_pool), K(config_name), K(if_not_exist));
876
  }
877
  return ret;
878
}
879

880
int ObUnitManager::inner_create_resource_pool_(
881
    share::ObResourcePool &resource_pool,
882
    const ObUnitConfigName &config_name,
883
    const bool if_not_exist,
884
    const uint64_t source_tenant_id,
885
    const common::ObIArray<share::ObUnitInfo> &source_units)
886
{
887
  int ret = OB_SUCCESS;
888
  LOG_INFO("start inner create resource pool", K(resource_pool), K(config_name),
889
           K(if_not_exist), K(source_tenant_id), K(source_units));
890
  ObUnitConfig *config = NULL;
891
  share::ObResourcePool *pool = NULL;
892
  bool is_bootstrap_pool = (ObUnitConfig::SYS_UNIT_CONFIG_ID == resource_pool.unit_config_id_);
893
  // source_tenant_id is used to recognize whether this resource pool is for clone tenant
894
  // if OB_INVALID_TENANT_ID == source_tenant_id means this is clone resource pool for clone tenant
895
  // if OB_INVALID_TENANT_ID != source_tenant_id means this is create resource pool for normal/restore tenant
896
  // so please DO NOT check whether source_tenant_id is valid or not
897
  bool is_clone_tenant = OB_INVALID_TENANT_ID != source_tenant_id;
898
  const char *module = is_clone_tenant ? "CLONE_RESOURCE_POOL" : "CREATE_RESOURCE_POOL";
899

900
  if (!check_inner_stat()) {
901
    ret = OB_INNER_STAT_ERROR;
902
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
903
  } else if (is_bootstrap_pool && OB_FAIL(check_bootstrap_pool(resource_pool))) {
904
    LOG_WARN("check bootstrap pool failed", K(resource_pool), K(ret));
905
  } else if (!is_bootstrap_pool && OB_FAIL(check_resource_pool(resource_pool, is_clone_tenant))) {
906
    LOG_WARN("check_resource_pool failed", K(resource_pool), K(is_clone_tenant), K(ret));
907
  } else if (config_name.is_empty()) {
908
    ret = OB_INVALID_ARGUMENT;
909
    LOG_USER_ERROR(OB_INVALID_ARGUMENT, "resource unit name");
910
    LOG_WARN("invalid config_name", K(config_name), K(ret));
911
  } else if (OB_FAIL(get_unit_config_by_name(config_name, config))) {
912
    if (OB_ENTRY_NOT_EXIST != ret) {
913
      LOG_WARN("get_unit_config_by_name failed", K(config_name), K(ret));
914
    } else {
915
      ret = OB_RESOURCE_UNIT_NOT_EXIST;
916
      LOG_USER_ERROR(OB_RESOURCE_UNIT_NOT_EXIST, to_cstring(config_name));
917
      LOG_WARN("config not exist", K(config_name), K(ret));
918
    }
919
  } else if (NULL == config) {
920
    ret = OB_ERR_UNEXPECTED;
921
    LOG_WARN("config is null", KP(config), K(ret));
922
  } else if (OB_SUCCESS == (ret = inner_get_resource_pool_by_name(resource_pool.name_, pool))) {
923
    if (NULL == pool) {
924
      ret = OB_ERR_UNEXPECTED;
925
      LOG_WARN("pool is null", K_(resource_pool.name), KP(pool), K(ret));
926
    } else if (if_not_exist) {
927
      LOG_USER_NOTE(OB_RESOURCE_POOL_EXIST, to_cstring(resource_pool.name_));
928
      LOG_INFO("resource_pool already exist, no need to create", K(resource_pool.name_));
929
    } else {
930
      ret = OB_RESOURCE_POOL_EXIST;
931
      LOG_USER_ERROR(OB_RESOURCE_POOL_EXIST, to_cstring(resource_pool.name_));
932
      LOG_WARN("resource_pool already exist", "name", resource_pool.name_, K(ret));
933
    }
934
  } else if (OB_ENTRY_NOT_EXIST != ret) {
935
    LOG_WARN("get resource pool by name failed", "name", resource_pool.name_, K(ret));
936
  } else {
937
    ret = OB_SUCCESS;
938
    common::ObMySQLTransaction trans;
939
    share::ObResourcePool *new_pool = NULL;
940
    const int64_t min_full_resource_pool_memory = GCONF.__min_full_resource_pool_memory;
941
    if (NULL == (new_pool = pool_allocator_.alloc())) {
942
      ret = OB_ALLOCATE_MEMORY_FAILED;
943
      LOG_ERROR("alloc memory failed", K(ret));
944
    } else if (REPLICA_TYPE_FULL == resource_pool.replica_type_
945
        && config->memory_size() < min_full_resource_pool_memory) {
946
      ret = OB_NOT_SUPPORTED;
947
      LOG_WARN("full resource pool min memory illegal", KR(ret), K(config->memory_size()),
948
          K(min_full_resource_pool_memory));
949
      LOG_USER_ERROR(OB_NOT_SUPPORTED, "unit MEMORY_SIZE less than __min_full_resource_pool_memory");
950
    } else {
951
      if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
952
        LOG_WARN("start transaction failed", K(ret));
953
      } else {
954
        if (OB_FAIL(new_pool->assign(resource_pool))) {
955
          LOG_WARN("failed to assign new_pool", K(ret));
956
        } else {
957
          new_pool->unit_config_id_ = config->unit_config_id();
958
          if (OB_INVALID_ID == new_pool->resource_pool_id_) {
959
            if (OB_FAIL(fetch_new_resource_pool_id(new_pool->resource_pool_id_))) {
960
              LOG_WARN("fetch_new_resource_pool_id failed", K(ret));
961
            }
962
          } else {
963
            // sys resource pool with pool_id set
964
          }
965
        }
966
        if (OB_FAIL(ret)) {
967
        } else if (OB_FAIL(ut_operator_.update_resource_pool(trans, *new_pool, false/*need_check_conflict_with_clone*/))) {
968
          LOG_WARN("update_resource_pool failed", "resource_pool", *new_pool, K(ret));
969
        } else if (OB_FAIL(update_pool_map(new_pool))) {
970
          LOG_WARN("update pool map failed", "resource_pool", *new_pool, K(ret));
971
        }
972

973
        if (OB_SUCCESS == ret && !is_bootstrap_resource_pool(new_pool->resource_pool_id_)) {
974
          if (is_clone_tenant) {
975
            if (OB_FAIL(check_new_pool_units_for_clone_tenant_(trans, *new_pool, source_units))) {
976
              LOG_WARN("fail to check new pool units for clone tenant", KR(ret),
977
                       K(source_tenant_id), K(source_units), "resource_pool", *new_pool);
978
            }
979
          } else if (OB_FAIL(allocate_new_pool_units_(trans, *new_pool, module))) {
980
            LOG_WARN("arrange pool units failed", K(module), KR(ret), "resource_pool", *new_pool);
981
          }
982
        }
983
      }
984

985
      if (trans.is_started()) {
986
        const bool commit = (OB_SUCC(ret));
987
        int temp_ret = OB_SUCCESS;
988
        if (OB_SUCCESS != (temp_ret = trans.end(commit))) {
989
          LOG_WARN("trans end failed", K(commit), K(temp_ret));
990
          ret = (OB_SUCCESS == ret) ? temp_ret : ret;
991
        }
992
      }
993
      if (OB_SUCC(ret)) {
994
        ret = OB_E(EventTable::EN_UNIT_MANAGER) OB_SUCCESS;
995
        DEBUG_SYNC(UNIT_MANAGER_WAIT_FOR_TIMEOUT);
996
      }
997

998
      if (OB_FAIL(ret)) {
999
        if (OB_INVALID_ID == new_pool->resource_pool_id_) {
1000
          // do nothing, fetch new resource pool id failed
1001
        } else {
1002
          int temp_ret = OB_SUCCESS; // avoid ret overwritten
1003
          // some error occur during doing the transaction, rollback change occur in memory
1004
          ObArray<ObUnit *> *units = NULL;
1005
          if (OB_SUCCESS == (temp_ret = get_units_by_pool(new_pool->resource_pool_id_, units))) {
1006
            if (NULL == units) {
1007
              temp_ret = OB_ERR_UNEXPECTED;
1008
              LOG_WARN("units is null", KP(units), K(temp_ret));
1009
            } else if (OB_SUCCESS != (temp_ret = delete_units_of_pool(
1010
                        new_pool->resource_pool_id_))) {
1011
              LOG_WARN("delete_units_of_pool failed", "resource_pool_id",
1012
                       new_pool->resource_pool_id_, K(temp_ret));
1013
            }
1014
          } else if (OB_ENTRY_NOT_EXIST != temp_ret) {
1015
            LOG_WARN("get_units_by_pool failed",
1016
                     "resource_pool_id", new_pool->resource_pool_id_, K(temp_ret));
1017
          } else {
1018
            temp_ret = OB_SUCCESS;
1019
          }
1020

1021
          share::ObResourcePool *temp_pool = NULL;
1022
          if (OB_SUCCESS != temp_ret) {
1023
          } else if (OB_SUCCESS != (temp_ret = get_resource_pool_by_id(
1024
                      new_pool->resource_pool_id_, temp_pool))) {
1025
            if (OB_ENTRY_NOT_EXIST != temp_ret) {
1026
              LOG_WARN("get_resource_pool_by_id failed", "pool_id", new_pool->resource_pool_id_,
1027
                       K(temp_ret));
1028
            } else {
1029
              temp_ret = OB_SUCCESS;
1030
              // do nothing, no need to delete from id_map and name_map
1031
            }
1032
          } else if (NULL == temp_pool) {
1033
            temp_ret = OB_ERR_UNEXPECTED;
1034
            LOG_WARN("temp_pool is null", KP(temp_pool), K(temp_ret));
1035
          } else if (OB_SUCCESS != (temp_ret = delete_resource_pool(
1036
                      new_pool->resource_pool_id_, new_pool->name_))) {
1037
            LOG_WARN("delete_resource_pool failed", "new pool", *new_pool, K(temp_ret));
1038
          }
1039
        }
1040
        // avoid memory leak
1041
        pool_allocator_.free(new_pool);
1042
        new_pool = NULL;
1043
      } else {
1044
        // inc unit config ref count at last
1045
        if (OB_FAIL(inc_config_ref_count(config->unit_config_id()))) {
1046
          LOG_WARN("inc_config_ref_count failed", "config id", config->unit_config_id(), K(ret));
1047
        } else if (OB_FAIL(insert_config_pool(config->unit_config_id(), new_pool))) {
1048
          LOG_WARN("insert config pool failed", "config id", config->unit_config_id(), K(ret));
1049
        } else {
1050
          ROOTSERVICE_EVENT_ADD("unit", is_clone_tenant ? "clone_resource_pool" : "create_resource_pool",
1051
                                "name", new_pool->name_,
1052
                                "unit", config_name,
1053
                                "zone_list", new_pool->zone_list_,
1054
                                "source_tenant_id", source_tenant_id);
1055
        }
1056
      }
1057
    }
1058
  }
1059
  LOG_INFO("finish inner create resource pool", K(resource_pool), K(config_name),
1060
           K(source_tenant_id), K(source_units), K(ret));
1061
  return ret;
1062
}
1063

1064
int ObUnitManager::check_new_pool_units_for_clone_tenant_(
1065
    ObISQLClient &client,
1066
    const share::ObResourcePool &pool,
1067
    const common::ObIArray<share::ObUnitInfo> &source_units)
1068
{
1069
  int ret = OB_SUCCESS;
1070
  ObUnitConfig *config = NULL;
1071
  ObArray<ObServerInfoInTable> servers_info_of_source_units;
1072
  ObArray<ObUnitPlacementStrategy::ObServerResource> server_resources;
1073
  std::string resource_not_enough_reason;
1074

1075
  if (!check_inner_stat()) {
1076
    ret = OB_INNER_STAT_ERROR;
1077
    LOG_WARN("check_inner_stat failed", KR(ret), K(inited_), K(loaded_));
1078
  } else if (OB_UNLIKELY(!pool.is_valid())
1079
             || OB_UNLIKELY(pool.zone_list_.count() <= 0)
1080
             || OB_UNLIKELY(source_units.count() <= 0)) {
1081
    ret = OB_INVALID_ARGUMENT;
1082
    LOG_WARN("invalid pool", KR(ret), K(pool), K(source_units));
1083
  } else if (OB_ISNULL(srv_rpc_proxy_)) {
1084
    ret = OB_ERR_UNEXPECTED;
1085
    LOG_WARN("srv_rpc_proxy_ is null", KR(ret), KP(srv_rpc_proxy_));
1086
  } else if (OB_FAIL(get_unit_config_by_id(pool.unit_config_id_, config))) {
1087
    LOG_WARN("get_unit_config_by_id failed", KR(ret), "unit_config_id", pool.unit_config_id_);
1088
  } else if (OB_ISNULL(config)) {
1089
    ret = OB_ERR_UNEXPECTED;
1090
    LOG_WARN("config is null", KR(ret), KP(config));
1091
  } else {
1092
    ObNotifyTenantServerResourceProxy notify_proxy(
1093
        *srv_rpc_proxy_,
1094
        &obrpc::ObSrvRpcProxy::notify_tenant_server_unit_resource);
1095
    for (int64_t i = 0; OB_SUCC(ret) && i < pool.zone_list_.count(); i++) {
1096
      // for each zone, check servers have enough resources
1097
      // ATTENTION:
1098
      //   The locations of clone tenant's units are certain according to source tenant's units
1099
      //   So there is no need to allocate servers for these units, but we have to check whether
1100
      //   these servers have enough resources.
1101
      //   We regard server inactive as a common case, let check success but log WARN in log file.
1102
      //   We can ignore server inactive because majority of snapshot restored successfully is enough,
1103
      //   it is ok currently to let it passed, just let log stream creation to handle this situation.
1104
      const ObZone &zone = pool.zone_list_.at(i);
1105
      servers_info_of_source_units.reuse();
1106
      server_resources.reuse();
1107

1108
      if (OB_FAIL(construct_server_resources_info_(
1109
                      zone,
1110
                      source_units,
1111
                      servers_info_of_source_units,
1112
                      server_resources))) {
1113
        LOG_WARN("fail to construct server resource infos", KR(ret), K(zone), K(source_units));
1114
      } else if (OB_FAIL(check_server_resources_and_persist_unit_info_(
1115
                             client,
1116
                             notify_proxy,
1117
                             zone,
1118
                             pool,
1119
                             servers_info_of_source_units,
1120
                             server_resources,
1121
                             config->unit_resource()))) {
1122
        LOG_WARN("fail to check and persist unit info", KR(ret), K(zone), K(pool),
1123
                 K(servers_info_of_source_units), K(server_resources), KPC(config));
1124
      }
1125
    }
1126
  }
1127
  return ret;
1128
}
1129

1130
int ObUnitManager::construct_server_resources_info_(
1131
    const ObZone &zone,
1132
    const ObIArray<share::ObUnitInfo> &source_units,
1133
    ObIArray<share::ObServerInfoInTable> &server_infos,
1134
    ObIArray<ObUnitPlacementStrategy::ObServerResource> &server_resources)
1135
{
1136
  int ret = OB_SUCCESS;
1137
  server_infos.reset();
1138
  server_resources.reset();
1139
  if (!check_inner_stat()) {
1140
    ret = OB_INNER_STAT_ERROR;
1141
    LOG_WARN("check_inner_stat failed", KR(ret), K(inited_), K(loaded_));
1142
  } else if (OB_UNLIKELY(zone.is_empty())
1143
             || OB_UNLIKELY(0 >= source_units.count())) {
1144
    ret = OB_INVALID_ARGUMENT;
1145
    LOG_WARN("invalid argument", KR(ret), K(zone), K(source_units));
1146
  } else {
1147
    ObArray<obrpc::ObGetServerResourceInfoResult> active_servers_resource_info_result;
1148
    for (int64_t i = 0; OB_SUCC(ret) && i < source_units.count(); i++) {
1149
      const share::ObUnit &unit = source_units.at(i).unit_;
1150
      if (zone == unit.zone_) {
1151
        const common::ObAddr &source_unit_server = unit.server_;
1152
        bool server_is_active = false;
1153
        if (OB_FAIL(SVR_TRACER.check_server_active(source_unit_server, server_is_active))) {
1154
          LOG_WARN("fail to check server active", KR(ret), K(zone), K(unit),
1155
                   K(source_unit_server), K(server_is_active));
1156
        } else if (server_is_active) {
1157
          ObServerInfoInTable server_info;
1158
          if (OB_FAIL(SVR_TRACER.get_server_info(source_unit_server, server_info))) {
1159
            LOG_WARN("fail to get server info", KR(ret), K(source_unit_server), K(server_info));
1160
          } else if (OB_FAIL(server_infos.push_back(server_info))) {
1161
            LOG_WARN("fail to add active server info into array", KR(ret), K(server_info));
1162
          }
1163
        } else {
1164
          // TODO@jingyu.cr:
1165
          //    If server not alive, we can not get correct cpu/mem/log_disk capacity, so just ignore them
1166
          //    Later, we can handle this case by ignoring zone with inactive server or find another valid
1167
          //    server to put unit on, just let other majority replicas successfully restored is ok.
1168
          LOG_WARN("[CLONE_TENANT_UNIT] server is inactive, create clone tenant resource pool may failed",
1169
                   K(zone), K(source_unit_server), K(server_is_active));
1170
        }
1171
      }
1172
    }
1173
    if (OB_FAIL(ret)) {
1174
    } else if (0 >= server_infos.count()) {
1175
      // just ignore, let snapshot creation decide whether to raise error
1176
      LOG_INFO("[CLONE_TENANT_UNIT] zone may have no valid servers to hold unit", K(zone), K(source_units));
1177
    } else if (OB_FAIL(get_servers_resource_info_via_rpc(server_infos, active_servers_resource_info_result))) {
1178
      LOG_WARN("fail to get server resource info via rpc", KR(ret), K(server_infos));
1179
    } else if (OB_FAIL(build_server_resources_(active_servers_resource_info_result, server_resources))) {
1180
      LOG_WARN("fail to build server resources", KR(ret), K(active_servers_resource_info_result));
1181
    }
1182
  }
1183
  return ret;
1184
}
1185

1186
int ObUnitManager::check_server_resources_and_persist_unit_info_(
1187
    ObISQLClient &client,
1188
    ObNotifyTenantServerResourceProxy &notify_proxy,
1189
    const ObZone &zone,
1190
    const share::ObResourcePool &pool,
1191
    const ObIArray<share::ObServerInfoInTable> &server_infos,
1192
    const ObIArray<ObUnitPlacementStrategy::ObServerResource> &server_resources,
1193
    const share::ObUnitResource &config)
1194
{
1195
  int ret = OB_SUCCESS;
1196
  bool is_server_valid = false;
1197
  bool is_resource_enough = false;
1198
  uint64_t new_unit_id = OB_INVALID_ID;
1199
  ObArray<ObAddr> new_servers; // not used
1200
  ObArray<ObUnit> units;
1201
  const char* module = "clone_resource_pool";
1202
  bool for_clone_tenant = true;
1203
  int64_t not_excluded_server_count = 0;
1204
  std::string resource_not_enough_reason;
1205
  ObArray<ObAddr> excluded_servers; // not used
1206
  ObArray<ObUnitPlacementStrategy::ObServerResource> valid_server_resources;
1207
  if (!check_inner_stat()) {
1208
    ret = OB_INNER_STAT_ERROR;
1209
    LOG_WARN("check_inner_stat failed", KR(ret), K(inited_), K(loaded_));
1210
  } else if (server_infos.count() != server_resources.count()) {
1211
    ret = OB_INVALID_ARGUMENT;
1212
    LOG_WARN("invalid servers_info and server_resources array", KR(ret), K(server_infos), K(server_resources));
1213
  } else if (OB_FAIL(construct_valid_servers_resource_(
1214
                         zone,
1215
                         config,
1216
                         excluded_servers,
1217
                         server_infos,
1218
                         server_resources,
1219
                         module,
1220
                         for_clone_tenant,
1221
                         not_excluded_server_count,
1222
                         resource_not_enough_reason,
1223
                         valid_server_resources))) {
1224
    LOG_WARN("fail to construct valid servers resource", KR(ret), K(zone), K(config),
1225
             K(excluded_servers), K(server_infos), K(module), K(for_clone_tenant));
1226
  } else {
1227
    for (int64_t i = 0; OB_SUCC(ret) && i < valid_server_resources.count(); i++) {
1228
      const ObUnitPlacementStrategy::ObServerResource &server_resource = valid_server_resources.at(i);
1229
      if (OB_FAIL(try_persist_unit_info_(
1230
                             notify_proxy,
1231
                             client,
1232
                             zone,
1233
                             pool,
1234
                             lib::Worker::CompatMode::INVALID/*compat_mode*/,
1235
                             0/*unit_group_id*/,
1236
                             server_resource.addr_,
1237
                             new_servers,
1238
                             units))) {
1239
        LOG_WARN("[CLONE_TENANT_UNIT] fail to persist unit info", KR(ret), K(zone), K(pool), K(server_resource));
1240
      }
1241
    }
1242
    int tmp_ret = OB_SUCCESS;
1243
    if (OB_TMP_FAIL(notify_proxy.wait())) {
1244
      LOG_WARN("fail to wait notify resource", KR(ret), K(tmp_ret));
1245
      ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
1246
    }
1247
    if (OB_FAIL(ret)) {
1248
      LOG_WARN("start to rollback unit persistence", KR(ret), K(units), K(pool));
1249
      if(OB_TMP_FAIL(rollback_persistent_units_(
1250
          units,
1251
          pool,
1252
          notify_proxy))) {
1253
        LOG_WARN("[CLONE_TENANT_UNIT] fail to rollback unit persistence",
1254
                 KR(ret), KR(tmp_ret), K(units), K(pool));
1255
      }
1256
    }
1257
  }
1258
  return ret;
1259
}
1260

1261
int ObUnitManager::convert_pool_name_list(
1262
    const common::ObIArray<common::ObString> &split_pool_list,
1263
    common::ObIArray<share::ObResourcePoolName> &split_pool_name_list)
1264
{
1265
  int ret = OB_SUCCESS;
1266
  split_pool_name_list.reset();
1267
  for (int64_t i = 0; OB_SUCC(ret) && i < split_pool_list.count(); ++i) {
1268
    share::ObResourcePoolName pool_name;
1269
    if (OB_FAIL(pool_name.assign(split_pool_list.at(i).ptr()))) {
1270
      LOG_WARN("fail to assign pool name", K(ret));
1271
    } else if (OB_FAIL(split_pool_name_list.push_back(pool_name))) {
1272
      LOG_WARN("fail to push back", K(ret));
1273
    }
1274
  }
1275
  return ret;
1276
}
1277

1278
int ObUnitManager::check_split_pool_name_condition(
1279
    const common::ObIArray<share::ObResourcePoolName> &split_pool_name_list)
1280
{
1281
  int ret = OB_SUCCESS;
1282
  // Check whether the pool name already exists,
1283
  // and check whether the pool name is duplicate
1284
  const int64_t POOL_NAME_SET_BUCKET_NUM = 16;
1285
  ObHashSet<share::ObResourcePoolName> pool_name_set;
1286
  if (OB_FAIL(pool_name_set.create(POOL_NAME_SET_BUCKET_NUM))) {
1287
    LOG_WARN("fail to create hash set", K(ret));
1288
  } else {
1289
    for (int64_t i = 0; OB_SUCC(ret) && i < split_pool_name_list.count(); ++i) {
1290
      const share::ObResourcePoolName &pool_name = split_pool_name_list.at(i);
1291
      share::ObResourcePool *pool = NULL;
1292
      int tmp_ret = inner_get_resource_pool_by_name(pool_name, pool);
1293
      if (OB_ENTRY_NOT_EXIST == tmp_ret) {
1294
        // good, go on next, this pool name not exist
1295
      } else if (OB_SUCCESS == tmp_ret) {
1296
        if (NULL == pool) {
1297
          ret = OB_ERR_UNEXPECTED;
1298
          LOG_WARN("pool is null", K(ret), KP(pool), K(pool_name));
1299
        } else {
1300
          ret = OB_RESOURCE_POOL_EXIST;
1301
          LOG_WARN("resource pool already exist", K(ret), K(pool_name));
1302
          LOG_USER_ERROR(OB_RESOURCE_POOL_EXIST, to_cstring(pool_name));
1303
        }
1304
      } else {
1305
        ret = OB_ERR_UNEXPECTED;
1306
        LOG_WARN("unexpected error when get pool by name", K(ret), K(pool_name));
1307
      }
1308
      if (OB_SUCC(ret)) {
1309
        int tmp_ret = pool_name_set.exist_refactored(pool_name);
1310
        if (OB_HASH_NOT_EXIST == tmp_ret) {
1311
          if (OB_FAIL(pool_name_set.set_refactored(pool_name))) {
1312
            LOG_WARN("fail to set", K(ret), K(pool_name));
1313
          }
1314
        } else if (OB_HASH_EXIST == tmp_ret) {
1315
          ret = OB_INVALID_ARGUMENT;
1316
          LOG_WARN("invalid argument to split resource pool with duplicated name");
1317
          LOG_USER_ERROR(OB_INVALID_ARGUMENT, "resource pool name");
1318
        } else {
1319
          ret = OB_ERR_UNEXPECTED;
1320
          LOG_WARN("unexpected error when set hashset", K(ret));
1321
        }
1322
      }
1323
    }
1324
  }
1325
  return ret;
1326
}
1327

1328
int ObUnitManager::check_split_pool_zone_condition(
1329
    const common::ObIArray<common::ObZone> &split_zone_list,
1330
    const share::ObResourcePool &pool)
1331
{
1332
  int ret = OB_SUCCESS;
1333
  const common::ObIArray<common::ObZone> &pool_zone_list = pool.zone_list_;
1334
  // Check whether the zone is included in the pool zone list,
1335
  // and check whether the zone is duplicated
1336
  const int64_t ZONE_SET_BUCKET_NUM = 16;
1337
  ObHashSet<common::ObZone> zone_set;
1338
  if (pool_zone_list.count() != split_zone_list.count()) {
1339
    ret = OB_INVALID_ARGUMENT;
1340
    LOG_WARN("invalid argument to split zone list", K(ret));
1341
    LOG_USER_ERROR(OB_INVALID_ARGUMENT, "corresponding zone vector");
1342
  } else if (OB_FAIL(zone_set.create(ZONE_SET_BUCKET_NUM))) {
1343
    LOG_WARN("fail to create hash set", K(ret));
1344
  } else {
1345
    for (int64_t i = 0; OB_SUCC(ret) && i < split_zone_list.count(); ++i) {
1346
      const common::ObZone &this_zone = split_zone_list.at(i);
1347
      if (!has_exist_in_array(pool_zone_list, this_zone)) {
1348
        ret = OB_INVALID_ARGUMENT;
1349
        LOG_WARN("invalid argument to non-exist zone in splitting zone vector", K(ret));
1350
        LOG_USER_ERROR(OB_INVALID_ARGUMENT, "non-exist zone in corresponding zone vector");
1351
      } else {
1352
        int tmp_ret = zone_set.exist_refactored(this_zone);
1353
        if (OB_HASH_NOT_EXIST == tmp_ret) {
1354
          if (OB_FAIL(zone_set.set_refactored(this_zone))) {
1355
            LOG_WARN("fail to set", K(ret), K(this_zone));
1356
          }
1357
        } else if (OB_HASH_EXIST == tmp_ret) {
1358
          ret = OB_INVALID_ARGUMENT;
1359
          LOG_WARN("invalid argument to duplicate zones in splitting zone vector", K(ret));
1360
          LOG_USER_ERROR(OB_INVALID_ARGUMENT, "duplidate zones in corresponding zone vector");
1361
        } else {
1362
          ret = OB_ERR_UNEXPECTED;
1363
          LOG_WARN("unexpected error when set hashset", K(ret));
1364
        }
1365
      }
1366
    }
1367
  }
1368
  return ret;
1369
}
1370

1371
int ObUnitManager::split_resource_pool(
1372
    const share::ObResourcePoolName &pool_name,
1373
    const common::ObIArray<common::ObString> &split_pool_list,
1374
    const common::ObIArray<common::ObZone> &split_zone_list)
1375
{
1376
  int ret = OB_SUCCESS;
1377
  LOG_INFO("start split resource pool", K(pool_name), K(split_pool_list), K(split_zone_list));
1378
  SpinWLockGuard guard(lock_);
1379
  share::ObResourcePool *pool = NULL;
1380
  common::ObArray<share::ObResourcePoolName> split_pool_name_list;
1381
  if (!check_inner_stat()) {
1382
    ret = OB_INNER_STAT_ERROR;
1383
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
1384
  } else if (pool_name.is_empty()
1385
             || split_pool_list.count() <= 0
1386
             || split_zone_list.count() <= 0) {
1387
    ret = OB_INVALID_ARGUMENT;
1388
    LOG_WARN("resource pool name is empty", K(ret), K(split_zone_list), K(split_pool_list));
1389
    LOG_USER_ERROR(OB_INVALID_ARGUMENT, "resource pool");
1390
  } else if (OB_FAIL(inner_get_resource_pool_by_name(pool_name, pool))) {
1391
    if (OB_ENTRY_NOT_EXIST != ret) {
1392
      LOG_WARN("get resource pool by name failed", K(ret), K(pool_name));
1393
    } else {
1394
      ret = OB_RESOURCE_POOL_NOT_EXIST;
1395
      LOG_WARN("resource pool not exist", K(ret), K(pool_name));
1396
      LOG_USER_ERROR(OB_RESOURCE_POOL_NOT_EXIST, to_cstring(pool_name));
1397
    }
1398
  } else if (OB_UNLIKELY(NULL == pool)) {
1399
    ret = OB_ERR_UNEXPECTED;
1400
    LOG_WARN("pool ptr is null", K(ret));
1401
  } else if (OB_FAIL(convert_pool_name_list(split_pool_list, split_pool_name_list))) {
1402
    LOG_WARN("fail to convert pool name list", K(ret));
1403
  } else if (OB_FAIL(check_split_pool_name_condition(split_pool_name_list))) {
1404
    LOG_WARN("fail to check pool list name duplicate", K(ret));
1405
  } else if (split_pool_name_list.count() != split_zone_list.count()) {
1406
    ret = OB_INVALID_ARGUMENT;
1407
    LOG_WARN("invalid argument to split pool and zone count", K(ret));
1408
    LOG_USER_ERROR(OB_INVALID_ARGUMENT, "split pool and zone count");
1409
  } else if (OB_FAIL(check_split_pool_zone_condition(split_zone_list, *pool))) {
1410
    LOG_WARN("fail to check split pool zone condition", K(ret));
1411
  } else if (OB_FAIL(do_split_resource_pool(
1412
          pool, split_pool_name_list, split_zone_list))) {
1413
    LOG_WARN("fail to do split resource pool", K(ret));
1414
  } else {
1415
    LOG_INFO("succeed to split resource pool", K(pool_name),
1416
             "new_pool_name", split_pool_list,
1417
             "corresponding_zone", split_zone_list);
1418
  }
1419
  return ret;
1420
}
1421

1422
int ObUnitManager::do_split_pool_persistent_info(
1423
    share::ObResourcePool *pool,
1424
    const common::ObIArray<share::ObResourcePoolName> &split_pool_name_list,
1425
    const common::ObIArray<common::ObZone> &split_zone_list,
1426
    common::ObIArray<share::ObResourcePool *> &allocate_pool_ptrs)
1427
{
1428
  int ret = OB_SUCCESS;
1429
  common::ObMySQLTransaction trans;
1430
  if (!check_inner_stat()) {
1431
    ret = OB_INNER_STAT_ERROR;
1432
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
1433
  } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
1434
    LOG_WARN("fail to start transaction", K(ret));
1435
  } else if (OB_UNLIKELY(split_zone_list.count() != split_pool_name_list.count()
1436
                         || NULL == pool)) {
1437
    ret = OB_INVALID_ARGUMENT;
1438
    LOG_WARN("invalid argument", K(ret));
1439
  } else {
1440
    // Write down the resource pool allocated during execution
1441
    allocate_pool_ptrs.reset();
1442
    for (int64_t i = 0; OB_SUCC(ret) && i < split_pool_name_list.count(); ++i) {
1443
      const share::ObResourcePoolName &new_pool_name = split_pool_name_list.at(i);
1444
      const common::ObZone &zone = split_zone_list.at(i);
1445
      share::ObResourcePool *new_pool = NULL;
1446
      if (NULL == (new_pool = pool_allocator_.alloc())) {
1447
        ret = OB_ALLOCATE_MEMORY_FAILED;
1448
        LOG_ERROR("fail to alloc memory", K(ret));
1449
      } else if (OB_FAIL(allocate_pool_ptrs.push_back(new_pool))) {
1450
        LOG_WARN("fail to push back", K(ret));
1451
      } else if (OB_FAIL(fill_splitting_pool_basic_info(new_pool_name, new_pool, zone, pool))) {
1452
        LOG_WARN("fail to fill splitting pool basic info", K(ret));
1453
      }
1454
    }
1455
    for (int64_t i = 0; OB_SUCC(ret) && i < allocate_pool_ptrs.count(); ++i) {
1456
      share::ObResourcePool *new_pool = allocate_pool_ptrs.at(i);
1457
      if (OB_UNLIKELY(NULL == new_pool)) {
1458
        ret = OB_ERR_UNEXPECTED;
1459
        LOG_WARN("pool ptr is null", K(ret));
1460
      } else if (new_pool->zone_list_.count() != 1) {
1461
        ret = OB_ERR_UNEXPECTED;
1462
        LOG_WARN("zone list count unexpected", K(ret), "zone_list", new_pool->zone_list_);
1463
      } else if (OB_FAIL(ut_operator_.update_resource_pool(trans, *new_pool, false/*need_check_conflict_with_clone*/))) {
1464
        LOG_WARN("fail to update resource pool", K(ret));
1465
      } else if (OB_FAIL(split_pool_unit_persistent_info(
1466
              trans, new_pool->zone_list_.at(0), new_pool, pool))) {
1467
        LOG_WARN("fail to split pool unit persistent info", K(ret));
1468
      }
1469
    }
1470
    if (OB_FAIL(ret)) {
1471
    } else if (ut_operator_.remove_resource_pool(trans, pool->resource_pool_id_)) {
1472
      LOG_WARN("fail to remove resource pool persistent info", K(ret),
1473
               "pool_id", pool->resource_pool_id_);
1474
    } else {} // all persistent infos update finished
1475
    const bool commit = (OB_SUCCESS == ret);
1476
    int tmp_ret = OB_SUCCESS;
1477
    if (OB_SUCCESS != (tmp_ret = trans.end(commit))) {
1478
      LOG_WARN("fail to end trans", K(tmp_ret), K(commit));
1479
      ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
1480
    }
1481
    if (OB_FAIL(ret)) {
1482
      // 1. The transaction did fail to commit.
1483
      //    the internal table and the memory state remain the same,
1484
      //    as long as the memory of allocate_pool_ptrs is released.
1485
      //
1486
      // 2. The transaction submission timed out, but the final submission was successful.
1487
      //    The internal table and memory state are inconsistent,
1488
      //    and the outer layer will call the reload unit manager.
1489
      //    Still need to release the memory of allocate_pool_ptrs
1490
      for (int64_t i = 0; i < allocate_pool_ptrs.count(); ++i) {
1491
        share::ObResourcePool *new_pool = allocate_pool_ptrs.at(i);
1492
        if (NULL != new_pool) {
1493
          pool_allocator_.free(new_pool);
1494
          new_pool = NULL;
1495
        }
1496
      }
1497
    }
1498
  }
1499
  return ret;
1500
}
1501

1502
int ObUnitManager::do_split_pool_inmemory_info(
1503
    share::ObResourcePool *pool,
1504
    common::ObIArray<share::ObResourcePool *> &allocate_pool_ptrs)
1505
{
1506
  int ret = OB_SUCCESS;
1507
  if (!check_inner_stat()) {
1508
    ret = OB_INNER_STAT_ERROR;
1509
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
1510
  } else if (NULL == pool) {
1511
    ret = OB_INVALID_ARGUMENT;
1512
    LOG_WARN("invalid argument", K(ret));
1513
  } else {
1514
    for (int64_t i = 0; OB_SUCC(ret) && i < allocate_pool_ptrs.count(); ++i) {
1515
      share::ObResourcePool *new_pool = allocate_pool_ptrs.at(i);
1516
      if (OB_UNLIKELY(NULL == new_pool)) {
1517
        ret = OB_ERR_UNEXPECTED;
1518
        LOG_WARN("pool ptr is null", K(ret));
1519
      } else if (new_pool->zone_list_.count() != 1) {
1520
        ret = OB_ERR_UNEXPECTED;
1521
        LOG_WARN("zone list count unexpected", K(ret), "zone_list", new_pool->zone_list_);
1522
      } else if (OB_FAIL(inc_config_ref_count(new_pool->unit_config_id_))) {
1523
        LOG_WARN("fail to inc config ref count", K(ret),
1524
                 "unit_config_id", new_pool->unit_config_id_);
1525
      } else if (OB_FAIL(insert_config_pool(new_pool->unit_config_id_, new_pool))) {
1526
        LOG_WARN("fail to insert config pool", K(ret),
1527
                 "unit_config_id", new_pool->unit_config_id_);
1528
      } else if (OB_FAIL(update_pool_map(new_pool))) {
1529
        LOG_WARN("fail to update pool map", K(ret),
1530
                 "resource_pool_id", new_pool->resource_pool_id_);
1531
      } else if (!new_pool->is_granted_to_tenant()) {
1532
        // bypass
1533
      } else if (OB_FAIL(insert_tenant_pool(new_pool->tenant_id_, new_pool))) {
1534
        LOG_WARN("fail to insert tenant pool", K(ret), "tenant_id", new_pool->tenant_id_);
1535
      }
1536
      if (OB_FAIL(ret)) {
1537
        // failed
1538
      } else if (OB_FAIL(split_pool_unit_inmemory_info(
1539
              new_pool->zone_list_.at(0), new_pool, pool))) {
1540
        LOG_WARN("fail to split pool unit inmemory info", K(ret));
1541
      } else {} // no more
1542
    }
1543
    common::ObArray<share::ObUnit *> *pool_units = NULL;
1544
    if (OB_FAIL(ret)) {
1545
    } else if (OB_FAIL(dec_config_ref_count(pool->unit_config_id_))) {
1546
      LOG_WARN("fail to dec config ref count", K(ret));
1547
    } else if (OB_FAIL(delete_config_pool(pool->unit_config_id_, pool))) {
1548
      LOG_WARN("fail to delete config pool", K(ret));
1549
    } else if (OB_FAIL(delete_resource_pool(pool->resource_pool_id_, pool->name_))) {
1550
      LOG_WARN("fail to delete resource pool", K(ret));
1551
    } else if (!pool->is_granted_to_tenant()) {
1552
      // bypass
1553
    } else if (OB_FAIL(delete_tenant_pool(pool->tenant_id_, pool))) {
1554
      LOG_WARN("fail to delete tenant pool", K(ret), "tenant_id", pool->tenant_id_);
1555
    }
1556
    if (OB_FAIL(ret)) {
1557
    } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, pool_units))) {
1558
      LOG_WARN("fail to get units by pool", K(ret), "pool_id", pool->resource_pool_id_);
1559
    } else if (OB_UNLIKELY(NULL == pool_units)) {
1560
      ret = OB_ERR_UNEXPECTED;
1561
      LOG_WARN("pool units ptr is null", K(ret));
1562
    } else if (OB_FAIL(pool_unit_map_.erase_refactored(pool->resource_pool_id_))) {
1563
      LOG_WARN("fail to erase map", K(ret), "pool_id", pool->resource_pool_id_);
1564
    } else {
1565
      pool_unit_allocator_.free(pool_units);
1566
      pool_units = NULL;
1567
      pool_allocator_.free(pool);
1568
      pool = NULL;
1569
    }
1570
    if (OB_FAIL(ret)) {
1571
      // reload
1572
      rootserver::ObRootService *root_service = NULL;
1573
      if (OB_UNLIKELY(NULL == (root_service = GCTX.root_service_))) {
1574
        ret = OB_ERR_UNEXPECTED;
1575
        LOG_WARN("rootservice is null", K(ret));
1576
      } else {
1577
        int tmp_ret = root_service->submit_reload_unit_manager_task();
1578
        if (OB_SUCCESS != tmp_ret) {
1579
          LOG_ERROR("fail to reload unit manager", K(tmp_ret));
1580
        }
1581
      }
1582
    }
1583
  }
1584
  return ret;
1585
}
1586

1587
int ObUnitManager::do_split_resource_pool(
1588
    share::ObResourcePool *pool,
1589
    const common::ObIArray<share::ObResourcePoolName> &split_pool_name_list,
1590
    const common::ObIArray<common::ObZone> &split_zone_list)
1591
{
1592
  int ret = OB_SUCCESS;
1593
  common::ObArray<share::ObResourcePool *> allocate_pool_ptrs;
1594
  if (!check_inner_stat()) {
1595
    ret = OB_INNER_STAT_ERROR;
1596
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
1597
  } else if (OB_UNLIKELY(NULL == pool)) {
1598
    ret = OB_INVALID_ARGUMENT;
1599
    LOG_WARN("invalid argument", K(ret));
1600
  } else if (split_pool_name_list.count() != split_zone_list.count()) {
1601
    ret = OB_ERR_UNEXPECTED;
1602
    LOG_WARN("split pool name and zone count not match",
1603
             K(ret), K(split_pool_name_list), K(split_zone_list));
1604
  } else if (OB_FAIL(do_split_pool_persistent_info(
1605
          pool, split_pool_name_list, split_zone_list, allocate_pool_ptrs))) {
1606
    LOG_WARN("fail to do split pool persistent info", K(ret));
1607
  } else if (OB_FAIL(do_split_pool_inmemory_info(pool, allocate_pool_ptrs))) {
1608
    LOG_WARN("fail to do split pool inmemory info", K(ret));
1609
  } else {} // no more
1610
  return ret;
1611
}
1612

1613
int ObUnitManager::fill_splitting_pool_basic_info(
1614
    const share::ObResourcePoolName &new_pool_name,
1615
    share::ObResourcePool *new_pool,
1616
    const common::ObZone &zone,
1617
    share::ObResourcePool *orig_pool)
1618
{
1619
  int ret = OB_SUCCESS;
1620
  if (!check_inner_stat()) {
1621
    ret = OB_INNER_STAT_ERROR;
1622
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
1623
  } else if (OB_UNLIKELY(new_pool_name.is_empty() || NULL == new_pool
1624
                         || zone.is_empty() || NULL == orig_pool)) {
1625
    ret = OB_INVALID_ARGUMENT;
1626
    LOG_WARN("invalid argument", K(ret), K(new_pool_name), KP(new_pool), K(zone), K(orig_pool));
1627
  } else {
1628
    new_pool->name_ = new_pool_name;
1629
    new_pool->unit_count_ = orig_pool->unit_count_;
1630
    new_pool->unit_config_id_ = orig_pool->unit_config_id_;
1631
    new_pool->tenant_id_ = orig_pool->tenant_id_;
1632
    new_pool->replica_type_ = orig_pool->replica_type_;
1633
    if (OB_FAIL(new_pool->zone_list_.push_back(zone))) {
1634
      LOG_WARN("fail to push back to zone list", K(ret));
1635
    } else if (OB_FAIL(fetch_new_resource_pool_id(new_pool->resource_pool_id_))) {
1636
      LOG_WARN("fail to fetch new resource pool id", K(ret));
1637
    } else {} // finish fill splitting pool basic info
1638
  }
1639
  return ret;
1640
}
1641

1642
int ObUnitManager::split_pool_unit_inmemory_info(
1643
    const common::ObZone &zone,
1644
    share::ObResourcePool *new_pool,
1645
    share::ObResourcePool *orig_pool)
1646
{
1647
  int ret = OB_SUCCESS;
1648
  common::ObArray<share::ObUnit *> *new_units = NULL;
1649
  common::ObArray<share::ObUnit *> *orig_units = NULL;
1650
  if (!check_inner_stat()) {
1651
    ret = OB_INNER_STAT_ERROR;
1652
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
1653
  } else if (OB_UNLIKELY(NULL == new_pool || NULL == orig_pool || zone.is_empty())) {
1654
    ret = OB_INVALID_ARGUMENT;
1655
    LOG_WARN("invalid argument", K(ret), KP(new_pool), K(orig_pool), K(zone));
1656
  } else if (OB_FAIL(get_units_by_pool(orig_pool->resource_pool_id_, orig_units))) {
1657
    LOG_WARN("fail to get units by pool", K(ret));
1658
  } else if (OB_UNLIKELY(NULL == orig_units)) {
1659
    ret = OB_ERR_UNEXPECTED;
1660
    LOG_WARN("units ptr is null", K(ret));
1661
  } else {
1662
    ret = get_units_by_pool(new_pool->resource_pool_id_, new_units);
1663
    if (OB_SUCCESS == ret) {
1664
      // got new units
1665
    } else if (OB_ENTRY_NOT_EXIST != ret) {
1666
      LOG_WARN("fail to get units by pool", K(ret));
1667
    } else {
1668
      if (NULL == (new_units = pool_unit_allocator_.alloc())) {
1669
        ret = OB_ALLOCATE_MEMORY_FAILED;
1670
        LOG_ERROR("fail to alloc memory", K(ret));
1671
      } else if (OB_FAIL(pool_unit_map_.set_refactored(new_pool->resource_pool_id_, new_units))) {
1672
        LOG_WARN("fail to set refactored", K(ret), "pool_id", new_pool->resource_pool_id_);
1673
      }
1674
    }
1675
    for (int64_t i = 0; OB_SUCC(ret) && i < orig_units->count(); ++i) {
1676
      share::ObUnit *unit = orig_units->at(i);
1677
      if (OB_UNLIKELY(NULL == unit)) {
1678
        ret = OB_ERR_UNEXPECTED;
1679
        LOG_WARN("unit ptr is null", K(ret));
1680
      } else if (zone != unit->zone_) {
1681
        // bypass
1682
      } else if (NULL == new_units) {
1683
        ret = OB_ERR_UNEXPECTED;
1684
        LOG_WARN("new units ptr is null", K(ret));
1685
      } else {
1686
        unit->resource_pool_id_ = new_pool->resource_pool_id_;
1687
        if (OB_FAIL(new_units->push_back(unit))) {
1688
          LOG_WARN("fail to push back", K(ret));
1689
        } else if (OB_FAIL(update_unit_load(unit, new_pool))) {
1690
          LOG_WARN("fail to update unit load", K(ret));
1691
        }
1692
      }
1693
    }
1694
  }
1695
  return ret;
1696
}
1697

1698
int ObUnitManager::split_pool_unit_persistent_info(
1699
    common::ObMySQLTransaction &trans,
1700
    const common::ObZone &zone,
1701
    share::ObResourcePool *new_pool,
1702
    share::ObResourcePool *orig_pool)
1703
{
1704
  int ret = OB_SUCCESS;
1705
  common::ObArray<share::ObUnit *> *units = NULL;
1706
  if (!check_inner_stat()) {
1707
    ret = OB_INNER_STAT_ERROR;
1708
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
1709
  } else if (OB_UNLIKELY(NULL == new_pool || NULL == orig_pool || zone.is_empty())) {
1710
    ret = OB_INVALID_ARGUMENT;
1711
    LOG_WARN("invalid argument", K(ret), KP(new_pool), KP(orig_pool), K(zone));
1712
  } else if (OB_FAIL(get_units_by_pool(orig_pool->resource_pool_id_, units))) {
1713
    LOG_WARN("fail to get units by pool", K(ret));
1714
  } else if (OB_UNLIKELY(NULL == units)) {
1715
    ret = OB_ERR_UNEXPECTED;
1716
    LOG_WARN("units ptr is null", K(ret));
1717
  } else {
1718
    for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
1719
      const share::ObUnit *unit = units->at(i);
1720
      if (OB_UNLIKELY(NULL == unit)) {
1721
        ret = OB_ERR_UNEXPECTED;
1722
        LOG_WARN("unit ptr is null", K(ret));
1723
      } else if (unit->zone_ != zone) {
1724
        // bypass
1725
      } else {
1726
        share::ObUnit new_unit = *unit;
1727
        new_unit.resource_pool_id_ = new_pool->resource_pool_id_;
1728
        if (OB_FAIL(ut_operator_.update_unit(trans, new_unit, false/*need_check_conflict_with_clone*/))) {
1729
          LOG_WARN("fail to update unit", K(ret), K(new_unit));
1730
        } else {
1731
          ROOTSERVICE_EVENT_ADD("unit", "split_pool",
1732
                                "unit_id", unit->unit_id_,
1733
                                "server", unit->server_,
1734
                                "prev_pool_id", orig_pool->resource_pool_id_,
1735
                                "curr_pool_id", new_pool->resource_pool_id_);
1736
        }
1737
      }
1738
    }
1739
  }
1740
  return ret;
1741
}
1742

1743
/* current_unit_num_per_zone: pool->unit_count_
1744
 * complete_unit_num_per_zone: pool->unit_count_ + unit_num_in_deleting
1745
 * has_unit_num_modification: when some unit in deleting this is true, else false
1746
 */
1747
int ObUnitManager::get_tenant_pools_complete_unit_num_and_status(
1748
    const uint64_t tenant_id,
1749
    const common::ObIArray<share::ObResourcePool *> &pools,
1750
    int64_t &complete_unit_num_per_zone,
1751
    int64_t &current_unit_num_per_zone,
1752
    bool &has_unit_num_modification)
1753
{
1754
  int ret = OB_SUCCESS;
1755
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
1756
    ret = OB_INVALID_ARGUMENT;
1757
    LOG_WARN("invalid argument", KR(ret), K(tenant_id));
1758
  } else if (OB_UNLIKELY(pools.count() <= 0)) {
1759
    ret = OB_INVALID_ARGUMENT;
1760
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(pools));
1761
  } else {
1762
    complete_unit_num_per_zone = -1;
1763
    current_unit_num_per_zone = -1;
1764
    has_unit_num_modification = false;
1765
    for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
1766
      int64_t my_complete_unit_num_per_zone = 0;
1767
      int64_t my_current_unit_num_per_zone = 0;
1768
      bool my_unit_num_modification = false;
1769
      const share::ObResourcePool *pool = pools.at(i);
1770
      if (OB_UNLIKELY(nullptr == pool)) {
1771
        ret = OB_ERR_UNEXPECTED;
1772
        LOG_WARN("pool ptr is null", KR(ret), KP(pool), K(tenant_id));
1773
      } else if (tenant_id != pool->tenant_id_) {
1774
        ret = OB_ERR_UNEXPECTED;
1775
        LOG_WARN("tenant id not match");
1776
      } else if (OB_FAIL(get_pool_complete_unit_num_and_status(
1777
              pool, my_complete_unit_num_per_zone,
1778
              my_current_unit_num_per_zone, my_unit_num_modification))) {
1779
        LOG_WARN("fail to get pool complete unit num and status", KR(ret));
1780
      } else if (-1 == complete_unit_num_per_zone) {
1781
        complete_unit_num_per_zone = my_complete_unit_num_per_zone;
1782
        current_unit_num_per_zone = my_current_unit_num_per_zone;
1783
        has_unit_num_modification = my_unit_num_modification;
1784
      } else if (complete_unit_num_per_zone != my_complete_unit_num_per_zone
1785
          || current_unit_num_per_zone != my_current_unit_num_per_zone
1786
          || has_unit_num_modification != my_unit_num_modification) {
1787
        ret = OB_ERR_UNEXPECTED;
1788
        LOG_WARN("tenant pools unit status unexpected", KR(ret), K(tenant_id));
1789
      }
1790
    }
1791
  }
1792
  return ret;
1793
}
1794

1795
int ObUnitManager::determine_alter_resource_tenant_unit_num_type(
1796
    const uint64_t tenant_id,
1797
    const common::ObIArray<share::ObResourcePool *> &pools,
1798
    const int64_t new_unit_num,
1799
    int64_t &old_unit_num,
1800
    AlterUnitNumType &alter_unit_num_type)
1801
{
1802
  int ret = OB_SUCCESS;
1803
  int64_t complete_unit_num_per_zone = 0;
1804
  old_unit_num = 0;
1805
  bool has_unit_num_modification = true;
1806

1807
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || new_unit_num <= 0)) {
1808
    ret = OB_INVALID_ARGUMENT;
1809
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(new_unit_num));
1810
  } else if (OB_FAIL(get_tenant_pools_complete_unit_num_and_status(
1811
          tenant_id, pools, complete_unit_num_per_zone,
1812
          old_unit_num, has_unit_num_modification))) {
1813
    LOG_WARN("fail to get tenant pools complete unit num and status", KR(ret), K(tenant_id));
1814
  } else if (OB_UNLIKELY(complete_unit_num_per_zone <= 0 || old_unit_num <= 0)) {
1815
    ret = OB_ERR_UNEXPECTED;
1816
    LOG_WARN("unexpected complete unit num", KR(ret),
1817
             K(complete_unit_num_per_zone), K(old_unit_num));
1818
  } else if (old_unit_num == new_unit_num) {
1819
    // when the new unit num is equal to the current unit num, do nothing and return
1820
    alter_unit_num_type = AUN_NOP;
1821
  } else {
1822
    if (has_unit_num_modification) { // a unit num change is taking place
1823
      if (new_unit_num == complete_unit_num_per_zone) {
1824
        // alter unit num to previous, it is a unit num rollback operation
1825
        alter_unit_num_type = AUN_ROLLBACK_SHRINK;
1826
      } else {
1827
        // alter unit num to a value not equal to the previous, an illegal operation
1828
        alter_unit_num_type = AUN_MAX;
1829
      }
1830
    } else { // no unit num change
1831
      if (new_unit_num > old_unit_num) {
1832
        alter_unit_num_type = AUN_EXPAND;
1833
      } else {
1834
        alter_unit_num_type = AUN_SHRINK;
1835
      }
1836
    }
1837
  }
1838
  return ret;
1839
}
1840

1841
int ObUnitManager::register_alter_resource_tenant_unit_num_rs_job(
1842
    const uint64_t tenant_id,
1843
    const int64_t new_unit_num,
1844
    const int64_t old_unit_num,
1845
    const AlterUnitNumType alter_unit_num_type,
1846
    const common::ObString &sql_text,
1847
    common::ObMySQLTransaction &trans)
1848
{
1849
  int ret = OB_SUCCESS;
1850
  uint64_t tenant_data_version = 0;
1851
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id)
1852
      || !(AUN_SHRINK == alter_unit_num_type || AUN_EXPAND == alter_unit_num_type))) {
1853
    ret = OB_INVALID_ARGUMENT;
1854
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(alter_unit_num_type));
1855
  } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) {
1856
    LOG_WARN("fail to get min data version", KR(ret), K(tenant_id));
1857
  } else if (tenant_data_version < DATA_VERSION_4_2_1_0) {
1858
    if (AUN_EXPAND == alter_unit_num_type) {
1859
      // skip, no rs job for expand task in version 4.1
1860
      // we do not need to rollback rs job, since in 4.1, there is no inprogress rs job at this step
1861
    } else if (OB_FAIL(register_shrink_tenant_pool_unit_num_rs_job(tenant_id,
1862
            new_unit_num, old_unit_num, sql_text, trans))) {
1863
      LOG_WARN("fail to execute register_shrink_tenant_pool_unit_num_rs_job", KR(ret),
1864
          K(tenant_id), K(new_unit_num));
1865
    }
1866
  } else { // tenant_data_version >= DATA_VERSION_4_2_1_0
1867
    // step 1: cancel rs job if exists
1868
    int64_t job_id = 0;
1869
    if (!ObShareUtil::is_tenant_enable_rebalance(tenant_id)) {
1870
      ret = OB_OP_NOT_ALLOW;
1871
      LOG_WARN("enable_rebalance is disabled, modify tenant unit num not allowed", KR(ret), K(tenant_id));
1872
      (void) print_user_error_(tenant_id);
1873
    } else if(OB_FAIL(cancel_alter_resource_tenant_unit_num_rs_job(tenant_id, trans))) {
1874
      LOG_WARN("fail to execute cancel_alter_resource_tenant_unit_num_rs_job",
1875
          KR(ret), K(tenant_id), K(sql_text));
1876
    } else {
1877
      ret = create_alter_resource_tenant_unit_num_rs_job(tenant_id,
1878
          new_unit_num, old_unit_num, job_id, sql_text, trans);
1879
      FLOG_INFO("[ALTER_RESOURCE_TENANT_UNIT_NUM NOTICE] create a new rs job", "type",
1880
          AUN_EXPAND == alter_unit_num_type ? "EXPAND UNIT_NUM" : "SHRINK UNIT_NUM",
1881
          KR(ret), K(tenant_id), K(job_id), K(alter_unit_num_type));
1882
    }
1883
  }
1884
  return ret;
1885
}
1886

1887
int ObUnitManager::find_alter_resource_tenant_unit_num_rs_job(
1888
    const uint64_t tenant_id,
1889
    int64_t &job_id,
1890
    common::ObISQLClient &sql_proxy)
1891
{
1892
  int ret = OB_SUCCESS;
1893
  ret = RS_JOB_FIND(
1894
      ALTER_RESOURCE_TENANT_UNIT_NUM,
1895
      job_id,
1896
      sql_proxy,
1897
      "tenant_id", tenant_id);
1898
  if (OB_ENTRY_NOT_EXIST == ret) {
1899
    ret = RS_JOB_FIND(
1900
        SHRINK_RESOURCE_TENANT_UNIT_NUM, // only created in version < 4.2
1901
        job_id,
1902
        sql_proxy,
1903
        "tenant_id", tenant_id);
1904
  }
1905
  return ret;
1906
}
1907
int ObUnitManager::register_shrink_tenant_pool_unit_num_rs_job(
1908
    const uint64_t tenant_id,
1909
    const int64_t new_unit_num,
1910
    const int64_t old_unit_num,
1911
    const common::ObString &sql_text,
1912
    common::ObMySQLTransaction &trans)
1913
{
1914
  int ret = OB_SUCCESS;
1915
  int64_t pos = 0;
1916
  int64_t job_id = 0;
1917
  if (!ObShareUtil::is_tenant_enable_rebalance(tenant_id)) {
1918
    ret = OB_OP_NOT_ALLOW;
1919
    LOG_WARN("enable_rebalance is disabled, modify tenant unit num not allowed", KR(ret), K(tenant_id));
1920
    (void) print_user_error_(tenant_id);
1921
  } else {
1922
    ret = create_alter_resource_tenant_unit_num_rs_job(
1923
        tenant_id,
1924
        new_unit_num,
1925
        old_unit_num,
1926
        job_id,
1927
        sql_text,
1928
        trans,
1929
        JOB_TYPE_SHRINK_RESOURCE_TENANT_UNIT_NUM);
1930
    FLOG_INFO("[ALTER_RESOURCE_TENANT_UNIT_NUM NOTICE] create a new rs job in Version < 4.2",
1931
        KR(ret), K(tenant_id), K(job_id), K(sql_text));
1932
  }
1933
  return ret ;
1934
}
1935

1936
void ObUnitManager::print_user_error_(const uint64_t tenant_id)
1937
{
1938
  const int64_t ERR_MSG_LEN = 256;
1939
  char err_msg[ERR_MSG_LEN] = {'\0'};
1940
  int ret = OB_SUCCESS;
1941
  int64_t pos = 0;
1942
  if (OB_FAIL(databuff_printf(err_msg, ERR_MSG_LEN, pos,
1943
      "Tenant (%lu) 'enable_rebalance' is disabled, alter tenant unit num", tenant_id))) {
1944
    if (OB_SIZE_OVERFLOW == ret) {
1945
      LOG_WARN("format to buff size overflow", KR(ret));
1946
    } else {
1947
      LOG_WARN("format new unit num failed", KR(ret));
1948
    }
1949
  } else {
1950
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, err_msg);
1951
  }
1952
}
1953

1954
int ObUnitManager::cancel_alter_resource_tenant_unit_num_rs_job(
1955
  const uint64_t tenant_id,
1956
  common::ObMySQLTransaction &trans)
1957
{
1958
  int ret = OB_SUCCESS;
1959
  int64_t job_id = 0;
1960
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
1961
    ret = OB_INVALID_ARGUMENT;
1962
    LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id));
1963
  } else if (OB_FAIL(find_alter_resource_tenant_unit_num_rs_job(tenant_id, job_id, trans))) {
1964
    if (OB_ENTRY_NOT_EXIST == ret) {
1965
      ret = OB_SUCCESS;
1966
      FLOG_INFO("[ALTER_RESOURCE_TENANT_UNIT_NUM NOTICE] there is no rs job in table",KR(ret), K(tenant_id));
1967
    } else {
1968
      LOG_WARN("fail to execute find_alter_resource_tenant_unit_num_rs_job", KR(ret), K(tenant_id));
1969
    }
1970
  } else {
1971
    ret = RS_JOB_COMPLETE(job_id, OB_CANCELED, trans);
1972
    FLOG_INFO("[ALTER_RESOURCE_TENANT_UNIT_NUM NOTICE] cancel an inprogress rs job", KR(ret),
1973
        K(tenant_id), K(job_id));
1974
  }
1975
  return ret;
1976
}
1977

1978
int ObUnitManager::create_alter_resource_tenant_unit_num_rs_job(
1979
    const uint64_t tenant_id,
1980
    const int64_t new_unit_num,
1981
    const int64_t old_unit_num,
1982
    int64_t &job_id,
1983
    const common::ObString &sql_text,
1984
    common::ObMySQLTransaction &trans,
1985
    ObRsJobType job_type)
1986
{
1987
  int ret = OB_SUCCESS;
1988
  job_id = 0;
1989
  const int64_t extra_info_len = common::MAX_ROOTSERVICE_EVENT_EXTRA_INFO_LENGTH;
1990
  HEAP_VAR(char[extra_info_len], extra_info) {
1991
    memset(extra_info, 0, extra_info_len);
1992
    int64_t pos = 0;
1993
    share::schema::ObSchemaGetterGuard schema_guard;
1994
    const ObSimpleTenantSchema *tenant_schema;
1995
    if (OB_FAIL(databuff_printf(extra_info, extra_info_len, pos,
1996
            "FROM: '%ld', TO: '%ld'", old_unit_num, new_unit_num))) {
1997
      if (OB_SIZE_OVERFLOW == ret) {
1998
        LOG_WARN("format to buff size overflow", K(ret));
1999
      } else {
2000
        LOG_WARN("format new unit num failed", K(ret));
2001
      }
2002
    } else if (OB_ISNULL(schema_service_)) {
2003
      ret = OB_NOT_INIT;
2004
      LOG_WARN("schema service is null", KR(ret), KP(schema_service_));
2005
    } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) {
2006
      LOG_WARN("fail to get tenant schema guard", KR(ret), K(tenant_id));
2007
    } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
2008
      LOG_WARN("fail to get tenant info", K(ret), K(tenant_id));
2009
    } else if (OB_ISNULL(tenant_schema)) {
2010
      ret = OB_ERR_UNEXPECTED;
2011
      LOG_WARN("tenant schema is null", KR(ret), KP(tenant_schema));
2012
    } else if (OB_FAIL(RS_JOB_CREATE_WITH_RET(
2013
        job_id,
2014
        job_type,
2015
        trans,
2016
        "tenant_id", tenant_id,
2017
        "tenant_name", tenant_schema->get_tenant_name(),
2018
        "sql_text", ObHexEscapeSqlStr(sql_text),
2019
        "extra_info", ObHexEscapeSqlStr(extra_info)))) {
2020
      LOG_WARN("fail to create rs job", KR(ret), K(tenant_id), K(job_type));
2021
    }
2022
  }
2023
  return ret;
2024
}
2025

2026
int ObUnitManager::rollback_alter_resource_tenant_unit_num_rs_job(
2027
    const uint64_t tenant_id,
2028
    const int64_t new_unit_num,
2029
    const int64_t old_unit_num,
2030
    const common::ObString &sql_text,
2031
    common::ObMySQLTransaction &trans)
2032
{
2033
  int ret = OB_SUCCESS;
2034
  int64_t job_id = 0;
2035
  uint64_t tenant_data_version = 0;
2036
  if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, tenant_data_version))) {
2037
    LOG_WARN("fail to get min data version", KR(ret), K(tenant_id));
2038
  } else if (tenant_data_version < DATA_VERSION_4_2_1_0) {
2039
    // in v4.1, it's allowed, since this will not track the number of logstreams
2040
    if (OB_FAIL(cancel_alter_resource_tenant_unit_num_rs_job(tenant_id, trans))) {
2041
      LOG_WARN("fail to execute cancel_alter_resource_tenant_unit_num_rs_job in v4.1", KR(ret), K(tenant_id));
2042
    }
2043
  } else if (!ObShareUtil::is_tenant_enable_rebalance(tenant_id)) {
2044
    ret = OB_OP_NOT_ALLOW;
2045
    LOG_WARN("enable_rebalance is disabled, modify tenant unit num not allowed", KR(ret), K(tenant_id));
2046
    (void) print_user_error_(tenant_id);
2047
  } else if (OB_FAIL(cancel_alter_resource_tenant_unit_num_rs_job(tenant_id, trans))) {
2048
    LOG_WARN("fail to execute cancel_alter_resource_tenant_unit_num_rs_job", KR(ret), K(tenant_id));
2049
  } else if (OB_FAIL(create_alter_resource_tenant_unit_num_rs_job(tenant_id,
2050
          new_unit_num, old_unit_num, job_id, sql_text, trans))) {
2051
    LOG_WARN("fail to execute create_alter_resource_tenant_unit_num_rs_job", KR(ret),
2052
        K(tenant_id), K(new_unit_num), K(old_unit_num), K(sql_text));
2053
  }
2054
  FLOG_INFO("[ALTER_RESOURCE_TENANT_UNIT_NUM NOTICE] rollback a SHRINK UNIT_NUM rs job",
2055
      KR(ret), K(tenant_id), K(job_id), K(new_unit_num));
2056
  return ret;
2057
}
2058

2059
int ObUnitManager::try_complete_shrink_tenant_pool_unit_num_rs_job(
2060
    const uint64_t tenant_id,
2061
    common::ObMySQLTransaction &trans)
2062
{
2063
  int ret = OB_SUCCESS;
2064
  // this function is called by ObDDLService::drop_resource_pool_pre
2065
  const int check_ret = OB_SUCCESS; // no need to check ls status
2066
  int64_t job_id = 0;
2067
  if (OB_FAIL(find_alter_resource_tenant_unit_num_rs_job(tenant_id, job_id, trans))) {
2068
    if (OB_ENTRY_NOT_EXIST == ret) {
2069
      ret = OB_SUCCESS;
2070
    } else {
2071
      LOG_WARN("fail to execute find_alter_resource_tenant_unit_num_rs_job", KR(ret), K(tenant_id));
2072
    }
2073
  } else {
2074
    ret = complete_shrink_tenant_pool_unit_num_rs_job_(
2075
          tenant_id, job_id, check_ret, trans);
2076
    FLOG_INFO("[ALTER_RESOURCE_TENANT_UNIT_NUM NOTICE] complete an inprogress rs job DROP_TENANT_FORCE",
2077
        KR(ret), K(tenant_id), K(job_id));
2078
  }
2079
  return ret;
2080
}
2081

2082
int ObUnitManager::complete_shrink_tenant_pool_unit_num_rs_job_(
2083
    const uint64_t tenant_id,
2084
    const int64_t job_id,
2085
    const int check_ret,
2086
    common::ObMySQLTransaction &trans)
2087
{
2088
  int ret = OB_SUCCESS;
2089
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id)) || job_id < 0) {
2090
    ret = OB_INVALID_ARGUMENT;
2091
    LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id), K(job_id));
2092
  } else if (OB_SUCC(RS_JOB_COMPLETE(job_id, check_ret, trans))) {
2093
    FLOG_INFO("[ALTER_RESOURCE_TENANT_UNIT_NUM NOTICE] complete an inprogressing rs job SHRINK UNIT_NUM",
2094
          KR(ret), K(tenant_id), K(job_id),  K(check_ret));
2095
  } else {
2096
    LOG_WARN("fail to complete rs job", KR(ret), K(tenant_id), K(job_id), K(check_ret));
2097
    if (OB_EAGAIN == ret) {
2098
      int64_t curr_job_id = 0;
2099
      if (OB_FAIL(find_alter_resource_tenant_unit_num_rs_job(tenant_id, curr_job_id, trans))) {
2100
        LOG_WARN("fail to find rs job", KR(ret), K(tenant_id), K(job_id), K(check_ret));
2101
        if (OB_ENTRY_NOT_EXIST == ret) {
2102
          FLOG_WARN("[ALTER_RESOURCE_TENANT_UNIT_NUM NOTICE] the specified rs job SHRINK UNIT_NUM might has "
2103
              "been already deleted in table manually",
2104
              KR(ret), K(tenant_id), K(job_id), K(check_ret));
2105
          ret = OB_SUCCESS;
2106
        }
2107
      } else {
2108
        ret = OB_EAGAIN;
2109
        FLOG_WARN("[ALTER_RESOURCE_TENANT_UNIT_NUM NOTICE] a non-checked rs job SHRINK UNIT_NUM cannot be committed",
2110
            KR(ret), K(job_id), K(curr_job_id));
2111
      }
2112
    }
2113
  }
2114
  return ret ;
2115
}
2116

2117
/* generate new unit group id for pools, we hold the assumption that
2118
 * each pool of pools comes from the same tenant and has the same unit num
2119
 */
2120
int ObUnitManager::generate_new_unit_group_id_array(
2121
    const uint64_t tenant_id,
2122
    const common::ObIArray<share::ObResourcePool *> &pools,
2123
    const int64_t new_unit_num,
2124
    common::ObIArray<uint64_t> &unit_group_id_array)
2125
{
2126
  int ret = OB_SUCCESS;
2127
  if (OB_UNLIKELY(pools.count() <= 0 || new_unit_num <= 0)) {
2128
    ret = OB_INVALID_ARGUMENT;
2129
    LOG_WARN("invalid argument", KR(ret), K(pools), K(new_unit_num));
2130
  } else {
2131
    {  // defense check
2132
      int64_t sample_unit_num = -1;
2133
      for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
2134
        const share::ObResourcePool *pool = pools.at(i);
2135
        if (OB_UNLIKELY(nullptr == pool)) {
2136
          ret = OB_ERR_UNEXPECTED;
2137
          LOG_WARN("pool ptr is null", KR(ret), K(tenant_id));
2138
        } else if (-1 == sample_unit_num) {
2139
          sample_unit_num = pool->unit_count_;
2140
        } else if (sample_unit_num != pool->unit_count_) {
2141
          ret = OB_ERR_UNEXPECTED;
2142
          LOG_WARN("unit num not match", KR(ret), K(tenant_id), KPC(pool), K(sample_unit_num));
2143
        }
2144
      }
2145
    }
2146
    const share::ObResourcePool *pool = pools.at(0);
2147
    int64_t unit_group_id_num = 0;
2148
    if (OB_FAIL(ret)) {
2149
      // failed, bypass
2150
    } else if (OB_UNLIKELY(nullptr == pool)) {
2151
      ret = OB_ERR_UNEXPECTED;
2152
      LOG_WARN("pool ptr is null", KR(ret));
2153
    } else if ((unit_group_id_num = new_unit_num - pool->unit_count_) <= 0) {
2154
      ret = OB_ERR_UNEXPECTED;
2155
      LOG_WARN("unit_group_id_num unexpected", KR(ret), K(unit_group_id_num), K(new_unit_num));
2156
    } else {
2157
      unit_group_id_array.reset();
2158
      for (int64_t i = 0; OB_SUCC(ret) && i < unit_group_id_num; ++i) {
2159
        uint64_t unit_group_id = OB_INVALID_ID;
2160
        if (OB_FAIL(fetch_new_unit_group_id(unit_group_id))) {
2161
          LOG_WARN("fail to fetch new unit group id", KR(ret));
2162
        } else if (OB_FAIL(unit_group_id_array.push_back(unit_group_id))) {
2163
          LOG_WARN("fail to push back", KR(ret));
2164
        }
2165
      }
2166
    }
2167
  }
2168
  return ret;
2169
}
2170

2171
/* when alter resource tenant and the new unit num is greater than the current unit num,
2172
 * this func is invoked.
2173
 */
2174
int ObUnitManager::expand_tenant_pools_unit_num_(
2175
    const uint64_t tenant_id,
2176
    common::ObIArray<share::ObResourcePool *> &pools,
2177
    const int64_t new_unit_num,
2178
    const int64_t old_unit_num,
2179
    const char *module,
2180
    const common::ObString &sql_text)
2181
{
2182
  int ret = OB_SUCCESS;
2183
  common::ObMySQLTransaction trans;
2184
  common::ObArray<uint64_t> new_unit_group_id_array;
2185
  ObArray<ObAddr> new_servers;
2186
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || new_unit_num <= 0)) {
2187
    ret = OB_INVALID_ARGUMENT;
2188
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(new_unit_num));
2189
  } else if (OB_FAIL(generate_new_unit_group_id_array(
2190
          tenant_id, pools, new_unit_num, new_unit_group_id_array))) {
2191
    LOG_WARN("fail to generate new unit group id array", KR(ret), K(pools), K(new_unit_num));
2192
  } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
2193
    LOG_WARN("fail to start transaction", KR(ret));
2194
  } else if (OB_FAIL(register_alter_resource_tenant_unit_num_rs_job(tenant_id,
2195
          new_unit_num, old_unit_num, AUN_EXPAND, sql_text, trans))) {
2196
    LOG_WARN("fail to register shrink tenant pool unit num rs job", KR(ret),
2197
        K(tenant_id), K(sql_text), K(old_unit_num), K(new_unit_num));
2198
  } else {
2199
    share::ObResourcePool new_pool;
2200
    for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
2201
      const bool new_allocate_pool = false;
2202
      const share::ObResourcePool *pool = pools.at(i);
2203
      new_pool.reset();
2204
      if (OB_UNLIKELY(nullptr == pool)) {
2205
        ret = OB_ERR_UNEXPECTED;
2206
        LOG_WARN("unexpected pool", KR(ret), K(tenant_id));
2207
      } else if (pool->tenant_id_ != tenant_id) {
2208
        ret = OB_ERR_UNEXPECTED;
2209
        LOG_WARN("tenant id not match", KR(ret),
2210
                 K(tenant_id), "pool_tenant_id", pool->tenant_id_);
2211
      } else if (OB_FAIL(allocate_pool_units_(
2212
              trans, *pool, pool->zone_list_, &new_unit_group_id_array, new_allocate_pool,
2213
              new_unit_num - pool->unit_count_, module, new_servers))) {
2214
        LOG_WARN("fail to allocate pool units", K(module), KR(ret), K(new_unit_num), KPC(pool));
2215
      } else if (OB_FAIL(new_pool.assign(*pool))) {
2216
        LOG_WARN("fail to assign new pool", KR(ret));
2217
      } else if (FALSE_IT(new_pool.unit_count_ = new_unit_num)) {
2218
        // false it, shall never be here
2219
      } else if (OB_FAIL(ut_operator_.update_resource_pool(trans, new_pool, true/*need_check_conflict_with_clone*/))) {
2220
        LOG_WARN("fail to update resource pool", KR(ret), K(tenant_id));
2221
      }
2222
    }
2223
    const bool commit = (OB_SUCCESS == ret);
2224
    int tmp_ret = OB_SUCCESS;
2225
    if (OB_SUCCESS != (tmp_ret = trans.end(commit))) {
2226
      LOG_WARN("fail to trans end", KR(tmp_ret), K(commit));
2227
      ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
2228
    }
2229
    for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
2230
      share::ObResourcePool *pool = pools.at(i);
2231
      if (OB_UNLIKELY(nullptr == pool)) {
2232
        ret = OB_ERR_UNEXPECTED;
2233
        LOG_WARN("unexpected pool", KR(ret), K(tenant_id));
2234
      } else {
2235
        pool->unit_count_ = new_unit_num;
2236
      }
2237
    }
2238
  }
2239
  return ret;
2240
}
2241

2242
int ObUnitManager::get_to_be_deleted_unit_group(
2243
    const uint64_t tenant_id,
2244
    const common::ObIArray<share::ObResourcePool *> &pools,
2245
    const int64_t new_unit_num,
2246
    const common::ObIArray<uint64_t> &deleted_unit_group_id_array,
2247
    common::ObIArray<uint64_t> &to_be_deleted_unit_group)
2248
{
2249
  int ret = OB_SUCCESS;
2250
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || pools.count() <= 0)) {
2251
    ret = OB_INVALID_ARGUMENT;
2252
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(pools));
2253
  } else {
2254
    common::ObArray<uint64_t> all_unit_group_id_array;
2255
    const share::ObResourcePool *pool = pools.at(0);
2256
    const bool is_active = false;
2257
    int64_t to_be_deleted_num = 0;
2258
    if (OB_UNLIKELY(nullptr == pool)) {
2259
      ret = OB_ERR_UNEXPECTED;
2260
      LOG_WARN("pool ptr is null", KR(ret), KP(pool));
2261
    } else if ((to_be_deleted_num = pool->unit_count_ - new_unit_num) <= 0) {
2262
      ret = OB_ERR_UNEXPECTED;
2263
      LOG_WARN("unit num unexpected", KR(ret), KPC(pool), K(tenant_id), K(new_unit_num));
2264
    } else if (OB_FAIL(inner_get_all_unit_group_id(tenant_id, is_active, all_unit_group_id_array))) {
2265
      LOG_WARN("fail to get all unit group id", KR(ret), K(tenant_id));
2266
    } else if (to_be_deleted_num > all_unit_group_id_array.count()) {
2267
      ret = OB_ERR_UNEXPECTED;
2268
      LOG_WARN("unit num to match", KR(ret), K(tenant_id), K(new_unit_num),
2269
               "old_unit_num", pool->unit_count_,
2270
               "tenant_unit_group_num", all_unit_group_id_array.count());
2271
    } else if (deleted_unit_group_id_array.count() <= 0) {
2272
      // deleted unit groups not specified by the client, we choose for automatically
2273
      // If some servers are inactive, the related unit_group should take priority to be deleted.
2274
      ObArray<uint64_t> sorted_unit_group_id_array;
2275
      // inactive unit_group_id first
2276
      ObArray<ObUnitInfo> pool_unit_infos;
2277
      for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
2278
        if (OB_UNLIKELY(nullptr == pools.at(i))) {
2279
          ret = OB_ERR_UNEXPECTED;
2280
          LOG_WARN("pool ptr is null", KR(ret), KP(pools.at(i)));
2281
        } else if (OB_FAIL(inner_get_unit_infos_of_pool_(pools.at(i)->resource_pool_id_, pool_unit_infos))) {
2282
          LOG_WARN("inner_get_unit_infos_of_pool failed", KR(ret), "resource_pool_id", pools.at(i)->resource_pool_id_);
2283
        } else if (pool_unit_infos.count() <= 0) {
2284
          ret = OB_ERR_UNEXPECTED;
2285
          LOG_WARN("pool_unit_infos is empty", KR(ret), "pool_id", pools.at(i)->resource_pool_id_);
2286
        } else {
2287
          ObServerInfoInTable server_info;
2288
          for (int64_t j = 0; OB_SUCC(ret) && j < pool_unit_infos.count(); ++j) {
2289
            ObUnit &unit = pool_unit_infos.at(j).unit_;
2290
            if (has_exist_in_array(sorted_unit_group_id_array, unit.unit_group_id_)) {
2291
              // skip, this unit_group_id has already been added
2292
            } else if (OB_FAIL(SVR_TRACER.get_server_info(unit.server_, server_info))) {
2293
              LOG_WARN("fail to get_server_info", KR(ret), K(unit));
2294
            } else if (!server_info.is_active()) {
2295
              if (OB_FAIL(sorted_unit_group_id_array.push_back(unit.unit_group_id_))) {
2296
                LOG_WARN("fail to push_back", KR(ret), K(unit));
2297
              }
2298
            } else {/*active server*/}
2299
          }
2300
        }
2301
      }
2302
      std::sort(sorted_unit_group_id_array.begin(), sorted_unit_group_id_array.end());
2303
      // then other active unit group id
2304
      for (int64_t i = 0; OB_SUCC(ret) && i < all_unit_group_id_array.count(); ++i) {
2305
        uint64_t ug_id = all_unit_group_id_array.at(i);
2306
        if (has_exist_in_array(sorted_unit_group_id_array, ug_id)) {
2307
          // skip, offline unit_group_id is already pushed
2308
        } else if (OB_FAIL(sorted_unit_group_id_array.push_back(ug_id))) {
2309
          LOG_WARN("fail to push back", KR(ret), K(ug_id));
2310
        }
2311
      }
2312
      for (int64_t i = 0; OB_SUCC(ret) && i < to_be_deleted_num; ++i) {
2313
        if (OB_FAIL(to_be_deleted_unit_group.push_back(sorted_unit_group_id_array.at(i)))) {
2314
          LOG_WARN("fail to push back", KR(ret));
2315
        }
2316
      }
2317
      LOG_INFO("Automatically determined on unit_group to delete for shrinking tenant unit num.",
2318
               KR(ret), K(tenant_id), K(new_unit_num),
2319
               K(to_be_deleted_unit_group), K(sorted_unit_group_id_array));
2320
    } else if (deleted_unit_group_id_array.count() == to_be_deleted_num) {
2321
      // the deleted unit groups are specified by the client
2322
      for (int64_t i = 0; OB_SUCC(ret) && i < deleted_unit_group_id_array.count(); ++i) {
2323
        const uint64_t unit_group_id = deleted_unit_group_id_array.at(i);
2324
        if (has_exist_in_array(all_unit_group_id_array, unit_group_id)) {
2325
          if (OB_FAIL(to_be_deleted_unit_group.push_back(unit_group_id))) {
2326
            LOG_WARN("fail to push back", KR(ret));
2327
          }
2328
        } else {
2329
          ret = OB_OP_NOT_ALLOW;
2330
          LOG_USER_ERROR(OB_OP_NOT_ALLOW, "delete unit group which does not belong to this tenant");
2331
        }
2332
      }
2333
    } else {
2334
      // deleted unit groups num is not match to the specified num.
2335
      ret = OB_OP_NOT_ALLOW;
2336
      LOG_USER_ERROR(OB_OP_NOT_ALLOW, "to be deleted unit num not match");
2337
    }
2338
  }
2339
  return ret;
2340
}
2341

2342
int ObUnitManager::check_shrink_tenant_pools_allowed(
2343
    const uint64_t tenant_id,
2344
    common::ObIArray<share::ObResourcePool *> &pools,
2345
    const int64_t unit_num,
2346
    bool &is_allowed)
2347
{
2348
  int ret = OB_SUCCESS;
2349
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || pools.count() <= 0)) {
2350
    ret = OB_INVALID_ARGUMENT;
2351
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(pools));
2352
  } else {
2353
    is_allowed = true;
2354
    for (int64_t i = 0; is_allowed && OB_SUCC(ret) && i < pools.count(); ++i) {
2355
      share::ObResourcePool *pool = pools.at(i);
2356
      bool my_is_allowed = true;
2357
      if (OB_UNLIKELY(nullptr == pool)) {
2358
        ret = OB_ERR_UNEXPECTED;
2359
        LOG_WARN("pool ptr is null", KR(ret), KP(pool));
2360
      } else if (tenant_id != pool->tenant_id_) {
2361
        ret = OB_ERR_UNEXPECTED;
2362
        LOG_WARN("tenant_id not match", KR(ret), K(tenant_id), "pool_tenant_id", pool->tenant_id_);
2363
      } else if (OB_FAIL(check_shrink_granted_pool_allowed(pool, unit_num, my_is_allowed))) {
2364
        LOG_WARN("fail to check shrink granted pool allowed", KR(ret));
2365
      } else {
2366
        is_allowed = my_is_allowed;
2367
      }
2368
    }
2369
  }
2370
  return ret;
2371
}
2372

2373
// There are two situations to consider in the shrinkage of the resource pool:
2374
// 1. There is no resource pool granted to tenants
2375
//    the specified number of units can be directly deleted when the resource pool shrinks.
2376
// 2. Resource pool that has been granted to tenants
2377
//    It is necessary to select the specified units from the resource pool,
2378
//    mark these units as the deleting state,
2379
//    and wait for all the replicas to be moved from such units before the whole process can be considered complete.
2380
//    The unit num of the pool that has been granted to the tenant needs to be scaled down.
2381
//    Preconditions:
2382
//    2.1 The reduced unit num cannot be less than the number of replicas of locality in the corresponding zone
2383
//    2.2 At present, the shrinking operation is performed when the unit is in a steady state,
2384
//        the shrinking is allowed when all units are not migrated.
2385
//        We avoid the unit being in both the migration and the deleting state, thereby reducing the complexity
2386
int ObUnitManager::shrink_tenant_pools_unit_num(
2387
    const uint64_t tenant_id,
2388
    common::ObIArray<share::ObResourcePool *> &pools,
2389
    const int64_t new_unit_num,
2390
    const int64_t old_unit_num,
2391
    const common::ObIArray<uint64_t> &delete_unit_group_id_array,
2392
     const common::ObString &sql_text)
2393
{
2394
  int ret = OB_SUCCESS;
2395
  common::ObMySQLTransaction trans;
2396
  bool is_allowed = false;
2397
  common::ObArray<uint64_t> to_be_deleted_unit_group;
2398

2399
  if (OB_FAIL(check_shrink_tenant_pools_allowed(
2400
          tenant_id, pools, new_unit_num, is_allowed))) {
2401
    LOG_WARN("fail to check shrink tenant pools allowed", KR(ret), K(tenant_id));
2402
  } else if (!is_allowed) {
2403
    ret = OB_OP_NOT_ALLOW;
2404
    LOG_WARN("cannot shrink tenant pools unit num", KR(ret), K(tenant_id));
2405
  } else if (OB_FAIL(get_to_be_deleted_unit_group(
2406
          tenant_id, pools, new_unit_num,
2407
          delete_unit_group_id_array, to_be_deleted_unit_group))) {
2408
    LOG_WARN("fail to get to be deleted unit group", KR(ret), K(tenant_id));
2409
  } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
2410
    LOG_WARN("fail to start transaction", KR(ret));
2411
  } else {
2412
    if (OB_FAIL(register_alter_resource_tenant_unit_num_rs_job(tenant_id,
2413
            new_unit_num, old_unit_num, AUN_SHRINK, sql_text, trans))) {
2414
      LOG_WARN("fail to register shrink tenant pool unit num rs job", KR(ret),
2415
          K(tenant_id), K(new_unit_num), K(sql_text), K(old_unit_num));
2416
    } else {
2417
      share::ObResourcePool new_pool;
2418
      for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
2419
        share::ObResourcePool *pool = pools.at(i);
2420
        common::ObArray<share::ObUnit *> *units = nullptr;
2421
        new_pool.reset();
2422
        if (OB_UNLIKELY(nullptr == pool || tenant_id != pool->tenant_id_)) {
2423
          ret = OB_ERR_UNEXPECTED;
2424
          LOG_WARN("pool info unexpected", KR(ret), K(tenant_id), KPC(pool));
2425
        } else if (OB_FAIL(new_pool.assign(*pool))) {
2426
          LOG_WARN("fail to assign new pool", KR(ret), K(tenant_id), KPC(pool));
2427
        } else if (FALSE_IT(new_pool.unit_count_ = new_unit_num)) {
2428
          // shall never be here
2429
        } else if (OB_FAIL(ut_operator_.update_resource_pool(trans, new_pool, true/*need_check_conflict_with_clone*/))) {
2430
          LOG_WARN("fail to update resource pool", KR(ret), K(tenant_id), K(new_pool));
2431
        } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
2432
          LOG_WARN("fail to get units by pool", KR(ret), KPC(pool), K(tenant_id));
2433
        } else if (OB_UNLIKELY(nullptr == units)) {
2434
          ret = OB_ERR_UNEXPECTED;
2435
          LOG_WARN("units ptr is null", KR(ret), K(tenant_id), KPC(pool));
2436
        } else {
2437
          ObUnit new_unit;
2438
          for (int64_t j = 0; OB_SUCC(ret) && j < units->count(); ++j) {
2439
            new_unit.reset();
2440
            const ObUnit *unit = units->at(j);
2441
            if (OB_UNLIKELY(nullptr == unit)) {
2442
              ret = OB_ERR_UNEXPECTED;
2443
              LOG_WARN("unit ptr is null", KR(ret));
2444
            } else if (!has_exist_in_array(to_be_deleted_unit_group, unit->unit_group_id_)) {
2445
              // bypass
2446
            } else if (FALSE_IT(new_unit = *unit)) {
2447
              // shall never be here
2448
            } else if (FALSE_IT(new_unit.status_ = ObUnit::UNIT_STATUS_DELETING)) {
2449
              // shall never be here
2450
            } else if (OB_FAIL(ut_operator_.update_unit(trans, new_unit, true/*need_check_conflict_with_clone*/))) {
2451
              LOG_WARN("fail to update unit", KR(ret), K(new_unit));
2452
            }
2453
          }
2454
        }
2455
      }
2456
    }
2457
    // however, we need to end this transaction
2458
    if (trans.is_started()) {
2459
      const bool commit = (OB_SUCCESS == ret);
2460
      int tmp_ret = OB_SUCCESS;
2461
      if (OB_SUCCESS != (tmp_ret = trans.end(commit))) {
2462
        LOG_WARN("trans end failed", K(tmp_ret), K(commit));
2463
        ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
2464
      }
2465
    }
2466
    // modify in memory pool/unit info
2467
    for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
2468
      share::ObResourcePool *pool = pools.at(i);
2469
      common::ObArray<share::ObUnit *> *units = nullptr;
2470
      if (OB_UNLIKELY(nullptr == pool || tenant_id != pool->tenant_id_)) {
2471
        ret = OB_ERR_UNEXPECTED;
2472
        LOG_WARN("pool info unexpected", KR(ret), K(tenant_id), KPC(pool));
2473
      } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
2474
        LOG_WARN("fail to get units by pool", KR(ret), KPC(pool), K(tenant_id));
2475
      } else if (OB_UNLIKELY(nullptr == units)) {
2476
        ret = OB_ERR_UNEXPECTED;
2477
        LOG_WARN("units ptr is null", KR(ret), K(tenant_id), KPC(pool));
2478
      } else {
2479
        pool->unit_count_ = new_unit_num;
2480
        for (int64_t j = 0; OB_SUCC(ret) && j < units->count(); ++j) {
2481
          ObUnit *unit = units->at(j);
2482
          if (OB_UNLIKELY(nullptr == unit)) {
2483
            ret = OB_ERR_UNEXPECTED;
2484
            LOG_WARN("unit ptr is null", KR(ret));
2485
          } else if (!has_exist_in_array(to_be_deleted_unit_group, unit->unit_group_id_)) {
2486
            // bypass
2487
          } else {
2488
            unit->status_ = ObUnit::UNIT_STATUS_DELETING;
2489
          }
2490
        }
2491
      }
2492
    }
2493
  }
2494
  return ret;
2495
}
2496

2497
int ObUnitManager::rollback_tenant_shrink_pools_unit_num(
2498
    const uint64_t tenant_id,
2499
    common::ObIArray<share::ObResourcePool *> &pools,
2500
    const int64_t new_unit_num,
2501
    const int64_t old_unit_num,
2502
    const common::ObString &sql_text)
2503
{
2504
  int ret = OB_SUCCESS;
2505
  common::ObMySQLTransaction trans;
2506
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || new_unit_num <= 0)) {
2507
    ret = OB_INVALID_ARGUMENT;
2508
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(new_unit_num));
2509
  } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
2510
    LOG_WARN("start transaction failed", K(ret));
2511
  } else {
2512
    if (OB_FAIL(rollback_alter_resource_tenant_unit_num_rs_job(tenant_id, new_unit_num,
2513
            old_unit_num, sql_text, trans))) {
2514
      LOG_WARN("rollback rs_job failed ", KR(ret), K(new_unit_num), K(old_unit_num), K(sql_text));
2515
    } else {
2516
      share::ObResourcePool new_pool;
2517
      for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
2518
        common::ObArray<share::ObUnit *> *units = nullptr;
2519
        const share::ObResourcePool *pool = pools.at(i);
2520
        new_pool.reset();
2521
        if (OB_UNLIKELY(nullptr == pool)) {
2522
          ret = OB_ERR_UNEXPECTED;
2523
          LOG_WARN("pool ptr is null", KR(ret), K(tenant_id));
2524
        } else if (OB_FAIL(new_pool.assign(*pool))) {
2525
          LOG_WARN("fail to assign new pool", KR(ret), KPC(pool));
2526
        } else if (FALSE_IT(new_pool.unit_count_ = new_unit_num)) {
2527
          // shall never be here
2528
        } else if (OB_FAIL(ut_operator_.update_resource_pool(trans, new_pool, false/*need_check_conflict_with_clone*/))) {
2529
          LOG_WARN("fail to update resource pool", KR(ret), K(new_pool));
2530
        } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
2531
          LOG_WARN("fail to get units by pool", KR(ret), "pool_id", pool->resource_pool_id_);
2532
        } else if (OB_UNLIKELY(nullptr == units)) {
2533
          ret = OB_ERR_UNEXPECTED;
2534
          LOG_WARN("units is null", KR(ret), K(tenant_id), "pool_id", pool->resource_pool_id_);
2535
        } else {
2536
          for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
2537
            const ObUnit *this_unit = units->at(i);
2538
            if (OB_UNLIKELY(NULL == this_unit)) {
2539
              ret = OB_ERR_UNEXPECTED;
2540
              LOG_WARN("unit ptr is null", K(ret));
2541
            } else if (ObUnit::UNIT_STATUS_ACTIVE == this_unit->status_) {
2542
              // go and process the unit in deleting
2543
            } else if (ObUnit::UNIT_STATUS_DELETING == this_unit->status_) {
2544
              ObUnit new_unit = *this_unit;
2545
              new_unit.status_ = ObUnit::UNIT_STATUS_ACTIVE;
2546
              if (OB_FAIL(ut_operator_.update_unit(trans, new_unit, true/*need_check_conflict_with_clone*/))) {
2547
                LOG_WARN("fail to update unit", K(ret), K(new_unit), "cur_unit", *this_unit);
2548
              }
2549
            } else {
2550
              ret = OB_ERR_UNEXPECTED;
2551
              LOG_WARN("unexpected unit status", K(ret), "unit", *this_unit);
2552
            }
2553
          }
2554
        }
2555
      }
2556
    }
2557
    const bool commit = (OB_SUCCESS == ret);
2558
    int tmp_ret = OB_SUCCESS;
2559
    if (OB_SUCCESS != (tmp_ret = trans.end(commit))) {
2560
      LOG_WARN("fail to trans end", KR(tmp_ret), K(commit));
2561
      ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
2562
    }
2563
  }
2564
  if (OB_SUCC(ret)) {
2565
    for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
2566
      common::ObArray<share::ObUnit *> *units = nullptr;
2567
      share::ObResourcePool *pool = pools.at(i);
2568
      if (OB_UNLIKELY(nullptr == pool)) {
2569
        ret = OB_ERR_UNEXPECTED;
2570
        LOG_WARN("pool ptr is null", KR(ret), K(tenant_id));
2571
      } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
2572
        LOG_WARN("fail to get units by pool", KR(ret), "pool_id", pool->resource_pool_id_);
2573
      } else if (OB_UNLIKELY(nullptr == units)) {
2574
        ret = OB_ERR_UNEXPECTED;
2575
        LOG_WARN("units is null", KR(ret), K(tenant_id), "pool_id", pool->resource_pool_id_);
2576
      } else {
2577
        pool->unit_count_ = new_unit_num;
2578
        for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
2579
          ObUnit *this_unit = units->at(i);
2580
          if (OB_UNLIKELY(NULL == this_unit)) {
2581
            ret = OB_ERR_UNEXPECTED;
2582
            LOG_WARN("unit ptr is null", K(ret));
2583
          } else if (ObUnit::UNIT_STATUS_ACTIVE == this_unit->status_) {
2584
            // go and process the unit in deleting
2585
          } else if (ObUnit::UNIT_STATUS_DELETING == this_unit->status_) {
2586
            this_unit->status_ = ObUnit::UNIT_STATUS_ACTIVE;
2587
          } else {
2588
            ret = OB_ERR_UNEXPECTED;
2589
            LOG_WARN("unexpected unit status", K(ret), "unit", *this_unit);
2590
          }
2591
        }
2592
      }
2593
    }
2594
  }
2595
  return ret;
2596
}
2597

2598
int ObUnitManager::alter_resource_tenant(
2599
    const uint64_t tenant_id,
2600
    const int64_t new_unit_num,
2601
    const common::ObIArray<uint64_t> &delete_unit_group_id_array,
2602
    const common::ObString &sql_text)
2603
{
2604
  int ret = OB_SUCCESS;
2605
  LOG_INFO("start to alter resource tenant", K(tenant_id));
2606
  SpinWLockGuard guard(lock_);
2607
  // related variables
2608
  common::ObArray<share::ObResourcePool *> *pools = nullptr;
2609
  const char *module = "ALTER_RESOURCE_TENANT";
2610
  AlterUnitNumType alter_unit_num_type = AUN_MAX;
2611
  int64_t old_unit_num = 0;
2612

2613
  if (OB_UNLIKELY(!check_inner_stat())) {
2614
    ret = OB_INNER_STAT_ERROR;
2615
    LOG_WARN("fail to check inner stat", KR(ret), K(inited_), K(loaded_));
2616
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || new_unit_num <= 0)) {
2617
    ret = OB_INVALID_ARGUMENT;
2618
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(new_unit_num));
2619
  } else if (OB_FAIL(get_pools_by_tenant_(tenant_id, pools))) {
2620
    LOG_WARN("fail to get pools by tenant", KR(ret), K(tenant_id));
2621
  } else if (OB_UNLIKELY(nullptr == pools)) {
2622
    ret = OB_ERR_UNEXPECTED;
2623
    LOG_WARN("pools ptr is null", KR(ret), K(tenant_id));
2624
  } else if (OB_FAIL(determine_alter_resource_tenant_unit_num_type(
2625
          tenant_id, *pools, new_unit_num, old_unit_num, alter_unit_num_type))) {
2626
    LOG_WARN("fail to do determine alter resource tenant unit num type", KR(ret));
2627
  } else if (AUN_NOP == alter_unit_num_type) {
2628
    if (delete_unit_group_id_array.count() > 0) {
2629
      ret = OB_NOT_SUPPORTED;
2630
      LOG_USER_ERROR(OB_NOT_SUPPORTED, "delete unit group without unit num change");
2631
    } else {} // good, nothing to do with unit group
2632
  } else if (AUN_ROLLBACK_SHRINK == alter_unit_num_type) {
2633
    if (delete_unit_group_id_array.count() > 0) {
2634
      ret = OB_NOT_SUPPORTED;
2635
      LOG_USER_ERROR(OB_NOT_SUPPORTED, "rollback shrink pool unit num combined with deleting unit");
2636
    } else if (OB_FAIL(rollback_tenant_shrink_pools_unit_num(
2637
            tenant_id, *pools, new_unit_num, old_unit_num, sql_text))) {
2638
      LOG_WARN("fail to rollback shrink pool unit num", K(ret),
2639
          K(new_unit_num), K(sql_text), K(old_unit_num));
2640
    }
2641
  } else if (AUN_EXPAND == alter_unit_num_type) {
2642
    // in 4.1, if enable_rebalance is false, this op can be executed successfully
2643
    // in 4.2, it will return OB_OP_NOT_ALLOW
2644
    if (delete_unit_group_id_array.count() > 0) {
2645
      ret = OB_NOT_SUPPORTED;
2646
      LOG_USER_ERROR(OB_NOT_SUPPORTED, "expand pool unit num combined with deleting unit");
2647
    } else if (OB_FAIL(expand_tenant_pools_unit_num_(
2648
            tenant_id, *pools, new_unit_num, old_unit_num, module, sql_text))) {
2649
      LOG_WARN("fail to expend pool unit num", K(module), KR(ret), K(new_unit_num), K(tenant_id),
2650
          KPC(pools), K(sql_text), K(old_unit_num));
2651
    }
2652
  } else if (AUN_SHRINK == alter_unit_num_type) {
2653
    // both 4.1 and 4.2 do not allow this op when enable_rebalance is false.
2654
    if (OB_FAIL(shrink_tenant_pools_unit_num(tenant_id, *pools, new_unit_num,
2655
            old_unit_num, delete_unit_group_id_array, sql_text))) {
2656
      LOG_WARN("fail to shrink pool unit num", K(ret), K(new_unit_num), K(sql_text), K(old_unit_num));
2657
    }
2658
  } else if (AUN_MAX == alter_unit_num_type) {
2659
    ret = OB_OP_NOT_ALLOW;
2660
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "alter unit group num while the previous operation is in progress");
2661
  } else {
2662
    ret = OB_ERR_UNEXPECTED;
2663
    LOG_WARN("unexpected alter unit group num type", KR(ret), K(alter_unit_num_type));
2664
  }
2665
  return ret;
2666
}
2667

2668
int ObUnitManager::merge_resource_pool(
2669
    const common::ObIArray<common::ObString> &old_pool_list,
2670
    const common::ObIArray<common::ObString> &new_pool_list)
2671
{
2672
  int ret = OB_SUCCESS;
2673
  LOG_INFO("start merge resource pool", K(old_pool_list), K(new_pool_list));
2674
  SpinWLockGuard guard(lock_);
2675
  common::ObArray<share::ObResourcePoolName> old_pool_name_list;
2676
  share::ObResourcePoolName merge_pool_name;
2677
  common::ObArray<share::ObResourcePool *> old_pool;//Pool to be merged
2678
  common::ObArray<common::ObZone> merge_zone_list;//zone list to be merged
2679
  if (!check_inner_stat()) {
2680
    ret = OB_INNER_STAT_ERROR;
2681
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
2682
  } else if (new_pool_list.count() <= 0
2683
             || old_pool_list.count() <= 0
2684
             || old_pool_list.count() < 2
2685
             //Preventing only one pool from being merged is meaningless
2686
             || new_pool_list.count() > 1) {
2687
    //Can only be merged into one resource pool
2688
    ret = OB_INVALID_ARGUMENT;
2689
    LOG_WARN("resource pool zone list is illeagle", K(old_pool_list), K(new_pool_list));
2690
    LOG_USER_ERROR(OB_INVALID_ARGUMENT, "resource pool");
2691
  } else if (OB_FAIL(convert_pool_name_list(old_pool_list, old_pool_name_list,
2692
                                            new_pool_list, merge_pool_name))) {//1.parse pool name
2693
    LOG_WARN("fail to convert pool name list", K(ret));
2694
  } else if (OB_FAIL(check_old_pool_name_condition(
2695
                     old_pool_name_list, merge_zone_list, old_pool))) {
2696
    //2. check the pool that in old_pool_list is whether valid
2697
    LOG_WARN("fail to check old pool name condition", K(ret));
2698
  } else if (OB_FAIL(check_merge_pool_name_condition(merge_pool_name))) {
2699
    //3. check the merge_pool_name is whether the new pool
2700
    LOG_WARN("fail to check merge pool name condition", K(ret));
2701
  } else if (OB_FAIL(do_merge_resource_pool(merge_pool_name, merge_zone_list, old_pool))) {
2702
    LOG_WARN("fail to do merge resource pool", K(ret));
2703
  } else {
2704
    LOG_INFO("success to merge resource pool", K(merge_pool_name),
2705
             "old_pool_name", old_pool_list);
2706
  }
2707
  return ret;
2708
}
2709

2710
int ObUnitManager::convert_pool_name_list(
2711
    const common::ObIArray<common::ObString> &old_pool_list,
2712
    common::ObIArray<share::ObResourcePoolName> &old_pool_name_list,
2713
    const common::ObIArray<common::ObString> &new_pool_list,
2714
    share::ObResourcePoolName &merge_pool_name)
2715
{
2716
  int ret = OB_SUCCESS;
2717
  old_pool_name_list.reset();
2718
  for (int64_t i = 0; OB_SUCC(ret) && i < old_pool_list.count(); ++i) {
2719
    share::ObResourcePoolName pool_name;
2720
    if (OB_FAIL(pool_name.assign(old_pool_list.at(i).ptr()))) {
2721
      LOG_WARN("fail to assign pool name", K(ret));
2722
    } else if (has_exist_in_array(old_pool_name_list, pool_name)) {//Check for duplication
2723
      ret = OB_OP_NOT_ALLOW;
2724
      LOG_WARN("not allow merge resource pool repeat", K(ret));
2725
      LOG_USER_ERROR(OB_OP_NOT_ALLOW, "merge resource pool repeat");
2726
    } else if (OB_FAIL(old_pool_name_list.push_back(pool_name))) {
2727
      LOG_WARN("fail to push back", K(ret));
2728
    }
2729
  }
2730
  if (OB_SUCC(ret)) {
2731
    if (OB_FAIL(merge_pool_name.assign(new_pool_list.at(0).ptr()))) {
2732
      LOG_WARN("fail to assign pool name", K(ret));
2733
    }
2734
  }
2735
  return ret;
2736
}
2737

2738
int ObUnitManager::check_merge_pool_name_condition(
2739
    const share::ObResourcePoolName &merge_pool_name)
2740
{
2741
  int ret = OB_SUCCESS;
2742
  // check the pool name is whether exist,
2743
  // and check the pool name is whether duplication.
2744
  share::ObResourcePool *pool = NULL;
2745
  int tmp_ret = inner_get_resource_pool_by_name(merge_pool_name, pool);
2746
  if (OB_ENTRY_NOT_EXIST == tmp_ret) {
2747
    // good, go on next, this pool name not exist
2748
  } else if (OB_SUCCESS == tmp_ret) {
2749
    if (NULL == pool) {
2750
      ret = OB_ERR_UNEXPECTED;
2751
      LOG_WARN("pool is null", K(ret), KP(pool), K(merge_pool_name));
2752
    } else {
2753
      ret = OB_RESOURCE_POOL_EXIST;
2754
      LOG_WARN("resource pool already exist", K(ret), K(merge_pool_name));
2755
      LOG_USER_ERROR(OB_RESOURCE_POOL_EXIST, to_cstring(merge_pool_name));
2756
    }
2757
  } else {
2758
    ret = OB_ERR_UNEXPECTED;
2759
    LOG_WARN("unexpected error when get pool by name", K(ret), K(merge_pool_name));
2760
  }
2761
  return ret;
2762
}
2763

2764
int ObUnitManager::check_old_pool_name_condition(
2765
    common::ObIArray<share::ObResourcePoolName> &old_pool_name_list,
2766
    common::ObIArray<common::ObZone> &merge_zone_list,
2767
    common::ObIArray<share::ObResourcePool*> &old_pool)
2768
{
2769
  int ret = OB_SUCCESS;
2770
  common::ObReplicaType replica_type = REPLICA_TYPE_MAX;
2771
  uint64_t tenant_id = OB_INVALID_ID;
2772
  share::ObUnitConfig *unit_config = NULL;
2773
  int64_t unit_count = 0;
2774
  const int64_t POOL_NAME_SET_BUCKET_NUM = 16;
2775
  ObHashSet<share::ObResourcePoolName> pool_name_set;
2776
  if (OB_FAIL(pool_name_set.create(POOL_NAME_SET_BUCKET_NUM))) {
2777
    LOG_WARN("fail to create hash set", K(ret));
2778
  } else {
2779
    for (int64_t i = 0; OB_SUCC(ret) && i < old_pool_name_list.count(); ++i) {
2780
      const share::ObResourcePoolName &pool_name = old_pool_name_list.at(i);
2781
      share::ObUnitConfig *this_unit_config = NULL;
2782
      share::ObResourcePool *pool = NULL;
2783
      if (OB_FAIL(inner_get_resource_pool_by_name(pool_name, pool))) {
2784
        LOG_WARN("fail to get resource pool by name", K(ret));
2785
      } else if (OB_ISNULL(pool)) {
2786
        ret = OB_ERR_UNEXPECTED;
2787
        LOG_WARN("pool is null", K(ret), K(pool), K(pool_name));
2788
      } else {
2789
        if (0 == i) {
2790
          replica_type = pool->replica_type_;
2791
          tenant_id = pool->tenant_id_;
2792
          unit_count = pool->unit_count_;
2793
          if (OB_FAIL(get_unit_config_by_id(pool->unit_config_id_, unit_config))) {
2794
            if (OB_ENTRY_NOT_EXIST == ret) {
2795
              ret = OB_RESOURCE_UNIT_NOT_EXIST;
2796
            }
2797
            LOG_WARN("can not find config for unit",
2798
                     "unit_config_id", pool->unit_config_id_,
2799
                      K(ret));
2800
          } else if (NULL == unit_config) {
2801
            ret = OB_ERR_UNEXPECTED;
2802
            LOG_WARN("config is null", K(*unit_config), K(ret));
2803
          } else {} //no more
2804
        } else {
2805
          if (replica_type != pool->replica_type_) {
2806
            // Determine whether the replica_type is the same
2807
            ret = OB_OP_NOT_ALLOW;
2808
            LOG_WARN("not allow pool replica type different",
2809
                      K(ret), K(replica_type), K(pool->replica_type_));
2810
            LOG_USER_ERROR(OB_OP_NOT_ALLOW, "pool replica type different");
2811
          } else if (tenant_id != pool->tenant_id_) {
2812
            // Determine whether the tenant_id is the same
2813
            ret = OB_OP_NOT_ALLOW;
2814
            LOG_WARN("not allow pool tenant id different",
2815
                      K(ret), K(tenant_id), K(pool->tenant_id_));
2816
            LOG_USER_ERROR(OB_OP_NOT_ALLOW, "pool tenant id different");
2817
          } else if (unit_count != pool->unit_count_) {
2818
            // Determine whether the unit_count is the same
2819
            ret = OB_OP_NOT_ALLOW;
2820
            LOG_WARN("not allow pool unit count different",
2821
                      K(ret), K(unit_count), K(pool->unit_count_));
2822
            LOG_USER_ERROR(OB_OP_NOT_ALLOW, "pool unit count different");
2823
          } else if (OB_FAIL(get_unit_config_by_id(pool->unit_config_id_, this_unit_config))) {
2824
            if (OB_ENTRY_NOT_EXIST == ret) {
2825
              ret = OB_RESOURCE_UNIT_NOT_EXIST;
2826
            }
2827
            LOG_WARN("can not find config for unit",
2828
                     "unit_config_id", pool->unit_config_id_,
2829
                      K(ret));
2830
          } else if (NULL == this_unit_config) {
2831
            ret = OB_ERR_UNEXPECTED;
2832
            LOG_WARN("config is null", KP(this_unit_config), K(ret));
2833
          } else {//Determine whether the unit config is the same
2834
            if (unit_config->unit_config_id() == this_unit_config->unit_config_id()) {//nothing todo
2835
            } else {
2836
              ret = OB_OP_NOT_ALLOW;
2837
              LOG_WARN("not allow pool unit config different",
2838
                        K(ret), K(*this_unit_config), K(*unit_config));
2839
              LOG_USER_ERROR(OB_OP_NOT_ALLOW, "pool unit config different");
2840
            }
2841
          }
2842
        }
2843
        if (OB_SUCC(ret)) {
2844
          int tmp_ret = pool_name_set.exist_refactored(pool_name);
2845
          if (OB_HASH_NOT_EXIST == tmp_ret) {
2846
            if (OB_FAIL(pool_name_set.set_refactored(pool_name))) {
2847
              LOG_WARN("fail to set", K(ret), K(pool_name));
2848
            }
2849
          } else if (OB_HASH_EXIST == tmp_ret) {
2850
            ret = OB_INVALID_ARGUMENT;
2851
            LOG_WARN("invalid argument to merge resource pool duplicate");
2852
            LOG_USER_ERROR(OB_INVALID_ARGUMENT, "resource pool name duplicate");
2853
          } else {
2854
            ret = OB_ERR_UNEXPECTED;
2855
            LOG_WARN("unexpected error when set hashset", K(ret));
2856
          }
2857
        }
2858
        if (OB_SUCC(ret)) {
2859
          // Judge whether the zone has crossover,
2860
          // it is implicitly judged whether the pool_name is repeated, no need to repeat the judgment
2861
          for (int64_t j = 0; OB_SUCC(ret) && j < pool->zone_list_.count(); ++j) {
2862
            const common::ObZone &this_zone = pool->zone_list_.at(j);
2863
            if (has_exist_in_array(merge_zone_list, this_zone)) {
2864
              ret = OB_OP_NOT_ALLOW;
2865
              LOG_WARN("not allow to merge resource pool with duplicated zone");
2866
              LOG_USER_ERROR(OB_OP_NOT_ALLOW, "resource pool with duplicated zone");
2867
            } else if (OB_FAIL(merge_zone_list.push_back(this_zone))) {
2868
              LOG_WARN("fail to push back", K(ret));
2869
            }
2870
          }
2871
        }
2872
        if (OB_SUCC(ret)) {
2873
          ObArray<ObUnit *> *units = nullptr;
2874
          if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
2875
            LOG_WARN("fail to get units by pool", KR(ret), KPC(pool));
2876
          } else if (OB_UNLIKELY(nullptr == units)) {
2877
            ret = OB_ERR_UNEXPECTED;
2878
            LOG_WARN("units ptr is null", KR(ret), KPC(pool));
2879
          } else {
2880
            for (int64_t j = 0; OB_SUCC(ret) && j < units->count(); ++j) {
2881
              ObUnit *unit = units->at(j);
2882
              if (OB_UNLIKELY(nullptr == unit)) {
2883
                ret = OB_ERR_UNEXPECTED;
2884
                LOG_WARN("unit ptr is null", KR(ret));
2885
              } else if (unit->status_ != ObUnit::UNIT_STATUS_ACTIVE) {
2886
                ret = OB_OP_NOT_ALLOW;
2887
                LOG_WARN("merging pools when any pool in shrinking not allowed", KR(ret), KPC(pool));
2888
                LOG_USER_ERROR(OB_OP_NOT_ALLOW, "merging pools when any pool in shrinking");
2889
              }
2890
            }
2891
          }
2892
        }
2893
        if (OB_SUCC(ret)) {
2894
          if (OB_FAIL(old_pool.push_back(pool))) {
2895
            LOG_WARN("fail to push back", K(ret));
2896
          }
2897
        }
2898
      }
2899
    }
2900
  }
2901
 return ret;
2902
}
2903

2904

2905
int ObUnitManager::do_merge_resource_pool(
2906
    const share::ObResourcePoolName &merge_pool_name,
2907
    const common::ObIArray<common::ObZone> &merge_zone_list,
2908
    common::ObIArray<share::ObResourcePool*> &old_pool)
2909
{
2910
  int ret = OB_SUCCESS;
2911
  share::ObResourcePool *allocate_pool_ptr = nullptr;
2912
  if (!check_inner_stat()) {
2913
    ret = OB_INNER_STAT_ERROR;
2914
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
2915
  } else if (OB_FAIL(do_merge_pool_persistent_info(
2916
                     allocate_pool_ptr, merge_pool_name,
2917
                     merge_zone_list, old_pool))) {
2918
    LOG_WARN("fail to do merge pool persistent info", K(ret));
2919
  } else if (OB_FAIL(do_merge_pool_inmemory_info(allocate_pool_ptr, old_pool))) {
2920
    LOG_WARN("fail to do merge pool inmemory info", K(ret));
2921
  } else {} //no more
2922
  return ret;
2923
}
2924

2925
int ObUnitManager::do_merge_pool_persistent_info(
2926
    share::ObResourcePool *&allocate_pool_ptr,
2927
    const share::ObResourcePoolName &merge_pool_name,
2928
    const common::ObIArray<common::ObZone> &merge_zone_list,
2929
    const common::ObIArray<share::ObResourcePool*> &old_pool)
2930
{
2931
  int ret = OB_SUCCESS;
2932
  common::ObMySQLTransaction trans;
2933
  if (!check_inner_stat()) {
2934
    ret = OB_INNER_STAT_ERROR;
2935
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
2936
  } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
2937
    LOG_WARN("fail to start transaction", K(ret));
2938
  } else {
2939
    if (NULL == (allocate_pool_ptr = pool_allocator_.alloc())) {
2940
      ret = OB_ALLOCATE_MEMORY_FAILED;
2941
      LOG_ERROR("fail to alloc memory", K(ret));
2942
    } else if (OB_FAIL(fill_merging_pool_basic_info(allocate_pool_ptr, merge_pool_name,
2943
                                                    merge_zone_list, old_pool))) {
2944
      // The specifications of the pools to be merged are the same, so select the first pool here
2945
      LOG_WARN("fail to fill merging pool basic info", K(ret));
2946
    } else if (OB_FAIL(ut_operator_.update_resource_pool(trans, *allocate_pool_ptr, false/*need_check_conflict_with_clone*/))) {
2947
      LOG_WARN("fail to update resource pool", K(ret));
2948
    } else {
2949
      for (int64_t i = 0; OB_SUCC(ret) && i < old_pool.count(); ++i) {
2950
        if (OB_FAIL(merge_pool_unit_persistent_info(
2951
                trans, allocate_pool_ptr, old_pool.at(i)))) {
2952
          LOG_WARN("fail to split pool unit persistent info", K(ret));
2953
        }
2954
      }
2955
    }
2956
    if (OB_SUCC(ret)) {
2957
      for (int64_t i = 0; i < old_pool.count() && OB_SUCC(ret); ++i) {
2958
        if (OB_FAIL(ut_operator_.remove_resource_pool(trans, old_pool.at(i)->resource_pool_id_))) {
2959
          LOG_WARN("fail to remove resource pool persistent info", K(ret),
2960
                   "pool_id", old_pool.at(i)->resource_pool_id_);
2961
        } else {} // all persistent infos update finished
2962
      }
2963
    }
2964
    const bool commit = (OB_SUCCESS == ret);
2965
    int tmp_ret = OB_SUCCESS;
2966
    if (OB_SUCCESS != (tmp_ret = trans.end(commit))) {
2967
      LOG_WARN("fail to end trans", K(tmp_ret), K(commit));
2968
      ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
2969
    }
2970
    if (OB_FAIL(ret)) {
2971
      // 1. The transaction did fail to commit.
2972
      //    the internal table and the memory state remain the same,
2973
      //    as long as the memory of allocate_pool_ptrs is released.
2974
      //
2975
      // 2. The transaction submission timed out, but the final submission was successful.
2976
      //    The internal table and memory state are inconsistent,
2977
      //    and the outer layer will call the reload unit manager.
2978
      //    Still need to release the memory of allocate_pool_ptrs
2979
      if (NULL != allocate_pool_ptr) {
2980
        pool_allocator_.free(allocate_pool_ptr);
2981
        allocate_pool_ptr = NULL;
2982
      }
2983
    }
2984
  }
2985
  return ret;
2986
}
2987

2988
int ObUnitManager::fill_merging_pool_basic_info(
2989
    share::ObResourcePool *&allocate_pool_ptr,
2990
    const share::ObResourcePoolName &merge_pool_name,
2991
    const common::ObIArray<common::ObZone> &merge_zone_list,
2992
    const common::ObIArray<share::ObResourcePool*> &old_pool)
2993
{
2994
  int ret = OB_SUCCESS;
2995
  if (!check_inner_stat()) {
2996
    ret = OB_INNER_STAT_ERROR;
2997
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
2998
  } else if (OB_UNLIKELY(old_pool.count() <= 1)) {
2999
    //It doesn't make sense to merge only one pool
3000
    ret = OB_INVALID_ARGUMENT;
3001
    LOG_WARN("invalid argument", K(ret));
3002
  }
3003
  if (OB_SUCC(ret)) {
3004
    share::ObResourcePool *orig_pool = old_pool.at(0);
3005
    if (OB_UNLIKELY(merge_pool_name.is_empty() || NULL == allocate_pool_ptr
3006
                    || NULL == orig_pool)) {
3007
      ret = OB_INVALID_ARGUMENT;
3008
      LOG_WARN("invalid argument", K(ret), K(merge_pool_name), KP(allocate_pool_ptr), K(orig_pool));
3009
    } else {
3010
      allocate_pool_ptr->name_ = merge_pool_name;
3011
      allocate_pool_ptr->unit_count_ = orig_pool->unit_count_;
3012
      allocate_pool_ptr->unit_config_id_ = orig_pool->unit_config_id_;
3013
      allocate_pool_ptr->tenant_id_ = orig_pool->tenant_id_;
3014
      allocate_pool_ptr->replica_type_ = orig_pool->replica_type_;
3015
      if (OB_FAIL(fetch_new_resource_pool_id(allocate_pool_ptr->resource_pool_id_))) {
3016
        LOG_WARN("fail to fetch new resource pool id", K(ret));
3017
      } else {
3018
        for (int64_t i = 0; i < merge_zone_list.count() && OB_SUCC(ret); ++i) {
3019
          if (OB_UNLIKELY(merge_zone_list.at(i).is_empty())) {
3020
            ret = OB_INVALID_ARGUMENT;
3021
            LOG_WARN("invalid argument", K(ret), K(merge_zone_list.at(i)));
3022
          } else if (OB_FAIL(allocate_pool_ptr->zone_list_.push_back(merge_zone_list.at(i)))) {
3023
            LOG_WARN("fail to push back to zone list", K(ret));
3024
          }
3025
        }
3026
      } // finish fill splitting pool basic info
3027
    }
3028
  }
3029
  return ret;
3030
}
3031

3032
int ObUnitManager::merge_pool_unit_persistent_info(
3033
    common::ObMySQLTransaction &trans,
3034
    share::ObResourcePool *new_pool,
3035
    share::ObResourcePool *orig_pool)
3036
{
3037
  int ret = OB_SUCCESS;
3038
  common::ObArray<share::ObUnit *> *units = NULL;
3039
  if (!check_inner_stat()) {
3040
    ret = OB_INNER_STAT_ERROR;
3041
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
3042
  } else if (OB_UNLIKELY(NULL == new_pool || NULL == orig_pool)) {
3043
    ret = OB_INVALID_ARGUMENT;
3044
    LOG_WARN("invalid argument", K(ret), KP(new_pool), KP(orig_pool));
3045
  } else if (OB_FAIL(get_units_by_pool(orig_pool->resource_pool_id_, units))) {
3046
    LOG_WARN("fail to get units by pool", K(ret));
3047
  } else if (OB_UNLIKELY(NULL == units)) {
3048
    ret = OB_ERR_UNEXPECTED;
3049
    LOG_WARN("units ptr is null", K(ret));
3050
  } else {
3051
    for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
3052
      const share::ObUnit *unit = units->at(i);
3053
      if (OB_UNLIKELY(NULL == unit)) {
3054
        ret = OB_ERR_UNEXPECTED;
3055
        LOG_WARN("unit ptr is null", K(ret));
3056
      } else {
3057
        share::ObUnit new_unit = *unit;
3058
        new_unit.resource_pool_id_ = new_pool->resource_pool_id_;
3059
        if (OB_FAIL(ut_operator_.update_unit(trans, new_unit, false/*need_check_conflict_with_clone*/))) {
3060
          LOG_WARN("fail to update unit", K(ret), K(new_unit));
3061
        }
3062
        ROOTSERVICE_EVENT_ADD("unit", "merge_pool",
3063
                              "unit_id", unit->unit_id_,
3064
                              "server", unit->server_,
3065
                              "prev_pool_id", orig_pool->resource_pool_id_,
3066
                              "curr_pool_id", new_pool->resource_pool_id_,K(ret));
3067
      }
3068
    }
3069
  }
3070
  return ret;
3071
}
3072

3073
int ObUnitManager::do_merge_pool_inmemory_info(
3074
    share::ObResourcePool *new_pool/*allocate_pool_ptr*/,
3075
    common::ObIArray<share::ObResourcePool*> &old_pool)
3076
{
3077
  int ret = OB_SUCCESS;
3078
  if (!check_inner_stat()) {
3079
    ret = OB_INNER_STAT_ERROR;
3080
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
3081
  } else if (NULL == new_pool) {
3082
    ret = OB_INVALID_ARGUMENT;
3083
    LOG_WARN("invalid argument", K(ret));
3084
  } else {
3085
    if (OB_FAIL(inc_config_ref_count(new_pool->unit_config_id_))) {
3086
      LOG_WARN("fail to inc config ref count", K(ret),
3087
               "unit_config_id", new_pool->unit_config_id_);
3088
    } else if (OB_FAIL(insert_config_pool(new_pool->unit_config_id_, new_pool))) {
3089
      LOG_WARN("fail to insert config pool", K(ret),
3090
               "unit_config_id", new_pool->unit_config_id_);
3091
    } else if (OB_FAIL(update_pool_map(new_pool))) {
3092
      LOG_WARN("fail to update pool map", K(ret),
3093
               "resource_pool_id", new_pool->resource_pool_id_);
3094
    } else if (!new_pool->is_granted_to_tenant()) {
3095
      // bypass
3096
    } else if (OB_FAIL(insert_tenant_pool(new_pool->tenant_id_, new_pool))) {
3097
      LOG_WARN("fail to insert tenant pool", K(ret), "tenant_id", new_pool->tenant_id_);
3098
    }
3099
    if (OB_FAIL(ret)) {
3100
      // failed
3101
    } else if (OB_FAIL(merge_pool_unit_inmemory_info(new_pool, old_pool))) {
3102
      LOG_WARN("fail to split pool unit inmemory info", K(ret));
3103
    } else {} // no more
3104
    for (int64_t i = 0 ; i < old_pool.count() && OB_SUCC(ret); ++i) {
3105
      common::ObArray<share::ObUnit *> *pool_units = NULL;
3106
      if (OB_FAIL(ret)) {
3107
      } else if (OB_FAIL(dec_config_ref_count(old_pool.at(i)->unit_config_id_))) {
3108
        LOG_WARN("fail to dec config ref count", K(ret));
3109
      } else if (OB_FAIL(delete_config_pool(old_pool.at(i)->unit_config_id_, old_pool.at(i)))) {
3110
        LOG_WARN("fail to delete config pool", K(ret));
3111
      } else if (OB_FAIL(delete_resource_pool(old_pool.at(i)->resource_pool_id_,
3112
                                              old_pool.at(i)->name_))) {
3113
        LOG_WARN("fail to delete resource pool", K(ret));
3114
      } else if (OB_INVALID_ID == old_pool.at(i)->tenant_id_) {
3115
        // bypass
3116
      } else if (OB_FAIL(delete_tenant_pool(old_pool.at(i)->tenant_id_, old_pool.at(i)))) {
3117
        LOG_WARN("fail to delete tenant pool", K(ret), "tenant_id", old_pool.at(i)->tenant_id_);
3118
      }
3119
      if (OB_FAIL(ret)) {
3120
      } else if (OB_FAIL(get_units_by_pool(old_pool.at(i)->resource_pool_id_, pool_units))) {
3121
        LOG_WARN("fail to get units by pool", K(ret), "pool_id", old_pool.at(i)->resource_pool_id_);
3122
      } else if (OB_UNLIKELY(NULL == pool_units)) {
3123
        ret = OB_ERR_UNEXPECTED;
3124
        LOG_WARN("pool units ptr is null", K(ret));
3125
      } else if (OB_FAIL(pool_unit_map_.erase_refactored(old_pool.at(i)->resource_pool_id_))) {
3126
        LOG_WARN("fail to erase map", K(ret), "pool_id", old_pool.at(i)->resource_pool_id_);
3127
      } else {
3128
        pool_unit_allocator_.free(pool_units);
3129
        pool_units = NULL;
3130
        pool_allocator_.free(old_pool.at(i));
3131
        old_pool.at(i) = NULL;
3132
      }
3133
    }
3134
    if (OB_FAIL(ret)) {
3135
      // reload
3136
      rootserver::ObRootService *root_service = NULL;
3137
      if (OB_UNLIKELY(NULL == (root_service = GCTX.root_service_))) {
3138
        ret = OB_ERR_UNEXPECTED;
3139
        LOG_WARN("rootservice is null", K(ret));
3140
      } else {
3141
        int tmp_ret = root_service->submit_reload_unit_manager_task();
3142
        if (OB_SUCCESS != tmp_ret) {
3143
          LOG_ERROR("fail to reload unit manager", K(tmp_ret));
3144
        }
3145
      }
3146
    }
3147
  }
3148
  return ret;
3149
}
3150

3151
int ObUnitManager::merge_pool_unit_inmemory_info(
3152
    share::ObResourcePool *new_pool/*allocate_pool_ptr*/,
3153
    common::ObIArray<share::ObResourcePool*> &old_pool)
3154
{
3155
  int ret = OB_SUCCESS;
3156
  common::ObArray<share::ObUnit *> *new_units = NULL;
3157
  if (!check_inner_stat()) {
3158
    ret = OB_INNER_STAT_ERROR;
3159
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
3160
  } else if (OB_UNLIKELY(NULL == new_pool)) {
3161
    ret = OB_INVALID_ARGUMENT;
3162
    LOG_WARN("invalid argument", K(ret), KP(new_pool));
3163
  } else {
3164
    ret = get_units_by_pool(new_pool->resource_pool_id_, new_units);
3165
    if (OB_SUCCESS == ret) {
3166
      // got new units
3167
    } else if (OB_ENTRY_NOT_EXIST != ret) {
3168
      LOG_WARN("fail to get units by pool", K(ret));
3169
    } else {
3170
      if (NULL == (new_units = pool_unit_allocator_.alloc())) {
3171
        ret = OB_ALLOCATE_MEMORY_FAILED;
3172
        LOG_ERROR("fail to alloc memory", K(ret));
3173
      } else if (OB_FAIL(pool_unit_map_.set_refactored(new_pool->resource_pool_id_, new_units))) {
3174
        LOG_WARN("fail to set refactored", K(ret), "pool_id", new_pool->resource_pool_id_);
3175
      }
3176
    }
3177
    for (int64_t i = 0; i < old_pool.count() && OB_SUCC(ret); ++i) {
3178
      common::ObArray<share::ObUnit *> *orig_units = NULL;
3179
      if (OB_ISNULL(old_pool.at(i))) {
3180
        ret = OB_INVALID_ARGUMENT;
3181
        LOG_WARN("invalid argument", K(ret), KP(old_pool.at(i)));
3182
      } else if (OB_FAIL(get_units_by_pool(old_pool.at(i)->resource_pool_id_, orig_units))) {
3183
        LOG_WARN("fail to get units by pool", K(ret));
3184
      } else if (OB_UNLIKELY(NULL == orig_units)) {
3185
        ret = OB_ERR_UNEXPECTED;
3186
        LOG_WARN("units ptr is null", K(ret));
3187
      } else {
3188
        for (int64_t i = 0; OB_SUCC(ret) && i < orig_units->count(); ++i) {
3189
          share::ObUnit *unit = orig_units->at(i);
3190
          if (OB_UNLIKELY(NULL == unit)) {
3191
            ret = OB_ERR_UNEXPECTED;
3192
            LOG_WARN("unit ptr is null", K(ret));
3193
          } else if (NULL == new_units) {
3194
            ret = OB_ERR_UNEXPECTED;
3195
            LOG_WARN("new units ptr is null", K(ret));
3196
          } else {
3197
            unit->resource_pool_id_ = new_pool->resource_pool_id_;
3198
            if (OB_FAIL(new_units->push_back(unit))) {
3199
              LOG_WARN("fail to push back", K(ret));
3200
            } else if (OB_FAIL(update_unit_load(unit, new_pool))) {
3201
              LOG_WARN("fail to update unit load", K(ret));
3202
            }
3203
          }
3204
        }
3205
      }
3206
    }
3207
  }
3208
  return ret;
3209
}
3210

3211
int ObUnitManager::alter_resource_pool(const share::ObResourcePool &alter_pool,
3212
                                       const ObUnitConfigName &config_name,
3213
                                       const common::ObIArray<uint64_t> &delete_unit_id_array)
3214
{
3215
  int ret = OB_SUCCESS;
3216
  LOG_INFO("start alter resource pool", K(alter_pool), K(config_name));
3217
  SpinWLockGuard guard(lock_);
3218
  share::ObResourcePool *pool = NULL;
3219
  share::ObResourcePool  pool_bak;
3220
  // don't invoke alter_pool.is_valid() here, alter_pool.unit_count may be 0
3221
  if (!check_inner_stat()) {
3222
    ret = OB_INNER_STAT_ERROR;
3223
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
3224
  } else if (alter_pool.name_.is_empty()) {
3225
    ret = OB_INVALID_ARGUMENT;
3226
    LOG_USER_ERROR(OB_INVALID_ARGUMENT, "resource pool name");
3227
    LOG_WARN("resource pool name is empty", "resource pool name", alter_pool.name_, K(ret));
3228
  } else if (alter_pool.unit_count_ < 0) {
3229
    ret = OB_INVALID_ARGUMENT;
3230
    LOG_USER_ERROR(OB_INVALID_ARGUMENT, "resource unit num");
3231
    LOG_WARN("invalid resource pool unit num", "unit num", alter_pool.unit_count_, K(ret));
3232
  } else if (OB_FAIL(inner_get_resource_pool_by_name(alter_pool.name_, pool))) {
3233
    if (OB_ENTRY_NOT_EXIST != ret) {
3234
      LOG_WARN("get resource pool by name failed", "resource_pool name", alter_pool.name_, K(ret));
3235
    } else {
3236
      ret = OB_RESOURCE_POOL_NOT_EXIST;
3237
      LOG_USER_ERROR(OB_RESOURCE_POOL_NOT_EXIST, to_cstring(alter_pool.name_));
3238
      LOG_WARN("resource pool not exist", "resource pool name", alter_pool.name_, K(ret));
3239
    }
3240
  } else if (NULL == pool) {
3241
    ret = OB_ERR_UNEXPECTED;
3242
    LOG_WARN("pool is null", KP(pool), K(ret));
3243
  } else if (REPLICA_TYPE_LOGONLY == pool->replica_type_
3244
             && alter_pool.unit_count_ > 1) {
3245
    ret = OB_NOT_SUPPORTED;
3246
    LOG_WARN("logonly resource pool should only have one unit on one zone", K(ret), K(alter_pool));
3247
  } else {
3248
    int64_t alter_count = 0;
3249
    if (!config_name.is_empty()) {
3250
      ++alter_count;
3251
    }
3252
    if (0 != alter_pool.unit_count_) {
3253
      ++alter_count;
3254
    }
3255
    if (alter_pool.zone_list_.count() > 0) {
3256
      ++alter_count;
3257
    }
3258
    if (alter_count > 1) {
3259
      ret = OB_NOT_SUPPORTED;
3260
      LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter unit_num, resource_unit, zone_list in one cmd");
3261
      LOG_WARN("only support alter one item one time", K(alter_pool), K(config_name), K(ret));
3262
    }
3263

3264
    if (OB_FAIL(ret)) {
3265
    } else if (OB_FAIL(pool_bak.assign(*pool))) {
3266
      LOG_WARN("failed to assign pool_bak", K(ret));
3267
    }
3268

3269
    //TODO: modiry replica_type not support;
3270
    // alter unit config
3271
    if (OB_FAIL(ret)) {
3272
    } else if (config_name.is_empty()) {
3273
      // config not change
3274
    } else if (OB_FAIL(alter_pool_unit_config(pool, config_name))) {
3275
      LOG_WARN("alter_pool_unit_config failed", "pool", *pool, K(config_name), K(ret));
3276
    }
3277

3278
    // alter unit num
3279
    if (OB_FAIL(ret)) {
3280
    } else if (0 == alter_pool.unit_count_) {
3281
      // unit num not change
3282
    } else if (OB_FAIL(alter_pool_unit_num(pool, alter_pool.unit_count_, delete_unit_id_array))) {
3283
      LOG_WARN("alter_pool_unit_num failed", "pool", *pool,
3284
               "unit_num", alter_pool.unit_count_, K(ret));
3285
    }
3286

3287
    // alter zone list
3288
    if (OB_FAIL(ret)) {
3289
    } else if (alter_pool.zone_list_.count() <=0) {
3290
      // zone list not change
3291
    } else if (OB_FAIL(alter_pool_zone_list(pool, alter_pool.zone_list_))) {
3292
      LOG_WARN("alter_pool_zone_list failed", "pool", *pool,
3293
               "zone_list", alter_pool.zone_list_, K(ret));
3294
    }
3295
  }
3296
  if (OB_SUCC(ret)) {
3297
    ROOTSERVICE_EVENT_ADD("unit", "alter_resource_pool",
3298
                          "name", pool_bak.name_);
3299
  }
3300
  LOG_INFO("finish alter resource pool", K(alter_pool), K(config_name), K(ret));
3301
  return ret;
3302
}
3303

3304
int ObUnitManager::drop_resource_pool(const uint64_t pool_id, const bool if_exist)
3305
{
3306
  int ret = OB_SUCCESS;
3307
  LOG_INFO("start drop resource pool", K(pool_id));
3308
  SpinWLockGuard guard(lock_);
3309
  share::ObResourcePool *pool = NULL;
3310
  if (!check_inner_stat()) {
3311
    ret = OB_INNER_STAT_ERROR;
3312
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
3313
  } else if (OB_INVALID_ID == pool_id) {
3314
    ret = OB_INVALID_ARGUMENT;
3315
    LOG_WARN("invalid argument", K(pool_id), K(ret));
3316
  } else if (OB_FAIL(get_resource_pool_by_id(pool_id, pool))) {
3317
    if (OB_ENTRY_NOT_EXIST != ret) {
3318
      LOG_WARN("get resource pool by id failed", K(pool_id), K(ret));
3319
    } else {
3320
      if (if_exist) {
3321
        ret = OB_SUCCESS;
3322
        LOG_USER_NOTE(OB_RESOURCE_POOL_NOT_EXIST, to_cstring(pool_id));
3323
        LOG_INFO("resource_pool not exist, but no need drop it", K(pool_id));
3324
      } else {
3325
        ret = OB_RESOURCE_POOL_NOT_EXIST;
3326
        LOG_USER_ERROR(OB_RESOURCE_POOL_NOT_EXIST, to_cstring(pool_id));
3327
        LOG_WARN("resource_pool not exist", K(pool_id), K(ret));
3328
      }
3329
    }
3330
  } else if (OB_FAIL(inner_drop_resource_pool(pool))) {
3331
    LOG_WARN("fail to inner drop resource pool", KR(ret), K(pool_id));
3332
  }
3333
  LOG_INFO("finish drop resource pool", K(pool_id), K(ret));
3334
  return ret;
3335
}
3336

3337
int ObUnitManager::drop_resource_pool(const ObResourcePoolName &name, const bool if_exist)
3338
{
3339
  int ret = OB_SUCCESS;
3340
  LOG_INFO("start drop resource pool", K(name));
3341
  SpinWLockGuard guard(lock_);
3342
  share::ObResourcePool *pool = NULL;
3343
  if (!check_inner_stat()) {
3344
    ret = OB_INNER_STAT_ERROR;
3345
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
3346
  } else if (name.is_empty()) {
3347
    ret = OB_INVALID_ARGUMENT;
3348
    LOG_USER_ERROR(OB_INVALID_ARGUMENT, "resource pool name");
3349
    LOG_WARN("invalid argument", K(name), K(ret));
3350
  } else if (OB_FAIL(inner_get_resource_pool_by_name(name, pool))) {
3351
    if (OB_ENTRY_NOT_EXIST != ret) {
3352
      LOG_WARN("get resource pool by name failed", K(name), K(ret));
3353
    } else {
3354
      if (if_exist) {
3355
        ret = OB_SUCCESS;
3356
        LOG_USER_NOTE(OB_RESOURCE_POOL_NOT_EXIST, to_cstring(name));
3357
        LOG_INFO("resource_pool not exist, but no need drop it", K(name));
3358
      } else {
3359
        ret = OB_RESOURCE_POOL_NOT_EXIST;
3360
        LOG_USER_ERROR(OB_RESOURCE_POOL_NOT_EXIST, to_cstring(name));
3361
        LOG_WARN("resource_pool not exist", K(name), K(ret));
3362
      }
3363
    }
3364
  } else if (OB_FAIL(inner_drop_resource_pool(pool))) {
3365
    LOG_WARN("fail to inner drop resource pool", KR(ret), K(name));
3366
  }
3367
  LOG_INFO("finish drop resource pool", K(name), K(ret));
3368
  return ret;
3369
}
3370

3371
int ObUnitManager::remove_resource_pool_unit_in_trans(const int64_t resource_pool_id,
3372
                                                     ObMySQLTransaction &trans)
3373
{
3374
  int ret = OB_SUCCESS;
3375
  int migrate_unit_ret = OB_CANCELED;
3376
  if (OB_INVALID_ID == resource_pool_id
3377
      || !trans.is_started()) {
3378
    ret = OB_INVALID_ARGUMENT;
3379
    LOG_WARN("invalid argument", K(ret), K(resource_pool_id),
3380
             "is_started", trans.is_started());
3381
  } else if (!check_inner_stat()) {
3382
    ret = OB_INNER_STAT_ERROR;
3383
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
3384
  } else if (OB_FAIL(ut_operator_.remove_units(trans, resource_pool_id))) {
3385
    LOG_WARN("remove_units failed", K(ret), K(resource_pool_id));
3386
  } else if (OB_FAIL(ut_operator_.remove_resource_pool(trans, resource_pool_id))) {
3387
    LOG_WARN("remove_resource_pool failed", K(ret), K(resource_pool_id));
3388
  } else if (OB_FAIL(complete_migrate_unit_rs_job_in_pool(resource_pool_id,
3389
                                                          migrate_unit_ret,
3390
                                                          trans))) {
3391
    LOG_WARN("failed to complete migrate unit in pool", K(ret), K(resource_pool_id));
3392
  }
3393
  return ret;
3394
}
3395
// site lock at the call
3396
int ObUnitManager::delete_resource_pool_unit(share::ObResourcePool *pool)
3397
{
3398
  int ret = OB_SUCCESS;
3399
  if (OB_ISNULL(pool)) {
3400
    ret = OB_INVALID_ARGUMENT;
3401
    LOG_WARN("pool is null or invalid", K(ret), K(pool));
3402
  } else if (!check_inner_stat()) {
3403
    ret = OB_INNER_STAT_ERROR;
3404
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
3405
  } else {
3406
    if (OB_FAIL(delete_units_of_pool(pool->resource_pool_id_))) {
3407
      LOG_WARN("delete_units_of_pool failed", "pool id", pool->resource_pool_id_, K(ret));
3408
    } else if (OB_FAIL(delete_resource_pool(pool->resource_pool_id_, pool->name_))) {
3409
      LOG_WARN("delete_resource_pool failed", "pool", *pool, K(ret));
3410
    } else {
3411
      const uint64_t config_id = pool->unit_config_id_;
3412
      if (OB_FAIL(dec_config_ref_count(config_id))) {
3413
        LOG_WARN("dec_config_ref_count failed", K(config_id), K(ret));
3414
      } else if (OB_FAIL(delete_config_pool(config_id, pool))) {
3415
        LOG_WARN("delete config pool failed", K(config_id), "pool", *pool, K(ret));
3416
      } else {
3417
        ROOTSERVICE_EVENT_ADD("unit", "drop_resource_pool",
3418
                              "name", pool->name_);
3419
      }
3420
      pool_allocator_.free(pool);
3421
      pool = NULL;
3422
    }
3423
  }
3424
  return ret;
3425
}
3426

3427
//After the 14x version,
3428
//the same tenant is allowed to have multiple unit specifications in a zone,
3429
//but it is necessary to ensure that these units can be scattered on each server in the zone,
3430
//and multiple units of the same tenant cannot be located on the same machine;
3431
int ObUnitManager::check_server_enough(const uint64_t tenant_id,
3432
                                       const ObIArray<ObResourcePoolName> &pool_names,
3433
                                       bool &enough)
3434
{
3435
  int ret = OB_SUCCESS;
3436
  enough = true;
3437
  share::ObResourcePool *pool = NULL;
3438
  ObArray<ObUnitInfo> unit_infos;
3439
  ObArray<ObUnitInfo> total_unit_infos;
3440
  common::ObArray<share::ObResourcePool *> *pools = NULL;;
3441
  if (!check_inner_stat()) {
3442
    ret = OB_INNER_STAT_ERROR;
3443
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
3444
  } else if (pool_names.count() <= 0 || !is_valid_tenant_id(tenant_id)) {
3445
    ret = OB_INVALID_ARGUMENT;
3446
    LOG_WARN("invalid argument", K(pool_names), K(tenant_id), K(ret));
3447
  } else {
3448
    //Count the number of newly added units
3449
    for (int64_t i = 0; i < pool_names.count() && OB_SUCC(ret); i++) {
3450
      unit_infos.reset();
3451
      if (OB_FAIL(inner_get_resource_pool_by_name(pool_names.at(i), pool))) {
3452
        LOG_WARN("fail to get resource pool by name", K(ret), "pool_name", pool_names.at(i));
3453
      } else if (OB_ISNULL(pool)) {
3454
        ret = OB_ERR_UNEXPECTED;
3455
        LOG_WARN("get invalid pool", K(ret), K(pool));
3456
      } else if (OB_FAIL(inner_get_unit_infos_of_pool_(pool->resource_pool_id_, unit_infos))) {
3457
        LOG_WARN("fail to get unit infos", K(ret), K(*pool));
3458
      } else {
3459
        for (int64_t j = 0; j < unit_infos.count() && OB_SUCC(ret); j++) {
3460
          if (OB_FAIL(total_unit_infos.push_back(unit_infos.at(j)))) {
3461
            LOG_WARN("fail to push back unit", K(ret), K(total_unit_infos), K(j), K(unit_infos));
3462
          } else {
3463
            LOG_DEBUG("add unit infos", K(ret), K(total_unit_infos), K(unit_infos));
3464
          }
3465
        }
3466
      } //end else
3467
    } // end for
3468
  }
3469
  //Count the number of existing units
3470
  if (FAILEDx(get_pools_by_tenant_(tenant_id, pools))) {
3471
    if (OB_ENTRY_NOT_EXIST == ret) {
3472
      // a new tenant, without resource pool already granted
3473
      ret = OB_SUCCESS;
3474
    } else {
3475
      LOG_WARN("fail to get pools by tenant", K(ret), K(tenant_id));
3476
    }
3477
  } else if (OB_UNLIKELY(NULL == pools)) {
3478
    ret = OB_ERR_UNEXPECTED;
3479
    LOG_WARN("pools is null", K(ret), KP(pools));
3480
  } else {
3481
    for (int64_t i = 0; i < pools->count() && OB_SUCC(ret); i++) {
3482
      unit_infos.reset();
3483
      const share::ObResourcePool *pool = pools->at(i);
3484
      if (OB_UNLIKELY(NULL == pool)) {
3485
        ret = OB_ERR_UNEXPECTED;
3486
        LOG_WARN("pool ptr is null", K(ret), KP(pool));
3487
      } else if (OB_FAIL(inner_get_unit_infos_of_pool_(pool->resource_pool_id_, unit_infos))) {
3488
        LOG_WARN("fail to get unit infos", K(ret), K(*pool));
3489
      } else {
3490
        for (int64_t j = 0; j < unit_infos.count() && OB_SUCC(ret); j++) {
3491
          if (OB_FAIL(total_unit_infos.push_back(unit_infos.at(j)))) {
3492
            LOG_WARN("fail to push back unit", K(ret), K(total_unit_infos), K(j), K(unit_infos));
3493
          } else {
3494
            LOG_WARN("add unit infos", K(ret), K(total_unit_infos), K(unit_infos));
3495
          }
3496
        }
3497
      }
3498
    }
3499
  }
3500
  ObArray<ObZoneInfo> zone_infos;
3501
  if (FAILEDx(zone_mgr_.get_zone(zone_infos))) {
3502
    LOG_WARN("fail to get zone infos", K(ret));
3503
  } else {
3504
    //Count the number of units in zone
3505
    for (int64_t i = 0; i < zone_infos.count() && OB_SUCC(ret) && enough; i++) {
3506
      const ObZone &zone = zone_infos.at(i).zone_;
3507
      int64_t unit_count = 0;
3508
      int64_t alive_server_count = 0;
3509
      for (int64_t j = 0; j < total_unit_infos.count() && OB_SUCC(ret); j++) {
3510
        if (total_unit_infos.at(j).unit_.zone_ == zone) {
3511
          unit_count ++;
3512
        }
3513
      }
3514
      if (unit_count > 0) {
3515
        if (OB_FAIL(SVR_TRACER.get_alive_servers_count(zone, alive_server_count))) {
3516
          LOG_WARN("fail to get alive server count", KR(ret), K(zone));
3517
        } else if (alive_server_count < unit_count) {
3518
          //ret = OB_UNIT_NUM_OVER_SERVER_COUNT;
3519
          enough = false;
3520
          LOG_WARN("resource pool unit num over zone server count", K(ret), K(unit_count), K(alive_server_count),
3521
                   K(total_unit_infos));
3522
        }
3523
      }
3524
    }
3525
  }
3526
  return ret;
3527
}
3528

3529
//The F/L scheme has new restrictions.
3530
//If the logonly replica exists in the locality before adding the Logonly pool, the change is not allowed
3531
int ObUnitManager::check_locality_for_logonly_unit(const share::schema::ObTenantSchema &tenant_schema,
3532
                                                   const ObIArray<ObResourcePoolName> &pool_names,
3533
                                                   bool &is_permitted)
3534
{
3535
  int ret = OB_SUCCESS;
3536
  is_permitted = true;
3537
  ObArray<ObZone> zone_with_logonly_unit;
3538
  ObArray<share::ObZoneReplicaNumSet> zone_locality;
3539
  if (!check_inner_stat()) {
3540
    ret = OB_INNER_STAT_ERROR;
3541
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
3542
  } else if (pool_names.count() <= 0) {
3543
    ret = OB_INVALID_ARGUMENT;
3544
    LOG_WARN("invalid argument", K(pool_names), K(ret));
3545
  } else {
3546
    FOREACH_CNT_X(pool_name, pool_names, OB_SUCCESS == ret) {
3547
      share::ObResourcePool *pool = NULL;
3548
      if (OB_FAIL(inner_get_resource_pool_by_name(*pool_name, pool))) {
3549
        LOG_WARN("get resource pool by name failed", "pool_name", *pool_name, K(ret));
3550
      } else if (NULL == pool) {
3551
        ret = OB_ERR_UNEXPECTED;
3552
        LOG_WARN("pool is null", KP(pool), K(ret));
3553
      } else if (REPLICA_TYPE_LOGONLY != pool->replica_type_) {
3554
        //nothing todo
3555
      } else {
3556
        for (int64_t i = 0; i < pool->zone_list_.count() && OB_SUCC(ret); i++) {
3557
          if (OB_FAIL(zone_with_logonly_unit.push_back(pool->zone_list_.at(i)))) {
3558
            LOG_WARN("fail to push back", K(ret));
3559
          }
3560
        }
3561
      }
3562
    }
3563
  }
3564
  if (OB_FAIL(ret)) {
3565
  } else if (OB_FAIL(tenant_schema.get_zone_replica_attr_array(zone_locality))) {
3566
    LOG_WARN("fail to get zone replica attr array", K(ret));
3567
  } else {
3568
    for (int64_t i = 0; i < zone_locality.count() && OB_SUCC(ret); i++) {
3569
      if ((zone_locality.at(i).replica_attr_set_.get_logonly_replica_num() == 1
3570
           || zone_locality.at(i).replica_attr_set_.get_encryption_logonly_replica_num() == 1)
3571
          && has_exist_in_array(zone_with_logonly_unit, zone_locality.at(i).zone_)) {
3572
        is_permitted = false;
3573
        ret = OB_NOT_SUPPORTED;
3574
        LOG_WARN("logonly replica already exist before logonly pool create", K(ret), K(zone_locality),
3575
                 K(zone_with_logonly_unit));
3576
      }
3577
    }
3578
  }
3579
  return ret;
3580
}
3581

3582
/* when expand zone resource for tenant this func is invoked,
3583
 * we need to check whether the tenant units are in deleting.
3584
 * if any tenant unit is in deleting,
3585
 * @is_allowed returns false
3586
 */
3587
int ObUnitManager::check_expand_zone_resource_allowed_by_old_unit_stat_(
3588
    const uint64_t tenant_id,
3589
    bool &is_allowed)
3590
{
3591
  int ret = OB_SUCCESS;
3592
  ObArray<share::ObResourcePool *> *cur_pool_array = nullptr;
3593
  if (OB_UNLIKELY(!check_inner_stat())) {
3594
    ret = OB_INNER_STAT_ERROR;
3595
    LOG_WARN("fail to check inner stat", KR(ret));
3596
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
3597
    ret = OB_INVALID_ARGUMENT;
3598
    LOG_WARN("invalid argument", KR(ret), K(tenant_id));
3599
  } else {
3600
    int tmp_ret = get_pools_by_tenant_(tenant_id, cur_pool_array);
3601
    if (OB_ENTRY_NOT_EXIST == tmp_ret) {
3602
      is_allowed = true;
3603
    } else if (OB_UNLIKELY(nullptr == cur_pool_array)) {
3604
      ret = OB_ERR_UNEXPECTED;
3605
      LOG_WARN("cur_pool is null", KR(ret), K(tenant_id));
3606
    } else {
3607
      is_allowed = true;
3608
      for (int64_t i = 0; is_allowed && OB_SUCC(ret) && i < cur_pool_array->count(); ++i) {
3609
        share::ObResourcePool *cur_pool = cur_pool_array->at(i);
3610
        ObArray<share::ObUnit *> *units = nullptr;
3611
        if (OB_UNLIKELY(nullptr == cur_pool)) {
3612
          ret = OB_ERR_UNEXPECTED;
3613
          LOG_WARN("cur pool is null", KR(ret));
3614
        } else if (OB_FAIL(get_units_by_pool(cur_pool->resource_pool_id_, units))) {
3615
          LOG_WARN("fail to get units by pool", KR(ret),
3616
                   "pool_id", cur_pool->resource_pool_id_);
3617
        } else if (OB_UNLIKELY(nullptr == units)) {
3618
          ret = OB_ERR_UNEXPECTED;
3619
          LOG_WARN("units ptrs is null", KR(ret), K(tenant_id), KPC(cur_pool));
3620
        } else {
3621
          for (int64_t j = 0; is_allowed && OB_SUCC(ret) && j < units->count(); ++j) {
3622
            if (OB_UNLIKELY(nullptr == units->at(j))) {
3623
              ret = OB_ERR_UNEXPECTED;
3624
              LOG_WARN("units ptrs is null", KR(ret), K(tenant_id), KPC(cur_pool));
3625
            } else {
3626
              is_allowed = ObUnit::UNIT_STATUS_ACTIVE == units->at(j)->status_;
3627
            }
3628
          }
3629
        }
3630
      }
3631
    }
3632
  }
3633
  return ret;
3634
}
3635

3636
int ObUnitManager::check_expand_zone_resource_allowed_by_new_unit_stat_(
3637
    const common::ObIArray<share::ObResourcePoolName> &pool_names)
3638
{
3639
  int ret = OB_SUCCESS;
3640
  for (int64_t i = 0; OB_SUCC(ret) && i < pool_names.count(); ++i) {
3641
    share::ObResourcePool *pool = NULL;
3642
    ObArray<ObUnit *> *units = nullptr;
3643
    if (OB_FAIL(inner_get_resource_pool_by_name(pool_names.at(i), pool))) {
3644
      LOG_WARN("get resource pool by name failed", "pool_name", pool_names.at(i), K(ret));
3645
    } else if (NULL == pool) {
3646
      ret = OB_ERR_UNEXPECTED;
3647
      LOG_WARN("pool is null", KP(pool), K(ret));
3648
    } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
3649
      LOG_WARN("fail to get units by pool", K(ret));
3650
    } else if (OB_UNLIKELY(nullptr == units)) {
3651
      ret = OB_ERR_UNEXPECTED;
3652
      LOG_WARN("units ptr is null", K(ret));
3653
    } else {
3654
      for (int64_t j = 0; OB_SUCC(ret) && j < units->count(); ++j) {
3655
        ObUnit *unit = units->at(j);
3656
        if (OB_UNLIKELY(nullptr == unit)) {
3657
          ret = OB_ERR_UNEXPECTED;
3658
          LOG_WARN("unit ptr is null", KR(ret), KPC(pool));
3659
        } else if (unit->status_ != ObUnit::UNIT_STATUS_ACTIVE) {
3660
          ret = OB_ERR_UNEXPECTED;
3661
          LOG_WARN("unexpected unit status", KR(ret), KPC(pool), KPC(unit));
3662
        } else {/* good */}
3663
      }
3664
    }
3665
  }
3666
  return ret;
3667
}
3668

3669
/* 1 when this is a tenant being created:
3670
 *   check the input pools, each input pool unit num shall be equal, otherwise illegal
3671
 * 2 when this is a tenant which exists:
3672
 *   check the input pools, each input pool unit num shall be equal to the pools already granted to the tenant.
3673
 */
3674
int ObUnitManager::check_tenant_pools_unit_num_legal_(
3675
    const uint64_t tenant_id,
3676
    const common::ObIArray<share::ObResourcePoolName> &input_pool_names,
3677
    bool &unit_num_legal,
3678
    int64_t &sample_unit_num)
3679
{
3680
  int ret = OB_SUCCESS;
3681
  ObArray<share::ObResourcePool *> *cur_pool_array = nullptr;
3682
  if (OB_UNLIKELY(!check_inner_stat())) {
3683
    ret = OB_INNER_STAT_ERROR;
3684
    LOG_WARN("fail to check inner stat", KR(ret));
3685
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id)
3686
                        || input_pool_names.count() <= 0)) {
3687
    ret = OB_INVALID_ARGUMENT;
3688
    LOG_WARN("invalid argument", KR(ret));
3689
  } else {
3690
    unit_num_legal = true;
3691
    sample_unit_num = -1;
3692
    int tmp_ret = get_pools_by_tenant_(tenant_id, cur_pool_array);
3693
    if (OB_ENTRY_NOT_EXIST == tmp_ret) {
3694
      // when create tenant pools belong to this tenant is empty
3695
    } else if (OB_UNLIKELY(nullptr == cur_pool_array)) {
3696
      ret = OB_ERR_UNEXPECTED;
3697
      LOG_WARN("cur_pool is null", KR(ret), K(tenant_id));
3698
    } else if (OB_UNLIKELY(cur_pool_array->count() <= 0)) {
3699
      ret = OB_ERR_UNEXPECTED;
3700
      LOG_WARN("pool_array", KR(ret));
3701
    } else if (OB_UNLIKELY(nullptr == cur_pool_array->at(0))) {
3702
      ret = OB_ERR_UNEXPECTED;
3703
      LOG_WARN("pool ptr is null", KR(ret));
3704
    } else {
3705
      sample_unit_num = cur_pool_array->at(0)->unit_count_;
3706
    }
3707
    for (int64_t i = 0; OB_SUCC(ret) && unit_num_legal && i < input_pool_names.count(); ++i) {
3708
      share::ObResourcePool *pool = NULL;
3709
      if (OB_FAIL(inner_get_resource_pool_by_name(input_pool_names.at(i), pool))) {
3710
        LOG_WARN("fail to get pool by name", KR(ret), "pool_name", input_pool_names.at(i));
3711
      } else if (OB_UNLIKELY(nullptr == pool)) {
3712
        ret = OB_ERR_UNEXPECTED;
3713
        LOG_WARN("pool ptr is null", KR(ret), "pool_name", input_pool_names.at(i));
3714
      } else if (-1 == sample_unit_num) {
3715
        sample_unit_num = pool->unit_count_;
3716
      } else if (sample_unit_num == pool->unit_count_) {
3717
        // this is good, unit num matched
3718
      } else {
3719
        unit_num_legal = false;
3720
      }
3721
    }
3722
  }
3723
  return ret;
3724
}
3725

3726
int ObUnitManager::get_pool_unit_group_id_(
3727
    const share::ObResourcePool &pool,
3728
    common::ObIArray<uint64_t> &new_unit_group_id_array)
3729
{
3730
  int ret = OB_SUCCESS;
3731
  if (!pool.is_granted_to_tenant()) {
3732
    for (int64_t i = 0; OB_SUCC(ret) && i < pool.unit_count_; ++i) {
3733
      if (OB_FAIL(new_unit_group_id_array.push_back(0/*not granted to tenant*/))) {
3734
        LOG_WARN("fail to push back", KR(ret));
3735
      }
3736
    }
3737
  } else {
3738
    const bool is_active = false;
3739
    if (OB_FAIL(inner_get_all_unit_group_id(pool.tenant_id_, is_active, new_unit_group_id_array))) {
3740
      LOG_WARN("fail to get all unit group id", KR(ret), K(pool));
3741
    }
3742
  }
3743
  return ret;
3744
}
3745

3746
/* get unit group id for a tenant
3747
 * 1 when bootstrap:
3748
 *   generate unit group id for sys unit group, assign 0 directly
3749
 * 2 when revoke pools for tenant:
3750
 *   after revokes pools, unit group id shall be set to 0 which representing these units
3751
 *   is not belong to any unit group
3752
 * 3 when grant pools for tenant:
3753
 *   3.1 when this is a tenant being created: fetch unit group id from inner table
3754
 *   3.2 when this is a tenant which exists: get unit group id from units granted to this tenant
3755
 */
3756
int ObUnitManager::get_tenant_pool_unit_group_id_(
3757
    const bool is_bootstrap,
3758
    const bool grant,
3759
    const uint64_t tenant_id,
3760
    const int64_t unit_group_num,
3761
    common::ObIArray<uint64_t> &new_unit_group_id_array)
3762
{
3763
  int ret = OB_SUCCESS;
3764
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || unit_group_num <= 0)) {
3765
    ret = OB_INVALID_ARGUMENT;
3766
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(unit_group_num));
3767
  } else if (is_bootstrap) {
3768
    if (OB_FAIL(new_unit_group_id_array.push_back(OB_SYS_UNIT_GROUP_ID))) {
3769
      LOG_WARN("fail to push back", KR(ret));
3770
    }
3771
  } else {
3772
    if (!grant) {
3773
      // when revoke pools, unit_group_id in units shall be modified to 0
3774
      for (int64_t i = 0; OB_SUCC(ret) && i < unit_group_num; ++i) {
3775
        if (OB_FAIL(new_unit_group_id_array.push_back(
3776
                0/* 0 means this unit doesn't belong to any unit group*/))) {
3777
          LOG_WARN("fail to push back", KR(ret));
3778
        }
3779
      }
3780
    } else {
3781
      // when grant pools, an unit group id greater than 0 is needed for every unit
3782
      ObArray<share::ObResourcePool *> *tenant_pool_array = nullptr;
3783
      int tmp_ret = get_pools_by_tenant_(tenant_id, tenant_pool_array);
3784
      if (OB_ENTRY_NOT_EXIST == tmp_ret) {
3785
        // need to fetch unit group from inner table, since this is invoked by create tenant
3786
        for (int64_t i = 0; OB_SUCC(ret) && i < unit_group_num; ++i) {
3787
          uint64_t unit_group_id = OB_INVALID_ID;
3788
          if (OB_FAIL(fetch_new_unit_group_id(unit_group_id))) {
3789
            LOG_WARN("fail to fetch new unit group id", KR(ret));
3790
          } else if (OB_FAIL(new_unit_group_id_array.push_back(unit_group_id))) {
3791
            LOG_WARN("fail to push back", KR(ret));
3792
          }
3793
        }
3794
      } else {
3795
        const bool is_active = false;
3796
        if (OB_FAIL(inner_get_all_unit_group_id(tenant_id, is_active, new_unit_group_id_array))) {
3797
          LOG_WARN("fail to get all unit group id array", KR(ret), K(tenant_id));
3798
        }
3799
      }
3800
    }
3801
  }
3802
  return ret;
3803
}
3804

3805
int ObUnitManager::inner_get_all_unit_group_id(
3806
    const uint64_t tenant_id,
3807
    const bool is_active,
3808
    common::ObIArray<uint64_t> &unit_group_array)
3809
{
3810
  int ret = OB_SUCCESS;
3811
  if (OB_UNLIKELY(!check_inner_stat())) {
3812
    ret = OB_INNER_STAT_ERROR;
3813
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
3814
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
3815
    ret = OB_INVALID_ARGUMENT;
3816
    LOG_WARN("invalid argument", KR(ret), K(tenant_id));
3817
  } else {
3818
    ObArray<share::ObResourcePool *> *cur_pool_array = nullptr;
3819
    if (OB_FAIL(get_pools_by_tenant_(tenant_id, cur_pool_array))) {
3820
      if (OB_ENTRY_NOT_EXIST == ret) {
3821
        // bypass
3822
        ret = OB_SUCCESS;
3823
      } else {
3824
        LOG_WARN("fail to get pools by tenant", KR(ret), K(tenant_id));
3825
      }
3826
    } else if (OB_ISNULL(cur_pool_array)) {
3827
      ret = OB_ERR_UNEXPECTED;
3828
      LOG_WARN("cur pool array ptr is null", KR(ret), K(tenant_id));
3829
    } else if (cur_pool_array->count() <= 0) {
3830
      ret = OB_ERR_UNEXPECTED;
3831
      LOG_WARN("cur_pool_array is empty", KR(ret), K(tenant_id));
3832
    } else {
3833
      share::ObResourcePool *cur_pool = cur_pool_array->at(0);
3834
      common::ObArray<ObUnit *> zone_sorted_unit_array;
3835
      if (OB_ISNULL(cur_pool)) {
3836
        ret = OB_ERR_UNEXPECTED;
3837
        LOG_WARN("cur_pool ptr is null", KR(ret), KP(cur_pool));
3838
      } else if (OB_FAIL(build_zone_sorted_unit_array_(cur_pool, zone_sorted_unit_array))) {
3839
        LOG_WARN("fail to build zone_sorted_unit_array", KR(ret), K(cur_pool));
3840
      } else {
3841
        for (int64_t j = 0; OB_SUCC(ret) && j < zone_sorted_unit_array.count(); ++j) {
3842
          ObUnit *unit = zone_sorted_unit_array.at(j);
3843
          if (OB_ISNULL(unit)) {
3844
            ret = OB_ERR_UNEXPECTED;
3845
            LOG_WARN("unit ptr is null", KR(ret), K(tenant_id), KPC(cur_pool), K(j));
3846
          } else if (has_exist_in_array(unit_group_array, unit->unit_group_id_)) {
3847
            //unit_count_ is small than unit group count while some unit is in deleting
3848
            break;
3849
          } else if (is_active && ObUnit::UNIT_STATUS_ACTIVE != unit->status_) {
3850
            //need active unit group
3851
            continue;
3852
          } else if (OB_FAIL(unit_group_array.push_back(unit->unit_group_id_))) {
3853
            LOG_WARN("fail to push back", KR(ret), K(unit));
3854
          }
3855
        }
3856
        if (OB_SUCC(ret) && OB_UNLIKELY(unit_group_array.count() < cur_pool->unit_count_)) {
3857
          ret = OB_ERR_UNEXPECTED;
3858
          LOG_WARN("pool unit count unexpected", KR(ret), KPC(cur_pool),
3859
                   K(unit_group_array));
3860
        }
3861
      }
3862
    }
3863
  }
3864
  return ret;
3865
}
3866

3867
int ObUnitManager::get_unit_in_group(
3868
    const uint64_t tenant_id,
3869
    const uint64_t unit_group_id,
3870
    const common::ObZone &zone,
3871
    share::ObUnitInfo &unit_info)
3872
{
3873
  int ret = OB_SUCCESS;
3874
  common::ObArray<share::ObUnitInfo> unit_info_array;
3875
  if (OB_UNLIKELY(zone.is_empty())) {
3876
    ret = OB_INVALID_ARGUMENT;
3877
    LOG_WARN("invalid argument", KR(ret));
3878
  } else if (OB_FAIL(get_unit_group(
3879
          tenant_id,
3880
          unit_group_id,
3881
          unit_info_array))) {
3882
    LOG_WARN("fail to get unit group", KR(ret), K(tenant_id), K(unit_group_id));
3883
  } else {
3884
    bool found = false;
3885
    for (int64_t i = 0; !found && OB_SUCC(ret) && i < unit_info_array.count(); ++i) {
3886
      const share::ObUnitInfo &this_unit_info = unit_info_array.at(i);
3887
      if (this_unit_info.unit_.zone_ != zone) {
3888
        // bypass
3889
      } else if (OB_FAIL(unit_info.assign(this_unit_info))) {
3890
        LOG_WARN("fail to assign unit info", KR(ret));
3891
      } else {
3892
        found = true;
3893
      }
3894
    }
3895
    if (OB_FAIL(ret)) {
3896
      // failed
3897
    } else if (!found) {
3898
      ret = OB_ENTRY_NOT_EXIST;
3899
    } else {} // good
3900
  }
3901
  return ret;
3902
}
3903

3904
int ObUnitManager::get_unit_group(
3905
    const uint64_t tenant_id,
3906
    const uint64_t unit_group_id,
3907
    common::ObIArray<share::ObUnitInfo> &unit_info_array)
3908
{
3909
  int ret = OB_SUCCESS;
3910
  SpinWLockGuard guard(lock_);
3911
  if (OB_UNLIKELY(!check_inner_stat())) {
3912
    ret = OB_INNER_STAT_ERROR;
3913
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
3914
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id)
3915
                         || 0 == unit_group_id
3916
                         || OB_INVALID_ID == unit_group_id)) {
3917
    ret = OB_INVALID_ARGUMENT;
3918
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(unit_group_id));
3919
  } else {
3920
    ObArray<share::ObResourcePool *> *cur_pool_array = nullptr;
3921
    ret = get_pools_by_tenant_(tenant_id, cur_pool_array);
3922
    if (OB_ENTRY_NOT_EXIST == ret) {
3923
      // bypass
3924
    } else if (OB_SUCCESS != ret) {
3925
      LOG_WARN("fail to get unit group", KR(ret), K(tenant_id), K(unit_group_id));
3926
    } else if (OB_UNLIKELY(nullptr == cur_pool_array)) {
3927
      ret = OB_ERR_UNEXPECTED;
3928
      LOG_WARN("cur pool array ptr is null", KR(ret), K(tenant_id), K(unit_group_id));
3929
    } else {
3930
      ObUnitInfo unit_info;
3931
      for (int64_t i = 0; OB_SUCC(ret) && i < cur_pool_array->count(); ++i) {
3932
        share::ObResourcePool *cur_pool = cur_pool_array->at(i);
3933
        ObArray<share::ObUnit *> *units = nullptr;
3934
        ObUnitConfig *config = nullptr;
3935
        if (OB_UNLIKELY(nullptr == cur_pool)) {
3936
          ret = OB_ERR_UNEXPECTED;
3937
          LOG_WARN("cur pool is null", KR(ret));
3938
        } else if (OB_FAIL(get_unit_config_by_id(cur_pool->unit_config_id_, config))) {
3939
          LOG_WARN("fail to get pool unit config", KR(ret), K(tenant_id), KPC(cur_pool));
3940
        } else if (OB_FAIL(get_units_by_pool(cur_pool->resource_pool_id_, units))) {
3941
          LOG_WARN("fail to get units by pool", KR(ret),
3942
                   "pool_id", cur_pool->resource_pool_id_);
3943
        } else if (OB_UNLIKELY(nullptr == units)) {
3944
          ret = OB_ERR_UNEXPECTED;
3945
          LOG_WARN("units ptrs is null", KR(ret), K(tenant_id), KPC(cur_pool));
3946
        } else {
3947
          for (int64_t j = 0; OB_SUCC(ret) && j < units->count(); ++j) {
3948
            if (OB_UNLIKELY(nullptr == units->at(j))) {
3949
              ret = OB_ERR_UNEXPECTED;
3950
              LOG_WARN("units ptrs is null", KR(ret), K(tenant_id), KPC(cur_pool));
3951
            } else if (units->at(j)->unit_group_id_ != unit_group_id) {
3952
              // unit group id not match
3953
            } else {
3954
              unit_info.reset();
3955
              unit_info.config_ = *config;
3956
              unit_info.unit_ = *(units->at(j));
3957
              if (OB_FAIL(unit_info.pool_.assign(*cur_pool))) {
3958
                LOG_WARN("fail to assign", KR(ret), KPC(cur_pool));
3959
              } else if (OB_FAIL(unit_info_array.push_back(unit_info))) {
3960
                LOG_WARN("fail to push back", K(unit_info));
3961
              }
3962
            }
3963
          }
3964
        }
3965
      }
3966
      if (OB_SUCC(ret)) {
3967
        if (unit_info_array.count() <= 0) {
3968
          ret = OB_ENTRY_NOT_EXIST;
3969
        }
3970
      }
3971
    }
3972
  }
3973
  return ret;
3974
}
3975

3976
// TODO(cangming.zl): need a new function to abstract the process of getting pools by pool_names
3977
// TODO: disable resource pools intersect for one tenant
3978
//       NEED to disable logics to handle resource pool intersect in server_balancer
3979
int ObUnitManager::grant_pools(common::ObMySQLTransaction &trans,
3980
                               common::ObIArray<uint64_t> &new_unit_group_id_array,
3981
                               const lib::Worker::CompatMode compat_mode,
3982
                               const ObIArray<ObResourcePoolName> &pool_names,
3983
                               const uint64_t tenant_id,
3984
                               const bool is_bootstrap,
3985
                               const uint64_t source_tenant_id,
3986
                               /*arg "const bool skip_offline_server" is no longer supported*/
3987
                               const bool check_data_version)
3988
{
3989
  int ret = OB_SUCCESS;
3990
  SpinWLockGuard guard(lock_);
3991
  const bool grant = true;
3992
  bool intersect = false;
3993
  bool server_enough = true;
3994
  bool is_grant_pool_allowed = false;
3995
  bool unit_num_legal = false;
3996
  int64_t legal_unit_num = -1;
3997
  if (!check_inner_stat()) {
3998
    ret = OB_INNER_STAT_ERROR;
3999
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), KR(ret));
4000
  } else if (pool_names.count() <= 0 || !is_valid_tenant_id(tenant_id)) {
4001
    ret = OB_INVALID_ARGUMENT;
4002
    LOG_WARN("invalid argument", K(pool_names), K(tenant_id), K(ret));
4003
  } else if (OB_FAIL(check_pool_ownership_(tenant_id, pool_names, true/*is_grant*/))) {
4004
    LOG_WARN("check pool ownership failed", KR(ret), K(pool_names));
4005
  } else if (OB_FAIL(check_pool_intersect_(tenant_id, pool_names, intersect))) {
4006
    LOG_WARN("check pool intersect failed", K(pool_names), KR(ret));
4007
  } else if (intersect) {
4008
    ret = OB_POOL_SERVER_INTERSECT;
4009
    LOG_USER_ERROR(OB_POOL_SERVER_INTERSECT, to_cstring(pool_names));
4010
    LOG_WARN("resource pool unit server intersect", K(pool_names), KR(ret));
4011
  } else if (!is_bootstrap
4012
      && OB_FAIL(check_server_enough(tenant_id, pool_names, server_enough))) {
4013
    LOG_WARN("fail to check server enough", KR(ret), K(tenant_id), K(pool_names));
4014
  } else if (!server_enough) {
4015
    ret = OB_UNIT_NUM_OVER_SERVER_COUNT;
4016
    LOG_WARN("resource pool unit num over zone server count", K(ret), K(pool_names), K(tenant_id));
4017
  } else if (OB_FAIL(check_expand_zone_resource_allowed_by_old_unit_stat_(
4018
          tenant_id, is_grant_pool_allowed))) {
4019
    LOG_WARN("fail to check grant pools allowed by unit stat", KR(ret), K(tenant_id));
4020
  } else if (!is_grant_pool_allowed) {
4021
    ret = OB_OP_NOT_ALLOW;
4022
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "grant pool when pools in shrinking");
4023
  } else if (OB_FAIL(check_expand_zone_resource_allowed_by_new_unit_stat_(pool_names))) {
4024
    LOG_WARN("fail to check grant pools allowed by unit stat", KR(ret));
4025
  } else if (OB_FAIL(check_tenant_pools_unit_num_legal_(
4026
          tenant_id, pool_names, unit_num_legal, legal_unit_num))) {
4027
    LOG_WARN("fail to check pools unit num legal", KR(ret), K(tenant_id), K(pool_names));
4028
  } else if (!unit_num_legal) {
4029
    ret = OB_OP_NOT_ALLOW;
4030
    LOG_WARN("pools belong to one tenant with different unit num are not allowed",
4031
             KR(ret), K(tenant_id), K(pool_names));
4032
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "pools belong to one tenant with different unit num are");
4033
  } else if (OB_FAIL(get_tenant_pool_unit_group_id_(
4034
          is_bootstrap, grant, tenant_id, legal_unit_num, new_unit_group_id_array))) {
4035
    LOG_WARN("fail to generate new unit group id", KR(ret), K(tenant_id), K(legal_unit_num));
4036
  } else if (OB_FAIL(do_grant_pools_(
4037
             trans, new_unit_group_id_array, compat_mode,
4038
             pool_names, tenant_id, is_bootstrap,
4039
             source_tenant_id, check_data_version))) {
4040
    LOG_WARN("do grant pools failed", KR(ret), K(grant), K(pool_names), K(tenant_id),
4041
                                         K(compat_mode), K(is_bootstrap), K(source_tenant_id));
4042
  }
4043
  LOG_INFO("grant resource pools to tenant", KR(ret), K(pool_names), K(tenant_id), K(is_bootstrap));
4044
  return ret;
4045
}
4046

4047
int ObUnitManager::revoke_pools(common::ObMySQLTransaction &trans,
4048
                                common::ObIArray<uint64_t> &new_unit_group_id_array,
4049
                                const ObIArray<ObResourcePoolName> &pool_names,
4050
                                const uint64_t tenant_id)
4051
{
4052
  int ret = OB_SUCCESS;
4053
  SpinWLockGuard guard(lock_);
4054
  const bool grant = false;
4055
  bool unit_num_legal = false;
4056
  int64_t legal_unit_num = -1;
4057
  if (!check_inner_stat()) {
4058
    ret = OB_INNER_STAT_ERROR;
4059
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4060
  } else if (pool_names.count() <= 0 || !is_valid_tenant_id(tenant_id)) {
4061
    ret = OB_INVALID_ARGUMENT;
4062
    LOG_WARN("invalid argument", K(pool_names), K(tenant_id), K(ret));
4063
  } else if (OB_FAIL(check_pool_ownership_(tenant_id, pool_names, false/*is_grant*/))) {
4064
    LOG_WARN("check pool ownership failed", KR(ret), K(pool_names));
4065
  } else if (OB_FAIL(check_tenant_pools_unit_num_legal_(
4066
          tenant_id, pool_names, unit_num_legal, legal_unit_num))) {
4067
    LOG_WARN("fail to check pools unit num legal", KR(ret), K(tenant_id), K(pool_names));
4068
  } else if (!unit_num_legal) {
4069
    ret = OB_OP_NOT_ALLOW;
4070
    LOG_WARN("pools belong to one tenant with different unit num are not allowed",
4071
             KR(ret), K(tenant_id), K(pool_names));
4072
  } else if (OB_FAIL(get_tenant_pool_unit_group_id_(
4073
          false /* is_bootstrap */, grant, tenant_id,
4074
          legal_unit_num, new_unit_group_id_array))) {
4075
    LOG_WARN("fail to generate new unit group id", KR(ret), K(tenant_id), K(legal_unit_num));
4076
  } else if (OB_FAIL(do_revoke_pools_(trans, new_unit_group_id_array, pool_names, tenant_id))) {
4077
    LOG_WARN("do revoke pools failed", KR(ret), K(grant), K(pool_names), K(tenant_id));
4078
  }
4079
  LOG_INFO("revoke resource pools from tenant", K(pool_names), K(ret));
4080
  return ret;
4081
}
4082

4083
int ObUnitManager::inner_get_pool_ids_of_tenant(const uint64_t tenant_id,
4084
                                                ObIArray<uint64_t> &pool_ids) const
4085
{
4086
  int ret = OB_SUCCESS;
4087
  if (!check_inner_stat()) {
4088
    ret = OB_INNER_STAT_ERROR;
4089
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4090
  } else if (!is_valid_tenant_id(tenant_id)) {
4091
    ret = OB_INVALID_ARGUMENT;
4092
    LOG_WARN("invalid argument", K(tenant_id), K(ret));
4093
  } else {
4094
    pool_ids.reuse();
4095
    ObArray<share::ObResourcePool  *> *pools = NULL;
4096
    if (OB_FAIL(get_pools_by_tenant_(tenant_id, pools))) {
4097
      if (OB_ENTRY_NOT_EXIST != ret) {
4098
        LOG_DEBUG("get_pools_by_tenant failed", K(tenant_id), K(ret));
4099
      } else {
4100
        // just return empty pool_ids
4101
        if (OB_GTS_TENANT_ID != tenant_id) {
4102
          LOG_INFO("tenant doesn't own any pool", K(tenant_id), KR(ret));
4103
        }
4104
        ret = OB_SUCCESS;
4105
      }
4106
    } else if (NULL == pools) {
4107
      ret = OB_ERR_UNEXPECTED;
4108
      LOG_WARN("pools is null", KP(pools), K(ret));
4109
    } else {
4110
      for (int64_t i = 0; OB_SUCC(ret) && i < pools->count(); ++i) {
4111
        if (NULL == pools->at(i)) {
4112
          ret = OB_ERR_UNEXPECTED;
4113
          LOG_WARN("pool is null", "pool", OB_P(pools->at(i)), K(ret));
4114
        } else if (OB_FAIL(pool_ids.push_back(pools->at(i)->resource_pool_id_))) {
4115
          LOG_WARN("push_back failed", K(ret));
4116
        }
4117
      }
4118
    }
4119
  }
4120
  return ret;
4121
}
4122

4123
int ObUnitManager::get_tenant_alive_servers_non_block(const uint64_t tenant_id,
4124
                                                      common::ObIArray<ObAddr> &servers)
4125
{
4126
  int ret = OB_SUCCESS;
4127
  if (!check_inner_stat()) {
4128
    ret = OB_INNER_STAT_ERROR;
4129
    LOG_WARN("check_inner_stat failed", KR(ret), K(inited_), K(loaded_));
4130
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
4131
    ret = OB_INVALID_ARGUMENT;
4132
    LOG_WARN("invalid argument", KR(ret), K(tenant_id));
4133
  } else {
4134
    ObArray<ObUnit> tenant_units;
4135
    uint64_t valid_tnt_id = is_meta_tenant(tenant_id) ? gen_user_tenant_id(tenant_id) : tenant_id;
4136
    // Try lock and get units from inmemory data. If lock failed, get from inner table.
4137
    if (lock_.try_rdlock()) {
4138
      // Get from inmemory data
4139
      ObArray<share::ObResourcePool *> *pools = nullptr;
4140
      if (OB_FAIL(get_pools_by_tenant_(valid_tnt_id, pools))) {
4141
        LOG_WARN("failed to get pools by tenant", KR(ret), K(valid_tnt_id));
4142
      } else if (OB_ISNULL(pools)) {
4143
        ret = OB_ERR_UNEXPECTED;
4144
        LOG_WARN("pools is nullptr", KR(ret), KP(pools));
4145
      } else {
4146
        for (int64_t i = 0; OB_SUCC(ret) && i < pools->count(); ++i) {
4147
          const share::ObResourcePool *pool = pools->at(i);
4148
          ObArray<ObUnit *> *pool_units;
4149
          if (OB_ISNULL(pool)) {
4150
            ret = OB_ERR_UNEXPECTED;
4151
            LOG_WARN("pool is nullptr", KR(ret), KP(pool));
4152
          } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, pool_units))) {
4153
            LOG_WARN("fail to get_units_by_pool", KR(ret),
4154
                    "pool_id", pool->resource_pool_id_);
4155
          } else if (OB_ISNULL(pool_units)) {
4156
            ret = OB_ERR_UNEXPECTED;
4157
            LOG_WARN("pool_units is nullptr", KR(ret), KP(pool_units));
4158
          } else {
4159
            ARRAY_FOREACH_X(*pool_units, idx, cnt, OB_SUCC(ret)) {
4160
              const ObUnit *unit = pool_units->at(idx);
4161
              if (OB_ISNULL(unit)) {
4162
                ret = OB_ERR_UNEXPECTED;
4163
                LOG_WARN("unit is nullptr", KR(ret), KP(unit));
4164
              } else if (OB_FAIL(tenant_units.push_back(*unit))) {
4165
                LOG_WARN("failed to push_back unit", KR(ret), KPC(unit));
4166
              }
4167
            }
4168
          }
4169
        }
4170
      }
4171
      lock_.unlock();
4172
    } else {
4173
      // Get from inner_table
4174
      tenant_units.reuse();
4175
      if (OB_FAIL(ut_operator_.get_units_by_tenant(valid_tnt_id, tenant_units))) {
4176
        LOG_WARN("fail to get_units_by_tenant from inner_table",
4177
                KR(ret), K(tenant_id), K(valid_tnt_id));
4178
      }
4179
    }
4180

4181
    // Filter alive servers
4182
    if (OB_SUCC(ret)) {
4183
      servers.reuse();
4184
      FOREACH_X(unit, tenant_units, OB_SUCC(ret)) {
4185
        bool is_alive = false;
4186
        if (OB_FAIL(SVR_TRACER.check_server_alive(unit->server_, is_alive))) {
4187
          LOG_WARN("check_server_alive failed", KR(ret), K(unit->server_));
4188
        } else if (is_alive) {
4189
          if (has_exist_in_array(servers, unit->server_)) {
4190
            // server exist
4191
          } else if (OB_FAIL(servers.push_back(unit->server_))) {
4192
            LOG_WARN("push_back failed", KR(ret), K(unit->server_));
4193
          }
4194
        }
4195
        if (OB_FAIL(ret) || !unit->migrate_from_server_.is_valid()) {
4196
          // skip
4197
        } else if (OB_FAIL(SVR_TRACER.check_server_alive(unit->migrate_from_server_, is_alive))) {
4198
          LOG_WARN("check_server_alive failed", KR(ret), K(unit->migrate_from_server_));
4199
        } else if (is_alive) {
4200
          if (has_exist_in_array(servers, unit->migrate_from_server_)) {
4201
            // server exist
4202
          } else if (OB_FAIL(servers.push_back(unit->migrate_from_server_))) {
4203
            LOG_WARN("push_back failed", KR(ret), K(unit->migrate_from_server_));
4204
          }
4205
        }
4206
      }
4207
    }
4208
  }
4209
  return ret;
4210
}
4211

4212
int ObUnitManager::get_pool_ids_of_tenant(const uint64_t tenant_id,
4213
                                          ObIArray<uint64_t> &pool_ids) const
4214
{
4215
  int ret = OB_SUCCESS;
4216
  SpinRLockGuard guard(lock_);
4217
  if (!check_inner_stat()) {
4218
    ret = OB_INNER_STAT_ERROR;
4219
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4220
  } else if (!is_valid_tenant_id(tenant_id)) {
4221
    ret = OB_INVALID_ARGUMENT;
4222
    LOG_WARN("invalid argument", K(tenant_id), K(ret));
4223
  } else if (OB_FAIL(inner_get_pool_ids_of_tenant(tenant_id, pool_ids))) {
4224
    LOG_WARN("fail to inner get pool ids of tenant", K(ret));
4225
  }
4226
  return ret;
4227
}
4228

4229
int ObUnitManager::get_pool_names_of_tenant(const uint64_t tenant_id,
4230
                                            ObIArray<ObResourcePoolName> &pool_names) const
4231
{
4232
  int ret = OB_SUCCESS;
4233
  SpinRLockGuard guard(lock_);
4234
  if (!check_inner_stat()) {
4235
    ret = OB_INNER_STAT_ERROR;
4236
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4237
  } else if (!is_valid_tenant_id(tenant_id)) {
4238
    ret = OB_INVALID_ARGUMENT;
4239
    LOG_WARN("invalid argument", K(tenant_id), K(ret));
4240
  } else {
4241
    ObArray<share::ObResourcePool  *> *pools = NULL;
4242
    if (OB_FAIL(get_pools_by_tenant_(tenant_id, pools))) {
4243
      if (OB_ENTRY_NOT_EXIST != ret) {
4244
        LOG_WARN("get_pools_by_tenant failed", K(tenant_id), K(ret));
4245
      } else {
4246
        // just return empty pool_ids
4247
        ret = OB_SUCCESS;
4248
        LOG_WARN("tenant doesn't own any pool", K(tenant_id), K(ret));
4249
      }
4250
    } else if (NULL == pools) {
4251
      ret = OB_ERR_UNEXPECTED;
4252
      LOG_WARN("pools is null", KP(pools), K(ret));
4253
    } else {
4254
      pool_names.reuse();
4255
      for (int64_t i = 0; OB_SUCC(ret) && i < pools->count(); ++i) {
4256
        if (NULL == pools->at(i)) {
4257
          ret = OB_ERR_UNEXPECTED;
4258
          LOG_WARN("pool is null", "pool", OB_P(pools->at(i)), K(ret));
4259
        } else if (OB_FAIL(pool_names.push_back(pools->at(i)->name_))) {
4260
          LOG_WARN("push_back failed", K(ret));
4261
        }
4262
      }
4263
    }
4264
  }
4265
  return ret;
4266
}
4267

4268

4269
int ObUnitManager::get_unit_config_by_pool_name(
4270
    const ObString &pool_name,
4271
    share::ObUnitConfig &unit_config) const
4272
{
4273
  int ret = OB_SUCCESS;
4274
  share::ObResourcePool *pool = NULL;
4275
  ObUnitConfig *config = NULL;
4276
  SpinRLockGuard guard(lock_);
4277
  if (!check_inner_stat()) {
4278
    ret = OB_INNER_STAT_ERROR;
4279
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4280
  } else if (OB_FAIL(inner_get_resource_pool_by_name(pool_name, pool))) {
4281
    if (OB_ENTRY_NOT_EXIST != ret) {
4282
      LOG_WARN("get resource pool by name failed", K(pool_name), K(ret));
4283
    } else {
4284
      ret = OB_RESOURCE_POOL_NOT_EXIST;
4285
      LOG_WARN("pool not exist", K(ret), K(pool_name));
4286
    }
4287
  } else if (NULL == pool) {
4288
    ret = OB_INVALID_ARGUMENT;
4289
    LOG_WARN("pool is null", KP(pool), K(ret));
4290
  } else if (OB_FAIL(get_unit_config_by_id(pool->unit_config_id_, config))) {
4291
    if (OB_ENTRY_NOT_EXIST == ret) {
4292
      ret = OB_RESOURCE_UNIT_NOT_EXIST;
4293
    }
4294
    LOG_WARN("can not find config for unit",
4295
             "unit_config_id", pool->unit_config_id_,
4296
             K(ret));
4297
  } else if (NULL == config) {
4298
    ret = OB_ERR_UNEXPECTED;
4299
    LOG_WARN("config is null", KP(config), K(ret));
4300
  } else {
4301
    unit_config = *config;
4302
  }
4303
  return ret;
4304
}
4305

4306

4307
int ObUnitManager::get_zones_of_pools(const ObIArray<ObResourcePoolName> &pool_names,
4308
                                      ObIArray<ObZone> &zones) const
4309
{
4310
  int ret = OB_SUCCESS;
4311
  SpinRLockGuard guard(lock_);
4312
  if (!check_inner_stat()) {
4313
    ret = OB_INNER_STAT_ERROR;
4314
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4315
  } else if (pool_names.count() <= 0) {
4316
    ret = OB_INVALID_ARGUMENT;
4317
    LOG_WARN("pool_names is empty", K(pool_names), K(ret));
4318
  } else {
4319
    for (int64_t i = 0; OB_SUCC(ret) && i < pool_names.count(); ++i) {
4320
      share::ObResourcePool  *pool = NULL;
4321
      if (pool_names.at(i).is_empty()) {
4322
        ret = OB_INVALID_ARGUMENT;
4323
        LOG_USER_ERROR(OB_INVALID_ARGUMENT, "resource pool name");
4324
        LOG_WARN("invalid pool name", "pool name", pool_names.at(i), K(ret));
4325
      } else if (OB_FAIL(inner_get_resource_pool_by_name(pool_names.at(i), pool))) {
4326
        if (OB_ENTRY_NOT_EXIST != ret) {
4327
          LOG_WARN("get resource pool by name failed", "name", pool_names.at(i), K(ret));
4328
        } else {
4329
          ret = OB_RESOURCE_POOL_NOT_EXIST;
4330
          LOG_USER_ERROR(OB_RESOURCE_POOL_NOT_EXIST, to_cstring(pool_names.at(i)));
4331
          LOG_WARN("pool not exist", "pool_name", pool_names.at(i), K(ret));
4332
        }
4333
      } else if (NULL == pool) {
4334
        ret = OB_ERR_UNEXPECTED;
4335
        LOG_WARN("pool is null", KP(pool), K(ret));
4336
      } else if (OB_FAIL(append(zones, pool->zone_list_))) {
4337
        LOG_WARN("append failed", "zone_list", pool->zone_list_, K(ret));
4338
      }
4339
    }
4340
  }
4341
  return ret;
4342
}
4343

4344
int ObUnitManager::get_pools(common::ObIArray<share::ObResourcePool> &pools) const
4345
{
4346
  int ret = OB_SUCCESS;
4347
  SpinWLockGuard guard(lock_);
4348
  if (!check_inner_stat()) {
4349
    ret = OB_INNER_STAT_ERROR;
4350
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4351
  } else {
4352
    ObHashMap<uint64_t, share::ObResourcePool  *>::const_iterator iter = id_pool_map_.begin();
4353
    for ( ; OB_SUCC(ret) && iter != id_pool_map_.end(); ++iter) {
4354
      if (NULL == iter->second) {
4355
        ret = OB_ERR_UNEXPECTED;
4356
        LOG_WARN("iter->second is null", KP(iter->second), K(ret));
4357
      } else if (OB_FAIL(pools.push_back(*(iter->second)))) {
4358
        LOG_WARN("push_back failed", K(ret));
4359
      }
4360
    }
4361
  }
4362
  return ret;
4363
}
4364

4365
int ObUnitManager::create_sys_units(const ObIArray<ObUnit> &sys_units)
4366
{
4367
  int ret = OB_SUCCESS;
4368
  SpinWLockGuard guard(lock_);
4369
  if (!check_inner_stat()) {
4370
    ret = OB_INNER_STAT_ERROR;
4371
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4372
  } else if (sys_units.count() <= 0) {
4373
    ret = OB_INVALID_ARGUMENT;
4374
    LOG_WARN("sys_units is empty", K(sys_units), K(ret));
4375
  } else {
4376
    for (int64_t i = 0; OB_SUCC(ret) && i < sys_units.count(); ++i) {
4377
      if (OB_FAIL(add_unit(*proxy_, sys_units.at(i)))) {
4378
        LOG_WARN("add_unit failed", "unit", sys_units.at(i), K(ret));
4379
      }
4380
    }
4381
  }
4382
  return ret;
4383
}
4384

4385
int ObUnitManager::inner_get_tenant_pool_zone_list(
4386
    const uint64_t tenant_id,
4387
    common::ObIArray<common::ObZone> &zone_list) const
4388
{
4389
  int ret = OB_SUCCESS;
4390
  ObArray<share::ObResourcePool *> *pools = NULL;
4391
  if (!check_inner_stat()) {
4392
    ret = OB_INNER_STAT_ERROR;
4393
    LOG_WARN("check inner stat failed", K(ret), K(inited_), K(loaded_));
4394
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
4395
    ret = OB_INVALID_ARGUMENT;
4396
    LOG_WARN("invalid argument", K(ret), K(tenant_id));
4397
  } else if (OB_FAIL(get_pools_by_tenant_(tenant_id, pools))) {
4398
    LOG_WARN("fail to get pools by tenant", K(ret), K(tenant_id));
4399
  } else if (OB_UNLIKELY(NULL == pools)) {
4400
    ret = OB_ERR_UNEXPECTED;
4401
    LOG_WARN("pools ptr is null", K(ret), K(tenant_id));
4402
  } else if (pools->count() <= 0) {
4403
    ret = OB_ERR_UNEXPECTED;
4404
    LOG_WARN("pools array is empty", K(ret), K(*pools));
4405
  } else {
4406
    zone_list.reset();
4407
    for (int64_t i = 0; OB_SUCC(ret) && i < pools->count(); ++i) {
4408
      const share::ObResourcePool *pool = pools->at(i);
4409
      if (NULL == pool) {
4410
        ret = OB_ERR_UNEXPECTED;
4411
        LOG_WARN("pool is null", K(ret), KP(pool));
4412
      } else if (OB_FAIL(append(zone_list, pool->zone_list_))) {
4413
        LOG_WARN("fail to append", K(ret));
4414
      } else {} // no more to do
4415
    }
4416
  }
4417
  return ret;
4418
}
4419

4420
int ObUnitManager::get_tenant_pool_zone_list(
4421
    const uint64_t tenant_id,
4422
    common::ObIArray<common::ObZone> &zone_list) const
4423
{
4424
  int ret = OB_SUCCESS;
4425
  SpinRLockGuard guard(lock_);
4426
  if (OB_FAIL(inner_get_tenant_pool_zone_list(tenant_id, zone_list))) {
4427
    LOG_WARN("fail to inner get tenant pool zone list", K(ret));
4428
  }
4429
  return ret;
4430
}
4431

4432
// cancel migrate units on server to other servers
4433
int ObUnitManager::cancel_migrate_out_units(const ObAddr &server)
4434
{
4435
  int ret = OB_SUCCESS;
4436
  SpinWLockGuard guard(lock_);
4437
  ObArray<uint64_t> migrate_units;
4438
  if (!check_inner_stat()) {
4439
    ret = OB_INNER_STAT_ERROR;
4440
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), K(ret));
4441
  } else if (!server.is_valid()) {
4442
    ret = OB_INVALID_ARGUMENT;
4443
    LOG_WARN("invalid argument", K(server), K(ret));
4444
  } else if (OB_FAIL(get_migrate_units_by_server(server, migrate_units))) {
4445
    LOG_WARN("get_migrate_units_by_server failed", K(server), K(ret));
4446
  } else {
4447
    const EndMigrateOp op = REVERSE;
4448
    for (int64_t i = 0; OB_SUCC(ret) && i < migrate_units.count(); ++i) {
4449
      if (OB_FAIL(end_migrate_unit(migrate_units.at(i), op))) {
4450
        LOG_WARN("end_migrate_unit failed", "unit_id", migrate_units.at(i), K(op), K(ret));
4451
      }
4452
    }
4453
  }
4454
  LOG_INFO("cancel migrate out units", K(server), K(ret));
4455
  return ret;
4456
}
4457

4458
int ObUnitManager::check_server_empty(const ObAddr &server, bool &empty) const
4459
{
4460
  int ret = OB_SUCCESS;
4461
  SpinRLockGuard guard(lock_);
4462
  ObArray<ObUnitLoad> *loads = NULL;
4463
  empty = false;
4464
  if (!check_inner_stat()) {
4465
    ret = OB_INNER_STAT_ERROR;
4466
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), K(ret));
4467
  } else if (!server.is_valid()) {
4468
    ret = OB_INVALID_ARGUMENT;
4469
    LOG_WARN("invalid server", K(server), K(ret));
4470
  } else if (OB_FAIL(get_loads_by_server(server, loads))) {
4471
    if (OB_ENTRY_NOT_EXIST != ret) {
4472
      LOG_WARN("get_loads_by_server failed", K(server), K(ret));
4473
    } else {
4474
      ret = OB_SUCCESS;
4475
      empty = true;
4476
    }
4477
  } else if (NULL == loads) {
4478
    ret = OB_ERR_UNEXPECTED;
4479
    LOG_WARN("loads is null", KP(loads), K(ret));
4480
  } else {
4481
    empty = false;
4482
  }
4483
  return ret;
4484
}
4485

4486
int ObUnitManager::finish_migrate_unit_not_in_tenant(
4487
                   share::ObResourcePool *pool)
4488
{
4489
  int ret = OB_SUCCESS;
4490
  SpinWLockGuard guard(lock_);
4491
  //Not in tenant unit
4492
  //Using pool, take the unit corresponding to the pool whose tenant_id is -1
4493
  if (-1 != pool->tenant_id_) {
4494
    //in tenant
4495
    //ignore in tenant unit
4496
  } else {
4497
    ObArray<ObUnitInfo> unit_infos;
4498
    if (OB_FAIL(inner_get_unit_infos_of_pool_(pool->resource_pool_id_, unit_infos))) {
4499
      LOG_WARN("fail to get units by pool", K(ret));
4500
    } else {
4501
      FOREACH_CNT_X(unit_info, unit_infos, OB_SUCC(ret)) {
4502
        if ((*unit_info).unit_.migrate_from_server_.is_valid()) {
4503
          if (OB_INVALID_ID == (*unit_info).unit_.unit_id_) {
4504
            ret = OB_INVALID_ARGUMENT;
4505
            LOG_WARN("invalid argument", K((*unit_info).unit_.unit_id_), K(ret));
4506
          } else {
4507
            const EndMigrateOp op = COMMIT;
4508
            if (OB_FAIL(end_migrate_unit((*unit_info).unit_.unit_id_, op))) {
4509
              LOG_WARN("end migrate unit failed", K(ret), K((*unit_info).unit_.unit_id_), K(op));
4510
            } else {
4511
              LOG_INFO("finish migrate unit not in tenant", K(ret),
4512
                       "unit_id", (*unit_info).unit_.unit_id_);
4513
            }
4514
          }
4515
        } else {}//ignore not in migrate unit
4516
      }
4517
    }
4518
  }
4519
  return ret;
4520
}
4521

4522
int ObUnitManager::finish_migrate_unit_not_in_locality(
4523
                   uint64_t tenant_id,
4524
                   share::schema::ObSchemaGetterGuard *schema_guard,
4525
                   ObArray<common::ObZone> zone_list)
4526
{
4527
  int ret = OB_SUCCESS;
4528
  SpinWLockGuard guard(lock_);
4529
  const ObTenantSchema *tenant_schema = NULL;
4530
  ObArray<ObUnitInfo> unit_infos;
4531
  ObArray<common::ObZone> zone_locality_list;
4532
  if (OB_FAIL((*schema_guard).get_tenant_info(tenant_id, tenant_schema))) {
4533
    LOG_WARN("fail to get tenant schema", K(ret), K(tenant_id));
4534
  } else if (OB_ISNULL(tenant_schema)) {
4535
    ret = OB_ERR_UNEXPECTED;
4536
    LOG_WARN("tenant schema is null", K(ret));
4537
  } else if (OB_FAIL(tenant_schema->get_zone_list(zone_locality_list))) {
4538
    LOG_WARN("fail to get zone replica attr array", K(ret));
4539
  } else {
4540
    for (int64_t i = 0; i < zone_list.count() && OB_SUCC(ret); i++) {
4541
      if (!has_exist_in_array(zone_locality_list, zone_list.at(i))) {
4542
        //Get the unit that is in the zone locality but not in the zone_list,
4543
        //the zone that is not in the locality
4544
        if (OB_FAIL(inner_get_zone_alive_unit_infos_by_tenant(
4545
                    tenant_id, zone_list.at(i), unit_infos))) {
4546
          LOG_WARN("fail to get zone alive unit infos by tenant", K(ret));
4547
        } else {
4548
          FOREACH_CNT_X(unit_info, unit_infos, OB_SUCC(ret)) {
4549
            if ((*unit_info).unit_.migrate_from_server_.is_valid()) {
4550
              if (OB_INVALID_ID == (*unit_info).unit_.unit_id_) {
4551
                ret = OB_INVALID_ARGUMENT;
4552
                LOG_WARN("invalid argument", K((*unit_info).unit_.unit_id_), K(ret));
4553
              } else {
4554
                const EndMigrateOp op = COMMIT;
4555
                if (OB_FAIL(end_migrate_unit((*unit_info).unit_.unit_id_, op))) {
4556
                  LOG_WARN("end migrate unit failed", K(ret),
4557
                            K((*unit_info).unit_.unit_id_), K(op));
4558
                } else {
4559
                  LOG_INFO("finish migrate unit not in locality", K(ret),
4560
                           "unit_id", (*unit_info).unit_.unit_id_);
4561
                }
4562
              }
4563
            } else {} //ignore not in migrate unit
4564
          } //end FOREACH
4565
        }
4566
      } else {} //ignore in locality unit
4567
    } //end for
4568
  }
4569
  return ret;
4570
}
4571

4572
int ObUnitManager::finish_migrate_unit(const uint64_t unit_id)
4573
{
4574
  int ret = OB_SUCCESS;
4575
  SpinWLockGuard guard(lock_);
4576
  if (!check_inner_stat()) {
4577
    ret = OB_INNER_STAT_ERROR;
4578
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4579
  } else if (OB_INVALID_ID == unit_id) {
4580
    ret = OB_INVALID_ARGUMENT;
4581
    LOG_WARN("invalid argument", K(unit_id), K(ret));
4582
  } else {
4583
    const EndMigrateOp op = COMMIT;
4584
    if (OB_FAIL(end_migrate_unit(unit_id, op))) {
4585
      LOG_WARN("end_migrate_unit failed", K(unit_id), K(op), K(ret));
4586
    }
4587
  }
4588
  LOG_INFO("finish migrate unit", K(unit_id), K(ret));
4589
  return ret;
4590
}
4591

4592
int ObUnitManager::inner_get_zone_alive_unit_infos_by_tenant(
4593
    const uint64_t tenant_id,
4594
    const common::ObZone &zone,
4595
    common::ObIArray<share::ObUnitInfo> &unit_infos) const
4596
{
4597
  int ret = OB_SUCCESS;
4598
  ObArray<uint64_t> rs_pool;
4599
  ObArray<ObUnitInfo> unit_array;
4600
  unit_infos.reset();
4601
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))
4602
      || OB_UNLIKELY(zone.is_empty())) {
4603
    ret = OB_INVALID_ARGUMENT;
4604
    LOG_WARN("invalid argument", K(ret), K(tenant_id), K(zone));
4605
  } else if (OB_FAIL(inner_get_pool_ids_of_tenant(tenant_id, rs_pool))) {
4606
    LOG_WARN("fail to get pool ids by tennat", K(ret), K(tenant_id));
4607
  } else {
4608
    FOREACH_X(pool, rs_pool, OB_SUCCESS == ret) {
4609
      unit_array.reuse();
4610
      if (!check_inner_stat()) {
4611
        ret = OB_INNER_STAT_ERROR;
4612
        LOG_WARN("check inner stat failed", K(ret), K(inited_), K(loaded_));
4613
      } else if (OB_UNLIKELY(NULL == pool)) {
4614
        ret = OB_ERR_UNEXPECTED;
4615
        LOG_WARN("pool is null", K(ret));
4616
      } else if (OB_FAIL(inner_get_unit_infos_of_pool_(*pool, unit_array))) {
4617
        LOG_WARN("fail to get unit infos of pool", K(ret));
4618
      } else if (unit_array.count() > 0) {
4619
        FOREACH_X(u, unit_array, OB_SUCCESS == ret) {
4620
          bool is_alive = false;
4621
          bool is_in_service = false;
4622
          if (OB_UNLIKELY(NULL == u)) {
4623
            ret = OB_ERR_UNEXPECTED;
4624
            LOG_WARN("unit is empty", K(ret));
4625
          } else if (zone != u->unit_.zone_) {
4626
            // do not belong to this zone
4627
            } else if (OB_FAIL(SVR_TRACER.check_server_alive(u->unit_.server_, is_alive))) {
4628
            LOG_WARN("check_server_alive failed", "server", u->unit_.server_, K(ret));
4629
          } else if (OB_FAIL(SVR_TRACER.check_in_service(u->unit_.server_, is_in_service))) {
4630
            LOG_WARN("check server in service failed", "server", u->unit_.server_, K(ret));
4631
          } else if (!is_alive || !is_in_service) {
4632
            // ignore unit on not-alive server
4633
          } else if (ObUnit::UNIT_STATUS_ACTIVE != u->unit_.status_) {
4634
            // ignore the unit which is in deleting status
4635
          } else if (OB_FAIL(unit_infos.push_back(*u))) {
4636
            LOG_WARN("fail to push back", K(ret));
4637
          } else {} // no more to do
4638
        }
4639
      } else {} // empty array
4640
    }
4641
  }
4642
  return ret;
4643
}
4644

4645
int ObUnitManager::get_all_unit_infos_by_tenant(const uint64_t tenant_id,
4646
                                                ObIArray<ObUnitInfo> &unit_infos)
4647
{
4648
  int ret = OB_SUCCESS;
4649
  SpinRLockGuard guard(lock_);
4650
  unit_infos.reset();
4651
  if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) {
4652
    LOG_WARN("invalid argument", KR(ret), K(tenant_id));
4653
  } else if (OB_FAIL(inner_get_all_unit_infos_by_tenant_(tenant_id, unit_infos))) {
4654
    LOG_WARN("fail to inner get unit infos of tenant", KR(ret), K(tenant_id));
4655
  }
4656
  return ret;
4657
}
4658

4659
int ObUnitManager::inner_get_all_unit_infos_by_tenant_(const uint64_t tenant_id,
4660
                                                       ObIArray<ObUnitInfo> &unit_infos)
4661
{
4662
  int ret = OB_SUCCESS;
4663
  share::schema::ObSchemaGetterGuard schema_guard;
4664
  const share::schema::ObTenantSchema *tenant_schema = NULL;
4665
  unit_infos.reset();
4666
  common::ObArray<common::ObZone> tenant_zone_list;
4667
  ObArray<uint64_t> rs_pool;
4668
  ObArray<ObUnitInfo> unit_array;
4669

4670
  if (!is_valid_tenant_id(tenant_id)) {
4671
    ret = OB_INVALID_ARGUMENT;
4672
    LOG_WARN("invalid argument", K(ret), K(tenant_id));
4673
  } else if (OB_ISNULL(schema_service_)) {
4674
    ret = OB_NOT_INIT;
4675
    LOG_WARN("schema service is null", K(schema_service_), KR(ret));
4676
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) {
4677
    LOG_WARN("fail to get tenant schema guard", KR(ret), K(tenant_id));
4678
  } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
4679
    LOG_WARN("fail to get tenant info", K(ret), K(tenant_id));
4680
  } else if (OB_UNLIKELY(NULL == tenant_schema)) {
4681
    ret = OB_ERR_UNEXPECTED;
4682
    LOG_WARN("tenant schema is null", K(ret), KP(tenant_schema));
4683
  } else if (OB_FAIL(tenant_schema->get_zone_list(tenant_zone_list))) {
4684
    LOG_WARN("fail to get zone list", K(ret));
4685
  } else if (OB_FAIL(inner_get_pool_ids_of_tenant(tenant_id, rs_pool))) {
4686
    LOG_WARN("fail to get pool ids of tenant", K(ret), K(tenant_id));
4687
  } else {
4688
    FOREACH_X(pool, rs_pool, OB_SUCCESS == ret) {
4689
      unit_array.reuse();
4690
      if (!check_inner_stat()) {
4691
        ret = OB_INNER_STAT_ERROR;
4692
        LOG_WARN("check inner stat failed", K(ret), K(inited_), K(loaded_));
4693
      } else if (OB_UNLIKELY(NULL == pool)) {
4694
        ret = OB_ERR_UNEXPECTED;
4695
        LOG_WARN("pool is null", K(ret));
4696
      } else if (OB_UNLIKELY(OB_INVALID_ID == *pool)) {
4697
        ret = OB_INVALID_ARGUMENT;
4698
        LOG_WARN("invalid argument", KR(ret));
4699
      } else if (OB_FAIL(inner_get_unit_infos_of_pool_(*pool, unit_array))) {
4700
        LOG_WARN("fail to get unit infos of pool", K(ret));
4701
      } else if (unit_array.count() > 0) {
4702
        FOREACH_X(u, unit_array, OB_SUCCESS == ret) {
4703
          if (OB_UNLIKELY(NULL == u)) {
4704
            ret = OB_ERR_UNEXPECTED;
4705
            LOG_WARN("empty is null", K(ret));
4706
          } else if (!has_exist_in_array(tenant_zone_list, u->unit_.zone_)) {
4707
            // this unit do not in tenant zone list, ignore
4708
          } else if (OB_FAIL(unit_infos.push_back(*u))) {
4709
            LOG_WARN("fail to push back", K(ret));
4710
          } else {} // no more to do
4711
        }
4712
      } else {} // do nothing
4713
    }
4714
  }
4715
  return ret;
4716
}
4717

4718
int ObUnitManager::commit_shrink_tenant_resource_pool(const uint64_t tenant_id)
4719
{
4720
  int ret = OB_SUCCESS;
4721
  SpinWLockGuard guard(lock_);
4722
  common::ObMySQLTransaction trans;
4723
  common::ObArray<share::ObResourcePool *> *pools = nullptr;
4724
  common::ObArray<common::ObArray<uint64_t>> resource_units;
4725

4726
  if (!check_inner_stat()) {
4727
    ret = OB_INNER_STAT_ERROR;
4728
    LOG_WARN("check_inner_stat failed", KR(ret), K(loaded_), K(inited_));
4729
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
4730
    ret = OB_INVALID_ARGUMENT;
4731
    LOG_WARN("invalid argument", KR(ret), K(tenant_id));
4732
  } else if (OB_FAIL(get_pools_by_tenant_(tenant_id, pools))) {
4733
    LOG_WARN("failed to get pool by tenant", KR(ret), K(tenant_id));
4734
  } else if (OB_UNLIKELY(NULL == pools)) {
4735
    ret = OB_ERR_UNEXPECTED;
4736
    LOG_WARN("pool ptr is null", KR(ret), KP(pools));
4737
  } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
4738
    LOG_WARN("start transaction failed", KR(ret));
4739
  } else if (OB_FAIL(commit_shrink_resource_pool_in_trans_(*pools, trans, resource_units))) {
4740
    LOG_WARN("failed to shrink in trans", KR(ret), KPC(pools));
4741
  }
4742
  const bool commit = (OB_SUCCESS == ret);
4743
  int tmp_ret = OB_SUCCESS;
4744
  if (OB_SUCCESS != (tmp_ret = trans.end(commit))) {
4745
    LOG_WARN("trans end failed", K(tmp_ret), K(commit));
4746
    ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
4747
  }
4748
  if (OB_SUCC(ret)) {
4749
    if (OB_UNLIKELY(pools->count() != resource_units.count())) {
4750
      ret = OB_ERR_UNEXPECTED;
4751
      LOG_WARN("pools count must equal to unit count", KR(ret), KPC(pools), K(resource_units));
4752
    }
4753
    for (int64_t i = 0; OB_SUCC(ret) && i < pools->count(); ++i) {
4754
      share::ObResourcePool *pool = pools->at(i);
4755
      if (OB_ISNULL(pool)) {
4756
        ret = OB_ERR_UNEXPECTED;
4757
        LOG_WARN("pool is null", KR(ret), K(i), KPC(pools));
4758
      } else if (OB_FAIL(delete_inmemory_units(pool->resource_pool_id_, resource_units.at(i)))) {
4759
        LOG_WARN("failed to delete inmemory units", KR(ret), KPC(pool), K(i), K(resource_units));
4760
      }
4761
    }
4762
  }
4763
  return ret;
4764
}
4765

4766
int ObUnitManager::commit_shrink_resource_pool_in_trans_(
4767
  const common::ObIArray<share::ObResourcePool *> &pools,
4768
  common::ObMySQLTransaction &trans,
4769
  common::ObIArray<common::ObArray<uint64_t>> &resource_units)
4770
{
4771
  int ret = OB_SUCCESS;
4772
  ObArray<ObUnit *> *units = NULL;
4773
  if (!check_inner_stat()) {
4774
    ret = OB_INNER_STAT_ERROR;
4775
    LOG_WARN("check_inner_stat failed", KR(ret), K(loaded_), K(inited_));
4776
  } else if (OB_UNLIKELY(0 == pools.count())) {
4777
    ret = OB_INVALID_ARGUMENT;
4778
    LOG_WARN("pool ptr is null", KR(ret), K(pools));
4779
  } else {
4780
    for (int64_t index = 0; OB_SUCC(ret) && index < pools.count(); ++index) {
4781
      // Commit shrink resource pool only needs to change the state of the unit,
4782
      // the state of the resource pool itself has been changed before,
4783
      // and there is no need to adjust it again.
4784
      units = NULL;
4785
      share::ObResourcePool *pool = pools.at(index);
4786
      if (OB_ISNULL(pool)) {
4787
        ret = OB_ERR_UNEXPECTED;
4788
        LOG_WARN("pool is null", KR(ret), K(index), K(pools));
4789
      } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
4790
        LOG_WARN("failed to get pool unit", KR(ret), KPC(pool));
4791
      } else if (OB_ISNULL(units)) {
4792
        ret = OB_ERR_UNEXPECTED;
4793
        LOG_WARN("units is null of the pool", KR(ret), KPC(pool));
4794
      } else {
4795
        common::ObArray<uint64_t> unit_ids;
4796
        const int64_t unit_count = pool->unit_count_;
4797
        if (unit_count <= 0) {
4798
          ret = OB_ERR_UNEXPECTED;
4799
          LOG_WARN("unit count unexpected", KR(ret), K(unit_count));
4800
        } else if (units->count() <= 0) {
4801
          ret = OB_ERR_UNEXPECTED;
4802
          LOG_WARN("zone unit ptrs has no element", KR(ret), "unit_cnt",
4803
                   units->count());
4804
        } else {
4805
          for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
4806
            const ObUnit *unit = units->at(i);
4807
            if (OB_ISNULL(unit)) {
4808
              ret = OB_ERR_UNEXPECTED;
4809
              LOG_WARN("unit ptr is null", KR(ret), KP(unit), K(i), KPC(units));
4810
            } else if (ObUnit::UNIT_STATUS_DELETING == unit->status_) {
4811
              if (OB_FAIL(ut_operator_.remove_unit(trans, *unit))) {
4812
                LOG_WARN("fail to remove unit", KR(ret), "unit", *unit);
4813
              } else if (OB_FAIL(unit_ids.push_back(unit->unit_id_))) {
4814
                LOG_WARN("fail to push back", KR(ret), KPC(unit));
4815
              } else {} // no more to do
4816
            } else {} // active unit, do nothing
4817
          }// end for each unit
4818
          if (FAILEDx(resource_units.push_back(unit_ids))) {
4819
            LOG_WARN("failed to push back units", KR(ret), K(unit_ids));
4820
          }
4821
        }
4822
      }
4823
    }//end for each pool
4824
  }
4825
  return ret;
4826
}
4827

4828
int ObUnitManager::get_deleting_units_of_pool(
4829
    const uint64_t resource_pool_id,
4830
    ObIArray<share::ObUnit> &units) const
4831
{
4832
  int ret = OB_SUCCESS;
4833
  SpinRLockGuard guard(lock_);
4834
  ObArray<ObUnit *> *inner_units = NULL;
4835
  if (!check_inner_stat()) {
4836
    ret = OB_INNER_STAT_ERROR;
4837
    LOG_WARN("check_inner_stat failed", K(ret), K(loaded_), K(inited_));
4838
  } else if (OB_UNLIKELY(OB_INVALID_ID == resource_pool_id)) {
4839
    ret = OB_INVALID_ARGUMENT;
4840
    LOG_WARN("invalid argument", K(ret), K(resource_pool_id));
4841
  } else if (OB_FAIL(get_units_by_pool(resource_pool_id, inner_units))) {
4842
      LOG_WARN("fail to get units by pool", K(ret), K(resource_pool_id));
4843
  } else if (NULL == inner_units) {
4844
    ret = OB_ERR_UNEXPECTED;
4845
    LOG_WARN("inner_units ptr is null", K(ret), KP(inner_units));
4846
  } else {
4847
    units.reset();
4848
    for (int64_t i = 0; OB_SUCC(ret) && i < inner_units->count(); ++i) {
4849
      const share::ObUnit *this_unit = inner_units->at(i);
4850
      if (NULL == this_unit) {
4851
        ret = OB_ERR_UNEXPECTED;
4852
        LOG_WARN("unit ptr is null", K(ret), KP(this_unit));
4853
      } else if (ObUnit::UNIT_STATUS_DELETING == this_unit->status_) {
4854
        if (OB_FAIL(units.push_back(*this_unit))) {
4855
          LOG_WARN("fail to push back", K(ret));
4856
        } else {} // no more to do
4857
      } else if (ObUnit::UNIT_STATUS_ACTIVE == this_unit->status_) {
4858
        // a normal unit, ignore
4859
      } else {
4860
        ret = OB_ERR_UNEXPECTED;
4861
        LOG_WARN("unexpected unit status", K(ret), "unit_status", this_unit->status_);
4862
      }
4863
    }
4864
    LOG_INFO("get deleting units of pool", K(ret), K(resource_pool_id), K(units));
4865
  }
4866
  return ret;
4867
}
4868

4869
int ObUnitManager::get_unit_infos_of_pool(const uint64_t resource_pool_id,
4870
                                          ObIArray<ObUnitInfo> &unit_infos) const
4871
{
4872
  int ret = OB_SUCCESS;
4873
  SpinRLockGuard guard(lock_);
4874
  if (!check_inner_stat()) {
4875
    ret = OB_INNER_STAT_ERROR;
4876
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4877
  } else if (OB_INVALID_ID == resource_pool_id) {
4878
    ret = OB_INVALID_ARGUMENT;
4879
    LOG_WARN("invalid argument", K(resource_pool_id), K(ret));
4880
  } else if (OB_FAIL(inner_get_unit_infos_of_pool_(resource_pool_id, unit_infos))) {
4881
    LOG_WARN("inner_get_unit_infos_of_pool failed", K(resource_pool_id), K(ret));
4882
  }
4883
  return ret;
4884
}
4885

4886
int ObUnitManager::inner_get_unit_infos_of_pool_(const uint64_t resource_pool_id,
4887
                                                ObIArray<ObUnitInfo> &unit_infos) const
4888
{
4889
  int ret = OB_SUCCESS;
4890
  if (!check_inner_stat()) {
4891
    ret = OB_INNER_STAT_ERROR;
4892
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4893
  } else if (OB_INVALID_ID == resource_pool_id) {
4894
    ret = OB_INVALID_ARGUMENT;
4895
    LOG_WARN("invalid argument", K(resource_pool_id), K(ret));
4896
  } else {
4897
    unit_infos.reuse();
4898
    ObUnitInfo unit_info;
4899
    share::ObResourcePool  *pool = NULL;
4900
    ObUnitConfig *config = NULL;
4901
    if (OB_FAIL(get_resource_pool_by_id(resource_pool_id, pool))) {
4902
      if (OB_ENTRY_NOT_EXIST == ret) {
4903
        ret = OB_RESOURCE_POOL_NOT_EXIST;
4904
      }
4905
      LOG_WARN("get_resource_pool_by_id failed", K(resource_pool_id), K(ret));
4906
    } else if (NULL == pool) {
4907
      ret = OB_ERR_UNEXPECTED;
4908
      LOG_WARN("pool is null", KP(pool), K(ret));
4909
    } else if (OB_FAIL(get_unit_config_by_id(pool->unit_config_id_, config))) {
4910
      if (OB_ENTRY_NOT_EXIST == ret) {
4911
        ret = OB_RESOURCE_UNIT_NOT_EXIST;
4912
      }
4913
      LOG_WARN("get_unit_config_by_id failed", "unit config id", pool->unit_config_id_, K(ret));
4914
    } else if (NULL == config) {
4915
      ret = OB_ERR_UNEXPECTED;
4916
      LOG_WARN("config is null", KP(config), K(ret));
4917
    } else {
4918
      unit_info.reset();
4919
      if (OB_FAIL(unit_info.pool_.assign(*pool))) {
4920
        LOG_WARN("failed to assign unit_info.pool_", K(ret));
4921
      } else {
4922
        unit_info.config_ = *config;
4923
        ObArray<ObUnit *> *units = NULL;
4924
        if (OB_FAIL(get_units_by_pool(resource_pool_id, units))) {
4925
          LOG_WARN("get_units_by_pool failed", K(resource_pool_id), K(ret));
4926
        } else if (NULL == units) {
4927
          ret = OB_ERR_UNEXPECTED;
4928
          LOG_WARN("units is null", KP(units), K(ret));
4929
        } else if (units->count() <= 0) {
4930
          ret = OB_ENTRY_NOT_EXIST;
4931
          LOG_WARN("units of resource pool not exist", K(resource_pool_id), K(ret));
4932
        } else if (OB_FAIL(unit_infos.reserve(units->count()))) {
4933
          LOG_WARN("failed to reserve for unit_infos", KR(ret));
4934
        } else {
4935
          for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
4936
            if (NULL == units->at(i)) {
4937
              ret = OB_ERR_UNEXPECTED;
4938
              LOG_WARN("unit is null", "unit", OB_P(units->at(i)), K(ret));
4939
            } else {
4940
              unit_info.unit_ = *units->at(i);
4941
              if (OB_FAIL(unit_infos.push_back(unit_info))) {
4942
                LOG_WARN("push_back failed", K(ret));
4943
              }
4944
            }
4945
          }
4946
        }
4947
      }
4948
    }
4949
  }
4950
  return ret;
4951
}
4952

4953
int ObUnitManager::inner_get_unit_info_by_id(const uint64_t unit_id, ObUnitInfo &unit_info) const
4954
{
4955
  int ret = OB_SUCCESS;
4956
  if (!check_inner_stat()) {
4957
    ret = OB_INNER_STAT_ERROR;
4958
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
4959
  } else if (OB_INVALID_ID == unit_id) {
4960
    ret = OB_INVALID_ARGUMENT;
4961
    LOG_WARN("invalid argument", K(unit_id), K(ret));
4962
  } else {
4963
    ObUnit *unit = NULL;
4964
    ObUnitConfig *config = NULL;
4965
    share::ObResourcePool  *pool = NULL;
4966
    if (OB_FAIL(get_unit_by_id(unit_id, unit))) {
4967
      LOG_WARN("get_unit_by_id failed", K(unit_id), K(ret));
4968
    } else if (NULL == unit) {
4969
      ret = OB_ERR_UNEXPECTED;
4970
      LOG_WARN("unit is null", KP(unit), K(ret));
4971
    } else if (OB_FAIL(get_resource_pool_by_id(unit->resource_pool_id_, pool))) {
4972
      if (OB_ENTRY_NOT_EXIST == ret) {
4973
        ret = OB_RESOURCE_POOL_NOT_EXIST;
4974
      }
4975
      LOG_WARN("get_resource_pool_by_id failed", "pool id", unit->resource_pool_id_, K(ret));
4976
    } else if (NULL == pool) {
4977
      ret = OB_ERR_UNEXPECTED;
4978
      LOG_WARN("pool is null", KP(pool), K(ret));
4979
    } else if (OB_FAIL(get_unit_config_by_id(pool->unit_config_id_, config))) {
4980
      if (OB_ENTRY_NOT_EXIST == ret) {
4981
        ret = OB_RESOURCE_UNIT_NOT_EXIST;
4982
      }
4983
      LOG_WARN("get_unit_config_by_id failed", "unit config id", pool->unit_config_id_, K(ret));
4984
    } else if (NULL == config) {
4985
      ret = OB_ERR_UNEXPECTED;
4986
      LOG_WARN("config is null", KP(config), K(ret));
4987
    } else {
4988
      if (OB_FAIL(unit_info.pool_.assign(*pool))) {
4989
        LOG_WARN("failed to assign unit_info.pool_", K(ret));
4990
      } else {
4991
        unit_info.unit_ = *unit;
4992
        unit_info.config_ = *config;
4993
      }
4994
    }
4995
  }
4996
  return ret;
4997
}
4998

4999
int ObUnitManager::extract_unit_ids(
5000
    const common::ObIArray<share::ObUnit *> &units,
5001
    common::ObIArray<uint64_t> &unit_ids)
5002
{
5003
  int ret = OB_SUCCESS;
5004
  unit_ids.reset();
5005
  for (int64_t i = 0; OB_SUCC(ret) && i < units.count(); ++i) {
5006
    if (OB_UNLIKELY(NULL == units.at(i))) {
5007
      ret = OB_ERR_UNEXPECTED;
5008
      LOG_WARN("unit ptr is null", K(ret));
5009
    } else if (OB_FAIL(unit_ids.push_back(units.at(i)->unit_id_))) {
5010
      LOG_WARN("fail to push back", K(ret));
5011
    } else {} // no more to do
5012
  }
5013
  return ret;
5014
}
5015

5016
int ObUnitManager::inner_get_unit_ids(ObIArray<uint64_t> &unit_ids) const
5017
{
5018
  int ret = OB_SUCCESS;
5019
  if (!check_inner_stat()) {
5020
    ret = OB_INNER_STAT_ERROR;
5021
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
5022
  }
5023
  for (ObHashMap<uint64_t, ObArray<ObUnit *> *>::const_iterator it = pool_unit_map_.begin();
5024
      OB_SUCCESS == ret && it != pool_unit_map_.end(); ++it) {
5025
    if (NULL == it->second) {
5026
      ret = OB_ERR_UNEXPECTED;
5027
      LOG_ERROR("pointer of ObArray<ObUnit *> is null", "pool_id", it->first, K(ret));
5028
    } else if (it->second->count() <= 0) {
5029
      ret = OB_ERR_UNEXPECTED;
5030
      LOG_ERROR("array of unit is empty", "pool_id", it->first, K(ret));
5031
    } else {
5032
      const ObArray<ObUnit *> units = *it->second;
5033
      for (int64_t i = 0; OB_SUCC(ret) && i < units.count(); ++i) {
5034
        uint64_t unit_id = units.at(i)->unit_id_;
5035
        if (OB_FAIL(unit_ids.push_back(unit_id))) {
5036
          LOG_WARN("fail push back it", K(unit_id), K(ret));
5037
        }
5038
      }
5039
    }
5040
  }
5041
  return ret;
5042
}
5043

5044
int ObUnitManager::get_unit_ids(ObIArray<uint64_t> &unit_ids) const
5045
{
5046
  int ret = OB_SUCCESS;
5047
  SpinRLockGuard guard(lock_);
5048
  if (OB_FAIL(inner_get_unit_ids(unit_ids))) {
5049
    LOG_WARN("fail to inner get unit ids", K(ret));
5050
  }
5051
  return ret;
5052
}
5053

5054
int ObUnitManager::calc_sum_load(const ObArray<ObUnitLoad> *unit_loads,
5055
                                 ObUnitConfig &sum_load)
5056
{
5057
  int ret = OB_SUCCESS;
5058
  sum_load.reset();
5059
  if (NULL == unit_loads) {
5060
    // all be zero
5061
  } else {
5062
    for (int64_t i = 0; OB_SUCC(ret) && i < unit_loads->count(); ++i) {
5063
      if (!unit_loads->at(i).is_valid()) {
5064
        ret = OB_INVALID_ARGUMENT;
5065
        LOG_WARN("invalid unit_load", "unit_load", unit_loads->at(i), K(ret));
5066
      } else {
5067
        sum_load += *unit_loads->at(i).unit_config_;
5068
      }
5069
    }
5070
  }
5071
  return ret;
5072
}
5073

5074
int ObUnitManager::check_resource_pool(
5075
    share::ObResourcePool &resource_pool,
5076
    const bool is_clone_tenant) const
5077
{
5078
  int ret = OB_SUCCESS;
5079
  if (!check_inner_stat()) {
5080
    ret = OB_INNER_STAT_ERROR;
5081
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
5082
  } else if (resource_pool.name_.is_empty()) {
5083
    ret = OB_INVALID_ARGUMENT;
5084
    LOG_USER_ERROR(OB_INVALID_ARGUMENT, "resource pool name");
5085
    LOG_WARN("invalid resource pool name", "resource pool name", resource_pool.name_, K(ret));
5086
  } else if (resource_pool.unit_count_ <= 0) {
5087
    ret = OB_INVALID_ARGUMENT;
5088
    LOG_USER_ERROR(OB_INVALID_ARGUMENT, "unit num");
5089
    LOG_WARN("invalid resource unit num", "unit num", resource_pool.unit_count_, K(ret));
5090
  } else {
5091
    // check zones in zone_list not intersected
5092
    for (int64_t i = 0; OB_SUCC(ret) && i < resource_pool.zone_list_.count(); ++i) {
5093
      for (int64_t j = i + 1; OB_SUCC(ret) && j < resource_pool.zone_list_.count(); ++j) {
5094
        if (resource_pool.zone_list_[i] == resource_pool.zone_list_[j]) {
5095
          ret = OB_ZONE_DUPLICATED;
5096
          LOG_USER_ERROR(OB_ZONE_DUPLICATED, to_cstring(resource_pool.zone_list_[i]),
5097
              to_cstring(resource_pool.zone_list_));
5098
          LOG_WARN("duplicate zone in zone list", "zone_list", resource_pool.zone_list_, K(ret));
5099
        }
5100
      }
5101
    }
5102
    // check zones in zone_list all existed
5103
    if (OB_SUCC(ret)) {
5104
      FOREACH_CNT_X(zone, resource_pool.zone_list_, OB_SUCCESS == ret) {
5105
        bool zone_exist = false;
5106
        if (OB_FAIL(zone_mgr_.check_zone_exist(*zone, zone_exist))) {
5107
          LOG_WARN("check_zone_exist failed", KPC(zone), K(ret));
5108
        } else if (!zone_exist) {
5109
          ret = OB_ZONE_INFO_NOT_EXIST;
5110
          LOG_USER_ERROR(OB_ZONE_INFO_NOT_EXIST, to_cstring(*zone));
5111
          LOG_WARN("zone not exist", "zone", *zone, K(ret));
5112
        }
5113
      }
5114
    }
5115
    // construct all zone as zone_list if zone_list is null
5116
    if (OB_SUCCESS == ret && 0 == resource_pool.zone_list_.count()) {
5117
      ObArray<ObZoneInfo> zone_infos;
5118
      if (OB_FAIL(zone_mgr_.get_zone(zone_infos))) {
5119
        LOG_WARN("get_zone failed", K(ret));
5120
      } else {
5121
        FOREACH_CNT_X(zone_info, zone_infos, OB_SUCCESS == ret) {
5122
          if (OB_FAIL(resource_pool.zone_list_.push_back(zone_info->zone_))) {
5123
            LOG_WARN("push_back failed", K(ret));
5124
          }
5125
        }
5126
        if (OB_SUCCESS == ret && resource_pool.zone_list_.count() <= 0) {
5127
          ret = OB_ERR_UNEXPECTED;
5128
          LOG_WARN("not active zone found", K(ret));
5129
        }
5130
      }
5131
    }
5132
    // check zones in zone_list has enough alive servers to support unit_num
5133
    // ATTENTION:
5134
    //    There is a limit that we have to persist all unit config successfully before create tenant.
5135
    //    For clone tenant, the locations of units to create is certain.
5136
    //    If majority of servers is down, clone tenant will fail, and this case can be detected before persist unit.
5137
    //    So for clone tenant, just skip this check, just make sure majority of teannt snapshot can restored is ok.
5138
    if (!is_clone_tenant) {
5139
      FOREACH_CNT_X(zone, resource_pool.zone_list_, OB_SUCCESS == ret) {
5140
        int64_t alive_server_count = 0;
5141
        if (OB_FAIL(SVR_TRACER.get_alive_servers_count(*zone, alive_server_count))) {
5142
          LOG_WARN("get_alive_servers failed", KR(ret), KPC(zone));
5143
        } else if (alive_server_count < resource_pool.unit_count_) {
5144
          ret = OB_UNIT_NUM_OVER_SERVER_COUNT;
5145
          LOG_WARN("resource pool unit num over zone server count", "unit_count",
5146
              resource_pool.unit_count_, K(alive_server_count), K(ret), "zone", *zone);
5147
        }
5148
      }
5149
    }
5150
  }
5151
  return ret;
5152
}
5153

5154
int ObUnitManager::allocate_new_pool_units_(
5155
    ObISQLClient &client,
5156
    const share::ObResourcePool  &pool,
5157
    const char *module)
5158
{
5159
  int ret = OB_SUCCESS;
5160
  if (!check_inner_stat()) {
5161
    ret = OB_INNER_STAT_ERROR;
5162
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
5163
  } else if (!pool.is_valid() || pool.zone_list_.count() <= 0) {
5164
    ret = OB_INVALID_ARGUMENT;
5165
    LOG_WARN("invalid pool", K(pool), K(ret));
5166
  } else {
5167
    const bool new_allocate_pool = true;
5168
    const int64_t delta_unit_num = pool.unit_count_;
5169
    ObArray<ObAddr> new_servers;
5170
    if (OB_FAIL(allocate_pool_units_(
5171
            client, pool, pool.zone_list_, nullptr, new_allocate_pool, delta_unit_num, module, new_servers))) {
5172
      LOG_WARN("allocate pool units failed", K(module), KR(ret), K(delta_unit_num), K(pool));
5173
    }
5174
  }
5175
  return ret;
5176
}
5177

5178
/* Notify creating or dropping unit on ObServer.
5179
 * 1. Specify @is_delete as true when dropping unit,
5180
 *    and only @tenant_id and @unit will be used in this case.
5181
 * 2. This function merely sends RPC call, so executor should make sure waiting is called later.
5182
 *    But there is one exception that when @unit is on this server where RS Leader locates,
5183
 *    notification will be locally executed without RPC, which means no need to wait proxy.
5184
 */
5185
int ObUnitManager::try_notify_tenant_server_unit_resource_(
5186
    const uint64_t tenant_id,
5187
    const bool is_delete,
5188
    ObNotifyTenantServerResourceProxy &notify_proxy,
5189
    const uint64_t unit_config_id,
5190
    const lib::Worker::CompatMode compat_mode,
5191
    const share::ObUnit &unit,
5192
    const bool if_not_grant,
5193
    const bool skip_offline_server,
5194
    const bool check_data_version)
5195
{
5196
  int ret = OB_SUCCESS;
5197
  bool is_alive = false;
5198
  if (!check_inner_stat()) {
5199
    ret = OB_INNER_STAT_ERROR;
5200
    LOG_WARN("check_inner_stat failed", K(ret), K(inited_), K(loaded_));
5201
  } else if (OB_FAIL(SVR_TRACER.check_server_alive(unit.server_, is_alive))) {
5202
    LOG_WARN("fail to get server_info", KR(ret), K(unit.server_));
5203
  } else if (!is_alive && (is_delete || skip_offline_server)) {
5204
    // do nothing
5205
    LOG_INFO("ignore not alive server when is_delete or skip_offline_server is true",
5206
              "server", unit.server_, K(is_delete), K(skip_offline_server));
5207
  } else if (!is_valid_tenant_id(tenant_id)) {
5208
    // do nothing, unit not granted
5209
  } else {
5210
    // STEP 1: Get and init notifying arg
5211
    obrpc::TenantServerUnitConfig tenant_unit_server_config;
5212
    if (!is_delete) {
5213
      const bool should_check_data_version = check_data_version && is_user_tenant(tenant_id);
5214
      if (OB_FAIL(build_notify_create_unit_resource_rpc_arg_(
5215
                    tenant_id, unit, compat_mode, unit_config_id, if_not_grant,
5216
                    tenant_unit_server_config))) {
5217
        LOG_WARN("fail to init tenant_unit_server_config", KR(ret), K(tenant_id), K(is_delete));
5218
      } else if (should_check_data_version
5219
                 && OB_FAIL(check_dest_data_version_is_loaded_(tenant_id, unit.server_))) {
5220
        LOG_WARN("fail to check dest data_version is loaded", KR(ret), K(tenant_id), "dst", unit.server_);
5221
      }
5222
    } else {
5223
      if (OB_FAIL(tenant_unit_server_config.init_for_dropping(tenant_id, is_delete))) {
5224
        LOG_WARN("fail to init tenant_unit_server_config", KR(ret), K(tenant_id), K(is_delete));
5225
      }
5226
    }
5227

5228
    // STEP 2: Do notification
5229
    if (OB_FAIL(ret)) {
5230
    } else if (OB_FAIL(do_notify_unit_resource_(
5231
                unit.server_, tenant_unit_server_config, notify_proxy))) {
5232
      LOG_WARN("failed to do_notify_unit_resource", "dst", unit.server_, K(tenant_unit_server_config));
5233
      if (OB_TENANT_EXIST == ret) {
5234
        ret = OB_TENANT_RESOURCE_UNIT_EXIST;
5235
        LOG_USER_ERROR(OB_TENANT_RESOURCE_UNIT_EXIST, tenant_id, to_cstring(unit.server_));
5236
      }
5237
    }
5238
  }
5239
  return ret;
5240
}
5241

5242
int ObUnitManager::check_dest_data_version_is_loaded_(
5243
    const uint64_t tenant_id, const ObAddr &addr)
5244
{
5245
 int ret = OB_SUCCESS;
5246
 ObTimeoutCtx ctx;
5247
 const int64_t DEFTAULT_TIMEOUT_TS = 5 * GCONF.rpc_timeout;
5248
 char ip_buf[OB_IP_STR_BUFF] = "";
5249
 if (OB_UNLIKELY(!check_inner_stat())) {
5250
   ret = OB_INNER_STAT_ERROR;
5251
   LOG_WARN("check_inner_stat failed", KR(ret), K(inited_), K(loaded_));
5252
 } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id
5253
            || !addr.is_valid()
5254
            || OB_ISNULL(proxy_))) {
5255
   ret = OB_INVALID_ARGUMENT;
5256
   LOG_WARN("invalid arg", KR(ret), K(tenant_id), K(addr), KP_(proxy));
5257
 } else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, DEFTAULT_TIMEOUT_TS))) {
5258
   LOG_WARN("fail to set default timeout ctx", KR(ret));
5259
 } else if (OB_UNLIKELY(!addr.ip_to_string(ip_buf, sizeof(ip_buf)))) {
5260
   ret = OB_ERR_UNEXPECTED;
5261
   LOG_WARN("fail to convert ip to string", KR(ret), K(tenant_id), K(addr));
5262
 } else {
5263
   const int64_t start_timeout_ts = ObTimeUtility::current_time();
5264
   const int64_t CHECK_INTERVAL_TS = 500 * 1000L; // 500ms
5265
   const int64_t SLEEP_TS = 100 * 1000L; // 100ms
5266
   ObSqlString sql;
5267
   if (OB_FAIL(sql.assign_fmt("SELECT IF(value = '0.0.0.0', 0, 1) AS loaded "
5268
                              "FROM %s WHERE tenant_id = %lu AND name = 'compatible' "
5269
                              "AND svr_ip = '%s' AND svr_port = %d",
5270
                              OB_ALL_VIRTUAL_TENANT_PARAMETER_INFO_TNAME,
5271
                              tenant_id, ip_buf, addr.get_port()))) {
5272
     LOG_WARN("fail to assign fmt", KR(ret), K(tenant_id), K(addr));
5273
   }
5274
   while (OB_SUCC(ret)) {
5275
     if (OB_UNLIKELY(ctx.is_timeouted())) {
5276
       ret = OB_TIMEOUT;
5277
       LOG_WARN("check dest data version timeout", KR(ret),
5278
                K(start_timeout_ts), "abs_timeout", ctx.get_abs_timeout());
5279
     } else {
5280
       SMART_VAR(ObMySQLProxy::MySQLResult, res) {
5281
         sqlclient::ObMySQLResult *result = NULL;
5282
         if (OB_FAIL(proxy_->read(res, OB_SYS_TENANT_ID, sql.ptr()))) {
5283
           LOG_WARN("fail to read by sql", KR(ret), K(sql));
5284
         } else if (OB_ISNULL(result = res.get_result())) {
5285
           ret = OB_ERR_UNEXPECTED;
5286
           LOG_WARN("result is null", KR(ret));
5287
         } else if (OB_FAIL(result->next())) {
5288
           if (OB_ITER_END == ret) {
5289
             ret = OB_SUCCESS;
5290
             if (REACH_TIME_INTERVAL(CHECK_INTERVAL_TS)) {
5291
               LOG_WARN_RET(OB_EAGAIN, "check data_version is loaded, but result is empty, try later",
5292
                            K(tenant_id), K(addr));
5293
             }
5294
           } else {
5295
             LOG_WARN("fail to get next row", KR(ret));
5296
           }
5297
         } else {
5298
           int64_t loaded = 0;
5299
           EXTRACT_INT_FIELD_MYSQL(*result, "loaded", loaded, int64_t);
5300
           if (OB_SUCC(ret)) {
5301
             if (1 == loaded) {
5302
               break;
5303
             } else if (REACH_TIME_INTERVAL(CHECK_INTERVAL_TS))
5304
               LOG_WARN_RET(OB_EAGAIN, "check data_version is loaded, but it's not refreshed yet, try later",
5305
                            K(tenant_id), K(addr));
5306
           }
5307
         }
5308
       } // end SMART_VAR
5309

5310
       if (OB_SUCC(ret)) {
5311
         ob_usleep(SLEEP_TS);
5312
       }
5313
     }
5314
   } // end while
5315
 }
5316
 return ret;
5317
}
5318

5319
int ObUnitManager::build_notify_create_unit_resource_rpc_arg_(
5320
    const uint64_t tenant_id,
5321
    const share::ObUnit &unit,
5322
    const lib::Worker::CompatMode compat_mode,
5323
    const uint64_t unit_config_id,
5324
    const bool if_not_grant,
5325
    obrpc::TenantServerUnitConfig &rpc_arg) const
5326
{
5327
  int ret = OB_SUCCESS;
5328
  // get unit_config
5329
  share::ObUnitConfig *unit_config = nullptr;
5330
  if (OB_FAIL(get_unit_config_by_id(unit_config_id, unit_config))) {
5331
    LOG_WARN("fail to get unit config by id", KR(ret));
5332
  } else if (OB_ISNULL(unit_config)) {
5333
    ret = OB_ERR_UNEXPECTED;
5334
    LOG_WARN("unit config is null", KR(ret), "unit_config_id", unit_config_id);
5335
  }
5336

5337
#ifdef OB_BUILD_TDE_SECURITY
5338
  // get root_key
5339
  obrpc::ObRootKeyResult root_key;
5340
  obrpc::ObRootKeyArg get_rootkey_arg;
5341
  if (OB_FAIL(ret)) {
5342
    // do nothing
5343
  } else if (OB_ISNULL(root_service_)) {
5344
    ret = OB_ERR_UNEXPECTED;
5345
    LOG_WARN("root_service_ is nullptr", KR(ret), KP(root_service_));
5346
  } else if (OB_FAIL(get_rootkey_arg.init_for_get(tenant_id))) {
5347
    LOG_WARN("failed to init get_root_key arg", KR(ret), K(tenant_id));
5348
  } else if (OB_FAIL(root_service_->handle_get_root_key(get_rootkey_arg, root_key))) {
5349
    LOG_WARN("fail to get root_key", KR(ret), K(get_rootkey_arg));
5350
  }
5351
#endif
5352

5353
  // init rpc_arg
5354
  const bool is_delete = false;
5355
  if (OB_FAIL(ret)) {
5356
  } else if (OB_FAIL(rpc_arg.init(tenant_id,
5357
                                  unit.unit_id_,
5358
                                  compat_mode,
5359
                                  *unit_config,
5360
                                  ObReplicaType::REPLICA_TYPE_FULL,
5361
                                  if_not_grant,
5362
                                  is_delete
5363
#ifdef OB_BUILD_TDE_SECURITY
5364
                                  , root_key
5365
#endif
5366
                                  ))) {
5367
    LOG_WARN("fail to init rpc_arg", KR(ret), K(tenant_id), K(is_delete));
5368
  }
5369
  return ret;
5370
}
5371

5372
int ObUnitManager::do_notify_unit_resource_(
5373
  const common::ObAddr server,
5374
  const obrpc::TenantServerUnitConfig &notify_arg,
5375
  ObNotifyTenantServerResourceProxy &notify_proxy)
5376
{
5377
  int ret = OB_SUCCESS;
5378
  if (GCONF.self_addr_ == server) {
5379
    // Directly call local interface without using RPC
5380
    if (OB_FAIL(omt::ObTenantNodeBalancer::get_instance().handle_notify_unit_resource(notify_arg))) {
5381
      LOG_WARN("fail to handle_notify_unit_resource", K(ret), K(notify_arg));
5382
    } else {
5383
      LOG_INFO("call notify resource to server (locally)", KR(ret), "dst", server, K(notify_arg));
5384
    }
5385
  } else {
5386
    int64_t start = ObTimeUtility::current_time();
5387
    int64_t rpc_timeout = NOTIFY_RESOURCE_RPC_TIMEOUT;
5388
    if (INT64_MAX != THIS_WORKER.get_timeout_ts()) {
5389
      rpc_timeout = max(rpc_timeout, THIS_WORKER.get_timeout_remain());
5390
    }
5391
    if (OB_FAIL(notify_proxy.call(server, rpc_timeout, notify_arg))) {
5392
      LOG_WARN("fail to call notify resource to server", KR(ret), K(rpc_timeout),
5393
               "dst", server, "cost", ObTimeUtility::current_time() - start);
5394
    } else {
5395
      LOG_INFO("call notify resource to server", KR(ret), "dst", server, K(notify_arg),
5396
               "cost", ObTimeUtility::current_time() - start);
5397
    }
5398
  }
5399
  return ret;
5400
}
5401

5402
int ObUnitManager::rollback_persistent_units_(
5403
      const common::ObArray<share::ObUnit> &units,
5404
      const share::ObResourcePool &pool,
5405
      ObNotifyTenantServerResourceProxy &notify_proxy)
5406
{
5407
  int ret = OB_SUCCESS;
5408
  int tmp_ret = OB_SUCCESS;
5409
  const bool is_delete = true;
5410
  ObArray<int> return_ret_array;
5411
  notify_proxy.reuse();
5412
  for (int64_t i = 0; i < units.count(); i++) {
5413
    const ObUnit & unit = units.at(i);
5414
    const lib::Worker::CompatMode dummy_mode = lib::Worker::CompatMode::INVALID;
5415
    if (OB_TMP_FAIL(try_notify_tenant_server_unit_resource_(
5416
        pool.tenant_id_, is_delete, notify_proxy,
5417
        pool.unit_config_id_, dummy_mode, unit,
5418
        false/*if_not_grant*/, false/*skip_offline_server*/,
5419
        false /*check_data_version*/))) {
5420
      ret = OB_SUCC(ret) ? tmp_ret : ret;
5421
      LOG_WARN("fail to try notify server unit resource", KR(ret), KR(tmp_ret),
5422
               K(is_delete), K(pool), K(dummy_mode), K(unit));
5423
    }
5424
  }
5425
  if (OB_TMP_FAIL(notify_proxy.wait_all(return_ret_array))) {
5426
    LOG_WARN("fail to wait notify resource", KR(ret), K(tmp_ret));
5427
    ret = OB_SUCC(ret) ? tmp_ret : ret;
5428
  } else if (OB_FAIL(ret)) {
5429
  } else {
5430
    // don't use arg/dest/result here because call() may has failure.
5431
    ObAddr invalid_addr;
5432
    for (int64_t i = 0; OB_SUCC(ret) && i < return_ret_array.count(); i++) {
5433
      const int ret_i = return_ret_array.at(i);
5434
      const ObAddr &addr = return_ret_array.count() != notify_proxy.get_dests().count() ?
5435
                           invalid_addr : notify_proxy.get_dests().at(i);
5436
      // if (OB_SUCCESS != ret_i && OB_TENANT_NOT_IN_SERVER != ret_i) {
5437
      if (OB_SUCCESS != ret_i && OB_TENANT_NOT_IN_SERVER != ret_i) {
5438
        ret = ret_i;
5439
        LOG_WARN("fail to mark tenant removed", KR(ret), KR(ret_i), K(addr));
5440
      }
5441
    }
5442
  }
5443
  LOG_WARN("rollback persistent unit", KR(ret), K(pool), K(units));
5444
  return ret;
5445
}
5446

5447
int ObUnitManager::get_tenant_unit_servers(
5448
    const uint64_t tenant_id,
5449
    const common::ObZone &zone,
5450
    common::ObIArray<common::ObAddr> &server_array) const
5451
{
5452
  int ret = OB_SUCCESS;
5453
  ObArray<share::ObResourcePool *> *pools = nullptr;
5454
  if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
5455
    ret = OB_INVALID_ARGUMENT;
5456
    LOG_WARN("invalid argument", K(ret), K(tenant_id));
5457
  } else {
5458
    server_array.reset();
5459
    int tmp_ret = get_pools_by_tenant_(tenant_id, pools);
5460
    if (OB_ENTRY_NOT_EXIST == tmp_ret) {
5461
      // pass, and return empty server array
5462
    } else if (OB_SUCCESS != tmp_ret) {
5463
      ret = tmp_ret;
5464
      LOG_WARN("fail to get pools by tenant", K(ret), K(tenant_id));
5465
    } else if (nullptr == pools) {
5466
      ret = OB_ERR_UNEXPECTED;
5467
      LOG_WARN("pools ptr is null, unexpected", K(ret), K(tenant_id));
5468
    } else {
5469
      common::ObArray<common::ObAddr> this_server_array;
5470
      for (int64_t i = 0; OB_SUCC(ret) && i < pools->count(); ++i) {
5471
        this_server_array.reset();
5472
        const share::ObResourcePool *pool = pools->at(i);
5473
        if (OB_UNLIKELY(nullptr == pool)) {
5474
          ret = OB_ERR_UNEXPECTED;
5475
          LOG_WARN("pool ptr is null", K(ret), K(tenant_id));
5476
        } else if (OB_FAIL(get_pool_servers(
5477
                pool->resource_pool_id_, zone, this_server_array))) {
5478
          LOG_WARN("fail to get pool server", K(ret),
5479
                   "pool_id", pool->resource_pool_id_, K(zone));
5480
        } else if (OB_FAIL(append(server_array, this_server_array))) {
5481
          LOG_WARN("fail to append", K(ret));
5482
        }
5483
      }
5484
    }
5485
  }
5486
  return ret;
5487
}
5488
ERRSIM_POINT_DEF(ERRSIM_UNIT_PERSISTENCE_ERROR);
5489

5490
// allocate unit on target zones for specified resource pool
5491
//
5492
// @param [in] client                     SQL client
5493
// @param [in] pool                       resource pool which new allocated units belong to
5494
// @param [in] zones                      target zones to allocate units
5495
// @param [in] unit_group_id_array        unit group id array for new allocated units
5496
// @param [in] new_allocate_pool          whether to allocate new pool
5497
// @param [in] increase_delta_unit_num    unit number to be increased
5498
// @param [in] module                     module name for print log
5499
// @param [out] new_servers               allocated servers for new units
5500
//
5501
// @ret OB_ZONE_RESOURCE_NOT_ENOUGH   zone resource not enough to hold all units
5502
// @ret OB_ZONE_SERVER_NOT_ENOUGH     zone server not enough to hold all units
5503
// @ret OB_SUCCESS                    success
5504
int ObUnitManager::allocate_pool_units_(
5505
    ObISQLClient &client,
5506
    const share::ObResourcePool &pool,
5507
    const common::ObIArray<common::ObZone> &zones,
5508
    common::ObIArray<uint64_t> *unit_group_id_array,
5509
    const bool new_allocate_pool,
5510
    const int64_t increase_delta_unit_num,
5511
    const char *module,
5512
    ObIArray<ObAddr> &new_servers)
5513
{
5514
  int ret = OB_SUCCESS;
5515
  ObUnitConfig *config = NULL;
5516
  lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::INVALID;
5517
  ObArray<ObServerInfoInTable> servers_info;
5518
  ObArray<ObServerInfoInTable> active_servers_info_of_zone;
5519
  ObArray<obrpc::ObGetServerResourceInfoResult> active_servers_resource_info_of_zone;
5520

5521
  if (!check_inner_stat()) {
5522
    ret = OB_INNER_STAT_ERROR;
5523
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
5524
  } else if (!pool.is_valid() || zones.count() <= 0) {
5525
    ret = OB_INVALID_ARGUMENT;
5526
    LOG_WARN("invalid resource pool or zones", K(pool), K(zones), KR(ret));
5527
  } else if (increase_delta_unit_num <= 0) {
5528
    ret = OB_INVALID_ARGUMENT;
5529
    LOG_WARN("invalid argument", K(increase_delta_unit_num), K(ret));
5530
  } else if (nullptr != unit_group_id_array
5531
      && unit_group_id_array->count() != increase_delta_unit_num) {
5532
    ret = OB_INVALID_ARGUMENT;
5533
    LOG_WARN("new unit group id array status not match",
5534
        KR(ret), K(increase_delta_unit_num), KP(unit_group_id_array));
5535
  } else if (OB_ISNULL(srv_rpc_proxy_) || OB_ISNULL(GCTX.sql_proxy_)) {
5536
    ret = OB_ERR_UNEXPECTED;
5537
    LOG_WARN("srv_rpc_proxy_ or GCTX.sql_proxy_ is null", KR(ret), KP(srv_rpc_proxy_), KP(GCTX.sql_proxy_));
5538
  } else if (is_valid_tenant_id(pool.tenant_id_)
5539
      && OB_FAIL(ObCompatModeGetter::get_tenant_mode(pool.tenant_id_, compat_mode))) {
5540
    LOG_WARN("fail to get tenant compat mode", KR(ret), K(pool.tenant_id_));
5541
  } else if (OB_FAIL(get_unit_config_by_id(pool.unit_config_id_, config))) {
5542
    LOG_WARN("get_unit_config_by_id failed", "unit_config_id", pool.unit_config_id_, KR(ret));
5543
  } else if (OB_ISNULL(config)) {
5544
    ret = OB_ERR_UNEXPECTED;
5545
    LOG_WARN("config is null", KP(config), K(ret));
5546
  } else {
5547
    ObNotifyTenantServerResourceProxy notify_proxy(
5548
        *srv_rpc_proxy_,
5549
        &obrpc::ObSrvRpcProxy::notify_tenant_server_unit_resource);
5550

5551
    ObArray<ObAddr> excluded_servers;
5552
    ObArray<ObUnit> units;
5553
    for (int64_t i = 0; OB_SUCC(ret) && i < zones.count(); ++i) { // for each zone
5554
      const ObZone &zone = zones.at(i);
5555
      excluded_servers.reuse();
5556
      active_servers_info_of_zone.reuse();
5557
      active_servers_resource_info_of_zone.reuse();
5558
      const bool ONLY_ACTIVE_SERVERS = true;
5559
      if (FAILEDx(get_excluded_servers(pool.resource_pool_id_, zone, module,
5560
              new_allocate_pool, excluded_servers))) {
5561
        LOG_WARN("get excluded servers fail", KR(ret), K(pool.resource_pool_id_), K(zone),
5562
            K(module), K(new_allocate_pool));
5563
      } else if (OB_FAIL(ObServerTableOperator::get_servers_info_of_zone(
5564
          *GCTX.sql_proxy_,
5565
          zone,
5566
          ONLY_ACTIVE_SERVERS,
5567
          active_servers_info_of_zone))) {
5568
        LOG_WARN("fail to get servers info of zone", KR(ret), K(zone));
5569
      } else if (OB_FAIL(get_servers_resource_info_via_rpc(
5570
          active_servers_info_of_zone,
5571
          active_servers_resource_info_of_zone))) {
5572
        LOG_WARN("fail to get active_servers_resource_info_of_zone", KR(ret), K(active_servers_info_of_zone));
5573
      }
5574

5575
      for (int64_t j = 0; OB_SUCC(ret) && j < increase_delta_unit_num; ++j) {
5576
        uint64_t unit_id = OB_INVALID_ID;
5577
        std::string resource_not_enough_reason;
5578
        ObAddr server;
5579
        if (OB_FAIL(choose_server_for_unit(
5580
            config->unit_resource(),
5581
            zone,
5582
            excluded_servers,
5583
            module,
5584
            active_servers_info_of_zone,
5585
            active_servers_resource_info_of_zone,
5586
            server,
5587
            resource_not_enough_reason))) {
5588
          LOG_WARN("choose server for unit failed", K(module), KR(ret), "unit_idx", j, K(increase_delta_unit_num),
5589
              K(zone), K(excluded_servers), KPC(config));
5590
          // handle return error info
5591
          if (OB_ZONE_RESOURCE_NOT_ENOUGH == ret) {
5592
            LOG_USER_ERROR(OB_ZONE_RESOURCE_NOT_ENOUGH,
5593
                to_cstring(zone), increase_delta_unit_num, resource_not_enough_reason.c_str());
5594
          } else if (OB_ZONE_SERVER_NOT_ENOUGH == ret) {
5595
            LOG_USER_ERROR(OB_ZONE_SERVER_NOT_ENOUGH, to_cstring(zone), increase_delta_unit_num);
5596
          }
5597
        } else if (OB_FAIL(excluded_servers.push_back(server))) {
5598
          LOG_WARN("push_back failed", K(ret));
5599
        } else if (OB_FAIL(try_persist_unit_info_(
5600
                               notify_proxy,
5601
                               client,
5602
                               zone,
5603
                               pool,
5604
                               compat_mode,
5605
                               (nullptr == unit_group_id_array) ? 0 : unit_group_id_array->at(j),
5606
                               server,
5607
                               new_servers,
5608
                               units))) {
5609
          LOG_WARN("fail to persist unit info", KR(ret), K(zone), K(pool), K(compat_mode), K(server));
5610
        }
5611
      }
5612
    }
5613
    int tmp_ret = OB_SUCCESS;
5614
    if (OB_TMP_FAIL(notify_proxy.wait())) {
5615
      LOG_WARN("fail to wait notify resource", K(ret), K(tmp_ret));
5616
      ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
5617
    } else if (OB_SUCC(ret)) {
5618
      // arg/dest/result can be used here.
5619
    }
5620
    if (is_valid_tenant_id(pool.tenant_id_)) {
5621
      ret = ERRSIM_UNIT_PERSISTENCE_ERROR ? : ret;
5622
    }
5623
    if (OB_FAIL(ret)) {
5624
      LOG_WARN("start to rollback unit persistence", KR(ret), K(units), K(pool));
5625
      if(OB_TMP_FAIL(rollback_persistent_units_(units, pool, notify_proxy))) {
5626
        LOG_WARN("fail to rollback unit persistence", KR(ret), KR(tmp_ret), K(units),
5627
            K(pool), K(compat_mode));
5628
      }
5629
    }
5630
  }
5631
  return ret;
5632
}
5633

5634
int ObUnitManager::try_persist_unit_info_(
5635
    ObNotifyTenantServerResourceProxy &notify_proxy,
5636
    ObISQLClient &client,
5637
    const ObZone &zone,
5638
    const share::ObResourcePool &pool,
5639
    const lib::Worker::CompatMode &compat_mode,
5640
    const uint64_t unit_group_id,
5641
    const ObAddr &server,
5642
    ObIArray<common::ObAddr> &new_servers,
5643
    ObIArray<share::ObUnit> &units)
5644
{
5645
  int ret = OB_SUCCESS;
5646
  ObUnit unit;
5647
  uint64_t new_unit_id = OB_INVALID_ID;
5648
  const bool is_delete = false; // is_delete is false when allocate new unit
5649
  if (!check_inner_stat()) {
5650
    ret = OB_INNER_STAT_ERROR;
5651
    LOG_WARN("check_inner_stat failed", KR(ret), K(inited_), K(loaded_));
5652
  } else if (OB_UNLIKELY(zone.is_empty())
5653
             || OB_UNLIKELY(!server.is_valid())) {
5654
    ret = OB_INVALID_ARGUMENT;
5655
    LOG_WARN("invalid argument", KR(ret), K(zone), K(server));
5656
  } else if (OB_FAIL(fetch_new_unit_id(new_unit_id))) {
5657
    LOG_WARN("fetch_new_unit_id failed", KR(ret));
5658
  } else {
5659
    unit.unit_id_ = new_unit_id;
5660
    unit.resource_pool_id_ = pool.resource_pool_id_;
5661
    unit.unit_group_id_ = unit_group_id;
5662
    unit.server_ = server;
5663
    unit.status_ = ObUnit::UNIT_STATUS_ACTIVE;
5664
    unit.replica_type_ = pool.replica_type_;
5665
    if (OB_FAIL(unit.zone_.assign(zone))) {
5666
      LOG_WARN("fail to assign zone", KR(ret), K(zone));
5667
    } else if (OB_FAIL(try_notify_tenant_server_unit_resource_(
5668
                           pool.tenant_id_, is_delete, notify_proxy,
5669
                           pool.unit_config_id_, compat_mode, unit, false/*if not grant*/,
5670
                           false/*skip offline server*/, true /*check_data_version*/))) {
5671
      LOG_WARN("fail to try notify server unit resource", KR(ret), K(pool), K(is_delete), K(unit));
5672
    } else if (OB_FAIL(add_unit(client, unit))) {
5673
      LOG_WARN("add_unit failed", KR(ret), K(unit), K(unit));
5674
    } else if (OB_FAIL(new_servers.push_back(server))) {
5675
      LOG_WARN("push_back failed", KR(ret), K(server));
5676
    } else if (OB_FAIL(units.push_back(unit))) {
5677
      LOG_WARN("fail to push an element into units", KR(ret), K(unit));
5678
    }
5679
  }
5680
  return ret;
5681
}
5682

5683
int ObUnitManager::get_excluded_servers(
5684
    const ObUnit &unit,
5685
    const ObUnitStat &unit_stat,
5686
    const char *module,
5687
    const ObIArray<ObServerInfoInTable> &servers_info, // servers info in unit.zone_
5688
    const ObIArray<obrpc::ObGetServerResourceInfoResult> &report_servers_resource_info, // active servers' resource info in unit.zone_
5689
    ObIArray<ObAddr> &servers) const
5690
{
5691
  int ret = OB_SUCCESS;
5692
  //Add all OBS whose disks do not meet the requirements
5693
  ObArray<share::ObServerStatus> server_list;
5694
  ObServerResourceInfo server_resource_info;
5695
  const bool new_allocate_pool = false;
5696
  if (OB_FAIL(get_excluded_servers(unit.resource_pool_id_, unit.zone_, module,
5697
          new_allocate_pool, servers))) {
5698
    LOG_WARN("fail to get excluded_servers", K(ret), K(unit), K(new_allocate_pool));
5699
  } else {
5700
    for (int64_t i = 0; i < servers_info.count() && OB_SUCC(ret); i++) {
5701
      const ObServerInfoInTable &server_info = servers_info.at(i);
5702
      const ObAddr &server = server_info.get_server();
5703
      bool is_exclude = false;
5704
      server_resource_info.reset();
5705
      if (!server_info.can_migrate_in()) {
5706
        is_exclude = true;
5707
        LOG_INFO("server can't migrate in, push into excluded_array", K(server_info), K(module));
5708
      } else if (OB_FAIL(ObRootUtils::get_server_resource_info(
5709
          report_servers_resource_info,
5710
          server,
5711
          server_resource_info))) {
5712
        // server which can be migrated in must have its resource_info
5713
        LOG_WARN("fail to get server_resource_info", KR(ret), K(report_servers_resource_info), K(server));
5714
      } else {
5715
        int64_t required_size = unit_stat.get_required_size() + server_resource_info.disk_in_use_;
5716
        int64_t total_size = server_resource_info.disk_total_;
5717
        if (total_size <= required_size || total_size <= 0) {
5718
          is_exclude = true;
5719
          LOG_INFO("server total size no bigger than required size", K(module), K(required_size),
5720
              K(total_size), K(unit_stat), K(server_resource_info));
5721
        } else if (required_size <= 0) {
5722
          //nothing todo
5723
        } else {
5724
          int64_t required_percent = (100 * required_size) / total_size;
5725
          int64_t limit_percent = GCONF.data_disk_usage_limit_percentage;
5726
          if (required_percent > limit_percent) {
5727
            is_exclude = true;
5728
            LOG_INFO("server disk percent will out of control;", K(module), K(required_percent), K(limit_percent),
5729
                     K(required_size), K(total_size));
5730
          }
5731
        }
5732
      }
5733
      if (!is_exclude) {
5734
        //nothing todo
5735
      } else if (has_exist_in_array(servers, server)) {
5736
        //nothing todo
5737
      } else if (OB_FAIL(servers.push_back(server))) {
5738
        LOG_WARN("fail to push back", KR(ret), K(server));
5739
      }
5740
    }
5741
  }
5742
  return ret;
5743
}
5744

5745
// get excluded servers for resource pool on target zone.
5746
//
5747
// 1. resource pool units servers on target zone
5748
// 2. tenant all resource pool units servers on target zone
5749
//
5750
// FIXME: 4.0 not support resource pool intersect in zone, here need not consider multiple pools of tenant
5751
//
5752
// @param [in] resource_pool_id   specified resource pool id
5753
// @param [in] zone               specified zone which may be empty that means all zones
5754
// @param [in] new_allocate_pool  new allocate pool 
5755
// @param [out] servers           returned excluded servers
5756
int ObUnitManager::get_excluded_servers(const uint64_t resource_pool_id,
5757
    const ObZone &zone,
5758
    const char *module,
5759
    const bool new_allocate_pool, 
5760
    ObIArray<ObAddr> &excluded_servers) const
5761
{
5762
  int ret = OB_SUCCESS;
5763
  share::ObResourcePool *pool = NULL;
5764
  int64_t tenant_id = OB_INVALID_ID;
5765
  common::ObArray<common::ObAddr> sys_standalone_servers;
5766
  common::ObArray<share::ObResourcePool *> *all_pools = NULL;
5767

5768
  excluded_servers.reset();
5769
  if (!check_inner_stat()) {
5770
    ret = OB_INNER_STAT_ERROR;
5771
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
5772
  } else if (OB_UNLIKELY(OB_INVALID_ID == resource_pool_id)) {
5773
    ret = OB_INVALID_ARGUMENT;
5774
    LOG_WARN("invalid resource pool id", KR(ret), K(resource_pool_id));
5775
  } else if (new_allocate_pool) {
5776
    //nothing todo
5777
  } else if (OB_FAIL(get_resource_pool_by_id(resource_pool_id, pool))) {
5778
    LOG_WARN("fail to get resource pool by id", K(ret), K(resource_pool_id));
5779
  } else if (FALSE_IT(tenant_id = pool->tenant_id_)) {
5780
    //nothing todo
5781
  } else if (! is_valid_tenant_id(tenant_id)) {
5782
    // The pool does not belong to any tenant, only get resource pool servers
5783
    if (OB_FAIL(get_pool_servers(resource_pool_id, zone, excluded_servers))) {
5784
      LOG_WARN("get_pool_servers failed", "resource pool id",
5785
          resource_pool_id, K(zone), KR(ret));
5786
    }
5787
  }
5788
  // get all tenant resource pool related servers on target zone
5789
  else if (OB_FAIL(get_tenant_unit_servers(tenant_id, zone, excluded_servers))) {
5790
    LOG_WARN("get tennat unit server fail", KR(ret), K(tenant_id), K(zone));
5791
  }
5792

5793
  if (OB_SUCC(ret) && GCONF.enable_sys_unit_standalone) {
5794
    // When the system tenant is deployed independently,
5795
    // the server where the unit of the system tenant is located is also required as the executed servers
5796
    if (OB_FAIL(get_tenant_unit_servers(OB_SYS_TENANT_ID, zone, sys_standalone_servers))) {
5797
      LOG_WARN("fail to get tenant unit servers", KR(ret), K(zone));
5798
    } else if (OB_FAIL(append(excluded_servers, sys_standalone_servers))) {
5799
      LOG_WARN("fail to append other excluded servers", K(ret));
5800
    }
5801
  }
5802

5803
  LOG_INFO("get tenant resource pool servers and sys standalone servers as excluded servers",
5804
      K(module), K(new_allocate_pool),
5805
      KR(ret),
5806
      K(resource_pool_id),
5807
      K(tenant_id),
5808
      K(zone),
5809
      "enable_sys_unit_standalone", GCONF.enable_sys_unit_standalone,
5810
      K(excluded_servers),
5811
      K(sys_standalone_servers));
5812
  return ret;
5813
}
5814

5815

5816
int ObUnitManager::get_pools_servers(const common::ObIArray<share::ObResourcePool  *> &pools,
5817
    common::hash::ObHashMap<common::ObAddr, int64_t> &server_ref_count_map) const
5818
{
5819
  int ret = OB_SUCCESS;
5820
  if (!check_inner_stat()) {
5821
    ret = OB_INNER_STAT_ERROR;
5822
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
5823
  } else if (pools.count() <= 0 || !server_ref_count_map.created()) {
5824
    ret = OB_INVALID_ARGUMENT;
5825
    LOG_WARN("pools is empty", K(pools),
5826
        "server_ref_count_map created", server_ref_count_map.created(), K(ret));
5827
  } else {
5828
    ObArray<ObAddr> servers;
5829
    const ObZone all_zone;
5830
    FOREACH_CNT_X(pool, pools, OB_SUCCESS == ret) {
5831
      servers.reuse();
5832
      if (NULL == *pool) {
5833
        ret = OB_INVALID_ARGUMENT;
5834
        LOG_WARN("pool is null", "pool", *pool, K(ret));
5835
      } else if (OB_FAIL(get_pool_servers((*pool)->resource_pool_id_, all_zone, servers))) {
5836
        LOG_WARN("get pool servers failed",
5837
            "pool id", (*pool)->resource_pool_id_, K(all_zone), K(ret));
5838
      } else {
5839
        FOREACH_CNT_X(server, servers, OB_SUCCESS == ret) {
5840
          int64_t server_ref_count = 0;
5841
          if (OB_FAIL(get_server_ref_count(server_ref_count_map, *server, server_ref_count))) {
5842
            if (OB_ENTRY_NOT_EXIST == ret) {
5843
              server_ref_count = 1;
5844
              ret = OB_SUCCESS;
5845
            } else {
5846
              LOG_WARN("get server ref count failed", "server", *server, K(ret));
5847
            }
5848
          } else {
5849
            ++server_ref_count;
5850
          }
5851
          if (OB_SUCC(ret)) {
5852
            if (OB_FAIL(set_server_ref_count(server_ref_count_map, *server, server_ref_count))) {
5853
              LOG_WARN("set server ref count failed",
5854
                  "server", *server, K(server_ref_count), K(ret));
5855
            }
5856
          }
5857
        }
5858
      }
5859
    }
5860
  }
5861
  return ret;
5862
}
5863

5864
int ObUnitManager::get_pool_servers(const uint64_t resource_pool_id,
5865
                                    const ObZone &zone,
5866
                                    ObIArray<ObAddr> &servers) const
5867
{
5868
  int ret = OB_SUCCESS;
5869
  ObArray<ObUnit *> *units = NULL;
5870
  if (!check_inner_stat()) {
5871
    ret = OB_INNER_STAT_ERROR;
5872
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
5873
  } else if (OB_INVALID_ID == resource_pool_id) {
5874
    // don't need to check zone, can be empty
5875
    ret = OB_INVALID_ARGUMENT;
5876
    LOG_WARN("invalid resource_pool_id", K(resource_pool_id), K(ret));
5877
  } else if (OB_FAIL(get_units_by_pool(resource_pool_id, units))) {
5878
    LOG_WARN("get_units_by_pool failed", K(resource_pool_id), K(ret));
5879
  } else if (NULL == units) {
5880
    ret = OB_ERR_UNEXPECTED;
5881
    LOG_WARN("units is null", KP(units), K(ret));
5882
  } else {
5883
    for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
5884
      if (NULL == units->at(i)) {
5885
        ret = OB_ERR_UNEXPECTED;
5886
        LOG_WARN("unit is null", "unit", OB_P(units->at(i)), K(ret));
5887
      } else if (!zone.is_empty() && zone != units->at(i)->zone_) {
5888
        continue;
5889
      } else {
5890
        if (OB_SUCCESS == ret && units->at(i)->migrate_from_server_.is_valid()) {
5891
          if (OB_FAIL(servers.push_back(units->at(i)->migrate_from_server_))) {
5892
            LOG_WARN("push_back failed", K(ret));
5893
          }
5894
        }
5895

5896
        if (OB_SUCCESS == ret && units->at(i)->server_.is_valid()) {
5897
          if (OB_FAIL(servers.push_back(units->at(i)->server_))) {
5898
            LOG_WARN("push_back failed", K(ret));
5899
          }
5900
        }
5901
      }
5902
    }
5903
  }
5904
  return ret;
5905
}
5906

5907
// choose server for target one unit
5908
//
5909
// @param [in] config                         target unit resource
5910
// @param [in] zone                           target zone
5911
// @param [in] excluded_servers               excluded servers which can not allocate unit
5912
// @param [in] module                         module name for log print
5913
// @param [out] server                        choosed server for unit
5914
// @param [out] resource_not_enough_reason    reason for resource not enough,
5915
//                                            is valid when ret = OB_ZONE_RESOURCE_NOT_ENOUGH
5916
//
5917
// @ret OB_SUCCESS                    on success
5918
// @ret OB_ZONE_RESOURCE_NOT_ENOUGH   zone resource not enough to hold new unit
5919
// @ret OB_ZONE_SERVER_NOT_ENOUGH     all valid servers are excluded, no server to hold new unit
5920
int ObUnitManager::choose_server_for_unit(
5921
    const ObUnitResource &config,
5922
    const ObZone &zone,
5923
    const ObArray<ObAddr> &excluded_servers,
5924
    const char *module,
5925
    const ObIArray<ObServerInfoInTable> &active_servers_info, // active_servers_info of the give zone,
5926
    const ObIArray<obrpc::ObGetServerResourceInfoResult> &active_servers_resource_info, // active_servers_resource_info of the give zone
5927
    ObAddr &choosed_server,
5928
    std::string &resource_not_enough_reason) const
5929
{
5930
  int ret = OB_SUCCESS;
5931
  ObArray<ObServerStatus> statuses;
5932
  ObArray<ObUnitPlacementStrategy::ObServerResource> server_resources;
5933
  ObArray<ObServerInfoInTable> servers_info;
5934
  ObArray<obrpc::ObGetServerResourceInfoResult> report_servers_resource_info;
5935
  if (!check_inner_stat()) {
5936
    ret = OB_INNER_STAT_ERROR;
5937
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
5938
  } else if (!config.is_valid() || zone.is_empty()) {
5939
    // excluded_servers can be empty
5940
    ret = OB_INVALID_ARGUMENT;
5941
    LOG_WARN("invalid config", KR(ret), K(config), K(zone));
5942
  } else if (OB_FAIL(build_server_resources_(active_servers_resource_info, server_resources))) {
5943
    LOG_WARN("fail to build server resources", KR(ret), K(active_servers_resource_info));
5944
  } else if (OB_FAIL(do_choose_server_for_unit_(config, zone, excluded_servers, active_servers_info,
5945
      server_resources, module, choosed_server, resource_not_enough_reason))) {
5946
    LOG_WARN("fail to choose server for unit", K(module), KR(ret), K(config), K(zone),
5947
        K(excluded_servers), K(servers_info), K(server_resources),
5948
        "resource_not_enough_reason", resource_not_enough_reason.c_str());
5949
  }
5950
  return ret;
5951
}
5952

5953
int ObUnitManager::check_server_status_valid_and_construct_log_(
5954
    const share::ObServerInfoInTable &server_info,
5955
    const bool for_clone_tenant,
5956
    bool &is_server_valid,
5957
    std::string &not_valid_reason) const
5958
{
5959
  int ret = OB_SUCCESS;
5960
  is_server_valid = false;
5961
  const ObAddr &server = server_info.get_server();
5962
  if (!check_inner_stat()) {
5963
    ret = OB_INNER_STAT_ERROR;
5964
    LOG_WARN("check_inner_stat failed", KR(ret), K(inited_), K(loaded_));
5965
  } else if (OB_UNLIKELY(!server_info.is_valid())) {
5966
    ret = OB_INVALID_ARGUMENT;
5967
    LOG_WARN("invalid argument", KR(ret), K(server_info));
5968
  } else if (!for_clone_tenant && !server_info.is_active()) {
5969
    // server is inactive
5970
    is_server_valid = false;
5971
    not_valid_reason = not_valid_reason + "server '" + to_cstring(server) + "' is not active\n";
5972
  } else if (server_info.is_migrate_in_blocked()) {
5973
    // server is block-migrate-in
5974
    is_server_valid = false;
5975
    not_valid_reason = not_valid_reason + "server '" + to_cstring(server) + "' is blocked migrate-in\n";
5976
  } else {
5977
    is_server_valid = true;
5978
  }
5979
  return ret;
5980
}
5981

5982
int ObUnitManager::check_server_resource_enough_and_construct_log_(
5983
    const ObUnitPlacementStrategy::ObServerResource &server_resource,
5984
    const share::ObUnitResource &config,
5985
    bool &is_resource_enough,
5986
    std::string &resource_not_enough_reason) const
5987
{
5988
  int ret = OB_SUCCESS;
5989
  is_resource_enough = false;
5990
  double hard_limit = 1.0;
5991
  const ObAddr &server = server_resource.get_server();
5992
  ObResourceType not_enough_resource = RES_MAX;
5993
  AlterResourceErr not_enough_resource_config = ALT_ERR;
5994

5995
  if (!check_inner_stat()) {
5996
    ret = OB_INNER_STAT_ERROR;
5997
    LOG_WARN("check_inner_stat failed", KR(ret), K(inited_), K(loaded_));
5998
  } else if (OB_FAIL(get_hard_limit(hard_limit))) {
5999
    LOG_WARN("get_hard_limit failed", KR(ret));
6000
  } else if (FALSE_IT(is_resource_enough = check_resource_enough_for_unit_(
6001
                             server_resource, config, hard_limit,
6002
                             not_enough_resource, not_enough_resource_config))) {
6003
    // shall never be here
6004
  } else if (!is_resource_enough) {
6005
    resource_not_enough_reason =
6006
        resource_not_enough_reason + "server '" + to_cstring(server) + "' "
6007
        + resource_type_to_str(not_enough_resource) + " resource not enough\n";
6008
  }
6009
  return ret;
6010
}
6011

6012
int ObUnitManager::construct_valid_servers_resource_(
6013
    const ObZone &zone,
6014
    const share::ObUnitResource &config,
6015
    const ObIArray<ObAddr> &excluded_servers,
6016
    const ObIArray<share::ObServerInfoInTable> &servers_info,
6017
    const ObIArray<ObUnitPlacementStrategy::ObServerResource> &server_resources,
6018
    const char *module,
6019
    const bool for_clone_tenant,
6020
    int64_t &not_excluded_server_count,
6021
    std::string &resource_not_enough_reason,
6022
    ObIArray<ObUnitPlacementStrategy::ObServerResource> &valid_server_resources) const
6023
{
6024
  int ret = OB_SUCCESS;
6025
  not_excluded_server_count = 0;
6026
  valid_server_resources.reset();
6027
  if (OB_UNLIKELY(servers_info.count() != server_resources.count())
6028
      || OB_UNLIKELY(zone.is_empty())) {
6029
    ret = OB_INVALID_ARGUMENT;
6030
    LOG_WARN("invalid argument", KR(ret), K(servers_info), K(server_resources), K(zone));
6031
  } else {
6032
    for (int64_t i = 0; OB_SUCC(ret) && i < servers_info.count(); ++i) { // for each active servers
6033
      ObResourceType not_enough_resource = RES_MAX;
6034
      AlterResourceErr not_enough_resource_config = ALT_ERR;
6035
      const ObServerInfoInTable &server_info = servers_info.at(i);
6036
      const ObAddr &server = server_info.get_server();
6037
      const ObUnitPlacementStrategy::ObServerResource &server_resource = server_resources.at(i);
6038
      bool is_server_valid = false;
6039
      bool is_resource_enough = false;
6040

6041
      if (has_exist_in_array(excluded_servers, server)) {
6042
        // excluded servers are expected, need not show in reason
6043
        continue;
6044
      } else {
6045
        not_excluded_server_count++;
6046
        if (OB_FAIL(check_server_status_valid_and_construct_log_(
6047
                        server_info,
6048
                        for_clone_tenant,
6049
                        is_server_valid,
6050
                        resource_not_enough_reason))) {
6051
          LOG_WARN("fail to check server status", KR(ret), K(server_info), K(for_clone_tenant));
6052
        } else if (!is_server_valid) {
6053
          if (for_clone_tenant) {
6054
            ret = OB_OP_NOT_ALLOW;
6055
            LOG_WARN("[CLONE_TENANT_UNIT] server with invalid status, can not create clone tenant resource pool",
6056
                     KR(ret), K(server_info), K(is_server_valid));
6057
            LOG_USER_ERROR(OB_OP_NOT_ALLOW, "server is block migrate in, create clone tenant resource pool");
6058
          } else {
6059
            LOG_WARN("[CHOOSE_SERVER_FOR_UNIT] server can not migrate in", K(module), K(i), K(server_info));
6060
            continue;
6061
          }
6062
        } else if (OB_FAIL(check_server_resource_enough_and_construct_log_(
6063
                               server_resource,
6064
                               config,
6065
                               is_resource_enough,
6066
                               resource_not_enough_reason))) {
6067
          LOG_WARN("fail to check server resource", KR(ret), K(server_resource), K(config));
6068
        } else if (is_resource_enough) {
6069
          LOG_INFO("[CHOOSE_SERVER_FOR_UNIT] find available server", K(module), K(i),
6070
                   K(server_resource), "request_unit_config", config);
6071
          if (OB_FAIL(valid_server_resources.push_back(server_resource))) {
6072
            LOG_WARN("failed to push into array", KR(ret), K(server_resource));
6073
          }
6074
        } else {
6075
          if (for_clone_tenant) {
6076
            ret = OB_ZONE_RESOURCE_NOT_ENOUGH;
6077
            LOG_WARN("resource not enough", KR(ret), K(server_resource), K(config));
6078
            LOG_USER_ERROR(OB_ZONE_RESOURCE_NOT_ENOUGH,
6079
                    to_cstring(zone), servers_info.count(), resource_not_enough_reason.c_str());
6080
          } else {
6081
            LOG_INFO("[CHOOSE_SERVER_FOR_UNIT] server resource not enough", K(module), K(i),
6082
                  "not_enough_resource", resource_type_to_str(not_enough_resource),
6083
                  "not_enough_resource_config", alter_resource_err_to_str(not_enough_resource_config),
6084
                  K(server_resource), "request_unit_config", config);
6085
          }
6086
        }
6087
      }
6088
    }
6089
  }
6090
  return ret;
6091
}
6092

6093
int ObUnitManager::do_choose_server_for_unit_(const ObUnitResource &config,
6094
                                              const ObZone &zone,
6095
                                              const ObArray<ObAddr> &excluded_servers,
6096
                                              const ObIArray<share::ObServerInfoInTable> &servers_info,
6097
                                              const ObIArray<ObUnitPlacementStrategy::ObServerResource> &server_resources,
6098
                                              const char *module,
6099
                                              ObAddr &choosed_server,
6100
                                              std::string &resource_not_enough_reason) const
6101
{
6102
  LOG_INFO("[CHOOSE_SERVER_FOR_UNIT] begin", K(module), K(zone), K(excluded_servers), K(config));
6103

6104
  int ret = OB_SUCCESS;
6105
  double hard_limit = 1.0;
6106
  ObArray<ObUnitPlacementStrategy::ObServerResource> valid_server_resources;
6107
  bool for_clone_tenant = false;
6108

6109
  choosed_server.reset();
6110

6111
  if (OB_UNLIKELY(zone.is_empty())) {
6112
    ret = OB_INVALID_ARGUMENT;
6113
    LOG_WARN("zone is empty, unexpected", KR(ret), K(zone));
6114
  } else if (servers_info.count() != server_resources.count()) {
6115
    ret = OB_INVALID_ARGUMENT;
6116
    LOG_WARN("invalid servers_info and server_resources array", KR(ret), K(servers_info), K(server_resources));
6117
  } else if (OB_FAIL(get_hard_limit(hard_limit))) {
6118
    LOG_WARN("get_hard_limit failed", K(ret));
6119
  } else {
6120
    int64_t not_excluded_server_count = 0;
6121
    // 1. construct valid servers resource
6122
    if (OB_FAIL(construct_valid_servers_resource_(
6123
                    zone,
6124
                    config,
6125
                    excluded_servers,
6126
                    servers_info,
6127
                    server_resources,
6128
                    module,
6129
                    for_clone_tenant,
6130
                    not_excluded_server_count,
6131
                    resource_not_enough_reason,
6132
                    valid_server_resources))) {
6133
      LOG_WARN("fail to construct valid servers resource", KR(ret), K(zone), K(config), K(excluded_servers),
6134
               K(servers_info), K(server_resources), K(module), K(for_clone_tenant));
6135
    } else {
6136
      if (0 == not_excluded_server_count) {
6137
        ret = OB_ZONE_SERVER_NOT_ENOUGH;
6138
        LOG_WARN("zone server not enough to hold all units", K(module), KR(ret), K(zone), K(excluded_servers),
6139
            K(servers_info));
6140
      } else if (valid_server_resources.count() <= 0) {
6141
        ret = OB_ZONE_RESOURCE_NOT_ENOUGH;
6142
        LOG_WARN("zone resource is not enough to hold a new unit", K(module), KR(ret), K(zone),
6143
            K(config), K(excluded_servers),
6144
            "resource_not_enough_reason", resource_not_enough_reason.c_str());
6145
      } else {
6146
        // 2. choose the server
6147
        ObUnitPlacementDPStrategy unit_placement;
6148
        if (OB_FAIL(unit_placement.choose_server(valid_server_resources, config, module, choosed_server))) {
6149
          LOG_WARN("failed to choose server for unit", K(ret), K(config));
6150
        }
6151
      }
6152
    }
6153
  }
6154
  LOG_INFO("[CHOOSE_SERVER_FOR_UNIT] end", K(module), KR(ret), K(choosed_server), K(zone), K(excluded_servers),
6155
      K(config), K(valid_server_resources),
6156
      "resource_not_enough_reason", resource_not_enough_reason.c_str());
6157
  return ret;
6158
}
6159

6160
int ObUnitManager::compute_server_resource_(
6161
    const obrpc::ObGetServerResourceInfoResult &report_server_resource_info,
6162
    ObUnitPlacementStrategy::ObServerResource &server_resource) const
6163
{
6164
  int ret = OB_SUCCESS;
6165
  ObUnitConfig sum_load;
6166
  ObArray<ObUnitLoad> *unit_loads = NULL;
6167
  const ObAddr &server = report_server_resource_info.get_server();
6168
  const ObServerResourceInfo &report_resource = report_server_resource_info.get_resource_info();
6169
  if (OB_UNLIKELY(!report_server_resource_info.is_valid())) {
6170
    ret = OB_ERR_UNEXPECTED;
6171
    LOG_WARN("invalid argument", KR(ret), K(report_server_resource_info));
6172
  } else if (OB_FAIL(get_loads_by_server(server, unit_loads))) {
6173
    if (OB_ENTRY_NOT_EXIST != ret) {
6174
      LOG_WARN("get_loads_by_server failed", "server", server, KR(ret));
6175
    } else {
6176
      ret = OB_SUCCESS;
6177
    }
6178
  } else if (OB_ISNULL(unit_loads)) {
6179
    ret = OB_ERR_UNEXPECTED;
6180
    LOG_WARN("unit_loads is null", KR(ret), KP(unit_loads));
6181
  } else if (OB_FAIL(calc_sum_load(unit_loads, sum_load))) {
6182
    LOG_WARN("calc_sum_load failed", KR(ret), KP(unit_loads));
6183
  }
6184

6185
  if (OB_SUCC(ret)) {
6186
    // Unit resource information is persisted on the observer side,
6187
    // The unit_load seen on the rs side is only the resource view managed by rs itself,
6188
    // It is actually inaccurate to judge whether the resources are sufficient based on the resource view of rs itself,
6189
    // Need to consider the persistent resources of the unit resource information on the observer side.
6190
    // The persistent information of the unit on the observer side is regularly reported to rs by the observer through the heartbeat.
6191
    // When performing allocation, rs reports the maximum value of resource information from its own resource view
6192
    // and observer side as a reference for unit resource allocation
6193
    server_resource.addr_ = server;
6194
    server_resource.assigned_[RES_CPU] = sum_load.min_cpu() > report_resource.report_cpu_assigned_
6195
                                         ? sum_load.min_cpu() : report_resource.report_cpu_assigned_;
6196
    server_resource.max_assigned_[RES_CPU] = sum_load.max_cpu() > report_resource.report_cpu_max_assigned_
6197
                                         ? sum_load.max_cpu() : report_resource.report_cpu_max_assigned_;
6198
    server_resource.capacity_[RES_CPU] = report_resource.cpu_;
6199
    server_resource.assigned_[RES_MEM] = sum_load.memory_size() > report_resource.report_mem_assigned_
6200
                                         ? static_cast<double>(sum_load.memory_size())
6201
                                         : static_cast<double>(report_resource.report_mem_assigned_);
6202
    server_resource.max_assigned_[RES_MEM] = server_resource.assigned_[RES_MEM];
6203
    server_resource.capacity_[RES_MEM] = static_cast<double>(report_resource.mem_total_);
6204
    server_resource.assigned_[RES_LOG_DISK] = static_cast<double>(sum_load.log_disk_size());
6205
    server_resource.max_assigned_[RES_LOG_DISK] = static_cast<double>(sum_load.log_disk_size());
6206
    server_resource.capacity_[RES_LOG_DISK] = static_cast<double>(report_resource.log_disk_total_);
6207
  }
6208

6209
  LOG_INFO("compute server resource", KR(ret),
6210
            "server", server,
6211
            K(server_resource),
6212
            "report_resource_info", report_resource,
6213
            "valid_unit_sum", sum_load,
6214
            "valid_unit_count", unit_loads != NULL ? unit_loads->count(): 0);
6215
  return ret;
6216
}
6217

6218
// check resource enough for unit
6219
//
6220
// @param [in] u                              demands resource that may have some invalid items, need not check valid
6221
// @param [out] not_enough_resource           returned resource type that is not enough
6222
// @param [out] not_enough_resource_config    returned resource config type that is not enough
6223
bool ObUnitManager::check_resource_enough_for_unit_(
6224
    const ObUnitPlacementStrategy::ObServerResource &r,
6225
    const ObUnitResource &u,
6226
    const double hard_limit,
6227
    ObResourceType &not_enough_resource,
6228
    AlterResourceErr &not_enough_resource_config) const
6229
{
6230
  bool is_enough = false; // default is false
6231

6232
  if (u.is_max_cpu_valid() &&
6233
      r.capacity_[RES_CPU] * hard_limit < r.max_assigned_[RES_CPU] + u.max_cpu()) {
6234
    not_enough_resource = RES_CPU;
6235
    not_enough_resource_config = MAX_CPU;
6236
  } else if (u.is_min_cpu_valid() &&
6237
             r.capacity_[RES_CPU] < r.assigned_[RES_CPU] + u.min_cpu()) {
6238
    not_enough_resource = RES_CPU;
6239
    not_enough_resource_config = MIN_CPU;
6240
  } else if (u.is_memory_size_valid() &&
6241
             r.capacity_[RES_MEM] < r.assigned_[RES_MEM] + u.memory_size()) {
6242
    not_enough_resource = RES_MEM;
6243
    not_enough_resource_config = MEMORY;
6244
  } else if (u.is_log_disk_size_valid() &&
6245
             r.capacity_[RES_LOG_DISK] < r.assigned_[RES_LOG_DISK] + u.log_disk_size()) {
6246
    not_enough_resource = RES_LOG_DISK;
6247
    not_enough_resource_config = LOG_DISK;
6248
  } else {
6249
    is_enough = true;
6250
    not_enough_resource = RES_MAX;
6251
    not_enough_resource_config = ALT_ERR;
6252
  }
6253

6254
  if (! is_enough) {
6255
    _LOG_INFO("server %s resource '%s' is not enough for unit. hard_limit=%.6g, server_resource=%s, "
6256
        "demands=%s",
6257
        resource_type_to_str(not_enough_resource),
6258
        alter_resource_err_to_str(not_enough_resource_config),
6259
        hard_limit,
6260
        to_cstring(r),
6261
        to_cstring(u));
6262
  }
6263
  return is_enough;
6264
}
6265

6266

6267
// demand_resource may have some invalid items, need not check valid for demand_resource
6268
int ObUnitManager::have_enough_resource(const obrpc::ObGetServerResourceInfoResult &report_server_resource_info,
6269
                                        const ObUnitResource &demand_resource,
6270
                                        const double hard_limit,
6271
                                        bool &is_enough,
6272
                                        AlterResourceErr &err_index) const
6273
{
6274
  int ret = OB_SUCCESS;
6275
  ObResourceType not_enough_resource = RES_MAX;
6276
  ObUnitPlacementStrategy::ObServerResource server_resource;
6277
  err_index = ALT_ERR;
6278

6279
  if (!check_inner_stat()) {
6280
    ret = OB_INNER_STAT_ERROR;
6281
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
6282
  } else if (!report_server_resource_info.is_valid() || hard_limit <= 0) {
6283
    ret = OB_ERR_UNEXPECTED;
6284
    LOG_WARN("invalid argument", K(report_server_resource_info), K(hard_limit), K(ret));
6285
  } else if (OB_FAIL(compute_server_resource_(report_server_resource_info, server_resource))) {
6286
    LOG_WARN("compute server resource fail", KR(ret), K(report_server_resource_info));
6287
  } else {
6288
    is_enough = check_resource_enough_for_unit_(server_resource, demand_resource, hard_limit,
6289
        not_enough_resource, err_index);
6290
  }
6291
  return ret;
6292
}
6293
int ObUnitManager::check_enough_resource_for_delete_server(
6294
    const ObAddr &server,
6295
    const ObZone &zone)
6296
{
6297
  int ret = OB_SUCCESS;
6298
  // get_servers_of_zone
6299
  ObArray<obrpc::ObGetServerResourceInfoResult> report_servers_resource_info;
6300
  ObArray<ObServerInfoInTable> servers_info_of_zone;
6301
  bool empty = false;
6302
  const bool ONLY_ACTIVE_SERVERS = true;
6303
  if (OB_ISNULL(GCTX.sql_proxy_)) {
6304
    ret = OB_ERR_UNEXPECTED;
6305
    LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_));
6306
  } else if (OB_UNLIKELY(!server.is_valid() || zone.is_empty())) {
6307
    ret = OB_INVALID_ARGUMENT;
6308
    LOG_WARN("invalid server or zone", KR(ret), K(server), K(zone));
6309
  } else if (OB_FAIL(ut_operator_.check_server_empty(server, empty))) {
6310
    // the validity of the server is checked here
6311
    LOG_WARN("fail to check whether the server is empty", KR(ret));
6312
  } else if (empty) {
6313
    //nothing todo
6314
  } else if (OB_FAIL(ObServerTableOperator::get_servers_info_of_zone(
6315
      *GCTX.sql_proxy_,
6316
      zone,
6317
      ONLY_ACTIVE_SERVERS,
6318
      servers_info_of_zone))) {
6319
    LOG_WARN("fail to get servers info of zone", KR(ret), K(zone));
6320
  } else if (OB_FAIL(get_servers_resource_info_via_rpc(servers_info_of_zone, report_servers_resource_info))) {
6321
    LOG_WARN("fail to get servers_resouce_info via rpc", KR(ret), K(servers_info_of_zone), K(report_servers_resource_info));
6322
  } else if (OB_FAIL(check_enough_resource_for_delete_server_(
6323
      server,
6324
      zone,
6325
      servers_info_of_zone,
6326
      report_servers_resource_info))) {
6327
    LOG_WARN("fail to check enough resource for delete server", KR(ret), K(server), K(zone),
6328
        K(servers_info_of_zone), K(report_servers_resource_info));
6329
  } else {}
6330
  return ret;
6331
}
6332
int ObUnitManager::get_servers_resource_info_via_rpc(
6333
    const ObIArray<share::ObServerInfoInTable> &servers_info,
6334
    ObIArray<obrpc::ObGetServerResourceInfoResult> &report_servers_resource_info) const
6335
{
6336
  int ret = OB_SUCCESS;
6337
  int tmp_ret = OB_SUCCESS;
6338
  ObTimeoutCtx ctx;
6339
  obrpc::ObGetServerResourceInfoArg arg;
6340
  ObArray<obrpc::ObGetServerResourceInfoResult> tmp_report_servers_resource_info;
6341
  report_servers_resource_info.reset();
6342
  if (OB_UNLIKELY(servers_info.count() < 0)) {
6343
    ret = OB_INVALID_ARGUMENT;
6344
    LOG_WARN("servers_info.count() should be >= 0", KR(ret), K(servers_info.count()));
6345
  } else if (0 == servers_info.count()) {
6346
    // do nothing
6347
  } else if (!ObHeartbeatService::is_service_enabled()) { // old logic
6348
    ObServerResourceInfo resource_info;
6349
    obrpc::ObGetServerResourceInfoResult result;
6350
    for (int64_t i = 0; OB_SUCC(ret) && i < servers_info.count(); i++) {
6351
      const ObAddr &server = servers_info.at(i).get_server();
6352
      resource_info.reset();
6353
      result.reset();
6354
      if (OB_FAIL(server_mgr_.get_server_resource_info(server, resource_info))) {
6355
        LOG_WARN("fail to get server resource info", KR(ret), K(server));
6356
      } else if (OB_FAIL(result.init(server, resource_info))) {
6357
        LOG_WARN("fail to init", KR(ret), K(server));
6358
      } else if (OB_FAIL(report_servers_resource_info.push_back(result))) {
6359
        LOG_WARN("fail to push an element into report_servers_resource_info", KR(ret), K(result));
6360
      }
6361
    }
6362
  } else { // new logic
6363
    if (OB_ISNULL(srv_rpc_proxy_)) {
6364
      ret = OB_ERR_UNEXPECTED;
6365
      LOG_WARN("srv_rpc_proxy_ is null", KR(ret), KP(srv_rpc_proxy_));
6366
    } else if (OB_FAIL(ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
6367
      LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
6368
    } else {
6369
      ObGetServerResourceInfoProxy proxy(*srv_rpc_proxy_, &obrpc::ObSrvRpcProxy::get_server_resource_info);
6370
      for (int64_t i = 0; OB_SUCC(ret) && i < servers_info.count(); i++) {
6371
        const ObServerInfoInTable & server_info = servers_info.at(i);
6372
        arg.reset();
6373
        if (OB_UNLIKELY(!server_info.is_valid())) {
6374
          ret = OB_INVALID_ARGUMENT;
6375
          LOG_WARN("invalid server_info", KR(ret), K(server_info));
6376
        } else {
6377
          const ObAddr &server = server_info.get_server();
6378
          const int64_t time_out = ctx.get_timeout();
6379
          if (OB_FAIL(arg.init(GCTX.self_addr()))) {
6380
            LOG_WARN("fail to init arg", KR(ret), K(GCTX.self_addr()));
6381
          } else if (OB_FAIL(proxy.call(
6382
              server,
6383
              time_out,
6384
              GCONF.cluster_id,
6385
              OB_SYS_TENANT_ID,
6386
              arg))) {
6387
            LOG_WARN("fail to send get_server_resource_info rpc",  KR(ret), KR(tmp_ret), K(server),
6388
                K(time_out), K(arg));
6389
          }
6390
        }
6391
      }
6392
      if (OB_TMP_FAIL(proxy.wait())) {
6393
        LOG_WARN("fail to wait all batch result", KR(ret), KR(tmp_ret));
6394
        ret = OB_SUCC(ret) ? tmp_ret : ret;
6395
      } else if (OB_SUCC(ret)) {
6396
        ARRAY_FOREACH_X(proxy.get_results(), idx, cnt, OB_SUCC(ret)) {
6397
          const obrpc::ObGetServerResourceInfoResult *rpc_result = proxy.get_results().at(idx);
6398
          if (OB_ISNULL(rpc_result)) {
6399
            ret = OB_ERR_UNEXPECTED;
6400
            LOG_WARN("rpc_result is null", KR(ret), KP(rpc_result));
6401
          } else if (OB_UNLIKELY(!rpc_result->is_valid())) {
6402
            ret = OB_INVALID_ARGUMENT;
6403
            LOG_WARN("rpc_result is invalid", KR(ret), KPC(rpc_result));
6404
          } else if (OB_FAIL(tmp_report_servers_resource_info.push_back(*rpc_result))) {
6405
            LOG_WARN("fail to push an element into tmp_report_servers_resource_info", KR(ret), KPC(rpc_result));
6406
          }
6407
        }
6408
      }
6409
    }
6410
    // get ordered report_servers_resource_info: since when processing resource_info,
6411
    // we assume servers_info.at(i).get_server() = report_servers_resource_info.at(i).get_server()
6412
    if (FAILEDx(order_report_servers_resource_info_(
6413
        servers_info,
6414
        tmp_report_servers_resource_info,
6415
        report_servers_resource_info ))) {
6416
      LOG_WARN("fail to order report_servers_resource_info", KR(ret),
6417
          K(servers_info.count()), K(tmp_report_servers_resource_info.count()),
6418
          K(servers_info), K(tmp_report_servers_resource_info));
6419
    }
6420
  }
6421
  return ret;
6422
}
6423

6424
int ObUnitManager::order_report_servers_resource_info_(
6425
    const ObIArray<share::ObServerInfoInTable> &servers_info,
6426
    const ObIArray<obrpc::ObGetServerResourceInfoResult> &report_servers_resource_info,
6427
    ObIArray<obrpc::ObGetServerResourceInfoResult> &ordered_report_servers_resource_info)
6428
{
6429
  // target: servers_info.at(i).get_server() = ordered_report_servers_resource_info.at(i).get_server()
6430
  int ret = OB_SUCCESS;
6431
  ordered_report_servers_resource_info.reset();
6432
  if (servers_info.count() != report_servers_resource_info.count()) {
6433
    ret = OB_INVALID_ARGUMENT;
6434
    LOG_WARN("the size of servers_info should be equal to the size of report_servers_resource_info",
6435
        KR(ret), K(servers_info.count()), K(report_servers_resource_info.count()),
6436
        K(servers_info), K(report_servers_resource_info));
6437
  } else {
6438
    for (int64_t i = 0; OB_SUCC(ret) && i < servers_info.count(); i++) {
6439
      bool find_server = false;
6440
      for (int64_t j = 0; OB_SUCC(ret) && !find_server && j < report_servers_resource_info.count(); j++) {
6441
        const obrpc::ObGetServerResourceInfoResult &server_resource_info = report_servers_resource_info.at(j);
6442
        if (servers_info.at(i).get_server() == server_resource_info.get_server()) {
6443
          find_server = true;
6444
          if (OB_FAIL(ordered_report_servers_resource_info.push_back(server_resource_info))) {
6445
            LOG_WARN("fail to push an element into ordered_report_servers_resource_info",
6446
                KR(ret), K(server_resource_info));
6447
          }
6448
        }
6449
      }
6450
      if(OB_SUCC(ret) && !find_server) {
6451
        ret = OB_ERR_UNEXPECTED;
6452
        LOG_WARN("server not exists in report_servers_resource_info",
6453
            K(servers_info.at(i)), K(report_servers_resource_info));
6454
      }
6455
    }
6456
  }
6457
  return ret;
6458
}
6459
int ObUnitManager::get_server_resource_info_via_rpc(
6460
    const share::ObServerInfoInTable &server_info,
6461
    obrpc::ObGetServerResourceInfoResult &report_server_resource_info) const
6462
{
6463
  int ret = OB_SUCCESS;
6464
  ObArray<share::ObServerInfoInTable> servers_info;
6465
  ObArray<obrpc::ObGetServerResourceInfoResult> report_resource_info_array;
6466
  report_server_resource_info.reset();
6467
  if (OB_UNLIKELY(!server_info.is_valid())) {
6468
    ret = OB_INVALID_ARGUMENT;
6469
    LOG_WARN("server_info is invalid", KR(ret), K(server_info));
6470
  } else if (OB_FAIL(servers_info.push_back(server_info))) {
6471
    LOG_WARN("fail to push an element into servers_info", KR(ret), K(server_info));
6472
  } else if (OB_FAIL(get_servers_resource_info_via_rpc(servers_info, report_resource_info_array))) {
6473
    LOG_WARN("fail to execute get_servers_resource_info_via_rpc", KR(ret), K(servers_info));
6474
  } else if (OB_UNLIKELY(1 != report_resource_info_array.count())) {
6475
    ret = OB_ERR_UNEXPECTED;
6476
    LOG_WARN("report_resource_info_array.count() should be one", KR(ret), K(report_resource_info_array.count()));
6477
  } else if (OB_FAIL(report_server_resource_info.assign(report_resource_info_array.at(0)))) {
6478
    LOG_WARN("fail to assign report_server_resource_info", KR(ret), K(report_resource_info_array.at(0)));
6479
  }
6480
  return ret;
6481
}
6482
int ObUnitManager::check_enough_resource_for_delete_server_(
6483
    const ObAddr &server,
6484
    const ObZone &zone,
6485
    const ObIArray<share::ObServerInfoInTable> &servers_info,
6486
    const ObIArray<obrpc::ObGetServerResourceInfoResult> &report_servers_resource_info)
6487
{
6488
  int ret = OB_SUCCESS;
6489
  ObArray<ObUnitManager::ObUnitLoad> *unit_loads = NULL;
6490
  ObArray<ObUnitPlacementStrategy::ObServerResource> initial_servers_resources;
6491
  SpinRLockGuard guard(lock_);
6492
  bool empty = false;
6493
  if (OB_UNLIKELY(zone.is_empty())) {
6494
    ret = OB_INVALID_ARGUMENT;
6495
    LOG_WARN("zone is invalid", KR(ret), K(zone), K(server));
6496
  } else if (OB_FAIL(ut_operator_.check_server_empty(server, empty))) {
6497
    LOG_WARN("fail to check server empty", K(ret));
6498
  } else if (empty) {
6499
    //nothing todo
6500
  } else {
6501
    if (OB_FAIL(get_loads_by_server(server, unit_loads))) {
6502
      LOG_WARN("fail to get loads by server", K(ret));
6503
    } else if (OB_FAIL(build_server_resources_(report_servers_resource_info, initial_servers_resources))) {
6504
      LOG_WARN("fail to build server resources", KR(ret), K(report_servers_resource_info));
6505
    } else {
6506
      for (int64_t i = 0; i < unit_loads->count() && OB_SUCC(ret); ++i) {
6507
        std::string resource_not_enough_reason;
6508
        if (OB_FAIL(check_server_have_enough_resource_for_delete_server_(
6509
            unit_loads->at(i),
6510
            zone,
6511
            servers_info,
6512
            initial_servers_resources,
6513
            resource_not_enough_reason))) {
6514
          LOG_WARN("fail to check server have enough resource for delete server",
6515
              K(ret),
6516
              K(zone),
6517
              K(initial_servers_resources),
6518
              K(i),
6519
              K(unit_loads),
6520
              K(resource_not_enough_reason.c_str()));
6521

6522
          // handle return error info
6523
          if (OB_ZONE_SERVER_NOT_ENOUGH == ret) {
6524
            std::string err_msg;
6525
            const ObUnit *unit = unit_loads->at(i).unit_;
6526
            uint64_t unit_id = (NULL == unit ? 0 : unit->unit_id_);
6527

6528
            err_msg = err_msg + "can not migrate out unit '" + to_cstring(unit_id) +
6529
                "', no other available servers on zone '" +  to_cstring(zone) +
6530
                "', delete server not allowed";
6531
            LOG_USER_ERROR(OB_DELETE_SERVER_NOT_ALLOWED, err_msg.c_str());
6532
          } else if (OB_ZONE_RESOURCE_NOT_ENOUGH == ret) {
6533
            std::string err_msg;
6534
            const ObUnit *unit = unit_loads->at(i).unit_;
6535
            uint64_t unit_id = (NULL == unit ? 0 : unit->unit_id_);
6536

6537
            err_msg = err_msg + "can not migrate out all units, zone '" + to_cstring(zone) +
6538
                "' resource not enough, delete server not allowed. "
6539
                "You can check resource info by views: DBA_OB_UNITS, GV$OB_UNITS, GV$OB_SERVERS.\n"
6540
                + resource_not_enough_reason.c_str();
6541
            LOG_USER_ERROR(OB_DELETE_SERVER_NOT_ALLOWED, err_msg.c_str());
6542
          }
6543
        }
6544
      } //end for unit_loads
6545
    }
6546
  }
6547
  return ret;
6548
}
6549
int ObUnitManager::check_server_have_enough_resource_for_delete_server_(
6550
    const ObUnitLoad &unit_load,
6551
    const common::ObZone &zone,
6552
    const ObIArray<share::ObServerInfoInTable> &servers_info,
6553
    ObIArray<ObUnitPlacementStrategy::ObServerResource> &initial_servers_resources,
6554
    std::string &resource_not_enough_reason)
6555
{
6556
  int ret = OB_SUCCESS;
6557
  double hard_limit = 1.0;
6558
  ObArray<ObUnitPlacementStrategy::ObServerResource> servers_resources;
6559
  ObArray<ObAddr> excluded_servers;
6560
  ObAddr choosed_server;
6561
  const ObUnitConfig *config = unit_load.unit_config_;
6562
  const char *module = "DELETE_SERVER";
6563
  const bool new_allocate_pool = false;
6564

6565
  if (!check_inner_stat()) {
6566
    ret = OB_INNER_STAT_ERROR;
6567
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
6568
  } else if (OB_ISNULL(config) || OB_UNLIKELY(! config->is_valid())) {
6569
    ret = OB_INVALID_ARGUMENT;
6570
    LOG_WARN("unit config ptr is null", KR(ret), KPC(config));
6571
  } else if (OB_UNLIKELY(zone.is_empty())) {
6572
    ret = OB_INVALID_ARGUMENT;
6573
    LOG_WARN("zone is empty, unexpected", KR(ret), K(zone));
6574
  } else if (servers_info.count() != initial_servers_resources.count()) {
6575
    ret = OB_INVALID_ARGUMENT;
6576
    LOG_WARN("servers_info does not match initial_servers_resources", KR(ret), K(servers_info),
6577
        K(initial_servers_resources));
6578
  } else if (OB_ISNULL(unit_load.unit_)) {
6579
    ret = OB_ERR_UNEXPECTED;
6580
    LOG_WARN("unit ptr is null", K(ret));
6581
  } else if (unit_load.unit_->migrate_from_server_.is_valid()) {
6582
    // In a state of migration, there must be a place to migrate
6583
  } else if (OB_FAIL(get_excluded_servers(unit_load.unit_->resource_pool_id_, unit_load.unit_->zone_, module,
6584
          new_allocate_pool, excluded_servers))) {
6585
    LOG_WARN("fail to get excluded server", K(module), KR(ret), KPC(unit_load.unit_), K(new_allocate_pool));
6586
  } else if (OB_FAIL(do_choose_server_for_unit_(config->unit_resource(), zone,
6587
      excluded_servers, servers_info, initial_servers_resources,
6588
      module, choosed_server, resource_not_enough_reason))) {
6589
    // choose right server for target unit
6590
    LOG_WARN("choose server for unit fail", K(module), KR(ret), K(zone), KPC(config),
6591
        K(excluded_servers), K(servers_info), K(initial_servers_resources),
6592
        "resource_not_enough_reason", resource_not_enough_reason.c_str());
6593
  } else {
6594
    // sum target unit resource config on choosed server resource
6595
    int64_t choosed_index = -1;
6596
    for (int64_t i = 0; choosed_index < 0 && i < initial_servers_resources.count(); i++) {
6597
      if (initial_servers_resources.at(i).addr_ == choosed_server) {
6598
        choosed_index = i;
6599
      }
6600
    }
6601

6602
    if (OB_UNLIKELY(choosed_index < 0 || choosed_index >= initial_servers_resources.count())) {
6603
      ret = OB_ERR_UNEXPECTED;
6604
      LOG_WARN("can not find choosed_server in initial_servers_resources, or invalid choosed_index", KR(ret),
6605
          K(choosed_index), K(choosed_server), K(initial_servers_resources));
6606
    } else if (OB_FAIL(sum_servers_resources(initial_servers_resources.at(choosed_index), *config))) {
6607
      LOG_WARN("fail to sum servers resources", KR(ret), K(initial_servers_resources),
6608
          KPC(config));
6609
    }
6610
  }
6611
  return ret;
6612
}
6613

6614
int ObUnitManager::build_server_resources_(
6615
    const ObIArray<obrpc::ObGetServerResourceInfoResult> &report_servers_resource_info,
6616
    ObIArray<ObUnitPlacementStrategy::ObServerResource> &servers_resources) const
6617
{
6618
  int ret = OB_SUCCESS;
6619
  for (int64_t i = 0; OB_SUCC(ret) && i < report_servers_resource_info.count(); ++i) {
6620
    const obrpc::ObGetServerResourceInfoResult &report_resource_info = report_servers_resource_info.at(i);
6621
    ObUnitPlacementStrategy::ObServerResource server_resource;
6622

6623
    if (OB_FAIL(compute_server_resource_(report_resource_info, server_resource))) {
6624
      LOG_WARN("compute server resource fail", KR(ret), K(report_resource_info));
6625
    } else if (OB_FAIL(servers_resources.push_back(server_resource))) {
6626
      LOG_WARN("fail to push back", KR(ret), K(server_resource));
6627
    }
6628
  }
6629
  return ret;
6630
}
6631

6632
int ObUnitManager::sum_servers_resources(ObUnitPlacementStrategy::ObServerResource &server_resource,
6633
                                         const share::ObUnitConfig &unit_config)
6634
{
6635
  int ret = OB_SUCCESS;
6636
  server_resource.assigned_[RES_CPU] = server_resource.assigned_[RES_CPU] +
6637
                                       unit_config.min_cpu();
6638
  server_resource.max_assigned_[RES_CPU] = server_resource.max_assigned_[RES_CPU] +
6639
                                           unit_config.max_cpu();
6640
  server_resource.assigned_[RES_MEM] = server_resource.assigned_[RES_MEM] +
6641
                                       static_cast<double>(unit_config.memory_size());
6642
  server_resource.max_assigned_[RES_MEM] = server_resource.max_assigned_[RES_MEM] +
6643
                                           static_cast<double>(unit_config.memory_size());
6644
  server_resource.assigned_[RES_LOG_DISK] = server_resource.assigned_[RES_LOG_DISK] +
6645
                                        static_cast<double>(unit_config.log_disk_size());
6646
  server_resource.max_assigned_[RES_LOG_DISK] = server_resource.max_assigned_[RES_LOG_DISK] +
6647
                                            static_cast<double>(unit_config.log_disk_size());
6648
  return ret;
6649
}
6650

6651
int ObUnitManager::add_unit(ObISQLClient &client, const ObUnit &unit)
6652
{
6653
  int ret = OB_SUCCESS;
6654
  if (!check_inner_stat()) {
6655
    ret = OB_INNER_STAT_ERROR;
6656
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
6657
  } else if (!unit.is_valid()) {
6658
    ret = OB_INVALID_ARGUMENT;
6659
    LOG_WARN("invalid argument", K(unit), K(ret));
6660
  } else {
6661
    ObUnit *new_unit = NULL;
6662
    if (OB_SUCCESS == (ret = get_unit_by_id(unit.unit_id_, new_unit))) {
6663
      ret = OB_ENTRY_EXIST;
6664
      LOG_WARN("unit already exist, can't add", K(unit), K(ret));
6665
    } else if (OB_ENTRY_NOT_EXIST != ret) {
6666
      LOG_WARN("get_unit_by_id failed", "unit_id", unit.unit_id_, K(ret));
6667
    } else {
6668
      ret = OB_SUCCESS;
6669
    }
6670

6671
    if (OB_FAIL(ret)) {
6672
    } else if (NULL == (new_unit = allocator_.alloc())) {
6673
      ret = OB_ALLOCATE_MEMORY_FAILED;
6674
      LOG_ERROR("alloc unit failed", K(ret));
6675
    } else {
6676
      if (OB_FAIL(ut_operator_.update_unit(client, unit, false/*need_check_conflict_with_clone*/))) {
6677
        LOG_WARN("update_unit failed", K(unit), K(ret));
6678
      } else {
6679
        *new_unit = unit;
6680
        if (OB_FAIL(insert_unit(new_unit))) {
6681
          LOG_WARN("insert_unit failed", "new unit", *new_unit, K(ret));
6682
        } else {
6683
          ROOTSERVICE_EVENT_ADD("unit", "create_unit",
6684
              "unit_id", unit.unit_id_,
6685
              "server", unit.server_);
6686
        }
6687
      }
6688
      if (OB_FAIL(ret)) {
6689
        //avoid memory leak
6690
        allocator_.free(new_unit);
6691
        new_unit = NULL;
6692
      }
6693
    }
6694
  }
6695
  return ret;
6696
}
6697

6698
int ObUnitManager::alter_pool_unit_config(share::ObResourcePool  *pool,
6699
                                          const ObUnitConfigName &config_name)
6700
{
6701
  int ret = OB_SUCCESS;
6702
  ObUnitConfig *config = NULL;
6703
  ObUnitConfig *alter_config = NULL;
6704
  common::ObSEArray<share::ObResourcePool *, 1> pools;
6705
  if (!check_inner_stat()) {
6706
    ret = OB_INNER_STAT_ERROR;
6707
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
6708
  } else if (NULL == pool) {
6709
    ret = OB_INVALID_ARGUMENT;
6710
    LOG_WARN("pool is null", K(ret));
6711
  } else if (config_name.is_empty()) {
6712
    ret = OB_INVALID_ARGUMENT;
6713
    LOG_WARN("config_name is empty", K(config_name), K(ret));
6714
  } else if (OB_FAIL(get_unit_config_by_id(pool->unit_config_id_, config))) {
6715
    if (OB_ENTRY_NOT_EXIST != ret) {
6716
      LOG_WARN("get_unit_config_by_id failed", "unit_config_id", pool->unit_config_id_, K(ret));
6717
    } else {
6718
      ret = OB_RESOURCE_UNIT_NOT_EXIST;
6719
      LOG_WARN("unit config not exist", "config_id", pool->unit_config_id_, K(ret));
6720
    }
6721
  } else if (NULL == config) {
6722
    ret = OB_ERR_UNEXPECTED;
6723
    LOG_WARN("config is null", KP(config), K(ret));
6724
  } else if (OB_FAIL(get_unit_config_by_name(config_name, alter_config))) {
6725
    if (OB_ENTRY_NOT_EXIST != ret) {
6726
      LOG_WARN("get_unit_config_by_name failed", K(config_name), K(ret));
6727
    } else {
6728
      ret = OB_RESOURCE_UNIT_NOT_EXIST;
6729
      LOG_USER_ERROR(OB_RESOURCE_UNIT_NOT_EXIST, to_cstring(config_name));
6730
      LOG_WARN("unit config not exist", K(config_name), K(ret));
6731
    }
6732
  } else if (NULL == alter_config) {
6733
    ret = OB_ERR_UNEXPECTED;
6734
    LOG_WARN("alter_config is null", KP(alter_config), K(ret));
6735
  } else if (config_name == config->name()) {
6736
    // do nothing
6737
  } else if (REPLICA_TYPE_FULL == pool->replica_type_
6738
      && alter_config->unit_resource().memory_size() < GCONF.__min_full_resource_pool_memory) {
6739
    ret = OB_NOT_SUPPORTED;
6740
    LOG_WARN("full resource pool memory size illegal", K(ret));
6741
    LOG_USER_ERROR(OB_NOT_SUPPORTED, "unit MEMORY_SIZE less than __min_full_resource_pool_memory");
6742
  } else if (OB_FAIL(pools.push_back(pool))) {
6743
    LOG_WARN("push back pool into array fail", KR(ret), K(pool), K(pools));
6744
  } else if (OB_FAIL(check_expand_resource_(
6745
      "ALTER_RESOURCE_POOL_UNIT_CONFIG",
6746
      pools,
6747
      config->unit_resource(),
6748
      alter_config->unit_resource()))) {
6749
    LOG_WARN("check_expand_config failed", KR(ret), KPC(pool), KPC(config), KPC(alter_config));
6750
  } else if (OB_FAIL(check_shrink_resource_(pools, config->unit_resource(),
6751
      alter_config->unit_resource()))) {
6752
    LOG_WARN("check_shrink_resource_ failed", KPC(pool), KPC(config), KPC(alter_config), KR(ret));
6753
  } else if (OB_FAIL(change_pool_config(pool, config, alter_config))) {
6754
    LOG_WARN("change_pool_config failed", "pool", *pool, "config", *config,
6755
        "alter_config", *alter_config, K(ret));
6756
  }
6757
  return ret;
6758
}
6759

6760
// shrink pool unit num should only work on not-granted pool.
6761
// routine that calls this function should ensure that.
6762
int ObUnitManager::shrink_pool_unit_num(
6763
    share::ObResourcePool *pool,
6764
    const int64_t alter_unit_num,
6765
    const common::ObIArray<uint64_t> &delete_unit_id_array)
6766
{
6767
  int ret = OB_SUCCESS;
6768
  if (OB_UNLIKELY(NULL == pool || alter_unit_num <= 0)) {
6769
    ret = OB_INVALID_ARGUMENT;
6770
    LOG_WARN("invalid argument", K(ret), KP(pool), K(alter_unit_num));
6771
  } else if (pool->is_granted_to_tenant()) {
6772
    ret = OB_ERR_UNEXPECTED;
6773
    LOG_WARN("unexpected pool status", KR(ret), KPC(pool));
6774
  } else {
6775
    ObArray<uint64_t> sort_unit_id_array;
6776
    for (int64_t i = 0; OB_SUCC(ret) && i < delete_unit_id_array.count(); ++i) {
6777
      if (OB_FAIL(sort_unit_id_array.push_back(delete_unit_id_array.at(i)))) {
6778
        LOG_WARN("fail to push back", K(ret));
6779
      }
6780
    }
6781
    std::sort(sort_unit_id_array.begin(), sort_unit_id_array.end());
6782
    for (int64_t i = 0; OB_SUCC(ret) && i < sort_unit_id_array.count() - 1; ++i) {
6783
      if (sort_unit_id_array.at(i) == sort_unit_id_array.at(i + 1)) {
6784
        ret = OB_NOT_SUPPORTED;
6785
        LOG_WARN("delete same unit more than once not supported", K(ret));
6786
        LOG_USER_ERROR(OB_NOT_SUPPORTED, "delete same unit more than once");
6787
      }
6788
    }
6789
    if (FAILEDx(shrink_not_granted_pool(pool, alter_unit_num, delete_unit_id_array))) {
6790
      LOG_WARN("fail to shrink not granted pool", KR(ret));
6791
    }
6792
  }
6793
  return ret;
6794
}
6795

6796
int ObUnitManager::build_sorted_zone_unit_ptr_array(
6797
    share::ObResourcePool *pool,
6798
    common::ObIArray<ZoneUnitPtr> &zone_unit_ptrs)
6799
{
6800
  int ret = OB_SUCCESS;
6801
  common::ObArray<share::ObUnit *> *units = NULL;
6802
  zone_unit_ptrs.reset();
6803
  if (OB_UNLIKELY(NULL == pool)) {
6804
    ret = OB_INVALID_ARGUMENT;
6805
    LOG_WARN("invalid argument", K(ret), KP(pool));
6806
  } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
6807
    LOG_WARN("fail to get units by pool", K(ret), "pool_id", pool->resource_pool_id_);
6808
  } else if (OB_UNLIKELY(NULL == units)) {
6809
    ret = OB_ERR_UNEXPECTED;
6810
    LOG_WARN("units ptr is null", K(ret), KP(units));
6811
  } else {
6812
    // traverse all unit in units
6813
    for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
6814
      ObUnit *this_unit = units->at(i);
6815
      if (OB_UNLIKELY(NULL == this_unit)) {
6816
        ret = OB_ERR_UNEXPECTED;
6817
        LOG_WARN("unit ptr is null", K(ret), KP(this_unit));
6818
      } else {
6819
        // aggregate units with the same zone
6820
        int64_t index = 0;
6821
        for (; OB_SUCC(ret) && index < zone_unit_ptrs.count(); ++index) {
6822
          if (this_unit->zone_ == zone_unit_ptrs.at(index).zone_) {
6823
            break;
6824
          }
6825
        }
6826
        if (OB_FAIL(ret)) {
6827
        } else if (index >= zone_unit_ptrs.count()) {
6828
          ZoneUnitPtr zone_unit_ptr;
6829
          zone_unit_ptr.zone_ = this_unit->zone_;
6830
          if (OB_FAIL(zone_unit_ptrs.push_back(zone_unit_ptr))) {
6831
            LOG_WARN("fail to push back", K(ret));
6832
          }
6833
        }
6834
        if (OB_FAIL(ret)) {
6835
        } else if (index >= zone_unit_ptrs.count()) {
6836
          ret = OB_ERR_UNEXPECTED;
6837
          LOG_WARN("unexpected index", K(ret), K(index),
6838
                   "zone_unit_count", zone_unit_ptrs.count());
6839
        } else if (this_unit->zone_ != zone_unit_ptrs.at(index).zone_) {
6840
          ret = OB_ERR_UNEXPECTED;
6841
          LOG_WARN("zone not match", K(ret), "left_zone", this_unit->zone_,
6842
                   "right_zone", zone_unit_ptrs.at(index).zone_);
6843
        } else if (OB_FAIL(zone_unit_ptrs.at(index).unit_ptrs_.push_back(this_unit))) {
6844
          LOG_WARN("fail to push back", K(ret), K(index));
6845
        } else {} // good, no more to do
6846
      }
6847
    }
6848
    for (int64_t i = 0; OB_SUCC(ret) && i < zone_unit_ptrs.count(); ++i) {
6849
      // sort each zone unit ptr using group id
6850
      ZoneUnitPtr &zone_unit_ptr = zone_unit_ptrs.at(i);
6851
      if (OB_FAIL(zone_unit_ptr.sort_by_unit_id_desc())) {
6852
        LOG_WARN("fail to sort unit", K(ret));
6853
      } else {} // good, unit num match
6854
    }
6855
  }
6856
  return ret;
6857
}
6858

6859
int ObUnitManager::check_shrink_unit_num_zone_condition(
6860
    share::ObResourcePool *pool,
6861
    const int64_t alter_unit_num,
6862
    const common::ObIArray<uint64_t> &delete_unit_id_array)
6863
{
6864
  int ret = OB_SUCCESS;
6865
  if (OB_UNLIKELY(NULL == pool || alter_unit_num <= 0)) {
6866
    ret = OB_INVALID_ARGUMENT;
6867
    LOG_WARN("invalid argument", K(ret), KP(pool), K(alter_unit_num));
6868
  } else if (delete_unit_id_array.count() <= 0) {
6869
    // good, we choose deleting set all by ourselves
6870
  } else if (pool->unit_count_ <= alter_unit_num) {
6871
    ret = OB_ERR_UNEXPECTED;
6872
    LOG_WARN("should be a shrink pool operation", K(ret),
6873
             "cur_unit_num", pool->unit_count_,
6874
             "new_unit_num", alter_unit_num);
6875
  } else {
6876
    int64_t delta = pool->unit_count_ - alter_unit_num;
6877
    const common::ObIArray<common::ObZone> &zone_list = pool->zone_list_;
6878
    common::ObArray<ZoneUnitPtr> delete_zone_unit_ptrs;
6879
    for (int64_t i = 0; OB_SUCC(ret) && i < delete_unit_id_array.count(); ++i) {
6880
      ObUnit *this_unit = NULL;
6881
      if (OB_FAIL(get_unit_by_id(delete_unit_id_array.at(i), this_unit))) {
6882
        LOG_WARN("fail to get unit by id", K(ret));
6883
      } else if (OB_UNLIKELY(NULL == this_unit)) {
6884
        ret = OB_ERR_UNEXPECTED;
6885
        LOG_WARN("unit ptr is null", K(ret), KP(this_unit));
6886
      } else if (this_unit->resource_pool_id_ != pool->resource_pool_id_) {
6887
        ret = OB_NOT_SUPPORTED;
6888
        LOG_WARN("do not support shrink unit belonging to other pool");
6889
        LOG_USER_ERROR(OB_NOT_SUPPORTED, "shrink unit belonging to other pool");
6890
      } else if (!has_exist_in_array(zone_list, this_unit->zone_)) {
6891
        ret = OB_ERR_UNEXPECTED;
6892
        LOG_WARN("unit zone not match", K(ret), K(zone_list), "unit_zone", this_unit->zone_);
6893
      } else {
6894
        int64_t index = 0;
6895
        for (; OB_SUCC(ret) && index < delete_zone_unit_ptrs.count(); ++index) {
6896
          if (this_unit->zone_ == delete_zone_unit_ptrs.at(index).zone_) {
6897
            break;
6898
          }
6899
        }
6900
        if (OB_FAIL(ret)) {
6901
        } else if (index >= delete_zone_unit_ptrs.count()) {
6902
          ZoneUnitPtr delete_zone_unit_ptr;
6903
          delete_zone_unit_ptr.zone_ = this_unit->zone_;
6904
          if (OB_FAIL(delete_zone_unit_ptrs.push_back(delete_zone_unit_ptr))) {
6905
            LOG_WARN("fail to push back", K(ret));
6906
          }
6907
        }
6908
        if (OB_FAIL(ret)) {
6909
        } else if (index >= delete_zone_unit_ptrs.count()) {
6910
          ret = OB_ERR_UNEXPECTED;
6911
          LOG_WARN("unexpected index", K(ret), K(index),
6912
                   "delete_zone_unit_count", delete_zone_unit_ptrs.count());
6913
        } else if (this_unit->zone_ != delete_zone_unit_ptrs.at(index).zone_) {
6914
          ret = OB_ERR_UNEXPECTED;
6915
          LOG_WARN("zone not match", K(ret), "left_zone", this_unit->zone_,
6916
                   "right_zone", delete_zone_unit_ptrs.at(index).zone_);
6917
        } else if (OB_FAIL(delete_zone_unit_ptrs.at(index).unit_ptrs_.push_back(this_unit))) {
6918
          LOG_WARN("fail to push back", K(ret), K(index));
6919
        } else {} // good, no more to do
6920
      }
6921
    }
6922
    if (OB_SUCC(ret)) {
6923
      if (delete_zone_unit_ptrs.count() != zone_list.count()) {
6924
        ret = OB_NOT_SUPPORTED;
6925
        LOG_WARN("do not support shrink unit num to different value on different zone", K(ret));
6926
        LOG_USER_ERROR(OB_NOT_SUPPORTED, "shrink unit num to different value on different zone");
6927
      }
6928
    }
6929
    if (OB_SUCC(ret)) {
6930
      for (int64_t i = 0; OB_SUCC(ret) && i < delete_zone_unit_ptrs.count(); ++i) {
6931
        if (delta != delete_zone_unit_ptrs.at(i).unit_ptrs_.count()) {
6932
          ret = OB_NOT_SUPPORTED;
6933
          LOG_WARN("shrink mismatching unit num and unit id list not support", K(ret));
6934
          LOG_USER_ERROR(OB_NOT_SUPPORTED, "shrink mismatching unit num and unit id list");
6935
        } else {} // good, go on to check next
6936
      }
6937
    }
6938
  }
6939
  return ret;
6940
}
6941

6942
int ObUnitManager::fill_delete_unit_ptr_array(
6943
    share::ObResourcePool *pool,
6944
    const common::ObIArray<uint64_t> &delete_unit_id_array,
6945
    const int64_t alter_unit_num,
6946
    common::ObIArray<ObUnit *> &output_delete_unit_ptr_array)
6947
{
6948
  int ret = OB_SUCCESS;
6949
  output_delete_unit_ptr_array.reset();
6950
  if (delete_unit_id_array.count() > 0) {
6951
    // The alter resource pool shrinkage specifies the deleted unit, just fill it in directly
6952
    for (int64_t i = 0; OB_SUCC(ret) && i < delete_unit_id_array.count(); ++i) {
6953
      ObUnit *unit = NULL;
6954
      if (OB_FAIL(get_unit_by_id(delete_unit_id_array.at(i), unit))) {
6955
        LOG_WARN("fail to get unit by id", K(ret));
6956
      } else if (NULL == unit) {
6957
        ret = OB_ERR_UNEXPECTED;
6958
        LOG_WARN("unit ptr is null", K(ret), KP(unit));
6959
      } else if (OB_FAIL(output_delete_unit_ptr_array.push_back(unit))) {
6960
        LOG_WARN("fail to push back", K(ret));
6961
      } else {} // no more to do
6962
    }
6963
  } else {
6964
    common::ObArray<ZoneUnitPtr> zone_unit_ptrs;
6965
    if (OB_UNLIKELY(NULL == pool)) {
6966
      ret = OB_ERR_UNEXPECTED;
6967
      LOG_WARN("invalid argument", K(ret), KP(pool));
6968
    } else if (OB_FAIL(build_sorted_zone_unit_ptr_array(pool, zone_unit_ptrs))) {
6969
      LOG_WARN("fail to build sorted zone unit ptr array", K(ret));
6970
    } else if (zone_unit_ptrs.count() <= 0) {
6971
      ret = OB_ERR_UNEXPECTED;
6972
      LOG_WARN("zone unit ptrs has no element", K(ret), "zone_unit_cnt", zone_unit_ptrs.count());
6973
    } else {
6974
      for (int64_t i = 0; OB_SUCC(ret) && i < zone_unit_ptrs.count(); ++i) {
6975
        const ZoneUnitPtr &zone_unit_ptr = zone_unit_ptrs.at(i);
6976
        for (int64_t j = alter_unit_num; OB_SUCC(ret) && j < zone_unit_ptr.unit_ptrs_.count(); ++j) {
6977
          ObUnit *unit = zone_unit_ptr.unit_ptrs_.at(j);
6978
          if (OB_UNLIKELY(NULL == unit)) {
6979
            ret = OB_ERR_UNEXPECTED;
6980
            LOG_WARN("unit ptr is null", K(ret), KP(unit), K(i), K(j));
6981
          } else if (OB_FAIL(output_delete_unit_ptr_array.push_back(unit))) {
6982
            LOG_WARN("fail to push back", K(ret));
6983
          } else {} // no more to do
6984
        }
6985
      }
6986
    }
6987
  }
6988
  return ret;
6989
}
6990

6991

6992
// the resource pool don't grant any tenants shrink directly.
6993
// step:
6994
// 1. clear __all_unit, change the unit num in __all_resouce_pool
6995
// 2. clear the unit info in memory structure, change the unit_num in memroy structure resource pool
6996
int ObUnitManager::shrink_not_granted_pool(
6997
    share::ObResourcePool *pool,
6998
    const int64_t alter_unit_num,
6999
    const common::ObIArray<uint64_t> &delete_unit_id_array)
7000
{
7001
  int ret = OB_SUCCESS;
7002
  common::ObMySQLTransaction trans;
7003
  if (OB_UNLIKELY(NULL == pool || alter_unit_num <= 0)) {
7004
    ret = OB_INVALID_ARGUMENT;
7005
    LOG_WARN("invalid argument", K(ret), KP(pool), K(alter_unit_num));
7006
  } else if (is_valid_tenant_id(pool->tenant_id_)) {
7007
    ret = OB_ERR_UNEXPECTED;
7008
    LOG_WARN("pool already grant to some tenant", K(ret), "tenant_id", pool->tenant_id_);
7009
  } else if (pool->unit_count_ <= alter_unit_num) {
7010
    ret = OB_ERR_UNEXPECTED;
7011
    LOG_WARN("should be a shrink pool operation", K(ret),
7012
             "cur_unit_num", pool->unit_count_,
7013
             "new_unit_num", alter_unit_num);
7014
  } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
7015
    LOG_WARN("start transaction failed", K(ret));
7016
  } else {
7017
    common::ObArray<ObUnit *> output_delete_unit_ptr_array;
7018
    common::ObArray<uint64_t> output_delete_unit_id_array;
7019
    share::ObResourcePool new_pool;
7020
    if (OB_FAIL(new_pool.assign(*pool))) {
7021
      LOG_WARN("fail to assign new pool", K(ret));
7022
    } else {
7023
      new_pool.unit_count_ = alter_unit_num;
7024
    }
7025
    if (OB_FAIL(ret)) {
7026
    } else if (OB_FAIL(ut_operator_.update_resource_pool(trans, new_pool, false/*need_check_conflict_with_clone*/))) {
7027
      LOG_WARN("fail to update resource pool", K(ret));
7028
    } else if (OB_FAIL(check_shrink_unit_num_zone_condition(pool, alter_unit_num, delete_unit_id_array))) {
7029
      LOG_WARN("fail to check shrink unit num zone condition", K(ret));
7030
    } else if (OB_FAIL(fill_delete_unit_ptr_array(
7031
            pool, delete_unit_id_array, alter_unit_num, output_delete_unit_ptr_array))) {
7032
      LOG_WARN("fail to fill delete unit id array", K(ret));
7033
    } else if (output_delete_unit_ptr_array.count() <= 0) {
7034
      ret = OB_ERR_UNEXPECTED;
7035
      LOG_WARN("zone unit ptrs has no element", K(ret),
7036
               "zone_unit_cnt", output_delete_unit_ptr_array.count());
7037
    } else {
7038
      for (int64_t i = 0; OB_SUCC(ret) && i < output_delete_unit_ptr_array.count(); ++i) {
7039
        const ObUnit *unit = output_delete_unit_ptr_array.at(i);
7040
        if (OB_UNLIKELY(NULL == unit)) {
7041
          ret = OB_ERR_UNEXPECTED;
7042
          LOG_WARN("unit ptr is null", K(ret), KP(unit));
7043
        } else if (OB_FAIL(ut_operator_.remove_unit(trans, *unit))) {
7044
          LOG_WARN("fail to remove unit", K(ret), "unit", *unit);
7045
        } else if (OB_FAIL(output_delete_unit_id_array.push_back(unit->unit_id_))) {
7046
          LOG_WARN("fail to push back", K(ret));
7047
        } else {} // no more to do
7048
      }
7049
    }
7050
    // however, we need to end this transaction
7051
    const bool commit = (OB_SUCCESS == ret);
7052
    int tmp_ret = OB_SUCCESS;
7053
    if (OB_SUCCESS != (tmp_ret = trans.end(commit))) {
7054
      LOG_WARN("trans end failed", K(tmp_ret), K(commit));
7055
      ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
7056
    }
7057
    if (OB_FAIL(ret)) {
7058
    } else if (OB_FAIL(delete_inmemory_units(pool->resource_pool_id_, output_delete_unit_id_array))) {
7059
      LOG_WARN("fail to delete unit groups", K(ret));
7060
    } else {
7061
      pool->unit_count_ = alter_unit_num;
7062
    }
7063
  }
7064
  return ret;
7065
}
7066

7067
int ObUnitManager::check_shrink_granted_pool_allowed_by_migrate_unit(
7068
    share::ObResourcePool *pool,
7069
    const int64_t alter_unit_num,
7070
    bool &is_allowed)
7071
{
7072
  int ret = OB_SUCCESS;
7073
  UNUSED(alter_unit_num);
7074
  is_allowed = true;
7075
  common::ObArray<share::ObUnit *> *units = NULL;
7076
  if (OB_UNLIKELY(NULL == pool || alter_unit_num <= 0)) {
7077
    ret = OB_INVALID_ARGUMENT;
7078
    LOG_WARN("invalid argument", K(ret), KP(pool), K(alter_unit_num));
7079
  } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
7080
    LOG_WARN("fail to get units by pool", K(ret));
7081
  } else if (OB_UNLIKELY(NULL == units)) {
7082
    ret = OB_ERR_UNEXPECTED;
7083
    LOG_WARN("unit ptr is null", K(ret));
7084
  } else {
7085
    for (int64_t i = 0; OB_SUCC(ret) && is_allowed && i < units->count(); ++i) {
7086
      const ObUnit *unit = units->at(i);
7087
      if (OB_UNLIKELY(NULL == unit)) {
7088
        ret = OB_ERR_UNEXPECTED;
7089
        LOG_WARN("unit ptr is null", K(ret), KP(unit));
7090
      } else if (unit->migrate_from_server_.is_valid()) {
7091
        is_allowed = false;
7092
      } else {} // unit not in migrating, check next
7093
    }
7094
  }
7095
  return ret;
7096
}
7097

7098
int ObUnitManager::check_all_pools_granted(
7099
    const common::ObIArray<share::ObResourcePool *> &pools,
7100
    bool &all_granted)
7101
{
7102
  int ret = OB_SUCCESS;
7103
  all_granted = true;
7104
  for (int64_t i = 0; all_granted && OB_SUCC(ret) && i < pools.count(); ++i) {
7105
    if (NULL == pools.at(i)) {
7106
      ret = OB_ERR_UNEXPECTED;
7107
      LOG_WARN("pool ptr is null", K(ret), K(i));
7108
    } else if (!is_valid_tenant_id(pools.at(i)->tenant_id_)) {
7109
      all_granted = false;
7110
    } else {} // go on check next
7111
  }
7112
  return ret;
7113
}
7114

7115
//alter 14x add the new locality F/L@ZONE;
7116
//temporarily logonly_unit = 1 does not support scaling,
7117
//so only the unit num of non-logonly_unit can be reduced;
7118
//Here it is only judged whether the number of units on non-logonly_unit satisfies locality;
7119
//The goal of this function is to determine whether the distribution of the current unit meets the requirements of locality
7120
//check in two parts:
7121
//1. First check whether the logonly unit meets the requirements
7122
//2. second check whether the non-logonly unit meets the requirements
7123
int ObUnitManager::do_check_shrink_granted_pool_allowed_by_locality(
7124
    const common::ObIArray<share::ObResourcePool *> &pools,
7125
    const common::ObIArray<common::ObZone> &schema_zone_list,
7126
    const ZoneLocalityIArray &zone_locality,
7127
    const ObIArray<int64_t> &new_unit_nums,
7128
    bool &is_allowed)
7129
{
7130
  int ret = OB_SUCCESS;
7131
  UNUSED(schema_zone_list);
7132
  is_allowed = true;
7133
  common::hash::ObHashMap<common::ObZone, UnitNum> zone_unit_num_map;
7134
  const int64_t BUCKET_SIZE = 2 * MAX_ZONE_NUM;
7135
  if (OB_UNLIKELY(pools.count() <= 0
7136
                  || new_unit_nums.count() <= 0
7137
                  || new_unit_nums.count() != pools.count())) {
7138
    ret = OB_INVALID_ARGUMENT;
7139
    LOG_WARN("invalid argument", K(ret), K(pools.count()), K(new_unit_nums.count()));
7140
  } else if (OB_FAIL(zone_unit_num_map.create(BUCKET_SIZE, ObModIds::OB_RS_UNIT_MANAGER))) {
7141
    LOG_WARN("fail to create map", K(ret));
7142
  } else {
7143
    for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
7144
      const int64_t pool_unit_num = new_unit_nums.at(i);
7145
      const share::ObResourcePool *pool = pools.at(i);
7146
      if (OB_UNLIKELY(nullptr == pool)) {
7147
        ret = OB_ERR_UNEXPECTED;
7148
        LOG_WARN("pool ptr is null", K(ret));
7149
      } else {
7150
        const common::ObIArray<common::ObZone> &pool_zone_list = pool->zone_list_;
7151
        for (int64_t j = 0; OB_SUCC(ret) && j < pool_zone_list.count(); ++j) {
7152
          const common::ObZone &zone = pool_zone_list.at(j);
7153
          const int32_t overwrite = 1;
7154
          UnitNum unit_num_set;
7155
          int tmp_ret = zone_unit_num_map.get_refactored(zone, unit_num_set);
7156
          if (OB_SUCCESS == tmp_ret) {
7157
            // bypass
7158
          } else if (OB_HASH_NOT_EXIST == tmp_ret) {
7159
            unit_num_set.reset();
7160
          } else {
7161
            ret = OB_ERR_UNEXPECTED;
7162
            LOG_WARN("fail to get from map", K(ret));
7163
          }
7164
          if (OB_FAIL(ret)) {
7165
          } else if (REPLICA_TYPE_FULL == pool->replica_type_) {
7166
            unit_num_set.full_unit_num_ += pool_unit_num;
7167
          } else if (REPLICA_TYPE_LOGONLY == pool->replica_type_) {
7168
            unit_num_set.logonly_unit_num_ += pool_unit_num;
7169
          } else {
7170
            ret = OB_ERR_UNEXPECTED;
7171
            LOG_WARN("pool replica type unexpected", K(ret), "pool", *pool);
7172
          }
7173
          if (OB_FAIL(ret)) {
7174
          } else if (OB_FAIL(zone_unit_num_map.set_refactored(zone, unit_num_set, overwrite))) {
7175
            LOG_WARN("fail to set map", K(ret));
7176
          }
7177
        }
7178
      }
7179
    }
7180
    for (int64_t i = 0; is_allowed && OB_SUCC(ret) && i < zone_locality.count(); ++i) {
7181
      const share::ObZoneReplicaAttrSet &zone_replica_attr = zone_locality.at(i);
7182
      int64_t full_unit_num = 0;
7183
      int64_t logonly_unit_num = 0;
7184
      for (int64_t j = 0; OB_SUCC(ret) && j < zone_replica_attr.zone_set_.count(); ++j) {
7185
        const common::ObZone &zone = zone_replica_attr.zone_set_.at(j);
7186
        UnitNum unit_num_set;
7187
        if (OB_FAIL(zone_unit_num_map.get_refactored(zone, unit_num_set))) {
7188
          LOG_WARN("fail to get refactored", K(ret));
7189
        } else {
7190
          full_unit_num += unit_num_set.full_unit_num_;
7191
          logonly_unit_num += unit_num_set.logonly_unit_num_;
7192
        }
7193
      }
7194
      if (OB_SUCC(ret)) {
7195
        const int64_t specific_num = zone_replica_attr.get_specific_replica_num();
7196
        const int64_t except_l_specific_num = specific_num
7197
                                              - zone_replica_attr.get_logonly_replica_num()
7198
                                              - zone_replica_attr.get_encryption_logonly_replica_num();
7199
        const int64_t total_unit_num = full_unit_num + logonly_unit_num;
7200
        is_allowed = (total_unit_num >= specific_num && full_unit_num >= except_l_specific_num);
7201
      }
7202
    }
7203
  }
7204
  return ret;
7205
}
7206

7207
// Scaling also needs to consider the locality of the tenant or table under the tenant.
7208
// There are several situations that need to be considered:
7209
// 1 Although the resource pool has been granted to a tenant,
7210
//   but the zone of the resource pool may not be in the tenant's zone list.
7211
//   For example: tenant has two pools : pool1(z1,z2,z3),pool2(z4,z5,z6),
7212
//                The locality of tenant is F@z1,F@z2,F@z3,F@z4,
7213
//                tenant the zone_list is (z1, z2, z3, z4). In this case,
7214
//                only z4 needs to be considered when shrinkg pool2 unit num.
7215
// 2 The locality of the tenant may include zone locality and region locality.
7216
//   For zone locality, it is only necessary to consider that
7217
//   the unit num of the zone after scaling is not less than the number of copies of the locality in this zone.
7218
// 3 For region locality,
7219
//   it is necessary to satisfy that the sum of unit num of all zones in the region under
7220
//   this tenant is not less than the number of locality replicas on the region.
7221
//   For example: tenant has two pools, pool1(z1,z2) and pool2(z3,z4),
7222
//                the current pool1_unit_num = 2, pool2_unit_num = 2,
7223
//                where z1, z4 belong to region SH, z2, z3 belong to region HZ, and locality is F{2},R{2}@SH,F{2},R{2}@HZ.
7224
//   We try to compress the unit_num of pool2 to 1,
7225
//   the total number of units in region SH after compression is 3,
7226
//   and the total number of units in region HZ is 3,
7227
//   which is not enough to accommodate the number of copies of locality.
7228
//   Therefore, compressing the unit num of pool2 to 1 is not allowed.
7229
//The specific implementation is to traverse each zone in the zone list of the resource pool:
7230
//1 The zone does not exist in the tenant's zone list, skip it directly;
7231
//2 The zone exists in the tenant's zone locality.
7232
//  It is necessary to compare whether the compressed unit num is enough to accommodate
7233
//  the number of locality replicas in the zone;
7234
//3 If the zone does not exist in the tenant's zone locality, save the unit num in the region_unit_num_container.
7235
//4 Save the zones of other resource pools under the tenant to the region zone container,
7236
//  and compare whether the unit num is sufficient
7237
int ObUnitManager::check_shrink_granted_pool_allowed_by_tenant_locality(
7238
    share::schema::ObSchemaGetterGuard &schema_guard,
7239
    const uint64_t tenant_id,
7240
    const common::ObIArray<share::ObResourcePool *> &pools,
7241
    const common::ObIArray<int64_t> &new_unit_nums,
7242
    bool &is_allowed)
7243
{
7244
  int ret = OB_SUCCESS;
7245
  const share::schema::ObTenantSchema *tenant_schema = NULL;
7246
  common::ObArray<common::ObZone> tenant_zone_list;
7247
  if (OB_UNLIKELY(! is_valid_tenant_id(tenant_id))) {
7248
    ret = OB_INVALID_ARGUMENT;
7249
    LOG_WARN("invalid argument", K(ret), K(tenant_id));
7250
  } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
7251
    LOG_WARN("fail to get tenant info", K(ret), K(tenant_id));
7252
  } else if (NULL == tenant_schema) {
7253
    ret = OB_ERR_UNEXPECTED;
7254
    LOG_WARN("tenant schema is null", K(ret), KP(tenant_schema));
7255
  } else if (OB_FAIL(tenant_schema->get_zone_list(tenant_zone_list))) {
7256
    LOG_WARN("fail to get zone list", K(ret));
7257
  } else {
7258
    ObArray<ObZoneReplicaNumSet> zone_locality;
7259
    if (OB_FAIL(tenant_schema->get_zone_replica_attr_array(zone_locality))) {
7260
      LOG_WARN("fail to assign", K(ret));
7261
    } else if (OB_FAIL(do_check_shrink_granted_pool_allowed_by_locality(
7262
                pools, tenant_zone_list, zone_locality, new_unit_nums, is_allowed))) {
7263
      LOG_WARN("fail to do check shrink by locality", K(ret));
7264
    } else {} // no more to do
7265
  }
7266
  return ret;
7267
}
7268

7269
//  Currently supports tenant-level and table-level locality,
7270
//  and check shrinkage needs to be checked at the same time two levels of locality, tenant and table.
7271
//  1 The tenant level is more intuitive.
7272
//    directly compare the zone/region locality involved with the reduced unit num.
7273
//  2 Three situations need to be considered at the table level:
7274
//    2.1 Virtual tables, index tables, etc. without entity partitions, ignore them
7275
//    2.2 The table with entity partition whose table locality is empty,
7276
//        inherited from tenant, has been checked, and skipped;
7277
//    2.3 Tables with entity partitions whose table locality is not empty are checked according to table locality
7278
int ObUnitManager::check_shrink_granted_pool_allowed_by_locality(
7279
    share::ObResourcePool *pool,
7280
    const int64_t alter_unit_num,
7281
    bool &is_allowed)
7282
{
7283
  int ret = OB_SUCCESS;
7284
  share::schema::ObSchemaGetterGuard schema_guard;
7285
  const share::schema::ObTenantSchema *tenant_schema = NULL;
7286
  common::ObArray<share::ObResourcePool *> *pool_list = nullptr;
7287
  common::ObArray<share::ObResourcePool *> new_pool_list;
7288
  common::ObArray<int64_t> new_unit_nums;
7289
  if (OB_UNLIKELY(NULL == pool || alter_unit_num <= 0)) {
7290
    ret = OB_INVALID_ARGUMENT;
7291
    LOG_WARN("invalid argument", K(ret), KP(pool), K(alter_unit_num));
7292
  } else if (OB_ISNULL(schema_service_)) {
7293
    ret = OB_NOT_INIT;
7294
    LOG_WARN("schema service is null", K(schema_service_), KR(ret));
7295
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(pool->tenant_id_, schema_guard))) {
7296
    LOG_WARN("fail to get tenant schema guard", KR(ret), K(pool->tenant_id_));
7297
  } else if (OB_FAIL(schema_guard.get_tenant_info(pool->tenant_id_, tenant_schema))) {
7298
    LOG_WARN("fail to get tenant info", K(ret), "tenant_id", pool->tenant_id_);
7299
  } else if (OB_UNLIKELY(NULL == tenant_schema)) {
7300
    ret = OB_ERR_UNEXPECTED;
7301
    LOG_WARN("tenant schema is null", K(ret), KP(tenant_schema));
7302
  } else if (OB_FAIL(get_pools_by_tenant_(tenant_schema->get_tenant_id(), pool_list))) {
7303
    LOG_WARN("fail to get pools by tenant", K(ret));
7304
  } else if (OB_UNLIKELY(nullptr == pool_list)) {
7305
    ret = OB_ERR_UNEXPECTED;
7306
    LOG_WARN("pool list ptr is null", K(ret));
7307
  } else {
7308
    for (int64_t i = 0; OB_SUCC(ret) && i < pool_list->count(); ++i) {
7309
      share::ObResourcePool *this_pool = pool_list->at(i);
7310
      if (OB_UNLIKELY(nullptr == this_pool)) {
7311
        ret = OB_ERR_UNEXPECTED;
7312
        LOG_WARN("this pool ptr is null", K(ret));
7313
      } else if (OB_FAIL(new_pool_list.push_back(this_pool))) {
7314
        LOG_WARN("fail to to push back", K(ret));
7315
      } else if (OB_FAIL(new_unit_nums.push_back(
7316
              this_pool->resource_pool_id_ == pool->resource_pool_id_
7317
              ? alter_unit_num : this_pool->unit_count_))) {
7318
        LOG_WARN("fail to push back", K(ret));
7319
      }
7320
    }
7321
    if (OB_FAIL(ret)) {
7322
    } else if (OB_FAIL(check_shrink_granted_pool_allowed_by_tenant_locality(
7323
            schema_guard, pool->tenant_id_, new_pool_list, new_unit_nums, is_allowed))) {
7324
      LOG_WARN("fail to check shrink by tenant locality", K(ret));
7325
    } else {} // no more to do
7326
  }
7327
  return ret;
7328
}
7329

7330
int ObUnitManager::check_shrink_granted_pool_allowed_by_alter_locality(
7331
    share::ObResourcePool *pool,
7332
    bool &is_allowed)
7333
{
7334
  int ret = OB_SUCCESS;
7335
  rootserver::ObRootService *root_service = NULL;
7336
  bool in_alter_locality = true;
7337
  if (OB_UNLIKELY(NULL == pool)) {
7338
    ret = OB_INVALID_ARGUMENT;
7339
    LOG_WARN("invalid argument", K(ret), KP(pool));
7340
  } else if (OB_UNLIKELY(!is_valid_tenant_id(pool->tenant_id_))) {
7341
    ret = OB_INVALID_ARGUMENT;
7342
    LOG_WARN("invalid tenant id", K(ret), "tenant_id", pool->tenant_id_);
7343
  } else if (OB_UNLIKELY(NULL == (root_service = GCTX.root_service_))) {
7344
    ret = OB_ERR_UNEXPECTED;
7345
    LOG_WARN("rootservice is null", K(ret));
7346
  } else if (OB_FAIL(root_service->check_tenant_in_alter_locality(
7347
          pool->tenant_id_, in_alter_locality))) {
7348
    LOG_WARN("fail to check tenant in alter locality", K(ret), "tenant_id", pool->tenant_id_);
7349
  } else if (in_alter_locality) {
7350
    is_allowed = false;
7351
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "shrink pool unit num while altering locality");
7352
  } else {} // no more to do
7353
  return ret;
7354
}
7355

7356
//  The shrinking operation of the granted resource pool requires the following pre-checks
7357
//  1 The design avoids simultaneous migration and shrinkage of units.
7358
//    Therefore, it is necessary to check whether all the units contained in the current resource pool are in the migration state before shrinking.
7359
//    If so, shrinking operations are temporarily not allowed, and the migration needs to be completed.
7360
//  2 The scaling operation also needs to consider locality.
7361
//    The number of copies of locality cannot exceed the number of units after scaling.
7362
//    For example, when the locality is F, R@zone1, the unit num in zone1 cannot be reduced to 1.
7363
//  3 Locality changes and scaling under the same tenant are not allowed to be performed at the same time
7364
int ObUnitManager::check_shrink_granted_pool_allowed(
7365
    share::ObResourcePool *pool,
7366
    const int64_t alter_unit_num,
7367
    bool &is_allowed)
7368
{
7369
  int ret = OB_SUCCESS;
7370
  if (OB_UNLIKELY(NULL == pool || alter_unit_num <= 0)) {
7371
    ret = OB_INVALID_ARGUMENT;
7372
    LOG_WARN("invalid argument", K(ret), KP(pool), K(alter_unit_num));
7373
  } else if (OB_GTS_TENANT_ID == pool->tenant_id_) {
7374
    is_allowed = true;
7375
  } else if (OB_FAIL(check_shrink_granted_pool_allowed_by_migrate_unit(pool, alter_unit_num, is_allowed))) {
7376
    LOG_WARN("fail to check by migrate unit", K(ret));
7377
  } else if (!is_allowed) {
7378
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "shrink pool unit num while unit migrating");
7379
  } else if (OB_FAIL(check_shrink_granted_pool_allowed_by_locality(pool, alter_unit_num, is_allowed))) {
7380
    LOG_WARN("fail to check by locality", K(ret));
7381
  } else if (!is_allowed) {
7382
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "shrink pool unit num causing not enough locality");
7383
  } else if (OB_FAIL(check_shrink_granted_pool_allowed_by_alter_locality(pool, is_allowed))) {
7384
    LOG_WARN("fail to check shrink granted pool allowed by alter locality", K(ret));
7385
  } else if (!is_allowed) {
7386
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "shrink pool unit num while altering locality");
7387
  }
7388
  return ret;
7389
}
7390

7391
int ObUnitManager::complete_migrate_unit_rs_job_in_pool(
7392
      const int64_t resource_pool_id,
7393
      const int result_ret,
7394
      common::ObMySQLTransaction &trans)
7395
{
7396
  int ret = OB_SUCCESS;
7397
  ObArray<ObUnit *> *units = NULL;
7398
  if (!inited_) {
7399
    ret = OB_NOT_INIT;
7400
    LOG_WARN("not init", K(ret));
7401
  } else if (OB_INVALID_ID == resource_pool_id) {
7402
    ret = OB_INVALID_ARGUMENT;
7403
    LOG_WARN("invalid argument", K(resource_pool_id), K(ret));
7404
  } else if (OB_FAIL(get_units_by_pool(resource_pool_id, units))) {
7405
    LOG_WARN("fail to get unit by pool", K(resource_pool_id), K(ret));
7406
  } else if (OB_UNLIKELY(NULL == units)) {
7407
    ret = OB_ERR_UNEXPECTED;
7408
    LOG_WARN("units is null", KP(units), K(ret));
7409
  } else {
7410
    for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
7411
      const ObUnit *unit = units->at(i);
7412
      if (OB_UNLIKELY(NULL == unit)) {
7413
        ret = OB_ERR_UNEXPECTED;
7414
        LOG_WARN("unit should not be null", K(ret));
7415
      } else if (unit->is_manual_migrate()) {
7416
        char ip_buf[common::MAX_IP_ADDR_LENGTH];
7417
        (void)unit->server_.ip_to_string(ip_buf, common::MAX_IP_ADDR_LENGTH);
7418
        int64_t job_id = 0;
7419
        int tmp_ret = RS_JOB_FIND(MIGRATE_UNIT, job_id, *proxy_,
7420
                                  "unit_id", unit->unit_id_,
7421
                                  "svr_ip", ip_buf,
7422
                                  "svr_port", unit->server_.get_port());
7423
        if (OB_SUCCESS == tmp_ret && job_id > 0) {
7424
          if (OB_FAIL(RS_JOB_COMPLETE(job_id, result_ret, trans))) {
7425
            LOG_WARN("all_rootservice_job update failed", K(ret), K(result_ret), K(job_id));
7426
          }
7427
        }
7428
      }
7429
    }
7430
  }
7431
  return ret;
7432
}
7433

7434
/* the routine that calls the expand_pool_unit_num shall
7435
 * ensure that this pool is not granted to any tenant
7436
 */
7437
int ObUnitManager::expand_pool_unit_num_(
7438
    share::ObResourcePool *pool,
7439
    const int64_t alter_unit_num)
7440
{
7441
  int ret = OB_SUCCESS;
7442
  const char *module = "ALTER_RESOURCE_POOL_UNIT_NUM";
7443
  const bool new_allocate_pool = false;
7444
  common::ObMySQLTransaction trans;
7445
  ObArray<ObAddr> new_servers;
7446
  ObArray<share::ObUnit *> *units = NULL;
7447
  ObArray<uint64_t> bak_unit_ids;
7448
  if (OB_UNLIKELY(NULL == pool || alter_unit_num <= pool->unit_count_)) {
7449
    ret = OB_INVALID_ARGUMENT;
7450
    LOG_WARN("pool ptr is null", K(ret), "pre_unit_cnt", pool->unit_count_, "cur_unit_cnt", alter_unit_num);
7451
  } else if (pool->is_granted_to_tenant()) {
7452
    ret = OB_ERR_UNEXPECTED;
7453
    LOG_WARN("unexpected pool status", KR(ret), KPC(pool));
7454
  } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
7455
    LOG_WARN("start transaction failed", K(ret));
7456
  } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
7457
    LOG_WARN("fail to get units by pool", K(ret));
7458
  } else if (OB_FAIL(extract_unit_ids(*units, bak_unit_ids))) {
7459
    LOG_WARN("fail to extract unit ids", K(ret));
7460
  } else if (OB_FAIL(allocate_pool_units_(
7461
          trans, *pool, pool->zone_list_, nullptr, new_allocate_pool,
7462
          alter_unit_num - pool->unit_count_, module, new_servers))) {
7463
    LOG_WARN("arrange units failed", "pool", *pool, K(new_allocate_pool), K(ret));
7464
  } else {
7465
    share::ObResourcePool new_pool;
7466
    if (OB_FAIL(new_pool.assign(*pool))) {
7467
      LOG_WARN("failed to assign new_poll", K(ret));
7468
    } else {
7469
      new_pool.unit_count_ = alter_unit_num;
7470
      if (OB_FAIL(ut_operator_.update_resource_pool(trans, new_pool, false/*need_check_conflict_with_clone*/))) {
7471
        LOG_WARN("update_resource_pool failed", K(new_pool), K(ret));
7472
      }
7473
    }
7474
  }
7475

7476
  if (trans.is_started()) {
7477
    const bool commit = (OB_SUCC(ret));
7478
    int temp_ret = OB_SUCCESS;
7479
    if (OB_SUCCESS != (temp_ret = trans.end(commit))) {
7480
      LOG_WARN("trans end failed", K(commit), K(temp_ret));
7481
      ret = (OB_SUCCESS == ret) ? temp_ret : ret;
7482
    }
7483
  }
7484

7485
  if (OB_FAIL(ret)) {
7486
    // avoid overwrite ret
7487
    int64_t temp_ret = OB_SUCCESS;
7488
    if (OB_SUCCESS != (temp_ret = delete_invalid_inmemory_units(pool->resource_pool_id_, bak_unit_ids))) {
7489
      LOG_WARN("delete unit groups failed", "resource pool id", pool->resource_pool_id_, K(temp_ret));
7490
    }
7491
  } else {
7492
    pool->unit_count_ = alter_unit_num;
7493
  }
7494
  return ret;
7495
}
7496

7497
int ObUnitManager::get_pool_complete_unit_num_and_status(
7498
    const share::ObResourcePool *pool,
7499
    int64_t &unit_num_per_zone,
7500
    int64_t &current_unit_num_per_zone,
7501
    bool &has_unit_num_modification)
7502
{
7503
  int ret = OB_SUCCESS;
7504
  common::ObArray<share::ObUnit *> *units = NULL;
7505
  if (OB_UNLIKELY(NULL == pool)) {
7506
    ret = OB_INVALID_ARGUMENT;
7507
    LOG_WARN("pool is null", K(ret));
7508
  } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
7509
    LOG_WARN("fail to get units by pool", K(ret), "pool_id", pool->resource_pool_id_);
7510
  } else if (OB_UNLIKELY(NULL == units)) {
7511
    ret = OB_ERR_UNEXPECTED;
7512
    LOG_WARN("units is null", K(ret), KP(units), "pool_id", pool->resource_pool_id_);
7513
  } else if (units->count() <= 0) {
7514
    ret = OB_ERR_UNEXPECTED;
7515
    LOG_WARN("total unit num unexpected", K(ret), "total_unit_num", units->count());
7516
  } else if (pool->zone_list_.count() <= 0) {
7517
    ret = OB_ERR_UNEXPECTED;
7518
    LOG_WARN("pool zone list count unexpected", K(ret), "pool_zone_list_count", pool->zone_list_.count());
7519
  } else {
7520
    int64_t total_unit_cnt = units->count();
7521
    int64_t zone_cnt = pool->zone_list_.count();
7522
    int64_t consult = total_unit_cnt / zone_cnt;
7523
    if (total_unit_cnt != consult * zone_cnt) {
7524
      // total_unit_cnt cannot be divisible by zone_cnt,
7525
      // the number of units in each zone of the resource pool is inconsistent
7526
      ret = OB_ERR_UNEXPECTED;
7527
      LOG_WARN("unexpected total unit cnt", K(ret), K(zone_cnt), K(total_unit_cnt));
7528
    } else {
7529
      unit_num_per_zone = consult;
7530
      current_unit_num_per_zone = pool->unit_count_;
7531
      has_unit_num_modification = false;
7532
      for (int64_t i = 0; OB_SUCC(ret) && i < units->count() && !has_unit_num_modification; ++i) {
7533
        const ObUnit *this_unit = units->at(i);
7534
        if (OB_UNLIKELY(NULL == this_unit)) {
7535
          ret = OB_ERR_UNEXPECTED;
7536
          LOG_WARN("unit ptr is null", K(ret), KP(this_unit));
7537
        } else if (ObUnit::UNIT_STATUS_ACTIVE == this_unit->status_) {
7538
          // an active unit, go and check next
7539
        } else if (ObUnit::UNIT_STATUS_DELETING == this_unit->status_) {
7540
          has_unit_num_modification = true;
7541
        } else {
7542
          ret = OB_ERR_UNEXPECTED;
7543
          LOG_WARN("unexpected unit status", K(ret), "unit_status", this_unit->status_);
7544
        }
7545
      }
7546
    }
7547
  }
7548
  return ret;
7549
}
7550

7551
// According to the unit num of the current resource pool
7552
// and the unit_num(alter_unit_num) of the resource pool after the change,
7553
// the change type of the current operation is determined.
7554
// Let the current num be cur_unit_num, and the num after the change should be alter_unit_num.
7555
// The principle is as follows:
7556
// 1 cur_unit_num == alter_unit_num, no change, means no operation, no modification
7557
// 2 cur_unit_num <alter_unit_num, and there is currently no modification operation of the resource pool and no other unit num,
7558
//   which is considered to be an expansion operation.
7559
// 3 cur_unit_num <alter_unit_num, and the resource pool is shrinking,
7560
//   the unit_num before shrinking is prev_unit_num,
7561
//   and alter_unit_num is equal to prev_unit_num,
7562
//   which is considered to be a shrinking rollback operation.
7563
// 4 cur_unit_num> alter_unit_num, and the resource pool has not performed other unit num modification operations,
7564
//   which is considered to be a shrinking operation
7565
int ObUnitManager::determine_alter_unit_num_type(
7566
    share::ObResourcePool *pool,
7567
    const int64_t alter_unit_num,
7568
    AlterUnitNumType &alter_unit_num_type)
7569
{
7570
  int ret = OB_SUCCESS;
7571
  int64_t complete_unit_num_per_zone = 0; // Contains the unit in the delete state
7572
  int64_t current_unit_num_per_zone = 0; // unit num in pool->unit_count
7573
  bool has_unit_num_modification = true;
7574
  if (OB_UNLIKELY(NULL == pool)) {
7575
    ret = OB_INVALID_ARGUMENT;
7576
    LOG_WARN("pool is null", K(ret));
7577
  } else if (alter_unit_num == pool->unit_count_) {
7578
    alter_unit_num_type = AUN_NOP; // do nothing
7579
  } else if (OB_FAIL(get_pool_complete_unit_num_and_status(
7580
          pool, complete_unit_num_per_zone,
7581
          current_unit_num_per_zone, has_unit_num_modification))) {
7582
    LOG_WARN("fail to get pool complete unit num and status", K(ret));
7583
  } else if (OB_UNLIKELY(complete_unit_num_per_zone <= 0)) {
7584
    ret = OB_ERR_UNEXPECTED;
7585
    LOG_WARN("unexpected complete_unit_num", K(ret), K(complete_unit_num_per_zone));
7586
  } else {
7587
    if (has_unit_num_modification) { // A unit num change is taking place
7588
      // pool not granted should not has unit in DELETING status
7589
      ret = OB_ERR_UNEXPECTED;
7590
      LOG_WARN("pool not granted is not expected to have unit in DELETING",
7591
               KR(ret), K(current_unit_num_per_zone), K(complete_unit_num_per_zone));
7592
    } else {
7593
      if (alter_unit_num > pool->unit_count_) {
7594
        alter_unit_num_type = AUN_EXPAND;
7595
      } else if (alter_unit_num < pool->unit_count_) {
7596
        alter_unit_num_type = AUN_SHRINK;
7597
      } else {
7598
        alter_unit_num_type = AUN_NOP;
7599
      }
7600
    }
7601
  }
7602
  return ret;
7603
}
7604

7605
// alter_pool_unit_num changes a single pool's unit_num,
7606
// and only works on pool not granted to tenant.
7607
int ObUnitManager::alter_pool_unit_num(
7608
    share::ObResourcePool *pool,
7609
    int64_t alter_unit_num,
7610
    const common::ObIArray<uint64_t> &delete_unit_id_array)
7611
{
7612
  int ret = OB_SUCCESS;
7613
  AlterUnitNumType alter_unit_num_type = AUN_MAX;
7614
  if (!check_inner_stat()) {
7615
    ret = OB_INNER_STAT_ERROR;
7616
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
7617
  } else if (NULL == pool) {
7618
    ret = OB_INVALID_ARGUMENT;
7619
    LOG_WARN("pool is null", K(ret));
7620
  } else if (alter_unit_num <= 0) {
7621
    ret = OB_INVALID_ARGUMENT;
7622
    LOG_WARN("invalid alter_unit_num", K(alter_unit_num), K(ret));
7623
  } else if (pool->is_granted_to_tenant()) {
7624
    ret = OB_OP_NOT_ALLOW;
7625
    LOG_WARN("alter pool unit num which is granted to a tenant is not allowed", KR(ret), KPC(pool));
7626
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "alter pool unit num which is granted to a tenant");
7627
  } else if (OB_FAIL(determine_alter_unit_num_type(pool, alter_unit_num, alter_unit_num_type))) {
7628
    LOG_WARN("fail to determine alter unit num type", K(ret));
7629
  } else if (AUN_NOP == alter_unit_num_type) {
7630
    // do nothing
7631
    if (delete_unit_id_array.count() > 0) {
7632
      ret = OB_NOT_SUPPORTED;
7633
      LOG_WARN("do not support deleting unit without unit num changed", K(ret));
7634
      LOG_USER_ERROR(OB_NOT_SUPPORTED, "delete unit without unit num change");
7635
    } else {} // good
7636
  } else if (AUN_EXPAND == alter_unit_num_type) {
7637
    if (delete_unit_id_array.count() > 0) {
7638
      ret = OB_NOT_SUPPORTED;
7639
      LOG_WARN("do not support expand pool unit num combined with deleting unit", K(ret));
7640
      LOG_USER_ERROR(OB_NOT_SUPPORTED, "expand pool unit num combined with deleting unit");
7641
    } else if (OB_FAIL(expand_pool_unit_num_(pool, alter_unit_num))) {
7642
      LOG_WARN("fail to expend pool unit num", K(ret), K(alter_unit_num));
7643
    }
7644
  } else if (AUN_SHRINK == alter_unit_num_type) {
7645
    if (OB_FAIL(shrink_pool_unit_num(pool, alter_unit_num, delete_unit_id_array))) {
7646
      LOG_WARN("fail to shrink pool unit num", K(ret), K(alter_unit_num));
7647
    }
7648
  } else if (AUN_ROLLBACK_SHRINK == alter_unit_num_type) {
7649
    ret = OB_ERR_UNEXPECTED;
7650
    LOG_WARN("unexpected alter_unit_num_type", KR(ret), K(alter_unit_num));
7651
  } else if (AUN_MAX == alter_unit_num_type) {
7652
    ret = OB_OP_NOT_ALLOW;
7653
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "alter unit num while the previous operation is in progress");
7654
    LOG_WARN("alter unit num not allowed", K(ret));
7655
  } else {
7656
    ret = OB_ERR_UNEXPECTED;
7657
    LOG_WARN("unexpected alter unit num type", K(ret), K(alter_unit_num_type));
7658
  }
7659
  return ret;
7660
}
7661

7662
int ObUnitManager::get_zone_pools_unit_num(
7663
    const common::ObZone &zone,
7664
    const common::ObIArray<share::ObResourcePoolName> &new_pool_name_list,
7665
    int64_t &total_unit_num,
7666
    int64_t &full_unit_num,
7667
    int64_t &logonly_unit_num)
7668
{
7669
  int ret = OB_SUCCESS;
7670
  SpinRLockGuard guard(lock_);
7671
  if (!check_inner_stat()) {
7672
    ret = OB_INNER_STAT_ERROR;
7673
    LOG_WARN("check_inner_stat failed", K(ret));
7674
  } else if (zone.is_empty()) {
7675
    ret = OB_INVALID_ARGUMENT;
7676
    LOG_WARN("invalid argument", K(ret), K(zone));
7677
  } else {
7678
    common::ObArray<share::ObResourcePool *> pool_list;
7679
    for (int64_t i = 0; OB_SUCC(ret) && i < new_pool_name_list.count(); ++i) {
7680
      const share::ObResourcePoolName &pool_name = new_pool_name_list.at(i);
7681
      share::ObResourcePool *pool = NULL;
7682
      if (OB_FAIL(inner_get_resource_pool_by_name(pool_name, pool))) {
7683
        LOG_WARN("fail to get resource pool by name", K(ret));
7684
      } else if (OB_UNLIKELY(NULL == pool)) {
7685
        ret = OB_ERR_UNEXPECTED;
7686
        LOG_WARN("pool ptr is null", K(ret), K(pool_name));
7687
      } else if (OB_FAIL(pool_list.push_back(pool))) {
7688
        LOG_WARN("fail to push back", K(ret));
7689
      } else {} // no more to do
7690
    }
7691
    if (OB_FAIL(ret)) {
7692
    } else if (OB_FAIL(inner_get_zone_pools_unit_num(
7693
            zone, pool_list, total_unit_num, full_unit_num, logonly_unit_num))) {
7694
      LOG_WARN("fail to inner get pools unit num", K(ret));
7695
    }
7696
  }
7697
  return ret;
7698
}
7699

7700
int ObUnitManager::check_can_remove_pool_zone_list(
7701
    const share::ObResourcePool *pool,
7702
    const common::ObIArray<common::ObZone> &to_be_removed_zones,
7703
    bool &can_remove)
7704
{
7705
  int ret = OB_SUCCESS;
7706
  can_remove = true;
7707
  common::ObArray<share::ObResourcePool *> *pool_list = NULL;
7708
  share::schema::ObSchemaGetterGuard schema_guard;
7709
  const share::schema::ObTenantSchema *tenant_schema = NULL;
7710
  if (OB_UNLIKELY(NULL == pool) || OB_UNLIKELY(to_be_removed_zones.count() <= 0)) {
7711
    ret = OB_INVALID_ARGUMENT;
7712
    LOG_WARN("invalid argument", K(ret), KP(pool), K(to_be_removed_zones));
7713
  } else if (!pool->is_granted_to_tenant()) {
7714
    can_remove = true; // this pool do not grant to any tenant, can remove zone unit
7715
  } else if (OB_FAIL(get_pools_by_tenant_(pool->tenant_id_, pool_list))) {
7716
    LOG_WARN("fail to get pools by tenant", K(ret));
7717
  } else if (OB_UNLIKELY(NULL == pool_list)) {
7718
    ret = OB_ERR_UNEXPECTED;
7719
    LOG_WARN("pool list ptr is null", K(ret));
7720
  } else if (OB_ISNULL(schema_service_)) {
7721
    ret = OB_NOT_INIT;
7722
    LOG_WARN("schema service is null", K(schema_service_), KR(ret));
7723
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(pool->tenant_id_, schema_guard))) {
7724
    LOG_WARN("fail to get tenant schema guard", KR(ret), K(pool->tenant_id_));
7725
  } else if (OB_FAIL(schema_guard.get_tenant_info(pool->tenant_id_, tenant_schema))) {
7726
    LOG_WARN("fail to get tenant schema", K(ret), "tenant_id", pool->tenant_id_);
7727
  } else if (NULL == tenant_schema) {
7728
    ret = OB_TENANT_NOT_EXIST;
7729
    LOG_WARN("tenant schema is null", K(ret));
7730
  } else if (!tenant_schema->get_previous_locality_str().empty()) {
7731
    can_remove = false;
7732
    LOG_WARN("alter pool zone list is not allowed while locality modification",
7733
             "tenant_id", tenant_schema->get_tenant_id());
7734
  } else {
7735
    for (int64_t i = 0; can_remove && OB_SUCC(ret) && i < to_be_removed_zones.count(); ++i) {
7736
      int64_t total_unit_num = 0;
7737
      int64_t full_unit_num = 0;
7738
      int64_t logonly_unit_num = 0;
7739
      const common::ObZone &zone = to_be_removed_zones.at(i);
7740
      bool enough = false;
7741
      if (OB_FAIL(inner_get_zone_pools_unit_num(
7742
              zone, *pool_list, total_unit_num, full_unit_num, logonly_unit_num))) {
7743
        LOG_WARN("fail to get zone pools unit num", K(ret));
7744
      } else if (total_unit_num != full_unit_num + logonly_unit_num) {
7745
        ret = OB_ERR_UNEXPECTED;
7746
        LOG_WARN("unit num value not match", K(ret),
7747
                 K(total_unit_num), K(full_unit_num), K(logonly_unit_num));
7748
      } else {
7749
        // Reduce the unit of this pool on to be remove zone
7750
        total_unit_num -= pool->unit_count_;
7751
        if (common::REPLICA_TYPE_FULL == pool->replica_type_) {
7752
          full_unit_num -= pool->unit_count_;
7753
        } else if (common::REPLICA_TYPE_LOGONLY == pool->replica_type_) {
7754
          logonly_unit_num -= pool->unit_count_;
7755
        } else {
7756
          ret = OB_ERR_UNEXPECTED;
7757
          LOG_WARN("do not support this pool type", K(ret), K(*pool));
7758
        }
7759
      }
7760
      if (OB_FAIL(ret)) {
7761
      } else if (OB_FAIL(inner_check_schema_zone_unit_enough(
7762
            zone, total_unit_num, full_unit_num, logonly_unit_num,
7763
            *tenant_schema, schema_guard, enough))) {
7764
        LOG_WARN("fail to inner check schema zone unit enough", K(ret));
7765
      } else if (!enough) {
7766
        can_remove = false;
7767
      } else {}
7768
    }
7769
  }
7770
  return ret;
7771
}
7772

7773
int ObUnitManager::inner_get_zone_pools_unit_num(
7774
    const common::ObZone &zone,
7775
    const common::ObIArray<share::ObResourcePool *> &pool_list,
7776
    int64_t &total_unit_num,
7777
    int64_t &full_unit_num,
7778
    int64_t &logonly_unit_num)
7779
{
7780
  int ret = OB_SUCCESS;
7781
  if (!check_inner_stat()) {
7782
    ret = OB_INNER_STAT_ERROR;
7783
    LOG_WARN("check inner stat failed", K(ret));
7784
  } else if (zone.is_empty()) {
7785
    ret = OB_INVALID_ARGUMENT;
7786
    LOG_WARN("invalid argument", K(ret), K(zone));
7787
  } else {
7788
    total_unit_num = 0;
7789
    full_unit_num = 0;
7790
    logonly_unit_num = 0;
7791
    for (int64_t i = 0; OB_SUCC(ret) && i < pool_list.count(); ++i) {
7792
      share::ObResourcePool *pool = pool_list.at(i);
7793
      if (OB_UNLIKELY(NULL == pool)) {
7794
        ret = OB_ERR_UNEXPECTED;
7795
        LOG_WARN("pool ptr is null", K(ret));
7796
      } else if (!has_exist_in_array(pool->zone_list_, zone)) {
7797
        //ignore
7798
      } else {
7799
        total_unit_num += pool->unit_count_;
7800
        if (common::REPLICA_TYPE_FULL == pool->replica_type_) {
7801
          full_unit_num += pool->unit_count_;
7802
        } else if (common::REPLICA_TYPE_LOGONLY == pool->replica_type_) {
7803
          logonly_unit_num += pool->unit_count_;
7804
        } else {
7805
          ret = OB_ERR_UNEXPECTED;
7806
          LOG_WARN("do not support this pool type", K(ret), K(*pool));
7807
        }
7808
      }
7809
    }
7810
  }
7811
  return ret;
7812
}
7813

7814
int ObUnitManager::alter_pool_zone_list(
7815
    share::ObResourcePool *pool,
7816
    const ObIArray<ObZone> &zone_list)
7817
{
7818
  // The requirement of alter pool_zone_list is currently only for cluster relocation.
7819
  // Before deleting the zone,
7820
  // the pool_zone_list that contains the zone in all resource pools needs to be removed from the zone_list..
7821
  int ret = OB_SUCCESS;
7822
  if (!check_inner_stat()) {
7823
    ret = OB_INNER_STAT_ERROR;
7824
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
7825
  } else if (OB_UNLIKELY(NULL == pool || zone_list.count() <= 0)) {
7826
    ret = OB_INVALID_ARGUMENT;
7827
    LOG_WARN("invalid argument", K(ret), KP(pool), "new zone list", zone_list);
7828
  } else {
7829
    ObArray<ObZone> new_zone_list;
7830
    //check repeat
7831
    for (int64_t i = 0; i < zone_list.count() && OB_SUCC(ret); ++i) {
7832
      if (has_exist_in_array(new_zone_list, zone_list.at(i))) {
7833
        ret = OB_OP_NOT_ALLOW;
7834
        LOG_USER_ERROR(OB_OP_NOT_ALLOW, "alter resource pool zone repeat");
7835
        LOG_WARN("alter resource pool zone repeat not allow", K(ret));
7836
      } else if (OB_FAIL(new_zone_list.push_back(zone_list.at(i)))) {
7837
        LOG_WARN("fail to push back", K(ret));
7838
      } else {}//no more to do
7839
    }
7840
    if (OB_SUCC(ret)) {
7841
      std::sort(new_zone_list.begin(), new_zone_list.end());
7842
      bool is_add_pool_zone = false;
7843
      bool is_remove_pool_zone = false;
7844
      for (int64_t i = 0; i < new_zone_list.count() && OB_SUCC(ret); ++i) {
7845
        if (!has_exist_in_array(pool->zone_list_, new_zone_list.at(i))) {
7846
          is_add_pool_zone = true;
7847
        } else {}//nothing todo
7848
      }
7849
      for (int64_t i = 0; i < pool->zone_list_.count() && OB_SUCC(ret); ++i) {
7850
        if (!has_exist_in_array(new_zone_list, pool->zone_list_.at(i))) {
7851
          is_remove_pool_zone = true;
7852
        } else {}//nothing todo
7853
      }
7854
      if (is_add_pool_zone && is_remove_pool_zone) {
7855
        ret = OB_OP_NOT_ALLOW;
7856
        LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Cannot add and delete zones at the same time");
7857
        LOG_WARN("Cannot add and delete zones at the same time", K(ret));
7858
      } else if (is_add_pool_zone) {
7859
        if (OB_FAIL(add_pool_zone_list(pool, new_zone_list))) {
7860
          LOG_WARN("fail to add pool zone list", K(ret));
7861
        }
7862
      } else {
7863
        if (OB_FAIL(remove_pool_zone_list(pool, new_zone_list))) {
7864
          LOG_WARN("fail to remoce pool zone list", K(ret));
7865
        }
7866
      }
7867
    }
7868
  }
7869
  return ret;
7870
}
7871

7872
int ObUnitManager::remove_pool_zone_list(
7873
    share::ObResourcePool *pool,
7874
    const ObIArray<ObZone> &zone_list)
7875
{
7876
  int ret = OB_SUCCESS;
7877
  bool can_remove = false;
7878
  common::ObArray<common::ObZone> zones_to_be_removed;
7879
  const common::ObIArray<common::ObZone> &prev_zone_list = pool->zone_list_;
7880
  if (OB_FAIL(cal_to_be_removed_pool_zone_list(
7881
          prev_zone_list, zone_list, zones_to_be_removed))) {
7882
    LOG_WARN("fail to calculate to be removed pool zone list", K(ret));
7883
  } else if (zones_to_be_removed.count() <= 0) {
7884
    // no zones need to be removed, return SUCC directly
7885
  } else if (OB_FAIL(check_can_remove_pool_zone_list(pool, zones_to_be_removed, can_remove))) {
7886
    LOG_WARN("fail to check can remove pool zon list", K(ret));
7887
  } else if (!can_remove) {
7888
    ret = OB_OP_NOT_ALLOW;
7889
    LOG_WARN("cannot alter resource pool zone list", K(ret),
7890
             "pool", *pool, K(zones_to_be_removed));
7891
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "alter resource pool zone list with not empty unit");
7892
  } else if (OB_FAIL(do_remove_pool_zone_list(pool, zone_list, zones_to_be_removed))) {
7893
    LOG_WARN("fail to do remove pool zone list", K(ret), "pool", *pool, K(zones_to_be_removed));
7894
  } else {} // no more to do
7895
  return ret;
7896
}
7897

7898
int ObUnitManager::add_pool_zone_list(
7899
    share::ObResourcePool *pool,
7900
    const ObIArray<ObZone> &zone_list)
7901
{
7902
  int ret = OB_SUCCESS;
7903
  bool can_add = false;
7904
  common::ObArray<common::ObZone> zones_to_be_add;
7905
  bool is_add_pool_zone_list_allowed = true;
7906
  if (OB_UNLIKELY(nullptr == pool)) {
7907
    ret = OB_INVALID_ARGUMENT;
7908
    LOG_WARN("invalid argument", KR(ret), KP(pool));
7909
  } else if (! pool->is_granted_to_tenant()) {
7910
    // good, can do add pool zone list directly
7911
    is_add_pool_zone_list_allowed = true;
7912
  } else {
7913
    if (OB_FAIL(check_expand_zone_resource_allowed_by_old_unit_stat_(
7914
            pool->tenant_id_, is_add_pool_zone_list_allowed))) {
7915
      LOG_WARN("fail to check grant pools allowed by unit stat",
7916
               KR(ret), "tenant_id", pool->tenant_id_);
7917
    }
7918
  }
7919

7920
  if (OB_FAIL(ret)) {
7921
    // bypass
7922
  } else if (!is_add_pool_zone_list_allowed) {
7923
    ret = OB_OP_NOT_ALLOW;
7924
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "add pool zone when granted pools of tenant in shrinking");
7925
  } else if (OB_FAIL(cal_to_be_add_pool_zone_list(
7926
          pool->zone_list_, zone_list, zones_to_be_add))) {
7927
    LOG_WARN("fail to calculate to be add pool zone list", K(ret));
7928
  } else if (zones_to_be_add.count() <= 0) {
7929
    //no zones need to be add, return SUCC directly
7930
  } else if (OB_FAIL(check_can_add_pool_zone_list_by_locality(pool, zones_to_be_add, can_add))) {
7931
    LOG_WARN("fail to check can add pool zone list", K(ret));
7932
  } else if (!can_add) {
7933
    ret = OB_OP_NOT_ALLOW;
7934
    LOG_WARN("cannot alter resource pool zone list", K(ret),
7935
             "pool", *pool, K(zones_to_be_add));
7936
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "alter resource pool zone list with not empty unit");
7937
  } else if (OB_FAIL(do_add_pool_zone_list(pool, zone_list, zones_to_be_add))) {
7938
    LOG_WARN("fail to do add pool zone list", K(ret), "pool", *pool, K(zones_to_be_add));
7939
  } else {} // no more to do
7940
  return ret;
7941
}
7942

7943
int ObUnitManager::cal_to_be_add_pool_zone_list(
7944
    const common::ObIArray<common::ObZone> &prev_zone_list,
7945
    const common::ObIArray<common::ObZone> &cur_zone_list,
7946
    common::ObIArray<common::ObZone> &to_be_add_zones) const
7947
{
7948
  int ret = OB_SUCCESS;
7949
  if (OB_UNLIKELY(prev_zone_list.count() <= 0 || cur_zone_list.count() <= 0)) {
7950
    ret = OB_INVALID_ARGUMENT;
7951
    LOG_WARN("invalid argument", K(ret),
7952
             "prev zone list", prev_zone_list,
7953
             "cur zone list", cur_zone_list);
7954
  } else {
7955
    to_be_add_zones.reset();
7956
    //Check if the added zone exists
7957
    for (int64_t i = 0; OB_SUCC(ret) && i < cur_zone_list.count(); ++i) {
7958
      const common::ObZone &this_zone = cur_zone_list.at(i);
7959
      bool zone_exist = false;
7960
      if (OB_FAIL(zone_mgr_.check_zone_exist(this_zone, zone_exist))) {
7961
        LOG_WARN("failed to check zone exists", K(ret), K(this_zone));
7962
      } else if (!zone_exist) {
7963
        ret = OB_ZONE_INFO_NOT_EXIST;
7964
        LOG_WARN("zone not exists", K(ret), K(this_zone));
7965
      } else {
7966
        if (has_exist_in_array(prev_zone_list, this_zone)) {
7967
          //still exist, do nothing
7968
        } else if (has_exist_in_array(to_be_add_zones, this_zone)) {
7969
          //just add one time
7970
          ret = OB_OP_NOT_ALLOW;
7971
          LOG_USER_ERROR(OB_OP_NOT_ALLOW, "add repeat zone");
7972
          LOG_WARN("not allow add repeat zone", K(ret));
7973
        } else if (OB_FAIL(to_be_add_zones.push_back(this_zone))) {
7974
          LOG_WARN("fail to push back", K(ret));
7975
        }
7976
      }
7977
    }
7978
  }
7979
  return ret;
7980
}
7981

7982
int ObUnitManager::check_can_add_pool_zone_list_by_locality(
7983
    const share::ObResourcePool *pool,
7984
    const common::ObIArray<common::ObZone> &to_be_add_zones,
7985
    bool &can_add)
7986
{
7987
  int ret = OB_SUCCESS;
7988
  can_add = true;
7989
  common::ObArray<share::ObResourcePool *> *pool_list = NULL;
7990
  share::schema::ObSchemaGetterGuard schema_guard;
7991
  const share::schema::ObTenantSchema *tenant_schema = NULL;
7992
  if (OB_UNLIKELY(NULL == pool) || OB_UNLIKELY(to_be_add_zones.count() <= 0)) {
7993
    ret = OB_INVALID_ARGUMENT;
7994
    LOG_WARN("ivnalid argument", K(ret), KP(pool), K(to_be_add_zones));
7995
  } else if (! pool->is_granted_to_tenant()) {
7996
    can_add = true; // not in tenant, can add zone unit
7997
  } else if (OB_FAIL(get_pools_by_tenant_(pool->tenant_id_, pool_list))) {
7998
    LOG_WARN("fail to get pools by tenant", K(ret));
7999
  } else if (OB_ISNULL(schema_service_)) {
8000
    ret = OB_NOT_INIT;
8001
    LOG_WARN("schema service is null", K(schema_service_), KR(ret));
8002
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(pool->tenant_id_, schema_guard))) {
8003
    LOG_WARN("fail to get tenant schema guard", KR(ret), K(pool->tenant_id_));
8004
  } else if (OB_FAIL(schema_guard.get_tenant_info(pool->tenant_id_, tenant_schema))) {
8005
    LOG_WARN("fail to get tenant schema", K(ret), "tenant_id", pool->tenant_id_);
8006
  } else if (NULL == tenant_schema) {
8007
    ret = OB_SCHEMA_ERROR;
8008
    LOG_WARN("tenant schema is null", K(ret));
8009
  } else if (!tenant_schema->get_previous_locality_str().empty()) {//No locality changes are allowed at this time
8010
    can_add = false;
8011
    LOG_WARN("alter pool zone list is not allowed while locality modification",
8012
             "tenant_id", tenant_schema->get_tenant_id());
8013
  } else {} //nothing todo
8014
  return ret;
8015
}
8016

8017
int ObUnitManager::do_add_pool_zone_list(
8018
    share::ObResourcePool *pool,
8019
    const common::ObIArray<common::ObZone> &new_zone_list,
8020
    const common::ObIArray<common::ObZone> &to_be_add_zones)
8021
{
8022
  int ret = OB_SUCCESS;
8023
  common::ObMySQLTransaction trans;
8024
  share::ObResourcePool new_pool;
8025
  const char *module = "ALTER_RESOURCE_POOL_ZONE_LIST";
8026
  if (!check_inner_stat()) {
8027
    ret = OB_INNER_STAT_ERROR;
8028
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
8029
  } else if (OB_UNLIKELY(NULL == pool)
8030
             || OB_UNLIKELY(to_be_add_zones.count() <= 0)
8031
             || OB_UNLIKELY(new_zone_list.count() <= 0)) {
8032
    ret = OB_INVALID_ARGUMENT;
8033
    LOG_WARN("invalid argument", K(ret), KP(pool), K(to_be_add_zones), K(new_zone_list));
8034
  } else {
8035
    if (OB_FAIL(new_pool.assign(*pool))) {
8036
      LOG_WARN("fail to assign new pool", K(ret));
8037
    } else if (OB_FAIL(new_pool.zone_list_.assign(new_zone_list))) {
8038
      LOG_WARN("fail to assign new pool zone list", K(ret));
8039
    } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
8040
      LOG_WARN("start transaction failed", K(ret));
8041
    } else {
8042
      if (OB_FAIL(increase_units_in_zones_(trans, *pool, to_be_add_zones, module))) {
8043
        LOG_WARN("fail to add units in zones", K(module), KR(ret),
8044
                 "pool id", pool->resource_pool_id_, K(to_be_add_zones));
8045
      } else if (OB_FAIL(pool->zone_list_.assign(new_zone_list))) {
8046
        LOG_WARN("fail to update pool zone list in memory", K(ret));
8047
      } else if (OB_FAIL(ut_operator_.update_resource_pool(trans, new_pool, false/*need_check_conflict_with_clone*/))) {
8048
        LOG_WARN("fail to update resource pool", K(ret));
8049
      } else {} //no more to do
8050
      if (trans.is_started()) {
8051
        const bool commit = (OB_SUCC(ret));
8052
        int tmp_ret = OB_SUCCESS;
8053
        if (OB_SUCCESS != (tmp_ret = trans.end(commit))) {
8054
          LOG_WARN("fail to end trans", K(commit), K(tmp_ret));
8055
          ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
8056
        }
8057
      } else {}//nothing todo
8058
    }
8059
  }
8060
  return ret;
8061
}
8062

8063
// allocate new units on new added zones for resource pool
8064
int ObUnitManager::increase_units_in_zones_(
8065
    common::ObISQLClient &client,
8066
    share::ObResourcePool &pool,
8067
    const common::ObIArray<common::ObZone> &to_be_add_zones,
8068
    const char *module)
8069
{
8070
  int ret = OB_SUCCESS;
8071
  ObArray<uint64_t> new_unit_group_id_array;
8072
  const bool new_allocate_pool = false;   // whether to alloacate new resource pool
8073
  const int64_t new_unit_count = pool.unit_count_;  // new unit count on every zone
8074
  ObArray<ObAddr> new_servers;
8075

8076
  for (int64_t i = 0; OB_SUCC(ret) && i < to_be_add_zones.count(); ++i) {
8077
    bool zone_exist = false;
8078
    if (OB_FAIL(zone_mgr_.check_zone_exist(to_be_add_zones.at(i), zone_exist))) {
8079
      LOG_WARN("failed to check zone exists", K(ret), K(to_be_add_zones.at(i)));
8080
    } else if (!zone_exist) {
8081
      ret = OB_ZONE_INFO_NOT_EXIST;
8082
      LOG_WARN("zone not exists", K(ret), K(to_be_add_zones.at(i)));
8083
    }
8084
  }
8085

8086
  if (OB_FAIL(ret)) {
8087
  } else if (OB_FAIL(get_pool_unit_group_id_(pool, new_unit_group_id_array))) {
8088
    LOG_WARN("fail to get pool unit group id", KR(ret), K(pool));
8089
  } else if (OB_FAIL(allocate_pool_units_(client, pool, to_be_add_zones, &new_unit_group_id_array,
8090
      new_allocate_pool, new_unit_count, module, new_servers))) {
8091
    LOG_WARN("fail to allocate new units for new added zones", K(module), KR(ret), K(to_be_add_zones), K(pool),
8092
        K(new_unit_count), K(new_unit_group_id_array));
8093
  }
8094

8095
  return ret;
8096
}
8097

8098
int ObUnitManager::do_remove_pool_zone_list(
8099
    share::ObResourcePool *pool,
8100
    const common::ObIArray<common::ObZone> &new_zone_list,
8101
    const common::ObIArray<common::ObZone> &to_be_removed_zones)
8102
{
8103
  int ret = OB_SUCCESS;
8104
  common::ObMySQLTransaction trans;
8105
  share::ObResourcePool new_pool;
8106
  common::ObArray<common::ObZone> new_zone_list1;
8107
  if (!check_inner_stat()) {
8108
    ret = OB_INNER_STAT_ERROR;
8109
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
8110
  } else if (OB_UNLIKELY(NULL == pool)
8111
             || OB_UNLIKELY(to_be_removed_zones.count() <= 0)
8112
             || OB_UNLIKELY(new_zone_list.count() <= 0)) {
8113
    ret = OB_INVALID_ARGUMENT;
8114
    LOG_WARN("invalid argument", K(ret), KP(pool), K(to_be_removed_zones), K(new_zone_list));
8115
  } else {
8116
    if (OB_FAIL(new_pool.assign(*pool))) {
8117
      LOG_WARN("fail to assign new pool", K(ret));
8118
    } else if (OB_FAIL(new_pool.zone_list_.assign(new_zone_list))) {
8119
      LOG_WARN("fail to assign new pool zone list", K(ret));
8120
    } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
8121
      LOG_WARN("start transaction failed", K(ret));
8122
    } else {
8123
      if (OB_FAIL(ut_operator_.remove_units_in_zones(
8124
              trans, pool->resource_pool_id_, to_be_removed_zones))) {
8125
        LOG_WARN("fail to remove units in zones", K(ret),
8126
                 "pool id", pool->resource_pool_id_, K(to_be_removed_zones));
8127
      } else if (OB_FAIL(ut_operator_.update_resource_pool(trans, new_pool, false/*need_check_conflict_with_clone*/))) {
8128
        LOG_WARN("fail to update resource pool", K(ret));
8129
      } else {} // no more to do
8130
      if (trans.is_started()) {
8131
        const bool commit = (OB_SUCC(ret));
8132
        int tmp_ret = OB_SUCCESS;
8133
        if (OB_SUCCESS != (tmp_ret = trans.end(commit))) {
8134
          LOG_WARN("fail to end trans", K(commit), K(tmp_ret));
8135
          ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
8136
        }
8137
      }
8138
      if (OB_FAIL(ret)) {
8139
      } else if (OB_FAIL(delete_units_in_zones(pool->resource_pool_id_, to_be_removed_zones))) {
8140
        LOG_WARN("fail to delete units in zones", K(ret),
8141
                 "pool id", pool->resource_pool_id_, K(to_be_removed_zones));
8142
      } else if (OB_FAIL(pool->zone_list_.assign(new_zone_list))) {
8143
        LOG_WARN("fail to update pool zone list in memory", K(ret));
8144
      } else {} // no more to do
8145
    }
8146
  }
8147
  return ret;
8148
}
8149

8150
int ObUnitManager::cal_to_be_removed_pool_zone_list(
8151
    const common::ObIArray<common::ObZone> &prev_zone_list,
8152
    const common::ObIArray<common::ObZone> &cur_zone_list,
8153
    common::ObIArray<common::ObZone> &to_be_removed_zones) const
8154
{
8155
  int ret = OB_SUCCESS;
8156
  if (OB_UNLIKELY(prev_zone_list.count() <= 0 || cur_zone_list.count() <= 0)) {
8157
    ret = OB_INVALID_ARGUMENT;
8158
    LOG_WARN("invalid argument", K(ret),
8159
             "prev zone list", prev_zone_list,
8160
             "cur zone list", cur_zone_list);
8161
  } else {
8162
    to_be_removed_zones.reset();
8163
    // Each zone in cur_zone_list must be included in prev_zone_list
8164
    for (int64_t i = 0; OB_SUCC(ret) && i < cur_zone_list.count(); ++i) {
8165
      const common::ObZone &this_zone = cur_zone_list.at(i);
8166
      if (!has_exist_in_array(prev_zone_list, this_zone)) {
8167
        ret = OB_NOT_SUPPORTED;
8168
        LOG_USER_ERROR(OB_NOT_SUPPORTED, "alter resource pool zone list with a new zone");
8169
        LOG_WARN("alter resource pool zone list with a new zone is not supported", K(ret));
8170
      } else {} // good
8171
    }
8172
    for (int64_t i = 0; OB_SUCC(ret) && i < prev_zone_list.count(); ++i) {
8173
      const common::ObZone &this_zone = prev_zone_list.at(i);
8174
      if (has_exist_in_array(cur_zone_list, this_zone)) {
8175
        // still exist, do nothing
8176
      } else if (OB_FAIL(to_be_removed_zones.push_back(this_zone))) {
8177
        LOG_WARN("fail to push back", K(ret));
8178
      } else {} // no more to do
8179
    }
8180
  }
8181
  return ret;
8182
}
8183

8184
int ObUnitManager::check_full_resource_pool_memory_condition(
8185
    const common::ObIArray<share::ObResourcePool *> &pools,
8186
    const int64_t memory_size) const
8187
{
8188
  int ret = OB_SUCCESS;
8189
  for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
8190
    const share::ObResourcePool *pool = pools.at(i);
8191
    if (OB_UNLIKELY(nullptr == pool)) {
8192
      ret = OB_ERR_UNEXPECTED;
8193
      LOG_WARN("pool ptr is null", K(ret), KP(pool));
8194
    } else if (REPLICA_TYPE_FULL != pool->replica_type_) {
8195
      // bypass
8196
    } else if (memory_size < GCONF.__min_full_resource_pool_memory) {
8197
      ret = OB_NOT_SUPPORTED;
8198
      LOG_WARN("full resource pool min memory illegal", KR(ret), K(memory_size),
8199
          "__min_full_resource_pool_memory", (int64_t)GCONF.__min_full_resource_pool_memory);
8200
      LOG_USER_ERROR(OB_NOT_SUPPORTED, "unit MEMORY_SIZE less than __min_full_resource_pool_memory");
8201
    }
8202
  }
8203
  return ret;
8204
}
8205

8206
ERRSIM_POINT_DEF(ERRSIM_USE_DUMMY_SERVER);
8207
int ObUnitManager::check_expand_resource_(
8208
    const char *module,
8209
    const common::ObIArray<share::ObResourcePool  *> &pools,
8210
    const share::ObUnitResource &old_resource,
8211
    const share::ObUnitResource &new_resource) const
8212
{
8213
  int ret = OB_SUCCESS;
8214
  common::hash::ObHashMap<ObAddr, int64_t> server_ref_count_map;
8215
  ObString err_str;
8216
  AlterResourceErr err_index = ALT_ERR;
8217
  int temp_ret = OB_SUCCESS;
8218
  if (!check_inner_stat()) {
8219
    ret = OB_INNER_STAT_ERROR;
8220
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), K(ret));
8221
  } else if (pools.count() <= 0) {
8222
    ret = OB_INVALID_ARGUMENT;
8223
    LOG_WARN("pools is empty", K(pools), K(ret));
8224
  } else if (!old_resource.is_valid() || !new_resource.is_valid()) {
8225
    ret = OB_INVALID_ARGUMENT;
8226
    LOG_WARN("invalid old_resource or invalid new_resource", K(old_resource), K(new_resource), K(ret));
8227
  } else if (!new_resource.has_expanded_resource_than(old_resource)) {
8228
    // skip, no need to check
8229
    LOG_INFO("new unit_resource has no expanded resource, skip check_expand_resource",
8230
             KR(ret), K(old_resource), K(new_resource));
8231
  } else if (OB_FAIL(server_ref_count_map.create(
8232
      SERVER_REF_COUNT_MAP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_SERVER_REF_COUNT_MAP))) {
8233
    LOG_WARN("pool_unit_map_ create failed",
8234
        "bucket_num", static_cast<int64_t>(SERVER_REF_COUNT_MAP_BUCKET_NUM), K(ret));
8235
  } else if (OB_FAIL(get_pools_servers(pools, server_ref_count_map))) {
8236
    LOG_WARN("get pools server failed", K(pools), K(ret));
8237
  } else {
8238
    bool can_expand = true;
8239
    const ObUnitResource delta = new_resource - old_resource;
8240
    ObUnitResource expand_resource;
8241
    ObServerInfoInTable server_info;
8242
    _LOG_INFO("[%s] check_expand_resource begin. old=%s, new=%s, delta=%s", module,
8243
        to_cstring(old_resource), to_cstring(new_resource), to_cstring(delta));
8244

8245
    FOREACH_X(iter, server_ref_count_map, OB_SUCCESS == ret) {
8246
      expand_resource = delta * (iter->second);
8247
      const ObAddr &server = iter->first;
8248
      server_info.reset();
8249
      _LOG_INFO("[%s] check_expand_resource. svr=%s, pools=%ld, expand_resource=%s", module,
8250
          to_cstring(server), iter->second, to_cstring(expand_resource));
8251
      if (OB_FAIL(SVR_TRACER.get_server_info(server, server_info))) {
8252
        LOG_WARN("fail to get server_info", KR(ret), K(server));
8253
      } else if (OB_UNLIKELY(!server_info.is_active())) {
8254
        ret = OB_OP_NOT_ALLOW;
8255
        LOG_WARN("server is inactive, cannot check_expand_resource", KR(ret), K(server), K(server_info));
8256
        const int64_t ERR_MSG_LEN = 256;
8257
        char err_msg[ERR_MSG_LEN] = {'\0'};
8258
        int tmp_ret = OB_SUCCESS;
8259
        int64_t pos = 0;
8260
        if (OB_TMP_FAIL(databuff_printf(err_msg, ERR_MSG_LEN, pos,
8261
              "Server %s is inactive, expanding resource",
8262
              to_cstring(OB_SUCCESS != ERRSIM_USE_DUMMY_SERVER ? ObAddr() : server)))) {
8263
          LOG_WARN("format err_msg failed", KR(tmp_ret), KR(ret));
8264
        } else {
8265
          LOG_USER_ERROR(OB_OP_NOT_ALLOW, err_msg);
8266
        }
8267
      } else if (OB_FAIL(check_expand_resource_(server_info, expand_resource, can_expand, err_index))) {
8268
        LOG_WARN("check expand resource failed", KR(ret), K(server_info));
8269
      } else if (!can_expand) {
8270
        const ObZone &zone = server_info.get_zone();
8271
        LOG_USER_ERROR(OB_MACHINE_RESOURCE_NOT_ENOUGH, to_cstring(zone), to_cstring(server),
8272
            alter_resource_err_to_str(err_index));
8273
        // return ERROR
8274
        ret = OB_MACHINE_RESOURCE_NOT_ENOUGH;
8275
      }
8276
    }
8277
  }
8278
  return ret;
8279
}
8280

8281
int ObUnitManager::check_expand_resource_(
8282
    const share::ObServerInfoInTable &server_info,
8283
    const ObUnitResource &expand_resource,
8284
    bool &can_expand,
8285
    AlterResourceErr &err_index) const
8286
{
8287
  int ret = OB_SUCCESS;
8288
  double hard_limit = 0;
8289
  bool can_hold_unit = false;
8290
  can_expand = true;
8291
  obrpc::ObGetServerResourceInfoResult report_server_resource_info;
8292
  // some item of expand_resource may be negative, so we don't check expand_resource here
8293
  if (!check_inner_stat()) {
8294
    ret = OB_INNER_STAT_ERROR;
8295
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), K(ret));
8296
  } else if (!server_info.is_valid()) {
8297
    ret = OB_INVALID_ARGUMENT;
8298
    LOG_WARN("invalid server_info", KR(ret), K(server_info));
8299
  } else if (OB_FAIL(get_hard_limit(hard_limit))) {
8300
    LOG_WARN("get_hard_limit failed", K(ret));
8301
  } else if (OB_FAIL(get_server_resource_info_via_rpc(server_info, report_server_resource_info))) {
8302
    LOG_WARN("get_server_resource_info_via_rpc failed", KR(ret), K(server_info));
8303
  } else if (OB_FAIL(have_enough_resource(
8304
      report_server_resource_info,
8305
      expand_resource,
8306
      hard_limit,
8307
      can_hold_unit,
8308
      err_index))) {
8309
    LOG_WARN("fail to check have enough resource", KR(ret), K(hard_limit),
8310
        K(report_server_resource_info), K(expand_resource));
8311
  } else if (!can_hold_unit) {
8312
    can_expand = false;
8313
    // don't need to set ret
8314
    LOG_WARN("find server can't hold expanded resource", KR(ret), K(server_info),
8315
        K(report_server_resource_info), K(expand_resource));
8316
  } else {
8317
    can_expand = true;
8318
  }
8319
  return ret;
8320
}
8321

8322
int ObUnitManager::check_shrink_resource_(const ObIArray<share::ObResourcePool *> &pools,
8323
                                          const ObUnitResource &resource,
8324
                                          const ObUnitResource &new_resource) const
8325
{
8326
  int ret = OB_SUCCESS;
8327
  if (!check_inner_stat()) {
8328
    ret = OB_INNER_STAT_ERROR;
8329
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), K(ret));
8330
  } else if (pools.count() <= 0) {
8331
    ret = OB_INVALID_ARGUMENT;
8332
    LOG_WARN("pools is empty", K(pools), K(ret));
8333
  } else if (!resource.is_valid() || !new_resource.is_valid()) {
8334
    ret = OB_INVALID_ARGUMENT;
8335
    LOG_WARN("invalid resource or invalid new_resource", K(resource), K(new_resource), K(ret));
8336
  } else if (!new_resource.has_shrunk_resource_than(resource)) {
8337
    // skip, no need to check
8338
    LOG_INFO("new unit_resource has no shrunk resource, skip check_shrink_resource",
8339
             KR(ret), K(resource), K(new_resource));
8340
  } else {
8341
    for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
8342
      const share::ObResourcePool *pool = pools.at(i);
8343
      if (OB_UNLIKELY(NULL == pool)) {
8344
        ret = OB_ERR_UNEXPECTED;
8345
        LOG_WARN("pool ptr is null", K(ret));
8346
      } else if (OB_FAIL(check_shrink_resource_(*pool, resource, new_resource))) {
8347
        LOG_WARN("fail to check shrink resource", KR(ret));
8348
      } else {} // no more to do
8349
    }
8350
  }
8351
  return ret;
8352
}
8353

8354
int ObUnitManager::check_shrink_resource_(const share::ObResourcePool &pool,
8355
                                          const ObUnitResource &resource,
8356
                                          const ObUnitResource &new_resource) const
8357
{
8358
  int ret = OB_SUCCESS;
8359
  if (!check_inner_stat()) {
8360
    ret = OB_INNER_STAT_ERROR;
8361
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), K(ret));
8362
  } else if (!pool.is_valid() || !resource.is_valid() || !new_resource.is_valid()) {
8363
    ret = OB_INVALID_ARGUMENT;
8364
    LOG_WARN("invalid pool or invalid resource or invalid new_resource", K(pool),
8365
        K(resource), K(new_resource), K(ret));
8366
  } else {
8367
    if (new_resource.max_cpu() < resource.max_cpu()) {
8368
      // cpu don't need check
8369
    }
8370

8371
    if (new_resource.memory_size() < resource.memory_size()) {
8372
      if (!pool.is_granted_to_tenant()) {
8373
        // do nothing
8374
      } else if (OB_FAIL(check_shrink_memory(pool,resource.memory_size(), new_resource.memory_size()))) {
8375
        LOG_WARN("check_shrink_memory failed", "tenant_id", pool.tenant_id_,
8376
            "old_resource_mem", resource.memory_size(),
8377
            "new_resource_mem", new_resource.memory_size(), KR(ret));
8378
      }
8379
    }
8380

8381
    if (new_resource.log_disk_size() < resource.log_disk_size()) {
8382
      // log disk don't need check.
8383
    } 
8384
  }
8385
  return ret;
8386
}
8387

8388
int ObUnitManager::check_shrink_memory(
8389
    const share::ObResourcePool &pool,
8390
    const int64_t old_memory,
8391
    const int64_t new_memory) const
8392
{
8393
  int ret = OB_SUCCESS;
8394
  ObArray<ObUnit *> *units = NULL;
8395
  ObArray<common::ObAddr> unit_servers;
8396

8397
  if (!check_inner_stat()) {
8398
    ret = OB_INNER_STAT_ERROR;
8399
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), K(ret));
8400
  } else if (! pool.is_granted_to_tenant()) {
8401
    ret = OB_INVALID_ARGUMENT;
8402
    LOG_WARN("pool is not granted to valid tenant, invalid pool", KR(ret), K(pool));
8403
  } else if (old_memory <= 0 || new_memory <= 0 || new_memory >= old_memory) {
8404
    ret = OB_INVALID_ARGUMENT;
8405
    LOG_WARN("invalid old_memory or invalid new_memory",
8406
        K(old_memory), K(new_memory), K(ret));
8407
  } else if (OB_FAIL(get_units_by_pool(pool.resource_pool_id_, units))) {
8408
    LOG_WARN("fail to get units by pool", K(ret));
8409
  } else if (OB_UNLIKELY(NULL == units)) {
8410
    ret = OB_ERR_UNEXPECTED;
8411
    LOG_WARN("units ptr is null", K(ret));
8412
  } else {
8413
    for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
8414
      ObUnit *unit = units->at(i);
8415
      if (OB_UNLIKELY(NULL == unit)) {
8416
        ret = OB_ERR_UNEXPECTED;
8417
        LOG_WARN("unit ptr is null", K(ret));
8418
      } else if (OB_FAIL(unit_servers.push_back(unit->server_))) {
8419
        LOG_WARN("fail to push back", K(ret));
8420
      } else if (!unit->migrate_from_server_.is_valid()) {
8421
        // this unit is not in migrating
8422
      } else if (OB_FAIL(unit_servers.push_back(unit->migrate_from_server_))) {
8423
        LOG_WARN("fail to push back", K(ret));
8424
      }
8425
    }
8426
    const double max_used_ratio = 0.9;
8427
    const double shrink_ratio = static_cast<double>(new_memory)
8428
        / static_cast<double>(old_memory);
8429
    int64_t max_used_memory = 0;
8430
    ObArray<ObTenantMemstoreInfoOperator::TenantServerMemInfo> mem_infos;
8431
    ObTenantMemstoreInfoOperator mem_info_operator(*proxy_);
8432
    if (OB_FAIL(ret)) {
8433
      // failed
8434
    } else if (OB_FAIL(mem_info_operator.get(pool.tenant_id_, unit_servers, mem_infos))) {
8435
      LOG_WARN("mem_info_operator get failed", K(ret));
8436
    } else {
8437
      FOREACH_CNT_X(mem_info, mem_infos, OB_SUCCESS == ret) {
8438
        max_used_memory = static_cast<int64_t>(static_cast<double>(mem_info->memstore_limit_)
8439
            * shrink_ratio * max_used_ratio);
8440
        if (mem_info->total_memstore_used_ > max_used_memory) {
8441
          ret = OB_NOT_SUPPORTED;
8442
          LOG_USER_ERROR(OB_NOT_SUPPORTED, "requested memory over 90 percent of total available memory");
8443
          LOG_WARN("new memory will cause memory use percentage over ninety percentage",
8444
              "mem_info", *mem_info, K(old_memory), K(new_memory),
8445
              K(max_used_ratio), K(max_used_memory), K(ret));
8446
        }
8447
      }
8448
    }
8449
  }
8450
  return ret;
8451
}
8452

8453
int ObUnitManager::change_pool_config(share::ObResourcePool *pool, ObUnitConfig *config,
8454
                                      ObUnitConfig *new_config)
8455
{
8456
  int ret = OB_SUCCESS;
8457
  if (!check_inner_stat()) {
8458
    ret = OB_INNER_STAT_ERROR;
8459
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), K(ret));
8460
  } else if (NULL == pool || NULL == config || NULL == new_config) {
8461
    ret = OB_INVALID_ARGUMENT;
8462
    LOG_WARN("invalid argument", KP(pool), KP(config), KP(new_config), K(ret));
8463
  } else {
8464
    share::ObResourcePool new_pool;
8465
    if (OB_FAIL(new_pool.assign(*pool))) {
8466
      LOG_WARN("failed to assign new_pool", K(ret));
8467
    }
8468
    if (OB_SUCC(ret)) {
8469
      new_pool.unit_config_id_ = new_config->unit_config_id();
8470
      if (OB_FAIL(ut_operator_.update_resource_pool(*proxy_, new_pool, false/*need_check_conflict_with_clone*/))) {
8471
        LOG_WARN("ut_operator_ update_resource_pool failed", K(new_pool), K(ret));
8472
      } else if (OB_FAIL(dec_config_ref_count(config->unit_config_id()))) {
8473
        LOG_WARN("dec_config_ref_count failed", "unit_config_id",
8474
            config->unit_config_id(), K(ret));
8475
      } else if (OB_FAIL(inc_config_ref_count(new_config->unit_config_id()))) {
8476
        LOG_WARN("inc_config_ref_count failed", "unit_config_id",
8477
            new_config->unit_config_id(), K(ret));
8478
      } else if (OB_FAIL(delete_config_pool(config->unit_config_id(), pool))) {
8479
        LOG_WARN("delete config pool failed", "config id", config->unit_config_id(), K(ret));
8480
      } else if (OB_FAIL(insert_config_pool(new_config->unit_config_id(), pool))) {
8481
        LOG_WARN("insert config pool failed", "config id", new_config->unit_config_id(), K(ret));
8482
      } else if (OB_FAIL(update_pool_load(pool, new_config))) {
8483
        LOG_WARN("update resource pool load failed", K(ret), "resource_pool", *pool,
8484
            "unit_config", *new_config);
8485
      } else {
8486
        pool->unit_config_id_ = new_config->unit_config_id();
8487
      }
8488
    }
8489
  }
8490
  return ret;
8491
}
8492

8493
// The zones of multiple pools have no intersection
8494
// 14x new semantics. If it is the source_pool used to store the copy of L, it can be compared
8495
int ObUnitManager::check_pool_intersect_(
8496
    const uint64_t tenant_id,
8497
    const ObIArray<ObResourcePoolName> &pool_names,
8498
    bool &intersect)
8499
{
8500
  int ret = OB_SUCCESS;
8501
  ObSEArray<ObString, OB_DEFAULT_REPLICA_NUM> zones;
8502
  common::ObArray<share::ObResourcePool *> *pools = NULL;;
8503
  intersect = false;
8504
  if (!check_inner_stat()) {
8505
    ret = OB_INNER_STAT_ERROR;
8506
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), KR(ret));
8507
  } else if (pool_names.count() <= 0 || !is_valid_tenant_id(tenant_id)) {
8508
    ret = OB_INVALID_ARGUMENT;
8509
    LOG_WARN("pool_names is empty", K(pool_names), K(tenant_id), KR(ret));
8510
  } else {
8511
    FOREACH_CNT_X(pool_name, pool_names, OB_SUCCESS == ret && !intersect) {
8512
      share::ObResourcePool *pool = NULL;
8513
      if (OB_FAIL(inner_get_resource_pool_by_name(*pool_name, pool))) {
8514
        LOG_WARN("get resource pool by name failed", "pool_name", *pool_name, KR(ret));
8515
      } else if (NULL == pool) {
8516
        ret = OB_ERR_UNEXPECTED;
8517
        LOG_WARN("pool is null", KP(pool), KR(ret));
8518
      } else {
8519
        FOREACH_CNT_X(zone, pool->zone_list_, OB_SUCCESS == ret && !intersect) {
8520
          if (NULL == zone) {
8521
            ret = OB_ERR_UNEXPECTED;
8522
            LOG_WARN("unit is null", KR(ret));
8523
          } else {
8524
            ObString zone_str;
8525
            zone_str.assign_ptr(zone->ptr(), static_cast<int32_t>(zone->size()));
8526
            if (has_exist_in_array(zones, zone_str)) {
8527
              intersect = true;
8528
            } else if (OB_FAIL(zones.push_back(zone_str))) {
8529
              LOG_WARN("push_back failed", KR(ret));
8530
            }
8531
          }
8532
        }  // end foreach zone
8533
      }
8534
    }  // end foreach pool
8535
    if (OB_FAIL(ret)) {
8536
    } else if (intersect) {
8537
    } else if (OB_FAIL(get_pools_by_tenant_(tenant_id, pools))) {
8538
      if (OB_ENTRY_NOT_EXIST == ret) {
8539
        // a new tenant, without resource pool already granted
8540
        ret = OB_SUCCESS;
8541
      } else {
8542
        LOG_WARN("fail to get pools by tenant", KR(ret), K(tenant_id));
8543
      }
8544
    } else if (OB_UNLIKELY(NULL == pools)) {
8545
      ret = OB_ERR_UNEXPECTED;
8546
      LOG_WARN("pools is null", KR(ret), KP(pools));
8547
    } else {
8548
      for (int64_t i = 0; !intersect && OB_SUCC(ret) && i < pools->count(); ++i) {
8549
        const share::ObResourcePool *pool = pools->at(i);
8550
        if (OB_UNLIKELY(NULL == pool)) {
8551
          ret = OB_ERR_UNEXPECTED;
8552
          LOG_WARN("pool ptr is null", KR(ret), KP(pool));
8553
        } else {
8554
          for (int64_t j = 0; !intersect && OB_SUCC(ret) && j < zones.count(); ++j) {
8555
            common::ObZone zone;
8556
            if (OB_FAIL(zone.assign(zones.at(j).ptr()))) {
8557
              LOG_WARN("fail to assign zone", KR(ret));
8558
            } else if (has_exist_in_array(pool->zone_list_, zone)) {
8559
              intersect = true;
8560
            } else {} // good
8561
          }
8562
        }
8563
      }
8564
    }
8565
  }
8566
  return ret;
8567
}
8568

8569
int ObUnitManager::check_pool_ownership_(const uint64_t tenant_id,
8570
                                        const common::ObIArray<share::ObResourcePoolName> &pool_names,
8571
                                        const bool grant)
8572
{
8573
  int ret = OB_SUCCESS;
8574
  share::ObResourcePool *pool = NULL;
8575
  for (int64_t i = 0; OB_SUCC(ret) && i < pool_names.count(); ++i) {
8576
    if (OB_FAIL(inner_get_resource_pool_by_name(pool_names.at(i), pool))) {
8577
      LOG_WARN("get resource pool by name failed", "pool_name", pool_names.at(i), KR(ret));
8578
    } else if (OB_ISNULL(pool)) {
8579
      ret = OB_ERR_UNEXPECTED;
8580
      LOG_WARN("pool is null", KP(pool), KR(ret));
8581
    }
8582
    if (OB_FAIL(ret)) {
8583
    } else if (grant) {
8584
      if (pool->is_granted_to_tenant()) {
8585
        ret = OB_RESOURCE_POOL_ALREADY_GRANTED;
8586
        LOG_USER_ERROR(OB_RESOURCE_POOL_ALREADY_GRANTED, to_cstring(pool_names.at(i)));
8587
        LOG_WARN("pool has already granted to other tenant, can't grant again",
8588
                  KR(ret), K(tenant_id), "pool", *pool);
8589
      } else {/*good*/}
8590
    } else {
8591
      if (!pool->is_granted_to_tenant()) {
8592
        ret = OB_ERR_UNEXPECTED;
8593
        LOG_WARN("find pool not granted to any tenant, can not revoke",
8594
            "pool", *pool, K(tenant_id), KR(ret));
8595
      } else if (pool->tenant_id_ != tenant_id) {
8596
        ret = OB_ERR_UNEXPECTED;
8597
        LOG_WARN("find pool already granted to other tenant, can not revoke",
8598
            "pool", *pool, K(tenant_id), KR(ret));
8599
      } else {/*good*/}
8600
    }
8601
  }
8602
  return ret;
8603
}
8604

8605
int ObUnitManager::construct_pool_units_to_grant_(
8606
    ObMySQLTransaction &trans,
8607
    const uint64_t tenant_id,
8608
    const share::ObResourcePool &new_pool,
8609
    const common::ObIArray<share::ObUnit *> &zone_sorted_unit_array,
8610
    const common::ObIArray<uint64_t> &new_ug_ids,
8611
    const lib::Worker::CompatMode &compat_mode,
8612
    ObNotifyTenantServerResourceProxy &notify_proxy,
8613
    const uint64_t source_tenant_id,
8614
    ObIArray<share::ObUnit> &pool_units,
8615
    const bool check_data_version)
8616
{
8617
  int ret = OB_SUCCESS;
8618
  pool_units.reset();
8619
  common::ObArray<share::ObUnitInfo> source_units;
8620
  common::ObArray<uint64_t> source_ug_ids;
8621
  if (!check_inner_stat()) {
8622
    ret = OB_INNER_STAT_ERROR;
8623
    LOG_WARN("check_inner_stat failed", KR(ret), K_(inited), K_(loaded));
8624
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))
8625
             || OB_UNLIKELY(0 >= zone_sorted_unit_array.count())
8626
             || OB_UNLIKELY(0 >= new_ug_ids.count())) {
8627
    ret = OB_INVALID_ARGUMENT;
8628
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(zone_sorted_unit_array),
8629
             K(new_ug_ids));
8630
  } else {
8631
    if (OB_INVALID_TENANT_ID != source_tenant_id) {
8632
      if (OB_FAIL(inner_get_all_unit_infos_by_tenant_(source_tenant_id, source_units))) {
8633
        LOG_WARN("fail to get source units of source tenant", KR(ret), K(source_tenant_id));
8634
      } else if (OB_FAIL(construct_source_unit_group_ids_(source_units, source_ug_ids))) {
8635
        LOG_WARN("fail to construct source unit group ids", KR(ret), K(source_units));
8636
      }
8637
    }
8638
    ObUnit new_unit;
8639
    for (int64_t j = 0; OB_SUCC(ret) && j < zone_sorted_unit_array.count(); ++j) {
8640
      ObUnit *unit = zone_sorted_unit_array.at(j);
8641
      uint64_t unit_group_id_to_set = 0;
8642
      new_unit.reset();
8643
      if (OB_ISNULL(unit)) {
8644
        ret = OB_ERR_UNEXPECTED;
8645
        LOG_WARN("unit ptr is null", KR(ret));
8646
      } else if (OB_FAIL(try_notify_tenant_server_unit_resource_(
8647
                             tenant_id, false /*is_delete*/, notify_proxy,
8648
                             new_pool.unit_config_id_, compat_mode, *unit,
8649
                             false/*if_not_grant*/, false/*skip_offline_server*/,
8650
                             check_data_version))) {
8651
        LOG_WARN("fail to try notify server unit resource", KR(ret),
8652
                 K(tenant_id), K(compat_mode), KPC(unit));
8653
      } else if (FALSE_IT(new_unit = *unit)) {
8654
        // shall never be here
8655
      } else if (OB_FAIL(construct_unit_group_id_for_unit_(
8656
                             source_tenant_id,
8657
                             source_units,
8658
                             source_ug_ids,
8659
                             *unit,
8660
                             j/*unit_index*/,
8661
                             new_pool.unit_count_,
8662
                             new_ug_ids,
8663
                             unit_group_id_to_set))) {
8664
        LOG_WARN("fail to construct unit group id for unit", KR(ret), K(source_tenant_id),
8665
                 K(source_units), K(source_ug_ids), KPC(unit), K(j), K(new_pool), K(new_ug_ids));
8666
      } else if (FALSE_IT(new_unit.unit_group_id_ = unit_group_id_to_set)) {
8667
        // shall never be here
8668
      } else if (OB_FAIL(ut_operator_.update_unit(trans, new_unit, false/*need_check_conflict_with_clone*/))) {
8669
        LOG_WARN("fail to update unit", KR(ret), K(new_unit));
8670
      } else if (OB_FAIL(pool_units.push_back(*unit))) {
8671
        LOG_WARN("fail to push an element into pool_units", KR(ret), KPC(unit));
8672
      }
8673
    }
8674
  }
8675
  return ret;
8676
}
8677

8678
int ObUnitManager::construct_source_unit_group_ids_(
8679
    const common::ObIArray<share::ObUnitInfo> &source_units,
8680
    common::ObIArray<uint64_t> &source_unit_group_ids)
8681
{
8682
  int ret = OB_SUCCESS;
8683
  source_unit_group_ids.reset();
8684
  if (OB_UNLIKELY(0 == source_units.count())) {
8685
    ret = OB_INVALID_ARGUMENT;
8686
    LOG_WARN("invalid source units", KR(ret), K(source_units));
8687
  } else {
8688
    for (int64_t i = 0; OB_SUCC(ret) && i < source_units.count(); ++i) {
8689
      uint64_t source_unit_group_id = 0;
8690
      if (!source_units.at(i).unit_.is_active_status()) {
8691
        // we can ensure all unit belongs to source tenant is active status, so it is unexpected to find an inactive unit
8692
        ret = OB_STATE_NOT_MATCH;
8693
        LOG_WARN("unit status is not expected", KR(ret), K(source_units), K(i));
8694
      } else {
8695
        source_unit_group_id = source_units.at(i).unit_.unit_group_id_;
8696
        if (!has_exist_in_array(source_unit_group_ids, source_unit_group_id)) {
8697
          if (OB_FAIL(source_unit_group_ids.push_back(source_unit_group_id))) {
8698
            LOG_WARN("fail to add source unit group id into array", KR(ret), K(source_unit_group_id));
8699
          }
8700
        }
8701
      }
8702
    }
8703
  }
8704
  return ret;
8705
}
8706

8707
int ObUnitManager::construct_unit_group_id_for_unit_(
8708
    const uint64_t source_tenant_id,
8709
    const common::ObIArray<share::ObUnitInfo> &source_units,
8710
    const common::ObIArray<uint64_t> &source_unit_group_ids,
8711
    const ObUnit &target_unit,
8712
    const int64_t unit_index,
8713
    const int64_t unit_num,
8714
    const common::ObIArray<uint64_t> &new_ug_ids,
8715
    uint64_t &unit_group_id)
8716
{
8717
  int ret = OB_SUCCESS;
8718
  unit_group_id = OB_INVALID_ID;
8719
  common::ObArray<uint64_t> sorted_new_ug_ids;
8720
  common::ObArray<uint64_t> sorted_source_ug_ids;
8721

8722
  if (!check_inner_stat()) {
8723
    ret = OB_INNER_STAT_ERROR;
8724
    LOG_WARN("check_inner_stat failed", KR(ret), K_(inited), K_(loaded));
8725
  } else if (OB_UNLIKELY(0 >= unit_num)
8726
             || OB_UNLIKELY(0 > unit_index)
8727
             || OB_UNLIKELY(new_ug_ids.count() != unit_num)
8728
             || OB_UNLIKELY(0 >= new_ug_ids.count())
8729
             || OB_UNLIKELY((source_unit_group_ids.count() != new_ug_ids.count() && 0!= source_unit_group_ids.count()))) {
8730
    ret = OB_INVALID_ARGUMENT;
8731
    LOG_WARN("invalid argument", KR(ret), K(unit_index), K(source_units), K(source_unit_group_ids),
8732
             K(unit_num), K(new_ug_ids));
8733
  } else if (OB_INVALID_TENANT_ID == source_tenant_id) {
8734
    // for tenant not clone, just allocate new unit group ids by order
8735
    unit_group_id = new_ug_ids.at(unit_index % (unit_num));
8736
  } else {
8737
    // for clone tenant, we have to choose new unit group id for this unit
8738
    if (OB_FAIL(sorted_new_ug_ids.assign(new_ug_ids))) {
8739
      LOG_WARN("fail to assign unit group ids", KR(ret), K(new_ug_ids));
8740
    } else if (OB_FAIL(sorted_source_ug_ids.assign(source_unit_group_ids))) {
8741
      LOG_WARN("fail to assign source unit group ids", KR(ret), K(source_unit_group_ids));
8742
    } else {
8743
      // construct sorted unit group ids first
8744
      std::sort(sorted_new_ug_ids.begin(), sorted_new_ug_ids.end());
8745
      std::sort(sorted_source_ug_ids.begin(), sorted_source_ug_ids.end());
8746
      // for clone tenant, we have to construct unit_group_id for its units
8747
      // the rule to allocate unit_group_id
8748
      //
8749
      // resume source tenant has source_units like below:
8750
      //     ========================================================
8751
      //     | server_addr |  source_unit_id | source_unit_group_id |
8752
      //     ========================================================
8753
      //     |   server A  |       1001      |         ug1          |
8754
      //     |------------------------------------------------------|
8755
      //     |   server B  |       1002      |         ug2          |
8756
      //     |------------------------------------------------------|
8757
      //     |   server C  |       1003      |         ug3          |
8758
      //     ========================================================
8759
      //
8760
      //  And we have 3 new unit group ids belongs to clone tenant, mark them as new_ug4, new_ug5, new_ug6.
8761
      //  And assume unit id for clone tenant is 1004(server B), 1005(server A), 1006(server C).
8762
      //  And assume ug1 > ug3 > ug2
8763
      //  And assume new_ug5 > new_ug6 > new_ug4
8764
      //
8765
      //  The logic to construct unit_group_id for unit(1004) is firstly to get the unit with same server(B), which is unit 1002.
8766
      //  Then caculate the order of source_unit_group_id(which is ug2) belongs to unit 1002 during all source_group_ids.(The smallest one)
8767
      //  Then get the smallest new unit_group_id which is new_ug4.
8768
      //  After all, the unit group id constructed for unit 1004 is new_ug4.
8769
      //
8770
      //  ATTENTION:
8771
      //    The main logic here is to gurantee a mapping relationship between source_ug_ids and new_ug_ids by comparing the order.
8772
      const common::ObAddr target_server = target_unit.server_;
8773
      uint64_t source_unit_group_id = 0;
8774
      for (int64_t i = 0; OB_SUCC(ret) && i < source_units.count(); ++i) {
8775
        if (target_server == source_units.at(i).unit_.server_) {
8776
          source_unit_group_id = source_units.at(i).unit_.unit_group_id_;
8777
          int64_t index = OB_INVALID_INDEX_INT64;
8778
          if (has_exist_in_array(sorted_source_ug_ids, source_unit_group_id, &index)) {
8779
            unit_group_id = sorted_new_ug_ids.at(index);
8780
            break;
8781
          } else {
8782
            ret = OB_ENTRY_NOT_EXIST;
8783
            LOG_WARN("can not find unit group id from source ug ids", KR(ret),
8784
                     K(sorted_source_ug_ids), K(source_unit_group_id));
8785
          }
8786
        }
8787
      }
8788
    }
8789
  }
8790
  if (OB_FAIL(ret)) {
8791
  } else if (OB_UNLIKELY(OB_INVALID_ID == unit_group_id)) {
8792
    ret = OB_ENTRY_NOT_EXIST;
8793
    LOG_WARN("can not find a valid unit group id", KR(ret), K(source_tenant_id), K(unit_group_id),
8794
             K(source_units), K(target_unit), K(sorted_source_ug_ids), K(sorted_new_ug_ids));
8795
  }
8796
  return ret;
8797
}
8798

8799
int ObUnitManager::do_grant_pools_(
8800
    ObMySQLTransaction &trans,
8801
    const common::ObIArray<uint64_t> &new_ug_ids,
8802
    const lib::Worker::CompatMode compat_mode,
8803
    const ObIArray<ObResourcePoolName> &pool_names,
8804
    const uint64_t tenant_id,
8805
    const bool is_bootstrap,
8806
    const uint64_t source_tenant_id,
8807
    const bool check_data_version)
8808
{
8809
  int ret = OB_SUCCESS;
8810
  if (!check_inner_stat()) {
8811
    ret = OB_INNER_STAT_ERROR;
8812
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), KR(ret));
8813
  } else if (!is_valid_tenant_id(tenant_id) || pool_names.count() <= 0) {
8814
    ret = OB_INVALID_ARGUMENT;
8815
    LOG_WARN("invalid argument", K(tenant_id), K(pool_names), KR(ret));
8816
  } else if (OB_UNLIKELY(nullptr == srv_rpc_proxy_)) {
8817
    ret = OB_ERR_UNEXPECTED;
8818
    LOG_WARN("srv_rpc_proxy_ ptr is null", KR(ret));
8819
  } else {
8820
    ObNotifyTenantServerResourceProxy notify_proxy(
8821
                                      *srv_rpc_proxy_,
8822
                                      &obrpc::ObSrvRpcProxy::notify_tenant_server_unit_resource);
8823
    share::ObResourcePool new_pool;
8824
    ObArray<share::ObResourcePool> pools;
8825
    ObArray<ObArray<ObUnit>> all_pool_units;
8826
    ObArray<ObUnit> pool_units;
8827
    for (int64_t i = 0; OB_SUCC(ret) && i < pool_names.count(); ++i) {
8828
      share::ObResourcePool *pool = NULL;
8829
      pool_units.reset();
8830
      common::ObArray<ObUnit *> zone_sorted_unit_array;
8831
      if (OB_FAIL(inner_get_resource_pool_by_name(pool_names.at(i), pool))) {
8832
        LOG_WARN("get resource pool by name failed", "pool_name", pool_names.at(i), KR(ret));
8833
      } else if (OB_ISNULL(pool)) {
8834
        ret = OB_ERR_UNEXPECTED;
8835
        LOG_WARN("pool is null", KP(pool), KR(ret));
8836
      } else if (pool->unit_count_ != new_ug_ids.count()) {
8837
        ret = OB_ERR_UNEXPECTED;
8838
        LOG_WARN("count not match", KR(ret), K(tenant_id),
8839
                 "pool", new_pool, K(new_ug_ids));
8840
      } else if (OB_FAIL(new_pool.assign(*pool))) {
8841
        LOG_WARN("failed to assign new_pool", KR(ret));
8842
      } else if (FALSE_IT(new_pool.tenant_id_ = tenant_id)) {
8843
        // shall never be here
8844
      } else if (OB_FAIL(ut_operator_.update_resource_pool(trans, new_pool,false/*need_check_conflict_with_clone*/))) {
8845
        LOG_WARN("update_resource_pool failed", K(new_pool), KR(ret));
8846
      } else if (is_bootstrap) {
8847
        //no need to notify unit and modify unit group id
8848
      } else if (OB_FAIL(build_zone_sorted_unit_array_(pool, zone_sorted_unit_array))) {
8849
        LOG_WARN("failed to generate zone_sorted_unit_array", KR(ret), KPC(pool));
8850
      } else if (OB_FAIL(construct_pool_units_to_grant_(
8851
                             trans,
8852
                             tenant_id,
8853
                             new_pool,
8854
                             zone_sorted_unit_array,
8855
                             new_ug_ids,
8856
                             compat_mode,
8857
                             notify_proxy,
8858
                             source_tenant_id,
8859
                             pool_units,
8860
                             check_data_version))) {
8861
        LOG_WARN("fail to construct pool units to grant", KR(ret), K(tenant_id), K(new_pool),
8862
                 K(zone_sorted_unit_array), K(new_ug_ids), K(compat_mode), K(source_tenant_id));
8863
      } else if (OB_FAIL(all_pool_units.push_back(pool_units))) {
8864
        LOG_WARN("fail to push an element into all_pool_units", KR(ret), K(pool_units));
8865
      } else if (OB_FAIL(pools.push_back(new_pool))) {
8866
        LOG_WARN("fail to push an element into pools", KR(ret), K(new_pool));
8867
      }
8868
    }
8869
    int tmp_ret = OB_SUCCESS;
8870
    if (OB_TMP_FAIL(notify_proxy.wait())) {
8871
      LOG_WARN("fail to wait notify resource", KR(ret), K(tmp_ret));
8872
      ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
8873
    } else if (OB_SUCC(ret)) {
8874
      // arg/dest/result can be used here.
8875
    }
8876
    ret = ERRSIM_UNIT_PERSISTENCE_ERROR ? : ret;
8877
    if (OB_FAIL(ret) && pools.count() == all_pool_units.count()) {
8878
      LOG_WARN("start to rollback unit persistence", KR(ret), K(pools), K(tenant_id));
8879
      for (int64_t i = 0; i < pools.count(); ++i) {
8880
        if (OB_TMP_FAIL(rollback_persistent_units_(all_pool_units.at(i), pools.at(i), notify_proxy))) {
8881
          LOG_WARN("fail to rollback unit persistence", KR(ret), KR(tmp_ret), K(all_pool_units.at(i)),
8882
              K(pools.at(i)), K(compat_mode));
8883
        }
8884
      }
8885
    }
8886
  }
8887
  return ret;
8888
}
8889

8890
int ObUnitManager::do_revoke_pools_(
8891
    ObMySQLTransaction &trans,
8892
    const common::ObIArray<uint64_t> &new_ug_ids,
8893
    const ObIArray<ObResourcePoolName> &pool_names,
8894
    const uint64_t tenant_id)
8895
{
8896
  int ret = OB_SUCCESS;
8897
  const lib::Worker::CompatMode dummy_mode = lib::Worker::CompatMode::INVALID;
8898
  if (!check_inner_stat()) {
8899
    ret = OB_INNER_STAT_ERROR;
8900
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), KR(ret));
8901
  } else if (!is_valid_tenant_id(tenant_id) || pool_names.count() <= 0) {
8902
    ret = OB_INVALID_ARGUMENT;
8903
    LOG_WARN("invalid argument", K(tenant_id), K(pool_names), KR(ret));
8904
  } else if (OB_UNLIKELY(nullptr == srv_rpc_proxy_)) {
8905
    ret = OB_ERR_UNEXPECTED;
8906
    LOG_WARN("srv_rpc_proxy_ ptr is null", KR(ret));
8907
  } else {
8908
    ObNotifyTenantServerResourceProxy notify_proxy(
8909
                                      *srv_rpc_proxy_,
8910
                                      &obrpc::ObSrvRpcProxy::notify_tenant_server_unit_resource);
8911
    share::ObResourcePool new_pool;
8912
    ObArray<share::ObResourcePool *> shrinking_pools;
8913
    ObArray<share::ObResourcePool> pools;
8914
    for (int64_t i = 0; OB_SUCC(ret) && i < pool_names.count(); ++i) {
8915
      share::ObResourcePool *pool = NULL;
8916
      bool is_shrinking = false;
8917
      common::ObArray<ObUnit *> zone_sorted_unit_array;
8918
      if (OB_FAIL(inner_get_resource_pool_by_name(pool_names.at(i), pool))) {
8919
        LOG_WARN("get resource pool by name failed", "pool_name", pool_names.at(i), KR(ret));
8920
      } else if (OB_ISNULL(pool)) {
8921
        ret = OB_ERR_UNEXPECTED;
8922
        LOG_WARN("pool is null", KP(pool), KR(ret));
8923
      } else if (pool->unit_count_ != new_ug_ids.count()) {
8924
        ret = OB_ERR_UNEXPECTED;
8925
        LOG_WARN("count not match", KR(ret), K(tenant_id),
8926
                 "pool", new_pool, K(new_ug_ids));
8927
      } else if (OB_FAIL(inner_check_pool_in_shrinking_(pool->resource_pool_id_, is_shrinking))) {   // TODO(cangming.zl): maybe need to move ahead
8928
        LOG_WARN("inner check pool in shrinking failed", KR(ret), "pool", *pool, K(is_shrinking));
8929
      } else if (is_shrinking && OB_FAIL(shrinking_pools.push_back(pool))) {
8930
        LOG_WARN("fail to push back shrinking resource pool before revoked", KR(ret), K(tenant_id), "pool", *pool);
8931
      } else if (OB_FAIL(new_pool.assign(*pool))) {
8932
        LOG_WARN("failed to assign new_pool", KR(ret));
8933
      } else if (FALSE_IT(new_pool.tenant_id_ = OB_INVALID_ID)) {
8934
        // shall never be here
8935
      } else if (OB_FAIL(ut_operator_.update_resource_pool(trans, new_pool, false/*need_check_conflict_with_clone*/))) {
8936
        LOG_WARN("update_resource_pool failed", K(new_pool), KR(ret));
8937
      } else if (OB_FAIL(build_zone_sorted_unit_array_(pool, zone_sorted_unit_array))) {
8938
        LOG_WARN("failed to generate zone_sorted_unit_array", KR(ret), KPC(pool));
8939
      } else {
8940
        ObUnit new_unit;
8941
        for (int64_t j = 0; OB_SUCC(ret) && j < zone_sorted_unit_array.count(); ++j) {
8942
          ObUnit *unit = zone_sorted_unit_array.at(j);
8943
          new_unit.reset();
8944
          if (OB_ISNULL(unit)) {
8945
            ret = OB_ERR_UNEXPECTED;
8946
            LOG_WARN("unit ptr is null", KR(ret));
8947
          } else if (OB_FAIL(try_notify_tenant_server_unit_resource_(
8948
                  tenant_id, true /*is_delete*/, notify_proxy,
8949
                  new_pool.unit_config_id_, dummy_mode, *unit,
8950
                  false/*if_not_grant*/, false/*skip_offline_server*/,
8951
                  false /*check_data_version*/))) {
8952
            LOG_WARN("fail to try notify server unit resource", KR(ret));
8953
          } else if (FALSE_IT(new_unit = *unit)) {
8954
            // shall never be here
8955
          } else if (FALSE_IT(new_unit.unit_group_id_ = new_ug_ids.at(j % (pool->unit_count_)))) {
8956
            // shall never be here
8957
          } else if (OB_FAIL(ut_operator_.update_unit(trans, new_unit, false/*need_check_conflict_with_clone*/))) {
8958
            LOG_WARN("fail to update unit", KR(ret));
8959
          }
8960
        }
8961
      }
8962
    }
8963
    // If some of the pools are shrinking, commit these shrinking pools now.
8964
    if (OB_SUCCESS != ret) {
8965
    } else if (shrinking_pools.count() > 0 && OB_FAIL(inner_commit_shrink_tenant_resource_pool_(trans, tenant_id, shrinking_pools))) {
8966
      LOG_WARN("failed to commit shrinking pools in revoking", KR(ret), K(tenant_id));
8967
    }
8968
    int tmp_ret = OB_SUCCESS;
8969
    if (OB_TMP_FAIL(notify_proxy.wait())) {
8970
      LOG_WARN("fail to wait notify resource", KR(ret), K(tmp_ret));
8971
      ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
8972
    } else if (OB_SUCC(ret)) {
8973
      // arg/dest/result can be used here.
8974
    }
8975
  }
8976
  return ret;
8977
}
8978

8979
int ObUnitManager::build_zone_sorted_unit_array_(const share::ObResourcePool *pool,
8980
                                             common::ObArray<share::ObUnit*> &zone_sorted_units)
8981
{
8982
  int ret = OB_SUCCESS;
8983
  ObArray<share::ObUnit*> *units;
8984
  if (!check_inner_stat()) {
8985
    ret = OB_INNER_STAT_ERROR;
8986
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), KR(ret));
8987
  } else if (OB_ISNULL(pool)) {
8988
    ret = OB_INVALID_ARGUMENT;
8989
    LOG_WARN("pool is nullptr", KR(ret));
8990
  } else if (OB_FAIL(get_units_by_pool(pool->resource_pool_id_, units))) {
8991
    LOG_WARN("fail to get units by pool", KR(ret));
8992
  } else if (OB_ISNULL(units)) {
8993
    ret = OB_ERR_UNEXPECTED;
8994
    LOG_WARN("units ptr is null", KR(ret));
8995
  } else if (OB_FAIL(zone_sorted_units.assign(*units))) {
8996
    LOG_WARN("fail to assign zone unit array", KR(ret));
8997
  } else {
8998
    UnitZoneOrderCmp cmp_operator;
8999
    std::sort(zone_sorted_units.begin(), zone_sorted_units.end(), cmp_operator);
9000
    // check unit count in each zone
9001
    ObUnit *curr_unit = NULL;
9002
    ObUnit *prev_unit = NULL;
9003
    int64_t unit_count_per_zone = 0;
9004
    for (int64_t j = 0; OB_SUCC(ret) && j < zone_sorted_units.count(); ++j) {
9005
      prev_unit = curr_unit;
9006
      curr_unit = zone_sorted_units.at(j);
9007
      if (OB_ISNULL(curr_unit)) {
9008
        ret = OB_ERR_UNEXPECTED;
9009
        LOG_WARN("unit ptr is null", KR(ret));
9010
      } else if (0 == j || curr_unit->zone_ == prev_unit->zone_) {
9011
        unit_count_per_zone += ObUnit::UNIT_STATUS_ACTIVE == curr_unit->status_ ? 1 : 0;
9012
      } else if (unit_count_per_zone != pool->unit_count_) {
9013
        ret = OB_ERR_UNEXPECTED;
9014
        LOG_WARN("zone unit num doesn't match resource pool's unit_count",
9015
                 KR(ret), "zone", prev_unit->zone_, K(unit_count_per_zone), K(pool->unit_count_));
9016
      } else {
9017
        unit_count_per_zone = ObUnit::UNIT_STATUS_ACTIVE == curr_unit->status_ ? 1 : 0;
9018
      }
9019
    }
9020
    if (OB_SUCC(ret) && unit_count_per_zone != pool->unit_count_) {
9021
      ret = OB_ERR_UNEXPECTED;
9022
      LOG_WARN("zone unit num doesn't match resource pool's unit_count",
9023
                KR(ret), "zone", curr_unit->zone_, K(unit_count_per_zone), K(pool->unit_count_));
9024
    }
9025
  }
9026
  return ret;
9027
}
9028

9029
// this function is only used to commit part of shrinking pools of a tenant when these pools are revoked.
9030
int ObUnitManager::inner_commit_shrink_tenant_resource_pool_(
9031
  common::ObMySQLTransaction &trans, const uint64_t tenant_id, const common::ObArray<share::ObResourcePool *> &pools)
9032
{
9033
  int ret = OB_SUCCESS;
9034
  if (!check_inner_stat()) {
9035
    ret = OB_INNER_STAT_ERROR;
9036
    LOG_WARN("check_inner_stat failed", KR(ret), K(loaded_), K(inited_));
9037
  } else if (OB_UNLIKELY(OB_INVALID_ID == tenant_id)) {
9038
    ret = OB_INVALID_ARGUMENT;
9039
    LOG_WARN("invalid argument", KR(ret), K(tenant_id));
9040
  } else if (pools.count() <= 0) {
9041
    ret = OB_INVALID_ARGUMENT;
9042
    LOG_WARN("pools is empty", KR(ret), K(pools));
9043
  } else {
9044
    // first check that pools are all owned by specified tenant
9045
    FOREACH_CNT_X(pool, pools, OB_SUCCESS == ret) {
9046
      if (OB_ISNULL(*pool)) {
9047
        ret = OB_ERR_UNEXPECTED;
9048
        LOG_WARN("pool is null pointer", KR(ret), KP(*pool));
9049
      } else if (tenant_id != (*pool)->tenant_id_) {
9050
        ret = OB_ERR_UNEXPECTED;
9051
        LOG_WARN("pool is not owned by specified tenant", KR(ret), K(tenant_id), KPC(*pool));
9052
      }
9053
    }
9054
    ObArray<ObArray<uint64_t>> resource_units;
9055
    if (FAILEDx(commit_shrink_resource_pool_in_trans_(pools, trans, resource_units))) {
9056
      LOG_WARN("failed to shrink in trans", KR(ret), K(pools));
9057
    } else if (OB_UNLIKELY(resource_units.count() <= 0)) {
9058
      ret = OB_ERR_UNEXPECTED;
9059
      LOG_WARN("resource units is empty", KR(ret), K(resource_units));
9060
    } else {/* good */}
9061
  }
9062
  return ret;
9063
}
9064

9065
int ObUnitManager::get_zone_units(const ObArray<share::ObResourcePool *> &pools,
9066
                                  ObArray<ZoneUnit> &zone_units) const
9067
{
9068
  int ret = OB_SUCCESS;
9069
  ObArray<ObZone> zones;
9070
  zone_units.reuse();
9071
  if (!check_inner_stat()) {
9072
    ret = OB_INNER_STAT_ERROR;
9073
    LOG_WARN("check_inner_stat failed", K_(inited), K_(loaded), K(ret));
9074
  } else if (pools.count() <= 0) {
9075
    ret = OB_INVALID_ARGUMENT;
9076
    LOG_WARN("pools is empty", K(pools), K(ret));
9077
  } else {
9078
    FOREACH_CNT_X(pool, pools, OB_SUCCESS == ret) {
9079
      if (NULL == *pool) {
9080
        ret = OB_INVALID_ARGUMENT;
9081
        LOG_WARN("pool is null", "pool", OB_P(*pool), K(ret));
9082
      } else {
9083
        FOREACH_CNT_X(pool_zone, (*pool)->zone_list_, OB_SUCCESS == ret) {
9084
          bool find = false;
9085
          FOREACH_CNT_X(zone, zones, !find) {
9086
            if (*zone == *pool_zone) {
9087
              find = true;
9088
            }
9089
          }
9090
          if (!find) {
9091
            if (OB_FAIL(zones.push_back(*pool_zone))) {
9092
              LOG_WARN("push_back failed", K(ret));
9093
            }
9094
          }
9095
        }
9096
      }
9097
    }
9098

9099
    ZoneUnit zone_unit;
9100
    ObArray<ObUnitInfo> unit_infos;
9101
    FOREACH_CNT_X(zone, zones, OB_SUCCESS == ret) {
9102
      zone_unit.reset();
9103
      zone_unit.zone_ = *zone;
9104
      FOREACH_CNT_X(pool, pools, OB_SUCCESS == ret) {
9105
        unit_infos.reuse();
9106
        if (NULL == *pool) {
9107
          ret = OB_INVALID_ARGUMENT;
9108
          LOG_WARN("pool is null", "pool", OB_P(*pool), K(ret));
9109
        } else if (OB_FAIL(inner_get_unit_infos_of_pool_((*pool)->resource_pool_id_, unit_infos))) {
9110
          LOG_WARN("inner_get_unit_infos_of_pool failed",
9111
              "pool id", (*pool)->resource_pool_id_, K(ret));
9112
        } else {
9113
          FOREACH_CNT_X(unit_info, unit_infos, OB_SUCCESS == ret) {
9114
            if (unit_info->unit_.zone_ == *zone) {
9115
              if (OB_FAIL(zone_unit.unit_infos_.push_back(*unit_info))) {
9116
                LOG_WARN("push_back failed", K(ret));
9117
              }
9118
            }
9119
          }
9120
        }
9121
      }
9122
      if (OB_SUCC(ret)) {
9123
        if (OB_FAIL(zone_units.push_back(zone_unit))) {
9124
          LOG_WARN("push_back failed", K(zone_unit), K(ret));
9125
        }
9126
      }
9127
    }
9128
  }
9129
  return ret;
9130
}
9131

9132
int ObUnitManager::get_tenants_of_server(const common::ObAddr &server,
9133
    common::hash::ObHashSet<uint64_t> &tenant_id_set) const
9134
{
9135
  int ret = OB_SUCCESS;
9136
  ObArray<ObUnitLoad> *unit_loads = NULL;
9137
  {
9138
    SpinRLockGuard guard(lock_);
9139
    if (!check_inner_stat()) {
9140
      ret = OB_INNER_STAT_ERROR;
9141
      LOG_WARN("check inner stat failed", K_(inited), K_(loaded), K(ret));
9142
    } else if (!server.is_valid()) {
9143
      ret = OB_INVALID_ARGUMENT;
9144
      LOG_WARN("server is invalid", K(server), K(ret));
9145
    } else if (OB_FAIL(get_loads_by_server(server, unit_loads))) {
9146
      if (OB_ENTRY_NOT_EXIST != ret) {
9147
        LOG_WARN("get_loads_by_server failed", K(server), K(ret));
9148
      } else {
9149
        ret = OB_SUCCESS;
9150
        // just return empty set
9151
      }
9152
    } else if (NULL == unit_loads) {
9153
      ret = OB_ERR_UNEXPECTED;
9154
      LOG_WARN("unit_loads is null", KP(unit_loads), K(ret));
9155
    }
9156
    if (OB_SUCC(ret) && !OB_ISNULL(unit_loads)) {
9157
      FOREACH_CNT_X(unit_load, *unit_loads, OB_SUCCESS == ret) {
9158
        if (!unit_load->is_valid()) {
9159
          ret = OB_ERR_UNEXPECTED;
9160
          LOG_WARN("invalid unit_load", "unit_load", *unit_load, K(ret));
9161
        } else {
9162
          const uint64_t tenant_id = unit_load->pool_->tenant_id_;
9163
          if (!is_valid_tenant_id(tenant_id)) {
9164
            //do nothing
9165
          } else if (OB_FAIL(tenant_id_set.set_refactored(tenant_id))) {
9166
            if (OB_HASH_EXIST == ret) {
9167
              ret = OB_SUCCESS;
9168
            } else {
9169
              LOG_WARN("set tenant id failed", K(tenant_id), K(ret));
9170
            }
9171
          }
9172
        }
9173
      }
9174
    }
9175
  }
9176
  return ret;
9177
}
9178

9179
int ObUnitManager::check_tenant_on_server(const uint64_t tenant_id,
9180
    const ObAddr &server, bool &on_server) const
9181
{
9182
  int ret = OB_SUCCESS;
9183
  if (!check_inner_stat()) {
9184
    ret = OB_INNER_STAT_ERROR;
9185
    LOG_WARN("check inner stat failed", K_(inited), K_(loaded), K(ret));
9186
  } else if (!is_valid_tenant_id(tenant_id) || !server.is_valid()) {
9187
    ret = OB_INVALID_ARGUMENT;
9188
    LOG_WARN("invalid tenant_id or invalid server", K(tenant_id), K(server), K(ret));
9189
  } else {
9190
    ObArray<uint64_t> pool_ids;
9191
    ObZone zone;
9192
    ObArray<ObAddr> servers;
9193
    if (OB_FAIL(get_pool_ids_of_tenant(tenant_id, pool_ids))) {
9194
      LOG_WARN("get_pool_ids_of_tenant failed", K(tenant_id), K(ret));
9195
    } else if (OB_FAIL(SVR_TRACER.get_server_zone(server, zone))) {
9196
      LOG_WARN("get_server_zone failed", K(server), K(ret));
9197
    } else {
9198
      SpinRLockGuard guard(lock_);
9199
      FOREACH_CNT_X(pool_id, pool_ids, OB_SUCCESS == ret && !on_server) {
9200
        if (OB_FAIL(get_pool_servers(*pool_id, zone, servers))) {
9201
          LOG_WARN("get_pool_servers failed", "pool_id", *pool_id, K(zone), K(ret));
9202
        } else if (has_exist_in_array(servers, server)) {
9203
          on_server = true;
9204
        }
9205
      }
9206
    }
9207
  }
9208
  return ret;
9209
}
9210

9211
int ObUnitManager::admin_migrate_unit(
9212
    const uint64_t unit_id,
9213
    const ObAddr &dst,
9214
    bool is_cancel)
9215
{
9216
  int ret = OB_SUCCESS;
9217
  ObUnitInfo unit_info;
9218
  ObArray<ObAddr> excluded_servers;
9219
  ObServerInfoInTable dst_server_info;
9220
  obrpc::ObGetServerResourceInfoResult report_dst_server_resource_info;
9221
  ObZone src_zone;
9222
  ObZone dst_zone;
9223
  double hard_limit = 0;
9224
  bool can_hold_unit = false;
9225
  SpinWLockGuard guard(lock_);
9226
  AlterResourceErr err_index = ALT_ERR;
9227
  const char *module = "ADMIN_MIGRATE_UNIT";
9228
  if (!check_inner_stat()) {
9229
    ret = OB_INNER_STAT_ERROR;
9230
    LOG_WARN("check inner stat failed", K_(inited), K_(loaded), KR(ret));
9231
  } else if (OB_INVALID_ID == unit_id) {
9232
    ret = OB_INVALID_ARGUMENT;
9233
    LOG_WARN("unit id is invalid", K(unit_id), KR(ret));
9234
  } else if (!dst.is_valid() && !is_cancel) {
9235
    ret = OB_INVALID_ARGUMENT;
9236
    LOG_WARN("dst is invalid", K(dst), KR(ret));
9237
  } else if (OB_FAIL(get_hard_limit(hard_limit))) {
9238
    LOG_WARN("get_hard_limit failed", KR(ret));
9239
  } else if (OB_FAIL(inner_get_unit_info_by_id(unit_id, unit_info))) {
9240
    if (OB_ENTRY_NOT_EXIST == ret) {
9241
      LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "unit_id not existed");
9242
    }
9243
    LOG_WARN("get unit info failed", K(unit_id), KR(ret));
9244
  } else if (ObUnit::UNIT_STATUS_ACTIVE != unit_info.unit_.status_) {
9245
    ret = OB_OP_NOT_ALLOW;
9246
    LOG_WARN("migrate a unit which is in deleting status", KR(ret), K(unit_id));
9247
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "migrate a unit which is in deleting status");
9248
  } else if (dst == unit_info.unit_.server_) {
9249
    // nothing need to do
9250
    LOG_INFO("migration dst same to src", KR(ret), K(dst), K(unit_info));
9251
  } else if (dst == unit_info.unit_.migrate_from_server_ || is_cancel) {
9252
    // cancel migrate unit
9253
    bool can_migrate_in = false;
9254
    if (is_cancel && !unit_info.unit_.migrate_from_server_.is_valid()) {
9255
	    ret = OB_ERR_UNEXPECTED;
9256
	    LOG_WARN("failed to cancel migrate unit, may be no migrate task", KR(ret), K(unit_info));
9257
      LOG_USER_ERROR(OB_ERR_UNEXPECTED,"no migrate task to cancel");
9258
	  } else if (OB_FAIL(SVR_TRACER.check_server_can_migrate_in(
9259
        unit_info.unit_.migrate_from_server_,
9260
        can_migrate_in))) {
9261
      LOG_WARN("fail to check server can_migrate_in", KR(ret), K(unit_info.unit_.migrate_from_server_));
9262
    } else if (OB_FAIL(cancel_migrate_unit(
9263
            unit_info.unit_, can_migrate_in, unit_info.pool_.tenant_id_ == OB_GTS_TENANT_ID))) {
9264
		LOG_WARN("failed to cancel migrate unit", KR(ret), K(unit_info), K(can_migrate_in));
9265
    }
9266
  } else if (OB_FAIL(SVR_TRACER.get_server_zone(unit_info.unit_.server_, src_zone))) {
9267
    LOG_WARN("get server zone failed", "server", unit_info.unit_.server_, KR(ret));
9268
  } else if (OB_FAIL(SVR_TRACER.get_server_zone(dst, dst_zone))) {
9269
    if (OB_ENTRY_NOT_EXIST == ret) {
9270
      LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "destination server not found in the cluster");
9271
    }
9272
    LOG_WARN("get server zone failed", "server", dst, KR(ret));
9273
  } else if (src_zone != dst_zone) {
9274
    ret = OB_NOT_SUPPORTED;
9275
    LOG_WARN("migrate unit between zones is not supported", KR(ret));
9276
    LOG_USER_ERROR(OB_NOT_SUPPORTED,"migrate unit between zones");
9277
  } else if (OB_FAIL(get_excluded_servers(unit_info.unit_.resource_pool_id_, unit_info.unit_.zone_,
9278
      module, false/*new_allocate_pool*/, excluded_servers))) {
9279
    LOG_WARN("get_excluded_servers failed", "unit", unit_info.unit_, KR(ret));
9280
  } else if (has_exist_in_array(excluded_servers, dst)) {
9281
    ret = OB_NOT_SUPPORTED;
9282
    LOG_USER_ERROR(OB_NOT_SUPPORTED,"hold two units of a tenant in the same server");
9283
    LOG_WARN("hold two units of a tenant in the same server is not supported", KR(ret));
9284
  } else if (OB_FAIL(SVR_TRACER.get_server_info(dst, dst_server_info))) {
9285
    LOG_WARN("get dst_server_info failed", KR(ret), K(dst));
9286
  } else if (!dst_server_info.can_migrate_in()) {
9287
    ret = OB_SERVER_MIGRATE_IN_DENIED;
9288
    LOG_WARN("server can not migrate in", K(dst), K(dst_server_info), KR(ret));
9289
  } else if (OB_FAIL(get_server_resource_info_via_rpc(dst_server_info, report_dst_server_resource_info))) {
9290
    LOG_WARN("fail to execute get_server_resource_info_via_rpc", KR(ret), K(dst_server_info));
9291
  } else if (OB_FAIL(have_enough_resource(
9292
      report_dst_server_resource_info,
9293
      unit_info.config_.unit_resource(),
9294
      hard_limit,
9295
      can_hold_unit,
9296
      err_index))) {
9297
    LOG_WARN("calculate_left_resource failed", KR(ret), K(report_dst_server_resource_info),
9298
        K(hard_limit), K(err_index));
9299
  } else if (!can_hold_unit) {
9300
    ret = OB_MACHINE_RESOURCE_NOT_ENOUGH;
9301
    if (OB_SUCCESS != ERRSIM_USE_DUMMY_SERVER) {
9302
      LOG_USER_ERROR(OB_MACHINE_RESOURCE_NOT_ENOUGH, "dummy_zone", "127.0.0.1:1000",
9303
          alter_resource_err_to_str(err_index));
9304
    } else {
9305
      LOG_USER_ERROR(OB_MACHINE_RESOURCE_NOT_ENOUGH, to_cstring(dst_zone), to_cstring(dst),
9306
          alter_resource_err_to_str(err_index));
9307
    }
9308
    LOG_WARN("left resource can't hold unit", "server", dst,
9309
        K(hard_limit), "config", unit_info.config_, KR(ret));
9310
  } else if (OB_FAIL(migrate_unit_(unit_id, dst, true/*is_manual*/))) {
9311
    LOG_WARN("migrate unit failed", K(unit_id), "destination", dst, KR(ret));
9312
  }
9313

9314
  return ret;
9315
}
9316

9317
int ObUnitManager::cancel_migrate_unit(
9318
    const share::ObUnit &unit,
9319
    const bool migrate_from_server_can_migrate_in,
9320
    const bool is_gts_unit)
9321
{
9322
  int ret = OB_SUCCESS;
9323
  if (!migrate_from_server_can_migrate_in && !is_gts_unit) {
9324
    ret = OB_SERVER_MIGRATE_IN_DENIED;
9325
    LOG_WARN("server can not migrate in", K(unit.migrate_from_server_), K(migrate_from_server_can_migrate_in), KR(ret));
9326
  } else {
9327
    const EndMigrateOp op = REVERSE;
9328
    if (OB_FAIL(end_migrate_unit(unit.unit_id_, op))) {
9329
      LOG_WARN("end_migrate_unit failed", "unit_id", unit.unit_id_, K(op), K(ret));
9330
    } else {
9331
      LOG_INFO("cancel migrate unit", K(unit));
9332
    }
9333
  }
9334
  return ret;
9335
}
9336

9337
int ObUnitManager::try_cancel_migrate_unit(const share::ObUnit &unit, bool &is_canceled)
9338
{
9339
  int ret = OB_SUCCESS;
9340
  bool migrate_from_server_can_migrate_in = false;
9341
  bool server_can_migrate_in = false;
9342
  is_canceled = false;
9343
  if (!check_inner_stat()) {
9344
    ret = OB_INNER_STAT_ERROR;
9345
    LOG_WARN("check inner stat failed", K_(inited), K_(loaded), K(ret));
9346
  } else if (!unit.is_valid()) {
9347
    ret = OB_INVALID_ARGUMENT;
9348
    LOG_WARN("invalid unit", K(unit), K(ret));
9349
  } else if (OB_FAIL(SVR_TRACER.check_server_can_migrate_in(unit.server_, server_can_migrate_in))) {
9350
    LOG_WARN("check_server_can_migrate_in failed", "server", unit.server_, K(ret));
9351
  } else if (server_can_migrate_in) {
9352
    // ignore, do nothing
9353
  } else if (OB_FAIL(SVR_TRACER.check_server_can_migrate_in(
9354
      unit.migrate_from_server_,
9355
      migrate_from_server_can_migrate_in))) {
9356
    LOG_WARN("get_server_status failed", "server", unit.migrate_from_server_, K(ret));
9357
  } else if (migrate_from_server_can_migrate_in) {
9358
    LOG_INFO("unit migrate_from_server can migrate in, "
9359
        "migrate unit back to migrate_from_server", K(unit), K(migrate_from_server_can_migrate_in));
9360
    const EndMigrateOp op = REVERSE;
9361
    if (OB_FAIL(end_migrate_unit(unit.unit_id_, op))) {
9362
      LOG_WARN("end_migrate_unit failed", "unit_id", unit.unit_id_, K(op), K(ret));
9363
    } else {
9364
      is_canceled = true;
9365
      LOG_INFO("reverse unit migrate success", K(ret), "unit_id", unit.unit_id_, K(op));
9366
    }
9367
  }
9368
  return ret;
9369
}
9370

9371
int ObUnitManager::get_hard_limit(double &hard_limit) const
9372
{
9373
  int ret = OB_SUCCESS;
9374
  if (!check_inner_stat()) {
9375
    ret = OB_INNER_STAT_ERROR;
9376
    LOG_WARN("check inner stat failed", K_(inited), K_(loaded), K(ret));
9377
  } else {
9378
    hard_limit = static_cast<double>(server_config_->resource_hard_limit) / 100;
9379
  }
9380
  return ret;
9381
}
9382

9383
// bug#11873101 issue/11873101
9384
// Before attempting to migrate the unit,
9385
// check whether the target unit space is sufficient,
9386
// if it is insufficient, do not migrate,
9387
// and return OB_OP_NOT_ALLOW
9388
int ObUnitManager::try_migrate_unit(const uint64_t unit_id,
9389
                                    const uint64_t tenant_id,
9390
                                    const ObUnitStat &unit_stat,
9391
                                    const ObIArray<ObUnitStat> &migrating_unit_stat,
9392
                                    const ObAddr &dst,
9393
                                    const ObServerResourceInfo &dst_resource_info,
9394
                                    const bool is_manual)
9395
{
9396
  int ret = OB_SUCCESS;
9397
  ObServerStatus server_status;
9398
  if (unit_id != unit_stat.get_unit_id()) {
9399
    ret = OB_INVALID_ARGUMENT;
9400
    LOG_WARN("invalid unit stat", K(unit_id), K(unit_stat), K(ret));
9401
  } else {
9402
    int64_t mig_required_size = 0;
9403
    for (int64_t i = 0; i < migrating_unit_stat.count(); ++i) {
9404
      mig_required_size +=  migrating_unit_stat.at(i).get_required_size();
9405
    }
9406
    // sstable Space constraints
9407
    int64_t required_size =
9408
        mig_required_size + unit_stat.get_required_size() + dst_resource_info.disk_in_use_;
9409
    int64_t total_size = dst_resource_info.disk_total_;
9410
    int64_t required_percent = (100 * required_size) / total_size;
9411
    int64_t limit_percent = GCONF.data_disk_usage_limit_percentage;
9412
    if (required_percent >= limit_percent) {
9413
      ret = OB_OP_NOT_ALLOW;
9414
      LOG_ERROR("migrate unit fail. dest server out of space",
9415
                K(unit_id), K(unit_stat), K(dst),
9416
                K(required_size), K(total_size), K(limit_percent), K(ret));
9417
    }
9418

9419
    if (FAILEDx(migrate_unit_(unit_id, dst, is_manual))) {
9420
      LOG_WARN("fail migrate unit", K(unit_id), K(dst), K(ret));
9421
    }
9422
  }
9423
  return ret;
9424
}
9425

9426
int ObUnitManager::migrate_unit_(const uint64_t unit_id, const ObAddr &dst, const bool is_manual)
9427
{
9428
  int ret = OB_SUCCESS;
9429
  LOG_INFO("start to migrate unit", KR(ret), K(unit_id), K(dst), K(is_manual));
9430
  ObUnit *unit = NULL;
9431
  share::ObResourcePool *pool = NULL;
9432
  ObZone zone;
9433
  if (!check_inner_stat()) {
9434
    ret = OB_INNER_STAT_ERROR;
9435
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
9436
  } else if (OB_INVALID_ID == unit_id || !dst.is_valid()) {
9437
    ret = OB_INVALID_ARGUMENT;
9438
    LOG_WARN("invalid argument", K(unit_id), K(dst), K(ret));
9439
  } else if (OB_FAIL(get_unit_by_id(unit_id, unit))) {
9440
    LOG_WARN("get_unit_by_id failed", K(unit_id), K(ret));
9441
  } else if (OB_ISNULL(unit)) {
9442
    ret = OB_ERR_UNEXPECTED;
9443
    LOG_WARN("unit is null", KP(unit), K(ret));
9444
  } else if (ObUnit::UNIT_STATUS_ACTIVE != unit->status_) {
9445
    ret = OB_OP_NOT_ALLOW;
9446
    LOG_WARN("cannot migrate unit which is in deleting", K(ret), K(unit_id));
9447
  } else if (unit->server_ == dst) {
9448
    ret = OB_INVALID_ARGUMENT;
9449
    LOG_WARN("unit->server same as migrate destination server",
9450
        "unit", *unit, K(dst), K(ret));
9451
  } else if (OB_FAIL(SVR_TRACER.get_server_zone(dst, zone))) {
9452
    LOG_WARN("get_server_zone failed", KR(ret), K(dst));
9453
  } else if (OB_UNLIKELY(zone != unit->zone_)) {
9454
    ret = OB_NOT_SUPPORTED;
9455
    LOG_WARN("migrate unit between zones not supported", KR(ret), KP(unit), K(dst), K(zone));
9456
  } else if (unit->migrate_from_server_.is_valid()) {
9457
    ret = OB_NOT_SUPPORTED;
9458
    LOG_WARN("unit is already migrating, cannot migrate any more", "unit", *unit, K(ret));
9459
    LOG_USER_ERROR(OB_NOT_SUPPORTED, "migrate unit already in migrating status");
9460
  } else if (OB_FAIL(get_resource_pool_by_id(unit->resource_pool_id_, pool))) {
9461
    LOG_WARN("get_resource_pool_by_id failed",
9462
        "resource pool id", unit->resource_pool_id_, K(ret));
9463
  } else if (OB_ISNULL(pool)) {
9464
    ret = OB_ERR_UNEXPECTED;
9465
    LOG_WARN("pool is null", KP(pool), K(ret));
9466
  } else {
9467
    const bool granted = pool->is_granted_to_tenant();
9468
    const ObAddr src = unit->server_;
9469
    ObUnit new_unit = *unit;
9470
    new_unit.server_ = dst;
9471
    new_unit.migrate_from_server_ = granted ? src : ObAddr();
9472
    new_unit.is_manual_migrate_ = is_manual;
9473

9474
    LOG_INFO("do migrate unit", KPC(unit), K(new_unit), KPC(pool), K(granted));
9475

9476
    // STEP 1: try notify unit persistence on destination ObServer
9477
    if (OB_FAIL(do_migrate_unit_notify_resource_(*pool, new_unit, is_manual, granted))) {
9478
      LOG_WARN("do_migrate_unit_notify_resource failed", KR(ret), KPC(pool), K(new_unit), K(is_manual), K(granted));
9479
    }
9480

9481
    // STEP 2: Update info in inner_table in trans
9482
    if (OB_FAIL(ret)) {
9483
    } else if (OB_FAIL(do_migrate_unit_in_trans_(*pool, new_unit, is_manual, granted))) {
9484
      LOG_WARN("do_migrate_unit_in_trans failed", KR(ret), KPC(pool), K(new_unit), K(is_manual), K(granted));
9485
    }
9486

9487
    // STEP 3: Update in-memory info (unit & unit_load & migrate_unit)
9488
    if (OB_FAIL(ret)) {
9489
    } else if (OB_FAIL(do_migrate_unit_inmemory_(new_unit, unit, is_manual, granted))) {
9490
      LOG_WARN("do_migrate_unit_inmemory failed", KR(ret), K(dst), KPC(unit), K(is_manual), K(granted));
9491
    }
9492

9493
    // STEP 4: migration succeed, do some postprocess
9494
    if (OB_SUCC(ret)) {
9495
      // wakeup rootbalance thread to make disaster_recovery process more quickly
9496
      root_balance_->wakeup();
9497
      // add migrate_unit rootservice event
9498
      ROOTSERVICE_EVENT_ADD("unit", "migrate_unit",
9499
          "unit_id", unit->unit_id_,
9500
          "migrate_from_server", unit->migrate_from_server_,
9501
          "server", unit->server_,
9502
          "tenant_id", pool->tenant_id_,
9503
          "manual_migrate", is_manual ? "YES" : "NO");
9504
    }
9505
  }
9506
  LOG_INFO("finish migrate unit", KR(ret), K(unit_id), K(dst), K(is_manual));
9507
  return ret;
9508
}
9509

9510
int ObUnitManager::do_migrate_unit_notify_resource_(const share::ObResourcePool &pool,
9511
                                                    const share::ObUnit &new_unit,
9512
                                                    const bool is_manual,
9513
                                                    const bool granted)
9514
{
9515
  int ret = OB_SUCCESS;
9516
  lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::INVALID;
9517
  if (!check_inner_stat()) {
9518
    ret = OB_INNER_STAT_ERROR;
9519
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
9520
  } else if (!granted) {
9521
    // do nothing. If unit is not granted, there's no need to notify observer.
9522
  } else if (OB_UNLIKELY(nullptr == srv_rpc_proxy_)) {
9523
    ret = OB_ERR_UNEXPECTED;
9524
    LOG_WARN("srv_rpc_proxy_ ptr is null", KR(ret));
9525
  } else if (OB_UNLIKELY(new_unit.resource_pool_id_ != pool.resource_pool_id_)) {
9526
    ret = OB_INVALID_ARGUMENT;
9527
    LOG_WARN("new_unit not belong to pool", KR(ret), K(pool), K(new_unit));
9528
  } else if (OB_FAIL(ObCompatModeGetter::get_tenant_mode(pool.tenant_id_, compat_mode))) {
9529
    LOG_WARN("fail to get tenant compat mode", KR(ret), K(pool));
9530
  } else {
9531
    ObNotifyTenantServerResourceProxy notify_proxy(*srv_rpc_proxy_,
9532
                                                  &obrpc::ObSrvRpcProxy::notify_tenant_server_unit_resource);
9533
    // only notify new unit resource on dst server here.
9534
    // Old unit on src server will be delete later when doing end_migrate
9535
    if (OB_FAIL(try_notify_tenant_server_unit_resource_(
9536
            pool.tenant_id_, false/*is_delete*/, notify_proxy, // is_delete is false when migrate unit
9537
            pool.unit_config_id_, compat_mode, new_unit, false/*if not grant*/,
9538
            false/*skip offline server*/,
9539
            true /*check_data_version*/))) {
9540
      LOG_WARN("fail to try notify server unit resource", K(ret));
9541
    }
9542
    int tmp_ret = OB_SUCCESS;
9543
    if (OB_TMP_FAIL(notify_proxy.wait())) {
9544
      LOG_WARN("fail to wait notify resource", K(ret), K(tmp_ret));
9545
      ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
9546
    } else if (OB_SUCC(ret)) {
9547
      // arg/dest/result can be used here.
9548
    }
9549

9550
    // Rollback persistent unit if persistence failed
9551
    ret = ERRSIM_UNIT_PERSISTENCE_ERROR ? : ret;
9552
    if (OB_FAIL(ret)) {
9553
      LOG_WARN("start to rollback unit persistence", KR(ret), K(new_unit), K(pool.tenant_id_));
9554
      int tmp_ret = OB_SUCCESS;
9555
      ObArray<ObUnit> units;
9556
      if (OB_TMP_FAIL(units.push_back(new_unit))) {
9557
        LOG_WARN("fail to push an element into units", KR(ret), KR(tmp_ret), K(new_unit));
9558
      } else if (OB_TMP_FAIL(rollback_persistent_units_(units, pool, notify_proxy))) {
9559
        LOG_WARN("fail to rollback unit persistence", KR(ret), KR(tmp_ret),
9560
            K(units), K(pool), K(compat_mode));
9561
      }
9562
    }
9563
  }
9564
  return ret;
9565
}
9566

9567
int ObUnitManager::do_migrate_unit_in_trans_(const share::ObResourcePool &pool,
9568
                                             const share::ObUnit &new_unit,
9569
                                             const bool is_manual,
9570
                                             const bool granted)
9571
{
9572
  int ret = OB_SUCCESS;
9573
  common::ObMySQLTransaction trans;
9574
  share::ObResourcePool real_pool;
9575
  if (!check_inner_stat()) {
9576
    ret = OB_INNER_STAT_ERROR;
9577
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
9578
  } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
9579
    LOG_WARN("failed to start trans", K(ret));
9580
  }
9581
  // Double check whether exactly unit is granted by querying pool from inner_table.
9582
  // Because during creating or dropping tenant, there's a period when inner_table
9583
  //     and persistence units are updated while in-memory info not updated yet.
9584
  // We need lock by SELECT FOR UPDATE to assure no other trans is still updating pool when checking.
9585
  if (OB_FAIL(ret)) {
9586
  } else if (OB_FAIL(ut_operator_.get_resource_pool(trans,
9587
                                                    pool.resource_pool_id_,
9588
                                                    true/*select_for_update*/,
9589
                                                    real_pool))) {
9590
    LOG_WARN("fail to get resource_pools from table", KR(ret));
9591
  } else {
9592
    if (pool.is_granted_to_tenant() != real_pool.is_granted_to_tenant()) {
9593
      ret = OB_ERR_UNEXPECTED;
9594
      LOG_WARN("in-memory pool info not accurate, cannot migrate unit now.",
9595
                KR(ret), K(new_unit), K(pool), K(real_pool));
9596
    }
9597
  }
9598
  // Create migrate RS_JOB if it's a manual migration:
9599
  // * If not granted, the job will be set as complete on creating.
9600
  // * If granted, the job will be completed when end_migrate_unit is called.
9601
  if (OB_FAIL(ret)) {
9602
  } else if (is_manual) {
9603
    char ip_buf[common::MAX_IP_ADDR_LENGTH];
9604
    const ObAddr &dst = new_unit.server_;
9605
    (void)dst.ip_to_string(ip_buf, common::MAX_IP_ADDR_LENGTH);
9606
    int64_t job_id = 0;
9607
    if (OB_FAIL(RS_JOB_CREATE_WITH_RET(
9608
        job_id,
9609
        ObRsJobType::JOB_TYPE_MIGRATE_UNIT,
9610
        trans,
9611
        "unit_id", new_unit.unit_id_,
9612
        "svr_ip", ip_buf,
9613
        "svr_port", dst.get_port(),
9614
        "tenant_id", pool.tenant_id_))) {
9615
      LOG_WARN("fail to create rs job MIGRATE_UNIT", KR(ret),
9616
               "tenant_id", pool.tenant_id_,
9617
               "unit_id", new_unit.unit_id_);
9618
    } else if (!granted) {
9619
      // not granted, migration can be done at once, so mark RS_JOB completed
9620
      if (OB_FAIL(RS_JOB_COMPLETE(job_id, OB_SUCCESS, trans))) {
9621
        LOG_WARN("all_rootservice_job update failed", K(ret), K(job_id));
9622
      }
9623
    }
9624
  }
9625
  // Update unit info
9626
  if (OB_FAIL(ret)) {
9627
  } else if (OB_FAIL(ut_operator_.update_unit(trans, new_unit, true/*need_check_conflict_with_clone*/))) {
9628
    LOG_WARN("update_unit failed", K(new_unit), K(ret));
9629
  }
9630
  // End this transaction
9631
  if (trans.is_started()) {
9632
    int tmp_ret = OB_SUCCESS;
9633
    if (OB_SUCCESS != (tmp_ret = trans.end(OB_SUCC(ret)))) {
9634
      ret = OB_SUCC(ret) ? tmp_ret : ret;
9635
      LOG_WARN("trans commit failed", K(tmp_ret), K(ret));
9636
    }
9637
  }
9638
  return ret;
9639
}
9640

9641
int ObUnitManager::do_migrate_unit_inmemory_(const share::ObUnit &new_unit,
9642
                                             share::ObUnit *unit,
9643
                                             const bool is_manual,
9644
                                             const bool granted)
9645
{
9646
  int ret = OB_SUCCESS;
9647
  if (!check_inner_stat()) {
9648
    ret = OB_INNER_STAT_ERROR;
9649
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
9650
  } else if (OB_ISNULL(unit)) {
9651
    ret = OB_ERR_UNEXPECTED;
9652
    LOG_WARN("unit ptr is null", KR(ret), KP(unit));
9653
  } else if (OB_UNLIKELY(unit->unit_id_ != new_unit.unit_id_)) {
9654
    ret = OB_ERR_UNEXPECTED;
9655
    LOG_WARN("old and new unit id not the same", KR(ret), KPC(unit), K(new_unit));
9656
  } else {
9657
    const ObAddr src = unit->server_;
9658
    const ObAddr dst = new_unit.server_;
9659
    // do update unit
9660
    *unit = new_unit;
9661
    // update unit_load
9662
    //    delete old unit load imediately if pool not granted
9663
    if (!granted) {
9664
      if (OB_FAIL(delete_unit_load(src, unit->unit_id_))) {
9665
        LOG_WARN("delete_unit_load failed", K(src), "unit_id", unit->unit_id_, KR(ret));
9666
      }
9667
    }
9668
    //    insert new unit load
9669
    ObUnitLoad load;
9670
    if (OB_FAIL(ret)) {
9671
    } else if (OB_FAIL(gen_unit_load(unit, load))) {
9672
      LOG_WARN("gen_unit_load failed", "unit", *unit, K(ret));
9673
    } else if (OB_FAIL(insert_unit_load(dst, load))) {
9674
      LOG_WARN("insert_unit_load failed", K(dst), K(ret));
9675
    }
9676
    // update migrate_unit list
9677
    if (OB_FAIL(ret)) {
9678
    } else if (granted) {
9679
      if (OB_FAIL(insert_migrate_unit(unit->migrate_from_server_, unit->unit_id_))) {
9680
        LOG_WARN("insert_migrate_unit failed", "unit", *unit, K(ret));
9681
      }
9682
    }
9683
  }
9684
  return ret;
9685
}
9686

9687
int ObUnitManager::inner_try_delete_migrate_unit_resource(
9688
    const uint64_t unit_id,
9689
    const common::ObAddr &migrate_from_server)
9690
{
9691
  int ret = OB_SUCCESS;
9692
  ObUnit *unit = NULL;
9693
  share::ObResourcePool *pool = NULL;
9694
  share::ObUnitConfig *unit_config = nullptr;
9695
  lib::Worker::CompatMode compat_mode = lib::Worker::CompatMode::INVALID;
9696
  bool is_alive = false;
9697
  if (!check_inner_stat()) {
9698
    ret = OB_INNER_STAT_ERROR;
9699
    LOG_WARN("inner stat unexpected", K(ret), K(inited_), K(loaded_));
9700
  } else if (OB_UNLIKELY(OB_INVALID_ID == unit_id)) {
9701
    ret = OB_INVALID_ARGUMENT;
9702
    LOG_WARN("invalid argument", K(ret), K(unit_id));
9703
  } else if (OB_FAIL(get_unit_by_id(unit_id, unit))) {
9704
    LOG_WARN("fail to get unit by id", K(ret), K(unit_id));
9705
  } else if (OB_ISNULL(unit)) {
9706
    ret = OB_ERR_UNEXPECTED;
9707
    LOG_WARN("unit ptr is null", K(ret), KP(unit));
9708
  } else if (!migrate_from_server.is_valid()) {
9709
    LOG_INFO("unit not in migrating, no need to delete src resource", K(unit_id));
9710
  } else if (OB_FAIL(SVR_TRACER.check_server_alive(migrate_from_server, is_alive))) {
9711
    LOG_WARN("fail to check server alive", K(ret), "server", migrate_from_server);
9712
  } else if (!is_alive) {
9713
    LOG_INFO("src server not alive, ignore notify",
9714
             K(unit_id), "server", migrate_from_server);
9715
  } else if (OB_FAIL(get_resource_pool_by_id(unit->resource_pool_id_, pool))) {
9716
    LOG_WARN("failed to get pool", K(ret), K(unit));
9717
  } else if (OB_ISNULL(pool)) {
9718
    ret = OB_ERR_UNEXPECTED;
9719
    LOG_WARN("pool ptr is null", K(ret), KP(pool));
9720
  } else if (OB_FAIL(get_unit_config_by_id(pool->unit_config_id_, unit_config))) {
9721
    LOG_WARN("fail to get unit config by id", K(ret));
9722
  } else if (OB_UNLIKELY(nullptr == unit_config)) {
9723
    ret = OB_ERR_UNEXPECTED;
9724
    LOG_WARN("unit config is null", K(ret), "unit_config_id", pool->unit_config_id_);
9725
  } else if (!pool->is_granted_to_tenant()) {
9726
    LOG_INFO("unit is not granted to any tenant", K(ret), "tenant_id", pool->tenant_id_);
9727
  } else if (OB_FAIL(ObCompatModeGetter::get_tenant_mode(pool->tenant_id_, compat_mode))) {
9728
    LOG_WARN("fail to get tenant compat mode", K(ret),
9729
             "tenant_id", pool->tenant_id_, K(unit_id), "pool", *pool);
9730
  } else {
9731
    const int64_t rpc_timeout = NOTIFY_RESOURCE_RPC_TIMEOUT;
9732
    obrpc::TenantServerUnitConfig tenant_unit_server_config;
9733
    ObNotifyTenantServerResourceProxy notify_proxy(
9734
                                      *srv_rpc_proxy_,
9735
                                      &obrpc::ObSrvRpcProxy::notify_tenant_server_unit_resource);
9736
    if (OB_FAIL(tenant_unit_server_config.init_for_dropping(pool->tenant_id_, true/*delete*/))) {
9737
      LOG_WARN("fail to init tenant server unit config", K(ret), "tenant_id", pool->tenant_id_);
9738
    } else if (OB_FAIL(notify_proxy.call(
9739
            migrate_from_server, rpc_timeout, tenant_unit_server_config))) {
9740
      LOG_WARN("fail to call notify resource to server",
9741
               K(ret), K(rpc_timeout), "unit", *unit, "dest", migrate_from_server);
9742
    }
9743
    int tmp_ret = OB_SUCCESS;
9744
    if (OB_TMP_FAIL(notify_proxy.wait())) {
9745
      LOG_WARN("fail to wait notify resource", K(ret), K(tmp_ret));
9746
      ret = (OB_SUCCESS == ret) ? tmp_ret : ret;
9747
    }
9748
    if (OB_SUCC(ret)) {
9749
      // arg/dest/result can be used here.
9750
      LOG_INFO("notify resource to server succeed", "unit", *unit, "dest", migrate_from_server);
9751
    }
9752
  }
9753
  return ret;
9754
}
9755

9756
int ObUnitManager::end_migrate_unit(const uint64_t unit_id, const EndMigrateOp end_migrate_op)
9757
{
9758
  int ret = OB_SUCCESS;
9759
  uint64_t tenant_id = OB_INVALID_ID;
9760
  if (!check_inner_stat()) {
9761
    ret = OB_INNER_STAT_ERROR;
9762
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
9763
  } else if (OB_INVALID_ID == unit_id) {
9764
    ret = OB_INVALID_ARGUMENT;
9765
    LOG_WARN("invalid argument", K(unit_id), K(ret));
9766
  } else if (end_migrate_op < COMMIT || end_migrate_op > REVERSE) {
9767
    ret = OB_INVALID_ARGUMENT;
9768
    LOG_WARN("invalid end_migrate_op", K(end_migrate_op), K(ret));
9769
  } else {
9770
    ObUnit *unit = NULL;
9771
    common::ObMySQLTransaction trans;
9772
    if (OB_FAIL(get_unit_by_id(unit_id, unit))) {
9773
      LOG_WARN("get_unit_by_id failed", K(unit_id), K(ret));
9774
    } else if (NULL == unit) {
9775
      ret = OB_ERR_UNEXPECTED;
9776
      LOG_WARN("unit is null", KP(unit), K(ret));
9777
    } else if (!unit->migrate_from_server_.is_valid()) {
9778
      // FIXME(jingqian): when can this happened, figure it out
9779
      ret = OB_SUCCESS;
9780
      LOG_WARN("unit is not in migrating status, maybe end_migrate_unit has ever called",
9781
               "unit", *unit, K(ret));
9782
    } else {
9783
      const ObAddr migrate_from_server = unit->migrate_from_server_;
9784
      const ObAddr unit_server = unit->server_;
9785
      const bool is_manual = unit->is_manual_migrate();
9786
      ObUnit new_unit = *unit;
9787
      new_unit.is_manual_migrate_ = false;  // clear manual_migrate
9788
      // generate new unit
9789
      if (COMMIT == end_migrate_op) {
9790
        new_unit.migrate_from_server_.reset();
9791
      } else if (ABORT == end_migrate_op) {
9792
        new_unit.server_ = unit->migrate_from_server_;
9793
        new_unit.migrate_from_server_.reset();
9794
      } else {
9795
        new_unit.server_ = unit->migrate_from_server_;
9796
        new_unit.migrate_from_server_ = unit->server_;
9797
      }
9798

9799
      // update unit in sys_table and in memory
9800
      if (OB_FAIL(ret)) {
9801
      } else if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
9802
        LOG_WARN("failed to start transaction ", K(ret));
9803
      } else if (OB_FAIL(ut_operator_.update_unit(trans, new_unit, true/*need_check_conflict_with_clone*/))) {
9804
        LOG_WARN("ut_operator update unit failed", K(new_unit), K(ret));
9805
      } else {
9806
        if (ABORT == end_migrate_op) {
9807
          if (OB_FAIL(delete_unit_load(unit->server_, unit->unit_id_))) {
9808
            LOG_WARN("delete_unit_load failed", "unit", *unit, K(ret));
9809
          }
9810
        } else if (COMMIT == end_migrate_op) {
9811
          if (OB_FAIL(delete_unit_load(unit->migrate_from_server_, unit->unit_id_))) {
9812
            LOG_WARN("delete_unit_load failed", "unit", *unit, K(ret));
9813
          }
9814
        }
9815
        if (OB_SUCC(ret)) {
9816
          *unit = new_unit;
9817
        }
9818
      }
9819

9820
      // delete migrating unit from migrate_units of migrate_from_server,
9821
      // if REVERSE == op, add migrating unit to migrate_units of unit_server
9822
      if (OB_FAIL(ret)) {
9823
      } else if (OB_FAIL(delete_migrate_unit(migrate_from_server, unit->unit_id_))) {
9824
        LOG_WARN("delete_migrate_unit failed", K(migrate_from_server),
9825
                 "unit_id", unit->unit_id_, K(ret));
9826
      } else if (REVERSE == end_migrate_op) {
9827
        if (OB_FAIL(insert_migrate_unit(unit_server, unit->unit_id_))) {
9828
          LOG_WARN("insert_migrate_unit failed", K(unit_server), "unit_id",
9829
                   unit->unit_id_, K(ret));
9830
        }
9831
      }
9832

9833
      if (OB_SUCC(ret)) {
9834
        int tmp_ret = OB_SUCCESS;
9835
        share::ObResourcePool *pool = NULL;
9836
        if (OB_SUCCESS != (tmp_ret = get_resource_pool_by_id(unit->resource_pool_id_, pool))) {
9837
          LOG_WARN("failed to get pool", K(tmp_ret), K(unit));
9838
        } else {
9839
          tenant_id = pool->tenant_id_;
9840
        }
9841
        ROOTSERVICE_EVENT_ADD("unit", "finish_migrate_unit",
9842
                              "unit_id", unit_id,
9843
                              "end_op", end_migrate_op_type_to_str(end_migrate_op),
9844
                              "migrate_from_server", migrate_from_server,
9845
                              "server", unit_server,
9846
                              "tenant_id", tenant_id,
9847
                              "manual_migrate", is_manual ? "YES" : "NO");
9848

9849
        // complete the job if exists
9850
        char ip_buf[common::MAX_IP_ADDR_LENGTH];
9851
        (void)unit_server.ip_to_string(ip_buf, common::MAX_IP_ADDR_LENGTH);
9852
        int64_t job_id = 0;
9853
        tmp_ret = RS_JOB_FIND(MIGRATE_UNIT, job_id, trans, "unit_id", unit_id,
9854
                              "svr_ip", ip_buf, "svr_port", unit_server.get_port());
9855
        if (OB_SUCCESS == tmp_ret && job_id > 0) {
9856
          tmp_ret = (end_migrate_op == COMMIT) ? OB_SUCCESS :
9857
              (end_migrate_op == REVERSE ? OB_CANCELED : OB_TIMEOUT);
9858
          if (OB_FAIL(RS_JOB_COMPLETE(job_id, tmp_ret, trans))) {
9859
            LOG_WARN("all_rootservice_job update failed", K(ret), K(job_id));
9860
          }
9861
        } else {
9862
          //Can not find the situation, only the user manually opened will write rs_job
9863
          LOG_WARN("no rs job", K(ret), K(tmp_ret), K(unit_id));
9864
        }
9865
      }
9866
      const bool commit = OB_SUCC(ret) ? true:false;
9867
      int tmp_ret = OB_SUCCESS ;
9868
      if (OB_SUCCESS != (tmp_ret = trans.end(commit))) {
9869
        LOG_WARN("tran commit failed", K(tmp_ret));
9870
      }
9871
      ret = OB_SUCC(ret) ? tmp_ret : ret;
9872

9873
      if (OB_SUCC(ret) && COMMIT == end_migrate_op && OB_INVALID_ID != tenant_id) {
9874
        (void)inner_try_delete_migrate_unit_resource(unit_id, migrate_from_server);
9875
      }
9876
    }
9877
  }
9878

9879
  LOG_INFO("end migrate unit", K(unit_id), K(end_migrate_op), K(ret));
9880
  return ret;
9881
}
9882

9883
#define INSERT_ITEM_TO_MAP(map, key, pvalue) \
9884
  do { \
9885
    if (OB_FAIL(ret)) { \
9886
    } else if (OB_FAIL(map.set_refactored(key, pvalue))) { \
9887
      if (OB_HASH_EXIST == ret) { \
9888
        LOG_WARN("key already exist", K(key), K(ret)); \
9889
      } else { \
9890
        LOG_WARN("map set failed", K(ret)); \
9891
      } \
9892
    } else { \
9893
    } \
9894
  } while (false)
9895

9896
#define SET_ITEM_TO_MAP(map, key, value) \
9897
  do { \
9898
    const int overwrite = 1; \
9899
    if (OB_FAIL(ret)) { \
9900
    } else if (OB_FAIL(map.set_refactored(key, value, overwrite))) { \
9901
      LOG_WARN("map set failed", K(ret)); \
9902
    } else { \
9903
    } \
9904
  } while (false)
9905

9906
#define INSERT_ARRAY_TO_MAP(map, key, array) \
9907
  do { \
9908
    if (OB_FAIL(ret)) { \
9909
    } else if (OB_FAIL(map.set_refactored(key, array))) { \
9910
      if (OB_HASH_EXIST == ret) { \
9911
        LOG_WARN("key already exist", K(key), K(ret)); \
9912
      } else { \
9913
        LOG_WARN("map set failed", K(ret)); \
9914
      } \
9915
    } else { \
9916
    } \
9917
  } while (false)
9918

9919
int ObUnitManager::build_unit_map(const ObIArray<ObUnit> &units)
9920
{
9921
  int ret = OB_SUCCESS;
9922
  // units is empty if invoked during bootstrap
9923
  if (!inited_) {
9924
    ret = OB_NOT_INIT;
9925
    LOG_WARN("not init", K(ret));
9926
  } else if (OB_FAIL(id_unit_map_.clear())) {
9927
    LOG_WARN("id_unit_map_ clear failed", K(ret));
9928
  } else {
9929
    for (int64_t i = 0; OB_SUCC(ret) && i < units.count(); ++i) {
9930
      ObUnit *unit = NULL;
9931
      if (NULL == (unit = allocator_.alloc())) {
9932
        ret = OB_ALLOCATE_MEMORY_FAILED;
9933
        LOG_ERROR("alloc unit failed", K(ret));
9934
      } else {
9935
        *unit = units.at(i);
9936
        if (OB_FAIL(insert_unit(unit))) {
9937
          LOG_WARN("insert_unit failed", "unit", *unit, K(ret));
9938
        }
9939

9940
        if (OB_FAIL(ret)) {
9941
          //avoid memory leak
9942
          allocator_.free(unit);
9943
          unit = NULL;
9944
        }
9945
      }
9946
    }
9947
  }
9948
  return ret;
9949
}
9950

9951
int ObUnitManager::build_config_map(const ObIArray<ObUnitConfig> &configs)
9952
{
9953
  int ret = OB_SUCCESS;
9954
  // configs is empty if invoked during bootstrap
9955
  if (!inited_) {
9956
    ret = OB_NOT_INIT;
9957
    LOG_WARN("not init", K(ret));
9958
  } else if (OB_FAIL(id_config_map_.clear())) {
9959
    LOG_WARN("id_config_map_ clear failed", K(ret));
9960
  } else if (OB_FAIL(name_config_map_.clear())) {
9961
    LOG_WARN("name_config_map_ clear failed", K(ret));
9962
  } else {
9963
    for (int64_t i = 0; OB_SUCC(ret) && i < configs.count(); ++i) {
9964
      ObUnitConfig *config = NULL;
9965
      if (NULL == (config = config_allocator_.alloc())) {
9966
        ret = OB_ALLOCATE_MEMORY_FAILED;
9967
        LOG_ERROR("alloc unit config failed", K(ret));
9968
      } else {
9969
        *config = configs.at(i);
9970
        if (OB_FAIL(insert_unit_config(config))) {
9971
          LOG_WARN("insert_unit_config failed", KP(config), K(ret));
9972
        }
9973

9974
        if (OB_FAIL(ret)) {
9975
          config_allocator_.free(config);
9976
          config = NULL;
9977
        }
9978
      }
9979
    }
9980
  }
9981
  return ret;
9982
}
9983

9984
int ObUnitManager::build_pool_map(const ObIArray<share::ObResourcePool> &pools)
9985
{
9986
  int ret = OB_SUCCESS;
9987
  // pools is empty if invoked during bootstrap
9988
  if (!inited_) {
9989
    ret = OB_NOT_INIT;
9990
    LOG_WARN("not init", K(ret));
9991
  } else if (OB_FAIL(id_pool_map_.clear())) {
9992
    LOG_WARN("id_pool_map_ clear failed", K(ret));
9993
  } else if (OB_FAIL(name_pool_map_.clear())) {
9994
    LOG_WARN("name_pool_map_ clear failed", K(ret));
9995
  } else {
9996
    for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
9997
      share::ObResourcePool *pool = NULL;
9998
      if (NULL == (pool = pool_allocator_.alloc())) {
9999
        ret = OB_ALLOCATE_MEMORY_FAILED;
10000
        LOG_ERROR("alloc resource pool failed", K(ret));
10001
      } else {
10002
        if (OB_FAIL(pool->assign(pools.at(i)))) {
10003
          LOG_WARN("failed to assign pool", K(ret));
10004
        } else {
10005
          INSERT_ITEM_TO_MAP(id_pool_map_, pool->resource_pool_id_, pool);
10006
          INSERT_ITEM_TO_MAP(name_pool_map_, pool->name_, pool);
10007
          if (OB_FAIL(ret)) {
10008
          } else if (OB_FAIL(inc_config_ref_count(pool->unit_config_id_))) {
10009
            LOG_WARN("inc_config_ref_count failed", "config id", pool->unit_config_id_, K(ret));
10010
          } else if (OB_FAIL(insert_config_pool(pool->unit_config_id_, pool))) {
10011
            LOG_WARN("insert config pool failed", "config id", pool->unit_config_id_, K(ret));
10012
          }
10013
        }
10014

10015
        if (OB_FAIL(ret)) {
10016
          pool_allocator_.free(pool);
10017
          pool = NULL;
10018
        }
10019
      }
10020
    }
10021
  }
10022
  return ret;
10023
}
10024

10025
int ObUnitManager::insert_unit_config(ObUnitConfig *config)
10026
{
10027
  int ret = OB_SUCCESS;
10028
  if (!inited_) {
10029
    ret = OB_NOT_INIT;
10030
    LOG_WARN("not init", K(ret));
10031
  } else if (NULL == config) {
10032
    ret = OB_INVALID_ARGUMENT;
10033
    LOG_WARN("config is null", KP(config), K(ret));
10034
  } else if (!config->is_valid()) {
10035
    ret = OB_INVALID_ARGUMENT;
10036
    LOG_WARN("invalid config", "config", *config, K(ret));
10037
  } else {
10038
    INSERT_ITEM_TO_MAP(id_config_map_, config->unit_config_id(), config);
10039
    INSERT_ITEM_TO_MAP(name_config_map_, config->name(), config);
10040
    int64_t ref_count = 0;
10041
    SET_ITEM_TO_MAP(config_ref_count_map_, config->unit_config_id(), ref_count);
10042
  }
10043
  return ret;
10044
}
10045

10046
int ObUnitManager::inc_config_ref_count(const uint64_t config_id)
10047
{
10048
  int ret = OB_SUCCESS;
10049
  if (!inited_) {
10050
    ret = OB_NOT_INIT;
10051
    LOG_WARN("not init", K(ret));
10052
  } else if (OB_INVALID_ID == config_id) {
10053
    ret = OB_INVALID_ARGUMENT;
10054
    LOG_WARN("invalid config_id", K(config_id), K(ret));
10055
  } else {
10056
    int64_t ref_count = 0;
10057
    if (OB_FAIL(get_config_ref_count(config_id, ref_count))) {
10058
      LOG_WARN("get_config_ref_count failed", K(config_id), K(ret));
10059
    } else {
10060
      ++ref_count;
10061
      SET_ITEM_TO_MAP(config_ref_count_map_, config_id, ref_count);
10062
    }
10063
  }
10064
  return ret;
10065
}
10066

10067
int ObUnitManager::dec_config_ref_count(const uint64_t config_id)
10068
{
10069
  int ret = OB_SUCCESS;
10070
  if (!inited_) {
10071
    ret = OB_NOT_INIT;
10072
    LOG_WARN("not init", K(ret));
10073
  } else if (OB_INVALID_ID == config_id) {
10074
    ret = OB_INVALID_ARGUMENT;
10075
    LOG_WARN("invalid config_id", K(config_id), K(ret));
10076
  } else {
10077
    int64_t ref_count = 0;
10078
    if (OB_FAIL(get_config_ref_count(config_id, ref_count))) {
10079
      LOG_WARN("get_config_ref_count failed", K(config_id), K(ret));
10080
    } else {
10081
      --ref_count;
10082
      SET_ITEM_TO_MAP(config_ref_count_map_, config_id, ref_count);
10083
    }
10084
  }
10085
  return ret;
10086
}
10087

10088
int ObUnitManager::update_pool_map(share::ObResourcePool *resource_pool)
10089
{
10090
  int ret = OB_SUCCESS;
10091
  if (!inited_) {
10092
    ret = OB_NOT_INIT;
10093
    LOG_WARN("not init", K(ret));
10094
  } else if (NULL == resource_pool) {
10095
    ret = OB_INVALID_ARGUMENT;
10096
    LOG_WARN("resource_pool is null", KP(resource_pool), K(ret));
10097
  } else if (!resource_pool->is_valid() || OB_INVALID_ID == resource_pool->resource_pool_id_) {
10098
    ret = OB_INVALID_ARGUMENT;
10099
    LOG_WARN("invalid resource_pool", "resource_pool", *resource_pool, K(ret));
10100
  } else {
10101
    INSERT_ITEM_TO_MAP(id_pool_map_, resource_pool->resource_pool_id_, resource_pool);
10102
    INSERT_ITEM_TO_MAP(name_pool_map_, resource_pool->name_, resource_pool);
10103
  }
10104
  return ret;
10105
}
10106

10107
int ObUnitManager::insert_unit(ObUnit *unit)
10108
{
10109
  int ret = OB_SUCCESS;
10110
  if (!inited_) {
10111
    ret = OB_NOT_INIT;
10112
    LOG_INFO("not init", K(ret));
10113
  } else if (NULL == unit) {
10114
    ret = OB_INVALID_ARGUMENT;
10115
    LOG_WARN("unit is null", KP(unit), K(ret));
10116
  } else if (!unit->is_valid()) {
10117
    ret = OB_INVALID_ARGUMENT;
10118
    LOG_WARN("invalid unit", "unit", *unit, K(ret));
10119
  } else {
10120
    ObArray<ObUnit *> *units = NULL;
10121
    if (OB_FAIL(get_units_by_pool(unit->resource_pool_id_, units))) {
10122
      if (OB_ENTRY_NOT_EXIST != ret) {
10123
        LOG_WARN("get_units_by_pool failed", K(ret));
10124
      } else {
10125
        ret = OB_SUCCESS;
10126
        if (NULL == (units = pool_unit_allocator_.alloc())) {
10127
          ret = OB_ALLOCATE_MEMORY_FAILED;
10128
          LOG_ERROR("alloc ObArray<ObUnit *> failed", K(ret));
10129
        } else {
10130
          if (OB_FAIL(units->push_back(unit))) {
10131
            LOG_WARN("push_back failed", K(ret));
10132
          } else {
10133
            INSERT_ARRAY_TO_MAP(pool_unit_map_, unit->resource_pool_id_, units);
10134
          }
10135
          if (OB_FAIL(ret)) {
10136
            pool_unit_allocator_.free(units);
10137
            units = NULL;
10138
          }
10139
        }
10140
      }
10141
    } else if (NULL == units) {
10142
      ret = OB_ERR_UNEXPECTED;
10143
      LOG_WARN("units is null", KP(units), K(ret));
10144
    } else if (OB_FAIL(units->push_back(unit))) {
10145
      LOG_WARN("push_back failed", K(ret));
10146
    }
10147

10148
    if (OB_SUCC(ret)) {
10149
      INSERT_ITEM_TO_MAP(id_unit_map_, unit->unit_id_, unit);
10150

10151
      if (OB_FAIL(ret)) {
10152
      } else if (OB_FAIL(insert_unit_loads(unit))) {
10153
        LOG_WARN("insert_unit_loads failed", "unit", *unit, K(ret));
10154
      }
10155
    }
10156
  }
10157
  return ret;
10158
}
10159

10160
int ObUnitManager::insert_unit_loads(ObUnit *unit)
10161
{
10162
  int ret = OB_SUCCESS;
10163
  ObUnitLoad load;
10164
  if (!inited_) {
10165
    ret = OB_NOT_INIT;
10166
    LOG_WARN("not init", K(ret));
10167
  } else if (NULL == unit) {
10168
    ret = OB_INVALID_ARGUMENT;
10169
    LOG_WARN("unit is null", KP(unit), K(ret));
10170
  } else if (!unit->is_valid()) {
10171
    ret = OB_INVALID_ARGUMENT;
10172
    LOG_WARN("invalid unit", "unit", *unit, K(ret));
10173
  } else if (OB_FAIL(gen_unit_load(unit, load))) {
10174
    LOG_WARN("gen_unit_load failed", "unit", *unit, K(ret));
10175
  } else if (OB_FAIL(insert_unit_load(unit->server_, load))) {
10176
    LOG_WARN("insert_unit_load failed", "server", unit->server_, K(ret));
10177
  } else {
10178
    if (OB_SUCCESS == ret && unit->migrate_from_server_.is_valid()) {
10179
      if (OB_FAIL(insert_unit_load(unit->migrate_from_server_, load))) {
10180
        LOG_WARN("insert_unit_load failed", "server", unit->migrate_from_server_, K(ret));
10181
      }
10182
    }
10183
  }
10184
  return ret;
10185
}
10186

10187
int ObUnitManager::insert_unit_load(const ObAddr &server, const ObUnitLoad &load)
10188
{
10189
  int ret = OB_SUCCESS;
10190
  ObArray<ObUnitLoad> *loads = NULL;
10191
  if (!inited_) {
10192
    ret = OB_NOT_INIT;
10193
    LOG_WARN("not init", K(ret));
10194
  } else if (!server.is_valid() || !load.is_valid()) {
10195
    ret = OB_INVALID_ARGUMENT;
10196
    LOG_WARN("invalid argument", K(server), K(load), K(ret));
10197
  } else if (OB_FAIL(get_loads_by_server(server, loads))) {
10198
    if (OB_ENTRY_NOT_EXIST != ret) {
10199
      LOG_WARN("get_loads_by_server failed", K(server), K(ret));
10200
    } else {
10201
      ret = OB_SUCCESS;
10202
      // not exist, alloc new array, add to hash map
10203
      if (NULL == (loads = load_allocator_.alloc())) {
10204
        ret = OB_ALLOCATE_MEMORY_FAILED;
10205
        LOG_ERROR("alloc ObArray<ObUnitLoad> failed", K(ret));
10206
      } else {
10207
        if (OB_FAIL(loads->push_back(load))) {
10208
          LOG_WARN("push_back failed", K(ret));
10209
        } else if (OB_FAIL(insert_load_array(server, loads))) {
10210
          LOG_WARN("insert_unit_load failed", K(server), K(ret));
10211
        }
10212
        if (OB_FAIL(ret)) {
10213
          // avoid memory leak
10214
          load_allocator_.free(loads);
10215
          loads = NULL;
10216
        }
10217
      }
10218
    }
10219
  } else if (NULL == loads) {
10220
    ret = OB_ERR_UNEXPECTED;
10221
    LOG_WARN("loads is null", KP(loads), K(ret));
10222
  } else {
10223
    if (OB_FAIL(loads->push_back(load))) {
10224
      LOG_WARN("push_back failed", K(ret));
10225
    }
10226
  }
10227
  return ret;
10228
}
10229

10230
int ObUnitManager::insert_load_array(const ObAddr &addr, ObArray<ObUnitLoad> *loads)
10231
{
10232
  int ret = OB_SUCCESS;
10233
  if (!inited_) {
10234
    ret = OB_NOT_INIT;
10235
    LOG_WARN("not init", K(ret));
10236
  } else if (!addr.is_valid() || NULL == loads) {
10237
    ret = OB_INVALID_ARGUMENT;
10238
    LOG_WARN("invalid argument", K(addr), KP(loads), K(ret));
10239
  } else if (loads->count() <= 0) {
10240
    ret = OB_INVALID_ARGUMENT;
10241
    LOG_WARN("loads is empty", "loads count", loads->count(), K(ret));
10242
  } else {
10243
    if (OB_FAIL(server_loads_.set_refactored(addr, loads))) {
10244
      if (OB_HASH_EXIST == ret) {
10245
        LOG_WARN("load array is not expect to exist", K(addr), K(ret));
10246
      } else {
10247
        LOG_WARN("set failed", K(addr), K(ret));
10248
      }
10249
    } else {
10250
      // do nothing
10251
    }
10252
  }
10253
  return ret;
10254
}
10255

10256
int ObUnitManager::update_pool_load(share::ObResourcePool *pool,
10257
    share::ObUnitConfig *new_config)
10258
{
10259
  int ret = OB_SUCCESS;
10260
  if (!inited_) {
10261
    ret = OB_NOT_INIT;
10262
    LOG_WARN("not init", K(ret));
10263
  } else if (NULL == pool || NULL == new_config) {
10264
    ret = OB_INVALID_ARGUMENT;
10265
    LOG_WARN("invalid argument", K(ret), KP(pool), KP(new_config));
10266
  } else {
10267
    FOREACH_X(sl, server_loads_, OB_SUCC(ret)) {
10268
      if (NULL == sl->second) {
10269
        ret = OB_ERR_UNEXPECTED;
10270
        LOG_WARN("NULL value", K(ret));
10271
      }
10272
      FOREACH_X(l, *sl->second, OB_SUCC(ret)) {
10273
        if (l->pool_ == pool) {
10274
          l->unit_config_ = new_config;
10275
        }
10276
      }
10277
    }
10278
  }
10279
  return ret;
10280
}
10281

10282
int ObUnitManager::gen_unit_load(ObUnit *unit, ObUnitLoad &load) const
10283
{
10284
  int ret = OB_SUCCESS;
10285
  if (!inited_) {
10286
    ret = OB_NOT_INIT;
10287
    LOG_WARN("not init", K(ret));
10288
  } else if (NULL == unit) {
10289
    ret = OB_INVALID_ARGUMENT;
10290
    LOG_WARN("unit is null", KP(unit), K(ret));
10291
  } else if (!unit->is_valid()) {
10292
    ret = OB_INVALID_ARGUMENT;
10293
    LOG_WARN("unit is invalid", "unit", *unit, K(ret));
10294
  } else {
10295
    ObUnitConfig *config = NULL;
10296
    share::ObResourcePool *pool = NULL;
10297
    if (OB_FAIL(get_resource_pool_by_id(unit->resource_pool_id_, pool))) {
10298
      LOG_WARN("get_resource_pool_by_id failed", "pool id", unit->resource_pool_id_, K(ret));
10299
    } else if (NULL == pool) {
10300
      ret = OB_ERR_UNEXPECTED;
10301
      LOG_WARN("pools is null", KP(pool), K(ret));
10302
    } else if (OB_FAIL(get_unit_config_by_id(pool->unit_config_id_, config))) {
10303
      LOG_WARN("get_unit_config_by_id failed", "unit config id", pool->unit_config_id_, K(ret));
10304
    } else if (NULL == config) {
10305
      ret = OB_ERR_UNEXPECTED;
10306
      LOG_WARN("config is null", KP(config), K(ret));
10307
    } else {
10308
      load.unit_ = unit;
10309
      load.pool_ = pool;
10310
      load.unit_config_ = config;
10311
    }
10312
  }
10313
  return ret;
10314
}
10315

10316
int ObUnitManager::gen_unit_load(const uint64_t unit_id, ObUnitLoad &load) const
10317
{
10318
  int ret = OB_SUCCESS;
10319
  ObUnit *unit = NULL;
10320
  if (!inited_) {
10321
    ret = OB_NOT_INIT;
10322
    LOG_WARN("not init", K(ret));
10323
  } else if (OB_INVALID_ID == unit_id) {
10324
    ret = OB_INVALID_ARGUMENT;
10325
    LOG_WARN("invalid unit_id", K(unit_id), K(ret));
10326
  } else if (OB_FAIL(get_unit_by_id(unit_id, unit))) {
10327
    LOG_WARN("get_unit_by_id failed", K(unit_id), K(ret));
10328
  } else if (NULL == unit) {
10329
    ret = OB_ERR_UNEXPECTED;
10330
    LOG_WARN("unit is null", KP(unit), K(ret));
10331
  } else if (OB_FAIL(gen_unit_load(unit, load))) {
10332
    LOG_WARN("gen_unit_load failed", "unit", *unit, K(ret));
10333
  }
10334
  return ret;
10335
}
10336

10337
int ObUnitManager::insert_tenant_pool(const uint64_t tenant_id, share::ObResourcePool *pool)
10338
{
10339
  int ret = OB_SUCCESS;
10340
  if (!inited_) {
10341
    ret = OB_NOT_INIT;
10342
    LOG_WARN("not init", K(ret));
10343
  } else if (!is_valid_tenant_id(tenant_id) || NULL == pool) {
10344
    ret = OB_INVALID_ARGUMENT;
10345
    LOG_WARN("invalid argument", K(tenant_id), KP(pool), K(ret));
10346
  } else if (OB_FAIL(insert_id_pool(tenant_pools_map_,
10347
      tenant_pools_allocator_, tenant_id, pool))) {
10348
    LOG_WARN("insert tenant pool failed", K(tenant_id), KP(pool), K(ret));
10349
  }
10350

10351
  return ret;
10352
}
10353

10354
int ObUnitManager::insert_config_pool(const uint64_t config_id, share::ObResourcePool *pool)
10355
{
10356
  int ret = OB_SUCCESS;
10357
  if (!inited_) {
10358
    ret = OB_NOT_INIT;
10359
    LOG_WARN("not init", K(ret));
10360
  } else if (OB_INVALID_ID == config_id || NULL == pool) {
10361
    ret = OB_INVALID_ARGUMENT;
10362
    LOG_WARN("invalid argument", K(config_id), KP(pool), K(ret));
10363
  } else if (OB_FAIL(insert_id_pool(config_pools_map_,
10364
      config_pools_allocator_, config_id, pool))) {
10365
    LOG_WARN("insert config pool failed", K(config_id), KP(pool), K(ret));
10366
  }
10367

10368
  return ret;
10369
}
10370

10371
int ObUnitManager::insert_id_pool(
10372
    common::hash::ObHashMap<uint64_t, common::ObArray<share::ObResourcePool *> *> &map,
10373
    common::ObPooledAllocator<common::ObArray<share::ObResourcePool *> > &allocator,
10374
    const uint64_t id,
10375
    share::ObResourcePool *pool)
10376
{
10377
  int ret = OB_SUCCESS;
10378
  if (!inited_) {
10379
    ret = OB_NOT_INIT;
10380
    LOG_WARN("not init", K(ret));
10381
  } else if (!map.created() || OB_INVALID_ID == id || NULL == pool) {
10382
    ret = OB_INVALID_ARGUMENT;
10383
    LOG_WARN("invalid argument", "map created", map.created(), K(id), KP(pool), K(ret));
10384
  } else {
10385
    ObArray<share::ObResourcePool *> *pools = NULL;
10386
    if (OB_FAIL(get_pools_by_id(map, id, pools))) {
10387
      if (OB_ENTRY_NOT_EXIST != ret) {
10388
        LOG_WARN("get_pools_by_id failed", K(id), K(ret));
10389
      } else {
10390
        ret = OB_SUCCESS;
10391
        if (NULL == (pools = allocator.alloc())) {
10392
          ret = OB_ALLOCATE_MEMORY_FAILED;
10393
          LOG_ERROR("alloc pools failed", K(ret));
10394
        } else if (OB_FAIL(pools->push_back(pool))) {
10395
          LOG_WARN("push_back failed", K(ret));
10396
        } else if (OB_FAIL(insert_id_pool_array(map, id, pools))) {
10397
          LOG_WARN("insert_id_pool_array failed", K(id), K(ret));
10398
        }
10399

10400
        // avoid memory leak
10401
        if (OB_SUCCESS != ret && NULL != pools) {
10402
          allocator.free(pools);
10403
          pools = NULL;
10404
        }
10405
      }
10406
    } else if (NULL == pools) {
10407
      ret = OB_ERR_UNEXPECTED;
10408
      LOG_WARN("pools is null", KP(pools), K(ret));
10409
    } else if (!has_exist_in_array(*pools, pool)) {
10410
      if (OB_FAIL(pools->push_back(pool))) {
10411
        LOG_WARN("push_back failed", K(ret));
10412
      }
10413
    } else {
10414
      ret = OB_ERR_UNEXPECTED;
10415
      LOG_WARN("resource pool already exists", K(ret), K(id), K(pools), K(pool));
10416
    }
10417
  }
10418
  return ret;
10419
}
10420

10421
int ObUnitManager::insert_id_pool_array(
10422
    common::hash::ObHashMap<uint64_t, common::ObArray<share::ObResourcePool *> *> &map,
10423
    const uint64_t id,
10424
    ObArray<share::ObResourcePool *> *pools)
10425
{
10426
  int ret = OB_SUCCESS;
10427
  if (!inited_) {
10428
    ret = OB_NOT_INIT;
10429
    LOG_WARN("not init", K(ret));
10430
  } else if (!map.created() || OB_INVALID_ID == id || NULL == pools) {
10431
    ret = OB_INVALID_ARGUMENT;
10432
    LOG_WARN("invalid argument", "map created", map.created(), K(id), KP(pools), K(ret));
10433
  } else {
10434
    if (OB_FAIL(map.set_refactored(id, pools))) {
10435
      if (OB_HASH_EXIST == ret) {
10436
        LOG_WARN("pools is not expect to exist", K(id), K(ret));
10437
      } else {
10438
        LOG_WARN("set failed", K(id), K(ret));
10439
      }
10440
    } else {
10441
      // do nothing
10442
    }
10443
  }
10444
  return ret;
10445
}
10446

10447
int ObUnitManager::insert_migrate_unit(const ObAddr &src_server, const uint64_t unit_id)
10448
{
10449
  int ret = OB_SUCCESS;
10450
  if (!inited_) {
10451
    ret = OB_NOT_INIT;
10452
    LOG_WARN("not init", K(ret));
10453
  } else if (!src_server.is_valid() || OB_INVALID_ID == unit_id) {
10454
    ret = OB_INVALID_ARGUMENT;
10455
    LOG_WARN("invalid argument", K(src_server), K(unit_id), K(ret));
10456
  } else {
10457
    ObArray<uint64_t> *migrate_units = NULL;
10458
    if (OB_FAIL(get_migrate_units_by_server(src_server, migrate_units))) {
10459
      if (OB_ENTRY_NOT_EXIST != ret) {
10460
        LOG_WARN("get_migrate_units_by_server failed", K(src_server), K(ret));
10461
      } else {
10462
        ret= OB_SUCCESS;
10463
        if (NULL == (migrate_units = migrate_units_allocator_.alloc())) {
10464
          ret = OB_ALLOCATE_MEMORY_FAILED;
10465
          LOG_ERROR("alloc  failed", K(ret));
10466
        } else if (OB_FAIL(migrate_units->push_back(unit_id))) {
10467
          LOG_WARN("push_back failed", K(ret));
10468
        } else {
10469
          INSERT_ARRAY_TO_MAP(server_migrate_units_map_, src_server, migrate_units);
10470
        }
10471

10472
        // avoid memory leak
10473
        if (OB_SUCCESS != ret && NULL != migrate_units) {
10474
          migrate_units_allocator_.free(migrate_units);
10475
          migrate_units = NULL;
10476
        }
10477
      }
10478
    } else if (NULL == migrate_units) {
10479
      ret = OB_ERR_UNEXPECTED;
10480
      LOG_WARN("migrate_units is null", KP(migrate_units), K(ret));
10481
    } else {
10482
      if (OB_FAIL(migrate_units->push_back(unit_id))) {
10483
        LOG_WARN("push_back failed", K(ret));
10484
      }
10485
    }
10486
  }
10487
  return ret;
10488
}
10489

10490
int ObUnitManager::set_server_ref_count(common::hash::ObHashMap<ObAddr, int64_t> &map,
10491
    const ObAddr &server, const int64_t ref_count) const
10492
{
10493
  int ret = OB_SUCCESS;
10494
  if (!inited_) {
10495
    ret = OB_NOT_INIT;
10496
    LOG_WARN("not init", K(ret));
10497
  } else if (!map.created() || !server.is_valid() || ref_count <= 0) {
10498
    ret = OB_INVALID_ARGUMENT;
10499
    LOG_WARN("invalid argument", "map_created", map.created(), K(server), K(ref_count), K(ret));
10500
  } else {
10501
    SET_ITEM_TO_MAP(map, server, ref_count);
10502
  }
10503
  return ret;
10504
}
10505

10506

10507
#undef INSERT_ITEM_TO_MAP
10508
#undef SET_ITEM_TO_MAP
10509
#undef INSERT_ARRAY_TO_MAP
10510

10511
#define DELETE_ITEM_FROM_MAP(map, key) \
10512
  do { \
10513
    if (OB_FAIL(ret)) { \
10514
    } else if (OB_FAIL(map.erase_refactored(key))) { \
10515
      LOG_WARN("map erase failed", K(key), K(ret)); \
10516
    } else { \
10517
    } \
10518
  } while (false)
10519

10520
int ObUnitManager::delete_unit_config(const uint64_t config_id,
10521
                                      const ObUnitConfigName &name)
10522
{
10523
  int ret = OB_SUCCESS;
10524
  if (!inited_) {
10525
    ret = OB_NOT_INIT;
10526
    LOG_WARN("not init", K(ret));
10527
  } else if (OB_INVALID_ID == config_id || name.is_empty()) {
10528
    ret = OB_INVALID_ARGUMENT;
10529
    LOG_WARN("invalid argument", K(config_id), K(name), K(ret));
10530
  } else {
10531
    DELETE_ITEM_FROM_MAP(id_config_map_, config_id);
10532
    DELETE_ITEM_FROM_MAP(name_config_map_, name);
10533
    DELETE_ITEM_FROM_MAP(config_ref_count_map_, config_id);
10534
  }
10535
  return ret;
10536
}
10537

10538
int ObUnitManager::delete_resource_pool(const uint64_t pool_id,
10539
                                        const ObResourcePoolName &name)
10540
{
10541
  int ret = OB_SUCCESS;
10542
  if (!inited_) {
10543
    ret = OB_NOT_INIT;
10544
    LOG_WARN("not init", K(ret));
10545
  } else if (OB_INVALID_ID == pool_id || name.is_empty()) {
10546
    ret = OB_INVALID_ARGUMENT;
10547
    LOG_WARN("invalid argument", K(pool_id), K(name), K(ret));
10548
  } else {
10549
    DELETE_ITEM_FROM_MAP(id_pool_map_, pool_id);
10550
    DELETE_ITEM_FROM_MAP(name_pool_map_, name);
10551
  }
10552
  return ret;
10553
}
10554

10555
int ObUnitManager::delete_units_in_zones(
10556
    const uint64_t resource_pool_id,
10557
    const common::ObIArray<common::ObZone> &to_be_removed_zones)
10558
{
10559
  int ret = OB_SUCCESS;
10560
  ObArray<ObUnit *> *units = NULL;
10561
  ObArray<ObUnit *> left_units;
10562
  if (!inited_) {
10563
    ret = OB_NOT_INIT;
10564
    LOG_WARN("not init", K(ret));
10565
  } else if (OB_UNLIKELY(OB_INVALID_ID == resource_pool_id)
10566
             || OB_UNLIKELY(to_be_removed_zones.count() <= 0)) {
10567
    ret = OB_INVALID_ARGUMENT;
10568
    LOG_WARN("invalid argument", K(ret), K(resource_pool_id), K(to_be_removed_zones));
10569
  } else if (OB_FAIL(get_units_by_pool(resource_pool_id, units))) {
10570
    LOG_WARN("fail to get units by pool", K(ret), K(resource_pool_id));
10571
  } else if (OB_UNLIKELY(NULL == units)) {
10572
    ret = OB_ERR_UNEXPECTED;
10573
    LOG_WARN("units is null", K(ret), KP(units));
10574
  } else {
10575
    for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
10576
      const ObUnit *unit = units->at(i);
10577
      if (OB_UNLIKELY(NULL == unit)) {
10578
        ret = OB_ERR_UNEXPECTED;
10579
        LOG_WARN("unit should not be null", "unit", OB_P(units->at(i)), K(ret));
10580
      } else if (!has_exist_in_array(to_be_removed_zones, unit->zone_)) {
10581
        if (OB_FAIL(left_units.push_back(units->at(i)))) {
10582
          LOG_WARN("fail to push back", K(ret));
10583
        } else {} // no more to do
10584
      } else if (OB_FAIL(delete_unit_loads(*unit))) {
10585
        LOG_WARN("fail to delete unit load", K(ret), "unit", *units->at(i));
10586
      } else {
10587
        DELETE_ITEM_FROM_MAP(id_unit_map_, unit->unit_id_);
10588
        if (OB_SUCC(ret)) {
10589
          if (unit->migrate_from_server_.is_valid()) {
10590
            //If the unit is being migrated, delete it from the state in memory
10591
            if (OB_FAIL(delete_migrate_unit(unit->migrate_from_server_,
10592
                                            unit->unit_id_))) {
10593
              LOG_WARN("failed to delete migrate unit", K(ret), "unit", *unit);
10594
            }
10595
          }
10596
        }
10597
        if (OB_SUCC(ret)) {
10598
          ROOTSERVICE_EVENT_ADD("unit", "drop_unit",
10599
              "unit_id", units->at(i)->unit_id_);
10600
          allocator_.free(units->at(i));
10601
          units->at(i) = NULL;;
10602
        }
10603
      }
10604
    }
10605
    if (OB_SUCC(ret)) {
10606
      if (left_units.count() <= 0) {
10607
        DELETE_ITEM_FROM_MAP(pool_unit_map_, resource_pool_id);
10608
        if (OB_SUCC(ret)) {
10609
          pool_unit_allocator_.free(units);
10610
          units = NULL;
10611
        }
10612
      } else {
10613
        if (OB_FAIL(units->assign(left_units))) {
10614
          LOG_WARN("assign failed", K(ret));
10615
        }
10616
      }
10617
    }
10618
  }
10619
  return ret;
10620
}
10621

10622
int ObUnitManager::delete_units_of_pool(const uint64_t resource_pool_id)
10623
{
10624
  int ret = OB_SUCCESS;
10625
  ObArray<ObUnit *> *units = NULL;
10626
  if (!inited_) {
10627
    ret = OB_NOT_INIT;
10628
    LOG_WARN("not init", K(ret));
10629
  } else if (OB_INVALID_ID == resource_pool_id) {
10630
    ret = OB_INVALID_ARGUMENT;
10631
    LOG_WARN("invalid argument", K(resource_pool_id), K(ret));
10632
  } else if (OB_FAIL(get_units_by_pool(resource_pool_id, units))) {
10633
    LOG_WARN("fail to get unit by pool", K(resource_pool_id), K(ret));
10634
  } else if (OB_UNLIKELY(NULL == units)) {
10635
    ret = OB_ERR_UNEXPECTED;
10636
    LOG_WARN("units is null", KP(units), K(ret));
10637
  } else {
10638
    for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
10639
      const ObUnit *unit = units->at(i);
10640
      if (OB_UNLIKELY(NULL == unit)) {
10641
        ret = OB_ERR_UNEXPECTED;
10642
        LOG_WARN("unit should not be null", K(ret));
10643
      } else if (OB_FAIL(delete_unit_loads(*unit))) {
10644
        LOG_WARN("fail to delete unit load", K(ret));
10645
      } else {
10646
        DELETE_ITEM_FROM_MAP(id_unit_map_, unit->unit_id_);
10647
        if (OB_SUCC(ret)) {
10648
          if (unit->migrate_from_server_.is_valid()) {
10649
            if (OB_FAIL(delete_migrate_unit(unit->migrate_from_server_,
10650
                                            unit->unit_id_))) {
10651
              LOG_WARN("failed to delete migrate unit", K(ret), "unit", *unit);
10652
            }
10653
          }
10654

10655
        }
10656

10657
        if (OB_SUCC(ret)) {
10658
          ROOTSERVICE_EVENT_ADD("unit", "drop_unit",
10659
                                "unit_id", units->at(i)->unit_id_);
10660
          allocator_.free(units->at(i));
10661
          units->at(i) = NULL;
10662
        }
10663
      }
10664
    }
10665
    if (OB_SUCC(ret)) {
10666
      DELETE_ITEM_FROM_MAP(pool_unit_map_, resource_pool_id);
10667
      if (OB_SUCC(ret)) {
10668
        pool_unit_allocator_.free(units);
10669
        units = NULL;
10670
      }
10671
    }
10672
  }
10673
  return ret;
10674
}
10675

10676
int ObUnitManager::delete_invalid_inmemory_units(
10677
    const uint64_t resource_pool_id,
10678
    const common::ObIArray<uint64_t> &valid_unit_ids)
10679
{
10680
  int ret = OB_SUCCESS;
10681
  ObArray<share::ObUnit *> *units = NULL;
10682
  ObArray<share::ObUnit *> left_units;
10683
  if (!inited_) {
10684
    ret = OB_NOT_INIT;
10685
    LOG_WARN("not init", K(ret));
10686
  } else if (OB_INVALID_ID == resource_pool_id || valid_unit_ids.count() <= 0) {
10687
    ret = OB_INVALID_ARGUMENT;
10688
    LOG_WARN("invalid argument", K(ret), K(resource_pool_id),
10689
             "valid_unit_cnt", valid_unit_ids.count());
10690
  } else if (OB_FAIL(get_units_by_pool(resource_pool_id, units))) {
10691
    LOG_WARN("fail to get units by pool", K(ret), K(resource_pool_id));
10692
  } else if (OB_UNLIKELY(NULL == units)) {
10693
    ret = OB_ERR_UNEXPECTED;
10694
    LOG_WARN("units is null", KP(units), K(ret));
10695
  } else {
10696
    for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
10697
      if (NULL == units->at(i)) {
10698
        ret = OB_ERR_UNEXPECTED;
10699
        LOG_WARN("unit ptr is null", K(ret));
10700
      } else if (has_exist_in_array(valid_unit_ids, units->at(i)->unit_id_)) {
10701
        if (OB_FAIL(left_units.push_back(units->at(i)))) {
10702
          LOG_WARN("push_back failed", K(ret));
10703
        }
10704
      } else if (OB_FAIL(delete_unit_loads(*units->at(i)))) {
10705
        LOG_WARN("fail to delete unit loads", K(ret), "unit", *units->at(i));
10706
      } else {
10707
        DELETE_ITEM_FROM_MAP(id_unit_map_, units->at(i)->unit_id_);
10708
        if (OB_SUCC(ret)) {
10709
          ROOTSERVICE_EVENT_ADD("unit", "drop_unit",
10710
              "unit_id", units->at(i)->unit_id_);
10711
          allocator_.free(units->at(i));
10712
          units->at(i) = NULL;
10713
        }
10714
      }
10715
    }
10716
    if (OB_SUCC(ret)) {
10717
      if (left_units.count() <= 0) {
10718
        DELETE_ITEM_FROM_MAP(pool_unit_map_, resource_pool_id);
10719
        if (OB_SUCC(ret)) {
10720
          pool_unit_allocator_.free(units);
10721
          units = NULL;
10722
        }
10723
      } else {
10724
        if (OB_FAIL(units->assign(left_units))) {
10725
          LOG_WARN("assign failed", K(ret));
10726
        }
10727
      }
10728
    }
10729
  }
10730
  return ret;
10731
}
10732

10733
int ObUnitManager::delete_inmemory_units(
10734
    const uint64_t resource_pool_id,
10735
    const common::ObIArray<uint64_t> &unit_ids)
10736
{
10737
  int ret = OB_SUCCESS;
10738
  if (!inited_) {
10739
    ret = OB_NOT_INIT;
10740
    LOG_WARN("not init", K(ret));
10741
  } else if (OB_INVALID_ID == resource_pool_id) {
10742
    ret = OB_INVALID_ARGUMENT;
10743
    LOG_WARN("invalid argument", K(resource_pool_id), K(ret));
10744
  } else {
10745
    ObArray<ObUnit *> *units = NULL;
10746
    ObArray<ObUnit *> left_units;
10747
    if (OB_FAIL(get_units_by_pool(resource_pool_id, units))) {
10748
      LOG_WARN("get_units_by_pool failed", K(resource_pool_id), K(ret));
10749
    } else if (NULL == units) {
10750
      ret = OB_ERR_UNEXPECTED;
10751
      LOG_WARN("units is null", KP(units), K(ret));
10752
    } else {
10753
      for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
10754
        if (NULL == units->at(i)) {
10755
          ret = OB_ERR_UNEXPECTED;
10756
          LOG_WARN("unit should not be null", "unit", OB_P(units->at(i)), K(ret));
10757
        } else if (!has_exist_in_array(unit_ids, units->at(i)->unit_id_)) {
10758
          if (OB_FAIL(left_units.push_back(units->at(i)))) {
10759
            LOG_WARN("push_back failed", K(ret));
10760
          }
10761
        } else {
10762
          if (OB_FAIL(delete_unit_loads(*units->at(i)))) {
10763
            LOG_WARN("delete_unit_load failed", "unit", *units->at(i), K(ret));
10764
          } else {
10765
            DELETE_ITEM_FROM_MAP(id_unit_map_, units->at(i)->unit_id_);
10766
            if (OB_SUCC(ret)) {
10767
              ROOTSERVICE_EVENT_ADD("unit", "drop_unit",
10768
                  "unit_id", units->at(i)->unit_id_);
10769
              allocator_.free(units->at(i));
10770
              units->at(i) = NULL;
10771
            }
10772
          }
10773
        }
10774
      }
10775
      if (OB_SUCC(ret)) {
10776
        // if all units of pool are deleted, delete item from hashmap
10777
        if (left_units.count() <= 0) {
10778
          DELETE_ITEM_FROM_MAP(pool_unit_map_, resource_pool_id);
10779
          if (OB_SUCC(ret)) {
10780
            pool_unit_allocator_.free(units);
10781
            units = NULL;
10782
          }
10783
        } else {
10784
          if (OB_FAIL(units->assign(left_units))) {
10785
            LOG_WARN("assign failed", K(ret));
10786
          }
10787
        }
10788
      }
10789
    }
10790
  }
10791
  return ret;
10792
}
10793

10794
int ObUnitManager::delete_unit_loads(const ObUnit &unit)
10795
{
10796
  int ret = OB_SUCCESS;
10797
  if (!inited_) {
10798
    ret = OB_NOT_INIT;
10799
    LOG_WARN("not init", K(ret));
10800
  } else if (!unit.is_valid()) {
10801
    ret = OB_INVALID_ARGUMENT;
10802
    LOG_WARN("invalid unit", K(unit), K(ret));
10803
  } else {
10804
    const uint64_t unit_id = unit.unit_id_;
10805
    if (OB_FAIL(delete_unit_load(unit.server_, unit_id))) {
10806
      LOG_WARN("delete_unit_load failed", "server", unit.server_, K(unit_id), K(ret));
10807
    } else {
10808
      if (unit.migrate_from_server_.is_valid()) {
10809
        if (OB_FAIL(delete_unit_load(unit.migrate_from_server_, unit_id))) {
10810
          LOG_WARN("delete_unit_load failed", "server", unit.migrate_from_server_,
10811
              K(unit_id), K(ret));
10812
        } else {
10813
          // do nothing
10814
        }
10815
      }
10816
    }
10817
  }
10818
  return ret;
10819
}
10820

10821
int ObUnitManager::delete_unit_load(const ObAddr &server, const uint64_t unit_id)
10822
{
10823
  int ret = OB_SUCCESS;
10824
  if (!inited_) {
10825
    ret = OB_NOT_INIT;
10826
    LOG_WARN("not init", K(ret));
10827
  } else if (!server.is_valid() || OB_INVALID_ID == unit_id) {
10828
    ret = OB_INVALID_ARGUMENT;
10829
    LOG_WARN("invalid argument", K(server), K(unit_id), K(ret));
10830
  } else {
10831
    ObArray<ObUnitLoad> *loads = NULL;
10832
    if (OB_FAIL(get_loads_by_server(server, loads))) {
10833
      LOG_WARN("get_loads_by_server failed", K(server), K(ret));
10834
    } else if (NULL == loads) {
10835
      ret = OB_ERR_UNEXPECTED;
10836
      LOG_WARN("loads is null", KP(loads), K(ret));
10837
    } else {
10838
      int64_t index = -1;
10839
      for (int64_t i = 0; OB_SUCC(ret) && i < loads->count(); ++i) {
10840
        if (loads->at(i).unit_->unit_id_ == unit_id) {
10841
          index = i;
10842
          break;
10843
        }
10844
      }
10845
      if (-1 == index) {
10846
        ret = OB_ENTRY_NOT_EXIST;
10847
        LOG_WARN("unit load not exist", K(server), K(unit_id), K(ret));
10848
      } else {
10849
        for (int64_t i = index; i < loads->count() - 1; ++i) {
10850
          loads->at(i) = loads->at(i+1);
10851
        }
10852
        loads->pop_back();
10853
        if (0 == loads->count()) {
10854
          load_allocator_.free(loads);
10855
          loads = NULL;
10856
          DELETE_ITEM_FROM_MAP(server_loads_, server);
10857
        }
10858
      }
10859
    }
10860
  }
10861
  return ret;
10862
}
10863

10864
int ObUnitManager::delete_tenant_pool(const uint64_t tenant_id, share::ObResourcePool *pool)
10865
{
10866
  int ret = OB_SUCCESS;
10867
  if (!inited_) {
10868
    ret = OB_NOT_INIT;
10869
    LOG_WARN("not init", K(ret));
10870
  } else if (!is_valid_tenant_id(tenant_id) || NULL == pool) {
10871
    ret = OB_INVALID_ARGUMENT;
10872
    LOG_WARN("invalid argument", K(tenant_id), KP(pool), K(ret));
10873
  } else if (OB_FAIL(delete_id_pool(tenant_pools_map_,
10874
      tenant_pools_allocator_, tenant_id, pool))) {
10875
    LOG_WARN("delete tenant pool failed", K(ret), K(tenant_id), "pool", *pool);
10876
  }
10877
  return ret;
10878
}
10879

10880
int ObUnitManager::delete_config_pool(const uint64_t config_id, share::ObResourcePool *pool)
10881
{
10882
  int ret = OB_SUCCESS;
10883
  if (!inited_) {
10884
    ret = OB_NOT_INIT;
10885
    LOG_WARN("not init", K(ret));
10886
  } else if (OB_INVALID_ID == config_id || NULL == pool) {
10887
    ret = OB_INVALID_ARGUMENT;
10888
    LOG_WARN("invalid argument", K(config_id), KP(pool), K(ret));
10889
  } else if (OB_FAIL(delete_id_pool(config_pools_map_,
10890
      config_pools_allocator_, config_id, pool))) {
10891
    LOG_WARN("delete config pool failed", K(ret), K(config_id), "pool", *pool);
10892
  }
10893
  return ret;
10894
}
10895

10896
int ObUnitManager::delete_id_pool(
10897
    common::hash::ObHashMap<uint64_t, common::ObArray<share::ObResourcePool *> *> &map,
10898
    common::ObPooledAllocator<common::ObArray<share::ObResourcePool *> > &allocator,
10899
    const uint64_t id,
10900
    share::ObResourcePool *pool)
10901
{
10902
  int ret = OB_SUCCESS;
10903
  if (!inited_) {
10904
    ret = OB_NOT_INIT;
10905
    LOG_WARN("not init", K(ret));
10906
  } else if (!map.created() || OB_INVALID_ID == id || NULL == pool) {
10907
    ret = OB_INVALID_ARGUMENT;
10908
    LOG_WARN("invalid argument", "map created", map.created(), K(id), KP(pool), K(ret));
10909
  } else {
10910
    ObArray<share::ObResourcePool *> *pools = NULL;
10911
    if (OB_FAIL(get_pools_by_id(map, id, pools))) {
10912
      LOG_WARN("get_pools_by_id failed", K(id), K(ret));
10913
    } else if (NULL == pools) {
10914
      ret = OB_ERR_UNEXPECTED;
10915
      LOG_WARN("pools is null", KP(pools), K(ret));
10916
    } else {
10917
      int64_t index = -1;
10918
      for (int64_t i = 0; i < pools->count(); ++i) {
10919
        if (pools->at(i) == pool) {
10920
          index = i;
10921
          break;
10922
        }
10923
      }
10924
      if (-1 == index) {
10925
        ret = OB_ENTRY_NOT_EXIST;
10926
        LOG_WARN("pool not exist", K(id), K(ret));
10927
      } else if (OB_FAIL(pools->remove(index))) {
10928
        LOG_WARN("remove failed", K(index), K(ret));
10929
      } else if (0 == pools->count()) {
10930
        allocator.free(pools);
10931
        pools = NULL;
10932
        DELETE_ITEM_FROM_MAP(map, id);
10933
      }
10934
    }
10935
  }
10936
  return ret;
10937
}
10938

10939
int ObUnitManager::delete_migrate_unit(const ObAddr &src_server,
10940
                                       const uint64_t unit_id)
10941
{
10942
  int ret = OB_SUCCESS;
10943
  if (!inited_) {
10944
    ret = OB_NOT_INIT;
10945
    LOG_WARN("not init", K(ret));
10946
  } else if (!src_server.is_valid() || OB_INVALID_ID == unit_id) {
10947
    ret = OB_INVALID_ARGUMENT;
10948
    LOG_WARN("invalid argument", K(src_server), K(unit_id), K(ret));
10949
  } else {
10950
    ObArray<uint64_t> *migrate_units = NULL;
10951
    if (OB_FAIL(get_migrate_units_by_server(src_server, migrate_units))) {
10952
      LOG_WARN("get_migrate_units_by_server failed", K(src_server), K(ret));
10953
    } else if (NULL == migrate_units) {
10954
      ret = OB_ERR_UNEXPECTED;
10955
      LOG_WARN("migrate_units is null", KP(migrate_units), K(ret));
10956
    } else {
10957
      int64_t index = -1;
10958
      for (int64_t i = 0; i < migrate_units->count(); ++i) {
10959
        if (migrate_units->at(i) == unit_id) {
10960
          index = i;
10961
          break;
10962
        }
10963
      }
10964
      if (-1 == index) {
10965
        ret = OB_ENTRY_NOT_EXIST;
10966
        LOG_WARN("migrate_unit not exist", K(unit_id), K(ret));
10967
      } else if (OB_FAIL(migrate_units->remove(index))) {
10968
        LOG_WARN("remove failed", K(index), K(ret));
10969
      } else if (0 == migrate_units->count()) {
10970
        migrate_units_allocator_.free(migrate_units);
10971
        migrate_units = NULL;
10972
        DELETE_ITEM_FROM_MAP(server_migrate_units_map_, src_server);
10973
      }
10974
    }
10975
  }
10976
  return ret;
10977
}
10978

10979
int ObUnitManager::inner_drop_resource_pool(share::ObResourcePool *pool)
10980
{
10981
  int ret = OB_SUCCESS;
10982

10983
  if (OB_ISNULL(pool)) {
10984
    ret = OB_ERR_UNEXPECTED;
10985
    LOG_WARN("pool is null", KP(pool), K(ret));
10986
  } else if (pool->is_granted_to_tenant()) {
10987
    ret = OB_RESOURCE_POOL_ALREADY_GRANTED;
10988
    LOG_USER_ERROR(OB_RESOURCE_POOL_ALREADY_GRANTED, to_cstring(pool->name_));
10989
    LOG_WARN("resource pool is granted to tenant, can't not delete it",
10990
             "tenant_id", pool->tenant_id_, K(ret));
10991
  } else {
10992
    common::ObMySQLTransaction trans;
10993
    if (OB_FAIL(trans.start(proxy_, OB_SYS_TENANT_ID))) {
10994
      LOG_WARN("start transaction failed", K(ret));
10995
    } else if (OB_FAIL(remove_resource_pool_unit_in_trans(pool->resource_pool_id_,
10996
                                                          trans))) {
10997
      LOG_WARN("failed to remove resource pool and unit", K(ret), K(pool));
10998
    }
10999
    if (trans.is_started()) {
11000
      const bool commit = (OB_SUCC(ret));
11001
      int temp_ret = OB_SUCCESS;
11002
      if (OB_SUCCESS != (temp_ret = trans.end(commit))) {
11003
        LOG_WARN("trans end failed", K(commit), K(temp_ret));
11004
        ret = (OB_SUCCESS == ret) ? temp_ret : ret;
11005
      }
11006
    }
11007

11008
    if (OB_FAIL(ret)) {
11009
    } else if (OB_FAIL(delete_resource_pool_unit(pool))) {
11010
      LOG_WARN("failed to delete resource pool or unit", K(ret), K(pool));
11011
    }
11012
  }
11013

11014
  return ret;
11015
}
11016

11017
#undef DELETE_ITEM_FROM_MAP
11018

11019
#define GET_ITEM_FROM_MAP(map, key, value) \
11020
  do { \
11021
    if (OB_FAIL(map.get_refactored(key, value))) { \
11022
      if (OB_HASH_NOT_EXIST == ret) { \
11023
        ret = OB_ENTRY_NOT_EXIST; \
11024
      } else { \
11025
        LOG_WARN("map get failed", K(key), K(ret)); \
11026
      } \
11027
    } else { \
11028
    } \
11029
  } while (false)
11030

11031
int ObUnitManager::update_unit_load(
11032
    share::ObUnit *unit,
11033
    share::ObResourcePool *new_pool)
11034
{
11035
  int ret = OB_SUCCESS;
11036
  if (OB_UNLIKELY(!inited_)) {
11037
    ret = OB_NOT_INIT;
11038
    LOG_WARN("not init", K(ret));
11039
  } else if (NULL == unit || NULL == new_pool) {
11040
    ret = OB_INVALID_ARGUMENT;
11041
    LOG_WARN("invalid argument", K(ret), KP(unit), KP(new_pool));
11042
  } else {
11043
    ObArray<ObUnitManager::ObUnitLoad> *server_load = NULL;
11044
    GET_ITEM_FROM_MAP(server_loads_, unit->server_, server_load);
11045
    if (OB_FAIL(ret)) {
11046
      LOG_WARN("fail to get server load", K(ret), "server", unit->server_);
11047
    } else if (NULL == server_load) {
11048
      ret = OB_ERR_UNEXPECTED;
11049
      LOG_WARN("loads ptr is null", K(ret));
11050
    } else {
11051
      FOREACH_X(l, *server_load, OB_SUCC(ret)) {
11052
        if (l->unit_ == unit) {
11053
          l->pool_ = new_pool;
11054
        }
11055
      }
11056
    }
11057
    if (OB_FAIL(ret)) {
11058
    } else if (unit->migrate_from_server_.is_valid()) {
11059
      ObArray<ObUnitManager::ObUnitLoad> *migrate_from_load = NULL;
11060
      GET_ITEM_FROM_MAP(server_loads_, unit->migrate_from_server_, migrate_from_load);
11061
      if (OB_FAIL(ret)) {
11062
         LOG_WARN("fail to get server load", K(ret),
11063
                  "migrate_from_server", unit->migrate_from_server_);
11064
      } else if (NULL == migrate_from_load) {
11065
        ret = OB_ERR_UNEXPECTED;
11066
        LOG_WARN("loads ptr is null", K(ret));
11067
      } else {
11068
        FOREACH_X(l, *migrate_from_load, OB_SUCC(ret)) {
11069
          if (l->unit_ == unit) {
11070
            l->pool_ = new_pool;
11071
          }
11072
        }
11073
      }
11074
    }
11075
  }
11076
  return ret;
11077
}
11078

11079
int ObUnitManager::get_unit_config_by_name(const ObUnitConfigName &name,
11080
                                           ObUnitConfig *&config) const
11081
{
11082
  int ret = OB_SUCCESS;
11083
  config = NULL;
11084
  if (!inited_) {
11085
    ret = OB_NOT_INIT;
11086
    LOG_WARN("not init", K(ret));
11087
  } else if (name.is_empty()) {
11088
    ret = OB_INVALID_ARGUMENT;
11089
    LOG_WARN("invalid name", K(name), K(ret));
11090
  } else {
11091
    GET_ITEM_FROM_MAP(name_config_map_, name, config);
11092
  }
11093
  return ret;
11094
}
11095

11096
int ObUnitManager::get_unit_config_by_id(const uint64_t config_id, ObUnitConfig *&config) const
11097
{
11098
  int ret = OB_SUCCESS;
11099
  config = NULL;
11100
  if (!inited_) {
11101
    ret = OB_NOT_INIT;
11102
    LOG_WARN("not init", K(ret));
11103
  } else if (OB_INVALID_ID == config_id) {
11104
    ret = OB_INVALID_ARGUMENT;
11105
    LOG_WARN("invalid argument", K(config_id), K(ret));
11106
  } else {
11107
    GET_ITEM_FROM_MAP(id_config_map_, config_id, config);
11108
  }
11109
  return ret;
11110
}
11111

11112
int ObUnitManager::get_config_ref_count(const uint64_t config_id, int64_t &ref_count) const
11113
{
11114
  int ret = OB_SUCCESS;
11115
  ref_count = 0;
11116
  if (!inited_) {
11117
    ret = OB_NOT_INIT;
11118
    LOG_WARN("not init", K(ret));
11119
  } else if (OB_INVALID_ID == config_id) {
11120
    ret = OB_INVALID_ARGUMENT;
11121
    LOG_WARN("invalid argument", K(config_id), K(ret));
11122
  } else {
11123
    GET_ITEM_FROM_MAP(config_ref_count_map_, config_id, ref_count);
11124
    if (OB_FAIL(ret)) {
11125
      if (OB_ENTRY_NOT_EXIST == ret) {
11126
      } else {
11127
        LOG_WARN("GET_ITEM_FROM_MAP failed", K(config_id), K(ret));
11128
      }
11129
    }
11130
  }
11131
  return ret;
11132
}
11133

11134
int ObUnitManager::get_server_ref_count(common::hash::ObHashMap<ObAddr, int64_t> &map,
11135
    const ObAddr &server, int64_t &ref_count) const
11136
{
11137
  int ret = OB_SUCCESS;
11138
  ref_count = 0;
11139
  if (!inited_) {
11140
    ret = OB_NOT_INIT;
11141
    LOG_WARN("not init", K(ret));
11142
  } else if (!map.created() || !server.is_valid()) {
11143
    ret = OB_INVALID_ARGUMENT;
11144
    LOG_WARN("invalid argument", "map created", map.created(), K(server), K(ret));
11145
  } else {
11146
    GET_ITEM_FROM_MAP(map, server, ref_count);
11147
    if (OB_FAIL(ret)) {
11148
      if (OB_ENTRY_NOT_EXIST == ret) {
11149
      } else {
11150
        LOG_WARN("GET_ITEM_FROM_MAP failed", K(server), K(ret));
11151
      }
11152
    }
11153
  }
11154
  return ret;
11155
}
11156

11157
int ObUnitManager::check_resource_pool_exist(const share::ObResourcePoolName &resource_pool_name,
11158
                                             bool &is_exist)
11159
{
11160
  int ret = OB_SUCCESS;
11161
  SpinRLockGuard guard(lock_);
11162
  share::ObResourcePool *pool = NULL;
11163
  is_exist = false;
11164

11165
  if (!check_inner_stat()) {
11166
    ret = OB_INNER_STAT_ERROR;
11167
    LOG_WARN("check_inner_stat failed", KR(ret), K(inited_), K(loaded_));
11168
  } else if (resource_pool_name.is_empty()) {
11169
    ret = OB_INVALID_ARGUMENT;
11170
    LOG_WARN("invalid argument", KR(ret), K(resource_pool_name));
11171
  } else if (OB_FAIL(inner_get_resource_pool_by_name(resource_pool_name, pool))) {
11172
    if (OB_ENTRY_NOT_EXIST != ret) {
11173
      LOG_WARN("inner_get_resource_pool_by_name failed", KR(ret), K(resource_pool_name));
11174
    } else {
11175
      ret = OB_SUCCESS;
11176
      // is_exist = false
11177
    }
11178
  } else if (OB_ISNULL(pool)) {
11179
    ret = OB_ERR_UNEXPECTED;
11180
    LOG_WARN("unexpected null", KR(ret), KP(pool), K(resource_pool_name));
11181
  } else {
11182
    is_exist = true;
11183
  }
11184

11185
  return ret;
11186
}
11187

11188
int ObUnitManager::inner_get_resource_pool_by_name(
11189
    const ObResourcePoolName &name,
11190
    share::ObResourcePool *&pool) const
11191
{
11192
  int ret = OB_SUCCESS;
11193
  pool = NULL;
11194
  if (!inited_) {
11195
    ret = OB_NOT_INIT;
11196
    LOG_WARN("not init", K(ret));
11197
  } else if (name.is_empty()) {
11198
    ret = OB_INVALID_ARGUMENT;
11199
    LOG_WARN("invalid name", K(name), K(ret));
11200
  } else {
11201
    GET_ITEM_FROM_MAP(name_pool_map_, name, pool);
11202
  }
11203
  return ret;
11204
}
11205

11206
int ObUnitManager::get_resource_pool_by_id(const uint64_t pool_id, share::ObResourcePool *&pool) const
11207
{
11208
  int ret = OB_SUCCESS;
11209
  pool = NULL;
11210
  if (!inited_) {
11211
    ret = OB_NOT_INIT;
11212
    LOG_WARN("not init", K(ret));
11213
  } else if (OB_INVALID_ID == pool_id) {
11214
    ret = OB_INVALID_ARGUMENT;
11215
    LOG_WARN("invalid argument", K(pool_id), K(ret));
11216
  } else {
11217
    GET_ITEM_FROM_MAP(id_pool_map_, pool_id, pool);
11218
  }
11219
  return ret;
11220
}
11221

11222
int ObUnitManager::get_units_by_pool(const uint64_t pool_id, ObArray<ObUnit *> *&units) const
11223
{
11224
  int ret = OB_SUCCESS;
11225
  units = NULL;
11226
  if (!inited_) {
11227
    ret = OB_NOT_INIT;
11228
    LOG_WARN("not init", K(ret));
11229
  } else if (OB_INVALID_ID == pool_id) {
11230
    ret = OB_INVALID_ARGUMENT;
11231
    LOG_WARN("invalid argument", K(pool_id), K(ret));
11232
  } else {
11233
    GET_ITEM_FROM_MAP(pool_unit_map_, pool_id, units);
11234
  }
11235
  return ret;
11236
}
11237

11238
int ObUnitManager::get_unit_by_id(const uint64_t unit_id, ObUnit *&unit) const
11239
{
11240
  int ret = OB_SUCCESS;
11241
  unit = NULL;
11242
  if (!inited_) {
11243
    ret = OB_NOT_INIT;
11244
    LOG_WARN("not init", K(ret));
11245
  } else if (OB_INVALID_ID == unit_id) {
11246
    ret = OB_INVALID_ARGUMENT;
11247
    LOG_WARN("invalid argument", K(unit_id), K(ret));
11248
  } else {
11249
    GET_ITEM_FROM_MAP(id_unit_map_, unit_id, unit);
11250
  }
11251
  return ret;
11252
}
11253

11254
int ObUnitManager::get_loads_by_server(const ObAddr &addr,
11255
                                       ObArray<ObUnitManager::ObUnitLoad> *&loads) const
11256
{
11257
  int ret = OB_SUCCESS;
11258
  loads = NULL;
11259
  if (!inited_) {
11260
    ret = OB_NOT_INIT;
11261
    LOG_WARN("not init", K(ret));
11262
  } else if (!addr.is_valid()) {
11263
    ret = OB_INVALID_ARGUMENT;
11264
    LOG_WARN("invalid argument", K(addr), K(ret));
11265
  } else {
11266
    GET_ITEM_FROM_MAP(server_loads_, addr, loads);
11267
  }
11268
  return ret;
11269
}
11270

11271
int ObUnitManager::get_pools_by_tenant_(const uint64_t tenant_id,
11272
                                       ObArray<share::ObResourcePool *> *&pools) const
11273
{
11274
  int ret = OB_SUCCESS;
11275
  // meta tenant has no self resource pool, here return user tenant resource pool
11276
  uint64_t valid_tnt_id = is_meta_tenant(tenant_id) ? gen_user_tenant_id(tenant_id) : tenant_id;
11277

11278
  pools = NULL;
11279
  if (!inited_) {
11280
    ret = OB_NOT_INIT;
11281
    LOG_WARN("not init", K(ret));
11282
  } else if (!is_valid_tenant_id(tenant_id)) {
11283
    ret = OB_INVALID_ARGUMENT;
11284
    LOG_WARN("invalid argument", K(tenant_id), K(ret));
11285
  } else if (OB_FAIL(get_pools_by_id(tenant_pools_map_, valid_tnt_id, pools))) {
11286
    if (OB_ENTRY_NOT_EXIST != ret) {
11287
      LOG_WARN("get_pools_by_id failed", K(valid_tnt_id), K(ret));
11288
    }
11289
  }
11290
  return ret;
11291
}
11292

11293
int ObUnitManager::get_pools_by_config(const uint64_t config_id,
11294
                                       ObArray<share::ObResourcePool *> *&pools) const
11295
{
11296
  int ret = OB_SUCCESS;
11297
  pools = NULL;
11298
  if (!inited_) {
11299
    ret = OB_NOT_INIT;
11300
    LOG_WARN("not init", K(ret));
11301
  } else if (OB_INVALID_ID == config_id) {
11302
    ret = OB_INVALID_ARGUMENT;
11303
    LOG_WARN("invalid argument", K(config_id), K(ret));
11304
  } else if (OB_FAIL(get_pools_by_id(config_pools_map_, config_id, pools))) {
11305
    if (OB_ENTRY_NOT_EXIST != ret) {
11306
      LOG_WARN("get_pools_by_id failed", K(config_id), K(ret));
11307
    }
11308
  }
11309
  return ret;
11310
}
11311

11312
int ObUnitManager::get_pools_by_id(
11313
    const common::hash::ObHashMap<uint64_t, common::ObArray<share::ObResourcePool *> *> &map,
11314
    const uint64_t id, ObArray<share::ObResourcePool *> *&pools) const
11315
{
11316
  int ret = OB_SUCCESS;
11317
  pools = NULL;
11318
  if (!inited_) {
11319
    ret = OB_NOT_INIT;
11320
    LOG_WARN("not init", K(ret));
11321
  } else if (!map.created() || OB_INVALID_ID == id) {
11322
    ret = OB_INVALID_ARGUMENT;
11323
    LOG_WARN("invalid argument", "map created", map.created(), K(id), K(ret));
11324
  } else {
11325
    GET_ITEM_FROM_MAP(map, id, pools);
11326
  }
11327
  return ret;
11328
}
11329

11330
int ObUnitManager::inner_check_single_logonly_pool_for_locality(
11331
    const share::ObResourcePool &pool,
11332
    const common::ObIArray<share::ObZoneReplicaAttrSet> &zone_locality,
11333
    bool &is_legal)
11334
{
11335
  int ret = OB_SUCCESS;
11336
  if (!inited_) {
11337
    ret = OB_NOT_INIT;
11338
    LOG_WARN("ObUnitManager not init", K(ret), K(inited_));
11339
  } else if (REPLICA_TYPE_LOGONLY != pool.replica_type_) {
11340
    ret = OB_ERR_UNEXPECTED;
11341
    LOG_WARN("pool replica type unexpected", K(ret), K(pool));
11342
  } else {
11343
    is_legal = true;
11344
    for (int64_t i = 0; is_legal && i < pool.zone_list_.count(); ++i) {
11345
      const common::ObZone &zone = pool.zone_list_.at(i);
11346
      for (int64_t j = 0; is_legal && j < zone_locality.count(); ++j) {
11347
        const common::ObIArray<common::ObZone> &zone_set = zone_locality.at(j).zone_set_;
11348
        if (zone_set.count() <= 1) {
11349
          // bypass Non-mixed locality
11350
        } else { // mixed locality
11351
          is_legal = !has_exist_in_array(zone_set, zone);
11352
        }
11353
      }
11354
    }
11355
  }
11356
  return ret;
11357
}
11358

11359
int ObUnitManager::inner_check_logonly_pools_for_locality(
11360
    const common::ObIArray<share::ObResourcePool *> &pools,
11361
    const common::ObIArray<share::ObZoneReplicaAttrSet> &zone_locality,
11362
    bool &is_legal)
11363
{
11364
  int ret = OB_SUCCESS;
11365
  if (!inited_) {
11366
    ret = OB_NOT_INIT;
11367
    LOG_WARN("ObUnitManager not init", K(ret), K(inited_));
11368
  } else if (OB_UNLIKELY(pools.count() <= 0)) {
11369
    ret = OB_INVALID_ARGUMENT;
11370
    LOG_WARN("invalid arguemnt", K(ret), "pool_count", pools.count());
11371
  } else {
11372
    is_legal = true;
11373
    for (int64_t i = 0; is_legal && OB_SUCC(ret) && i < pools.count(); ++i) {
11374
      share::ObResourcePool *pool = pools.at(i);
11375
      if (OB_UNLIKELY(nullptr == pool)) {
11376
        ret = OB_ERR_UNEXPECTED;
11377
        LOG_WARN("pool ptr is null", K(ret), KP(pool));
11378
      } else if (REPLICA_TYPE_LOGONLY != pool->replica_type_) {
11379
        // bypass, since this is not logonly pool
11380
      } else if (OB_FAIL(inner_check_single_logonly_pool_for_locality(
11381
              *pool, zone_locality, is_legal))) {
11382
        LOG_WARN("fail to inner check single logonly pool for locality", K(ret));
11383
      }
11384
    }
11385
  }
11386
  return ret;
11387
}
11388

11389
int ObUnitManager::inner_check_pools_unit_num_enough_for_locality(
11390
    const common::ObIArray<share::ObResourcePool *> &pools,
11391
    const common::ObIArray<common::ObZone> &schema_zone_list,
11392
    const ZoneLocalityIArray &zone_locality,
11393
    bool &is_enough)
11394
{
11395
  int ret = OB_SUCCESS;
11396
  if (!inited_) {
11397
    ret = OB_NOT_INIT;
11398
    LOG_WARN("ObUnitManager not init", K(ret), K(inited_));
11399
  } else if (pools.count() <= 0) {
11400
    ret = OB_INVALID_ARGUMENT;
11401
    LOG_WARN("invalid argument", K(ret), K(pools.count()));
11402
  } else {
11403
    common::ObArray<int64_t> unit_nums;
11404
    for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
11405
      share::ObResourcePool *pool = pools.at(i);
11406
      if (NULL == pool) {
11407
        ret = OB_ERR_UNEXPECTED;
11408
        LOG_WARN("pool ptr is null", K(ret), KP(pool));
11409
      } else if (OB_FAIL(unit_nums.push_back(pool->unit_count_))) {
11410
        LOG_WARN("fail to push back", K(ret));
11411
      } else {} // no more to do
11412
    }
11413
    if (OB_FAIL(ret)) {
11414
    } else if (OB_FAIL(do_check_shrink_granted_pool_allowed_by_locality(
11415
            pools, schema_zone_list, zone_locality, unit_nums, is_enough))) {
11416
      LOG_WARN("fail to check pools unit num enough for locality", K(ret));
11417
    } else {} // no more to do
11418
  }
11419
  return ret;
11420
}
11421

11422
// The legality check consists of two parts
11423
// 1 In the locality of the mixed scene, there can be no logonly resource pool
11424
// 2 Can the unit num of the resource pool fit all locality copies
11425
int ObUnitManager::check_pools_unit_legality_for_locality(
11426
    const common::ObIArray<share::ObResourcePoolName> &pools,
11427
    const common::ObIArray<common::ObZone> &schema_zone_list,
11428
    const ZoneLocalityIArray &zone_locality,
11429
    bool &is_legal)
11430
{
11431
  int ret = OB_SUCCESS;
11432
  if (!inited_) {
11433
    ret = OB_NOT_INIT;
11434
    LOG_WARN("ObUnitManager not init", K(ret), K(inited_));
11435
  } else if (pools.count() <= 0) {
11436
    ret = OB_INVALID_ARGUMENT;
11437
    LOG_WARN("invalid argument", K(ret), K(pools.count()));
11438
  } else {
11439
    SpinRLockGuard guard(lock_);
11440
    common::ObArray<share::ObResourcePool *> pool_ptrs;
11441
    for (int64_t i = 0; OB_SUCC(ret) && i < pools.count(); ++i) {
11442
      share::ObResourcePool *pool_ptr = NULL;
11443
      if (OB_FAIL(inner_get_resource_pool_by_name(pools.at(i), pool_ptr))) {
11444
        LOG_WARN("fail to get resource pool by name", K(ret), "pool_name", pools.at(i));
11445
      } else if (OB_FAIL(pool_ptrs.push_back(pool_ptr))) {
11446
        LOG_WARN("fail to push back", K(ret));
11447
      } else {} // no more to do
11448
    }
11449
    if (OB_FAIL(ret)) {
11450
    } else if (OB_FAIL(inner_check_logonly_pools_for_locality(
11451
            pool_ptrs, zone_locality, is_legal))) {
11452
      LOG_WARN("fail to check logonly pools for locality", K(ret));
11453
    } else if (!is_legal) {
11454
      // no need to check any more
11455
    } else if (OB_FAIL(inner_check_pools_unit_num_enough_for_locality(
11456
            pool_ptrs, schema_zone_list, zone_locality, is_legal))) {
11457
      LOG_WARN("fail to check pools unit num enough for locality", K(ret));
11458
    } else {} // no more to do
11459
  }
11460
  return ret;
11461
}
11462

11463
int ObUnitManager::get_migrate_units_by_server(const ObAddr &server,
11464
                                               common::ObIArray<uint64_t> &migrate_units) const
11465
{
11466
  int ret = OB_SUCCESS;
11467
  ObArray<uint64_t> *units= NULL;
11468
  if (OB_FAIL(get_migrate_units_by_server(server, units))) {
11469
    if (OB_ENTRY_NOT_EXIST == ret) {
11470
      ret = OB_SUCCESS; // no migrating units
11471
    } else {
11472
      LOG_WARN("fail get migrate units by server", K(server), K(ret));
11473
    }
11474
  } else {
11475
    for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
11476
      const uint64_t unit_id = units->at(i);
11477
      if (OB_FAIL(migrate_units.push_back(unit_id))) {
11478
        LOG_WARN("fail push back unit id to array", K(unit_id), K(i), K(ret));
11479
      }
11480
    }
11481
  }
11482
  return ret;
11483
}
11484

11485
int ObUnitManager::get_migrate_units_by_server(const ObAddr &server,
11486
                                               common::ObArray<uint64_t> *&migrate_units) const
11487
{
11488
  int ret = OB_SUCCESS;
11489
  migrate_units = NULL;
11490
  if (!inited_) {
11491
    ret = OB_NOT_INIT;
11492
    LOG_WARN("not init", K(ret));
11493
  } else if (!server.is_valid()) {
11494
    ret = OB_INVALID_ARGUMENT;
11495
    LOG_WARN("invalid server", K(server), K(ret));
11496
  } else {
11497
    GET_ITEM_FROM_MAP(server_migrate_units_map_, server, migrate_units);
11498
  }
11499
  return ret;
11500
}
11501

11502
#undef DELETE_ITEM_FROM_MAP
11503

11504
int ObUnitManager::fetch_new_unit_config_id(uint64_t &config_id)
11505
{
11506
  int ret = OB_SUCCESS;
11507
  if (!inited_) {
11508
    ret = OB_NOT_INIT;
11509
    LOG_WARN("not init", K(ret));
11510
  } else {
11511
    ObMaxIdFetcher id_fetcher(*proxy_);
11512
    uint64_t combine_id = OB_INVALID_ID;
11513
    if (OB_FAIL(id_fetcher.fetch_new_max_id(
11514
        OB_SYS_TENANT_ID, OB_MAX_USED_UNIT_CONFIG_ID_TYPE, combine_id))) {
11515
      LOG_WARN("fetch_max_id failed", "id_type", OB_MAX_USED_UNIT_CONFIG_ID_TYPE, K(ret));
11516
    } else {
11517
      config_id = combine_id;
11518
    }
11519
  }
11520
  return ret;
11521
}
11522

11523
int ObUnitManager::fetch_new_resource_pool_id(uint64_t &resource_pool_id)
11524
{
11525
  int ret = OB_SUCCESS;
11526
  if (!inited_) {
11527
    ret = OB_NOT_INIT;
11528
    LOG_WARN("not init", K(ret));
11529
  } else {
11530
    uint64_t combine_id = OB_INVALID_ID;
11531
    ObMaxIdFetcher id_fetcher(*proxy_);
11532
    if (OB_FAIL(id_fetcher.fetch_new_max_id(OB_SYS_TENANT_ID,
11533
        OB_MAX_USED_RESOURCE_POOL_ID_TYPE, combine_id))) {
11534
      LOG_WARN("fetch_new_max_id failed", "id_type", OB_MAX_USED_RESOURCE_POOL_ID_TYPE, K(ret));
11535
    } else {
11536
      resource_pool_id = combine_id;
11537
    }
11538
  }
11539
  return ret;
11540
}
11541

11542
int ObUnitManager::fetch_new_unit_group_id(uint64_t &unit_group_id)
11543
{
11544
  int ret = OB_SUCCESS;
11545
  if (OB_UNLIKELY(!inited_)) {
11546
    ret = OB_NOT_INIT;
11547
    LOG_WARN("not init", K(ret));
11548
  } else {
11549
    uint64_t combine_id = OB_INVALID_ID;
11550
    ObMaxIdFetcher id_fetcher(*proxy_);
11551
    if (OB_FAIL(id_fetcher.fetch_new_max_id(OB_SYS_TENANT_ID,
11552
        OB_MAX_USED_UNIT_GROUP_ID_TYPE, combine_id))) {
11553
      LOG_WARN("fetch_new_max_id failed", "id_type", OB_MAX_USED_UNIT_ID_TYPE, K(ret));
11554
    } else {
11555
      unit_group_id = combine_id;
11556
    }
11557
  }
11558
  return ret;
11559
}
11560

11561
int ObUnitManager::fetch_new_unit_id(uint64_t &unit_id)
11562
{
11563
  int ret = OB_SUCCESS;
11564
  if (!inited_) {
11565
    ret = OB_NOT_INIT;
11566
    LOG_WARN("not init", K(ret));
11567
  } else {
11568
    uint64_t combine_id = OB_INVALID_ID;
11569
    ObMaxIdFetcher id_fetcher(*proxy_);
11570
    if (OB_FAIL(id_fetcher.fetch_new_max_id(OB_SYS_TENANT_ID,
11571
        OB_MAX_USED_UNIT_ID_TYPE, combine_id))) {
11572
      LOG_WARN("fetch_new_max_id failed", "id_type", OB_MAX_USED_UNIT_ID_TYPE, K(ret));
11573
    } else {
11574
      unit_id = combine_id;
11575
    }
11576
  }
11577
  return ret;
11578
}
11579

11580
int ObUnitManager::check_bootstrap_pool(const share::ObResourcePool &pool)
11581
{
11582
  int ret = OB_SUCCESS;
11583
  if (!pool.is_valid()) {
11584
    ret = OB_INVALID_ARGUMENT;
11585
    LOG_WARN("invalid pool", K(pool), K(ret));
11586
  } else if (ObUnitConfig::SYS_UNIT_CONFIG_ID == pool.unit_config_id_) {
11587
    // good
11588
  } else {
11589
    ret = OB_INVALID_ARGUMENT;
11590
    LOG_WARN("pool not sys pool", K(pool), K(ret));
11591
  }
11592
  return ret;
11593
}
11594

11595
int ObUnitManager::inner_get_tenant_zone_full_unit_num(
11596
    const int64_t tenant_id,
11597
    const common::ObZone &zone,
11598
    int64_t &unit_num)
11599
{
11600
  int ret = OB_SUCCESS;
11601
  ObArray<share::ObResourcePool *> *pools = NULL;
11602
  if (OB_UNLIKELY(!inited_)) {
11603
    ret = OB_NOT_INIT;
11604
    LOG_WARN("not init", K(ret));
11605
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id))) {
11606
    ret = OB_INVALID_ARGUMENT;
11607
    LOG_WARN("invalid argument", K(ret), K(tenant_id));
11608
  } else if (OB_FAIL(get_pools_by_tenant_(tenant_id, pools))) {
11609
    LOG_WARN("fail to get pools by tenant", K(ret), K(tenant_id));
11610
  } else if (OB_UNLIKELY(NULL == pools)) {
11611
    ret = OB_ERR_UNEXPECTED;
11612
    LOG_WARN("pools ptr is null", K(ret), KP(pools));
11613
  } else {
11614
    bool find = false;
11615
    for (int64_t i = 0; !find && OB_SUCC(ret) && i < pools->count(); ++i) {
11616
      share::ObResourcePool *pool = pools->at(i);
11617
      if (NULL == pool) {
11618
        ret = OB_ERR_UNEXPECTED;
11619
        LOG_WARN("pool ptr is null", K(ret), KP(pool));
11620
      } else if (!has_exist_in_array(pool->zone_list_, zone)) {
11621
        // not in this pool
11622
      } else if (REPLICA_TYPE_FULL == pool->replica_type_) {
11623
        unit_num = pool->unit_count_;
11624
        find = true;
11625
      } else {} // not a full replica type resource pool, go on to check next
11626
    }
11627
    if (OB_FAIL(ret)) {
11628
    } else if (!find) {
11629
      unit_num = 0;
11630
    }
11631
  }
11632
  return ret;
11633
}
11634

11635
int ObUnitManager::get_tenant_zone_unit_loads(
11636
    const int64_t tenant_id,
11637
    const common::ObZone &zone,
11638
    const common::ObReplicaType replica_type,
11639
    common::ObIArray<ObUnitManager::ObUnitLoad> &unit_loads)
11640
{
11641
  int ret = OB_SUCCESS;
11642
  ObArray<share::ObResourcePool *> *pools = NULL;
11643
  unit_loads.reset();
11644
  if (OB_UNLIKELY(!inited_)) {
11645
    ret = OB_NOT_INIT;
11646
    LOG_WARN("not init", K(ret));
11647
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id)
11648
                         || zone.is_empty()
11649
                         || !ObReplicaTypeCheck::is_replica_type_valid(replica_type))) {
11650
    ret = OB_INVALID_ARGUMENT;
11651
    LOG_WARN("invalid argument", K(ret), K(tenant_id), K(zone), K(replica_type));
11652
  } else if (OB_FAIL(get_pools_by_tenant_(tenant_id, pools))) {
11653
    if (OB_ENTRY_NOT_EXIST == ret) {
11654
      ret = OB_SUCCESS;
11655
    } else {
11656
      LOG_WARN("fail to get pools by tenant", K(ret), K(tenant_id));
11657
    }
11658
  } else if (OB_UNLIKELY(NULL == pools)) {
11659
    ret = OB_ERR_UNEXPECTED;
11660
    LOG_WARN("pool ptr is null", K(ret), KP(pools));
11661
  } else {
11662
    unit_loads.reset();
11663
    share::ObResourcePool *target_pool = NULL;
11664
    for (int64_t i = 0; NULL == target_pool && OB_SUCC(ret) && i < pools->count(); ++i) {
11665
      share::ObResourcePool *pool = pools->at(i);
11666
      if (NULL == pool) {
11667
        ret = OB_ERR_UNEXPECTED;
11668
        LOG_WARN("pool ptr is null", K(ret), KP(pool));
11669
      } else if (!has_exist_in_array(pool->zone_list_, zone)) {
11670
        // not in this pool
11671
      } else if (replica_type == pool->replica_type_) {
11672
        target_pool = pool;
11673
      } else {} // not a full replica type resource pool, go on to check next
11674
    }
11675
    if (OB_FAIL(ret)) {
11676
    } else if (NULL == target_pool) {
11677
      ret = OB_ENTRY_NOT_EXIST;
11678
    } else {
11679
      ObArray<share::ObUnit *> *units = NULL;
11680
      if (OB_FAIL(get_units_by_pool(target_pool->resource_pool_id_, units))) {
11681
        LOG_WARN("fail to get units by pool", K(ret),
11682
                 "pool_id", target_pool->resource_pool_id_);
11683
      } else if (OB_UNLIKELY(NULL == units)) {
11684
        ret = OB_ERR_UNEXPECTED;
11685
        LOG_WARN("units ptr is null", K(ret), KP(units));
11686
      } else {
11687
        for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
11688
          ObUnitLoad unit_load;
11689
          ObUnit *unit = units->at(i);
11690
          if (NULL == unit) {
11691
            ret = OB_ERR_UNEXPECTED;
11692
            LOG_WARN("unit ptr is null", K(ret));
11693
          } else if (unit->zone_ != zone) {
11694
            // not this zone, ignore
11695
          } else if (OB_FAIL(gen_unit_load(unit, unit_load))) {
11696
            LOG_WARN("fail to gen unit load", K(ret));
11697
          } else if (OB_FAIL(unit_loads.push_back(unit_load))) {
11698
            LOG_WARN("fail to push back", K(ret));
11699
          } else {} // no more to do
11700
        }
11701
      }
11702
    }
11703
  }
11704
  return ret;
11705
}
11706

11707
int ObUnitManager::get_tenant_zone_all_unit_loads(
11708
    const int64_t tenant_id,
11709
    const common::ObZone &zone,
11710
    common::ObIArray<ObUnitManager::ObUnitLoad> &unit_loads)
11711
{
11712
  int ret = OB_SUCCESS;
11713
  ObArray<share::ObResourcePool *> *pools = NULL;
11714
  unit_loads.reset();
11715
  if (OB_UNLIKELY(!inited_)) {
11716
    ret = OB_NOT_INIT;
11717
    LOG_WARN("not init", K(ret));
11718
  } else if (OB_UNLIKELY(!is_valid_tenant_id(tenant_id) || zone.is_empty())) {
11719
    ret = OB_INVALID_ARGUMENT;
11720
    LOG_WARN("invalid argument", K(ret), K(tenant_id), K(zone));
11721
  } else if (OB_FAIL(get_pools_by_tenant_(tenant_id, pools))) {
11722
    if (OB_ENTRY_NOT_EXIST == ret) {
11723
      ret = OB_SUCCESS;
11724
    } else {
11725
      LOG_WARN("fail to get pools by tenant", K(ret), K(tenant_id));
11726
    }
11727
  } else if (OB_UNLIKELY(NULL == pools)) {
11728
    ret = OB_ERR_UNEXPECTED;
11729
    LOG_WARN("pool ptr is null", K(ret), KP(pools));
11730
  } else {
11731
    unit_loads.reset();
11732
    common::ObArray<share::ObResourcePool *> target_pools;
11733
    for (int64_t i = 0; OB_SUCC(ret) && i < pools->count(); ++i) {
11734
      share::ObResourcePool *pool = pools->at(i);
11735
      if (NULL == pool) {
11736
        ret = OB_ERR_UNEXPECTED;
11737
        LOG_WARN("pool ptr is null", K(ret), KP(pool));
11738
      } else if (!has_exist_in_array(pool->zone_list_, zone)) {
11739
        // not in this pool
11740
      } else if (OB_FAIL(target_pools.push_back(pool))) {
11741
        LOG_WARN("fail to push back", K(ret));
11742
      } else {} // no more to do
11743
    }
11744
    for (int64_t i = 0; OB_SUCC(ret) && i < target_pools.count(); ++i) {
11745
      ObArray<share::ObUnit *> *units = NULL;
11746
      share::ObResourcePool *target_pool = target_pools.at(i);
11747
      if (OB_UNLIKELY(NULL == target_pool)) {
11748
        ret = OB_ERR_UNEXPECTED;
11749
        LOG_WARN("target pool ptr is null", K(ret), KP(target_pool));
11750
      } else if (OB_FAIL(get_units_by_pool(target_pool->resource_pool_id_, units))) {
11751
        LOG_WARN("fail to get units by pool", K(ret),
11752
                 "pool_id", target_pool->resource_pool_id_);
11753
      } else if (OB_UNLIKELY(NULL == units)) {
11754
        ret = OB_ERR_UNEXPECTED;
11755
        LOG_WARN("units ptr is null", K(ret), KP(units));
11756
      } else {
11757
        for (int64_t i = 0; OB_SUCC(ret) && i < units->count(); ++i) {
11758
          ObUnitLoad unit_load;
11759
          ObUnit *unit = units->at(i);
11760
          if (NULL == unit) {
11761
            ret = OB_ERR_UNEXPECTED;
11762
            LOG_WARN("unit ptr is null", K(ret));
11763
          } else if (unit->zone_ != zone) {
11764
            // not this zone, ignore
11765
          } else if (OB_FAIL(gen_unit_load(unit, unit_load))) {
11766
            LOG_WARN("fail to gen unit load", K(ret));
11767
          } else if (OB_FAIL(unit_loads.push_back(unit_load))) {
11768
            LOG_WARN("fail to push back", K(ret));
11769
          } else {} // no more to do
11770
        }
11771
      }
11772
    }
11773
    if (OB_FAIL(ret)) {
11774
    } else if (unit_loads.count() <= 0) {
11775
      ret = OB_ENTRY_NOT_EXIST;
11776
    }
11777
  }
11778
  return ret;
11779
}
11780

11781
int ObUnitManager::get_logonly_unit_by_tenant(const int64_t tenant_id,
11782
                                              ObIArray<ObUnitInfo> &logonly_unit_infos)
11783
{
11784
  int ret = OB_SUCCESS;
11785
  share::schema::ObSchemaGetterGuard schema_guard;
11786
  if (OB_ISNULL(schema_service_)) {
11787
    ret = OB_NOT_INIT;
11788
    LOG_WARN("schema service is null", K(schema_service_), KR(ret));
11789
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) {
11790
    LOG_WARN("fail to get tenant schema guard", KR(ret), K(tenant_id));
11791
  } else if (OB_FAIL(get_logonly_unit_by_tenant(schema_guard, tenant_id, logonly_unit_infos))) {
11792
    LOG_WARN("get logonly unit by tenant fail", KR(ret), K(tenant_id));
11793
  }
11794
  return ret;
11795
}
11796

11797
int ObUnitManager::get_logonly_unit_by_tenant(share::schema::ObSchemaGetterGuard &schema_guard,
11798
                                              const int64_t tenant_id,
11799
                                              ObIArray<ObUnitInfo> &logonly_unit_infos)
11800
{
11801
  int ret = OB_SUCCESS;
11802
  const ObTenantSchema *tenant_schema = NULL;
11803
  logonly_unit_infos.reset();
11804
  ObArray<ObUnitInfo> unit_infos;
11805
  SpinRLockGuard guard(lock_);
11806
  if (!check_inner_stat()) {
11807
    ret = OB_INNER_STAT_ERROR;
11808
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
11809
  } else if (!is_valid_tenant_id(tenant_id)) {
11810
    ret = OB_INVALID_ARGUMENT;
11811
    LOG_WARN("invalid argument", K(ret), K(tenant_id));
11812
  } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
11813
    LOG_WARN("fail to get tenant info", K(ret), K(tenant_id));
11814
  } else if (OB_ISNULL(tenant_schema)) {
11815
    ret = OB_ERR_UNEXPECTED;
11816
    LOG_WARN("get invalid tenant_schema", K(ret), K(tenant_id));
11817
  } else if (OB_FAIL(inner_get_active_unit_infos_of_tenant(*tenant_schema, unit_infos))) {
11818
    LOG_WARN("fail to get active unit", K(ret), K(tenant_id));
11819
  } else {
11820
    for (int64_t i = 0; i < unit_infos.count() && OB_SUCC(ret); i++) {
11821
      if (REPLICA_TYPE_LOGONLY != unit_infos.at(i).unit_.replica_type_) {
11822
        //nothing todo
11823
      } else if (OB_FAIL(logonly_unit_infos.push_back(unit_infos.at(i)))) {
11824
        LOG_WARN("fail to push back", K(ret), K(unit_infos));
11825
      }
11826
    }
11827
  }
11828
  return ret;
11829
}
11830

11831
int ObUnitManager::get_unit_infos(const common::ObIArray<share::ObResourcePoolName> &pools,
11832
                                  ObIArray<ObUnitInfo> &unit_infos)
11833
{
11834
  int ret = OB_SUCCESS;
11835
  SpinRLockGuard guard(lock_);
11836
  unit_infos.reset();
11837
  if (pools.count() <= 0) {
11838
    ret = OB_INVALID_ARGUMENT;
11839
    LOG_WARN("invalid argument", K(ret), K(pools));
11840
  } else {
11841
    ObArray<ObUnitInfo> pool_units;
11842
    share::ObResourcePool *pool = NULL;
11843
    for (int64_t i = 0; i < pools.count() && OB_SUCC(ret); i++) {
11844
      pool_units.reset();
11845
      if (OB_FAIL(inner_get_resource_pool_by_name(pools.at(i), pool))) {
11846
        LOG_WARN("fail to get resource pool", K(ret), K(i), K(pools));
11847
      } else if (OB_ISNULL(pool)) {
11848
        ret = OB_ERR_UNEXPECTED;
11849
        LOG_WARN("get invalid pool", K(ret), K(pools));
11850
      } else if (OB_FAIL(inner_get_unit_infos_of_pool_(pool->resource_pool_id_, pool_units))) {
11851
        LOG_WARN("fail to get unit infos", K(ret), K(*pool));
11852
      } else {
11853
        for (int64_t j = 0; j < pool_units.count() && OB_SUCC(ret); j++) {
11854
          if (OB_FAIL(unit_infos.push_back(pool_units.at(j)))) {
11855
            LOG_WARN("fail to push back", K(ret), K(pool_units));
11856
          }
11857
        }
11858
      }
11859
    }
11860
  }
11861
  return ret;
11862
}
11863

11864
int ObUnitManager::get_servers_by_pools(
11865
    const common::ObIArray<share::ObResourcePoolName> &pools,
11866
    common::ObIArray<ObAddr> &addrs)
11867
{
11868
  int ret = OB_SUCCESS;
11869
  addrs.reset();
11870
  ObArray<share::ObUnitInfo> unit_infos;
11871
  if (OB_FAIL(get_unit_infos(pools, unit_infos))) {
11872
    LOG_WARN("fail to get unit infos", KR(ret), K(pools));
11873
  }
11874
  for (int64_t i = 0; OB_SUCC(ret) && i < unit_infos.count(); i++) {
11875
    const share::ObUnitInfo &unit_info = unit_infos.at(i);
11876
    if (OB_FAIL(addrs.push_back(unit_info.unit_.server_))) {
11877
      LOG_WARN("fail to push back addr", KR(ret), K(unit_info));
11878
    }
11879
  } // end for
11880
  return ret;
11881
}
11882

11883
int ObUnitManager::inner_get_active_unit_infos_of_tenant(const ObTenantSchema &tenant_schema,
11884
                                                  ObIArray<ObUnitInfo> &unit_info)
11885
{
11886
  int ret = OB_SUCCESS;
11887
  ObArray<uint64_t> pool_ids;
11888
  uint64_t tenant_id = tenant_schema.get_tenant_id();
11889
  common::ObArray<common::ObZone> tenant_zone_list;
11890
  ObArray<share::ObResourcePool  *> *pools = NULL;
11891
  if (!check_inner_stat()) {
11892
    ret = OB_INNER_STAT_ERROR;
11893
    LOG_WARN("check_inner_stat failed", K(inited_), K(loaded_), K(ret));
11894
  } else if (OB_FAIL(tenant_schema.get_zone_list(tenant_zone_list))) {
11895
    LOG_WARN("fail to get zone list", K(ret));
11896
  } else if (OB_FAIL(get_pools_by_tenant_(tenant_id, pools))) {
11897
    if (OB_ENTRY_NOT_EXIST != ret) {
11898
      LOG_WARN("get_pools_by_tenant failed", K(tenant_id), K(ret));
11899
    } else {
11900
      // just return empty pool_ids
11901
      ret = OB_SUCCESS;
11902
      LOG_WARN("tenant doesn't own any pool", K(tenant_id), K(ret));
11903
    }
11904
  } else if (NULL == pools) {
11905
    ret = OB_ERR_UNEXPECTED;
11906
    LOG_WARN("pools is null", KP(pools), K(ret));
11907
  } else {
11908
    for (int64_t i = 0; OB_SUCC(ret) && i < pools->count(); ++i) {
11909
      if (NULL == pools->at(i)) {
11910
        ret = OB_ERR_UNEXPECTED;
11911
        LOG_WARN("pool is null", "pool", OB_P(pools->at(i)), K(ret));
11912
      } else if (OB_FAIL(pool_ids.push_back(pools->at(i)->resource_pool_id_))) {
11913
        LOG_WARN("push_back failed", K(ret));
11914
      }
11915
    }
11916
  }
11917
  if (OB_SUCC(ret)) {
11918
    ObArray<ObUnitInfo> unit_in_pool;
11919
    for (int64_t i = 0; i < pool_ids.count() && OB_SUCC(ret); i++) {
11920
      uint64_t pool_id = pool_ids.at(i);
11921
      unit_in_pool.reset();
11922
      if (OB_FAIL(inner_get_unit_infos_of_pool_(pool_id, unit_in_pool))) {
11923
        LOG_WARN("fail to inner get unit infos", K(ret), K(pool_id));
11924
      } else {
11925
        for (int64_t j = 0; j < unit_in_pool.count() && OB_SUCC(ret); j++) {
11926
          if (ObUnit::UNIT_STATUS_ACTIVE != unit_in_pool.at(j).unit_.status_) {
11927
            //nothing todo
11928
          } else if (!has_exist_in_array(tenant_zone_list, unit_in_pool.at(j).unit_.zone_)) {
11929
            //nothing todo
11930
          } else if (OB_FAIL(unit_info.push_back(unit_in_pool.at(j)))) {
11931
            LOG_WARN("fail to push back", K(ret), K(unit_in_pool), K(j));
11932
          }
11933
        }
11934
      }
11935
    }
11936
  }
11937
  return ret;
11938
}
11939

11940
}//end namespace share
11941
}//end namespace oceanbase
11942