oceanbase

1
/**
2
 * Copyright (c) 2021 OceanBase
3
 * OceanBase CE is licensed under Mulan PubL v2.
4
 * You can use this software according to the terms and conditions of the Mulan PubL v2.
5
 * You may obtain a copy of Mulan PubL v2 at:
6
 *          http://license.coscl.org.cn/MulanPubL-2.0
7
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10
 * See the Mulan PubL v2 for more details.
11
 */
12

13
#define USING_LOG_PREFIX RS
14

15
#include "observer/ob_server.h"
16
#include "rootserver/ob_root_service.h"
17

18
#include "share/ob_define.h"
19
#include "lib/time/ob_time_utility.h"
20
#include "lib/string/ob_sql_string.h"
21
#include "lib/utility/ob_preload.h"
22
#include "lib/worker.h"
23
#include "lib/oblog/ob_log_module.h"
24
#include "lib/container/ob_se_array_iterator.h"
25
#include "lib/container/ob_array_iterator.h"
26
#include "lib/file/file_directory_utils.h"
27
#include "lib/encrypt/ob_encrypted_helper.h"
28

29
#include "share/ob_srv_rpc_proxy.h"
30
#include "share/ob_thread_mgr.h"
31
#include "common/ob_timeout_ctx.h"
32
#include "common/object/ob_object.h"
33
#include "share/ob_cluster_version.h"
34

35
#include "share/ob_define.h"
36
#include "share/ob_version.h"
37
#include "share/schema/ob_schema_service.h"
38
#include "share/schema/ob_multi_version_schema_service.h"
39
#include "share/schema/ob_schema_utils.h"
40
#include "share/schema/ob_schema_mgr.h"
41
#include "share/ob_lease_struct.h"
42
#include "share/ob_common_rpc_proxy.h"
43
#include "share/config/ob_config_helper.h"
44
#include "share/config/ob_config_manager.h"
45
#include "share/inner_table/ob_inner_table_schema.h"
46
#include "share/schema/ob_part_mgr_util.h"
47
#include "share/ob_web_service_root_addr.h"
48
#include "share/ob_inner_config_root_addr.h"
49
#include "share/ob_global_stat_proxy.h"
50
#include "share/ob_autoincrement_service.h"
51
#include "share/ob_time_zone_info_manager.h"
52
#include "share/ob_server_status.h"
53
#include "share/ob_index_builder_util.h"
54
#include "observer/ob_server_struct.h"
55
#include "observer/omt/ob_tenant_config_mgr.h"
56
#include "observer/ob_server_event_history_table_operator.h"
57
#include "share/ob_upgrade_utils.h"
58
#include "share/deadlock/ob_deadlock_inner_table_service.h"
59
#ifdef OB_BUILD_TDE_SECURITY
60
#include "share/ob_master_key_getter.h"
61
#endif
62
#ifdef OB_BUILD_ARBITRATION
63
#include "share/arbitration_service/ob_arbitration_service_utils.h" // ObArbitrationServiceUtils
64
#endif
65
#include "share/ob_max_id_fetcher.h" // ObMaxIdFetcher
66
#include "share/backup/ob_backup_config.h"
67
#include "share/backup/ob_backup_helper.h"
68
#include "share/scheduler/ob_sys_task_stat.h"
69

70
#include "sql/executor/ob_executor_rpc_proxy.h"
71
#include "sql/engine/cmd/ob_user_cmd_executor.h"
72
#include "sql/engine/px/ob_px_util.h"
73
#include "observer/dbms_job/ob_dbms_job_master.h"
74
#include "observer/dbms_scheduler/ob_dbms_sched_job_master.h"
75

76
#include "rootserver/ob_bootstrap.h"
77
#include "rootserver/ob_schema2ddl_sql.h"
78
#include "rootserver/ob_index_builder.h"
79
#include "rootserver/ob_mlog_builder.h"
80
#include "rootserver/ob_update_rs_list_task.h"
81
#include "rootserver/ob_resource_weight_parser.h"
82
#include "rootserver/ob_rs_job_table_operator.h"
83
#include "rootserver/restore/ob_restore_util.h"
84
#include "rootserver/ob_root_utils.h"
85
#include "rootserver/ob_vertical_partition_builder.h"
86
#include "rootserver/ob_ddl_sql_generator.h"
87
#include "rootserver/ddl_task/ob_ddl_task.h"
88
#include "rootserver/ddl_task/ob_constraint_task.h"
89
#include "share/ob_ddl_sim_point.h"
90
#include "storage/ob_file_system_router.h"
91
#include "storage/tx/ob_ts_mgr.h"
92
#include "lib/stat/ob_diagnose_info.h"
93
#include "rootserver/ob_cluster_event.h"        // CLUSTER_EVENT_ADD_CONTROL
94
#include "observer/omt/ob_tenant_timezone_mgr.h"
95
#include "share/backup/ob_backup_io_adapter.h"
96
#include "share/ob_global_context_operator.h"
97

98
#include "share/ls/ob_ls_table_operator.h"  // for ObLSTableOperator
99
#include "share/ls/ob_ls_status_operator.h"//ls_status_operator
100
#include "share/ob_max_id_fetcher.h" //ObMaxIdFetcher
101
#include "observer/ob_service.h"
102
#include "storage/ob_file_system_router.h"
103
#include "storage/ddl/ob_ddl_heart_beat_task.h"
104
#include "rootserver/freeze/ob_major_freeze_helper.h"
105
#include "share/restore/ob_physical_restore_table_operator.h"//ObPhysicalRestoreTableOperator
106
#include "share/ob_cluster_event_history_table_operator.h"//CLUSTER_EVENT_INSTANCE
107
#include "share/scn.h"
108
#include "share/restore/ob_recover_table_util.h"
109
#include "rootserver/backup/ob_backup_proxy.h" //ObBackupServiceProxy
110
#include "logservice/palf_handle_guard.h"
111
#include "logservice/ob_log_service.h"
112
#include "rootserver/restore/ob_recover_table_initiator.h"
113
#include "rootserver/ob_heartbeat_service.h"
114
#include "share/tenant_snapshot/ob_tenant_snapshot_table_operator.h"
115
#include "share/restore/ob_tenant_clone_table_operator.h"
116
#include "rootserver/tenant_snapshot/ob_tenant_snapshot_util.h"
117
#include "rootserver/restore/ob_tenant_clone_util.h"
118

119
#include "parallel_ddl/ob_create_table_helper.h" // ObCreateTableHelper
120
#include "parallel_ddl/ob_create_view_helper.h"  // ObCreateViewHelper
121

122
namespace oceanbase
123
{
124

125
using namespace common;
126
using namespace obrpc;
127
using namespace share;
128
using namespace share::schema;
129
using namespace storage;
130
using namespace dbms_job;
131

132
namespace rootserver
133
{
134

135
#define PUSH_BACK_TO_ARRAY_AND_SET_RET(array, msg)                              \
136
  do {                                                                          \
137
    if (OB_FAIL(array.push_back(msg))) {                                        \
138
      LOG_WARN("push reason array error", KR(ret), K(array), K(msg));           \
139
    }                                                                           \
140
  } while(0)
141

142
ObRootService::ObStatusChangeCallback::ObStatusChangeCallback(ObRootService &root_service)
143
    : root_service_(root_service)
144
{
145
}
146

147
ObRootService::ObStatusChangeCallback::~ObStatusChangeCallback()
148
{
149
}
150

151
int ObRootService::ObStatusChangeCallback::wakeup_balancer()
152
{
153
  int ret = OB_SUCCESS;
154
  if (!root_service_.is_inited()) {
155
    ret = OB_NOT_INIT;
156
    LOG_WARN("root_service not init", K(ret));
157
  } else {
158
    root_service_.get_root_balancer().wakeup();
159
  }
160
  return ret;
161
}
162

163
int ObRootService::ObStatusChangeCallback::wakeup_daily_merger()
164
{
165
  int ret = OB_SUCCESS;
166
  if (!root_service_.is_inited()) {
167
    ret = OB_NOT_INIT;
168
    LOG_WARN("root_service not init", K(ret));
169
  } else {
170
    //root_service_.get_daily_merge_scheduler().wakeup();
171
  }
172
  return ret;
173
}
174

175
int ObRootService::ObStatusChangeCallback::on_server_status_change(const common::ObAddr &server)
176
{
177
  int ret = OB_SUCCESS;
178
  if (!root_service_.is_inited()) {
179
    ret = OB_NOT_INIT;
180
    LOG_WARN("root service not inited", K(ret));
181
  } else if (!server.is_valid()) {
182
    ret = OB_INVALID_ARGUMENT;
183
    LOG_WARN("invalid server", K(server), K(ret));
184
  } else if (OB_FAIL(root_service_.submit_update_all_server_task(server))) {
185
    LOG_WARN("root service commit task failed", K(server), K(ret));
186
  }
187
  LOG_INFO("on_server_status_change finish", KR(ret), K(server));
188
  return ret;
189
}
190

191
int ObRootService::ObStatusChangeCallback::on_start_server(const common::ObAddr &server)
192
{
193
  int ret = OB_SUCCESS;
194
  if (!root_service_.is_inited()) {
195
    ret = OB_NOT_INIT;
196
    LOG_WARN("root service not inited", K(ret));
197
  } else if (!server.is_valid()) {
198
    ret = OB_INVALID_ARGUMENT;
199
    LOG_WARN("invalid server", K(ret), K(server));
200
  } else if (OB_FAIL(root_service_.submit_start_server_task(server))) {
201
    LOG_WARN("fail to submit start server task", K(ret));
202
  }
203
  return ret;
204
}
205

206
int ObRootService::ObStatusChangeCallback::on_stop_server(const common::ObAddr &server)
207
{
208
  int ret = OB_SUCCESS;
209
  if (!root_service_.is_inited()) {
210
    ret = OB_NOT_INIT;
211
    LOG_WARN("root service not inited", K(ret));
212
  } else if (!server.is_valid()) {
213
    ret = OB_INVALID_ARGUMENT;
214
    LOG_WARN("invalid server", K(ret), K(server));
215
  } else if (OB_FAIL(root_service_.submit_stop_server_task(server))) {
216
    LOG_WARN("fail to submit stop server task", K(ret));
217
  }
218
  return ret;
219
}
220

221
int ObRootService::ObStatusChangeCallback::on_offline_server(const common::ObAddr &server)
222
{
223
  int ret = OB_SUCCESS;
224
  if (!root_service_.is_inited()) {
225
    ret = OB_NOT_INIT;
226
    LOG_WARN("root service not inited", K(ret));
227
  } else if (!server.is_valid()) {
228
    ret = OB_INVALID_ARGUMENT;
229
    LOG_WARN("invalid server", K(ret), K(server));
230
  } else if (OB_FAIL(root_service_.submit_offline_server_task(server))) {
231
    LOG_WARN("fail to submit stop server task", K(ret));
232
  }
233
  return ret;
234
}
235

236
int ObRootService::ObServerChangeCallback::on_server_change()
237
{
238
  int ret = OB_SUCCESS;
239
  if (!root_service_.is_inited()) {
240
    ret = OB_NOT_INIT;
241
    LOG_WARN("root service not inited", K(ret));
242
  } else if (OB_FAIL(root_service_.submit_update_all_server_config_task())) {
243
    LOG_WARN("root service commit task failed", K(ret));
244
  }
245
  return ret;
246
}
247

248
int ObRootService::ObStartStopServerTask::process()
249
{
250
  int ret = OB_SUCCESS;
251
  bool exist = false;
252
  if (!server_.is_valid()) {
253
    ret = OB_INVALID_ARGUMENT;
254
    LOG_WARN("invalid argument", K(ret), K(server_));
255
  // still use server manager here, since this func. will be called only in version < 4.2
256
  } else if (OB_FAIL(root_service_.get_server_mgr().is_server_exist(server_, exist))) {
257
    LOG_WARN("fail to check server exist", KR(ret), K(server_));
258
  } else if (!exist) {
259
    // server does not exist, ignore
260
  } else {
261
    obrpc::ObAdminServerArg arg;
262
    if (OB_FAIL(arg.servers_.push_back(server_))) {
263
      LOG_WARN("fail to push back", K(ret), K(server_));
264
    } else if (start_) {
265
      if (OB_FAIL(root_service_.start_server(arg))) {
266
        LOG_WARN("fail to start server", K(ret), K(arg));
267
      }
268
    } else {
269
      if (OB_FAIL(root_service_.stop_server(arg))) {
270
        LOG_WARN("fail to stop server", K(ret), K(arg));
271
      }
272
    }
273
  }
274
  return ret;
275
}
276

277
int64_t ObRootService::ObStartStopServerTask::get_deep_copy_size() const
278
{
279
  return sizeof(*this);
280
}
281

282
ObAsyncTask *ObRootService::ObStartStopServerTask::deep_copy(
283
    char *buf, const int64_t buf_size) const
284
{
285
  ObAsyncTask *task = NULL;
286
  int ret = OB_SUCCESS;
287
  const int64_t need_size = get_deep_copy_size();
288
  if (NULL == buf) {
289
    ret = OB_INVALID_ARGUMENT;
290
    LOG_WARN("buf is null", K(ret));
291
  } else if (buf_size < need_size) {
292
    ret = OB_INVALID_ARGUMENT;
293
    LOG_WARN("buf is not long enough", K(need_size), K(buf_size), K(ret));
294
  } else {
295
    task = new (buf) ObStartStopServerTask(root_service_, server_, start_);
296
  }
297
  return task;
298
}
299

300
int ObRootService::ObOfflineServerTask::process()
301
{
302
  int ret = OB_SUCCESS;
303
  ObRsListArg arg;
304
  ObLSInfo ls_info;
305
  const int64_t cluster_id = GCONF.cluster_id;
306
  const int64_t tenant_id = OB_SYS_TENANT_ID;
307
  if (!server_.is_valid()) {
308
    ret = OB_INVALID_ARGUMENT;
309
    LOG_WARN("invalid argument", KR(ret), K(server_));
310
  } else if (OB_FAIL(root_service_.get_lst_operator().get(
311
                                      cluster_id,
312
                                      tenant_id,
313
                                      SYS_LS,
314
                                      share::ObLSTable::DEFAULT_MODE,
315
                                      ls_info))) {
316
    LOG_WARN("fail to get", KR(ret));
317
  } else {
318
    arg.master_rs_ = GCONF.self_addr_;
319
    FOREACH_CNT_X(replica, ls_info.get_replicas(), OB_SUCCESS == ret) {
320
      if (replica->get_server() == GCONF.self_addr_
321
          || (replica->is_in_service()
322
          && ObReplicaTypeCheck::is_paxos_replica_V2(replica->get_replica_type()))) {
323
        if (OB_FAIL(arg.rs_list_.push_back(replica->get_server()))) {
324
          LOG_WARN("fail to push back", KR(ret));
325
        }
326
      }
327
    }
328
    if (OB_FAIL(ret)) {
329
    } else if (OB_FAIL(root_service_.get_rpc_proxy().to(server_).broadcast_rs_list(arg))) {
330
      LOG_DEBUG("fail to broadcast rs list", KR(ret));
331
    } else {
332
      LOG_INFO("broadcast rs list success", K(arg), K_(server));
333
    }
334
  }
335
  return ret;
336
}
337

338
int64_t ObRootService::ObOfflineServerTask::get_deep_copy_size() const
339
{
340
  return sizeof(*this);
341
}
342

343
ObAsyncTask *ObRootService::ObOfflineServerTask::deep_copy(
344
    char *buf, const int64_t buf_size) const
345
{
346
  ObAsyncTask *task = NULL;
347
  int ret = OB_SUCCESS;
348
  const int64_t need_size = get_deep_copy_size();
349
  if (NULL == buf) {
350
    ret = OB_INVALID_ARGUMENT;
351
    LOG_WARN("buf is null", K(ret));
352
  } else if (buf_size < need_size) {
353
    ret = OB_INVALID_ARGUMENT;
354
    LOG_WARN("buf is not long enough", K(need_size), K(buf_size), K(ret));
355
  } else {
356
    task = new (buf) ObOfflineServerTask(root_service_, server_);
357
  }
358
  return task;
359
}
360

361
int ObRootService::ObMinorFreezeTask::process()
362
{
363
  int ret = OB_SUCCESS;
364
  ObAddr rs_addr;
365
  DEBUG_SYNC(BEFORE_DO_MINOR_FREEZE);
366
  if (OB_ISNULL(GCTX.rs_rpc_proxy_) || OB_ISNULL(GCTX.rs_mgr_)) {
367
    ret = OB_ERR_UNEXPECTED;
368
    LOG_WARN("invalid global context", K(ret));
369
  } else if (OB_FAIL(GCTX.rs_mgr_->get_master_root_server(rs_addr))) {
370
    LOG_WARN("get rootservice address failed", K(ret));
371
  } else if (OB_FAIL(GCTX.rs_rpc_proxy_->to(rs_addr).timeout(GCONF.rpc_timeout)
372
                     .root_minor_freeze(arg_))) {
373
    LOG_WARN("minor freeze rpc failed", K(ret), K_(arg));
374
  } else {
375
    LOG_INFO("minor freeze rpc success", K(ret), K_(arg));
376
  }
377
  return ret;
378
}
379

380
int64_t ObRootService::ObMinorFreezeTask::get_deep_copy_size() const
381
{
382
  return sizeof(*this);
383
}
384

385
ObAsyncTask *ObRootService::ObMinorFreezeTask::deep_copy(char *buf, const int64_t buf_size) const
386
{
387
  ObAsyncTask *task = NULL;
388
  int ret = OB_SUCCESS;
389
  const int64_t need_size = get_deep_copy_size();
390
  if (NULL == buf) {
391
    ret = OB_INVALID_ARGUMENT;
392
    LOG_WARN("buf is null", K(ret));
393
  } else if (buf_size < need_size) {
394
    ret = OB_INVALID_ARGUMENT;
395
    LOG_WARN("buf is not long enough", K(need_size), K(buf_size), K(ret));
396
  } else {
397
    task = new(buf) ObMinorFreezeTask(arg_);
398
  }
399
  return task;
400
}
401

402
////////////////////////////////////////////////////////////////
403

404
bool ObRsStatus::can_start_service() const
405
{
406
  bool bret = false;
407
  SpinRLockGuard guard(lock_);
408
  status::ObRootServiceStatus rs_status = ATOMIC_LOAD(&rs_status_);
409
  if (status::INIT == rs_status) {
410
    bret = true;
411
  }
412
  return bret;
413
}
414

415
bool ObRsStatus::is_start() const
416
{
417
  bool bret = false;
418
  SpinRLockGuard guard(lock_);
419
  status::ObRootServiceStatus stat = ATOMIC_LOAD(&rs_status_);
420
  if (status::STARTING == stat || status::IN_SERVICE == stat
421
      || status::FULL_SERVICE == stat || status::STARTED == stat) {
422
    bret = true;
423
  }
424
  return bret;
425
}
426

427
bool ObRsStatus::is_stopping() const
428
{
429
  bool bret = false;
430
  SpinRLockGuard guard(lock_);
431
  status::ObRootServiceStatus stat = ATOMIC_LOAD(&rs_status_);
432
  if (status::STOPPING == stat) {
433
    bret = true;
434
  }
435
  return bret;
436
}
437

438
bool ObRsStatus::need_do_restart() const
439
{
440
  bool bret = false;
441
  SpinRLockGuard guard(lock_);
442
  status::ObRootServiceStatus stat = ATOMIC_LOAD(&rs_status_);
443
  if (status::IN_SERVICE == stat) {
444
    bret = true;
445
  }
446
  return bret;
447
}
448

449
bool ObRsStatus::is_full_service() const
450
{
451
  SpinRLockGuard guard(lock_);
452
  bool bret = false;
453
  status::ObRootServiceStatus stat = ATOMIC_LOAD(&rs_status_);
454
  if (status::FULL_SERVICE == stat || status::STARTED == stat) {
455
    bret = true;
456
  }
457
  return bret;
458
}
459

460
bool ObRsStatus::in_service() const
461
{
462
  bool bret = false;
463
  SpinRLockGuard guard(lock_);
464
  status::ObRootServiceStatus stat = ATOMIC_LOAD(&rs_status_);
465
  if (status::IN_SERVICE == stat
466
      || status::FULL_SERVICE == stat
467
      || status::STARTED == stat) {
468
    bret = true;
469
  }
470
  return bret;
471
}
472

473
bool ObRsStatus::is_need_stop() const
474
{
475
  SpinRLockGuard guard(lock_);
476
  status::ObRootServiceStatus stat = ATOMIC_LOAD(&rs_status_);
477
  return status::NEED_STOP == stat;
478
}
479

480
status::ObRootServiceStatus ObRsStatus::get_rs_status() const
481
{
482
  SpinRLockGuard guard(lock_);
483
  return ATOMIC_LOAD(&rs_status_);
484
}
485

486
//RS need stop after leader revoke
487
int ObRsStatus::revoke_rs()
488
{
489
  int ret = OB_SUCCESS;
490
  FLOG_INFO("[ROOTSERVICE_NOTICE] try to revoke rs");
491
  SpinWLockGuard guard(lock_);
492
  if (status::IN_SERVICE == rs_status_ || status::FULL_SERVICE == rs_status_) {
493
    rs_status_ = status::NEED_STOP;
494
    FLOG_INFO("[ROOTSERVICE_NOTICE] rs_status is setted to need_stop", K_(rs_status));
495
  } else if (status::STOPPING != rs_status_) {
496
    rs_status_ = status::STOPPING;
497
    FLOG_INFO("[ROOTSERVICE_NOTICE] rs_status is setted to stopping", K_(rs_status));
498
  }
499
  return ret;
500
}
501

502
int ObRsStatus::try_set_stopping()
503
{
504
  int ret = OB_SUCCESS;
505
  FLOG_INFO("[ROOTSERVICE_NOTICE] try set rs_status to stopping");
506
  SpinWLockGuard guard(lock_);
507
  if (status::NEED_STOP == rs_status_) {
508
    rs_status_ = status::STOPPING;
509
    FLOG_INFO("[ROOTSERVICE_NOTICE] rs_status is setted to stopping");
510
  }
511
  return ret;
512
}
513

514
int ObRsStatus::set_rs_status(const status::ObRootServiceStatus status)
515
{
516
  int ret = OB_SUCCESS;
517
  SpinWLockGuard guard(lock_);
518
  const char* new_status_str = NULL;
519
  const char* old_status_str = NULL;
520
  if (OB_FAIL(get_rs_status_str(status, new_status_str))) {
521
    FLOG_WARN("fail to get rs status", KR(ret), K(status));
522
  } else if (OB_FAIL(get_rs_status_str(rs_status_, old_status_str))) {
523
    FLOG_WARN("fail to get rs status", KR(ret), K(rs_status_));
524
  } else if (OB_ISNULL(new_status_str) || OB_ISNULL(old_status_str)) {
525
    ret = OB_ERR_UNEXPECTED;
526
    FLOG_WARN("error unexpect", KR(ret), K(new_status_str), K(old_status_str));
527
  }
528
  if (OB_SUCC(ret)) {
529
    switch(rs_status_) {
530
      case status::INIT:
531
        {
532
          if (status::STARTING == status
533
              || status::STOPPING == status) {
534
            //rs.stop() will be executed while obs exit
535
            rs_status_ = status;
536
            FLOG_INFO("[ROOTSERVICE_NOTICE] success to set rs status",
537
                      K(new_status_str), K(old_status_str), K(rs_status_));
538
          } else {
539
            ret = OB_OP_NOT_ALLOW;
540
            FLOG_WARN("can't set rs status", KR(ret));
541
          }
542
          break;
543
        }
544
      case status::STARTING:
545
        {
546
          if (status::IN_SERVICE == status
547
              || status::STOPPING == status) {
548
            rs_status_ = status;
549
            FLOG_INFO("[ROOTSERVICE_NOTICE] success to set rs status",
550
                      K(new_status_str), K(old_status_str), K(rs_status_));
551
          } else {
552
            ret = OB_OP_NOT_ALLOW;
553
            FLOG_WARN("can't set rs status", KR(ret));
554
          }
555
          break;
556
        }
557
      case status::IN_SERVICE:
558
        {
559
          if (status::FULL_SERVICE == status
560
              || status::NEED_STOP == status
561
              || status::STOPPING == status) {
562
            rs_status_ = status;
563
            FLOG_INFO("[ROOTSERVICE_NOTICE] success to set rs status",
564
                      K(new_status_str), K(old_status_str), K(rs_status_));
565
          } else {
566
            ret = OB_OP_NOT_ALLOW;
567
            FLOG_WARN("can't set rs status", KR(ret));
568
          }
569
          break;
570
        }
571
      case status::FULL_SERVICE:
572
        {
573
          if (status::STARTED == status
574
              || status::NEED_STOP == status
575
              || status::STOPPING == status) {
576
            rs_status_ = status;
577
            FLOG_INFO("[ROOTSERVICE_NOTICE] success to set rs status",
578
                      K(new_status_str), K(old_status_str), K(rs_status_));
579
          } else {
580
            ret = OB_OP_NOT_ALLOW;
581
            FLOG_WARN("can't set rs status", KR(ret));
582
          }
583
          break;
584
        }
585
      case status::STARTED:
586
        {
587
          if (status::STOPPING == status) {
588
            rs_status_ = status;
589
            FLOG_INFO("[ROOTSERVICE_NOTICE] success to set rs status",
590
                      K(new_status_str), K(old_status_str), K(rs_status_));
591
          } else {
592
            ret = OB_OP_NOT_ALLOW;
593
            FLOG_WARN("can't set rs status", KR(ret));
594
          }
595
          break;
596
        }
597
      case status::NEED_STOP:
598
        {
599
          if (status::STOPPING == status) {
600
            rs_status_ = status;
601
            FLOG_INFO("[ROOTSERVICE_NOTICE] success to set rs status",
602
                      K(new_status_str), K(old_status_str), K(rs_status_));
603
          } else {
604
            ret = OB_OP_NOT_ALLOW;
605
            FLOG_WARN("can't set rs status", KR(ret));
606
          }
607
          break;
608
        }
609
      case status::STOPPING:
610
        {
611
          if (status::INIT == status
612
              || status::STOPPING == status) {
613
            rs_status_ = status;
614
            FLOG_INFO("[ROOTSERVICE_NOTICE] success to set rs status",
615
                      K(new_status_str), K(old_status_str), K(rs_status_));
616
          } else {
617
            ret = OB_OP_NOT_ALLOW;
618
            FLOG_WARN("can't set rs status", KR(ret));
619
          }
620
          break;
621
        }
622
      default:
623
        ret = OB_ERR_UNEXPECTED;
624
        FLOG_WARN("invalid rs status", KR(ret), K(rs_status_));
625
    }
626
  }
627
  return ret;
628
}
629

630
////////////////////////////////////////////////////////////////
631
ObRootService::ObRootService()
632
: inited_(false), server_refreshed_(false),
633
    debug_(false),
634
    self_addr_(), config_(NULL), config_mgr_(NULL),
635
    rpc_proxy_(), common_proxy_(), sql_proxy_(), restore_ctx_(NULL), rs_mgr_(NULL),
636
    schema_service_(NULL), status_change_cb_(*this),
637
    server_change_callback_(*this),
638
    server_manager_(), hb_checker_(),
639
    server_checker_(),
640
    rs_list_change_cb_(*this),
641
    server_zone_op_service_(),
642
    root_minor_freeze_(),
643
    lst_operator_(NULL),
644
    zone_manager_(), ddl_service_(), unit_manager_(server_manager_, zone_manager_),
645
    root_balancer_(), empty_server_checker_(), lost_replica_checker_(), thread_checker_(),
646
    vtable_location_getter_(unit_manager_),
647
    addr_agent_(NULL), root_inspection_(),
648
    upgrade_executor_(),
649
    upgrade_storage_format_executor_(), create_inner_schema_executor_(),
650
    bootstrap_lock_(), broadcast_rs_list_lock_(ObLatchIds::RS_BROADCAST_LOCK),
651
    task_queue_(),
652
    inspect_task_queue_(),
653
    restart_task_(*this),
654
    refresh_server_task_(*this),
655
    check_server_task_(task_queue_, server_checker_),
656
    self_check_task_(*this),
657
    load_ddl_task_(*this),
658
    refresh_io_calibration_task_(*this),
659
    event_table_clear_task_(ROOTSERVICE_EVENT_INSTANCE,
660
                            SERVER_EVENT_INSTANCE,
661
                            DEALOCK_EVENT_INSTANCE,
662
                            task_queue_),
663
    inspector_task_(*this),
664
    purge_recyclebin_task_(*this),
665
    ddl_scheduler_(),
666
    snapshot_manager_(),
667
    core_meta_table_version_(0),
668
    update_rs_list_timer_task_(*this),
669
    update_all_server_config_task_(*this),
670
    baseline_schema_version_(0),
671
    start_service_time_(0),
672
    rs_status_(),
673
    fail_count_(0),
674
    schema_history_recycler_(),
675
#ifdef OB_BUILD_TDE_SECURITY
676
    master_key_mgr_(),
677
#endif
678
    disaster_recovery_task_executor_(),
679
    disaster_recovery_task_mgr_(),
680
    global_ctx_task_(*this)
681
{
682
}
683

684
ObRootService::~ObRootService()
685
{
686
  if (inited_) {
687
    destroy();
688
  }
689
}
690

691
int ObRootService::fake_init(ObServerConfig &config,
692
                             ObConfigManager &config_mgr,
693
                             ObSrvRpcProxy &srv_rpc_proxy,
694
                             ObCommonRpcProxy &common_proxy,
695
                             ObAddr &self,
696
                             ObMySQLProxy &sql_proxy,
697
                             ObRsMgr &rs_mgr,
698
                             ObMultiVersionSchemaService *schema_service,
699
                             ObLSTableOperator &lst_operator)
700
{
701
  int ret = OB_SUCCESS;
702
  if (inited_) {
703
    ret = OB_INIT_TWICE;
704
    LOG_WARN("rootservice already inited", K(ret));
705
  } else if (!self.is_valid()) {
706
    ret = OB_INVALID_ARGUMENT;
707
    LOG_WARN("invalid self address", K(self), K(ret));
708
  } else if (NULL == schema_service) {
709
    ret = OB_INVALID_ARGUMENT;
710
    LOG_WARN("schema_service must not null", KP(schema_service), K(ret));
711
  } else {
712
    LOG_INFO("start to init rootservice");
713
    config_ = &config;
714
    config_mgr_ = &config_mgr;
715

716
    rpc_proxy_ = srv_rpc_proxy;
717
    common_proxy_ = common_proxy;
718

719
    const bool rpc_active = false;
720
    common_proxy_.active(rpc_active);
721
    rpc_proxy_.active(rpc_active);
722

723
    self_addr_ = self;
724

725
    sql_proxy_.assign(sql_proxy);
726
    sql_proxy_.set_inactive();
727
    oracle_sql_proxy_.set_inactive();
728

729
    rs_mgr_ = &rs_mgr;
730
    addr_agent_ = &rs_mgr.get_addr_agent();
731
    schema_service_ = schema_service;
732
    lst_operator_ = &lst_operator;
733
  }
734

735
  // init server management related
736
  if (OB_SUCC(ret)) {
737
    if (OB_FAIL(server_manager_.init(
738
                status_change_cb_, server_change_callback_, sql_proxy_, unit_manager_, zone_manager_,
739
                config, self, rpc_proxy_))) {
740
      LOG_WARN("init server manager failed", K(ret));
741
    } else if (OB_FAIL(hb_checker_.init(server_manager_))) {
742
      LOG_WARN("init heartbeat checker failed", K(ret));
743
    } else if (OB_FAIL(server_checker_.init(server_manager_, self))) {
744
      LOG_WARN("init server checker failed", K(self), K(ret));
745
    }
746
  }
747

748
  // init ddl service
749
  if (OB_SUCC(ret)) {
750
    if (OB_FAIL(ddl_service_.init(rpc_proxy_, common_proxy_, sql_proxy_, *schema_service,
751
                                  lst_operator, zone_manager_, unit_manager_,
752
                                  snapshot_manager_))) {
753
      LOG_WARN("ddl_service_ init failed", K(ret));
754
    }
755
  }
756

757
  if (OB_SUCC(ret)) {
758
    if (OB_FAIL(dbms_job::ObDBMSJobMaster::get_instance()
759
          .init(&sql_proxy_, schema_service_))) {
760
      LOG_WARN("failed to init ObDBMSJobMaster", K(ret));
761
    }
762
  }
763

764
  if (OB_SUCC(ret)) {
765
    if (OB_FAIL(dbms_scheduler::ObDBMSSchedJobMaster::get_instance()
766
          .init(&unit_manager_, &sql_proxy_, schema_service_))) {
767
      LOG_WARN("failed to init ObDBMSSchedJobMaster", K(ret));
768
    }
769
  }
770

771
  if (OB_SUCC(ret)) {
772
    inited_ = true;
773
  }
774

775
  LOG_INFO("init rootservice", K(ret));
776
  return ret;
777
}
778
int ObRootService::init(ObServerConfig &config,
779
                        ObConfigManager &config_mgr,
780
                        ObSrvRpcProxy &srv_rpc_proxy,
781
                        ObCommonRpcProxy &common_proxy,
782
                        ObAddr &self,
783
                        ObMySQLProxy &sql_proxy,
784
                        observer::ObRestoreCtx &restore_ctx,
785
                        ObRsMgr &rs_mgr,
786
                        ObMultiVersionSchemaService *schema_service,
787
			                  ObLSTableOperator &lst_operator)
788
{
789
  int ret = OB_SUCCESS;
790
  FLOG_INFO("[ROOTSERVICE_NOTICE] begin to init rootservice");
791
  if (inited_) {
792
    ret = OB_INIT_TWICE;
793
    FLOG_WARN("rootservice already inited", KR(ret));
794
  } else if (!self.is_valid()) {
795
    ret = OB_INVALID_ARGUMENT;
796
    FLOG_WARN("invalid self address", K(self), KR(ret));
797
  } else if (NULL == schema_service) {
798
    ret = OB_INVALID_ARGUMENT;
799
    FLOG_WARN("schema_service must not null", KP(schema_service), KR(ret));
800
  } else {
801
    config_ = &config;
802
    config_mgr_ = &config_mgr;
803

804
    rpc_proxy_ = srv_rpc_proxy;
805
    common_proxy_ = common_proxy;
806

807
    const bool rpc_active = false;
808
    common_proxy_.active(rpc_active);
809
    rpc_proxy_.active(rpc_active);
810

811
    self_addr_ = self;
812

813
    restore_ctx_ = &restore_ctx;
814

815
    sql_proxy_.assign(sql_proxy);
816
    sql_proxy_.set_inactive();
817

818
    if (OB_FAIL(oracle_sql_proxy_.init(sql_proxy.get_pool()))) {
819
      FLOG_WARN("init oracle sql proxy failed", KR(ret));
820
    } else {
821
      oracle_sql_proxy_.set_inactive();
822
    }
823

824
    rs_mgr_ = &rs_mgr;
825
    addr_agent_ = &rs_mgr.get_addr_agent();
826
    schema_service_ = schema_service;
827
    lst_operator_ = &lst_operator;
828
  }
829

830
  // init inner queue
831
  if (FAILEDx(task_queue_.init(
832
              config_->rootservice_async_task_thread_count,
833
              config_->rootservice_async_task_queue_size,
834
              "RSAsyncTask"))) {
835
    FLOG_WARN("init inner queue failed",
836
             "thread_count", static_cast<int64_t>(config_->rootservice_async_task_thread_count),
837
             "queue_size", static_cast<int64_t>(config_->rootservice_async_task_queue_size), KR(ret));
838
  } else if (OB_FAIL(inspect_task_queue_.init(1/*only for the inspection of RS*/,
839
                                              config_->rootservice_async_task_queue_size,
840
                                              "RSInspectTask"))) {
841
    FLOG_WARN("init inner queue failed",
842
              "thread_count", 1,
843
              "queue_size", static_cast<int64_t>(config_->rootservice_async_task_queue_size), KR(ret));
844
  } else if (OB_FAIL(zone_manager_.init(sql_proxy_))) {
845
    // init zone manager
846
    FLOG_WARN("init zone manager failed", KR(ret));
847
  } else if (OB_FAIL(server_manager_.init(status_change_cb_, server_change_callback_, sql_proxy_,
848
                                          unit_manager_, zone_manager_, config, self, rpc_proxy_))) {
849
    // init server management related
850
    FLOG_WARN("init server manager failed", KR(ret));
851
  } else if (OB_FAIL(server_zone_op_service_.init(
852
      server_change_callback_,
853
      rpc_proxy_,
854
      lst_operator,
855
      unit_manager_,
856
      sql_proxy_
857
#ifdef OB_BUILD_TDE_SECURITY
858
      , &master_key_mgr_
859
#endif
860
      ))) {
861
    FLOG_WARN("init server zone op service failed", KR(ret));
862
  } else if (OB_FAIL(hb_checker_.init(server_manager_))) {
863
    FLOG_WARN("init heartbeat checker failed", KR(ret));
864
  } else if (OB_FAIL(server_checker_.init(server_manager_, self))) {
865
    FLOG_WARN("init server checker failed", KR(ret), K(self));
866
  } else if (OB_FAIL(root_minor_freeze_.init(rpc_proxy_, unit_manager_))) {
867
    // init root minor freeze
868
    FLOG_WARN("init root_minor_freeze_ failed", KR(ret));
869
  } else if (OB_FAIL(ddl_service_.init(rpc_proxy_, common_proxy_, sql_proxy_, *schema_service,
870
                                       lst_operator, zone_manager_, unit_manager_,
871
                                       snapshot_manager_))) {
872
    // init ddl service
873
    FLOG_WARN("init ddl_service_ failed", KR(ret));
874
  } else if (OB_FAIL(unit_manager_.init(sql_proxy_, *config_, rpc_proxy_, *schema_service,
875
                                        root_balancer_, *this))) {
876
    // init unit manager
877
    FLOG_WARN("init unit_manager failed", KR(ret));
878
  } else if (OB_FAIL(snapshot_manager_.init(self_addr_))) {
879
    FLOG_WARN("init snapshot manager failed", KR(ret));
880
  } else if (OB_FAIL(root_inspection_.init(*schema_service_, zone_manager_, sql_proxy_, &common_proxy_))) {
881
    FLOG_WARN("init root inspection failed", KR(ret));
882
  } else if (OB_FAIL(upgrade_executor_.init(*schema_service_,
883
             root_inspection_, sql_proxy_, rpc_proxy_, common_proxy_))) {
884
    FLOG_WARN("init upgrade_executor failed", KR(ret));
885
  } else if (OB_FAIL(upgrade_storage_format_executor_.init(*this, ddl_service_))) {
886
    FLOG_WARN("init upgrade storage format executor failed", KR(ret));
887
  } else if (OB_FAIL(create_inner_schema_executor_.init(*schema_service_, sql_proxy_, common_proxy_))) {
888
    FLOG_WARN("init create inner role executor failed", KR(ret));
889
  } else if (OB_FAIL(thread_checker_.init())) {
890
    FLOG_WARN("init thread checker failed", KR(ret));
891
  } else if (OB_FAIL(empty_server_checker_.init(
892
      server_manager_,
893
      unit_manager_,
894
      *lst_operator_,
895
      *schema_service_,
896
      server_zone_op_service_))) {
897
    FLOG_WARN("init empty server checker failed", KR(ret));
898
  } else if (OB_FAIL(lost_replica_checker_.init(*lst_operator_, *schema_service_))) {
899
    FLOG_WARN("init empty server checker failed", KR(ret));
900
  } else if (OB_FAIL(root_balancer_.init(*config_, *schema_service_, unit_manager_,
901
                                           server_manager_, zone_manager_, rpc_proxy_,
902
                                           self_addr_, sql_proxy, disaster_recovery_task_mgr_))) {
903
    FLOG_WARN("init root balancer failed", KR(ret));
904
  } else if (OB_FAIL(ROOTSERVICE_EVENT_INSTANCE.init(sql_proxy, self_addr_))) {
905
    FLOG_WARN("init rootservice event history failed", KR(ret));
906
  } else if (OB_FAIL(THE_RS_JOB_TABLE.init(&sql_proxy, self_addr_))) {
907
    FLOG_WARN("init THE_RS_JOB_TABLE failed", KR(ret));
908
  } else if (OB_FAIL(ddl_scheduler_.init(this))) {
909
    FLOG_WARN("init ddl task scheduler failed", KR(ret));
910
  } else if (OB_FAIL(schema_history_recycler_.init(*schema_service_,
911
                                                   zone_manager_,
912
                                                   sql_proxy_))) {
913
    FLOG_WARN("fail to init schema history recycler failed", KR(ret));
914
  } else if (OB_FAIL(dbms_job::ObDBMSJobMaster::get_instance().init(&sql_proxy_,
915
                                                                    schema_service_))) {
916
    FLOG_WARN("init ObDBMSJobMaster failed", KR(ret));
917
  } else if (OB_FAIL(dbms_scheduler::ObDBMSSchedJobMaster::get_instance().init(&unit_manager_,
918
                                                                               &sql_proxy_,
919
                                                                               schema_service_))) {
920
    FLOG_WARN("init ObDBMSSchedJobMaster failed", KR(ret));
921
#ifdef OB_BUILD_TDE_SECURITY
922
  } else if (OB_FAIL(master_key_mgr_.init(&zone_manager_, schema_service_))) {
923
    FLOG_WARN("init master key mgr failed", KR(ret));
924
#endif
925
  } else if (OB_FAIL(disaster_recovery_task_executor_.init(lst_operator,
926
                                                           rpc_proxy_))) {
927
    FLOG_WARN("init disaster recovery task executor failed", KR(ret));
928
  } else if (OB_FAIL(disaster_recovery_task_mgr_.init(self,
929
                                                      *config_,
930
                                                      disaster_recovery_task_executor_,
931
                                                      &rpc_proxy_,
932
                                                      &sql_proxy_,
933
                                                      schema_service_))) {
934
    FLOG_WARN("init disaster recovery task mgr failed", KR(ret));
935
  }
936
  if (OB_SUCC(ret)) {
937
    inited_ = true;
938
    FLOG_INFO("[ROOTSERVICE_NOTICE] init rootservice success", KR(ret), K_(inited));
939
  } else {
940
    LOG_ERROR("[ROOTSERVICE_NOTICE] fail to init root service", KR(ret));
941
    LOG_DBA_ERROR(OB_ERR_ROOTSERVICE_START, "msg", "rootservice init() has failure", KR(ret));
942
  }
943

944
  return ret;
945
}
946

947
void ObRootService::destroy()
948
{
949
  int ret = OB_SUCCESS;
950
  int fail_ret = OB_SUCCESS;
951
  FLOG_INFO("[ROOTSERVICE_NOTICE] start to destroy rootservice");
952
  if (in_service()) {
953
    if (OB_FAIL(stop_service())) {
954
      FLOG_WARN("stop service failed", KR(ret));
955
      fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
956
    }
957
  }
958

959
  if (OB_FAIL(lost_replica_checker_.destroy())) {
960
    FLOG_WARN("lost replica checker failed", KR(ret));
961
    fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
962
  } else {
963
    FLOG_INFO("lost replica checker destroy");
964
  }
965

966
  // continue executing while error happen
967
  if (OB_FAIL(root_balancer_.destroy())) {
968
    FLOG_WARN("root balance destroy failed", KR(ret));
969
    fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
970
  } else {
971
    FLOG_INFO("root balance destroy");
972
  }
973

974
  if (OB_FAIL(empty_server_checker_.destroy())) {
975
    FLOG_WARN("empty server checker destroy failed", KR(ret));
976
    fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
977
  } else {
978
    FLOG_INFO("empty server checker destroy");
979
  }
980

981
  if (OB_FAIL(thread_checker_.destroy())) {
982
    FLOG_WARN("rs_monitor_check : thread checker destroy failed", KR(ret));
983
    fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
984
  } else {
985
    FLOG_INFO("rs_monitor_check : thread checker destroy");
986
  }
987
  if (OB_FAIL(schema_history_recycler_.destroy())) {
988
    FLOG_WARN("schema history recycler destroy failed", KR(ret));
989
    fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
990
  } else {
991
    FLOG_INFO("schema history recycler destroy");
992
  }
993

994
  task_queue_.destroy();
995
  FLOG_INFO("inner queue destroy");
996
  inspect_task_queue_.destroy();
997
  FLOG_INFO("inspect queue destroy");
998
  ddl_builder_.destroy();
999
  FLOG_INFO("ddl builder destroy");
1000
  if (OB_FAIL(hb_checker_.destroy())) {
1001
    FLOG_WARN("heartbeat checker destroy failed", KR(ret));
1002
    fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
1003
  } else {
1004
    FLOG_INFO("heartbeat checker destroy");
1005
  }
1006

1007
  ROOTSERVICE_EVENT_INSTANCE.destroy();
1008
  FLOG_INFO("event table operator destroy");
1009

1010
  dbms_job::ObDBMSJobMaster::get_instance().destroy();
1011
  FLOG_INFO("ObDBMSJobMaster destroy");
1012

1013
  ddl_scheduler_.destroy();
1014
  FLOG_INFO("ddl task scheduler destroy");
1015

1016
#ifdef OB_BUILD_TDE_SECURITY
1017
  if (OB_FAIL(master_key_mgr_.destroy())) {
1018
    FLOG_WARN("master key mgr destroy failed", KR(ret));
1019
    fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
1020
  } else {
1021
    FLOG_INFO("master key mgr destroy");
1022
  }
1023
#endif
1024

1025
  if (OB_FAIL(disaster_recovery_task_mgr_.destroy())) {
1026
    FLOG_WARN("disaster recovery task mgr destroy failed", KR(ret));
1027
    fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
1028
  } else {
1029
    FLOG_INFO("disaster recovery task mgr destroy");
1030
  }
1031

1032
  dbms_scheduler::ObDBMSSchedJobMaster::get_instance().destroy();
1033
  FLOG_INFO("ObDBMSSchedJobMaster destroy");
1034
  TG_DESTROY(lib::TGDefIDs::GlobalCtxTimer);
1035
  FLOG_INFO("global ctx timer destroyed");
1036

1037

1038
  if (OB_SUCC(ret)) {
1039
    if (inited_) {
1040
      inited_ = false;
1041
    }
1042
  }
1043

1044
  FLOG_INFO("[ROOTSERVICE_NOTICE] destroy rootservice end", KR(ret));
1045
  if (OB_SUCCESS != fail_ret) {
1046
    LOG_DBA_WARN(OB_ERR_ROOTSERVICE_STOP, "msg", "rootservice destroy() has failure", KR(fail_ret));
1047
  }
1048
}
1049

1050
int ObRootService::start_service()
1051
{
1052
  int ret = OB_SUCCESS;
1053
  start_service_time_ = ObTimeUtility::current_time();
1054
  ROOTSERVICE_EVENT_ADD("root_service", "start_rootservice", K_(self_addr));
1055
  FLOG_INFO("[ROOTSERVICE_NOTICE] start to start rootservice", K_(start_service_time));
1056
  if (!inited_) {
1057
    ret = OB_NOT_INIT;
1058
    FLOG_WARN("rootservice not inited", KR(ret));
1059
  } else if (OB_FAIL(rs_status_.set_rs_status(status::STARTING))) {
1060
    FLOG_WARN("fail to set rs status", KR(ret));
1061
  } else if (!ObRootServiceRoleChecker::is_rootserver()) {
1062
    ret = OB_NOT_MASTER;
1063
    FLOG_WARN("not master", KR(ret));
1064
  } else {
1065
    sql_proxy_.set_active();
1066
    oracle_sql_proxy_.set_active();
1067
    const bool rpc_active = true;
1068
    common_proxy_.active(rpc_active);
1069
    rpc_proxy_.active(rpc_active);
1070
    ddl_service_.restart();
1071
    server_manager_.reset();
1072
    zone_manager_.reset();
1073
    OTC_MGR.reset_version_has_refreshed();
1074

1075
    if (OB_FAIL(hb_checker_.start())) {
1076
      FLOG_WARN("hb checker start failed", KR(ret));
1077
    } else if (OB_FAIL(task_queue_.start())) {
1078
      FLOG_WARN("inner queue start failed", KR(ret));
1079
    } else if (OB_FAIL(inspect_task_queue_.start())) {
1080
      FLOG_WARN("inspect queue start failed", KR(ret));
1081
    } else if (OB_FAIL(ddl_builder_.start())) {
1082
      FLOG_WARN("start ddl builder failed", KR(ret));
1083
    } else if (OB_FAIL(TG_START(lib::TGDefIDs::GlobalCtxTimer))) {
1084
      FLOG_WARN("init global ctx timer fail", KR(ret));
1085
    } else if (OB_FAIL(global_ctx_task_.schedule(lib::TGDefIDs::GlobalCtxTimer))) {
1086
      FLOG_WARN("failed to schedule global ctx task", KR(ret));
1087
    } else if (OB_FAIL(lst_operator_->set_callback_for_rs(rs_list_change_cb_))) {
1088
      FLOG_WARN("lst_operator set as rs leader failed", KR(ret));
1089
    } else if (OB_FAIL(rs_status_.set_rs_status(status::IN_SERVICE))) {
1090
      FLOG_WARN("fail to set rs status", KR(ret));
1091
    } else if (OB_FAIL(schedule_refresh_server_timer_task(0))) {
1092
      FLOG_WARN("failed to schedule refresh_server task", KR(ret));
1093
    } else if (OB_FAIL(schedule_restart_timer_task(0))) {
1094
      FLOG_WARN("failed to schedule restart task", KR(ret));
1095
    } else if (OB_FAIL(schema_service_->get_ddl_epoch_mgr().remove_all_ddl_epoch())) {
1096
      FLOG_WARN("fail to remove ddl epoch", KR(ret));
1097
    } else if (debug_) {
1098
      if (OB_FAIL(init_debug_database())) {
1099
        FLOG_WARN("init_debug_database failed", KR(ret));
1100
      }
1101
    }
1102
  }
1103

1104
  ROOTSERVICE_EVENT_ADD("root_service", "finish_start_rootservice",
1105
                        "result", ret, K_(self_addr));
1106

1107
  if (OB_FAIL(ret)) {
1108
    // increase fail count for self checker and print log.
1109
    update_fail_count(ret);
1110
    FLOG_WARN("start service failed, do stop service", KR(ret));
1111
    int tmp_ret = OB_SUCCESS;
1112
    if (OB_SUCCESS != (tmp_ret = rs_status_.set_rs_status(status::STOPPING))) {
1113
      FLOG_WARN("fail to set status", KR(tmp_ret));
1114
    } else if (OB_SUCCESS != (tmp_ret = stop_service())) {
1115
      FLOG_WARN("stop service failed", KR(tmp_ret));
1116
    }
1117
  }
1118

1119
  FLOG_INFO("[ROOTSERVICE_NOTICE] rootservice start_service finished", KR(ret));
1120
  return ret;
1121
}
1122

1123
int ObRootService::stop_service()
1124
{
1125
  int ret = OB_SUCCESS;
1126
  FLOG_INFO("[ROOTSERVICE_NOTICE] stop service begin");
1127
  if (OB_FAIL(stop())) {
1128
    FLOG_WARN("fail to stop thread", KR(ret));
1129
  } else {
1130
    wait();
1131
  }
1132
  if (FAILEDx(rs_status_.set_rs_status(status::INIT))) {
1133
    FLOG_WARN("fail to set rs status", KR(ret));
1134
  }
1135
  FLOG_INFO("[ROOTSERVICE_NOTICE] stop service finished", KR(ret));
1136
  return ret;
1137
}
1138

1139
int ObRootService::stop()
1140
{
1141
  int ret = OB_SUCCESS;
1142
  int fail_ret = OB_SUCCESS;
1143
  start_service_time_ = 0;
1144
  int64_t start_time = ObTimeUtility::current_time();
1145
  ROOTSERVICE_EVENT_ADD("root_service", "stop_rootservice", K_(self_addr));
1146
  FLOG_INFO("[ROOTSERVICE_NOTICE] start to stop rootservice", K(start_time));
1147
  if (!inited_) {
1148
    ret = OB_NOT_INIT;
1149
    FLOG_WARN("rootservice not inited", KR(ret));
1150
    fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
1151
  } else if (OB_FAIL(rs_status_.set_rs_status(status::STOPPING))) {
1152
    FLOG_WARN("fail to set rs status", KR(ret));
1153
    fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
1154
  } else {
1155
    // set to rpc ls table as soon as possible
1156
    if (OB_FAIL(lst_operator_->set_callback_for_obs(
1157
        common_proxy_,
1158
        rpc_proxy_,
1159
        *rs_mgr_,
1160
        sql_proxy_))) {
1161
      FLOG_WARN("set as rs follower failed", KR(ret));
1162
      fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
1163
    } else {
1164
      FLOG_INFO("set old rs to follower finished");
1165
    }
1166
    //full_service_ = false;
1167
    server_refreshed_ = false;
1168
    //in_service_ = false;
1169
    sql_proxy_.set_inactive();
1170
    FLOG_INFO("sql_proxy set inactive finished");
1171
    oracle_sql_proxy_.set_inactive();
1172
    FLOG_INFO("oracle_sql_proxy set inactive finished");
1173
    const bool rpc_active = false;
1174
    common_proxy_.active(rpc_active);
1175
    FLOG_INFO("commom_proxy set inactive finished");
1176
    rpc_proxy_.active(rpc_active);
1177
    FLOG_INFO("rpc_proxy set inactive finished");
1178

1179
    int tmp_ret = OB_SUCCESS;
1180
    if (OB_SUCCESS != (tmp_ret = upgrade_executor_.stop())) {
1181
      FLOG_WARN("upgrade_executor stop failed", KR(tmp_ret));
1182
      fail_ret = OB_SUCCESS == fail_ret ? tmp_ret : fail_ret;
1183
    } else {
1184
      FLOG_INFO("upgrade_executor stop finished");
1185
    }
1186
    if (OB_SUCCESS != (tmp_ret = upgrade_storage_format_executor_.stop())) {
1187
      FLOG_WARN("fail to stop upgrade storage format executor", KR(tmp_ret));
1188
      fail_ret = OB_SUCCESS == fail_ret ? tmp_ret : fail_ret;
1189
    } else {
1190
      FLOG_INFO("upgrade_storage_format_executor stop finished");
1191
    }
1192

1193
    if (OB_SUCCESS != (tmp_ret = create_inner_schema_executor_.stop())) {
1194
      FLOG_WARN("fail to stop create inner schema executor", KR(tmp_ret));
1195
      fail_ret = OB_SUCCESS == fail_ret ? tmp_ret : fail_ret;
1196
    } else {
1197
      FLOG_INFO("create_inner_schema_executor stop finished");
1198
    }
1199

1200
    if (OB_SUCC(ret)) {
1201
      if (OB_FAIL(stop_timer_tasks())) {
1202
        FLOG_WARN("stop timer tasks failed", KR(ret));
1203
        fail_ret = OB_SUCCESS == fail_ret ? ret : fail_ret;
1204
      } else {
1205
        FLOG_INFO("stop timer tasks success");
1206
      }
1207
    }
1208

1209
    if (OB_SUCC(ret)) {
1210
      // ddl_service may be trying refresh schema, stop it
1211
      ddl_service_.stop();
1212
      FLOG_INFO("ddl service stop");
1213
      root_minor_freeze_.stop();
1214
      FLOG_INFO("minor freeze stop");
1215
      root_inspection_.stop();
1216
      FLOG_INFO("root inspection stop");
1217
    }
1218
    if (OB_SUCC(ret)) {
1219
      ddl_builder_.stop();
1220
      FLOG_INFO("ddl builder stop");
1221
      task_queue_.stop();
1222
      FLOG_INFO("task_queue stop");
1223
      inspect_task_queue_.stop();
1224
      FLOG_INFO("inspect queue stop");
1225
      root_balancer_.stop();
1226
      FLOG_INFO("root_balancer stop");
1227
      empty_server_checker_.stop();
1228
      FLOG_INFO("empty_server_checker stop");
1229
      lost_replica_checker_.stop();
1230
      FLOG_INFO("lost_replica_checker stop");
1231
      thread_checker_.stop();
1232
      FLOG_INFO("rs_monitor_check : thread_checker stop");
1233
      schema_history_recycler_.stop();
1234
      FLOG_INFO("schema_history_recycler stop");
1235
      hb_checker_.stop();
1236
      FLOG_INFO("hb_checker stop");
1237
      ddl_scheduler_.stop();
1238
      FLOG_INFO("ddl task scheduler stop");
1239
      dbms_job::ObDBMSJobMaster::get_instance().stop();
1240
      FLOG_INFO("dbms job master stop");
1241
#ifdef OB_BUILD_TDE_SECURITY
1242
      master_key_mgr_.stop();
1243
      FLOG_INFO("master key mgr stop");
1244
#endif
1245
      disaster_recovery_task_mgr_.stop();
1246
      FLOG_INFO("disaster_recovery_task_mgr stop");
1247
      dbms_scheduler::ObDBMSSchedJobMaster::get_instance().stop();
1248
      FLOG_INFO("dbms sched job master stop");
1249
      TG_STOP(lib::TGDefIDs::GlobalCtxTimer);
1250
      FLOG_INFO("global ctx timer stop");
1251
    }
1252
  }
1253

1254
  ROOTSERVICE_EVENT_ADD("root_service", "finish_stop_thread", KR(ret), K_(self_addr));
1255
  FLOG_INFO("[ROOTSERVICE_NOTICE] finish stop rootservice", KR(ret));
1256
  if (OB_SUCCESS != fail_ret) {
1257
    LOG_DBA_WARN(OB_ERR_ROOTSERVICE_STOP, "msg", "rootservice stop() has failure", KR(fail_ret));
1258
  }
1259
  return ret;
1260
}
1261

1262
void ObRootService::wait()
1263
{
1264
  FLOG_INFO("[ROOTSERVICE_NOTICE] wait rootservice begin");
1265
  int64_t start_time = ObTimeUtility::current_time();
1266
  FLOG_INFO("start to wait all thread exit");
1267
  root_balancer_.wait();
1268
  FLOG_INFO("root balancer exit success");
1269
  empty_server_checker_.wait();
1270
  FLOG_INFO("empty server checker exit success");
1271
  lost_replica_checker_.wait();
1272
  FLOG_INFO("lost replica checker exit success");
1273
  thread_checker_.wait();
1274
  FLOG_INFO("rs_monitor_check : thread checker exit success");
1275
  schema_history_recycler_.wait();
1276
  FLOG_INFO("schema_history_recycler exit success");
1277
  hb_checker_.wait();
1278
  FLOG_INFO("hb checker exit success");
1279
  task_queue_.wait();
1280
  FLOG_INFO("task queue exit success");
1281
  inspect_task_queue_.wait();
1282
  FLOG_INFO("inspect queue exit success");
1283
  ddl_scheduler_.wait();
1284
  FLOG_INFO("ddl task scheduler exit success");
1285
#ifdef OB_BUILD_TDE_SECURITY
1286
  master_key_mgr_.wait();
1287
  FLOG_INFO("master key mgr exit success");
1288
#endif
1289
  disaster_recovery_task_mgr_.wait();
1290
  FLOG_INFO("rebalance task mgr exit success");
1291
  TG_WAIT(lib::TGDefIDs::GlobalCtxTimer);
1292
  FLOG_INFO("global ctx timer exit success");
1293
  ddl_service_.get_index_name_checker().reset_all_cache();
1294
  FLOG_INFO("reset index name checker success");
1295
  ddl_service_.get_non_partitioned_tablet_allocator().reset_all_cache();
1296
  FLOG_INFO("reset non partitioned tablet allocator success");
1297
  ObUpdateRsListTask::clear_lock();
1298
  THE_RS_JOB_TABLE.reset_max_job_id();
1299
  int64_t cost = ObTimeUtility::current_time() - start_time;
1300
  ROOTSERVICE_EVENT_ADD("root_service", "finish_wait_stop", K(cost));
1301
  FLOG_INFO("[ROOTSERVICE_NOTICE] rootservice wait finished", K(start_time), K(cost));
1302
  if (cost > 10 * 60 * 1000 * 1000L) { // 10min
1303
    int ret = OB_ERROR;
1304
    LOG_ERROR("cost too much time to wait rs stop", KR(ret), K(start_time), K(cost));
1305
  }
1306
}
1307

1308
int ObRootService::reload_config()
1309
{
1310
  int ret = OB_SUCCESS;
1311
  if (!inited_) {
1312
    ret = OB_NOT_INIT;
1313
    LOG_WARN("not init", K(ret));
1314
  } else {
1315
    if (OB_FAIL(addr_agent_->reload())) {
1316
      LOG_WARN("root address agent reload failed", K(ret));
1317
    }
1318
  }
1319
  return ret;
1320
}
1321

1322
int ObRootService::submit_update_all_server_config_task()
1323
{
1324
  int ret = OB_SUCCESS;
1325
  ObUpdateAllServerConfigTask task(*this);
1326
  if (!inited_) {
1327
    ret = OB_NOT_INIT;
1328
    LOG_WARN("not init", K(ret));
1329
  } else if (OB_FAIL(task_queue_.add_async_task(task))) {
1330
    LOG_WARN("fail to add async task", K(ret));
1331
  } else {
1332
    LOG_INFO("ass async task for update all server config");
1333
  }
1334
  return ret;
1335
}
1336

1337
int ObRootService::submit_ddl_single_replica_build_task(ObAsyncTask &task)
1338
{
1339
  int ret = OB_SUCCESS;
1340
  if (OB_UNLIKELY(!inited_)) {
1341
    ret = OB_NOT_INIT;
1342
    LOG_WARN("ObRootService has not been inited", K(ret));
1343
  } else if (OB_FAIL(ddl_builder_.push_task(task))) {
1344
    LOG_WARN("add task to ddl builder failed", K(ret));
1345
  }
1346
  return ret;
1347
}
1348

1349
int ObRootService::submit_update_all_server_task(const ObAddr &server)
1350
{
1351
  int ret = OB_SUCCESS;
1352
  if (!inited_) {
1353
    ret = OB_NOT_INIT;
1354
    LOG_WARN("not init", K(ret));
1355
  } else if (!server.is_valid()) {
1356
    ret = OB_INVALID_ARGUMENT;
1357
    LOG_WARN("invalid server", K(server), K(ret));
1358
  } else {
1359
    const bool with_rootserver = (server == self_addr_);
1360
    if (!ObHeartbeatService::is_service_enabled()) {
1361
      ObAllServerTask task(server_manager_, disaster_recovery_task_mgr_, server, with_rootserver);
1362
      if (OB_FAIL(task_queue_.add_async_task(task))) {
1363
        LOG_WARN("inner queue push task failed", K(ret));
1364
      }
1365
    }
1366
  }
1367

1368
  // FIXME: @wanhong.wwh: If self is RS and self status change, submit_update_rslist_task
1369
  if (OB_SUCC(ret)) {
1370
    if (!in_service()) {
1371
      LOG_INFO("self is not RS, need not submit update rslist task in update_all_server_task",
1372
          K(server));
1373
    } else {
1374
      LOG_INFO("self is RS and self status change, submit update rslist task", K(server));
1375
      if (OB_FAIL(submit_update_rslist_task())) {
1376
        LOG_WARN("submit update rslist task failed", KR(ret));
1377
      }
1378
    }
1379
  }
1380
  return ret;
1381
}
1382

1383
int ObRootService::submit_start_server_task(const common::ObAddr &server)
1384
{
1385
  int ret = OB_SUCCESS;
1386
  if (!inited_) {
1387
    ret = OB_NOT_INIT;
1388
    LOG_WARN("not init", K(ret));
1389
  } else if (!server.is_valid()) {
1390
    ret = OB_INVALID_ARGUMENT;
1391
    LOG_WARN("invalid server", K(ret), K(server));
1392
  } else {
1393
    const bool start = true;
1394
    ObStartStopServerTask task(*this, server, start);
1395
    if (OB_FAIL(task_queue_.add_async_task(task))) {
1396
      LOG_WARN("inner queue push task failed", K(ret));
1397
    }
1398
  }
1399
  return ret;
1400
}
1401

1402
int ObRootService::submit_stop_server_task(const common::ObAddr &server)
1403
{
1404
  int ret = OB_SUCCESS;
1405
  if (!inited_) {
1406
    ret = OB_NOT_INIT;
1407
    LOG_WARN("not init", K(ret));
1408
  } else if (!server.is_valid()) {
1409
    ret = OB_INVALID_ARGUMENT;
1410
    LOG_WARN("invalid server", K(ret), K(server));
1411
  } else {
1412
    const bool start = false;
1413
    ObStartStopServerTask task(*this, server, start);
1414
    if (OB_FAIL(task_queue_.add_async_task(task))) {
1415
      LOG_WARN("inner queue push task failed", K(ret));
1416
    }
1417
  }
1418
  return ret;
1419
}
1420

1421
int ObRootService::submit_offline_server_task(const common::ObAddr &server)
1422
{
1423
  int ret = OB_SUCCESS;
1424
  if (!inited_) {
1425
    ret = OB_NOT_INIT;
1426
    LOG_WARN("not init", K(ret));
1427
  } else if (!server.is_valid()) {
1428
    ret = OB_INVALID_ARGUMENT;
1429
    LOG_WARN("invalid server", K(ret), K(server));
1430
  } else {
1431
    ObOfflineServerTask task(*this, server);
1432
    if (OB_FAIL(task_queue_.add_async_task(task))) {
1433
      LOG_WARN("inner queue push task failed", K(ret));
1434
    }
1435
  }
1436
  return ret;
1437
}
1438

1439
int ObRootService::submit_upgrade_task(
1440
    const obrpc::ObUpgradeJobArg &arg)
1441
{
1442
  int ret = OB_SUCCESS;
1443
  ObUpgradeTask task(upgrade_executor_);
1444
  task.set_retry_times(0); //not repeat
1445
  if (!inited_) {
1446
    ret = OB_NOT_INIT;
1447
    LOG_WARN("not init", KR(ret));
1448
  } else if (OB_FAIL(task.init(arg))) {
1449
    LOG_WARN("task init failed", KR(ret), K(arg));
1450
  } else if (OB_FAIL(upgrade_executor_.can_execute())) {
1451
    LOG_WARN("can't run task now", KR(ret), K(arg));
1452
  } else if (OB_FAIL(task_queue_.add_async_task(task))) {
1453
    LOG_WARN("submit upgrade task fail", KR(ret), K(arg));
1454
  } else {
1455
    LOG_INFO("submit upgrade task success", KR(ret), K(arg));
1456
  }
1457
  return ret;
1458
}
1459

1460
int ObRootService::submit_upgrade_storage_format_version_task()
1461
{
1462
  int ret = OB_SUCCESS;
1463
  ObUpgradeStorageFormatVersionTask task(upgrade_storage_format_executor_);
1464
  task.set_retry_times(0);
1465
  if (OB_UNLIKELY(!inited_)) {
1466
    ret = OB_NOT_INIT;
1467
    LOG_WARN("ObRootService has not been inited", K(ret));
1468
  } else if (OB_FAIL(upgrade_storage_format_executor_.can_execute())) {
1469
    LOG_WARN("cannot run task now", K(ret));
1470
  } else if (OB_FAIL(task_queue_.add_async_task(task))) {
1471
    LOG_WARN("submit upgrade storage format version", K(ret));
1472
  } else {
1473
    LOG_INFO("submit upgrade storage format version success", K(ret), K(common::lbt()));
1474
  }
1475
  return ret;
1476
}
1477

1478
int ObRootService::submit_create_inner_schema_task()
1479
{
1480
  int ret = OB_SUCCESS;
1481
  ObCreateInnerSchemaTask task(create_inner_schema_executor_);
1482
  task.set_retry_times(0);
1483
  if (OB_UNLIKELY(!inited_)) {
1484
    ret = OB_NOT_INIT;
1485
    LOG_WARN("ObRootService has not been inited", K(ret));
1486
  } else if (OB_FAIL(create_inner_schema_executor_.can_execute())) {
1487
    LOG_WARN("cannot run task now", K(ret));
1488
  } else if (OB_FAIL(task_queue_.add_async_task(task))) {
1489
    LOG_WARN("submit create inner role task", K(ret));
1490
  } else {
1491
    LOG_INFO("submit create inner role task success", K(ret), K(common::lbt()));
1492
  }
1493
  return ret;
1494
}
1495

1496
int ObRootService::schedule_check_server_timer_task()
1497
{
1498
  int ret = OB_SUCCESS;
1499
  if (!inited_) {
1500
    ret = OB_NOT_INIT;
1501
    LOG_WARN("not init", K(ret));
1502
  } else if (!ObHeartbeatService::is_service_enabled()) {
1503
    if (OB_FAIL(task_queue_.add_timer_task(check_server_task_, config_->server_check_interval, true))) {
1504
      LOG_WARN("failed to add check_server task", K(ret));
1505
    }
1506
  } else {
1507
    LOG_TRACE("no need to schedule ObCheckServerTask in version >= 4.2");
1508
  }
1509
  return ret;
1510
}
1511

1512
int ObRootService::schedule_recyclebin_task(int64_t delay)
1513
{
1514
  int ret = OB_SUCCESS;
1515
  const bool did_repeat = false;
1516

1517
  if (OB_FAIL(get_inspect_task_queue().add_timer_task(
1518
              purge_recyclebin_task_, delay, did_repeat))) {
1519
    if (OB_CANCELED != ret) {
1520
      LOG_ERROR("schedule purge recyclebin task failed", KR(ret), K(delay), K(did_repeat));
1521
    } else {
1522
      LOG_WARN("schedule purge recyclebin task failed", KR(ret), K(delay), K(did_repeat));
1523
    }
1524
  }
1525

1526
  return ret;
1527
}
1528

1529
int ObRootService::schedule_inspector_task()
1530
{
1531
  int ret = OB_SUCCESS;
1532
  int64_t inspect_interval = ObInspector::INSPECT_INTERVAL;
1533
  int64_t delay = 1 * 60 * 1000 * 1000;
1534
  int64_t purge_interval = GCONF._recyclebin_object_purge_frequency;
1535
  int64_t expire_time = GCONF.recyclebin_object_expire_time;
1536
  if (purge_interval > 0 && expire_time > 0) {
1537
    delay = purge_interval;
1538
  }
1539
  if (!inited_) {
1540
    ret = OB_NOT_INIT;
1541
    LOG_WARN("not init", KR(ret));
1542
  } else if (!inspect_task_queue_.exist_timer_task(inspector_task_)
1543
             && OB_FAIL(inspect_task_queue_.add_timer_task(inspector_task_, inspect_interval, true))) {
1544
    LOG_WARN("failed to add inspect task", KR(ret));
1545
  } else if (!inspect_task_queue_.exist_timer_task(purge_recyclebin_task_)
1546
             && OB_FAIL(inspect_task_queue_.add_timer_task(purge_recyclebin_task_,
1547
                                                           delay, false))) {
1548
    LOG_WARN("failed to add purge recyclebin task", KR(ret));
1549
  } else {
1550
    LOG_INFO("schedule inspector task", K(inspect_interval), K(purge_interval));
1551
  }
1552
  return ret;
1553
}
1554

1555
int ObRootService::schedule_self_check_task()
1556
{
1557
  int ret = OB_SUCCESS;
1558
  const bool did_repeat = false;
1559
  const int64_t delay = 5L * 1000L * 1000L; //5s
1560
  if (!inited_) {
1561
    ret = OB_NOT_INIT;
1562
    LOG_WARN("not init", K(ret));
1563
  } else if (task_queue_.exist_timer_task(self_check_task_)) {
1564
    // ignore error
1565
    LOG_WARN("already have one self_check_task, ignore this");
1566
  } else if (OB_FAIL(task_queue_.add_timer_task(self_check_task_, delay, did_repeat))) {
1567
    LOG_WARN("fail to add timer task", K(ret));
1568
  } else {
1569
    LOG_INFO("add self_check task success");
1570
  }
1571
  return ret;
1572
}
1573

1574
int ObRootService::schedule_update_rs_list_task()
1575
{
1576
  int ret = OB_SUCCESS;
1577
  const bool did_repeat = true;
1578
  if (!inited_) {
1579
    ret = OB_NOT_INIT;
1580
    LOG_WARN("not init", K(ret));
1581
  } else if (task_queue_.exist_timer_task(update_rs_list_timer_task_)) {
1582
    // ignore error
1583
    LOG_WARN("already have one update rs list timer task , ignore this");
1584
  } else if (OB_FAIL(task_queue_.add_timer_task(update_rs_list_timer_task_,
1585
                                                ObUpdateRsListTimerTask::RETRY_INTERVAL,
1586
                                                did_repeat))) {
1587
    LOG_WARN("fail to add timer task", K(ret));
1588
  } else {
1589
    LOG_INFO("add update rs list task success");
1590
  }
1591
  return ret;
1592
}
1593
ERRSIM_POINT_DEF(ALL_SERVER_SCHEDULE_ERROR);
1594
int ObRootService::schedule_update_all_server_config_task()
1595
{
1596
  int ret = OB_SUCCESS;
1597
  const bool did_repeat = true;
1598
  if (OB_UNLIKELY(!inited_)) {
1599
    ret = OB_NOT_INIT;
1600
    LOG_WARN("not init", KR(ret), K(inited_));
1601
  } else if (task_queue_.exist_timer_task(update_all_server_config_task_)) {
1602
    // ignore error
1603
    LOG_WARN("already have one update_all_server_config task , ignore this");
1604
  } else if (OB_FAIL(task_queue_.add_timer_task(
1605
      update_all_server_config_task_,
1606
      ALL_SERVER_SCHEDULE_ERROR ? (ObUpdateAllServerConfigTask::RETRY_INTERVAL / 2) : ObUpdateAllServerConfigTask::RETRY_INTERVAL,
1607
      did_repeat))) {
1608
    LOG_WARN("fail to add timer task", KR(ret));
1609
  } else {
1610
    LOG_INFO("add update server config task success");
1611
  }
1612
  return ret;
1613
}
1614

1615
int ObRootService::schedule_load_ddl_task()
1616
{
1617
  int ret = OB_SUCCESS;
1618
  const bool did_repeat = false;
1619
#ifdef ERRSIM
1620
  const int64_t delay = 1000L * 1000L; //1s
1621
#else
1622
  const int64_t delay = 5L * 1000L * 1000L; //5s
1623
#endif
1624
  if (OB_UNLIKELY(!inited_)) {
1625
    ret = OB_NOT_INIT;
1626
    LOG_WARN("not init", K(ret));
1627
  } else if (task_queue_.exist_timer_task(load_ddl_task_)) {
1628
    // ignore error
1629
    LOG_WARN("load ddl task already exist", K(ret));
1630
  } else if (OB_FAIL(task_queue_.add_timer_task(load_ddl_task_, delay, did_repeat))) {
1631
    LOG_WARN("fail to add timer task", K(ret));
1632
  } else {
1633
    LOG_INFO("succeed to add load ddl task");
1634
  }
1635
  return ret;
1636
}
1637

1638
int ObRootService::schedule_refresh_io_calibration_task()
1639
{
1640
  int ret = OB_SUCCESS;
1641
  const bool did_repeat = false;
1642
  const int64_t delay = 5L * 1000L * 1000L; //5s
1643
  if (OB_UNLIKELY(!inited_)) {
1644
    ret = OB_NOT_INIT;
1645
    LOG_WARN("not init", K(ret));
1646
  } else if (task_queue_.exist_timer_task(refresh_io_calibration_task_)) {
1647
    // ignore error
1648
    LOG_WARN("refresh io calibration task already exist", K(ret));
1649
  } else if (OB_FAIL(task_queue_.add_timer_task(refresh_io_calibration_task_, delay, did_repeat))) {
1650
    LOG_WARN("fail to add timer task", K(ret));
1651
  } else {
1652
    LOG_INFO("succeed to add refresh io calibration task");
1653
  }
1654
  return ret;
1655
}
1656

1657
int ObRootService::submit_update_rslist_task(const bool force_update)
1658
{
1659
  int ret = OB_SUCCESS;
1660
  if (OB_UNLIKELY(!inited_)) {
1661
    ret = OB_NOT_INIT;
1662
    LOG_WARN("not init", KR(ret));
1663
  } else {
1664
    if (ObUpdateRsListTask::try_lock()) {
1665
      bool task_added = false;
1666
      ObUpdateRsListTask task;
1667
      if (OB_FAIL(task.init(*lst_operator_, addr_agent_, zone_manager_,
1668
                            broadcast_rs_list_lock_,
1669
                            force_update, self_addr_))) {
1670
        LOG_WARN("task init failed", KR(ret));
1671
      } else if (OB_FAIL(task_queue_.add_async_task(task))) {
1672
        LOG_WARN("inner queue push task failed", K(ret));
1673
      } else {
1674
        task_added = true;
1675
        LOG_INFO("added async task to update rslist", K(force_update));
1676
      }
1677
      if (!task_added) {
1678
        ObUpdateRsListTask::unlock();
1679
      }
1680
    } else {
1681
      LOG_WARN("fail to submit update rslist task, need retry", K(force_update));
1682
    }
1683
  }
1684
  return ret;
1685
}
1686

1687
int ObRootService::submit_report_core_table_replica_task()
1688
{
1689
  int ret = OB_SUCCESS;
1690
  if (OB_UNLIKELY(!inited_)) {
1691
    ret = OB_NOT_INIT;
1692
    LOG_WARN("not init", K(ret));
1693
  } else {
1694
    ObReportCoreTableReplicaTask task(*this);
1695
    if (OB_FAIL(task_queue_.add_async_task(task))) {
1696
      LOG_WARN("inner queue push task failed", K(ret));
1697
    } else {} // no more to do
1698
  }
1699
  return ret;
1700
}
1701

1702
int ObRootService::submit_reload_unit_manager_task()
1703
{
1704
  int ret = OB_SUCCESS;
1705
  if (OB_UNLIKELY(!inited_)) {
1706
    ret = OB_NOT_INIT;
1707
    LOG_WARN("not init", K(ret));
1708
  } else {
1709
    ObReloadUnitManagerTask task(*this, unit_manager_);
1710
    if (OB_FAIL(task_queue_.add_async_task(task))) {
1711
      LOG_WARN("inner queue push reload_unit task failed", K(ret));
1712
    } else {
1713
      LOG_INFO("submit reload unit task success", K(ret));
1714
    }
1715
  }
1716
  return ret;
1717
}
1718

1719
int ObRootService::schedule_restart_timer_task(const int64_t delay)
1720
{
1721
  int ret = OB_SUCCESS;
1722
  if (!inited_) {
1723
    ret = OB_NOT_INIT;
1724
    LOG_WARN("not init", K(ret));
1725
  } else {
1726
    const bool did_repeat = false;
1727
    if (OB_FAIL(task_queue_.add_timer_task(restart_task_,
1728
                                           delay, did_repeat))) {
1729
      LOG_WARN("schedule restart task failed", K(ret), K(delay), K(did_repeat));
1730
    } else {
1731
      LOG_INFO("submit restart task success", K(delay));
1732
    }
1733
  }
1734
  return ret;
1735
}
1736

1737
int ObRootService::schedule_refresh_server_timer_task(const int64_t delay)
1738
{
1739
  int ret = OB_SUCCESS;
1740
  if (!inited_) {
1741
    ret = OB_NOT_INIT;
1742
    LOG_WARN("not init", K(ret));
1743
  } else {
1744
    const bool did_repeat = false;
1745
    if (OB_FAIL(task_queue_.add_timer_task(refresh_server_task_,
1746
                                           delay, did_repeat))) {
1747
      LOG_WARN("schedule restart task failed", K(ret), K(delay), K(did_repeat));
1748
    } else {
1749
      LOG_INFO("schedule refresh server timer task", K(delay));
1750
    }
1751
  }
1752
  return ret;
1753
}
1754

1755
int ObRootService::update_rslist()
1756
{
1757
  int ret = OB_SUCCESS;
1758
  ObUpdateRsListTask task;
1759
  ObTimeoutCtx ctx;
1760
  ctx.set_timeout(config_->rpc_timeout);
1761
  const bool force_update = true;
1762
  if (OB_FAIL(task.init(*lst_operator_, addr_agent_,
1763
                        zone_manager_, broadcast_rs_list_lock_, force_update, self_addr_))) {
1764
    LOG_WARN("task init failed", K(ret), K(force_update));
1765
  } else if (OB_FAIL(task.process_without_lock())) {
1766
    LOG_WARN("failed to update rslist", K(ret));
1767
  } else {
1768
    LOG_INFO("broadcast root address succeed");
1769
  }
1770
  return ret;
1771
}
1772

1773
//only used in bootstrap
1774
int ObRootService::update_all_server_and_rslist()
1775
{
1776
  int ret = OB_SUCCESS;
1777
  if (!inited_) {
1778
    ret = OB_NOT_INIT;
1779
    LOG_WARN("not init", K(ret));
1780
  } else {
1781
    SpinWLockGuard rs_list_guard(broadcast_rs_list_lock_);
1782
    ret = update_rslist();
1783
    if (OB_FAIL(ret)) {
1784
      LOG_INFO("fail to update rslist, ignore it", KR(ret));
1785
      ret = OB_SUCCESS;
1786
    }
1787
  }
1788

1789
  if (OB_SUCC(ret)) {
1790
    ObArray<ObAddr> servers;
1791
    ObZone empty_zone; // empty zone for all servers
1792
    if (OB_FAIL(SVR_TRACER.get_servers_of_zone(empty_zone, servers))) {
1793
      LOG_WARN("get server list failed", K(ret));
1794
    } else {
1795
      FOREACH_X(s, servers, OB_SUCC(ret)) {
1796
        const bool with_rootserver = (*s == self_addr_);
1797
        ObAllServerTask task(server_manager_, disaster_recovery_task_mgr_, *s, with_rootserver);
1798
        if (OB_FAIL(task.process())) {
1799
          LOG_WARN("sync server status to __all_server table failed",
1800
                   K(ret), "server", *s);
1801
        }
1802
      }
1803
    }
1804
  }
1805
  return ret;
1806
}
1807

1808
int ObRootService::request_heartbeats()
1809
{
1810
  int ret = OB_SUCCESS;
1811
  ObServerManager::ObServerStatusArray statuses;
1812
  ObZone zone; // empty zone means all zone
1813
  if (!inited_) {
1814
    ret = OB_NOT_INIT;
1815
    LOG_WARN("not init", K(ret));
1816
  } else if (OB_FAIL(server_manager_.get_server_statuses(zone, statuses))) {
1817
    LOG_WARN("get_server_statuses failed", K(zone), K(ret));
1818
  } else {
1819
    const int64_t rpc_timeout = 250 * 1000; // 250ms
1820
    ObLeaseRequest lease_request;
1821
    // should continue even some failed, so don't look at condition OB_SUCCESS == ret
1822
    FOREACH_CNT(status, statuses) {
1823
      if (ObServerStatus::OB_HEARTBEAT_LEASE_EXPIRED == status->hb_status_
1824
          || (ObServerStatus::OB_SERVER_ADMIN_DELETING == status->admin_status_
1825
              && !status->is_alive())) {
1826
        uint64_t server_id = OB_INVALID_ID;
1827
        lease_request.reset();
1828
        int temp_ret = OB_SUCCESS;
1829
        bool to_alive = false;
1830
        if (OB_SUCCESS != (temp_ret = rpc_proxy_.to(status->server_).timeout(rpc_timeout)
1831
                           .request_heartbeat(lease_request))) {
1832
          LOG_WARN("request_heartbeat failed", "server", status->server_,
1833
                   K(rpc_timeout), K(temp_ret));
1834
        } else if (OB_SUCCESS != (temp_ret = server_manager_.receive_hb(
1835
                    lease_request, server_id, to_alive))) {
1836
          LOG_WARN("receive hb failed", K(lease_request), K(temp_ret));
1837
        }
1838
        ret = (OB_SUCCESS != ret) ? ret : temp_ret;
1839
      }
1840
    }
1841
  }
1842
  return ret;
1843
}
1844

1845
int ObRootService::self_check()
1846
{
1847
  int ret = OB_SUCCESS;
1848
  if (!inited_) {
1849
    ret = OB_NOT_INIT;
1850
    LOG_WARN("not init", K(ret));
1851
  } else if (!in_service()) {
1852
    // nothing todo
1853
  } else if (GCONF.in_upgrade_mode()) {
1854
    // nothing todo
1855
  } else if (OB_FAIL(root_inspection_.check_all())) {  //ignore failed
1856
    LOG_WARN("root_inspection check_all failed", K(ret));
1857
    if (OB_FAIL(schedule_self_check_task())) {
1858
      if (OB_CANCELED != ret) {
1859
        LOG_ERROR("fail to schedule self check task", K(ret));
1860
      }
1861
    }
1862
  }
1863
  return ret;
1864
}
1865

1866
int ObRootService::after_restart()
1867
{
1868
  ObCurTraceId::init(GCONF.self_addr_);
1869

1870
  // avoid concurrent with bootstrap
1871
  FLOG_INFO("[ROOTSERVICE_NOTICE] try to get lock for bootstrap in after_restart");
1872
  ObLatchRGuard guard(bootstrap_lock_, ObLatchIds::RS_BOOTSTRAP_LOCK);
1873

1874
  // NOTE: Following log print after lock
1875
  FLOG_INFO("[ROOTSERVICE_NOTICE] start to do restart task");
1876

1877
  int ret = OB_SUCCESS;
1878
  if (!inited_) {
1879
    ret = OB_NOT_INIT;
1880
    FLOG_WARN("rootservice not init", KR(ret));
1881
  } else if (!ObRootServiceRoleChecker::is_rootserver()) {
1882
    ret = OB_NOT_MASTER;
1883
    FLOG_WARN("not master", KR(ret));
1884
  } else if (need_do_restart() && OB_FAIL(do_restart())) {
1885
    FLOG_WARN("do restart failed, retry again", KR(ret));
1886
  } else if (OB_FAIL(do_after_full_service())) {
1887
    FLOG_WARN("fail to do after full service", KR(ret));
1888
  }
1889

1890
  int64_t cost = ObTimeUtility::current_time() - start_service_time_;
1891
  if (OB_FAIL(ret)) {
1892
    FLOG_WARN("do restart task failed, retry again", KR(ret), K(cost));
1893
  } else if (OB_FAIL(rs_status_.set_rs_status(status::STARTED))) {
1894
    FLOG_WARN("fail to set rs status", KR(ret));
1895
  } else {
1896
    FLOG_INFO("do restart task success, finish restart", KR(ret), K(cost), K_(start_service_time));
1897
  }
1898

1899
  if (OB_FAIL(ret)) {
1900
    rs_status_.try_set_stopping();
1901
    if (rs_status_.is_stopping()) {
1902
      // need stop
1903
      FLOG_INFO("rs_status_ is set to stopping");
1904
    } else {
1905
      const int64_t RETRY_TIMES = 3;
1906
      int64_t tmp_ret = OB_SUCCESS;
1907
      for (int64_t i = 0; i < RETRY_TIMES; ++i) {
1908
        if (OB_SUCCESS != (tmp_ret = schedule_restart_timer_task(config_->rootservice_ready_check_interval))) {
1909
          FLOG_WARN("fail to schedule_restart_timer_task at this retry", KR(tmp_ret), K(i));
1910
        } else {
1911
          FLOG_INFO("success to schedule_restart_timer_task");
1912
          break;
1913
        }
1914
      }
1915
      if (OB_SUCCESS != tmp_ret) {
1916
        LOG_ERROR("fatal error, fail to add restart task", KR(tmp_ret));
1917
        if (OB_FAIL(rs_status_.set_rs_status(status::STOPPING))) {
1918
          LOG_ERROR("fail to set rs status", KR(ret));
1919
        }
1920
      }
1921
    }
1922
  }
1923

1924
  // NOTE: Following log print after lock
1925
  FLOG_INFO("[ROOTSERVICE_NOTICE] finish do restart task", KR(ret));
1926
  return ret;
1927
}
1928

1929
int ObRootService::do_after_full_service() {
1930
  int ret = OB_SUCCESS;
1931
  ObGlobalStatProxy global_proxy(sql_proxy_, OB_SYS_TENANT_ID);
1932
  if (OB_FAIL(global_proxy.get_baseline_schema_version(baseline_schema_version_))) {
1933
    LOG_WARN("fail to get baseline schema version", KR(ret));
1934
  }
1935

1936
  if (OB_FAIL(ret)) {
1937
  } else if (OB_FAIL(schedule_self_check_task())) {
1938
    LOG_WARN("fail to schedule self check task", K(ret));
1939
  } else {
1940
    LOG_INFO("schedule self check to root_inspection success");
1941
  }
1942

1943
  // force broadcast rs list again to make sure rootserver list be updated
1944
  if (OB_SUCC(ret)) {
1945
    int tmp_ret = update_rslist();
1946
    if (OB_SUCCESS != tmp_ret) {
1947
      LOG_WARN("broadcast root address failed", K(tmp_ret));
1948
    }
1949
  }
1950
  return ret;
1951
}
1952

1953
////////////////////////////////////////////////////////////////
1954
int ObRootService::execute_bootstrap(const obrpc::ObBootstrapArg &arg)
1955
{
1956
  int ret = OB_SUCCESS;
1957
  const obrpc::ObServerInfoList &server_list = arg.server_list_;
1958
  BOOTSTRAP_LOG(INFO, "STEP_1.1:execute_bootstrap start to executor.");
1959
  if (!inited_) {
1960
    ret = OB_NOT_INIT;
1961
    LOG_WARN("root_service not inited", K(ret));
1962
  } else if (!sql_proxy_.is_inited() || !sql_proxy_.is_active()) {
1963
    ret = OB_INVALID_ARGUMENT;
1964
    LOG_WARN("sql_proxy not inited or not active", "sql_proxy inited",
1965
             sql_proxy_.is_inited(), "sql_proxy active", sql_proxy_.is_active(), K(ret));
1966
  } else if (server_list.count() <= 0) {
1967
    ret = OB_INVALID_ARGUMENT;
1968
    LOG_WARN("server_list is empty", K(server_list), K(ret));
1969
  } else if (OB_UNLIKELY(nullptr == lst_operator_)) {
1970
    ret = OB_ERR_UNEXPECTED;
1971
    LOG_WARN("lst_operator_ ptr is null", KR(ret), KP(lst_operator_));
1972
  } else {
1973
    update_cpu_quota_concurrency_in_memory_();
1974
    // avoid bootstrap and do_restart run concurrently
1975
    FLOG_INFO("[ROOTSERVICE_NOTICE] try to get lock for bootstrap in execute_bootstrap");
1976
    ObLatchWGuard guard(bootstrap_lock_, ObLatchIds::RS_BOOTSTRAP_LOCK);
1977
    FLOG_INFO("[ROOTSERVICE_NOTICE] success to get lock for bootstrap in execute_bootstrap");
1978
    ObBootstrap bootstrap(rpc_proxy_, *lst_operator_, ddl_service_, unit_manager_,
1979
                          *config_, arg, common_proxy_);
1980
    if (OB_FAIL(bootstrap.execute_bootstrap(server_zone_op_service_))) {
1981
      LOG_ERROR("failed to execute_bootstrap", K(server_list), K(ret));
1982
    }
1983

1984
    BOOTSTRAP_LOG(INFO, "start to do_restart");
1985
    ObGlobalStatProxy global_proxy(sql_proxy_, OB_SYS_TENANT_ID);
1986
    ObArray<ObAddr> self_addr;
1987
    if (OB_FAIL(ret)) {
1988
    } else if (OB_FAIL(do_restart())) {
1989
      LOG_WARN("do restart task failed", K(ret));
1990
    } else if (OB_FAIL(check_ddl_allowed())) {
1991
      LOG_WARN("fail to check ddl allowed", K(ret));
1992
    } else if (OB_FAIL(update_all_server_and_rslist())) {
1993
      LOG_WARN("failed to update all_server and rslist", K(ret));
1994
    } else if (OB_FAIL(zone_manager_.reload())) {
1995
      LOG_WARN("failed to reload zone manager", K(ret));
1996
    } else if (OB_FAIL(set_cluster_version())) {
1997
      LOG_WARN("set cluster version failed", K(ret));
1998
    } else if (OB_FAIL(pl::ObPLPackageManager::load_all_sys_package(sql_proxy_))) {
1999
      LOG_WARN("load all system package failed", K(ret));
2000
    } else if (OB_FAIL(finish_bootstrap())) {
2001
      LOG_WARN("failed to finish bootstrap", K(ret));
2002
    } else if (OB_FAIL(update_baseline_schema_version())) {
2003
      LOG_WARN("failed to update baseline schema version", K(ret));
2004
    } else if (OB_FAIL(global_proxy.get_baseline_schema_version(
2005
                       baseline_schema_version_))) {
2006
      LOG_WARN("fail to get baseline schema version", KR(ret));
2007
    } else if (OB_FAIL(set_cpu_quota_concurrency_config_())) {
2008
      LOG_WARN("failed to update cpu_quota_concurrency", K(ret));
2009
    }
2010

2011
    if (OB_SUCC(ret)) {
2012
      char ori_min_server_version[OB_SERVER_VERSION_LENGTH] = {'\0'};
2013
      uint64_t ori_cluster_version = GET_MIN_CLUSTER_VERSION();
2014
      share::ObServerInfoInTable::ObBuildVersion build_version;
2015
      if (OB_INVALID_INDEX == ObClusterVersion::print_version_str(
2016
          ori_min_server_version, OB_SERVER_VERSION_LENGTH, ori_cluster_version)) {
2017
         ret = OB_INVALID_ARGUMENT;
2018
         LOG_WARN("fail to print version str", KR(ret), K(ori_cluster_version));
2019
      } else if (OB_FAIL(observer::ObService::get_build_version(build_version))) {
2020
        LOG_WARN("fail to get build version", KR(ret));
2021
      } else {
2022
        CLUSTER_EVENT_SYNC_ADD("BOOTSTRAP", "BOOTSTRAP_SUCCESS",
2023
                               "cluster_version", ori_min_server_version,
2024
                               "build_version", build_version.ptr());
2025
      }
2026
    }
2027

2028
    //clear bootstrap flag, regardless failure or success
2029
    int tmp_ret = OB_SUCCESS;
2030
    if (OB_SUCCESS != (tmp_ret = clear_special_cluster_schema_status())) {
2031
      LOG_WARN("failed to clear special cluster schema status",
2032
                KR(ret), K(tmp_ret));
2033
    }
2034
    ret = OB_SUCC(ret) ? tmp_ret : ret;
2035
  }
2036
  BOOTSTRAP_LOG(INFO, "execute_bootstrap finished", K(ret));
2037
  return ret;
2038
}
2039

2040
#ifdef OB_BUILD_TDE_SECURITY
2041
int ObRootService::check_sys_tenant_initial_master_key_valid()
2042
{
2043
  int ret = OB_SUCCESS;
2044
  const int64_t start = ObTimeUtility::current_time();
2045
  const int64_t MAX_WAIT_US = 120L * 1000L * 1000L; //120s
2046
  const int64_t end = start + MAX_WAIT_US;
2047
  const int64_t IDLING_US = 100L * 1000L; // 100ms
2048
  while (OB_SUCC(ret)) {
2049
    if (ObTimeUtility::current_time() >= end) {
2050
      ret = OB_TIMEOUT;
2051
      LOG_WARN("wait sys tenant initial master key valid timeout", KR(ret));
2052
    } else {
2053
      bool has_available_master_key = false;
2054
      if (OB_FAIL(master_key_mgr_.check_if_tenant_has_available_master_keys(
2055
              OB_SYS_TENANT_ID, has_available_master_key))) {
2056
        LOG_WARN("fail to check if tenant has available master key", KR(ret));
2057
      } else if (!has_available_master_key) {
2058
        ob_usleep(std::min(IDLING_US, end - ObTimeUtility::current_time()));
2059
      } else {
2060
        break;
2061
      }
2062
    }
2063
  }
2064
  return ret;
2065
}
2066
#endif
2067

2068
int ObRootService::check_config_result(const char *name, const char* value)
2069
{
2070
  int ret = OB_SUCCESS;
2071
  const int64_t start = ObTimeUtility::current_time();
2072
  const uint64_t DEFAULT_WAIT_US = 120 * 1000 * 1000L; //120s
2073
  int64_t timeout = DEFAULT_WAIT_US;
2074
  if (INT64_MAX != THIS_WORKER.get_timeout_ts()) {
2075
    timeout = MAX(DEFAULT_WAIT_US, THIS_WORKER.get_timeout_remain());
2076
  }
2077
  ObSqlString sql;
2078
  HEAP_VAR(ObMySQLProxy::MySQLResult, res) {
2079
    common::sqlclient::ObMySQLResult *result = NULL;
2080
    if (OB_FAIL(sql.assign_fmt("SELECT count(*) as count FROM %s "
2081
                               "WHERE name = '%s' and value != '%s'",
2082
                               "__all_virtual_tenant_parameter_stat", name, value))) {
2083
      LOG_WARN("fail to append sql", K(ret));
2084
    }
2085
    while(OB_SUCC(ret) || OB_ERR_WAIT_REMOTE_SCHEMA_REFRESH == ret /* remote schema not ready, return -4029 on remote */) {
2086
      if (ObTimeUtility::current_time() - start > timeout) {
2087
        ret = OB_TIMEOUT;
2088
        LOG_WARN("sync config info use too much time", K(ret), K(name), K(value),
2089
                 "cost_us", ObTimeUtility::current_time() - start);
2090
      } else {
2091
        if (OB_FAIL(sql_proxy_.read(res, sql.ptr()))) {
2092
          LOG_WARN("fail to execute sql", K(ret), K(sql));
2093
        } else if (NULL == (result = res.get_result())) {
2094
          ret = OB_ERR_UNEXPECTED;
2095
          LOG_WARN("fail to get sql result", K(ret));
2096
        } else if (OB_FAIL(result->next())) {
2097
          LOG_WARN("fail to get result", K(ret));
2098
        } else {
2099
          int32_t count = OB_INVALID_COUNT;
2100
          EXTRACT_INT_FIELD_MYSQL(*result, "count", count, int32_t);
2101
          if (OB_SUCC(ret)) {
2102
            if (count == 0) { break; }
2103
          }
2104
        }
2105
      }
2106
    } // while end
2107
  }
2108
  return ret;
2109
}
2110

2111
// DDL exection depends on full_service & major_freeze_done state. the sequence of these two status in bootstrap is:
2112
// 1.rs do_restart: major_freeze_launcher start
2113
// 2.rs do_restart success: full_service is true
2114
// 3.root_major_freeze success: major_freeze_done is true (need full_service is true)
2115
// the success of do_restart does not mean to execute DDL, therefor, add wait to bootstrap, to avoid bootstrap failure cause by DDL failure
2116
int ObRootService::check_ddl_allowed()
2117
{
2118
  int ret = OB_SUCCESS;
2119
  const int64_t SLEEP_INTERVAL_US = 1 * 1000 * 1000; //1s
2120
  while (OB_SUCC(ret) && !is_ddl_allowed()) {
2121
    if (!in_service() && !is_start()) {
2122
      ret = OB_RS_SHUTDOWN;
2123
      LOG_WARN("rs shutdown", K(ret));
2124
    } else if (THIS_WORKER.is_timeout()) {
2125
      ret = OB_TIMEOUT;
2126
      LOG_WARN("wait too long", K(ret));
2127
    } else {
2128
      ob_usleep(SLEEP_INTERVAL_US);
2129
    }
2130
  }
2131
  return ret;
2132
}
2133

2134
// used by bootstrap
2135
int ObRootService::update_baseline_schema_version()
2136
{
2137
  int ret = OB_SUCCESS;
2138
  ObMySQLTransaction trans;
2139
  int64_t baseline_schema_version = OB_INVALID_VERSION;
2140
  if (!inited_) {
2141
    ret = OB_NOT_INIT;
2142
    LOG_WARN("not init", K(ret));
2143
  } else if (OB_FAIL(trans.start(&sql_proxy_, OB_SYS_TENANT_ID))) {
2144
    LOG_WARN("trans start failed", K(ret));
2145
  } else if (OB_FAIL(ddl_service_.get_schema_service().
2146
                     get_tenant_refreshed_schema_version(OB_SYS_TENANT_ID,
2147
                                                         baseline_schema_version))) {
2148
    LOG_WARN("fail to get refreshed schema version", K(ret));
2149
  } else {
2150
    ObGlobalStatProxy proxy(trans, OB_SYS_TENANT_ID);
2151
    if (OB_FAIL(proxy.set_baseline_schema_version(baseline_schema_version))) {
2152
      LOG_WARN("set_baseline_schema_version failed", K(baseline_schema_version), K(ret));
2153
    }
2154
  }
2155
  int temp_ret = OB_SUCCESS;
2156
  if (!trans.is_started()) {
2157
  } else if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCCESS == ret))) {
2158
    LOG_WARN("trans end failed", "commit", OB_SUCCESS == ret, K(temp_ret));
2159
    ret = (OB_SUCCESS == ret) ? temp_ret : ret;
2160
  }
2161
    LOG_DEBUG("update_baseline_schema_version finish", K(ret), K(temp_ret),
2162
              K(baseline_schema_version));
2163
  return ret;
2164
}
2165

2166
int ObRootService::finish_bootstrap()
2167
{
2168
  int ret = OB_SUCCESS;
2169
  if (!inited_) {
2170
    ret = OB_NOT_INIT;
2171
    LOG_WARN("not init", K(ret));
2172
  } else {
2173
    const int64_t tenant_id = OB_SYS_TENANT_ID;
2174
    int64_t new_schema_version = OB_INVALID_VERSION;
2175
    ObMultiVersionSchemaService &multi_schema_service = ddl_service_.get_schema_service();
2176
    share::schema::ObSchemaService *tmp_schema_service = multi_schema_service.get_schema_service();
2177
    if (OB_ISNULL(tmp_schema_service)) {
2178
      ret = OB_ERR_UNEXPECTED;
2179
      LOG_WARN("schema service is null", K(ret), KP(tmp_schema_service));
2180
    } else {
2181
      ObMySQLProxy &sql_proxy = ddl_service_.get_sql_proxy();
2182
      share::schema::ObDDLSqlService ddl_sql_service(*tmp_schema_service);
2183
      share::schema::ObSchemaOperation schema_operation;
2184
      schema_operation.op_type_ = share::schema::OB_DDL_FINISH_BOOTSTRAP;
2185
      schema_operation.tenant_id_ = tenant_id;
2186
      if (OB_FAIL(multi_schema_service.gen_new_schema_version(tenant_id, new_schema_version))) {
2187
        LOG_WARN("fail to gen new schema_version", K(ret), K(tenant_id), K(new_schema_version));
2188
      } else if (OB_FAIL(ddl_sql_service.log_nop_operation(schema_operation,
2189
                                                           new_schema_version,
2190
                                                           schema_operation.ddl_stmt_str_,
2191
                                                           sql_proxy))) {
2192
        LOG_WARN("log finish bootstrap operation failed", K(ret), K(schema_operation));
2193
      } else if (OB_FAIL(ddl_service_.refresh_schema(OB_SYS_TENANT_ID))) {
2194
        LOG_WARN("failed to refresh_schema", K(ret));
2195
      } else {
2196
        LOG_INFO("finish bootstrap", K(ret), K(new_schema_version));
2197
      }
2198
    }
2199
  }
2200
  return ret;
2201
}
2202

2203
void ObRootService::construct_lease_expire_time(
2204
     const ObLeaseRequest &lease_request,
2205
     share::ObLeaseResponse &lease_response,
2206
     const share::ObServerStatus &server_status)
2207
{
2208
  UNUSED(lease_request);
2209
  const int64_t now = ObTimeUtility::current_time();
2210
  // if force_stop_hb is true,
2211
  // then lease_expire_time_ won't be changed
2212
  lease_response.heartbeat_expire_time_ = is_full_service() && server_status.force_stop_hb_
2213
                                          ? server_status.last_hb_time_ + config_->lease_time
2214
                                          : now + config_->lease_time;
2215
  lease_response.lease_expire_time_ = lease_response.heartbeat_expire_time_;
2216
}
2217

2218
int ObRootService::renew_lease(const ObLeaseRequest &lease_request, ObLeaseResponse &lease_response)
2219
{
2220
  int ret = OB_SUCCESS;
2221
  ObServerStatus server_stat;
2222
  uint64_t server_id = OB_INVALID_ID;
2223
  bool to_alive = false;
2224
  DEBUG_SYNC(HANG_HEART_BEAT_ON_RS);
2225
  if (!inited_) {
2226
    ret = OB_NOT_INIT;
2227
    LOG_WARN("not init", K(ret));
2228
  } else if (!lease_request.is_valid()) {
2229
    ret = OB_INVALID_ARGUMENT;
2230
    LOG_WARN("invalid lease_request", K(lease_request), K(ret));
2231
  } else if (OB_FAIL(server_manager_.receive_hb(lease_request, server_id, to_alive))) {
2232
    LOG_WARN("server manager receive hb failed", K(lease_request), K(ret));
2233
  } else if (OB_ISNULL(schema_service_)) {
2234
    ret = OB_ERR_UNEXPECTED;
2235
    LOG_WARN("schema_service is null", K(ret));
2236
  } else {
2237
    // before __all_zone load, it may fail, ignore it
2238
    int temp_ret = OB_SUCCESS;
2239
    int64_t lease_info_version = 0;
2240
    bool is_stopped = false;
2241
    lease_response.rs_server_status_ = RSS_INVALID;
2242
    if (is_full_service()) {
2243
      if (OB_FAIL(zone_manager_.get_lease_info_version(lease_info_version))) {
2244
        LOG_WARN("get_lease_info_version failed", K(ret));
2245
      } else if (OB_FAIL(server_manager_.get_server_status(
2246
          lease_request.server_, server_stat))) {
2247
        // get server_stat for construct_lease_expire_time only!
2248
        LOG_WARN("get server status failed", K(ret), "server", lease_request.server_);
2249
      }
2250
      if (!ObHeartbeatService::is_service_enabled()) {
2251
        if (FAILEDx(server_manager_.is_server_stopped(lease_request.server_, is_stopped))) {
2252
          LOG_WARN("check_server_stopped failed", KR(ret), "server", lease_request.server_);
2253
        } else {
2254
          lease_response.rs_server_status_ = is_stopped ? RSS_IS_STOPPED : RSS_IS_WORKING;
2255
        }
2256
      }
2257
#ifdef OB_BUILD_TDE_SECURITY
2258
      if (OB_SUCCESS != (temp_ret = master_key_mgr_.input_server_master_key(
2259
              lease_request.server_, lease_request.tenant_max_flushed_key_version_))) {
2260
        LOG_WARN("fail to input server master key", KR(temp_ret), K(lease_request));
2261
      }
2262
#endif
2263
    }
2264
    if (OB_SUCC(ret)) {
2265
      lease_response.version_ = ObLeaseResponse::LEASE_VERSION;
2266
      construct_lease_expire_time(lease_request, lease_response, server_stat);
2267
      lease_response.lease_info_version_ = lease_info_version;
2268
      lease_response.server_id_ = server_id;
2269
      lease_response.force_frozen_status_ = to_alive;
2270
      lease_response.baseline_schema_version_ = baseline_schema_version_;
2271
      (void)OTC_MGR.get_lease_response(lease_response);
2272

2273
      // after split schema, the schema_version is not used, but for the legality detection, set schema_version to sys's schema_version
2274
      if (OB_SUCCESS != (temp_ret = schema_service_->get_tenant_schema_version(OB_SYS_TENANT_ID, lease_response.schema_version_))) {
2275
        LOG_WARN("fail to get tenant schema version", K(temp_ret));
2276
      }
2277

2278
      if (OB_SUCCESS != (temp_ret = schema_service_->get_refresh_schema_info(
2279
              lease_response.refresh_schema_info_))) {
2280
        LOG_WARN("fail to get refresh_schema_info", K(temp_ret));
2281
      }
2282

2283
#ifdef OB_BUILD_TDE_SECURITY
2284
      if (OB_SUCCESS != (temp_ret = master_key_mgr_.get_all_tenant_master_key(
2285
              lease_request.zone_,
2286
              lease_response.tenant_max_key_version_))) {
2287
        LOG_WARN("fail to get all tenant master key", KR(temp_ret),
2288
                 "server", lease_request.server_, "zone", lease_request.zone_);
2289
      }
2290
#endif
2291
      LOG_TRACE("lease_request", K(lease_request), K(lease_response));
2292
    }
2293
  }
2294
  return ret;
2295
}
2296

2297
int ObRootService::report_sys_ls(const share::ObLSReplica &replica)
2298
{
2299
  int ret = OB_SUCCESS;
2300
  ObInMemoryLSTable *inmemory_ls = NULL;
2301
  ObRole role = FOLLOWER;
2302
  bool inner_table_only = false;
2303
  LOG_INFO("receive request to report sys ls", K(replica));
2304
  if (OB_UNLIKELY(!inited_) || OB_ISNULL(lst_operator_)) {
2305
    ret = OB_NOT_INIT;
2306
    LOG_WARN("rootservice not inited", KR(ret));
2307
  } else if (OB_FAIL(lst_operator_->get_role(OB_SYS_TENANT_ID, SYS_LS, role))) {
2308
    LOG_WARN("fail to get local role by lst_operator", KR(ret));
2309
  } else if (OB_UNLIKELY(!is_strong_leader(role))) {
2310
    ret = OB_RS_NOT_MASTER;
2311
    LOG_WARN("local role is not leader", KR(ret), K(role));
2312
  } else if (OB_UNLIKELY(!replica.is_valid())) {
2313
    ret = OB_INVALID_ARGUMENT;
2314
    LOG_WARN("invalid replica", KR(ret), K(replica));
2315
  } else if (OB_ISNULL(inmemory_ls = lst_operator_->get_inmemory_ls())) {
2316
    ret = OB_ERR_UNEXPECTED;
2317
    LOG_WARN("fail to get inmemory ls", KR(ret), K(replica));
2318
  } else if (OB_FAIL(inmemory_ls->update(replica, inner_table_only))) {
2319
    LOG_WARN("update sys ls failed", KR(ret), K(replica));
2320
  } else {
2321
    LOG_INFO("update sys ls on rs success", K(replica));
2322
  }
2323
  return ret;
2324
}
2325

2326
int ObRootService::remove_sys_ls(const obrpc::ObRemoveSysLsArg &arg)
2327
{
2328
  int ret = OB_SUCCESS;
2329
  ObInMemoryLSTable *inmemory_ls = NULL;
2330
  ObRole role = FOLLOWER;
2331
  bool inner_table_only = false;
2332
  LOG_INFO("receive request to remove sys ls", K(arg));
2333
  if (OB_UNLIKELY(!inited_) || OB_ISNULL(lst_operator_)) {
2334
    ret = OB_NOT_INIT;
2335
    LOG_WARN("rootservice not inited", KR(ret));
2336
  } else if (OB_FAIL(lst_operator_->get_role(OB_SYS_TENANT_ID, SYS_LS, role))) {
2337
    LOG_WARN("fail to get local role by lst_operator", KR(ret));
2338
  } else if (OB_UNLIKELY(!is_strong_leader(role))) {
2339
    ret = OB_RS_NOT_MASTER;
2340
    LOG_WARN("local role is not leader", KR(ret), K(role));
2341
  } else if (OB_UNLIKELY(!arg.is_valid())) {
2342
    ret = OB_INVALID_ARGUMENT;
2343
    LOG_WARN("invalid server", KR(ret), K(arg));
2344
  } else if (OB_ISNULL(inmemory_ls = lst_operator_->get_inmemory_ls())) {
2345
    ret = OB_ERR_UNEXPECTED;
2346
    LOG_WARN("fail to get inmemory ls", KR(ret), K(arg));
2347
  } else if (OB_FAIL(inmemory_ls->remove(
2348
      OB_SYS_TENANT_ID,
2349
      SYS_LS,
2350
      arg.server_,
2351
      inner_table_only))) {
2352
    LOG_WARN("remove sys ls failed", KR(ret), K(arg));
2353
  } else {
2354
    LOG_INFO("remove sys ls on rs success", K(arg));
2355
  }
2356
  return ret;
2357
}
2358

2359
int ObRootService::fetch_location(
2360
    const obrpc::ObFetchLocationArg &arg,
2361
    obrpc::ObFetchLocationResult &res)
2362
{
2363
  int ret = OB_SUCCESS;
2364
  const ObVtableLocationType &vtable_type = arg.get_vtable_type();
2365
  ObSArray<ObAddr> servers;
2366
  if (!inited_) {
2367
    ret = OB_NOT_INIT;
2368
    LOG_WARN("rootservice not init", KR(ret));
2369
  } else if (!arg.is_valid()) {
2370
    ret = OB_INVALID_ARGUMENT;
2371
    LOG_WARN("invalid arg", KR(ret), K(arg));
2372
  } else if (OB_FAIL(vtable_location_getter_.get(vtable_type, servers))) {
2373
    LOG_WARN("vtable_location_getter get failed", KR(ret), K(arg));
2374
  } else if (OB_FAIL(res.set_servers(servers))) {
2375
    LOG_WARN("fail to assign servers", KR(ret), K(servers), K(arg));
2376
  }
2377
  return ret;
2378
}
2379

2380
////////////////////////////////////////////////////////////////
2381

2382
int ObRootService::create_resource_unit(const obrpc::ObCreateResourceUnitArg &arg)
2383
{
2384
  int ret = OB_SUCCESS;
2385
  const bool if_not_exist = arg.get_if_not_exist();
2386
  LOG_INFO("receive create_resource_unit request", K(arg));
2387

2388
  if (!inited_) {
2389
    ret = OB_NOT_INIT;
2390
    LOG_WARN("not init", K(ret));
2391
  } else if (OB_FAIL(unit_manager_.create_unit_config(arg.get_unit_config(), if_not_exist))) {
2392
    LOG_WARN("create_unit_config failed", K(arg), K(if_not_exist), KR(ret));
2393
    int mysql_error = -common::ob_mysql_errno(ret);
2394
    if (OB_TIMEOUT == ret || OB_TIMEOUT == mysql_error) {
2395
      int tmp_ret = OB_SUCCESS;
2396
      if (OB_SUCCESS != (tmp_ret = submit_reload_unit_manager_task())) {
2397
        if (OB_CANCELED != tmp_ret) {
2398
          LOG_ERROR("fail to reload unit_manager, please try 'alter system reload unit', please try 'alter system reload unit'", K(tmp_ret));
2399
        }
2400
      }
2401
    }
2402
  }
2403

2404
  LOG_INFO("finish create_resource_unit", K(arg), KR(ret));
2405
  ROOTSERVICE_EVENT_ADD("root_service", "create_resource_unit", K(ret), K(arg));
2406
  return ret;
2407
}
2408

2409
int ObRootService::alter_resource_unit(const obrpc::ObAlterResourceUnitArg &arg)
2410
{
2411
  int ret = OB_SUCCESS;
2412
  if (!inited_) {
2413
    ret = OB_NOT_INIT;
2414
    LOG_WARN("not init", K(ret));
2415
  } else {
2416
    LOG_INFO("receive alter_resource_unit request", K(arg));
2417
    if (OB_FAIL(unit_manager_.alter_unit_config(arg.get_unit_config()))) {
2418
      LOG_WARN("alter_unit_config failed", K(arg), KR(ret));
2419
      int mysql_error = -common::ob_mysql_errno(ret);
2420
      if (OB_TIMEOUT == ret || OB_TIMEOUT == mysql_error) {
2421
        int tmp_ret = OB_SUCCESS;
2422
        if (OB_SUCCESS != (tmp_ret = submit_reload_unit_manager_task())) {
2423
          if (OB_CANCELED != tmp_ret) {
2424
            LOG_ERROR("fail to reload unit_manager, please try 'alter system reload unit', please try 'alter system reload unit'", K(tmp_ret));
2425
          }
2426
        }
2427
      }
2428
    }
2429
    LOG_INFO("finish alter_resource_unit", K(arg), KR(ret));
2430
  }
2431
  ROOTSERVICE_EVENT_ADD("root_service", "alter_resource_unit", K(ret), K(arg));
2432
  return ret;
2433
}
2434

2435
int ObRootService::drop_resource_unit(const obrpc::ObDropResourceUnitArg &arg)
2436
{
2437
  int ret = OB_SUCCESS;
2438
  const bool if_exist = arg.if_exist_;
2439
  if (!inited_) {
2440
    ret = OB_NOT_INIT;
2441
    LOG_WARN("not init", K(ret));
2442
  } else if (!arg.is_valid()) {
2443
    ret = OB_INVALID_ARGUMENT;
2444
    LOG_WARN("invalid arg", K(arg), K(ret));
2445
  } else {
2446
    LOG_INFO("receive drop_resource_unit request", K(arg));
2447
    if (OB_FAIL(unit_manager_.drop_unit_config(arg.unit_name_, if_exist))) {
2448
      LOG_WARN("drop_unit_config failed", "unit_config", arg.unit_name_, K(if_exist), K(ret));
2449
      int mysql_error = -common::ob_mysql_errno(ret);
2450
      if (OB_TIMEOUT == ret || OB_TIMEOUT == mysql_error) {
2451
        int tmp_ret = OB_SUCCESS;
2452
        if (OB_SUCCESS != (tmp_ret = submit_reload_unit_manager_task())) {
2453
          if (OB_CANCELED != tmp_ret) {
2454
            LOG_ERROR("fail to reload unit_manager, please try 'alter system reload unit'", K(tmp_ret));
2455
          }
2456
        }
2457
      }
2458
    }
2459
    LOG_INFO("finish drop_resource_unit", K(arg), K(ret));
2460
  }
2461
  ROOTSERVICE_EVENT_ADD("root_service", "drop_resource_unit", K(ret), K(arg));
2462
  return ret;
2463
}
2464

2465
int ObRootService::clone_resource_pool(const obrpc::ObCloneResourcePoolArg &arg)
2466
{
2467
  int ret = OB_SUCCESS;
2468
  bool is_compatible = false;
2469
  LOG_INFO("receive clone_resource_pool request", K(arg));
2470
  if (!inited_) {
2471
    ret = OB_NOT_INIT;
2472
    LOG_WARN("rootservice not init", KR(ret), K_(inited));
2473
  } else if (!arg.is_valid()) {
2474
    ret = OB_INVALID_ARGUMENT;
2475
    LOG_WARN("invalid argument to clone resource pool", KR(ret), K(arg));
2476
  } else if (OB_FAIL(ObShareUtil::check_compat_version_for_clone_tenant(
2477
                         arg.get_source_tenant_id(),
2478
                         is_compatible))) {
2479
    LOG_WARN("fail to check compat version", KR(ret), K(arg));
2480
  } else if (!is_compatible) {
2481
    ret = OB_STATE_NOT_MATCH;
2482
    LOG_WARN("clone tenant or sys tenant data version is below 4.3", KR(ret), K(arg), K(is_compatible));
2483
  } else {
2484
    share::ObResourcePool pool_to_clone;
2485
    pool_to_clone.name_ = arg.get_pool_name();
2486
    pool_to_clone.resource_pool_id_ = arg.get_resource_pool_id();
2487
    if (OB_FAIL(unit_manager_.clone_resource_pool(pool_to_clone, arg.get_unit_config_name(), arg.get_source_tenant_id()))) {
2488
      LOG_WARN("clone_resource_pool failed", KR(ret), K(pool_to_clone), K(arg));
2489
      int tmp_ret = OB_SUCCESS;
2490
      if (OB_SUCCESS != (tmp_ret = submit_reload_unit_manager_task())) {
2491
        if (OB_CANCELED != tmp_ret) {
2492
          LOG_ERROR("fail to reload unit_manager, please try 'alter system reload unit'", K(tmp_ret));
2493
        }
2494
      }
2495
    }
2496
  }
2497
  LOG_INFO("finish clone_resource_pool", KR(ret), K(arg));
2498
  ROOTSERVICE_EVENT_ADD("root_service", "clone_resource_pool", KR(ret), K(arg));
2499
  return ret;
2500
}
2501

2502
int ObRootService::create_resource_pool(const obrpc::ObCreateResourcePoolArg &arg)
2503
{
2504
  int ret = OB_SUCCESS;
2505
  const bool if_not_exist = arg.if_not_exist_;
2506
  if (!inited_) {
2507
    ret = OB_NOT_INIT;
2508
    LOG_WARN("not init", K(ret));
2509
  } else if (!arg.is_valid()) {
2510
    ret = OB_MISS_ARGUMENT;
2511
    if (arg.pool_name_.empty()) {
2512
      LOG_USER_ERROR(OB_MISS_ARGUMENT, "resource pool name");
2513
    } else if (arg.unit_.empty()) {
2514
      LOG_USER_ERROR(OB_MISS_ARGUMENT, "unit");
2515
    } else if (arg.unit_num_ <= 0) {
2516
      LOG_USER_ERROR(OB_MISS_ARGUMENT, "unit_num");
2517
    }
2518
    LOG_WARN("missing arg to create resource pool", K(arg), K(ret));
2519
  } else if (REPLICA_TYPE_LOGONLY != arg.replica_type_
2520
             && REPLICA_TYPE_FULL != arg.replica_type_) {
2521
    ret = OB_NOT_SUPPORTED;
2522
    LOG_WARN("only full/logonly pool are supported", K(ret), K(arg));
2523
  } else if (REPLICA_TYPE_LOGONLY == arg.replica_type_
2524
             && arg.unit_num_> 1) {
2525
    ret = OB_NOT_SUPPORTED;
2526
    LOG_WARN("logonly resource pool should only have one unit on one zone", K(ret), K(arg));
2527
  } else if (0 == arg.unit_.case_compare(OB_STANDBY_UNIT_CONFIG_TEMPLATE_NAME)) {
2528
    ret = OB_OP_NOT_ALLOW;
2529
    LOG_WARN("can not create resource pool use standby unit config template", K(ret), K(arg));
2530
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "create resource pool use stanby unit config template");
2531
  } else {
2532
    LOG_INFO("receive create_resource_pool request", K(arg));
2533
    share::ObResourcePool pool;
2534
    pool.name_ = arg.pool_name_;
2535
    pool.unit_count_ = arg.unit_num_;
2536
    pool.replica_type_ = arg.replica_type_;
2537
    if (OB_FAIL(pool.zone_list_.assign(arg.zone_list_))) {
2538
      LOG_WARN("assign failed", K(ret));
2539
    } else if (OB_FAIL(unit_manager_.create_resource_pool(pool, arg.unit_, if_not_exist))) {
2540
      LOG_WARN("create_resource_pool failed", K(pool),
2541
               "unit_config", arg.unit_, K(if_not_exist), K(ret));
2542
      int mysql_error = -common::ob_mysql_errno(ret);
2543
      if (OB_TIMEOUT == ret || OB_TIMEOUT == mysql_error) {
2544
        int tmp_ret = OB_SUCCESS;
2545
        if (OB_SUCCESS != (tmp_ret = submit_reload_unit_manager_task())) {
2546
          if (OB_CANCELED != tmp_ret) {
2547
            LOG_ERROR("fail to reload unit_manager, please try 'alter system reload unit'", K(tmp_ret));
2548
          }
2549
        }
2550
      }
2551
    }
2552
    LOG_INFO("finish create_resource_pool", K(arg), K(ret));
2553
  }
2554
  ROOTSERVICE_EVENT_ADD("root_service", "create_resource_pool", K(ret), K(arg));
2555
  return ret;
2556
}
2557

2558
int ObRootService::split_resource_pool(const obrpc::ObSplitResourcePoolArg &arg)
2559
{
2560
  int ret = OB_SUCCESS;
2561
  if (!inited_) {
2562
    ret = OB_NOT_INIT;
2563
    LOG_WARN("not init", K(ret));
2564
  } else if (!arg.is_valid()) {
2565
    ret = OB_INVALID_ARGUMENT;
2566
    LOG_WARN("invalid argument", K(ret), K(arg));
2567
  } else {
2568
    LOG_INFO("receive split resource pool request", K(arg));
2569
    share::ObResourcePoolName pool_name = arg.pool_name_;
2570
    const common::ObIArray<common::ObString> &split_pool_list = arg.split_pool_list_;
2571
    const common::ObIArray<common::ObZone> &zone_list = arg.zone_list_;
2572
    if (OB_FAIL(unit_manager_.split_resource_pool(pool_name, split_pool_list, zone_list))) {
2573
      LOG_WARN("fail to split resource pool", K(ret));
2574
      int mysql_error = -common::ob_mysql_errno(ret);
2575
      if (OB_TIMEOUT == ret || OB_TIMEOUT == mysql_error) {
2576
        int tmp_ret = OB_SUCCESS;
2577
        if (OB_SUCCESS != (tmp_ret = submit_reload_unit_manager_task())) {
2578
          if (OB_CANCELED != tmp_ret) {
2579
            LOG_ERROR("fail to reload unit_mgr, please try 'alter system reload unit'", K(tmp_ret));
2580
          }
2581
        }
2582
      }
2583
    }
2584
    LOG_INFO("finish split_resource_pool", K(ret), K(arg));
2585
  }
2586
  ROOTSERVICE_EVENT_ADD("root_service", "split_resource_pool", K(ret), K(arg));
2587
  return ret;
2588
}
2589

2590
int ObRootService::alter_resource_tenant(const obrpc::ObAlterResourceTenantArg &arg)
2591
{
2592
  int ret = OB_SUCCESS;
2593
  if (OB_UNLIKELY(!inited_)) {
2594
    ret = OB_NOT_INIT;
2595
    LOG_WARN("not init", KR(ret));
2596
  } else if (OB_UNLIKELY(!arg.is_valid())) {
2597
    ret = OB_INVALID_ARGUMENT;
2598
    LOG_WARN("invalid argument", KR(ret), K(arg));
2599
  } else if (OB_UNLIKELY(nullptr == schema_service_)) {
2600
    ret = OB_ERR_UNEXPECTED;
2601
    LOG_WARN("schema_service ptr is null", KR(ret));
2602
  } else {
2603
    LOG_INFO("receive alter resource tenant request", K(arg));
2604
    const ObString &target_tenant_name = arg.tenant_name_;
2605
    const int64_t new_unit_num = arg.unit_num_;
2606
    const common::ObIArray<uint64_t> &delete_unit_group_id_array = arg.unit_group_id_array_;
2607
    share::schema::ObSchemaGetterGuard schema_guard;
2608
    uint64_t target_tenant_id = OB_INVALID_ID;
2609
    int tmp_ret = OB_SUCCESS;
2610

2611
    if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
2612
      LOG_WARN("fail to get tenant schema guard", KR(ret), "tenant_id", OB_SYS_TENANT_ID);
2613
    } else if (OB_FAIL(schema_guard.get_tenant_id(target_tenant_name, target_tenant_id))) {
2614
      LOG_WARN("fail to get tenant id", KR(ret), K(target_tenant_name));
2615
    } else if (OB_UNLIKELY(OB_INVALID_ID == target_tenant_id)) {
2616
      ret = OB_ERR_UNEXPECTED;
2617
      LOG_WARN("target_tenant_id value unexpected", KR(ret), K(target_tenant_name), K(target_tenant_id));
2618
    } else if (OB_FAIL(unit_manager_.alter_resource_tenant(
2619
            target_tenant_id, new_unit_num, delete_unit_group_id_array, arg.ddl_stmt_str_))) {
2620
      LOG_WARN("fail to alter resource tenant", KR(ret), K(target_tenant_id),
2621
               K(new_unit_num), K(delete_unit_group_id_array));
2622
      if (OB_TMP_FAIL(submit_reload_unit_manager_task())) {
2623
        if (OB_CANCELED != tmp_ret) {
2624
          LOG_ERROR("fail to reload unit_mgr, please try 'alter system reload unit'", KR(ret), KR(tmp_ret));
2625
        }
2626
      }
2627
    }
2628
    LOG_INFO("finish alter_resource_tenant", KR(ret), K(arg));
2629
  }
2630

2631
  ROOTSERVICE_EVENT_ADD("root_service", "alter_resource_tenant", K(ret), K(arg));
2632
  return ret;
2633
}
2634

2635
int ObRootService::merge_resource_pool(const obrpc::ObMergeResourcePoolArg &arg)
2636
{
2637
  int ret = OB_SUCCESS;
2638
  if (!inited_) {
2639
    ret = OB_NOT_INIT;
2640
    LOG_WARN("not init", K(ret));
2641
  } else if (!arg.is_valid()) {
2642
    ret = OB_INVALID_ARGUMENT;
2643
    LOG_WARN("invalid argument", K(ret), K(arg));
2644
  } else {
2645
    LOG_INFO("receive merge resource pool request", K(arg));
2646
    const common::ObIArray<common::ObString> &old_pool_list = arg.old_pool_list_;
2647
    const common::ObIArray<common::ObString> &new_pool_list = arg.new_pool_list_;
2648
    if (OB_FAIL(unit_manager_.merge_resource_pool(old_pool_list, new_pool_list))) {
2649
      LOG_WARN("fail to merge resource pool", K(ret));
2650
      int tmp_ret = OB_SUCCESS;
2651
      if (OB_SUCCESS != (tmp_ret = submit_reload_unit_manager_task())) {//ensure submit task all case
2652
        if (OB_CANCELED != tmp_ret) {
2653
          LOG_ERROR("fail to reload unit_mgr, please try 'alter system reload unit'", KR(ret), K(tmp_ret));
2654
        }
2655
      }
2656
    }
2657
    LOG_INFO("finish merge_resource_pool", K(ret), K(arg));
2658
  }
2659
  ROOTSERVICE_EVENT_ADD("root_service", "merge_resource_pool", K(ret), K(arg));
2660
  return ret;
2661
}
2662

2663
int ObRootService::alter_resource_pool(const obrpc::ObAlterResourcePoolArg &arg)
2664
{
2665
  int ret = OB_SUCCESS;
2666
  if (!inited_) {
2667
    ret = OB_NOT_INIT;
2668
    LOG_WARN("not init", K(ret));
2669
  } else if (!arg.is_valid()) {
2670
    ret = OB_MISS_ARGUMENT;
2671
    if (arg.pool_name_.empty()) {
2672
      LOG_USER_ERROR(OB_MISS_ARGUMENT, "resource pool name");
2673
    }
2674
    LOG_WARN("missing arg to alter resource pool", K(arg), K(ret));
2675
  } else if (0 == arg.unit_.case_compare(OB_STANDBY_UNIT_CONFIG_TEMPLATE_NAME)) {
2676
    ret = OB_OP_NOT_ALLOW;
2677
    LOG_WARN("can not alter resource pool use standby unit config template", K(ret), K(arg));
2678
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "alter resource pool use stanby unit config template");
2679
  } else {
2680
    LOG_INFO("receive alter_resource_pool request", K(arg));
2681
    share::ObResourcePool pool;
2682
    pool.name_ = arg.pool_name_;
2683
    pool.unit_count_ = arg.unit_num_;
2684
    if (OB_FAIL(pool.zone_list_.assign(arg.zone_list_))) {
2685
      LOG_WARN("assign failed", K(ret));
2686
    } else if (OB_FAIL(unit_manager_.alter_resource_pool(
2687
            pool, arg.unit_, arg.delete_unit_id_array_))) {
2688
      LOG_WARN("alter_resource_pool failed", K(pool), K(arg), "resource unit", arg.unit_, K(ret));
2689
      int tmp_ret = OB_SUCCESS;
2690
      if (OB_TMP_FAIL(submit_reload_unit_manager_task())) {//ensure submit task all case
2691
        if (OB_CANCELED != tmp_ret) {
2692
          LOG_ERROR("fail to reload unit_mgr, please try 'alter system reload unit'", KR(ret), K(tmp_ret));
2693
        }
2694
      }
2695
    }
2696
    LOG_INFO("finish alter_resource_pool", K(arg), K(ret));
2697
  }
2698
  ROOTSERVICE_EVENT_ADD("root_service", "alter_resource_pool", K(ret), K(arg));
2699
  return ret;
2700
}
2701

2702
int ObRootService::drop_resource_pool(const obrpc::ObDropResourcePoolArg &arg)
2703
{
2704
  int ret = OB_SUCCESS;
2705
  const bool if_exist = arg.if_exist_;
2706
  if (!inited_) {
2707
    ret = OB_NOT_INIT;
2708
    LOG_WARN("not init", K(ret));
2709
  } else if (!arg.is_valid()) {
2710
    ret = OB_MISS_ARGUMENT;
2711
    if (arg.pool_name_.empty()) {
2712
      LOG_USER_ERROR(OB_MISS_ARGUMENT, "resource pool name");
2713
    }
2714
    LOG_WARN("missing arg to drop resource pool", K(arg), K(ret));
2715
  } else {
2716
    LOG_INFO("receive drop_resource_pool request", K(arg));
2717
    if (OB_INVALID_ID != arg.pool_id_) {
2718
      if (OB_FAIL(unit_manager_.drop_resource_pool(arg.pool_id_, if_exist))) {
2719
        LOG_WARN("drop_resource_pool failed", "pool", arg.pool_id_, K(if_exist), KR(ret));
2720
      }
2721
    } else {
2722
      if (OB_FAIL(unit_manager_.drop_resource_pool(arg.pool_name_, if_exist))) {
2723
        LOG_WARN("drop_resource_pool failed", "pool", arg.pool_name_, K(if_exist), KR(ret));
2724
      }
2725
    }
2726

2727
    if (OB_FAIL(ret)) {
2728
      int mysql_error = -common::ob_mysql_errno(ret);
2729
      if (OB_TIMEOUT == ret || OB_TIMEOUT == mysql_error) {
2730
        int tmp_ret = OB_SUCCESS;
2731
        if (OB_SUCCESS != (tmp_ret = submit_reload_unit_manager_task())) {
2732
          if (OB_CANCELED != tmp_ret) {
2733
            LOG_ERROR("fail to reload unit_manager, please try 'alter system reload unit'", K(tmp_ret));
2734
          }
2735
        }
2736
      }
2737
    }
2738
    LOG_INFO("finish drop_resource_pool", K(arg), K(ret));
2739
  }
2740
  ROOTSERVICE_EVENT_ADD("root_service", "drop_resource_pool", K(ret), K(arg));
2741
  return ret;
2742
}
2743

2744
int ObRootService::check_tenant_in_alter_locality(
2745
    const uint64_t tenant_id,
2746
    bool &in_alter_locality)
2747
{
2748
  int ret = OB_SUCCESS;
2749
  if (OB_UNLIKELY(!inited_)) {
2750
    ret = OB_NOT_INIT;
2751
    LOG_WARN("not init", K(ret));
2752
  } else if (OB_UNLIKELY(OB_INVALID_ID == tenant_id)) {
2753
    ret = OB_INVALID_ARGUMENT;
2754
    LOG_WARN("invalid argument", K(ret));
2755
  } else if (OB_FAIL(ddl_service_.check_tenant_in_alter_locality(tenant_id, in_alter_locality))) {
2756
    LOG_WARN("fail to check tenant in alter locality", K(ret));
2757
  } else {} // no more to do
2758
  return ret;
2759
}
2760

2761
int ObRootService::create_tenant(const ObCreateTenantArg &arg, UInt64 &tenant_id)
2762
{
2763
  LOG_INFO("receive create tenant arg", K(arg), "timeout_ts", THIS_WORKER.get_timeout_ts());
2764
  int ret = OB_SUCCESS;
2765
  int tmp_ret = OB_SUCCESS;
2766
  bool compatible_with_clone_tenant = false;
2767
  const ObString &tenant_name = arg.tenant_schema_.get_tenant_name_str();
2768
  // when recovering table, it needs to create tmp tenant
2769
  const bool tmp_tenant = arg.is_tmp_tenant_for_recover_;
2770
  if (!inited_) {
2771
    ret = OB_NOT_INIT;
2772
    LOG_WARN("not init", KR(ret));
2773
  } else if (!tmp_tenant && OB_FAIL(ObResolverUtils::check_not_supported_tenant_name(tenant_name))) {
2774
    LOG_WARN("unsupported tenant name", KR(ret), K(tenant_name));
2775
  } else if (arg.is_clone_tenant()
2776
             && OB_FAIL(ObShareUtil::check_compat_version_for_clone_tenant(
2777
                            arg.source_tenant_id_,
2778
                            compatible_with_clone_tenant))) {
2779
    LOG_WARN("fail to check compatible version with clone tenant", KR(ret), K(arg));
2780
  } else if (arg.is_clone_tenant() && !compatible_with_clone_tenant) {
2781
    ret = OB_OP_NOT_ALLOW;
2782
    LOG_WARN("create clone tenant with data version below 4.3 not allowed",
2783
             KR(ret), K(arg), K(compatible_with_clone_tenant));
2784
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "create clone tenant with data version below 4.3");
2785
  } else if (OB_FAIL(ddl_service_.create_tenant(arg, tenant_id))) {
2786
    LOG_WARN("fail to create tenant", KR(ret), K(arg));
2787
    if (OB_TMP_FAIL(submit_reload_unit_manager_task())) {
2788
      if (OB_CANCELED != tmp_ret) {
2789
        LOG_ERROR("fail to reload unit_mgr, please try 'alter system reload unit'", KR(ret), KR(tmp_ret));
2790
      }
2791
    }
2792
  }
2793
  LOG_INFO("finish create tenant", KR(ret), K(tenant_id), K(arg), "timeout_ts", THIS_WORKER.get_timeout_ts());
2794
  return ret;
2795
}
2796

2797
int ObRootService::create_tenant_end(const ObCreateTenantEndArg &arg)
2798
{
2799
  LOG_DEBUG("receive create tenant end arg", K(arg));
2800
  int ret = OB_SUCCESS;
2801
  if (!inited_) {
2802
    ret = OB_NOT_INIT;
2803
    LOG_WARN("not init", K(ret));
2804
  } else if (!arg.is_valid()) {
2805
    ret = OB_INVALID_ARGUMENT;
2806
    LOG_WARN("invalid arg", K(ret));
2807
  } else if (OB_FAIL(ddl_service_.create_tenant_end(arg.tenant_id_))) {
2808
    LOG_WARN("fail to create tenant end", K(ret), K(arg));
2809
  } else {
2810
    LOG_INFO("success to create tenant end", K(ret), K(arg));
2811
  }
2812
  return ret;
2813
}
2814

2815
int ObRootService::commit_alter_tenant_locality(
2816
    const rootserver::ObCommitAlterTenantLocalityArg &arg)
2817
{
2818
  int ret = OB_SUCCESS;
2819
  LOG_INFO("commit alter tenant locality", K(arg));
2820
  if (OB_UNLIKELY(!inited_)) {
2821
    ret = OB_NOT_INIT;
2822
    LOG_WARN("not init", K(ret));
2823
  } else if (!arg.is_valid()) {
2824
    ret = OB_INVALID_ARGUMENT;
2825
    LOG_WARN("invalid argument", K(ret), K(arg));
2826
  } else if (OB_FAIL(ddl_service_.commit_alter_tenant_locality(arg))) {
2827
    LOG_WARN("fail to commit alter tenant locality", K(ret));
2828
  } else {
2829
    LOG_INFO("commit alter tenant locality succeed", K(ret));
2830
  }
2831
  return ret;
2832
}
2833

2834

2835
int ObRootService::drop_tenant(const ObDropTenantArg &arg)
2836
{
2837
  int ret = OB_SUCCESS;
2838
  if (!inited_) {
2839
    ret = OB_NOT_INIT;
2840
    LOG_WARN("not init", K(ret));
2841
  } else if (!arg.is_valid()) {
2842
    ret = OB_INVALID_ARGUMENT;
2843
    LOG_WARN("invalid arg", K(arg), K(ret));
2844
  } else if (OB_FAIL(ddl_service_.drop_tenant(arg))) {
2845
    LOG_WARN("ddl_service_ drop_tenant failed", K(arg), K(ret));
2846
  }
2847
  return ret;
2848
}
2849

2850
int ObRootService::flashback_tenant(const ObFlashBackTenantArg &arg)
2851
{
2852
  int ret = OB_SUCCESS;
2853
  if (!inited_) {
2854
    ret = OB_NOT_INIT;
2855
    LOG_WARN("not init", K(ret));
2856
  } else if (!arg.is_valid()) {
2857
    ret = OB_INVALID_ARGUMENT;
2858
    LOG_WARN("invalid argument", K(arg), K(ret));
2859
  } else if (OB_FAIL(ObResolverUtils::check_not_supported_tenant_name(arg.new_tenant_name_))) {
2860
    LOG_WARN("unsupported tenant name", KR(ret), "new_tenant_name", arg.new_tenant_name_);
2861
  } else if (OB_FAIL(ddl_service_.flashback_tenant(arg))) {
2862
    LOG_WARN("failed to flash back tenant", K(ret));
2863
  }
2864
  LOG_INFO("flashback tenant success");
2865
  return ret;
2866
}
2867

2868
int ObRootService::purge_tenant(const ObPurgeTenantArg &arg)
2869
{
2870
  int ret = OB_SUCCESS;
2871
  if (!inited_) {
2872
    ret = OB_NOT_INIT;
2873
    LOG_WARN("not init", K(ret));
2874
  } else if (!arg.is_valid()) {
2875
    ret = OB_INVALID_ARGUMENT;
2876
    LOG_WARN("invalid argument", K(arg), K(ret));
2877
  } else if (OB_FAIL(ddl_service_.purge_tenant(arg))) {
2878
    LOG_WARN("failed to purge tenant", K(ret));
2879
  }
2880
  LOG_INFO("purge tenant success");
2881
  return ret;
2882
}
2883

2884
int ObRootService::modify_tenant(const ObModifyTenantArg &arg)
2885
{
2886
  LOG_DEBUG("receive modify tenant arg", K(arg));
2887
  int ret = OB_NOT_SUPPORTED;
2888
  if (!inited_) {
2889
    ret = OB_NOT_INIT;
2890
    LOG_WARN("not init", K(ret));
2891
  } else if (!arg.is_valid()) {
2892
    ret = OB_INVALID_ARGUMENT;
2893
    LOG_WARN("invalid arg", K(arg), K(ret));
2894
  } else if (OB_FAIL(ObResolverUtils::check_not_supported_tenant_name(arg.new_tenant_name_))) {
2895
    LOG_WARN("unsupported tenant name", KR(ret), "new_tenant_name", arg.new_tenant_name_);
2896
  } else if (OB_FAIL(ddl_service_.modify_tenant(arg))) {
2897
    LOG_WARN("ddl service modify tenant failed", K(arg), K(ret));
2898
  } else {
2899
    root_balancer_.wakeup();
2900
  }
2901
  // weak leader coordinator while modify primary zone
2902
  //if (OB_SUCC(ret)
2903
  //    && arg.alter_option_bitset_.has_member(obrpc::ObModifyTenantArg::PRIMARY_ZONE)) {
2904
  //  leader_coordinator_.signal();
2905
  //}
2906
  return ret;
2907
}
2908

2909
int ObRootService::lock_tenant(const obrpc::ObLockTenantArg &arg)
2910
{
2911
  int ret = OB_SUCCESS;
2912
  LOG_INFO("receive lock tenant request", K(arg));
2913
  if (!inited_) {
2914
    ret = OB_NOT_INIT;
2915
    LOG_WARN("not init", K(ret));
2916
  } else if (!arg.is_valid()) {
2917
    ret = OB_INVALID_ARGUMENT;
2918
    LOG_WARN("invalid arg", K(arg), K(ret));
2919
  } else if (OB_FAIL(ddl_service_.lock_tenant(arg.tenant_name_, arg.is_locked_))) {
2920
    LOG_WARN("ddl_service lock_tenant failed", K(arg), K(ret));
2921
  }
2922
  LOG_INFO("finish lock tenant", K(arg), K(ret));
2923
  return ret;
2924
}
2925

2926
int ObRootService::add_system_variable(const ObAddSysVarArg &arg)
2927
{
2928
  int ret = OB_SUCCESS;
2929
  if (!inited_) {
2930
    ret = OB_NOT_INIT;
2931
    LOG_WARN("not init", K(ret));
2932
  } else if (OB_UNLIKELY(!arg.is_valid())) {
2933
    ret = OB_INVALID_ARGUMENT;
2934
    LOG_WARN("invalid sysvar arg", K(arg));
2935
  } else if (OB_FAIL(ddl_service_.add_system_variable(arg))) {
2936
    LOG_WARN("add system variable failed", K(ret));
2937
  }
2938
  return ret;
2939
}
2940

2941
int ObRootService::modify_system_variable(const obrpc::ObModifySysVarArg &arg)
2942
{
2943
  int ret = OB_SUCCESS;
2944
  if (!inited_) {
2945
    ret = OB_NOT_INIT;
2946
    LOG_WARN("not init", K(ret));
2947
  } else if (OB_UNLIKELY(!arg.is_valid())) {
2948
    ret = OB_INVALID_ARGUMENT;
2949
    LOG_WARN("invalid sysvar arg", K(arg));
2950
  } else if (OB_FAIL(ddl_service_.modify_system_variable(arg))) {
2951
    LOG_WARN("modify system variable failed", K(ret));
2952
  }
2953
  return ret;
2954
}
2955

2956
int ObRootService::create_database(const ObCreateDatabaseArg &arg, UInt64 &db_id)
2957
{
2958
  int ret = OB_SUCCESS;
2959
  if (!inited_) {
2960
    ret = OB_NOT_INIT;
2961
    LOG_WARN("not init", K(ret));
2962
  } else if (!arg.is_valid()) {
2963
    ret = OB_INVALID_ARGUMENT;
2964
    LOG_WARN("invalid arg", K(arg), K(ret));
2965
  } else {
2966
    ObDatabaseSchema copied_db_schema = arg.database_schema_;
2967
    if (OB_FAIL(ddl_service_.create_database(arg.if_not_exist_,
2968
                                             copied_db_schema, &arg.ddl_stmt_str_))) {
2969
      LOG_WARN("create_database failed", "if_not_exist", arg.if_not_exist_,
2970
               K(copied_db_schema), "ddl_stmt_str", arg.ddl_stmt_str_, K(ret));
2971
    } else {
2972
      db_id = copied_db_schema.get_database_id();
2973
    }
2974
  }
2975
  return ret;
2976
}
2977

2978
int ObRootService::alter_database(const ObAlterDatabaseArg &arg)
2979
{
2980
  int ret = OB_SUCCESS;
2981
  if (!inited_) {
2982
    ret = OB_NOT_INIT;
2983
    LOG_WARN("not init", K(ret));
2984
  } else if (!arg.is_valid()) {
2985
    ret = OB_INVALID_ARGUMENT;
2986
    LOG_WARN("invalid arg", K(arg), K(ret));
2987
  } else if (common::STANDBY_CLUSTER == ObClusterInfoGetter::get_cluster_role_v2()) {
2988
    const int64_t tenant_id = arg.database_schema_.get_tenant_id();
2989
    ObSchemaGetterGuard schema_guard;
2990
    uint64_t database_id = OB_INVALID_ID;
2991
    if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(
2992
            tenant_id, schema_guard))) {
2993
      LOG_WARN("get_schema_guard with version in inner table failed", K(ret), K(tenant_id));
2994
    } else if (OB_FAIL(schema_guard.get_database_id(tenant_id,
2995
            arg.database_schema_.get_database_name_str(), database_id))) {
2996
      LOG_WARN("failed to get database id", K(ret), K(tenant_id), K(arg));
2997
    }
2998
  }
2999
  if (OB_FAIL(ret)) {
3000
  } else if (OB_FAIL(ddl_service_.alter_database(arg))) {
3001
    LOG_WARN("alter database failed", K(arg), K(ret));
3002
  }
3003
  return ret;
3004
}
3005

3006
int ObRootService::create_tablegroup(const ObCreateTablegroupArg &arg, UInt64 &tg_id)
3007
{
3008
  LOG_INFO("receive create tablegroup arg", K(arg));
3009
  int ret = OB_SUCCESS;
3010
  if (!inited_) {
3011
    ret = OB_NOT_INIT;
3012
    LOG_WARN("not init", K(ret));
3013
  } else if (!arg.is_valid()) {
3014
    ret = OB_INVALID_ARGUMENT;
3015
    LOG_WARN("invalid arg", K(arg), K(ret));
3016
  } else {
3017
    ObTablegroupSchema copied_tg_schema;
3018
    if (OB_FAIL(copied_tg_schema.assign(arg.tablegroup_schema_))) {
3019
      LOG_WARN("failed to assign tablegroup schema", K(ret), K(arg));
3020
    } else if (OB_FAIL(ddl_service_.create_tablegroup(
3021
            arg.if_not_exist_, copied_tg_schema, &arg.ddl_stmt_str_))) {
3022
      LOG_WARN("create_tablegroup failed", "if_not_exist", arg.if_not_exist_,
3023
               K(copied_tg_schema), "ddl_stmt_str", arg.ddl_stmt_str_, K(ret));
3024
    } else {
3025
      tg_id = copied_tg_schema.get_tablegroup_id();
3026
    }
3027
  }
3028
  return ret;
3029
}
3030

3031
int ObRootService::handle_security_audit(const ObSecurityAuditArg &arg)
3032
{
3033
  int ret = OB_SUCCESS;
3034
  if (OB_UNLIKELY(!inited_)) {
3035
    ret = OB_NOT_INIT;
3036
    LOG_WARN("not init", K(ret));
3037
  } else if (OB_FAIL(ddl_service_.handle_security_audit(arg))) {
3038
    LOG_WARN("handle audit request failed", K(ret), K(arg));
3039
  }
3040
  return ret;
3041
}
3042

3043

3044
int ObRootService::parallel_ddl_pre_check_(const uint64_t tenant_id)
3045
{
3046
  int ret = OB_SUCCESS;
3047
  bool is_dropped = false;
3048
  if (OB_UNLIKELY(!inited_)) {
3049
    ret = OB_NOT_INIT;
3050
    LOG_WARN("not init", KR(ret));
3051
  } else if (OB_UNLIKELY(OB_INVALID_TENANT_ID == tenant_id)) {
3052
    ret = OB_INVALID_ARGUMENT;
3053
    LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id));
3054
  } else if (OB_FAIL(schema_service_->check_if_tenant_has_been_dropped(tenant_id, is_dropped))) {
3055
    LOG_WARN("fail to check if tenant has been dropped", KR(ret), K(tenant_id));
3056
  } else if (is_dropped) {
3057
    ret = OB_TENANT_HAS_BEEN_DROPPED;
3058
    LOG_WARN("tenant has been dropped", KR(ret), K(tenant_id));
3059
  } else if (!schema_service_->is_tenant_refreshed(tenant_id)) {
3060
    // use this err to trigger DDL retry and release current thread.
3061
    ret = OB_ERR_PARALLEL_DDL_CONFLICT;
3062
    LOG_WARN("tenant' schema not refreshed yet, need retry", KR(ret), K(tenant_id));
3063
  }
3064
  return ret;
3065
}
3066

3067
int ObRootService::parallel_create_table(const ObCreateTableArg &arg, ObCreateTableRes &res)
3068
{
3069
  LOG_TRACE("receive create table arg", K(arg));
3070
  int64_t begin_time = ObTimeUtility::current_time();
3071
  const uint64_t tenant_id = arg.exec_tenant_id_;
3072
  int ret = OB_SUCCESS;
3073
  if (OB_UNLIKELY(!inited_)) {
3074
    ret = OB_NOT_INIT;
3075
    LOG_WARN("not init", KR(ret));
3076
  } else if (OB_UNLIKELY(!arg.is_valid())) {
3077
    ret = OB_INVALID_ARGUMENT;
3078
    LOG_WARN("invalid arg", KR(ret), K(arg));
3079
  } else if (OB_FAIL(parallel_ddl_pre_check_(tenant_id))) {
3080
    LOG_WARN("pre check failed before parallel ddl execute", KR(ret), K(tenant_id));
3081
  } else if (arg.schema_.is_view_table()) {
3082
    ObCreateViewHelper create_view_helper(schema_service_, tenant_id, arg, res);
3083
    if (OB_FAIL(create_view_helper.init(ddl_service_))) {
3084
      LOG_WARN("fail to init create view helper", KR(ret), K(tenant_id));
3085
    } else if (OB_FAIL(create_view_helper.execute())) {
3086
      LOG_WARN("fail to execute create view", KR(ret), K(tenant_id));
3087
    }
3088
  } else {
3089
    ObCreateTableHelper create_table_helper(schema_service_, tenant_id, arg, res);
3090
    if (OB_FAIL(create_table_helper.init(ddl_service_))) {
3091
      LOG_WARN("fail to init create table helper", KR(ret), K(tenant_id));
3092
    } else if (OB_FAIL(create_table_helper.execute())) {
3093
      LOG_WARN("fail to execute create table", KR(ret), K(tenant_id));
3094
    }
3095
  }
3096
  int64_t cost = ObTimeUtility::current_time() - begin_time;
3097
  LOG_TRACE("finish create table", KR(ret), K(arg), K(cost));
3098
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "parallel create table",
3099
                        K(tenant_id),
3100
                        "ret", ret,
3101
                        "trace_id", *ObCurTraceId::get_trace_id(),
3102
                        "table_id", res.table_id_,
3103
                        "schema_version", res.schema_version_,
3104
                        K(cost));
3105
  return ret;
3106
}
3107

3108
int ObRootService::gen_container_table_schema_(const ObCreateTableArg &arg,
3109
                                               ObSchemaGetterGuard &schema_guard,
3110
                                               ObTableSchema &mv_table_schema,
3111
                                               ObArray<ObTableSchema> &table_schemas)
3112
{
3113
  int ret = OB_SUCCESS;
3114
  SMART_VAR(ObTableSchema, container_table_schema) {
3115
    if (arg.mv_ainfo_.count() >= 2) {
3116
      ret = OB_ERR_UNEXPECTED;
3117
      LOG_WARN("container table should be less than two", KR(ret), K(arg.mv_ainfo_.count()));
3118
    }
3119

3120
    for (int64_t i = 0; OB_SUCC(ret) && i < arg.mv_ainfo_.count(); i ++) {
3121
      container_table_schema.reset();
3122
      char buf[OB_MAX_TABLE_NAME_LENGTH];
3123
      memset(buf, 0, OB_MAX_TABLE_NAME_LENGTH);
3124

3125
      if (OB_SUCC(ret)) {
3126
        if (OB_FAIL(container_table_schema.assign(arg.mv_ainfo_.at(i).container_table_schema_))) {
3127
          LOG_WARN("fail to assign index schema", KR(ret));
3128
        } else if (OB_FAIL(databuff_printf(buf, OB_MAX_TABLE_NAME_LENGTH, "__mv_container_%ld", mv_table_schema.get_table_id()))) {
3129
          LOG_WARN("fail to print table name", KR(ret));
3130
        } else if (OB_FAIL(container_table_schema.set_table_name(buf))) {
3131
          LOG_WARN("fail to set table_name", KR(ret));
3132
        } else {
3133
          container_table_schema.set_database_id(mv_table_schema.get_database_id());
3134
        }
3135
      }
3136

3137
      if (OB_SUCC(ret)) {
3138
        ObArray<ObSchemaType> conflict_schema_types;
3139
        if (!arg.is_alter_view_
3140
            && OB_FAIL(schema_guard.check_oracle_object_exist(container_table_schema.get_tenant_id(),
3141
                   container_table_schema.get_database_id(), container_table_schema.get_table_name_str(),
3142
                   TABLE_SCHEMA, INVALID_ROUTINE_TYPE, arg.if_not_exist_, conflict_schema_types))) {
3143
          LOG_WARN("fail to check oracle_object exist", K(ret), K(container_table_schema));
3144
        } else if (conflict_schema_types.count() > 0) {
3145
          ret = OB_ERR_EXIST_OBJECT;
3146
          LOG_WARN("Name is already used by an existing object",
3147
                   K(ret), K(container_table_schema), K(conflict_schema_types));
3148
        }
3149
      }
3150
      if (OB_SUCC(ret)) { // check same table_name
3151
        bool table_exist = false;
3152
        bool object_exist = false;
3153
        uint64_t synonym_id = OB_INVALID_ID;
3154
        ObSchemaGetterGuard::CheckTableType check_type = ObSchemaGetterGuard::ALL_NON_HIDDEN_TYPES;
3155

3156
        if (FAILEDx(schema_guard.check_synonym_exist_with_name(container_table_schema.get_tenant_id(),
3157
                container_table_schema.get_database_id(),
3158
                container_table_schema.get_table_name_str(),
3159
                object_exist,
3160
                synonym_id))) {
3161
          LOG_WARN("fail to check synonym exist", K(container_table_schema), KR(ret));
3162
        } else if (object_exist) {
3163
          ret = OB_ERR_EXIST_OBJECT;
3164
          LOG_WARN("Name is already used by an existing object", K(container_table_schema), KR(ret));
3165
        } else if (OB_FAIL(schema_guard.check_table_exist(container_table_schema.get_tenant_id(),
3166
                container_table_schema.get_database_id(),
3167
                container_table_schema.get_table_name_str(),
3168
                false, /*is index*/
3169
                check_type,
3170
                table_exist))) {
3171
          LOG_WARN("check table exist failed", KR(ret), K(container_table_schema));
3172
        } else if (table_exist) {
3173
          ret = OB_ERR_TABLE_EXIST;
3174
          LOG_WARN("table exist", KR(ret), K(container_table_schema), K(arg.if_not_exist_));
3175
        }
3176
      }
3177

3178
      if (OB_SUCC(ret)) {
3179
        if (OB_FAIL(ddl_service_.generate_schema(arg, container_table_schema))) {
3180
          LOG_WARN("fail to generate container table schema", KR(ret));
3181
        } else {
3182
          //table_schema.get_view_schema().set_container_table_id(container_table_schema.get_table_id());
3183
          mv_table_schema.set_data_table_id(container_table_schema.get_table_id());
3184
        }
3185
      }
3186
      if (OB_SUCC(ret)) {
3187
        if (OB_FAIL(table_schemas.push_back(container_table_schema))) {
3188
          LOG_WARN("push_back failed", KR(ret));
3189
        }
3190
      }
3191
    }
3192
  }
3193
  return ret;
3194
}
3195

3196
int ObRootService::create_table(const ObCreateTableArg &arg, ObCreateTableRes &res)
3197
{
3198
  LOG_DEBUG("receive create table arg", K(arg));
3199
  int ret = OB_SUCCESS;
3200
  int64_t begin_time = ObTimeUtility::current_time();
3201
  LOG_INFO("receive create table ddl", K(begin_time));
3202
  RS_TRACE(create_table_begin);
3203
  if (!inited_) {
3204
    ret = OB_NOT_INIT;
3205
    LOG_WARN("not init", K(ret));
3206
  } else if (!arg.is_valid()) {
3207
    ret = OB_INVALID_ARGUMENT;
3208
    LOG_WARN("invalid arg", K(arg), K(ret));
3209
  } else {
3210
    ObArray<ObTableSchema> table_schemas;
3211
    ObSchemaGetterGuard schema_guard;
3212
    const ObDatabaseSchema *db_schema = NULL;
3213
    schema_guard.set_session_id(arg.schema_.get_session_id());
3214
    ObSchemaService *schema_service = schema_service_->get_schema_service();
3215
    ObTableSchema table_schema;
3216
    bool is_oracle_mode = false;
3217
    int64_t ddl_task_id = 0;
3218
    // generate base table schema
3219
    if (OB_FAIL(table_schema.assign(arg.schema_))) {
3220
      LOG_WARN("fail to assign schema", K(ret));
3221
    } else if (OB_ISNULL(schema_service)) {
3222
      ret = OB_ERR_UNEXPECTED;
3223
      LOG_WARN("schema_service is null", KP(schema_service), K(ret));
3224
    } else if (OB_FAIL(generate_table_schema_in_tenant_space(arg, table_schema))) {
3225
      LOG_WARN("fail to generate table schema in tenant space", K(ret), K(arg));
3226
    } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(
3227
               table_schema.get_tenant_id(), schema_guard))) {
3228
      LOG_WARN("get_schema_guard with version in inner table failed", K(ret));
3229
    } else if (OB_FAIL(check_parallel_ddl_conflict(schema_guard, arg))) {
3230
      LOG_WARN("check parallel ddl conflict failed", K(ret));
3231
    } else if (OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(
3232
                table_schema.get_tenant_id(), is_oracle_mode))) {
3233
      LOG_WARN("fail to check is oracle mode", K(ret));
3234
    } else if (OB_INVALID_ID == table_schema.get_database_id()) {
3235
      ObString database_name = arg.db_name_;
3236
      if (OB_FAIL(schema_guard.get_database_schema(table_schema.get_tenant_id(),
3237
                                                   database_name,
3238
                                                   db_schema))) {
3239
        LOG_WARN("get databas schema failed", K(arg));
3240
      } else if (OB_ISNULL(db_schema)) {
3241
        ret = OB_ERR_BAD_DATABASE;
3242
        LOG_USER_ERROR(OB_ERR_BAD_DATABASE, database_name.length(), database_name.ptr());
3243
      } else if (!arg.is_inner_ && db_schema->is_in_recyclebin()) {
3244
        ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
3245
        LOG_WARN("Can't not create table of db in recyclebin", K(ret), K(arg), K(*db_schema));
3246
      } else if (OB_INVALID_ID == db_schema->get_database_id()) {
3247
        ret = OB_ERR_BAD_DATABASE;
3248
        LOG_WARN("database id is invalid", "tenant_id",
3249
                 table_schema.get_tenant_id(), K(database_name), K(*db_schema), K(ret));
3250
      } else {
3251
        table_schema.set_database_id(db_schema->get_database_id());
3252
      }
3253
    } else {
3254
      // for view, database_id must be filled
3255
    }
3256
    if (OB_SUCC(ret)) {
3257
      bool table_exist = false;
3258
      bool object_exist = false;
3259
      uint64_t synonym_id = OB_INVALID_ID;
3260
      ObSchemaGetterGuard::CheckTableType check_type = ObSchemaGetterGuard::ALL_NON_HIDDEN_TYPES;
3261
      if (table_schema.is_mysql_tmp_table()) {
3262
        check_type = ObSchemaGetterGuard::TEMP_TABLE_TYPE;
3263
      } else if (0 == table_schema.get_session_id()) {
3264
        //if session_id <> 0 during create table, need to exclude the existence of temporary table with the same table_name,
3265
        //if there is, need to throw error.
3266
        check_type = ObSchemaGetterGuard::NON_TEMP_WITH_NON_HIDDEN_TABLE_TYPE;
3267
      }
3268
      if (OB_SUCC(ret)) {
3269
        ObArray<ObSchemaType> conflict_schema_types;
3270
        if (!arg.is_alter_view_
3271
            && OB_FAIL(schema_guard.check_oracle_object_exist(table_schema.get_tenant_id(),
3272
                   table_schema.get_database_id(), table_schema.get_table_name_str(),
3273
                   TABLE_SCHEMA, INVALID_ROUTINE_TYPE, arg.if_not_exist_, conflict_schema_types))) {
3274
          LOG_WARN("fail to check oracle_object exist", K(ret), K(table_schema));
3275
        } else if (conflict_schema_types.count() > 0) {
3276
          ret = OB_ERR_EXIST_OBJECT;
3277
          LOG_WARN("Name is already used by an existing object",
3278
                   K(ret), K(table_schema), K(conflict_schema_types));
3279
        }
3280
      }
3281
      if (FAILEDx(schema_guard.check_synonym_exist_with_name(table_schema.get_tenant_id(),
3282
                                                             table_schema.get_database_id(),
3283
                                                             table_schema.get_table_name_str(),
3284
                                                             object_exist,
3285
                                                             synonym_id))) {
3286
        LOG_WARN("fail to check synonym exist", K(table_schema), K(ret));
3287
      } else if (object_exist) {
3288
        ret = OB_ERR_EXIST_OBJECT;
3289
        LOG_WARN("Name is already used by an existing object", K(table_schema), K(ret));
3290
      } else if (OB_FAIL(schema_guard.check_table_exist(table_schema.get_tenant_id(),
3291
                                                        table_schema.get_database_id(),
3292
                                                        table_schema.get_table_name_str(),
3293
                                                        false, /*is index*/
3294
                                                        check_type,
3295
                                                        table_exist))) {
3296
        LOG_WARN("check table exist failed", K(ret), K(table_schema));
3297
      } else if (table_exist) {
3298
        if (table_schema.is_view_table() && arg.if_not_exist_) {
3299
          //create or replace view ...
3300
          //create user table will drop the old view and recreate it in trans
3301
          const ObSimpleTableSchemaV2 *simple_table_schema = nullptr;
3302
          if (OB_FAIL(schema_guard.get_simple_table_schema(
3303
                      table_schema.get_tenant_id(),
3304
                      table_schema.get_database_id(),
3305
                      table_schema.get_table_name_str(),
3306
                      false, /*is index*/
3307
                      simple_table_schema))) {
3308
            LOG_WARN("failed to get table schema", K(ret));
3309
          } else if (OB_ISNULL(simple_table_schema)) {
3310
            ret = OB_ERR_UNEXPECTED;
3311
            LOG_WARN("simple_table_schema is null", K(ret));
3312
          } else if (simple_table_schema->get_table_type() == SYSTEM_VIEW
3313
                     || simple_table_schema->get_table_type() == USER_VIEW
3314
                     || simple_table_schema->get_table_type() == MATERIALIZED_VIEW) {
3315
            ret = OB_SUCCESS;
3316
          } else {
3317
            if (is_oracle_mode) {
3318
              ret = OB_ERR_EXIST_OBJECT;
3319
              LOG_WARN("name is already used by an existing object",
3320
                       K(ret), K(table_schema.get_table_name_str()));
3321
            } else { // mysql mode
3322
              const ObDatabaseSchema *db_schema = nullptr;
3323
              if (OB_FAIL(schema_guard.get_database_schema(
3324
                          table_schema.get_tenant_id(),
3325
                          table_schema.get_database_id(),
3326
                          db_schema))) {
3327
                LOG_WARN("get db schema failed", K(ret), K(table_schema.get_database_id()));
3328
              } else if (OB_ISNULL(db_schema)) {
3329
                ret = OB_ERR_UNEXPECTED;
3330
                LOG_WARN("db schema is null", K(ret));
3331
              } else {
3332
                ret = OB_ERR_WRONG_OBJECT;
3333
                LOG_USER_ERROR(OB_ERR_WRONG_OBJECT,
3334
                    to_cstring(db_schema->get_database_name_str()),
3335
                    to_cstring(table_schema.get_table_name_str()), "VIEW");
3336
                LOG_WARN("table exist", K(ret), K(table_schema));
3337
              }
3338
            }
3339
          }
3340
        } else {
3341
          ret = OB_ERR_TABLE_EXIST;
3342
          LOG_WARN("table exist", K(ret), K(table_schema), K(arg.if_not_exist_));
3343
        }
3344
      } else if (!table_exist && table_schema.is_view_table() && arg.is_alter_view_) {
3345
        // the origin view must exist while alter view
3346
        const ObSimpleDatabaseSchema *simple_db_schema = nullptr;
3347
        if (OB_FAIL(schema_guard.get_database_schema(
3348
                    table_schema.get_tenant_id(),
3349
                    table_schema.get_database_id(),
3350
                    simple_db_schema))) {
3351
          LOG_WARN("get db schema failed", K(ret), K(table_schema.get_database_id()));
3352
        } else if (OB_ISNULL(simple_db_schema)) {
3353
          ret = OB_ERR_UNEXPECTED;
3354
          LOG_WARN("db schema is null", K(ret));
3355
        } else {
3356
          ret = OB_TABLE_NOT_EXIST;
3357
          LOG_USER_ERROR(OB_TABLE_NOT_EXIST,
3358
                         to_cstring(simple_db_schema->get_database_name_str()),
3359
                         to_cstring(table_schema.get_table_name_str()));
3360
          LOG_WARN("table not exist", K(ret), K(table_schema));
3361
        }
3362
      }
3363
    }
3364
    RS_TRACE(generate_schema_start);
3365
    //bool can_hold_new_table = false;
3366
    common::hash::ObHashMap<ObString, uint64_t> mock_fk_parent_table_map; // name, count
3367
    ObArray<ObMockFKParentTableSchema> tmp_mock_fk_parent_table_schema_array;
3368
    ObArray<ObMockFKParentTableSchema> mock_fk_parent_table_schema_array;
3369
    if (OB_FAIL(ret)) {
3370
      //do nothing
3371
    } else if (OB_FAIL(mock_fk_parent_table_map.create(16, "MockFKParentTbl"))) {
3372
      LOG_WARN("fail to create mock_fk_parent_table_map", K(ret));
3373
    } else if (OB_FAIL(ddl_service_.generate_schema(arg, table_schema))) {
3374
      LOG_WARN("generate_schema for table failed", K(ret));
3375
      //} else if (OB_FAIL(check_rs_capacity(table_schema, can_hold_new_table))) {
3376
      //  LOG_WARN("fail to check rs capacity", K(ret), K(table_schema));
3377
      //} else if (!can_hold_new_table) {
3378
      //  ret = OB_PARTITION_CNT_REACH_ROOTSERVER_LIMIT;
3379
      //  LOG_WARN("reach rs's limits, rootserver can only hold limited replicas");
3380
    } else if (OB_FAIL(table_schemas.push_back(table_schema))) {
3381
      LOG_WARN("push_back failed", K(ret));
3382
    } else if (OB_FAIL(gen_container_table_schema_(arg, schema_guard, table_schema, table_schemas))) {
3383
      LOG_WARN("fail to gen container table schema", KR(ret));
3384
    }
3385

3386
    if (OB_SUCC(ret)) {
3387
      RS_TRACE(generate_schema_index);
3388
      res.table_id_ = table_schema.get_table_id();
3389
      // generate index schemas
3390
      ObIndexBuilder index_builder(ddl_service_);
3391
      ObTableSchema index_schema;
3392
      for (int64_t i = 0; OB_SUCC(ret) && i < arg.index_arg_list_.size(); ++i) {
3393
        index_schema.reset();
3394
        ObCreateIndexArg &index_arg = const_cast<ObCreateIndexArg&>(arg.index_arg_list_.at(i));
3395
        //if we pass the table_schema argument, the create_index_arg can not set database_name
3396
        //and table_name, which will used from get data table schema in generate_schema
3397
        if (!index_arg.index_schema_.is_partitioned_table()
3398
            && !table_schema.is_partitioned_table()) {
3399
          if (INDEX_TYPE_NORMAL_GLOBAL == index_arg.index_type_) {
3400
            index_arg.index_type_ = INDEX_TYPE_NORMAL_GLOBAL_LOCAL_STORAGE;
3401
          } else if (INDEX_TYPE_UNIQUE_GLOBAL == index_arg.index_type_) {
3402
            index_arg.index_type_ = INDEX_TYPE_UNIQUE_GLOBAL_LOCAL_STORAGE;
3403
          } else if (INDEX_TYPE_SPATIAL_GLOBAL == index_arg.index_type_) {
3404
            index_arg.index_type_ = INDEX_TYPE_SPATIAL_GLOBAL_LOCAL_STORAGE;
3405
          }
3406
        }
3407
        // the global index has generated column schema during resolve, RS no need to generate index schema,
3408
        // just assign column schema
3409
        if (INDEX_TYPE_NORMAL_GLOBAL == index_arg.index_type_
3410
            || INDEX_TYPE_UNIQUE_GLOBAL == index_arg.index_type_
3411
            || INDEX_TYPE_SPATIAL_GLOBAL == index_arg.index_type_) {
3412
          if (OB_FAIL(index_schema.assign(index_arg.index_schema_))) {
3413
            LOG_WARN("fail to assign schema", K(ret));
3414
          }
3415
        }
3416
        const bool global_index_without_column_info = false;
3417
        ObSEArray<ObColumnSchemaV2 *, 1> gen_columns;
3418
        ObIAllocator *allocator = index_arg.index_schema_.get_allocator();
3419
        if (OB_FAIL(ret)) {
3420
        } else if (OB_ISNULL(allocator)) {
3421
          ret = OB_ERR_UNEXPECTED;
3422
          LOG_WARN("invalid allocator", K(ret));
3423
        } else if (OB_FAIL(ObIndexBuilderUtil::adjust_expr_index_args(index_arg, table_schema, *allocator, gen_columns))) {
3424
            LOG_WARN("fail to adjust expr index args", K(ret));
3425
        } else if (OB_FAIL(index_builder.generate_schema(index_arg,
3426
                                                         table_schema,
3427
                                                         global_index_without_column_info,
3428
                                                         true, /*generate_id*/
3429
                                                         index_schema))) {
3430
          LOG_WARN("generate_schema for index failed", K(index_arg), K(table_schema), K(ret));
3431
        }
3432
        if (OB_SUCC(ret)) {
3433
          uint64_t new_table_id = OB_INVALID_ID;
3434
          if (OB_FAIL(schema_service->fetch_new_table_id(table_schema.get_tenant_id(), new_table_id))) {
3435
            LOG_WARN("failed to fetch_new_table_id", "tenant_id", table_schema.get_tenant_id(), K(ret));
3436
          } else {
3437
            index_schema.set_table_id(new_table_id);
3438
            //index_schema.set_data_table_id(table_id);
3439
            if (OB_FAIL(table_schemas.push_back(index_schema))) {
3440
              LOG_WARN("push_back failed", K(ret));
3441
            }
3442
          }
3443
        }
3444
      }
3445
      RS_TRACE(generate_schema_lob);
3446
      if (OB_FAIL(ret) || table_schema.is_view_table() || table_schema.is_external_table()) {
3447
        // do nothing
3448
      } else if (OB_FAIL(ddl_service_.build_aux_lob_table_schema_if_need(table_schema, table_schemas))) {
3449
        LOG_WARN("fail to build_aux_lob_table_schema_if_need", K(ret), K(table_schema));
3450
      }
3451
      if (OB_SUCC(ret)) {
3452
        for (int64_t i = 0; OB_SUCC(ret) && i < arg.foreign_key_arg_list_.count(); i++) {
3453
          const ObCreateForeignKeyArg &foreign_key_arg = arg.foreign_key_arg_list_.at(i);
3454
          ObForeignKeyInfo foreign_key_info;
3455
          // check for duplicate constraint names of foregin key
3456
          if (foreign_key_arg.foreign_key_name_.empty()) {
3457
            ret = OB_ERR_UNEXPECTED;
3458
            LOG_WARN("fk name is empty", K(ret));
3459
          } else {
3460
            bool is_foreign_key_name_exist = true;
3461
            if (OB_FAIL(ddl_service_.check_constraint_name_is_exist(
3462
                        schema_guard, table_schema, foreign_key_arg.foreign_key_name_, true, is_foreign_key_name_exist))) {
3463
              LOG_WARN("fail to check foreign key name is exist or not", K(ret), K(foreign_key_arg.foreign_key_name_));
3464
            } else if(is_foreign_key_name_exist) {
3465
              if (is_oracle_mode) {
3466
                ret = OB_ERR_CONSTRAINT_NAME_DUPLICATE;
3467
                LOG_WARN("fk name is duplicate", K(ret), K(foreign_key_arg.foreign_key_name_));
3468
              } else { // mysql mode
3469
                ret = OB_ERR_DUP_KEY;
3470
                LOG_USER_ERROR(OB_ERR_DUP_KEY,
3471
                    table_schema.get_table_name_str().length(),
3472
                    table_schema.get_table_name_str().ptr());
3473
              }
3474
            }
3475
          }
3476
          // end of check for duplicate constraint names of foregin key
3477
          const ObTableSchema *parent_schema = NULL;
3478
          if (OB_SUCC(ret)) {
3479
            // get parent table schema.
3480
            // TODO: is it necessory to determine whether it is case sensitive by check sys variable
3481
            // check whether it belongs to self reference, if so, the parent schema is child schema.
3482
            if (0 == foreign_key_arg.parent_table_.case_compare(table_schema.get_table_name_str())
3483
                  && 0 == foreign_key_arg.parent_database_.case_compare(arg.db_name_)) {
3484
              parent_schema = &table_schema;
3485
              if (CONSTRAINT_TYPE_PRIMARY_KEY == foreign_key_arg.ref_cst_type_) {
3486
                if (is_oracle_mode) {
3487
                  for (ObTableSchema::const_constraint_iterator iter = parent_schema->constraint_begin(); iter != parent_schema->constraint_end(); ++iter) {
3488
                    if (CONSTRAINT_TYPE_PRIMARY_KEY == (*iter)->get_constraint_type()) {
3489
                      foreign_key_info.ref_cst_type_ = CONSTRAINT_TYPE_PRIMARY_KEY;
3490
                      foreign_key_info.ref_cst_id_ = (*iter)->get_constraint_id();
3491
                      break;
3492
                    }
3493
                  }
3494
                } else {
3495
                  foreign_key_info.ref_cst_type_ = CONSTRAINT_TYPE_PRIMARY_KEY;
3496
                  foreign_key_info.ref_cst_id_ = common::OB_INVALID_ID;
3497
                }
3498
              } else if (CONSTRAINT_TYPE_UNIQUE_KEY == foreign_key_arg.ref_cst_type_) {
3499
                if (OB_FAIL(ddl_service_.get_uk_cst_id_for_self_ref(table_schemas, foreign_key_arg, foreign_key_info))) {
3500
                  LOG_WARN("failed to get uk cst id for self ref", K(ret), K(foreign_key_arg));
3501
                }
3502
              } else {
3503
                ret = OB_ERR_UNEXPECTED;
3504
                LOG_WARN("invalid foreign key ref cst type", K(ret), K(foreign_key_arg));
3505
              }
3506
            } else if (OB_FAIL(schema_guard.get_table_schema(table_schema.get_tenant_id(),
3507
                                                             foreign_key_arg.parent_database_,
3508
                                                             foreign_key_arg.parent_table_,
3509
                                                             false, parent_schema))) {
3510
              LOG_WARN("failed to get parent table schema", K(ret), K(foreign_key_arg));
3511
            } else {
3512
              foreign_key_info.ref_cst_type_ = foreign_key_arg.ref_cst_type_;
3513
              foreign_key_info.ref_cst_id_ = foreign_key_arg.ref_cst_id_;
3514
            }
3515
          }
3516
          const ObMockFKParentTableSchema *tmp_mock_fk_parent_table_ptr = NULL;
3517
          ObMockFKParentTableSchema mock_fk_parent_table_schema;
3518
          if (OB_SUCC(ret)) {
3519
            if (foreign_key_arg.is_parent_table_mock_) {
3520
              uint64_t dup_name_mock_fk_parent_table_count = 0;
3521
              if (NULL != parent_schema) {
3522
                ret = OB_ERR_PARALLEL_DDL_CONFLICT;
3523
                LOG_WARN("the mock parent table is conflict with the real parent table, need retry",
3524
                    K(ret), K(foreign_key_arg), K(parent_schema->get_table_id()));
3525
              } else if (OB_FAIL(mock_fk_parent_table_map.get_refactored(foreign_key_arg.parent_table_, dup_name_mock_fk_parent_table_count))) {
3526
                if (OB_HASH_NOT_EXIST == ret) {
3527
                  ret = OB_SUCCESS;
3528
                  if (OB_FAIL(mock_fk_parent_table_map.set_refactored(foreign_key_arg.parent_table_, ++dup_name_mock_fk_parent_table_count))) {
3529
                    LOG_WARN("failed to insert into mock_fk_parent_table_map", K(ret), K(foreign_key_arg), K(dup_name_mock_fk_parent_table_count));
3530
                  }
3531
                } else {
3532
                  LOG_WARN("get_refactored from mock_fk_parent_table_map failed", K(ret), K(foreign_key_arg));
3533
                }
3534
              } else {
3535
                //already had dup name mock_fk_parent_table in tmp_mock_fk_parent_table_schema_array
3536
                int64_t count = 0;
3537
                for (int64_t i = 0; i < tmp_mock_fk_parent_table_schema_array.count(); ++i) {
3538
                  if (0 == tmp_mock_fk_parent_table_schema_array.at(i).get_mock_fk_parent_table_name().case_compare(foreign_key_arg.parent_table_)) {
3539
                    if (++count == dup_name_mock_fk_parent_table_count) {
3540
                      tmp_mock_fk_parent_table_ptr = &tmp_mock_fk_parent_table_schema_array.at(i);
3541
                      break;
3542
                    }
3543
                  }
3544
                }
3545
                if (OB_ISNULL(tmp_mock_fk_parent_table_ptr)) {
3546
                  ret = OB_ERR_UNEXPECTED;
3547
                  LOG_WARN("tmp_mock_fk_parent_table_ptr is null", K(ret), K(foreign_key_arg), K(tmp_mock_fk_parent_table_schema_array));
3548
                } else if (OB_FAIL(mock_fk_parent_table_map.set_refactored(foreign_key_arg.parent_table_, ++dup_name_mock_fk_parent_table_count, true/*overwrite*/))) {
3549
                  LOG_WARN("failed to insert into mock_fk_parent_table_map", K(ret), K(foreign_key_arg), K(dup_name_mock_fk_parent_table_count));
3550
                }
3551
              }
3552
              if (OB_FAIL(ret)) {
3553
              } else if (OB_FAIL(ddl_service_.gen_mock_fk_parent_table_for_create_fk(
3554
                         schema_guard, table_schema.get_tenant_id(), foreign_key_arg, tmp_mock_fk_parent_table_ptr, foreign_key_info, mock_fk_parent_table_schema))) {
3555
                LOG_WARN("failed to generate_mock_fk_parent_table_schema", K(ret), K(table_schema.get_tenant_id()), K(foreign_key_arg));
3556
              }
3557
            } else if (OB_ISNULL(parent_schema)) {
3558
              ret = OB_TABLE_NOT_EXIST;
3559
              LOG_WARN("parent table is not exist", K(ret), K(foreign_key_arg));
3560
            } else if (false == parent_schema->is_tmp_table()
3561
                           && 0 != parent_schema->get_session_id()
3562
                           && OB_INVALID_ID != schema_guard.get_session_id()) {
3563
              ret = OB_TABLE_NOT_EXIST;
3564
              LOG_USER_ERROR(OB_TABLE_NOT_EXIST, to_cstring(foreign_key_arg.parent_database_), to_cstring(foreign_key_arg.parent_table_));
3565
            } else if (!arg.is_inner_ && parent_schema->is_in_recyclebin()) {
3566
              ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
3567
              LOG_WARN("parent table is in recyclebin", K(ret), K(foreign_key_arg));
3568
            } else if (parent_schema->get_table_id() != table_schema.get_table_id()) {
3569
              // no need to update sync_versin_for_cascade_table while the refrence table is itself
3570
              if (OB_FAIL(table_schema.add_depend_table_id(parent_schema->get_table_id()))) {
3571
                LOG_WARN("failed to add depend table id", K(ret), K(foreign_key_arg));
3572
              }
3573
            }
3574
          }
3575
          // get child column schema.
3576
          if (OB_SUCC(ret)) {
3577
            foreign_key_info.child_table_id_ = res.table_id_;
3578
            foreign_key_info.parent_table_id_ = foreign_key_arg.is_parent_table_mock_ ? mock_fk_parent_table_schema.get_mock_fk_parent_table_id() : parent_schema->get_table_id();
3579
            for (int64_t j = 0; OB_SUCC(ret) && j < foreign_key_arg.child_columns_.count(); j++) {
3580
              const ObString &column_name = foreign_key_arg.child_columns_.at(j);
3581
              const ObColumnSchemaV2 *column_schema = table_schema.get_column_schema(column_name);
3582
              if (OB_ISNULL(column_schema)) {
3583
                ret = OB_ERR_COLUMN_NOT_FOUND;
3584
                LOG_WARN("child column is not exist", K(ret), K(column_name));
3585
              } else if (OB_FAIL(foreign_key_info.child_column_ids_.push_back(column_schema->get_column_id()))) {
3586
                LOG_WARN("failed to push child column id", K(ret), K(column_name));
3587
              }
3588
            }
3589
          }
3590
          // get parent column schema.
3591
          if (OB_SUCC(ret) && !foreign_key_arg.is_parent_table_mock_) {
3592
            for (int64_t j = 0; OB_SUCC(ret) && j < foreign_key_arg.parent_columns_.count(); j++) {
3593
              const ObString &column_name = foreign_key_arg.parent_columns_.at(j);
3594
              const ObColumnSchemaV2 *column_schema = parent_schema->get_column_schema(column_name);
3595
              if (OB_ISNULL(column_schema)) {
3596
                ret = OB_ERR_COLUMN_NOT_FOUND;
3597
                LOG_WARN("parent column is not exist", K(ret), K(column_name));
3598
              } else if (OB_FAIL(foreign_key_info.parent_column_ids_.push_back(column_schema->get_column_id()))) {
3599
                LOG_WARN("failed to push parent column id", K(ret), K(column_name));
3600
              }
3601
            }
3602
          }
3603
          // get reference option and foreign key name.
3604
          if (OB_SUCC(ret)) {
3605
            foreign_key_info.update_action_ = foreign_key_arg.update_action_;
3606
            foreign_key_info.delete_action_ = foreign_key_arg.delete_action_;
3607
            foreign_key_info.foreign_key_name_ = foreign_key_arg.foreign_key_name_;
3608
            foreign_key_info.enable_flag_ = foreign_key_arg.enable_flag_;
3609
            foreign_key_info.validate_flag_ = foreign_key_arg.validate_flag_;
3610
            foreign_key_info.rely_flag_ = foreign_key_arg.rely_flag_;
3611
            foreign_key_info.is_parent_table_mock_ = foreign_key_arg.is_parent_table_mock_;
3612
            foreign_key_info.name_generated_type_ = foreign_key_arg.name_generated_type_;
3613
          }
3614
          // add foreign key info.
3615
          if (OB_SUCC(ret)) {
3616
            if (OB_FAIL(schema_service->fetch_new_constraint_id(table_schema.get_tenant_id(),
3617
                                                                foreign_key_info.foreign_key_id_))) {
3618
              LOG_WARN("failed to fetch new foreign key id", K(ret), K(foreign_key_arg));
3619
            } else if (OB_FAIL(table_schema.add_foreign_key_info(foreign_key_info))) {
3620
              LOG_WARN("failed to push foreign key info", K(ret), K(foreign_key_info));
3621
            } else if (foreign_key_info.is_parent_table_mock_
3622
                       && MOCK_FK_PARENT_TABLE_OP_INVALID != mock_fk_parent_table_schema.get_operation_type()) {
3623
              if (OB_FAIL(mock_fk_parent_table_schema.add_foreign_key_info(foreign_key_info))) {
3624
                LOG_WARN("failed to push foreign key info", K(ret), K(foreign_key_info));
3625
              } else if (ObMockFKParentTableOperationType::MOCK_FK_PARENT_TABLE_OP_CREATE_TABLE_BY_ADD_FK_IN_CHILD_TBALE == mock_fk_parent_table_schema.get_operation_type()) {
3626
                if (OB_FAIL(tmp_mock_fk_parent_table_schema_array.push_back(mock_fk_parent_table_schema))) {
3627
                  LOG_WARN("failed to push mock_fk_parent_table_schema to tmp_mock_fk_parent_table_schema_array", K(ret), K(mock_fk_parent_table_schema));
3628
                }
3629
              } else { // ObMockFKParentTableOperationType::MOCK_FK_PARENT_TABLE_OP_CREATE_TABLE_BY_ADD_FK_IN_CHILD_TBALE != mock_fk_parent_table_schema.get_operation_type()
3630
                if (OB_FAIL(mock_fk_parent_table_schema_array.push_back(mock_fk_parent_table_schema))) {
3631
                  LOG_WARN("failed to push mock_fk_parent_table_schema to mock_fk_parent_table_schema_array", K(ret), K(mock_fk_parent_table_schema));
3632
                } else if (OB_FAIL(mock_fk_parent_table_map.erase_refactored(mock_fk_parent_table_schema.get_mock_fk_parent_table_name()))) {
3633
                  LOG_WARN("failed to delete from mock_fk_parent_table_map", K(ret), K(mock_fk_parent_table_schema.get_mock_fk_parent_table_name()));
3634
                }
3635
              }
3636
            }
3637
          }
3638
        } // for
3639
        if (OB_SUCC(ret)) {
3640
          // push back to mock_fk_parent_table_schema_array with the last one of all dup name mock_fk_parent_table_schema
3641
          if (!tmp_mock_fk_parent_table_schema_array.empty()) {
3642
            for (int64_t i = 0; OB_SUCC(ret) && i < tmp_mock_fk_parent_table_schema_array.count(); ++i) {
3643
              uint64_t dup_name_mock_fk_parent_table_count = 0;
3644
              ObString mock_fk_parent_table_name;
3645
              if (OB_FAIL(mock_fk_parent_table_map.get_refactored(tmp_mock_fk_parent_table_schema_array.at(i).get_mock_fk_parent_table_name(), dup_name_mock_fk_parent_table_count))) {
3646
                if (OB_HASH_NOT_EXIST == ret) {
3647
                  ret = OB_SUCCESS;
3648
                  continue;
3649
                } else {
3650
                  LOG_WARN("get_refactored from mock_fk_parent_table_map failed", K(ret), K(tmp_mock_fk_parent_table_schema_array.at(i)));
3651
                }
3652
              } else {
3653
                mock_fk_parent_table_name = tmp_mock_fk_parent_table_schema_array.at(i).get_mock_fk_parent_table_name();
3654
                int64_t j = i;
3655
                uint64_t count = 0;
3656
                for (; count < dup_name_mock_fk_parent_table_count && j < tmp_mock_fk_parent_table_schema_array.count(); ++j) {
3657
                  if (0 == mock_fk_parent_table_name.case_compare(tmp_mock_fk_parent_table_schema_array.at(j).get_mock_fk_parent_table_name())) {
3658
                    ++count;
3659
                  }
3660
                }
3661
                if (--j >= tmp_mock_fk_parent_table_schema_array.count()) {
3662
                  ret = OB_ERR_UNEXPECTED;
3663
                  LOG_WARN("j >= tmp_mock_fk_parent_table_schema_array.count()", K(ret), K(j), K(tmp_mock_fk_parent_table_schema_array.count()));
3664
                } else {
3665
                  for (int64_t k = 0; OB_SUCC(ret) && k < tmp_mock_fk_parent_table_schema_array.at(j).get_foreign_key_infos().count(); ++k) {
3666
                    tmp_mock_fk_parent_table_schema_array.at(j).get_foreign_key_infos().at(k).parent_table_id_ = tmp_mock_fk_parent_table_schema_array.at(j).get_mock_fk_parent_table_id();
3667
                  }
3668
                }
3669
                if (OB_FAIL(ret)) {
3670
                } else if (OB_FAIL(mock_fk_parent_table_schema_array.push_back(tmp_mock_fk_parent_table_schema_array.at(j)))) {
3671
                  LOG_WARN("fail to push back to mock_fk_parent_table_schema_array", K(ret), K(tmp_mock_fk_parent_table_schema_array.at(j)));
3672
                } else if (OB_FAIL(mock_fk_parent_table_map.erase_refactored(mock_fk_parent_table_name))) {
3673
                  LOG_WARN("failed to delete from mock_fk_parent_table_map", K(mock_fk_parent_table_name), K(ret));
3674
                }
3675
              }
3676
            }
3677
          }
3678
        }
3679
        if (OB_SUCC(ret)) {
3680
          // deal with new table name which is the same to mock_fk_parent_table_name, replace mock_parent_table with this new table
3681
          const ObMockFKParentTableSchema *ori_mock_parent_table_schema_ptr = NULL;
3682
          if (OB_FAIL(schema_guard.get_mock_fk_parent_table_schema_with_name(
3683
              table_schema.get_tenant_id(),
3684
              table_schema.get_database_id(),
3685
              table_schema.get_table_name_str(),
3686
              ori_mock_parent_table_schema_ptr))) {
3687
            LOG_WARN("failed to check_mock_fk_parent_table_exist_with_name");
3688
          } else if (OB_NOT_NULL(ori_mock_parent_table_schema_ptr)) {
3689
            ObMockFKParentTableSchema mock_fk_parent_table_schema;
3690
            ObArray<const share::schema::ObTableSchema*> index_schemas;
3691
            for (int64_t i = 1; OB_SUCC(ret) && i < table_schemas.count(); ++i) {
3692
              if (table_schemas.at(i).is_unique_index()
3693
                  && OB_FAIL(index_schemas.push_back(&table_schemas.at(i)))) {
3694
                LOG_WARN("failed to push back index_schemas", K(ret));
3695
              }
3696
            }
3697
            if (FAILEDx(ddl_service_.gen_mock_fk_parent_table_for_replacing_mock_fk_parent_table(
3698
                schema_guard, ori_mock_parent_table_schema_ptr->get_mock_fk_parent_table_id(), table_schema, index_schemas,
3699
                mock_fk_parent_table_schema))) {
3700
              LOG_WARN("failed to gen_mock_fk_parent_table_for_replacing_mock_fk_parent_table", K(ret));
3701
            } else if (OB_FAIL(mock_fk_parent_table_schema_array.push_back(mock_fk_parent_table_schema))) {
3702
              LOG_WARN("failed to push mock_fk_parent_table_schema", K(ret), K(mock_fk_parent_table_schema));
3703
            }
3704
          }
3705
        }
3706
      } // check foreign key info end.
3707
    }
3708
    RS_TRACE(generate_schema_finish);
3709
    if (OB_SUCC(ret)) {
3710
      //table schema may be updated during analyse index schema, so reset table_schema
3711
      const bool is_standby = PRIMARY_CLUSTER != ObClusterInfoGetter::get_cluster_role_v2();
3712
      if (OB_FAIL(table_schemas.at(0).assign(table_schema))) {
3713
        LOG_WARN("fail to assign schema", K(ret));
3714
      } else if (OB_FAIL(ddl_service_.create_user_tables(
3715
                                      arg.if_not_exist_,
3716
                                      arg.ddl_stmt_str_,
3717
                                      arg.error_info_,
3718
                                      table_schemas,
3719
                                      schema_guard,
3720
                                      arg.sequence_ddl_arg_,
3721
                                      arg.last_replay_log_id_,
3722
                                      &arg.dep_infos_,
3723
                                      mock_fk_parent_table_schema_array,
3724
                                      ddl_task_id))) {
3725
        LOG_WARN("create_user_tables failed", "if_not_exist", arg.if_not_exist_,
3726
                 "ddl_stmt_str", arg.ddl_stmt_str_, K(ret));
3727
      }
3728
    }
3729
    if (OB_ERR_TABLE_EXIST == ret) {
3730
      //create table xx if not exist (...)
3731
      //create or replace view xx as ...
3732
      if (arg.if_not_exist_) {
3733
        ret = OB_SUCCESS;
3734
        LOG_INFO("table is exist, no need to create again, ",
3735
                 "tenant_id", table_schema.get_tenant_id(),
3736
                 "database_id", table_schema.get_database_id(),
3737
                 "table_name", table_schema.get_table_name());
3738
      } else {
3739
        ret = OB_ERR_TABLE_EXIST;
3740
        LOG_USER_ERROR(OB_ERR_TABLE_EXIST, table_schema.get_table_name_str().length(),
3741
            table_schema.get_table_name_str().ptr());
3742
        LOG_WARN("table is exist, cannot create it twice,",
3743
                 "tenant_id", table_schema.get_tenant_id(),
3744
                 "database_id", table_schema.get_database_id(),
3745
                 "table_name", table_schema.get_table_name(), K(ret));
3746
      }
3747
    }
3748
    // check vertical partition
3749
    // is_primary_vp_table()
3750
    // get_aux_vp_tid_array()
3751
    // is_aux_vp_table()
3752
    // get_vp_store_column_ids
3753
    // get_vp_column_ids_without_rowkey
3754
    if (OB_SUCC(ret)) {
3755
      ObSchemaGetterGuard new_schema_guard;
3756
      const ObTableSchema *new_table_schema = NULL;
3757
      const uint64_t arg_vp_cnt = arg.vertical_partition_arg_list_.count();
3758

3759
      if (arg_vp_cnt == 0) {
3760
        LOG_INFO("avg_vp_cnt is 0");
3761
        // do-nothing
3762
      } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(
3763
                         table_schema.get_tenant_id(), new_schema_guard))) {
3764
        LOG_WARN("fail to get schema guard with version in inner table",
3765
                 K(ret), K(table_schema.get_tenant_id()));
3766
      } else if (OB_FAIL(new_schema_guard.get_table_schema(table_schema.get_tenant_id(),
3767
                                                           table_schema.get_table_id(),
3768
                                                           new_table_schema))) {
3769
        LOG_WARN("fail to get table schema", K(ret), K(table_schema));
3770
      } else if (NULL == new_table_schema) {
3771
        ret = OB_ERR_UNEXPECTED;
3772
        LOG_WARN("NULL ptr", K(ret));
3773
      } else if (!new_table_schema->is_primary_vp_table()) {
3774
        ret = OB_ERR_UNEXPECTED;
3775
        LOG_WARN("is_primary_vp_table is invalid", K(ret), K(arg_vp_cnt), K(new_table_schema->is_primary_vp_table()));
3776
      } else {
3777
        ObSEArray<uint64_t, 16> aux_vp_tid_array;
3778
        if (OB_FAIL(new_table_schema->get_aux_vp_tid_array(aux_vp_tid_array))) {
3779
          LOG_WARN("failed to get_aux_vp_tid_array", K(*new_table_schema));
3780
        } else if (!((arg_vp_cnt == (aux_vp_tid_array.count()+ 1)
3781
                      || (arg_vp_cnt == aux_vp_tid_array.count())))) {
3782
          ret = OB_ERR_UNEXPECTED;
3783
          LOG_WARN("arg_vp_cnt is not equal to aux_vp_cnt_ or (aux_vp_cnt_+1)",
3784
                   K(ret), K(arg_vp_cnt), K(aux_vp_tid_array.count()));
3785
        } else {
3786
          // check primary partition table include get_vp_store_column_ids and vertical partition column information
3787
          ObArray<share::schema::ObColDesc> columns;
3788
          const ObColumnSchemaV2 *column_schema = NULL;
3789
          const ObCreateVertialPartitionArg primary_vp_arg = arg.vertical_partition_arg_list_.at(0);
3790
          int64_t arg_pri_vp_col_cnt = primary_vp_arg.vertical_partition_columns_.count();
3791
          if (OB_FAIL(new_table_schema->get_vp_store_column_ids(columns))) {
3792
            LOG_WARN("get_vp_store_column_ids failed", K(ret));
3793
          }
3794
          for (int64_t i = 0; OB_SUCC(ret) && i < columns.count(); ++i) {
3795
            LOG_INFO("column info", K(columns.at(i).col_id_), K(columns.at(i).col_type_));
3796
            if (NULL == (column_schema = new_table_schema->get_column_schema(columns.at(i).col_id_))) {
3797
              ret = OB_ERR_BAD_FIELD_ERROR;
3798
              LOG_WARN("get_column_schema failed", K(columns.at(i)), K(ret));
3799
            } else {
3800
              ObString column_name = column_schema->get_column_name();
3801
              LOG_INFO("column info", K(column_name),
3802
                  K(column_schema->get_column_id()), K(column_schema->get_table_id()));
3803
              if (column_schema->is_primary_vp_column()) {
3804
                for (int64_t j = 0; OB_SUCC(ret) && j < primary_vp_arg.vertical_partition_columns_.count(); ++j) {
3805
                  ObString pri_vp_col = primary_vp_arg.vertical_partition_columns_.at(j);
3806
                  if (0 == column_name.case_compare(pri_vp_col)) {
3807
                    arg_pri_vp_col_cnt--;
3808
                    LOG_INFO("primary vp", K(column_name));
3809
                    break;
3810
                  }
3811
                }
3812
              } else {
3813
                LOG_INFO("non-primary vp", K(column_name));
3814
              }
3815
            }
3816
          }
3817
          if (OB_SUCC(ret) && (0 != arg_pri_vp_col_cnt)) {
3818
            ret = OB_ERR_UNEXPECTED;
3819
            LOG_WARN("mismatch primary vp column", K(ret));
3820
            for (int64_t j = 0; j < arg_pri_vp_col_cnt; ++j) {
3821
              ObString pri_vp_col = primary_vp_arg.vertical_partition_columns_.at(j);
3822
              LOG_INFO("arg primary vp", K(pri_vp_col));
3823
            }
3824
          }
3825

3826
          // verify secondary partition table
3827
          if (OB_SUCC(ret)) {
3828
            int64_t N = aux_vp_tid_array.count();
3829
            for (int64_t i = 0; OB_SUCC(ret) && i < N; i++) {
3830
              const ObTableSchema *aux_vp_table_schema = NULL;
3831
              ObArray<share::schema::ObColDesc> vp_columns;
3832
              ObArray<share::schema::ObColDesc> store_columns;
3833
              if (OB_FAIL(new_schema_guard.get_table_schema(table_schema.get_tenant_id(),
3834
                          aux_vp_tid_array.at(i), aux_vp_table_schema))) {
3835
                LOG_WARN("get_table_schema failed", "table id", aux_vp_tid_array.at(i), K(ret));
3836
              } else if (NULL == aux_vp_table_schema) {
3837
                ret = OB_ERR_UNEXPECTED;
3838
                LOG_WARN("aux vp table is null", K(ret));
3839
              } else if (!aux_vp_table_schema->is_aux_vp_table()
3840
                  || AUX_VERTIAL_PARTITION_TABLE != aux_vp_table_schema->get_table_type()) {
3841
                ret = OB_ERR_UNEXPECTED;
3842
                LOG_WARN("aux vp table type is incorrect", K(ret), K(aux_vp_table_schema->is_aux_vp_table()));
3843
              } else if (OB_FAIL(aux_vp_table_schema->get_vp_column_ids(vp_columns))) {
3844
                ret = OB_ERR_UNEXPECTED;
3845
                LOG_WARN("failed to get aux vp table columns", K(ret), K(*aux_vp_table_schema));
3846
              } else if (OB_FAIL(aux_vp_table_schema->get_vp_store_column_ids(store_columns))) {
3847
                ret = OB_ERR_UNEXPECTED;
3848
                LOG_WARN("failed to get aux vp table columns", K(ret), K(*aux_vp_table_schema));
3849
              } else {
3850
                LOG_INFO("table info", K(aux_vp_table_schema->get_table_name()), K(aux_vp_table_schema->get_table_id()),
3851
                        K(aux_vp_table_schema->get_data_table_id()), K(ret));
3852
                const ObColumnSchemaV2 *column_schema = NULL;
3853

3854
                for (int64_t k = 0; OB_SUCC(ret) && k < vp_columns.count(); ++k) {
3855
                  LOG_INFO("column info", K(vp_columns.at(k).col_id_), K(vp_columns.at(k).col_type_));
3856
                  if (NULL == (column_schema = aux_vp_table_schema->get_column_schema(vp_columns.at(k).col_id_))) {
3857
                    ret = OB_ERR_BAD_FIELD_ERROR;
3858
                    LOG_WARN("get_column_schema failed", K(vp_columns.at(k)), K(ret));
3859
                  } else {
3860
                    LOG_INFO("column info", K(column_schema->get_column_name()), K(column_schema->get_column_id()),
3861
                        K(column_schema->get_table_id()), K(ret));
3862
                  }
3863
                }
3864
                // verify get_vp_store_column_ids return all vertical partition columns,
3865
                // include vertical partition columns of primary key.
3866
                for (int64_t k = 0; OB_SUCC(ret) && k < store_columns.count(); ++k) {
3867
                  LOG_INFO("column info", K(store_columns.at(k).col_id_), K(store_columns.at(k).col_type_));
3868
                  if (NULL == (column_schema = aux_vp_table_schema->get_column_schema(store_columns.at(k).col_id_))) {
3869
                    ret = OB_ERR_BAD_FIELD_ERROR;
3870
                    LOG_WARN("get_column_schema failed", K(store_columns.at(k)), K(ret));
3871
                  } else {
3872
                    LOG_INFO("column info", K(column_schema->get_column_name()), K(column_schema->get_column_id()),
3873
                        K(column_schema->get_table_id()), K(ret));
3874
                  }
3875
                }
3876
              }
3877
            }
3878
          }
3879
        }
3880
      }
3881
    }
3882
    if (OB_SUCC(ret)) {
3883
      uint64_t tenant_id = table_schema.get_tenant_id();
3884
      if (OB_FAIL(schema_service_->get_tenant_schema_version(tenant_id, res.schema_version_))) {
3885
        LOG_WARN("failed to get tenant schema version", K(ret));
3886
      } else {
3887
        res.task_id_ = ddl_task_id;
3888
      }
3889
    }
3890
  }
3891

3892
  RS_TRACE(create_table_end);
3893
  FORCE_PRINT_TRACE(THE_RS_TRACE, "[create table]");
3894
  int64_t cost = ObTimeUtility::current_time() - begin_time;
3895
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "create table",
3896
                        "tenant_id", arg.schema_.get_tenant_id(),
3897
                        "ret", ret,
3898
                        "trace_id", *ObCurTraceId::get_trace_id(),
3899
                        "table_id", res.table_id_,
3900
                        "schema_version", res.schema_version_,
3901
                        K(cost));
3902
  LOG_INFO("finish create table ddl", K(ret), K(cost), "ddl_event_info", ObDDLEventInfo());
3903
  return ret;
3904
}
3905

3906
// create sys_table by specify table_id for tenant:
3907
// 1. can not create table cross tenant except sys tenant.
3908
// 2. part_type of sys table only support non-partition or only level hash_like part type.
3909
// 3. sys table's tablegroup and database must be oceanbase
3910
int ObRootService::generate_table_schema_in_tenant_space(
3911
    const ObCreateTableArg &arg,
3912
    ObTableSchema &table_schema)
3913
{
3914
  int ret = OB_SUCCESS;
3915
  const uint64_t tenant_id = arg.exec_tenant_id_;
3916
  const uint64_t table_id = table_schema.get_table_id();
3917
  const ObPartitionLevel part_level = table_schema.get_part_level();
3918
  const ObPartitionFuncType part_func_type = table_schema.get_part_option().get_part_func_type();
3919
  if (!inited_) {
3920
    ret = OB_NOT_INIT;
3921
    LOG_WARN("not init", K(ret));
3922
  } else if (OB_INVALID_ID == table_id || !is_inner_table(table_id)) {
3923
    // skip
3924
  } else if (OB_SYS_TENANT_ID != arg.exec_tenant_id_) {
3925
    //FIXME: this restriction should be removed later.
3926
    // only enable sys tenant create sys table
3927
    ret = OB_OP_NOT_ALLOW;
3928
    LOG_WARN("only sys tenant can create tenant space table", K(ret), K(arg));
3929
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "non-sys tenant creating system tables");
3930
  } else if (table_schema.is_view_table()) {
3931
    // no need specify tenant_id while specify table_id creating sys table
3932
    if (OB_SYS_TENANT_ID != tenant_id) {
3933
      ret = OB_OP_NOT_ALLOW;
3934
      LOG_WARN("create sys view with ordinary tenant not allowed", K(ret), K(table_schema));
3935
    }
3936
  } else if (part_level > ObPartitionLevel::PARTITION_LEVEL_ONE
3937
             || !is_hash_like_part(part_func_type)) {
3938
    // sys tables do not write __all_part table, so sys table only support non-partition or only level hash_like part type.
3939
    ret = OB_OP_NOT_ALLOW;
3940
    LOG_WARN("sys table's partition option is invalid", K(ret), K(arg));
3941
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "invalid partition option to system table");
3942
  } else if (0 != table_schema.get_tablegroup_name().case_compare(OB_SYS_TABLEGROUP_NAME)) {
3943
    // sys tables's tablegroup must be oceanbase
3944
    ret = OB_OP_NOT_ALLOW;
3945
    LOG_WARN("sys table's tablegroup should be oceanbase", K(ret), K(arg));
3946
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "invalid tablegroup to system table");
3947
  } else if (0 != arg.db_name_.case_compare(OB_SYS_DATABASE_NAME)) {
3948
    // sys tables's database  must be oceanbase
3949
    ret = OB_OP_NOT_ALLOW;
3950
    LOG_WARN("sys table's database should be oceanbase", K(ret), K(arg));
3951
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "invalid database to sys table");
3952
  } else {
3953
    table_schema.set_tenant_id(tenant_id);
3954
    table_schema.set_table_id(table_id);
3955
    table_schema.set_tablegroup_id(OB_SYS_TABLEGROUP_ID);
3956
    table_schema.set_tablegroup_name(OB_SYS_TABLEGROUP_NAME);
3957
    table_schema.set_database_id(OB_SYS_DATABASE_ID);
3958
  }
3959
  return ret;
3960
}
3961

3962
int ObRootService::maintain_obj_dependency_info(const obrpc::ObDependencyObjDDLArg &arg)
3963
{
3964
  LOG_DEBUG("receive maintain obj dependency info arg", K(arg));
3965
  int ret = OB_SUCCESS;
3966
  if (!inited_) {
3967
    ret = OB_NOT_INIT;
3968
    LOG_WARN("not init", K(ret));
3969
  } else if (!arg.is_valid()) {
3970
    ret = OB_INVALID_ARGUMENT;
3971
    LOG_WARN("invalid arg", K(arg), K(ret));
3972
  } else if (OB_FAIL(ddl_service_.maintain_obj_dependency_info(arg))) {
3973
    LOG_WARN("failed to maintain obj dependency info", K(ret), K(arg));
3974
  }
3975
  return ret;
3976
}
3977

3978
int ObRootService::mview_complete_refresh(const obrpc::ObMViewCompleteRefreshArg &arg,
3979
                                          obrpc::ObMViewCompleteRefreshRes &res)
3980
{
3981
  LOG_DEBUG("receive mview complete refresh arg", K(arg));
3982
  int ret = OB_SUCCESS;
3983
  const uint64_t tenant_id = arg.tenant_id_;
3984
  uint64_t compat_version = 0;
3985
  if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
3986
    LOG_WARN("fail to get data version", KR(ret), K(tenant_id));
3987
  } else if (compat_version < DATA_VERSION_4_3_0_0) {
3988
    ret = OB_NOT_SUPPORTED;
3989
    LOG_WARN("version lower than 4.3 does not support this operation", KR(ret));
3990
    LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant's data version is below 4.3.0.0, mview complete refresh is ");
3991
  } else if (!inited_) {
3992
    ret = OB_NOT_INIT;
3993
    LOG_WARN("not init", KR(ret));
3994
  } else if (!arg.is_valid()) {
3995
    ret = OB_INVALID_ARGUMENT;
3996
    LOG_WARN("invalid arg", KR(ret), K(arg));
3997
  } else {
3998
    ObSchemaGetterGuard schema_guard;
3999
    if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
4000
      LOG_WARN("get schema guard in inner table failed", KR(ret), K(tenant_id));
4001
    } else if (OB_FAIL(check_parallel_ddl_conflict(schema_guard, arg))) {
4002
      LOG_WARN("check parallel ddl conflict failed", KR(ret), K(arg));
4003
    } else if (OB_FAIL(ddl_service_.mview_complete_refresh(arg, res, schema_guard))) {
4004
      LOG_WARN("failed to mview complete refresh", KR(ret), K(arg));
4005
    }
4006
  }
4007
  return ret;
4008
}
4009

4010
int ObRootService::execute_ddl_task(const obrpc::ObAlterTableArg &arg,
4011
                                    common::ObSArray<uint64_t> &obj_ids)
4012
{
4013
  LOG_DEBUG("receive execute ddl task arg", K(arg));
4014
  int ret = OB_SUCCESS;
4015
  if (!inited_) {
4016
    ret = OB_NOT_INIT;
4017
    LOG_WARN("not init", K(ret));
4018
  } else if (!arg.is_valid()) {
4019
    ret = OB_INVALID_ARGUMENT;
4020
    LOG_WARN("invalid arg", K(arg), K(ret));
4021
  } else {
4022
    switch (arg.ddl_task_type_) {
4023
      case share::REBUILD_INDEX_TASK: {
4024
        if (OB_FAIL(ddl_service_.rebuild_hidden_table_index_in_trans(
4025
            const_cast<obrpc::ObAlterTableArg &>(arg), obj_ids))) {
4026
          LOG_WARN("failed to rebuild hidden table index in trans", K(ret));
4027
        }
4028
        break;
4029
      }
4030
      case share::REBUILD_CONSTRAINT_TASK: {
4031
        if (OB_FAIL(ddl_service_.rebuild_hidden_table_constraints_in_trans(
4032
            const_cast<obrpc::ObAlterTableArg &>(arg), obj_ids))) {
4033
          LOG_WARN("failed to rebuild hidden table constraints in trans", K(ret));
4034
        }
4035
        break;
4036
      }
4037
      case share::REBUILD_FOREIGN_KEY_TASK: {
4038
        if (OB_FAIL(ddl_service_.rebuild_hidden_table_foreign_key_in_trans(
4039
            const_cast<obrpc::ObAlterTableArg &>(arg), obj_ids))) {
4040
          LOG_WARN("failed to rebuild hidden table foreign key in trans", K(ret));
4041
        }
4042
        break;
4043
      }
4044
      case share::MAKE_DDL_TAKE_EFFECT_TASK: {
4045
        if (OB_FAIL(ddl_service_.swap_orig_and_hidden_table_state(
4046
            const_cast<obrpc::ObAlterTableArg &>(arg)))) {
4047
          LOG_WARN("failed to swap orig and hidden table state", K(ret));
4048
        }
4049
        break;
4050
      }
4051
      case share::CLEANUP_GARBAGE_TASK: {
4052
        if (OB_FAIL(ddl_service_.cleanup_garbage(
4053
            const_cast<obrpc::ObAlterTableArg &>(arg)))) {
4054
          LOG_WARN("failed to cleanup garbage", K(ret));
4055
        }
4056
        break;
4057
      }
4058
      case share::MODIFY_FOREIGN_KEY_STATE_TASK: {
4059
        if (OB_FAIL(ddl_service_.modify_hidden_table_fk_state(
4060
            const_cast<obrpc::ObAlterTableArg &>(arg)))) {
4061
          LOG_WARN("failed to modify hidden table fk state", K(ret));
4062
        }
4063
        break;
4064
      }
4065
      case share::DELETE_COLUMN_FROM_SCHEMA: {
4066
        if (OB_FAIL(ddl_service_.delete_column_from_schema(const_cast<ObAlterTableArg &>(arg)))) {
4067
          LOG_WARN("fail to set column to no minor status", K(ret), K(arg));
4068
        }
4069
        break;
4070
      }
4071
      // remap all index tables to hidden table and take effect concurrently.
4072
      case share::REMAP_INDEXES_AND_TAKE_EFFECT_TASK: {
4073
        if (OB_FAIL(ddl_service_.remap_index_tablets_and_take_effect(
4074
            const_cast<obrpc::ObAlterTableArg &>(arg)))) {
4075
          LOG_WARN("fail to remap index tables to hidden table and take effect", K(ret));
4076
        }
4077
        break;
4078
      }
4079
      case share::UPDATE_AUTOINC_SCHEMA: {
4080
        if (OB_FAIL(ddl_service_.update_autoinc_schema(const_cast<ObAlterTableArg &>(arg)))) {
4081
          LOG_WARN("fail to update autoinc schema", K(ret), K(arg));
4082
        }
4083
        break;
4084
      }
4085
      case share::MODIFY_NOT_NULL_COLUMN_STATE_TASK: {
4086
        if (OB_FAIL(ddl_service_.modify_hidden_table_not_null_column_state(arg))) {
4087
          LOG_WARN("failed to modify hidden table cst state", K(ret));
4088
        }
4089
        break;
4090
      }
4091
      case share::MAKE_RECOVER_RESTORE_TABLE_TASK_TAKE_EFFECT: {
4092
        if (OB_FAIL(ddl_service_.make_recover_restore_tables_visible(const_cast<ObAlterTableArg &>(arg)))) {
4093
          LOG_WARN("make recovert restore task visible failed", K(ret), K(arg));
4094
        }
4095
        break;
4096
      }
4097
      default:
4098
        ret = OB_ERR_UNEXPECTED;
4099
        LOG_WARN("unknown ddl task type", K(ret), K(arg.ddl_task_type_));
4100
    }
4101
  }
4102
  return ret;
4103
}
4104

4105
int ObRootService::precheck_interval_part(const obrpc::ObAlterTableArg &arg)
4106
{
4107
  int ret = OB_SUCCESS;
4108
  ObSchemaGetterGuard schema_guard;
4109
  const ObAlterTableArg::AlterPartitionType op_type = arg.alter_part_type_;
4110
  const ObSimpleTableSchemaV2 *simple_table_schema = NULL;
4111
  const AlterTableSchema &alter_table_schema = arg.alter_table_schema_;
4112
  int64_t tenant_id = alter_table_schema.get_tenant_id();
4113

4114
  if (!alter_table_schema.is_interval_part()
4115
      || obrpc::ObAlterTableArg::ADD_PARTITION != op_type) {
4116
  } else if (OB_ISNULL(schema_service_)) {
4117
    ret = OB_ERR_UNEXPECTED;
4118
    LOG_WARN("error unexpected, schema service must not be NULL", K(ret));
4119
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) {
4120
    LOG_WARN("fail to get schema guard", K(ret));
4121
  } else if (OB_FAIL(schema_guard.get_simple_table_schema(tenant_id,
4122
             alter_table_schema.get_table_id(), simple_table_schema))) {
4123
    LOG_WARN("get table schema failed", KR(ret), K(tenant_id), K(alter_table_schema));
4124
  } else if (OB_ISNULL(simple_table_schema)) {
4125
    ret = OB_TABLE_NOT_EXIST;
4126
    LOG_WARN("simple_table_schema is null", K(ret), K(alter_table_schema));
4127
  } else if (simple_table_schema->get_schema_version() < alter_table_schema.get_schema_version()) {
4128
  } else if (simple_table_schema->get_interval_range() != alter_table_schema.get_interval_range()
4129
             || simple_table_schema->get_transition_point() != alter_table_schema.get_transition_point()) {
4130
    ret = OB_ERR_INTERVAL_PARTITION_ERROR;
4131
    LOG_WARN("interval_range or transition_point is changed", KR(ret), \
4132
             KPC(simple_table_schema), K(alter_table_schema));
4133
  } else {
4134
    int64_t j = 0;
4135
    const ObRowkey *rowkey_orig= NULL;
4136
    bool is_all_exist = true;
4137
    ObPartition **inc_part_array = alter_table_schema.get_part_array();
4138
    ObPartition **orig_part_array = simple_table_schema->get_part_array();
4139
    if (OB_ISNULL(inc_part_array)) {
4140
      ret = OB_ERR_UNEXPECTED;
4141
      LOG_WARN("ptr is null", K(ret), K(alter_table_schema), KPC(simple_table_schema));
4142
    } else if (OB_ISNULL(orig_part_array)) {
4143
      ret = OB_ERR_UNEXPECTED;
4144
      LOG_WARN("ptr is null", K(ret), K(alter_table_schema), KPC(simple_table_schema));
4145
    }
4146
    for (int64_t i = 0; is_all_exist && OB_SUCC(ret) && i < alter_table_schema.get_part_option().get_part_num(); ++i) {
4147
      const ObRowkey *rowkey_cur = NULL;
4148
      if (OB_ISNULL(inc_part_array[i])) {
4149
        ret = OB_ERR_UNEXPECTED;
4150
        LOG_WARN("ptr is null", K(ret), K(alter_table_schema), KPC(simple_table_schema));
4151
      } else if (OB_UNLIKELY(NULL == (rowkey_cur = &inc_part_array[i]->get_high_bound_val()))) {
4152
        ret = OB_ERR_UNEXPECTED;
4153
        LOG_WARN("ptr is null", K(ret), K(alter_table_schema), KPC(simple_table_schema));
4154
      }
4155
      while (is_all_exist && OB_SUCC(ret) && j < simple_table_schema->get_part_option().get_part_num()) {
4156
        if (OB_ISNULL(orig_part_array[j])) {
4157
          ret = OB_ERR_UNEXPECTED;
4158
          LOG_WARN("ptr is null", K(ret), K(alter_table_schema), KPC(simple_table_schema));
4159
        } else if (OB_UNLIKELY(NULL == (rowkey_orig = &orig_part_array[j]->get_high_bound_val()))) {
4160
          ret = OB_ERR_UNEXPECTED;
4161
          LOG_WARN("ptr is null", K(ret), K(alter_table_schema), KPC(simple_table_schema));
4162
        } else if (*rowkey_orig < *rowkey_cur) {
4163
          j++;
4164
        } else {
4165
          break;
4166
        }
4167
      }
4168
      if (OB_FAIL(ret)) {
4169
      } else if (*rowkey_orig != *rowkey_cur) {
4170
        is_all_exist = false;
4171
      }
4172
    }
4173
    if (OB_FAIL(ret)) {
4174
    } else if (is_all_exist) {
4175
      LOG_INFO("all interval part for add is exist", K(alter_table_schema), KPC(simple_table_schema));
4176
      ret = OB_ERR_INTERVAL_PARTITION_EXIST;
4177
    }
4178
  }
4179
  return ret;
4180
}
4181

4182
int ObRootService::create_hidden_table(const obrpc::ObCreateHiddenTableArg &arg,
4183
                                       obrpc::ObCreateHiddenTableRes &res)
4184
{
4185
  LOG_DEBUG("receive create hidden table arg", K(arg));
4186
  int ret = OB_SUCCESS;
4187
  const uint64_t tenant_id = arg.tenant_id_;
4188
  uint64_t compat_version = 0;
4189
  if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
4190
    LOG_WARN("fail to get data version", K(ret), K(tenant_id));
4191
  } else if (compat_version < DATA_VERSION_4_1_0_0) {
4192
    ret = OB_NOT_SUPPORTED;
4193
    LOG_WARN("version 4.0 does not support this operation", K(ret));
4194
  } else if (OB_UNLIKELY(!inited_)) {
4195
    ret = OB_NOT_INIT;
4196
    LOG_WARN("not init", K(ret));
4197
  } else if (OB_UNLIKELY(!arg.is_valid())) {
4198
    ret = OB_INVALID_ARGUMENT;
4199
    LOG_WARN("invalid arg", K(ret), K(arg));
4200
  } else if (OB_FAIL(DDL_SIM(arg.tenant_id_, arg.task_id_, CREATE_HIDDEN_TABLE_RPC_FAILED))) {
4201
    LOG_WARN("ddl sim failure", K(ret), K(arg));
4202
  } else if (OB_FAIL(DDL_SIM(arg.tenant_id_, arg.task_id_, CREATE_HIDDEN_TABLE_RPC_SLOW))) {
4203
    LOG_WARN("ddl sim failure", K(ret), K(arg));
4204
  } else if (OB_FAIL(ddl_service_.create_hidden_table(arg, res))) {
4205
    LOG_WARN("do create hidden table in trans failed", K(ret), K(arg));
4206
  }
4207
  char tenant_id_buffer[128];
4208
  snprintf(tenant_id_buffer, sizeof(tenant_id_buffer), "orig_tenant_id:%ld, target_tenant_id:%ld",
4209
            arg.tenant_id_, arg.dest_tenant_id_);
4210
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "create hidden table",
4211
                        "tenant_id", tenant_id_buffer,
4212
                        "ret", ret,
4213
                        "trace_id", *ObCurTraceId::get_trace_id(),
4214
                        "task_id", res.task_id_,
4215
                        "table_id", arg.table_id_,
4216
                        "schema_version", res.schema_version_);
4217
  LOG_INFO("finish create hidden table ddl", K(ret), K(arg), K(res), "ddl_event_info", ObDDLEventInfo());
4218
  return ret;
4219
}
4220

4221
int ObRootService::update_ddl_task_active_time(const obrpc::ObUpdateDDLTaskActiveTimeArg &arg)
4222
{
4223
  LOG_DEBUG("receive recv ddl task status arg", K(arg));
4224
  int ret = OB_SUCCESS;
4225
  const int64_t task_id = arg.task_id_;
4226
  const uint64_t tenant_id = arg.tenant_id_;
4227
  uint64_t compat_version = 0;
4228
  if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
4229
    LOG_WARN("fail to get data version", K(ret), K(tenant_id));
4230
  } else if (compat_version < DATA_VERSION_4_1_0_0) {
4231
    ret = OB_NOT_SUPPORTED;
4232
    LOG_WARN("version 4.0 does not support this operation", K(ret));
4233
  } else if (OB_UNLIKELY(!inited_)) {
4234
    ret = OB_NOT_INIT;
4235
    LOG_WARN("not init", K(ret));
4236
  } else if (OB_UNLIKELY(!arg.is_valid())) {
4237
    ret = OB_INVALID_ARGUMENT;
4238
    LOG_WARN("invalid arg", K(ret), K(arg));
4239
  } else if (OB_FAIL(ddl_scheduler_.update_ddl_task_active_time(ObDDLTaskID(tenant_id, task_id)))) {
4240
    LOG_WARN("fail to set RegTaskTime map", K(ret), K(tenant_id), K(task_id));
4241
  }
4242
  return ret;
4243
}
4244

4245
int ObRootService::abort_redef_table(const obrpc::ObAbortRedefTableArg &arg)
4246
{
4247
  LOG_DEBUG("receive abort redef table arg", K(arg));
4248
  int ret = OB_SUCCESS;
4249
  const int64_t task_id = arg.task_id_;
4250
  const uint64_t tenant_id = arg.tenant_id_;
4251
  uint64_t compat_version = 0;
4252
  if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
4253
    LOG_WARN("fail to get data version", K(ret), K(tenant_id));
4254
  } else if (compat_version < DATA_VERSION_4_1_0_0) {
4255
    ret = OB_NOT_SUPPORTED;
4256
    LOG_WARN("version 4.0 does not support this operation", K(ret));
4257
  } else if (OB_UNLIKELY(!inited_)) {
4258
    ret = OB_NOT_INIT;
4259
    LOG_WARN("not init", K(ret));
4260
  } else if (OB_UNLIKELY(!arg.is_valid())) {
4261
    ret = OB_INVALID_ARGUMENT;
4262
    LOG_WARN("invalid arg", K(ret), K(arg));
4263
  } else if (OB_FAIL(DDL_SIM(arg.tenant_id_, arg.task_id_, ABORT_REDEF_TABLE_RPC_FAILED))) {
4264
    LOG_WARN("ddl sim failure", K(ret), K(arg));
4265
  } else if (OB_FAIL(DDL_SIM(arg.tenant_id_, arg.task_id_, ABORT_REDEF_TABLE_RPC_SLOW))) {
4266
    LOG_WARN("ddl sim failure", K(ret), K(arg));
4267
  } else if (OB_FAIL(ddl_scheduler_.abort_redef_table(ObDDLTaskID(tenant_id, task_id)))) {
4268
    LOG_WARN("cancel task failed", K(ret), K(tenant_id), K(task_id));
4269
  }
4270
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "abort redef table",
4271
                        "tenant_id", arg.tenant_id_,
4272
                        "ret", ret,
4273
                        "trace_id", *ObCurTraceId::get_trace_id(),
4274
                        "task_id", arg.task_id_);
4275
  LOG_INFO("finish abort redef table ddl", K(ret), K(arg), "ddl_event_info", ObDDLEventInfo());
4276
  return ret;
4277
}
4278

4279
int ObRootService::finish_redef_table(const obrpc::ObFinishRedefTableArg &arg)
4280
{
4281
  LOG_DEBUG("receive finish redef table arg", K(arg));
4282
  int ret = OB_SUCCESS;
4283
  const int64_t task_id = arg.task_id_;
4284
  const uint64_t tenant_id = arg.tenant_id_;
4285
  uint64_t compat_version = 0;
4286
  if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
4287
    LOG_WARN("fail to get data version", K(ret), K(tenant_id));
4288
  } else if (compat_version < DATA_VERSION_4_1_0_0) {
4289
    ret = OB_NOT_SUPPORTED;
4290
    LOG_WARN("version 4.0 does not support this operation", K(ret));
4291
  } else if (OB_UNLIKELY(!inited_)) {
4292
    ret = OB_NOT_INIT;
4293
    LOG_WARN("not init", K(ret));
4294
  } else if (OB_UNLIKELY(!arg.is_valid())) {
4295
    ret = OB_INVALID_ARGUMENT;
4296
    LOG_WARN("invalid arg", K(ret), K(arg));
4297
  } else if (OB_FAIL(DDL_SIM(arg.tenant_id_, arg.task_id_, FINISH_REDEF_TABLE_RPC_FAILED))) {
4298
    LOG_WARN("ddl sim failure", K(ret), K(arg));
4299
  } else if (OB_FAIL(DDL_SIM(arg.tenant_id_, arg.task_id_, FINISH_REDEF_TABLE_RPC_SLOW))) {
4300
    LOG_WARN("ddl sim failure", K(ret), K(arg));
4301
  } else if (OB_FAIL(ddl_scheduler_.finish_redef_table(ObDDLTaskID(tenant_id, task_id)))) {
4302
    LOG_WARN("failed to finish redef table", K(ret), K(task_id), K(tenant_id));
4303
  }
4304
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "finish redef table",
4305
                        "tenant_id", arg.tenant_id_,
4306
                        "ret", ret,
4307
                        "trace_id", *ObCurTraceId::get_trace_id(),
4308
                        "task_id", arg.task_id_);
4309
  LOG_INFO("finish abort redef table ddl", K(ret), K(arg), "ddl_event_info", ObDDLEventInfo());
4310
  return ret;
4311
}
4312

4313
int ObRootService::copy_table_dependents(const obrpc::ObCopyTableDependentsArg &arg)
4314
{
4315
  LOG_INFO("receive copy table dependents arg", K(arg));
4316
  int ret = OB_SUCCESS;
4317
  const int64_t task_id = arg.task_id_;
4318
  const uint64_t tenant_id = arg.tenant_id_;
4319
  const bool is_copy_indexes = arg.copy_indexes_;
4320
  const bool is_copy_triggers = arg.copy_triggers_;
4321
  const bool is_copy_constraints = arg.copy_constraints_;
4322
  const bool is_copy_foreign_keys = arg.copy_foreign_keys_;
4323
  const bool is_ignore_errors = arg.ignore_errors_;
4324
  uint64_t compat_version = 0;
4325
  if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
4326
    LOG_WARN("fail to get data version", K(ret), K(tenant_id));
4327
  } else if (compat_version < DATA_VERSION_4_1_0_0) {
4328
    ret = OB_NOT_SUPPORTED;
4329
    LOG_WARN("version 4.0 does not support this operation", K(ret));
4330
  } else if (OB_UNLIKELY(!inited_)) {
4331
    ret = OB_NOT_INIT;
4332
    LOG_WARN("not init", K(ret));
4333
  } else if (OB_UNLIKELY(!arg.is_valid())) {
4334
    ret = OB_INVALID_ARGUMENT;
4335
    LOG_WARN("invalid arg", K(ret), K(arg));
4336
  } else if (OB_FAIL(DDL_SIM(arg.tenant_id_, arg.task_id_, COPY_TABLE_DEPENDENTS_RPC_FAILED))) {
4337
    LOG_WARN("ddl sim failure", K(ret), K(arg));
4338
  } else if (OB_FAIL(DDL_SIM(arg.tenant_id_, arg.task_id_, COPY_TABLE_DEPENDENTS_RPC_SLOW))) {
4339
    LOG_WARN("ddl sim failure", K(ret), K(arg));
4340
  } else if (OB_FAIL(ddl_scheduler_.copy_table_dependents(ObDDLTaskID(tenant_id, task_id),
4341
                                                          is_copy_constraints,
4342
                                                          is_copy_indexes,
4343
                                                          is_copy_triggers,
4344
                                                          is_copy_foreign_keys,
4345
                                                          is_ignore_errors))) {
4346
    LOG_WARN("failed to copy table dependents", K(ret), K(arg));
4347
  }
4348
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "copy table dependents",
4349
                        "tenant_id", tenant_id,
4350
                        "ret", ret,
4351
                        "trace_id", *ObCurTraceId::get_trace_id(),
4352
                        "task_id", task_id);
4353
  LOG_INFO("finish copy table dependents ddl", K(ret), K(arg), "ddl_event_info", ObDDLEventInfo());
4354
  return ret;
4355
}
4356

4357
int ObRootService::start_redef_table(const obrpc::ObStartRedefTableArg &arg, obrpc::ObStartRedefTableRes &res)
4358
{
4359
  LOG_DEBUG("receive start redef table arg", K(arg));
4360
  int ret = OB_SUCCESS;
4361
  const uint64_t tenant_id = arg.orig_tenant_id_;
4362
  uint64_t compat_version = 0;
4363
  if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
4364
    LOG_WARN("fail to get data version", K(ret), K(tenant_id));
4365
  } else if (compat_version < DATA_VERSION_4_1_0_0) {
4366
    ret = OB_NOT_SUPPORTED;
4367
    LOG_WARN("version 4.0 does not support this operation", K(ret));
4368
  } else if (OB_UNLIKELY(!inited_)) {
4369
    ret = OB_NOT_INIT;
4370
    LOG_WARN("not init", K(ret));
4371
  } else if (OB_UNLIKELY(!arg.is_valid())) {
4372
    ret = OB_INVALID_ARGUMENT;
4373
    LOG_WARN("invalid arg", K(ret), K(arg));
4374
  } else if (OB_FAIL(ddl_scheduler_.start_redef_table(arg, res))) {
4375
    LOG_WARN("start redef table failed", K(ret));
4376
  }
4377
  char tenant_id_buffer[128];
4378
  snprintf(tenant_id_buffer, sizeof(tenant_id_buffer), "orig_tenant_id:%ld, target_tenant_id:%ld",
4379
            arg.orig_tenant_id_, arg.target_tenant_id_);
4380
  char table_id_buffer[128];
4381
  snprintf(table_id_buffer, sizeof(table_id_buffer), "orig_table_id:%ld, target_table_id:%ld",
4382
            arg.orig_table_id_, arg.target_table_id_);
4383
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "redef table",
4384
                        "tenant_id", tenant_id_buffer,
4385
                        "ret", ret,
4386
                        "trace_id", *ObCurTraceId::get_trace_id(),
4387
                        "task_id", res.task_id_,
4388
                        "table_id", table_id_buffer,
4389
                        "schema_version", res.schema_version_);
4390
  LOG_INFO("finish redef table ddl", K(arg), K(ret), K(res), "ddl_event_info", ObDDLEventInfo());
4391
  return ret;
4392
}
4393

4394
int ObRootService::recover_restore_table_ddl(const obrpc::ObRecoverRestoreTableDDLArg &arg)
4395
{
4396
  int ret = OB_SUCCESS;
4397
  uint64_t compat_version = 0;
4398
  if (OB_FAIL(GET_MIN_DATA_VERSION(arg.src_tenant_id_, compat_version))) {
4399
    LOG_WARN("fail to get data version", K(ret), K(arg));
4400
  } else if (compat_version < DATA_VERSION_4_2_1_0) {
4401
    ret = OB_NOT_SUPPORTED;
4402
    LOG_WARN("version 4.0 does not support this operation", K(ret));
4403
  } else if (OB_UNLIKELY(!inited_)) {
4404
    ret = OB_NOT_INIT;
4405
    LOG_WARN("not init", K(ret));
4406
  } else if (OB_UNLIKELY(!arg.is_valid())) {
4407
    ret = OB_INVALID_ARGUMENT;
4408
    LOG_WARN("invalid arg", K(ret), K(arg));
4409
  } else if (OB_FAIL(ddl_service_.recover_restore_table_ddl_task(arg))) {
4410
    LOG_WARN("recover restore table ddl task failed", K(ret), K(arg));
4411
  }
4412
  LOG_INFO("recover restore table ddl finish", K(ret), K(arg));
4413
  return ret;
4414
}
4415

4416
int ObRootService::alter_table(const obrpc::ObAlterTableArg &arg, obrpc::ObAlterTableRes &res)
4417
{
4418
  LOG_DEBUG("receive alter table arg", K(arg));
4419
  int ret = OB_SUCCESS;
4420
  bool is_oracle_mode = false;
4421
  ObSchemaGetterGuard schema_guard;
4422
  const uint64_t tenant_id = arg.alter_table_schema_.get_tenant_id();
4423
  ObAlterTableArg &nonconst_arg = const_cast<ObAlterTableArg &>(arg);
4424
  if (!inited_) {
4425
    ret = OB_NOT_INIT;
4426
    LOG_WARN("not init", K(ret));
4427
  } else if (!arg.is_valid()) {
4428
    ret = OB_INVALID_ARGUMENT;
4429
    LOG_WARN("invalid arg", K(arg), K(ret));
4430
  } else if (OB_FAIL(precheck_interval_part(arg))) {
4431
    if (ret != OB_ERR_INTERVAL_PARTITION_EXIST) {
4432
      LOG_WARN("fail to precheck_interval_part", K(arg), KR(ret));
4433
    }
4434
  } else {
4435
    if (OB_FAIL(ret)) {
4436
    } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
4437
      LOG_WARN("get schema guard in inner table failed", K(ret));
4438
    } else if (OB_FAIL(check_parallel_ddl_conflict(schema_guard, arg))) {
4439
      LOG_WARN("check parallel ddl conflict failed", K(ret));
4440
    } else if (OB_FAIL(table_allow_ddl_operation(arg))) {
4441
      LOG_WARN("table can't do ddl now", K(ret));
4442
    } else if (nonconst_arg.is_add_to_scheduler_) {
4443
      ObDDLTaskRecord task_record;
4444
      ObArenaAllocator allocator(lib::ObLabel("DdlTaskTmp"));
4445
      ObDDLType ddl_type = ObDDLType::DDL_INVALID;
4446
      const ObTableSchema *orig_table_schema = nullptr;
4447
      schema_guard.set_session_id(arg.session_id_);
4448
      if (obrpc::ObAlterTableArg::DROP_PARTITION == nonconst_arg.alter_part_type_) {
4449
        ddl_type = ObDDLType::DDL_DROP_PARTITION;
4450
      } else if (obrpc::ObAlterTableArg::DROP_SUB_PARTITION == nonconst_arg.alter_part_type_) {
4451
        ddl_type = ObDDLType::DDL_DROP_SUB_PARTITION;
4452
      } else if (obrpc::ObAlterTableArg::TRUNCATE_PARTITION == nonconst_arg.alter_part_type_) {
4453
        ddl_type = ObDDLType::DDL_TRUNCATE_PARTITION;
4454
      } else if (obrpc::ObAlterTableArg::TRUNCATE_SUB_PARTITION == nonconst_arg.alter_part_type_) {
4455
        ddl_type = ObDDLType::DDL_TRUNCATE_SUB_PARTITION;
4456
      } else if (obrpc::ObAlterTableArg::RENAME_PARTITION == nonconst_arg.alter_part_type_) {
4457
        ddl_type = ObDDLType::DDL_RENAME_PARTITION;
4458
      } else if (obrpc::ObAlterTableArg::RENAME_SUB_PARTITION == nonconst_arg.alter_part_type_) {
4459
        ddl_type = ObDDLType::DDL_RENAME_SUB_PARTITION;
4460
      } else {
4461
        ret = OB_ERR_UNEXPECTED;
4462
        LOG_WARN("unexpected ddl type", K(ret), K(nonconst_arg.alter_part_type_), K(nonconst_arg));
4463
      }
4464
      if (OB_FAIL(ret)) {
4465
      } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id,
4466
                                                        nonconst_arg.alter_table_schema_.get_database_name(),
4467
                                                        nonconst_arg.alter_table_schema_.get_origin_table_name(),
4468
                                                        false  /* is_index*/,
4469
                                                        orig_table_schema))) {
4470
        LOG_WARN("fail to get and check table schema", K(ret));
4471
      } else if (OB_ISNULL(orig_table_schema)) {
4472
        ret = OB_TABLE_NOT_EXIST;
4473
        LOG_WARN("table not exist", K(ret), K(tenant_id), K(nonconst_arg.alter_table_schema_));
4474
      } else {
4475
        ObCreateDDLTaskParam param(tenant_id,
4476
                                   ddl_type,
4477
                                   nullptr,
4478
                                   nullptr,
4479
                                   orig_table_schema->get_table_id(),
4480
                                   orig_table_schema->get_schema_version(),
4481
                                   arg.parallelism_,
4482
                                   arg.consumer_group_id_,
4483
                                   &allocator,
4484
                                   &arg,
4485
                                   0 /*parent task id*/);
4486
        if (OB_FAIL(ddl_scheduler_.create_ddl_task(param, sql_proxy_, task_record))) {
4487
          LOG_WARN("submit ddl task failed", K(ret), K(arg));
4488
        } else if (OB_FAIL(ddl_scheduler_.schedule_ddl_task(task_record))) {
4489
          LOG_WARN("fail to schedule ddl task", K(ret), K(task_record));
4490
        } else {
4491
          res.ddl_type_ = ddl_type;
4492
          res.task_id_ = task_record.task_id_;
4493
        }
4494
      }
4495
    } else if (OB_FAIL(ddl_service_.alter_table(nonconst_arg, res))) {
4496
      LOG_WARN("alter_user_table failed", K(arg), K(ret));
4497
    } else {
4498
      const ObSimpleTableSchemaV2 *simple_table_schema = NULL;
4499
      // there are multiple DDL except alter table, ctas, comment on, eg.
4500
      // but only alter_table specify table_id, so if no table_id, it indicates DDL is not alter table, skip.
4501
      if (OB_INVALID_ID == arg.alter_table_schema_.get_table_id()) {
4502
        // skip
4503
      } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
4504
        LOG_WARN("get schema guard in inner table failed", K(ret));
4505
      } else if (OB_FAIL(schema_guard.get_simple_table_schema(tenant_id, arg.alter_table_schema_.get_table_id(), simple_table_schema))) {
4506
        LOG_WARN("fail to get table schema", K(ret), K(arg.alter_table_schema_.get_table_id()));
4507
      } else if (OB_ISNULL(simple_table_schema)) {
4508
        ret = OB_ERR_UNEXPECTED;
4509
        LOG_WARN("simple_table_schema is NULL ptr", K(ret), K(simple_table_schema), K(ret));
4510
      } else {
4511
        res.schema_version_ = simple_table_schema->get_schema_version();
4512
      }
4513
    }
4514
  }
4515
  char table_id_buffer[256];
4516
  snprintf(table_id_buffer, sizeof(table_id_buffer), "table_id:%ld, hidden_table_id:%ld",
4517
            arg.table_id_, arg.hidden_table_id_);
4518
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "alter table",
4519
                        K(tenant_id),
4520
                        "ret", ret,
4521
                        "trace_id", *ObCurTraceId::get_trace_id(),
4522
                        "task_id", res.task_id_,
4523
                        "table_id", table_id_buffer,
4524
                        "schema_version", res.schema_version_);
4525
  LOG_INFO("finish alter table ddl", K(ret), K(arg), K(res), "ddl_event_info", ObDDLEventInfo());
4526
  return ret;
4527
}
4528

4529
int ObRootService::create_index(const ObCreateIndexArg &arg, obrpc::ObAlterTableRes &res)
4530
{
4531
  int ret = OB_SUCCESS;
4532
  ObSchemaGetterGuard schema_guard;
4533
  LOG_DEBUG("receive create index arg", K(arg));
4534
  if (!inited_) {
4535
    ret = OB_NOT_INIT;
4536
    LOG_WARN("not init", K(ret));
4537
  } else if (!arg.is_valid()) {
4538
    ret = OB_INVALID_ARGUMENT;
4539
    LOG_WARN("invalid arg", K(arg), K(ret));
4540
  } else {
4541
    ObIndexBuilder index_builder(ddl_service_);
4542
    if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(arg.tenant_id_, schema_guard))) {
4543
      LOG_WARN("get schema guard in inner table failed", K(ret));
4544
    } else if (OB_FAIL(check_parallel_ddl_conflict(schema_guard, arg))) {
4545
      LOG_WARN("check parallel ddl conflict failed", K(ret));
4546
    } else if (OB_FAIL(index_builder.create_index(arg, res))) {
4547
      LOG_WARN("create_index failed", K(arg), K(ret));
4548
    }
4549
  }
4550
  char table_id_buffer[256];
4551
  snprintf(table_id_buffer, sizeof(table_id_buffer), "data_table_id:%ld, index_table_id:%ld",
4552
            arg.data_table_id_, arg.index_table_id_);
4553
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "create index",
4554
                        "tenant_id", arg.tenant_id_,
4555
                        "ret", ret,
4556
                        "trace_id", *ObCurTraceId::get_trace_id(),
4557
                        "task_id", res.task_id_,
4558
                        "table_id", table_id_buffer,
4559
                        "schema_version", res.schema_version_);
4560
  LOG_INFO("finish create index ddl", K(ret), K(arg), K(res), "ddl_event_info", ObDDLEventInfo());
4561
  return ret;
4562
}
4563

4564
int ObRootService::create_mlog(const obrpc::ObCreateMLogArg &arg, obrpc::ObCreateMLogRes &res)
4565
{
4566
  int ret = OB_SUCCESS;
4567
  if (!inited_) {
4568
    ret = OB_NOT_INIT;
4569
    LOG_WARN("not init", KR(ret));
4570
  } else if (!arg.is_valid()) {
4571
    ret = OB_INVALID_ARGUMENT;
4572
    LOG_WARN("invalid arg", KR(ret), K(arg));
4573
  } else {
4574
    ObSchemaGetterGuard schema_guard;
4575
    ObMLogBuilder mlog_builder(ddl_service_);
4576
    if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(
4577
        arg.tenant_id_, schema_guard))) {
4578
      LOG_WARN("get schema guard in inner table failed", K(ret));
4579
    } else if (OB_FAIL(check_parallel_ddl_conflict(schema_guard, arg))) {
4580
      LOG_WARN("check parallel ddl conflict failed", K(ret));
4581
    } else if (OB_FAIL(mlog_builder.init())) {
4582
      LOG_WARN("failed to init mlog builder", KR(ret));
4583
    } else if (OB_FAIL(mlog_builder.create_mlog(schema_guard, arg, res))) {
4584
      LOG_WARN("failed to create mlog", KR(ret), K(arg));
4585
    }
4586
  }
4587
  return ret;
4588
}
4589

4590
int ObRootService::drop_table(const obrpc::ObDropTableArg &arg, obrpc::ObDDLRes &res)
4591
{
4592
  int ret = OB_SUCCESS;
4593
  uint64_t target_object_id = OB_INVALID_ID;
4594
  int64_t schema_version = OB_INVALID_SCHEMA_VERSION;
4595
  bool need_add_to_ddl_scheduler = arg.is_add_to_scheduler_;
4596
  const uint64_t tenant_id = arg.tenant_id_;
4597
  ObSchemaGetterGuard schema_guard;
4598
  if (!inited_) {
4599
    ret = OB_NOT_INIT;
4600
    LOG_WARN("not init", K(ret));
4601
  } else if (!arg.is_valid()) {
4602
    ret = OB_INVALID_ARGUMENT;
4603
    LOG_WARN("invalid arg", K(arg), K(ret));
4604
  } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
4605
    LOG_WARN("fail to get schema guard with version in inner table", K(ret), K(arg));
4606
  } else if (need_add_to_ddl_scheduler) {
4607
    // to decide wherther to add to ddl scheduler.
4608
    // 1. do not add to scheduler if all tables do not exist.
4609
    // 2. do not add to scheduler if all existed tables are temporary tables.
4610
    need_add_to_ddl_scheduler = arg.tables_.count() == 0 ? false : true;
4611
    for (int64_t i = 0; OB_SUCC(ret) && need_add_to_ddl_scheduler && i < arg.tables_.count(); ++i) {
4612
      int tmp_ret = OB_SUCCESS;
4613
      const ObTableItem &table_item = arg.tables_.at(i);
4614
      const ObTableSchema *table_schema = nullptr;
4615
      if (OB_SUCCESS != (tmp_ret = ddl_service_.check_table_exists(tenant_id,
4616
                                                                   table_item,
4617
                                                                   arg.table_type_,
4618
                                                                   schema_guard,
4619
                                                                   &table_schema))) {
4620
        LOG_INFO("check table exist failed, generate error msg in ddl service later", K(ret), K(tmp_ret));
4621
      }
4622
      if (OB_FAIL(ret)) {
4623
      } else if (nullptr != table_schema) {
4624
        if (table_schema->is_tmp_table()) {
4625
          // do nothing.
4626
        } else if (OB_INVALID_ID == target_object_id || OB_INVALID_SCHEMA_VERSION == schema_version) {
4627
          // regard table_id, schema_version of the the first table as the tag to submit ddl task.
4628
          target_object_id = table_schema->get_table_id();
4629
          schema_version = table_schema->get_schema_version();
4630
        }
4631
      }
4632
    }
4633
    // all tables do not exist, or all existed tables are temporary tables.
4634
    if (OB_INVALID_ID == target_object_id || OB_INVALID_SCHEMA_VERSION == schema_version) {
4635
      need_add_to_ddl_scheduler = false;
4636
    }
4637
  }
4638

4639
  if (OB_FAIL(ret)) {
4640
  } else if (need_add_to_ddl_scheduler) {
4641
    ObDDLTaskRecord task_record;
4642
    ObArenaAllocator allocator(lib::ObLabel("DdlTaskTmp"));
4643
    ObCreateDDLTaskParam param(tenant_id,
4644
                               ObDDLType::DDL_DROP_TABLE,
4645
                               nullptr,
4646
                               nullptr,
4647
                               target_object_id,
4648
                               schema_version,
4649
                               arg.parallelism_,
4650
                               arg.consumer_group_id_,
4651
                               &allocator,
4652
                               &arg,
4653
                               0 /* parent task id*/);
4654
    if (OB_UNLIKELY(OB_INVALID_ID == target_object_id || OB_INVALID_SCHEMA_VERSION == schema_version)) {
4655
      ret = OB_ERR_UNEXPECTED;
4656
      LOG_WARN("error unexpected", K(ret), K(arg), K(target_object_id), K(schema_version));
4657
    } else if (OB_FAIL(ddl_scheduler_.create_ddl_task(param, sql_proxy_, task_record))) {
4658
      LOG_WARN("submit ddl task failed", K(ret), K(arg));
4659
    } else if (OB_FAIL(ddl_scheduler_.schedule_ddl_task(task_record))) {
4660
      LOG_WARN("fail to schedule ddl task", K(ret), K(task_record));
4661
    } else {
4662
      res.tenant_id_ = tenant_id;
4663
      res.schema_id_ = target_object_id;
4664
      res.task_id_ = task_record.task_id_;
4665
    }
4666
  } else if (OB_FAIL(ddl_service_.drop_table(arg, res))) {
4667
    LOG_WARN("ddl service failed to drop table", K(ret), K(arg), K(res));
4668
  }
4669
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "drop table",
4670
                        "tenant_id", arg.tenant_id_,
4671
                        "ret", ret,
4672
                        "trace_id", *ObCurTraceId::get_trace_id(),
4673
                        "task_id", res.task_id_,
4674
                        "session_id", arg.session_id_,
4675
                        "schema_version", res.schema_id_);
4676
  LOG_INFO("finish drop table ddl", K(ret), K(arg), "ddl_event_info", ObDDLEventInfo());
4677
  return ret;
4678
}
4679

4680
int ObRootService::drop_database(const obrpc::ObDropDatabaseArg &arg, ObDropDatabaseRes &drop_database_res)
4681
{
4682
  int ret = OB_SUCCESS;
4683
  uint64_t database_id = 0;
4684
  int64_t schema_version = 0;
4685
  bool need_add_to_scheduler = arg.is_add_to_scheduler_;
4686
  const uint64_t tenant_id = arg.tenant_id_;
4687
  if (!inited_) {
4688
    ret = OB_NOT_INIT;
4689
    LOG_WARN("not init", K(ret));
4690
  } else if (!arg.is_valid()) {
4691
    ret = OB_INVALID_ARGUMENT;
4692
    LOG_WARN("invalid arg", K(arg), K(ret));
4693
  } else if (need_add_to_scheduler) {
4694
    ObSchemaGetterGuard schema_guard;
4695
    if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
4696
      LOG_WARN("get schema guard in inner table failed", K(ret), K(tenant_id));
4697
    } else if (OB_FAIL(schema_guard.get_schema_version(tenant_id, schema_version))) {
4698
      LOG_WARN("fail to get schema version", K(ret), K(arg));
4699
    } else if (OB_FAIL(schema_guard.get_database_id(tenant_id, arg.database_name_, database_id))) {
4700
      LOG_WARN("fail to get database id");
4701
    } else if (OB_INVALID_ID == database_id) {
4702
      // drop database if exists xxx.
4703
      need_add_to_scheduler = false;
4704
    }
4705
  }
4706

4707
  if (OB_FAIL(ret)) {
4708
  } else if (need_add_to_scheduler) {
4709
    ObDDLTaskRecord task_record;
4710
    ObArenaAllocator allocator(lib::ObLabel("DdlTaskTmp"));
4711
    ObCreateDDLTaskParam param(tenant_id,
4712
                                ObDDLType::DDL_DROP_DATABASE,
4713
                                nullptr,
4714
                                nullptr,
4715
                                database_id,
4716
                                schema_version,
4717
                                arg.parallelism_,
4718
                                arg.consumer_group_id_,
4719
                                &allocator,
4720
                                &arg,
4721
                                0 /* parent task id*/);
4722
    if (OB_FAIL(ddl_scheduler_.create_ddl_task(param, sql_proxy_, task_record))) {
4723
      LOG_WARN("submit ddl task failed", K(ret), K(arg));
4724
    } else if (OB_FAIL(ddl_scheduler_.schedule_ddl_task(task_record))) {
4725
      LOG_WARN("fail to schedule ddl task", K(ret), K(task_record));
4726
    } else {
4727
      drop_database_res.ddl_res_.tenant_id_ = tenant_id;
4728
      drop_database_res.ddl_res_.schema_id_ = database_id;
4729
      drop_database_res.ddl_res_.task_id_ = task_record.task_id_;
4730
    }
4731
  } else if (OB_FAIL(ddl_service_.drop_database(arg, drop_database_res))) {
4732
    LOG_WARN("ddl_service_ drop_database failed", K(arg), K(ret));
4733
  }
4734
  return ret;
4735
}
4736

4737

4738
int ObRootService::drop_tablegroup(const obrpc::ObDropTablegroupArg &arg)
4739
{
4740
  int ret = OB_SUCCESS;
4741
  if (!inited_) {
4742
    ret = OB_NOT_INIT;
4743
    LOG_WARN("not init", K(ret));
4744
  } else if (!arg.is_valid()) {
4745
    ret = OB_INVALID_ARGUMENT;
4746
    LOG_WARN("invalid arg", K(arg), K(ret));
4747
  } else if (OB_FAIL(ddl_service_.drop_tablegroup(arg))) {
4748
    LOG_WARN("ddl_service_ drop_tablegroup failed", K(arg), K(ret));
4749
  }
4750
  return ret;
4751
}
4752

4753
int ObRootService::alter_tablegroup(const obrpc::ObAlterTablegroupArg &arg)
4754
{
4755
  LOG_DEBUG("receive alter tablegroup arg", K(arg));
4756
  const ObTablegroupSchema *tablegroup_schema = NULL;
4757
  ObSchemaGetterGuard schema_guard;
4758
  uint64_t tablegroup_id = OB_INVALID_ID;
4759
  const uint64_t tenant_id = arg.tenant_id_;
4760
  int ret = OB_SUCCESS;
4761
  if (!inited_) {
4762
    ret = OB_NOT_INIT;
4763
    LOG_WARN("not init", K(ret));
4764
  } else if (!arg.is_valid()) {
4765
    ret = OB_INVALID_ARGUMENT;
4766
    LOG_WARN("invalid arg", K(arg), K(ret));
4767
  } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
4768
    LOG_WARN("get schema guard in inner table failed", K(ret));
4769
  } else if (OB_FAIL(schema_guard.get_tablegroup_id(tenant_id,
4770
                                                    arg.tablegroup_name_,
4771
                                                    tablegroup_id))) {
4772
    LOG_WARN("fail to get tablegroup id", K(ret));
4773
  } else if (OB_INVALID_ID == tablegroup_id) {
4774
    ret = OB_TABLEGROUP_NOT_EXIST;
4775
    LOG_WARN("get invalid tablegroup schema", KR(ret), K(arg));
4776
  } else if (OB_FAIL(schema_guard.get_tablegroup_schema(tenant_id, tablegroup_id, tablegroup_schema))) {
4777
    LOG_WARN("fail to get tablegroup schema", K(ret), K(tenant_id), K(ret));
4778
  } else if (OB_ISNULL(tablegroup_schema)) {
4779
    ret = OB_TABLEGROUP_NOT_EXIST;
4780
    LOG_WARN("get invalid tablegroup schema", K(ret));
4781
  } else if (tablegroup_schema->is_in_splitting()) {
4782
    ret = OB_OP_NOT_ALLOW;
4783
    LOG_WARN("tablegroup is splitting, refuse to alter now", K(ret), K(tablegroup_id));
4784
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tablegroup is splitting, alter tablegroup");
4785
  }
4786
  if (OB_FAIL(ret)) {
4787
  } else if (OB_FAIL(ddl_service_.alter_tablegroup(arg))) {
4788
    LOG_WARN("ddl_service_ alter tablegroup failed", K(arg), K(ret));
4789
  } else {
4790
  }
4791
  return ret;
4792
}
4793

4794
int ObRootService::drop_index(const obrpc::ObDropIndexArg &arg, obrpc::ObDropIndexRes &res)
4795
{
4796
  int ret = OB_SUCCESS;
4797
  if (!inited_) {
4798
    ret = OB_NOT_INIT;
4799
    LOG_WARN("not init", K(ret));
4800
  } else if (!arg.is_valid()) {
4801
    ret = OB_INVALID_ARGUMENT;
4802
    LOG_WARN("invalid arg", K(arg), K(ret));
4803
  } else {
4804
    ObIndexBuilder index_builder(ddl_service_);
4805
    if (OB_FAIL(index_builder.drop_index(arg, res))) {
4806
      LOG_WARN("index_builder drop_index failed", K(arg), K(ret));
4807
    }
4808
  }
4809
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "drop index",
4810
                        "tenant_id", res.tenant_id_,
4811
                        "ret", ret,
4812
                        "trace_id", *ObCurTraceId::get_trace_id(),
4813
                        "task_id", res.task_id_,
4814
                        "table_id", arg.index_table_id_,
4815
                        "schema_version", res.schema_version_);
4816
  LOG_INFO("finish drop index ddl", K(ret), K(arg), "ddl_event_info", ObDDLEventInfo());
4817
  return ret;
4818
}
4819

4820
int ObRootService::rebuild_index(const obrpc::ObRebuildIndexArg &arg, obrpc::ObAlterTableRes &res)
4821
{
4822
  int ret = OB_SUCCESS;
4823
  if (!inited_) {
4824
    ret = OB_NOT_INIT;
4825
    LOG_WARN("not init", K(ret));
4826
  } else if (!arg.is_valid()) {
4827
    ret = OB_INVALID_ARGUMENT;
4828
    LOG_WARN("invalid arg", K(arg), K(ret));
4829
  } else if (OB_FAIL(ddl_service_.rebuild_index(arg, res))) {
4830
    LOG_WARN("ddl_service rebuild index failed", K(arg), K(ret));
4831
  }
4832
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "rebuild index",
4833
                        "tenant_id", arg.tenant_id_,
4834
                        "ret", ret,
4835
                        "trace_id", *ObCurTraceId::get_trace_id(),
4836
                        "task_id", res.task_id_,
4837
                        "table_id", arg.index_table_id_,
4838
                        "schema_version", res.schema_version_);
4839
  LOG_INFO("finish rebuild index ddl", K(ret), K(arg), K(res), "ddl_event_info", ObDDLEventInfo());
4840
  return ret;
4841
}
4842

4843
int ObRootService::flashback_index(const ObFlashBackIndexArg &arg) {
4844
  int ret = OB_SUCCESS;
4845
  if (!inited_) {
4846
    ret = OB_NOT_INIT;
4847
    LOG_WARN("not init", K(ret));
4848
  } else if (!arg.is_valid()) {
4849
    ret = OB_INVALID_ARGUMENT;
4850
    LOG_WARN("invalid argument", K(arg), K(ret));
4851
  } else if (OB_FAIL(ddl_service_.flashback_index(arg))) {
4852
    LOG_WARN("failed to flashback index", K(ret));
4853
  }
4854

4855
  return ret;
4856
}
4857

4858
int ObRootService::purge_index(const ObPurgeIndexArg &arg)
4859
{
4860
  int ret = OB_SUCCESS;
4861
  if (!inited_) {
4862
    ret = OB_NOT_INIT;
4863
    LOG_WARN("not init", K(ret));
4864
  } else if (!arg.is_valid()) {
4865
    ret = OB_INVALID_ARGUMENT;
4866
    LOG_WARN("invalid argument", K(arg), K(ret));
4867
  } else if (OB_FAIL(ddl_service_.purge_index(arg))) {
4868
    LOG_WARN("failed to purge index", K(ret));
4869
  }
4870

4871
  return ret;
4872
}
4873

4874
int ObRootService::rename_table(const obrpc::ObRenameTableArg &arg)
4875
{
4876
  int ret = OB_SUCCESS;
4877
  if (!inited_) {
4878
    ret = OB_NOT_INIT;
4879
    LOG_WARN("not init", K(ret));
4880
  } else if (!arg.is_valid()) {
4881
    ret = OB_INVALID_ARGUMENT;
4882
    LOG_WARN("invalid arg", K(arg), K(ret));
4883
  } else if (OB_FAIL(ddl_service_.rename_table(arg))){
4884
    LOG_WARN("rename table failed", K(ret));
4885
  }
4886
  return ret;
4887
}
4888

4889
int ObRootService::truncate_table(const obrpc::ObTruncateTableArg &arg, obrpc::ObDDLRes &res)
4890
{
4891
  int ret = OB_SUCCESS;
4892
  if (!inited_) {
4893
    ret = OB_NOT_INIT;
4894
    LOG_WARN("not init", K(ret));
4895
  } else if (!arg.is_valid()) {
4896
    ret = OB_INVALID_ARGUMENT;
4897
    LOG_WARN("invalid arg", K(arg), K(ret));
4898
  } else {
4899
    SCN frozen_scn;
4900
    if (OB_FAIL(ObMajorFreezeHelper::get_frozen_scn(arg.tenant_id_, frozen_scn))) {
4901
      LOG_WARN("get_frozen_scn failed", K(ret));
4902
    } else if (arg.is_add_to_scheduler_) {
4903
      ObDDLTaskRecord task_record;
4904
      ObArenaAllocator allocator(lib::ObLabel("DdlTaskTmp"));
4905
      ObSchemaGetterGuard schema_guard;
4906
      const ObTableSchema *table_schema = nullptr;
4907
      const uint64_t tenant_id = arg.tenant_id_;
4908
      if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
4909
        LOG_WARN("get schema guard in inner table failed", K(ret));
4910
      } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, arg.database_name_,
4911
                                                       arg.table_name_, false /* is_index */,
4912
                                                       table_schema))) {
4913
        LOG_WARN("fail to get table schema", K(ret));
4914
      } else if (OB_ISNULL(table_schema)) {
4915
        ret = OB_TABLE_NOT_EXIST;
4916
        LOG_WARN("table not exist", K(ret), K(arg));
4917
      } else {
4918
        ObCreateDDLTaskParam param(tenant_id,
4919
                                   ObDDLType::DDL_TRUNCATE_TABLE,
4920
                                   nullptr,
4921
                                   nullptr,
4922
                                   table_schema->get_table_id(),
4923
                                   table_schema->get_schema_version(),
4924
                                   arg.parallelism_,
4925
                                   arg.consumer_group_id_,
4926
                                   &allocator,
4927
                                   &arg,
4928
                                   0 /* parent task id*/);
4929
        if (OB_FAIL(GCTX.root_service_->get_ddl_scheduler().create_ddl_task(param, sql_proxy_, task_record))) {
4930
          LOG_WARN("submit ddl task failed", K(ret), K(arg));
4931
        } else if (OB_FAIL(ddl_scheduler_.schedule_ddl_task(task_record))) {
4932
          LOG_WARN("fail to schedule ddl task", K(ret), K(task_record));
4933
        } else {
4934
          res.tenant_id_ = tenant_id;
4935
          res.schema_id_ = table_schema->get_table_id();
4936
          res.task_id_ = task_record.task_id_;
4937
        }
4938
      }
4939
    } else if (OB_FAIL(ddl_service_.truncate_table(arg, res, frozen_scn))) {
4940
      LOG_WARN("ddl service failed to truncate table", K(arg), K(ret), K(frozen_scn));
4941
    }
4942
  }
4943
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "truncate table",
4944
                        "tenant_id", arg.tenant_id_,
4945
                        "ret", ret,
4946
                        "trace_id", *ObCurTraceId::get_trace_id(),
4947
                        "task_id", res.task_id_,
4948
                        "table_id", arg.table_name_,
4949
                        "schema_version", res.schema_id_);
4950
  LOG_INFO("finish truncate table ddl", K(ret), K(arg), K(res), "ddl_event_info", ObDDLEventInfo());
4951
  return ret;
4952
}
4953

4954
/*
4955
 * new parallel truncate table
4956
 */
4957
int ObRootService::truncate_table_v2(const obrpc::ObTruncateTableArg &arg, obrpc::ObDDLRes &res)
4958
{
4959
  int ret = OB_SUCCESS;
4960
  if (!inited_) {
4961
    ret = OB_NOT_INIT;
4962
    LOG_WARN("not init", K(ret));
4963
  } else if (!arg.is_valid()) {
4964
    ret = OB_INVALID_ARGUMENT;
4965
    LOG_WARN("invalid arg", K(arg), K(ret));
4966
  } else {
4967
    SCN frozen_scn;
4968
    if (OB_FAIL(ObMajorFreezeHelper::get_frozen_scn(arg.tenant_id_, frozen_scn))) {
4969
      LOG_WARN("get_frozen_scn failed", K(ret));
4970
    } else if (OB_FAIL(ddl_service_.new_truncate_table(arg, res, frozen_scn))) {
4971
      LOG_WARN("ddl service failed to truncate table", K(arg), K(ret));
4972
    }
4973
    ROOTSERVICE_EVENT_ADD("ddl scheduler", "truncate table new",
4974
                          "tenant_id", arg.tenant_id_,
4975
                          "ret", ret,
4976
                          "trace_id", *ObCurTraceId::get_trace_id(),
4977
                          "task_id", res.task_id_,
4978
                          "table_name", arg.table_name_,
4979
                          "schema_version", res.schema_id_,
4980
                          frozen_scn);
4981
    LOG_INFO("finish new truncate table ddl", K(ret), K(arg), K(res), "ddl_event_info", ObDDLEventInfo());
4982
  }
4983
  return ret;
4984
}
4985

4986
int ObRootService::create_table_like(const ObCreateTableLikeArg &arg)
4987
{
4988
  int ret = OB_SUCCESS;
4989
  if (!inited_) {
4990
    ret = OB_NOT_INIT;
4991
    LOG_WARN("not init", K(ret));
4992
  } else if (!arg.is_valid()) {
4993
    ret = OB_INVALID_ARGUMENT;
4994
    LOG_WARN("invalid arg", K(arg), K(ret));
4995
  } else {
4996
    if (OB_FAIL(ddl_service_.create_table_like(arg))) {
4997
      if (OB_ERR_TABLE_EXIST == ret) {
4998
        //create table xx if not exist like
4999
        if (arg.if_not_exist_) {
5000
          LOG_USER_NOTE(OB_ERR_TABLE_EXIST,
5001
                        arg.new_table_name_.length(), arg.new_table_name_.ptr());
5002
          LOG_WARN("table is exist, no need to create again", K(arg), K(ret));
5003
          ret = OB_SUCCESS;
5004
        } else {
5005
          ret = OB_ERR_TABLE_EXIST;
5006
          LOG_USER_ERROR(OB_ERR_TABLE_EXIST, arg.new_table_name_.length(), arg.new_table_name_.ptr());
5007
          LOG_WARN("table is exist, cannot create it twice", K(arg), K(ret));
5008
        }
5009
      }
5010
    }
5011
  }
5012
  return ret;
5013
}
5014

5015
/**
5016
 * recyclebin related
5017
 */
5018
int ObRootService::flashback_table_from_recyclebin(const ObFlashBackTableFromRecyclebinArg &arg)
5019
{
5020
  int ret = OB_SUCCESS;
5021
  if (!inited_) {
5022
    ret = OB_NOT_INIT;
5023
    LOG_WARN("not init", K(ret));
5024
  } else if (!arg.is_valid()) {
5025
    ret = OB_INVALID_ARGUMENT;
5026
    LOG_WARN("invalid argument", K(arg), K(ret));
5027
  } else if (OB_FAIL(ddl_service_.flashback_table_from_recyclebin(arg))) {
5028
    LOG_WARN("failed to flash back table", K(ret));
5029
  }
5030
  return ret;
5031
}
5032

5033
int ObRootService::flashback_table_to_time_point(const obrpc::ObFlashBackTableToScnArg &arg)
5034
{
5035
  int ret = OB_SUCCESS;
5036
  LOG_INFO("receive flashback table arg", K(arg));
5037

5038
  if (!inited_) {
5039
    ret = OB_NOT_INIT;
5040
    LOG_WARN("not init", K(ret));
5041
  } else if (!arg.is_valid()) {
5042
    ret = OB_INVALID_ARGUMENT;
5043
    LOG_WARN("invalid argument", K(ret), K(arg));
5044
  } else if (OB_FAIL(ddl_service_.flashback_table_to_time_point(arg))) {
5045
    LOG_WARN("failed to flash back table", K(ret));
5046
  }
5047
  return ret;
5048
}
5049

5050
int ObRootService::purge_table(const ObPurgeTableArg &arg)
5051
{
5052
  int ret = OB_SUCCESS;
5053
  if (!inited_) {
5054
    ret = OB_NOT_INIT;
5055
    LOG_WARN("not init", K(ret));
5056
  } else if (!arg.is_valid()) {
5057
    ret = OB_INVALID_ARGUMENT;
5058
    LOG_WARN("invalid argument", K(arg), K(ret));
5059
  } else if (OB_FAIL(ddl_service_.purge_table(arg))) {
5060
    LOG_WARN("failed to purge table", K(ret));
5061
  }
5062
  return ret;
5063
}
5064

5065
int ObRootService::flashback_database(const ObFlashBackDatabaseArg &arg)
5066
{
5067
  int ret = OB_SUCCESS;
5068
  if (!inited_) {
5069
    ret = OB_NOT_INIT;
5070
    LOG_WARN("not init", K(ret));
5071
  } else if (!arg.is_valid()) {
5072
    ret = OB_INVALID_ARGUMENT;
5073
    LOG_WARN("invalid argument", K(arg), K(ret));
5074
  } else if (OB_FAIL(ddl_service_.flashback_database(arg))) {
5075
    LOG_WARN("failed to flash back database", K(ret));
5076
  }
5077
  return ret;
5078
}
5079

5080
int ObRootService::purge_database(const ObPurgeDatabaseArg &arg)
5081
{
5082
  int ret = OB_SUCCESS;
5083
  if (!inited_) {
5084
    ret = OB_NOT_INIT;
5085
    LOG_WARN("not init", K(ret));
5086
  } else if (!arg.is_valid()) {
5087
    ret = OB_INVALID_ARGUMENT;
5088
    LOG_WARN("invalid argument", K(arg), K(ret));
5089
  } else if (OB_FAIL(ddl_service_.purge_database(arg))) {
5090
    LOG_WARN("failed to purge database", K(ret));
5091
  }
5092
  return ret;
5093
}
5094

5095
int ObRootService::purge_expire_recycle_objects(const ObPurgeRecycleBinArg &arg, Int64 &affected_rows)
5096
{
5097
  int ret = OB_SUCCESS;
5098
  int64_t purged_objects = 0;
5099
  if (!inited_) {
5100
    ret = OB_NOT_INIT;
5101
    LOG_WARN("not init", K(ret));
5102
  } else if (OB_FAIL(ddl_service_.purge_tenant_expire_recycle_objects(arg, purged_objects))) {
5103
    LOG_WARN("failed to purge expire recyclebin objects", K(ret), K(arg));
5104
  } else {
5105
    affected_rows = purged_objects;
5106
  }
5107
  return ret;
5108
}
5109

5110
int ObRootService::optimize_table(const ObOptimizeTableArg &arg)
5111
{
5112
  int ret = OB_SUCCESS;
5113
  ObSchemaGetterGuard schema_guard;
5114
  LOG_INFO("receive optimize table request", K(arg));
5115
  lib::Worker::CompatMode mode;
5116
  if (!arg.is_valid()) {
5117
    ret = OB_INVALID_ARGUMENT;
5118
    LOG_WARN("invalid arguments", K(ret), K(arg));
5119
  } else if (OB_ISNULL(schema_service_)) {
5120
    ret = OB_ERR_UNEXPECTED;
5121
    LOG_WARN("error unexpected, schema service must not be NULL", K(ret));
5122
  } else if (OB_FAIL(ObCompatModeGetter::get_tenant_mode(arg.tenant_id_, mode))) {
5123
    LOG_WARN("fail to get tenant mode", K(ret));
5124
  } else {
5125
    const int64_t all_core_table_id = OB_ALL_CORE_TABLE_TID;
5126
    for (int64_t i = 0; OB_SUCC(ret) && i < arg.tables_.count(); ++i) {
5127
      SMART_VAR(obrpc::ObAlterTableArg, alter_table_arg) {
5128
        ObSqlString sql;
5129
        const obrpc::ObTableItem &table_item = arg.tables_.at(i);
5130
        const ObTableSchema *table_schema = nullptr;
5131
        alter_table_arg.is_alter_options_ = true;
5132
        alter_table_arg.alter_table_schema_.set_origin_database_name(table_item.database_name_);
5133
        alter_table_arg.alter_table_schema_.set_origin_table_name(table_item.table_name_);
5134
        alter_table_arg.alter_table_schema_.set_tenant_id(arg.tenant_id_);
5135
        alter_table_arg.skip_sys_table_check_ = true;
5136
        //exec_tenant_id_ is used in standby cluster
5137
        alter_table_arg.exec_tenant_id_ = arg.exec_tenant_id_;
5138
        if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(arg.tenant_id_, schema_guard))) {
5139
          LOG_WARN("fail to get tenant schema guard", K(ret));
5140
        } else if (OB_FAIL(schema_guard.get_table_schema(arg.tenant_id_, table_item.database_name_, table_item.table_name_, false/*is index*/, table_schema))) {
5141
          LOG_WARN("fail to get table schema", K(ret));
5142
        } else if (nullptr == table_schema) {
5143
          // skip deleted table
5144
        } else if (all_core_table_id == table_schema->get_table_id()) {
5145
          // do nothing
5146
        } else {
5147
          if (lib::Worker::CompatMode::MYSQL == mode) {
5148
            if (OB_FAIL(sql.append_fmt("OPTIMIZE TABLE `%.*s`",
5149
                table_item.table_name_.length(), table_item.table_name_.ptr()))) {
5150
              LOG_WARN("fail to assign sql stmt", K(ret));
5151
            }
5152
          } else if (lib::Worker::CompatMode::ORACLE == mode) {
5153
            if (OB_FAIL(sql.append_fmt("ALTER TABLE \"%.*s\" SHRINK SPACE",
5154
                table_item.table_name_.length(), table_item.table_name_.ptr()))) {
5155
              LOG_WARN("fail to append fmt", K(ret));
5156
            }
5157
          } else {
5158
            ret = OB_ERR_UNEXPECTED;
5159
            LOG_WARN("error unexpected, unknown mode", K(ret), K(mode));
5160
          }
5161
          if (OB_SUCC(ret)) {
5162
            alter_table_arg.ddl_stmt_str_ = sql.string();
5163
            obrpc::ObAlterTableRes res;
5164
            if (OB_FAIL(alter_table_arg.alter_table_schema_.alter_option_bitset_.add_member(ObAlterTableArg::PROGRESSIVE_MERGE_ROUND))) {
5165
              LOG_WARN("fail to add member", K(ret));
5166
            } else if (OB_FAIL(alter_table(alter_table_arg, res))) {
5167
              LOG_WARN("fail to alter table", K(ret), K(alter_table_arg));
5168
            }
5169
          }
5170
        }
5171
      }
5172
    }
5173
  }
5174
  return ret;
5175
}
5176

5177
int ObRootService::calc_column_checksum_repsonse(const obrpc::ObCalcColumnChecksumResponseArg &arg)
5178
{
5179
  int ret = OB_SUCCESS;
5180
  if (OB_UNLIKELY(!inited_)) {
5181
    ret = OB_NOT_INIT;
5182
    LOG_WARN("not inited", K(ret));
5183
  } else if (OB_UNLIKELY(!arg.is_valid())) {
5184
    ret = OB_INVALID_ARGUMENT;
5185
    LOG_WARN("invalid arguments", K(ret), K(arg));
5186
  } else if (OB_FAIL(DDL_SIM(arg.tenant_id_, arg.task_id_, PROCESS_COLUMN_CHECKSUM_RESPONSE_SLOW))) {
5187
    LOG_WARN("ddl sim failure: procesc column checksum response slow", K(ret));
5188
  } else if (OB_FAIL(ddl_scheduler_.on_column_checksum_calc_reply(
5189
          arg.tablet_id_, ObDDLTaskKey(arg.tenant_id_, arg.target_table_id_, arg.schema_version_), arg.ret_code_))) {
5190
    LOG_WARN("handle column checksum calc response failed", K(ret), K(arg));
5191
  }
5192
  return ret;
5193
}
5194

5195
int ObRootService::refresh_config()
5196
{
5197
  int ret = OB_SUCCESS;
5198
  int64_t local_config_version = 0;
5199
  if (!inited_) {
5200
    ret = OB_NOT_INIT;
5201
    LOG_WARN("not init", K(ret));
5202
  } else if (OB_FAIL(zone_manager_.get_config_version(local_config_version))) {
5203
    LOG_WARN("get_config_version failed", K(ret));
5204
  } else {
5205
    LOG_INFO("receive refresh config");
5206
    const int64_t now = ObTimeUtility::current_time();
5207
    const int64_t new_config_version = max(local_config_version + 1, now);
5208
    if (OB_FAIL(zone_manager_.update_config_version(new_config_version))) {
5209
      LOG_WARN("update_config_version failed", K(new_config_version), K(ret));
5210
    } else if (OB_FAIL(config_mgr_->got_version(new_config_version))) {
5211
      LOG_WARN("got_version failed", K(new_config_version), K(ret));
5212
    } else {
5213
      LOG_INFO("root service refresh_config succeed", K(new_config_version));
5214
    }
5215
  }
5216
  ROOTSERVICE_EVENT_ADD("root_service", "refresh_config", K(ret));
5217
  return ret;
5218
}
5219

5220
int ObRootService::root_minor_freeze(const ObRootMinorFreezeArg &arg)
5221
{
5222
  int ret = OB_SUCCESS;
5223
  LOG_INFO("receive minor freeze request", K(arg));
5224

5225
  if (!inited_) {
5226
    ret = OB_NOT_INIT;
5227
    LOG_WARN("not init", K(ret));
5228
  } else if (!arg.is_valid()) {
5229
    ret = OB_INVALID_ARGUMENT;
5230
    LOG_WARN("invalid arg", K(arg), K(ret));
5231
  } else if (OB_FAIL(root_minor_freeze_.try_minor_freeze(arg))) {
5232
    LOG_WARN("minor freeze failed", K(ret), K(arg));
5233
  }
5234
  ROOTSERVICE_EVENT_ADD("root_service", "root_minor_freeze", K(ret), K(arg));
5235
  return ret;
5236
}
5237

5238
int ObRootService::update_index_status(const obrpc::ObUpdateIndexStatusArg &arg)
5239
{
5240
  int ret = OB_SUCCESS;
5241
  if (!inited_) {
5242
    ret = OB_NOT_INIT;
5243
    LOG_WARN("not init", K(ret));
5244
  } else if (!arg.is_valid()) {
5245
    ret = OB_INVALID_ARGUMENT;
5246
    LOG_WARN("invalid arg", K(arg), K(ret));
5247
  } else if (OB_FAIL(ddl_service_.update_index_status(arg))) {
5248
    LOG_WARN("update index table status failed", K(ret), K(arg));
5249
  }
5250
  return ret;
5251
}
5252

5253
int ObRootService::update_mview_status(const obrpc::ObUpdateMViewStatusArg &arg)
5254
{
5255
  int ret = OB_SUCCESS;
5256
  const uint64_t tenant_id = arg.exec_tenant_id_;
5257
  uint64_t compat_version = 0;
5258
  if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compat_version))) {
5259
    LOG_WARN("fail to get data version", KR(ret), K(tenant_id));
5260
  } else if (compat_version < DATA_VERSION_4_3_0_0) {
5261
    ret = OB_NOT_SUPPORTED;
5262
    LOG_WARN("version lower than 4.3 does not support this operation", KR(ret));
5263
    LOG_USER_ERROR(OB_NOT_SUPPORTED, "tenant's data version is below 4.3.0.0, update mview status is ");
5264
  } else if (!inited_) {
5265
    ret = OB_NOT_INIT;
5266
    LOG_WARN("not init", K(ret));
5267
  } else if (!arg.is_valid()) {
5268
    ret = OB_INVALID_ARGUMENT;
5269
    LOG_WARN("invalid arg", KR(ret), K(arg));
5270
  } else if (OB_FAIL(ddl_service_.update_mview_status(arg))) {
5271
    LOG_WARN("update mview table status failed", KR(ret), K(arg));
5272
  }
5273
  return ret;
5274
}
5275

5276
int ObRootService::clone_tenant(const obrpc::ObCloneTenantArg &arg,
5277
                                obrpc::ObCloneTenantRes &res)
5278
{
5279
  int ret = OB_SUCCESS;
5280
  res.reset();
5281
  int64_t refreshed_schema_version = OB_INVALID_VERSION;
5282
  const ObString &clone_tenant_name = arg.get_new_tenant_name();
5283
  const ObString &source_tenant_name = arg.get_source_tenant_name();
5284
  const ObString &tenant_snapshot_name = arg.get_tenant_snapshot_name();
5285
  const bool is_fork_tenant = tenant_snapshot_name.empty();
5286
  uint64_t source_tenant_id = OB_INVALID_TENANT_ID;
5287
  int64_t job_id = OB_INVALID_ID;
5288
  ObTenantSnapItem tenant_snapshot_item;
5289
  bool is_unit_config_exist = false;
5290
  bool is_resource_pool_exist = false;
5291

5292
  if (!inited_) {
5293
    ret = OB_NOT_INIT;
5294
    LOG_WARN("not init", KR(ret));
5295
  } else if (!arg.is_valid()) {
5296
    ret = OB_INVALID_ARGUMENT;
5297
    LOG_WARN("invalid arg", KR(ret), K(arg));
5298
  } else if (OB_ISNULL(schema_service_)) {
5299
    ret = OB_ERR_UNEXPECTED;
5300
    LOG_WARN("unexpected schema_service_", KR(ret), KP(schema_service_));
5301
  } else if (GCTX.is_standby_cluster() || GCONF.in_upgrade_mode()) {
5302
    ret = OB_OP_NOT_ALLOW;
5303
    LOG_WARN("clone tenant while in standby cluster or in upgrade mode is not allowed", KR(ret));
5304
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "clone tenant while in standby cluster or in upgrade mode");
5305
  } else if (OB_FAIL(ObResolverUtils::check_not_supported_tenant_name(clone_tenant_name))) {
5306
    LOG_WARN("unsupported clone tenant name", KR(ret), K(clone_tenant_name));
5307
  } else {
5308
    ObSchemaGetterGuard schema_guard;
5309
    const ObTenantSchema *clone_tenant_schema = NULL;
5310
    const ObTenantSchema *source_tenant_schema = NULL;
5311
    if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(
5312
                      OB_SYS_TENANT_ID, schema_guard))) {
5313
      LOG_WARN("fail to get sys tenant's schema guard", KR(ret));
5314
    } else if (OB_FAIL(schema_guard.get_schema_version(OB_SYS_TENANT_ID, refreshed_schema_version))) {
5315
      LOG_WARN("fail to get sys schema version", KR(ret));
5316
    } else if (OB_FAIL(schema_guard.get_tenant_info(clone_tenant_name, clone_tenant_schema))) {
5317
      LOG_WARN("fail to get clone tenant schema", KR(ret), K(clone_tenant_name));
5318
    } else if (OB_NOT_NULL(clone_tenant_schema)) {
5319
      ret = OB_TENANT_EXIST;
5320
      LOG_WARN("clone tenant already exists", KR(ret), K(clone_tenant_name));
5321
      LOG_USER_ERROR(OB_TENANT_EXIST, to_cstring(clone_tenant_name));
5322
    } else if (OB_FAIL(schema_guard.get_tenant_info(source_tenant_name, source_tenant_schema))) {
5323
      LOG_WARN("fail to get source tenant info", KR(ret), K(source_tenant_name));
5324
    } else if (OB_ISNULL(source_tenant_schema)) {
5325
      ret = OB_TENANT_NOT_EXIST;
5326
      LOG_WARN("source tenant not exists", KR(ret), K(source_tenant_name));
5327
      LOG_USER_ERROR(OB_TENANT_NOT_EXIST, source_tenant_name.length(), source_tenant_name.ptr());
5328
    } else {
5329
      source_tenant_id = source_tenant_schema->get_tenant_id();
5330
    }
5331
  }
5332

5333
  if (FAILEDx(ObTenantSnapshotUtil::check_source_tenant_info(source_tenant_id,
5334
                                                      ObTenantSnapshotUtil::RESTORE_OP))) {
5335
    LOG_WARN("source tenant can not do cloning", KR(ret), K(source_tenant_id));
5336
  } else if (OB_FAIL(unit_manager_.check_unit_config_exist(arg.get_unit_config_name(), is_unit_config_exist))) {
5337
    LOG_WARN("fail to check unit config exist", KR(ret), K(arg));
5338
  } else if (!is_unit_config_exist) {
5339
    ret = OB_RESOURCE_UNIT_NOT_EXIST;
5340
    LOG_USER_ERROR(OB_RESOURCE_UNIT_NOT_EXIST, to_cstring(arg.get_unit_config_name()));
5341
    LOG_WARN("config not exist", KR(ret), K(arg));
5342
  } else if (OB_FAIL(unit_manager_.check_resource_pool_exist(arg.get_resource_pool_name(), is_resource_pool_exist))) {
5343
    LOG_WARN("fail to check resource pool exist", KR(ret), K(arg));
5344
  } else if (is_resource_pool_exist) {
5345
    ret = OB_RESOURCE_POOL_EXIST;
5346
    LOG_USER_ERROR(OB_RESOURCE_POOL_EXIST, to_cstring(arg.get_resource_pool_name()));
5347
    LOG_WARN("resource_pool already exist", "name", arg.get_resource_pool_name(), K(ret));
5348
  } else if (is_fork_tenant) { // fork tenant (clone tenant without snapshot)
5349
    // precheck
5350
    if (OB_FAIL(ObTenantSnapshotUtil::check_log_archive_ready(source_tenant_id, source_tenant_name))) {
5351
      LOG_WARN("check log archive ready failed", KR(ret), K(source_tenant_id), K(source_tenant_name));
5352
    }
5353
  } else { // !is_fork_tenant (clone tenant with snapshot)
5354
    if (OB_FAIL(ObTenantSnapshotUtil::get_tenant_snapshot_info(sql_proxy_, source_tenant_id,
5355
                                                tenant_snapshot_name, tenant_snapshot_item))) {
5356
      LOG_WARN("get tenant snapshot info failed", KR(ret), K(source_tenant_id), K(tenant_snapshot_name));
5357
      if (OB_TENANT_SNAPSHOT_NOT_EXIST == ret) {
5358
        ret = OB_OP_NOT_ALLOW;
5359
        LOG_USER_ERROR(OB_OP_NOT_ALLOW, "tenant snapshot not exist in source tenant, clone tenant");
5360
      }
5361
    }
5362
  }
5363

5364
  if (OB_SUCC(ret)) {
5365
    ObCloneJob clone_job;
5366
    bool has_job = false;
5367
    SCN gts_scn;
5368
    ObDDLSQLTransaction trans(schema_service_, false /*end_signal*/);
5369
    if (OB_FAIL(trans.start(&sql_proxy_, OB_SYS_TENANT_ID, refreshed_schema_version))) {
5370
      LOG_WARN("failed to start trans", KR(ret));
5371
    } else if (OB_FAIL(OB_TS_MGR.get_ts_sync(OB_SYS_TENANT_ID, GCONF.rpc_timeout, gts_scn))) {
5372
      LOG_WARN("fail to get ts sync", KR(ret));
5373
    } else if (FALSE_IT(job_id = gts_scn.get_val_for_tx())) {
5374
    } else if (OB_FAIL(ObTenantCloneUtil::fill_clone_job(job_id, arg, source_tenant_id, source_tenant_name,
5375
                                                tenant_snapshot_item, clone_job))) {
5376
      LOG_WARN("fail to fill clone job", KR(ret), K(job_id), K(arg), K(source_tenant_id), K(tenant_snapshot_item));
5377
    } else if (OB_FAIL(ObTenantCloneUtil::record_clone_job(trans, clone_job))) {
5378
      LOG_WARN("fail to record clone job", KR(ret), K(clone_job));
5379
    } else {
5380
      res.set_job_id(job_id);
5381
    }
5382

5383
    if (trans.is_started()) {
5384
      int tmp_ret = OB_SUCCESS;
5385
      if (OB_TMP_FAIL(trans.end(OB_SUCC(ret)))) {
5386
        LOG_WARN("trans end failed", "is_commit", OB_SUCCESS == ret, KR(tmp_ret), KR(ret));
5387
        ret = (OB_SUCC(ret)) ? tmp_ret : ret;
5388
      }
5389
    }
5390
  }
5391

5392
  if (OB_SUCC(ret)) {
5393
    int tmp_ret = OB_SUCCESS;
5394
    if (OB_TMP_FAIL(ObTenantCloneUtil::notify_clone_scheduler(OB_SYS_TENANT_ID))) {
5395
      LOG_WARN("notify clone scheduler failed", KR(tmp_ret));
5396
    }
5397
  }
5398

5399
  LOG_INFO("[RESTORE] clone tenant start", KR(ret), K(arg));
5400
  ROOTSERVICE_EVENT_ADD("clone", "clone_start",
5401
                        "job_id", job_id,
5402
                        K(ret),
5403
                        "clone_tenant_name", clone_tenant_name,
5404
                        "source_tenant_name", source_tenant_name);
5405

5406
  if (OB_SUCC(ret)) {
5407
    const char *status_str = ObTenantCloneStatus::get_clone_status_str(
5408
                                      ObTenantCloneStatus::Status::CLONE_SYS_LOCK);
5409
    ROOTSERVICE_EVENT_ADD("clone", "change_clone_status",
5410
                          "job_id", job_id,
5411
                          K(ret),
5412
                          "prev_clone_status", "NULL",
5413
                          "cur_clone_status", status_str);
5414
  }
5415
  return ret;
5416
}
5417

5418
int ObRootService::init_debug_database()
5419
{
5420
  const schema_create_func *creator_ptr_array[] = {
5421
    core_table_schema_creators,
5422
    sys_table_schema_creators,
5423
    NULL};
5424

5425
  int ret = OB_SUCCESS;
5426
  HEAP_VAR(char[OB_MAX_SQL_LENGTH], sql) {
5427
    if (!inited_) {
5428
      ret = OB_NOT_INIT;
5429
      LOG_WARN("not init", K(ret));
5430
    }
5431

5432
    ObTableSchema table_schema;
5433
    ObSqlString create_func_sql;
5434
    ObSqlString del_sql;
5435
    for (const schema_create_func **creator_ptr_ptr = creator_ptr_array;
5436
         OB_SUCCESS == ret && NULL != *creator_ptr_ptr; ++creator_ptr_ptr) {
5437
      for (const schema_create_func *creator_ptr = *creator_ptr_ptr;
5438
           OB_SUCCESS == ret && NULL != *creator_ptr; ++creator_ptr) {
5439
        table_schema.reset();
5440
        create_func_sql.reset();
5441
        del_sql.reset();
5442
        if (OB_FAIL((*creator_ptr)(table_schema))) {
5443
          LOG_WARN("create table schema failed", K(ret));
5444
          ret = OB_SCHEMA_ERROR;
5445
        } else {
5446
          int64_t affected_rows = 0;
5447
          // ignore create function result
5448
          int temp_ret = OB_SUCCESS;
5449
          if (OB_SUCCESS != (temp_ret = create_func_sql.assign(
5450
                      "create function time_to_usec(t timestamp) "
5451
                      "returns bigint(20) deterministic begin return unix_timestamp(t); end;"))) {
5452
            LOG_WARN("create_func_sql assign failed", K(temp_ret));
5453
          } else if (OB_SUCCESS != (temp_ret = sql_proxy_.write(
5454
                      create_func_sql.ptr(), affected_rows))) {
5455
            LOG_WARN("execute sql failed", K(create_func_sql), K(temp_ret));
5456
          } else if (OB_SUCCESS != (temp_ret = create_func_sql.assign(
5457
                      "create function usec_to_time(u bigint(20)) "
5458
                      "returns timestamp deterministic begin return from_unixtime(u); end;"))) {
5459
            LOG_WARN("create_func_sql assign failed", K(temp_ret));
5460
          } else if (OB_SUCCESS != (temp_ret = sql_proxy_.write(
5461
                      create_func_sql.ptr(), affected_rows))) {
5462
            LOG_WARN("execute sql failed", K(create_func_sql), K(temp_ret));
5463
          }
5464

5465
          memset(sql, 0, sizeof(sql));
5466
          if (OB_FAIL(del_sql.assign_fmt(
5467
                      "DROP table IF EXISTS %s", table_schema.get_table_name()))) {
5468
            LOG_WARN("assign sql failed", K(ret));
5469
          } else if (OB_FAIL(sql_proxy_.write(del_sql.ptr(), affected_rows))) {
5470
            LOG_WARN("execute sql failed", K(ret));
5471
          } else if (OB_FAIL(ObSchema2DDLSql::convert(
5472
                      table_schema, sql, sizeof(sql)))) {
5473
            LOG_WARN("convert table schema to create table sql failed", K(ret));
5474
          } else if (OB_FAIL(sql_proxy_.write(sql, affected_rows))) {
5475
            LOG_WARN("execute sql failed", K(ret), K(sql));
5476
          }
5477
        }
5478
      }
5479
    }
5480

5481
    LOG_INFO("init debug database finish.", K(ret));
5482
  }
5483
  return ret;
5484
}
5485

5486
int ObRootService::do_restart()
5487
{
5488
  int ret = OB_SUCCESS;
5489

5490
  const int64_t tenant_id = OB_SYS_TENANT_ID;
5491
  SpinWLockGuard rs_list_guard(broadcast_rs_list_lock_);
5492

5493
  // NOTE: following log print after lock
5494
  FLOG_INFO("[ROOTSERVICE_NOTICE] start do_restart");
5495

5496
  if (!inited_) {
5497
    ret = OB_NOT_INIT;
5498
    FLOG_WARN("not init", KR(ret));
5499
  } else if (!ObRootServiceRoleChecker::is_rootserver()) {
5500
    ret = OB_NOT_MASTER;
5501
    FLOG_WARN("not master", KR(ret));
5502
  }
5503

5504
  // renew master rootservice, ignore error
5505
  if (OB_SUCC(ret)) {
5506
    int tmp_ret = rs_mgr_->renew_master_rootserver();
5507
    if (OB_SUCCESS != tmp_ret) {
5508
      FLOG_WARN("renew master rootservice failed", KR(tmp_ret));
5509
    }
5510
  }
5511

5512
  //fetch root partition info
5513
  if (FAILEDx(fetch_sys_tenant_ls_info())) {
5514
    FLOG_WARN("fetch root partition info failed", KR(ret));
5515
  } else {
5516
    FLOG_INFO("fetch root partition info succeed", KR(ret));
5517
  }
5518

5519
  // broadcast root server address, ignore error
5520
  // not full service, can not update rs_list success
5521
  //if (OB_SUCC(ret)) {
5522
  //  int tmp_ret = update_rslist();
5523
  //  if (OB_SUCCESS != tmp_ret) {
5524
  //    FLOG_WARN("failed to update rslist but ignored", KR(tmp_ret));
5525
  //  }
5526
  //}
5527
  if (FAILEDx(submit_update_rslist_task(true))) {
5528
    FLOG_WARN("submit_update_rslist_task failed", KR(ret));
5529
  } else {
5530
    FLOG_INFO("submit_update_rslist_task succeed");
5531
  }
5532

5533
  if (OB_SUCC(ret)) {
5534
    //standby cluster trigger load_refresh_schema_status by heartbeat.
5535
    //due to switchover, primary cluster need to load schema_status too.
5536
    ObSchemaStatusProxy *schema_status_proxy = GCTX.schema_status_proxy_;
5537
    if (OB_ISNULL(schema_status_proxy)) {
5538
      ret = OB_ERR_UNEXPECTED;
5539
      FLOG_WARN("schema_status_proxy is null", KR(ret));
5540
    } else if (OB_FAIL(schema_status_proxy->load_refresh_schema_status())) {
5541
      FLOG_WARN("fail to load refresh schema status", KR(ret));
5542
    } else {
5543
      FLOG_INFO("load schema status success");
5544
    }
5545
  }
5546

5547
  bool load_frozen_status = true;
5548
  const bool refresh_server_need_retry = false; // no need retry
5549
  // try fast recover
5550
  if (OB_SUCC(ret)) {
5551
    int tmp_ret = refresh_server(load_frozen_status, refresh_server_need_retry);
5552
    if (OB_SUCCESS != tmp_ret) {
5553
      FLOG_WARN("refresh server failed", KR(tmp_ret), K(load_frozen_status));
5554
    }
5555
    tmp_ret = refresh_schema(load_frozen_status);
5556
    if (OB_SUCCESS != tmp_ret) {
5557
      FLOG_WARN("refresh schema failed", KR(tmp_ret), K(load_frozen_status));
5558
    }
5559
  }
5560
  load_frozen_status = false;
5561
  // refresh schema
5562
  if (FAILEDx(refresh_schema(load_frozen_status))) {
5563
    FLOG_WARN("refresh schema failed", KR(ret), K(load_frozen_status));
5564
  } else {
5565
    FLOG_INFO("success to refresh schema", K(load_frozen_status));
5566
  }
5567

5568
  // refresh server manager
5569
  if (FAILEDx(refresh_server(load_frozen_status, refresh_server_need_retry))) {
5570
    FLOG_WARN("refresh server failed", KR(ret), K(load_frozen_status));
5571
  } else {
5572
    FLOG_INFO("success to refresh server", K(load_frozen_status));
5573
  }
5574

5575
  // add other reload logic here
5576
  if (FAILEDx(zone_manager_.reload())) {
5577
    FLOG_WARN("zone_manager_ reload failed", KR(ret));
5578
  } else {
5579
    FLOG_INFO("success to reload zone_manager_");
5580
  }
5581

5582
  // start timer tasks
5583
  if (FAILEDx(start_timer_tasks())) {
5584
    FLOG_WARN("start timer tasks failed", KR(ret));
5585
  } else {
5586
    FLOG_INFO("success to start timer tasks");
5587
  }
5588

5589
  DEBUG_SYNC(BEFORE_UNIT_MANAGER_LOAD);
5590
  if (FAILEDx(unit_manager_.load())) {
5591
    FLOG_WARN("unit_manager_ load failed", KR(ret));
5592
  } else {
5593
    FLOG_INFO("load unit_manager success");
5594
  }
5595

5596
  /*
5597
   * FIXME: wanhong.wwh: need re-implement
5598
  if (OB_SUCC(ret)) {
5599
    if (OB_FAIL(set_core_table_unit_id())) {
5600
      LOG_WARN("START_SERVICE: set core table partition unit failed", K(ret));
5601
    } else {
5602
      ObTaskController::get().allow_next_syslog();
5603
      LOG_INFO("START_SERVICE: set core table unit id success");
5604
    }
5605
  }
5606
  */
5607

5608
  if (FAILEDx(schema_history_recycler_.start())) {
5609
    FLOG_WARN("schema_history_recycler start failed", KR(ret));
5610
  } else {
5611
    FLOG_INFO("success to start schema_history_recycler");
5612
  }
5613

5614
  if (FAILEDx(root_balancer_.start())) {
5615
    FLOG_WARN("root balancer start failed", KR(ret));
5616
  } else {
5617
    FLOG_INFO("success to start root balancer");
5618
  }
5619

5620
  if (FAILEDx(thread_checker_.start())) {
5621
    FLOG_WARN("rs_monitor_check: start thread checker failed", KR(ret));
5622
  } else {
5623
    FLOG_INFO("success to start thread checker");
5624
  }
5625

5626
  if (FAILEDx(empty_server_checker_.start())) {
5627
    FLOG_WARN("start empty server checker failed", KR(ret));
5628
  } else {
5629
    FLOG_INFO("success to start empty server checker");
5630
  }
5631

5632
  if (FAILEDx(lost_replica_checker_.start())) {
5633
    FLOG_WARN("start lost replica checker failed", KR(ret));
5634
  } else {
5635
    FLOG_INFO("start lost replica checker success");
5636
  }
5637

5638
  // broadcast root server address again, this task must be in the end part of do_restart,
5639
  // because system may work properly without it.
5640
  //if (FAILEDx(update_rslist())) {
5641
  //  FLOG_WARN("broadcast root address failed but ignored", KR(ret));
5642
    // it's ok ret be overwritten, update_rslist_task will retry until succeed
5643
  if (FAILEDx(submit_update_rslist_task(true))) {
5644
    FLOG_WARN("submit_update_rslist_task failed", KR(ret));
5645
  } else {
5646
    FLOG_INFO("submit_update_rslist_task succeed");
5647
  }
5648
  //} else {
5649
  //  FLOG_INFO("broadcast root address succeed");
5650
  //}
5651

5652
  if (FAILEDx(report_single_replica(tenant_id, SYS_LS))) {
5653
    FLOG_WARN("report all_core_table replica failed, but ignore",
5654
              KR(ret), K(tenant_id), K(SYS_LS));
5655
    // it's ok ret be overwritten, report single all_core will retry until succeed
5656
    if (OB_FAIL(submit_report_core_table_replica_task())) {
5657
      FLOG_WARN("submit all core table replica task failed", KR(ret));
5658
    } else {
5659
      FLOG_INFO("submit all core table replica task succeed");
5660
    }
5661
  } else {
5662
    FLOG_INFO("report all_core_table replica finish");
5663
  }
5664

5665
  if (FAILEDx(ddl_scheduler_.start())) {
5666
    FLOG_WARN("fail to start ddl task scheduler", KR(ret));
5667
  } else {
5668
    FLOG_INFO("success to start ddl task scheduler");
5669
  }
5670

5671
  if (FAILEDx(dbms_job::ObDBMSJobMaster::get_instance().start())) {
5672
    FLOG_WARN("failed to start dbms job master", KR(ret));
5673
  } else {
5674
    FLOG_INFO("success to start dbms job master");
5675
  }
5676

5677
  if (FAILEDx(dbms_scheduler::ObDBMSSchedJobMaster::get_instance().start())) {
5678
    FLOG_WARN("failed to start dbms sched job master", KR(ret));
5679
  } else {
5680
    FLOG_INFO("success to start dbms sched job mstart");
5681
  }
5682

5683
  if (OB_SUCC(ret)) {
5684
    upgrade_executor_.start();
5685
    FLOG_INFO("success to start upgrade_executor_", KR(ret));
5686
    upgrade_storage_format_executor_.start();
5687
    FLOG_INFO("success to start upgrade_storage_format_executor_", KR(ret));
5688
    create_inner_schema_executor_.start();
5689
    FLOG_INFO("success to start create_inner_schema_executor_", KR(ret));
5690
  }
5691

5692
  // to avoid increase rootservice_epoch while fail to restart RS,
5693
  // put it and the end of restart RS.
5694
  if (FAILEDx(init_sequence_id())) {
5695
    FLOG_WARN("fail to init sequence id", KR(ret));
5696
  } else {
5697
    FLOG_INFO("success to init sequenxe id");
5698
  }
5699

5700

5701
#ifdef OB_BUILD_TDE_SECURITY
5702
  if (FAILEDx(master_key_mgr_.start())) {
5703
    FLOG_WARN("fail to start master key manager", KR(ret));
5704
  } else {
5705
    FLOG_INFO("success to start master key manager");
5706
  }
5707
#endif
5708

5709
  if (FAILEDx(disaster_recovery_task_mgr_.start())) {
5710
    FLOG_WARN("disaster recovery task manager start failed", KR(ret));
5711
  } else {
5712
    FLOG_INFO("success to start disaster recovery task manager");
5713
  }
5714

5715
  if (FAILEDx(rs_status_.set_rs_status(status::FULL_SERVICE))) {
5716
    FLOG_WARN("fail to set rs status", KR(ret));
5717
  } else {
5718
    FLOG_INFO("full_service !!! start to work!!");
5719
    ROOTSERVICE_EVENT_ADD("root_service", "full_rootservice",
5720
                          "result", ret, K_(self_addr));
5721
    root_balancer_.set_active();
5722
    root_minor_freeze_.start();
5723
    FLOG_INFO("root_minor_freeze_ started");
5724
    root_inspection_.start();
5725
    FLOG_INFO("root_inspection_ started");
5726
    int64_t now = ObTimeUtility::current_time();
5727
    core_meta_table_version_ = now;
5728
    EVENT_SET(RS_START_SERVICE_TIME, now);
5729
    // reset fail count for self checker and print log.
5730
    reset_fail_count();
5731
  }
5732

5733
  if (OB_FAIL(ret)) {
5734
    update_fail_count(ret);
5735
  }
5736

5737
  FLOG_INFO("[ROOTSERVICE_NOTICE] finish do_restart", KR(ret));
5738
  return ret;
5739
}
5740

5741
bool ObRootService::in_service() const
5742
{
5743
  return rs_status_.in_service();
5744
}
5745

5746
bool ObRootService::is_full_service() const
5747
{
5748
  return rs_status_.is_full_service();
5749
}
5750

5751
bool ObRootService::is_start() const
5752
{
5753
  return rs_status_.is_start();
5754
}
5755

5756
bool ObRootService::is_stopping() const
5757
{
5758
  return rs_status_.is_stopping();
5759
}
5760

5761
bool ObRootService::is_need_stop() const
5762
{
5763
  return rs_status_.is_need_stop();
5764
}
5765

5766
bool ObRootService::can_start_service() const
5767
{
5768
  return rs_status_.can_start_service();
5769
}
5770

5771
int ObRootService::set_rs_status(const status::ObRootServiceStatus status)
5772
{
5773
  return rs_status_.set_rs_status(status);
5774
}
5775

5776
bool ObRootService::need_do_restart() const
5777
{
5778
  return rs_status_.need_do_restart();
5779
}
5780

5781
int ObRootService::revoke_rs()
5782
{
5783
  return rs_status_.revoke_rs();
5784
}
5785
int ObRootService::check_parallel_ddl_conflict(
5786
    share::schema::ObSchemaGetterGuard &schema_guard,
5787
    const obrpc::ObDDLArg &arg)
5788
{
5789
  int ret = OB_SUCCESS;
5790
  int64_t schema_version = OB_INVALID_VERSION;
5791

5792
  if (arg.is_need_check_based_schema_objects()) {
5793
    for (int64_t i = 0; OB_SUCC(ret) && (i < arg.based_schema_object_infos_.count()); ++i) {
5794
      const ObBasedSchemaObjectInfo &info = arg.based_schema_object_infos_.at(i);
5795
      if (OB_FAIL(schema_guard.get_schema_version(
5796
          info.schema_type_,
5797
          info.schema_tenant_id_ == OB_INVALID_TENANT_ID ? arg.exec_tenant_id_: info.schema_tenant_id_,
5798
          info.schema_id_,
5799
          schema_version))) {
5800
        LOG_WARN("failed to get_schema_version", K(ret), K(arg.exec_tenant_id_), K(info));
5801
      } else if (OB_INVALID_VERSION == schema_version) {
5802
        ret = OB_ERR_PARALLEL_DDL_CONFLICT;
5803
        LOG_WARN("schema_version is OB_INVALID_VERSION", K(ret), K(info));
5804
      } else if (schema_version != info.schema_version_) {
5805
        ret = OB_ERR_PARALLEL_DDL_CONFLICT;
5806
        LOG_WARN("schema_version is not equal to info.schema_version_", K(ret), K(schema_version), K(info));
5807
      }
5808
    }
5809
  }
5810

5811
  return ret;
5812
}
5813

5814
int ObRootService::init_sequence_id()
5815
{
5816
  int ret = OB_SUCCESS;
5817
  ObMySQLTransaction trans;
5818
  if (OB_UNLIKELY(!inited_)) {
5819
    ret = OB_NOT_INIT;
5820
    LOG_WARN("not init", K(ret));
5821
  } else if (OB_ISNULL(schema_service_)) {
5822
    ret = OB_ERR_UNEXPECTED;
5823
    LOG_WARN("schema_service is null", K(ret), K(schema_service_));
5824
  } else if (OB_FAIL(trans.start(&sql_proxy_, OB_SYS_TENANT_ID))) {
5825
    LOG_WARN("trans start failed", K(ret));
5826
  } else {
5827
    ObGlobalStatProxy proxy(trans, OB_SYS_TENANT_ID);
5828
    ObSchemaService *schema_service = schema_service_->get_schema_service();
5829
    int64_t rootservice_epoch = 0;
5830
    int64_t schema_version = OB_INVALID_VERSION;
5831
    ObRefreshSchemaInfo schema_info;
5832
    //increase sequence_id can trigger every observer refresh schema while restart RS
5833
    if (OB_FAIL(proxy.inc_rootservice_epoch())) {
5834
      LOG_WARN("fail to increase rootservice_epoch", K(ret));
5835
    } else if (OB_FAIL(proxy.get_rootservice_epoch(rootservice_epoch))) {
5836
      LOG_WARN("fail to get rootservice start times", K(ret), K(rootservice_epoch));
5837
    } else if (rootservice_epoch <= 0) {
5838
      ret = OB_ERR_UNEXPECTED;
5839
      LOG_WARN("invalid rootservice_epoch", K(ret), K(rootservice_epoch));
5840
    } else if (OB_FAIL(schema_service->init_sequence_id(rootservice_epoch))) {
5841
      LOG_WARN("init sequence id failed", K(ret), K(rootservice_epoch));
5842
    } else if (OB_FAIL(schema_service_->get_tenant_refreshed_schema_version(OB_SYS_TENANT_ID, schema_version))) {
5843
      LOG_WARN("fail to get sys tenant refreshed schema version", K(ret));
5844
    } else if (schema_version <= OB_CORE_SCHEMA_VERSION + 1) {
5845
      // in bootstrap and new schema mode, to avoid write failure while schema_version change,
5846
      // only actively refresh schema at the end of bootstrap, and make heartbeat'srefresh_schema_info effective.
5847
    } else if (OB_FAIL(schema_service->set_refresh_schema_info(schema_info))) {
5848
      LOG_WARN("fail to set refresh schema info", K(ret), K(schema_info));
5849
    }
5850

5851
    int temp_ret = OB_SUCCESS;
5852
    if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCCESS == ret))) {
5853
      LOG_WARN("trans end failed", "commit", OB_SUCCESS == ret, K(temp_ret));
5854
      ret = (OB_SUCCESS == ret) ? temp_ret : ret;
5855
    }
5856
  }
5857
  return ret;
5858
}
5859

5860
int ObRootService::load_server_manager()
5861
{
5862
  int ret = OB_SUCCESS;
5863
  if (OB_UNLIKELY(!inited_)) {
5864
    ret = OB_NOT_INIT;
5865
    LOG_WARN("not init", K(ret));
5866
  } else if (OB_FAIL(server_manager_.load_server_manager())) {
5867
    LOG_WARN("fail to load server manager", K(ret));
5868
  } else {} // no more to do
5869
  return ret;
5870
}
5871

5872
ERRSIM_POINT_DEF(ERROR_EVENT_TABLE_CLEAR_INTERVAL);
5873
int ObRootService::start_timer_tasks()
5874
{
5875
  int ret = OB_SUCCESS;
5876
  if (!inited_) {
5877
    ret = OB_NOT_INIT;
5878
    LOG_WARN("not init", K(ret));
5879
  }
5880

5881
  if (OB_SUCCESS == ret && !task_queue_.exist_timer_task(event_table_clear_task_)) {
5882
    const int64_t delay = ERROR_EVENT_TABLE_CLEAR_INTERVAL ? 10 * 1000 * 1000 :
5883
      ObEventHistoryTableOperator::EVENT_TABLE_CLEAR_INTERVAL;
5884
    if (OB_FAIL(task_queue_.add_repeat_timer_task_schedule_immediately(event_table_clear_task_, delay))) {
5885
      LOG_WARN("start event table clear task failed", K(delay), K(ret));
5886
    } else {
5887
      LOG_INFO("added event_table_clear_task", K(delay));
5888
    }
5889
  }
5890

5891
  if (OB_SUCC(ret) && !task_queue_.exist_timer_task(update_rs_list_timer_task_)) {
5892
    if (OB_FAIL(schedule_update_rs_list_task())) {
5893
      LOG_WARN("failed to schedule update rs list task", K(ret));
5894
    } else {
5895
      LOG_INFO("add update rs list timer task");
5896
    }
5897
  }
5898

5899
  if (OB_SUCC(ret) && !task_queue_.exist_timer_task(update_all_server_config_task_)) {
5900
    if (OB_FAIL(schedule_update_all_server_config_task())) {
5901
      LOG_WARN("fail to schedule update_all_server_config_task", KR(ret));
5902
    } else {
5903
      LOG_INFO("add update_all_server_config_task");
5904
    }
5905
  }
5906

5907
  if (OB_SUCC(ret)) {
5908
    if (OB_FAIL(schedule_inspector_task())) {
5909
      LOG_WARN("start inspector fail", K(ret));
5910
    } else {
5911
      LOG_INFO("start inspector success");
5912
    }
5913
  }
5914

5915
  if (OB_SUCC(ret) && !task_queue_.exist_timer_task(check_server_task_)) {
5916
    if (OB_FAIL(schedule_check_server_timer_task())) {
5917
      LOG_WARN("start all server checker fail", K(ret));
5918
    } else {
5919
      LOG_INFO("start all server checker success");
5920
    }
5921
  }
5922

5923
  if (OB_SUCC(ret)) {
5924
    if (OB_FAIL(schedule_load_ddl_task())) {
5925
      LOG_WARN("schedule load ddl task failed", K(ret));
5926
    }
5927
  }
5928

5929
  if (OB_SUCC(ret)) {
5930
    if (OB_FAIL(schedule_refresh_io_calibration_task())) {
5931
      LOG_WARN("schedule refresh io calibration task failed", K(ret));
5932
    }
5933
  }
5934

5935
  LOG_INFO("start all timer tasks finish", K(ret));
5936
  return ret;
5937
}
5938

5939
int ObRootService::stop_timer_tasks()
5940
{
5941
  int ret = OB_SUCCESS;
5942
  if (!inited_) {
5943
    ret = OB_NOT_INIT;
5944
    LOG_WARN("not init", K(ret));
5945
  } else {
5946
    task_queue_.cancel_timer_task(restart_task_);
5947
    task_queue_.cancel_timer_task(check_server_task_);
5948
    task_queue_.cancel_timer_task(event_table_clear_task_);
5949
    task_queue_.cancel_timer_task(self_check_task_);
5950
    task_queue_.cancel_timer_task(update_rs_list_timer_task_);
5951
    task_queue_.cancel_timer_task(update_all_server_config_task_);
5952
    inspect_task_queue_.cancel_timer_task(inspector_task_);
5953
    inspect_task_queue_.cancel_timer_task(purge_recyclebin_task_);
5954
  }
5955

5956
  //stop other timer tasks here
5957
  LOG_INFO("stop all timer tasks finish", K(ret));
5958
  return ret;
5959
}
5960

5961
int ObRootService::fetch_sys_tenant_ls_info()
5962
{
5963
  int ret = OB_SUCCESS;
5964
  ObLSReplica replica;
5965
  ObRole role = FOLLOWER;
5966
  bool inner_table_only = false;
5967
  ObMemberList member_list;
5968
  // TODO: automatically decide to use rpc or inmemory
5969
  ObLSTable* inmemory_ls;
5970
  if (OB_UNLIKELY(!inited_)) {
5971
    ret = OB_NOT_INIT;
5972
    LOG_WARN("not init", KR(ret));
5973
  } else if (OB_ISNULL(GCTX.ob_service_) || OB_ISNULL(GCTX.lst_operator_)) {
5974
    ret = OB_ERR_UNEXPECTED;
5975
    LOG_WARN("ob_service is null", KR(ret));
5976
  } else {
5977
    inmemory_ls = GCTX.lst_operator_->get_inmemory_ls();
5978
    if (OB_ISNULL(inmemory_ls)) {
5979
      ret = OB_ERR_UNEXPECTED;
5980
      LOG_WARN("inmemory ls_table is null", KR(ret), KP(inmemory_ls));
5981
    } else if (OB_FAIL(inmemory_ls->get_role(
5982
          OB_SYS_TENANT_ID,
5983
          SYS_LS,
5984
          role))) {
5985
      LOG_WARN("get role from ObLS failed", KR(ret));
5986
    } else if (OB_FAIL(inmemory_ls->get_member_list(
5987
          OB_SYS_TENANT_ID,
5988
          SYS_LS,
5989
          member_list))) {
5990
      LOG_WARN("get sys_tenant ls member list failed", KR(ret));
5991
    } else if (!is_strong_leader(role)) {
5992
      ret = OB_ENTRY_NOT_EXIST;
5993
      LOG_WARN("local role should be leader now", KR(ret), K(role));
5994
    } else if (OB_FAIL(GCTX.ob_service_->fill_ls_replica(
5995
          OB_SYS_TENANT_ID,
5996
          SYS_LS,
5997
          replica))) {
5998
      LOG_WARN("fail to fill log stream replica", KR(ret), K(replica));
5999
    } else if (OB_FAIL(inmemory_ls->update(replica, inner_table_only))) {
6000
      LOG_WARN("fail to update ls replica", KR(ret), K(replica));
6001
    }
6002
  }
6003

6004
  for (int64_t i = 0; OB_SUCC(ret) && i < member_list.get_member_number(); i++) {
6005
    ObAddr addr;
6006
    if (OB_FAIL(member_list.get_server_by_index(i, addr))) {
6007
      LOG_WARN("fail to get server", KR(ret), K(i), K(member_list));
6008
    } else if (self_addr_ == addr){
6009
      continue;
6010
    } else {
6011
      replica.reset();
6012
      int temp_ret = OB_SUCCESS;
6013
      if (OB_SUCCESS != (temp_ret = rpc_proxy_.to(addr).timeout(config_->rpc_timeout)
6014
                         .by(OB_SYS_TENANT_ID).fetch_sys_ls(replica))) {
6015
        LOG_WARN("fetch sys_ls failed", K(temp_ret), "server", addr);
6016
      } else if (replica.is_strong_leader()) {
6017
        ret = OB_ENTRY_EXIST;
6018
        LOG_WARN("role should be follower", KR(ret), K(replica));
6019
      } else if (OB_FAIL(inmemory_ls->update(replica, inner_table_only))) {
6020
        LOG_WARN("update sys_ls info failed", KR(ret), K(replica));
6021
      } else {
6022
        LOG_INFO("update sys_tenant ls replica succeed", K(replica), "server", addr);
6023
      }
6024
    }
6025
  }
6026
  return ret;
6027
}
6028

6029
int ObRootService::not_implement()
6030
{
6031
  int ret = OB_NOT_IMPLEMENT;
6032
  bt("not implement");
6033
  LOG_WARN("rpc not implemented", K(ret));
6034
  return ret;
6035
}
6036

6037
ObRootService::ObRestartTask::ObRestartTask(ObRootService &root_service)
6038
:ObAsyncTimerTask(root_service.task_queue_),
6039
    root_service_(root_service)
6040
{
6041
  set_retry_times(0);  // don't retry when failed
6042
}
6043

6044
ObRootService::ObRestartTask::~ObRestartTask()
6045
{
6046
}
6047

6048
int ObRootService::ObRestartTask::process()
6049
{
6050
  int ret = OB_SUCCESS;
6051
  FLOG_INFO("after_restart task begin to process");
6052
  if (OB_FAIL(root_service_.after_restart())) {
6053
    LOG_WARN("root service after restart failed", K(ret));
6054
  }
6055
  FLOG_INFO("after_restart task process finish", KR(ret));
6056
  return ret;
6057
}
6058

6059
ObAsyncTask *ObRootService::ObRestartTask::deep_copy(char *buf, const int64_t buf_size) const
6060
{
6061
  ObRestartTask *task = NULL;
6062
  if (NULL == buf || buf_size < static_cast<int64_t>(sizeof(*this))) {
6063
    LOG_WARN_RET(OB_BUF_NOT_ENOUGH, "buffer not large enough", K(buf_size));
6064
  } else {
6065
    task = new(buf) ObRestartTask(root_service_);
6066
  }
6067
  return task;
6068
}
6069

6070
ObRootService::ObRefreshServerTask::ObRefreshServerTask(ObRootService &root_service)
6071
:ObAsyncTimerTask(root_service.task_queue_),
6072
    root_service_(root_service)
6073
{
6074
  set_retry_times(0);  // don't retry when process failed
6075
}
6076

6077
int ObRootService::ObRefreshServerTask::process()
6078
{
6079
  int ret = OB_SUCCESS;
6080
  const bool load_frozen_status = true;
6081
  const bool need_retry = true;
6082
  FLOG_INFO("refresh server task process");
6083
  ObLatchRGuard guard(root_service_.bootstrap_lock_, ObLatchIds::RS_BOOTSTRAP_LOCK);
6084
  if (OB_FAIL(root_service_.refresh_server(load_frozen_status, need_retry))) {
6085
    FLOG_WARN("refresh server failed", K(ret), K(load_frozen_status));
6086
  } else {}
6087
  return ret;
6088
}
6089

6090
ObAsyncTask *ObRootService::ObRefreshServerTask::deep_copy(char *buf, const int64_t buf_size) const
6091
{
6092
  ObRefreshServerTask *task = NULL;
6093
  if (NULL == buf || buf_size < static_cast<int64_t>(sizeof(*this))) {
6094
    LOG_WARN_RET(OB_BUF_NOT_ENOUGH, "buffer not large enough", K(buf_size));
6095
  } else {
6096
    task = new(buf) ObRefreshServerTask(root_service_);
6097
  }
6098
  return task;
6099
}
6100

6101
//-----Functions for managing privileges------
6102
int ObRootService::create_user(obrpc::ObCreateUserArg &arg,
6103
                               common::ObSArray<int64_t> &failed_index)
6104
{
6105
  int ret = OB_SUCCESS;
6106
  failed_index.reset();
6107
  if (!inited_) {
6108
    ret = OB_NOT_INIT;
6109
    LOG_WARN("not init", K(ret));
6110
  } else if (!arg.is_valid()) {
6111
    ret = OB_INVALID_ARGUMENT;
6112
    LOG_WARN("invalid arg", K(arg), K(ret));
6113
  } else if (OB_FAIL(ddl_service_.create_user(arg, failed_index))){
6114
    LOG_WARN("create user failed", K(ret), K(arg));
6115
  }
6116
  return ret;
6117
}
6118

6119
int ObRootService::drop_user(const ObDropUserArg &arg,
6120
                             common::ObSArray<int64_t> &failed_index)
6121
{
6122
  int ret = OB_SUCCESS;
6123
  failed_index.reset();
6124
  if (!inited_) {
6125
    ret = OB_NOT_INIT;
6126
    LOG_WARN("not init", K(ret));
6127
  } else if (!arg.is_valid()) {
6128
    ret = OB_INVALID_ARGUMENT;
6129
    LOG_WARN("invalid arg", K(arg), K(ret));
6130
  } else if (OB_FAIL(ddl_service_.drop_user(arg, failed_index))) {
6131
    LOG_WARN("drop user failed", K(ret), K(arg));
6132
  }
6133
  return ret;
6134
}
6135

6136
int ObRootService::rename_user(const obrpc::ObRenameUserArg &arg,
6137
                               common::ObSArray<int64_t> &failed_index)
6138
{
6139
  int ret = OB_SUCCESS;
6140
  failed_index.reset();
6141
  if (!inited_) {
6142
    ret = OB_NOT_INIT;
6143
    LOG_WARN("not init", K(ret));
6144
  } else if (!arg.is_valid()) {
6145
    ret = OB_INVALID_ARGUMENT;
6146
    LOG_WARN("invalid arg", K(arg), K(ret));
6147
  } else if (OB_FAIL(ddl_service_.rename_user(arg, failed_index))){
6148
    LOG_WARN("rename user failed", K(arg), K(ret));
6149
  }
6150
  return ret;
6151
}
6152

6153
int ObRootService::alter_role(const obrpc::ObAlterRoleArg &arg)
6154
{
6155
  int ret = OB_SUCCESS;
6156
  if (!inited_) {
6157
    ret = OB_NOT_INIT;
6158
    LOG_WARN("not init", K(ret));
6159
  } else if(!arg.is_valid()) {
6160
    ret = OB_INVALID_ARGUMENT;
6161
    LOG_WARN("invalid arg", K(arg), K(ret));
6162
  } else if (OB_FAIL(ddl_service_.alter_role(arg))) {
6163
    LOG_WARN("alter role failed", K(arg), K(ret));
6164
  }
6165
  return ret;
6166
}
6167

6168
int ObRootService::set_passwd(const obrpc::ObSetPasswdArg &arg)
6169
{
6170
  int ret = OB_SUCCESS;
6171
  if (!inited_) {
6172
    ret = OB_NOT_INIT;
6173
    LOG_WARN("not init", K(ret));
6174
  } else if (!arg.is_valid()) {
6175
    ret = OB_INVALID_ARGUMENT;
6176
    LOG_WARN("invalid arg", K(arg), K(ret));
6177
  } else if (OB_FAIL(ddl_service_.set_passwd(arg))){
6178
    LOG_WARN("set passwd failed",  K(arg), K(ret));
6179
  }
6180
  return ret;
6181
}
6182

6183
int ObRootService::grant(const ObGrantArg &arg)
6184
{
6185
  int ret = OB_SUCCESS;
6186
  if (!inited_) {
6187
    ret = OB_NOT_INIT;
6188
    LOG_WARN("not init", K(ret));
6189
  } else if (!arg.is_valid()) {
6190
    ret = OB_INVALID_ARGUMENT;
6191
    LOG_WARN("invalid arg", K(arg), K(ret));
6192
  } else {
6193
    if (OB_FAIL(ddl_service_.grant(arg))) {
6194
      LOG_WARN("Grant user failed", K(arg), K(ret));
6195
    }
6196
  }
6197
  return ret;
6198
}
6199

6200
int ObRootService::revoke_user(const ObRevokeUserArg &arg)
6201
{
6202
  int ret = OB_SUCCESS;
6203
  if (!inited_) {
6204
    ret = OB_NOT_INIT;
6205
    LOG_WARN("not init", K(ret));
6206
  } else if (!arg.is_valid()) {
6207
    ret = OB_INVALID_ARGUMENT;
6208
    LOG_WARN("invalid arg", K(arg), K(ret));
6209
  } else if (OB_FAIL(ddl_service_.revoke(arg))) {
6210
    LOG_WARN("revoke privilege failed", K(ret), K(arg));
6211
  }
6212
  return ret;
6213
}
6214

6215
int ObRootService::lock_user(const ObLockUserArg &arg, ObSArray<int64_t> &failed_index)
6216
{
6217
  int ret = OB_SUCCESS;
6218
  failed_index.reset();
6219
  if (!inited_) {
6220
    ret = OB_NOT_INIT;
6221
    LOG_WARN("not init", K(ret));
6222
  } else if (!arg.is_valid()) {
6223
    ret = OB_INVALID_ARGUMENT;
6224
    LOG_WARN("invalid arg", K(arg), K(ret));
6225
  } else if (OB_FAIL(ddl_service_.lock_user(arg, failed_index))){
6226
    LOG_WARN("lock user failed", K(arg), K(ret));
6227
  }
6228
  return ret;
6229
}
6230

6231
int ObRootService::alter_user_profile(const ObAlterUserProfileArg &arg)
6232
{
6233
  int ret = OB_SUCCESS;
6234
  if (!inited_) {
6235
    ret = OB_NOT_INIT;
6236
    LOG_WARN("not init", K(ret));
6237
  } else if (!arg.is_valid()) {
6238
    ret = OB_INVALID_ARGUMENT;
6239
    LOG_WARN("invalid arg", K(arg), K(ret));
6240
  } else if (OB_FAIL(ddl_service_.alter_user_profile(arg))){
6241
    LOG_WARN("lock user failed", K(arg), K(ret));
6242
  }
6243
  return ret;
6244
}
6245

6246
int ObRootService::create_directory(const obrpc::ObCreateDirectoryArg &arg)
6247
{
6248
  int ret = OB_SUCCESS;
6249
  if (!inited_) {
6250
    ret = OB_NOT_INIT;
6251
    LOG_WARN("not init", K(ret));
6252
  } else if (OB_FAIL(ddl_service_.create_directory(arg, &arg.ddl_stmt_str_))) {
6253
    LOG_WARN("create directory failed", K(arg.schema_), K(ret));
6254
  }
6255
  return ret;
6256
}
6257

6258
int ObRootService::drop_directory(const obrpc::ObDropDirectoryArg &arg)
6259
{
6260
  int ret = OB_SUCCESS;
6261
  if (!inited_) {
6262
    ret = OB_NOT_INIT;
6263
    LOG_WARN("not init", K(ret));
6264
  } else if (OB_FAIL(ddl_service_.drop_directory(arg, &arg.ddl_stmt_str_))) {
6265
    LOG_WARN("drop directory failed", K(arg.directory_name_), K(ret));
6266
  }
6267
  return ret;
6268
}
6269

6270
////////////////////////////////////////////////////////////////
6271
// row level security
6272
////////////////////////////////////////////////////////////////
6273

6274
int ObRootService::handle_rls_policy_ddl(const obrpc::ObRlsPolicyDDLArg &arg)
6275
{
6276
  int ret = OB_SUCCESS;
6277
  uint64_t data_version = 0;
6278
  if (!inited_) {
6279
    ret = OB_NOT_INIT;
6280
    LOG_WARN("not init", K(ret));
6281
  } else if (OB_FAIL(GET_MIN_DATA_VERSION(arg.exec_tenant_id_, data_version))) {
6282
    LOG_WARN("failed to get min data version", K(ret));
6283
  } else if (data_version < DATA_VERSION_4_1_0_0) {
6284
    ret = OB_NOT_SUPPORTED;
6285
    LOG_USER_ERROR(OB_NOT_SUPPORTED, "dbms_rls");
6286
  } else if (OB_FAIL(ddl_service_.handle_rls_policy_ddl(arg))) {
6287
    LOG_WARN("do rls policy ddl failed", K(arg), K(ret));
6288
  }
6289
  return ret;
6290
}
6291

6292
int ObRootService::handle_rls_group_ddl(const obrpc::ObRlsGroupDDLArg &arg)
6293
{
6294
  int ret = OB_SUCCESS;
6295
  uint64_t data_version = 0;
6296
  if (!inited_) {
6297
    ret = OB_NOT_INIT;
6298
    LOG_WARN("not init", K(ret));
6299
  } else if (OB_FAIL(GET_MIN_DATA_VERSION(arg.exec_tenant_id_, data_version))) {
6300
    LOG_WARN("failed to get min data version", K(ret));
6301
  } else if (data_version < DATA_VERSION_4_1_0_0) {
6302
    ret = OB_NOT_SUPPORTED;
6303
    LOG_USER_ERROR(OB_NOT_SUPPORTED, "dbms_rls");
6304
  } else if (OB_FAIL(ddl_service_.handle_rls_group_ddl(arg))) {
6305
    LOG_WARN("do rls group ddl failed", K(arg), K(ret));
6306
  }
6307
  return ret;
6308
}
6309

6310
int ObRootService::handle_rls_context_ddl(const obrpc::ObRlsContextDDLArg &arg)
6311
{
6312
  int ret = OB_SUCCESS;
6313
  uint64_t data_version = 0;
6314
  if (!inited_) {
6315
    ret = OB_NOT_INIT;
6316
    LOG_WARN("not init", K(ret));
6317
  } else if (OB_FAIL(GET_MIN_DATA_VERSION(arg.exec_tenant_id_, data_version))) {
6318
    LOG_WARN("failed to get min data version", K(ret));
6319
  } else if (data_version < DATA_VERSION_4_1_0_0) {
6320
    ret = OB_NOT_SUPPORTED;
6321
    LOG_USER_ERROR(OB_NOT_SUPPORTED, "dbms_rls");
6322
  } else if (OB_FAIL(ddl_service_.handle_rls_context_ddl(arg))) {
6323
    LOG_WARN("do rls context ddl failed", K(arg), K(ret));
6324
  }
6325
  return ret;
6326
}
6327

6328
int ObRootService::revoke_database(const ObRevokeDBArg &arg)
6329
{
6330
  int ret = OB_SUCCESS;
6331
  if (!inited_) {
6332
    ret = OB_NOT_INIT;
6333
    LOG_WARN("not init", K(ret));
6334
  } else if (!arg.is_valid()) {
6335
    ret = OB_INVALID_ARGUMENT;
6336
    LOG_WARN("invalid arg", K(arg), K(ret));
6337
  } else {
6338
    ObOriginalDBKey db_key(arg.tenant_id_, arg.user_id_, arg.db_);
6339
    if (OB_FAIL(ddl_service_.revoke_database(db_key, arg.priv_set_))) {
6340
      LOG_WARN("Revoke db failed", K(arg), K(ret));
6341
    }
6342
  }
6343
  return ret;
6344
}
6345

6346
int ObRootService::revoke_table(const ObRevokeTableArg &arg)
6347
{
6348
  int ret = OB_SUCCESS;
6349
  if (!inited_) {
6350
    ret = OB_NOT_INIT;
6351
    LOG_WARN("not init", K(ret));
6352
  } else if (!arg.is_valid()) {
6353
    ret = OB_INVALID_ARGUMENT;
6354
    LOG_WARN("invalid arg", K(arg), K(ret));
6355
  } else {
6356
    ObTablePrivSortKey table_priv_key(arg.tenant_id_, arg.user_id_, arg.db_, arg.table_);
6357
    ObObjPrivSortKey obj_priv_key(arg.tenant_id_,
6358
                                  arg.obj_id_,
6359
                                  arg.obj_type_,
6360
                                  OBJ_LEVEL_FOR_TAB_PRIV,
6361
                                  arg.grantor_id_,
6362
                                  arg.user_id_);
6363
    OZ (ddl_service_.revoke_table(table_priv_key,
6364
                                  arg.priv_set_,
6365
                                  obj_priv_key,
6366
                                  arg.obj_priv_array_,
6367
                                  arg.revoke_all_ora_));
6368
  }
6369
  return ret;
6370
}
6371

6372
int ObRootService::revoke_syspriv(const ObRevokeSysPrivArg &arg)
6373
{
6374
  int ret = OB_SUCCESS;
6375
  if (!inited_) {
6376
    ret = OB_NOT_INIT;
6377
    LOG_WARN("not init", K(ret));
6378
  } else if (!arg.is_valid()) {
6379
    ret = OB_INVALID_ARGUMENT;
6380
    LOG_WARN("invalid arg", K(arg), K(ret));
6381
  } else if (OB_FAIL(ddl_service_.revoke_syspriv(arg.tenant_id_,
6382
                                                 arg.grantee_id_,
6383
                                                 arg.sys_priv_array_,
6384
                                                 arg.role_ids_,
6385
                                                 &arg.ddl_stmt_str_))) {
6386
    LOG_WARN("revoke privilege failed", K(ret), K(arg));
6387
  }
6388
  return ret;
6389
}
6390

6391
//-----End of functions for managing privileges-----
6392

6393
//-----Functions for managing outlines-----
6394
int ObRootService::create_outline(const ObCreateOutlineArg &arg)
6395
{
6396
  int ret = OB_SUCCESS;
6397
  if (!inited_) {
6398
    ret = OB_NOT_INIT;
6399
    LOG_WARN("not init", K(ret));
6400
  } else if (!arg.is_valid()) {
6401
    ret = OB_INVALID_ARGUMENT;
6402
    LOG_WARN("invalid arg", K(arg), K(ret));
6403
  } else {
6404
    ObOutlineInfo outline_info = arg.outline_info_;
6405
    const bool is_or_replace = arg.or_replace_;
6406
    uint64_t tenant_id = outline_info.get_tenant_id();
6407
    ObString database_name = arg.db_name_;
6408
    ObSchemaGetterGuard schema_guard;
6409
    const ObDatabaseSchema *db_schema = NULL;
6410
    if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
6411
      LOG_WARN("get schema guard in inner table failed", K(ret));
6412
    } else if (database_name == OB_MOCK_DEFAULT_DATABASE_NAME) {
6413
      // if not specify database, set default database name and database id;
6414
      outline_info.set_database_id(OB_MOCK_DEFAULT_DATABASE_ID);
6415
    } else if (OB_FAIL(schema_guard.get_database_schema(tenant_id, database_name, db_schema))) {
6416
      LOG_WARN("get database schema failed", K(ret));
6417
    } else if (NULL == db_schema) {
6418
      ret = OB_ERR_BAD_DATABASE;
6419
      LOG_USER_ERROR(OB_ERR_BAD_DATABASE, database_name.length(), database_name.ptr());
6420
    } else if (db_schema->is_in_recyclebin()) {
6421
      ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
6422
      LOG_WARN("Can't not create outline of db in recyclebin", K(ret), K(arg), K(*db_schema));
6423
    } else if (OB_INVALID_ID == db_schema->get_database_id()) {
6424
      ret = OB_ERR_BAD_DATABASE;
6425
      LOG_WARN("database id is invalid", K(tenant_id), K(*db_schema), K(ret));
6426
    } else {
6427
      outline_info.set_database_id(db_schema->get_database_id());
6428
    }
6429

6430
    bool is_update = false;
6431
    if (OB_SUCC(ret)) {
6432
      if (OB_FAIL(ddl_service_.check_outline_exist(outline_info, is_or_replace, is_update))) {
6433
        LOG_WARN("failed to check_outline_exist", K(outline_info), K(ret));
6434
      }
6435
    }
6436

6437
    if (OB_SUCC(ret)) {
6438
      if (OB_FAIL(ddl_service_.create_outline(outline_info, is_update, &arg.ddl_stmt_str_, schema_guard))) {
6439
        LOG_WARN("create_outline failed", K(outline_info), K(is_update), K(ret));
6440
      }
6441
    }
6442
  }
6443
  return ret;
6444
}
6445

6446
int ObRootService::create_user_defined_function(const obrpc::ObCreateUserDefinedFunctionArg &arg)
6447
{
6448
  int ret = OB_SUCCESS;
6449
  bool exist = false;
6450
  uint64_t udf_id = OB_INVALID_ID;
6451
  ObUDF udf_info_ = arg.udf_;
6452
  if (!inited_) {
6453
    ret = OB_NOT_INIT;
6454
    LOG_WARN("not init", K(ret));
6455
  } else if (!arg.is_valid()) {
6456
    ret = OB_INVALID_ARGUMENT;
6457
    LOG_WARN("invalid arg", K(arg), K(ret));
6458
  } else if (OB_FAIL(ddl_service_.check_udf_exist(arg.udf_.get_tenant_id(), arg.udf_.get_name_str(), exist, udf_id))) {
6459
    LOG_WARN("failed to check_udf_exist", K(arg.udf_.get_tenant_id()), K(arg.udf_.get_name_str()), K(exist), K(ret));
6460
  } else if (exist) {
6461
    ret = OB_UDF_EXISTS;
6462
    LOG_USER_ERROR(OB_UDF_EXISTS, arg.udf_.get_name_str().length(), arg.udf_.get_name_str().ptr());
6463
  } else if (OB_FAIL(ddl_service_.create_user_defined_function(udf_info_, arg.ddl_stmt_str_))) {
6464
    LOG_WARN("failed to create udf", K(arg), K(ret));
6465
  } else {/*do nothing*/}
6466
  return ret;
6467
}
6468

6469
int ObRootService::drop_user_defined_function(const obrpc::ObDropUserDefinedFunctionArg &arg)
6470
{
6471
  int ret = OB_SUCCESS;
6472
  if (!inited_) {
6473
    ret = OB_NOT_INIT;
6474
    LOG_WARN("not init", K(ret));
6475
  } else if (!arg.is_valid()) {
6476
    ret = OB_INVALID_ARGUMENT;
6477
    LOG_WARN("invalid arg", K(arg), K(ret));
6478
  } else if (OB_FAIL(ddl_service_.drop_user_defined_function(arg))) {
6479
    LOG_WARN("failed to alter udf", K(arg), K(ret));
6480
  } else {/*do nothing*/}
6481

6482
  return ret;
6483
}
6484

6485
bool ObRootService::is_sys_tenant(const ObString &tenant_name)
6486
{
6487
  return (0 == tenant_name.case_compare(OB_SYS_TENANT_NAME)
6488
          || 0 == tenant_name.case_compare(OB_DIAG_TENANT_NAME)
6489
          || 0 == tenant_name.case_compare(OB_GTS_TENANT_NAME));
6490
}
6491

6492
int ObRootService::alter_outline(const ObAlterOutlineArg &arg)
6493
{
6494
  int ret = OB_SUCCESS;
6495
  if (!inited_) {
6496
    ret = OB_NOT_INIT;
6497
    LOG_WARN("not init", K(ret));
6498
  } else if (!arg.is_valid()) {
6499
    ret = OB_INVALID_ARGUMENT;
6500
    LOG_WARN("invalid arg", K(arg), K(ret));
6501
  } else if (OB_FAIL(ddl_service_.alter_outline(arg))) {
6502
    LOG_WARN("failed to alter outline", K(arg), K(ret));
6503
  } else {/*do nothing*/}
6504
  return ret;
6505
}
6506

6507
int ObRootService::drop_outline(const obrpc::ObDropOutlineArg &arg)
6508
{
6509
  int ret = OB_SUCCESS;
6510
  if (!inited_) {
6511
    ret = OB_NOT_INIT;
6512
    LOG_WARN("not init", K(ret));
6513
  } else if (!arg.is_valid()) {
6514
    ret = OB_INVALID_ARGUMENT;
6515
    LOG_WARN("invalid arg", K(arg), K(ret));
6516
  } else {
6517
    if (OB_FAIL(ddl_service_.drop_outline(arg))) {
6518
      LOG_WARN("ddl service failed to drop outline", K(arg), K(ret));
6519
    }
6520
  }
6521
  return ret;
6522
}
6523
//-----End of functions for managing outlines-----
6524

6525
int ObRootService::create_routine(const ObCreateRoutineArg &arg)
6526
{
6527
  int ret = OB_SUCCESS;
6528
  if (!inited_) {
6529
    ret = OB_NOT_INIT;
6530
    LOG_WARN("not init", K(ret));
6531
  } else if (!arg.is_valid()) {
6532
    ret = OB_INVALID_ARGUMENT;
6533
    LOG_WARN("invalid arg", K(arg), K(ret));
6534
  } else {
6535
    ObRoutineInfo routine_info = arg.routine_info_;
6536
    const ObRoutineInfo* old_routine_info = NULL;
6537
    uint64_t tenant_id = routine_info.get_tenant_id();
6538
    ObString database_name = arg.db_name_;
6539
    bool is_or_replace = false;
6540
    bool is_inner = false;
6541
    ObSchemaGetterGuard schema_guard;
6542
    const ObDatabaseSchema *db_schema = NULL;
6543
    const ObUserInfo *user_info = NULL;
6544
    bool is_oracle_mode = false;
6545
    if (OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(
6546
                tenant_id, is_oracle_mode))) {
6547
      LOG_WARN("fail to check is oracle mode", K(ret));
6548
    } else {
6549
      is_or_replace = is_oracle_mode ? arg.is_or_replace_ : arg.is_need_alter_;
6550
      is_inner = is_oracle_mode ? false : arg.is_or_replace_;
6551
    }
6552

6553
    if (OB_FAIL(ret)) {
6554
    } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
6555
      LOG_WARN("get schema guard in inner table failed", K(ret));
6556
    } else if (OB_FAIL(schema_guard.get_database_schema(tenant_id, database_name, db_schema))) {
6557
      LOG_WARN("get database schema failed", K(ret));
6558
    } else if (NULL == db_schema) {
6559
      ret = OB_ERR_BAD_DATABASE;
6560
      LOG_USER_ERROR(OB_ERR_BAD_DATABASE, database_name.length(), database_name.ptr());
6561
    } else if (!is_inner && db_schema->is_in_recyclebin()) {
6562
      ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
6563
      LOG_WARN("Can't not create routine of db in recyclebin", K(ret), K(arg), K(*db_schema));
6564
    } else if (OB_INVALID_ID == db_schema->get_database_id()) {
6565
      ret = OB_ERR_BAD_DATABASE;
6566
      LOG_WARN("database id is invalid", K(tenant_id), K(*db_schema), K(ret));
6567
    } else {
6568
      routine_info.set_database_id(db_schema->get_database_id());
6569
    }
6570
    if (OB_SUCC(ret)
6571
        && database_name.case_compare(OB_SYS_DATABASE_NAME) != 0
6572
        && is_oracle_mode) {
6573
      if (OB_FAIL(schema_guard.get_user_info(
6574
          tenant_id, database_name, ObString(OB_DEFAULT_HOST_NAME), user_info))) {
6575
        LOG_WARN("failed to get user info", K(ret), K(database_name));
6576
      } else if (OB_ISNULL(user_info)) {
6577
        ret = OB_USER_NOT_EXIST;
6578
        LOG_WARN("user is does not exist", K(ret), K(database_name));
6579
      } else if (OB_INVALID_ID == user_info->get_user_id()) {
6580
        ret = OB_USER_NOT_EXIST;
6581
        LOG_WARN("user id is invalid", K(ret), K(database_name));
6582
      } else {
6583
        routine_info.set_owner_id(user_info->get_user_id());
6584
      }
6585
    }
6586
    if (OB_SUCC(ret)) {
6587
      ObArray<ObSchemaType> conflict_schema_types;
6588
      if (OB_FAIL(schema_guard.check_oracle_object_exist(tenant_id,
6589
          db_schema->get_database_id(), routine_info.get_routine_name(), ROUTINE_SCHEMA,
6590
          routine_info.get_routine_type(), is_or_replace, conflict_schema_types))) {
6591
        LOG_WARN("fail to check oracle_object exist", K(ret), K(routine_info.get_routine_name()));
6592
      } else if (conflict_schema_types.count() > 0) {
6593
        // 这里检查 oracle 模式下新对象的名字是否已经被其他对象占用了
6594
        ret = OB_ERR_EXIST_OBJECT;
6595
        LOG_WARN("Name is already used by an existing object in oralce mode",
6596
                 K(ret), K(routine_info.get_routine_name()),
6597
                 K(conflict_schema_types));
6598
      }
6599
    }
6600
    bool exist = false;
6601
    if (OB_SUCC(ret)) {
6602
      if (routine_info.get_routine_type() == ROUTINE_PROCEDURE_TYPE) {
6603
        if (OB_FAIL(schema_guard.check_standalone_procedure_exist(tenant_id, db_schema->get_database_id(),
6604
                                                                  routine_info.get_routine_name(), exist))) {
6605
          LOG_WARN("failed to check procedure info exist", K(routine_info), K(ret));
6606
        } else if (exist && !is_or_replace) {
6607
          ret = OB_ERR_SP_ALREADY_EXISTS;
6608
          LOG_USER_ERROR(OB_ERR_SP_ALREADY_EXISTS, "PROCEDURE",
6609
                         routine_info.get_routine_name().length(), routine_info.get_routine_name().ptr());
6610
        } else if (exist && is_or_replace) {
6611
          if (OB_FAIL(schema_guard.get_standalone_procedure_info(tenant_id, db_schema->get_database_id(),
6612
                                                                 routine_info.get_routine_name(), old_routine_info))) {
6613
            LOG_WARN("failed to get standalone procedure info", K(routine_info), K(ret));
6614
          } else if (OB_ISNULL(old_routine_info)) {
6615
            ret = OB_ERR_UNEXPECTED;
6616
            LOG_WARN("old routine info is NULL", K(ret));
6617
          }
6618
        }
6619
      } else {
6620
        if (OB_FAIL(schema_guard.check_standalone_function_exist(tenant_id, db_schema->get_database_id(),
6621
                                                                 routine_info.get_routine_name(), exist))) {
6622
          LOG_WARN("failed to check function info exist", K(routine_info), K(ret));
6623
        } else if (exist && !is_or_replace) {
6624
          ret = OB_ERR_SP_ALREADY_EXISTS;
6625
          LOG_USER_ERROR(OB_ERR_SP_ALREADY_EXISTS, "FUNCTION",
6626
                         routine_info.get_routine_name().length(), routine_info.get_routine_name().ptr());
6627
        } else if (exist && is_or_replace) {
6628
          if (OB_FAIL(schema_guard.get_standalone_function_info(tenant_id, db_schema->get_database_id(),
6629
                                                                routine_info.get_routine_name(), old_routine_info))) {
6630
            LOG_WARN("failed to get standalone function info", K(routine_info), K(ret));
6631
          } else if (OB_ISNULL(old_routine_info)) {
6632
            ret = OB_ERR_UNEXPECTED;
6633
            LOG_WARN("old routine info is NULL", K(ret));
6634
          }
6635
        }
6636
      }
6637
      if (OB_SUCC(ret)) {
6638
        ObErrorInfo error_info = arg.error_info_;
6639
        ObSArray<ObDependencyInfo> &dep_infos =
6640
                      const_cast<ObSArray<ObDependencyInfo> &>(arg.dependency_infos_);
6641
        if (OB_FAIL(ddl_service_.create_routine(routine_info,
6642
                                                old_routine_info,
6643
                                                (exist && is_or_replace),
6644
                                                error_info,
6645
                                                dep_infos,
6646
                                                &arg.ddl_stmt_str_,
6647
                                                schema_guard))) {
6648
          LOG_WARN("failed to replace routine", K(routine_info), K(ret));
6649
        }
6650
      }
6651
    }
6652
  }
6653
  return ret;
6654
}
6655

6656
int ObRootService::alter_routine(const ObCreateRoutineArg &arg)
6657
{
6658
  int ret = OB_SUCCESS;
6659
  if (!inited_) {
6660
    ret = OB_NOT_INIT;
6661
    LOG_WARN("not init", K(ret));
6662
  } else if (!arg.is_valid()) {
6663
    ret = OB_INVALID_ARGUMENT;
6664
    LOG_WARN("invalid arg", K(arg), K(ret));
6665
  } else {
6666
    ObErrorInfo error_info = arg.error_info_;
6667
    const ObRoutineInfo *routine_info = NULL;
6668
    ObSchemaGetterGuard schema_guard;
6669
    bool is_oracle_mode = false;
6670
    const uint64_t tenant_id = arg.routine_info_.get_tenant_id();
6671
    if (OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(
6672
                tenant_id, is_oracle_mode))) {
6673
      LOG_WARN("fail to check is oracle mode", K(ret));
6674
    } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(
6675
                                  tenant_id, schema_guard))) {
6676
      LOG_WARN("get schema guard in inner table failed", K(ret), K(tenant_id));
6677
    } else if (OB_FAIL(schema_guard.get_routine_info(
6678
               tenant_id, arg.routine_info_.get_routine_id(), routine_info))) {
6679
      LOG_WARN("failed to get routine info", K(ret), K(tenant_id));
6680
    } else if (OB_ISNULL(routine_info)) {
6681
      ret = OB_ERR_SP_DOES_NOT_EXIST;
6682
      LOG_WARN("routine info is not exist!", K(ret), K(arg.routine_info_));
6683
    }
6684
    if (OB_FAIL(ret)) {
6685
    } else if ((is_oracle_mode && arg.is_or_replace_) ||
6686
               (!is_oracle_mode && arg.is_need_alter_)) {
6687
      if (OB_FAIL(create_routine(arg))) {
6688
        LOG_WARN("failed to alter routine with create", K(ret));
6689
      }
6690
    } else {
6691
      if (OB_FAIL(ddl_service_.alter_routine(*routine_info, error_info, &arg.ddl_stmt_str_,
6692
                                             schema_guard))) {
6693
        LOG_WARN("alter routine failed", K(ret), K(arg.routine_info_), K(error_info));
6694
      }
6695
    }
6696
  }
6697
  return ret;
6698
}
6699

6700
int ObRootService::drop_routine(const ObDropRoutineArg &arg)
6701
{
6702
  int ret = OB_SUCCESS;
6703
  if (!inited_) {
6704
    ret = OB_NOT_INIT;
6705
    LOG_WARN("not init", K(ret));
6706
  } else if (!arg.is_valid()) {
6707
    ret = OB_INVALID_ARGUMENT;
6708
    LOG_WARN("invalid arg", K(arg), K(ret));
6709
  } else {
6710
    uint64_t tenant_id = arg.tenant_id_;
6711
    const ObString &db_name = arg.db_name_;
6712
    const ObString &routine_name = arg.routine_name_;
6713
    ObRoutineType routine_type = arg.routine_type_;
6714
    ObSchemaGetterGuard schema_guard;
6715
    const ObDatabaseSchema *db_schema = NULL;
6716
    /*!
6717
     * 兼容mysql行为:
6718
     * create database test;
6719
     * use test;
6720
     * drop database test;
6721
     * drop function if exists no_such_func; -- warning 1035
6722
     * drop procedure if exists no_such_proc; -- error 1046
6723
     * drop function no_such_func; --error 1035
6724
     * drop procedure no_such_proc; --error 1046
6725
     */
6726
    if (db_name.empty()) {
6727
      ret = OB_ERR_NO_DB_SELECTED;
6728
      LOG_WARN("no database selected", K(ret), K(db_name));
6729
    } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
6730
      LOG_WARN("get schema guard in inner table failed", K(ret));
6731
    } else if (OB_FAIL(schema_guard.get_database_schema(tenant_id, db_name, db_schema))) {
6732
      LOG_WARN("get database schema failed", K(ret));
6733
    } else if (NULL == db_schema) {
6734
      ret = OB_ERR_BAD_DATABASE;
6735
      LOG_USER_ERROR(OB_ERR_BAD_DATABASE, db_name.length(), db_name.ptr());
6736
    } else if (db_schema->is_in_recyclebin()) {
6737
      ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
6738
      LOG_WARN("Can't not create procedure of db in recyclebin", K(ret), K(arg), K(*db_schema));
6739
    } else if (OB_INVALID_ID == db_schema->get_database_id()) {
6740
      ret = OB_ERR_BAD_DATABASE;
6741
      LOG_WARN("database id is invalid", K(tenant_id), K(*db_schema), K(ret));
6742
    }
6743

6744
    if (OB_SUCC(ret)) {
6745
      bool exist = false;
6746
      const ObRoutineInfo *routine_info = NULL;
6747
      if (ROUTINE_PROCEDURE_TYPE == routine_type) {
6748
        if (OB_FAIL(schema_guard.check_standalone_procedure_exist(tenant_id, db_schema->get_database_id(),
6749
                                                                  routine_name, exist))) {
6750
          LOG_WARN("failed to check standalone procedure info exist", K(routine_name), K(ret));
6751
        } else if (exist) {
6752
          if (OB_FAIL(schema_guard.get_standalone_procedure_info(tenant_id, db_schema->get_database_id(),
6753
                                                                 routine_name, routine_info))) {
6754
            LOG_WARN("get procedure info failed", K(ret));
6755
          }
6756
        } else if (!arg.if_exist_) {
6757
          ret = OB_ERR_SP_DOES_NOT_EXIST;
6758
          LOG_USER_ERROR(OB_ERR_SP_DOES_NOT_EXIST, "PROCEDURE", db_name.length(), db_name.ptr(),
6759
                         routine_name.length(), routine_name.ptr());
6760
        }
6761
      } else {
6762
        if (OB_FAIL(schema_guard.check_standalone_function_exist(tenant_id, db_schema->get_database_id(),
6763
                                                                 routine_name, exist))) {
6764
          LOG_WARN("failed to check standalone function info exist", K(routine_name), K(ret));
6765
        } else if (exist) {
6766
          if (OB_FAIL(schema_guard.get_standalone_function_info(tenant_id, db_schema->get_database_id(),
6767
                                                                routine_name, routine_info))) {
6768
            LOG_WARN("get function info failed", K(ret));
6769
          }
6770
        } else if (!arg.if_exist_) {
6771
          ret = OB_ERR_SP_DOES_NOT_EXIST;
6772
          LOG_USER_ERROR(OB_ERR_SP_DOES_NOT_EXIST, "FUNCTION", db_name.length(), db_name.ptr(),
6773
                         routine_name.length(), routine_name.ptr());
6774
        }
6775
      }
6776

6777
      if (OB_SUCC(ret) && !OB_ISNULL(routine_info)) {
6778
        ObErrorInfo error_info = arg.error_info_;
6779
        if (OB_FAIL(ddl_service_.drop_routine(*routine_info,
6780
                                              error_info,
6781
                                              &arg.ddl_stmt_str_,
6782
                                              schema_guard))) {
6783
          LOG_WARN("drop routine failed", K(ret), K(routine_name), K(routine_info));
6784
        }
6785
      }
6786
    }
6787
    if (OB_ERR_NO_DB_SELECTED == ret && ROUTINE_FUNCTION_TYPE == routine_type) {
6788
      if (arg.if_exist_) {
6789
        ret = OB_SUCCESS;
6790
        LOG_USER_WARN(OB_ERR_SP_DOES_NOT_EXIST, "FUNCTION (UDF)",
6791
                      db_name.length(), db_name.ptr(),
6792
                      routine_name.length(), routine_name.ptr());
6793
      } else {
6794
        ret = OB_ERR_SP_DOES_NOT_EXIST;
6795
        LOG_USER_ERROR(OB_ERR_SP_DOES_NOT_EXIST, "FUNCTION (UDF)",
6796
                      db_name.length(), db_name.ptr(),
6797
                      routine_name.length(), routine_name.ptr());
6798
        LOG_WARN("FUNCTION (UDF) does not exists", K(ret), K(routine_name), K(db_name));
6799
      }
6800
    }
6801
  }
6802
  return ret;
6803
}
6804

6805
int ObRootService::create_udt(const ObCreateUDTArg &arg)
6806
{
6807
  int ret = OB_SUCCESS;
6808
  if (!inited_) {
6809
    ret = OB_NOT_INIT;
6810
    LOG_WARN("not init", K(ret));
6811
  } else if (!arg.is_valid()) {
6812
    ret = OB_INVALID_ARGUMENT;
6813
    LOG_WARN("invalid arg", K(arg), K(ret));
6814
  } else {
6815
    ObUDTTypeInfo udt_info = arg.udt_info_;
6816
    const ObUDTTypeInfo* old_udt_info = NULL;
6817
    uint64_t tenant_id = udt_info.get_tenant_id();
6818
    ObString database_name = arg.db_name_;
6819
    bool is_or_replace = arg.is_or_replace_;
6820
    ObSchemaGetterGuard schema_guard;
6821
    const ObDatabaseSchema *db_schema = NULL;
6822
    if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
6823
      LOG_WARN("get schema guard in inner table failed", K(ret));
6824
    } else if (OB_FAIL(schema_guard.get_database_schema(tenant_id, database_name, db_schema))) {
6825
      LOG_WARN("get database schema failed", K(ret));
6826
    } else if (NULL == db_schema) {
6827
      ret = OB_ERR_BAD_DATABASE;
6828
      LOG_USER_ERROR(OB_ERR_BAD_DATABASE, database_name.length(), database_name.ptr());
6829
    } else if (db_schema->is_in_recyclebin()) {
6830
      ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
6831
      LOG_WARN("Can't not create udt of db in recyclebin", K(ret), K(arg), K(*db_schema));
6832
    } else if (OB_INVALID_ID == db_schema->get_database_id()) {
6833
      ret = OB_ERR_BAD_DATABASE;
6834
      LOG_WARN("database id is invalid", K(tenant_id), K(*db_schema), K(ret));
6835
    } else {
6836
      udt_info.set_database_id(db_schema->get_database_id());
6837
    }
6838
    if (OB_SUCC(ret)) {
6839
      ObArray<ObSchemaType> conflict_schema_types;
6840
      if (OB_FAIL(schema_guard.check_oracle_object_exist(tenant_id, db_schema->get_database_id(),
6841
          udt_info.get_type_name(), UDT_SCHEMA, INVALID_ROUTINE_TYPE, is_or_replace,
6842
          conflict_schema_types))) {
6843
        LOG_WARN("fail to check oracle_object exist", K(ret), K(udt_info.get_type_name()));
6844
      } else if (1 == conflict_schema_types.count()
6845
                 && UDT_SCHEMA == conflict_schema_types.at(0) && udt_info.is_object_type()) {
6846
        // skip
6847
      } else if (conflict_schema_types.count() > 0) {
6848
        ret = OB_ERR_EXIST_OBJECT;
6849
        LOG_WARN("Name is already used by an existing object", K(ret), K(udt_info.get_type_name()),
6850
            K(conflict_schema_types));
6851
      }
6852
    }
6853
    bool exist = false;
6854
    ObUDTTypeCode type_code = udt_info.is_object_body_ddl()
6855
      ? UDT_TYPE_OBJECT_BODY : static_cast<ObUDTTypeCode>(udt_info.get_typecode());
6856
    if (OB_SUCC(ret)) {
6857
      if (OB_FAIL(schema_guard.check_udt_exist(tenant_id,
6858
                                              db_schema->get_database_id(),
6859
                                              OB_INVALID_ID,
6860
                                              type_code,
6861
                                              udt_info.get_type_name(),
6862
                                              exist))) {
6863
        LOG_WARN("failed to check udt info exist", K(udt_info), K(ret));
6864
      } else if (exist && !is_or_replace) {
6865
        if (!udt_info.is_object_type()) {
6866
          ret = OB_ERR_SP_ALREADY_EXISTS;
6867
          LOG_USER_ERROR(OB_ERR_SP_ALREADY_EXISTS, "UDT",
6868
                        udt_info.get_type_name().length(), udt_info.get_type_name().ptr());
6869
        }
6870
      } else {
6871
        // do nothing
6872
      }
6873
      if (exist && OB_SUCC(ret)) {
6874
        if (OB_FAIL(schema_guard.get_udt_info(tenant_id,
6875
                                              db_schema->get_database_id(),
6876
                                              OB_INVALID_ID,
6877
                                              udt_info.get_type_name(),
6878
                                              type_code,
6879
                                              old_udt_info))) {
6880
          LOG_WARN("failed to get udt info", K(udt_info), K(ret));
6881
        } else if (OB_ISNULL(old_udt_info)) {
6882
          ret = OB_ERR_UNEXPECTED;
6883
          LOG_WARN("old udt info is NULL", K(ret));
6884
        }
6885
      }
6886
      if (OB_SUCC(ret)
6887
          && is_inner_pl_udt_id(udt_info.get_type_id())
6888
          && type_code != UDT_TYPE_OBJECT_BODY) {
6889
        if (!exist) {
6890
          if (OB_FAIL(schema_guard.get_udt_info(tenant_id, udt_info.get_type_id(), old_udt_info))) {
6891
            LOG_WARN("failed to get udt info", K(ret), K(tenant_id), K(udt_info.get_type_id()));
6892
          } else if (OB_NOT_NULL(old_udt_info)) {
6893
            ret = OB_ERR_UNEXPECTED;
6894
            LOG_WARN("type id already used by other udt", K(ret), KPC(old_udt_info));
6895
          }
6896
        } else {
6897
          CK (OB_NOT_NULL(old_udt_info));
6898
          OV (old_udt_info->get_type_id() == udt_info.get_type_id(), OB_ERR_UNEXPECTED, KPC(old_udt_info), K(udt_info));
6899
        }
6900
      }
6901
      // 这儿检查object是否能插入，当name相同，且不是replace的时候，只有一种情况可以插入
6902
      // 新的body不为空，但是老的body为空，这说明需要增加body信息
6903
      if (OB_SUCC(ret) && exist && !is_or_replace) {
6904
        ret = OB_ERR_SP_ALREADY_EXISTS;
6905
        LOG_USER_ERROR(OB_ERR_SP_ALREADY_EXISTS, "UDT",
6906
                    udt_info.get_type_name().length(), udt_info.get_type_name().ptr());
6907
      }
6908

6909
      if (OB_SUCC(ret)) {
6910
        ObErrorInfo error_info = arg.error_info_;
6911
        ObSArray<ObRoutineInfo> &public_routine_infos =
6912
                      const_cast<ObSArray<ObRoutineInfo> &>(arg.public_routine_infos_);
6913
        ObSArray<ObDependencyInfo> &dep_infos =
6914
                     const_cast<ObSArray<ObDependencyInfo> &>(arg.dependency_infos_);
6915
        if (OB_FAIL(ddl_service_.create_udt(udt_info,
6916
                                            old_udt_info,
6917
                                            (exist && is_or_replace),
6918
                                            public_routine_infos,
6919
                                            error_info,
6920
                                            schema_guard,
6921
                                            dep_infos,
6922
                                            &arg.ddl_stmt_str_))) {
6923
          LOG_WARN("failed to create udt", K(udt_info), K(ret));
6924
        }
6925
      }
6926
    }
6927
  }
6928
  return ret;
6929
}
6930

6931
int ObRootService::drop_udt(const ObDropUDTArg &arg)
6932
{
6933
  int ret = OB_SUCCESS;
6934
  if (!inited_) {
6935
    ret = OB_NOT_INIT;
6936
    LOG_WARN("not init", K(ret));
6937
  } else if (!arg.is_valid()) {
6938
    ret = OB_INVALID_ARGUMENT;
6939
    LOG_WARN("invalid arg", K(arg), K(ret));
6940
  } else {
6941
    uint64_t tenant_id = arg.tenant_id_;
6942
    const ObString &db_name = arg.db_name_;
6943
    const ObString &udt_name = arg.udt_name_;
6944
    ObSchemaGetterGuard schema_guard;
6945
    const ObDatabaseSchema *db_schema = NULL;
6946
    if (db_name.empty()) {
6947
      ret = OB_ERR_NO_DB_SELECTED;
6948
      LOG_WARN("no database selected", K(ret), K(db_name));
6949
    } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
6950
      LOG_WARN("get schema guard in inner table failed", K(ret));
6951
    } else if (OB_FAIL(schema_guard.get_database_schema(tenant_id, db_name, db_schema))) {
6952
      LOG_WARN("get database schema failed", K(ret));
6953
    } else if (NULL == db_schema) {
6954
      ret = OB_ERR_BAD_DATABASE;
6955
      LOG_USER_ERROR(OB_ERR_BAD_DATABASE, db_name.length(), db_name.ptr());
6956
    } else if (db_schema->is_in_recyclebin()) {
6957
      ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
6958
      LOG_WARN("Can't not create udt of db in recyclebin", K(ret), K(arg), K(*db_schema));
6959
    } else if (OB_INVALID_ID == db_schema->get_database_id()) {
6960
      ret = OB_ERR_BAD_DATABASE;
6961
      LOG_WARN("database id is invalid", K(tenant_id), K(*db_schema), K(ret));
6962
    }
6963

6964
    if (OB_SUCC(ret)) {
6965
      if (!arg.is_type_body_) {
6966
        bool exist = false;
6967
        const ObUDTTypeInfo *udt_info = NULL;
6968
        ObUDTTypeInfo udt;
6969
        const ObUDTTypeCode type_code = ObUDTTypeCode::UDT_TYPE_OBJECT;
6970
        if (OB_FAIL(schema_guard.check_udt_exist(tenant_id,
6971
                                                db_schema->get_database_id(),
6972
                                                OB_INVALID_ID, type_code,
6973
                                                udt_name, exist))) {
6974
            LOG_WARN("failed to check udt info exist", K(udt_name), K(ret));
6975
        } else if (exist) {
6976
          if (OB_FAIL(schema_guard.get_udt_info(tenant_id,
6977
                                                db_schema->get_database_id(),
6978
                                                OB_INVALID_ID,
6979
                                                udt_name, type_code, udt_info))) {
6980
            LOG_WARN("get udt info failed", K(ret));
6981
          }
6982
        } else if (!arg.if_exist_) {
6983
          ret = OB_ERR_SP_DOES_NOT_EXIST;
6984
          LOG_USER_ERROR(OB_ERR_SP_DOES_NOT_EXIST, "UDT", db_name.length(), db_name.ptr(),
6985
                        udt_name.length(), udt_name.ptr());
6986
        }
6987

6988
        if (OB_SUCC(ret) && !OB_ISNULL(udt_info)) {
6989
          if (OB_FAIL(udt.assign(*udt_info))) {
6990
            LOG_WARN("assign udt info failed", K(ret), KPC(udt_info));
6991
          } else if (udt.is_object_type()) {
6992
              udt.clear_property_flag(ObUDTTypeFlag::UDT_FLAG_OBJECT_TYPE_BODY);
6993
              udt.set_object_ddl_type(ObUDTTypeFlag::UDT_FLAG_OBJECT_TYPE_SPEC);
6994
          }
6995
          if (OB_SUCC(ret)
6996
           && OB_FAIL(ddl_service_.drop_udt(udt, schema_guard, &arg.ddl_stmt_str_))) {
6997
            LOG_WARN("drop udt failed", K(ret), K(udt_name), K(*udt_info));
6998
          }
6999
        }
7000
      } else {
7001
        bool exist = false;
7002
        const ObUDTTypeInfo *udt_info = NULL;
7003
        ObUDTTypeInfo udt;
7004
        const ObUDTTypeCode type_code = ObUDTTypeCode::UDT_TYPE_OBJECT_BODY;
7005
        if (OB_FAIL(schema_guard.check_udt_exist(tenant_id,
7006
                                                db_schema->get_database_id(),
7007
                                                OB_INVALID_ID, type_code,
7008
                                                udt_name, exist))) {
7009
            LOG_WARN("failed to check udt info exist", K(udt_name), K(ret));
7010
        } else if (exist) {
7011
          if (OB_FAIL(schema_guard.get_udt_info(tenant_id,
7012
                                                db_schema->get_database_id(),
7013
                                                OB_INVALID_ID,
7014
                                                udt_name, type_code, udt_info))) {
7015
            LOG_WARN("get udt info failed", K(ret));
7016
          }
7017
        } else if (!arg.if_exist_) {
7018
          ret = OB_ERR_SP_DOES_NOT_EXIST;
7019
          LOG_USER_ERROR(OB_ERR_SP_DOES_NOT_EXIST, "UDT", db_name.length(), db_name.ptr(),
7020
                        udt_name.length(), udt_name.ptr());
7021
        }
7022
        if (OB_SUCC(ret) && exist && !OB_ISNULL(udt_info)) {
7023
          if (OB_FAIL(udt.assign(*udt_info))) {
7024
            LOG_WARN("assign udt info failed", K(ret), KPC(udt_info));
7025
          } else if (udt.is_object_type()) {
7026
              udt.clear_property_flag(ObUDTTypeFlag::UDT_FLAG_OBJECT_TYPE_SPEC);
7027
              udt.set_object_ddl_type(ObUDTTypeFlag::UDT_FLAG_OBJECT_TYPE_BODY);
7028
          }
7029
          if (OB_SUCC(ret)
7030
            && OB_FAIL(ddl_service_.drop_udt(udt, schema_guard, &arg.ddl_stmt_str_))) {
7031
            LOG_WARN("drop udt failed", K(ret), K(udt_name), K(*udt_info));
7032
          }
7033
        }
7034
      }
7035
    }
7036
  }
7037
  return ret;
7038
}
7039

7040
//----Functions for managing dblinks----
7041

7042
int ObRootService::create_dblink(const obrpc::ObCreateDbLinkArg &arg)
7043
{
7044
  int ret = OB_SUCCESS;
7045
  if (OB_FAIL(ddl_service_.create_dblink(arg, &arg.ddl_stmt_str_))) {
7046
    LOG_WARN("create_dblink failed", K(arg.dblink_info_), K(ret));
7047
  }
7048
  return ret;
7049
}
7050

7051
int ObRootService::drop_dblink(const obrpc::ObDropDbLinkArg &arg)
7052
{
7053
  int ret = OB_SUCCESS;
7054
  if (OB_FAIL(ddl_service_.drop_dblink(arg, &arg.ddl_stmt_str_))) {
7055
    LOG_WARN("drop_dblink failed", K(arg.tenant_id_), K(arg.dblink_name_), K(ret));
7056
  }
7057
  return ret;
7058
}
7059

7060

7061
//----End of functions for managing dblinks----
7062

7063
//-----Functions for managing synonyms-----
7064
int ObRootService::create_synonym(const ObCreateSynonymArg &arg)
7065
{
7066
  int ret = OB_SUCCESS;
7067
  if (!inited_) {
7068
    ret = OB_NOT_INIT;
7069
    LOG_WARN("not init", K(ret));
7070
  } else {
7071
    ObSynonymInfo synonym_info = arg.synonym_info_;
7072
    uint64_t tenant_id = synonym_info.get_tenant_id();
7073
    ObString database_name = arg.db_name_;
7074
    ObString obj_database_name = arg.obj_db_name_;
7075
    ObSchemaGetterGuard schema_guard;
7076
    const ObDatabaseSchema *db_schema = NULL;
7077
    const ObDatabaseSchema *obj_db_schema = NULL;
7078
    if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
7079
      LOG_WARN("get schema guard in inner table failed", K(ret));
7080
    } else if (OB_FAIL(schema_guard.get_database_schema(tenant_id, database_name, db_schema))) {
7081
      LOG_WARN("get database schema failed", K(database_name), K(ret));
7082
    } else if (OB_FAIL(schema_guard.get_database_schema(tenant_id, obj_database_name, obj_db_schema))) {
7083
      LOG_WARN("get database schema failed", K(obj_database_name), K(ret));
7084
    } else if (OB_ISNULL(db_schema)) {
7085
      ret = OB_ERR_BAD_DATABASE;
7086
      LOG_USER_ERROR(OB_ERR_BAD_DATABASE, database_name.length(), database_name.ptr());
7087
    } else if (OB_ISNULL(obj_db_schema)) {
7088
      ret = OB_ERR_BAD_DATABASE;
7089
      LOG_USER_ERROR(OB_ERR_BAD_DATABASE, obj_database_name.length(), obj_database_name.ptr());
7090
    } else if (OB_UNLIKELY(db_schema->is_in_recyclebin())
7091
               || OB_UNLIKELY(obj_db_schema->is_in_recyclebin())) {
7092
      ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
7093
      LOG_WARN("can't create synonym of db in recyclebin", K(arg), KPC(db_schema), KPC(obj_db_schema), K(ret));
7094
    } else if (OB_UNLIKELY(OB_INVALID_ID == db_schema->get_database_id())
7095
               || OB_UNLIKELY(OB_INVALID_ID == obj_db_schema->get_database_id())) {
7096
      ret = OB_ERR_BAD_DATABASE;
7097
      LOG_WARN("database id is invalid", K(tenant_id), KPC(db_schema), KPC(obj_db_schema), K(ret));
7098
    } else {
7099
      synonym_info.set_database_id(db_schema->get_database_id());
7100
      synonym_info.set_object_database_id(obj_db_schema->get_database_id());
7101
    }
7102
    bool is_update = false;
7103
    if (OB_SUCC(ret)) {
7104
      ObArray<ObSchemaType> conflict_schema_types;
7105
      if (OB_FAIL(schema_guard.check_oracle_object_exist(tenant_id, synonym_info.get_database_id(),
7106
          synonym_info.get_synonym_name_str(), SYNONYM_SCHEMA, INVALID_ROUTINE_TYPE,
7107
          arg.or_replace_, conflict_schema_types))) {
7108
        LOG_WARN("fail to check oracle_object exist", K(ret), K(synonym_info));
7109
      } else if (conflict_schema_types.count() > 0) {
7110
        ret = OB_ERR_EXIST_OBJECT;
7111
        LOG_WARN("Name is already used by an existing object",
7112
                 K(ret), K(synonym_info), K(conflict_schema_types));
7113
      }
7114
    }
7115
    if (OB_SUCC(ret)) {
7116
      // can not delete, it will update synonym_info between check_synonym_exist.
7117
      if (OB_FAIL(ddl_service_.check_synonym_exist(synonym_info, arg.or_replace_, is_update))) {
7118
        LOG_WARN("failed to check_synonym_exist", K(synonym_info), K(arg.or_replace_), K(is_update), K(ret));
7119
      }
7120
    }
7121
    if (OB_SUCC(ret)) {
7122
      if (OB_FAIL(ddl_service_.create_synonym(synonym_info, arg.dependency_info_, &arg.ddl_stmt_str_, is_update, schema_guard))) {
7123
        LOG_WARN("create_synonym failed", K(synonym_info), K(ret));
7124
      }
7125
    }
7126
  }
7127
  return ret;
7128
}
7129

7130
int ObRootService::drop_synonym(const obrpc::ObDropSynonymArg &arg)
7131
{
7132
  int ret = OB_SUCCESS;
7133
  if (!inited_) {
7134
    ret = OB_NOT_INIT;
7135
    LOG_WARN("not init", K(ret));
7136
  } else if (OB_FAIL(ddl_service_.drop_synonym(arg))) {
7137
    LOG_WARN("ddl service failed to drop synonym", K(arg), K(ret));
7138
  }
7139
  return ret;
7140
}
7141
//-----End of functions for managing synonyms-----
7142

7143
#ifdef OB_BUILD_SPM
7144
//-----Functions for managing plan_baselines-----
7145
int ObRootService::accept_plan_baseline(const ObModifyPlanBaselineArg &arg)
7146
{
7147
  int ret = OB_SUCCESS;
7148
  ObZone null_zone;
7149
  ObSEArray<ObAddr, 8> server_list;
7150
  if (!inited_) {
7151
    ret = OB_NOT_INIT;
7152
    LOG_WARN("not init", K(ret));
7153
  } else if (OB_FAIL(SVR_TRACER.get_alive_servers(null_zone, server_list))) {
7154
    LOG_WARN("fail to get alive server", K(ret));
7155
  } else {
7156
    for (int64_t i = 0; OB_SUCC(ret) && i < server_list.count(); i++) {
7157
      if (OB_FAIL(rpc_proxy_.to(server_list.at(i))
7158
                            .by(arg.tenant_id_)
7159
                            .as(arg.tenant_id_)
7160
                            .svr_accept_plan_baseline(arg))) {
7161
        LOG_WARN("fail to accept plan baseline", K(ret), K(server_list.at(i)));
7162
        ret = OB_SUCCESS;
7163
      } else { /*do nothing*/}
7164
    }
7165
  }
7166
  return ret;
7167
}
7168

7169
int ObRootService::cancel_evolve_task(const ObModifyPlanBaselineArg &arg)
7170
{
7171
  int ret = OB_SUCCESS;
7172
  ObZone null_zone;
7173
  ObSEArray<ObAddr, 8> server_list;
7174
  if (!inited_) {
7175
    ret = OB_NOT_INIT;
7176
    LOG_WARN("not init", K(ret));
7177
  } else if (OB_FAIL(SVR_TRACER.get_alive_servers(null_zone, server_list))) {
7178
    LOG_WARN("fail to get alive server", K(ret));
7179
  } else {
7180
    for (int64_t i = 0; OB_SUCC(ret) && i < server_list.count(); i++) {
7181
      if (OB_FAIL(rpc_proxy_.to(server_list.at(i))
7182
                            .by(arg.tenant_id_)
7183
                            .as(arg.tenant_id_)
7184
                            .svr_cancel_evolve_task(arg))) {
7185
        LOG_WARN("fail to accept plan baseline", K(ret), K(server_list.at(i)));
7186
        ret = OB_SUCCESS;
7187
      } else { /*do nothing*/}
7188
    }
7189
  }
7190
  return ret;
7191
}
7192

7193
int ObRootService::admin_load_baseline(const obrpc::ObLoadPlanBaselineArg &arg)
7194
{
7195
  int ret = OB_SUCCESS;
7196
  if (!inited_) {
7197
    ret = OB_NOT_INIT;
7198
    LOG_WARN("not init", K(ret));
7199
  } else {
7200
    ObSystemAdminCtx ctx;
7201
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
7202
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
7203
    } else {
7204
      ObAdminLoadBaseline admin_util(ctx);
7205
      if (OB_FAIL(admin_util.execute(arg))) {
7206
        LOG_WARN("dispatch flush cache failed", K(arg), K(ret));
7207
      }
7208
      ROOTSERVICE_EVENT_ADD("root_service", "admin_load_baseline", K(ret), K(arg));
7209
    }
7210
  }
7211
  return ret;
7212
}
7213

7214
int ObRootService::admin_load_baseline_v2(const obrpc::ObLoadPlanBaselineArg &arg,
7215
                                          obrpc::ObLoadBaselineRes &res)
7216
{
7217
  int ret = OB_SUCCESS;
7218
  if (!inited_) {
7219
    ret = OB_NOT_INIT;
7220
    LOG_WARN("not init", K(ret));
7221
  } else {
7222
    ObSystemAdminCtx ctx;
7223
    uint64_t load_count = 0;
7224
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
7225
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
7226
    } else {
7227
      ObAdminLoadBaselineV2 admin_util(ctx);
7228
      if (OB_FAIL(admin_util.execute(arg, load_count))) {
7229
        LOG_WARN("dispatch flush cache failed", K(arg), K(ret));
7230
      } else {
7231
        res.load_count_ = load_count;
7232
      }
7233
      ROOTSERVICE_EVENT_ADD("root_service", "admin_load_baseline", K(ret), K(arg));
7234
    }
7235
  }
7236
  return ret;
7237
}
7238

7239
//-----End of functions for managing plan_baselines-----
7240
#endif
7241

7242
int ObRootService::admin_sync_rewrite_rules(const obrpc::ObSyncRewriteRuleArg &arg)
7243
{
7244
  int ret = OB_SUCCESS;
7245
  if (!inited_) {
7246
    ret = OB_NOT_INIT;
7247
    LOG_WARN("not init", K(ret));
7248
  } else {
7249
    ObSystemAdminCtx ctx;
7250
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
7251
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
7252
    } else {
7253
      ObAdminSyncRewriteRules admin_util(ctx);
7254
      if (OB_FAIL(admin_util.execute(arg))) {
7255
        LOG_WARN("dispatch sync rewrite rules failed", K(arg), K(ret));
7256
      }
7257
      ROOTSERVICE_EVENT_ADD("root_service", "admin_sync_rewrite_rules", K(ret), K(arg));
7258
    }
7259
  }
7260
  return ret;
7261
}
7262

7263
int ObRootService::create_package(const obrpc::ObCreatePackageArg &arg)
7264
{
7265
  int ret = OB_SUCCESS;
7266
  if (!inited_) {
7267
    ret = OB_NOT_INIT;
7268
    LOG_WARN("not init", K(ret));
7269
  } else if (!arg.is_valid()) {
7270
    ret = OB_INVALID_ARGUMENT;
7271
    LOG_WARN("invalid arg", K(arg), K(ret));
7272
  } else {
7273
    ObPackageInfo new_package_info = arg.package_info_;
7274
    const ObPackageInfo *old_package_info = NULL;
7275
    uint64_t tenant_id = new_package_info.get_tenant_id();
7276
    ObString database_name = arg.db_name_;
7277
    ObSchemaGetterGuard schema_guard;
7278
    const ObDatabaseSchema *db_schema = NULL;
7279
    const ObUserInfo *user_info = NULL;
7280
    bool is_oracle_mode = false;
7281
    if (OB_FAIL(ObCompatModeGetter::check_is_oracle_mode_with_tenant_id(
7282
                tenant_id, is_oracle_mode))) {
7283
      LOG_WARN("fail to check is oracle mode", K(ret));
7284
    } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
7285
      LOG_WARN("get schema guard in inner table failed", K(ret));
7286
    } else if (OB_FAIL(schema_guard.get_database_schema(tenant_id, database_name, db_schema))) {
7287
      LOG_WARN("get database schema failed", K(ret));
7288
    } else if (NULL == db_schema) {
7289
      ret = OB_ERR_BAD_DATABASE;
7290
      LOG_USER_ERROR(OB_ERR_BAD_DATABASE, database_name.length(), database_name.ptr());
7291
    } else if (db_schema->is_in_recyclebin()) {
7292
      ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
7293
      LOG_WARN("Can't not create package of db in recyclebin", K(ret), K(arg), K(*db_schema));
7294
    } else if (OB_INVALID_ID == db_schema->get_database_id()) {
7295
      ret = OB_ERR_BAD_DATABASE;
7296
      LOG_WARN("database id is invalid", K(tenant_id), K(*db_schema), K(ret));
7297
    } else {
7298
      new_package_info.set_database_id(db_schema->get_database_id());
7299
    }
7300
    if (OB_SUCC(ret)
7301
        && database_name.case_compare(OB_SYS_DATABASE_NAME) != 0
7302
        && is_oracle_mode) {
7303
      if (OB_FAIL(schema_guard.get_user_info(
7304
        tenant_id, database_name, ObString(OB_DEFAULT_HOST_NAME), user_info))) {
7305
        LOG_WARN("failed to get user info", K(ret), K(database_name));
7306
      } else if (OB_ISNULL(user_info)) {
7307
        ret = OB_USER_NOT_EXIST;
7308
        LOG_WARN("user is does not exist", K(ret), K(database_name));
7309
      } else if (OB_INVALID_ID == user_info->get_user_id()) {
7310
        ret = OB_USER_NOT_EXIST;
7311
        LOG_WARN("user id is invalid", K(ret), K(database_name));
7312
      } else {
7313
        new_package_info.set_owner_id(user_info->get_user_id());
7314
      }
7315
    }
7316
    if (OB_SUCC(ret)) {
7317
      ObArray<ObSchemaType> conflict_schema_types;
7318
      // package body 查重只比较有没有重复的 package body，package 查重需要比较除 package body 以外的对象
7319
      if (PACKAGE_TYPE == new_package_info.get_type()
7320
          && OB_FAIL(schema_guard.check_oracle_object_exist(tenant_id,
7321
             db_schema->get_database_id(), new_package_info.get_package_name(), PACKAGE_SCHEMA,
7322
             INVALID_ROUTINE_TYPE, arg.is_replace_, conflict_schema_types))) {
7323
        LOG_WARN("fail to check object exist", K(ret), K(new_package_info.get_package_name()));
7324
      } else if (conflict_schema_types.count() > 0) {
7325
        // 这里检查 oracle 模式下新对象的名字是否已经被其他对象占用了
7326
        ret = OB_ERR_EXIST_OBJECT;
7327
        LOG_WARN("Name is already used by an existing object in oralce mode",
7328
                 K(ret), K(new_package_info.get_package_name()),
7329
                 K(conflict_schema_types));
7330
      }
7331
    }
7332
    if (OB_SUCC(ret)) {
7333
      if (OB_FAIL(schema_guard.get_package_info(tenant_id, db_schema->get_database_id(), new_package_info.get_package_name(),
7334
                                                new_package_info.get_type(), new_package_info.get_compatibility_mode(),
7335
                                                old_package_info))) {
7336
        LOG_WARN("failed to check package info exist", K(new_package_info), K(ret));
7337
      } else if (OB_ISNULL(old_package_info) || arg.is_replace_) {
7338
        bool need_create = true;
7339
        // 对于系统包, 为了避免多次重建, 比较下新的系统包与已经存在的系统包是否相同
7340
        if (OB_NOT_NULL(old_package_info) && OB_SYS_TENANT_ID == tenant_id) {
7341
          if (old_package_info->get_source().length() == new_package_info.get_source().length()
7342
              && (0 == MEMCMP(old_package_info->get_source().ptr(),
7343
                              new_package_info.get_source().ptr(),
7344
                              old_package_info->get_source().length()))
7345
              && old_package_info->get_exec_env() == new_package_info.get_exec_env()) {
7346
            need_create = false;
7347
            LOG_INFO("do not recreate package with same source",
7348
                     K(ret),
7349
                     K(old_package_info->get_source()),
7350
                     K(new_package_info.get_source()), K(need_create));
7351
          } else {
7352
            LOG_INFO("recreate package with diff source",
7353
                     K(ret),
7354
                     K(old_package_info->get_source()),
7355
                     K(new_package_info.get_source()), K(need_create));
7356
          }
7357
        }
7358
        if (need_create) {
7359
          ObSArray<ObRoutineInfo> &public_routine_infos = const_cast<ObSArray<ObRoutineInfo> &>(arg.public_routine_infos_);
7360
          ObErrorInfo error_info = arg.error_info_;
7361
          ObSArray<ObDependencyInfo> &dep_infos =
7362
                               const_cast<ObSArray<ObDependencyInfo> &>(arg.dependency_infos_);
7363
          if (OB_FAIL(ddl_service_.create_package(schema_guard,
7364
                                                  old_package_info,
7365
                                                  new_package_info,
7366
                                                  public_routine_infos,
7367
                                                  error_info,
7368
                                                  dep_infos,
7369
                                                  &arg.ddl_stmt_str_))) {
7370
            LOG_WARN("create package failed", K(ret), K(new_package_info));
7371
          }
7372
        }
7373
      } else {
7374
        ret = OB_ERR_PACKAGE_ALREADY_EXISTS;
7375
        const char *type = (new_package_info.get_type() == PACKAGE_TYPE ? "PACKAGE" : "PACKAGE BODY");
7376
        LOG_USER_ERROR(OB_ERR_PACKAGE_ALREADY_EXISTS, type,
7377
                       database_name.length(), database_name.ptr(),
7378
                       new_package_info.get_package_name().length(), new_package_info.get_package_name().ptr());
7379
      }
7380
    }
7381
  }
7382
  return ret;
7383
}
7384

7385
int ObRootService::alter_package(const obrpc::ObAlterPackageArg &arg)
7386
{
7387
  int ret = OB_SUCCESS;
7388
  if (!inited_) {
7389
    ret = OB_NOT_INIT;
7390
    LOG_WARN("not init", K(ret));
7391
  } else if (!arg.is_valid()) {
7392
    ret = OB_INVALID_ARGUMENT;
7393
    LOG_WARN("invalid arg", K(arg), K(ret));
7394
  } else {
7395
    uint64_t tenant_id = arg.tenant_id_;
7396
    const ObString &db_name = arg.db_name_;
7397
    const ObString &package_name = arg.package_name_;
7398
    ObPackageType package_type = arg.package_type_;
7399
    int64_t compatible_mode =  arg.compatible_mode_;
7400
    ObSchemaGetterGuard schema_guard;
7401
    const ObDatabaseSchema *db_schema = NULL;
7402
    if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
7403
      LOG_WARN("get schema guard in inner table failed", K(ret));
7404
    } else if (OB_FAIL(schema_guard.get_database_schema(tenant_id, db_name, db_schema))) {
7405
      LOG_WARN("get database schema failed", K(ret));
7406
    } else if (NULL == db_schema) {
7407
      ret = OB_ERR_BAD_DATABASE;
7408
      LOG_WARN("database id is invalid", K(tenant_id), K(db_name), K(ret));
7409
    } else if (db_schema->is_in_recyclebin()) {
7410
      ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
7411
      LOG_WARN("Can't not create package of db in recyclebin", K(ret), K(arg), K(*db_schema));
7412
    }
7413
    if (OB_SUCC(ret)) {
7414
      bool exist = false;
7415
      ObSArray<ObRoutineInfo> &public_routine_infos = const_cast<ObSArray<ObRoutineInfo> &>(arg.public_routine_infos_);
7416
      if (OB_FAIL(schema_guard.check_package_exist(tenant_id, db_schema->get_database_id(),
7417
                                                   package_name, package_type, compatible_mode, exist))) {
7418
        LOG_WARN("failed to check package info exist", K(package_name), K(ret));
7419
      } else if (exist) {
7420
        const ObPackageInfo *package_info = NULL;
7421
        if (OB_FAIL(schema_guard.get_package_info(tenant_id, db_schema->get_database_id(), package_name, package_type,
7422
                                                  compatible_mode, package_info))) {
7423
          LOG_WARN("get package info failed", K(ret));
7424
        } else if (OB_ISNULL(package_info)) {
7425
          ret = OB_ERR_UNEXPECTED;
7426
          LOG_WARN("package info is null", K(db_schema->get_database_id()), K(package_name), K(package_type), K(ret));
7427
        } else if (OB_FAIL(ddl_service_.alter_package(schema_guard,
7428
                                                      *package_info,
7429
                                                      public_routine_infos,
7430
                                                      const_cast<ObErrorInfo &>(arg.error_info_),
7431
                                                      &arg.ddl_stmt_str_))) {
7432
          LOG_WARN("drop package failed", K(ret), K(package_name));
7433
        }
7434
      } else {
7435
        ret = OB_ERR_PACKAGE_DOSE_NOT_EXIST;
7436
        const char *type = (package_type == PACKAGE_TYPE ? "PACKAGE" : "PACKAGE BODY");
7437
        LOG_USER_ERROR(OB_ERR_PACKAGE_DOSE_NOT_EXIST, type,
7438
                       db_schema->get_database_name_str().length(), db_schema->get_database_name(),
7439
                       package_name.length(), package_name.ptr());
7440
      }
7441
    }
7442
  }
7443

7444
  return ret;
7445
}
7446

7447
int ObRootService::drop_package(const obrpc::ObDropPackageArg &arg)
7448
{
7449
  int ret = OB_SUCCESS;
7450
  if (!inited_) {
7451
    ret = OB_NOT_INIT;
7452
    LOG_WARN("not init", K(ret));
7453
  } else if (!arg.is_valid()) {
7454
    ret = OB_INVALID_ARGUMENT;
7455
    LOG_WARN("invalid arg", K(arg), K(ret));
7456
  } else {
7457
    uint64_t tenant_id = arg.tenant_id_;
7458
    const ObString &db_name = arg.db_name_;
7459
    const ObString &package_name = arg.package_name_;
7460
    ObPackageType package_type = arg.package_type_;
7461
    int64_t compatible_mode = arg.compatible_mode_;
7462
    ObSchemaGetterGuard schema_guard;
7463
    const ObDatabaseSchema *db_schema = NULL;
7464
    if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
7465
      LOG_WARN("get schema guard in inner table failed", K(ret));
7466
    } else if (OB_FAIL(schema_guard.get_database_schema(tenant_id, db_name, db_schema))) {
7467
      LOG_WARN("get database schema failed", K(ret));
7468
    } else if (NULL == db_schema) {
7469
      ret = OB_ERR_BAD_DATABASE;
7470
      LOG_USER_ERROR(OB_ERR_BAD_DATABASE, db_name.length(), db_name.ptr());
7471
    } else if (db_schema->is_in_recyclebin()) {
7472
      ret = OB_ERR_OPERATION_ON_RECYCLE_OBJECT;
7473
      LOG_WARN("Can't not create package of db in recyclebin", K(ret), K(arg), K(*db_schema));
7474
    } else if (OB_INVALID_ID == db_schema->get_database_id()) {
7475
      ret = OB_ERR_BAD_DATABASE;
7476
      LOG_WARN("database id is invalid", K(tenant_id), K(*db_schema), K(ret));
7477
    }
7478
    if (OB_SUCC(ret)) {
7479
      bool exist = false;
7480
      if (OB_FAIL(schema_guard.check_package_exist(tenant_id, db_schema->get_database_id(),
7481
          package_name, package_type, compatible_mode, exist))) {
7482
        LOG_WARN("failed to check package info exist", K(package_name), K(ret));
7483
      } else if (exist) {
7484
        const ObPackageInfo *package_info = NULL;
7485
        ObErrorInfo error_info = arg.error_info_;
7486
        if (OB_FAIL(schema_guard.get_package_info(tenant_id, db_schema->get_database_id(), package_name, package_type, compatible_mode, package_info))) {
7487
          LOG_WARN("get package info failed", K(ret));
7488
        } else if (OB_FAIL(ddl_service_.drop_package(*package_info,
7489
                                                     error_info,
7490
                                                     &arg.ddl_stmt_str_))) {
7491
          LOG_WARN("drop package failed", K(ret), K(package_name));
7492
        }
7493
      } else {
7494
        ret = OB_ERR_PACKAGE_DOSE_NOT_EXIST;
7495
        const char *type = (package_type == PACKAGE_TYPE ? "PACKAGE" : "PACKAGE BODY");
7496
        LOG_USER_ERROR(OB_ERR_PACKAGE_DOSE_NOT_EXIST, type,
7497
                       db_name.length(), db_name.ptr(),
7498
                       package_name.length(), package_name.ptr());
7499
      }
7500
    }
7501
  }
7502
  return ret;
7503
}
7504

7505
int ObRootService::create_trigger(const obrpc::ObCreateTriggerArg &arg)
7506
{
7507
  int ret = OB_SUCCESS;
7508
  ObSchemaGetterGuard schema_guard;
7509
  if (!inited_) {
7510
    ret = OB_NOT_INIT;
7511
    LOG_WARN("not init", K(ret));
7512
  } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(
7513
                     arg.trigger_info_.get_tenant_id(), schema_guard))) {
7514
    LOG_WARN("get schema guard with version in inner table failed", K(ret));
7515
  } else if (OB_FAIL(ddl_service_.create_trigger(arg, schema_guard, NULL))) {
7516
    LOG_WARN("failed to create trigger", K(ret));
7517
  }
7518
  return ret;
7519
}
7520

7521
int ObRootService::create_trigger_with_res(const obrpc::ObCreateTriggerArg &arg,
7522
                                           obrpc::ObCreateTriggerRes &res)
7523
{
7524
  int ret = OB_SUCCESS;
7525
  ObSchemaGetterGuard schema_guard;
7526
  if (!inited_) {
7527
    ret = OB_NOT_INIT;
7528
    LOG_WARN("not init", K(ret));
7529
  } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(
7530
                     arg.trigger_info_.get_tenant_id(), schema_guard))) {
7531
    LOG_WARN("get schema guard with version in inner table failed", K(ret));
7532
  } else if (OB_FAIL(check_parallel_ddl_conflict(schema_guard, arg))) {
7533
    LOG_WARN("check parallel ddl conflict failed", K(ret));
7534
  } else if (OB_FAIL(ddl_service_.create_trigger(arg, schema_guard, &res))) {
7535
    LOG_WARN("failed to create trigger", K(ret));
7536
  }
7537
  return ret;
7538
}
7539

7540
int ObRootService::alter_trigger(const obrpc::ObAlterTriggerArg &arg)
7541
{
7542
  int ret = OB_SUCCESS;
7543
  if (!inited_) {
7544
    ret = OB_NOT_INIT;
7545
    LOG_WARN("not init", K(ret));
7546
  } else if (!arg.is_valid()) {
7547
    ret = OB_INVALID_ARGUMENT;
7548
    LOG_WARN("invalid arg", K(arg), K(ret));
7549
  } else if (OB_FAIL(ddl_service_.alter_trigger(arg))) {
7550
    LOG_WARN("failed to alter trigger", K(ret));
7551
  }
7552
  return ret;
7553
}
7554

7555
int ObRootService::drop_trigger(const obrpc::ObDropTriggerArg &arg)
7556
{
7557
  int ret = OB_SUCCESS;
7558
  if (!inited_) {
7559
    ret = OB_NOT_INIT;
7560
    LOG_WARN("not init", K(ret));
7561
  } else
7562
  if (!arg.is_valid()) {
7563
    ret = OB_INVALID_ARGUMENT;
7564
    LOG_WARN("invalid arg", K(arg), K(ret));
7565
  } else if (OB_FAIL(ddl_service_.drop_trigger(arg))) {
7566
    LOG_WARN("failed to drop trigger", K(ret));
7567
  }
7568
  return ret;
7569
}
7570

7571
////////////////////////////////////////////////////////////////
7572
// sequence
7573
////////////////////////////////////////////////////////////////
7574
int ObRootService::do_sequence_ddl(const obrpc::ObSequenceDDLArg &arg)
7575
{
7576
  int ret = OB_SUCCESS;
7577
  if (!inited_) {
7578
    ret = OB_NOT_INIT;
7579
    LOG_WARN("not init", K(ret));
7580
  } else if (OB_FAIL(ddl_service_.do_sequence_ddl(arg))) {
7581
    LOG_WARN("do sequence ddl failed", K(arg), K(ret));
7582
  }
7583
  return ret;
7584
}
7585

7586
////////////////////////////////////////////////////////////////
7587
// context
7588
////////////////////////////////////////////////////////////////
7589
int ObRootService::do_context_ddl(const obrpc::ObContextDDLArg &arg)
7590
{
7591
  int ret = OB_SUCCESS;
7592
  if (!inited_) {
7593
    ret = OB_NOT_INIT;
7594
    LOG_WARN("not init", K(ret));
7595
  } else if (OB_FAIL(ddl_service_.do_context_ddl(arg))) {
7596
    LOG_WARN("do context ddl failed", K(arg), K(ret));
7597
  }
7598
  return ret;
7599
}
7600

7601
////////////////////////////////////////////////////////////////
7602
// schema revise
7603
////////////////////////////////////////////////////////////////
7604
int ObRootService::schema_revise(const obrpc::ObSchemaReviseArg &arg)
7605
{
7606
  int ret = OB_SUCCESS;
7607
  if (!inited_) {
7608
    ret = OB_NOT_INIT;
7609
    LOG_WARN("not init", K(ret));
7610
  } else if (OB_FAIL(ddl_service_.do_schema_revise(arg))) {
7611
    LOG_WARN("schema revise failed", K(arg), K(ret));
7612
  }
7613
  return ret;
7614
}
7615

7616
////////////////////////////////////////////////////////////////
7617
// keystore
7618
////////////////////////////////////////////////////////////////
7619

7620
int ObRootService::do_keystore_ddl(const obrpc::ObKeystoreDDLArg &arg)
7621
{
7622
  int ret = OB_SUCCESS;
7623
  if (!inited_) {
7624
    ret = OB_NOT_INIT;
7625
    LOG_WARN("not init", K(ret));
7626
#ifndef OB_BUILD_TDE_SECURITY
7627
  } else if (OB_FAIL(ddl_service_.do_keystore_ddl(arg))) {
7628
    LOG_WARN("do sequence ddl failed", K(arg), K(ret));
7629
  }
7630
#else
7631
  } else {
7632
    // exclude add server
7633
    common::ObArray<uint64_t> tenant_id_array;
7634
    common::ObArray<std::pair<uint64_t, uint64_t> > max_key_version;
7635
    SpinRLockGuard sync_guard(master_key_mgr_.sync());
7636
    if (OB_FAIL(ddl_service_.do_keystore_ddl(arg))) {
7637
      LOG_WARN("do sequence ddl failed", K(arg), K(ret));
7638
    } else if (arg.type_ == ObKeystoreDDLArg::ALTER_KEYSTORE_SET_KEY) {
7639
      if (OB_FAIL(tenant_id_array.push_back(arg.exec_tenant_id_))) {
7640
        LOG_WARN("fail to push back", KR(ret));
7641
      } else if (OB_FAIL(ObMasterKeyGetter::instance().get_latest_key_versions(
7642
            tenant_id_array, max_key_version))) {
7643
        LOG_WARN("fail to get latest key versions", KR(ret));
7644
      } else if (max_key_version.count() != 1 || max_key_version.at(0).second <= 0) {
7645
        ret = OB_ERR_UNEXPECTED;
7646
        LOG_WARN("max key version unexpected", KR(ret), K(arg), K(max_key_version));
7647
      } else if (OB_FAIL(master_key_mgr_.forward_tenant_max_key_version(
7648
              arg.exec_tenant_id_, max_key_version.at(0).second))) {
7649
        LOG_WARN("fail to forward tenant max key version", KR(ret), K(arg), K(max_key_version));
7650
      }
7651
    } else {
7652
      // no new master key generated, ignore
7653
    }
7654
  }
7655
#endif
7656
  return ret;
7657
}
7658

7659
////////////////////////////////////////////////////////////////
7660
// label security policy
7661
////////////////////////////////////////////////////////////////
7662

7663
int ObRootService::handle_label_se_policy_ddl(const obrpc::ObLabelSePolicyDDLArg &arg)
7664
{
7665
  int ret = OB_SUCCESS;
7666
  if (!inited_) {
7667
    ret = OB_NOT_INIT;
7668
    LOG_WARN("not init", K(ret));
7669
  } else if (OB_FAIL(ddl_service_.handle_label_se_policy_ddl(arg))) {
7670
    LOG_WARN("do label security policy ddl failed", K(arg), K(ret));
7671
  }
7672
  return ret;
7673
}
7674

7675
int ObRootService::handle_label_se_component_ddl(const obrpc::ObLabelSeComponentDDLArg &arg)
7676
{
7677
  int ret = OB_SUCCESS;
7678
  if (!inited_) {
7679
    ret = OB_NOT_INIT;
7680
    LOG_WARN("not init", K(ret));
7681
  } else if (OB_FAIL(ddl_service_.handle_label_se_component_ddl(arg))) {
7682
    LOG_WARN("do label security policy ddl failed", K(arg), K(ret));
7683
  }
7684
  return ret;
7685
}
7686

7687
int ObRootService::handle_label_se_label_ddl(const obrpc::ObLabelSeLabelDDLArg &arg)
7688
{
7689
  int ret = OB_SUCCESS;
7690
  if (!inited_) {
7691
    ret = OB_NOT_INIT;
7692
    LOG_WARN("not init", K(ret));
7693
  } else if (OB_FAIL(ddl_service_.handle_label_se_label_ddl(arg))) {
7694
    LOG_WARN("do label security policy ddl failed", K(arg), K(ret));
7695
  }
7696
  return ret;
7697
}
7698

7699
int ObRootService::handle_label_se_user_level_ddl(const obrpc::ObLabelSeUserLevelDDLArg &arg)
7700
{
7701
  int ret = OB_SUCCESS;
7702
  if (!inited_) {
7703
    ret = OB_NOT_INIT;
7704
    LOG_WARN("not init", K(ret));
7705
  } else if (OB_FAIL(ddl_service_.handle_label_se_user_level_ddl(arg))) {
7706
    LOG_WARN("do label security policy ddl failed", K(arg), K(ret));
7707
  }
7708
  return ret;
7709
}
7710
// tablespace
7711
////////////////////////////////////////////////////////////////
7712
int ObRootService::do_tablespace_ddl(const obrpc::ObTablespaceDDLArg &arg)
7713
{
7714
  int ret = OB_SUCCESS;
7715
  if (!inited_) {
7716
    ret = OB_NOT_INIT;
7717
    LOG_WARN("not init", K(ret));
7718
  } else if (OB_FAIL(ddl_service_.do_tablespace_ddl(arg))) {
7719
    LOG_WARN("do sequence ddl failed", K(arg), K(ret));
7720
  }
7721

7722
  return ret;
7723
}
7724
////////////////////////////////////////////////////////////////
7725
// server & zone management
7726
////////////////////////////////////////////////////////////////
7727
int ObRootService::add_server_for_bootstrap_in_version_smaller_than_4_2_0(
7728
      const common::ObAddr &server,
7729
      const common::ObZone &zone)
7730
{
7731
  return server_manager_.add_server(server, zone);
7732
}
7733
int ObRootService::add_server(const obrpc::ObAdminServerArg &arg)
7734
{
7735
  int ret = OB_SUCCESS;
7736
  ObTimeoutCtx ctx;
7737
  if (OB_UNLIKELY(!inited_)) {
7738
    ret = OB_NOT_INIT;
7739
    LOG_WARN("not init", KR(ret), K(inited_));
7740
  } else if (!arg.is_valid()) {
7741
    ret = OB_INVALID_ARGUMENT;
7742
    LOG_WARN("invalid arg", KR(ret), K(arg));
7743
  } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
7744
      LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
7745
  } else {}
7746
  if (OB_SUCC(ret)) {
7747
    if (!ObHeartbeatService::is_service_enabled()) { // the old logic
7748
      LOG_INFO("sys tenant data version < 4.2, add_server", K(arg),
7749
          "timeout_ts", ctx.get_timeout());
7750
      if (OB_FAIL(old_add_server(arg))) {
7751
        LOG_WARN("fail to add server by using old logic", KR(ret), K(arg));
7752
      }
7753
    } else { // the new logic
7754
      LOG_INFO("sys tenant data version >= 4.2, add_server", K(arg),
7755
          "timeout_ts", ctx.get_timeout());
7756
      if (OB_FAIL(server_zone_op_service_.add_servers(arg.servers_, arg.zone_))) {
7757
        LOG_WARN("fail to add servers", KR(ret), K(arg));
7758
      }
7759
      int tmp_ret = OB_SUCCESS;
7760
      if (OB_TMP_FAIL(load_server_manager())) {
7761
        // ** FIXME (linqiucen.lqc): temp. solution.
7762
        // ** This will be removed if we do not need whitelist in server_manager
7763
        LOG_WARN("fail to load server_manager, please try 'ALTER SYSTEM RELOAD SERVER;'", KR(ret), KR(tmp_ret));
7764
        ret = OB_SUCC(ret) ? tmp_ret : ret;
7765
      }
7766
    }
7767
  }
7768
  FLOG_INFO("add server", KR(ret), K(arg));
7769
  return ret;
7770
}
7771
int ObRootService::old_add_server(const obrpc::ObAdminServerArg &arg)
7772
{
7773
  int ret = OB_SUCCESS;
7774
  uint64_t sys_data_version = 0;
7775
  // argument
7776
#ifdef OB_BUILD_TDE_SECURITY
7777
  ObWaitMasterKeyInSyncArg wms_in_sync_arg;
7778
  // master key mgr sync
7779
#endif
7780
  if (!inited_) {
7781
    ret = OB_NOT_INIT;
7782
    LOG_WARN("not init", K(ret));
7783
  } else if (!arg.is_valid()) {
7784
    ret = OB_INVALID_ARGUMENT;
7785
    LOG_WARN("invalid arg", K(arg), K(ret));
7786
  } else if (OB_FAIL(GET_MIN_DATA_VERSION(OB_SYS_TENANT_ID, sys_data_version))) {
7787
    LOG_WARN("fail to get sys data version", KR(ret));
7788
#ifdef OB_BUILD_TDE_SECURITY
7789
  } else if (OB_FAIL(server_zone_op_service_.construct_rs_list_arg(wms_in_sync_arg.rs_list_arg_))) {
7790
    LOG_WARN("fail to construct rs list arg", KR(ret));
7791
#endif
7792
  } else {
7793
    LOG_INFO("add_server", K(arg), "timeout_ts", THIS_WORKER.get_timeout_ts());
7794
    ObCheckServerEmptyArg new_arg(ObCheckServerEmptyArg::ADD_SERVER,
7795
                                  sys_data_version);
7796
    ObCheckDeploymentModeArg dp_arg;
7797
    dp_arg.single_zone_deployment_on_ = OB_FILE_SYSTEM_ROUTER.is_single_zone_deployment_on();
7798

7799
#ifdef OB_BUILD_TDE_SECURITY
7800
    SpinRLockGuard sync_guard(master_key_mgr_.sync());
7801
#endif
7802
    for (int64_t i = 0; OB_SUCC(ret) && i < arg.servers_.count(); ++i) {
7803
      const ObAddr &addr = arg.servers_[i];
7804
      Bool is_empty(false);
7805
      Bool is_deployment_mode_match(false);
7806
      if (OB_FAIL(rpc_proxy_.to(addr).is_empty_server(new_arg, is_empty))) {
7807
        LOG_WARN("fail to check is server empty", K(ret), K(addr));
7808
      } else if (!is_empty) {
7809
        ret = OB_OP_NOT_ALLOW;
7810
        LOG_WARN("add non-empty server not allowed", K(ret));
7811
        LOG_USER_ERROR(OB_OP_NOT_ALLOW, "add non-empty server");
7812
      } else if (OB_FAIL([&](){
7813
                           int ret = OB_SUCCESS;
7814
                           ret = rpc_proxy_.to(addr).check_deployment_mode_match(
7815
                             dp_arg, is_deployment_mode_match);
7816
                           return ret;}())) {
7817
        LOG_WARN("fail to check deployment mode match", K(ret));
7818
      } else if (!is_deployment_mode_match) {
7819
        ret = OB_OP_NOT_ALLOW;
7820
        LOG_WARN("add server deployment mode mot match not allowed", K(ret));
7821
        LOG_USER_ERROR(OB_OP_NOT_ALLOW, "add server deployment mode not match");
7822
#ifdef OB_BUILD_TDE_SECURITY
7823
      } else if (OB_FAIL(server_zone_op_service_.master_key_checking_for_adding_server(
7824
            addr,
7825
            arg.zone_,
7826
            wms_in_sync_arg))) {
7827
        LOG_WARN("master key checking for adding server is failed", KR(ret), K(addr), K(arg.zone_));
7828
#endif
7829
      }
7830
      if (OB_SUCC(ret)) {
7831
        if (OB_FAIL(server_manager_.add_server(addr, arg.zone_))) {
7832
          LOG_WARN("add_server failed", "server", addr,  "zone", arg.zone_, K(ret));
7833
        }
7834
      }
7835
    }
7836
  }
7837
  ROOTSERVICE_EVENT_ADD("root_service", "add_server", K(ret), K(arg));
7838
  return ret;
7839
}
7840
int ObRootService::delete_server(const obrpc::ObAdminServerArg &arg)
7841
{
7842
  int ret = OB_SUCCESS;
7843
  ObTimeoutCtx ctx;
7844
  if (OB_UNLIKELY(!inited_)) {
7845
    ret = OB_NOT_INIT;
7846
    LOG_WARN("not init", KR(ret), K(inited_));
7847
  } else if (!arg.is_valid()) {
7848
    ret = OB_INVALID_ARGUMENT;
7849
    LOG_WARN("invalid arg", KR(ret), K(arg));
7850
  } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
7851
    LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
7852
  } else {}
7853
  if (OB_SUCC(ret)) {
7854
    if (!ObHeartbeatService::is_service_enabled()) { // the old logic
7855
      LOG_INFO("sys tenant data version < 4.2, delete_server", K(arg),
7856
          "timeout_ts", ctx.get_timeout());
7857
      if (OB_FAIL(old_delete_server(arg))) {
7858
        LOG_WARN("fail to delete server by using the old logic", KR(ret), K(arg));
7859
      }
7860
    } else { // the new logic
7861
      LOG_INFO("sys tenant data version >= 4.2, delete_server", K(arg),
7862
          "timeout_ts", ctx.get_timeout());
7863
      if (OB_FAIL(server_zone_op_service_.delete_servers(arg.servers_, arg.zone_))) {
7864
        LOG_WARN("fail to delete servers", KR(ret), K(arg));
7865
      }
7866
      int tmp_ret = OB_SUCCESS;
7867
      if (OB_TMP_FAIL(load_server_manager())) {
7868
        // ** FIXME (linqiucen.lqc): temp. solution.
7869
        // ** This will be removed if we do not need whitelist in server_manager
7870
        LOG_WARN("fail to load server_manager, please try 'ALTER SYSTEM RELOAD SERVER;'", KR(ret), KR(tmp_ret));
7871
        ret = OB_SUCC(ret) ? tmp_ret : ret;
7872
      } else {
7873
        root_balancer_.wakeup();
7874
        empty_server_checker_.wakeup();
7875
        lost_replica_checker_.wakeup();
7876
        LOG_INFO("delete server and load server manager successfully", K(arg));
7877
      }
7878
    }
7879
  }
7880
  FLOG_INFO("delete server", KR(ret), K(arg));
7881
  return ret;
7882
}
7883
int ObRootService::old_delete_server(const obrpc::ObAdminServerArg &arg)
7884
{
7885
  int ret = OB_SUCCESS;
7886
  bool has_enough = false;
7887
  if (!inited_) {
7888
    ret = OB_NOT_INIT;
7889
    LOG_WARN("not init", K(ret));
7890
  } else if (!arg.is_valid()) {
7891
    ret = OB_INVALID_ARGUMENT;
7892
    LOG_WARN("invalid arg", K(arg), K(ret));
7893
  } else if (OB_FAIL(check_server_have_enough_resource_for_delete_server(arg.servers_, arg.zone_))) {
7894
    LOG_WARN("not enough resource, cannot delete servers", K(ret), K(arg));
7895
  } else if (OB_FAIL(check_all_ls_has_leader("delete server"))) {
7896
    LOG_WARN("fail to check all ls has leader", KR(ret), K(arg));
7897
  } else if (OB_FAIL(server_manager_.delete_server(arg.servers_, arg.zone_))) {
7898
    LOG_WARN("delete_server failed", "servers", arg.servers_, "zone", arg.zone_, K(ret));
7899
  } else {
7900
    root_balancer_.wakeup();
7901
    empty_server_checker_.wakeup();
7902
    lost_replica_checker_.wakeup();
7903
  }
7904
  ROOTSERVICE_EVENT_ADD("root_service", "delete_server", K(ret), K(arg));
7905
  return ret;
7906
}
7907

7908
int ObRootService::check_server_have_enough_resource_for_delete_server(
7909
                   const ObIArray<ObAddr> &servers,
7910
                   const ObZone &zone)
7911
{
7912
  int ret = OB_SUCCESS;
7913
  common::ObZone tmp_zone;
7914
  if (OB_UNLIKELY(!inited_)) {
7915
    ret = OB_NOT_INIT;
7916
    LOG_WARN("not init", K(ret));
7917
  } else {
7918
    FOREACH_CNT_X(server, servers, OB_SUCC(ret)) {
7919
      if (zone.is_empty()) {
7920
        // still use server manager here, since this func. will be called only in version < 4.2
7921
        if (OB_FAIL(server_manager_.get_server_zone(*server, tmp_zone))) {
7922
          LOG_WARN("fail to get server zone", K(ret));
7923
        }
7924
      } else {
7925
        if (OB_FAIL(server_manager_.get_server_zone(*server, tmp_zone))) {
7926
          LOG_WARN("fail to get server zone", K(ret));
7927
        } else if (tmp_zone != zone) {
7928
          ret = OB_SERVER_ZONE_NOT_MATCH;
7929
          LOG_WARN("delete server not in zone", K(ret));
7930
        }
7931
      }
7932
      if (OB_SUCC(ret)) {
7933
        if (OB_FAIL(unit_manager_.check_enough_resource_for_delete_server(
7934
                *server, tmp_zone))) {
7935
          LOG_WARN("fail to check enouch resource", K(ret));
7936
        }
7937
      }
7938
    }//end for each
7939
  }
7940
  return ret;
7941
}
7942

7943
int ObRootService::cancel_delete_server(const obrpc::ObAdminServerArg &arg)
7944
{
7945
  int ret = OB_SUCCESS;
7946
  ObTimeoutCtx ctx;
7947
  if (OB_UNLIKELY(!inited_)) {
7948
    ret = OB_NOT_INIT;
7949
    LOG_WARN("not init", KR(ret), K(inited_));
7950
  } else if (!arg.is_valid()) {
7951
    ret = OB_INVALID_ARGUMENT;
7952
    LOG_WARN("invalid arg", KR(ret), K(arg));
7953
  } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
7954
    LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
7955
  } else {}
7956
  if (OB_SUCC(ret)) {
7957
    if (!ObHeartbeatService::is_service_enabled()) { // the old logic
7958
      LOG_INFO("sys tenant data version < 4.2, cancel_delete_server", K(arg),
7959
          "timeout_ts", ctx.get_timeout());
7960
      if (OB_FAIL(old_cancel_delete_server(arg))) {
7961
        LOG_WARN("fail to cancel delete server by using the old logic", KR(ret), K(arg));
7962
      }
7963
    } else { // the new logic
7964
      LOG_INFO("sys tenant data version >= 4.2, cancel_delete_server", K(arg),
7965
          "timeout_ts", ctx.get_timeout());
7966
      if (OB_FAIL(server_zone_op_service_.cancel_delete_servers(arg.servers_, arg.zone_))) {
7967
        LOG_WARN("fail to cancel delete servers", KR(ret), K(arg));
7968
      }
7969
      int tmp_ret = OB_SUCCESS;
7970
      if (OB_TMP_FAIL(load_server_manager())) {
7971
        // ** FIXME (linqiucen.lqc): temp. solution.
7972
        // ** This will be removed if we do not need whitelist in server_manager
7973
        LOG_WARN("fail to load server_manager, please try 'ALTER SYSTEM RELOAD SERVER;'", KR(ret), KR(tmp_ret));
7974
        ret = OB_SUCC(ret) ? tmp_ret : ret;
7975
      } else {
7976
        root_balancer_.wakeup();
7977
      }
7978
    }
7979
  }
7980
  FLOG_INFO("cancel delete server", KR(ret), K(arg));
7981
  return ret;
7982
}
7983

7984
int ObRootService::old_cancel_delete_server(const obrpc::ObAdminServerArg &arg)
7985
{
7986
  int ret = OB_SUCCESS;
7987
  if (!inited_) {
7988
    ret = OB_NOT_INIT;
7989
    LOG_WARN("not init", K(ret));
7990
  } else if (!arg.is_valid()) {
7991
    ret = OB_INVALID_ARGUMENT;
7992
    LOG_WARN("invalid arg", K(arg), K(ret));
7993
  } else {
7994
    for (int64_t i = 0; OB_SUCC(ret) && i < arg.servers_.count(); ++i) {
7995
      const bool commit = false;
7996
      if (OB_FAIL(ret)) {
7997
      } else if (OB_FAIL(server_manager_.end_delete_server(arg.servers_[i], arg.zone_, commit))) {
7998
        LOG_WARN("delete_server failed", "server", arg.servers_[i],
7999
                 "zone", arg.zone_, K(commit), K(ret));
8000
      } else {
8001
        // TODO: @wanhong.wwh NEED support cancel DR task
8002
        root_balancer_.wakeup();
8003
      }
8004
    }
8005
  }
8006
  ROOTSERVICE_EVENT_ADD("root_service", "cancel_delete_server", K(ret), K(arg));
8007
  return ret;
8008
}
8009

8010
int ObRootService::start_server(const obrpc::ObAdminServerArg &arg)
8011
{
8012
  int ret = OB_SUCCESS;
8013
  ObTimeoutCtx ctx;
8014
  if (OB_UNLIKELY(!inited_)) {
8015
    ret = OB_NOT_INIT;
8016
    LOG_WARN("not init", KR(ret), K(inited_));
8017
  } else if (!arg.is_valid()) {
8018
    ret = OB_INVALID_ARGUMENT;
8019
    LOG_WARN("invalid arg", KR(ret), K(arg));
8020
  } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
8021
      LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
8022
  } else {}
8023
  if (OB_SUCC(ret)) {
8024
    if (!ObHeartbeatService::is_service_enabled()) { // the old logic
8025
      LOG_INFO("sys tenant data version < 4.2, start_server", K(arg),
8026
          "timeout_ts", ctx.get_timeout());
8027
      if (OB_FAIL(server_manager_.start_server_list(arg.servers_, arg.zone_))) {
8028
        LOG_WARN("fail to start server by using old logic", KR(ret), K(arg));
8029
      }
8030
    } else { // the new logic
8031
      LOG_INFO("sys tenant data version >= 4.2, start_server", K(arg),
8032
          "timeout_ts", ctx.get_timeout());
8033
      if (OB_FAIL(server_zone_op_service_.start_servers(arg.servers_, arg.zone_))) {
8034
        LOG_WARN("fail to start servers", KR(ret), K(arg));
8035
      }
8036
      int tmp_ret = OB_SUCCESS;
8037
      if (OB_TMP_FAIL(load_server_manager())) {
8038
        // ** FIXME (linqiucen.lqc): temp. solution.
8039
        // ** This will be removed if we do not need whitelist in server_manager
8040
        LOG_WARN("fail to load server_manager, please try 'ALTER SYSTEM RELOAD SERVER;'", KR(ret), KR(tmp_ret));
8041
        ret = OB_SUCC(ret) ? tmp_ret : ret;
8042
      }
8043
    }
8044
  }
8045
  FLOG_INFO("start server", KR(ret), K(arg));
8046
  return ret;
8047
}
8048

8049
int ObRootService::stop_server(const obrpc::ObAdminServerArg &arg)
8050
{
8051
  int ret = OB_SUCCESS;
8052
  ObTimeoutCtx ctx;
8053
  if (OB_UNLIKELY(!inited_)) {
8054
    ret = OB_NOT_INIT;
8055
    LOG_WARN("not init", KR(ret), K(inited_));
8056
  } else if (OB_UNLIKELY(!arg.is_valid())) {
8057
    ret = OB_INVALID_ARGUMENT;
8058
    LOG_WARN("invalid arg", KR(ret), K(arg));
8059
  } else if (OB_FAIL(rootserver::ObRootUtils::get_rs_default_timeout_ctx(ctx))) {
8060
      LOG_WARN("fail to get timeout ctx", KR(ret), K(ctx));
8061
  } else {
8062
    if (!ObHeartbeatService::is_service_enabled()) { // the old logic
8063
      LOG_INFO("sys tenant data version < 4.2, stop_server", K(arg),
8064
          "timeout_ts", ctx.get_timeout());
8065
      if (OB_FAIL(server_zone_op_service_.stop_server_precheck(arg.servers_, arg.op_))) {
8066
        LOG_WARN("fail to precheck stop server", KR(ret), K(arg));
8067
      } else if (OB_FAIL(server_manager_.stop_server_list(arg.servers_, arg.zone_))) {
8068
        LOG_WARN("stop server failed", "server", arg.servers_, "zone", arg.zone_, KR(ret));
8069
      }
8070
    } else {
8071
      LOG_INFO("sys tenant data version >= 4.2, stop_server", K(arg),
8072
          "timeout_ts", ctx.get_timeout());
8073
      if (OB_FAIL(server_zone_op_service_.stop_servers(arg.servers_, arg.zone_, arg.op_))) {
8074
        LOG_WARN("stop server failed", KR(ret), K(arg));
8075
      }
8076
      int tmp_ret = OB_SUCCESS;
8077
      if (OB_TMP_FAIL(load_server_manager())) {
8078
        // ** FIXME (linqiucen.lqc): temp. solution.
8079
        // ** This will be removed if we do not need whitelist in server_manager
8080
        LOG_WARN("fail to load server_manager, please try 'ALTER SYSTEM RELOAD SERVER;'", KR(ret), KR(tmp_ret));
8081
        ret = OB_SUCC(ret) ? tmp_ret : ret;
8082
      }
8083
      if (OB_TMP_FAIL(try_notify_switch_leader(obrpc::ObNotifySwitchLeaderArg::STOP_SERVER))) {
8084
        LOG_WARN("failed to notify switch leader", KR(ret), KR(tmp_ret));
8085
      }
8086
    }
8087
  }
8088
  FLOG_INFO("stop server", KR(ret), K(arg));
8089
  return ret;
8090
}
8091

8092
#ifdef OB_BUILD_TDE_SECURITY
8093
int ObRootService::try_check_encryption_zone_cond(
8094
    const obrpc::ObAdminZoneArg &arg)
8095
{
8096
  int ret = OB_SUCCESS;
8097
  bool has_available_master_key = false;
8098
  if (OB_UNLIKELY(!inited_)) {
8099
    ret = OB_NOT_INIT;
8100
    LOG_WARN("not init", K(ret));
8101
  } else if (OB_UNLIKELY(!arg.is_valid())) {
8102
    ret = OB_INVALID_ARGUMENT;
8103
    LOG_WARN("invalid argument", KR(ret), K(arg));
8104
  } else if (arg.zone_type_ != ZONE_TYPE_ENCRYPTION) {
8105
    // good, no need to check
8106
  } else if (OB_FAIL(master_key_mgr_.check_if_tenant_has_available_master_keys(
8107
          OB_SYS_TENANT_ID, has_available_master_key))) {
8108
    LOG_WARN("fail to check if tenant has available master key", KR(ret));
8109
  } else if (!has_available_master_key) {
8110
    ret = OB_OP_NOT_ALLOW;
8111
    LOG_WARN("add encryption zone without available master key in sys tenant is not allowed", KR(ret));
8112
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "add encryption zone without available master key in sys tenant");
8113
  }
8114
  return ret;
8115
}
8116
#endif
8117

8118
int ObRootService::add_zone(const obrpc::ObAdminZoneArg &arg)
8119
{
8120
  int ret = OB_SUCCESS;
8121
  if (!inited_) {
8122
    ret = OB_NOT_INIT;
8123
    LOG_WARN("not init", K(ret));
8124
  } else if (!arg.is_valid()) {
8125
    ret = OB_INVALID_ARGUMENT;
8126
    LOG_WARN("invalid arg", K(arg), K(ret));
8127
#ifdef OB_BUILD_TDE_SECURITY
8128
  } else if (OB_FAIL(try_check_encryption_zone_cond(arg))) {
8129
    LOG_WARN("fail to check encryption zone", KR(ret), K(arg));
8130
#endif
8131
  } else if (OB_FAIL(zone_manager_.add_zone(arg.zone_, arg.region_, arg.idc_, arg.zone_type_))) {
8132
    LOG_WARN("failed to add zone", K(ret), K(arg));
8133
  } else {
8134
    LOG_INFO("add zone ok", K(arg));
8135
  }
8136
  ROOTSERVICE_EVENT_ADD("root_service", "add_zone", K(ret), "sql_text", ObHexEscapeSqlStr(arg.sql_stmt_str_));
8137
  return ret;
8138
}
8139

8140
int ObRootService::delete_zone(const obrpc::ObAdminZoneArg &arg)
8141
{
8142
  int ret = OB_SUCCESS;
8143
  if (!inited_) {
8144
    ret = OB_NOT_INIT;
8145
    LOG_WARN("not init", K(ret));
8146
  } else if (!arg.is_valid()) {
8147
    ret = OB_INVALID_ARGUMENT;
8148
    LOG_WARN("invalid arg", K(arg), K(ret));
8149
  } else {
8150
    // @note to avoid deadlock risk, put it beside ZoneManager::write_lock_. ObServerManager::add_server also call the interfaces of zone_mgr.
8151
    // it does not matter while add server after check.
8152
    int64_t alive_count = 0;
8153
    int64_t not_alive_count = 0;
8154
    ObArray<ObServerInfoInTable> servers_info;
8155
    if (OB_ISNULL(GCTX.sql_proxy_)) {
8156
      ret = OB_ERR_UNEXPECTED;
8157
      LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_));
8158
    } else if (OB_FAIL(ObServerTableOperator::get(*GCTX.sql_proxy_, servers_info))) {
8159
      LOG_WARN("fail to get servers_info", KR(ret), KP(GCTX.sql_proxy_));
8160
    } else if (OB_FAIL(ObRootUtils::get_server_count(servers_info, arg.zone_, alive_count, not_alive_count))) {
8161
      LOG_WARN("failed to get server count of the zone", KR(ret), K(arg.zone_), K(servers_info));
8162
    } else {
8163
      LOG_INFO("current server count of zone",
8164
               "zone", arg.zone_, K(alive_count), K(not_alive_count));
8165
      if (alive_count > 0 || not_alive_count > 0) {
8166
        ret = OB_ERR_ZONE_NOT_EMPTY;
8167
        LOG_USER_ERROR(OB_ERR_ZONE_NOT_EMPTY, alive_count, not_alive_count);
8168
      } else if (OB_FAIL(zone_manager_.delete_zone(arg.zone_))) {
8169
        LOG_WARN("delete zone failed", K(ret), K(arg));
8170
      } else {
8171
        LOG_INFO("delete zone ok", K(arg));
8172
      }
8173
    }
8174
  }
8175
  ROOTSERVICE_EVENT_ADD("root_service", "delete_zone", K(ret), "sql_text", ObHexEscapeSqlStr(arg.sql_stmt_str_));
8176
  return ret;
8177
}
8178

8179
int ObRootService::start_zone(const obrpc::ObAdminZoneArg &arg)
8180
{
8181
  int ret = OB_SUCCESS;
8182
  if (!inited_) {
8183
    ret = OB_NOT_INIT;
8184
    LOG_WARN("not init", K(ret));
8185
  } else if (!arg.is_valid()) {
8186
    ret = OB_INVALID_ARGUMENT;
8187
    LOG_WARN("invalid arg", K(arg), K(ret));
8188
  } else if (OB_FAIL(zone_manager_.start_zone(arg.zone_))) {
8189
    LOG_WARN("failed to start zone", K(ret), K(arg));
8190
  } else {
8191
    LOG_INFO("start zone ok", K(arg));
8192
  }
8193
  ROOTSERVICE_EVENT_ADD("root_service", "start_zone", K(ret), "sql_text", ObHexEscapeSqlStr(arg.sql_stmt_str_));
8194
  return ret;
8195
}
8196

8197
//check whether has a object that all primary_zones were stopped after this time stop.
8198
//basic idea:
8199
//1. get the intersection tenant of to_stop_list and stopped_server_list. if tenant is only on stopp_list, it is no matter with this time stop.
8200
//   therefor, only need to consider the intersection of two sets;
8201
//2. set the union of to_stop_list and stopped_list to zone_list. set the intersection tenant of to_stop_list and stopped_server_list to tenant_ids,
8202
//   check whether the primary zone of every tenant in tenant_ids is the subset of zone_list. if it is the subset, then the leader of the object may switch
8203
//   to non_primary_zone observer, this situation is not allowed; if it is not the subset, than still has replica in primary zone can be leader.
8204
int ObRootService::check_can_stop(const ObZone &zone,
8205
                                  const ObIArray<ObAddr> &servers,
8206
                                  const bool is_stop_zone)
8207
{
8208
  int ret = OB_SUCCESS;
8209
  int64_t start_time = ObTimeUtility::current_time();
8210
  ObArray<ObAddr> to_stop_list;
8211
  ObArray<ObZone> stopped_zone_list;
8212
  ObArray<ObAddr> stopped_server_list;
8213
  ObArray<ObServerInfoInTable> servers_info_in_table;
8214
  if ((!is_stop_zone && (0 == servers.count() || zone.is_empty()))
8215
      || (is_stop_zone && zone.is_empty())) {
8216
    ret = OB_INVALID_ARGUMENT;
8217
    LOG_WARN("invalid argument", KR(ret), K(servers), K(zone));
8218
  } else if (OB_FAIL(ObRootUtils::get_stopped_zone_list(stopped_zone_list,
8219
                                                        stopped_server_list))) {
8220
    LOG_WARN("fail to get stopped zone list", KR(ret));
8221
  } else if (0 >= stopped_server_list.count()) {
8222
    //nothing todo
8223
  } else {
8224
    if (!is_stop_zone) {
8225
      if (OB_FAIL(to_stop_list.assign(servers))) {
8226
        LOG_WARN("fail to push back", KR(ret), K(servers));
8227
      }
8228
    } else if (OB_ISNULL(GCTX.sql_proxy_)) {
8229
      ret = OB_ERR_UNEXPECTED;
8230
      LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_));
8231
    } else if (OB_FAIL(ObServerTableOperator::get(*GCTX.sql_proxy_, servers_info_in_table))) {
8232
      LOG_WARN("fail to get servers_info_in_table", KR(ret), KP(GCTX.sql_proxy_));
8233
    } else if (OB_FAIL(ObRootUtils::get_servers_of_zone(servers_info_in_table, zone, to_stop_list))) {
8234
      LOG_WARN("fail to get servers of zone", KR(ret), K(zone));
8235
    }
8236
    ObArray<uint64_t> tenant_ids;
8237
    bool is_in = false;
8238
    if (OB_FAIL(ret)) {
8239
    } else if (OB_FAIL(ObRootUtils::get_tenant_intersection(unit_manager_,  //get intersection tenant of to_stop_list and stopped_server_list
8240
                                                            to_stop_list,
8241
                                                            stopped_server_list,
8242
                                                            tenant_ids))) {
8243
      LOG_WARN("fail to get tenant intersections", KR(ret));
8244
    } else if (!has_exist_in_array(stopped_zone_list, zone)
8245
               && OB_FAIL(stopped_zone_list.push_back(zone))) {
8246
      LOG_WARN("fail to push back", KR(ret), K(zone));
8247
    } else if (OB_FAIL(ObRootUtils::check_primary_region_in_zonelist(schema_service_, //check whether stop primary region of the tenant
8248
                                                                     &ddl_service_,
8249
                                                                     unit_manager_,
8250
                                                                     zone_manager_,
8251
                                                                     tenant_ids,
8252
                                                                     stopped_zone_list,
8253
                                                                     is_in))) {
8254
      LOG_WARN("fail to check tenant stop primary region", KR(ret));
8255
    } else if (!is_in) {
8256
      //nothing todo
8257
    } else {
8258
      ret = OB_OP_NOT_ALLOW;
8259
      LOG_WARN("stop all primary region is not allowed", KR(ret), K(zone), K(servers));
8260
    }
8261
  }
8262
  int64_t cost = ObTimeUtility::current_time() - start_time;
8263
  LOG_INFO("check can stop zone/server", KR(ret), K(zone), K(servers), K(cost));
8264
  return ret;
8265
}
8266

8267
int ObRootService::stop_zone(const obrpc::ObAdminZoneArg &arg)
8268
{
8269
  int ret = OB_SUCCESS;
8270
  ObArray<ObAddr> empty_server;
8271
  HEAP_VAR(ObZoneInfo, zone_info) {
8272
    bool zone_active = false;
8273
    bool zone_exist = false;
8274
    if (OB_UNLIKELY(!inited_)) {
8275
      ret = OB_NOT_INIT;
8276
      LOG_WARN("not init", K(ret));
8277
    } else if (!arg.is_valid()) {
8278
      ret = OB_INVALID_ARGUMENT;
8279
      LOG_WARN("invalid arg", K(arg), K(ret));
8280
    } else if (OB_FAIL(zone_manager_.check_zone_exist(arg.zone_, zone_exist))) {
8281
      LOG_WARN("fail to check zone exist", K(ret));
8282
    } else if (!zone_exist) {
8283
      ret = OB_ENTRY_NOT_EXIST;
8284
      LOG_WARN("zone not exist");
8285
      LOG_USER_ERROR(OB_ENTRY_NOT_EXIST, "zone not exist");
8286
    } else if (OB_FAIL(zone_manager_.check_zone_active(arg.zone_, zone_active))) {
8287
      LOG_WARN("fail to check zone active", KR(ret), K(arg.zone_));
8288
    } else if (!zone_active) {
8289
      //nothing todo
8290
    } else if (ObAdminZoneArg::ISOLATE == arg.op_) {
8291
      if (OB_FAIL(check_can_stop(arg.zone_, empty_server, true/*is_stop_zone*/))) {
8292
        LOG_WARN("fail to check zone can stop", KR(ret), K(arg));
8293
        if (OB_OP_NOT_ALLOW == ret) {
8294
          LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Stop all server in primary region is disabled");
8295
        }
8296
      }
8297
    } else {
8298
      //stop zone/force stop zone
8299
      if (ObRootUtils::have_other_stop_task(arg.zone_)) {
8300
        ret = OB_STOP_SERVER_IN_MULTIPLE_ZONES;
8301
        LOG_WARN("cannot stop zone when other stop task already exist", KR(ret), K(arg));
8302
        LOG_USER_ERROR(OB_STOP_SERVER_IN_MULTIPLE_ZONES,
8303
                       "cannot stop server or stop zone in multiple zones");
8304
      }
8305
    }
8306

8307
    if (OB_FAIL(ret)) {
8308
    } else if (OB_FAIL(zone_manager_.get_zone(arg.zone_, zone_info))) {
8309
      LOG_WARN("fail to get zone info", K(ret));
8310
    } else {
8311
      common::ObZoneType zone_type = static_cast<ObZoneType>(zone_info.zone_type_.value_);
8312
      if (ObAdminZoneArg::ISOLATE == arg.op_) {
8313
        // isolate server no need to check count and status of replicas, it can not kill observer savely
8314
      } else if (common::ZONE_TYPE_READONLY == zone_type) {
8315
        // do not need to check anything for readonly zone
8316
      } else if (common::ZONE_TYPE_READWRITE == zone_type
8317
                 || common::ZONE_TYPE_ENCRYPTION == zone_type) {
8318
        ObArray<ObAddr> server_list;
8319
        ObArray<ObServerInfoInTable> servers_info;
8320
        if (OB_ISNULL(GCTX.sql_proxy_)) {
8321
          ret = OB_ERR_UNEXPECTED;
8322
          LOG_WARN("GCTX.sql_proxy_ is null", KR(ret), KP(GCTX.sql_proxy_));
8323
        } else if (OB_FAIL(ObServerTableOperator::get(*GCTX.sql_proxy_, servers_info))) {
8324
          LOG_WARN("fail to get servers_info", KR(ret), KP(GCTX.sql_proxy_));
8325
        } else if (OB_FAIL(ObRootUtils::get_servers_of_zone(servers_info, arg.zone_, server_list))) {
8326
          LOG_WARN("get servers of zone failed", KR(ret), K(arg.zone_), K(servers_info));
8327
        } else if (server_list.count() <= 0) {
8328
          //do not need to check anyting while zone is empty
8329
        } else if (OB_FAIL(check_majority_and_log_in_sync(
8330
            server_list,
8331
            arg.force_stop_,/*skip_log_sync_check*/
8332
            "stop zone"))) {
8333
          LOG_WARN("fail to check majority and log in-sync", KR(ret), K(arg));
8334
        }
8335
      } else {
8336
        ret = OB_ERR_UNEXPECTED;
8337
        LOG_WARN("invalid zone type", K(ret), "zone", arg.zone_, K(zone_type));
8338
      }
8339
      if (OB_SUCC(ret)) {
8340
        if (OB_FAIL(zone_manager_.stop_zone(arg.zone_))) {
8341
          LOG_WARN("stop zone failed", K(arg), K(ret));
8342
        } else {
8343
          LOG_INFO("stop zone ok", K(arg));
8344
          int tmp_ret = OB_SUCCESS;
8345
          if (OB_TMP_FAIL(try_notify_switch_leader(obrpc::ObNotifySwitchLeaderArg::STOP_ZONE))) {
8346
            LOG_WARN("failed to notify switch leader", KR(ret), KR(tmp_ret));
8347
          }
8348
        }
8349
      } else {
8350
        //set other error code to 4179
8351
      }
8352
    }
8353
    ROOTSERVICE_EVENT_ADD("root_service", "stop_zone", K(ret), "sql_text", ObHexEscapeSqlStr(arg.sql_stmt_str_));
8354
  }
8355
  return ret;
8356
}
8357

8358
int ObRootService::alter_zone(const obrpc::ObAdminZoneArg &arg)
8359
{
8360
  int ret = OB_SUCCESS;
8361
  if (!inited_) {
8362
    ret = OB_NOT_INIT;
8363
    LOG_WARN("not init", K(ret));
8364
  } else if (!arg.is_valid()) {
8365
    ret = OB_INVALID_ARGUMENT;
8366
    LOG_WARN("invalid arg", K(arg), K(ret));
8367
  } else if (OB_FAIL(zone_manager_.alter_zone(arg))) {
8368
    LOG_WARN("failed to alter zone", K(ret), K(arg));
8369
  } else {
8370
    LOG_INFO("alter zone ok", K(arg));
8371
  }
8372
  ROOTSERVICE_EVENT_ADD("root_service", "alter_zone", K(ret), "sql_text", ObHexEscapeSqlStr(arg.sql_stmt_str_));
8373
  return ret;
8374
}
8375

8376

8377
int ObRootService::generate_stop_server_log_in_sync_dest_server_array(
8378
    const common::ObIArray<common::ObAddr> &alive_server_array,
8379
    const common::ObIArray<common::ObAddr> &excluded_server_array,
8380
    common::ObIArray<common::ObAddr> &dest_server_array)
8381
{
8382
  int ret = OB_SUCCESS;
8383
  if (OB_UNLIKELY(!inited_)) {
8384
    ret = OB_NOT_INIT;
8385
    LOG_WARN("not init", K(ret));
8386
  } else {
8387
    dest_server_array.reset();
8388
    for (int64_t i = 0; OB_SUCC(ret) && i < alive_server_array.count(); ++i) {
8389
      const common::ObAddr &server = alive_server_array.at(i);
8390
      if (has_exist_in_array(excluded_server_array, server)) {
8391
        // in this excluded server array
8392
      } else {
8393
        if (OB_FAIL(dest_server_array.push_back(server))) {
8394
          LOG_WARN("fail to push back", K(ret));
8395
        }
8396
      }
8397
    }
8398
  }
8399
  return ret;
8400
}
8401

8402
int ObRootService::try_notify_switch_leader(const obrpc::ObNotifySwitchLeaderArg::SwitchLeaderComment &comment)
8403
{
8404
  int ret = OB_SUCCESS;
8405
  ObServerManager::ObServerArray server_list;
8406
  ObZone zone;
8407
  obrpc::ObNotifySwitchLeaderArg arg;
8408
  if (OB_UNLIKELY(!inited_)) {
8409
    ret = OB_NOT_INIT;
8410
    LOG_WARN("not init", K(ret));
8411
  } else if (OB_FAIL(SVR_TRACER.get_alive_servers(zone, server_list))) {
8412
    LOG_WARN("failed to get server list", KR(ret), K(zone));
8413
  } else if (OB_FAIL(arg.init(OB_INVALID_TENANT_ID, ObLSID(), ObAddr(), comment))) {
8414
    LOG_WARN("failed to init switch leader arg", KR(ret), K(comment));
8415
  } else if (OB_FAIL(ObRootUtils::notify_switch_leader(&rpc_proxy_, OB_SYS_TENANT_ID, arg, server_list))) {
8416
    LOG_WARN("failed to notify switch leader", KR(ret), K(arg), K(server_list));
8417
  }
8418
  return ret;
8419
}
8420
////////////////////////////////////////////////////////////////
8421
// system admin command (alter system ...)
8422
////////////////////////////////////////////////////////////////
8423
int ObRootService::init_sys_admin_ctx(ObSystemAdminCtx &ctx)
8424
{
8425
  int ret = OB_SUCCESS;
8426
  if (!inited_) {
8427
    ret = OB_NOT_INIT;
8428
    LOG_WARN("not init", K(ret));
8429
  } else {
8430
    ctx.rs_status_ = &rs_status_;
8431
    ctx.rpc_proxy_ = &rpc_proxy_;
8432
    ctx.sql_proxy_ = &sql_proxy_;
8433
    ctx.server_mgr_ = &server_manager_;
8434
    ctx.zone_mgr_ = &zone_manager_;
8435
    ctx.schema_service_ = schema_service_;
8436
    ctx.ddl_service_ = &ddl_service_;
8437
    ctx.config_mgr_ = config_mgr_;
8438
    ctx.unit_mgr_ = &unit_manager_;
8439
    ctx.root_inspection_ = &root_inspection_;
8440
    ctx.root_service_ = this;
8441
    ctx.root_balancer_ = &root_balancer_;
8442
    ctx.upgrade_storage_format_executor_ = &upgrade_storage_format_executor_;
8443
    ctx.create_inner_schema_executor_ = &create_inner_schema_executor_;
8444
    ctx.inited_ = true;
8445
  }
8446
  return ret;
8447
}
8448

8449
int ObRootService::admin_switch_replica_role(const obrpc::ObAdminSwitchReplicaRoleArg &arg)
8450
{
8451
  int ret = OB_SUCCESS;
8452
  if (!inited_) {
8453
    ret = OB_NOT_INIT;
8454
    LOG_WARN("not init", K(ret));
8455
  } else if (!arg.is_valid()) {
8456
    ret = OB_INVALID_ARGUMENT;
8457
    LOG_WARN("invalid arg", K(arg), K(ret));
8458
  } else {
8459
    ObSystemAdminCtx ctx;
8460
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8461
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8462
    } else {
8463
      ObAdminSwitchReplicaRole admin_util(ctx);
8464
      if (OB_FAIL(admin_util.execute(arg))) {
8465
        LOG_WARN("execute switch replica role failed", K(arg), K(ret));
8466
      }
8467
    }
8468
  }
8469
  ROOTSERVICE_EVENT_ADD("root_service", "admin_switch_replica_role", K(ret), K(arg));
8470
  return ret;
8471
}
8472

8473
int ObRootService::admin_switch_rs_role(const obrpc::ObAdminSwitchRSRoleArg &arg)
8474
{
8475
  int ret = OB_NOT_SUPPORTED;
8476
  UNUSED(arg);
8477
  return ret;
8478
}
8479

8480
int ObRootService::admin_change_replica(const obrpc::ObAdminChangeReplicaArg &arg)
8481
{
8482
  int ret = OB_NOT_SUPPORTED;
8483
  UNUSED(arg);
8484
  return ret;
8485
}
8486

8487
int ObRootService::admin_drop_replica(const obrpc::ObAdminDropReplicaArg &arg)
8488
{
8489
  int ret = OB_NOT_SUPPORTED;
8490
  UNUSED(arg);
8491
  return ret;
8492
}
8493

8494
int ObRootService::admin_migrate_replica(const obrpc::ObAdminMigrateReplicaArg &arg)
8495
{
8496
  int ret = OB_NOT_SUPPORTED;
8497
  UNUSED(arg);
8498
  return ret;
8499
}
8500

8501
int ObRootService::admin_report_replica(const obrpc::ObAdminReportReplicaArg &arg)
8502
{
8503
  int ret = OB_SUCCESS;
8504
  if (!inited_) {
8505
    ret = OB_NOT_INIT;
8506
    LOG_WARN("not init", K(ret));
8507
  } else if (!arg.is_valid()) {
8508
    ret = OB_INVALID_ARGUMENT;
8509
    LOG_WARN("invalid arg", K(arg), K(ret));
8510
  } else {
8511
    ObSystemAdminCtx ctx;
8512
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8513
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8514
    } else {
8515
      ObAdminReportReplica admin_util(ctx);
8516
      if (OB_FAIL(admin_util.execute(arg))) {
8517
        LOG_WARN("execute report replica failed", K(arg), K(ret));
8518
      }
8519
    }
8520
  }
8521
  ROOTSERVICE_EVENT_ADD("root_service", "admin_report_replica", K(ret), K(arg));
8522
  return ret;
8523
}
8524

8525
int ObRootService::admin_flush_cache(const obrpc::ObAdminFlushCacheArg &arg)
8526
{
8527
  int ret = OB_SUCCESS;
8528
  if (!inited_) {
8529
    ret = OB_NOT_INIT;
8530
    LOG_WARN("not init", K(ret));
8531
  } else if (!arg.is_valid()) {
8532
    ret = OB_INVALID_ARGUMENT;
8533
    LOG_WARN("invalid arg", K(arg), K(ret));
8534
  } else {
8535
    ObSystemAdminCtx ctx;
8536
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8537
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8538
    } else {
8539
      ObAdminFlushCache admin_util(ctx);
8540
      if (OB_FAIL(admin_util.execute(arg))) {
8541
        LOG_WARN("dispatch flush cache failed", K(arg), K(ret));
8542
      }
8543
      ROOTSERVICE_EVENT_ADD("root_service", "admin_flush_cache", K(ret), K(arg));
8544
    }
8545
  }
8546
  return ret;
8547
}
8548

8549
int ObRootService::admin_recycle_replica(const obrpc::ObAdminRecycleReplicaArg &arg)
8550
{
8551
  int ret = OB_SUCCESS;
8552
  if (!inited_) {
8553
    ret = OB_NOT_INIT;
8554
    LOG_WARN("not init", K(ret));
8555
  } else if (!arg.is_valid()) {
8556
    ret = OB_INVALID_ARGUMENT;
8557
    LOG_WARN("invalid arg", K(arg), K(ret));
8558
  } else {
8559
    ObSystemAdminCtx ctx;
8560
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8561
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8562
    } else {
8563
      ObAdminRecycleReplica admin_util(ctx);
8564
      if (OB_FAIL(admin_util.execute(arg))) {
8565
        LOG_WARN("execute recycle replica failed", K(arg), K(ret));
8566
      }
8567
    }
8568
  }
8569
  ROOTSERVICE_EVENT_ADD("root_service", "admin_recycle_replica", K(ret), K(arg));
8570
  return ret;
8571
}
8572

8573
int ObRootService::admin_merge(const obrpc::ObAdminMergeArg &arg)
8574
{
8575
  int ret = OB_SUCCESS;
8576
  if (!inited_) {
8577
    ret = OB_NOT_INIT;
8578
    LOG_WARN("not init", K(ret));
8579
  } else if (!arg.is_valid()) {
8580
    ret = OB_INVALID_ARGUMENT;
8581
    LOG_WARN("invalid arg", K(arg), K(ret));
8582
  } else {
8583
    ObSystemAdminCtx ctx;
8584
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8585
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8586
    } else {
8587
      ObAdminMerge admin_util(ctx);
8588
      if (OB_FAIL(admin_util.execute(arg))) {
8589
        LOG_WARN("execute merge control failed", K(arg), K(ret));
8590
      }
8591
    }
8592
  }
8593
  ROOTSERVICE_EVENT_ADD("root_service", "admin_merge", K(ret), K(arg));
8594
  return ret;
8595
}
8596

8597
int ObRootService::admin_recovery(const obrpc::ObAdminRecoveryArg &arg)
8598
{
8599
  int ret = OB_NOT_SUPPORTED;
8600
  if (OB_UNLIKELY(!inited_)) {
8601
    ret = OB_NOT_INIT;
8602
    LOG_WARN("ObRootService has not been inited", K(ret));
8603
  } else if (OB_UNLIKELY(!arg.is_valid())) {
8604
    ret = OB_INVALID_ARGUMENT;
8605
    LOG_WARN("invalid arguments", K(ret), K(arg));
8606
  } else {
8607
    ObSystemAdminCtx ctx;
8608
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8609
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8610
    } else {
8611
      ObAdminZoneFastRecovery admin_util(ctx);
8612
      if (OB_FAIL(admin_util.execute(arg))) {
8613
        LOG_WARN("execute merge control failed", K(arg), K(ret));
8614
      }
8615
    }
8616
  }
8617
  ROOTSERVICE_EVENT_ADD("root_service", "admin_recovery", K(ret), K(arg));
8618
  return ret;
8619
}
8620

8621
int ObRootService::admin_clear_roottable(const obrpc::ObAdminClearRoottableArg &arg)
8622
{
8623
  int ret = OB_SUCCESS;
8624
  if (!inited_) {
8625
    ret = OB_NOT_INIT;
8626
    LOG_WARN("not init", K(ret));
8627
  } else if (!arg.is_valid()) {
8628
    ret = OB_INVALID_ARGUMENT;
8629
    LOG_WARN("invalid arg", K(arg), K(ret));
8630
  } else {
8631
    ObSystemAdminCtx ctx;
8632
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8633
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8634
    } else {
8635
      ObAdminClearRoottable admin_util(ctx);
8636
      if (OB_FAIL(admin_util.execute(arg))) {
8637
        LOG_WARN("execute clear root table failed", K(arg), K(ret));
8638
      }
8639
    }
8640
  }
8641
  ROOTSERVICE_EVENT_ADD("root_service", "admin_clear_roottable", K(ret), K(arg));
8642
  return ret;
8643
}
8644

8645
int ObRootService::admin_refresh_schema(const obrpc::ObAdminRefreshSchemaArg &arg)
8646
{
8647
  int ret = OB_SUCCESS;
8648
  if (!inited_) {
8649
    ret = OB_NOT_INIT;
8650
    LOG_WARN("not init", K(ret));
8651
  } else if (!arg.is_valid()) {
8652
    ret = OB_INVALID_ARGUMENT;
8653
    LOG_WARN("invalid arg", K(arg), K(ret));
8654
  } else {
8655
    ObSystemAdminCtx ctx;
8656
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8657
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8658
    } else {
8659
      ObAdminRefreshSchema admin_util(ctx);
8660
      if (OB_FAIL(admin_util.execute(arg))) {
8661
        LOG_WARN("execute refresh schema failed", K(arg), K(ret));
8662
      }
8663
    }
8664
  }
8665
  ROOTSERVICE_EVENT_ADD("root_service", "admin_refresh_schema", K(ret), K(arg));
8666
  return ret;
8667
}
8668

8669
int ObRootService::admin_set_config(obrpc::ObAdminSetConfigArg &arg)
8670
{
8671
  int ret = OB_SUCCESS;
8672
  if (!inited_) {
8673
    ret = OB_NOT_INIT;
8674
    LOG_WARN("not init", K(ret));
8675
  } else if (!arg.is_valid()) {
8676
    ret = OB_INVALID_ARGUMENT;
8677
    LOG_WARN("invalid arg", K(arg), K(ret));
8678
  } else if (arg.is_backup_config_) {
8679
    if (OB_FAIL(admin_set_backup_config(arg))) {
8680
      LOG_WARN("fail to set backup config", K(ret), K(arg));
8681
    }
8682
  } else {
8683
    ObSystemAdminCtx ctx;
8684
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8685
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8686
    } else {
8687
      ObLatchWGuard guard(set_config_lock_, ObLatchIds::CONFIG_LOCK);
8688
      ObAdminSetConfig admin_util(ctx);
8689
      if (OB_FAIL(admin_util.execute(arg))) {
8690
        LOG_WARN("execute set config failed", K(arg), K(ret));
8691
      }
8692
    }
8693
  }
8694
  // Add event one by one if more than one parameters are set
8695
  for (int i = 0; i < arg.items_.count(); i++) {
8696
    ROOTSERVICE_EVENT_ADD_TRUNCATE("root_service", "admin_set_config", K(ret), "arg", arg.items_.at(i), "is_inner", arg.is_inner_);
8697
  }
8698
  return ret;
8699
}
8700

8701
int ObRootService::admin_clear_location_cache(
8702
    const obrpc::ObAdminClearLocationCacheArg &arg)
8703
{
8704
  int ret = OB_SUCCESS;
8705
  if (!inited_) {
8706
    ret = OB_NOT_INIT;
8707
    LOG_WARN("not init", K(ret));
8708
  } else if (!arg.is_valid()) {
8709
    ret = OB_INVALID_ARGUMENT;
8710
    LOG_WARN("invalid arg", K(arg), K(ret));
8711
  } else {
8712
    ObSystemAdminCtx ctx;
8713
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8714
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8715
    } else {
8716
      ObAdminClearLocationCache admin_util(ctx);
8717
      if (OB_FAIL(admin_util.execute(arg))) {
8718
        LOG_WARN("execute clear location cache failed", K(arg), K(ret));
8719
      }
8720
    }
8721
  }
8722
  ROOTSERVICE_EVENT_ADD("root_service", "admin_clear_location_cache", K(ret), K(arg));
8723
  return ret;
8724
}
8725

8726
int ObRootService::admin_refresh_memory_stat(const ObAdminRefreshMemStatArg &arg)
8727
{
8728
  int ret = OB_SUCCESS;
8729
  if (!inited_) {
8730
    ret = OB_NOT_INIT;
8731
    LOG_WARN("not init", K(ret));
8732
  } else {
8733
    ObSystemAdminCtx ctx;
8734
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8735
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8736
    } else {
8737
      ObAdminRefreshMemStat admin_util(ctx);
8738
      if (OB_FAIL(admin_util.execute(arg))) {
8739
        LOG_WARN("execute refresh memory stat failed", K(ret));
8740
      }
8741
    }
8742
  }
8743
  ROOTSERVICE_EVENT_ADD("root_service", "admin_refresh_memory_stat", K(ret));
8744
  return ret;
8745
}
8746

8747
int ObRootService::admin_wash_memory_fragmentation(const ObAdminWashMemFragmentationArg &arg)
8748
{
8749
  int ret = OB_SUCCESS;
8750
  if (!inited_) {
8751
    ret = OB_NOT_INIT;
8752
    LOG_WARN("not init", K(ret));
8753
  } else {
8754
    ObSystemAdminCtx ctx;
8755
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8756
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8757
    } else {
8758
      ObAdminWashMemFragmentation admin_util(ctx);
8759
      if (OB_FAIL(admin_util.execute(arg))) {
8760
        LOG_WARN("execute refresh memory stat failed", K(ret));
8761
      }
8762
    }
8763
  }
8764
  ROOTSERVICE_EVENT_ADD("root_service", "admin_wash_memory_fragmentation", K(ret));
8765
  return ret;
8766
}
8767

8768
int ObRootService::admin_refresh_io_calibration(const obrpc::ObAdminRefreshIOCalibrationArg &arg)
8769
{
8770
  int ret = OB_SUCCESS;
8771
  if (OB_UNLIKELY(!inited_)) {
8772
    ret = OB_NOT_INIT;
8773
    LOG_WARN("not init", K(ret));
8774
  } else if (OB_UNLIKELY(!arg.is_valid())) {
8775
    ret = OB_INVALID_ARGUMENT;
8776
    LOG_WARN("invalid argument", K(ret), K(arg));
8777
  } else {
8778
    ObSystemAdminCtx ctx;
8779
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8780
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8781
    } else {
8782
      ObAdminRefreshIOCalibration admin_util(ctx);
8783
      if (OB_FAIL(admin_util.execute(arg))) {
8784
        LOG_WARN("execute refresh io calibration failed", K(ret));
8785
      }
8786
    }
8787
  }
8788
  ROOTSERVICE_EVENT_ADD("root_service", "admin_refresh_io_calibration", K(ret));
8789
  return ret;
8790
}
8791

8792
int ObRootService::admin_reload_unit()
8793
{
8794
  int ret = OB_SUCCESS;
8795
  if (!inited_) {
8796
    ret = OB_NOT_INIT;
8797
    LOG_WARN("not init", K(ret));
8798
  } else {
8799
    ObSystemAdminCtx ctx;
8800
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8801
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8802
    } else {
8803
      ObAdminReloadUnit admin_util(ctx);
8804
      if (OB_FAIL(admin_util.execute())) {
8805
        LOG_WARN("execute reload unit failed", K(ret));
8806
      }
8807
    }
8808
  }
8809
  ROOTSERVICE_EVENT_ADD("root_service", "admin_reload_unit", K(ret));
8810
  return ret;
8811
}
8812

8813
int ObRootService::admin_reload_server()
8814
{
8815
  int ret = OB_SUCCESS;
8816
  if (!inited_) {
8817
    ret = OB_NOT_INIT;
8818
    LOG_WARN("not init", K(ret));
8819
  } else {
8820
    ObSystemAdminCtx ctx;
8821
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8822
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8823
    } else {
8824
      ObAdminReloadServer admin_util(ctx);
8825
      if (OB_FAIL(admin_util.execute())) {
8826
        LOG_WARN("execute reload server failed", K(ret));
8827
      }
8828
    }
8829
  }
8830
  ROOTSERVICE_EVENT_ADD("root_service", "admin_reload_server", K(ret));
8831
  return ret;
8832
}
8833

8834
int ObRootService::admin_reload_zone()
8835
{
8836
  int ret = OB_SUCCESS;
8837
  if (!inited_) {
8838
    ret = OB_NOT_INIT;
8839
    LOG_WARN("not init", K(ret));
8840
  } else {
8841
    ObSystemAdminCtx ctx;
8842
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8843
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8844
    } else {
8845
      ObAdminReloadZone admin_util(ctx);
8846
      if (OB_FAIL(admin_util.execute())) {
8847
        LOG_WARN("execute reload zone failed", K(ret));
8848
      }
8849
    }
8850
  }
8851
  ROOTSERVICE_EVENT_ADD("root_service", "admin_reload_zone", K(ret));
8852
  return ret;
8853
}
8854

8855
int ObRootService::admin_clear_merge_error(const obrpc::ObAdminMergeArg &arg)
8856
{
8857
  int ret = OB_SUCCESS;
8858
  LOG_INFO("admin receive clear_merge_error request");
8859
  if (!inited_) {
8860
    ret = OB_NOT_INIT;
8861
    LOG_WARN("not init", K(ret));
8862
  } else {
8863
    ObSystemAdminCtx ctx;
8864
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8865
      LOG_WARN("init_sys_admin_ctx failed", KR(ret));
8866
    } else {
8867
      ObAdminClearMergeError admin_util(ctx);
8868
      if (OB_FAIL(admin_util.execute(arg))) {
8869
        LOG_WARN("execute clear merge error failed", KR(ret), K(arg));
8870
      }
8871
      ROOTSERVICE_EVENT_ADD("root_service", "clear_merge_error", KR(ret), K(arg));
8872
    }
8873
  }
8874
  return ret;
8875
}
8876

8877
#ifdef OB_BUILD_ARBITRATION
8878
int ObRootService::admin_add_arbitration_service(const obrpc::ObAdminAddArbitrationServiceArg &arg)
8879
{
8880
  int ret = OB_SUCCESS;
8881
  if (!inited_) {
8882
    ret = OB_NOT_INIT;
8883
    LOG_WARN("not init", KR(ret));
8884
  } else if (!arg.is_valid()) {
8885
    ret = OB_INVALID_ARGUMENT;
8886
    LOG_WARN("invalid arg", KR(ret), K(arg));
8887
  } else if (OB_FAIL(share::ObArbitrationServiceUtils::add_arbitration_service(sql_proxy_, arg))) {
8888
    LOG_WARN("fail to add arbitration service", KR(ret), K(arg));
8889
  }
8890
  ROOTSERVICE_EVENT_ADD("arb_service", "admin_add_arbitration_service", K(ret), K(arg));
8891
  return ret;
8892
}
8893

8894
int ObRootService::admin_remove_arbitration_service(const obrpc::ObAdminRemoveArbitrationServiceArg &arg)
8895
{
8896
  int ret = OB_SUCCESS;
8897
  if (!inited_) {
8898
    ret = OB_NOT_INIT;
8899
    LOG_WARN("not init", KR(ret));
8900
  } else if (!arg.is_valid()) {
8901
    ret = OB_INVALID_ARGUMENT;
8902
    LOG_WARN("invalid arg", KR(ret), K(arg));
8903
  } else if (OB_FAIL(share::ObArbitrationServiceUtils::remove_arbitration_service(sql_proxy_, arg))) {
8904
    LOG_WARN("fail to remove arbitration service", KR(ret), K(arg));
8905
  }
8906
  ROOTSERVICE_EVENT_ADD("arb_service", "admin_remove_arbitration_service", K(ret), K(arg));
8907
  return ret;
8908
}
8909

8910
int ObRootService::admin_replace_arbitration_service(const obrpc::ObAdminReplaceArbitrationServiceArg &arg)
8911
{
8912
  int ret = OB_SUCCESS;
8913
  if (!inited_) {
8914
    ret = OB_NOT_INIT;
8915
    LOG_WARN("not init", KR(ret));
8916
  } else if (!arg.is_valid()) {
8917
    ret = OB_INVALID_ARGUMENT;
8918
    LOG_WARN("invalid arg", KR(ret), K(arg));
8919
  } else if (OB_FAIL(share::ObArbitrationServiceUtils::replace_arbitration_service(sql_proxy_, arg))) {
8920
    LOG_WARN("fail to replace arbitration service", KR(ret), K(arg));
8921
  }
8922
  ROOTSERVICE_EVENT_ADD("arb_service", "admin_replace_arbitration_service", K(ret), K(arg));
8923
  return ret;
8924
}
8925

8926
int ObRootService::remove_cluster_info_from_arb_server(const obrpc::ObRemoveClusterInfoFromArbServerArg &arg)
8927
{
8928
  int ret = OB_SUCCESS;
8929
  if (!inited_) {
8930
    ret = OB_NOT_INIT;
8931
    LOG_WARN("not init", KR(ret));
8932
  } else if (!arg.is_valid()) {
8933
    ret = OB_INVALID_ARGUMENT;
8934
    LOG_WARN("invalid arg", KR(ret), K(arg));
8935
  } else if (OB_FAIL(ObArbitrationServiceUtils::remove_cluster_info_from_arb_server(arg.get_arbitration_service()))) {
8936
    LOG_WARN("fail to remove cluster info from arb server", KR(ret), K(arg));
8937
  }
8938
  return ret;
8939
}
8940
#endif
8941

8942
int ObRootService::admin_migrate_unit(const obrpc::ObAdminMigrateUnitArg &arg)
8943
{
8944
  int ret = OB_SUCCESS;
8945
  if (!inited_) {
8946
    ret = OB_NOT_INIT;
8947
    LOG_WARN("not init", K(ret));
8948
  } else if (!arg.is_valid()) {
8949
    ret = OB_INVALID_ARGUMENT;
8950
    LOG_WARN("invalid arg", K(arg), K(ret));
8951
  } else {
8952
    ObSystemAdminCtx ctx;
8953
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8954
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8955
    } else {
8956
      ObAdminMigrateUnit admin_util(ctx);
8957
      if (OB_FAIL(admin_util.execute(arg))) {
8958
        LOG_WARN("execute migrate unit failed", K(arg), K(ret));
8959
      }
8960
    }
8961
  }
8962
  ROOTSERVICE_EVENT_ADD("root_service", "admin_migrate_unit", K(ret), K(arg));
8963
  return ret;
8964
}
8965

8966
int ObRootService::admin_upgrade_virtual_schema()
8967
{
8968
  int ret = OB_SUCCESS;
8969
  if (!inited_) {
8970
    ret = OB_NOT_INIT;
8971
    LOG_WARN("not init", K(ret));
8972
  } else {
8973
    ObSystemAdminCtx ctx;
8974
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8975
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8976
    } else {
8977
      ObAdminUpgradeVirtualSchema admin_util(ctx);
8978
      if (OB_FAIL(admin_util.execute())) {
8979
        LOG_WARN("upgrade virtual schema failed", K(ret));
8980
      }
8981
    }
8982
  }
8983
  ROOTSERVICE_EVENT_ADD("root_service", "admin_upgrade_virtual_schema", K(ret));
8984
  return ret;
8985
}
8986

8987
int ObRootService::admin_upgrade_cmd(const obrpc::Bool &arg)
8988
{
8989
  int ret = OB_SUCCESS;
8990
  if (!inited_) {
8991
    ret = OB_NOT_INIT;
8992
    LOG_WARN("not init", K(ret));
8993
  } else {
8994
    ObSystemAdminCtx ctx;
8995
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
8996
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
8997
    } else {
8998
      ObAdminUpgradeCmd admin_util(ctx);
8999
      if (OB_FAIL(admin_util.execute(arg))) {
9000
        LOG_WARN("begin upgrade failed", K(ret));
9001
      }
9002
    }
9003
  }
9004
  ROOTSERVICE_EVENT_ADD("root_service", "admin_upgrade_cmd", K(ret), K(arg));
9005
  return ret;
9006
}
9007

9008
int ObRootService::admin_rolling_upgrade_cmd(const obrpc::ObAdminRollingUpgradeArg &arg)
9009
{
9010
  int ret = OB_SUCCESS;
9011
  if (!inited_) {
9012
    ret = OB_NOT_INIT;
9013
    LOG_WARN("not init", K(ret));
9014
  } else {
9015
    ObSystemAdminCtx ctx;
9016
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
9017
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
9018
    } else {
9019
      ObAdminRollingUpgradeCmd admin_util(ctx);
9020
      if (OB_FAIL(admin_util.execute(arg))) {
9021
        LOG_WARN("begin upgrade failed", K(ret));
9022
      }
9023
    }
9024
  }
9025
  ROOTSERVICE_EVENT_ADD("root_service", "admin_rolling_upgrade_cmd", K(ret), K(arg));
9026
  return ret;
9027
}
9028

9029
int ObRootService::physical_restore_tenant(const obrpc::ObPhysicalRestoreTenantArg &arg, obrpc::Int64 &res_job_id)
9030
{
9031
  int ret = OB_SUCCESS;
9032
  bool has_standby_cluster = false;
9033
  res_job_id = OB_INVALID_ID;
9034
  int64_t current_timestamp = ObTimeUtility::current_time();
9035
  int64_t start_ts = ObTimeUtility::current_time();
9036
  int64_t job_id = OB_INVALID_ID;
9037
  int64_t refreshed_schema_version = OB_INVALID_VERSION;
9038
  ObSchemaGetterGuard schema_guard;
9039
  int64_t restore_concurrency = 0;
9040
  omt::ObTenantConfigGuard tenant_config(TENANT_CONF(MTL_ID()));
9041
  if (tenant_config.is_valid()) {
9042
    restore_concurrency = tenant_config->ha_high_thread_score;
9043
  }
9044
  if (0 == restore_concurrency) {
9045
    restore_concurrency = OB_DEFAULT_RESTORE_CONCURRENCY;
9046
  }
9047

9048
  if (!inited_) {
9049
    ret = OB_NOT_INIT;
9050
    LOG_WARN("not init", K(ret));
9051
  } else if (!arg.is_valid()) {
9052
    ret = OB_INVALID_ARGUMENT;
9053
    LOG_WARN("invalid arg", K(arg), K(ret));
9054
  } else if (GCTX.is_standby_cluster() || GCONF.in_upgrade_mode()) {
9055
    ret = OB_OP_NOT_ALLOW;
9056
    LOG_WARN("restore tenant while in standby cluster or "
9057
             "in upgrade mode is not allowed", KR(ret));
9058
    LOG_USER_ERROR(OB_OP_NOT_ALLOW,
9059
                   "restore tenant while in standby cluster or in upgrade mode");
9060
  } else if (0 == restore_concurrency) {
9061
    ret = OB_OP_NOT_ALLOW;
9062
    LOG_WARN("restore tenant when restore_concurrency is 0 not allowed", KR(ret));
9063
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "restore tenant when restore_concurrency is 0");
9064
  } else if (OB_FAIL(ObResolverUtils::check_not_supported_tenant_name(arg.tenant_name_))) {
9065
    LOG_WARN("unsupported tenant name", KR(ret), "tenant_name", arg.tenant_name_);
9066
  } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(
9067
                     OB_SYS_TENANT_ID, schema_guard))) {
9068
    LOG_WARN("fail to get sys tenant's schema guard", KR(ret));
9069
  } else if (OB_FAIL(schema_guard.get_schema_version(OB_SYS_TENANT_ID, refreshed_schema_version))) {
9070
    LOG_WARN("fail to get sys schema version", KR(ret));
9071
  } else {
9072
    HEAP_VAR(ObPhysicalRestoreJob, job_info) {
9073
      // just to check sys tenant's schema with latest schema version
9074
      ObDDLSQLTransaction trans(schema_service_, false /*end_signal*/);
9075
      if (OB_FAIL(trans.start(&sql_proxy_, OB_SYS_TENANT_ID, refreshed_schema_version))) {
9076
        LOG_WARN("failed to start trans, ", K(ret));
9077
      } else if (OB_FAIL(RS_JOB_CREATE_EXT(job_id, RESTORE_TENANT, trans,
9078
                         "sql_text", ObHexEscapeSqlStr(arg.get_sql_stmt())))) {
9079
        LOG_WARN("fail create rs job", K(ret), K(arg));
9080
      } else if (job_id < 0) {
9081
        ret = OB_ERR_UNEXPECTED;
9082
        LOG_WARN("invalid job_id", K(ret), K(job_id));
9083
      } else if (OB_FAIL(ObRestoreUtil::fill_physical_restore_job(job_id, arg, job_info))) {
9084
        LOG_WARN("fail to fill physical restore job", K(ret), K(job_id), K(arg));
9085
      } else {
9086
        job_info.set_restore_start_ts(start_ts);
9087
        res_job_id = job_id;
9088
      }
9089
      if (FAILEDx(check_restore_tenant_valid(job_info, schema_guard))) {
9090
        LOG_WARN("failed to check restore tenant vailid", KR(ret), K(job_info));
9091
      }
9092

9093
      if (FAILEDx(ObRestoreUtil::record_physical_restore_job(trans, job_info))) {
9094
        LOG_WARN("fail to record physical restore job", K(ret), K(job_id), K(arg));
9095
      } else {
9096
        //TODO wakeup restore scheduler
9097
      }
9098

9099
      if (trans.is_started()) {
9100
        int temp_ret = OB_SUCCESS;
9101
        if (OB_SUCCESS != (temp_ret = trans.end(OB_SUCC(ret)))) {
9102
          LOG_WARN("trans end failed", "is_commit", OB_SUCCESS == ret, K(temp_ret));
9103
          ret = (OB_SUCC(ret)) ? temp_ret : ret;
9104
        }
9105
      }
9106
    }
9107
  }
9108
  LOG_INFO("[RESTORE] physical restore tenant start", K(arg), K(ret));
9109
  ROOTSERVICE_EVENT_ADD("physical_restore", "restore_start", K(ret),
9110
                        "tenant_name", arg.tenant_name_);
9111
  if (OB_SUCC(ret)) {
9112
    const char *status_str = ObPhysicalRestoreTableOperator::get_restore_status_str(
9113
                             static_cast<PhysicalRestoreStatus>(PHYSICAL_RESTORE_CREATE_TENANT));
9114
    ROOTSERVICE_EVENT_ADD("physical_restore", "change_restore_status",
9115
                          "job_id", job_id,
9116
                          "tenant_name", arg.tenant_name_,
9117
                          "status", status_str);
9118
  }
9119
  return ret;
9120
}
9121

9122
int ObRootService::check_restore_tenant_valid(const share::ObPhysicalRestoreJob &job_info,
9123
    share::schema::ObSchemaGetterGuard &schema_guard)
9124
{
9125
  int ret = OB_SUCCESS;
9126
  if (OB_UNLIKELY(!inited_)) {
9127
    ret = OB_NOT_INIT;
9128
    LOG_WARN("not init", K(ret));
9129
  } else if (OB_UNLIKELY(!job_info.is_valid())) {
9130
    ret = OB_INVALID_ARGUMENT;
9131
    LOG_WARN("invalid arg", KR(ret), K(job_info));
9132
  } else {
9133
    //check tenant if exist
9134
    const ObTenantSchema *tenant_schema = NULL;
9135
    const ObString &tenant_name = job_info.get_tenant_name();
9136
    if (OB_FAIL(schema_guard.get_tenant_info(tenant_name, tenant_schema))) {
9137
      LOG_WARN("fail to get tenant schema", KR(ret), K(job_info));
9138
    } else if (OB_NOT_NULL(tenant_schema)) {
9139
      ret = OB_OP_NOT_ALLOW;
9140
      LOG_WARN("restore tenant with existed tenant name is not allowed", KR(ret), K(tenant_name));
9141
      LOG_USER_ERROR(OB_OP_NOT_ALLOW, "restore tenant with existed tenant name is");
9142
    } else {
9143
      // check if resource pool contains any E zone
9144
      ObSqlString pool_list_str;
9145
      ObArray<ObString> pool_list;
9146
      ObArray<ObResourcePoolName> pools;
9147
      ObArray<ObZone> zones;
9148
      if (OB_FAIL(pool_list_str.assign(job_info.get_pool_list()))) {
9149
        LOG_WARN("failed to assign pool list", KR(ret), K(job_info));
9150
      } else if (OB_FAIL(ObRestoreScheduler::assign_pool_list(pool_list_str.ptr(), pool_list))) {
9151
        LOG_WARN("failed to assgin pool list", KR(ret), K(pool_list_str));
9152
      } else if (OB_FAIL(ObUnitManager::convert_pool_name_list(pool_list, pools))) {
9153
         LOG_WARN("fail to convert pools", KR(ret), K(pool_list));
9154
      } else if (OB_FAIL(unit_manager_.get_zones_of_pools(pools, zones))) {
9155
        LOG_WARN("fail to get zones of pools", KR(ret), K(pools));
9156
      } else {
9157
        HEAP_VAR(ObZoneInfo, info) {
9158
          for (int64_t i = 0; OB_SUCC(ret) && i < zones.count(); i++) {
9159
            const ObZone &zone = zones.at(i);
9160
            info.reset();
9161
            if (OB_FAIL(zone_manager_.get_zone(zone, info))) {
9162
              LOG_WARN("fail to get zone info", KR(ret), K(zone));
9163
            } else if (static_cast<int64_t>(ObZoneType::ZONE_TYPE_ENCRYPTION) == info.zone_type_.value_) {
9164
              ret = OB_NOT_SUPPORTED;
9165
              LOG_WARN("restore tenant with encrypted zone is not supported", KR(ret), K(info));
9166
            }
9167
          }
9168
        }
9169
      }
9170
    }
9171
  }
9172
  //TODO check if need check R replica
9173
  return ret;
9174
}
9175

9176
int ObRootService::run_job(const obrpc::ObRunJobArg &arg)
9177
{
9178
  int ret = OB_SUCCESS;
9179
  if (!inited_) {
9180
    ret = OB_NOT_INIT;
9181
    LOG_WARN("not init", K(ret));
9182
  } else if (!arg.is_valid()) {
9183
    ret = OB_INVALID_ARGUMENT;
9184
    LOG_WARN("invalid arg", K(arg), K(ret));
9185
  } else {
9186
    ObSystemAdminCtx ctx;
9187
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
9188
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
9189
    } else {
9190
      ObAdminRunJob admin_util(ctx);
9191
      if (OB_FAIL(admin_util.execute(arg))) {
9192
        LOG_WARN("run job failed", K(arg), K(ret));
9193
      }
9194
    }
9195
  }
9196
  ROOTSERVICE_EVENT_ADD("root_service", "admin_run_job", K(ret), K(arg));
9197
  return ret;
9198
}
9199

9200
int ObRootService::run_upgrade_job(const obrpc::ObUpgradeJobArg &arg)
9201
{
9202
  int ret = OB_SUCCESS;
9203
  int64_t version = arg.version_;
9204
  if (!inited_) {
9205
    ret = OB_NOT_INIT;
9206
    LOG_WARN("not init", KR(ret), K(arg));
9207
  } else if (!arg.is_valid()) {
9208
    ret = OB_INVALID_ARGUMENT;
9209
    LOG_WARN("invalid arg", K(arg), KR(ret));
9210
  } else if (ObUpgradeJobArg::UPGRADE_POST_ACTION == arg.action_
9211
             && !ObUpgradeChecker::check_data_version_exist(version)) {
9212
    ret = OB_NOT_SUPPORTED;
9213
    LOG_WARN("unsupported version to run upgrade job", KR(ret), K(version));
9214
    LOG_USER_ERROR(OB_NOT_SUPPORTED, "run upgrade job with such version is");
9215
  } else if (ObUpgradeJobArg::STOP_UPGRADE_JOB == arg.action_) {
9216
    if (OB_FAIL(upgrade_executor_.stop())) {
9217
      LOG_WARN("fail to stop upgrade task", KR(ret));
9218
    } else {
9219
      upgrade_executor_.start();
9220
    }
9221
  } else if (OB_FAIL(submit_upgrade_task(arg))) {
9222
    LOG_WARN("fail to submit upgrade task", KR(ret), K(arg));
9223
  }
9224
  ROOTSERVICE_EVENT_ADD("root_service", "admin_run_upgrade_job", KR(ret), K(arg));
9225
  return ret;
9226
}
9227

9228
int ObRootService::upgrade_table_schema(const obrpc::ObUpgradeTableSchemaArg &arg)
9229
{
9230
  int ret = OB_SUCCESS;
9231
  const int64_t start = ObTimeUtility::current_time();
9232
  FLOG_INFO("[UPGRADE] start to upgrade table", K(arg));
9233
  if (!inited_) {
9234
    ret = OB_NOT_INIT;
9235
    LOG_WARN("not init", KR(ret));
9236
  } else if (!arg.is_valid()) {
9237
    ret = OB_INVALID_ARGUMENT;
9238
    LOG_WARN("arg is invalid", KR(ret), K(arg));
9239
  } else if (!arg.upgrade_virtual_schema()) {
9240
    // upgrade single system table
9241
    if (OB_FAIL(ddl_service_.upgrade_table_schema(arg))) {
9242
      LOG_WARN("fail to upgrade table schema", KR(ret), K(arg));
9243
    }
9244
  } else {
9245
    // upgrade all virtual table/sys view
9246
    ObSystemAdminCtx ctx;
9247
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
9248
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
9249
    } else {
9250
      ObAdminUpgradeVirtualSchema admin_util(ctx);
9251
      int64_t upgrade_cnt = 0;
9252
      if (OB_FAIL(admin_util.execute(arg.get_tenant_id(), upgrade_cnt))) {
9253
        LOG_WARN("upgrade virtual schema failed", KR(ret), K(arg));
9254
      }
9255
    }
9256
  }
9257
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "update table schema",
9258
                        "tenant_id", arg.get_tenant_id(),
9259
                        "ret", ret,
9260
                        "trace_id", *ObCurTraceId::get_trace_id(),
9261
                        "table_id", arg.get_table_id());
9262
  FLOG_INFO("[UPGRADE] finish upgrade table", KR(ret), K(arg),
9263
            "cost_us", ObTimeUtility::current_time() - start, "ddl_event_info", ObDDLEventInfo());
9264
  return ret;
9265
}
9266

9267
int ObRootService::broadcast_ds_action(const obrpc::ObDebugSyncActionArg &arg)
9268
{
9269
  LOG_INFO("receive broadcast debug sync actions", K(arg));
9270
  int ret = OB_SUCCESS;
9271
  ObArray<ObAddr> server_list;
9272
  const ObZone all_zone;
9273
  if (!inited_) {
9274
    ret = OB_NOT_INIT;
9275
    LOG_WARN("not init", K(ret));
9276
  } else if (!arg.is_valid()) {
9277
    ret = OB_INVALID_ARGUMENT;
9278
    LOG_WARN("invalid arg", K(arg), K(ret));
9279
  } else if (OB_FAIL(SVR_TRACER.get_alive_servers(all_zone, server_list))) {
9280
    LOG_WARN("get all alive servers failed", K(all_zone), K(ret));
9281
  } else {
9282
    FOREACH_X(s, server_list, OB_SUCCESS == ret) {
9283
      if (OB_FAIL(rpc_proxy_.to(*s).timeout(config_->rpc_timeout).set_debug_sync_action(arg))) {
9284
        LOG_WARN("set server's global sync action failed", K(ret), "server", *s, K(arg));
9285
      }
9286
    }
9287
  }
9288
  return ret;
9289
}
9290

9291
int ObRootService::check_dangling_replica_finish(const obrpc::ObCheckDanglingReplicaFinishArg &arg)
9292
{
9293
  UNUSED(arg);
9294
  return OB_NOT_SUPPORTED;
9295
}
9296

9297
int ObRootService::fetch_alive_server(const ObFetchAliveServerArg &arg,
9298
                                      ObFetchAliveServerResult &result)
9299
{
9300
  LOG_DEBUG("receive fetch alive server request");
9301
  ObZone empty_zone; // for all server
9302
  int ret = OB_SUCCESS;
9303
  if (!inited_) {
9304
    ret = OB_NOT_INIT;
9305
    LOG_WARN("not init", K(ret));
9306
  } else if (!arg.is_valid()) {
9307
    ret = OB_INVALID_ARGUMENT;
9308
    LOG_WARN("invalid argument", K(ret), K(arg));
9309
  } else if (!server_refreshed_) {
9310
    ret = OB_SERVER_IS_INIT;
9311
    LOG_WARN("RS is initializing, server not refreshed, can not process this request");
9312
  } else if (arg.cluster_id_ != config_->cluster_id) {
9313
    ret = OB_ERR_UNEXPECTED;
9314
    LOG_WARN("cluster id mismatch",
9315
             K(ret), K(arg), "cluster_id", static_cast<int64_t>(config_->cluster_id));
9316
  } else if (OB_FAIL(SVR_TRACER.get_servers_by_status(empty_zone, result.active_server_list_,
9317
                                                           result.inactive_server_list_))) {
9318
    LOG_WARN("get alive servers failed", K(ret));
9319
  }
9320
  return ret;
9321
}
9322

9323
int ObRootService::refresh_server(const bool load_frozen_status, const bool need_retry)
9324
{
9325
  int ret = OB_SUCCESS;
9326
  if (!inited_) {
9327
    ret = OB_NOT_INIT;
9328
    LOG_WARN("not init", K(ret));
9329
  } else if (!in_service()|| (server_refreshed_ && load_frozen_status)) {
9330
    // no need to refresh again for fast recover
9331
  } else {
9332
    {
9333
      ObTimeoutCtx ctx;
9334
      if (load_frozen_status) {
9335
        ctx.set_timeout(config_->rpc_timeout);
9336
      }
9337
      if (OB_FAIL(load_server_manager())) {
9338
        LOG_WARN("build server manager failed", K(ret), K(load_frozen_status));
9339
      } else {
9340
        LOG_INFO("build server manager succeed", K(load_frozen_status));
9341
      }
9342
      if (FAILEDx(SVR_TRACER.refresh())) {
9343
        LOG_WARN("fail to refresh all server tracer", KR(ret));
9344
      }
9345
    }
9346
    // request heartbeats from observers
9347
    if (OB_SUCC(ret) && !ObHeartbeatService::is_service_enabled()) {
9348
      int temp_ret = OB_SUCCESS;
9349
      if (OB_SUCCESS != (temp_ret = request_heartbeats())) {
9350
        LOG_WARN("request heartbeats failed", K(temp_ret));
9351
      } else {
9352
        LOG_INFO("request heartbeats succeed");
9353
      }
9354
    }
9355
    if (OB_SUCC(ret)) {
9356
      server_refreshed_ = true;
9357
      LOG_INFO("refresh server success", K(load_frozen_status));
9358
    } else if (need_retry) {
9359
      LOG_INFO("refresh server failed, retry", K(ret), K(load_frozen_status));
9360
      int tmp_ret = OB_SUCCESS;
9361
      if (OB_SUCCESS != (tmp_ret = schedule_refresh_server_timer_task(
9362
                  ObRefreshServerTask::REFRESH_SERVER_INTERVAL))) {
9363
        if (OB_CANCELED != ret) {
9364
          LOG_ERROR("schedule refresh server task failed", K(ret));
9365
        } else {
9366
          LOG_WARN("schedule refresh server task failed, because the server is stopping", K(ret));
9367
        }
9368
      } else {
9369
        ObTaskController::get().allow_next_syslog();
9370
        LOG_INFO("schedule refresh server task again", KR(tmp_ret), KR(ret));
9371
      }
9372
    } else {} // no more to do
9373
  }
9374
  return ret;
9375
}
9376

9377
int ObRootService::refresh_schema(const bool load_frozen_status)
9378
{
9379
  int ret = OB_SUCCESS;
9380
  if (!inited_) {
9381
    ret = OB_NOT_INIT;
9382
    LOG_WARN("not init", K(ret));
9383
  } else {
9384
    ObTimeoutCtx ctx;
9385
    int64_t schema_version = OB_INVALID_VERSION;
9386
    if (load_frozen_status) {
9387
      ctx.set_timeout(config_->rpc_timeout);
9388
    }
9389
    ObArray<uint64_t> tenant_ids; //depend on sys schema while start RS
9390
    if (OB_FAIL(tenant_ids.push_back(OB_SYS_TENANT_ID))) {
9391
      LOG_WARN("fail to refresh sys schema", K(ret));
9392
    } else if (OB_FAIL(schema_service_->refresh_and_add_schema(tenant_ids))) {
9393
      LOG_WARN("refresh schema failed", K(ret), K(load_frozen_status));
9394
    } else if (OB_FAIL(schema_service_->get_tenant_schema_version(OB_SYS_TENANT_ID, schema_version))) {
9395
      LOG_WARN("fail to get max schema version", K(ret));
9396
    } else {
9397
      LOG_INFO("refresh schema with new mode succeed", K(load_frozen_status), K(schema_version));
9398
    }
9399
    if (OB_SUCC(ret)) {
9400
      ObSchemaService *schema_service = schema_service_->get_schema_service();
9401
      if (NULL == schema_service) {
9402
        ret = OB_ERR_SYS;
9403
        LOG_WARN("schema_service can't be null", K(ret), K(schema_version));
9404
      } else {
9405
        schema_service->set_refreshed_schema_version(schema_version);
9406
        LOG_INFO("set schema version succeed", K(ret), K(schema_service), K(schema_version));
9407
      }
9408
    }
9409
  }
9410
  return ret;
9411
}
9412

9413
int ObRootService::set_cluster_version()
9414
{
9415
  int ret = OB_SUCCESS;
9416
  int64_t affected_rows = 0;
9417
  char sql[1024] = {0};
9418
  ObMySQLProxy &sql_proxy = ddl_service_.get_sql_proxy();
9419

9420
  snprintf(sql, sizeof(sql), "alter system set min_observer_version = '%s'", PACKAGE_VERSION);
9421
  if (OB_FAIL(sql_proxy.write(OB_SYS_TENANT_ID, sql, affected_rows))) {
9422
    LOG_WARN("execute sql failed", K(sql));
9423
  }
9424

9425
  return ret;
9426
}
9427

9428
int ObRootService::admin_set_tracepoint(const obrpc::ObAdminSetTPArg &arg)
9429
{
9430
  int ret = OB_SUCCESS;
9431
  if (!inited_) {
9432
    ret = OB_NOT_INIT;
9433
    LOG_WARN("not init", K(ret));
9434
  } else if (!arg.is_valid()) {
9435
    ret = OB_INVALID_ARGUMENT;
9436
    LOG_WARN("invalid arg", K(arg), K(ret));
9437
  } else {
9438
    ObSystemAdminCtx ctx;
9439
    if (OB_FAIL(init_sys_admin_ctx(ctx))) {
9440
      LOG_WARN("init_sys_admin_ctx failed", K(ret));
9441
    } else {
9442
      ObAdminSetTP admin_util(ctx, arg);
9443
      if (OB_FAIL(admin_util.execute(arg))) {
9444
        LOG_WARN("execute report replica failed", K(arg), K(ret));
9445
      }
9446
    }
9447
  }
9448
  ROOTSERVICE_EVENT_ADD("root_service", "admin_set_tracepoint", K(ret), K(arg));
9449
  return ret;
9450
}
9451

9452
// RS may receive refresh time zone from observer with old binary during upgrade.
9453
// do notiong
9454
int ObRootService::refresh_time_zone_info(const obrpc::ObRefreshTimezoneArg &arg)
9455
{
9456
  int ret = OB_SUCCESS;
9457
  UNUSED(arg);
9458
  ROOTSERVICE_EVENT_ADD("root_service", "refresh_time_zone_info", K(ret), K(arg));
9459
  return ret;
9460
}
9461

9462
int ObRootService::request_time_zone_info(const ObRequestTZInfoArg &arg, ObRequestTZInfoResult &result)
9463
{
9464
  UNUSED(arg);
9465
  int ret = OB_SUCCESS;
9466
  uint64_t tenant_id = OB_SYS_TENANT_ID;
9467

9468
  ObTZMapWrap tz_map_wrap;
9469
  ObTimeZoneInfoManager *tz_info_mgr = NULL;
9470
  if (!inited_) {
9471
    ret = OB_NOT_INIT;
9472
    LOG_WARN("not init", K(ret));
9473
  } else if (OB_FAIL(OTTZ_MGR.get_tenant_timezone(tenant_id, tz_map_wrap, tz_info_mgr))) {
9474
    LOG_WARN("get tenant timezone failed", K(ret));
9475
  } else if (OB_ISNULL(tz_info_mgr)) {
9476
    ret = OB_ERR_UNEXPECTED;
9477
    LOG_WARN("get_tz_mgr failed", K(ret), K(tz_info_mgr));
9478
  } else if (OB_FAIL(tz_info_mgr->response_time_zone_info(result))) {
9479
    LOG_WARN("fail to response tz_info", K(ret));
9480
  } else {
9481
    LOG_INFO("rs success to response lastest tz_info to server", "server", arg.obs_addr_, "last_version", result.last_version_);
9482
  }
9483
  return ret;
9484
}
9485

9486
bool ObRootService::check_config(const ObConfigItem &item, const char *&err_info)
9487
{
9488
  bool bret = true;
9489
  err_info = NULL;
9490
  if (!inited_) {
9491
    bret = false;
9492
    LOG_WARN_RET(OB_NOT_INIT, "service not init");
9493
  } else if (0 == STRCMP(item.name(), MIN_OBSERVER_VERSION)) {
9494
    if (OB_SUCCESS != ObClusterVersion::is_valid(item.str())) {
9495
      LOG_WARN_RET(OB_INVALID_ERROR, "fail to parse min_observer_version value");
9496
      bret = false;
9497
    }
9498
  } else if (0 == STRCMP(item.name(), __BALANCE_CONTROLLER)) {
9499
    ObString balance_troller_str(item.str());
9500
    ObRootBalanceHelp::BalanceController switch_info;
9501
    int tmp_ret = ObRootBalanceHelp::parse_balance_info(balance_troller_str, switch_info);
9502
    if (OB_SUCCESS != tmp_ret) {
9503
      LOG_WARN_RET(tmp_ret, "fail to parse balance switch", K(balance_troller_str));
9504
      bret = false;
9505
    }
9506
  }
9507
  return bret;
9508
}
9509

9510
int ObRootService::report_replica()
9511
{
9512
  int ret = OB_SUCCESS;
9513
  ObArray<ObAddr> server_list;
9514
  ObZone null_zone;
9515
  if (!inited_) {
9516
    ret = OB_NOT_INIT;
9517
    LOG_WARN("not init", K(ret));
9518
  } else if (OB_FAIL(SVR_TRACER.get_alive_servers(null_zone, server_list))) {
9519
    LOG_WARN("fail to get alive server", K(ret));
9520
  } else {
9521
    FOREACH_CNT(server, server_list) {
9522
      if (OB_ISNULL(server)) {
9523
        ret = OB_ERR_UNEXPECTED;
9524
        LOG_WARN("get invalid server", K(ret));
9525
      } else if (OB_FAIL(rpc_proxy_.to(*server).report_replica())) {
9526
        LOG_WARN("fail to force observer report replica", K(ret), K(*server));
9527
      }
9528
      ret = OB_SUCCESS;
9529
    }
9530
  }
9531
  return ret;
9532
}
9533

9534
int ObRootService::report_single_replica(
9535
    const int64_t tenant_id,
9536
    const ObLSID &ls_id)
9537
{
9538
  int ret = OB_SUCCESS;
9539
  ObArray<ObAddr> server_list;
9540
  ObZone null_zone;
9541
  obrpc::ObReportSingleReplicaArg arg;
9542
  if (OB_UNLIKELY(!inited_)) {
9543
    ret = OB_NOT_INIT;
9544
    LOG_WARN("not init", KR(ret));
9545
  } else if (OB_FAIL(SVR_TRACER.get_alive_servers(null_zone, server_list))) {
9546
    LOG_WARN("fail to get alive server", KR(ret));
9547
  } else if (OB_INVALID_TENANT_ID == tenant_id || !ls_id.is_valid()) {
9548
    ret = OB_INVALID_ARGUMENT;
9549
    LOG_WARN("invalid argument", KR(ret), K(tenant_id), K(ls_id));
9550
  } else {
9551
    arg.tenant_id_ = tenant_id;
9552
    arg.ls_id_ = ls_id;
9553
    FOREACH_CNT(server, server_list) {
9554
      if (OB_ISNULL(server)) {
9555
        ret = OB_ERR_UNEXPECTED;
9556
        LOG_WARN("get invalid server", KR(ret));
9557
//      } else if (OB_FAIL(rpc_proxy_.to(*server).report_single_replica(arg))) {//TODO(xiuming):delete it
9558
//        LOG_WARN("fail to force observer report replica", KR(ret), K(server), K(arg));
9559
      }
9560
      ret = OB_SUCCESS;
9561
    }
9562
  }
9563
  return ret;
9564
}
9565

9566
int ObRootService::update_all_server_config()
9567
{
9568
  int ret = OB_SUCCESS;
9569
  ObZone empty_zone;
9570
  ObArray<ObAddr> server_list;
9571
  ObArray<ObAddr> config_all_server_list;
9572
  ObArray<ObAddr> empty_excluded_server_list;
9573
  bool need_update = true;
9574
  HEAP_VAR(ObAdminSetConfigItem, all_server_config) {
9575
    auto &value = all_server_config.value_;
9576
    int64_t pos = 0;
9577
    if (!inited_) {
9578
      ret = OB_NOT_INIT;
9579
      LOG_WARN("not init", K(ret));
9580
    } else if (OB_UNLIKELY(!SVR_TRACER.has_build())) {
9581
      need_update = false;
9582
    } else if (OB_FAIL(SVR_TRACER.get_servers_of_zone(empty_zone, server_list))) {
9583
      LOG_WARN("fail to get server", K(ret));
9584
    } else if (OB_UNLIKELY(0 == server_list.size())) {
9585
      need_update = false;
9586
      LOG_WARN("no servers in all_server_tracer");
9587
    } else if (OB_FAIL(all_server_config.name_.assign(config_->all_server_list.name()))) {
9588
      LOG_WARN("fail to assign name", K(ret));
9589
    } else if (OB_FAIL(ObShareUtil::parse_all_server_list(empty_excluded_server_list, config_all_server_list))) {
9590
      LOG_WARN("fail to parse all_server_list from GCONF", KR(ret));
9591
    } else if (ObRootUtils::is_subset(server_list, config_all_server_list)
9592
        && ObRootUtils::is_subset(config_all_server_list, server_list)) {
9593
      need_update = false;
9594
      LOG_TRACE("server_list is the same as config_all_server_list, no need to update GCONF.all_server_list",
9595
          K(server_list), K(config_all_server_list));
9596
    } else {
9597
      LOG_INFO("GCONF.all_server_list should be updated", K(config_all_server_list), K(server_list));
9598
      char ip_port_buf[MAX_IP_PORT_LENGTH];
9599
      for (int64_t i = 0; i < server_list.count() - 1; i++) {
9600
        if (OB_FAIL(server_list.at(i).ip_port_to_string(ip_port_buf, MAX_IP_PORT_LENGTH))) {
9601
          LOG_WARN("fail to print ip port", K(ret), "server", server_list.at(i));
9602
        } else if (OB_FAIL(databuff_printf(value.ptr(), OB_MAX_CONFIG_VALUE_LEN, pos, "%s", ip_port_buf))) {
9603
          LOG_WARN("fail to databuff_printf", K(ret), K(i), K(server_list));
9604
        } else if (OB_FAIL(databuff_printf(value.ptr(), OB_MAX_CONFIG_VALUE_LEN, pos, "%c", ','))) {
9605
          LOG_WARN("fail to print char", K(ret), K(i), K(server_list));
9606
        }
9607
      }
9608
      if (OB_SUCC(ret) && 0 < server_list.count()) {
9609
        if (OB_FAIL(server_list.at(server_list.count() - 1).ip_port_to_string(ip_port_buf, MAX_IP_PORT_LENGTH))) {
9610
          LOG_WARN("fail to print ip port", K(ret), "server", server_list.at(server_list.count() - 1));
9611
        } else if (OB_FAIL(databuff_printf(value.ptr(), OB_MAX_CONFIG_VALUE_LEN, pos, "%s", ip_port_buf))) {
9612
          LOG_WARN("fail to databuff_printf", K(ret), K(server_list), K(ip_port_buf));
9613
        }
9614
      }
9615
    }
9616
    if (OB_SIZE_OVERFLOW == ret) {
9617
      LOG_ERROR("can't print server addr to buffer, size overflow", K(ret), K(server_list));
9618
    }
9619
    if (need_update && OB_SUCC(ret)) {
9620
      ObAdminSetConfigArg arg;
9621
      arg.is_inner_ = true;
9622
      if (OB_FAIL(arg.items_.push_back(all_server_config))) {
9623
        LOG_WARN("fail to push back", K(ret));
9624
      } else if (OB_FAIL(admin_set_config(arg))) {
9625
        LOG_WARN("fail to set config", K(ret));
9626
      } else {
9627
        LOG_INFO("update all server config success", K(arg));
9628
      }
9629
    }
9630
  }
9631
  return ret;
9632
}
9633
/////////////////////////
9634
ObRootService::ObReportCoreTableReplicaTask::ObReportCoreTableReplicaTask(ObRootService &root_service)
9635
: ObAsyncTimerTask(root_service.task_queue_),
9636
    root_service_(root_service)
9637
{
9638
  set_retry_times(INT64_MAX); //retry until success
9639
}
9640

9641
int ObRootService::ObReportCoreTableReplicaTask::process()
9642
{
9643
  int ret = OB_SUCCESS;
9644
  const int64_t tenant_id = OB_SYS_TENANT_ID;
9645
  if (OB_FAIL(root_service_.report_single_replica(tenant_id, SYS_LS))) {
9646
    LOG_WARN("fail to report single replica", K(ret), K(tenant_id), K(SYS_LS));
9647
  } else {
9648
    LOG_INFO("report all_core table succeed");
9649
  }
9650
  return ret;
9651
}
9652

9653
ObAsyncTask *ObRootService::ObReportCoreTableReplicaTask::deep_copy(char *buf, const int64_t buf_size) const
9654
{
9655
  ObReportCoreTableReplicaTask *task = NULL;
9656
  if (NULL == buf || buf_size < static_cast<int64_t>(sizeof(*this))) {
9657
    LOG_WARN_RET(OB_BUF_NOT_ENOUGH, "buffer not large enough", K(buf_size), KP(buf));
9658
  } else {
9659
    task = new (buf) ObReportCoreTableReplicaTask(root_service_);
9660
  }
9661
  return task;
9662
}
9663
//////////////ObReloadUnitManagerTask
9664
ObRootService::ObReloadUnitManagerTask::ObReloadUnitManagerTask(ObRootService &root_service,
9665
                                                                ObUnitManager &unit_manager)
9666
: ObAsyncTimerTask(root_service.task_queue_),
9667
    root_service_(root_service),
9668
    unit_manager_(unit_manager)
9669
{
9670
  set_retry_times(INT64_MAX); // retry until success
9671
}
9672

9673
int ObRootService::ObReloadUnitManagerTask::process()
9674
{
9675
  int ret = OB_SUCCESS;
9676
  if (OB_FAIL(unit_manager_.load())) {
9677
    LOG_WARN("fail to reload unit_manager", K(ret));
9678
  } else {
9679
    LOG_INFO("reload unit_manger succeed");
9680
  }
9681
  return ret;
9682
}
9683

9684
ObAsyncTask *ObRootService::ObReloadUnitManagerTask::deep_copy(char *buf, const int64_t buf_size) const
9685
{
9686
  ObReloadUnitManagerTask *task = NULL;
9687
  if (NULL == buf || buf_size < static_cast<int64_t>(sizeof(*this))) {
9688
    LOG_WARN_RET(OB_BUF_NOT_ENOUGH, "buffer not large enough", K(buf_size), KP(buf));
9689
  } else {
9690
    task = new (buf) ObReloadUnitManagerTask(root_service_, unit_manager_);
9691
  }
9692
  return task;
9693
}
9694

9695
ObRootService::ObLoadDDLTask::ObLoadDDLTask(ObRootService &root_service)
9696
  : ObAsyncTimerTask(root_service.task_queue_), root_service_(root_service)
9697
{
9698
  set_retry_times(INT64_MAX);
9699
}
9700

9701
int ObRootService::ObLoadDDLTask::process()
9702
{
9703
  int ret = OB_SUCCESS;
9704
  ObDDLScheduler &ddl_scheduler = root_service_.get_ddl_scheduler();
9705
  if (OB_FAIL(ddl_scheduler.recover_task())) {
9706
    LOG_WARN("load ddl task failed", K(ret));
9707
  }
9708
  return ret;
9709
}
9710

9711
ObAsyncTask *ObRootService::ObLoadDDLTask::deep_copy(char *buf, const int64_t buf_size) const
9712
{
9713
  ObLoadDDLTask *task = nullptr;
9714
  if (nullptr == buf || buf_size < static_cast<int64_t>(sizeof(*this))) {
9715
    LOG_WARN_RET(OB_BUF_NOT_ENOUGH, "buf is not enough", K(buf_size), "request_size", sizeof(*this));
9716
  } else {
9717
    task = new (buf) ObLoadDDLTask(root_service_);
9718
  }
9719
  return task;
9720
}
9721

9722
ObRootService::ObRefreshIOCalibrationTask::ObRefreshIOCalibrationTask(ObRootService &root_service)
9723
  : ObAsyncTimerTask(root_service.task_queue_), root_service_(root_service)
9724
{
9725
  set_retry_times(INT64_MAX);
9726
}
9727

9728
int ObRootService::ObRefreshIOCalibrationTask::process()
9729
{
9730
  int ret = OB_SUCCESS;
9731
  obrpc::ObAdminRefreshIOCalibrationArg arg;
9732
  arg.only_refresh_ = true;
9733
  if (OB_FAIL(root_service_.admin_refresh_io_calibration(arg))) {
9734
    LOG_WARN("refresh io calibration failed", K(ret), K(arg));
9735
  } else {
9736
    LOG_INFO("refresh io calibration succeeded");
9737
  }
9738
  return ret;
9739
}
9740

9741
ObAsyncTask *ObRootService::ObRefreshIOCalibrationTask::deep_copy(char *buf, const int64_t buf_size) const
9742
{
9743
  ObRefreshIOCalibrationTask *task = nullptr;
9744
  if (nullptr == buf || buf_size < static_cast<int64_t>(sizeof(*this))) {
9745
    LOG_WARN_RET(OB_BUF_NOT_ENOUGH, "buf is not enough", K(buf_size), "request_size", sizeof(*this));
9746
  } else {
9747
    task = new (buf) ObRefreshIOCalibrationTask(root_service_);
9748
  }
9749
  return task;
9750
}
9751

9752
////////////////////
9753
ObRootService::ObSelfCheckTask::ObSelfCheckTask(ObRootService &root_service)
9754
:ObAsyncTimerTask(root_service.task_queue_),
9755
    root_service_(root_service)
9756
{
9757
  set_retry_times(0);  // don't retry when failed
9758
}
9759

9760
int ObRootService::ObSelfCheckTask::process()
9761
{
9762
  int ret = OB_SUCCESS;
9763
  if (OB_FAIL(root_service_.self_check())) {
9764
    LOG_WARN("fail to do root inspection check, please check it", K(ret));
9765
  } else {
9766
    LOG_INFO("self check success!");
9767
  }
9768
  return ret;
9769
}
9770

9771
ObAsyncTask *ObRootService::ObSelfCheckTask::deep_copy(char *buf, const int64_t buf_size) const
9772
{
9773
  ObSelfCheckTask *task = NULL;
9774
  if (NULL == buf || buf_size < static_cast<int64_t>(sizeof(*this))) {
9775
    LOG_WARN_RET(OB_BUF_NOT_ENOUGH, "buffer not large enough", K(buf_size));
9776
  } else {
9777
    task = new(buf) ObSelfCheckTask(root_service_);
9778
  }
9779
  return task;
9780
}
9781

9782
/////////////////////////
9783
ObRootService::ObUpdateAllServerConfigTask::ObUpdateAllServerConfigTask(ObRootService &root_service)
9784
: ObAsyncTimerTask(root_service.task_queue_),
9785
    root_service_(root_service)
9786
{
9787
  set_retry_times(INT64_MAX); // retry until success
9788
}
9789

9790
int ObRootService::ObUpdateAllServerConfigTask::process()
9791
{
9792
  int ret = OB_SUCCESS;
9793
  if (OB_FAIL(root_service_.update_all_server_config())) {
9794
    LOG_WARN("fail to update all server config", K(ret));
9795
  } else {
9796
    LOG_INFO("update all server config success");
9797
  }
9798
  return ret;
9799
}
9800

9801
ObAsyncTask *ObRootService::ObUpdateAllServerConfigTask::deep_copy(char *buf, const int64_t buf_size) const
9802
{
9803
  ObUpdateAllServerConfigTask *task = NULL;
9804
  if (NULL == buf || buf_size < static_cast<int64_t>(sizeof(*this))) {
9805
    LOG_WARN_RET(OB_BUF_NOT_ENOUGH, "buffer not large enough", K(buf_size), KP(buf));
9806
  } else {
9807
    task = new (buf) ObUpdateAllServerConfigTask(root_service_);
9808
  }
9809
  return task;
9810
}
9811

9812
/////////////////////////
9813
int ObRootService::admin_clear_balance_task(const obrpc::ObAdminClearBalanceTaskArg &args)
9814
{
9815
  // TODO: @wanhong.wwh NEED SUPPORT
9816
  UNUSEDx(args);
9817
  return 0;
9818
}
9819

9820
status::ObRootServiceStatus ObRootService::get_status() const
9821
{
9822
  return rs_status_.get_rs_status();
9823
}
9824

9825
int ObRootService::table_allow_ddl_operation(const obrpc::ObAlterTableArg &arg)
9826
{
9827
  int ret = OB_SUCCESS;
9828
  const ObTableSchema *schema = NULL;
9829
  ObSchemaGetterGuard schema_guard;
9830
  const AlterTableSchema &alter_table_schema = arg.alter_table_schema_;
9831
  const uint64_t tenant_id = alter_table_schema.get_tenant_id();
9832
  const ObString &origin_database_name = alter_table_schema.get_origin_database_name();
9833
  const ObString &origin_table_name = alter_table_schema.get_origin_table_name();
9834
  schema_guard.set_session_id(arg.session_id_);
9835
  if (arg.is_refresh_sess_active_time()) {
9836
    //do nothing
9837
  } else if (!arg.is_valid()) {
9838
    ret = OB_INVALID_ARGUMENT;
9839
    LOG_WARN("invali argument", K(ret), K(arg));
9840
  } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(tenant_id, schema_guard))) {
9841
    LOG_WARN("get schema guard in inner table failed", K(ret));
9842
  } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, origin_database_name,
9843
                                                   origin_table_name, false, schema))) {
9844
    LOG_WARN("fail to get table schema", K(ret), K(tenant_id), K(origin_database_name), K(origin_table_name));
9845
  } else if (OB_ISNULL(schema)) {
9846
    ret = OB_TABLE_NOT_EXIST;
9847
    LOG_WARN("invalid schema", K(ret));
9848
    LOG_USER_ERROR(OB_TABLE_NOT_EXIST, to_cstring(origin_database_name), to_cstring(origin_table_name));
9849
  } else if (schema->is_in_splitting()) {
9850
    //TODO ddl must not execute on splitting table due to split not unstable
9851
    ret = OB_OP_NOT_ALLOW;
9852
    LOG_WARN("table is physical or logical split can not split", K(ret), K(schema));
9853
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "table is in physial or logical split, ddl operation");
9854
  } else if (schema->is_ctas_tmp_table()) {
9855
    if (!alter_table_schema.alter_option_bitset_.has_member(ObAlterTableArg::SESSION_ID)) {
9856
      //to prevet alter table after failed to create table, the table is invisible.
9857
      ret = OB_OP_NOT_ALLOW;
9858
      LOG_WARN("try to alter invisible table schema", K(schema->get_session_id()), K(arg));
9859
      LOG_USER_ERROR(OB_OP_NOT_ALLOW, "try to alter invisible table");
9860
    }
9861
  } else if (schema->has_mlog_table() || schema->is_mlog_table()) {
9862
    if (OB_FAIL(ObResolverUtils::check_allowed_alter_operations_for_mlog(
9863
        tenant_id, arg, *schema))) {
9864
      LOG_WARN("failed to check allowed alter operation for mlog",
9865
          KR(ret), K(tenant_id), K(arg));
9866
    }
9867
  }
9868
  return ret;
9869
}
9870

9871
// ask each server to update statistic
9872
int ObRootService::update_stat_cache(const obrpc::ObUpdateStatCacheArg &arg)
9873
{
9874
  int ret = OB_SUCCESS;
9875
  ObZone null_zone;
9876
  ObSEArray<ObAddr, 8> server_list;
9877
  bool evict_plan_failed = false;
9878
  if (!inited_) {
9879
    ret = OB_NOT_INIT;
9880
    LOG_WARN("not init", K(ret));
9881
  } else if (OB_FAIL(SVR_TRACER.get_alive_servers(null_zone, server_list))) {
9882
    LOG_WARN("fail to get alive server", K(ret));
9883
  } else {
9884
    for (int64_t i = 0; OB_SUCC(ret) && i < server_list.count(); i++) {
9885
      if (OB_FAIL(rpc_proxy_.to(server_list.at(i)).update_local_stat_cache(arg))) {
9886
        LOG_WARN("fail to update table statistic", K(ret), K(server_list.at(i)));
9887
        // OB_SQL_PC_NOT_EXIST represent evict plan failed
9888
        if (OB_SQL_PC_NOT_EXIST == ret) {
9889
          ret = OB_SUCCESS;
9890
          evict_plan_failed = true;
9891
        }
9892
      } else { /*do nothing*/}
9893
    }
9894
  }
9895
  if (OB_SUCC(ret) && evict_plan_failed) {
9896
    ret = OB_SQL_PC_NOT_EXIST;
9897
  }
9898
  return ret;
9899
}
9900

9901
int ObRootService::check_weak_read_version_refresh_interval(int64_t refresh_interval, bool &valid)
9902
{
9903
  int ret = OB_SUCCESS;
9904
  ObSchemaGetterGuard sys_schema_guard;
9905
  ObArray<uint64_t> tenant_ids;
9906
  valid = true;
9907

9908
  if (OB_ISNULL(GCTX.schema_service_)) {
9909
    ret = OB_ERR_UNEXPECTED;
9910
    LOG_WARN("schema service is null", KR(ret));
9911
  } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, sys_schema_guard))) {
9912
    LOG_WARN("get sys schema guard failed", KR(ret));
9913
  } else if (OB_FAIL(sys_schema_guard.get_tenant_ids(tenant_ids))) {
9914
    LOG_WARN("get tenant ids failed", KR(ret));
9915
  } else {
9916
    ObSchemaGetterGuard schema_guard;
9917
    const ObSimpleTenantSchema *tenant_schema = NULL;
9918
    const ObSysVarSchema *var_schema = NULL;
9919
    ObObj obj;
9920
    int64_t session_max_stale_time = 0;
9921
    uint64_t tenant_id = OB_INVALID_TENANT_ID;
9922
    for (int64_t i = 0; OB_SUCC(ret) && valid && i < tenant_ids.count(); i++) {
9923
      tenant_id = tenant_ids[i];
9924
      if (OB_FAIL(sys_schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
9925
        LOG_WARN("fail to get tenant schema", KR(ret), K(tenant_id));
9926
      } else if (OB_ISNULL(tenant_schema)) {
9927
        ret = OB_SUCCESS;
9928
        LOG_WARN("tenant schema is null, skip and continue", KR(ret), K(tenant_id));
9929
      } else if (!tenant_schema->is_normal()) {
9930
        ret = OB_SUCCESS;
9931
        LOG_WARN("tenant schema is not normal, skip and continue", KR(ret), K(tenant_id));
9932
      } else if (OB_FAIL(GCTX.schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) {
9933
        LOG_WARN("get schema guard failed", KR(ret), K(tenant_id));
9934
      } else if (OB_FAIL(schema_guard.get_tenant_system_variable(tenant_id,
9935
                         OB_SV_MAX_READ_STALE_TIME, var_schema))) {
9936
        LOG_WARN("get tenant system variable failed", KR(ret), K(tenant_id));
9937
      } else if (OB_ISNULL(var_schema)) {
9938
        ret = OB_ERR_UNEXPECTED;
9939
        LOG_WARN("var schema is null", KR(ret), K(tenant_id));
9940
      } else if (OB_FAIL(var_schema->get_value(NULL, NULL, obj))) {
9941
        LOG_WARN("get value failed", KR(ret), K(tenant_id), K(obj));
9942
      } else if (OB_FAIL(obj.get_int(session_max_stale_time))) {
9943
        LOG_WARN("get int failed", KR(ret), K(tenant_id), K(obj));
9944
      } else if (session_max_stale_time != share::ObSysVarFactory::INVALID_MAX_READ_STALE_TIME
9945
                 && refresh_interval > session_max_stale_time) {
9946
        valid = false;
9947
        LOG_USER_ERROR(OB_INVALID_ARGUMENT,
9948
                       "weak_read_version_refresh_interval is larger than ob_max_read_stale_time");
9949
      }
9950
    }
9951
  }
9952
  return ret;
9953
}
9954

9955
int ObRootService::set_config_pre_hook(obrpc::ObAdminSetConfigArg &arg)
9956
{
9957
  int ret = OB_SUCCESS;
9958
  if (!arg.is_valid()) {
9959
    ret = OB_INVALID_ARGUMENT;
9960
    LOG_WARN("invalid argument", K(ret), K(arg));
9961
  }
9962
  FOREACH_X(item, arg.items_, OB_SUCCESS == ret) {
9963
    bool valid = true;
9964
    if (item->name_.is_empty()) {
9965
      ret = OB_INVALID_ARGUMENT;
9966
      LOG_WARN("empty config name", "item", *item, K(ret));
9967
    } else if (0 == STRCMP(item->name_.ptr(), _TX_SHARE_MEMORY_LIMIT_PERCENTAGE)) {
9968
      ret = check_tx_share_memory_limit_(*item);
9969
    } else if (0 == STRCMP(item->name_.ptr(), MEMSTORE_LIMIT_PERCENTAGE)) {
9970
      ret = check_memstore_limit_(*item);
9971
    } else if (0 == STRCMP(item->name_.ptr(), TENANT_MEMSTORE_LIMIT_PERCENTAGE)) {
9972
      ret = check_tenant_memstore_limit_(*item);
9973
    } else if (0 == STRCMP(item->name_.ptr(), _TX_DATA_MEMORY_LIMIT_PERCENTAGE)) {
9974
      ret = check_tx_data_memory_limit_(*item);
9975
    } else if (0 == STRCMP(item->name_.ptr(), _MDS_MEMORY_LIMIT_PERCENTAGE)) {
9976
      ret = check_mds_memory_limit_(*item);
9977
    } else if (0 == STRCMP(item->name_.ptr(), FREEZE_TRIGGER_PERCENTAGE)) {
9978
      ret = check_freeze_trigger_percentage_(*item);
9979
    } else if (0 == STRCMP(item->name_.ptr(), WRITING_THROTTLEIUNG_TRIGGER_PERCENTAGE)) {
9980
      ret = check_write_throttle_trigger_percentage(*item);
9981
    } else if (0 == STRCMP(item->name_.ptr(), WEAK_READ_VERSION_REFRESH_INTERVAL)) {
9982
      int64_t refresh_interval = ObConfigTimeParser::get(item->value_.ptr(), valid);
9983
      if (valid && OB_FAIL(check_weak_read_version_refresh_interval(refresh_interval, valid))) {
9984
        LOG_WARN("check refresh interval failed ", KR(ret), K(*item));
9985
      } else if (!valid) {
9986
        ret = OB_INVALID_ARGUMENT;
9987
        LOG_WARN("config invalid", KR(ret), K(*item));
9988
      }
9989
    } else if (0 == STRCMP(item->name_.ptr(), PARTITION_BALANCE_SCHEDULE_INTERVAL)) {
9990
      const int64_t DEFAULT_BALANCER_IDLE_TIME = 10 * 1000 * 1000L; // 10s
9991
      for (int i = 0; i < item->tenant_ids_.count() && valid; i++) {
9992
        const uint64_t tenant_id = item->tenant_ids_.at(i);
9993
        omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
9994
        int64_t balancer_idle_time = tenant_config.is_valid() ? tenant_config->balancer_idle_time : DEFAULT_BALANCER_IDLE_TIME;
9995
        int64_t interval = ObConfigTimeParser::get(item->value_.ptr(), valid);
9996
        if (valid) {
9997
          if (0 == interval) {
9998
            valid = true;
9999
          } else if (interval >= balancer_idle_time) {
10000
            valid = true;
10001
          } else {
10002
            valid = false;
10003
            char err_msg[DEFAULT_BUF_LENGTH];
10004
            (void)snprintf(err_msg, sizeof(err_msg), "partition_balance_schedule_interval of tenant %ld, "
10005
                "it should not be less than balancer_idle_time", tenant_id);
10006
            LOG_USER_ERROR(OB_INVALID_ARGUMENT, err_msg);
10007
          }
10008
        }
10009
        if (!valid) {
10010
          ret = OB_INVALID_ARGUMENT;
10011
          LOG_WARN("config invalid", KR(ret), K(*item), K(balancer_idle_time), K(tenant_id));
10012
        }
10013
      }
10014
    } else if (0 == STRCMP(item->name_.ptr(), BALANCER_IDLE_TIME)) {
10015
      const int64_t DEFAULT_PARTITION_BALANCE_SCHEDULE_INTERVAL = 2 * 3600 * 1000 * 1000L; // 2h
10016
      for (int i = 0; i < item->tenant_ids_.count() && valid; i++) {
10017
        const uint64_t tenant_id = item->tenant_ids_.at(i);
10018
        omt::ObTenantConfigGuard tenant_config(TENANT_CONF(tenant_id));
10019
        int64_t interval = tenant_config.is_valid()
10020
            ? tenant_config->partition_balance_schedule_interval
10021
            : DEFAULT_PARTITION_BALANCE_SCHEDULE_INTERVAL;
10022
        int64_t idle_time = ObConfigTimeParser::get(item->value_.ptr(), valid);
10023
        if (valid && (idle_time > interval)) {
10024
          valid = false;
10025
          char err_msg[DEFAULT_BUF_LENGTH];
10026
          (void)snprintf(err_msg, sizeof(err_msg), "balancer_idle_time of tenant %ld, "
10027
              "it should not be longer than partition_balance_schedule_interval", tenant_id);
10028
          LOG_USER_ERROR(OB_INVALID_ARGUMENT, err_msg);
10029
        }
10030
        if (!valid) {
10031
          ret = OB_INVALID_ARGUMENT;
10032
          LOG_WARN("config invalid", KR(ret), K(*item), K(interval), K(tenant_id));
10033
        }
10034
      }
10035
    } else if (0 == STRCMP(item->name_.ptr(), LOG_DISK_UTILIZATION_LIMIT_THRESHOLD)) {
10036
      // check log_disk_utilization_limit_threshold
10037
      for (int i = 0; i < item->tenant_ids_.count() && valid; i++) {
10038
        valid = valid && ObConfigLogDiskLimitThresholdIntChecker::check(item->tenant_ids_.at(i), *item);
10039
        if (!valid) {
10040
          ret = OB_INVALID_ARGUMENT;
10041
          LOG_USER_ERROR(OB_INVALID_ARGUMENT, "log_disk_utilization_limit_threshold should be greater than log_disk_throttling_percentage "
10042
                        "when log_disk_throttling_percentage is not equal to 100");
10043
          LOG_WARN("config invalid", "item", *item, K(ret), K(i), K(item->tenant_ids_.at(i)));
10044
        }
10045
      }
10046
    } else if (0 == STRCMP(item->name_.ptr(), LOG_DISK_THROTTLING_PERCENTAGE)) {
10047
      // check log_disk_throttling_percentage
10048
      for (int i = 0; i < item->tenant_ids_.count() && valid; i++) {
10049
        valid = valid && ObConfigLogDiskThrottlingPercentageIntChecker::check(item->tenant_ids_.at(i), *item);
10050
        if (!valid) {
10051
          ret = OB_INVALID_ARGUMENT;
10052
          LOG_USER_ERROR(OB_INVALID_ARGUMENT, "log_disk_throttling_percentage should be equal to 100 or smaller than log_disk_utilization_limit_threshold");
10053
          LOG_WARN("config invalid", "item", *item, K(ret), K(i), K(item->tenant_ids_.at(i)));
10054
        }
10055
      }
10056
    }
10057
  }
10058
  return ret;
10059
}
10060

10061
#define CHECK_TENANTS_CONFIG_WITH_FUNC(FUNCTOR, LOG_INFO)                                  \
10062
  do {                                                                                     \
10063
    bool valid = true;                                                                     \
10064
    for (int i = 0; i < item.tenant_ids_.count() && valid; i++) {                          \
10065
      valid = valid && FUNCTOR::check(item.tenant_ids_.at(i), item);                       \
10066
      if (!valid) {                                                                        \
10067
        ret = OB_INVALID_ARGUMENT;                                                         \
10068
        LOG_USER_ERROR(OB_INVALID_ARGUMENT, LOG_INFO);                                     \
10069
        LOG_WARN("config invalid", "item", item, K(ret), K(i), K(item.tenant_ids_.at(i))); \
10070
      }                                                                                    \
10071
    }                                                                                      \
10072
  } while (0)
10073

10074
#define CHECK_CLUSTER_CONFIG_WITH_FUNC(FUNCTOR, LOG_INFO)                                  \
10075
  do {                                                                                     \
10076
    bool valid = true;                                                                     \
10077
    for (int i = 0; i < tenant_ids.count() && valid; i++) {                                \
10078
      valid = valid && FUNCTOR::check(tenant_ids.at(i), item);                             \
10079
      if (!valid) {                                                                        \
10080
        ret = OB_INVALID_ARGUMENT;                                                         \
10081
        LOG_USER_ERROR(OB_INVALID_ARGUMENT, LOG_INFO);                                     \
10082
        LOG_WARN("config invalid", "item", item, K(ret), K(i), K(tenant_ids.at(i)));       \
10083
      }                                                                                    \
10084
    }                                                                                      \
10085
  } while (0)
10086

10087
int ObRootService::check_tx_share_memory_limit_(obrpc::ObAdminSetConfigItem &item)
10088
{
10089
  int ret = OB_SUCCESS;
10090
  // There is a prefix "Incorrect arguments to " before user log so the warn log looked kinds of wired
10091
  const char *warn_log = "tenant config _tx_share_memory_limit_percentage. "
10092
                         "It should larger than or equal with any single module in it(Memstore, TxData, Mds)";
10093
  CHECK_TENANTS_CONFIG_WITH_FUNC(ObConfigTxShareMemoryLimitChecker, warn_log);
10094
  return ret;
10095
}
10096

10097
int ObRootService::check_memstore_limit_(obrpc::ObAdminSetConfigItem &item)
10098
{
10099
  int ret = OB_SUCCESS;
10100
  const char *warn_log = "cluster config memstore_limit_percentage. "
10101
                         "It should less than or equal with all tenant's _tx_share_memory_limit_percentage";
10102
  ObArray<uint64_t> tenant_ids;
10103
  ObSchemaGetterGuard schema_guard;
10104
  if (OB_UNLIKELY(!inited_)) {
10105
    ret = OB_NOT_INIT;
10106
    LOG_WARN("not inited", KR(ret));
10107
  } else if (OB_ISNULL(schema_service_)) {
10108
    ret = OB_ERR_UNEXPECTED;
10109
    LOG_WARN("schema service is null", KR(ret));
10110
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
10111
    LOG_WARN("get schema guard failed", KR(ret));
10112
  } else if (OB_FAIL(schema_guard.get_tenant_ids(tenant_ids))) {
10113
    LOG_WARN("failed to get all tenant ids", KR(ret), K(tenant_ids));
10114
  } else {
10115
    CHECK_CLUSTER_CONFIG_WITH_FUNC(ObConfigMemstoreLimitChecker, warn_log);
10116
  }
10117
  return ret;
10118
}
10119

10120
int ObRootService::check_tenant_memstore_limit_(obrpc::ObAdminSetConfigItem &item)
10121
{
10122
  int ret = OB_SUCCESS;
10123
  const char *warn_log = "tenant config _memstore_limit_percentage. "
10124
    "It should less than or equal with _tx_share_memory_limit_percentage";
10125
  CHECK_TENANTS_CONFIG_WITH_FUNC(ObConfigMemstoreLimitChecker, warn_log);
10126
  return ret;
10127
}
10128

10129
int ObRootService::check_tx_data_memory_limit_(obrpc::ObAdminSetConfigItem &item)
10130
{
10131
  int ret = OB_SUCCESS;
10132
  const char *warn_log = "tenant config _tx_data_memory_limit_percentage. "
10133
                         "It should less than or equal with _tx_share_memory_limit_percentage";
10134
  CHECK_TENANTS_CONFIG_WITH_FUNC(ObConfigTxDataLimitChecker, warn_log);
10135
  return ret;
10136
}
10137

10138
int ObRootService::check_mds_memory_limit_(obrpc::ObAdminSetConfigItem &item)
10139
{
10140
  int ret = OB_SUCCESS;
10141
  const char *warn_log = "tenant config _mds_memory_limit_percentage. "
10142
                         "It should less than or equal with _tx_share_memory_limit_percentage";
10143
  CHECK_TENANTS_CONFIG_WITH_FUNC(ObConfigMdsLimitChecker, warn_log);
10144
  return ret;
10145
}
10146

10147
int ObRootService::check_freeze_trigger_percentage_(obrpc::ObAdminSetConfigItem &item)
10148
{
10149
  int ret = OB_SUCCESS;
10150
  const char *warn_log = "tenant freeze_trigger_percentage "
10151
                         "which should smaller than writing_throttling_trigger_percentage";
10152
  CHECK_TENANTS_CONFIG_WITH_FUNC(ObConfigFreezeTriggerIntChecker, warn_log);
10153
  return ret;
10154
}
10155

10156
int ObRootService::check_write_throttle_trigger_percentage(obrpc::ObAdminSetConfigItem &item)
10157
{
10158
  int ret = OB_SUCCESS;
10159
  const char *warn_log = "tenant writing_throttling_trigger_percentage "
10160
                         "which should greater than freeze_trigger_percentage";
10161
  CHECK_TENANTS_CONFIG_WITH_FUNC(ObConfigWriteThrottleTriggerIntChecker, warn_log);
10162
  return ret;
10163
}
10164

10165
#undef CHECK_TENANTS_CONFIG_WITH_FUNC
10166
#undef CHECK_CLUSTER_CONFIG_WITH_FUNC
10167

10168
int ObRootService::set_config_post_hook(const obrpc::ObAdminSetConfigArg &arg)
10169
{
10170
  int ret = OB_SUCCESS;
10171
  if (!arg.is_valid()) {
10172
    ret = OB_INVALID_ARGUMENT;
10173
    LOG_WARN("invalid argument", K(ret), K(arg));
10174
  }
10175
  FOREACH_X(item, arg.items_, OB_SUCCESS == ret) {
10176
    if (item->name_.is_empty()) {
10177
      ret = OB_INVALID_ARGUMENT;
10178
      LOG_WARN("empty config name", "item", *item, K(ret));
10179
    } else if (0 == STRCMP(item->name_.ptr(), ENABLE_REBALANCE)
10180
               || 0 == STRCMP(item->name_.ptr(), ENABLE_REREPLICATION)) {
10181
      // TODO: @wanhong.wwh SUPPORT clear DR task after disable rebalance and rereplication
10182
    } else if (0 == STRCMP(item->name_.ptr(), MERGER_CHECK_INTERVAL)) {
10183
      //daily_merge_scheduler_.wakeup();
10184
    } else if (0 == STRCMP(item->name_.ptr(), ENABLE_AUTO_LEADER_SWITCH)) {
10185
      //wake_up leader_cooridnator
10186
    } else if (0 == STRCMP(item->name_.ptr(), OBCONFIG_URL)) {
10187
      int tmp_ret = OB_SUCCESS;
10188
      bool force_update = true;
10189
      if (OB_SUCCESS != (tmp_ret = submit_update_rslist_task(force_update))) {
10190
        LOG_WARN("fail to submit update rs list task", KR(ret), K(tmp_ret));
10191
      }
10192
      LOG_INFO("obconfig_url parameters updated, force submit update rslist task", KR(tmp_ret),
10193
          KPC(item));
10194
    } else if (0 == STRCMP(item->name_.ptr(), SCHEMA_HISTORY_RECYCLE_INTERVAL)) {
10195
      schema_history_recycler_.wakeup();
10196
      LOG_INFO("schema_history_recycle_interval parameters updated, wakeup schema_history_recycler",
10197
               KPC(item));
10198
    }
10199
  }
10200
  return ret;
10201
}
10202

10203
//ensure execute on DDL thread
10204
int ObRootService::force_create_sys_table(const obrpc::ObForceCreateSysTableArg &arg)
10205
{
10206
  return OB_NOT_SUPPORTED;
10207
}
10208

10209
// set tenant's locality
10210
// TODO
10211
//  1. set all locality of normal tenant to DEFAULT first.
10212
//  2. verify that replica distribution satifies the new locality
10213
int ObRootService::force_set_locality(const obrpc::ObForceSetLocalityArg &arg)
10214
{
10215
  LOG_INFO("receive force set locality arg", K(arg));
10216
  int ret = OB_SUCCESS;
10217
  ObSchemaGetterGuard schema_guard;
10218
  const ObTenantSchema *tenant_schema = NULL;
10219
  const uint64_t tenant_id = arg.exec_tenant_id_;
10220
  if (!inited_) {
10221
    LOG_WARN("not init", KR(ret));
10222
  } else if (!arg.is_valid()
10223
             || is_meta_tenant(tenant_id)) {
10224
    ret = OB_INVALID_ARGUMENT;
10225
    LOG_WARN("invalid arg", KR(ret), K(arg));
10226
  } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(OB_SYS_TENANT_ID, schema_guard))) {
10227
    LOG_WARN("fail to get schema guard with version in inner table", KR(ret));
10228
  } else if (OB_FAIL(schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
10229
    LOG_WARN("fail to get tenant schema", KR(ret), K(tenant_id));
10230
  } else if (OB_ISNULL(tenant_schema)) {
10231
    ret = OB_TENANT_NOT_EXIST;
10232
    LOG_WARN("tenant not exist", KR(ret));
10233
  } else {
10234
    ObTenantSchema new_tenant;
10235
    if (OB_FAIL(new_tenant.assign(*tenant_schema))) {
10236
      LOG_WARN("fail to assgin tenant schema", KR(ret), KPC(tenant_schema));
10237
    } else if (OB_FAIL(new_tenant.set_locality(arg.locality_))) {
10238
      LOG_WARN("fail to set locality", KR(ret), K(arg));
10239
    } else if (OB_FAIL(new_tenant.set_previous_locality(ObString("")))) {
10240
      LOG_WARN("fail to reset previous locality", KR(ret), K(arg));
10241
    } else if (OB_FAIL(ddl_service_.force_set_locality(schema_guard, new_tenant))) {
10242
      LOG_WARN("fail to force set locality", K(ret), K(new_tenant));
10243
    }
10244
  }
10245
  LOG_INFO("force set locality", K(arg));
10246
  return ret;
10247
}
10248

10249
int ObRootService::clear_special_cluster_schema_status()
10250
{
10251
  int ret = OB_SUCCESS;
10252
  if (OB_UNLIKELY(!inited_)) {
10253
    ret = OB_NOT_INIT;
10254
    LOG_WARN("not init", KR(ret));
10255
  } else if (OB_ISNULL(schema_service_)) {
10256
    ret = OB_ERR_UNEXPECTED;
10257
    LOG_WARN("schema service is null", KR(ret));
10258
  } else {
10259
    ObSchemaService *schema_service = schema_service_->get_schema_service();
10260
    if (OB_ISNULL(schema_service)) {
10261
      ret = OB_ERR_UNEXPECTED;
10262
      LOG_WARN("schema service is null", K(ret));
10263
    } else {
10264
      schema_service->set_cluster_schema_status(
10265
          ObClusterSchemaStatus::NORMAL_STATUS);
10266
    }
10267
  }
10268
  return ret;
10269
}
10270

10271
int ObRootService::get_is_in_bootstrap(bool &is_bootstrap) const
10272
{
10273
  int ret = OB_SUCCESS;
10274
  is_bootstrap = false;
10275
  if (OB_UNLIKELY(!inited_)) {
10276
    ret = OB_NOT_INIT;
10277
    LOG_WARN("not init", KR(ret));
10278
  } else if (OB_ISNULL(schema_service_)) {
10279
    ret = OB_ERR_UNEXPECTED;
10280
    LOG_WARN("schema service is null", KR(ret));
10281
  } else {
10282
    ObSchemaService *schema_service = schema_service_->get_schema_service();
10283
    if (OB_ISNULL(schema_service_)) {
10284
      ret = OB_ERR_UNEXPECTED;
10285
      LOG_WARN("schema service is null", KR(ret));
10286
    } else if (ObClusterSchemaStatus::BOOTSTRAP_STATUS
10287
        == schema_service->get_cluster_schema_status()) {
10288
      is_bootstrap = true;
10289
    }
10290
  }
10291
  return ret;
10292
}
10293

10294
int ObRootService::log_nop_operation(const obrpc::ObDDLNopOpreatorArg &arg)
10295
{
10296
  int ret = OB_SUCCESS;
10297
  LOG_INFO("start to log nop operation", K(arg));
10298
  if (!inited_) {
10299
    ret = OB_NOT_INIT;
10300
    LOG_WARN("not init", K(ret));
10301
  } else if (OB_FAIL(ddl_service_.log_nop_operation(arg))) {
10302
    LOG_WARN("failed to log nop operation", K(ret), K(arg));
10303
  }
10304
  return ret;
10305
}
10306

10307
// if tenant_id =  OB_INVALID_TENANT_ID, indicates refresh all tenants's schema;
10308
// otherwise, refresh specify tenant's schema. ensure schema_version not fallback by outer layer logic.
10309
int ObRootService::broadcast_schema(const obrpc::ObBroadcastSchemaArg &arg)
10310
{
10311
  int ret = OB_SUCCESS;
10312
  LOG_INFO("receieve broadcast_schema request", K(arg));
10313
  if (!inited_) {
10314
    ret = OB_NOT_INIT;
10315
    LOG_WARN("not init", K(ret));
10316
  } else if (OB_ISNULL(schema_service_)
10317
             || OB_ISNULL(schema_service_->get_schema_service())) {
10318
    ret = OB_ERR_UNEXPECTED;
10319
    LOG_WARN("schema_service is null", K(ret), KP_(schema_service));
10320
  } else {
10321
    ObRefreshSchemaInfo schema_info;
10322
    ObSchemaService *schema_service = schema_service_->get_schema_service();
10323
    if (OB_INVALID_TENANT_ID != arg.tenant_id_) {
10324
      // tenant_id is valid, just refresh specify tenant's schema.
10325
      schema_info.set_tenant_id(arg.tenant_id_);
10326
      schema_info.set_schema_version(arg.schema_version_);
10327
    } else {
10328
      // tenant_id =  OB_INVALID_TENANT_ID, indicates refresh all tenants's schema;
10329
      if (OB_FAIL(schema_service->inc_sequence_id())) {
10330
        LOG_WARN("increase sequence_id failed", K(ret));
10331
      }
10332
    }
10333
    if (OB_FAIL(ret)) {
10334
    } else if (OB_FAIL(schema_service->inc_sequence_id())) {
10335
      LOG_WARN("increase sequence_id failed", K(ret));
10336
    } else if (OB_FAIL(schema_service->set_refresh_schema_info(schema_info))) {
10337
      LOG_WARN("fail to set refresh schema info", K(ret), K(schema_info));
10338
    }
10339
  }
10340
  LOG_INFO("end broadcast_schema request", K(ret), K(arg));
10341
  return ret;
10342
}
10343

10344
/*
10345
 * standby_cluster, will return local tenant's schema_version
10346
 * primary_cluster, will return tenant's newest schema_version
10347
 *   - schema_version = OB_CORE_SCHEMA_VERSION, indicate the tenant is garbage.
10348
 *   - schema_version = OB_INVALID_VERSION, indicate that it is failed to get schame_version.
10349
 *   - schema_version > OB_CORE_SCHEMA_VERSION, indicate that the schema_version is valid.
10350
 */
10351
int ObRootService::get_tenant_schema_versions(
10352
    const obrpc::ObGetSchemaArg &arg,
10353
    obrpc::ObTenantSchemaVersions &tenant_schema_versions)
10354
{
10355
  int ret = OB_SUCCESS;
10356
  tenant_schema_versions.reset();
10357
  ObSchemaGetterGuard schema_guard;
10358
  ObArray<uint64_t> tenant_ids;
10359
  if (!inited_) {
10360
    ret = OB_NOT_INIT;
10361
    LOG_WARN("not init", KR(ret));
10362
  } else if (OB_ISNULL(schema_service_)) {
10363
    ret = OB_ERR_UNEXPECTED;
10364
    LOG_WARN("schema_service is null", K(ret));
10365
  } else if (OB_FAIL(ddl_service_.get_tenant_schema_guard_with_version_in_inner_table(
10366
                     OB_SYS_TENANT_ID, schema_guard))) {
10367
    LOG_WARN("fail to get schema guard", KR(ret));
10368
  } else if (OB_FAIL(schema_guard.get_tenant_ids(tenant_ids))) {
10369
    LOG_WARN("fail to get tenant ids", KR(ret));
10370
  } else {
10371
    int64_t tenant_id = OB_INVALID_TENANT_ID;
10372
    int64_t schema_version = 0;
10373
    for (int64_t i = 0; i < tenant_ids.count() && OB_SUCC(ret); i++) {
10374
      ObSchemaGetterGuard tenant_schema_guard;
10375
      tenant_id = tenant_ids.at(i);
10376
      schema_version = 0;
10377
      if (OB_SYS_TENANT_ID == tenant_id
10378
          || STANDBY_CLUSTER == ObClusterInfoGetter::get_cluster_role_v2()) {
10379
        // 对于备库，由于schema_status不在DDL线程推进，且能接受最终一致，
10380
        // 故只需取本地schema版本即可
10381
        if (OB_FAIL(schema_service_->get_tenant_refreshed_schema_version(
10382
                    tenant_id, schema_version))) {
10383
          LOG_WARN("fail to get tenant refreshed schema version", K(ret), K(tenant_id));
10384
        }
10385
      } else {
10386
        // for primary cluster, need to get newest schema_version from inner table.
10387
        ObRefreshSchemaStatus schema_status;
10388
        schema_status.tenant_id_ = tenant_id;
10389
        int64_t version_in_inner_table = OB_INVALID_VERSION;
10390
        bool is_restore = false;
10391
        if (OB_FAIL(schema_service_->check_tenant_is_restore(&schema_guard, tenant_id, is_restore))) {
10392
          LOG_WARN("fail to check tenant is restore", KR(ret), K(tenant_id));
10393
        } else if (is_restore) {
10394
          ObSchemaStatusProxy *schema_status_proxy = GCTX.schema_status_proxy_;
10395
          if (OB_ISNULL(schema_status_proxy)) {
10396
            ret = OB_ERR_UNEXPECTED;
10397
            LOG_WARN("schema_status_proxy is null", KR(ret));
10398
          } else if (OB_FAIL(schema_status_proxy->get_refresh_schema_status(tenant_id, schema_status))) {
10399
            LOG_WARN("failed to get tenant refresh schema status", KR(ret), K(tenant_id));
10400
          } else if (OB_INVALID_VERSION != schema_status.readable_schema_version_) {
10401
            ret = OB_EAGAIN;
10402
            LOG_WARN("tenant's sys replicas are not restored yet, try later", KR(ret), K(tenant_id));
10403
          }
10404
        }
10405
        if (FAILEDx(schema_service_->get_schema_version_in_inner_table(
10406
                    sql_proxy_, schema_status, version_in_inner_table))) {
10407
          // failed tenant creation, inner table is empty, return OB_CORE_SCHEMA_VERSION
10408
          if (OB_EMPTY_RESULT == ret) {
10409
            LOG_INFO("create tenant maybe failed", K(ret), K(tenant_id));
10410
            schema_version = OB_CORE_SCHEMA_VERSION;
10411
            ret = OB_SUCCESS;
10412
          } else {
10413
            LOG_WARN("fail to get latest schema version in inner table", K(ret));
10414
          }
10415
        } else if (OB_FAIL(schema_service_->get_tenant_refreshed_schema_version(
10416
                           tenant_id, schema_version))) {
10417
          LOG_WARN("fail to get tenant refreshed schema version", K(ret), K(tenant_id));
10418
        } else if (schema_version < version_in_inner_table) {
10419
          ObArray<uint64_t> tenant_ids;
10420
          if (OB_FAIL(tenant_ids.push_back(tenant_id))) {
10421
            LOG_WARN("fail to push back tenant_id", K(ret), K(tenant_id));
10422
          } else if (OB_FAIL(schema_service_->refresh_and_add_schema(tenant_ids))) {
10423
            LOG_WARN("fail to refresh schema", K(ret), K(tenant_id));
10424
          } else if (OB_FAIL(schema_service_->get_tenant_refreshed_schema_version(
10425
                             tenant_id, schema_version))) {
10426
            LOG_WARN("fail to get tenant refreshed schema version", K(ret), K(tenant_id));
10427
          } else if (schema_version < version_in_inner_table) {
10428
            ret = OB_ERR_UNEXPECTED;
10429
            LOG_WARN("local version is still less than version in table",
10430
                     K(ret), K(tenant_id), K(schema_version), K(version_in_inner_table));
10431
          } else {}
10432
        } else {}
10433
      }
10434
      if (OB_FAIL(ret)) {
10435
      } else if (OB_FAIL(tenant_schema_versions.add(tenant_id, schema_version))) {
10436
        LOG_WARN("fail to add tenant schema version", KR(ret), K(tenant_id), K(schema_version));
10437
      }
10438
      if (OB_FAIL(ret) && arg.ignore_fail_ && OB_SYS_TENANT_ID != tenant_id) {
10439
        int64_t invalid_schema_version = OB_INVALID_SCHEMA_VERSION;
10440
        if (OB_FAIL(tenant_schema_versions.add(tenant_id, invalid_schema_version))) {
10441
          LOG_WARN("fail to add tenant schema version", KR(ret), K(tenant_id), K(schema_version));
10442
        }
10443
      }
10444
    } // end for
10445
  }
10446
  return ret;
10447
}
10448

10449
int ObRootService::generate_user(const ObClusterRole &cluster_role,
10450
                                 const char* user_name,
10451
                                 const char* user_passwd)
10452
{
10453
  int ret = OB_SUCCESS;
10454
  ObSqlString ddl_stmt_str;
10455
  int64_t affected_row = 0;
10456
  ObString passwd(user_passwd);
10457
  ObString encry_passwd;
10458
  char enc_buf[ENC_BUF_LEN] = {0};
10459
  if (OB_ISNULL(user_name) || OB_ISNULL(user_passwd)) {
10460
    ret = OB_INVALID_ARGUMENT;
10461
    LOG_WARN("invalid argument", KR(ret), K(user_name), K(user_passwd));
10462
  } else if (PRIMARY_CLUSTER != cluster_role) {
10463
    LOG_INFO("standby cluster, no need to create user", K(cluster_role));
10464
  } else if (OB_FAIL(sql::ObCreateUserExecutor::encrypt_passwd(passwd, encry_passwd, enc_buf, ENC_BUF_LEN))) {
10465
    LOG_WARN("Encrypt passwd failed", K(ret));
10466
  } else if (OB_FAIL(ObDDLSqlGenerator::gen_create_user_sql(ObAccountArg(user_name, OB_SYS_HOST_NAME),
10467
                                                     encry_passwd, ddl_stmt_str))) {
10468
    LOG_WARN("fail to gen create user sql", KR(ret));
10469
  } else if (OB_FAIL(sql_proxy_.write(ddl_stmt_str.ptr(), affected_row))) {
10470
    LOG_WARN("execute sql failed", K(ret), K(ddl_stmt_str));
10471
  } else {
10472
    LOG_INFO("create user success", K(user_name), K(affected_row));
10473
  }
10474
  ddl_stmt_str.reset();
10475
  if (OB_FAIL(ret) || PRIMARY_CLUSTER != cluster_role) {
10476
    //nothing todo
10477
  } else if (OB_FAIL(ddl_stmt_str.assign_fmt("grant select on *.* to '%s'",
10478
                                             user_name))) {
10479
    LOG_WARN("fail to assign fmt", KR(ret));
10480
  } else if (OB_FAIL(sql_proxy_.write(ddl_stmt_str.ptr(), affected_row))) {
10481
    LOG_WARN("fail to write", KR(ret), K(ddl_stmt_str));
10482
  } else {
10483
    LOG_INFO("grant privilege success", K(ddl_stmt_str));
10484
  }
10485
  return ret;
10486
}
10487

10488
int ObRootService::get_recycle_schema_versions(
10489
    const obrpc::ObGetRecycleSchemaVersionsArg &arg,
10490
    obrpc::ObGetRecycleSchemaVersionsResult &result)
10491
{
10492
  int ret = OB_SUCCESS;
10493
  LOG_INFO("receive get recycle schema versions request", K(arg));
10494
  bool is_standby = GCTX.is_standby_cluster();
10495
  bool in_service = is_full_service();
10496
  if (OB_UNLIKELY(!inited_)) {
10497
    ret = OB_NOT_INIT;
10498
    LOG_WARN("not init", K(ret));
10499
  } else if (!arg.is_valid()) {
10500
    ret = OB_INVALID_ARGUMENT;
10501
    LOG_WARN("arg is invalid", K(ret), K(arg));
10502
  } else if (!is_standby || !in_service) {
10503
    ret = OB_STATE_NOT_MATCH;
10504
    LOG_WARN("should be standby cluster and rs in service",
10505
             KR(ret), K(is_standby), K(in_service));
10506
  } else if (OB_FAIL(schema_history_recycler_.get_recycle_schema_versions(arg, result))) {
10507
    LOG_WARN("fail to get recycle schema versions", KR(ret), K(arg));
10508
  }
10509
  LOG_INFO("get recycle schema versions", KR(ret), K(arg), K(result));
10510
  return ret;
10511
}
10512
int ObRootService::do_profile_ddl(const obrpc::ObProfileDDLArg &arg)
10513
{
10514
  int ret = OB_SUCCESS;
10515
  if (!inited_) {
10516
    ret = OB_NOT_INIT;
10517
    LOG_WARN("not init", K(ret));
10518
  } else if (OB_FAIL(ddl_service_.handle_profile_ddl(arg))) {
10519
    LOG_WARN("handle ddl failed", K(arg), K(ret));
10520
  }
10521
  return ret;
10522
}
10523

10524
int ObRootService::rebuild_index_in_restore(
10525
    const obrpc::ObRebuildIndexInRestoreArg &arg)
10526
{
10527
  int ret = OB_NOT_SUPPORTED;
10528
  UNUSED(arg);
10529
  return ret;
10530
}
10531

10532
int ObRootService::handle_archive_log(const obrpc::ObArchiveLogArg &arg)
10533
{
10534
  int ret = OB_SUCCESS;
10535
  LOG_INFO("handle_archive_log", K(arg));
10536
  if (!inited_) {
10537
    ret = OB_NOT_INIT;
10538
    LOG_WARN("not init", K(ret));
10539
  } else if (OB_FAIL(ObBackupServiceProxy::handle_archive_log(arg))) {
10540
    LOG_WARN("failed to handle archive log", K(ret));
10541
  }
10542
  return ret;
10543
}
10544

10545
int ObRootService::handle_backup_database(const obrpc::ObBackupDatabaseArg &in_arg)
10546
{
10547
  int ret = OB_SUCCESS;
10548
	if (!inited_) {
10549
    ret = OB_NOT_INIT;
10550
    LOG_WARN("not init", K(ret));
10551
  } else if (OB_FAIL(ObBackupServiceProxy::handle_backup_database(in_arg))) {
10552
    LOG_WARN("failed to handle backup database", K(ret), K(in_arg));
10553
  }
10554
  FLOG_INFO("handle_backup_database", K(ret), K(in_arg));
10555
  return ret;
10556
}
10557

10558
int ObRootService::handle_validate_database(const obrpc::ObBackupManageArg &arg)
10559
{
10560
  int ret = OB_NOT_SUPPORTED;
10561
  LOG_ERROR("not supported now", K(ret), K(arg));
10562
  return ret;
10563
}
10564

10565
int ObRootService::handle_validate_backupset(const obrpc::ObBackupManageArg &arg)
10566
{
10567
  int ret = OB_NOT_SUPPORTED;
10568
  LOG_ERROR("not supported now", K(ret), K(arg));
10569
  return ret;
10570
}
10571

10572
int ObRootService::handle_cancel_validate(const obrpc::ObBackupManageArg &arg)
10573
{
10574
  int ret = OB_NOT_SUPPORTED;
10575
  LOG_ERROR("not supported now", K(ret), K(arg));
10576
  return ret;
10577
}
10578

10579

10580
int ObRootService::disaster_recovery_task_reply(
10581
    const obrpc::ObDRTaskReplyResult &arg)
10582
{
10583
  int ret = OB_SUCCESS;
10584
  DEBUG_SYNC(BEFORE_RS_DEAL_WITH_RPC);
10585
  FLOG_INFO("[DRTASK_NOTICE] receive disaster recovery task reply", K(arg));
10586
  if (OB_UNLIKELY(!inited_)) {
10587
    ret = OB_NOT_INIT;
10588
    LOG_WARN("not init", K(ret));
10589
  } else if (OB_UNLIKELY(!arg.is_valid())) {
10590
    ret = OB_INVALID_ARGUMENT;
10591
    LOG_WARN("invalid arg", KR(ret), K(arg));
10592
  } else if (OB_FAIL(disaster_recovery_task_mgr_.deal_with_task_reply(arg))) {
10593
    LOG_WARN("fail to execute over", KR(ret), K(arg));
10594
  }
10595
  return ret;
10596
}
10597

10598
int ObRootService::handle_backup_manage(const obrpc::ObBackupManageArg &arg)
10599
{
10600
  int ret = OB_SUCCESS;
10601

10602
  if (!inited_) {
10603
    ret = OB_NOT_INIT;
10604
    LOG_WARN("not init", K(ret));
10605
  } else {
10606
    switch (arg.type_) {
10607
    case ObBackupManageArg::CANCEL_BACKUP: {
10608
      if (OB_FAIL(handle_backup_database_cancel(arg))) {
10609
        LOG_WARN("failed to handle backup database cancel", K(ret), K(arg));
10610
      }
10611
      break;
10612
    };
10613
    case ObBackupManageArg::VALIDATE_DATABASE: {
10614
      if (OB_FAIL(handle_validate_database(arg))) {
10615
        LOG_WARN("failed to handle validate database", K(ret), K(arg));
10616
      }
10617
      break;
10618
    };
10619
    case ObBackupManageArg::VALIDATE_BACKUPSET: {
10620
      if (OB_FAIL(handle_validate_backupset(arg))) {
10621
        LOG_WARN("failed to handle validate backupset", K(ret), K(arg));
10622
      }
10623
      break;
10624
    };
10625
    case ObBackupManageArg::CANCEL_VALIDATE: {
10626
      if (OB_FAIL(handle_cancel_validate(arg))) {
10627
        LOG_WARN("failed to handle cancel validate", K(ret), K(arg));
10628
      }
10629
      break;
10630
    };
10631
    case ObBackupManageArg::CANCEL_BACKUP_BACKUPSET: {
10632
      if (OB_FAIL(handle_cancel_backup_backup(arg))) {
10633
        LOG_WARN("failed to handle cancel backup backup", K(ret), K(arg));
10634
      }
10635
      break;
10636
    }
10637
    case ObBackupManageArg::CANCEL_BACKUP_BACKUPPIECE: {
10638
      if (OB_FAIL(handle_cancel_backup_backup(arg))) {
10639
        LOG_WARN("failed to handle cancel backup backup", K(ret), K(arg));
10640
      }
10641
      break;
10642
    }
10643
    case ObBackupManageArg::CANCEL_ALL_BACKUP_FORCE: {
10644
      if (OB_FAIL(handle_cancel_all_backup_force(arg))) {
10645
        LOG_WARN("failed to handle cancel all backup force", K(ret), K(arg));
10646
      }
10647
      break;
10648
    };
10649
    default: {
10650
      ret = OB_INVALID_ARGUMENT;
10651
      LOG_ERROR("invalid backup manage arg", K(ret), K(arg));
10652
      break;
10653
    }
10654
    }
10655
  }
10656

10657
  FLOG_INFO("finish handle_backup_manage", K(ret), K(arg));
10658
  return ret;
10659
}
10660

10661
int ObRootService::handle_backup_delete(const obrpc::ObBackupCleanArg &arg)
10662
{
10663
  int ret = OB_SUCCESS;
10664
	if (!inited_) {
10665
    ret = OB_NOT_INIT;
10666
    LOG_WARN("not init", K(ret));
10667
  } else if (OB_FAIL(ObBackupServiceProxy::handle_backup_delete(arg))) {
10668
    LOG_WARN("failed to handle backup delete", K(ret), K(arg));
10669
  }
10670
  return ret;
10671
}
10672

10673
int ObRootService::handle_delete_policy(const obrpc::ObDeletePolicyArg &arg)
10674
{
10675
  int ret = OB_SUCCESS;
10676
	if (!inited_) {
10677
    ret = OB_NOT_INIT;
10678
    LOG_WARN("not init", K(ret));
10679
	} else if (OB_FAIL(ObBackupServiceProxy::handle_delete_policy(arg))) {
10680
    LOG_WARN("failed to handle delete policy", K(ret), K(arg));
10681
  }
10682
  return ret;
10683
}
10684

10685
int ObRootService::handle_backup_database_cancel(
10686
    const obrpc::ObBackupManageArg &arg)
10687
{
10688
  int ret = OB_SUCCESS;
10689
  if (!inited_) {
10690
    ret = OB_NOT_INIT;
10691
    LOG_WARN("not init", KR(ret));
10692
  } else if (ObBackupManageArg::CANCEL_BACKUP != arg.type_) {
10693
    ret = OB_INVALID_ARGUMENT;
10694
    LOG_WARN("handle backup database cancel get invalid argument", K(ret), K(arg));
10695
  } else if (OB_FAIL(ObBackupServiceProxy::handle_backup_database_cancel(arg))) {
10696
    LOG_WARN("failed to start schedule backup cancel", K(ret), K(arg));
10697
  }
10698
  return ret;
10699
}
10700

10701
int ObRootService::check_backup_scheduler_working(Bool &is_working)
10702
{
10703
  int ret = OB_NOT_SUPPORTED;
10704
  is_working = true;
10705

10706
  FLOG_INFO("not support check backup scheduler working, should not use anymore", K(ret), K(is_working));
10707
  return ret;
10708
}
10709

10710
ObRootService::ObTenantGlobalContextCleanTimerTask::ObTenantGlobalContextCleanTimerTask(
10711
                                               ObRootService &root_service)
10712
  : root_service_(root_service)
10713
{
10714
}
10715

10716
int ObRootService::ObTenantGlobalContextCleanTimerTask::schedule(int tg_id)
10717
{
10718
  return TG_SCHEDULE(tg_id, *this, SCHEDULE_PERIOD, true);
10719
}
10720

10721
void ObRootService::ObTenantGlobalContextCleanTimerTask::runTimerTask()
10722
{
10723
  int ret = OB_SUCCESS;
10724
  if (OB_FAIL(root_service_.clean_global_context())) {
10725
    LOG_WARN("failed to clean global context", K(ret));
10726
  }
10727
}
10728

10729
int ObRootService::handle_cancel_backup_backup(const obrpc::ObBackupManageArg &arg)
10730
{
10731
  int ret = OB_NOT_SUPPORTED;
10732
  LOG_ERROR("not supported now", K(ret), K(arg));
10733
  return ret;
10734
}
10735

10736
int ObRootService::handle_cancel_all_backup_force(const obrpc::ObBackupManageArg &arg)
10737
{
10738
  int ret = OB_NOT_SUPPORTED;
10739
  LOG_ERROR("not support now", K(ret), K(arg));
10740
  return ret;
10741
}
10742

10743
void ObRootService::reset_fail_count()
10744
{
10745
  ATOMIC_STORE(&fail_count_, 0);
10746
}
10747

10748
void ObRootService::update_fail_count(int ret)
10749
{
10750
  int64_t count = ATOMIC_AAF(&fail_count_, 1);
10751
  if (count > OB_ROOT_SERVICE_START_FAIL_COUNT_UPPER_LIMIT
10752
      && REACH_TIME_INTERVAL(60 * 1000 * 1000)) {
10753
    LOG_ERROR("rs_monitor_check : fail to start root service", KR(ret), K(count));
10754
  } else {
10755
    LOG_WARN("rs_monitor_check : fail to start root service", KR(ret), K(count));
10756
  }
10757
  LOG_DBA_WARN(OB_ERR_ROOTSERVICE_START, "msg", "rootservice start()/do_restart() has failure",
10758
               KR(ret), "fail_cnt", count);
10759
}
10760

10761
int ObRootService::send_physical_restore_result(const obrpc::ObPhysicalRestoreResult &res)
10762
{
10763
  int ret = OB_SUCCESS;
10764
  if (!inited_) {
10765
    ret = OB_NOT_INIT;
10766
    LOG_WARN("not init", K(ret));
10767
  } else if (!res.is_valid()) {
10768
    ret = OB_INVALID_ARGUMENT;
10769
    LOG_WARN("invalid arg", K(ret), K(res));
10770
  } else {
10771
    ret = OB_NOT_SUPPORTED;
10772
    //TODO set physical restore result
10773
  }
10774
  LOG_INFO("get physical restore job's result", K(ret), K(res));
10775
  return ret;
10776
}
10777

10778
int ObRootService::create_restore_point(const obrpc::ObCreateRestorePointArg &arg)
10779
{
10780
  int ret = OB_NOT_SUPPORTED;
10781
  UNUSED(arg);
10782
  LOG_WARN("craete restpre point is not supported now", K(ret));
10783
  return ret;
10784
}
10785

10786
int ObRootService::drop_restore_point(const obrpc::ObDropRestorePointArg &arg)
10787
{
10788
  int ret = OB_NOT_SUPPORTED;
10789
  UNUSED(arg);
10790
  LOG_WARN("drop restpre point is not supported now", K(ret));
10791
  return ret;
10792
}
10793

10794
int ObRootService::build_ddl_single_replica_response(const obrpc::ObDDLBuildSingleReplicaResponseArg &arg)
10795
{
10796
  int ret = OB_SUCCESS;
10797
  LOG_INFO("receive build ddl single replica response", K(arg));
10798
  ObDDLTaskInfo info;
10799
  info.row_scanned_ = arg.row_scanned_;
10800
  info.row_inserted_ = arg.row_inserted_;
10801
  if (OB_UNLIKELY(!inited_)) {
10802
    ret = OB_NOT_INIT;
10803
    LOG_WARN("not inited", K(ret));
10804
  } else if (OB_UNLIKELY(!arg.is_valid())) {
10805
    ret = OB_INVALID_ARGUMENT;
10806
    LOG_WARN("invalid arguments", K(ret), K(arg));
10807
  } else if (OB_FAIL(DDL_SIM(arg.tenant_id_, arg.task_id_, PROCESS_BUILD_SSTABLE_RESPONSE_SLOW))) {
10808
    LOG_WARN("ddl sim failure: procesc build sstable response slow", K(ret));
10809
  } else if (OB_FAIL(ddl_scheduler_.on_sstable_complement_job_reply(
10810
      arg.tablet_id_/*source tablet id*/, ObDDLTaskKey(arg.dest_tenant_id_, arg.dest_schema_id_, arg.dest_schema_version_), arg.snapshot_version_, arg.execution_id_, arg.ret_code_, info))) {
10811
    LOG_WARN("handle column checksum calc response failed", K(ret), K(arg));
10812
  }
10813
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "build ddl single replica response",
10814
                        "tenant_id", arg.tenant_id_,
10815
                        "ret", ret,
10816
                        "trace_id", *ObCurTraceId::get_trace_id(),
10817
                        "task_id", arg.task_id_,
10818
                        "tablet_id_", arg.tablet_id_,
10819
                        "snapshot_version_", arg.snapshot_version_,
10820
                        arg.source_table_id_);
10821
  LOG_INFO("finish build ddl single replica response ddl", K(ret), K(arg), "ddl_event_info", ObDDLEventInfo());
10822
  return ret;
10823
}
10824

10825
int ObRootService::purge_recyclebin_objects(int64_t purge_each_time)
10826
{
10827
  int ret = OB_SUCCESS;
10828
  // always passed
10829
  int64_t expire_timeval = GCONF.recyclebin_object_expire_time;
10830
  ObSEArray<uint64_t, 16> tenant_ids;
10831
  ObSchemaGetterGuard guard;
10832
  if (OB_ISNULL(schema_service_)) {
10833
    ret = OB_ERR_UNEXPECTED;
10834
    LOG_WARN("schema_serviece_ is null", KR(ret));
10835
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, guard))) {
10836
    LOG_WARN("fail to get sys schema guard", KR(ret));
10837
  } else if (OB_FAIL(guard.get_tenant_ids(tenant_ids))) {
10838
    LOG_WARN("get all tenants failed", KR(ret));
10839
  } else {
10840
    const int64_t current_time = ObTimeUtility::current_time();
10841
    obrpc::Int64 expire_time = current_time - expire_timeval;
10842
    const int64_t SLEEP_INTERVAL = 100 * 1000;  //100ms interval of send rpc
10843
    const int64_t PURGE_EACH_RPC = 10;          //delete count per rpc
10844
    obrpc::Int64 affected_rows = 0;
10845
    obrpc::ObPurgeRecycleBinArg arg;
10846
    int64_t purge_sum = purge_each_time;
10847
    const bool is_standby = PRIMARY_CLUSTER != ObClusterInfoGetter::get_cluster_role_v2();
10848
    const ObSimpleTenantSchema *simple_tenant = NULL;
10849
    //ignore ret
10850
    for (int i = 0; i < tenant_ids.count() && in_service() && purge_sum > 0; ++i) {
10851
      int64_t purge_time = GCONF._recyclebin_object_purge_frequency;
10852
      const uint64_t tenant_id = tenant_ids.at(i);
10853
      if (purge_time <= 0) {
10854
        break;
10855
      }
10856
      if (OB_SYS_TENANT_ID != tenant_id && is_standby) {
10857
        // standby cluster won't purge recyclebin automacially.
10858
        LOG_TRACE("user tenant won't purge recyclebin automacially in standby cluster", K(tenant_id));
10859
        continue;
10860
      } else if (OB_FAIL(guard.get_tenant_info(tenant_id, simple_tenant))) {
10861
        LOG_WARN("fail to get simple tenant schema", KR(ret), K(tenant_id));
10862
      } else if (OB_ISNULL(simple_tenant)) {
10863
        ret = OB_TENANT_NOT_EXIST;
10864
        LOG_WARN("simple tenant schema not exist", KR(ret), K(tenant_id));
10865
      } else if (!simple_tenant->is_normal()) {
10866
        // only deal with normal tenant.
10867
        LOG_TRACE("tenant which isn't normal won't purge recyclebin automacially", K(tenant_id));
10868
        continue;
10869
      }
10870
      // ignore error code of different tenant
10871
      ret = OB_SUCCESS;
10872
      affected_rows = 0;
10873
      arg.tenant_id_ = tenant_id;
10874
      arg.expire_time_ = expire_time;
10875
      arg.auto_purge_ = true;
10876
      arg.exec_tenant_id_ = tenant_id;
10877
      LOG_INFO("start purge recycle objects of tenant", K(arg), K(purge_sum));
10878
      while (OB_SUCC(ret) && in_service() && purge_sum > 0) {
10879
        int64_t cal_timeout = 0;
10880
        int64_t start_time = ObTimeUtility::current_time();
10881
        arg.purge_num_ = purge_sum > PURGE_EACH_RPC ? PURGE_EACH_RPC : purge_sum;
10882
        if (OB_FAIL(schema_service_->cal_purge_need_timeout(arg, cal_timeout))) {
10883
          LOG_WARN("fail to cal purge need timeout", KR(ret), K(arg));
10884
        } else if (0 == cal_timeout) {
10885
          LOG_INFO("cal purge need timeout is zero, just exit", K(tenant_id), K(purge_sum));
10886
          break;
10887
        } else if (OB_FAIL(common_proxy_.timeout(cal_timeout).purge_expire_recycle_objects(arg, affected_rows))) {
10888
          LOG_WARN("purge reyclebin objects failed", KR(ret),
10889
              K(current_time), K(expire_time), K(affected_rows), K(arg));
10890
        } else {
10891
          purge_sum -= affected_rows;
10892
          if (arg.purge_num_ != affected_rows) {
10893
            int64_t cost_time = ObTimeUtility::current_time() - start_time;
10894
            LOG_INFO("purge recycle objects", KR(ret), K(tenant_id), K(cost_time), K(purge_sum),
10895
                                              K(cal_timeout), K(expire_time), K(current_time), K(affected_rows));
10896
            if (OB_SUCC(ret) && in_service()) {
10897
              ob_usleep(SLEEP_INTERVAL);
10898
            }
10899
            break;
10900
          }
10901
        }
10902
        int64_t cost_time = ObTimeUtility::current_time() - start_time;
10903
        LOG_INFO("purge recycle objects", KR(ret), K(tenant_id), K(cost_time), K(purge_sum),
10904
                                          K(cal_timeout), K(expire_time), K(current_time), K(affected_rows));
10905
        if (OB_SUCC(ret) && in_service()) {
10906
          ob_usleep(SLEEP_INTERVAL);
10907
        }
10908
      }
10909
    }
10910
  }
10911
  return ret;
10912
}
10913

10914
int ObRootService::flush_opt_stat_monitoring_info(const obrpc::ObFlushOptStatArg &arg)
10915
{
10916
  int ret = OB_SUCCESS;
10917
  ObZone empty_zone;
10918
  ObSEArray<ObAddr, 8> server_list;
10919
  if (!inited_) {
10920
    ret = OB_NOT_INIT;
10921
    LOG_WARN("not init", K(ret));
10922
  } else if (OB_FAIL(SVR_TRACER.get_alive_servers(empty_zone, server_list))) {
10923
    LOG_WARN("fail to get alive server", KR(ret));
10924
  } else {
10925
    for (int64_t i = 0; OB_SUCC(ret) && i < server_list.count(); ++i) {
10926
      if (OB_FAIL(rpc_proxy_.to(server_list.at(i)).flush_local_opt_stat_monitoring_info(arg))) {
10927
        LOG_WARN("fail to update table statistic", K(ret), K(server_list.at(i)));
10928
      } else { /*do nothing*/}
10929
    }
10930
  }
10931
  return ret;
10932
}
10933

10934
int ObRootService::clean_global_context()
10935
{
10936
  int ret = OB_SUCCESS;
10937
  ObArray<uint64_t> tenant_ids;
10938
  ObSchemaGetterGuard schema_guard;
10939
  if (OB_UNLIKELY(!inited_)) {
10940
    ret = OB_NOT_INIT;
10941
    LOG_WARN("not inited", KR(ret));
10942
  } else if (OB_ISNULL(schema_service_)) {
10943
    ret = OB_ERR_UNEXPECTED;
10944
    LOG_WARN("schema service is null", KR(ret));
10945
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
10946
    LOG_WARN("get schema guard failed", KR(ret));
10947
  } else if (OB_FAIL(schema_guard.get_tenant_ids(tenant_ids))) {
10948
    LOG_WARN("failed to get all tenant ids", KR(ret), K(tenant_ids));
10949
  } else {
10950
    ObGlobalContextOperator ctx_operator;
10951
    if (OB_FAIL(ctx_operator.clean_global_context(tenant_ids, sql_proxy_, *schema_service_))) {
10952
      LOG_WARN("failed to clean global context", K(ret));
10953
    }
10954
  }
10955
  return ret;
10956
}
10957

10958
int ObRootService::admin_set_backup_config(const obrpc::ObAdminSetConfigArg &arg)
10959
{
10960
  int ret = OB_SUCCESS;
10961
  if (!arg.is_valid()) {
10962
    ret = OB_ERR_UNEXPECTED;
10963
    LOG_WARN("invalid backup config arg", K(ret));
10964
  } else if (!arg.is_backup_config_) {
10965
    ret = OB_ERR_UNEXPECTED;
10966
    LOG_WARN("admin set config type not backup config", K(ret), K(arg));
10967
  }
10968
  share::BackupConfigItemPair config_item;
10969
  share::ObBackupConfigParserMgr config_parser_mgr;
10970
  ARRAY_FOREACH_X(arg.items_, i , cnt, OB_SUCC(ret)) {
10971
    const ObAdminSetConfigItem &item = arg.items_.at(i);
10972
    uint64_t exec_tenant_id = OB_INVALID_TENANT_ID;
10973
    ObMySQLTransaction trans;
10974
    config_parser_mgr.reset();
10975
    if ((common::is_sys_tenant(item.exec_tenant_id_) && item.tenant_name_.is_empty())
10976
        || (common::is_user_tenant(item.exec_tenant_id_) && !item.tenant_name_.is_empty())
10977
        || common::is_meta_tenant(item.exec_tenant_id_)) {
10978
      ret = OB_NOT_SUPPORTED;
10979
      LOG_WARN("backup config only support user tenant", K(ret));
10980
    } else if (!item.tenant_name_.is_empty()) {
10981
      schema::ObSchemaGetterGuard guard;
10982
      if (OB_ISNULL(schema_service_)) {
10983
        ret = OB_ERR_UNEXPECTED;
10984
        LOG_WARN("schema service must not be null", K(ret));
10985
      } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, guard))) {
10986
        LOG_WARN("fail to get tenant schema guard", K(ret));
10987
      } else if (OB_FAIL(guard.get_tenant_id(ObString(item.tenant_name_.ptr()), exec_tenant_id))) {
10988
        LOG_WARN("fail to get tenant id", K(ret));
10989
      }
10990
    } else {
10991
      exec_tenant_id = item.exec_tenant_id_;
10992
    }
10993

10994
    if (OB_FAIL(ret)) {
10995
    } else if (OB_FAIL(trans.start(&sql_proxy_, gen_meta_tenant_id(exec_tenant_id)))) {
10996
      LOG_WARN("fail to start trans", K(ret));
10997
    } else {
10998
      common::ObSqlString name;
10999
      common::ObSqlString value;
11000
      if (OB_FAIL(name.assign(item.name_.ptr()))) {
11001
        LOG_WARN("fail to assign name", K(ret));
11002
      } else if (OB_FAIL(value.assign(item.value_.ptr()))) {
11003
        LOG_WARN("fail to assign value", K(ret));
11004
      } else if (OB_FAIL(config_parser_mgr.init(name, value, exec_tenant_id))) {
11005
        LOG_WARN("fail to init backup config parser mgr", K(ret), K(item));
11006
      } else if (OB_FAIL(config_parser_mgr.update_inner_config_table(rpc_proxy_, trans))) {
11007
        LOG_WARN("fail to update inner config table", K(ret));
11008
      }
11009

11010
      if (OB_SUCC(ret)) {
11011
        if (OB_FAIL(trans.end(true))) {
11012
          LOG_WARN("fail to commit trans", K(ret));
11013
        }
11014
      } else {
11015
        int tmp_ret = OB_SUCCESS;
11016
        if (OB_SUCCESS != (tmp_ret = trans.end(false))) {
11017
          LOG_WARN("fail to rollback trans", K(tmp_ret));
11018
        }
11019
      }
11020
    }
11021
  }
11022
  return ret;
11023
}
11024

11025
int ObRootService::cancel_ddl_task(const ObCancelDDLTaskArg &arg)
11026
{
11027
  int ret = OB_SUCCESS;
11028
  LOG_INFO("receive cancel ddl task", K(arg));
11029
  if (OB_UNLIKELY(!arg.is_valid())) {
11030
    ret = OB_INVALID_ARGUMENT;
11031
    LOG_WARN("invalid arguments", K(ret), K(arg));
11032
  } else if (OB_FAIL(SYS_TASK_STATUS_MGR.cancel_task(arg.get_task_id()))) {
11033
    LOG_WARN("cancel task failed", K(ret));
11034
  } else {
11035
    LOG_INFO("succeed to cancel ddl task", K(arg));
11036
  }
11037
  ROOTSERVICE_EVENT_ADD("ddl scheduler", "cancel ddl task",
11038
                        "tenant_id", MTL_ID(),
11039
                        "ret", ret,
11040
                        "trace_id", *ObCurTraceId::get_trace_id(),
11041
                        "task_id", arg.get_task_id());
11042
  LOG_INFO("finish cancel ddl task ddl", K(ret), K(arg), "ddl_event_info", ObDDLEventInfo());
11043
  return ret;
11044
}
11045

11046
int ObRootService::check_majority_and_log_in_sync(
11047
    const ObIArray<ObAddr> &to_stop_servers,
11048
    const bool skip_log_sync_check,
11049
    const char *print_str)
11050
{
11051
  int ret = OB_SUCCESS;
11052
  ObLSStatusOperator ls_status_op;
11053
  bool need_retry = false;
11054
  const int64_t CHECK_RETRY_INTERVAL = 100 * 1000; // 100ms
11055
  const int64_t RESERVED_TIME = 500 * 1000; // 500ms
11056
  int64_t start_time = ObTimeUtility::current_time();
11057
  int64_t abs_timeout_us = OB_INVALID_TIMESTAMP;
11058
  ObTimeoutCtx ctx;
11059
  const int64_t DEFAULT_RETRY_TIMEOUT = GCONF.internal_sql_execute_timeout;
11060
  LOG_INFO("check majority and log in sync start",
11061
      K(to_stop_servers), K(skip_log_sync_check), K(DEFAULT_RETRY_TIMEOUT));
11062

11063
  if (OB_UNLIKELY(!inited_)) {
11064
    ret = OB_NOT_INIT;
11065
    LOG_WARN("not init", KR(ret));
11066
  } else if (OB_ISNULL(schema_service_)) {
11067
    ret = OB_ERR_UNEXPECTED;
11068
    LOG_WARN("schema_service is null", KR(ret));
11069
  } else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, DEFAULT_RETRY_TIMEOUT))) {
11070
    LOG_WARN("failed to set default timeout ctx", KR(ret), K(DEFAULT_RETRY_TIMEOUT));
11071
  } else {
11072
    abs_timeout_us = ctx.get_abs_timeout() - RESERVED_TIME;
11073
    if (OB_FAIL(ctx.set_abs_timeout(abs_timeout_us))) {
11074
      LOG_WARN("fail to set abs timeout", KR(ret), K(abs_timeout_us));
11075
    } else {
11076
      do {
11077
        if (need_retry) {
11078
          ob_usleep(CHECK_RETRY_INTERVAL);
11079
        }
11080
        if (OB_FAIL(ls_status_op.check_all_ls_has_majority_and_log_sync(
11081
            to_stop_servers,
11082
            skip_log_sync_check,
11083
            print_str,
11084
            *schema_service_,
11085
            sql_proxy_,
11086
            need_retry))) {
11087
          LOG_WARN("fail to get and check all ls_paxos_info", KR(ret),
11088
              K(to_stop_servers), K(skip_log_sync_check));
11089
        }
11090
      } while ((OB_OP_NOT_ALLOW == ret) && need_retry);
11091
    }
11092
  }
11093
  LOG_INFO("check majority and log in sync finish", K(to_stop_servers),
11094
      K(skip_log_sync_check), "cost_time", ObTimeUtility::current_time() - start_time);
11095
  return ret;
11096
}
11097

11098
int ObRootService::check_all_ls_has_leader(const char *print_str)
11099
{
11100
  int ret = OB_SUCCESS;
11101
  ObLSStatusOperator ls_status_op;
11102
  const int64_t CHECK_RETRY_INTERVAL = 100 * 1000; // 100ms
11103
  const int64_t RESERVED_TIME = 500 * 1000; // 500ms
11104
  int64_t start_time = ObTimeUtility::current_time();
11105
  int64_t abs_timeout_us = OB_INVALID_TIMESTAMP;
11106
  bool has_ls_without_leader = false;
11107
  ObTimeoutCtx ctx;
11108
  const int64_t DEFAULT_RETRY_TIMEOUT = GCONF.internal_sql_execute_timeout;
11109
  ObSqlString last_error_msg;
11110

11111
  if (OB_UNLIKELY(!inited_)) {
11112
    ret = OB_NOT_INIT;
11113
    LOG_WARN("not init", KR(ret));
11114
  } else if (OB_FAIL(ObShareUtil::set_default_timeout_ctx(ctx, DEFAULT_RETRY_TIMEOUT))) {
11115
    LOG_WARN("failed to set default timeout ctx", KR(ret), K(DEFAULT_RETRY_TIMEOUT));
11116
  } else {
11117
    abs_timeout_us = ctx.get_abs_timeout() - RESERVED_TIME;
11118
    if (OB_FAIL(ctx.set_abs_timeout(abs_timeout_us))) {
11119
      LOG_WARN("fail to set abs timeout", KR(ret), K(abs_timeout_us));
11120
    } else {
11121
      do {
11122
        if (has_ls_without_leader) {
11123
          ob_usleep(CHECK_RETRY_INTERVAL);
11124
        }
11125
        if (OB_FAIL(ls_status_op.check_all_ls_has_leader(
11126
            sql_proxy_,
11127
            print_str,
11128
            has_ls_without_leader,
11129
            last_error_msg))) {
11130
          LOG_WARN("fail to check all ls has leader", KR(ret), K(print_str));
11131
        }
11132
      } while (OB_OP_NOT_ALLOW == ret && has_ls_without_leader);
11133
    }
11134
  }
11135
  if (OB_TIMEOUT == ret) {
11136
    ret = OB_OP_NOT_ALLOW;
11137
    if (!last_error_msg.empty()) {
11138
      LOG_USER_ERROR(OB_OP_NOT_ALLOW, last_error_msg.ptr());
11139
    } else {
11140
      LOG_WARN("fail to check all ls has leader because inner sql timeout", KR(ret), K(print_str));
11141
      char err_msg[OB_TMP_BUF_SIZE_256];
11142
      (void)snprintf(err_msg, sizeof(err_msg), "check leader for all LS timeout, %s", print_str);
11143
      LOG_USER_ERROR(OB_OP_NOT_ALLOW, err_msg);
11144
    }
11145
  }
11146
  LOG_INFO("check all ls has leader finish", KR(ret), K(abs_timeout_us), K(start_time),
11147
      "cost_time", ObTimeUtility::current_time() - start_time);
11148
  return ret;
11149
}
11150

11151
void ObRootService::update_cpu_quota_concurrency_in_memory_()
11152
{
11153
  {
11154
    omt::ObTenantConfigGuard tenant_config(TENANT_CONF(OB_SYS_TENANT_ID));
11155
    tenant_config->cpu_quota_concurrency = MAX(10, tenant_config->cpu_quota_concurrency);
11156
  }
11157
}
11158

11159
int ObRootService::set_cpu_quota_concurrency_config_()
11160
{
11161
  int64_t affected_rows = 0;
11162
  int ret = OB_SUCCESS;
11163
  if (OB_FAIL(sql_proxy_.write("ALTER SYSTEM SET cpu_quota_concurrency = 10;", affected_rows))) {
11164
    LOG_WARN("update cpu_quota_concurrency failed", K(ret));
11165
  } else if (OB_FAIL(check_config_result("cpu_quota_concurrency", "10"))) {
11166
    LOG_WARN("failed to check config same", K(ret));
11167
  }
11168
  return ret;
11169
}
11170

11171
int ObRootService::handle_recover_table(const obrpc::ObRecoverTableArg &arg)
11172
{
11173
  int ret = OB_SUCCESS;
11174
  LOG_INFO("succeed received recover table arg", K(arg));
11175
  uint64_t data_version = 0;
11176
  if (!inited_) {
11177
    ret = OB_NOT_INIT;
11178
    LOG_WARN("not init", K(ret));
11179
  } else if (!arg.is_valid()) {
11180
    ret = OB_INVALID_ARGUMENT;
11181
    LOG_WARN("invalid argument", K(ret), K(arg));
11182
  } else if (GCTX.is_standby_cluster()) {
11183
    ret = OB_OP_NOT_ALLOW;
11184
    LOG_WARN("recover table in standby tenant is not allowed", K(ret), K(arg));
11185
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover table in standby tenant");
11186
  } else if (GCONF.in_upgrade_mode()) {
11187
    ret = OB_OP_NOT_ALLOW;
11188
    LOG_WARN("recover table in upgrade mode is not allowed", K(ret), K(arg));
11189
    LOG_USER_ERROR(OB_OP_NOT_ALLOW, "Cluster is in upgrade mode, recover table is");
11190
  } else if (OB_FAIL(ObRecoverTableUtil::check_compatible(arg.tenant_id_))) {
11191
    LOG_WARN("check recover table compatible failed", K(ret), K(arg));
11192
  } else {
11193
    ObRecoverTableInitiator initiator;
11194
    bool is_exist = false;
11195
    if (OB_FAIL(initiator.init(schema_service_, &sql_proxy_))) {
11196
      LOG_WARN("failed to init ObRecoverTableInitiator", K(ret));
11197
    } else if (ObRecoverTableArg::Action::INITIATE == arg.action_
11198
        && OB_FAIL(initiator.is_recover_job_exist(arg.tenant_id_, is_exist))) {
11199
      LOG_WARN("failed to check recover job exist", K(ret), K(arg));
11200
    } else if (is_exist) {
11201
      ret = OB_OP_NOT_ALLOW;
11202
      LOG_WARN("recover job is exist", K(ret), K(arg));
11203
      LOG_USER_ERROR(OB_OP_NOT_ALLOW, "recover table when recover table job exists is");
11204
    } else if (OB_FAIL(initiator.initiate_recover_table(arg))) {
11205
      LOG_WARN("failed to initiate table recover", K(ret), K(arg));
11206
    } else {
11207
      LOG_INFO("[RECOVER_TABLE] initiate recover table succeed", K(arg));
11208
    }
11209
  }
11210
  return ret;
11211
}
11212

11213
int ObRootService::recompile_all_views_batch(const obrpc::ObRecompileAllViewsBatchArg &arg)
11214
{
11215
  int ret = OB_SUCCESS;
11216
  int64_t start_time = ObTimeUtility::current_time();
11217
  if (!inited_) {
11218
    ret = OB_NOT_INIT;
11219
    LOG_WARN("not init", K(ret));
11220
  } else if (!arg.is_valid()) {
11221
    ret = OB_INVALID_ARGUMENT;
11222
    LOG_WARN("invalid arg", K(arg), K(ret));
11223
  } else if (OB_FAIL(ddl_service_.recompile_all_views_batch(arg.tenant_id_, arg.view_ids_))) {
11224
    LOG_WARN("failed to recompile all views", K(ret), K(arg.tenant_id_));
11225
  }
11226
  LOG_INFO("recompile all views batch finish", KR(ret), K(start_time),
11227
      "cost_time", ObTimeUtility::current_time() - start_time);
11228
  return ret;
11229
}
11230

11231
int ObRootService::try_add_dep_infos_for_synonym_batch(const obrpc::ObTryAddDepInofsForSynonymBatchArg &arg)
11232
{
11233
  int ret = OB_SUCCESS;
11234
  int64_t start_time = ObTimeUtility::current_time();
11235
  if (!inited_) {
11236
    ret = OB_NOT_INIT;
11237
    LOG_WARN("not init", K(ret));
11238
  } else if (!arg.is_valid()) {
11239
    ret = OB_INVALID_ARGUMENT;
11240
    LOG_WARN("invalid arg", K(arg), K(ret));
11241
  } else if (OB_FAIL(ddl_service_.try_add_dep_info_for_all_synonyms_batch(arg.tenant_id_, arg.synonym_ids_))) {
11242
    LOG_WARN("failed to add synonym dep info", K(ret), K(arg.tenant_id_));
11243
  }
11244
  LOG_INFO("add dep infos for synonym batch finish", KR(ret), K(start_time),
11245
      "cost_time", ObTimeUtility::current_time() - start_time);
11246
  return ret;
11247
}
11248

11249
#ifdef OB_BUILD_TDE_SECURITY
11250
int ObRootService::handle_get_root_key(const obrpc::ObRootKeyArg &arg,
11251
                                       obrpc::ObRootKeyResult &result)
11252
{
11253
  int ret = OB_SUCCESS;
11254
  ObRootKey root_key;
11255
  if (!inited_) {
11256
    ret = OB_NOT_INIT;
11257
    LOG_WARN("not init", K(ret));
11258
  } else if (OB_UNLIKELY(arg.is_set_ || !arg.is_valid())) {
11259
    ret = OB_INVALID_ARGUMENT;
11260
    LOG_WARN("invalid arg", K(arg), K(ret));
11261
  } else if (OB_FAIL(ObMasterKeyGetter::instance().get_root_key(arg.tenant_id_, root_key))) {
11262
    LOG_WARN("failed to get root key", K(ret));
11263
  } else if (obrpc::RootKeyType::INVALID != root_key.key_type_) {
11264
    result.key_type_ = root_key.key_type_;
11265
    result.root_key_ = root_key.key_;
11266
  } else if (OB_FAIL(get_root_key_from_obs_(arg, result))) {
11267
    LOG_WARN("failed to get root key from obs", K(ret));
11268
  }
11269
  return ret;
11270
}
11271

11272
int ObRootService::get_root_key_from_obs_(const obrpc::ObRootKeyArg &arg,
11273
                                          obrpc::ObRootKeyResult &result)
11274
{
11275
  int ret = OB_SUCCESS;
11276
  ObZone empty_zone;
11277
  ObArray<ObAddr> active_server_list;
11278
  const ObSimpleTenantSchema *simple_tenant = NULL;
11279
  ObSchemaGetterGuard guard;
11280
  const uint64_t tenant_id = arg.tenant_id_;
11281
  bool enable_default = false;
11282
  if (OB_ISNULL(schema_service_)) {
11283
    ret = OB_ERR_UNEXPECTED;
11284
    LOG_WARN("schema_serviece_ is null", KR(ret));
11285
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, guard))) {
11286
    LOG_WARN("fail to get sys schema guard", KR(ret));
11287
  } else if (OB_FAIL(guard.get_tenant_info(tenant_id, simple_tenant))) {
11288
    LOG_WARN("fail to get simple tenant schema", KR(ret), K(tenant_id));
11289
  } else if (OB_NOT_NULL(simple_tenant) && simple_tenant->is_normal()) {
11290
    enable_default = true;
11291
  }
11292
  if (OB_FAIL(ret)) {
11293
  } else if (OB_FAIL(SVR_TRACER.get_alive_servers(empty_zone, active_server_list))) {
11294
    LOG_WARN("get alive servers failed", KR(ret));
11295
  } else if (OB_FAIL(ObDDLService::notify_root_key(rpc_proxy_, arg, active_server_list, result,
11296
                                                   enable_default, true/*skip_call_rs*/))) {
11297
    LOG_WARN("failed to notify root key", KR(ret));
11298
  }
11299
  return ret;
11300
}
11301

11302
int ObRootService::reload_master_key(const obrpc::ObReloadMasterKeyArg &arg,
11303
                                     obrpc::ObReloadMasterKeyResult &result)
11304
{
11305
  int ret = OB_SUCCESS;
11306
  uint64_t max_version = 0;
11307
  if (!inited_) {
11308
    ret = OB_NOT_INIT;
11309
    LOG_WARN("not init", K(ret));
11310
  } else if (OB_UNLIKELY(!arg.is_valid())) {
11311
    ret = OB_INVALID_ARGUMENT;
11312
    LOG_WARN("invalid arg", K(arg), K(ret));
11313
  } else if (OB_FAIL(master_key_mgr_.reload_tenant_max_key_version(arg.tenant_id_, max_version))) {
11314
    LOG_WARN("failed to reload master key version", K(ret), K(arg));
11315
  } else {
11316
    result.tenant_id_ = arg.tenant_id_;
11317
    result.master_key_id_ = max_version;
11318
  }
11319
  return ret;
11320
}
11321
#endif
11322

11323
} // end namespace rootserver
11324
} // end namespace oceanbase
11325