oceanbase

1
/**
2
 * Copyright (c) 2021 OceanBase
3
 * OceanBase CE is licensed under Mulan PubL v2.
4
 * You can use this software according to the terms and conditions of the Mulan PubL v2.
5
 * You may obtain a copy of Mulan PubL v2 at:
6
 *          http://license.coscl.org.cn/MulanPubL-2.0
7
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10
 * See the Mulan PubL v2 for more details.
11
 */
12

13
#define USING_LOG_PREFIX RS
14

15
#include "ob_root_inspection.h"
16

17
#include "lib/string/ob_sql_string.h"
18
#include "lib/mysqlclient/ob_mysql_proxy.h"
19
#include "lib/utility/ob_tracepoint.h"
20
#include "share/inner_table/ob_inner_table_schema.h"
21
#include "share/schema/ob_multi_version_schema_service.h"
22
#include "share/schema/ob_schema_getter_guard.h"
23
#include "share/schema/ob_schema_utils.h"
24
#include "share/ob_zone_info.h"
25
#include "share/system_variable/ob_system_variable_factory.h"
26
#include "share/system_variable/ob_system_variable_init.h"
27
#include "rootserver/ob_root_utils.h"
28
#include "rootserver/ob_zone_manager.h"
29
#include "rootserver/ob_ddl_operator.h"
30
#include "rootserver/ob_root_service.h"
31
#include "observer/ob_server_struct.h"
32
#include "observer/ob_sql_client_decorator.h"
33
#include "share/ob_primary_zone_util.h"
34
#include "share/ob_upgrade_utils.h"
35
#include "share/rc/ob_context.h"
36
#include "share/schema/ob_schema_mgr.h"
37
#include "share/ob_schema_status_proxy.h"//ObSchemaStatusProxy
38
#include "share/ob_global_stat_proxy.h"//ObGlobalStatProxy
39
#include "share/ob_tenant_info_proxy.h" // ObAllTenantInfoProxy
40
#include "share/schema/ob_table_schema.h"
41

42
namespace oceanbase
43
{
44
using namespace common;
45
using namespace common::sqlclient;
46
using namespace share;
47
using namespace share::schema;
48
using namespace sql;
49
namespace rootserver
50
{
51
////////////////////////////////////////////////////////////////
52
int ObTenantChecker::inspect(bool &passed, const char* &warning_info)
53
{
54
  int ret = OB_SUCCESS;
55
  int tmp_ret = OB_SUCCESS;
56
  passed = true;
57
  UNUSED(warning_info);
58
  if (OB_SUCCESS != (tmp_ret = alter_tenant_primary_zone_())) {
59
    ret = OB_SUCC(ret) ? tmp_ret : ret;
60
    LOG_WARN("fail to alter tenant primary_zone", KR(ret), KR(tmp_ret));
61
  }
62

63
  if (OB_SUCCESS != (tmp_ret = check_create_tenant_end_())) {
64
    ret = OB_SUCC(ret) ? tmp_ret : ret;
65
    LOG_WARN("fail to check create tenant end", KR(ret), KR(tmp_ret));
66
  }
67

68
  if (OB_SUCCESS != (tmp_ret = check_garbage_tenant_(passed))) {
69
    ret = OB_SUCC(ret) ? tmp_ret : ret;
70
    LOG_WARN("fail to check garbage tenant", KR(ret), KR(tmp_ret));
71
  }
72
  return ret;
73
}
74

75
// If the primary_zone of tenant is null, need to set to 'RANDOM'
76
int ObTenantChecker::alter_tenant_primary_zone_()
77
{
78
  int ret = OB_SUCCESS;
79
  ObSchemaGetterGuard schema_guard;
80
  ObArray<uint64_t> tenant_ids;
81
  if (OB_ISNULL(schema_service_)) {
82
    ret = OB_ERR_UNEXPECTED;
83
    LOG_WARN("schema service not init", K(ret));
84
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
85
    LOG_WARN("get_schema_guard failed", K(ret));
86
  } else if (OB_FAIL(schema_guard.get_tenant_ids(tenant_ids))) {
87
    LOG_WARN("get_tenant_ids failed", K(ret));
88
  } else {
89
    const ObTenantSchema *tenant_schema = NULL;
90
    int64_t affected_rows = 0;
91
    FOREACH_CNT_X(tenant_id, tenant_ids, OB_SUCCESS == ret) {
92
      if (OB_ISNULL(tenant_id)) {
93
        ret = OB_ERR_UNEXPECTED;
94
        LOG_WARN("tenant_id is null", K(ret));
95
      } else if (OB_FAIL(schema_guard.get_tenant_info(*tenant_id, tenant_schema))) {
96
        LOG_WARN("fail to get tenant info", K(ret), K(*tenant_id));
97
      } else if (OB_ISNULL(tenant_schema)) {
98
        ret = OB_ERR_UNEXPECTED;
99
        LOG_WARN("tenant schema is null", K(ret), K(*tenant_id));
100
      } else if (tenant_schema->get_primary_zone().empty()) {
101
        ObSqlString sql;
102
        if (OB_FAIL(sql.append_fmt("ALTER TENANT %s set primary_zone = RANDOM",
103
                                   tenant_schema->get_tenant_name()))) {
104
          LOG_WARN("fail to generate sql", K(ret), K(*tenant_id));
105
        } else if (OB_ISNULL(sql_proxy_)) {
106
          ret = OB_ERR_UNEXPECTED;
107
          LOG_WARN("sql_proxy is null", K(ret));
108
        } else if (OB_FAIL(sql_proxy_->write(sql.ptr(), affected_rows))) {
109
          LOG_WARN("execute sql failed", K(ret));
110
        } else {
111
          ROOTSERVICE_EVENT_ADD("inspector", "alter_tenant_primary_zone",
112
                                "tenant_id", tenant_schema->get_tenant_id(),
113
                                "tenant", tenant_schema->get_tenant_name());
114
        }
115
      }
116
    }
117
  }
118
  return ret;
119
}
120

121
int ObTenantChecker::check_create_tenant_end_()
122
{
123
  int ret = OB_SUCCESS;
124
  ObSchemaGetterGuard schema_guard;
125
  ObArray<uint64_t> tenant_ids;
126
  if (OB_ISNULL(schema_service_)) {
127
    ret = OB_ERR_UNEXPECTED;
128
    LOG_WARN("schema service not init", K(ret));
129
  } else if (!schema_service_->is_sys_full_schema()) {
130
    // skip
131
  } else if (GCTX.is_standby_cluster()) {
132
    // skip
133
  } else if (OB_FAIL(schema_service_->get_tenant_ids(tenant_ids))) {
134
    LOG_WARN("get_tenant_ids failed", K(ret));
135
  } else if (OB_ISNULL(GCTX.root_service_)) {
136
    ret = OB_ERR_UNEXPECTED;
137
    LOG_WARN("rootservice is null", KR(ret));
138
  } else {
139
    const ObSimpleTenantSchema *tenant_schema = NULL;
140
    int64_t schema_version = OB_INVALID_VERSION;
141
    int64_t baseline_schema_version = OB_INVALID_VERSION;
142
    FOREACH_CNT(tenant_id, tenant_ids) {
143
      // overwrite ret
144
      if (!GCTX.root_service_->is_full_service()) {
145
        ret = OB_CANCELED;
146
        LOG_WARN("rs is not in full service", KR(ret));
147
        break;
148
      } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(*tenant_id, schema_guard))) {
149
        LOG_WARN("get_schema_guard failed", KR(ret), K(*tenant_id));
150
      } else if (OB_FAIL(schema_guard.get_schema_version(*tenant_id, schema_version))) {
151
        LOG_WARN("fail to get tenant schema version", KR(ret), K(*tenant_id));
152
      } else if (!share::schema::ObSchemaService::is_formal_version(schema_version)) {
153
        // tenant is still in creating
154
      } else if (OB_FAIL(schema_guard.get_tenant_info(*tenant_id, tenant_schema))) {
155
        LOG_WARN("fail to get tenant schema", KR(ret), K(*tenant_id));
156
      } else if (OB_ISNULL(tenant_schema)) {
157
        ret = OB_ERR_UNEXPECTED;
158
        LOG_WARN("tenant not exist", KR(ret), K(*tenant_id));
159
      } else if (OB_FAIL(schema_service_->get_baseline_schema_version(*tenant_id, false/*auto update*/,
160
                                                                      baseline_schema_version))) {
161
        LOG_WARN("fail to get baseline schema_version", KR(ret), K(*tenant_id));
162
      } else if (OB_INVALID_VERSION == baseline_schema_version) {
163
        //baseline_schema_version is not valid, just skip to create this kind of tenant
164
      } else if (tenant_schema->is_creating()) {
165
        obrpc::ObCreateTenantEndArg arg;
166
        arg.exec_tenant_id_ = OB_SYS_TENANT_ID;
167
        arg.tenant_id_ = *tenant_id;
168
        if (OB_FAIL(rpc_proxy_.create_tenant_end(arg))) {
169
          LOG_WARN("fail to execute create tenant end", KR(ret), K(*tenant_id));
170
        } else {
171
          LOG_INFO("execute create_tenant_end", KR(ret), K(*tenant_id), K(schema_version));
172
          ROOTSERVICE_EVENT_ADD("inspector", "tenant_checker", "info", "execute create_tenant_end", "tenant_id", *tenant_id);
173
        }
174
      }
175
    }
176
  }
177
  return ret;
178
}
179

180
int ObTenantChecker::check_garbage_tenant_(bool &passed)
181
{
182
  int ret = OB_SUCCESS;
183
  if (OB_ISNULL(schema_service_)) {
184
    ret = OB_ERR_UNEXPECTED;
185
    LOG_WARN("schema service not init", KR(ret));
186
  } else if (!schema_service_->is_sys_full_schema()) {
187
    // skip
188
  } else {
189
    obrpc::ObGetSchemaArg arg;
190
    obrpc::ObTenantSchemaVersions result;
191
    ObSchemaGetterGuard schema_guard;
192
    arg.ignore_fail_ = true;
193
    arg.exec_tenant_id_ = OB_SYS_TENANT_ID;
194
    const int64_t rpc_timeout = GCONF.rpc_timeout * 10;
195
    // There may have multi RootService, so we won't force drop tenant here.
196
    if (OB_FAIL(rpc_proxy_.timeout(rpc_timeout).get_tenant_schema_versions(arg, result))) {
197
      LOG_WARN("fail to get tenant schema versions", KR(ret));
198
    } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
199
      LOG_WARN("get_schema_guard failed", KR(ret));
200
    }
201
    for (int64_t i = 0; OB_SUCC(ret) && i < result.tenant_schema_versions_.count(); i++) {
202
      TenantIdAndSchemaVersion &tenant = result.tenant_schema_versions_.at(i);
203
      int tmp_ret = OB_SUCCESS;
204
      if (!GCTX.root_service_->is_full_service()) {
205
        ret = OB_CANCELED;
206
        LOG_WARN("rs is not in full service", KR(ret));
207
      } else if (!ObSchemaService::is_formal_version(tenant.schema_version_)) {
208
        const ObSimpleTenantSchema *tenant_schema = NULL;
209
        uint64_t tenant_id = tenant.tenant_id_;
210
        if (OB_SUCCESS != (tmp_ret = schema_guard.get_tenant_info(tenant_id, tenant_schema))) {
211
          LOG_WARN("fail to get tenant info", KR(tmp_ret), K(tenant_id));
212
        } else if (OB_ISNULL(tenant_schema)) {
213
          tmp_ret = OB_TENANT_NOT_EXIST;
214
        } else if (tenant_schema->is_restore()) {
215
          LOG_INFO("tenant is in restore", KPC(tenant_schema));
216
        } else if (tenant_schema->is_creating()) {
217
          LOG_ERROR("the tenant may be in the process of creating, if the error reports continuously, please check", K(tenant_id));
218
          LOG_DBA_WARN(OB_ERR_ROOT_INSPECTION, "msg", "the tenant may be in the process of creating, if the error reports continuously, please check",
219
                       K(tenant_id), "tenant_name", tenant_schema->get_tenant_name());
220
          ROOTSERVICE_EVENT_ADD("inspector", "tenant_checker",
221
                                "info", "the tenant may be in the process of creating, if the error reports continuously, please check",
222
                                "tenant_id", tenant_id,
223
                                "tenant_name", tenant_schema->get_tenant_name_str());
224
        }
225
        ret = OB_SUCC(ret) ? tmp_ret : ret;
226
      }
227
    } // end for
228
    passed = OB_SUCC(ret);
229
  }
230
  return ret;
231
}
232

233
////////////////////////////////////////////////////////////////
234
ObTableGroupChecker::ObTableGroupChecker(share::schema::ObMultiVersionSchemaService &schema_service)
235
    : schema_service_(schema_service),
236
      check_part_option_map_(),
237
      part_option_not_match_set_(),
238
      allocator_(ObModIds::OB_SCHEMA),
239
      is_inited_(false)
240
{
241
}
242

243
int ObTableGroupChecker::init()
244
{
245
  int ret = OB_SUCCESS;
246
  if (is_inited_) {
247
    ret = OB_INIT_TWICE;
248
    LOG_WARN("init twice", K(ret));
249
  } else if (OB_FAIL(check_part_option_map_.create(TABLEGROUP_BUCKET_NUM, ObModIds::OB_HASH_BUCKET_TABLEGROUP_MAP))) {
250
    LOG_WARN("init check_part_option_map failed", K(ret));
251
  } else if (OB_FAIL(part_option_not_match_set_.create(TABLEGROUP_BUCKET_NUM))) {
252
    LOG_WARN("init part_option_not_match_set failed", K(ret));
253
  } else {
254
    is_inited_ = true;
255
  }
256
  return ret;
257
}
258

259
ObTableGroupChecker::~ObTableGroupChecker()
260
{
261
}
262

263
int ObTableGroupChecker::inspect(bool &passed, const char* &warning_info)
264
{
265
  int ret = OB_SUCCESS;
266
  passed = true;
267
  ObArray<uint64_t> tenant_ids;
268
  if (!is_inited_) {
269
    ret = OB_NOT_INIT;
270
    LOG_WARN("tablegroup checker is not init", K(ret));
271
  } else if (OB_FAIL(schema_service_.get_tenant_ids(tenant_ids))) {
272
    LOG_WARN("fail to get tenant ids", KR(ret));
273
  } else {
274
    int tmp_ret = OB_SUCCESS;
275
    FOREACH(tenant_id, tenant_ids) { // ignore error for each tenant
276
      if (OB_SUCCESS != (tmp_ret = inspect_(*tenant_id, passed))) {
277
        LOG_WARN("inspect tablegroup options by tenant failed",
278
                 KR(tmp_ret), "tenant_id", *tenant_id);
279
      }
280
      ret = OB_SUCC(ret) ? tmp_ret : ret;
281
    }
282
    if (!passed) {
283
      warning_info = "tablegroup has tables that have different primary_zone/locality/part_option";
284
    }
285
  }
286
  return ret;
287
}
288

289
int ObTableGroupChecker::inspect_(
290
    const uint64_t tenant_id,
291
    bool &passed)
292
{
293
  int ret = OB_SUCCESS;
294
  ObSchemaGetterGuard schema_guard;
295
  allocator_.reset();
296
  check_part_option_map_.reuse();
297
  part_option_not_match_set_.reuse();
298
  ObArray<uint64_t> table_ids;
299
  if (!is_inited_) {
300
    ret = OB_NOT_INIT;
301
    LOG_WARN("tablegroup checker is not init", KR(ret));
302
  } else if (OB_FAIL(schema_service_.get_tenant_schema_guard(tenant_id, schema_guard))) {
303
    LOG_WARN("get schema guard failed", KR(ret), K(tenant_id));
304
  } else if (OB_FAIL(schema_guard.get_table_ids_in_tenant(tenant_id, table_ids))) {
305
    LOG_WARN("fail to get table_ids", KR(ret), K(tenant_id));
306
  } else if (OB_ISNULL(GCTX.root_service_)) {
307
    ret = OB_ERR_UNEXPECTED;
308
    LOG_WARN("rootservice is null", KR(ret));
309
  } else if (!GCTX.root_service_->is_full_service()) {
310
    ret = OB_CANCELED;
311
    LOG_WARN("rs is not in full service", KR(ret));
312
  } else {
313
    for (int64_t i = 0; OB_SUCC(ret) && i < table_ids.count(); i++) {
314
      const uint64_t table_id = table_ids.at(i);
315
      const ObTableSchema *table = NULL;
316
      // schema guard cannot be used repeatedly in iterative logic,
317
      // otherwise it will cause a memory hike in schema cache
318
      if (!GCTX.root_service_->is_full_service()) {
319
        ret = OB_CANCELED;
320
        LOG_WARN("rs is not in full service", KR(ret));
321
      } else if (OB_FAIL(schema_service_.get_tenant_schema_guard(tenant_id, schema_guard))) {
322
        LOG_WARN("get schema guard failed", K(ret), K(tenant_id));
323
      } else if (OB_FAIL(schema_guard.get_table_schema(tenant_id, table_id, table))) {
324
        LOG_WARN("get table schema failed", K(ret), KT(table_id));
325
      } else if (OB_ISNULL(table)) {
326
        ret = OB_ERR_UNEXPECTED;
327
        LOG_WARN("table not exist", KR(ret), K(table_id));
328
      } else if (is_sys_table(table->get_table_id()) || !table->has_partition()) {
329
        // skip, check the partitioned user table
330
      } else if (OB_FAIL(check_part_option(*table, schema_guard))) {
331
        LOG_WARN("check part option fail", KR(ret), KPC(table));
332
      } else {}
333
    }
334
  }
335
  if (OB_SUCC(ret)) {
336
    if (part_option_not_match_set_.size() > 0) {
337
      passed = false;
338
      LOG_WARN("tables part option in one tablegroup are not the same", K(tenant_id), K_(part_option_not_match_set));
339
      ROOTSERVICE_EVENT_ADD("inspector", "check_part_option", K(tenant_id), K_(part_option_not_match_set));
340
    }
341
  }
342
  return ret;
343
}
344

345
// Check the partition_option of tables in the same tablegroup:
346
// 1. For tablegroups created before 2.0, the part_type and part_num of tables in tablegroup should be same.
347
// 2. For tablegroups created after 2.0:
348
//    1) tablegroup is nonpartition, Allow "non partitioned table" or "partitioned table with 1 number of partitions" in tablegroup.
349
//       in addition, the partition_num, partition_type, partition_value and number of expression vectors of tables must be same.
350
//    2) tablegroup is partitioned, the partition_num, partition_type, partition_value and number of expression vectors
351
//       of tables in tablegroup should be same.
352
int ObTableGroupChecker::check_part_option(const ObSimpleTableSchemaV2 &table, ObSchemaGetterGuard &schema_guard)
353
{
354
  int ret = OB_SUCCESS;
355
  int tmp_ret = OB_SUCCESS;
356
  ObSqlString user_error;
357
  const uint64_t tenant_id = table.get_tenant_id();
358
  const uint64_t tablegroup_id = table.get_tablegroup_id();
359
  if (!is_inited_) {
360
    ret = OB_NOT_INIT;
361
    LOG_WARN("tablegroup checker is not init", KR(ret));
362
  } else if (OB_INVALID_ID == tablegroup_id) {
363
    // skip check while tablegroup_id is default value
364
  } else if (!table.is_user_table()) {
365
    // only check user table
366
  } else if (OB_HASH_NOT_EXIST != (tmp_ret = part_option_not_match_set_.exist_refactored(tablegroup_id))) {
367
   //skip check while already in part_option_not_match_set_
368
   if (OB_HASH_EXIST != tmp_ret) {
369
     ret = tmp_ret;
370
     LOG_WARN("fail to check if tablegroup_id exist", KR(ret), K(tablegroup_id));
371
   }
372
  } else {
373
    const ObSimpleTableSchemaV2 *table_in_map = NULL;
374
    bool is_matched = true;
375
    const ObTablegroupSchema *tablegroup = NULL;
376
    const ObSimpleTableSchemaV2 *primary_table_schema = NULL;
377
    if (OB_FAIL(schema_guard.get_tablegroup_schema(tenant_id, tablegroup_id, tablegroup))) {
378
      LOG_WARN("fail to get tablegroup schema", KR(ret), K(tenant_id), KT(tablegroup_id));
379
    } else if (OB_ISNULL(tablegroup)) {
380
      ret = OB_ERR_UNEXPECTED;
381
      LOG_WARN("tablegroup schema is null", KR(ret), KT(tablegroup_id));
382
    } else if (tablegroup->get_sharding() == OB_PARTITION_SHARDING_NONE) {
383
      //no need to check,just ignore
384
    } else if (tablegroup->get_sharding() == OB_PARTITION_SHARDING_PARTITION
385
              || tablegroup->get_sharding() == OB_PARTITION_SHARDING_ADAPTIVE) {
386
      bool check_sub_part = tablegroup->get_sharding() == OB_PARTITION_SHARDING_PARTITION ? false : true;
387
      if (OB_FAIL(check_part_option_map_.get_refactored(tablegroup_id, table_in_map))) {
388
        //set to the map while not in check_part_option_map_
389
        if (OB_HASH_NOT_EXIST == ret) {
390
          ObSimpleTableSchemaV2 *new_table_schema = NULL;
391
          if (OB_FAIL(schema_guard.get_primary_table_schema_in_tablegroup(tenant_id, tablegroup_id, primary_table_schema))) {
392
            LOG_WARN("fail to get primary table schema in tablegroup", KR(ret), K(tablegroup_id));
393
          } else if (OB_ISNULL(primary_table_schema)) {
394
            ret = OB_ERR_UNEXPECTED;
395
            LOG_WARN("primary table schema is NULL", KR(ret), K(tenant_id), K(tablegroup_id));
396
          } else if (OB_FAIL(ObSchemaUtils::alloc_schema(allocator_, *primary_table_schema, new_table_schema))) {
397
            LOG_WARN("alloc schema failed", KR(ret), KPC(primary_table_schema));
398
          } else if (OB_FAIL(check_part_option_map_.set_refactored(tablegroup_id, new_table_schema))) {
399
            LOG_WARN("set table_schema in hashmap fail", KR(ret), K(tablegroup_id), KPC(primary_table_schema));
400
          }
401
        } else {
402
          LOG_WARN("check tablegroup_id in hashmap fail", KR(ret), K(tablegroup_id));
403
        }
404
      } else if (OB_ISNULL(table_in_map)) {
405
        ret = OB_ERR_UNEXPECTED;
406
        LOG_WARN("table_is_map is NULL", KR(ret), K(tablegroup_id));
407
      } else if (OB_FAIL(ObSimpleTableSchemaV2::compare_partition_option(table, *table_in_map, check_sub_part, is_matched, &user_error))) {
408
        LOG_WARN("fail to check partition option", KR(ret), K(table), KPC(table_in_map));
409
      }
410
    } else {
411
      ret = OB_ERR_UNEXPECTED;
412
      LOG_WARN("sharding type not suit", KR(ret), KPC(tablegroup));
413
    }
414
    if (OB_FAIL(ret) || is_matched) {
415
      //skip
416
    } else if (OB_FAIL(part_option_not_match_set_.set_refactored(tablegroup_id))) {
417
      LOG_WARN("set tablegroup_id in hashset fail", KR(ret));
418
    } else if (OB_ISNULL(table_in_map)) {
419
      ret = OB_ERR_UNEXPECTED;
420
      LOG_WARN("table_is_map is NULL", KR(ret), K(tablegroup_id));
421
    } else {
422
      LOG_INFO("tables in one tablegroup have different part/subpart option",
423
               K(tablegroup_id), "table_id", table.get_table_id(), K(user_error), K(table_in_map->get_table_id()));
424
    }
425
  }
426
  return ret;
427
}
428

429
////////////////////////////////////////////////////////////////
430
ObInspector::ObInspector(ObRootService &rs)
431
    :ObAsyncTimerTask(rs.get_inspect_task_queue()),
432
    rs_(rs)
433
{}
434

435
int ObInspector::process()
436
{
437
  // @todo ObTaskController::get().switch_task(share::ObTaskType::ROOT_SERVICE);
438
  int ret = OB_SUCCESS;
439
  ObTableGroupChecker tablegroup_checker(rs_.get_schema_service());
440
  ObRootInspection system_schema_checker;
441
  ObTenantChecker tenant_checker(rs_.get_schema_service(), rs_.get_sql_proxy(), rs_.get_common_rpc_proxy());
442

443
  ret = OB_E(EventTable::EN_STOP_ROOT_INSPECTION) OB_SUCCESS;
444
  if (OB_FAIL(ret)) {
445
  } else if (OB_FAIL(tablegroup_checker.init())) {
446
    LOG_WARN("init tablegroup_checker failed", K(ret));
447
  } else if (OB_FAIL(system_schema_checker.init(rs_.get_schema_service(), rs_.get_zone_mgr(),
448
                                         rs_.get_sql_proxy()))) {
449
    LOG_WARN("init root inspection failed", K(ret));
450
  } else {
451
    ObInspectionTask *inspection_tasks[] = {
452
      &tablegroup_checker,
453
      &system_schema_checker,
454
      &tenant_checker
455
    };
456
    bool passed = true;
457
    const char* warning_info = NULL;
458
    int N = ARRAYSIZEOF(inspection_tasks);
459
    for (int i = 0; i < N; ++i) {
460
      passed = true;
461
      warning_info = NULL;
462
      int tmp_ret = inspection_tasks[i]->inspect(passed, warning_info);
463
      if (OB_SUCCESS != tmp_ret) {
464
        LOG_WARN("inpection task failed", K(tmp_ret), K(i), "task", inspection_tasks[i]->get_task_name());
465
      } else if (passed) {
466
        LOG_INFO("inspection task succ", K(i), "task", inspection_tasks[i]->get_task_name());
467
      } else {
468
        LOG_ERROR(warning_info);
469
        ROOTSERVICE_EVENT_ADD("inspector", inspection_tasks[i]->get_task_name(),
470
                              "info", (warning_info == NULL ? "": warning_info));
471
      }
472
    }
473
  }
474
  return ret;
475
}
476

477
ObAsyncTask *ObInspector::deep_copy(char *buf, const int64_t buf_size) const
478
{
479
  ObInspector *task = NULL;
480
  if (NULL == buf || buf_size < static_cast<int64_t>(sizeof(*this))) {
481
    LOG_WARN_RET(OB_BUF_NOT_ENOUGH, "buffer not large enough", K(buf_size));
482
  } else {
483
    task = new(buf) ObInspector(rs_);
484
  }
485
  return task;
486
}
487

488
////////////////////////////////////////////////////////////////
489
ObPurgeRecyclebinTask::ObPurgeRecyclebinTask(ObRootService &rs)
490
    :ObAsyncTimerTask(rs.get_inspect_task_queue()),
491
    root_service_(rs)
492
{}
493

494
int ObPurgeRecyclebinTask::process()
495
{
496
  LOG_INFO("purge recyclebin task begin");
497
  int ret = OB_SUCCESS;
498
  const int64_t PURGE_EACH_TIME = 1000;
499
  int64_t delay = 1 * 60 * 1000 * 1000;
500
  int64_t expire_time = GCONF.recyclebin_object_expire_time;
501
  int64_t purge_interval = GCONF._recyclebin_object_purge_frequency;
502
  if (expire_time > 0 && purge_interval > 0) {
503
   if (OB_FAIL(root_service_.purge_recyclebin_objects(PURGE_EACH_TIME))) {
504
      LOG_WARN("fail to purge recyclebin objects", KR(ret));
505
    }
506
    delay = purge_interval;
507
  }
508
  // the error code is only for outputtion log, the function will return success.
509
  // the task no need retry, because it will be triggered periodically.
510
  if (OB_FAIL(root_service_.schedule_recyclebin_task(delay))) {
511
    LOG_WARN("schedule purge recyclebin task failed", KR(ret), K(delay));
512
  } else {
513
    LOG_INFO("submit purge recyclebin task success", K(delay));
514
  }
515
  LOG_INFO("purge recyclebin task end", K(delay));
516
  return OB_SUCCESS;
517
}
518

519
ObAsyncTask *ObPurgeRecyclebinTask::deep_copy(char *buf, const int64_t buf_size) const
520
{
521
  ObPurgeRecyclebinTask *task = NULL;
522
  if (NULL == buf || buf_size < static_cast<int64_t>(sizeof(*this))) {
523
    LOG_WARN_RET(OB_BUF_NOT_ENOUGH, "buffer not large enough", K(buf_size));
524
  } else {
525
    task = new(buf) ObPurgeRecyclebinTask(root_service_);
526
  }
527
  return task;
528
}
529

530
ObRootInspection::ObRootInspection()
531
  : inited_(false), stopped_(false), zone_passed_(false),
532
    sys_param_passed_(false), sys_stat_passed_(false),
533
    sys_table_schema_passed_(false), data_version_passed_(false),
534
    all_checked_(false), all_passed_(false), can_retry_(false),
535
    sql_proxy_(NULL), rpc_proxy_(NULL), schema_service_(NULL),
536
    zone_mgr_(NULL)
537
{
538
}
539

540
ObRootInspection::~ObRootInspection()
541
{
542
}
543

544
int ObRootInspection::init(ObMultiVersionSchemaService &schema_service,
545
                           ObZoneManager &zone_mgr,
546
                           ObMySQLProxy &sql_proxy,
547
                           obrpc::ObCommonRpcProxy *rpc_proxy)
548
{
549
  int ret = OB_SUCCESS;
550
  if (inited_) {
551
    ret = OB_INIT_TWICE;
552
    LOG_WARN("init twice", K(ret));
553
  } else {
554
    schema_service_ = &schema_service;
555
    zone_mgr_ = &zone_mgr;
556
    sql_proxy_ = &sql_proxy;
557
    stopped_ = false;
558
    zone_passed_ = false;
559
    sys_param_passed_ = false;
560
    sys_stat_passed_ = false;
561
    sys_table_schema_passed_ = false;
562
    data_version_passed_ = false;
563
    all_checked_ = false;
564
    all_passed_ = false;
565
    can_retry_ = false;
566
    rpc_proxy_ = rpc_proxy;
567
    inited_ = true;
568
  }
569
  return ret;
570
}
571

572
int ObRootInspection::check_all()
573
{
574
  int ret = OB_SUCCESS;
575
  if (!inited_) {
576
    ret = OB_NOT_INIT;
577
    LOG_WARN("not init", K(ret));
578
  } else if (OB_FAIL(check_cancel())) {
579
    LOG_WARN("check_cancel failed", K(ret));
580
  } else if (OB_ISNULL(schema_service_)) {
581
    ret = OB_ERR_UNEXPECTED;
582
    LOG_WARN("schema_service is null", K(ret));
583
  } else if (!schema_service_->is_tenant_full_schema(OB_SYS_TENANT_ID)) {
584
    ret = OB_EAGAIN;
585
    LOG_WARN("schema is not ready, try again", K(ret));
586
  } else {
587
    can_retry_ = false;
588
    int tmp = OB_SUCCESS;
589

590
    // check __all_zone
591
    if (OB_SUCCESS != (tmp = check_zone())) {
592
      LOG_WARN("check_zone failed", K(tmp));
593
      ret = (OB_SUCCESS == ret) ? tmp : ret;
594
    }
595
    zone_passed_ = (OB_SUCCESS == tmp);
596

597
    // check sys stat
598
    tmp = OB_SUCCESS;
599
    if (OB_SUCCESS != (tmp = check_sys_stat_())) {
600
      LOG_WARN("check_sys_stat failed", K(tmp));
601
      ret = (OB_SUCCESS == ret) ? tmp : ret;
602
    }
603
    sys_stat_passed_ = (OB_SUCCESS == tmp);
604

605
    // check sys param
606
    tmp = OB_SUCCESS;
607
    if (OB_SUCCESS != (tmp = check_sys_param_())) {
608
      LOG_WARN("check_sys_param failed", K(tmp));
609
      ret = (OB_SUCCESS == ret) ? tmp : ret;
610
    }
611
    sys_param_passed_ = (OB_SUCCESS == tmp);
612

613
    // check sys schema
614
    tmp = OB_SUCCESS;
615
    if (OB_SUCCESS != (tmp = check_sys_table_schemas_())) {
616
      LOG_WARN("check_sys_table_schemas failed", K(tmp));
617
      ret = (OB_SUCCESS == ret) ? tmp : ret;
618
    }
619
    sys_table_schema_passed_ = (OB_SUCCESS == tmp);
620

621
    // check tenant's data version
622
    tmp = OB_SUCCESS;
623
    if (OB_SUCCESS != (tmp = check_data_version_())) {
624
      LOG_WARN("check_data_version failed", K(tmp));
625
      ret = (OB_SUCCESS == ret) ? tmp : ret;
626
    }
627
    data_version_passed_ = (OB_SUCCESS == tmp);
628

629
    // upgrade job may still running, in order to avoid the upgrade process error stuck,
630
    // ignore the 4674 error
631
    for (int64_t i = 0; i < UPGRADE_JOB_TYPE_COUNT; i++) {
632
      tmp = OB_SUCCESS;
633
      ObRsJobType job_type = upgrade_job_type_array[i];
634
      if (job_type > JOB_TYPE_INVALID && job_type < JOB_TYPE_MAX) {
635
        if (OB_SUCCESS != (tmp = check_cancel())) {
636
          LOG_WARN("check_cancel failed", KR(ret), K(tmp));
637
          ret = (OB_SUCCESS == ret) ? tmp : ret;
638
          break;
639
        } else if (OB_SUCCESS != (tmp = ObUpgradeUtils::check_upgrade_job_passed(job_type))) {
640
          LOG_WARN("fail to check upgrade job passed", K(tmp), K(job_type));
641
          if (OB_RUN_JOB_NOT_SUCCESS != tmp) {
642
            ret = (OB_SUCCESS == ret) ? tmp : ret;
643
          } else {
644
            LOG_WARN("upgrade job may still running, check with __all_virtual_uprade_inspection",
645
                     K(ret), K(tmp), "job_type", ObRsJobTableOperator::get_job_type_str(job_type));
646
          }
647
        }
648
      }
649
    }
650

651
    all_checked_ = true;
652
    all_passed_ = OB_SUCC(ret);
653
  }
654
  return ret;
655
}
656

657
int ObRootInspection::check_tenant(const uint64_t tenant_id)
658
{
659
  int ret = OB_SUCCESS;
660
  if (!inited_) {
661
    ret = OB_NOT_INIT;
662
    LOG_WARN("not init", KR(ret));
663
  } else if (OB_FAIL(check_cancel())) {
664
    LOG_WARN("check_cancel failed", KR(ret));
665
  } else {
666
    int tmp_ret = OB_SUCCESS;
667
    if (OB_TMP_FAIL(check_sys_stat_(tenant_id))) {
668
      LOG_WARN("fail to check sys stat", KR(tmp_ret), K(tenant_id));
669
      ret = OB_SUCC(ret) ? tmp_ret : ret;
670
    }
671
    if (OB_TMP_FAIL(check_sys_param_(tenant_id))) {
672
      LOG_WARN("fail to check param", KR(tmp_ret), K(tenant_id));
673
      ret = OB_SUCC(ret) ? tmp_ret : ret;
674
    }
675
    if (OB_TMP_FAIL(check_sys_table_schemas_(tenant_id))) {
676
      LOG_WARN("fail to check sys table", KR(tmp_ret), K(tenant_id));
677
      ret = OB_SUCC(ret) ? tmp_ret : ret;
678
    }
679
  }
680
  return ret;
681
}
682

683
int ObRootInspection::inspect(bool &passed, const char* &warning_info)
684
{
685
  int ret = OB_SUCCESS;
686
  if (!GCONF.in_upgrade_mode()) {
687
    ret = check_all();
688
    if (OB_SUCC(ret)) {
689
      passed = all_passed_;
690
      warning_info = "system metadata error";
691
    }
692
  } else {
693
    passed = true;
694
  }
695
  return ret;
696
}
697

698

699
// standby tenant may stay at lower data version,
700
// root_inspection won't check standby tenant's schema.
701
int ObRootInspection::construct_tenant_ids_(
702
    common::ObIArray<uint64_t> &tenant_ids)
703
{
704
  int ret = OB_SUCCESS;
705
  tenant_ids.reset();
706
  ObArray<uint64_t> standby_tenants;
707
  ObArray<uint64_t> tmp_tenants;
708
  if (!inited_) {
709
    ret = OB_NOT_INIT;
710
    LOG_WARN("not init", KR(ret));
711
  } else if (OB_FAIL(check_cancel())) {
712
    LOG_WARN("check_cancel failed", KR(ret));
713
  } else if (OB_FAIL(ObTenantUtils::get_tenant_ids(schema_service_, tmp_tenants))) {
714
    LOG_WARN("get_tenant_ids failed", KR(ret));
715
  } else {
716
    bool is_standby = false;
717
    for (int64_t i = 0; OB_SUCC(ret) && i < tmp_tenants.count(); i++) {
718
      const uint64_t tenant_id = tmp_tenants.at(i);
719
      if (OB_FAIL(ObAllTenantInfoProxy::is_standby_tenant(sql_proxy_, tenant_id, is_standby))) {
720
        LOG_WARN("fail to check is standby tenant", KR(ret), K(tenant_id));
721
      } else if (is_standby) {
722
        // skip
723
      } else if (OB_FAIL(tenant_ids.push_back(tenant_id))) {
724
        LOG_WARN("fail to push back tenant_id", KR(ret), K(tenant_id));
725
      }
726
    } // end for
727
  }
728
  return ret;
729
}
730

731
int ObRootInspection::check_zone()
732
{
733
  int ret = OB_SUCCESS;
734
  ObSqlString extra_cond;
735
  HEAP_VAR(ObGlobalInfo, global_zone_info) {
736
    ObArray<ObZoneInfo> zone_infos;
737
    ObArray<const char *> global_zone_item_names;
738
    if (!inited_) {
739
      ret = OB_NOT_INIT;
740
      LOG_WARN("not init", K(ret));
741
    } else if (OB_FAIL(check_cancel())) {
742
      LOG_WARN("check_cancel failed", K(ret));
743
    } else if (OB_FAIL(extra_cond.assign_fmt("zone = '%s'", global_zone_info.zone_.ptr()))) {
744
      LOG_WARN("extra_cond assign_fmt failed", K(ret));
745
    } else if (OB_FAIL(get_names(global_zone_info.list_, global_zone_item_names))) {
746
      LOG_WARN("get global zone item names failed", K(ret));
747
    } else if (OB_FAIL(check_names(OB_SYS_TENANT_ID, OB_ALL_ZONE_TNAME, global_zone_item_names, extra_cond))) {
748
      LOG_WARN("check global zone item names failed", "table_name", OB_ALL_ZONE_TNAME,
749
          K(global_zone_item_names), K(extra_cond), K(ret));
750
    } else if (OB_FAIL(zone_mgr_->get_zone(zone_infos))) {
751
      LOG_WARN("zone manager get_zone failed", K(ret));
752
    } else {
753
      ObArray<const char *> zone_item_names;
754
      FOREACH_CNT_X(zone_info, zone_infos, OB_SUCCESS == ret) {
755
        zone_item_names.reuse();
756
        extra_cond.reset();
757
        if (OB_FAIL(check_cancel())) {
758
          LOG_WARN("check_cancel failed", K(ret));
759
        } else if (OB_FAIL(extra_cond.assign_fmt("zone = '%s'", zone_info->zone_.ptr()))) {
760
          LOG_WARN("extra_cond assign_fmt failed", K(ret));
761
        } else if (OB_FAIL(get_names(zone_info->list_, zone_item_names))) {
762
          LOG_WARN("get zone item names failed", K(ret));
763
        } else if (OB_FAIL(check_names(OB_SYS_TENANT_ID, OB_ALL_ZONE_TNAME, zone_item_names, extra_cond))) {
764
          LOG_WARN("check zone item names failed", "table_name", OB_ALL_ZONE_TNAME,
765
              K(zone_item_names), "zone_info", *zone_info, K(extra_cond), K(ret));
766
        }
767
      }
768
    }
769
  }
770
  return ret;
771
}
772

773
int ObRootInspection::check_sys_stat_()
774
{
775
  int ret = OB_SUCCESS;
776
  ObArray<uint64_t> tenant_ids;
777
  if (!inited_) {
778
    ret = OB_NOT_INIT;
779
    LOG_WARN("not init", KR(ret));
780
  } else if (OB_ISNULL(schema_service_)) {
781
    ret = OB_ERR_UNEXPECTED;
782
    LOG_WARN("schema_service is null", KR(ret));
783
  } else if (OB_FAIL(check_cancel())) {
784
    LOG_WARN("check_cancel failed", KR(ret));
785
  } else if (OB_FAIL(construct_tenant_ids_(tenant_ids))) {
786
    LOG_WARN("get_tenant_ids failed", KR(ret));
787
  } else {
788
    int backup_ret = OB_SUCCESS;
789
    int tmp_ret = OB_SUCCESS;
790
    FOREACH_CNT_X(tenant_id, tenant_ids, OB_SUCC(ret)) {
791
      if (OB_FAIL(check_cancel())) {
792
        LOG_WARN("check_cancel failed", KR(ret));
793
      } else if (OB_TMP_FAIL(check_sys_stat_(*tenant_id))) {
794
        LOG_WARN("fail to check sys stat", KR(tmp_ret), K(*tenant_id));
795
        backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret;
796
      }
797
    } // end foreach
798
    ret = OB_SUCC(ret) ? backup_ret : ret;
799
  }
800
  return ret;
801
}
802

803
int ObRootInspection::check_sys_stat_(const uint64_t tenant_id)
804
{
805
  int ret = OB_SUCCESS;
806
  ObArray<const char *> sys_stat_names;
807
  ObSqlString extra_cond;
808
  ObSysStat sys_stat;
809
  const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id);
810
  if (!inited_) {
811
    ret = OB_NOT_INIT;
812
    LOG_WARN("not init", KR(ret));
813
  } else if (OB_ISNULL(schema_service_)) {
814
    ret = OB_ERR_UNEXPECTED;
815
    LOG_WARN("schema_service is null", KR(ret));
816
  } else if (OB_FAIL(check_cancel())) {
817
    LOG_WARN("check_cancel failed", KR(ret));
818
  } else if (OB_FAIL(check_tenant_status_(tenant_id))) {
819
    LOG_WARN("fail to check tenant status", KR(ret), K(tenant_id));
820
  } else if (OB_FAIL(sys_stat.set_initial_values(tenant_id))) {
821
    LOG_WARN("set initial values failed", KR(ret), K(tenant_id));
822
  } else if (OB_FAIL(extra_cond.assign_fmt("tenant_id = %lu",
823
             ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id)))) {
824
    LOG_WARN("extra_cond assign_fmt failed", KR(ret), K(tenant_id));
825
  } else if (OB_FAIL(get_names(sys_stat.item_list_, sys_stat_names))) {
826
    LOG_WARN("get sys stat names failed", KR(ret), K(tenant_id));
827
  } else if (OB_FAIL(check_names(tenant_id, OB_ALL_SYS_STAT_TNAME, sys_stat_names, extra_cond))) {
828
    LOG_WARN("check all sys stat names failed", K(ret), K(tenant_id),
829
             "table_name", OB_ALL_SYS_STAT_TNAME, K(sys_stat_names));
830
  }
831
  return ret;
832
}
833

834
int ObRootInspection::check_sys_param_()
835
{
836
  int ret = OB_SUCCESS;
837
  ObArray<uint64_t> tenant_ids;
838
  if (!inited_) {
839
    ret = OB_NOT_INIT;
840
    LOG_WARN("not init", KR(ret));
841
  } else if (OB_ISNULL(schema_service_)) {
842
    ret = OB_ERR_UNEXPECTED;
843
    LOG_WARN("schema_service is null", KR(ret));
844
  } else if (OB_FAIL(check_cancel())) {
845
    LOG_WARN("check_cancel failed", KR(ret));
846
  } else if (OB_FAIL(construct_tenant_ids_(tenant_ids))) {
847
    LOG_WARN("get_tenant_ids failed", KR(ret));
848
  } else {
849
    int backup_ret = OB_SUCCESS;
850
    int tmp_ret = OB_SUCCESS;
851
    FOREACH_CNT_X(tenant_id, tenant_ids, OB_SUCC(ret)) {
852
      if (OB_FAIL(check_cancel())) {
853
        LOG_WARN("check_cancel failed", K(ret));
854
      } else if (OB_TMP_FAIL(check_sys_param_(*tenant_id))) {
855
        LOG_WARN("fail to check sys param", KR(tmp_ret), K(*tenant_id));
856
        backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret;
857
      }
858
    } // end foreach
859
    ret = OB_SUCC(ret) ? backup_ret : ret;
860
  }
861
  return ret;
862
}
863

864
int ObRootInspection::check_sys_param_(const uint64_t tenant_id)
865
{
866
  int ret = OB_SUCCESS;
867
  ObArray<uint64_t> tenant_ids;
868
  ObArray<const char *> sys_param_names;
869
  ObSqlString extra_cond;
870
  const uint64_t exec_tenant_id = ObSchemaUtils::get_exec_tenant_id(tenant_id);
871
  if (!inited_) {
872
    ret = OB_NOT_INIT;
873
    LOG_WARN("not init", KR(ret));
874
  } else if (OB_ISNULL(schema_service_)) {
875
    ret = OB_ERR_UNEXPECTED;
876
    LOG_WARN("schema_service is null", KR(ret));
877
  } else if (OB_FAIL(check_cancel())) {
878
    LOG_WARN("check_cancel failed", KR(ret));
879
  } else if (OB_FAIL(check_tenant_status_(tenant_id))) {
880
    LOG_WARN("fail to check tenant status", KR(ret), K(tenant_id));
881
  } else if (OB_FAIL(extra_cond.assign_fmt("tenant_id = %lu",
882
             ObSchemaUtils::get_extract_tenant_id(exec_tenant_id, tenant_id)))) {
883
    LOG_WARN("extra_cond assign_fmt failed", KR(ret), K(tenant_id));
884
  } else if (OB_FAIL(get_sys_param_names(sys_param_names))) {
885
    LOG_WARN("get sys param names failed", KR(ret), K(tenant_id));
886
  } else if (OB_FAIL(check_names(tenant_id, OB_ALL_SYS_VARIABLE_TNAME,
887
             sys_param_names, extra_cond))) {
888
    LOG_WARN("check all sys params names failed", KR(ret), K(tenant_id),
889
             "table_name", OB_ALL_SYS_VARIABLE_TNAME, K(sys_param_names), K(extra_cond));
890
  }
891
  if (OB_SCHEMA_ERROR != ret) {
892
  } else if (GCONF.in_upgrade_mode()) {
893
    LOG_WARN("check sys_variable failed", KR(ret));
894
  } else {
895
    LOG_DBA_ERROR(OB_ERR_ROOT_INSPECTION, "msg", "system variables are unmatched", KR(ret));
896
  }
897
  return ret;
898
}
899

900
template<typename Item>
901
int ObRootInspection::get_names(const ObDList<Item> &list, ObIArray<const char*> &names)
902
{
903
  int ret = OB_SUCCESS;
904
  if (!inited_) {
905
    ret = OB_NOT_INIT;
906
    LOG_WARN("not init", K(ret));
907
  } else if (list.get_size() <= 0) {
908
    ret = OB_INVALID_ARGUMENT;
909
    LOG_WARN("list is empty", K(ret));
910
  } else {
911
    const Item *it = list.get_first();
912
    while (OB_SUCCESS == ret && it != list.get_header()) {
913
      if (OB_FAIL(names.push_back(it->name_))) {
914
        LOG_WARN("push_back failed", K(ret));
915
      } else {
916
        it = it->get_next();
917
      }
918
    }
919
  }
920
  return ret;
921
}
922

923
int ObRootInspection::get_sys_param_names(ObIArray<const char *> &names)
924
{
925
  int ret = OB_SUCCESS;
926
  if (!inited_) {
927
    ret = OB_NOT_INIT;
928
    LOG_WARN("not init", K(ret));
929
  } else {
930
    const int64_t param_count = ObSysVariables::get_amount();
931
    for (int64_t i = 0; OB_SUCC(ret) && i < param_count; ++i) {
932
      if (OB_FAIL(names.push_back(ObSysVariables::get_name(i).ptr()))) {
933
        LOG_WARN("push_back failed", K(ret));
934
      }
935
    }
936
  }
937
  return ret;
938
}
939

940
int ObRootInspection::check_names(const uint64_t tenant_id,
941
                                  const char *table_name,
942
                                  const ObIArray<const char *> &names,
943
                                  const ObSqlString &extra_cond)
944
{
945
  int ret = OB_SUCCESS;
946
  if (!inited_) {
947
    ret = OB_NOT_INIT;
948
    LOG_WARN("not init", K(ret));
949
  } else if (OB_FAIL(check_cancel())) {
950
    LOG_WARN("check_cancel failed", K(ret));
951
  } else if (NULL == table_name || names.count() <= 0) {
952
    // extra_cond can be empty, so wo don't check it here
953
    ret = OB_INVALID_ARGUMENT;
954
    LOG_WARN("table_name is null or names is empty", KP(table_name), K(names), K(ret));
955
  } else {
956
    ObArray<Name> fetch_names; // Get the data of the internal table
957
    ObArray<Name> extra_names; // data inner table more than hard code
958
    ObArray<Name> miss_names; // data inner table less than hard code
959
    if (OB_FAIL(calc_diff_names(tenant_id, table_name, names, extra_cond,
960
        fetch_names, extra_names, miss_names))) {
961
      LOG_WARN("fail to calc diff names", K(ret), KP(table_name), K(names), K(extra_cond));
962
    } else {
963
      if (fetch_names.count() <= 0) {
964
        // don't need to set ret
965
        LOG_WARN("maybe tenant or zone has been deleted, ignore it",
966
                 K(tenant_id), K(table_name), K(extra_cond));
967
      } else {
968
        if (extra_names.count() > 0) {
969
          // don't need to set ret
970
          LOG_WARN("some item exist in table, but not hard coded",
971
                   K(tenant_id), K(table_name), K(extra_names));
972
        }
973
        if (miss_names.count() > 0) {
974
          ret = OB_ENTRY_NOT_EXIST;
975
          LOG_WARN("some item exist in hard code, but not exist in inner table",
976
                   K(ret), K(tenant_id), K(table_name), K(miss_names));
977
        }
978
      }
979
    }
980
  }
981
  return ret;
982
}
983

984
int ObRootInspection::calc_diff_names(const uint64_t tenant_id,
985
                                      const char *table_name,
986
                                      const ObIArray<const char *> &names,
987
                                      const ObSqlString &extra_cond,
988
                                      ObIArray<Name> &fetch_names, /* data reading from inner table*/
989
                                      ObIArray<Name> &extra_names, /* data inner table more than hard code*/
990
                                      ObIArray<Name> &miss_names /* data inner table less than hard code*/)
991
{
992
  int ret = OB_SUCCESS;
993
  ObRefreshSchemaStatus schema_status;
994
  schema_status.tenant_id_ = tenant_id;
995
  fetch_names.reset();
996
  if (!inited_) {
997
    ret = OB_NOT_INIT;
998
    LOG_WARN("not init", K(ret));
999
  } else if (OB_INVALID_TENANT_ID == tenant_id) {
1000
    ret = OB_INVALID_ARGUMENT;
1001
    LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id));
1002
  } else if (OB_FAIL(check_cancel())) {
1003
    LOG_WARN("check_cancel failed", KR(ret), K(tenant_id));
1004
  } else if (NULL == table_name || names.count() <= 0) {
1005
    // extra_cond can be empty, don't need to check it
1006
    ret = OB_INVALID_ARGUMENT;
1007
    LOG_WARN("table_name is null or names is empty",
1008
             KR(ret), K(tenant_id), KP(table_name), K(names));
1009
  } else if (GCTX.is_standby_cluster() && is_user_tenant(tenant_id)) {
1010
    if (OB_ISNULL(GCTX.schema_status_proxy_)) {
1011
      ret = OB_ERR_UNEXPECTED;
1012
      LOG_WARN("schema status proxy is null", K(ret));
1013
    } else if (OB_FAIL(GCTX.schema_status_proxy_->get_refresh_schema_status(tenant_id, schema_status))) {
1014
      LOG_WARN("fail to get schema status", KR(ret), K(tenant_id));
1015
    }
1016
  }
1017

1018
  if (OB_SUCC(ret)) {
1019
    const uint64_t exec_tenant_id = schema_status.tenant_id_;
1020
    int64_t snapshot_timestamp = schema_status.snapshot_timestamp_;
1021
    ObSqlString sql;
1022
    ObSQLClientRetryWeak sql_client_retry_weak(sql_proxy_,
1023
                                               snapshot_timestamp);
1024
    if (OB_FAIL(sql.append_fmt("SELECT name FROM %s%s%s", table_name,
1025
        (extra_cond.empty()) ? "" : " WHERE ", extra_cond.ptr()))) {
1026
      LOG_WARN("append_fmt failed", KR(ret), K(tenant_id), K(table_name), K(extra_cond));
1027
    } else {
1028
      SMART_VAR(ObMySQLProxy::MySQLResult, res) {
1029
        ObMySQLResult *result = NULL;
1030
        if (OB_FAIL(sql_client_retry_weak.read(res, exec_tenant_id, sql.ptr()))) {
1031
          LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(sql));
1032
          can_retry_ = true;
1033
        } else if (OB_ISNULL(result = res.get_result())) {
1034
          ret = OB_ERR_UNEXPECTED;
1035
          LOG_WARN("result is not expected to be NULL",
1036
                   KR(ret), K(tenant_id), "result", OB_P(result));
1037
        } else {
1038
          //only for filling the out parameter,
1039
          //Ensure that there is no '\ 0' character in the middle of the corresponding string
1040
          int64_t tmp_real_str_len = 0;
1041
          Name name;
1042
          while (OB_SUCC(ret)) {
1043
            if (OB_FAIL(result->next())) {
1044
              if (OB_ITER_END == ret) {
1045
                ret = OB_SUCCESS;
1046
                break;
1047
              } else {
1048
                LOG_WARN("get next result failed", KR(ret), K(tenant_id));
1049
              }
1050
            } else {
1051
              EXTRACT_STRBUF_FIELD_MYSQL(*result, "name", name.ptr(),
1052
                  static_cast<int64_t>(NAME_BUF_LEN), tmp_real_str_len);
1053
              (void) tmp_real_str_len; // make compiler happy
1054
              if (OB_FAIL(fetch_names.push_back(name))) {
1055
                LOG_WARN("push_back failed", KR(ret), K(tenant_id));
1056
              }
1057
            }
1058
          }
1059
        }
1060
      }
1061
    }
1062
    if (OB_SUCC(ret)) {
1063
      if (fetch_names.count() <= 0) {
1064
        LOG_WARN("maybe tenant or zone has been deleted, ignore it",
1065
                 KR(ret), K(schema_status), K(table_name), K(extra_cond));
1066
      } else {
1067
        extra_names.reset();
1068
        miss_names.reset();
1069
        FOREACH_CNT_X(fetch_name, fetch_names, OB_SUCCESS == ret) {
1070
          bool found = false;
1071
          FOREACH_CNT_X(name, names, OB_SUCC(ret)) {
1072
            if (Name(*name) == *fetch_name) {
1073
              found = true;
1074
              break;
1075
            }
1076
          }
1077
          if (!found) {
1078
            if (OB_FAIL(extra_names.push_back(*fetch_name))) {
1079
              LOG_WARN("fail to push name into fetch_names",
1080
                       KR(ret), K(tenant_id), K(*fetch_name), K(fetch_names));
1081
            }
1082
          }
1083
        }
1084
        FOREACH_CNT_X(name, names, OB_SUCCESS == ret) {
1085
          bool found = false;
1086
          FOREACH_CNT_X(fetch_name, fetch_names, OB_SUCCESS == ret) {
1087
            if (Name(*name) == *fetch_name) {
1088
              found = true;
1089
              break;
1090
            }
1091
          }
1092
          if (!found) {
1093
            if (OB_FAIL(miss_names.push_back(Name(*name)))) {
1094
              LOG_WARN("fail to push name into miss_names",
1095
                       KR(ret), K(tenant_id), K(*name), K(miss_names));
1096
            }
1097
          }
1098
        }
1099
      }
1100
    }
1101
  }
1102
  return ret;
1103
}
1104

1105
int ObRootInspection::check_sys_table_schemas_()
1106
{
1107
  int ret = OB_SUCCESS;
1108
  ObArray<uint64_t> tenant_ids;
1109
  if (OB_UNLIKELY(!inited_)) {
1110
    ret = OB_NOT_INIT;
1111
    LOG_WARN("not init", KR(ret));
1112
  } else if (OB_FAIL(check_cancel())) {
1113
    LOG_WARN("check_cancel failed", KR(ret));
1114
  } else if (OB_ISNULL(schema_service_)) {
1115
    ret = OB_ERR_UNEXPECTED;
1116
    LOG_WARN("schema_service is null", KR(ret));
1117
  } else if (OB_FAIL(construct_tenant_ids_(tenant_ids))) {
1118
    LOG_WARN("get_tenant_ids failed", KR(ret));
1119
  } else {
1120
    int backup_ret = OB_SUCCESS;
1121
    int tmp_ret = OB_SUCCESS;
1122
    FOREACH_X(tenant_id, tenant_ids, OB_SUCC(ret)) {
1123
      if (OB_FAIL(check_cancel())) {
1124
        LOG_WARN("check_cancel failed", KR(ret));
1125
      } else if (OB_TMP_FAIL(check_sys_table_schemas_(*tenant_id))) {
1126
        LOG_WARN("fail to check sys table schemas by tenant", KR(tmp_ret), K(*tenant_id));
1127
        backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret;
1128
      }
1129
    } // end foreach
1130
    ret = OB_SUCC(ret) ? backup_ret : ret;
1131
  }
1132
  return ret;
1133
}
1134

1135
int ObRootInspection::check_sys_table_schemas_(
1136
    const uint64_t tenant_id)
1137
{
1138
  int ret = OB_SUCCESS;
1139
  int64_t schema_version = OB_INVALID_VERSION;
1140
  if (OB_UNLIKELY(!inited_)) {
1141
    ret = OB_NOT_INIT;
1142
    LOG_WARN("not init", KR(ret));
1143
  } else if (OB_FAIL(check_cancel())) {
1144
    LOG_WARN("check_cancel failed", KR(ret));
1145
  } else if (OB_UNLIKELY(
1146
             is_virtual_tenant_id(tenant_id)
1147
             || OB_INVALID_TENANT_ID == tenant_id)) {
1148
    ret = OB_INVALID_ARGUMENT;
1149
    LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id));
1150
  } else {
1151
    const schema_create_func *creator_ptr_array[] = {
1152
      share::all_core_table_schema_creator,
1153
      share::core_table_schema_creators,
1154
      share::sys_table_schema_creators,
1155
      share::virtual_table_schema_creators,
1156
      share::sys_view_schema_creators,
1157
      share::core_index_table_schema_creators,
1158
      share::sys_index_table_schema_creators,
1159
      NULL };
1160

1161
    int back_ret = OB_SUCCESS;
1162
    int tmp_ret = OB_SUCCESS;
1163
    ObTableSchema table_schema;
1164
    bool exist = false;
1165
    for (const schema_create_func **creator_ptr_ptr = creator_ptr_array;
1166
         OB_SUCC(ret) && OB_NOT_NULL(*creator_ptr_ptr); ++creator_ptr_ptr) {
1167
      for (const schema_create_func *creator_ptr = *creator_ptr_ptr;
1168
           OB_SUCC(ret) && OB_NOT_NULL(*creator_ptr); ++creator_ptr) {
1169
        table_schema.reset();
1170
        if (OB_FAIL(check_cancel())) {
1171
          LOG_WARN("check_cancel failed", KR(ret));
1172
        } else if (OB_FAIL(check_tenant_status_(tenant_id))) {
1173
          LOG_WARN("fail to check tenant status", KR(ret), K(tenant_id));
1174
        } else if (OB_FAIL((*creator_ptr)(table_schema))) {
1175
          LOG_WARN("create table schema failed", KR(ret));
1176
        } else if (!is_sys_tenant(tenant_id)
1177
                   && OB_FAIL(ObSchemaUtils::construct_tenant_space_full_table(
1178
                              tenant_id, table_schema))) {
1179
          LOG_WARN("fail to construct tenant space table", KR(ret), K(tenant_id));
1180
        } else if (OB_FAIL(ObSysTableChecker::is_inner_table_exist(
1181
                   tenant_id, table_schema, exist))) {
1182
          LOG_WARN("fail to check inner table exist",
1183
                   KR(ret), K(tenant_id), K(table_schema));
1184
        } else if (!exist) {
1185
          // skip
1186
        } else {
1187
          if (OB_TMP_FAIL(check_table_schema(tenant_id, table_schema))) {
1188
            // don't print table_schema, otherwise log will be too much
1189
            LOG_WARN("check table schema failed", KR(tmp_ret), K(tenant_id),
1190
                     "table_id", table_schema.get_table_id(), "table_name", table_schema.get_table_name());
1191
            back_ret = OB_SUCCESS == back_ret ? tmp_ret : back_ret;
1192
          }
1193

1194
          if (OB_TMP_FAIL(check_sys_view_(tenant_id, table_schema))) {
1195
            LOG_WARN("check sys view failed", KR(tmp_ret), K(tenant_id),
1196
                     "table_id", table_schema.get_table_id(), "table_name", table_schema.get_table_name());
1197
            back_ret = OB_SUCCESS == back_ret ? tmp_ret : back_ret;
1198
            // sql may has occur other error except OB_SCHEMA_ERROR, we should not continue is such situation.
1199
            if (OB_SCHEMA_ERROR != tmp_ret) {
1200
              ret = OB_SUCC(ret) ? back_ret : tmp_ret;
1201
            }
1202
          }
1203
        }
1204
      } // end for
1205
    } // end for
1206
    ret = OB_SUCC(ret) ? back_ret : ret;
1207
  }
1208
  if (OB_SCHEMA_ERROR != ret) {
1209
  } else if (GCONF.in_upgrade_mode()) {
1210
    LOG_WARN("check sys table schema failed", KR(ret), K(tenant_id));
1211
  } else {
1212
    LOG_ERROR("check sys table schema failed", KR(ret), K(tenant_id));
1213
    LOG_DBA_ERROR(OB_ERR_ROOT_INSPECTION, "msg", "inner tables are unmatched", KR(ret), K(tenant_id));
1214
  }
1215
  return ret;
1216
}
1217

1218
int ObRootInspection::check_table_schema(
1219
    const uint64_t tenant_id,
1220
    const ObTableSchema &hard_code_table)
1221
{
1222
  int ret = OB_SUCCESS;
1223
  const ObTableSchema *table = NULL;
1224
  ObSchemaGetterGuard schema_guard;
1225
  if (OB_UNLIKELY(!inited_)) {
1226
    ret = OB_NOT_INIT;
1227
    LOG_WARN("not init", KR(ret));
1228
  } else if (OB_UNLIKELY(
1229
             is_virtual_tenant_id(tenant_id)
1230
             || OB_INVALID_TENANT_ID == tenant_id)) {
1231
    ret = OB_INVALID_ARGUMENT;
1232
    LOG_WARN("invalid tenant_id", KR(ret), K(tenant_id));
1233
  } else if (OB_UNLIKELY(!hard_code_table.is_valid())) {
1234
    ret = OB_INVALID_ARGUMENT;
1235
    LOG_WARN("invalid table_schema", KR(ret), K(tenant_id), K(hard_code_table));
1236
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(tenant_id, schema_guard))) {
1237
    LOG_WARN("failed to get schema guard", KR(ret), K(tenant_id));
1238
  } else if (OB_FAIL(schema_guard.get_table_schema(
1239
             tenant_id, hard_code_table.get_table_id(), table))) {
1240
    LOG_WARN("get_table_schema failed", KR(ret), K(tenant_id),
1241
             "table_id", hard_code_table.get_table_id(),
1242
             "table_name", hard_code_table.get_table_name());
1243
    // fail may cause by load table schema sql, set retry flag.
1244
    can_retry_ = true;
1245
  } else if (OB_ISNULL(table)) {
1246
    ret = OB_SCHEMA_ERROR;
1247
    LOG_WARN("table should not be null", KR(ret), K(tenant_id),
1248
             "table_id", hard_code_table.get_table_id(),
1249
             "table_name", hard_code_table.get_table_name());
1250
    can_retry_ = true;
1251
  } else if (OB_FAIL(check_table_schema(hard_code_table, *table))) {
1252
    LOG_WARN("fail to check table schema", KR(ret), K(tenant_id), K(hard_code_table), KPC(table));
1253
  }
1254
  return ret;
1255
}
1256

1257
int ObRootInspection::check_table_schema(const ObTableSchema &hard_code_table,
1258
                                         const ObTableSchema &inner_table)
1259
{
1260
  int ret = OB_SUCCESS;
1261
  if (!hard_code_table.is_valid()
1262
      || !inner_table.is_valid()) {
1263
    ret = OB_INVALID_ARGUMENT;
1264
    LOG_WARN("invalid table_schema", K(hard_code_table), K(inner_table), K(ret));
1265
  } else if (OB_FAIL(check_table_options_(inner_table, hard_code_table))) {
1266
    LOG_WARN("check_table_options failed", "table_id", hard_code_table.get_table_id(), K(ret));
1267
  } else if (!inner_table.is_view_table()) { //view table do not check column info
1268
    if (hard_code_table.get_column_count() != inner_table.get_column_count()) {
1269
      ret = OB_SCHEMA_ERROR;
1270
      LOG_WARN("column count mismatch", "table_id", inner_table.get_table_id(),
1271
          "table_name",inner_table.get_table_name(), "table_column_cnt",inner_table.get_column_count(),
1272
          "hard_code_table_column_cnt", hard_code_table.get_column_count(), K(ret));
1273
    } else {
1274
      int back_ret = OB_SUCCESS;
1275
      for (int64_t i = 0; OB_SUCC(ret) && i < hard_code_table.get_column_count(); ++i) {
1276
        const ObColumnSchemaV2 *hard_code_column = hard_code_table.get_column_schema_by_idx(i);
1277
        const ObColumnSchemaV2 *column = NULL;
1278
        if (NULL == hard_code_column) {
1279
          ret = OB_SCHEMA_ERROR;
1280
          LOG_WARN("hard_code_column is null", "hard_code_column", OB_P(hard_code_column), K(ret));
1281
        } else if (NULL == (column = inner_table.get_column_schema(
1282
            hard_code_column->get_column_name()))) {
1283
          ret = OB_SCHEMA_ERROR;
1284
          LOG_WARN("hard code column not found", "table_id", hard_code_table.get_table_id(),
1285
              "table_name", hard_code_table.get_table_name(), "column",
1286
              hard_code_column->get_column_name(), K(ret));
1287
        } else {
1288
          const bool ignore_column_id = is_virtual_table(hard_code_table.get_table_id());
1289
          if (OB_FAIL(check_column_schema_(hard_code_table.get_table_name(),
1290
              *column, *hard_code_column, ignore_column_id))) {
1291
            LOG_WARN("column schema mismatch with hard code column schema",
1292
                "table_name",inner_table.get_table_name(), "column", *column,
1293
                "hard_code_column", *hard_code_column, K(ret));
1294
          }
1295
        }
1296
        back_ret = OB_SUCCESS == back_ret ? ret : back_ret;
1297
        ret = OB_SUCCESS;
1298
      }
1299
      ret = back_ret;
1300
    }
1301
  }
1302
  return ret;
1303
}
1304

1305
int ObRootInspection::check_and_get_system_table_column_diff(
1306
    const share::schema::ObTableSchema &table_schema,
1307
    const share::schema::ObTableSchema &hard_code_schema,
1308
    common::ObIArray<uint64_t> &add_column_ids,
1309
    common::ObIArray<uint64_t> &alter_column_ids)
1310
{
1311
  int ret = OB_SUCCESS;
1312
  add_column_ids.reset();
1313
  alter_column_ids.reset();
1314
  if (!table_schema.is_valid() || !hard_code_schema.is_valid()) {
1315
    ret = OB_INVALID_ARGUMENT;
1316
    LOG_WARN("invalid table_schema", KR(ret), K(table_schema), K(hard_code_schema));
1317
  } else if (table_schema.get_tenant_id() != hard_code_schema.get_tenant_id()
1318
             || table_schema.get_table_id() != hard_code_schema.get_table_id()
1319
             || 0 != table_schema.get_table_name_str().compare(hard_code_schema.get_table_name_str())
1320
             || !is_system_table(table_schema.get_table_id())) {
1321
    ret = OB_INVALID_ARGUMENT;
1322
    LOG_WARN("invalid table_schema", KR(ret),
1323
             "tenant_id", table_schema.get_tenant_id(),
1324
             "table_id", table_schema.get_table_id(),
1325
             "table_name", table_schema.get_table_name(),
1326
             "hard_code_tenant_id", hard_code_schema.get_tenant_id(),
1327
             "hard_code_table_id", hard_code_schema.get_table_id(),
1328
             "hard_code_table_name", hard_code_schema.get_table_name());
1329
  } else {
1330
    const uint64_t tenant_id = table_schema.get_tenant_id();
1331
    const uint64_t table_id = table_schema.get_table_id();
1332
    const ObColumnSchemaV2 *column = NULL;
1333
    const ObColumnSchemaV2 *hard_code_column = NULL;
1334
    ObColumnSchemaV2 tmp_column; // check_column_can_be_altered_online() may change dst_column, is ugly.
1335
    bool ignore_column_id = false;
1336

1337
    // case 1. check if columns should be dropped.
1338
    // case 2. check if column can be altered online.
1339
    for (int64_t i = 0; OB_SUCC(ret) && i < table_schema.get_column_count(); i++) {
1340
      column = table_schema.get_column_schema_by_idx(i);
1341
      if (OB_ISNULL(column)) {
1342
        ret = OB_ERR_UNEXPECTED;
1343
        LOG_WARN("column schema is null", KR(ret), K(tenant_id), K(table_id), K(i));
1344
      } else if (OB_ISNULL(hard_code_column = hard_code_schema.get_column_schema(column->get_column_id()))) {
1345
        ret = OB_NOT_SUPPORTED; // case 1
1346
        LOG_WARN("can't drop system table's column", KR(ret),
1347
                 K(tenant_id), K(table_id),
1348
                 "table_name", table_schema.get_table_name(),
1349
                 "column_id", column->get_column_id(),
1350
                 "column_name", column->get_column_name());
1351
      } else {
1352
        // case 2
1353
        int tmp_ret = check_column_schema_(table_schema.get_table_name_str(),
1354
                                           *column,
1355
                                           *hard_code_column,
1356
                                           ignore_column_id);
1357
        if (OB_SUCCESS == tmp_ret) {
1358
          // not changed
1359
        } else if (OB_SCHEMA_ERROR != tmp_ret) {
1360
          ret = tmp_ret;
1361
          LOG_WARN("fail to check column schema", KR(ret),
1362
                   K(tenant_id), K(table_id), KPC(column), KPC(hard_code_column));
1363
        } else if (OB_FAIL(tmp_column.assign(*hard_code_column))) {
1364
          LOG_WARN("fail to assign hard code column schema", KR(ret),
1365
                   K(tenant_id), K(table_id),  "column_id", hard_code_column->get_column_id());
1366
        } else if (OB_FAIL(table_schema.check_column_can_be_altered_online(column, &tmp_column))) {
1367
          LOG_WARN("fail to check alter column online", KR(ret),
1368
                   K(tenant_id), K(table_id),
1369
                   "table_name", table_schema.get_table_name(),
1370
                   "column_id", column->get_column_id(),
1371
                   "column_name", column->get_column_name());
1372
        } else if (OB_FAIL(alter_column_ids.push_back(column->get_column_id()))) {
1373
          LOG_WARN("fail to push back column_id", KR(ret), K(tenant_id), K(table_id),
1374
                   "column_id", column->get_column_id());
1375
        }
1376
      }
1377
    } // end for
1378

1379
    // case 3: check if columns should be added.
1380
    for (int64_t i = 0; OB_SUCC(ret) && i < hard_code_schema.get_column_count(); i++) {
1381
      hard_code_column = hard_code_schema.get_column_schema_by_idx(i);
1382
      if (OB_ISNULL(hard_code_column)) {
1383
        ret = OB_ERR_UNEXPECTED;
1384
        LOG_WARN("column schema is null", KR(ret), K(tenant_id), K(table_id), K(i));
1385
      } else if (OB_NOT_NULL(column = table_schema.get_column_schema(hard_code_column->get_column_id()))) {
1386
        // column exist, just skip
1387
      } else {
1388
        const uint64_t hard_code_column_id = hard_code_column->get_column_id();
1389
        const ObColumnSchemaV2 *last_column = NULL;
1390
        if (table_schema.get_column_count() <= 0
1391
            || OB_ISNULL(last_column = table_schema.get_column_schema_by_idx(
1392
                         table_schema.get_column_count() - 1))) {
1393
          ret = OB_ERR_UNEXPECTED;
1394
          LOG_WARN("invalid column count or column", KR(ret), K(table_schema));
1395
        } else if (table_schema.get_max_used_column_id() >= hard_code_column_id
1396
                  || last_column->get_column_id() >= hard_code_column_id) {
1397
          ret = OB_NOT_SUPPORTED;
1398
          LOG_WARN("column should be added at last", KR(ret), KPC(hard_code_column), K(table_schema));
1399
        } else if (OB_FAIL(add_column_ids.push_back(hard_code_column_id))) {
1400
          LOG_WARN("fail to push back column_id", KR(ret), K(tenant_id), K(table_id),
1401
                   "column_id", hard_code_column_id);
1402
        }
1403
      }
1404
    } // end for
1405
  }
1406
  return ret;
1407
}
1408

1409
bool ObRootInspection::check_str_with_lower_case_(const ObString &str)
1410
{
1411
  bool bret = false;
1412
  if (str.length() > 0) {
1413
    for (int64_t i = 0; !bret && i < str.length(); i++) {
1414
      if (str.ptr()[i] >= 'a' && str.ptr()[i] <= 'z') {
1415
        bret = true;
1416
      }
1417
    }
1418
  }
1419
  return bret;
1420
}
1421

1422
int ObRootInspection::check_sys_view_(
1423
    const uint64_t tenant_id,
1424
    const ObTableSchema &hard_code_table)
1425
{
1426
  int ret = OB_SUCCESS;
1427
  if (OB_ISNULL(GCTX.root_service_)) {
1428
    ret = OB_NOT_INIT;
1429
    LOG_WARN("not init", KR(ret));
1430
  } else if (hard_code_table.is_view_table()) {
1431
    // check view definition
1432
    const ObString &table_name = hard_code_table.get_table_name();
1433
    const uint64_t table_id = hard_code_table.get_table_id();
1434
    const uint64_t database_id = hard_code_table.get_database_id();
1435
    bool is_oracle = is_oracle_sys_database_id(database_id);
1436
    bool check_lower_case = !is_mysql_database_id(database_id);
1437
    SMART_VAR(ObMySQLProxy::MySQLResult, res) {
1438
      common::sqlclient::ObMySQLResult *result = NULL;
1439
      ObSqlString sql;
1440
      // case 0: check expansion of sys view definition
1441
      if (is_oracle) {
1442
        if (OB_FAIL(sql.assign_fmt("SELECT FIELD FROM \"%s\".\"%s\" WHERE TABLE_ID = %lu",
1443
                                   OB_ORA_SYS_SCHEMA_NAME,
1444
                                   OB_TENANT_VIRTUAL_TABLE_COLUMN_ORA_TNAME,
1445
                                   table_id))) {
1446
          LOG_WARN("failed to assign sql", KR(ret), K(sql));
1447
        } else if (OB_FAIL(GCTX.root_service_->get_oracle_sql_proxy().read(res, tenant_id, sql.ptr()))) {
1448
          LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(table_name), K(sql));
1449
        }
1450
      } else {
1451
        if (OB_FAIL(sql.assign_fmt("SELECT FIELD FROM `%s`.`%s` WHERE TABLE_ID = %lu",
1452
                                   OB_SYS_DATABASE_NAME,
1453
                                   OB_TENANT_VIRTUAL_TABLE_COLUMN_TNAME,
1454
                                   table_id))) {
1455
          LOG_WARN("failed to assign sql", KR(ret), K(sql));
1456
        } else if (!is_oracle && OB_FAIL(GCTX.root_service_->get_sql_proxy().read(res, tenant_id, sql.ptr()))) {
1457
          LOG_WARN("execute sql failed", KR(ret), K(tenant_id), K(table_name), K(sql));
1458
        }
1459
      }
1460
      if (OB_FAIL(ret)) {
1461
        if (OB_ERR_VIEW_INVALID == ret) {
1462
          ret = OB_SCHEMA_ERROR;
1463
          LOG_ERROR("check sys view: expand failed", KR(ret), K(tenant_id), K(table_name));
1464
        } else {
1465
          LOG_WARN("check sys view: expand failed", KR(ret), K(tenant_id), K(table_name));
1466
        }
1467
      } else if (OB_ISNULL(result = res.get_result())) {
1468
        ret = OB_ERR_UNEXPECTED;
1469
        LOG_WARN("failed to get sql result", KR(ret), K(tenant_id));
1470
      } else if (check_lower_case) {
1471
        // case 1: check column name with lower case
1472
        ObString col_name;
1473
        while (OB_SUCC(ret) && OB_SUCC(result->next())) {
1474
          if (OB_FAIL(result->get_varchar(0L, col_name))) {
1475
            LOG_WARN("fail to get filed", KR(ret), K(tenant_id), K(table_name));
1476
          } else if (check_str_with_lower_case_(col_name)) {
1477
            ret = OB_SCHEMA_ERROR;
1478
            LOG_ERROR("check sys view: column name should be uppercase",
1479
                      KR(ret), K(tenant_id), K(table_name), K(col_name));
1480
          }
1481
        } // end while
1482
        if (OB_ITER_END == ret) {
1483
          ret = OB_SUCCESS;
1484
        } else {
1485
          ret = OB_SUCC(ret) ? OB_ERR_UNEXPECTED : ret;
1486
          LOG_WARN("iterate failed", KR(ret));
1487
        }
1488
        // case 2: check view name with lower case
1489
        if (OB_SUCC(ret) && check_str_with_lower_case_(hard_code_table.get_table_name())) {
1490
          ret = OB_SCHEMA_ERROR;
1491
          LOG_ERROR("check sys view: table name should be uppercase", KR(ret), K(tenant_id), K(table_name));
1492
        }
1493
      }
1494
    }
1495
  }
1496
  return ret;
1497
}
1498

1499
int ObRootInspection::check_table_options_(const ObTableSchema &table,
1500
                                           const ObTableSchema &hard_code_table)
1501
{
1502
  int ret = OB_SUCCESS;
1503
  if (!table.is_valid() || !hard_code_table.is_valid()) {
1504
    ret = OB_INVALID_ARGUMENT;
1505
    LOG_WARN("invalid table or invalid hard_code_table", K(table), K(hard_code_table), K(ret));
1506
  } else if (table.get_table_id() != hard_code_table.get_table_id()) {
1507
    ret = OB_SCHEMA_ERROR;
1508
    LOG_WARN("table id not match", "table_id", table.get_table_id(),
1509
        "hard_code table_id", hard_code_table.get_table_id(), K(ret));
1510
  } else if (table.get_table_name_str() != hard_code_table.get_table_name_str()) {
1511
    ret = OB_SCHEMA_ERROR;
1512
    LOG_WARN("table name mismatch with hard code table",
1513
        "table_id", table.get_table_id(), "table_name", table.get_table_name(),
1514
        "hard_code_table name", hard_code_table.get_table_name(), K(ret));
1515
  } else {
1516
    const ObString &table_name = table.get_table_name_str();
1517

1518
    if (table.get_tenant_id() != hard_code_table.get_tenant_id()) {
1519
      ret = OB_SCHEMA_ERROR;
1520
      LOG_WARN("tenant_id mismatch", K(table_name), "in_memory", table.get_tenant_id(),
1521
          "hard_code", hard_code_table.get_tenant_id(), K(ret));
1522
    } else if (table.get_database_id() != hard_code_table.get_database_id()) {
1523
      ret = OB_SCHEMA_ERROR;
1524
      LOG_WARN("database_id mismatch", K(table_name), "in_memory", table.get_database_id(),
1525
          "hard_code", hard_code_table.get_database_id(), K(ret));
1526
    } else if (table.get_tablegroup_id() != hard_code_table.get_tablegroup_id()) {
1527
      ret = OB_SCHEMA_ERROR;
1528
      LOG_WARN("tablegroup_id mismatch", K(table_name), "in_memory", table.get_tablegroup_id(),
1529
          "hard_code", hard_code_table.get_tablegroup_id(), K(ret));
1530
    } else if (table.get_auto_increment() != hard_code_table.get_auto_increment()) {
1531
      ret = OB_SCHEMA_ERROR;
1532
      LOG_WARN("auto_increment mismatch", K(table_name), "in_memory", table.get_auto_increment(),
1533
          "hard_code", hard_code_table.get_auto_increment(), K(ret));
1534
    } else if (table.is_read_only() != hard_code_table.is_read_only()) {
1535
      ret = OB_SCHEMA_ERROR;
1536
      LOG_WARN("read_only mismatch", K(table_name), "in_memory", table.is_read_only(),
1537
          "hard code", hard_code_table.is_read_only(), K(ret));
1538
    } else if (table.get_load_type() != hard_code_table.get_load_type()) {
1539
      ret = OB_SCHEMA_ERROR;
1540
      LOG_WARN("load_type mismatch", K(table_name), "in_memory", table.get_load_type(),
1541
          "hard_code", hard_code_table.get_load_type(), K(ret));
1542
    } else if (table.get_table_type() != hard_code_table.get_table_type()) {
1543
      ret = OB_SCHEMA_ERROR;
1544
      LOG_WARN("table_type mismatch", K(table_name), "in_memory", table.get_table_type(),
1545
          "hard_code", hard_code_table.get_table_type(), K(ret));
1546
    } else if (table.get_index_type() != hard_code_table.get_index_type()) {
1547
      ret = OB_SCHEMA_ERROR;
1548
      LOG_WARN("index_type mismatch", K(table_name), "in_memory", table.get_index_type(),
1549
          "hard_code", hard_code_table.get_index_type(), K(ret));
1550
    } else if (table.get_index_using_type() != hard_code_table.get_index_using_type()) {
1551
      ret = OB_SCHEMA_ERROR;
1552
      LOG_WARN("index_using_type mismatch", K(table_name), "in_memory", table.get_index_using_type(),
1553
          "hard_code", hard_code_table.get_index_using_type(), K(ret));
1554
    } else if (table.get_def_type() != hard_code_table.get_def_type()) {
1555
      ret = OB_SCHEMA_ERROR;
1556
      LOG_WARN("def_type mismatch", K(table_name), "in_memory", table.get_def_type(),
1557
          "hard_code", hard_code_table.get_def_type(), K(ret));
1558
    } else if (table.get_data_table_id() != hard_code_table.get_data_table_id()) {
1559
      ret = OB_SCHEMA_ERROR;
1560
      LOG_WARN("data_table_id mismatch", K(table_name), "in_memory", table.get_data_table_id(),
1561
          "hard_code", hard_code_table.get_data_table_id(), K(ret));
1562
    } else if (table.get_tablegroup_name() != hard_code_table.get_tablegroup_name()) {
1563
      ret = OB_SCHEMA_ERROR;
1564
      LOG_WARN("tablegroup_name mismatch", K(table_name), "in_memory", table.get_tablegroup_name(),
1565
          "hard_code", hard_code_table.get_tablegroup_name(), K(ret));
1566
    } else if (table.get_view_schema() != hard_code_table.get_view_schema()) {
1567
      ret = OB_SCHEMA_ERROR;
1568
      LOG_WARN("view_schema mismatch", K(table_name), "in_memory", table.get_view_schema(),
1569
          "hard_code", hard_code_table.get_view_schema(), K(ret));
1570
    } else if (table.get_part_level() != hard_code_table.get_part_level()) {
1571
      ret = OB_SCHEMA_ERROR;
1572
      LOG_WARN("part_level mismatch", K(table_name), "in_memory", table.get_part_level(),
1573
          "hard_code", hard_code_table.get_part_level(), K(ret));
1574
    } else if ((table.get_part_option().get_part_func_expr_str()
1575
        != hard_code_table.get_part_option().get_part_func_expr_str())
1576
        || (table.get_part_option().get_part_func_type()
1577
        != hard_code_table.get_part_option().get_part_func_type())) {
1578
      ret = OB_SCHEMA_ERROR;
1579
      LOG_WARN("part_expr mismatch", K(table_name), "in_memory",
1580
          table.get_part_option(), "hard_code", hard_code_table.get_part_option(), K(ret));
1581
    } else if ((table.get_sub_part_option().get_part_func_expr_str()
1582
        != hard_code_table.get_sub_part_option().get_part_func_expr_str())
1583
        || (table.get_sub_part_option().get_part_func_type()
1584
        != hard_code_table.get_sub_part_option().get_part_func_type())) {
1585
      ret = OB_SCHEMA_ERROR;
1586
      LOG_WARN("sub_part_expr mismatch", K(table_name), "in_memory",
1587
          table.get_sub_part_option(), "hard_code", hard_code_table.get_sub_part_option(), K(ret));
1588
    } else if (table.is_view_table()) {
1589
      // view table do not check column info
1590
    } else if (table.get_max_used_column_id() < hard_code_table.get_max_used_column_id()) {
1591
      ret = OB_SCHEMA_ERROR;
1592
      LOG_WARN("max_used_column_id mismatch", K(table_name), "in_memory",
1593
          table.get_max_used_column_id(), "hard_code",
1594
          hard_code_table.get_max_used_column_id(), K(ret));
1595
    } else if (table.get_rowkey_column_num() != hard_code_table.get_rowkey_column_num()) {
1596
      ret = OB_SCHEMA_ERROR;
1597
      LOG_WARN("rowkey_column_num mismatch", K(table_name), "in_memory",
1598
          table.get_rowkey_column_num(), "hard_code",
1599
          hard_code_table.get_rowkey_column_num(), K(ret));
1600
    } else if (table.get_index_column_num() != hard_code_table.get_index_column_num()) {
1601
      ret = OB_SCHEMA_ERROR;
1602
      LOG_WARN("index_column_num mismatch", K(table_name), "in_memory",
1603
          table.get_index_column_num(), "hard_code",
1604
          hard_code_table.get_index_column_num(), K(ret));
1605
    } else if (table.get_rowkey_split_pos() != hard_code_table.get_rowkey_split_pos()) {
1606
      ret = OB_SCHEMA_ERROR;
1607
      LOG_WARN("rowkey_split_pos mismatch", K(table_name), "in_memory",
1608
          table.get_rowkey_split_pos(), "hard_code",
1609
          hard_code_table.get_rowkey_split_pos(), K(ret));
1610
    } else if (table.get_partition_key_column_num()
1611
        != hard_code_table.get_partition_key_column_num()) {
1612
      ret = OB_SCHEMA_ERROR;
1613
      LOG_WARN("partition_key_column_num mismatch", K(table_name), "in_memory",
1614
          table.get_partition_key_column_num(), "hard_code",
1615
          hard_code_table.get_partition_key_column_num(), K(ret));
1616
    } else if (table.get_subpartition_key_column_num()
1617
        != hard_code_table.get_subpartition_key_column_num()) {
1618
      ret = OB_SCHEMA_ERROR;
1619
      LOG_WARN("partition_key_column_num mismatch", K(table_name), "in_memory",
1620
          table.get_subpartition_key_column_num(), "hard_code",
1621
          hard_code_table.get_subpartition_key_column_num(), K(ret));
1622
    } else if (table.get_autoinc_column_id() != hard_code_table.get_autoinc_column_id()) {
1623
      ret = OB_SCHEMA_ERROR;
1624
      LOG_WARN("autoinc_column_id mismatch", K(table_name), "in_memory",
1625
          table.get_autoinc_column_id(), "hard_code",
1626
          hard_code_table.get_autoinc_column_id(), K(ret));
1627
    }
1628

1629
    // options may be different between different ob instance, don't check
1630
    // block_size
1631
    // is_user_bloomfilter
1632
    // progressive_merge_num
1633
    // replica_num
1634
    // index_status
1635
    // name_case_mode
1636
    // charset_type
1637
    // collation_type
1638
    // schema_version
1639
    // comment
1640
    // compress_func_name
1641
    // expire_info
1642
    // zone_list
1643
    // primary_zone
1644
    // part_expr.part_num_
1645
    // sub_part_expr.part_num_
1646
    // store_format
1647
    // row_store_type
1648
    // progressive_merge_round
1649
    // storage_format_version
1650
  }
1651
  return ret;
1652
}
1653

1654
int ObRootInspection::check_column_schema_(const ObString &table_name,
1655
                                           const ObColumnSchemaV2 &column,
1656
                                           const ObColumnSchemaV2 &hard_code_column,
1657
                                           const bool ignore_column_id)
1658
{
1659
  int ret = OB_SUCCESS;
1660
  if (table_name.empty() || !column.is_valid() || !hard_code_column.is_valid()) {
1661
    ret = OB_INVALID_ARGUMENT;
1662
    LOG_WARN("table_name is empty or invalid column or invalid hard_code_column",
1663
             KR(ret), K(table_name), K(column), K(hard_code_column));
1664
  } else {
1665
#define CMP_COLUMN_ATTR(attr) \
1666
  if (OB_SUCC(ret)) { \
1667
    if (column.get_##attr() != hard_code_column.get_##attr()) { \
1668
      ret = OB_SCHEMA_ERROR; \
1669
      LOG_WARN(#attr " mismatch", KR(ret), K(table_name), "column_name", column.get_column_name(), \
1670
               "in_memory", column.get_##attr(), "hard_code", hard_code_column.get_##attr()); \
1671
    } \
1672
  }
1673

1674
#define CMP_COLUMN_IS_ATTR(attr) \
1675
  if (OB_SUCC(ret)) { \
1676
    if (column.is_##attr() != hard_code_column.is_##attr()) { \
1677
      ret = OB_SCHEMA_ERROR; \
1678
      LOG_WARN(#attr " mismatch", KR(ret), K(table_name), "column_name", column.get_column_name(), \
1679
               "in_memory", column.is_##attr(), "hard_code", hard_code_column.is_##attr()); \
1680
    } \
1681
  }
1682
    if (OB_SUCC(ret)) {
1683
      if (column.get_column_name_str() != hard_code_column.get_column_name_str()) {
1684
        ret = OB_SCHEMA_ERROR;
1685
        LOG_WARN("column_name mismatch", KR(ret), K(table_name),
1686
                 "in_memory", column.get_column_name(),
1687
                 "hard_code", hard_code_column.get_column_name());
1688
      }
1689
    }
1690

1691
    if (!ignore_column_id) {
1692
      CMP_COLUMN_ATTR(column_id);
1693
    }
1694
    CMP_COLUMN_ATTR(tenant_id);
1695
    CMP_COLUMN_ATTR(table_id);
1696
    // don't need to check schema version
1697
    CMP_COLUMN_ATTR(rowkey_position);
1698
    CMP_COLUMN_ATTR(index_position);
1699
    CMP_COLUMN_ATTR(order_in_rowkey);
1700
    CMP_COLUMN_ATTR(tbl_part_key_pos);
1701
    CMP_COLUMN_ATTR(meta_type);
1702
    CMP_COLUMN_ATTR(accuracy);
1703
    CMP_COLUMN_ATTR(data_length);
1704
    CMP_COLUMN_IS_ATTR(nullable);
1705
    CMP_COLUMN_IS_ATTR(zero_fill);
1706
    CMP_COLUMN_IS_ATTR(autoincrement);
1707
    CMP_COLUMN_IS_ATTR(hidden);
1708
    CMP_COLUMN_IS_ATTR(on_update_current_timestamp);
1709
    CMP_COLUMN_ATTR(charset_type);
1710
    // don't need to check orig default value
1711
    if (ObString("row_store_type") == column.get_column_name()
1712
        && (ObString("__all_table") == table_name || ObString("__all_table_history") == table_name)) {
1713
      // row_store_type may have two possible default values
1714
    } else {
1715
      CMP_COLUMN_ATTR(cur_default_value);
1716
    }
1717
    CMP_COLUMN_ATTR(comment);
1718

1719
  }
1720

1721
#undef CMP_COLUMN_IS_ATTR
1722
#undef CMP_COLUMN_INT_ATTR
1723
  return ret;
1724
}
1725

1726
int ObRootInspection::check_data_version_()
1727
{
1728
  int ret = OB_SUCCESS;
1729
  ObArray<uint64_t> tenant_ids;
1730
  if (OB_UNLIKELY(!inited_)) {
1731
    ret = OB_NOT_INIT;
1732
    LOG_WARN("not init", KR(ret));
1733
  } else if (OB_FAIL(check_cancel())) {
1734
    LOG_WARN("check_cancel failed", KR(ret));
1735
  } else if (OB_ISNULL(schema_service_)) {
1736
    ret = OB_ERR_UNEXPECTED;
1737
    LOG_WARN("schema_service is null", KR(ret));
1738
  } else if (OB_FAIL(construct_tenant_ids_(tenant_ids))) {
1739
    LOG_WARN("get_tenant_ids failed", KR(ret));
1740
  } else {
1741
    int backup_ret = OB_SUCCESS;
1742
    int tmp_ret = OB_SUCCESS;
1743
    FOREACH_X(tenant_id, tenant_ids, OB_SUCC(ret)) {
1744
      if (OB_FAIL(check_cancel())) {
1745
        LOG_WARN("check_cancel failed", KR(ret));
1746
      } else if (OB_FAIL(check_tenant_status_(*tenant_id))) {
1747
        LOG_WARN("fail to check tenant status", KR(ret), K(*tenant_id));
1748
      } else if (OB_TMP_FAIL(check_data_version_(*tenant_id))) {
1749
        LOG_WARN("fail to check data version by tenant", KR(tmp_ret), K(*tenant_id));
1750
        backup_ret = OB_SUCCESS == backup_ret ? tmp_ret : backup_ret;
1751
      }
1752
    } // end foreach
1753
    ret = OB_SUCC(ret) ? backup_ret : ret;
1754
  }
1755
  return ret;
1756
}
1757

1758
int ObRootInspection::check_data_version_(
1759
    const uint64_t tenant_id)
1760
{
1761
  int ret = OB_SUCCESS;
1762
  if (OB_UNLIKELY(!inited_)) {
1763
    ret = OB_NOT_INIT;
1764
    LOG_WARN("not init", KR(ret));
1765
  } else if (OB_FAIL(check_cancel())) {
1766
    LOG_WARN("check_cancel failed", KR(ret));
1767
  } else {
1768
    share::ObGlobalStatProxy proxy(*sql_proxy_, tenant_id);
1769
    uint64_t target_data_version = 0;
1770
    uint64_t current_data_version = 0;
1771
    uint64_t compatible_version = 0;
1772
    bool for_update = false;
1773
    if (OB_FAIL(proxy.get_target_data_version(for_update, target_data_version))) {
1774
      LOG_WARN("fail to get target data version", KR(ret), K(tenant_id));
1775
    } else if (OB_FAIL(proxy.get_current_data_version(current_data_version))) {
1776
      LOG_WARN("fail to get current data version", KR(ret), K(tenant_id));
1777
    } else if (OB_FAIL(GET_MIN_DATA_VERSION(tenant_id, compatible_version))) {
1778
      LOG_WARN("fail to get min data version", KR(ret), K(tenant_id));
1779
    } else if (target_data_version != current_data_version
1780
               || target_data_version != compatible_version
1781
               || target_data_version != DATA_CURRENT_VERSION) {
1782
      ret = OB_STATE_NOT_MATCH;
1783
      LOG_WARN("data_version not match, upgrade process should be run",
1784
               KR(ret), K(tenant_id), K(target_data_version),
1785
               K(current_data_version), K(compatible_version));
1786
    }
1787
  }
1788
  return ret;
1789
}
1790

1791
int ObRootInspection::check_cancel()
1792
{
1793
  int ret = OB_SUCCESS;
1794
  if (stopped_) {
1795
    ret = OB_CANCELED;
1796
  } else if (OB_ISNULL(GCTX.root_service_)) {
1797
    ret = OB_ERR_UNEXPECTED;
1798
    LOG_WARN("rootservice is null", KR(ret));
1799
  } else if (!GCTX.root_service_->is_full_service()) {
1800
    ret = OB_CANCELED;
1801
  }
1802
  return ret;
1803
}
1804

1805
int ObRootInspection::check_tenant_status_(const uint64_t tenant_id)
1806
{
1807
  int ret = OB_SUCCESS;
1808
  ObSchemaGetterGuard guard;
1809
  const ObSimpleTenantSchema *tenant = NULL;
1810
  int64_t schema_version = OB_INVALID_VERSION;
1811
  if (OB_ISNULL(schema_service_)) {
1812
    ret = OB_NOT_INIT;
1813
    LOG_WARN("schema service is null", KR(ret));
1814
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, guard))) {
1815
    LOG_WARN("fail to get schema guard", KR(ret));
1816
  } else if (OB_FAIL(guard.get_tenant_info(tenant_id, tenant))) {
1817
    LOG_WARN("fail to get tenant schema", KR(ret), K(tenant_id));
1818
  } else if (OB_ISNULL(tenant)) {
1819
    // tenant may has been dropped;
1820
    ret = OB_EAGAIN;
1821
    LOG_WARN("tenant may be dropped, don't continue", KR(ret), K(tenant_id));
1822
  } else if (!tenant->is_normal()) {
1823
    ret = OB_EAGAIN;
1824
    LOG_WARN("tenant status is not noraml, should check next round", KR(ret), K(tenant_id));
1825
  } else if (OB_FAIL(schema_service_->get_tenant_refreshed_schema_version(tenant_id, schema_version))) {
1826
    LOG_WARN("fail to get tenant schema version", KR(ret), K(tenant_id));
1827
  } else if (!ObSchemaService::is_formal_version(schema_version)) {
1828
    ret = OB_EAGAIN;
1829
    LOG_WARN("schema version is not formal, observer may be restarting or inner table schema changed, "
1830
             "should check next round", KR(ret), K(tenant_id), K(schema_version));
1831
  }
1832
  return ret;
1833
}
1834

1835
ObUpgradeInspection::ObUpgradeInspection()
1836
  : inited_(false), schema_service_(NULL), root_inspection_(NULL)
1837
{
1838
}
1839

1840
ObUpgradeInspection::~ObUpgradeInspection()
1841
{
1842
}
1843

1844
int ObUpgradeInspection::init(ObMultiVersionSchemaService &schema_service,
1845
                              ObRootInspection &root_inspection)
1846
{
1847
  int ret = OB_SUCCESS;
1848
  if (inited_) {
1849
    ret = OB_INIT_TWICE;
1850
    LOG_WARN("init twice", K(ret));
1851
  } else {
1852
    schema_service_ = &schema_service;
1853
    root_inspection_ = &root_inspection;
1854
    inited_ = true;
1855
  }
1856
  return ret;
1857
}
1858

1859
int ObUpgradeInspection::inner_get_next_row(common::ObNewRow *&row)
1860
{
1861
  int ret = OB_SUCCESS;
1862
  ObSchemaGetterGuard schema_guard;
1863
  if (NULL == allocator_) {
1864
    ret = OB_NOT_INIT;
1865
    LOG_WARN("not init, allocator is null", K(ret));
1866
  } else if (!inited_) {
1867
    ret = OB_NOT_INIT;
1868
    LOG_WARN("not init", K(ret));
1869
  } else if (OB_FAIL(schema_service_->get_tenant_schema_guard(OB_SYS_TENANT_ID, schema_guard))) {
1870
    LOG_WARN("get schema guard error", K(ret));
1871
  } else if (!start_to_read_) {
1872
    const ObTableSchema *table_schema = NULL;
1873
    const uint64_t table_id = OB_ALL_VIRTUAL_UPGRADE_INSPECTION_TID;
1874
    if (OB_FAIL(schema_guard.get_table_schema(OB_SYS_TENANT_ID, table_id, table_schema))) {
1875
      LOG_WARN("get_table_schema failed", K(table_id), K(ret));
1876
    } else if (NULL == table_schema) {
1877
      ret = OB_ERR_UNEXPECTED;
1878
      LOG_WARN("table_schema is null", KP(table_schema), K(ret));
1879
    } else {
1880
      ObArray<Column> columns;
1881

1882
#define ADD_ROW(name, info) \
1883
  do { \
1884
    columns.reuse(); \
1885
    if (OB_FAIL(ret)) { \
1886
    } else if (OB_FAIL(get_full_row(table_schema, name, info, columns))) { \
1887
      LOG_WARN("get_full_row failed", "table_schema", *table_schema, \
1888
          K(name), K(info), K(ret)); \
1889
    } else if (OB_FAIL(project_row(columns, cur_row_))) { \
1890
      LOG_WARN("project_row failed", K(columns), K(ret)); \
1891
    } else if (OB_FAIL(scanner_.add_row(cur_row_))) { \
1892
      LOG_WARN("add_row failed", K(cur_row_), K(ret)); \
1893
    } \
1894
  } while (false)
1895

1896
#define CHECK_RESULT(checked, value) (checked ? (value ? "succeed" : "failed") : "checking")
1897

1898
      ADD_ROW("zone_check", CHECK_RESULT(root_inspection_->is_all_checked(),
1899
          root_inspection_->is_zone_passed()));
1900
      ADD_ROW("sys_stat_check", CHECK_RESULT(root_inspection_->is_all_checked(),
1901
          root_inspection_->is_sys_stat_passed()));
1902
      ADD_ROW("sys_param_check", CHECK_RESULT(root_inspection_->is_all_checked(),
1903
          root_inspection_->is_sys_param_passed()));
1904
      ADD_ROW("sys_table_schema_check", CHECK_RESULT(root_inspection_->is_all_checked(),
1905
          root_inspection_->is_sys_table_schema_passed()));
1906
      ADD_ROW("data_version_check", CHECK_RESULT(root_inspection_->is_all_checked(),
1907
          root_inspection_->is_data_version_passed()));
1908

1909
      bool upgrade_job_passed = true;
1910
      for (int64_t i = 0; OB_SUCC(ret) && i < UPGRADE_JOB_TYPE_COUNT; i++) {
1911
        int tmp = OB_SUCCESS;
1912
        ObRsJobType job_type = upgrade_job_type_array[i];
1913
        if (job_type > JOB_TYPE_INVALID && job_type < JOB_TYPE_MAX) {
1914
          if (OB_SUCCESS != (tmp = ObUpgradeUtils::check_upgrade_job_passed(job_type))) {
1915
            LOG_WARN("fail to check upgrade job passed", K(tmp), K(job_type));
1916
            upgrade_job_passed = false;
1917
          }
1918
          ADD_ROW(ObRsJobTableOperator::get_job_type_str(job_type),
1919
                  CHECK_RESULT(root_inspection_->is_all_checked(), (OB_SUCCESS == tmp)));
1920
        }
1921
      }
1922

1923
      ADD_ROW("all_check", CHECK_RESULT(root_inspection_->is_all_checked(),
1924
              (root_inspection_->is_all_passed() && upgrade_job_passed)));
1925

1926
#undef CHECK_RESULT
1927
#undef ADD_ROW
1928
    }
1929
    if (OB_SUCC(ret)) {
1930
      scanner_it_ = scanner_.begin();
1931
      start_to_read_ = true;
1932
    }
1933
  }
1934
  if (OB_SUCC(ret)) {
1935
    if (OB_FAIL(scanner_it_.get_next_row(cur_row_))) {
1936
      if (OB_ITER_END != ret) {
1937
        LOG_WARN("get_next_row failed", K(ret));
1938
      }
1939
    } else {
1940
      row = &cur_row_;
1941
    }
1942
  }
1943
  return ret;
1944

1945
}
1946

1947
int ObUpgradeInspection::get_full_row(const share::schema::ObTableSchema *table,
1948
                                      const char *name, const char *info,
1949
                                      ObIArray<Column> &columns)
1950
{
1951
  int ret = OB_SUCCESS;
1952
  if (!inited_) {
1953
    ret = OB_NOT_INIT;
1954
    LOG_WARN("not init", K(ret));
1955
  } else if (NULL == table || NULL == name || NULL == info) {
1956
    ret = OB_INVALID_ARGUMENT;
1957
    LOG_WARN("invalid argument", KP(table), KP(name), KP(info), K(ret));
1958
  } else {
1959
    ADD_COLUMN(set_varchar, table, "name", name, columns);
1960
    ADD_COLUMN(set_varchar, table, "info", info, columns);
1961
  }
1962

1963
  return ret;
1964
}
1965

1966
}//end namespace rootserver
1967
}//end namespace oceanbase
1968