oceanbase

1
// Copyright (c) 2021 OceanBase
2
// OceanBase is licensed under Mulan PubL v2.
3
// You can use this software according to the terms and conditions of the Mulan PubL v2.
4
// You may obtain a copy of Mulan PubL v2 at:
5
//          http://license.coscl.org.cn/MulanPubL-2.0
6
// THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
7
// EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
8
// MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
9
// See the Mulan PubL v2 for more details.
10
#include <cstdio>
11
#include <gtest/gtest.h>
12
#include <signal.h>
13
#define private public
14
#include "env/ob_simple_log_cluster_env.h"
15
#undef private
16

17
const std::string TEST_NAME = "arb_service";
18

19
using namespace oceanbase::common;
20
using namespace oceanbase;
21
namespace oceanbase
22
{
23
using namespace logservice;
24

25
int64_t ARB_TIMEOUT_ARG = 2 * 1000 * 1000L;
26

27
namespace logservice
28
{
29

30
void ObArbitrationService::update_arb_timeout_()
31
{
32
  arb_timeout_us_ = ARB_TIMEOUT_ARG;
33
  if (REACH_TIME_INTERVAL(2 * 1000 * 1000)) {
34
    CLOG_LOG_RET(WARN, OB_ERR_UNEXPECTED, "update_arb_timeout_", K_(self), K_(arb_timeout_us));
35
  }
36
}
37
}
38

39
namespace unittest
40
{
41

42
class TestObSimpleLogClusterArbService : public ObSimpleLogClusterTestEnv
43
{
44
public:
45
  TestObSimpleLogClusterArbService() :  ObSimpleLogClusterTestEnv()
46
  {}
47
};
48

49
int64_t ObSimpleLogClusterTestBase::member_cnt_ = 3;
50
int64_t ObSimpleLogClusterTestBase::node_cnt_ = 5;
51
bool ObSimpleLogClusterTestBase::need_add_arb_server_ = true;
52
std::string ObSimpleLogClusterTestBase::test_name_ = TEST_NAME;
53

54
TEST_F(TestObSimpleLogClusterArbService, test_2f1a_degrade_upgrade)
55
{
56
  oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_4_1_0_0;
57
  SET_CASE_LOG_FILE(TEST_NAME, "arb_2f1a_degrade_upgrade");
58
  OB_LOGGER.set_log_level("TRACE");
59
  MockLocCB loc_cb;
60
  int ret = OB_SUCCESS;
61
  PALF_LOG(INFO, "begin test_2f1a_degrade_upgrade");
62
	int64_t leader_idx = 0;
63
  int64_t arb_replica_idx = -1;
64
  PalfHandleImplGuard leader;
65
  std::vector<PalfHandleImplGuard*> palf_list;
66
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
67
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
68
  common::ObMember dummy_member;
69
	EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, arb_replica_idx, leader_idx, leader));
70
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
71
  const int64_t another_f_idx = (leader_idx+1)%3;
72
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
73
  // 为备副本设置location cb，用于备副本找leader
74
  palf_list[another_f_idx]->get_palf_handle_impl()->set_location_cache_cb(&loc_cb);
75
  block_net(leader_idx, another_f_idx);
76
  // do not check OB_SUCCESS, may return OB_NOT_MASTER during degrading member
77
  submit_log(leader, 100, id);
78

79
  PALF_LOG(INFO, "CASE[1] degrade caused by block_net ");
80
  EXPECT_TRUE(is_degraded(leader, another_f_idx));
81

82
  loc_cb.leader_ = leader.palf_handle_impl_->self_;
83
  unblock_net(leader_idx, another_f_idx);
84
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10, id));
85

86
  EXPECT_TRUE(is_upgraded(leader, id));
87
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10, id));
88

89
  // set clog disk error
90
  ObTenantEnv::set_tenant(get_cluster()[another_f_idx]->get_tenant_base());
91
  logservice::coordinator::ObFailureDetector *detector = MTL(logservice::coordinator::ObFailureDetector *);
92
  if (NULL != detector) {
93
    PALF_LOG(INFO, "set clog full event");
94
    detector->has_add_clog_full_event_ = true;
95
  }
96

97
  PALF_LOG(INFO, "CASE[2] degrade caused by clog disk error");
98
  EXPECT_TRUE(is_degraded(leader, another_f_idx));
99

100
  if (NULL != detector) {
101
    detector->has_add_clog_full_event_ = false;
102
  }
103

104
  EXPECT_TRUE(is_upgraded(leader, id));
105
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1, id));
106

107
  // test disable sync
108
  PALF_LOG(INFO, "CASE[3] degrade caused by disable_sync");
109
  palf_list[another_f_idx]->palf_handle_impl_->disable_sync();
110
  EXPECT_TRUE(is_degraded(leader, another_f_idx));
111
  palf_list[another_f_idx]->palf_handle_impl_->enable_sync();
112
  EXPECT_TRUE(is_upgraded(leader, id));
113

114
  PALF_LOG(INFO, "CASE[4] degrade caused by disable_vote");
115
  // test disbale vote
116
  palf_list[another_f_idx]->palf_handle_impl_->disable_vote(false/*no need check log missing*/);
117
  EXPECT_TRUE(is_degraded(leader, another_f_idx));
118
  palf_list[another_f_idx]->palf_handle_impl_->enable_vote();
119
  EXPECT_TRUE(is_upgraded(leader, id));
120

121
  // test revoking the leader when arb service is degrading
122
  block_all_net(another_f_idx);
123
  const common::ObAddr follower_addr = get_cluster()[another_f_idx]->get_addr();
124
  LogConfigChangeArgs args(common::ObMember(follower_addr, 1), 0, DEGRADE_ACCEPTOR_TO_LEARNER);
125
  int64_t ele_epoch;
126
  common::ObRole ele_role;
127
  int64_t proposal_id = leader.palf_handle_impl_->state_mgr_.get_proposal_id();
128
  leader.palf_handle_impl_->election_.get_role(ele_role, ele_epoch);
129
  LogConfigVersion config_version;
130
  EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config_(args, proposal_id, ele_epoch, config_version));
131
  EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.alive_paxos_memberlist_.contains(follower_addr));
132
  EXPECT_EQ(leader.palf_handle_impl_->config_mgr_.state_, 1);
133

134
  // reset status supposing the lease is expried
135
  block_net(leader_idx, another_f_idx);
136
  leader.palf_handle_impl_->config_mgr_.reset_status();
137
  EXPECT_TRUE(is_degraded(leader, another_f_idx));
138
  unblock_net(leader_idx, another_f_idx);
139
  unblock_all_net(another_f_idx);
140

141
  revert_cluster_palf_handle_guard(palf_list);
142
  leader.reset();
143
  delete_paxos_group(id);
144
  PALF_LOG(INFO, "end test_2f1a_degrade_upgrade", K(id));
145
}
146

147
TEST_F(TestObSimpleLogClusterArbService, test_4f1a_degrade_upgrade)
148
{
149
  SET_CASE_LOG_FILE(TEST_NAME, "arb_4f1a_degrade_upgrade");
150
  OB_LOGGER.set_log_level("TRACE");
151
  MockLocCB loc_cb;
152
  int ret = OB_SUCCESS;
153
  PALF_LOG(INFO, "begin test_4f1a_degrade_upgrade");
154
	int64_t leader_idx = 0;
155
  int64_t arb_replica_idx = -1;
156
  PalfHandleImplGuard leader;
157
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
158
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
159
  common::ObMember dummy_member;
160
  std::vector<PalfHandleImplGuard*> palf_list;
161
	EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, arb_replica_idx, leader_idx, leader));
162
  LogConfigVersion config_version;
163
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
164
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(get_cluster()[3]->get_addr(), 1), 3, config_version, CONFIG_CHANGE_TIMEOUT));
165
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
166
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(get_cluster()[4]->get_addr(), 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
167
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
168

169
  const int64_t another_f1_idx = (leader_idx+3)%5;
170
  const int64_t another_f2_idx = (leader_idx+4)%5;
171
  palf_list[another_f1_idx]->palf_handle_impl_->set_location_cache_cb(&loc_cb);
172
  palf_list[another_f2_idx]->palf_handle_impl_->set_location_cache_cb(&loc_cb);
173
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
174
  sleep(2);
175
  block_all_net(another_f1_idx);
176
  block_all_net(another_f2_idx);
177

178

179
  EXPECT_TRUE(is_degraded(leader, another_f1_idx));
180
  EXPECT_TRUE(is_degraded(leader, another_f2_idx));
181

182
  unblock_all_net(another_f1_idx);
183
  unblock_all_net(another_f2_idx);
184
  loc_cb.leader_ = leader.palf_handle_impl_->self_;
185
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10, id));
186

187
  EXPECT_TRUE(is_upgraded(leader, id));
188

189
  revert_cluster_palf_handle_guard(palf_list);
190
  leader.reset();
191
  delete_paxos_group(id);
192
  PALF_LOG(INFO, "end test_4f1a_degrade_upgrade", K(id));
193
}
194

195
TEST_F(TestObSimpleLogClusterArbService, test_2f1a_reconfirm_degrade_upgrade)
196
{
197
  SET_CASE_LOG_FILE(TEST_NAME, "arb_2f1a_reconfirm_test");
198
  OB_LOGGER.set_log_level("TRACE");
199
  int ret = OB_SUCCESS;
200
  PALF_LOG(INFO, "begin test_2f1a_reconfirm_degrade_upgrade");
201
  MockLocCB loc_cb;
202
	int64_t leader_idx = 0;
203
  int64_t arb_replica_idx = -1;
204
  PalfHandleImplGuard leader;
205
  std::vector<PalfHandleImplGuard*> palf_list;
206
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
207
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
208
  common::ObMember dummy_member;
209
	EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, arb_replica_idx, leader_idx, leader));
210
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
211
  const int64_t another_f_idx = (leader_idx+1)%3;
212
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
213
  sleep(2);
214
  palf_list[leader_idx]->palf_handle_impl_->set_location_cache_cb(&loc_cb);
215
  palf_list[another_f_idx]->palf_handle_impl_->set_location_cache_cb(&loc_cb);
216
  // block net of old leader, new leader will be elected
217
  // and degrade in RECONFIRM state
218
  ARB_TIMEOUT_ARG = 15 * 1000 * 1000;
219
  block_net(leader_idx, another_f_idx);
220
  block_net(leader_idx, arb_replica_idx);
221
  // block_net后会理解进行降级操作，导致旧主上有些单副本写成功的日志被committed
222
  submit_log(leader, 20, id);
223
  // submit some logs which will be truncated
224

225
  EXPECT_TRUE(is_degraded(*palf_list[another_f_idx], leader_idx));
226

227
  int64_t new_leader_idx = -1;
228
  PalfHandleImplGuard new_leader;
229
  EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx));
230
  loc_cb.leader_ = new_leader.palf_handle_impl_->self_;
231
  unblock_net(leader_idx, another_f_idx);
232
  unblock_net(leader_idx, arb_replica_idx);
233
  EXPECT_EQ(OB_SUCCESS, submit_log(new_leader, 100, id));
234

235
  EXPECT_TRUE(is_upgraded(new_leader, id));
236
  revert_cluster_palf_handle_guard(palf_list);
237
  leader.reset();
238
  new_leader.reset();
239
  delete_paxos_group(id);
240
  ARB_TIMEOUT_ARG = 2 * 1000 * 1000;
241
  PALF_LOG(INFO, "end test_2f1a_reconfirm_degrade_upgrade", K(id));
242
}
243

244
TEST_F(TestObSimpleLogClusterArbService, test_4f1a_reconfirm_degrade_upgrade)
245
{
246
  SET_CASE_LOG_FILE(TEST_NAME, "arb_4f1a_reconfirm_test");
247
  OB_LOGGER.set_log_level("TRACE");
248
  MockLocCB loc_cb;
249
  int ret = OB_SUCCESS;
250
  PALF_LOG(INFO, "begin test_4f1a_reconfirm_degrade_upgrade");
251
	int64_t leader_idx = 0;
252
  int64_t arb_replica_idx = -1;
253
  auto cluster = get_cluster();
254
  PalfHandleImplGuard leader;
255
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
256
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
257
  common::ObMember dummy_member;
258
  std::vector<PalfHandleImplGuard*> palf_list;
259

260
	EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, arb_replica_idx, leader_idx, leader));
261
  LogConfigVersion config_version;
262
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
263
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(get_cluster()[3]->get_addr(), 1), 3, config_version, CONFIG_CHANGE_TIMEOUT));
264
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
265
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(get_cluster()[4]->get_addr(), 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
266

267
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
268

269
  const int64_t another_f1_idx = 3;
270
  const int64_t another_f2_idx = 4;
271
  palf_list[leader_idx]->palf_handle_impl_->set_location_cache_cb(&loc_cb);
272
  palf_list[another_f1_idx]->palf_handle_impl_->set_location_cache_cb(&loc_cb);
273
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
274
  sleep(2);
275
  // stop leader and a follower
276
  block_all_net(leader_idx);
277
  block_all_net(another_f1_idx);
278

279
  //EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
280
  // wait for new leader is elected
281
  int64_t new_leader_idx = leader_idx;
282
  PalfHandleImplGuard new_leader;
283
  while (leader_idx == new_leader_idx) {
284
    new_leader.reset();
285
    EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx));
286
  }
287

288
  EXPECT_TRUE(is_degraded(new_leader, another_f1_idx));
289
  EXPECT_TRUE(is_degraded(new_leader, leader_idx));
290

291
  loc_cb.leader_ = new_leader.palf_handle_impl_->self_;
292
  // restart two servers
293
  unblock_all_net(leader_idx);
294
  unblock_all_net(another_f1_idx);
295

296
  EXPECT_EQ(OB_SUCCESS, submit_log(new_leader, 100, id));
297

298
  EXPECT_TRUE(is_upgraded(new_leader, id));
299
  leader.reset();
300
  new_leader.reset();
301
  revert_cluster_palf_handle_guard(palf_list);
302
  delete_paxos_group(id);
303
  PALF_LOG(INFO, "end test_4f1a_reconfirm_degrade_upgrade", K(id));
304
}
305

306
TEST_F(TestObSimpleLogClusterArbService, test_2f1a_config_change)
307
{
308
  SET_CASE_LOG_FILE(TEST_NAME, "arb_2f1a_config_change");
309
  OB_LOGGER.set_log_level("DEBUG");
310
  MockLocCB loc_cb;
311
  int ret = OB_SUCCESS;
312
  PALF_LOG(INFO, "begin arb_2f1a_config_change");
313
	int64_t leader_idx = 0;
314
  int64_t arb_replica_idx = -1;
315
  PalfHandleImplGuard leader;
316
  std::vector<PalfHandleImplGuard*> palf_list;
317
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
318
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
319
  common::ObMember dummy_member;
320
	EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, arb_replica_idx, leader_idx, leader));
321
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
322
  // 为备副本设置location cb，用于备副本找leader
323
  const int64_t another_f_idx = (leader_idx+1)%3;
324
  loc_cb.leader_ = leader.palf_handle_impl_->self_;
325
  palf_list[another_f_idx]->get_palf_handle_impl()->set_location_cache_cb(&loc_cb);
326
  palf_list[3]->get_palf_handle_impl()->set_location_cache_cb(&loc_cb);
327
  palf_list[4]->get_palf_handle_impl()->set_location_cache_cb(&loc_cb);
328
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
329
  sleep(2);
330

331
  LogConfigVersion config_version;
332
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
333
  // replace member
334
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_member(
335
      ObMember(palf_list[3]->palf_handle_impl_->self_, 1),
336
      ObMember(palf_list[another_f_idx]->palf_handle_impl_->self_, 1),
337
      config_version,
338
      CONFIG_CHANGE_TIMEOUT));
339

340
  // add learner
341
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(
342
      ObMember(palf_list[4]->palf_handle_impl_->self_, 1),
343
      CONFIG_CHANGE_TIMEOUT));
344

345
  // switch learner
346
  EXPECT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->switch_learner_to_acceptor(
347
      ObMember(palf_list[4]->palf_handle_impl_->self_, 1),
348
      2,
349
      config_version,
350
      CONFIG_CHANGE_TIMEOUT));
351
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->switch_learner_to_acceptor(
352
      ObMember(palf_list[4]->palf_handle_impl_->self_, 1),
353
      3,
354
      config_version,
355
      CONFIG_CHANGE_TIMEOUT));
356
  revert_cluster_palf_handle_guard(palf_list);
357
  leader.reset();
358
  delete_paxos_group(id);
359
  PALF_LOG(INFO, "end arb_2f1a_config_change", K(id));
360
}
361

362
TEST_F(TestObSimpleLogClusterArbService, test_2f1a_arb_with_highest_version)
363
{
364
  oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_4_1_0_0;
365
  SET_CASE_LOG_FILE(TEST_NAME, "test_2f1a_arb_with_highest_version");
366
  OB_LOGGER.set_log_level("DEBUG");
367
  MockLocCB loc_cb;
368
  int ret = OB_SUCCESS;
369
  PALF_LOG(INFO, "begin test_2f1a_arb_with_highest_version");
370
	int64_t leader_idx = 0;
371
  int64_t arb_replica_idx = -1;
372
  PalfHandleImplGuard leader;
373
  std::vector<PalfHandleImplGuard*> palf_list;
374
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
375
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
376
  common::ObMember dummy_member;
377
	EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, arb_replica_idx, leader_idx, leader));
378
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
379
  // 为备副本设置location cb，用于备副本找leader
380
  const int64_t another_f_idx = (leader_idx+1)%3;
381
  loc_cb.leader_ = leader.palf_handle_impl_->self_;
382
  palf_list[another_f_idx]->get_palf_handle_impl()->set_location_cache_cb(&loc_cb);
383
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 500, id));
384
  sleep(2);
385

386
  LogConfigChangeArgs args(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 0, ADD_LEARNER);
387
  int64_t proposal_id = 0;
388
  int64_t election_epoch = 0;
389
  LogConfigVersion config_version;
390
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->config_mgr_.start_change_config(proposal_id, election_epoch, args.type_));
391
  EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config(args, proposal_id, election_epoch, config_version));
392
  // learner list and state_ has been changed
393
  EXPECT_TRUE(config_version.is_valid());
394
  EXPECT_EQ(1, leader.palf_handle_impl_->config_mgr_.state_);
395
  // only send config log to arb member
396
  ObMemberList member_list;
397
  member_list.add_server(get_cluster()[2]->get_addr());
398
  const int64_t prev_log_proposal_id = leader.palf_handle_impl_->config_mgr_.reconfig_barrier_.prev_log_proposal_id_;
399
  const LSN prev_lsn = leader.palf_handle_impl_->config_mgr_.reconfig_barrier_.prev_lsn_;
400
  const int64_t prev_mode_pid = leader.palf_handle_impl_->config_mgr_.reconfig_barrier_.prev_mode_pid_;
401
  const LogConfigMeta config_meta = leader.palf_handle_impl_->config_mgr_.log_ms_meta_;
402
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->log_engine_.submit_change_config_meta_req( \
403
    member_list, proposal_id, prev_log_proposal_id, prev_lsn, prev_mode_pid, config_meta));
404
  sleep(1);
405
  // check if arb member has received and persisted the config log
406
  while (true) {
407
    PalfHandleLiteGuard arb_member;
408
    if (OB_FAIL(get_arb_member_guard(id, arb_member))) {
409
    } else if (arb_member.palf_handle_lite_->config_mgr_.persistent_config_version_ == config_version) {
410
      break;
411
    } else {
412
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->log_engine_.submit_change_config_meta_req( \
413
        member_list, proposal_id, prev_log_proposal_id, prev_lsn, prev_mode_pid, config_meta));
414
    }
415
    ::ob_usleep(10 * 1000);
416
  }
417
  EXPECT_GT(config_version, leader.palf_handle_impl_->config_mgr_.persistent_config_version_);
418
  EXPECT_GT(config_version, palf_list[1]->palf_handle_impl_->config_mgr_.persistent_config_version_);
419

420
  // restart cluster, close a follower, restart leader
421
  revert_cluster_palf_handle_guard(palf_list);
422
  leader.reset();
423
  // block_net, so two F cann't reach majority
424
  block_net(another_f_idx, leader_idx);
425
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
426

427
  const int64_t restart_finish_time_us_ = common::ObTimeUtility::current_time();
428
  PalfHandleImplGuard new_leader;
429
  int64_t new_leader_idx;
430
  get_leader(id, new_leader, new_leader_idx);
431
  EXPECT_EQ(OB_SUCCESS, submit_log(new_leader, 500, id));
432
  PALF_LOG(ERROR, "RTO", "RTO", common::ObTimeUtility::current_time() - restart_finish_time_us_);
433

434
  new_leader.reset();
435
  // must delete paxos group in here, otherwise memory of
436
  // MockLocCB will be relcaimed and core dump will occur
437
  // blacklist will not be deleted after reboot, clean it manually
438
  unblock_net(another_f_idx, leader_idx);
439
  delete_paxos_group(id);
440
  PALF_LOG(INFO, "end test_2f1a_arb_with_highest_version", K(id));
441
}
442

443
TEST_F(TestObSimpleLogClusterArbService, test_2f1a_defensive)
444
{
445
  SET_CASE_LOG_FILE(TEST_NAME, "test_2f1a_defensive");
446
  OB_LOGGER.set_log_level("DEBUG");
447
  MockLocCB loc_cb;
448
  int ret = OB_SUCCESS;
449
  PALF_LOG(INFO, "begin test_2f1a_defensive");
450
	int64_t leader_idx = 0;
451
  int64_t arb_replica_idx = -1;
452
  PalfHandleImplGuard leader;
453
  std::vector<PalfHandleImplGuard*> palf_list;
454
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
455
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
456
  common::ObMember dummy_member;
457
	EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, arb_replica_idx, leader_idx, leader));
458
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
459
  // 为备副本设置location cb，用于备副本找leader
460
  const int64_t another_f_idx = (leader_idx+1)%3;
461
  loc_cb.leader_ = leader.palf_handle_impl_->self_;
462
  palf_list[another_f_idx]->get_palf_handle_impl()->set_location_cache_cb(&loc_cb);
463
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
464
  sleep(2);
465
  const int64_t added_member_idx = 3;
466
  const common::ObMember added_member = ObMember(palf_list[added_member_idx]->palf_handle_impl_->self_, 1);
467

468
  // add a member, do not allow to append logs until config log reaches majority
469
  LogConfigVersion cur_config_version;
470
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(cur_config_version));
471
  LogConfigChangeArgs args(added_member, 3, cur_config_version, ADD_MEMBER);
472
  int64_t proposal_id = 0;
473
  int64_t election_epoch = 0;
474
  LogConfigVersion config_version;
475
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->config_mgr_.start_change_config(proposal_id, election_epoch, args.type_));
476
  EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config(args, proposal_id, election_epoch, config_version));
477
  // do not allow to append log when changing config with arb
478
  // EXPECT_TRUE(leader.palf_handle_impl_->state_mgr_.is_changing_config_with_arb());
479
  while (true) {
480
    if (OB_SUCC(leader.palf_handle_impl_->config_mgr_.change_config(args, proposal_id, election_epoch, config_version))) {
481
      break;
482
    } else {
483
      (void) leader.palf_handle_impl_->config_mgr_.pre_sync_config_log_and_mode_meta(args.server_, proposal_id);
484
      ::ob_usleep(10 * 1000);
485
    }
486
  }
487

488
  // flashback one follower
489
  LogEntryHeader header_origin;
490
  SCN base_scn;
491
  base_scn.set_base();
492
  SCN flashback_scn;
493
  palf::AccessMode unused_access_mode;
494
  int64_t mode_version;
495
  EXPECT_EQ(OB_SUCCESS, get_middle_scn(50, leader, flashback_scn, header_origin));
496
  switch_append_to_flashback(leader, mode_version);
497
  sleep(1);
498
  EXPECT_EQ(OB_SUCCESS, palf_list[another_f_idx]->palf_handle_impl_->flashback(mode_version, flashback_scn, CONFIG_CHANGE_TIMEOUT));
499

500
  // remove another follower
501
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(added_member, 2, CONFIG_CHANGE_TIMEOUT));
502

503
  revert_cluster_palf_handle_guard(palf_list);
504
  leader.reset();
505
  delete_paxos_group(id);
506
  PALF_LOG(INFO, "end test_2f1a_defensive", K(id));
507
}
508

509
int get_palf_handle_lite(const int64_t tenant_id,
510
                         const int64_t palf_id,
511
                         ObSimpleArbServer *server,
512
                         IPalfHandleImplGuard &handle_guard)
513
{
514
  int ret = OB_SUCCESS;
515
  PalfEnvLiteGuard env_guard;
516
  if (NULL == server) {
517
    ret = OB_INVALID_ARGUMENT;
518
  } else if (OB_FAIL(server->get_palf_env_lite(tenant_id, env_guard))) {
519
    PALF_LOG(ERROR, "get_palf_env_lite failed", K(tenant_id), K(palf_id));
520
  } else if (OB_FAIL(env_guard.palf_env_lite_->get_palf_handle_impl(palf_id, handle_guard))) {
521
    PALF_LOG(ERROR, "get_palf_handle_impl failed", K(tenant_id), K(palf_id));
522
  } else {
523
  }
524
  return ret;
525
}
526

527
using namespace palflite;
528

529
TEST_F(TestObSimpleLogClusterArbService, test_multi_meta_block)
530
{
531
  SET_CASE_LOG_FILE(TEST_NAME, "test_mutli_meta_block");
532
  OB_LOGGER.set_log_level("INFO");
533
  MockLocCB loc_cb;
534
  int ret = OB_SUCCESS;
535
  PALF_LOG(INFO, "begin test_multi_meta_block");
536
	int64_t leader_idx = 0;
537
  int64_t arb_replica_idx = -1;
538
  PalfHandleImplGuard leader;
539
  std::vector<PalfHandleImplGuard*> palf_list;
540
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
541
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
542
  common::ObMember dummy_member;
543
	EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, arb_replica_idx, leader_idx, leader));
544
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
545
  // 为备副本设置location cb，用于备副本找leader
546
  const int64_t another_f_idx = (leader_idx+1)%3;
547
  loc_cb.leader_ = leader.palf_handle_impl_->self_;
548
  palf_list[another_f_idx]->get_palf_handle_impl()->set_location_cache_cb(&loc_cb);
549
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
550
  sleep(2);
551
  ObSimpleArbServer *arb_server = dynamic_cast<ObSimpleArbServer*>(get_cluster()[arb_replica_idx]);
552
  IPalfHandleImplGuard arb_guard;
553
  ASSERT_EQ(OB_SUCCESS, get_palf_handle_lite(OB_SERVER_TENANT_ID, id, arb_server, arb_guard));
554
  PalfHandleLite *arb_palf = dynamic_cast<PalfHandleLite *>(arb_guard.palf_handle_impl_);
555
  LogEngine *log_engine = &arb_palf->log_engine_;
556
  LSN meta_tail = log_engine->log_meta_storage_.log_tail_;
557
  LogStorage *meta_storage = &log_engine->log_meta_storage_;
558
  {
559
    while (1) {
560
      if (meta_storage->log_tail_ < LSN(meta_storage->logical_block_size_)) {
561
        EXPECT_EQ(OB_SUCCESS, log_engine->append_log_meta_(log_engine->log_meta_));
562
      } else {
563
        break;
564
      }
565
    }
566
  }
567
  meta_tail = log_engine->log_meta_storage_.log_tail_;
568
  ASSERT_EQ(meta_tail, LSN(log_engine->log_meta_storage_.logical_block_size_));
569
  revert_cluster_palf_handle_guard(palf_list);
570
  arb_guard.reset();
571
  leader.reset();
572
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
573
  {
574
    ObSimpleArbServer *arb_server = dynamic_cast<ObSimpleArbServer*>(get_cluster()[arb_replica_idx]);
575
    PalfHandleImplGuard leader;
576
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
577
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
578
    IPalfHandleImplGuard arb_guard;
579
    ASSERT_EQ(OB_SUCCESS, get_palf_handle_lite(OB_SERVER_TENANT_ID, id, arb_server, arb_guard));
580
    PalfHandleLite *arb_palf = dynamic_cast<PalfHandleLite *>(arb_guard.palf_handle_impl_);
581
    LogEngine *log_engine = &arb_palf->log_engine_;
582
    LogStorage *meta_storage = &log_engine->log_meta_storage_;
583
    EXPECT_EQ(OB_SUCCESS, log_engine->append_log_meta_(log_engine->log_meta_));
584
    LSN meta_tail = log_engine->log_meta_storage_.log_tail_;
585
    ASSERT_NE(meta_tail, LSN(log_engine->log_meta_storage_.logical_block_size_));
586
  }
587
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
588
  {
589
    ObSimpleArbServer *arb_server = dynamic_cast<ObSimpleArbServer*>(get_cluster()[arb_replica_idx]);
590
    PalfHandleImplGuard leader;
591
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
592
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
593
    IPalfHandleImplGuard arb_guard;
594
    ASSERT_EQ(OB_SUCCESS, get_palf_handle_lite(OB_SERVER_TENANT_ID, id, arb_server, arb_guard));
595
    PalfHandleLite *arb_palf = dynamic_cast<PalfHandleLite *>(arb_guard.palf_handle_impl_);
596
    LogEngine *log_engine = &arb_palf->log_engine_;
597
    LSN meta_tail = log_engine->log_meta_storage_.log_tail_;
598
    LogStorage *meta_storage = &log_engine->log_meta_storage_;
599
    while (1) {
600
      if (meta_storage->log_tail_ < LSN(32 * meta_storage->logical_block_size_)) {
601
        EXPECT_EQ(OB_SUCCESS, log_engine->append_log_meta_(log_engine->log_meta_));
602
      } else {
603
        break;
604
      }
605
    }
606
  }
607
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
608
  {
609
    ObSimpleArbServer *arb_server = dynamic_cast<ObSimpleArbServer*>(get_cluster()[arb_replica_idx]);
610
    PalfHandleImplGuard leader;
611
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
612
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 4000, id));
613
    IPalfHandleImplGuard arb_guard;
614
    ASSERT_EQ(OB_SUCCESS, get_palf_handle_lite(OB_SERVER_TENANT_ID, id, arb_server, arb_guard));
615
    PalfHandleLite *arb_palf = dynamic_cast<PalfHandleLite *>(arb_guard.palf_handle_impl_);
616
    LogEngine *log_engine = &arb_palf->log_engine_;
617
    LSN meta_tail = log_engine->log_meta_storage_.log_tail_;
618
    LogStorage *meta_storage = &log_engine->log_meta_storage_;
619
    while (1) {
620
      if (meta_storage->log_tail_ < LSN(34 * meta_storage->logical_block_size_ + 4*4*1024)) {
621
        EXPECT_EQ(OB_SUCCESS, log_engine->append_log_meta_(log_engine->log_meta_));
622
      } else {
623
        break;
624
      }
625
    }
626
  }
627
  delete_paxos_group(id);
628
  PALF_LOG(INFO, "end test_mutli_meta_block", K(id));
629
}
630

631
// 1. 2F1A, the leader starts to degrade another F
632
// 2. after the config log has been accepted by another F, the leader revoked
633
// 3. the previous leader has been elected as the new leader
634
// 4. reconfirm may fail because leader's config_version is not same to that of the follower
635
TEST_F(TestObSimpleLogClusterArbService, test_2f1a_degrade_when_no_leader)
636
{
637
  SET_CASE_LOG_FILE(TEST_NAME, "test_2f1a_degrade_when_no_leader");
638
  MockLocCB loc_cb;
639
  int ret = OB_SUCCESS;
640
  PALF_LOG(INFO, "begin test_2f1a_degrade_when_no_leader");
641
	int64_t leader_idx = 0;
642
  int64_t arb_replica_idx = -1;
643
  PalfHandleImplGuard leader;
644
  std::vector<PalfHandleImplGuard*> palf_list;
645
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
646
  common::ObMember dummy_member;
647
	EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, arb_replica_idx, leader_idx, leader));
648
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
649
  // 为备副本设置location cb，用于备副本找leader
650
  const int64_t another_f_idx = (leader_idx+1)%3;
651
  loc_cb.leader_ = leader.palf_handle_impl_->self_;
652
  palf_list[another_f_idx]->get_palf_handle_impl()->set_location_cache_cb(&loc_cb);
653
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
654
  sleep(2);
655

656
  const common::ObAddr b_addr = palf_list[another_f_idx]->palf_handle_impl_->self_;
657
  LogConfigChangeArgs args(ObMember(b_addr, 1), 0, DEGRADE_ACCEPTOR_TO_LEARNER);
658
  int64_t proposal_id = 0;
659
  int64_t election_epoch = 0;
660
  LogConfigVersion config_version;
661
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->config_mgr_.start_change_config(proposal_id, election_epoch, args.type_));
662
  EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config(args, proposal_id, election_epoch, config_version));
663

664
  // leader appended config meta
665
  EXPECT_FALSE(palf_list[leader_idx]->get_palf_handle_impl()->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
666
  EXPECT_TRUE(palf_list[another_f_idx]->get_palf_handle_impl()->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
667

668
  // block all networks of arb member, and the network from the follower to the leader
669
  block_net(arb_replica_idx, another_f_idx, true);
670
  block_net(arb_replica_idx, leader_idx, true);
671
  block_net(another_f_idx, leader_idx, true);
672

673
  // waiting for leader revoke
674
  while (leader.palf_handle_impl_->state_mgr_.role_ == common::ObRole::LEADER) {
675
    sleep(1);
676
  }
677

678
  // unblock_net
679
  unblock_net(another_f_idx, leader_idx);
680
  unblock_net(arb_replica_idx, leader_idx);
681

682
  common::ObMemberList leader_member_list;
683
  int64_t leader_replica_num = 0;
684
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->config_mgr_.get_log_sync_member_list( \
685
      leader_member_list, leader_replica_num));
686
  EXPECT_EQ(1, leader_member_list.get_member_number());
687
  EXPECT_EQ(1, leader_replica_num);
688

689
  int64_t new_leader_idx = 0;
690
  PalfHandleImplGuard new_leader;
691
  EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx));
692

693
  EXPECT_EQ(leader.palf_handle_impl_->self_, new_leader.palf_handle_impl_->self_);
694

695
  // waiting for upgrading
696
  is_upgraded(leader, id);
697
  EXPECT_EQ(OB_SUCCESS, new_leader.palf_handle_impl_->config_mgr_.get_log_sync_member_list( \
698
      leader_member_list, leader_replica_num));
699
  EXPECT_EQ(2, leader_member_list.get_member_number());
700
  EXPECT_EQ(2, leader_replica_num);
701

702
  revert_cluster_palf_handle_guard(palf_list);
703
  leader.reset();
704
  new_leader.reset();
705
  delete_paxos_group(id);
706
  PALF_LOG(INFO, "end test_2f1a_degrade_when_no_leader", K(id));
707
}
708

709
TEST_F(TestObSimpleLogClusterArbService, test_2f1a_upgrade_when_no_leader)
710
{
711
  SET_CASE_LOG_FILE(TEST_NAME, "test_2f1a_upgrade_when_no_leader");
712
//  OB_LOGGER.set_log_level("TRACE");
713
  MockLocCB loc_cb;
714
  int ret = OB_SUCCESS;
715
  PALF_LOG(INFO, "begin test_2f1a_upgrade_when_no_leader");
716
	int64_t leader_idx = 0;
717
  int64_t arb_replica_idx = -1;
718
  PalfHandleImplGuard leader;
719
  std::vector<PalfHandleImplGuard*> palf_list;
720
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
721
  common::ObMember dummy_member;
722
	EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, arb_replica_idx, leader_idx, leader));
723
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
724
  // 为备副本设置location cb，用于备副本找leader
725
  const int64_t another_f_idx = (leader_idx+1)%3;
726
  loc_cb.leader_ = leader.palf_handle_impl_->self_;
727
  palf_list[another_f_idx]->get_palf_handle_impl()->set_location_cache_cb(&loc_cb);
728
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
729
  sleep(2);
730

731
  // block the network from the follower to the leader
732
  block_net(another_f_idx, leader_idx, true);
733
  is_degraded(leader, another_f_idx);
734

735
  // upgrade follower manually
736
  int64_t proposal_id;
737
  int64_t election_epoch;
738
  LogConfigVersion config_version;
739
  LogConfigChangeArgs args(common::ObMember(get_cluster()[another_f_idx]->get_addr(), 1), 0, UPGRADE_LEARNER_TO_ACCEPTOR);
740
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->config_mgr_.start_change_config(proposal_id, election_epoch, args.type_));
741

742
  block_net(arb_replica_idx, leader_idx, true);
743
  EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config(args, proposal_id, election_epoch, config_version));
744
  EXPECT_EQ(1, leader.palf_handle_impl_->config_mgr_.state_);
745
  EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config(args, proposal_id, election_epoch, config_version));
746
  EXPECT_EQ(1, leader.palf_handle_impl_->config_mgr_.state_);
747

748
  // waiting for leader revoke
749
  while (leader.palf_handle_impl_->state_mgr_.role_ == LEADER) {
750
    sleep(1);
751
  }
752

753
  // avoid the follower is elected to be leader
754
  block_net(arb_replica_idx, another_f_idx, true);
755
  unblock_all_net(leader_idx);
756

757
  // waiting for leader takeover
758
  while (!leader.palf_handle_impl_->state_mgr_.is_leader_active()) {
759
    sleep(1);
760
  }
761
  // waiting for upgrading
762
  is_upgraded(leader, id);
763

764
  revert_cluster_palf_handle_guard(palf_list);
765
  leader.reset();
766
  delete_paxos_group(id);
767
  PALF_LOG(INFO, "end test_2f1a_upgrade_when_no_leader", K(id));
768
}
769

770
TEST_F(TestObSimpleLogClusterArbService, test_1f1a_create_palf_group)
771
{
772
  oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_4_1_0_0;
773
  SET_CASE_LOG_FILE(TEST_NAME, "test_1f1a_create_palf_group");
774
  PALF_LOG(INFO, "begin test_1f1a_create_palf_group");
775
  OB_LOGGER.set_log_level("TRACE");
776
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
777
  MockLocCB loc_cb;
778
  int ret = OB_SUCCESS;
779
  int64_t leader_idx = 0;
780
  int64_t arb_replica_idx = -1;
781
  PalfHandleImplGuard leader;
782
  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
783
  common::ObMemberList member_list = get_member_list();
784
  member_list.remove_server(get_cluster()[1]->get_addr());
785
  const int64_t member_cnt = 2;
786
  const common::ObMember &arb_member = get_arb_member();
787
  EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, &loc_cb, member_list, member_cnt, arb_member, arb_replica_idx, leader_idx, false, leader));
788
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
789

790
  LogConfigVersion config_version;
791
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
792
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(get_cluster()[1]->get_addr(), 1), 2, config_version, CONFIG_CHANGE_TIMEOUT));
793

794
  EXPECT_EQ(2, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_);
795
  EXPECT_EQ(2, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.get_member_number());
796

797
  leader.reset();
798
  delete_paxos_group(id);
799
  PALF_LOG(INFO, "end test_2f1a_degrade_upgrade", K(id));
800
}
801

802
// 1. 2F1A
803
// 2. lock_memberlist(just renew barrier)
804
// 3. submit and commit logs
805
// 4. kill leader
806
// 5. check committed_end_lsn
807
TEST_F(TestObSimpleLogClusterArbService, test_lock_memberlist_opt)
808
{
809
  SET_CASE_LOG_FILE(TEST_NAME, "test_lock_memberlist_opt");
810
  int ret = OB_SUCCESS;
811
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
812
  PALF_LOG(INFO, "begin test_repeat_lock_memberlist", K(id));
813
	int64_t leader_idx = 0, arb_replica_idx = 0;
814
  PalfHandleImplGuard leader;
815
  oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_4_2_0_0;
816
	EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_arb(id, arb_replica_idx, leader_idx, leader));
817
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
818
  const int64_t another_f_idx = (leader_idx+1)%3;
819

820
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10, id));
821
  EXPECT_UNTIL_EQ(leader.palf_handle_impl_->get_max_lsn(), leader.palf_handle_impl_->get_end_lsn());
822

823
  // 2. renew_barrier
824
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->config_mgr_.renew_config_change_barrier());
825

826
  // 3. submit and commit logs
827
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10, id));
828
  const LSN max_lsn = leader.palf_handle_impl_->get_max_lsn();
829
  EXPECT_UNTIL_EQ(leader.palf_handle_impl_->get_max_lsn(), leader.palf_handle_impl_->get_end_lsn());
830

831
  // 4. kill leader
832
  block_all_net(leader_idx);
833

834
  // 5. check committed_end_lsn
835
  int64_t new_leader_idx = -1;
836
  PalfHandleImplGuard new_leader;
837
  EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx));
838
  EXPECT_UNTIL_EQ(leader.palf_handle_impl_->get_max_lsn(), leader.palf_handle_impl_->get_end_lsn());
839
  EXPECT_EQ(max_lsn, leader.palf_handle_impl_->get_end_lsn());
840
  unblock_all_net(leader_idx);
841

842
  leader.reset();
843
  new_leader.reset();
844
  delete_paxos_group(id);
845
  PALF_LOG(INFO, "end test_lock_memberlist_opt", K(id));
846
}
847
} // end unittest
848
} // end oceanbase
849

850
int main(int argc, char **argv)
851
{
852
  RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME);
853
}
854