oceanbase

1
/**
2
 * Copyright (c) 2021 OceanBase
3
 * OceanBase CE is licensed under Mulan PubL v2.
4
 * You can use this software according to the terms and conditions of the Mulan PubL v2.
5
 * You may obtain a copy of Mulan PubL v2 at:
6
 *          http://license.coscl.org.cn/MulanPubL-2.0
7
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10
 * See the Mulan PubL v2 for more details.
11
 */
12

13
#include <cstdio>
14
#include <gtest/gtest.h>
15
#include <signal.h>
16
#include <share/scn.h>
17
#define private public
18
#include "env/ob_simple_log_cluster_env.h"
19
#undef private
20

21
const std::string TEST_NAME = "config_change_mock_ele";
22

23
using namespace oceanbase::common;
24
using namespace oceanbase;
25
namespace oceanbase
26
{
27
using namespace logservice;
28
namespace unittest
29
{
30
class TestObSimpleLogClusterConfigChangeMockEle : public ObSimpleLogClusterTestEnv
31
{
32
public:
33
  TestObSimpleLogClusterConfigChangeMockEle() :  ObSimpleLogClusterTestEnv()
34
  {}
35
};
36

37
int64_t ObSimpleLogClusterTestBase::member_cnt_ = 3;
38
int64_t ObSimpleLogClusterTestBase::node_cnt_ = 7;
39
std::string ObSimpleLogClusterTestBase::test_name_ = TEST_NAME;
40
bool ObSimpleLogClusterTestBase::need_add_arb_server_  = false;
41

42
MockLocCB loc_cb;
43

44
// switch leader after appending config log
45
TEST_F(TestObSimpleLogClusterConfigChangeMockEle, switch_leader_during_removing_member1)
46
{
47
  int ret = OB_SUCCESS;
48
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
49
  SET_CASE_LOG_FILE(TEST_NAME, "switch_leader_during_removing_member1");
50
  PALF_LOG(INFO, "begin test switch_leader_during_removing_member1", K(id));
51
  {
52
    int64_t leader_idx = 0;
53
    PalfHandleImplGuard leader;
54
    std::vector<PalfHandleImplGuard*> palf_list;
55
    EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_mock_election(id, leader_idx, leader));
56
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 200, id));
57
    EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
58

59
    const int64_t b_idx = (leader_idx + 1) % 3;
60
    const int64_t c_idx = (leader_idx + 2) % 3;
61
    PalfHandleImplGuard *a_handle = palf_list[leader_idx];
62
    PalfHandleImplGuard *b_handle = palf_list[b_idx];
63
    PalfHandleImplGuard *c_handle = palf_list[c_idx];
64

65
    // 1. A starts to remove B
66
    int64_t remove_b_pid = 0;
67
    int64_t remove_b_ele_epoch = 0;
68
    LogConfigVersion remove_b_version;
69
    const common::ObAddr b_addr = get_cluster()[b_idx]->get_addr();
70
    const common::ObAddr c_addr = get_cluster()[c_idx]->get_addr();
71
    LogConfigChangeArgs args(common::ObMember(b_addr, 1), 2, REMOVE_MEMBER);
72
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->config_mgr_.start_change_config(remove_b_pid, remove_b_ele_epoch, args.type_));
73
    EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config_(args, remove_b_pid, remove_b_ele_epoch, remove_b_version));
74

75
    EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
76
    EXPECT_TRUE(b_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
77
    EXPECT_TRUE(c_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
78

79
    // 2. B is elected to be the leader
80
    block_net(leader_idx, b_idx);
81
    for (auto srv: get_cluster()) {
82
      srv->set_leader(id, b_addr);
83
    }
84
    EXPECT_UNTIL_EQ(true, b_handle->palf_handle_impl_->state_mgr_.is_leader_active());
85

86
    // 3. A can not remove B successfully
87
    EXPECT_EQ(OB_NOT_MASTER, leader.palf_handle_impl_->config_mgr_.change_config_(args, remove_b_pid, remove_b_ele_epoch, remove_b_version));
88
    unblock_net(leader_idx, b_idx);
89

90
    // 4. A's memberlist will be reset
91
    EXPECT_UNTIL_EQ(true, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
92
    leader.reset();
93

94
    // 5. B remove C
95
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
96
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
97
    LogConfigChangeArgs remove_c_args(common::ObMember(c_addr, 1), 2, REMOVE_MEMBER);
98
    int64_t remove_c_pid = 0;
99
    int64_t remove_c_ele_epoch = 0;
100
    LogConfigVersion remove_c_version;
101
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->config_mgr_.start_change_config(remove_c_pid, remove_c_ele_epoch, remove_c_args.type_));
102
    EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config_(remove_c_args, remove_c_pid, remove_c_ele_epoch, remove_c_version));
103

104
    EXPECT_TRUE(a_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(c_addr));
105
    EXPECT_FALSE(b_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(c_addr));
106
    EXPECT_TRUE(c_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(c_addr));
107

108
    // 6. the leader B revokes and takeover again
109
    for (auto srv: get_cluster()) {
110
      const ObAddr addr1(ObAddr::IPV4, "0.0.0.0", 0);
111
      srv->set_leader(id, addr1);
112
    }
113
    EXPECT_UNTIL_EQ(false, b_handle->palf_handle_impl_->state_mgr_.is_leader_active());
114
    for (auto srv: get_cluster()) {
115
      srv->set_leader(id, b_addr, 3);
116
    }
117
    EXPECT_UNTIL_EQ(true, b_handle->palf_handle_impl_->state_mgr_.is_leader_active());
118

119
    // 7. C will be removed successfully
120
    EXPECT_FALSE(a_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(c_addr));
121
    EXPECT_FALSE(b_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(c_addr));
122

123
    revert_cluster_palf_handle_guard(palf_list);
124
  }
125
  delete_paxos_group(id);
126
  PALF_LOG(INFO, "end test switch_leader_during_removing_member1", K(id));
127
}
128

129
// switch leader after sending config meta to followers
130
TEST_F(TestObSimpleLogClusterConfigChangeMockEle, switch_leader_during_removing_member2)
131
{
132
  int ret = OB_SUCCESS;
133
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
134
  SET_CASE_LOG_FILE(TEST_NAME, "switch_leader_during_removing_member2");
135
  PALF_LOG(INFO, "begin test switch_leader_during_removing_member2", K(id));
136
  {
137
    int64_t leader_idx = 0;
138
    PalfHandleImplGuard leader;
139
    std::vector<PalfHandleImplGuard*> palf_list;
140
    EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_mock_election(id, leader_idx, leader));
141
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 200, id));
142
    EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
143

144
    const int64_t b_idx = (leader_idx + 1) % 3;
145
    const int64_t c_idx = (leader_idx + 2) % 3;
146
    PalfHandleImplGuard *b_handle = palf_list[b_idx];
147
    PalfHandleImplGuard *c_handle = palf_list[c_idx];
148

149
    // 1. A starts to remove B
150
    int64_t remove_b_pid = 0;
151
    int64_t remove_b_ele_epoch = 0;
152
    LogConfigVersion remove_b_version;
153
    const common::ObAddr b_addr = get_cluster()[b_idx]->get_addr();
154
    LogConfigChangeArgs args(common::ObMember(b_addr, 1), 2, REMOVE_MEMBER);
155
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->config_mgr_.start_change_config(remove_b_pid, remove_b_ele_epoch, args.type_));
156
    EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config_(args, remove_b_pid, remove_b_ele_epoch, remove_b_version));
157

158
    EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
159
    EXPECT_TRUE(b_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
160
    EXPECT_TRUE(c_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
161

162
    // 1. A sends config meta to C
163
    block_net(leader_idx, b_idx);
164
    while (-1 == leader.palf_handle_impl_->config_mgr_.last_submit_config_log_time_us_ ||
165
           c_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr)) {
166
      EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config_(args, remove_b_pid, remove_b_ele_epoch, remove_b_version));
167
      usleep(500);
168
    }
169

170
    EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
171
    EXPECT_UNTIL_EQ(true, b_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
172
    EXPECT_UNTIL_EQ(false, c_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
173

174
    // 2. B is elected to be the leader
175
    for (auto srv: get_cluster()) {
176
      srv->set_leader(id, b_addr);
177
    }
178
    EXPECT_UNTIL_EQ(true, b_handle->palf_handle_impl_->state_mgr_.is_leader_active());
179

180
    // 3. A can not remove B successfully
181
    EXPECT_EQ(OB_NOT_MASTER, leader.palf_handle_impl_->config_mgr_.change_config_(args, remove_b_pid, remove_b_ele_epoch, remove_b_version));
182
    unblock_net(leader_idx, b_idx);
183

184
    // 4. A's memberlist will be reset
185
    EXPECT_UNTIL_EQ(true, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
186

187
    revert_cluster_palf_handle_guard(palf_list);
188
  }
189
  delete_paxos_group(id);
190
  PALF_LOG(INFO, "end test switch_leader_during_removing_member2", K(id));
191
}
192

193
TEST_F(TestObSimpleLogClusterConfigChangeMockEle, switch_leader_during_removing_member3)
194
{
195
  int ret = OB_SUCCESS;
196
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
197
  SET_CASE_LOG_FILE(TEST_NAME, "switch_leader_during_removing_member3");
198
  PALF_LOG(INFO, "begin test switch_leader_during_removing_member3", K(id));
199
  {
200
    int64_t leader_idx = 0;
201
    PalfHandleImplGuard leader;
202
    std::vector<PalfHandleImplGuard*> palf_list;
203
    EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_mock_election(id, leader_idx, leader));
204
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 200, id));
205
    EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
206

207
    const int64_t b_idx = (leader_idx + 1) % 3;
208
    const int64_t c_idx = (leader_idx + 2) % 3;
209
    PalfHandleImplGuard *a_handle = palf_list[leader_idx];
210
    PalfHandleImplGuard *b_handle = palf_list[b_idx];
211
    PalfHandleImplGuard *c_handle = palf_list[c_idx];
212

213
    // 1. A starts to remove B
214
    int64_t remove_b_pid = 0;
215
    int64_t remove_b_ele_epoch = 0;
216
    LogConfigVersion remove_b_version;
217
    const common::ObAddr b_addr = get_cluster()[b_idx]->get_addr();
218
    const common::ObAddr c_addr = get_cluster()[c_idx]->get_addr();
219
    LogConfigChangeArgs args(common::ObMember(b_addr, 1), 2, REMOVE_MEMBER);
220
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->config_mgr_.start_change_config(remove_b_pid, remove_b_ele_epoch, args.type_));
221
    EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config_(args, remove_b_pid, remove_b_ele_epoch, remove_b_version));
222

223
    EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
224
    EXPECT_TRUE(b_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
225
    EXPECT_TRUE(c_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
226

227
    // A sends config meta to C
228
    block_net(leader_idx, b_idx);
229
    while (-1 == leader.palf_handle_impl_->config_mgr_.last_submit_config_log_time_us_ ||
230
           c_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr)) {
231
      EXPECT_EQ(OB_EAGAIN, leader.palf_handle_impl_->config_mgr_.change_config_(args, remove_b_pid, remove_b_ele_epoch, remove_b_version));
232
      usleep(500);
233
    }
234

235
    EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
236
    EXPECT_UNTIL_EQ(true, b_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
237
    EXPECT_UNTIL_EQ(false, c_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
238

239
    // 2. C is elected to be the leader
240
    for (auto srv: get_cluster()) {
241
      srv->set_leader(id, c_addr);
242
    }
243
    EXPECT_UNTIL_EQ(true, c_handle->palf_handle_impl_->state_mgr_.is_leader_active());
244

245
    // 3. A can not remove B successfully
246
    EXPECT_EQ(OB_NOT_MASTER, leader.palf_handle_impl_->config_mgr_.change_config_(args, remove_b_pid, remove_b_ele_epoch, remove_b_version));
247
    unblock_net(leader_idx, b_idx);
248

249
    // 4. A's memberlist will be re-sync by C successfully
250
    EXPECT_UNTIL_EQ(false, a_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
251
    EXPECT_UNTIL_EQ(false, c_handle->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(b_addr));
252

253
    leader.reset();
254
    revert_cluster_palf_handle_guard(palf_list);
255
  }
256
  delete_paxos_group(id);
257
  PALF_LOG(INFO, "end test switch_leader_during_removing_member3", K(id));
258
}
259

260
// 1. remove D from member list (ABCD) and match_lsn_map, committed_end_lsn is 100 and last_submit_end_lsn is 200.
261
// 2. because committed_end_lsn is smaller than last_submit_end_lsn, the leader will use prev_member_list (ABCD) to generate committed_end_lsn
262
// 3. D is not in match_lsn_map, the leader may can not generate committed_end_lsn
263
TEST_F(TestObSimpleLogClusterConfigChangeMockEle, test_committed_end_lsn_after_removing_member)
264
{
265
  int ret = OB_SUCCESS;
266
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
267
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
268
  SET_CASE_LOG_FILE(TEST_NAME, "test_committed_end_lsn_after_removing_member");
269
  PALF_LOG(INFO, "begin test test_committed_end_lsn_after_removing_member", K(id));
270
  {
271
    int64_t leader_idx = 0;
272
    PalfHandleImplGuard leader;
273
    std::vector<PalfHandleImplGuard*> palf_list;
274
    EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_mock_election(id, leader_idx, leader));
275
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 200, id));
276
    EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
277

278
    const int64_t b_idx = (leader_idx + 1) % 4;
279
    const int64_t c_idx = (leader_idx + 2) % 4;
280
    const int64_t d_idx = (leader_idx + 3) % 4;
281
    const common::ObAddr b_addr = get_cluster()[b_idx]->get_addr();
282
    const common::ObAddr c_addr = get_cluster()[c_idx]->get_addr();
283
    const common::ObAddr d_addr = get_cluster()[d_idx]->get_addr();
284
    PalfHandleImplGuard *a_handle = palf_list[leader_idx];
285
    PalfHandleImplGuard *b_handle = palf_list[b_idx];
286
    PalfHandleImplGuard *c_handle = palf_list[c_idx];
287
    PalfHandleImplGuard *d_handle = palf_list[d_idx];
288
    LogConfigVersion config_version;
289
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
290
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(common::ObMember(d_addr, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
291

292
    // 1. leader can not commit logs
293
    block_pcode(leader_idx, ObRpcPacketCode::OB_LOG_PUSH_RESP);
294
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
295
    sleep(1);
296
    EXPECT_GT(leader.palf_handle_impl_->sw_.last_submit_lsn_, leader.palf_handle_impl_->sw_.committed_end_lsn_);
297

298
    // 2. remove D
299
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(common::ObMember(d_addr, 1), 3, CONFIG_CHANGE_TIMEOUT));
300
    EXPECT_GT(leader.palf_handle_impl_->sw_.last_submit_lsn_, leader.palf_handle_impl_->sw_.committed_end_lsn_);
301
    EXPECT_EQ(leader.palf_handle_impl_->config_mgr_.reconfig_barrier_.prev_end_lsn_, leader.palf_handle_impl_->sw_.last_submit_end_lsn_);
302
    EXPECT_GT(leader.palf_handle_impl_->config_mgr_.reconfig_barrier_.prev_end_lsn_, leader.palf_handle_impl_->sw_.committed_end_lsn_);
303

304
    // 3. leader can commit logs
305
    unblock_pcode(leader_idx, ObRpcPacketCode::OB_LOG_PUSH_RESP);
306

307
    // 4. check if the leader can commit logs after D has been removed from match_lsn_map
308
    EXPECT_UNTIL_EQ(leader.palf_handle_impl_->sw_.committed_end_lsn_, leader.palf_handle_impl_->sw_.last_submit_end_lsn_);
309

310
    leader.reset();
311
    revert_cluster_palf_handle_guard(palf_list);
312
  }
313
  delete_paxos_group(id);
314
  PALF_LOG(INFO, "end test test_committed_end_lsn_after_removing_member", K(id));
315
}
316

317
TEST_F(TestObSimpleLogClusterConfigChangeMockEle, test_remove_if_another_rebuild)
318
{
319
  int ret = OB_SUCCESS;
320
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
321
  const int64_t CONFIG_CHANGE_TIMEOUT = 4 * 1000 * 1000L; // 10s
322
  SET_CASE_LOG_FILE(TEST_NAME, "test_remove_if_another_rebuild");
323
  PALF_LOG(INFO, "begin test test_remove_if_another_rebuild", K(id));
324
  {
325
    int64_t leader_idx = 0;
326
    PalfHandleImplGuard leader;
327
    std::vector<PalfHandleImplGuard*> palf_list;
328
    EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_mock_election(id, leader_idx, leader));
329
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 200, id));
330
    EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
331

332
    const int64_t b_idx = (leader_idx + 1) % 4;
333
    const int64_t c_idx = (leader_idx + 2) % 4;
334
    const int64_t d_idx = (leader_idx + 3) % 4;
335
    const common::ObAddr b_addr = get_cluster()[b_idx]->get_addr();
336
    const common::ObAddr c_addr = get_cluster()[c_idx]->get_addr();
337
    const common::ObAddr d_addr = get_cluster()[d_idx]->get_addr();
338
    PalfHandleImplGuard *a_handle = palf_list[leader_idx];
339
    PalfHandleImplGuard *b_handle = palf_list[b_idx];
340
    PalfHandleImplGuard *c_handle = palf_list[c_idx];
341
    PalfHandleImplGuard *d_handle = palf_list[d_idx];
342
    LogConfigVersion config_version;
343

344
    // 1. disable vote
345
    EXPECT_EQ(OB_SUCCESS, c_handle->palf_handle_impl_->disable_vote(false));
346
    EXPECT_EQ(OB_TIMEOUT, leader.palf_handle_impl_->remove_member(common::ObMember(b_addr, 1), 2, CONFIG_CHANGE_TIMEOUT));
347
    EXPECT_EQ(OB_SUCCESS, c_handle->palf_handle_impl_->enable_vote());
348

349
    // 2. disable sync
350
    EXPECT_EQ(OB_SUCCESS, c_handle->palf_handle_impl_->disable_sync());
351
    EXPECT_EQ(OB_TIMEOUT, leader.palf_handle_impl_->remove_member(common::ObMember(b_addr, 1), 2, CONFIG_CHANGE_TIMEOUT));
352
    EXPECT_EQ(OB_SUCCESS, c_handle->palf_handle_impl_->enable_sync());
353

354
    // 3. need rebuild
355
    c_handle->palf_handle_impl_->last_rebuild_lsn_ = LSN(leader.get_palf_handle_impl()->get_max_lsn().val_ + 100000);
356
    EXPECT_EQ(OB_TIMEOUT, leader.palf_handle_impl_->remove_member(common::ObMember(b_addr, 1), 2, CONFIG_CHANGE_TIMEOUT));
357

358
    leader.reset();
359
    revert_cluster_palf_handle_guard(palf_list);
360
  }
361
  delete_paxos_group(id);
362
  PALF_LOG(INFO, "end test test_committed_end_lsn_after_removing_member", K(id));
363
}
364

365
// 1. (ABCD), A is the leader, D crashed
366
// 2. remove C from member list (ABCD) and match_lsn_map, committed_end_lsn is 100 and last_submit_end_lsn is 200.
367
// 2. because committed_end_lsn is smaller than last_submit_end_lsn, the leader will use prev_member_list (ABCD) to generate committed_end_lsn
368
// 3. C is not in match_lsn_map, D crashed, the leader may can not generate committed_end_lsn
369
TEST_F(TestObSimpleLogClusterConfigChangeMockEle, test_committed_end_lsn_after_removing_member2)
370
{
371
  int ret = OB_SUCCESS;
372
  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
373
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
374
  SET_CASE_LOG_FILE(TEST_NAME, "test_committed_end_lsn_after_removing_member");
375
  PALF_LOG(INFO, "begin test test_committed_end_lsn_after_removing_member", K(id));
376
  {
377
    int64_t leader_idx = 0;
378
    PalfHandleImplGuard leader;
379
    std::vector<PalfHandleImplGuard*> palf_list;
380
    EXPECT_EQ(OB_SUCCESS, create_paxos_group_with_mock_election(id, leader_idx, leader));
381
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 200, id));
382
    EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
383

384
    const int64_t b_idx = (leader_idx + 1) % 4;
385
    const int64_t c_idx = (leader_idx + 2) % 4;
386
    const int64_t d_idx = (leader_idx + 3) % 4;
387
    const common::ObAddr b_addr = get_cluster()[b_idx]->get_addr();
388
    const common::ObAddr c_addr = get_cluster()[c_idx]->get_addr();
389
    const common::ObAddr d_addr = get_cluster()[d_idx]->get_addr();
390
    PalfHandleImplGuard *a_handle = palf_list[leader_idx];
391
    PalfHandleImplGuard *b_handle = palf_list[b_idx];
392
    PalfHandleImplGuard *c_handle = palf_list[c_idx];
393
    PalfHandleImplGuard *d_handle = palf_list[d_idx];
394
    LogConfigVersion config_version;
395
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
396
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(common::ObMember(d_addr, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
397

398
    // 1. D crashed
399
    block_all_net(d_idx);
400

401
    // 2. leader can not commit logs
402
    block_pcode(leader_idx, ObRpcPacketCode::OB_LOG_PUSH_RESP);
403
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
404
    sleep(1);
405
    EXPECT_GT(leader.palf_handle_impl_->sw_.last_submit_lsn_, leader.palf_handle_impl_->sw_.committed_end_lsn_);
406

407
    // 3. remove C
408
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(common::ObMember(c_addr, 1), 3, CONFIG_CHANGE_TIMEOUT));
409
    EXPECT_GT(leader.palf_handle_impl_->sw_.last_submit_lsn_, leader.palf_handle_impl_->sw_.committed_end_lsn_);
410
    EXPECT_EQ(leader.palf_handle_impl_->config_mgr_.reconfig_barrier_.prev_end_lsn_, leader.palf_handle_impl_->sw_.last_submit_end_lsn_);
411
    EXPECT_GT(leader.palf_handle_impl_->config_mgr_.reconfig_barrier_.prev_end_lsn_, leader.palf_handle_impl_->sw_.committed_end_lsn_);
412

413
    // 3. leader can commit logs
414
    unblock_pcode(leader_idx, ObRpcPacketCode::OB_LOG_PUSH_RESP);
415

416
    // 4. check if the leader can commit logs after C has been removed from match_lsn_map
417
    EXPECT_UNTIL_EQ(leader.palf_handle_impl_->sw_.committed_end_lsn_, leader.palf_handle_impl_->sw_.last_submit_end_lsn_);
418

419
    leader.reset();
420
    revert_cluster_palf_handle_guard(palf_list);
421
  }
422
  delete_paxos_group(id);
423
  PALF_LOG(INFO, "end test test_committed_end_lsn_after_removing_member", K(id));
424
}
425

426
} // end unittest
427
} // end oceanbase
428

429
int main(int argc, char **argv)
430
{
431
  RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME);
432
}
433