oceanbase

1
/**
2
 * Copyright (c) 2021 OceanBase
3
 * OceanBase CE is licensed under Mulan PubL v2.
4
 * You can use this software according to the terms and conditions of the Mulan PubL v2.
5
 * You may obtain a copy of Mulan PubL v2 at:
6
 *          http://license.coscl.org.cn/MulanPubL-2.0
7
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10
 * See the Mulan PubL v2 for more details.
11
 */
12

13
#include <cstdio>
14
#include <gtest/gtest.h>
15
#include <signal.h>
16
#include <share/scn.h>
17
#define private public
18
#include "logservice/palf/log_config_mgr.h"
19
#include "env/ob_simple_log_cluster_env.h"
20
#undef private
21

22
const std::string TEST_NAME = "config_change";
23

24
using namespace oceanbase::common;
25
using namespace oceanbase;
26
namespace oceanbase
27
{
28
using namespace logservice;
29
namespace unittest
30
{
31
class TestObSimpleLogClusterConfigChange : public ObSimpleLogClusterTestEnv
32
{
33
public:
34
  TestObSimpleLogClusterConfigChange() :  ObSimpleLogClusterTestEnv()
35
  {}
36
};
37

38
int64_t ObSimpleLogClusterTestBase::member_cnt_ = 3;
39
int64_t ObSimpleLogClusterTestBase::node_cnt_ = 7;
40
std::string ObSimpleLogClusterTestBase::test_name_ = TEST_NAME;
41
bool ObSimpleLogClusterTestBase::need_add_arb_server_  = false;
42

43
bool check_children_valid(const std::vector<PalfHandleImplGuard*> &palf_list, const LogLearnerList &all_learner)
44
{
45
  int ret = OB_SUCCESS;
46
  LogLearnerList all_children;
47
  for (auto palf : palf_list) {
48
    const LogLearnerList &self_children = palf->palf_handle_impl_->config_mgr_.children_;
49
    int64_t children_cnt = self_children.get_member_number();
50
    for (int i = 0; i < children_cnt; ++i) {
51
      LogLearner tmp_server;
52
      if (OB_FAIL(self_children.get_learner(i, tmp_server))) {
53
      } else if (OB_FAIL(all_children.add_learner(tmp_server))) {
54
      }
55
    }
56
  }
57
  bool bool_ret = all_children.learner_addr_equal(all_learner);
58
  PALF_LOG(INFO, "check_children", K(ret), K(all_children), K(all_learner));
59
  return bool_ret;
60
}
61

62
bool check_parent(const std::vector<PalfHandleImplGuard*> &palf_list, const LogLearnerList &all_learner, const ObAddr &parent)
63
{
64
  bool bool_ret = true;
65
  for (auto palf : palf_list) {
66
    const ObAddr &self = palf->palf_handle_impl_->self_;
67
    if (all_learner.contains(self)) {
68
      const ObAddr &my_parent = palf->palf_handle_impl_->config_mgr_.parent_;
69
      if (my_parent.is_valid() && (my_parent == parent)) {
70
        continue;
71
      } else {
72
        bool_ret = false;
73
        break;
74
      }
75
    }
76
  }
77
  return bool_ret;
78
}
79

80
int check_log_sync(const std::vector<PalfHandleImplGuard*> &palf_list,
81
                   const common::ObMemberList &member_list,
82
                   const LogLearnerList &learner_list,
83
                   PalfHandleImplGuard &leader)
84
{
85
  int ret = OB_SUCCESS;
86
  const int64_t max_flushed_proposal_id = leader.palf_handle_impl_->sw_.max_flushed_log_pid_;
87
  const LSN max_flushed_end_lsn = leader.palf_handle_impl_->sw_.max_flushed_end_lsn_;
88
  PALF_LOG(INFO, "before check_log_sync", K(max_flushed_proposal_id), K(max_flushed_end_lsn));
89
  for (auto palf : palf_list) {
90
    const common::ObAddr parent_addr = palf->palf_handle_impl_->config_mgr_.parent_;
91
    // for paxos member, log must sync with leader
92
    // for learner, sync with parent's committed_end_lsn
93
    const int64_t this_max_flushed_proposal_id = palf->palf_handle_impl_->sw_.max_flushed_log_pid_;
94
    const LSN this_max_flushed_end_lsn = palf->palf_handle_impl_->sw_.max_flushed_end_lsn_;
95
    const common::ObAddr self = palf->palf_handle_impl_->self_;
96
    if (member_list.contains(self)) {
97
      PALF_LOG(INFO, "before check", K(max_flushed_proposal_id), K(max_flushed_end_lsn), K(this_max_flushed_proposal_id),
98
          K(this_max_flushed_end_lsn), K(self));
99
      EXPECT_EQ(max_flushed_proposal_id, this_max_flushed_proposal_id);
100
      EXPECT_EQ(max_flushed_end_lsn, this_max_flushed_end_lsn) \
101
          << max_flushed_end_lsn.val_ << "," << this_max_flushed_end_lsn.val_;
102
    } else if (!learner_list.contains(self)) {
103
    } else if (!parent_addr.is_valid()) {
104
      ret = OB_ERR_UNEXPECTED;
105
      break;
106
    } else {
107
      for (auto parent: palf_list) {
108
        if (parent->palf_handle_impl_->self_ == parent_addr) {
109
          const LSN parent_committed_end_lsn = parent->palf_handle_impl_->sw_.committed_end_lsn_;
110
          EXPECT_EQ(parent_committed_end_lsn, this_max_flushed_end_lsn) \
111
              << parent_committed_end_lsn.val_ << "," << this_max_flushed_end_lsn.val_;
112
          if (parent_committed_end_lsn != this_max_flushed_end_lsn) {
113
            PALF_LOG(ERROR, "log not sync", K(parent_addr), "self", palf->palf_handle_impl_->self_,
114
                K(parent_committed_end_lsn), K(this_max_flushed_end_lsn));
115
          }
116
        }
117
      }
118
    }
119
  }
120
  return ret;
121
}
122

123
MockLocCB loc_cb;
124

125
TEST_F(TestObSimpleLogClusterConfigChange, split_brain)
126
{
127
  SET_CASE_LOG_FILE(TEST_NAME, "split_brain");
128
  std::vector<PalfHandleImplGuard*> palf_list;
129
  int ret = OB_SUCCESS;
130
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
131
  PALF_LOG(INFO, "begin test split_brain", K(id));
132
	int64_t leader_idx = 0;
133
  // 1. A, B, C
134
  // 2. B block_net
135
  // 3. A, C, D
136
  // 4. A, D. delete C
137
  // 5. unblock_net B, create new C
138
  // 6. may be split-brain {B, C} {A, D}
139
  {
140
    PalfHandleImplGuard leader;
141
    const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
142
    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, &loc_cb, leader_idx, leader));
143
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 200, id));
144
    PALF_LOG(INFO, "after submit_log");
145
    const int64_t follower_B_idx = (leader_idx + 1);
146
    const int64_t follower_C_idx = (leader_idx + 2);
147
    const int64_t follower_D_idx = (leader_idx + 3);
148
    // step 2
149
    EXPECT_EQ(OB_SUCCESS, get_cluster()[follower_B_idx]->simple_close(false));
150
    // step 3
151
    const ObAddr follower_b_addr = get_cluster()[follower_B_idx]->get_addr();
152
    PALF_LOG(INFO, "before remove member");
153
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(follower_b_addr, 1), 3, CONFIG_CHANGE_TIMEOUT));
154
    PALF_LOG(INFO, "after remove member");
155

156
    PalfHandleImplGuard new_leader;
157
    int64_t new_leader_idx;
158
    EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx));
159
    loc_cb.leader_ = get_cluster()[new_leader_idx]->get_addr();
160
    PALF_LOG(INFO, "set leader for loc_cb", "leader", loc_cb.leader_);
161

162
    LogConfigVersion config_version;
163
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
164
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(get_cluster()[follower_D_idx]->get_addr(), 1), 3, config_version, CONFIG_CHANGE_TIMEOUT));
165
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
166
    // step 4
167
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(get_cluster()[follower_C_idx]->get_addr(), 1), 3, CONFIG_CHANGE_TIMEOUT));
168
    EXPECT_EQ(OB_SUCCESS, get_cluster()[follower_C_idx]->get_palf_env()->remove_palf_handle_impl(id));
169
    // step 5
170
    const int64_t node_id = follower_B_idx * 2 + get_node_idx_base();
171
    EXPECT_EQ(OB_SUCCESS, get_cluster()[follower_B_idx]->simple_init(get_test_name(), follower_b_addr, node_id, false));
172
    EXPECT_EQ(OB_SUCCESS, get_cluster()[follower_B_idx]->simple_start(false));
173
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
174
    PalfBaseInfo palf_base_info;
175
    IPalfHandleImpl* follower_C_handle = NULL;
176
    palf_base_info.generate_by_default();
177
    //PalfHandleImplGuard leader;
178
    palf_base_info.prev_log_info_.scn_ = share::SCN::min_scn();
179
    EXPECT_EQ(OB_SUCCESS, get_cluster()[follower_C_idx]->get_palf_env()->create_palf_handle_impl(id, palf::AccessMode::APPEND, palf_base_info, follower_C_handle));
180
    get_cluster()[follower_C_idx]->get_palf_env()->revert_palf_handle_impl(follower_C_handle);
181
    sleep(5);
182
    // check if split-brain happens
183
    int64_t leader_cnt = 0;
184
    EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
185
    for (auto palf: palf_list) {
186
      ObRole role;
187
      palf::ObReplicaState unused_state;
188
      palf->palf_handle_impl_->state_mgr_.get_role_and_state(role, unused_state);
189
      if (role == LEADER) {
190
        leader_cnt += 1;
191
      }
192
    }
193
    EXPECT_EQ(leader_cnt, 1);
194
    revert_cluster_palf_handle_guard(palf_list);
195
  }
196
  delete_paxos_group(id);
197
  PALF_LOG(INFO, "end test split_brain", K(id));
198
}
199

200
TEST_F(TestObSimpleLogClusterConfigChange, test_config_change_defensive)
201
{
202
  SET_CASE_LOG_FILE(TEST_NAME, "config_change_defensive");
203
  int ret = OB_SUCCESS;
204
  PALF_LOG(INFO, "begin test_config_change_defensive");
205
  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
206
  {
207
    int64_t leader_idx = 0;
208
    PalfHandleImplGuard leader;
209
    std::vector<PalfHandleImplGuard*> palf_list;
210
    const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
211
    common::ObMember dummy_member;
212
    const LogConfigChangeArgs args(dummy_member, 1, STARTWORKING);
213
	EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, &loc_cb, leader_idx, leader));
214
    EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
215
    const int64_t lag_follower_idx = (leader_idx+1)%3;
216
    const int64_t remove_follower_idx = (leader_idx+2)%3;
217
    block_net(leader_idx, lag_follower_idx);
218
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
219
    sleep(1);
220
    // remove member who is in majority
221
    EXPECT_EQ(OB_TIMEOUT, leader.palf_handle_impl_->remove_member(ObMember(palf_list[remove_follower_idx]->palf_handle_impl_->self_, 1), 2, CONFIG_CHANGE_TIMEOUT / 2));
222
    unblock_net(leader_idx, lag_follower_idx);
223
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
224
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(palf_list[remove_follower_idx]->palf_handle_impl_->self_, 1), 2, CONFIG_CHANGE_TIMEOUT));
225

226
    int64_t new_leader_idx;
227
    PalfHandleImplGuard new_leader;
228
    EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx));
229
    loc_cb.leader_ = get_cluster()[new_leader_idx]->get_addr();
230
    PALF_LOG(INFO, "set leader for loc_cb", "leader", get_cluster()[new_leader_idx]->get_addr());
231

232
    LogConfigVersion config_version;
233
    ASSERT_EQ(OB_SUCCESS, new_leader.palf_handle_impl_->get_config_version(config_version));
234
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[remove_follower_idx]->palf_handle_impl_->self_, 1), 3, config_version, CONFIG_CHANGE_TIMEOUT));
235
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10, id));
236
    // majority of members should be normal
237
    palf_list[lag_follower_idx]->palf_handle_impl_->disable_vote(false);
238
    EXPECT_EQ(OB_TIMEOUT, leader.palf_handle_impl_->remove_member(ObMember(palf_list[remove_follower_idx]->palf_handle_impl_->self_, 1), 2, CONFIG_CHANGE_TIMEOUT / 2));
239
    palf_list[lag_follower_idx]->palf_handle_impl_->enable_vote();
240

241
    block_net(leader_idx, lag_follower_idx);
242

243
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
244
    sleep(3);
245

246
    // add member without any logs
247
    // we will send config log to 3 before add_member(3, 4), so 3 can know who is leader and fetch log from it.
248
    // so just comment this line, after leader forbit fetch log req that comes from node who is not follower or
249
    // children of leader, we will uncomment this line
250
    // EXPECT_EQ(OB_TIMEOUT, leader.add_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 4, CONFIG_CHANGE_TIMEOUT / 2));
251
    unblock_net(leader_idx, lag_follower_idx);
252
    ASSERT_EQ(OB_SUCCESS, new_leader.palf_handle_impl_->get_config_version(config_version));
253
    EXPECT_EQ(OB_SUCCESS, new_leader.palf_handle_impl_->add_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
254
    //test add_member_with_check
255
    EXPECT_EQ(OB_SUCCESS, new_leader.palf_handle_impl_->remove_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 3, CONFIG_CHANGE_TIMEOUT));
256
    LogConfigVersion invalid_config_version;
257
    int64_t follower_idx = (new_leader_idx +1) % 3;
258
    EXPECT_EQ(OB_STATE_NOT_MATCH, new_leader.palf_handle_impl_->add_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 4, invalid_config_version, CONFIG_CHANGE_TIMEOUT));
259
    EXPECT_EQ(OB_SUCCESS, new_leader.palf_handle_impl_->get_config_version(config_version));
260
    EXPECT_EQ(OB_NOT_MASTER, palf_list[follower_idx]->palf_handle_impl_->add_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
261
    EXPECT_EQ(true, config_version.is_valid());
262
    EXPECT_EQ(OB_SUCCESS, new_leader.palf_handle_impl_->add_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
263

264
    revert_cluster_palf_handle_guard(palf_list);
265
  }
266
  delete_paxos_group(id);
267
  PALF_LOG(INFO, "end test_config_change_defensive", K(id));
268
}
269

270
TEST_F(TestObSimpleLogClusterConfigChange, test_change_replica_num)
271
{
272
  SET_CASE_LOG_FILE(TEST_NAME, "change_replica_num");
273
  int ret = OB_SUCCESS;
274
	const int64_t id = ATOMIC_FAA(&palf_id_, 1);
275
  PALF_LOG(INFO, "begin test_change_replica_num", K(id));
276
  {
277
	  int64_t leader_idx = 0;
278
    PalfHandleImplGuard leader;
279
    const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
280
    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, &loc_cb, leader_idx, leader));
281
    PalfHandleImplGuard new_leader;
282
    int64_t new_leader_idx;
283
    EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx));
284
    loc_cb.leader_ = get_cluster()[new_leader_idx]->get_addr();
285
    PALF_LOG(INFO, "set leader for loc_cb", "leader", get_cluster()[new_leader_idx]->get_addr());
286

287
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
288
    // 3->5
289
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->change_replica_num(get_member_list(), 3, 5, CONFIG_CHANGE_TIMEOUT));
290
    LogConfigVersion config_version;
291
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
292
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(get_cluster()[4]->get_addr(), 1), 5, config_version, CONFIG_CHANGE_TIMEOUT));
293
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
294
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(get_cluster()[5]->get_addr(), 1), 5, config_version, CONFIG_CHANGE_TIMEOUT));
295
    common::ObMemberList curr_member_list;
296
    int64_t curr_replica_num;
297
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(get_cluster()[4]->get_addr(), 1), 5, CONFIG_CHANGE_TIMEOUT));
298
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(get_cluster()[5]->get_addr(), 1), 5, CONFIG_CHANGE_TIMEOUT));
299
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_paxos_member_list(curr_member_list, curr_replica_num));
300
    // 5->3
301
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->change_replica_num(curr_member_list, curr_replica_num, 3, CONFIG_CHANGE_TIMEOUT));
302
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 200, id));
303
    // 3->4
304
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_paxos_member_list(curr_member_list, curr_replica_num));
305
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->change_replica_num(curr_member_list, curr_replica_num, 4, CONFIG_CHANGE_TIMEOUT));
306
  }
307
  delete_paxos_group(id);
308
  PALF_LOG(INFO, "end test_change_replica_num", K(id));
309
}
310

311
TEST_F(TestObSimpleLogClusterConfigChange, test_basic_config_change)
312
{
313
  SET_CASE_LOG_FILE(TEST_NAME, "config_change");
314
  int ret = OB_SUCCESS;
315
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
316
  PALF_LOG(INFO, "begin test config change", K(id));
317
  {
318
	  int64_t leader_idx = 0;
319
    std::vector<PalfHandleImplGuard*> palf_list;
320
    const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
321
    PalfHandleImplGuard leader;
322
    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, &loc_cb, leader_idx, leader));
323
    PalfHandleImplGuard new_leader;
324
    int64_t new_leader_idx;
325
    EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx));
326
    loc_cb.leader_ = get_cluster()[new_leader_idx]->get_addr();
327
    PALF_LOG(INFO, "set leader for loc_cb", "leader", get_cluster()[new_leader_idx]->get_addr());
328
    EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
329
    // add member when no log
330
    LogConfigVersion config_version;
331
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
332
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
333
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 3, CONFIG_CHANGE_TIMEOUT));
334
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(palf_list[(leader_idx+1)%3]->palf_handle_impl_->self_, 1), 2, CONFIG_CHANGE_TIMEOUT));
335
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(palf_list[(leader_idx+2)%3]->palf_handle_impl_->self_, 1), 1, CONFIG_CHANGE_TIMEOUT));
336
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
337
    EXPECT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->add_member(ObMember(palf_list[(leader_idx+1)%3]->palf_handle_impl_->self_, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
338
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[(leader_idx+1)%3]->palf_handle_impl_->self_, 1), 2, config_version, CONFIG_CHANGE_TIMEOUT));
339
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
340
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[(leader_idx+2)%3]->palf_handle_impl_->self_, 1), 3, config_version, CONFIG_CHANGE_TIMEOUT));
341
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
342
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
343
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
344

345
    // add member when contains log
346
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
347
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[4]->palf_handle_impl_->self_, 1), 5, config_version, CONFIG_CHANGE_TIMEOUT));
348
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
349

350
    // remove member
351
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(palf_list[4]->palf_handle_impl_->self_, 1), 4, CONFIG_CHANGE_TIMEOUT));
352

353
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
354

355

356
    EXPECT_EQ(OB_STATE_NOT_MATCH, leader.palf_handle_impl_->replace_member(ObMember(palf_list[5]->palf_handle_impl_->self_, 1),
357
                                                                           ObMember(palf_list[3]->palf_handle_impl_->self_, 1),
358
                                                                           config_version,
359
                                                                           CONFIG_CHANGE_TIMEOUT));
360
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
361
    // replace member
362
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_member(ObMember(palf_list[5]->palf_handle_impl_->self_, 1),
363
                                                                   ObMember(palf_list[3]->palf_handle_impl_->self_, 1),
364
                                                                   config_version,
365
                                                                   CONFIG_CHANGE_TIMEOUT));
366

367
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
368
    // switch acceptor to learner
369
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->switch_acceptor_to_learner(ObMember(palf_list[5]->palf_handle_impl_->self_, 1), 3, CONFIG_CHANGE_TIMEOUT));
370
    // add learner
371
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), CONFIG_CHANGE_TIMEOUT));
372
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(ObMember(palf_list[4]->palf_handle_impl_->self_, 1), CONFIG_CHANGE_TIMEOUT));
373
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(ObMember(palf_list[6]->palf_handle_impl_->self_, 1), CONFIG_CHANGE_TIMEOUT));
374

375
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
376
    revert_cluster_palf_handle_guard(palf_list);
377
  }
378
  delete_paxos_group(id);
379
  PALF_LOG(INFO, "end test config change", K(id));
380
}
381

382
TEST_F(TestObSimpleLogClusterConfigChange, test_basic_config_change_for_migration)
383
{
384
  SET_CASE_LOG_FILE(TEST_NAME, "config_change_for_migration");
385
  int ret = OB_SUCCESS;
386
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
387
  PALF_LOG(INFO, "begin test config change", K(id));
388
  {
389
	  int64_t leader_idx = 0;
390
    std::vector<PalfHandleImplGuard*> palf_list;
391
    const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
392
    PalfHandleImplGuard leader;
393
    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, &loc_cb, leader_idx, leader));
394
    PalfHandleImplGuard new_leader;
395
    int64_t new_leader_idx;
396
    EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx));
397
    loc_cb.leader_ = get_cluster()[new_leader_idx]->get_addr();
398
    PALF_LOG(INFO, "set leader for loc_cb", "leader", get_cluster()[new_leader_idx]->get_addr());
399
    EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
400
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
401
    const common::ObAddr &addr2 = get_cluster()[(leader_idx+2)%3]->get_addr();
402
    const common::ObAddr &addr3 = get_cluster()[3]->get_addr();
403
    const common::ObAddr &addr4 = get_cluster()[4]->get_addr();
404
    const common::ObAddr &addr5 = get_cluster()[5]->get_addr();
405
    // 1. replicate an FULL replica
406
    {
407
      PALF_LOG(INFO, "CASE1: replicate an FULL replica", K(id));
408
      common::ObMember added_member = ObMember(addr3, 1);
409
      added_member.set_migrating();
410
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(added_member, CONFIG_CHANGE_TIMEOUT));
411
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_member));
412
      EXPECT_EQ(3, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_);
413

414
      // clean
415
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_learner(added_member, CONFIG_CHANGE_TIMEOUT));
416
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_member));
417
      // add again
418
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(added_member, CONFIG_CHANGE_TIMEOUT));
419
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_member));
420

421
      LogConfigVersion config_version;
422
      ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
423
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->switch_learner_to_acceptor(added_member, 4, config_version, CONFIG_CHANGE_TIMEOUT));
424
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_member));
425
      // member with flag do not exist
426
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(added_member));
427
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(added_member.get_server()));
428
      EXPECT_EQ(4, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_);
429
      // reentrant, do not get config_version again
430
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->switch_learner_to_acceptor(added_member, 4, config_version, CONFIG_CHANGE_TIMEOUT));
431
      // reset environment
432
      added_member.reset_migrating();
433
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(added_member, 3, CONFIG_CHANGE_TIMEOUT));
434
    }
435

436
    // 2. migrate an FULL replica (addr2 -> addr3)
437
    {
438
      PALF_LOG(INFO, "CASE2: migrate an FULL replica", K(id));
439
      common::ObMember added_member = ObMember(addr3, 1);
440
      added_member.set_migrating();
441
      common::ObMember replaced_member = ObMember(addr2, 1);
442
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(added_member, CONFIG_CHANGE_TIMEOUT));
443
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_member));
444
      EXPECT_EQ(3, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_);
445
      LogConfigVersion config_version;
446
      ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
447
      LogConfigChangeArgs args(added_member, 0, config_version, SWITCH_LEARNER_TO_ACCEPTOR_AND_NUM);
448
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->one_stage_config_change_(args, CONFIG_CHANGE_TIMEOUT));
449
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_member));
450
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(added_member));
451
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(added_member.get_server()));
452
      EXPECT_EQ(4, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_);
453

454
      ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
455
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_member_with_learner(added_member, replaced_member, config_version, CONFIG_CHANGE_TIMEOUT));
456
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_member));
457
      // member with flag do not exist
458
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(added_member));
459
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(added_member.get_server()));
460
      EXPECT_EQ(3, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_);
461
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(replaced_member.get_server()));
462
      // reentrant and check
463
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_member_with_learner(added_member, replaced_member, config_version, CONFIG_CHANGE_TIMEOUT));
464
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_member));
465
      // member with flag do not exist
466
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(added_member));
467
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(added_member.get_server()));
468
      EXPECT_EQ(3, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_);
469
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(replaced_member.get_server()));
470
      // reset environment
471
      added_member.reset_migrating();
472
      ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
473
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_member(replaced_member, added_member, config_version, CONFIG_CHANGE_TIMEOUT));
474
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(added_member.get_server()));
475
      EXPECT_EQ(3, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_);
476
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_memberlist_.contains(replaced_member.get_server()));
477
    }
478

479
    // 3. replicate an READONLY replica
480
    {
481
      PALF_LOG(INFO, "CASE3: replicate an READONLY replica", K(id));
482
      // learner's addr must be different from members'
483
      common::ObMember migrating_member = ObMember(addr2, 1);
484
      EXPECT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->add_learner(migrating_member, CONFIG_CHANGE_TIMEOUT));
485

486
      common::ObMember added_migrating_learner = ObMember(addr3, 1);
487
      common::ObMember added_learner = ObMember(addr3, 1);
488
      added_migrating_learner.set_migrating();
489
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(added_migrating_learner, CONFIG_CHANGE_TIMEOUT));
490
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_migrating_learner));
491
      EXPECT_EQ(3, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_);
492
      ObMemberList added_learners, removed_learners;
493
      EXPECT_EQ(OB_SUCCESS, added_learners.add_member(added_learner));
494
      EXPECT_EQ(OB_SUCCESS, removed_learners.add_member(added_migrating_learner));
495
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_learners(added_learners, removed_learners, CONFIG_CHANGE_TIMEOUT));
496
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_learner));
497
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_migrating_learner));
498
      // reentrant and check
499
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_learners(added_learners, removed_learners, CONFIG_CHANGE_TIMEOUT));
500
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_learner));
501
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_migrating_learner));
502
      // reset environment
503
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_learner(added_learner, CONFIG_CHANGE_TIMEOUT));
504
    }
505

506
    // 4. migrate an READONLY replica, addr4 -> addr3
507
    {
508
      PALF_LOG(INFO, "CASE4: migrate an READONLY replica", K(id));
509
      common::ObMember removed_learner = ObMember(addr4, 1);
510
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(removed_learner, CONFIG_CHANGE_TIMEOUT));
511
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(removed_learner));
512

513
      common::ObMember added_migrating_learner = ObMember(addr3, 1);
514
      common::ObMember added_learner = ObMember(addr3, 1);
515
      added_migrating_learner.set_migrating();
516
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(added_migrating_learner, CONFIG_CHANGE_TIMEOUT));
517
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_migrating_learner));
518
      EXPECT_EQ(3, leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.log_sync_replica_num_);
519
      ObMemberList added_learners, removed_learners;
520
      EXPECT_EQ(OB_SUCCESS, added_learners.add_member(added_learner));
521
      EXPECT_EQ(OB_SUCCESS, removed_learners.add_member(added_migrating_learner));
522
      EXPECT_EQ(OB_SUCCESS, removed_learners.add_member(removed_learner));
523
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_learners(added_learners, removed_learners, CONFIG_CHANGE_TIMEOUT));
524
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_learner));
525
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_migrating_learner));
526
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(removed_learner));
527
      // reentrant and check
528
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_learners(added_learners, removed_learners, CONFIG_CHANGE_TIMEOUT));
529
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_learner));
530
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(added_migrating_learner));
531
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(removed_learner));
532
      // reset environment
533
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_learner(added_learner, CONFIG_CHANGE_TIMEOUT));
534
    }
535
    // 5. replace_learners (addr3, addr4) -> (addr3, addr5)
536
    {
537
      PALF_LOG(INFO, "CASE5: replace_learners", K(id));
538
      const common::ObMember member2 = ObMember(addr2, 1);
539
      const common::ObMember member3 = ObMember(addr3, 1);
540
      const common::ObMember member4 = ObMember(addr4, 1);
541
      const common::ObMember member5 = ObMember(addr5, 1);
542
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(member3, CONFIG_CHANGE_TIMEOUT));
543
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(member4, CONFIG_CHANGE_TIMEOUT));
544
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member3));
545
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member4));
546

547
      ObMemberList added_learners, removed_learners;
548
      EXPECT_EQ(OB_SUCCESS, added_learners.add_member(member3));
549
      EXPECT_EQ(OB_SUCCESS, added_learners.add_member(member5));
550
      EXPECT_EQ(OB_SUCCESS, removed_learners.add_member(member2));
551
      EXPECT_EQ(OB_SUCCESS, removed_learners.add_member(member4));
552
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_learners(added_learners, removed_learners, CONFIG_CHANGE_TIMEOUT));
553
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member3));
554
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member5));
555
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member2));
556
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member4));
557
      // reentrant and check
558
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_learners(added_learners, removed_learners, CONFIG_CHANGE_TIMEOUT));
559
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member3));
560
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member5));
561
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member2));
562
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member4));
563
      // reset environment
564
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_learner(member3, CONFIG_CHANGE_TIMEOUT));
565
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_learner(member5, CONFIG_CHANGE_TIMEOUT));
566
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member3));
567
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(member5));
568
    }
569
    // 6. defensive
570
    {
571
      PALF_LOG(INFO, "CASE6: defensive", K(id));
572
      const common::ObMember member2 = ObMember(addr2, 1);
573
      const common::ObMember member3 = ObMember(addr3, 1);
574
      const common::ObMember member4 = ObMember(addr4, 1);
575
      const common::ObMember member5 = ObMember(addr5, 1);
576
      common::ObMember migrating_member3 = member3;
577
      migrating_member3.set_migrating();
578

579
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(migrating_member3, CONFIG_CHANGE_TIMEOUT));
580
      EXPECT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->add_learner(member3, CONFIG_CHANGE_TIMEOUT));
581
      LogConfigVersion config_version;
582
      ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
583
      EXPECT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->add_member(member3, 4, config_version, CONFIG_CHANGE_TIMEOUT));
584
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_learner(member3, CONFIG_CHANGE_TIMEOUT));
585
      EXPECT_TRUE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(migrating_member3));
586
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_learner(migrating_member3, CONFIG_CHANGE_TIMEOUT));
587
      EXPECT_FALSE(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.learnerlist_.contains(migrating_member3));
588

589
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(migrating_member3, CONFIG_CHANGE_TIMEOUT));
590
      ObMemberList added_learners, removed_learners;
591
      EXPECT_EQ(OB_SUCCESS, added_learners.add_member(member3));
592
      EXPECT_EQ(OB_SUCCESS, removed_learners.add_member(member4));
593
      EXPECT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->replace_learners(added_learners, removed_learners, CONFIG_CHANGE_TIMEOUT));
594

595
      common::ObMember migrating_member2 = member2;
596
      migrating_member2.set_migrating();
597
      EXPECT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->remove_member(migrating_member2, 2, CONFIG_CHANGE_TIMEOUT));
598
    }
599
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
600
    revert_cluster_palf_handle_guard(palf_list);
601
  }
602
  delete_paxos_group(id);
603
  PALF_LOG(INFO, "end test config change", K(id));
604
}
605

606
TEST_F(TestObSimpleLogClusterConfigChange, test_replace_member)
607
{
608
  SET_CASE_LOG_FILE(TEST_NAME, "replace_member");
609
  int ret = OB_SUCCESS;
610
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
611
  PALF_LOG(INFO, "begin test replace_member", K(id));
612
  {
613
    std::vector<PalfHandleImplGuard*> palf_list;
614
	  int64_t leader_idx = 0;
615
    PalfHandleImplGuard leader;
616
    const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
617
	  EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, &loc_cb, leader_idx, leader));
618
    PalfHandleImplGuard new_leader;
619
    int64_t new_leader_idx;
620
    EXPECT_EQ(OB_SUCCESS, get_leader(id, new_leader, new_leader_idx));
621
    loc_cb.leader_ = get_cluster()[new_leader_idx]->get_addr();
622
    PALF_LOG(INFO, "set leader for loc_cb", "leader", get_cluster()[new_leader_idx]->get_addr());
623
    EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
624
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
625
    // replace member when no log
626
    LogConfigVersion config_version;
627
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
628
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
629
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 3, CONFIG_CHANGE_TIMEOUT));
630
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(palf_list[(leader_idx+1)%3]->palf_handle_impl_->self_, 1), 2, CONFIG_CHANGE_TIMEOUT));
631
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(palf_list[(leader_idx+2)%3]->palf_handle_impl_->self_, 1), 1, CONFIG_CHANGE_TIMEOUT));
632
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
633
    EXPECT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->add_member(ObMember(palf_list[(leader_idx+1)%3]->palf_handle_impl_->self_, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
634
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[(leader_idx+1)%3]->palf_handle_impl_->self_, 1), 2, config_version, CONFIG_CHANGE_TIMEOUT));
635
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
636
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[(leader_idx+2)%3]->palf_handle_impl_->self_, 1), 3, config_version, CONFIG_CHANGE_TIMEOUT));
637
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
638
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
639
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
640

641
    // add member when contains log
642
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
643
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[4]->palf_handle_impl_->self_, 1), 5, config_version, CONFIG_CHANGE_TIMEOUT));
644
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
645

646
    // remove member
647
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(palf_list[4]->palf_handle_impl_->self_, 1), 4, CONFIG_CHANGE_TIMEOUT));
648

649
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
650

651
    // replace member
652
    ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
653
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->replace_member(ObMember(palf_list[5]->palf_handle_impl_->self_, 1),
654
                                               ObMember(palf_list[3]->palf_handle_impl_->self_, 1),
655
                                               config_version,
656
                                               CONFIG_CHANGE_TIMEOUT));
657

658
    // switch acceptor to learner
659
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->switch_acceptor_to_learner(ObMember(palf_list[5]->palf_handle_impl_->self_, 1), 3, CONFIG_CHANGE_TIMEOUT));
660
    // add learner
661
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), CONFIG_CHANGE_TIMEOUT));
662
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(ObMember(palf_list[4]->palf_handle_impl_->self_, 1), CONFIG_CHANGE_TIMEOUT));
663
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(ObMember(palf_list[6]->palf_handle_impl_->self_, 1), CONFIG_CHANGE_TIMEOUT));
664

665
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
666
    revert_cluster_palf_handle_guard(palf_list);
667
  }
668
  delete_paxos_group(id);
669
  PALF_LOG(INFO, "end test replace_member", K(id));
670
}
671

672
TEST_F(TestObSimpleLogClusterConfigChange, learner)
673
{
674
  SET_CASE_LOG_FILE(TEST_NAME, "learner");
675
  int ret = OB_SUCCESS;
676
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
677
	int64_t leader_idx = 0;
678
  int64_t log_ts = 1;
679
  PalfHandleImplGuard leader;
680
  LogLearnerList all_learner;
681
  std::vector<PalfHandleImplGuard*> palf_list;
682
  std::vector<ObRegion> region_list;
683
  common::ObRegion default_region(DEFAULT_REGION_NAME);
684
  LogMemberRegionMap region_map;
685
  EXPECT_EQ(OB_SUCCESS, region_map.init("localmap", OB_MAX_MEMBER_NUMBER));
686
  region_list.push_back(ObRegion("BEIJING"));
687
  region_list.push_back(ObRegion("SHANGHAI"));
688
  region_list.push_back(ObRegion("TIANJIN"));
689
  region_list.push_back(ObRegion("SHENZHEN"));
690
  region_list.push_back(ObRegion("GUANGZHOU"));
691
  const ObMemberList &node_list = get_node_list();
692
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
693
	EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, &loc_cb, leader_idx, leader));
694
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
695
  loc_cb.leader_ = get_cluster()[leader_idx]->get_addr();
696
  PALF_LOG(INFO, "set leader for loc_cb", "leader", get_cluster()[leader_idx]->get_addr());
697

698
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
699
  // case 1: set region and switch_acceptor_to_learner
700
  // add_learner
701
  for (int64_t i = 3; i < ObSimpleLogClusterTestBase::node_cnt_; ++i) {
702
    PalfHandleImplGuard tmp_handle;
703
    common::ObMember added_learner;
704
    EXPECT_EQ(OB_SUCCESS, node_list.get_member_by_index(i, added_learner));
705
    LogLearner learner(added_learner.get_server(), 1);
706
    EXPECT_EQ(OB_SUCCESS, all_learner.add_learner(learner));
707
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(added_learner, CONFIG_CHANGE_TIMEOUT));
708
  }
709
  // check children_cnt
710
  while (false == check_children_valid(palf_list, all_learner))
711
  {
712
    sleep(1);
713
    PALF_LOG(INFO, "check_children_valid 1");
714
  }
715
  // change region of one follower
716
  bool has_change_region = false;
717
  int64_t diff_region_follower_idx = -1;
718
  int64_t another_follower_idx = -1;
719
  for (int i = 0; i < ObSimpleLogClusterTestBase::member_cnt_; i++) {
720
    const bool not_leader = palf_list[i]->palf_handle_impl_->self_ != leader.palf_handle_impl_->self_;
721
    if (!has_change_region && not_leader) {
722
      EXPECT_EQ(OB_SUCCESS, palf_list[i]->palf_handle_impl_->set_region(region_list[0]));
723
      region_map.insert(palf_list[i]->palf_handle_impl_->self_, region_list[0]);
724
      has_change_region = true;
725
      diff_region_follower_idx = i;
726
    } else {
727
      if (not_leader) {
728
        another_follower_idx = i;
729
      }
730
      region_map.insert(palf_list[0]->palf_handle_impl_->self_, default_region);
731
    }
732
  }
733
  // notify leader region of follower i has changed
734
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->set_paxos_member_region_map(region_map));
735
  // check children_cnt again
736
  while (false == check_children_valid(palf_list, all_learner))
737
  {
738
    sleep(1);
739
    PALF_LOG(INFO, "check_children_valid 2");
740
  }
741
  // after setting region of a follower, parents of all learners should be another follower
742
  EXPECT_GE(another_follower_idx, 0);
743
  EXPECT_LE(another_follower_idx, 2);
744
  ObAddr curr_parent = palf_list[another_follower_idx]->palf_handle_impl_->self_;
745
  while (false == check_parent(palf_list, all_learner, curr_parent))
746
  {
747
    sleep(1);
748
    PALF_LOG(INFO, "check_parent 1");
749
  }
750
  // continue submitting log
751
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 20, id));
752
  PALF_LOG(INFO, "all_learner", K(all_learner));
753
  // EXPECT_EQ(OB_SUCCESS, check_log_sync(palf_list, get_member_list(), all_learner, leader));
754

755
  // switch current unique parent to learner
756
  EXPECT_EQ(OB_SUCCESS, all_learner.add_learner(LogLearner(curr_parent, 1)));
757
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->switch_acceptor_to_learner(ObMember(curr_parent, 1), 2, CONFIG_CHANGE_TIMEOUT));
758
  // after switch follower 1 to learner, a learner will be registered to leader, and other learners will
759
  // be registerd to this learner
760
  while (false == check_children_valid(palf_list, all_learner))
761
  {
762
    sleep(1);
763
    PALF_LOG(INFO, "check_children_valid 3");
764
  }
765
  // check learner topology
766
  ObAddr leaderschild;
767
  PalfHandleImplGuard leaderschild_handle;
768
  LogLearnerList expect_children;
769
  EXPECT_EQ(1, leader.palf_handle_impl_->config_mgr_.children_.get_member_number());
770
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->config_mgr_.children_.get_server_by_index(0, leaderschild));
771
  EXPECT_EQ(OB_SUCCESS, get_palf_handle_guard(palf_list, leaderschild, leaderschild_handle));
772
  expect_children = all_learner;
773
  EXPECT_EQ(OB_SUCCESS, expect_children.remove_learner(leaderschild));
774
  EXPECT_TRUE(expect_children.learner_addr_equal(leaderschild_handle.palf_handle_impl_->config_mgr_.children_));
775
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 20, id));
776
  // EXPECT_EQ(OB_SUCCESS, check_log_sync(palf_list, get_member_list(), all_learner, leader));
777
  // learners' regions are different from paxos member, so parent of all learners is leader
778
  // set regions
779
  for (int64_t i = 3; i < ObSimpleLogClusterTestBase::node_cnt_; ++i) {
780
    PalfHandleImplGuard tmp_handle;
781
    common::ObMember learner;
782
    EXPECT_EQ(OB_SUCCESS, node_list.get_member_by_index(i, learner));
783
    EXPECT_EQ(OB_SUCCESS, get_palf_handle_guard(palf_list, learner.get_server(), tmp_handle));
784
    EXPECT_EQ(OB_SUCCESS, tmp_handle.palf_handle_impl_->set_region(region_list[i-2]));
785
  }
786
  sleep(1);
787
  // check children_cnt
788
  while (false == check_children_valid(palf_list, all_learner))
789
  {
790
    sleep(1);
791
    PALF_LOG(INFO, "check_children_valid 4");
792
  }
793
  while (false == check_parent(palf_list, all_learner, leader.palf_handle_impl_->self_))
794
  {
795
    sleep(1);
796
    PALF_LOG(INFO, "check_parent 2");
797
  }
798

799
  // switch leader, after switching leader, the parent of all learners is the new leader
800
  const int64_t new_leader_idx = diff_region_follower_idx;
801
  PalfHandleImplGuard new_leader;
802
  EXPECT_EQ(OB_SUCCESS, switch_leader(id, 0, new_leader));
803

804
  while (false == check_children_valid(palf_list, all_learner))
805
  {
806
    sleep(1);
807
    PALF_LOG(INFO, "check_children_valid 5");
808
  }
809
  while (false == check_parent(palf_list, all_learner, new_leader.palf_handle_impl_->self_))
810
  {
811
    sleep(1);
812
    PALF_LOG(INFO, "check_parent 3");
813
  }
814

815
  revert_cluster_palf_handle_guard(palf_list);
816
  PALF_LOG(INFO, "end test learner", K(id));
817
}
818

819
TEST_F(TestObSimpleLogClusterConfigChange, test_config_change_lock)
820
{
821
  SET_CASE_LOG_FILE(TEST_NAME, "config_change_lock");
822
  int ret = OB_SUCCESS;
823
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
824
  PALF_LOG(INFO, "begin test config change", K(id));
825
	int64_t leader_idx = 0;
826
  int64_t log_ts = 1;
827
  PalfHandleImplGuard leader;
828
  std::vector<PalfHandleImplGuard*> palf_list;
829
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
830
	ASSERT_EQ(OB_SUCCESS, create_paxos_group(id, &loc_cb, leader_idx, leader));
831
  ASSERT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
832

833
  ASSERT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
834
  loc_cb.leader_ = get_cluster()[leader_idx]->get_addr();
835
  PALF_LOG(INFO, "set leader for loc_cb", "leader", loc_cb.leader_);
836
  int64_t lock_owner_out = -1;
837
  bool lock_stat = false;
838
  //ASSERT_EQ(OB_NOT_SUPPORTED, leader.palf_handle_impl_->get_config_change_lock_stat(lock_owner_out, lock_stat));
839
  //ASSERT_EQ(OB_NOT_SUPPORTED, leader.palf_handle_impl_->try_lock_config_change(1, CONFIG_CHANGE_TIMEOUT));
840
  //ASSERT_EQ(OB_NOT_SUPPORTED, leader.palf_handle_impl_->unlock_config_change(1, CONFIG_CHANGE_TIMEOUT));
841
  //oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_4_2_0_0;
842
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_change_lock_stat(lock_owner_out, lock_stat));
843
  ASSERT_EQ(OB_INVALID_CONFIG_CHANGE_LOCK_OWNER, lock_owner_out);
844
  ASSERT_EQ(false, lock_stat);
845

846
  //invalid arguments
847
  ASSERT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->try_lock_config_change(-1, CONFIG_CHANGE_TIMEOUT));
848
  ASSERT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->try_lock_config_change(1, 0));
849
  ASSERT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->unlock_config_change(-1, CONFIG_CHANGE_TIMEOUT));
850
  ASSERT_EQ(OB_INVALID_ARGUMENT, leader.palf_handle_impl_->unlock_config_change(1, 0));
851

852
  //test try lock
853
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->try_lock_config_change(1, CONFIG_CHANGE_TIMEOUT));
854
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->try_lock_config_change(1, CONFIG_CHANGE_TIMEOUT));
855
  ASSERT_EQ(OB_TRY_LOCK_CONFIG_CHANGE_CONFLICT, leader.palf_handle_impl_->try_lock_config_change(2, CONFIG_CHANGE_TIMEOUT));
856

857
  ASSERT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
858

859
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_change_lock_stat(lock_owner_out, lock_stat));
860
  ASSERT_EQ(1, lock_owner_out);
861
  ASSERT_EQ(true, lock_stat);
862

863
  //test unlock
864
  ASSERT_EQ(OB_STATE_NOT_MATCH, leader.palf_handle_impl_->unlock_config_change(2, CONFIG_CHANGE_TIMEOUT));
865
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->unlock_config_change(1, CONFIG_CHANGE_TIMEOUT));
866
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->unlock_config_change(1, CONFIG_CHANGE_TIMEOUT));
867
  ASSERT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
868

869
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_change_lock_stat(lock_owner_out, lock_stat));
870
  ASSERT_EQ(1, lock_owner_out);
871
  ASSERT_EQ(false, lock_stat);
872

873
  //changing stat, test locking stat
874
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->try_lock_config_change(3, CONFIG_CHANGE_TIMEOUT));
875
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_change_lock_stat(lock_owner_out, lock_stat));
876
  ASSERT_EQ(3, lock_owner_out);
877
  ASSERT_EQ(true, lock_stat);
878

879
  //block add_member
880

881
  LogConfigVersion config_version;
882
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
883

884
  //ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), CONFIG_CHANGE_TIMEOUT));
885
  ASSERT_EQ(OB_EAGAIN, leader.palf_handle_impl_->add_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
886
  ASSERT_EQ(OB_EAGAIN, leader.palf_handle_impl_->remove_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 3, CONFIG_CHANGE_TIMEOUT));
887
  ASSERT_EQ(OB_EAGAIN, leader.palf_handle_impl_->remove_member(ObMember(palf_list[2]->palf_handle_impl_->self_, 1), 3, CONFIG_CHANGE_TIMEOUT));
888
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
889
  ASSERT_EQ(OB_EAGAIN, leader.palf_handle_impl_->add_member(ObMember(palf_list[2]->palf_handle_impl_->self_, 1), 3, config_version, CONFIG_CHANGE_TIMEOUT));
890
  ASSERT_EQ(OB_EAGAIN, leader.palf_handle_impl_->switch_acceptor_to_learner(ObMember(palf_list[2]->palf_handle_impl_->self_, 1), 2, CONFIG_CHANGE_TIMEOUT));
891
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
892
  ASSERT_EQ(OB_EAGAIN, leader.palf_handle_impl_->switch_learner_to_acceptor(ObMember(palf_list[2]->palf_handle_impl_->self_, 1), 3, config_version, CONFIG_CHANGE_TIMEOUT));
893
  ASSERT_EQ(OB_EAGAIN, leader.palf_handle_impl_->change_replica_num(get_member_list(), 3, 4, CONFIG_CHANGE_TIMEOUT));
894

895

896
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->unlock_config_change(3, CONFIG_CHANGE_TIMEOUT));
897
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_change_lock_stat(lock_owner_out, lock_stat));
898
  ASSERT_EQ(3, lock_owner_out);
899
  ASSERT_EQ(false, lock_stat);
900

901
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
902
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
903
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(palf_list[3]->palf_handle_impl_->self_, 1), 3, CONFIG_CHANGE_TIMEOUT));
904
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->remove_member(ObMember(palf_list[2]->palf_handle_impl_->self_, 1), 3, CONFIG_CHANGE_TIMEOUT));
905
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
906
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_member(ObMember(palf_list[2]->palf_handle_impl_->self_, 1), 3, config_version, CONFIG_CHANGE_TIMEOUT));
907
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->switch_acceptor_to_learner(ObMember(palf_list[2]->palf_handle_impl_->self_, 1), 2, CONFIG_CHANGE_TIMEOUT));
908
  ASSERT_EQ(OB_STATE_NOT_MATCH, leader.palf_handle_impl_->switch_learner_to_acceptor(ObMember(palf_list[2]->palf_handle_impl_->self_, 1), 3, config_version, CONFIG_CHANGE_TIMEOUT));
909
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
910
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->switch_learner_to_acceptor(ObMember(palf_list[2]->palf_handle_impl_->self_, 1), 3, config_version, CONFIG_CHANGE_TIMEOUT));
911
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->change_replica_num(get_member_list(), 3, 4, CONFIG_CHANGE_TIMEOUT));
912
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->change_replica_num(get_member_list(), 4, 3, CONFIG_CHANGE_TIMEOUT));
913

914
  //switch leader
915

916
  ASSERT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
917
  revert_cluster_palf_handle_guard(palf_list);
918
  PALF_LOG(INFO, "end test config change lock", K(id));
919
}
920

921
TEST_F(TestObSimpleLogClusterConfigChange, test_switch_leader)
922
{
923
  SET_CASE_LOG_FILE(TEST_NAME, "switch_leader");
924
  int ret = OB_SUCCESS;
925
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
926
  PALF_LOG(INFO, "begin test switch_leader", K(id));
927
	int64_t leader_idx = 0;
928
  PalfHandleImplGuard leader;
929
  std::vector<PalfHandleImplGuard*> palf_list;
930
  oceanbase::common::ObClusterVersion::get_instance().cluster_version_ = CLUSTER_VERSION_4_2_0_0;
931
  EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
932
  ASSERT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
933
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 200, id));
934
  // try lock config change
935
  //
936
  const int64_t follower_idx = (leader_idx+1)%3;
937
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
938

939
  int64_t lock_owner_out = -1;
940
  bool lock_stat = false;
941
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->try_lock_config_change(1, CONFIG_CHANGE_TIMEOUT));
942
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_change_lock_stat(lock_owner_out, lock_stat));
943
  ASSERT_EQ(1, lock_owner_out);
944
  ASSERT_EQ(true, lock_stat);
945

946
  PalfHandleImplGuard new_leader;
947
  const int64_t new_leader_idx = (leader_idx+1)%3;
948
  PalfHandleImplGuard &follower = *palf_list[new_leader_idx];
949
  ASSERT_EQ(OB_NOT_MASTER, follower.palf_handle_impl_->get_config_change_lock_stat(lock_owner_out, lock_stat));
950
  EXPECT_EQ(OB_SUCCESS, switch_leader(id, new_leader_idx, new_leader));
951
  sleep(5);
952
  ObRole role;
953
  int64_t curr_proposal_id = 0;
954
  bool is_pending_stat = false;
955
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_role(role, curr_proposal_id, is_pending_stat));
956
  EXPECT_EQ(ObRole::FOLLOWER, role);
957

958
  lock_owner_out = -1;
959
  lock_stat = false;
960

961
  ASSERT_EQ(OB_SUCCESS, new_leader.palf_handle_impl_->get_config_change_lock_stat(lock_owner_out, lock_stat));
962
  ASSERT_EQ(1, lock_owner_out);
963
  ASSERT_EQ(true, lock_stat);
964
  EXPECT_EQ(OB_SUCCESS, submit_log(new_leader, 200, id));
965

966

967
  ASSERT_EQ(OB_SUCCESS, new_leader.palf_handle_impl_->unlock_config_change(1, CONFIG_CHANGE_TIMEOUT));
968
  sleep(3);
969
  //switch leader during lock_config_change
970

971
  LogConfigChangeArgs args(2, ConfigChangeLockType::LOCK_PAXOS_MEMBER_CHANGE, TRY_LOCK_CONFIG_CHANGE);
972
  LogConfigVersion config_version;
973
  const int64_t proposal_id = new_leader.palf_handle_impl_->state_mgr_.get_proposal_id();
974
  const int64_t leader_epoch = new_leader.palf_handle_impl_->state_mgr_.get_leader_epoch();
975
  ASSERT_EQ(OB_EAGAIN, new_leader.palf_handle_impl_->config_mgr_.change_config(args, proposal_id, leader_epoch, config_version));
976
  ASSERT_EQ(LogConfigMgr::ConfigChangeState::CHANGING, new_leader.palf_handle_impl_->config_mgr_.state_);
977
  ASSERT_EQ(OB_EAGAIN, new_leader.palf_handle_impl_->get_config_change_lock_stat(lock_owner_out, lock_stat));
978
  //block_net
979
  const int64_t follower_idx1 = (new_leader_idx+1)%3;
980
  const int64_t follower_idx2 = (new_leader_idx+2)%3;
981
  block_net(follower_idx1, new_leader_idx, true);
982
  block_net(follower_idx2, new_leader_idx, true);
983
  //send info to follower
984
  ASSERT_EQ(OB_EAGAIN, new_leader.palf_handle_impl_->config_mgr_.change_config(args, proposal_id, leader_epoch, config_version));
985
  sleep(1);
986

987
  unblock_net(follower_idx1, new_leader_idx);
988
  unblock_net(follower_idx2, new_leader_idx);
989
  PalfHandleImplGuard old_leader;
990
  EXPECT_EQ(OB_SUCCESS, switch_leader(id, leader_idx, old_leader));
991

992
  EXPECT_EQ(OB_SUCCESS, old_leader.palf_handle_impl_->get_role(role, curr_proposal_id, is_pending_stat));
993
  EXPECT_EQ(ObRole::LEADER, role);
994

995
  //check config change stat
996
  sleep(1);
997
  ASSERT_EQ(OB_SUCCESS, old_leader.palf_handle_impl_->get_config_change_lock_stat(lock_owner_out, lock_stat));
998
  ASSERT_EQ(2, lock_owner_out);
999
  ASSERT_EQ(true, lock_stat);
1000

1001

1002
  revert_cluster_palf_handle_guard(palf_list);
1003
  PALF_LOG(INFO, "end test switch_leader", K(id));
1004
}
1005

1006
// 1. 3F(beijing), 4R(shanghai)
1007
// 2. the client submits logs to F replicas, but R replicas can not receive logs
1008
// 3. switch a R to F, the R replica must be one of the children of another R.
1009
//    Due to step 2, the R will not receive the reconfiguration log
1010
// 4. enable the remaining R re-register parents
1011
// 5. check loop between R replicas
1012
TEST_F(TestObSimpleLogClusterConfigChange, learner_loop)
1013
{
1014
  SET_CASE_LOG_FILE(TEST_NAME, "learner_loop");
1015
  int ret = OB_SUCCESS;
1016
  const int64_t CONFIG_CHANGE_TIMEOUT = 10 * 1000 * 1000L; // 10s
1017
	const int64_t id = ATOMIC_AAF(&palf_id_, 1);
1018
	int64_t leader_idx = 0;
1019
  PalfHandleImplGuard leader;
1020
  LogLearnerList all_learner;
1021
  const ObMemberList &node_list = get_node_list();
1022
  std::vector<PalfHandleImplGuard*> palf_list;
1023
  common::ObRegion beijing_region("BEIJING");
1024
  common::ObRegion shanghai_region("SHANGHAI");
1025

1026
	EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, &loc_cb, leader_idx, leader));
1027
  EXPECT_EQ(OB_SUCCESS, get_cluster_palf_handle_guard(id, palf_list));
1028
  loc_cb.leader_ = get_cluster()[leader_idx]->get_addr();
1029

1030
  // 1. init
1031
  for (int64_t i = 3; i < ObSimpleLogClusterTestBase::node_cnt_; ++i) {
1032
    common::ObMember added_learner;
1033
    EXPECT_EQ(OB_SUCCESS, node_list.get_member_by_index(i, added_learner));
1034
    LogLearner learner(added_learner.get_server(), 1);
1035
    EXPECT_EQ(OB_SUCCESS, all_learner.add_learner(learner));
1036
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->add_learner(added_learner, CONFIG_CHANGE_TIMEOUT));
1037
  }
1038

1039
  // set region, version 42x
1040
  // for (int i = 0; i < ObSimpleLogClusterTestBase::node_cnt_; i++) {
1041
  //   const common::ObAddr addr = palf_list[i]->palf_handle_impl_->self_;
1042
  //   if (leader.palf_handle_impl_->config_mgr_.alive_paxos_memberlist_.contains(addr)) {
1043
  //     get_cluster()[0]->get_locality_manager()->set_server_region(addr, beijing_region);
1044
  //   } else {
1045
  //     get_cluster()[0]->get_locality_manager()->set_server_region(addr, shanghai_region);
1046
  //   }
1047
  // }
1048
  // for (auto palf_handle: palf_list) { palf_handle->palf_handle_impl_->update_self_region_(); }
1049

1050
  // set region, version 421, master
1051
  LogMemberRegionMap region_map;
1052
  EXPECT_EQ(OB_SUCCESS, region_map.init("localmap", OB_MAX_MEMBER_NUMBER));
1053
  for (int i = 0; i < ObSimpleLogClusterTestBase::member_cnt_; i++) {
1054
    const common::ObAddr addr = palf_list[i]->palf_handle_impl_->self_;
1055
    if (leader.palf_handle_impl_->config_mgr_.alive_paxos_memberlist_.contains(addr)) {
1056
      EXPECT_EQ(OB_SUCCESS, palf_list[i]->palf_handle_impl_->set_region(beijing_region));
1057
      region_map.insert(addr, beijing_region);
1058
    } else {
1059
      EXPECT_EQ(OB_SUCCESS, palf_list[i]->palf_handle_impl_->set_region(shanghai_region));
1060
      region_map.insert(addr, shanghai_region);
1061
    }
1062
  }
1063
  // notify leader region of follower i has changed
1064
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->set_paxos_member_region_map(region_map));
1065

1066
  // check topo
1067
  EXPECT_UNTIL_EQ(true, check_children_valid(palf_list, all_learner));
1068
  EXPECT_UNTIL_EQ(1, leader.palf_handle_impl_->config_mgr_.children_.get_member_number());
1069
  EXPECT_UNTIL_EQ(0, palf_list[1]->palf_handle_impl_->config_mgr_.children_.get_member_number());
1070
  EXPECT_UNTIL_EQ(0, palf_list[2]->palf_handle_impl_->config_mgr_.children_.get_member_number());
1071
  ObAddr same_parent, any_child;
1072
  int64_t any_child_idx = -1;
1073
  for (int i = 0; i < ObSimpleLogClusterTestBase::node_cnt_; i++)
1074
  {
1075
    const common::ObAddr addr = palf_list[i]->palf_handle_impl_->self_;
1076
    if (leader.palf_handle_impl_->config_mgr_.children_.contains(addr)) {
1077
      EXPECT_UNTIL_EQ(palf_list[i]->palf_handle_impl_->config_mgr_.parent_, leader.palf_handle_impl_->self_);
1078
      same_parent = addr;
1079
      PALF_LOG(INFO, "SAME_PARENT", K(id), K(addr), K(same_parent));
1080
      break;
1081
    }
1082
  }
1083
  EXPECT_TRUE(same_parent.is_valid());
1084
  for (int i = 0; i < ObSimpleLogClusterTestBase::node_cnt_; i++)
1085
  {
1086
    const common::ObAddr addr = palf_list[i]->palf_handle_impl_->self_;
1087
    if (all_learner.contains(addr) && addr != same_parent) {
1088
      EXPECT_UNTIL_EQ(same_parent, palf_list[i]->palf_handle_impl_->config_mgr_.parent_);
1089
      any_child = addr;
1090
      any_child_idx = i;
1091
      PALF_LOG(INFO, "CHECK_PARENT", K(id), K(addr), K(same_parent));
1092
    }
1093
  }
1094
  // 2. replicating logs to all F replicas
1095
  EXPECT_NE(-1, any_child_idx);
1096
  EXPECT_UNTIL_EQ(leader.palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.config_version_,
1097
      palf_list[any_child_idx]->palf_handle_impl_->config_mgr_.log_ms_meta_.curr_.config_.config_version_);
1098
  EXPECT_UNTIL_EQ(leader.palf_handle_impl_->get_max_lsn().val_, leader.palf_handle_impl_->get_end_lsn().val_);
1099
  for (int i = 0; i < ObSimpleLogClusterTestBase::node_cnt_; i++) {
1100
    const common::ObAddr &addr = palf_list[i]->palf_handle_impl_->self_;
1101
    if (true == all_learner.contains(addr)) {
1102
      block_pcode(i, ObRpcPacketCode::OB_LOG_PUSH_REQ);
1103
    }
1104
  }
1105
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
1106
  EXPECT_UNTIL_EQ(leader.palf_handle_impl_->get_max_lsn().val_, leader.palf_handle_impl_->get_end_lsn().val_);
1107

1108
  // 3. switch a R replica to F
1109
  LogConfigVersion config_version;
1110
  ASSERT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_config_version(config_version));
1111
  EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->switch_learner_to_acceptor(ObMember(any_child, -1), 4, config_version, CONFIG_CHANGE_TIMEOUT));
1112

1113
  // 4. enable the remaining R re-register parents
1114
  leader.palf_handle_impl_->config_mgr_.children_.reset();
1115
  for (auto palf_handle: palf_list) {
1116
    const common::ObAddr addr = palf_handle->palf_handle_impl_->self_;
1117
    if (true == all_learner.contains(addr) && addr != any_child) {
1118
      palf_handle->palf_handle_impl_->config_mgr_.retire_parent_();
1119
      palf_handle->palf_handle_impl_->config_mgr_.register_parent_();
1120
    }
1121
  }
1122

1123
  // 5. check loop
1124
  for (int i = 0; i < ObSimpleLogClusterTestBase::node_cnt_; i++) {
1125
    const common::ObAddr &addr = palf_list[i]->palf_handle_impl_->self_;
1126
    if (true == all_learner.contains(addr)) {
1127
      block_pcode(i, ObRpcPacketCode::OB_LOG_PUSH_REQ);
1128
    }
1129
  }
1130
  sleep(2);
1131

1132
  EXPECT_EQ(OB_SUCCESS, all_learner.remove_learner(any_child));
1133
  for (auto palf_handle: palf_list) {
1134
    const common::ObAddr addr = palf_handle->palf_handle_impl_->self_;
1135
    if (true == all_learner.contains(addr)) {
1136
      EXPECT_UNTIL_EQ(true, palf_handle->palf_handle_impl_->config_mgr_.parent_.is_valid());
1137
      EXPECT_UNTIL_EQ(false, palf_handle->palf_handle_impl_->config_mgr_.children_.contains(palf_handle->palf_handle_impl_->config_mgr_.parent_));
1138
    }
1139
  }
1140

1141
  revert_cluster_palf_handle_guard(palf_list);
1142
  PALF_LOG(INFO, "end test learner_loop", K(id));
1143
}
1144

1145
} // end unittest
1146
} // end oceanbase
1147

1148
int main(int argc, char **argv)
1149
{
1150
  RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME);
1151
}
1152