oceanbase

1
/**
2
 * Copyright (c) 2021 OceanBase
3
 * OceanBase CE is licensed under Mulan PubL v2.
4
 * You can use this software according to the terms and conditions of the Mulan PubL v2.
5
 * You may obtain a copy of Mulan PubL v2 at:
6
 *          http://license.coscl.org.cn/MulanPubL-2.0
7
 * THIS SOFTWARE IS PROVIDED ON AN "AS IS" BASIS, WITHOUT WARRANTIES OF ANY KIND,
8
 * EITHER EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO NON-INFRINGEMENT,
9
 * MERCHANTABILITY OR FIT FOR A PARTICULAR PURPOSE.
10
 * See the Mulan PubL v2 for more details.
11
 */
12

13
#include "lib/ob_define.h"
14
#include "lib/ob_errno.h"
15
#include <cstdio>
16
#include <gtest/gtest.h>
17
#include <signal.h>
18
#include <stdexcept>
19
#define private public
20
#define protected public
21
#include "env/ob_simple_log_cluster_env.h"
22
#undef private
23
#undef protected
24
#include "logservice/palf/log_reader_utils.h"
25
#include "logservice/palf/log_define.h"
26
#include "logservice/palf/log_group_entry_header.h"
27
#include "logservice/palf/log_io_worker.h"
28
#include "logservice/palf/lsn.h"
29

30
const std::string TEST_NAME = "log_restart";
31
using namespace oceanbase::common;
32
using namespace oceanbase;
33
namespace oceanbase
34
{
35
using namespace logservice;
36
namespace unittest
37
{
38
class TestObSimpleLogClusterRestart : public ObSimpleLogClusterTestEnv
39
{
40
public:
41
  TestObSimpleLogClusterRestart() : ObSimpleLogClusterTestEnv()
42
  {
43
    int ret = init();
44
    if (OB_SUCCESS != ret) {
45
      throw std::runtime_error("TestObSimpleLogClusterLogEngine init failed");
46
    }
47
  }
48
  ~TestObSimpleLogClusterRestart()
49
  {
50
    destroy();
51
  }
52
  int init()
53
  {
54
    return OB_SUCCESS;
55
  }
56
  void destroy()
57
  {}
58
  int64_t id_;
59
  PalfHandleImplGuard leader_;
60
};
61

62
int64_t ObSimpleLogClusterTestBase::member_cnt_ = 1;
63
int64_t ObSimpleLogClusterTestBase::node_cnt_ = 1;
64
std::string ObSimpleLogClusterTestBase::test_name_ = TEST_NAME;
65
bool ObSimpleLogClusterTestBase::need_add_arb_server_  = false;
66
constexpr int64_t timeout_ts_us = 3 * 1000 * 1000;
67

68

69
TEST_F(TestObSimpleLogClusterRestart, read_block_in_flashback)
70
{
71
  disable_hot_cache_ = true;
72
  SET_CASE_LOG_FILE(TEST_NAME, "read_block_in_flashback");
73
  OB_LOGGER.set_log_level("TRACE");
74
  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
75
  int64_t leader_idx = 0;
76
  PalfHandleImplGuard leader;
77
  PalfEnv *palf_env = NULL;
78
  EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
79

80
  EXPECT_EQ(OB_SUCCESS, submit_log(leader, 2 * 32 + 2, id, MAX_LOG_BODY_SIZE));
81
  EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.get_palf_handle_impl()->get_max_lsn()));
82

83
  block_id_t min_block_id, max_block_id;
84
  LogStorage *log_storage = &leader.get_palf_handle_impl()->log_engine_.log_storage_;
85
  EXPECT_EQ(OB_SUCCESS, log_storage->get_block_id_range(min_block_id, max_block_id));
86
  EXPECT_EQ(2, max_block_id);
87
  SCN scn;
88
  char block_name_tmp[OB_MAX_FILE_NAME_LENGTH];
89
  EXPECT_EQ(OB_SUCCESS, block_id_to_tmp_string(max_block_id, block_name_tmp, OB_MAX_FILE_NAME_LENGTH));
90
  char block_name[OB_MAX_FILE_NAME_LENGTH];
91
  EXPECT_EQ(OB_SUCCESS, block_id_to_string(max_block_id, block_name, OB_MAX_FILE_NAME_LENGTH));
92
  ::renameat(log_storage->block_mgr_.dir_fd_, block_name, log_storage->block_mgr_.dir_fd_, block_name_tmp);
93
  EXPECT_EQ(-1, ::openat(log_storage->block_mgr_.dir_fd_, block_name, LOG_READ_FLAG));
94
  EXPECT_EQ(OB_NEED_RETRY, read_log(leader));
95
  EXPECT_EQ(OB_NEED_RETRY, log_storage->get_block_min_scn(max_block_id, scn));
96

97
  // 测试边界场景，read_log_tail_为文件中间，最后一个文件完全被flashback掉, 此时log_tail_是最后一个文件头
98
  log_storage->log_tail_ = LSN(2*PALF_BLOCK_SIZE);
99
  EXPECT_EQ(OB_NEED_RETRY, read_log(leader));
100
  EXPECT_EQ(OB_NEED_RETRY, log_storage->get_block_min_scn(max_block_id, scn));
101

102
  // 测试边界场景，read_log_tail_最后一个文件头，最后一个文件完全被flashback掉
103
  log_storage->log_tail_ = LSN(2*PALF_BLOCK_SIZE);
104
  log_storage->readable_log_tail_ = LSN(2*PALF_BLOCK_SIZE);
105
  EXPECT_EQ(OB_ITER_END, read_log(leader));
106
  EXPECT_EQ(OB_ERR_OUT_OF_UPPER_BOUND, log_storage->get_block_min_scn(max_block_id, scn));
107

108
 // 不太好模拟这种场景，考虑引入debug sync
109
 // // 测试边界场景，readable_log_tail_还没改变前检验是否可读通过，直接读文件时报错文件不存在。
110
 // log_storage->log_tail_ = LSN(3*PALF_BLOCK_SIZE);
111
 // log_storage->readable_log_tail_ = LSN(3*PALF_BLOCK_SIZE);
112
 // // 设置max_block_id_为1是为了构造check_read_out_of_bound返回OB_ERR_OUT_OF_UPPER_BOUND的场景
113
 // log_storage->block_mgr_.max_block_id_ = 1;
114
 // // log_storage返回OB_ERR_OUT_OF_UPPER_BOUND, iterator将其转换为OB_ITER_END
115
 // EXPECT_EQ(OB_ITER_END, read_log(leader));
116
 // EXPECT_EQ(OB_ERR_OUT_OF_UPPER_BOUND, log_storage->get_block_min_scn(max_block_id, scn));
117
}
118

119
TEST_F(TestObSimpleLogClusterRestart, restart_when_first_log_block_is_empty)
120
{
121
  SET_CASE_LOG_FILE(TEST_NAME, "restart_when_first_log_block_is_empty");
122
  OB_LOGGER.set_log_level("TRACE");
123
  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
124
  int64_t leader_idx = 0;
125
  // 创建日志流后不写入任何数据
126
  {
127
    PalfHandleImplGuard leader;
128
    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
129
  }
130
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
131
  // 测试truncate场景
132
  {
133
    PalfHandleImplGuard leader;
134
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
135
    EXPECT_EQ(LSN(PALF_INITIAL_LSN_VAL), leader.palf_handle_impl_->get_max_lsn());
136
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1, id, MAX_LOG_BODY_SIZE));
137
    EXPECT_EQ(OB_SUCCESS, wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader));
138
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->log_engine_.truncate(LSN(0)));
139
  }
140
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
141
  LSN rebuild_lsn(2*PALF_BLOCK_SIZE);
142
  // 测试rebuild场景
143
  {
144
    PalfHandleImplGuard leader;
145
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
146
    EXPECT_EQ(LSN(PALF_INITIAL_LSN_VAL), leader.palf_handle_impl_->get_max_lsn());
147
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 40, id, MAX_LOG_BODY_SIZE));
148
    EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.palf_handle_impl_->get_max_lsn()));
149
    PalfBaseInfo base_info;
150
    base_info.generate_by_default();
151
    base_info.curr_lsn_ = rebuild_lsn;
152
    base_info.prev_log_info_.accum_checksum_ = 10000;
153
    base_info.prev_log_info_.log_id_ = 100;
154
    base_info.prev_log_info_.lsn_ = rebuild_lsn - 4096;
155
    base_info.prev_log_info_.log_proposal_id_ = 2;
156
    base_info.prev_log_info_.scn_ = leader.palf_handle_impl_->get_max_scn();
157

158
    leader.palf_handle_impl_->state_mgr_.role_ = FOLLOWER;
159
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->disable_sync());
160
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->advance_base_info(base_info, true));
161
    while (leader.palf_handle_impl_->log_engine_.log_storage_.get_end_lsn() != rebuild_lsn) {
162
      sleep(1);
163
      PALF_LOG(INFO, "has not finish rebuild", K(leader.palf_handle_impl_->log_engine_.log_storage_));
164
    }
165
  }
166
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
167
  // 测试flashback场景
168
  {
169
    PalfHandleImplGuard leader;
170
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
171
    EXPECT_EQ(rebuild_lsn, leader.palf_handle_impl_->get_max_lsn());
172
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1, id, MAX_LOG_BODY_SIZE));
173
    EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.palf_handle_impl_->get_max_lsn()));
174
    int64_t mode_version;
175
    switch_append_to_flashback(leader, mode_version);
176
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->flashback(mode_version, SCN::min_scn(), 10*1000*1000));
177
  }
178
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
179
  {
180
    PalfHandleImplGuard leader;
181
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
182
    EXPECT_EQ(rebuild_lsn, leader.palf_handle_impl_->get_max_lsn());
183
    int64_t mode_version;
184
    switch_flashback_to_append(leader, mode_version);
185
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1, id, MAX_LOG_BODY_SIZE));
186
    EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.palf_handle_impl_->get_max_lsn()));
187
  }
188
}
189

190
TEST_F(TestObSimpleLogClusterRestart, test_restart)
191
{
192
  SET_CASE_LOG_FILE(TEST_NAME, "test_restart");
193
  int64_t id = ATOMIC_AAF(&palf_id_, 1);
194
  int64_t leader_idx = 0;
195
  char meta_fd[OB_MAX_FILE_NAME_LENGTH] = {'\0'};
196
  char log_fd[OB_MAX_FILE_NAME_LENGTH] = {'\0'};
197
  ObServerLogBlockMgr *pool = NULL;
198
  {
199
    PalfHandleImplGuard leader;
200
    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
201
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1, id, MAX_LOG_BODY_SIZE));
202
    wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader);
203
    LogEngine *log_engine = &leader.palf_handle_impl_->log_engine_;
204
    char *meta_log_dir = log_engine->log_meta_storage_.block_mgr_.log_dir_;
205
    char *log_dir = log_engine->log_storage_.block_mgr_.log_dir_;
206
    EXPECT_EQ(OB_SUCCESS, get_log_pool(leader_idx, pool));
207
    char *pool_dir = pool->log_pool_path_;
208
    snprintf(meta_fd, OB_MAX_FILE_NAME_LENGTH, "mv %s/%d %s/%d", meta_log_dir, 0, pool_dir, 10000000);
209
    snprintf(log_fd, OB_MAX_FILE_NAME_LENGTH, "mv %s/%d %s/%d", log_dir, 0, pool_dir, 100000001);
210
    system(meta_fd);
211
  }
212
  OB_LOGGER.set_log_level("TRACE");
213
  sleep(1);
214
  EXPECT_EQ(OB_ERR_UNEXPECTED, restart_paxos_groups());
215
  system(log_fd);
216
  PALF_LOG(INFO, "first restart_paxos_groups, after meta dir is empty while log dir is not");
217
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
218

219
  // 验证切文件过程中宕机重启
220
  {
221
    PalfHandleImplGuard leader;
222
    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
223
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
224
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 33, id, MAX_LOG_BODY_SIZE));
225
    wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader);
226
    block_id_t min_block_id, max_block_id;
227
    LogStorage *log_storage = &leader.palf_handle_impl_->log_engine_.log_storage_;
228
    LogStorage *meta_storage = &leader.get_palf_handle_impl()->log_engine_.log_meta_storage_;
229
    EXPECT_EQ(OB_SUCCESS, log_storage->get_block_id_range(min_block_id, max_block_id));
230
    EXPECT_EQ(1, max_block_id);
231
    // 模拟只switch block，但没有更新manifest, 此时manifest依旧是1, 宕机重启后由于2号文件为空，manifest会被更新为2
232
    EXPECT_EQ(OB_SUCCESS, log_storage->truncate(LSN(PALF_BLOCK_SIZE)));
233
    EXPECT_EQ(OB_SUCCESS, log_storage->update_manifest_(1));
234
    EXPECT_EQ(PALF_BLOCK_SIZE, log_storage->curr_block_writable_size_);
235
    EXPECT_EQ(1, lsn_2_block(meta_storage->log_block_header_.min_lsn_, PALF_BLOCK_SIZE));
236
  }
237
  PALF_LOG(INFO, "second restart_paxos_groups after restart in process of switch block");
238
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
239
  {
240
    PalfHandleImplGuard leader;
241
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
242
    //检查manifest是否为3
243
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1, id, MAX_LOG_BODY_SIZE));
244
    LogStorage *meta_storage = &leader.get_palf_handle_impl()->log_engine_.log_meta_storage_;
245
    EXPECT_EQ(2, lsn_2_block(meta_storage->log_block_header_.min_lsn_, PALF_BLOCK_SIZE));
246
  }
247
  PALF_LOG(INFO, "third restart_paxos_groups");
248
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
249
  // 验证重启后新建日志流
250
  {
251
    PalfHandleImplGuard leader;
252
    id = ATOMIC_AAF(&palf_id_, 1);
253
    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
254
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 66, id, MAX_LOG_BODY_SIZE));
255
    wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader);
256
    EXPECT_EQ(OB_ITER_END, read_log(leader));
257
  }
258
  // 验证truncate或flashback过程中，修改完manifest后，删除文件前宕机重启（删除1个文件）
259
  {
260
    PalfHandleImplGuard leader;
261
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
262
    block_id_t min_block_id, max_block_id;
263
    // 此时manifest为3
264
    LogStorage *log_storage = &leader.palf_handle_impl_->log_engine_.log_storage_;
265
    LogStorage *meta_storage = &leader.get_palf_handle_impl()->log_engine_.log_meta_storage_;
266
    EXPECT_EQ(OB_SUCCESS, log_storage->get_block_id_range(min_block_id, max_block_id));
267
    EXPECT_EQ(2, max_block_id);
268
    EXPECT_EQ(3, lsn_2_block(meta_storage->log_block_header_.min_lsn_, PALF_BLOCK_SIZE));
269
    // truncate 或 flashback会先更新manifest为2
270
    EXPECT_EQ(OB_SUCCESS, log_storage->update_manifest_(2));
271
    EXPECT_EQ(2, lsn_2_block(meta_storage->log_block_header_.min_lsn_, PALF_BLOCK_SIZE));
272
  }
273
  PALF_LOG(INFO, "fourth restart_paxos_groups after modify manifest while not delete block");
274
  // 验证truncate或flashback过程中，修改完manifest后，truncaet/flashback正好将最后一个文件清空
275
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
276
  {
277
    PalfHandleImplGuard leader;
278
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
279
    block_id_t min_block_id, max_block_id;
280
    // 此时manifest为2
281
    LogStorage *log_storage = &leader.palf_handle_impl_->log_engine_.log_storage_;
282
    LogStorage *meta_storage = &leader.get_palf_handle_impl()->log_engine_.log_meta_storage_;
283
    EXPECT_EQ(OB_SUCCESS, log_storage->get_block_id_range(min_block_id, max_block_id));
284
    EXPECT_EQ(2, max_block_id);
285
    // 尽管manifest为2，但在这种场景下，2号文件是可以删除的
286
    EXPECT_EQ(2, lsn_2_block(meta_storage->log_block_header_.min_lsn_, PALF_BLOCK_SIZE));
287
    EXPECT_EQ(OB_SUCCESS, log_storage->truncate(LSN(2*PALF_BLOCK_SIZE)));
288
    EXPECT_EQ(OB_SUCCESS, log_storage->update_manifest_(2));
289
  }
290
  PALF_LOG(INFO, "five restart_paxos_groups after modify manifest and last block is empty");
291
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
292
  {
293
    PalfHandleImplGuard leader;
294
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
295
    block_id_t min_block_id, max_block_id;
296
    // 重启之后，由于磁盘上最大的文件为2，同时该文件为空，此时会更新manifest为3
297
    LogStorage *log_storage = &leader.palf_handle_impl_->log_engine_.log_storage_;
298
    LogStorage *meta_storage = &leader.get_palf_handle_impl()->log_engine_.log_meta_storage_;
299
    EXPECT_EQ(OB_SUCCESS, log_storage->get_block_id_range(min_block_id, max_block_id));
300
    EXPECT_EQ(2, max_block_id);
301
    EXPECT_EQ(3, lsn_2_block(meta_storage->log_block_header_.min_lsn_, PALF_BLOCK_SIZE));
302
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1, id, MAX_LOG_BODY_SIZE));
303
    wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader);
304
    EXPECT_EQ(3, lsn_2_block(meta_storage->log_block_header_.min_lsn_, PALF_BLOCK_SIZE));
305
  }
306
  PALF_LOG(INFO, "six restart_paxos_groups");
307
  // 验证base lsn 大于持久化的committed 位点
308
  {
309
    PalfHandleImplGuard leader;
310
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
311
    LogStorage *log_storage = &leader.palf_handle_impl_->log_engine_.log_storage_;
312
    LogIOWorker *iow = leader.palf_handle_impl_->log_engine_.log_io_worker_;
313
    int64_t epoch = leader.palf_handle_impl_->log_engine_.palf_epoch_;
314
    int64_t palf_id = leader.palf_handle_impl_->palf_id_;
315
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 29, id, MAX_LOG_BODY_SIZE));
316
    EXPECT_EQ(OB_SUCCESS, wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader));
317
    // 预期log_tail接近文件2的尾部
318
    EXPECT_LE(LSN(3*PALF_BLOCK_SIZE) - log_storage->log_tail_, 5*1024*1024);
319
    EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.palf_handle_impl_->get_end_lsn()));
320
    sleep(1);
321
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1, id, 1000));
322
    wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader);
323
    EXPECT_EQ(OB_SUCCESS, wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader));
324
    IOTaskConsumeCond cond(palf_id, epoch);
325
    EXPECT_EQ(OB_SUCCESS, iow->submit_io_task(&cond));
326
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10, id, MAX_LOG_BODY_SIZE));
327
    while (1) {
328
      if (leader.palf_handle_impl_->sw_.last_submit_end_lsn_ < leader.palf_handle_impl_->get_max_lsn()) {
329
        usleep(5000);
330
        leader.palf_handle_impl_->sw_.freeze_mode_ = FEEDBACK_FREEZE_MODE;
331
        leader.palf_handle_impl_->sw_.feedback_freeze_last_log_();
332
        PALF_LOG(INFO, "has log in sw", "last_submit_end_lsn", leader.palf_handle_impl_->sw_.last_submit_end_lsn_,
333
                 "max_lsn", leader.palf_handle_impl_->get_max_lsn());
334
      } else {
335
        break;
336
      }
337
    }
338
    cond.cond_.signal();
339
    EXPECT_EQ(OB_SUCCESS, wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader));
340
    PALF_LOG(INFO, "after wait_lsn_until_flushed", "end_lsn:", leader.palf_handle_impl_->get_end_lsn(),
341
             "max_lsn:", leader.palf_handle_impl_->get_end_lsn());
342
    EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.palf_handle_impl_->get_max_lsn()));
343
    EXPECT_GE(leader.palf_handle_impl_->get_max_lsn(), LSN(3*PALF_BLOCK_SIZE));
344
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->set_base_lsn(LSN(3*PALF_BLOCK_SIZE)));
345
  }
346
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
347
  PALF_LOG(INFO, "seven restart_paxos_groups after committed lsn is smaller than base lsn");
348
  // 验证rebuild过程中持久化palf_base_info后，宕机重启
349
  {
350
    PalfHandleImplGuard leader;
351
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
352
    LogStorage *log_storage = &leader.palf_handle_impl_->log_engine_.log_storage_;
353
    LogIOWorker *iow = leader.palf_handle_impl_->log_engine_.log_io_worker_;
354
    int64_t epoch = leader.palf_handle_impl_->log_engine_.palf_epoch_;
355
    int64_t palf_id = leader.palf_handle_impl_->palf_id_;
356
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10, id, MAX_LOG_BODY_SIZE));
357
    EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.palf_handle_impl_->get_end_lsn()));
358
    sleep(1);
359
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 1, id, 1000));
360
    wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader);
361
    EXPECT_EQ(OB_SUCCESS, wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader));
362
    IOTaskConsumeCond cond(palf_id, epoch);
363
    EXPECT_EQ(OB_SUCCESS, iow->submit_io_task(&cond));
364
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10, id, MAX_LOG_BODY_SIZE));
365
    while (1) {
366
      if (leader.palf_handle_impl_->sw_.last_submit_end_lsn_ < leader.palf_handle_impl_->get_max_lsn()) {
367
        usleep(5000);
368
        leader.palf_handle_impl_->sw_.freeze_mode_ = FEEDBACK_FREEZE_MODE;
369
        leader.palf_handle_impl_->sw_.feedback_freeze_last_log_();
370
        PALF_LOG(INFO, "has log in sw", "last_submit_end_lsn", leader.palf_handle_impl_->sw_.last_submit_end_lsn_,
371
                 "max_lsn", leader.palf_handle_impl_->get_max_lsn());
372
      } else {
373
        break;
374
      }
375
    }
376
    cond.cond_.signal();
377
    EXPECT_EQ(OB_SUCCESS, wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader));
378
    PALF_LOG(INFO, "after wait_lsn_until_flushed", "end_lsn:", leader.palf_handle_impl_->get_end_lsn(),
379
             "max_lsn:", leader.palf_handle_impl_->get_end_lsn());
380
    EXPECT_EQ(OB_SUCCESS, wait_until_has_committed(leader, leader.palf_handle_impl_->get_max_lsn()));
381
    EXPECT_GE(leader.palf_handle_impl_->get_max_lsn(), LSN(3*PALF_BLOCK_SIZE));
382
    PalfBaseInfo base_info;
383
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->get_base_info(
384
      leader.palf_handle_impl_->get_max_lsn(), base_info));
385
    LogSnapshotMeta snapshot;
386
    base_info.prev_log_info_.lsn_ = LSN(10*PALF_BLOCK_SIZE - 10*1024);
387
    EXPECT_EQ(OB_SUCCESS, snapshot.generate(LSN(10*PALF_BLOCK_SIZE), base_info.prev_log_info_));
388
    FlushMetaCbCtx meta_ctx;
389
    meta_ctx.type_ = SNAPSHOT_META;
390
    meta_ctx.base_lsn_ = snapshot.base_lsn_;
391
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->log_engine_.submit_flush_snapshot_meta_task(meta_ctx, snapshot));
392
    sleep(2);
393
  }
394
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
395
  PALF_LOG(INFO, "seven restart_paxos_groups after committed lsn is smaller than base lsn");
396
  {
397
    PalfHandleImplGuard leader;
398
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
399
    EXPECT_EQ(LSN(10*PALF_BLOCK_SIZE), leader.palf_handle_impl_->get_max_lsn());
400
    EXPECT_EQ(LSN(10*PALF_BLOCK_SIZE), leader.palf_handle_impl_->log_engine_.log_storage_.get_end_lsn());
401
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 10, id, 1000));
402
    EXPECT_EQ(OB_SUCCESS, wait_lsn_until_flushed(leader.palf_handle_impl_->get_max_lsn(), leader));
403
  }
404
}
405

406
TEST_F(TestObSimpleLogClusterRestart, advance_base_lsn_with_restart)
407
{
408
  SET_CASE_LOG_FILE(TEST_NAME, "advance_base_lsn_with_restart");
409
  OB_LOGGER.set_log_level("INFO");
410
  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
411
  PALF_LOG(INFO, "start advance_base_lsn", K(id));
412
  int64_t leader_idx = 0;
413
  int64_t log_ts = 1;
414
  {
415
    PalfHandleImplGuard leader;
416
    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
417
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, id));
418
    sleep(2);
419
    LSN log_tail =
420
        leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_;
421
    int count = (LSN(PALF_META_BLOCK_SIZE) - log_tail)/4096;
422
    for (int64_t i = 0; i < count; i++) {
423
      EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->enable_vote());
424
    }
425
    while (LSN(PALF_META_BLOCK_SIZE) !=
426
        leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_)
427
    {
428
      sleep(1);
429
    }
430
    sleep(1);
431
    EXPECT_EQ(LSN(PALF_META_BLOCK_SIZE), leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_);
432
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->log_engine_.log_meta_storage_.block_mgr_.switch_next_block(1));
433
  }
434
  EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
435
  {
436
    PalfHandleImplGuard leader;
437
    EXPECT_EQ(OB_SUCCESS, get_leader(id, leader, leader_idx));
438
    EXPECT_LT(LSN(PALF_META_BLOCK_SIZE), leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_);
439
    EXPECT_EQ(OB_SUCCESS, leader.palf_handle_impl_->set_base_lsn(LSN(0)));
440
    sleep(1);
441
    EXPECT_LT(LSN(PALF_META_BLOCK_SIZE) + 4096, leader.palf_handle_impl_->log_engine_.log_meta_storage_.log_tail_);
442
  }
443
}
444

445
TEST_F(TestObSimpleLogClusterRestart, restart_and_clear_tmp_files)
446
{
447
  SET_CASE_LOG_FILE(TEST_NAME, "restart_and_clear_tmp_files");
448
  ObTimeGuard guard("restart_and_clear_tmp_files", 0);
449
  const int64_t id = ATOMIC_AAF(&palf_id_, 1);
450
  int64_t leader_idx = 0;
451
  std::string logserver_dir;
452
  {
453
    PalfHandleImplGuard leader;
454
    EXPECT_EQ(OB_SUCCESS, create_paxos_group(id, leader_idx, leader));
455
    guard.click("create");
456
    logserver_dir = leader.palf_env_impl_->log_dir_;
457
    EXPECT_EQ(OB_SUCCESS, submit_log(leader, 100, leader_idx, 1 * 1024 * 1024));
458
    guard.click("submit_log");
459
    while (leader.palf_handle_impl_->get_end_lsn()
460
           < LSN(100 * 1024 * 1024ul)) {
461
      usleep(100 * 1000);
462
    }
463
  }
464
  const std::string base_dir = logserver_dir;
465
  const std::string tmp_1_dir = base_dir + "/10000.tmp/log/";
466
  const std::string mkdir_tmp_1 = "mkdir -p " + tmp_1_dir;
467
  const std::string dir_2 = base_dir + "/10000000";
468
  const std::string dir_normal_file = base_dir + "/10000000/log/1";
469
  const std::string dir_normal_file1 = base_dir + "/10000000/meta/";
470
  const std::string mkdir_2 = "mkdir -p " + dir_normal_file;
471
  const std::string mkdir_3 = "mkdir -p " + dir_normal_file1;
472
  system(mkdir_tmp_1.c_str());
473
  system(mkdir_2.c_str());
474
  system(mkdir_3.c_str());
475
  int ret = OB_SUCCESS;
476
  guard.click("mkdir");
477
  EXPECT_EQ(OB_ERR_UNEXPECTED, restart_paxos_groups());
478
  CLOG_LOG(INFO, "after restart_paxos_groups after exist tmp dir");
479
  guard.click("restart");
480
  const std::string rm_dir_2 = "rm -rf " + dir_2;
481
  system(rm_dir_2.c_str());
482
  guard.click("rm_dir");
483
  if (OB_FAIL(restart_paxos_groups())) {
484
    PALF_LOG(ERROR, "restart_paxos_groups failed", K(ret));
485
  } else {
486
    {
487
      CLOG_LOG(INFO, "after restart_paxos_groups after remove tmp dir");
488
      guard.click("restart");
489
      bool result = false;
490
      EXPECT_EQ(OB_SUCCESS,
491
                common::FileDirectoryUtils::is_exists(tmp_1_dir.c_str(), result));
492
      EXPECT_EQ(result, false);
493
      PalfHandleImplGuard leader1;
494
      EXPECT_EQ(OB_SUCCESS, get_leader(id, leader1, leader_idx));
495
      guard.click("get_leader");
496
      LogStorage *log_storage =
497
          &leader1.palf_handle_impl_->log_engine_.log_storage_;
498
      LSN lsn_origin_log_tail = log_storage->get_log_tail_guarded_by_lock_();
499
      EXPECT_EQ(OB_SUCCESS, submit_log(leader1, 10, leader_idx, 1 * 1024 * 1024));
500
      while (log_storage->log_tail_ == lsn_origin_log_tail) {
501
        usleep(1 * 1000);
502
        PALF_LOG(INFO, "log_tail is same", KPC(log_storage), K(lsn_origin_log_tail));
503
      }
504
      guard.click("submit_log");
505
      EXPECT_EQ(OB_ITER_END, read_log(leader1));
506
      guard.click("read_log");
507
      PALF_LOG(INFO, "finish read_log", KPC(log_storage), K(lsn_origin_log_tail), KPC(leader1.palf_handle_impl_));
508
    }
509
    // 验证tenant下有临时文件的场景，该临时文件需要归还给log_pool
510
    {
511
      PalfHandleImplGuard leader1;
512
      int64_t leader_idx1 = 0;
513
      EXPECT_EQ(OB_SUCCESS, get_leader(id, leader1, leader_idx1));
514
      std::string palf_log_dir = leader1.palf_handle_impl_->log_engine_.log_storage_.block_mgr_.log_dir_;
515
      ObISimpleLogServer *i_server = get_cluster()[leader_idx1];
516
      ObSimpleLogServer *server = dynamic_cast<ObSimpleLogServer*>(i_server);
517
      std::string log_pool = server->log_block_pool_.log_pool_path_;
518
      const block_id_t min_block_id = server->log_block_pool_.min_block_id_;
519
      char src[1024] = {'\0'};
520
      char dest[1024] = {'\0'};
521
      block_id_to_tmp_string(10000, dest, 1024);
522
      block_id_to_string(min_block_id, src, 1024);
523
      std::string src_str = log_pool + "/" + src;
524
      std::string dest_str = palf_log_dir + "/" + dest;
525
      std::string mv_system = "mv " + src_str + " " + dest_str;
526
      system(mv_system.c_str());
527
      bool result1 = false;
528
      EXPECT_EQ(OB_SUCCESS,
529
                common::FileDirectoryUtils::is_exists(dest_str.c_str(), result1));
530
      EXPECT_EQ(true, result1);
531
      leader1.reset();
532
      EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
533
      EXPECT_EQ(OB_SUCCESS,
534
                common::FileDirectoryUtils::is_exists(dest_str.c_str(), result1));
535
      EXPECT_EQ(false, result1);
536
    }
537
    EXPECT_EQ(OB_SUCCESS, remove_dir());
538
    EXPECT_EQ(OB_SUCCESS, restart_paxos_groups());
539
  }
540
  EXPECT_EQ(OB_SUCCESS, ret);
541
  PALF_LOG(INFO, "end test restart", K(id), K(guard));
542
}
543

544
} // namespace unittest
545
} // namespace oceanbase
546

547
int main(int argc, char **argv)
548
{
549
  RUN_SIMPLE_LOG_CLUSTER_TEST(TEST_NAME);
550
}
551