cubefs

1
// Copyright 2018 The CubeFS Authors.
2
//
3
// Licensed under the Apache License, Version 2.0 (the "License");
4
// you may not use this file except in compliance with the License.
5
// You may obtain a copy of the License at
6
//
7
//     http://www.apache.org/licenses/LICENSE-2.0
8
//
9
// Unless required by applicable law or agreed to in writing, software
10
// distributed under the License is distributed on an "AS IS" BASIS,
11
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
12
// implied. See the License for the specific language governing
13
// permissions and limitations under the License.
14

15
package cmd
16

17
import (
18
	"fmt"
19
	"sort"
20
	"strconv"
21

22
	"github.com/cubefs/cubefs/proto"
23
	"github.com/cubefs/cubefs/sdk/master"
24
	"github.com/spf13/cobra"
25
)
26

27
const (
28
	cmdMetaPartitionUse   = "metapartition [COMMAND]"
29
	cmdMetaPartitionShort = "Manage meta partition"
30
)
31

32
func newMetaPartitionCmd(client *master.MasterClient) *cobra.Command {
33
	cmd := &cobra.Command{
34
		Use:   cmdMetaPartitionUse,
35
		Short: cmdMetaPartitionShort,
36
	}
37
	cmd.AddCommand(
38
		newMetaPartitionGetCmd(client),
39
		newListCorruptMetaPartitionCmd(client),
40
		newMetaPartitionDecommissionCmd(client),
41
		newMetaPartitionReplicateCmd(client),
42
		newMetaPartitionDeleteReplicaCmd(client),
43
	)
44
	return cmd
45
}
46

47
const (
48
	cmdMetaPartitionGetShort           = "Display detail information of a meta partition"
49
	cmdCheckCorruptMetaPartitionShort  = "Check out corrupt meta partitions"
50
	cmdMetaPartitionDecommissionShort  = "Decommission a replication of the meta partition to a new address"
51
	cmdMetaPartitionReplicateShort     = "Add a replication of the meta partition on a new address"
52
	cmdMetaPartitionDeleteReplicaShort = "Delete a replication of the meta partition on a fixed address"
53
)
54

55
func newMetaPartitionGetCmd(client *master.MasterClient) *cobra.Command {
56
	cmd := &cobra.Command{
57
		Use:   CliOpInfo + " [META PARTITION ID]",
58
		Short: cmdMetaPartitionGetShort,
59
		Args:  cobra.MinimumNArgs(1),
60
		Run: func(cmd *cobra.Command, args []string) {
61
			var (
62
				err         error
63
				partitionID uint64
64
				partition   *proto.MetaPartitionInfo
65
			)
66
			defer func() {
67
				errout(err)
68
			}()
69
			if partitionID, err = strconv.ParseUint(args[0], 10, 64); err != nil {
70
				return
71
			}
72
			if partition, err = client.ClientAPI().GetMetaPartition(partitionID); err != nil {
73
				return
74
			}
75
			stdout(formatMetaPartitionInfo(partition))
76
		},
77
	}
78
	return cmd
79
}
80

81
func newListCorruptMetaPartitionCmd(client *master.MasterClient) *cobra.Command {
82
	cmd := &cobra.Command{
83
		Use:   CliOpCheck,
84
		Short: cmdCheckCorruptMetaPartitionShort,
85
		Long: `If the meta nodes are marked as "Inactive", it means the nodes has been not available for a long time. It is suggested to eliminate
86
the network, disk or other problems first. If the bad nodes can never be "active" again, they are called corrupt nodes. And the 
87
"decommission" command can be used to discard the corrupt nodes. However, if more than half replicas of a partition are on 
88
the corrupt nodes, the few remaining replicas can not reach an agreement with one leader. In this case, you can use the 
89
"metapartition reset" command to fix the problem, however this action may lead to data loss, be careful to do this. The 
90
"reset" command will be released in next version.`,
91
		Run: func(cmd *cobra.Command, args []string) {
92
			var (
93
				diagnosis *proto.MetaPartitionDiagnosis
94
				metaNodes []*proto.MetaNodeInfo
95
				err       error
96
			)
97
			defer func() {
98
				errout(err)
99
			}()
100
			if diagnosis, err = client.AdminAPI().DiagnoseMetaPartition(); err != nil {
101
				return
102
			}
103
			stdout("[Inactive Meta nodes]:\n")
104
			stdout("%v\n", formatMetaNodeDetailTableHeader())
105
			sort.SliceStable(diagnosis.InactiveMetaNodes, func(i, j int) bool {
106
				return diagnosis.InactiveMetaNodes[i] < diagnosis.InactiveMetaNodes[j]
107
			})
108
			for _, addr := range diagnosis.InactiveMetaNodes {
109
				var node *proto.MetaNodeInfo
110
				node, err = client.NodeAPI().GetMetaNode(addr)
111
				metaNodes = append(metaNodes, node)
112
			}
113
			sort.SliceStable(metaNodes, func(i, j int) bool {
114
				return metaNodes[i].ID < metaNodes[j].ID
115
			})
116
			for _, node := range metaNodes {
117
				stdout("%v\n", formatMetaNodeDetail(node, true))
118
			}
119

120
			stdout("\n")
121
			stdout("[Corrupt meta partitions](no leader):\n")
122
			stdout("%v\n", partitionInfoTableHeader)
123
			sort.SliceStable(diagnosis.CorruptMetaPartitionIDs, func(i, j int) bool {
124
				return diagnosis.CorruptMetaPartitionIDs[i] < diagnosis.CorruptMetaPartitionIDs[j]
125
			})
126
			for _, pid := range diagnosis.CorruptMetaPartitionIDs {
127
				var partition *proto.MetaPartitionInfo
128
				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
129
					err = fmt.Errorf("Partition not found, err:[%v] ", err)
130
					return
131
				}
132
				stdout("%v\n", formatMetaPartitionInfoRow(partition))
133
			}
134

135
			stdout("\n")
136
			stdout("%v\n", "[Meta partition lack replicas]:")
137
			stdout("%v\n", partitionInfoTableHeader)
138
			sort.SliceStable(diagnosis.LackReplicaMetaPartitionIDs, func(i, j int) bool {
139
				return diagnosis.LackReplicaMetaPartitionIDs[i] < diagnosis.LackReplicaMetaPartitionIDs[j]
140
			})
141
			for _, pid := range diagnosis.LackReplicaMetaPartitionIDs {
142
				var partition *proto.MetaPartitionInfo
143
				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
144
					err = fmt.Errorf("Partition not found, err:[%v] ", err)
145
					return
146
				}
147
				if partition != nil {
148
					stdout("%v\n", formatMetaPartitionInfoRow(partition))
149
				}
150
			}
151

152
			stdout("\n")
153
			stdout("%v\n", "[Bad meta partitions(decommission not completed)]:")
154
			badPartitionTablePattern := "%-8v    %-10v\n"
155
			stdout(badPartitionTablePattern, "PATH", "PARTITION ID")
156
			for _, bmpv := range diagnosis.BadMetaPartitionIDs {
157
				sort.SliceStable(bmpv.PartitionIDs, func(i, j int) bool {
158
					return bmpv.PartitionIDs[i] < bmpv.PartitionIDs[j]
159
				})
160
				for _, pid := range bmpv.PartitionIDs {
161
					stdout(badPartitionTablePattern, bmpv.Path, pid)
162
				}
163
			}
164

165
			stdout("\n")
166
			stdout("%v\n", "[Meta Partition has unavailable replica]:")
167
			stdout("%v\n", badReplicaPartitionInfoTableHeader)
168
			sort.SliceStable(diagnosis.BadReplicaMetaPartitionIDs, func(i, j int) bool {
169
				return diagnosis.BadReplicaMetaPartitionIDs[i] < diagnosis.BadReplicaMetaPartitionIDs[j]
170
			})
171
			for _, pid := range diagnosis.BadReplicaMetaPartitionIDs {
172
				var partition *proto.MetaPartitionInfo
173
				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
174
					err = fmt.Errorf("Partition not found, err:[%v] ", err)
175
					return
176
				}
177
				if partition != nil {
178
					badReplicaMpInfoRow := formatBadReplicaMpInfoRow(partition)
179
					if "" != badReplicaMpInfoRow {
180
						stdout("%v\n", badReplicaMpInfoRow)
181
					}
182
				}
183
			}
184

185
			stdout("\n")
186
			stdout("%v\n", "[Partition with replica inode count not equal]:")
187
			stdout("%v\n", inodeCountNotEqualInfoTableHeader)
188
			sort.SliceStable(diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs, func(i, j int) bool {
189
				return diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs[i] < diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs[j]
190
			})
191
			for _, pid := range diagnosis.InodeCountNotEqualReplicaMetaPartitionIDs {
192
				var partition *proto.MetaPartitionInfo
193
				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
194
					err = fmt.Errorf("Partition not found, err:[%v] ", err)
195
					return
196
				}
197
				if partition != nil {
198
					stdout("%v\n", formatMetaPartitionReplicaInodeNotEqualInfoRow(partition))
199
				}
200
			}
201

202
			stdout("\n")
203
			stdout("%v\n", "[Partition with replica max inode not equal]:")
204
			stdout("%v\n", maxInodeNotEqualInfoTableHeader)
205
			sort.SliceStable(diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs, func(i, j int) bool {
206
				return diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs[i] < diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs[j]
207
			})
208
			for _, pid := range diagnosis.MaxInodeNotEqualReplicaMetaPartitionIDs {
209
				var partition *proto.MetaPartitionInfo
210
				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
211
					err = fmt.Errorf("Partition not found, err:[%v] ", err)
212
					return
213
				}
214
				if partition != nil {
215
					stdout("%v\n", formatMetaPartitionReplicaInodeNotEqualInfoRow(partition))
216
				}
217
			}
218

219
			stdout("\n")
220
			stdout("%v\n", "[Partition with replica dentry count not equal]:")
221
			stdout("%v\n", dentryCountNotEqualInfoTableHeader)
222
			sort.SliceStable(diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs, func(i, j int) bool {
223
				return diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs[i] < diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs[j]
224
			})
225
			for _, pid := range diagnosis.DentryCountNotEqualReplicaMetaPartitionIDs {
226
				var partition *proto.MetaPartitionInfo
227
				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
228
					err = fmt.Errorf("Partition not found, err:[%v] ", err)
229
					return
230
				}
231
				if partition != nil {
232
					stdout("%v\n", formatMetaPartitionReplicaDentryNotEqualInfoRow(partition))
233
				}
234
			}
235

236
			stdout("\n")
237
			stdout("%v\n", "[Partition with excessive replicas]:")
238
			stdout("%v\n", partitionInfoTableHeader)
239
			sort.SliceStable(diagnosis.ExcessReplicaMetaPartitionIDs, func(i, j int) bool {
240
				return diagnosis.ExcessReplicaMetaPartitionIDs[i] < diagnosis.ExcessReplicaMetaPartitionIDs[j]
241
			})
242
			for _, pid := range diagnosis.ExcessReplicaMetaPartitionIDs {
243
				var partition *proto.MetaPartitionInfo
244
				if partition, err = client.ClientAPI().GetMetaPartition(pid); err != nil {
245
					err = fmt.Errorf("Partition not found, err:[%v] ", err)
246
					return
247
				}
248
				if partition != nil {
249
					stdout("%v\n", formatMetaPartitionInfoRow(partition))
250
				}
251
			}
252
		},
253
	}
254
	return cmd
255
}
256

257
func newMetaPartitionDecommissionCmd(client *master.MasterClient) *cobra.Command {
258
	var clientIDKey string
259
	cmd := &cobra.Command{
260
		Use:   CliOpDecommission + " [ADDRESS] [META PARTITION ID]",
261
		Short: cmdMetaPartitionDecommissionShort,
262
		Args:  cobra.MinimumNArgs(2),
263
		Run: func(cmd *cobra.Command, args []string) {
264
			var (
265
				err         error
266
				partitionID uint64
267
			)
268
			defer func() {
269
				errout(err)
270
			}()
271
			address := args[0]
272
			partitionID, err = strconv.ParseUint(args[1], 10, 64)
273
			if err = client.AdminAPI().DecommissionMetaPartition(partitionID, address, clientIDKey); err != nil {
274
				return
275
			}
276
			stdout("Decommission meta partition successfully\n")
277
		},
278
		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
279
			if len(args) != 0 {
280
				return nil, cobra.ShellCompDirectiveNoFileComp
281
			}
282
			return validMetaNodes(client, toComplete), cobra.ShellCompDirectiveNoFileComp
283
		},
284
	}
285
	cmd.Flags().StringVar(&clientIDKey, CliFlagClientIDKey, client.ClientIDKey(), CliUsageClientIDKey)
286
	return cmd
287
}
288

289
func newMetaPartitionReplicateCmd(client *master.MasterClient) *cobra.Command {
290
	var clientIDKey string
291
	cmd := &cobra.Command{
292
		Use:   CliOpReplicate + " [ADDRESS] [META PARTITION ID]",
293
		Short: cmdMetaPartitionReplicateShort,
294
		Args:  cobra.MinimumNArgs(2),
295
		Run: func(cmd *cobra.Command, args []string) {
296
			var (
297
				err         error
298
				partitionID uint64
299
			)
300
			defer func() {
301
				errout(err)
302
			}()
303
			address := args[0]
304
			partitionID, err = strconv.ParseUint(args[1], 10, 64)
305
			if err = client.AdminAPI().AddMetaReplica(partitionID, address, clientIDKey); err != nil {
306
				return
307
			}
308
			stdout("Add replication successfully\n")
309
		},
310
		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
311
			if len(args) != 0 {
312
				return nil, cobra.ShellCompDirectiveNoFileComp
313
			}
314
			return validMetaNodes(client, toComplete), cobra.ShellCompDirectiveNoFileComp
315
		},
316
	}
317
	cmd.Flags().StringVar(&clientIDKey, CliFlagClientIDKey, client.ClientIDKey(), CliUsageClientIDKey)
318
	return cmd
319
}
320

321
func newMetaPartitionDeleteReplicaCmd(client *master.MasterClient) *cobra.Command {
322
	var clientIDKey string
323
	cmd := &cobra.Command{
324
		Use:   CliOpDelReplica + " [ADDRESS] [META PARTITION ID]",
325
		Short: cmdMetaPartitionDeleteReplicaShort,
326
		Args:  cobra.MinimumNArgs(2),
327
		Run: func(cmd *cobra.Command, args []string) {
328
			var (
329
				err         error
330
				partitionID uint64
331
			)
332
			defer func() {
333
				errout(err)
334
			}()
335
			address := args[0]
336
			partitionID, err = strconv.ParseUint(args[1], 10, 64)
337
			if err != nil {
338
				return
339
			}
340
			if err = client.AdminAPI().DeleteMetaReplica(partitionID, address, clientIDKey); err != nil {
341
				return
342
			}
343
			stdout("Delete replication successfully\n")
344
		},
345
		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
346
			if len(args) != 0 {
347
				return nil, cobra.ShellCompDirectiveNoFileComp
348
			}
349
			return validMetaNodes(client, toComplete), cobra.ShellCompDirectiveNoFileComp
350
		},
351
	}
352
	cmd.Flags().StringVar(&clientIDKey, CliFlagClientIDKey, client.ClientIDKey(), CliUsageClientIDKey)
353
	return cmd
354
}
355