glusterfs

1
#!/bin/python
2
"""
3
  Copyright (c) 2006-2012 Red Hat, Inc. <http://www.redhat.com>
4
  This file is part of GlusterFS.
5

6
  This file is licensed to you under your choice of the GNU Lesser
7
  General Public License, version 3 or any later version (LGPLv3 or
8
  later), or the GNU General Public License, version 2 (GPLv2), in all
9
  cases as published by the Free Software Foundation.
10

11
"""
12

13
from __future__ import print_function
14
import os
15
import string
16
import sys
17

18

19
class NFSRequest:
20
        def requestIsEntryOp (self):
21
                op = self.op
22
                if op == "CREATE" or op == "LOOKUP" or op == "REMOVE" or op == "LINK" or op == "RENAME" or op == "MKDIR" or op == "RMDIR" or op == "SYMLINK" or op == "MKNOD":
23
                        return 1
24
                else:
25
                        return 0
26

27
        def __init__ (self, logline, linecount):
28
                self.calllinecount = 0
29
                self.xid = ""
30
                self.op = ""
31
                self.opdata = ""
32
                self.replydata = ""
33
                self.replylinecount = 0
34
                self.timestamp = ""
35
                self.entryname = ""
36
                self.gfid = ""
37
                self.replygfid = ""
38

39
                tokens = logline.strip ().split (" ")
40
                self.timestamp = tokens[0] + " " + tokens[1]
41
                if "XID:" not in tokens:
42
                        return None
43

44
                if  "args:" not in tokens:
45
                        return None
46

47
                self.calllinecount = linecount
48

49
                xididx = tokens.index ("XID:")
50
                self.xid = tokens [xididx+1].strip(",") 
51

52
                opidx = tokens.index ("args:")
53
                self.op = tokens [opidx-1].strip (":")
54
                self.opdata = " ".join(tokens [opidx+1:])
55
                if self.requestIsEntryOp ():
56
                        nameidx = tokens.index ("name:")
57
                        self.entryname = tokens[nameidx + 1].strip (",")
58
                gfididx = tokens.index ("gfid")
59
                self.gfid = tokens[gfididx +1].strip(",")
60

61

62
        def getXID (self):
63
                return self.xid
64

65
        def setReply (self, logline, linecount):
66
                tokens = logline.strip ().split (" ")
67
                timestamp = tokens[0] + " " + tokens[1]
68
                statidx = tokens.index ("NFS:")
69
                self.replydata = " TimeStamp: " + timestamp + " " + " ".join (tokens [statidx+1:])
70
                self.replylinecount = linecount
71
                if "gfid" in tokens:
72
                        gfididx = tokens.index ("gfid")
73
                        self.replygfid = tokens [gfididx + 1].strip(",")
74

75
        def dump (self):
76
                print("ReqLine: " + str(self.calllinecount) + " TimeStamp: " + self.timestamp + ", XID: " + self.xid + " " + self.op + " ARGS: " + self.opdata + " RepLine: " + str(self.replylinecount) + " " + self.replydata)
77

78
class NFSLogAnalyzer:
79

80
        def __init__ (self, optn, trackfilename, tracknamefh, stats):
81
                self.stats = stats
82
                self.xid_request_map = {}
83
                self.orphan_replies = {}
84
                self.rqlist = []
85
                self.CALL = 1
86
                self.REPLY = 2
87
                self.optn = optn
88
                self.trackfilename = trackfilename
89
                self.tracknamefh = tracknamefh
90
                self.trackedfilehandles = []
91

92
        def handle_call_line (self, logline, linecount):
93
                newreq = NFSRequest (logline, linecount)
94
                xid = newreq.getXID ()
95
                if (self.optn == SYNTHESIZE):
96
                        self.xid_request_map [xid] = newreq
97
                        self.rqlist.append(newreq)
98
                elif self.optn == TRACKFILENAME:
99
                        if newreq.requestIsEntryOp():
100
                                if newreq.entryname == self.trackfilename:
101
                                        self.xid_request_map [xid] = newreq
102
                                        self.rqlist.append(newreq)
103
                                else:
104
                                        del newreq
105
                        elif self.tracknamefh == ENABLE_TRACKNAME_FH:
106
                                if len (self.trackedfilehandles) > 0:
107
                                        if newreq.gfid in self.trackedfilehandles:
108
                                                self.xid_request_map [xid] = newreq
109
                                                self.rqlist.append(newreq)
110
                                        else:
111
                                                del newreq
112
                                else:
113
                                        del newreq
114
                        else:
115
                                del newreq
116

117

118
        def handle_reply_line (self, logline, linecount):
119
                tokens = logline.strip ().split (" ")
120

121
                xididx = tokens.index ("XID:")
122
                xid = tokens [xididx + 1].strip(",")
123
                if xid not in self.xid_request_map.keys ():
124
                        self.orphan_replies [xid] = logline
125
                else:
126
                        rq = self.xid_request_map [xid]
127
                        rq.setReply (logline, linecount)
128
                        if rq.requestIsEntryOp() and rq.entryname == self.trackfilename:
129
                                self.trackedfilehandles.append (rq.replygfid)
130

131
        def analyzeLine (self, logline, linecount):
132
                tokens = logline.strip ().split (" ")
133
                msgtype = 0
134

135
                if "XID:" not in tokens:
136
                        return
137

138
                if  "args:" in tokens:
139
                        msgtype = self.CALL
140
                elif "NFS:" in tokens:
141
                        msgtype = self.REPLY
142

143
                if msgtype == self.CALL:
144
                        self.handle_call_line (logline, linecount)
145
                elif msgtype == self.REPLY:
146
                        self.handle_reply_line (logline, linecount)
147

148
        def getStats (self):
149
                if self.stats == 0:
150
                        return
151
                rcount = len (self.xid_request_map.keys ())
152
                orphancount = len (self.orphan_replies.keys ())
153
                print("Requests: " + str(rcount) + ", Orphans: " + str(orphancount))
154

155
        def dump (self):
156
                self.getStats ()
157
                for rq in self.rqlist:
158
                        rq.dump ()
159
                        del rq
160

161
                self.rqlist = []
162
                self.orphan_replies = {}
163
                self.xid_request_map = {}
164

165

166
linecount = 0
167

168
SYNTHESIZE = 1
169
TRACKFILENAME = 2
170

171
ENABLESTATS = 1
172
DISABLESTATS = 0
173

174
ENABLE_TRACKNAME_FH = 1
175
DISABLE_TRACKNAME_FH = 0
176

177
progmsgcount = 1000
178
dumpinterval = 200000
179
operation = SYNTHESIZE
180
stats = ENABLESTATS
181
tracknamefh = DISABLE_TRACKNAME_FH
182
trackfilename = ""
183

184
"""
185
Print the progress of the analysing operations every X number of lines read from
186
the logs, where X is the argument provided to this option.
187

188
Use this to print a status message every say 10000 lines processed or 100000
189
lines processed to know how much longer the processing will go on for.
190

191

192
USAGE: --progress <NUMLINES>
193
"""
194
if "--progress" in sys.argv:
195
        idx = sys.argv.index ("--progress")
196
        progmsgcount = int(sys.argv[idx+1])
197

198
"""
199
The replies for a NFS request can be separated by hundreds and even thousands
200
of other NFS requests and replies. These can be spread over many hundreds and
201
thousands of log lines. This script maintains a memory dict to map each request
202
to its reply using the XID. Because this is in-core, there is a limit to the
203
number of entries in the dict. At regular intervals, it dumps the mapped
204
requests and the replies into the stdout. The requests whose replies were not
205
found at the point of dumping are left as orphans, i.e. without info about the
206
replies. Use this option to tune the number of lines to maximize the number of
207
requests whose replies are found while balancing the dict size with memory
208
on the machine. The default works fine for most cases.
209

210
USAGE: --dump <NUMLINES>
211
"""
212
if "--dump" in sys.argv:
213
        idx = sys.argv.index ("--dump")
214
        dumpinterval = int(sys.argv[idx+1])
215

216
"""
217
The default operation of the script is to output all the requests mapped to
218
their replies in a single line. This operation mode can be changed by this
219
argument. It is used to print only those operations that were performed on the
220
filename given as the argument to this option. Only those entry operations are
221
printed which contain this filename.
222

223
USAGE: --trackfilename <filename>
224
"""
225
if "--trackfilename" in sys.argv:
226
        idx = sys.argv.index ("--trackfilename")
227
        trackfilename = sys.argv[idx + 1]
228
        operation = TRACKFILENAME
229

230
"""
231
At every dump interval, some stats are printed about the dumped lines.
232
Use this option to disable printing that to avoid cluttering the
233
output.
234
"""
235
if "--nostats" in sys.argv:
236
        stats = DISABLESTATS
237

238
"""
239
While tracking a file using --trackfilename, we're only given those
240
operations which contain the filename. This excludes a large number
241
of operations which operate on that file using its filehandle instead of
242
the filename. This option enables outputting those operations also. It
243
tracks every single file handle that was ever seen in the log for a given
244
filename.
245

246
USAGE: --trackfilename
247
"""
248
if "--tracknamefh" in sys.argv:
249
        tracknamefh = ENABLE_TRACKNAME_FH
250

251
la = NFSLogAnalyzer (operation, trackfilename, tracknamefh, stats)
252

253
for line in sys.stdin:
254
        linecount = linecount + 1
255
        if linecount % dumpinterval == 0:
256
                sys.stderr.write ("Dumping data..\n")
257
                la.dump ()
258

259
        if linecount % progmsgcount == 0:
260
                sys.stderr.write ("Integrating line: "+ str(linecount) + "\n")
261
        la.analyzeLine (line, linecount)
262