FWCore/Services/scripts/edmTracerCompactLogViewer.py - master - githubmirror/cmssw

FWCore/Services/scripts/edmTracerCompactLogViewer.py
1 541 строка64 KB
Chris Jones
fix problem with json output for module centric view
03 авг 2026, 17:37
03 авг 2026, 17:370e8109c
#!/usr/bin/env python3
from builtins import range
from itertools import groupby
from operator import attrgetter,itemgetter
import sys
import jsonpickle
from collections import defaultdict
#----------------------------------------------
def printHelp():
    s = '''
To Use: Add the Tracer Service to the cmsRun job use something like this
  in the configuration:

  process.add_(cms.Service("Tracer", fileName = cms.untracked.string("tracer.log")))

  After running the job, execute this script and pass the name of the
  Tracer log file to the script.

  This script will output a more human readable form of the data in the Tracer log file.'''
    return s

#these values come from tracer_setupFile.cc
#enum class Step : char {
#  preSourceTransition = 'S',
#  postSourceTransition = 's',
#  preModulePrefetching = 'P',
#  postModulePrefetching = 'p',
#  preModuleEventAcquire = 'A',
#  postModuleEventAcquire = 'a',
#  preModuleTransition = 'M',
#  preEventReadFromSource = 'R',
#  postEventReadFromSource = 'r',
#  preModuleEventDelayedGet = 'D',
#  postModuleEventDelayedGet = 'd',
#  postModuleTransition = 'm',
#  preESModulePrefetching = 'Q',
#  postESModulePrefetching = 'q',
#  preESModule = 'N',
#  postESModule = 'n',
#  preESModuleAcquire = 'B',
#  postESModuleAcquire = 'b',
#  preFrameworkTransition = 'F',
#  postFrameworkTransition = 'f'
#};


#Special names
kSourceFindEvent = "sourceFindEvent"
kSourceDelayedRead ="sourceDelayedRead"
#this value is defined in the framework itself
kLargestLumiNumber = 4294967295

#these values must match the enum class Phase in tracer_setupFile.cc
class Phase (object):
  destruction = -16
  endJob = -12
  endStream = -11
  writeProcessBlock = -10
  endProcessBlock = -9
  globalWriteRun = -7
  globalEndRun = -6
  streamEndRun = -5
  globalWriteLumi = -4
  globalEndLumi = -3
  streamEndLumi = -2
  clearEvent = -1
  Event = 0
  streamBeginLumi = 2
  globalBeginLumi = 3
  streamBeginRun = 5
  globalBeginRun = 6
  accessInputProcessBlock = 8
  beginProcessBlock = 9
  openFile = 10
  beginStream = 11
  beginJob  = 12
  esSync = 13
  esSyncEnqueue = 14
  getNextTransition = 15
  construction = 16
  finalizeEDModules = 17
  finalizeEventSetupConfiguration = 18
  scheduleConsistencyCheck = 19
  createRunLumiEvents = 20
  finishSchedule = 21
  constructESModules = 22
  startServices = 23
  processPython = 24

#used for json output
class Activity (object):
  prefetch = 0
  acquire = 1
  process = 2
  delayedGet = 3
  externalWork = 4
  temporary = 100


transitionToNames_ = {
    Phase.processPython: 'process python',
    Phase.startServices: 'start Services',
    Phase.constructESModules: 'construct ESModules',
    Phase.finishSchedule: 'finish schedule',
    Phase.createRunLumiEvents: 'create run/lumi/events',
    Phase.scheduleConsistencyCheck: 'schedule consistency check',
    Phase.finalizeEventSetupConfiguration : 'finalize EventSetup configuration',
    Phase.finalizeEDModules: 'finalize EDModules',
    Phase.construction: 'construction',
    Phase.destruction: 'destruction',
    Phase.beginJob: 'begin job',
    Phase.endJob: 'end job',
    Phase.beginStream: 'begin stream',
    Phase.endStream: 'end stream',
    Phase.beginProcessBlock: 'begin process block',
    Phase.endProcessBlock: 'end process block',
    Phase.accessInputProcessBlock: 'access input process block',
    Phase.writeProcessBlock: 'write process block',
    Phase.globalBeginRun: 'global begin run',
    Phase.globalEndRun: 'global end run',
    Phase.globalWriteRun: 'global write run',
    Phase.streamBeginRun: 'stream begin run',
    Phase.streamEndRun: 'stream end run',
    Phase.globalBeginLumi: 'global begin lumi',
    Phase.globalEndLumi: 'global end lumi',
    Phase.globalWriteLumi: 'global write lumi',
    Phase.streamBeginLumi: 'stream begin lumi',
    Phase.streamEndLumi: 'stream end lumi',
    Phase.esSyncEnqueue: 'EventSetup synchronization',
    Phase.esSync: 'EventSetup synchronization',
    Phase.Event: 'event',
    Phase.clearEvent: 'clear event',
    Phase.getNextTransition: 'get next transition'
}

def transitionName(transition):
    return transitionToNames_[transition]

transitionToIndent_ = {
    Phase.processPython: 0,
    Phase.startServices: 0,
    Phase.constructESModules: 0,
    Phase.finishSchedule: 0,
    Phase.createRunLumiEvents: 0,
    Phase.scheduleConsistencyCheck: 0,
    Phase.finalizeEventSetupConfiguration : 0,
    Phase.finalizeEDModules: 0,
    Phase.construction: 0,
    Phase.destruction: 0,
    Phase.endJob: 0,
    Phase.beginJob: 0,
    Phase.beginStream: 0,
    Phase.endStream: 0,
    Phase.beginProcessBlock: 1,
    Phase.endProcessBlock: 1,
    Phase.accessInputProcessBlock: 1,
    Phase.writeProcessBlock: 1,
    Phase.globalBeginRun: 1,
    Phase.globalEndRun: 1,
    Phase.globalWriteRun: 1,
    Phase.streamBeginRun: 1,
    Phase.streamEndRun: 1,
    Phase.globalBeginLumi: 2,
    Phase.globalEndLumi: 2,
    Phase.globalWriteLumi: 2,
    Phase.streamBeginLumi: 2,
    Phase.streamEndLumi: 2,
    Phase.Event: 3,
    Phase.clearEvent: 3,
    Phase.esSyncEnqueue: 1,
    Phase.esSync: 1,
    Phase.getNextTransition: 1
}
def transitionIndentLevel(transition):
    return transitionToIndent_[transition]

globalTransitions_ = {
    Phase.processPython,
    Phase.startServices,
    Phase.constructESModules,
    Phase.finishSchedule,
    Phase.createRunLumiEvents,
    Phase.scheduleConsistencyCheck,
    Phase.finalizeEventSetupConfiguration,
    Phase.finalizeEDModules,
    Phase.construction,
    Phase.destruction,
    Phase.endJob,
    Phase.beginJob,
    Phase.beginProcessBlock,
    Phase.endProcessBlock,
    Phase.accessInputProcessBlock,
    Phase.writeProcessBlock,
    Phase.globalBeginRun,
    Phase.globalEndRun,
    Phase.globalWriteRun,
    Phase.globalBeginLumi,
    Phase.globalEndLumi,
    Phase.globalWriteLumi,
    Phase.esSyncEnqueue,
    Phase.esSync,
    Phase.getNextTransition
}
def transitionIsGlobal(transition):
    return transition in globalTransitions_;

def textPrefix_(time, indentLevel):
    #using 11 spaces for time should accomodate a job that runs 24 hrs
    return f'{time:>11} '+"++"*indentLevel

class FrameworkTransitionParser (object):
    def __init__(self, payload):
        self.transition = int(payload[0])
        self.index = int(payload[1])
        self.sync = (int(payload[2]), int(payload[3]), int(payload[4]))
        self.time = int(payload[5])
    def indentLevel(self):
        return transitionIndentLevel(self.transition)
    def textPrefix(self):
        return textPrefix_(self.time, self.indentLevel())
    def syncText(self):
        if self.transition == Phase.globalBeginRun or Phase.globalEndRun == self.transition:
            return f'run={self.sync[0]}'
        if self.transition == Phase.globalWriteRun:
            return f'run={self.sync[0]}'
        if self.transition == Phase.streamBeginRun or Phase.streamEndRun == self.transition:
            return f'run={self.sync[0]}'
        if self.transition == Phase.globalBeginLumi or Phase.globalEndLumi == self.transition:
            return f'run={self.sync[0]} lumi={self.sync[1]}'
        if self.transition == Phase.globalWriteLumi:
            return f'run={self.sync[0]} lumi={self.sync[1]}'
        if self.transition == Phase.streamBeginLumi or Phase.streamEndLumi == self.transition:
            return f'run={self.sync[0]} lumi={self.sync[1]}'
        if self.transition == Phase.Event:
            return f'run={self.sync[0]} lumi={self.sync[1]} event={self.sync[2]}'
        if self.transition == Phase.esSyncEnqueue or self.transition == Phase.esSync:
            return f'run={self.sync[0]} lumi={self.sync[1]}'
        if self.transition == Phase.beginJob:
            return ''
        if self.transition == Phase.beginProcessBlock or self.transition == Phase.endProcessBlock or self.transition == Phase.writeProcessBlock or self.transition == Phase.accessInputProcessBlock:
            return ''
        if self.transition in [Phase.finalizeEDModules, Phase.finalizeEventSetupConfiguration, Phase.scheduleConsistencyCheck, Phase.createRunLumiEvents, Phase.finishSchedule,  Phase.constructESModules, Phase.startServices, Phase.processPython]:
            return ''
        if self.transition == Phase.construction or self.transition == Phase.destruction:
            return ''
    def textPostfix(self):
        return f'{transitionName(self.transition)} : id={self.index} {self.syncText()}'
    def text(self, context):
        return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'

def findMatchingTransition(sync, containers):
    for i in range(len(containers)):
        if containers[i][-1].sync == sync:
            return i
    #need more exhausting search
    for i in range(len(containers)):
        for t in containers[i]:
            if t.sync == sync:
                return i

    print("find failed",sync, containers)
    return None

def popQueuedTransitions(sync, container):
    results = []
    for i in range(len(container)):
        if sync == container[i].sync:
            results.append(container[i])
            results.append(container[i+1])
            del container[i]
            del container[i]
            break
    return results
        
transitionsToFindMatch_ = {
    Phase.globalEndRun,
    Phase.globalEndLumi,
    Phase.globalWriteRun,
    Phase.globalWriteLumi
}

class PreFrameworkTransitionParser (FrameworkTransitionParser):
    def __init__(self, payload):
        super().__init__(payload)
    def textSpecial(self):
        return "starting"
    def jsonInfo(self, data):
        if transitionIsGlobal(self.transition):
            index = 0
            if self.transition == Phase.esSync:
                if self.sync[1] == kLargestLumiNumber:
                    #at end run transition
                    index = findMatchingTransition(list(self.sync), data.allGlobals())
                    container = data.indexedGlobal(index)
                    container[-1].setFinish(self.time)
                else:
                    data._queued[-1].setFinish(self.time)
                    data._queued.append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
                    return
            elif self.transition==Phase.globalBeginRun:
                index = self.index
                #find associated es queued items
                queued = data._queued
                q = popQueuedTransitions(list(self.sync), queued)
                container = data.indexedGlobal(index)
                #find source, should be previous
                last = container[-1]
                if last.type==Phase.globalBeginRun and last.isSrc:
                    last.sync=list(self.sync)
                container.append(q[0])
                container.append(q[1])
            elif self.transition==Phase.globalBeginLumi:
                index = self.index
                #find associated es queued items
                queued = data._queued
                q = popQueuedTransitions(list(self.sync), queued)
                container = data.indexedGlobal(index)
                #find source, should be previous
                last = container[-1]
                if last.type==Phase.globalBeginLumi and last.isSrc:
                    last.sync=list(self.sync)
                container.append(q[0])
                container.append(q[1])
            elif self.transition in transitionsToFindMatch_:
                index = findMatchingTransition(list(self.sync), data.allGlobals())
            container = data.indexedGlobal(index)
        else:
            container = data.indexedStream(self.index)
            if self.transition == Phase.Event:
                #find source, should be previous
                last = container[-1]
                if last.type==Phase.Event and last.isSrc:
                    last.sync=list(self.sync)
            index = self.index
        container.append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
        

class PostFrameworkTransitionParser (FrameworkTransitionParser):
    def __init__(self, payload):
        super().__init__(payload)
    def textSpecial(self):
        return "finished"
    def jsonInfo(self, data):
        if transitionIsGlobal(self.transition):
            if self.transition == Phase.esSync and self.sync[1] != kLargestLumiNumber:
                data._queued[-1].setFinish(self.time)
                return
            index = findMatchingTransition(list(self.sync), data.allGlobals())
            container = data.indexedGlobal(index)
        else:
            container = data.indexedStream(self.index)
        container[-1].setFinish(self.time)


class QueuingFrameworkTransitionParser (FrameworkTransitionParser):
    def __init__(self, payload):
        super().__init__(payload)
    def textSpecial(self):
        return "queuing"
    def jsonInfo(self, data):
        index = -1
        if self.sync[1] == kLargestLumiNumber:
            #find the mtching open run
            index = findMatchingTransition([self.sync[0],0,0], data.allGlobals())
            data.indexedGlobal(index).append( jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))
        else:
            data._queued.append(jsonTransition(type=self.transition, id = index, sync=list(self.sync), start=self.time , finish=0))

class SourceTransitionParser(object):
    def __init__(self, payload):
        self.transition = int(payload[0])
        if self.transition == Phase.getNextTransition:
            self.time = int(payload[1])
            self.index = -1
            return
        self.index = int(payload[1])
        self.time = int(payload[2])
    def indentLevel(self):
        if self.transition == Phase.globalBeginRun:
            return 1
        if self.transition == Phase.globalBeginLumi:
            return 2
        if self.transition == Phase.Event:
            return 3
        if self.transition == Phase.construction:
            return 1
        if self.transition == Phase.getNextTransition:
            return 1
        return None
    def textPrefix(self):
        return textPrefix_(self.time, self.indentLevel())
    def textPostfix(self):
        return f'source during {transitionName(self.transition)} : id={self.index}'
    def text(self, context):
        return f'{self.textPrefix()} {self.textSpecial()}: {self.textPostfix()}'

class PreSourceTransitionParser(SourceTransitionParser):
    def __init__(self, payload, moduleCentric):
        self._moduleCentric = moduleCentric
        super().__init__(payload)
    def textSpecial(self):
        return "starting"
    def jsonInfo(self, data):
        if self.transition == Phase.getNextTransition:
            data._nextTrans.append(jsonTransition(type=self.transition, id=self.index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
            if self._moduleCentric:
                #this all goes to a module ID sorted container so not knowing actual index is OK
                data.findOpenSlotInModGlobals(0,0).append(data._nextTrans[-1])
            return
        elif self.transition == Phase.construction:
            index = 0
            container = data.indexedGlobal(index)
        elif self.transition == Phase.Event:
            index = self.index
            container = data.indexedStream(index)
        else:
            index = self.index
            container = data.indexedGlobal(index)
        nextTrans = data._nextTrans
        if nextTrans:
            data._nextTrans = []
            for t in nextTrans:
                t.id=index
                #find proper time order in the container
                transStartTime = t.start
                inserted = False
                for i in range(-1, -1*len(container), -1):
                    if transStartTime > container[i].start:
                        if i == -1:
                            container.append(t)
                            inserted = True
                            break
                        else:
                            container.insert(i+1,t)
                            inserted = True
                            break
                if not inserted:
                    container.insert(0,t)
        container.append(jsonTransition(type=self.transition, id=index, sync=[0,0,0], start=self.time, finish=0, isSrc=True))
        if self._moduleCentric:
            if self.transition == Phase.Event:
                data.findOpenSlotInModStreams(index,0).append(container[-1])
            else:
                data.findOpenSlotInModGlobals(index,0).append(container[-1])

class PostSourceTransitionParser(SourceTransitionParser):
    def __init__(self, payload, moduleCentric):
        super().__init__(payload)
        self._moduleCentric = moduleCentric
    def textSpecial(self):
        return "finished"
    def jsonInfo(self, data):
        index = self.index
        if self.transition == Phase.Event:
            container = data.indexedStream(index)
        elif self.transition == Phase.getNextTransition:
            data._nextTrans[-1].setFinish(self.time)
            return
        elif self.transition == Phase.construction:
            pre = None
            for i, g in enumerate(data.allGlobals()):
                for t in reversed(g):
                    if t.type != Phase.construction:
                        break
                    if t.isSrc:
                        pre = t
                        break
                if pre:
                    pre.setFinish(self.time)
                    break
            return
        else:
            container = data.indexedGlobal(index)

        container[-1].setFinish(self.time)

class EDModuleTransitionParser(object):
    def __init__(self, payload, moduleNames):
        self.transition = int(payload[0])
        self.index = int(payload[1])
        self.moduleID = int(payload[2])
        self.moduleName = moduleNames[self.moduleID][0]
        self.moduleType = moduleNames[self.moduleID][1]
        self.callID = int(payload[3])
        self.requestingModuleID = int(payload[4])
        self.requestingCallID = int(payload[5])
        self.requestingModuleName = None
        if self.requestingModuleID != 0:
            self.requestingModuleName = moduleNames[self.requestingModuleID][0]
        self.time = int(payload[6])
    def baseIndentLevel(self):
        return transitionIndentLevel(self.transition)
    def textPrefix(self, context):
        indent = 0
        if self.requestingModuleID != 0:
            indent = context[(self.transition, self.index, self.requestingModuleID, self.requestingCallID)]
        context[(self.transition, self.index, self.moduleID, self.callID)] = indent+1
        return textPrefix_(self.time, indent+1+self.baseIndentLevel())
    def textPostfix(self):
        return f'{self.moduleName} during {transitionName(self.transition)} : id={self.index}'
    def textIfTransform(self):
        if self.callID:
            return f' transform {self.callID-1}'
        return ''
    def text(self, context):
        return f'{self.textPrefix(context)} {self.textSpecial()}{self.textIfTransform()}: {self.textPostfix()}'
    def _preJson(self, activity, data, mayUseTemp = False):
        index = self.index
        found = False
        if mayUseTemp:
            compare = lambda x: x.type == self.transition and x.id == self.index and hasattr(x, 'mod') and x.mod == self.moduleID and x.call == self.callID and x.requestingMod == self.requestingModuleID and x.requestingCall == self.requestingCallID and (x.act == Activity.temporary or x.act == Activity.externalWork)
            if transitionIsGlobal(self.transition):
                item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
            else:
                item,slot = data.findLastInModStreams(index, self.moduleID, compare)
            if slot:
                if item.act == Activity.temporary:
                    slot.pop()
                else:
                    item.setFinish(self.time)
                found = True
        if not found:
            if transitionIsGlobal(self.transition):
                slot = data.findOpenSlotInModGlobals(index, self.moduleID)
            else:
                slot = data.findOpenSlotInModStreams(index, self.moduleID)
        slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, requestingMod=self.requestingModuleID, requestingCall=self.requestingCallID, activity=activity, start=self.time))
        return slot[-1]
    def _postJson(self, data, injectAfter = None):
        compare = lambda x: x.id == self.index and x.mod == self.moduleID and x.call == self.callID and x.type == self.transition and x.requestingMod == self.requestingModuleID and x.requestingCall == self.requestingCallID
        index = self.index
        if transitionIsGlobal(self.transition):
            item,slot = data.findLastInModGlobals(index, self.moduleID, compare)
        else:
            item,slot = data.findLastInModStreams(index, self.moduleID, compare)
        if item is None:
            print(f"failed to find {self.moduleID} for {self.transition} in {self.index}")
        else:
            item.setFinish(self.time)
            if injectAfter:
                slot.append(injectAfter)

class PreEDModuleTransitionParser(EDModuleTransitionParser):
    def __init__(self, payload, names, moduleCentric):
        super().__init__(payload, names)
        self._moduleCentric = moduleCentric
    def textSpecial(self):
        return "starting action"
    def jsonInfo(self, data):
        return self._preJson(Activity.process, data, mayUseTemp=self._moduleCentric)

class PostEDModuleTransitionParser(EDModuleTransitionParser):
    def __init__(self, payload, names):
        super().__init__(payload, names)
    def textSpecial(self):
        return "finished action"
    def jsonInfo(self, data):
        return self._postJson(data)
        
class PreEDModulePrefetchingParser(EDModuleTransitionParser):
    def __init__(self, payload, names, moduleCentric):
        super().__init__(payload, names)
        self._moduleCentric = moduleCentric
    def textSpecial(self):
        return "starting prefetch"
    def jsonInfo(self, data):
        #the total time in prefetching isn't useful, but seeing the start is
        entry = self._preJson(Activity.prefetch, data)
        if self._moduleCentric:
            return entry
        entry.setFinish(-1)
        return entry


class PostEDModulePrefetchingParser(EDModuleTransitionParser):
    def __init__(self, payload, names, moduleCentric):
        super().__init__(payload, names)
        self._moduleCentric = moduleCentric
    def textSpecial(self):
        return "finished prefetch"
    def jsonInfo(self, data):
        if self._moduleCentric:
            #inject a dummy at end of the same slot to guarantee module run is in that slot
            return self._postJson(data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, requestingMod=self.requestingModuleID, requestingCall=self.requestingCallID, activity=Activity.temporary, start=self.time))
        pass

class PreEDModuleAcquireParser(EDModuleTransitionParser):
    def __init__(self, payload, names, moduleCentric):
        super().__init__(payload, names)
        self._moduleCentric = moduleCentric
    def textSpecial(self):
        return "starting acquire"
    def jsonInfo(self, data):
        return self._preJson(Activity.acquire, data, mayUseTemp=self._moduleCentric)

class PostEDModuleAcquireParser(EDModuleTransitionParser):
    def __init__(self, payload, names, moduleCentric):
        super().__init__(payload, names)
        self._moduleCentric = moduleCentric
    def textSpecial(self):
        return "finished acquire"
    def jsonInfo(self, data):
        if self._moduleCentric:
            #inject an external work at end of the same slot to guarantee module run is in that slot
            return self._postJson(data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, requestingMod=self.requestingModuleID, requestingCall=self.requestingCallID, activity=Activity.externalWork, start=self.time))
        return self._postJson(data)

class PreEDModuleEventDelayedGetParser(EDModuleTransitionParser):
    def __init__(self, payload, names, moduleCentric):
        super().__init__(payload, names)
        self._moduleCentric = moduleCentric
    def textSpecial(self):
        return "starting delayed get"
    def jsonInfo(self, data):
        #a pre delayed get is effectively a post transition as the system assumes one state per module per transition instance
        if self._moduleCentric:
            #inject a dummy at end of the same slot to guarantee module run is in that slot
            return self._postJson(data, jsonModuleTransition(type=self.transition, id=self.index, modID=self.moduleID, callID=self.callID, requestingMod=self.requestingModuleID, requestingCall=self.requestingCallID, activity=Activity.temporary, start=self.time))
        return self._postJson(data)
        #return self._preJson(Activity.delayedGet, data)

class PostEDModuleEventDelayedGetParser(EDModuleTransitionParser):
    def __init__(self, payload, names, moduleCentric):
        super().__init__(payload, names)
        self._moduleCentric = moduleCentric
    def textSpecial(self):
        return "finished delayed get"
    def jsonInfo(self, data):
        return self._preJson(Activity.process, data, mayUseTemp=self._moduleCentric)
        #return self._postJson(data)

class PreEventReadFromSourceParser(EDModuleTransitionParser):
    def __init__(self, payload, names, moduleCentric):
        super().__init__(payload, names)
        #shift the module id to be the request
        self.requestingModuleID = self.moduleID
        self.requestingCallID = self.callID
        self.moduleID = 0 #this is the source
        self.callID = 0
        self._moduleCentric = moduleCentric
    def textSpecial(self):
        return "starting read from source"
    def jsonInfo(self, data):
        slot = self._preJson(Activity.process, data, mayUseTemp=self._moduleCentric)
        return slot

class PostEventReadFromSourceParser(EDModuleTransitionParser):
    def __init__(self, payload, names):
        super().__init__(payload, names)
        #shift the module id to be the request
        self.requestingModuleID = self.moduleID
        self.requestingCallID = self.callID
        self.moduleID = 0 #this is the source
        self.callID = 0
    def textSpecial(self):
        return "finished read from source"
    def jsonInfo(self, data):
        return self._postJson(data)

class ESModuleTransitionParser(object):
    def __init__(self, payload, moduleNames, esModuleNames, recordNames):
        self.transition = int(payload[0])
        self.index = int(payload[1])
        self.moduleID = int(payload[2])
        self.moduleName = esModuleNames[self.moduleID][0]
        self.moduleType = esModuleNames[self.moduleID][1]
        self.recordID = int(payload[3])
        if self.transition != Phase.constructESModules:
            self.recordName = recordNames[self.recordID]
        else:
            self.recordName = '<N/A>'
        self.callID = int(payload[4])
        self.requestingModuleID = int(payload[5])
        self.requestingCallID = int(payload[6])
        self.requestingModuleName = None
        if self.requestingModuleID < 0 :
            self.requestingModuleName = esModuleNames[-1*self.requestingModuleID][0]
        elif self.requestingModuleID != 0:
            self.requestingModuleName = moduleNames[self.requestingModuleID][0]
        else:
            self.requestingModuleName = '<N/A>'
        self.time = int(payload[7])
    def baseIndentLevel(self):
        return transitionIndentLevel(self.transition)
    def textPrefix(self, context):
        indent = 0
        indent = context[(self.transition, self.index, self.requestingModuleID, self.requestingCallID)]
        context[(self.transition, self.index, -1*self.moduleID, self.callID)] = indent+1
        return textPrefix_(self.time, indent+1+self.baseIndentLevel())
    def textPostfix(self):
        return f'esmodule {self.moduleName} in record {self.recordName} during {transitionName(self.transition)} : id={self.index}'
    def text(self, context):
        return f'{self.textPrefix(context)} {self.textSpecial()}: {self.textPostfix()}'
    def _preJson(self, activity, data):
        index = self.index
        if transitionIsGlobal(self.transition):
            slot = data.findOpenSlotInModGlobals(index, -1*self.moduleID)
        else:
            slot = data.findOpenSlotInModStreams(index, -1*self.moduleID)
        slot.append(jsonModuleTransition(type=self.transition, id=self.index, modID=-1*self.moduleID, callID=self.callID, requestingMod=self.requestingModuleID, requestingCall=self.requestingCallID, activity=activity, start=self.time))
        return slot[-1]
    def _postJson(self, data):
        compare = lambda x: x.id == self.index and x.mod == -1*self.moduleID and x.call == self.callID
        index = self.index
        if transitionIsGlobal(self.transition):
            item,s = data.findLastInModGlobals(index, -1*self.moduleID, compare)
        else:
            item,s = data.findLastInModStreams(index, -1*self.moduleID, compare)
        if item is None:
            print(f"failed to find {-1*self.moduleID} for {self.transition} in {self.index}")
            return
        item.setFinish(self.time)


class PreESModuleTransitionParser(ESModuleTransitionParser):
    def __init__(self, payload, names, esNames, recordNames):
        super().__init__(payload, names, esNames, recordNames)
    def textSpecial(self):
        return "starting action"
    def jsonInfo(self, data):
        return self._preJson(Activity.process, data)

class PostESModuleTransitionParser(ESModuleTransitionParser):
    def __init__(self, payload, names, esNames, recordNames):
        super().__init__(payload, names, esNames, recordNames)
    def textSpecial(self):
        return "finished action"
    def jsonInfo(self, data):
        return self._postJson(data)

class PreESModulePrefetchingParser(ESModuleTransitionParser):
    def __init__(self, payload, names, esNames, recordNames, moduleCentric):
        super().__init__(payload, names, esNames, recordNames)
        self._moduleCentric = moduleCentric
    def textSpecial(self):
        return "starting prefetch"
    def jsonInfo(self, data):
        entry = self._preJson(Activity.prefetch, data)
        if not self._moduleCentric:
            entry.setFinish(-1)
        return entry

class PostESModulePrefetchingParser(ESModuleTransitionParser):
    def __init__(self, payload, names, esNames, recordNames, moduleCentric):
        super().__init__(payload, names, esNames, recordNames)
        self._moduleCentric = moduleCentric
    def textSpecial(self):
        return "finished prefetch"
    def jsonInfo(self, data):
        if self._moduleCentric:
            return self._postJson(data)
        pass

class PreESModuleAcquireParser(ESModuleTransitionParser):
    def __init__(self, payload, names, recordNames):
        super().__init__(payload, names, recordNames)
    def textSpecial(self):
        return "starting acquire"
    def jsonInfo(self, data):
        return self._preJson(Activity.acquire, data)

class PostESModuleAcquireParser(ESModuleTransitionParser):
    def __init__(self, payload, names, esNames, recordNames):
        super().__init__(payload, names, esNames, recordNames)
    def textSpecial(self):
        return "finished acquire"
    def jsonInfo(self, data):
        return self._postJson(data)


def lineParserFactory (step, payload, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric):
    if step == 'F':
        parser = PreFrameworkTransitionParser(payload)
        if parser.transition == Phase.esSyncEnqueue:
            return QueuingFrameworkTransitionParser(payload)
        return parser
    if step == 'f':
        return PostFrameworkTransitionParser(payload)
    if step == 'S':
        return PreSourceTransitionParser(payload, moduleCentric)
    if step == 's':
        return PostSourceTransitionParser(payload, moduleCentric)
    if frameworkOnly:
        return None
    if step == 'M':
        return PreEDModuleTransitionParser(payload, moduleNames, moduleCentric)
    if step == 'm':
        return PostEDModuleTransitionParser(payload, moduleNames)
    if step == 'P':
        return PreEDModulePrefetchingParser(payload, moduleNames, moduleCentric)
    if step == 'p':
        return PostEDModulePrefetchingParser(payload, moduleNames, moduleCentric)
    if step == 'A':
        return PreEDModuleAcquireParser(payload, moduleNames, moduleCentric)
    if step == 'a':
        return PostEDModuleAcquireParser(payload, moduleNames, moduleCentric)
    if step == 'D':
        return PreEDModuleEventDelayedGetParser(payload, moduleNames, moduleCentric)
    if step == 'd':
        return PostEDModuleEventDelayedGetParser(payload, moduleNames, moduleCentric)
    if step == 'R':
        return PreEventReadFromSourceParser(payload, moduleNames, moduleCentric)
    if step == 'r':
        return PostEventReadFromSourceParser(payload, moduleNames)
    if step == 'N':
        return PreESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
    if step == 'n':
        return PostESModuleTransitionParser(payload, moduleNames, esModuleNames, recordNames)
    if step == 'Q':
        return PreESModulePrefetchingParser(payload, moduleNames, esModuleNames, recordNames, moduleCentric)
    if step == 'q':
        return PostESModulePrefetchingParser(payload, moduleNames, esModuleNames, recordNames, moduleCentric)
    if step == 'B':
        return PreESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)
    if step == 'b':
        return PostESModuleAcquireParser(payload, moduleNames, esModuleNames, recordNames)

    
#----------------------------------------------
def processingStepsFromFile(f,moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric):
    for rawl in f:
        l = rawl.strip()
        if not l or l[0] == '#':
            continue
        (step,payload) = tuple(l.split(None,1))
        payload=payload.split()

        parser = lineParserFactory(step, payload, moduleNames, esModuleNames, recordNames, frameworkOnly, moduleCentric)
        if parser:
            yield parser
    return

class TracerCompactFileParser(object):
    def __init__(self,f, frameworkOnly, moduleCentric):
        streamBeginRun = str(Phase.streamBeginRun)
        numStreams = 0
        numStreamsFromSource = 0
        moduleNames = {}
        esModuleNames = {}
        recordNames = {}
        for rawl in f:
            l = rawl.strip()
            if l and l[0] == 'M':
                i = l.split(' ')
                if i[3] == streamBeginRun:
                    #found global begin run
                    numStreams = int(i[1])+1
                    break
            if numStreams == 0 and l and l[0] == 'S':
                s = int(l.split(' ')[1])
                if s > numStreamsFromSource:
                  numStreamsFromSource = s
            if len(l) > 5 and l[0:2] == "#M":
                (id,name,mtype)=tuple(l[2:].split())
                moduleNames[int(id)] = (name,mtype)
                continue
            if len(l) > 5 and l[0:2] == "#N":
                (id,name,mtype)=tuple(l[2:].split())
                esModuleNames[int(id)] = (name,mtype)
                continue
            if len(l) > 5 and l[0:2] == "#R":
                (id,name)=tuple(l[2:].split())
                recordNames[int(id)] = name
                continue

        self._f = f
        self._frameworkOnly = frameworkOnly
        self._moduleCentric = moduleCentric
        if numStreams == 0:
          numStreams = numStreamsFromSource +2
        self.numStreams =numStreams
        self._moduleNames = moduleNames
        self._esModuleNames = esModuleNames
        self._recordNames = recordNames
        self.maxNameSize =0
        self.maxTypeSize =0
        for k,v in moduleNames.items():
            self.maxNameSize = max(self.maxNameSize,len(v[0]))
            self.maxTypeSize = max(self.maxTypeSize,len(v[1]))
        for k,v in esModuleNames.items():
            self.maxNameSize = max(self.maxNameSize,len(v[0]))
            self.maxTypeSize = max(self.maxTypeSize,len(v[1]))
        self.maxNameSize = max(self.maxNameSize,len(kSourceDelayedRead))
        self.maxNameSize = max(self.maxNameSize, len('streamBeginLumi'))

    def processingSteps(self):
        """Create a generator which can step through the file and return each processing step.
        Using a generator reduces the memory overhead when parsing a large file.
            """
        self._f.seek(0)
        return processingStepsFromFile(self._f,self._moduleNames, self._esModuleNames, self._recordNames, self._frameworkOnly, self._moduleCentric)

def textOutput( parser ):
    context = {}
    for p in parser.processingSteps():
        print(p.text(context))
    

class Containers(object):
    def __init__(self):
        self._modGlobals = [[]]
        self._modStreams = [[]]
        self._globals = [[]]
        self._streams = [[]]
        self._queued = []
        self._nextTrans = []
    def _extendIfNeeded(self, container, index):
        while len(container) < index+1:
            container.append([])
    def allGlobals(self):
        return self._globals
    def indexedGlobal(self, index):
        self._extendIfNeeded(self._globals, index)
        return self._globals[index]
    def allStreams(self):
        return self._streams
    def indexedStream(self, index):
        self._extendIfNeeded(self._streams, index)
        return self._streams[index]
    def _findOpenSlot(self, index, fullContainer):
        self._extendIfNeeded(fullContainer, index)
        container = fullContainer[index]
        #find open slot
        foundOpenSlot = False
        for slot in container:
            if len(slot) == 0:
                foundOpenSlot = True
                break
            if slot[-1].finish != 0:
                foundOpenSlot = True
                break
        if not foundOpenSlot:
            container.append([])
            slot = container[-1]
        return slot
    def findOpenSlotInModGlobals(self, index, modID):
        return self._findOpenSlot(index, self._modGlobals)
    def findOpenSlotInModStreams(self, index, modID):
        return self._findOpenSlot(index, self._modStreams)
    def _findLastIn(self, index, fullContainer, comparer):
        container = fullContainer[index]
        #find slot containing the pre
        for slot in container:
            if comparer(slot[-1]):
                return (slot[-1],slot)
        return (None,None)
    def findLastInModGlobals(self, index, modID, comparer):
        return self._findLastIn(index, self._modGlobals, comparer)
    def findLastInModStreams(self, index, modID, comparer):
        return self._findLastIn(index, self._modStreams, comparer)
    

class ModuleCentricContainers(object):
    def __init__(self):
        self._modules= []
        self._globals = [[]]
        self._streams = [[]]
        self._queued = []
        self._nextTrans = []
        self._moduleIDOffset = 0
    def _moduleID2Index(self, modID):
        return modID + self._moduleIDOffset
    def _extendIfNeeded(self, container, index):
        while len(container) < index+1:
            container.append([])
    def _extendModulesIfNeeded(self, container, index):
        while index + self._moduleIDOffset < 0:
            container.insert(0,[])
            self._moduleIDOffset +=1
        self._extendIfNeeded(container, self._moduleID2Index(index))
    def allGlobals(self):
        return self._globals
    def indexedGlobal(self, index):
        self._extendIfNeeded(self._globals, index)
        return self._globals[index]
    def allStreams(self):
        return self._streams
    def indexedStream(self, index):
        self._extendIfNeeded(self._streams, index)
        return self._streams[index]
    def _findOpenSlot(self, index, fullContainer):
        self._extendModulesIfNeeded(fullContainer, index)
        container = fullContainer[self._moduleID2Index(index)]
        #find open slot
        foundOpenSlot = False
        for slot in container:
            if len(slot) == 0:
                foundOpenSlot = True
                break
            if slot[-1].finish != 0:
                foundOpenSlot = True
                break
        if not foundOpenSlot:
            container.append([])
            slot = container[-1]
        return slot
    def findOpenSlotInModGlobals(self, index, modID):
        return self._findOpenSlot(modID, self._modules)
    def findOpenSlotInModStreams(self, index, modID):
        return self._findOpenSlot(modID, self._modules)
    def _findLastIn(self, index, fullContainer, comparer):
        if not fullContainer:
            return (None, None)
        if len(fullContainer) > self._moduleID2Index(index):
            container = fullContainer[self._moduleID2Index(index)]
        else:
            return (None, None)
        #find slot containing the pre
        for slot in container:
            if slot is not None and comparer(slot[-1]):
                return (slot[-1],slot)
        return (None, None)
    def findLastInModGlobals(self, index, modID, comparer):
        return self._findLastIn(modID, self._modules, comparer)
    def findLastInModStreams(self, index, modID, comparer):
        return self._findLastIn(modID, self._modules, comparer)

    
class JsonTransition(object):
    def __init__(self, type, id, sync, start, finish, isSrc=False):
        self.type = type
        self.id = id
        self.sync = sync
        self.start = start
        self.finish = finish
        self.isSrc = isSrc
    def setFinish(self, finish):
        self.finish = finish

def jsonTransition(type, id, sync, start, finish, isSrc=False):
    return JsonTransition(type, id, sync, start, finish, isSrc)

class JsonModuleTransition(object):
    def __init__(self, type, id, modID, callID, activity, requestingMod, requestingCall, start, finish=0):
        self.type = type
        self.id = id
        self.mod = modID
        self.call = callID
        self.requestingMod = requestingMod
        self.requestingCall = requestingCall
        self.act = activity
        self.start = start
        self.finish = finish
    def setFinish(self, finish):
        self.finish = finish

def jsonModuleTransition(type, id, modID, callID, activity, requestingMod, requestingCall, start, finish=0):
    return JsonModuleTransition(type, id, modID, callID, activity, requestingMod, requestingCall, start, finish)

def startTime(x):
    return x.start
def jsonInfo(parser):
    if parser._moduleCentric:
        data = ModuleCentricContainers()
    else:
        data = Containers()
    for p in parser.processingSteps():
        p.jsonInfo(data)
    #make sure everything is sorted
    for g in data.allGlobals():
        g.sort(key=startTime)
    final = {"transitions" : [] , "modules": [], "esModules": []}
    final["transitions"].append({ "name":"Global", "slots": []})
    globals = final["transitions"][-1]["slots"]
    for i, g in enumerate(data.allGlobals()):
        globals.append(g)
        if not parser._moduleCentric and not parser._frameworkOnly:
            if len(data._modGlobals) < i+1:
                break
            for mod in data._modGlobals[i]:
                globals.append(mod)
    for i,s in enumerate(data.allStreams()):
        final["transitions"].append({"name": f"Stream {i}", "slots":[]})
        stream = final["transitions"][-1]["slots"]
        stream.append(s)
        if not parser._moduleCentric and not parser._frameworkOnly:
            for mod in data._modStreams[i]:
                stream.append(mod)
    if parser._moduleCentric:
        modules = []
        for i,m in parser._moduleNames.items():
            modules.append({"name": f"{m[0]}", "slots":[]})
            slots = modules[-1]["slots"]
            foundSlots = data._modules[data._moduleID2Index(i)]
            time = 0
            for s in foundSlots:
                slots.append(s)
                for t in s:
                    #protect against source having framework transitions
                    if not hasattr(t,'act') or t.act !=Activity.prefetch:
                        time += t.finish-t.start
            modules[-1]['time']=time
        for i,m in parser._esModuleNames.items():
            modules.append({"name": f"{m[0]}", "slots":[]})
            slots = modules[-1]["slots"]
            foundSlots = data._modules[data._moduleID2Index(-1*i)]
            time = 0
            for s in foundSlots:
                slots.append(s)
                for t in s:
                    if t.act !=Activity.prefetch:
                        time += t.finish-t.start
            modules[-1]['time']=time
        modules.sort(key= lambda x : x['time'], reverse=True)
        for m in modules:
            final['transitions'].append(m)

    if not parser._frameworkOnly:
        max = 0
        for k in parser._moduleNames.keys():
            if k > max:
                max = k
        
        final["modules"] =[['',]]*(max+1)
        final["modules"][0] = ['source','unknown']
        for k,v in parser._moduleNames.items():
            final["modules"][k]=[v[0],v[1]]
        
        max = 0
        for k in parser._esModuleNames.keys():
            if k > max:
                max = k
        final["esModules"] = [['','']]*(max+1)
        for k,v in parser._esModuleNames.items():
            final["esModules"][k] = [v[0],v[1]]
    return final

# ============   
transitionToName = {
    -16: "destructor",
    -15: 0, #unused -15
    -14: 0, #unused -14
    -13: 0, #unused -13
    -12: "end_job",
    -11: "end_stream",
    -10: "write_process_block",
    -9: "end_process_block",
    -8: 0, #unused -8
    -7: "write_end_run",
    -6: "global_end_run",
    -5: "stream_end_run",
    -4: "write_end_lumi",
    -3: "global_end_lumi",
    -2: "stream_end_lumi",
    -1: "clear_event",
    0: "event",
    1: 0, #unused 1
    2: "stream_begin_lumi",
    3: "global_begin_lumi",
    4: 0, #unused 4
    5: "stream_begin_run", 
    6: "global_begin_run",
    7: 0, #unused 7
    8: "access_input_process_block",
    9: "begin_process_block",
    10: "open_file",
    11: "begin_stream", 
    12: "begin_job",
    13: "eventsetup_synch",
    14: "eventsetup_sych_enqueue",
    15: "find_next_transition",
    16: "construction",
    17: "finalize_edmodules",
    18: "finalize_eventsetup_configuration",
    19: "schedule_consistency_check",
    20: "create_run/lumi/events",
    21: "finish_schedule",
    22: "construct_esmodules",
    23: "start_services",
    24: "process_python"
}

def typeValueToName(id:int) -> str:
    return transitionToName[id]

class ComponentToName:
    def __init__(self, edmodules,esmodules):
        self._edmodules = edmodules
        self._esmodules = esmodules
    def modToName(self, mod):
        if 0 > mod:
            return self._esmodules[-1*mod][0]
        return self._edmodules[mod][0]
    def modToType(self,mod):
        if 0 > mod:
            return self._esmodules[-1*mod][1]
        return self._edmodules[mod][1]

def globalName(isSource:bool) -> str:
    if isSource:
        return 'source'
    return 'framework'

activityToName = {
    0:"prefetch",
    1:"acquire",
    2:"process",
    3:"delayedGet",
    4:"externalWork"
}

def moduleCategories(namer, mod:int, type:int, act:int):
    return namer.modToName(mod)+","+namer.modToType(mod)+','+typeValueToName(type)+","+activityToName[act]

def moduleTransition(componentNamer:ComponentToName, event:JsonModuleTransition, pid:int, tid:int, moduleCentric:bool):
    if moduleCentric:
        name = activityToName[event.act]
    else:
        name = componentNamer.modToName(event.mod)
    if event.finish == 0:
        return dict(name=name, cat=moduleCategories(componentNamer,event.mod, event.type,event.act), ph="i", ts=event.start, pid=pid, tid=tid)
    duration = (event.finish-event.start)
    if duration < 0.1:
        duration = 1.0
    return dict(name=name, cat=moduleCategories(componentNamer,event.mod, event.type,event.act), ph="X", ts=event.start, dur=duration, pid=pid, tid=tid)
def globalHeaderTransition(e:JsonTransition, sourceType, pid=1):
    if e.isSrc:
        name = 'source'
        cat = "source,"+sourceType+','+typeValueToName(e.type)
    else:
        name = typeValueToName(e.type)
        cat = "framework"
    if e.finish == 0:
        return dict(name=name, cat=cat, ph="i", ts=e.start, pid=pid, tid=1, args={"sync":e.sync})
    return dict(name=name, cat=cat, ph="X", ts=e.start, dur=(e.finish-e.start), pid=pid, tid=1, args={"sync":e.sync})
def streamHeaderTransition(e, streamID:int, sourceType:str):
    if e.isSrc:
        name = 'source'
        cat = "source,"+sourceType+','+typeValueToName(e.type)
    else:
        name = typeValueToName(e.type)
        cat = "framework"
    if e.finish == 0:
        return dict(name=name, cat=cat, ph="i", ts=e.start, pid=streamID+2, tid=1, args={"sync":e.sync})
    return dict(name=name, cat=cat, ph="X", ts=e.start, dur=(e.finish-e.start), pid=streamID+2, tid=1, args={"sync":e.sync})


def convertToChromeTraceFormat(parsed, moduleCentric:bool):
    componentNamer = ComponentToName(edmodules=parsed['modules'], esmodules=parsed['esModules'])
    sourceType = parsed['modules'][0][1]
    name = "Global"
    if moduleCentric:
        name = ". Global"
    traceEvents = [{"name": "process_name", "ph": "M", "pid":1, "tid":0,"args": { "name":name}},
                    {"name":"program start", "ph":"i", "ts":0, "pid":1,"tid":1}]
    #global is first transition
    globalTransitions = parsed["transitions"][0]
    #the first slot holds the framework and source transition information
    for e in globalTransitions["slots"][0]:
            traceEvents.append(globalHeaderTransition(e, sourceType))
    for index,slot in enumerate(globalTransitions["slots"][1:]):
        for e in slot:
            traceEvents.append(moduleTransition(componentNamer, e, 1, index+2, moduleCentric))
    if moduleCentric:
        for pid,stream in enumerate(parsed["transitions"][1:]):
            name = stream["name"]
            if name[:6] == "Stream":
                name = f". {name}"
            traceEvents.append(dict(name="process_name", ph="M", pid=pid+2, tid=1, args={"name": name}))
            #the first slot holds the framework and source transition information
            if stream["name"][:6] == "Stream":
                for e in stream["slots"][0]:
                    traceEvents.append(streamHeaderTransition(e,pid,sourceType))
            elif stream["name"] == "source":
                for index,slot in enumerate(stream["slots"]):
                    for e in slot:
                        if hasattr(e, 'isSrc'):
                            traceEvents.append(globalHeaderTransition(e, sourceType, pid+2))
                        else:
                            traceEvents.append(moduleTransition(componentNamer, e, pid+2, index+1, moduleCentric))
            else:
                for index,slot in enumerate(stream["slots"]):
                    for e in slot:
                        traceEvents.append(moduleTransition(componentNamer, e, pid+2, index+1, moduleCentric))
    else:
        for streamid,stream in enumerate(parsed["transitions"][1:]):
            traceEvents.append(dict(name="process_name", ph="M", pid=streamid+2, tid=streamid, args={"name": f"Stream{streamid}"}))
            #the first slot holds the framework and source transition information
            for e in stream["slots"][0]:
                traceEvents.append(streamHeaderTransition(e,streamid, sourceType))
            for index,slot in enumerate(stream["slots"][1:]):
                for e in slot:
                    traceEvents.append(moduleTransition(componentNamer, e, streamid+2, index+2, moduleCentric))
    trace = dict(traceEvents=traceEvents)

    return trace
#=======================================
import unittest

class DummyFile(list):
    def __init__(self):
        super()
    def seek(self, i):
        pass

class TestModuleCommand(unittest.TestCase):
    def setUp(self):
        self.tracerFile = DummyFile()
        t = [0]
        def incr(t):
            t[0] += 1
            return t[0]
        
        self.tracerFile.extend([
            '#R 1 Record',
            '#S 0 source SourceType',
            '#M 0 source SourceType',
            '#M 1 Module ModuleType',
            '#N 1 ESModule ESModuleType',
             f'F {Phase.processPython} 0 0 0 0 {incr(t)}',
             f'f {Phase.processPython} 0 0 0 0 {incr(t)}',
             f'F {Phase.startServices} 0 0 0 0 {incr(t)}',
             f'f {Phase.startServices} 0 0 0 0 {incr(t)}',
             f'F {Phase.constructESModules} 0 0 0 0 {incr(t)}',
             f'N {Phase.constructESModules} 0 1 0 0 0 0 {incr(t)}',
             f'n {Phase.constructESModules} 0 1 0 0 0 0 {incr(t)}',
             f'f {Phase.constructESModules} 0 0 0 0 {incr(t)}',
             f'S {Phase.construction} 0 {incr(t)}',
             f's {Phase.construction} 0 {incr(t)}',
             f'M {Phase.construction} 0 1 0 0 0 {incr(t)}',
             f'm {Phase.construction} 0 1 0 0 0 {incr(t)}',
             f'F {Phase.finishSchedule} 0 0 0 0 {incr(t)}',
             f'f {Phase.finishSchedule} 0 0 0 0 {incr(t)}',
             f'F {Phase.createRunLumiEvents} 0 0 0 0 {incr(t)}',
             f'f {Phase.createRunLumiEvents} 0 0 0 0 {incr(t)}',
             f'F {Phase.scheduleConsistencyCheck} 0 0 0 0 {incr(t)}',
             f'f {Phase.scheduleConsistencyCheck} 0 0 0 0 {incr(t)}',
             f'F {Phase.finalizeEventSetupConfiguration} 0 0 0 0 {incr(t)}',
             f'f {Phase.finalizeEventSetupConfiguration} 0 0 0 0 {incr(t)}',
             f'F {Phase.finalizeEDModules} 0 0 0 0 {incr(t)}',
             f'f {Phase.finalizeEDModules} 0 0 0 0 {incr(t)}',
             f'F {Phase.beginJob} 0 0 0 0 {incr(t)}',
             f'M {Phase.beginJob} 0 1 0 0 0 {incr(t)}',
             f'm {Phase.beginJob} 0 1 0 0 0 {incr(t)}',
             f'f {Phase.beginJob} 0 0 0 0 {incr(t)}',
             f'F {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
             f'f {Phase.beginProcessBlock} 0 0 0 0 {incr(t)}',
             f'S {Phase.getNextTransition} {incr(t)}',
             f's {Phase.getNextTransition} {incr(t)}',
             f'F {Phase.esSyncEnqueue} -1 1 0 0 {incr(t)}',
             f'F {Phase.esSync} -1 1 0 0 {incr(t)}',
             f'f {Phase.esSync} -1 1 0 0 {incr(t)}',
             f'S {Phase.globalBeginRun} 0 {incr(t)}',
             f's {Phase.globalBeginRun} 0 {incr(t)}',
             f'S {Phase.getNextTransition} {incr(t)}',
             f's {Phase.getNextTransition} {incr(t)}',
             f'F {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
             f'P {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
             f'p {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
             f'M {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
             f'm {Phase.globalBeginRun} 0 1 0 0 0 {incr(t)}',
             f'f {Phase.globalBeginRun} 0 1 0 0 {incr(t)}',
             f'F {Phase.esSyncEnqueue} -1 1 1 0 {incr(t)}',
             f'F {Phase.esSync} -1 1 1 0 {incr(t)}',
             f'f {Phase.esSync} -1 1 1 0 {incr(t)}',
             f'S {Phase.getNextTransition} {incr(t)}',
             f's {Phase.getNextTransition} {incr(t)}',
             f'F {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
             f'M {Phase.streamBeginRun} 0 1 0 0 0 {incr(t)}',
             f'm {Phase.streamBeginRun} 0 1 0 0 0 {incr(t)}',
             f'f {Phase.streamBeginRun} 0 1 0 0 {incr(t)}',
             f'F {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
             f'M {Phase.streamBeginRun} 1 1 0 0 0 {incr(t)}',
             f'm {Phase.streamBeginRun} 1 1 0 0 0 {incr(t)}',
             f'f {Phase.streamBeginRun} 1 1 0 0 {incr(t)}',
             f'S {Phase.globalBeginLumi} 0 {incr(t)}',
             f's {Phase.globalBeginLumi} 0 {incr(t)}',
             f'S {Phase.getNextTransition} {incr(t)}',
             f's {Phase.getNextTransition} {incr(t)}',
             f'F {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
             f'P {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
             f'p {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
             f'M {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
             f'm {Phase.globalBeginLumi} 0 1 0 0 0 {incr(t)}',
             f'f {Phase.globalBeginLumi} 0 1 1 0 {incr(t)}',
             f'F {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
             f'f {Phase.streamBeginLumi} 0 1 1 0 {incr(t)}',
             f'F {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
             f'f {Phase.streamBeginLumi} 1 1 1 0 {incr(t)}',
             f'S {Phase.Event} 0 {incr(t)}',
             f's {Phase.Event} 0 {incr(t)}',
             f'S {Phase.getNextTransition} {incr(t)}',
             f's {Phase.getNextTransition} {incr(t)}',
             f'F {Phase.Event} 0 1 1 1 {incr(t)}',
             f'S {Phase.Event} 1 {incr(t)}',
             f's {Phase.Event} 1 {incr(t)}',
             f'F {Phase.Event} 1 1 1 2 {incr(t)}',
             f'P {Phase.Event} 0 1 0 0 0 {incr(t)}',
             f'p {Phase.Event} 0 1 0 0 0 {incr(t)}',
             f'Q {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
             f'q {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
             f'R {Phase.Event} 0 1 0 0 0 {incr(t)}',
             f'r {Phase.Event} 0 1 0 0 0 {incr(t)}',
             f'N {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
             f'n {Phase.Event} 0 1 1 0 1 0 {incr(t)}',
             f'P {Phase.Event} 1 1 0 0 0 {incr(t)}',
             f'p {Phase.Event} 1 1 0 0 0 {incr(t)}',
             f'M {Phase.Event} 0 1 0 0 0 {incr(t)}',
             f'M {Phase.Event} 1 1 0 0 0 {incr(t)}',
             f'm {Phase.Event} 1 1 0 0 0 {incr(t)}',
             f'm {Phase.Event} 0 1 0 0 0 {incr(t)}',
             f'f {Phase.Event} 0 1 1 1 {incr(t)}',
             f'f {Phase.Event} 1 1 1 2 {incr(t)}'])

        None
    def testContainers(self):
        c = Containers()
        c.indexedGlobal(2)
        self.assertEqual(len(c.allGlobals()), 3)
        c.indexedStream(2)
        self.assertEqual(len(c.allStreams()), 3)
        slot = c.findOpenSlotInModGlobals(2, 1)
        self.assertEqual(len(c._modGlobals),3)
        self.assertEqual(len(slot),0)
        slot.append({"start":1, "finish":0, "id":1})
        def testFind(item):
            return item["id"]==1
        item,s = c.findLastInModGlobals(2, 1, testFind)
        self.assertEqual(item["id"],1)
        self.assertEqual(slot,s)
        slot = c.findOpenSlotInModStreams(2, 1)
        self.assertEqual(len(c._modStreams),3)
        self.assertEqual(len(slot),0)
        slot.append({"start":1, "finish":0, "id":1})
        item,s = c.findLastInModStreams(2, 1, testFind)
        self.assertEqual(item["id"],1)
        self.assertEqual(slot,s)
    def testFrameworkOnly(self):
        parser = TracerCompactFileParser(self.tracerFile, True, False)
        j = jsonInfo(parser)
        #print(j)
        self.assertEqual(len(j["modules"]), 0)
        self.assertEqual(len(j["esModules"]), 0)
        self.assertEqual(len(j['transitions']), 3)
        self.assertEqual(j['transitions'][0]['name'], "Global")
        self.assertEqual(j['transitions'][1]['name'], "Stream 0")
        self.assertEqual(j['transitions'][2]['name'], "Stream 1")
        self.assertEqual(len(j["transitions"][0]["slots"]), 1)
        self.assertEqual(len(j["transitions"][0]["slots"][0]), 22)
        self.assertEqual(len(j["transitions"][1]["slots"]), 1)
        self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
        self.assertEqual(len(j["transitions"][2]["slots"]), 1)
        self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
    def testFull(self):
        parser = TracerCompactFileParser(self.tracerFile, False, False)
        j = jsonInfo(parser)
        #print(jsonpickle.encode(j, unpicklable=False, indent=2))
        self.assertEqual(len(j["modules"]), 2)
        self.assertEqual(len(j["esModules"]), 2)
        self.assertEqual(len(j['transitions']), 3)
        self.assertEqual(j['transitions'][0]['name'], "Global")
        self.assertEqual(j['transitions'][1]['name'], "Stream 0")
        self.assertEqual(j['transitions'][2]['name'], "Stream 1")
        self.assertEqual(len(j["transitions"][0]["slots"]), 2)
        self.assertEqual(len(j["transitions"][0]["slots"][0]), 22)
        self.assertEqual(len(j["transitions"][0]["slots"][1]), 7)
        self.assertEqual(len(j["transitions"][1]["slots"]), 2)
        self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
        self.assertEqual(len(j["transitions"][1]["slots"][1]), 6)
        self.assertEqual(len(j["transitions"][2]["slots"]), 2)
        self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
        self.assertEqual(len(j["transitions"][2]["slots"][1]), 3)
    def testModuleCentric(self):
        parser = TracerCompactFileParser(self.tracerFile, False, True)
        j = jsonInfo(parser)
        #print(jsonpickle.encode(j, unpicklable=False, indent=2))
        self.assertEqual(len(j["modules"]), 2)
        self.assertEqual(len(j["esModules"]), 2)
        self.assertEqual(len(j['transitions']), 6)
        self.assertEqual(j['transitions'][0]['name'], "Global")
        self.assertEqual(j['transitions'][1]['name'], "Stream 0")
        self.assertEqual(j['transitions'][2]['name'], "Stream 1")
        self.assertEqual(j['transitions'][3]['name'], "source")
        self.assertEqual(j['transitions'][4]['name'], "Module")
        self.assertEqual(j['transitions'][5]['name'], "ESModule")
        self.assertEqual(len(j["transitions"][0]["slots"]), 1)
        self.assertEqual(len(j["transitions"][0]["slots"][0]), 22)
        self.assertEqual(len(j["transitions"][1]["slots"]), 1)
        self.assertEqual(len(j["transitions"][1]["slots"][0]), 5)
        self.assertEqual(len(j["transitions"][2]["slots"]), 1)
        self.assertEqual(len(j["transitions"][2]["slots"][0]), 5)
        self.assertEqual(len(j["transitions"][4]["slots"]), 2)
        self.assertEqual(len(j["transitions"][4]["slots"][0]), 10)
        self.assertEqual(len(j["transitions"][4]["slots"][1]), 2)
        self.assertTrue(j["transitions"][4]["slots"][1][-1].finish != 0.0)
        self.assertEqual(len(j["transitions"][5]["slots"]), 1)
        self.assertEqual(len(j["transitions"][5]["slots"][0]), 3)

def runTests():
    return unittest.main(argv=sys.argv[:1])

#=======================================
if __name__=="__main__":
    import argparse
    import re
    import sys

    # Program options
    parser = argparse.ArgumentParser(description='Convert a compact tracer file into human readable output.',
                                     formatter_class=argparse.RawDescriptionHelpFormatter,
                                     epilog=printHelp())
    parser.add_argument('filename',
                        type=argparse.FileType('r'), # open file
                        help='file to process')
    parser.add_argument('-f', '--frameworkOnly',
                        action='store_true',
                        help='''Output only the framework transitions, excluding the individual module transitions.''')
    parser.add_argument('-j', '--json',
                        action='store_true',
                        help='''Write output in json format.''' )
    parser.add_argument('-w', '--web',
                        type=str,
                        default='',
                        help='''If specified, writes chrome trace json formatted output to the specified file. The file can be used with a compatible viewer (Chrome has one built in).''')
    parser.add_argument('-m', '--module_centric',
                        action = 'store_true',
                        help='''For --json or --web, organize data by module instead of by global/stream.''' )
    parser.add_argument('-t', '--test',
                        action='store_true',
                        help='''Run internal tests.''')

    args = parser.parse_args()
    if args.test:
        runTests()
    else :
        parser = TracerCompactFileParser(args.filename, args.frameworkOnly, args.module_centric)
        if args.json or args.web:
            parsed = jsonInfo(parser)
            if args.json:
                import jsonpickle
                j = jsonpickle.encode(parsed, unpicklable=False, indent=2)
                print(j)
            if args.web:
                import json
                web = convertToChromeTraceFormat(parsed, args.module_centric)
                j = json.dumps(web, indent=2)
                f=open(args.web, 'w')
                f.write(j)
                f.close()
        else:
            textOutput(parser)