pyCmdrange.cmdrangeBackEnd

42 ''' 43 Replace all QB_* tokens with the appropriate values from the object's frame range or job stage 44 45 fRange is either the job's frame range value, or the agenda item's specific frame range 46 47 @return: Both the cmd with all QB_FRAME* tokens evaluted, and a dictionary of all tokens and their values. 48 49 @rtype: C{tuple} 50 ''' 51 padding = int(padding) 52 fStep = 1 53 54 if not re.search('[-,]', fRange): 55 # find the trivial single-frame case, with neither a '-' nor a ',' 56 fEnd = fStart = fRange 57 58 elif fRange.count('-') and not fRange.count(','): 59 if not fRange.startswith('-'): 60 # most common case, simple frame range beginning with a positive number 61 (fStart, fEnd) = fRange.split('-', 1) 62 elif fRange.count('-') == 1: 63 # a single negative number 64 fStart = fEnd = fRange 65 else: 66 # rare, a frame range whose start is a negative number 67 m = re.search('(-\d+)-(-?\d+.*)', fRange) 68 (fStart, fEnd) = m.groups() 69 70 if fEnd.count('x'): 71 (fEnd, fStep) = fEnd.split('x') 72 else: 73 # a disjointed frame range containing a comma 74 fList = qb.rangesplit(fRange) 75 fStart = fList[0] 76 fEnd = fList[-1] 77 78 tokens = {} 79 tokens['QB_FRAME_START'] = tokens['QB_FRAME_NUMBER'] = str('%0*d' % (padding, int(fStart))) 80 tokens['QB_FRAME_END'] = str('%0*d' % (padding, int(fEnd))) 81 tokens['QB_FRAME_STEP'] = str(fStep) 82 tokens['QB_FRAME_RANGE'] = fRange 83 84 for rgx in QB_TOKEN_RGX: 85 cmd = rgx.sub(tokens[rgx.pattern], cmd) 86 87 return (cmd, tokens)

91 ''' 92 A backend for a python-based cmdrange jobtype which allows for extensible log parsing. 93 ''' 94

95 - def __init__(self, job):

96 super(CmdRangeBackEnd, self).__init__(job) 97 98 # submit the job with job['package']['dev'] = True to set the logging level to DEBUG 99 self.logging = logging.getLogger('%s' % self.__class__.__name__)

100

101 - def executeWork(self):

102 ''' 103 Now that we've got a job from the supervisor and have fired up the python 104 interpreter in the child process, it's time to ask the supervisor for an 105 agenda item to process. 106 107 Everything we need to know to get the work done is in either the job's or 108 the work's package dict. 109 ''' 110 if self.dev: 111 pp = pprint.PrettyPrinter(indent=2, width=1) 112 113 while True: 114 work_status = 1 115 116 backendUtils.bannerPrint('Requesting work', fhList=[sys.stdout, sys.stderr]) 117 work = qb.requestwork() 118 119 if self.dev: 120 print "WORK:" 121 pp.pprint(work) 122 123 # Deal with the minimal set of work statuses 124 if work['status'] == "failed": 125 # preflights failed, so skip this agenda item and mark it failed 126 print 'preflights for work [%s:%s] failed' % (self.job['id'], work['name']) 127 work['status'] = 'failed' 128 qb.reportwork(work) 129 continue 130 elif work['status'] == 'complete': 131 work_status = 0 132 break 133 elif work['status'] == 'pending': 134 # preempted -- bail out 135 print 'job %s has been preempted' % self.job['id'] 136 work_status = 0 137 qb.reportjob('pending') 138 break 139 elif work['status'] == 'blocked': 140 # blocked -- perhaps part of a dependency chain 141 print 'job %s has been blocked' % self.job['id'] 142 work_status = 0 143 qb.reportjob('blocked') 144 break 145 elif work['status'] == 'waiting': 146 # waiting -- rare, come back in QB_WAITING_TIMEOUT seconds 147 print 'job %s will be back in %s seconds' % (self.job['id'], self.QB_WAITING_TIMEOUT) 148 time.sleep(self.QB_WAITING_TIMEOUT) 149 continue 150 151 if not work['package']: 152 work['package'] = {} 153 154 #============================================================ 155 # Do the actual work 156 #============================================================ 157 backendUtils.bannerPrint('Processing work: %s' % work['name']) 158 159 cmd = self.job['package']['cmdline'] 160 (cmd, qbTokens) = evaluateQBTokens(cmd, work['name'], self.job['package'].get('padding', 1)) 161 162 retCode = self.runCmd(work, cmd, qbTokens) 163 164 if work['status'] == 'failed': 165 backendUtils.flushPrint('ERROR: Processing work failed\n', fhList=[sys.stdout, sys.stderr]) 166 work_status = 1 167 else: 168 work_status = retCode 169 170 if work.get('resultpackage') is None: 171 work['resultpackage'] = {} 172 # ----------------------------------------------------------- 173 # set the work status, then report it back to the supervisor 174 # so that it can update the server-side agenda 175 # ----------------------------------------------------------- 176 if work_status != 0: 177 # either the work or the job instance itself has failed 178 work['status'] = 'failed' 179 elif self.outputPaths_required and len(work.get('resultpackage', {}).get('outputPaths', '')) == 0: 180 work['status'] = 'failed' 181 backendUtils.flushPrint('WARNING: no "regex_outputPaths" match was found, setting agenda item status to "failed".', fhList=[sys.stdout, sys.stderr]) 182 else: 183 # mark the work as complete, and reset both the failure counter and timer 184 work['status'] = 'complete' 185 186 # report the work status back to the supervisor 187 backendUtils.bannerPrint('Reporting work as %(status)s: %(name)s ' % work, fhList=[sys.stdout, sys.stderr]) 188 189 qb.reportwork(work)

Source Code for Module pyCmdrange.cmdrangeBackEnd