Package pyCmdrange :: Module cmdrangeFrontEnd
[hide private]
[frames] | no frames]

Source Code for Module pyCmdrange.cmdrangeFrontEnd

  1  #!/bin/env python 
  2  ''' 
  3  A convenience class for building Qube command range (pyCmdrange) jobs.   
  4   
  5  Jobs of this type are typically used for iterating over a range of frame numbers, and have an agenda 
  6  comprised of integers, floats, or in the case of a chunkSize > 1, a string like "1-10". 
  7   
  8  The command line is scanned for several QB_* tokens; these tokens are replace at runtime with the 
  9  current frame number(s), step-frame values, etc. 
 10   
 11  The jobtype has an optional 'padding' value for padding the numeric values (all except QB_FRAME_RANGE) 
 12   
 13  The tokens for this jobtype are: 
 14   
 15      QB_FRAME_NUMBER     - the current frame number, also the first frame number in the current frame chunk if chunkSize > 1 
 16      QB_FRAME_START      - the first frame number in the chunk 
 17      QB_FRAME_END        - the last frame number in the chunk 
 18      QB_FRAME_STEP       - the "step" or "by" value, eg, "render on 2's" 
 19      QB_FRAME_RANGE      - the frame range for the chunk, NOT the entire job 
 20   
 21  Examples: 
 22   
 23      Command: render -s QB_FRAME_START -e QB_FRAME_END -b QB_FRAME_STEP  
 24   
 25      Render a sequence from frame 1 -> 100, no chunks (chunkSize=1), padding set to 3 
 26   
 27          When rendering frame 12: 
 28   
 29              QB_FRAME_NUMBER 12 
 30              QB_FRAME_START  12 
 31              QB_FRAME_END    12 
 32              QB_FRAME_STEP   1 
 33              QB_FRAME_RANGE  12 
 34   
 35          Command evaluates to: render -s 012 -e 012 -b 1 
 36       
 37   
 38      Render a sequence from frame 1 -> 100, in 25-frame chunks, padding set to 1 
 39   
 40          The second chunk will be "26-50".  At runtime, the QB_* tokens evaluate as: 
 41   
 42              QB_FRAME_NUMBER 26 
 43              QB_FRAME_START  26 
 44              QB_FRAME_END    50 
 45              QB_FRAME_STEP   1 
 46              QB_FRAME_RANGE  26-50 
 47   
 48          Command evaluates to: render -s 26 -e 50 -b 1 
 49   
 50  ''' 
 51  #====================================== 
 52  #  $Revision: #3 $ 
 53  #  $Change: 12347 $ 
 54  #====================================== 
 55   
 56  import sys 
 57  import os 
 58   
 59  try: 
 60      import qb 
 61  except: 
 62      if 'QBDIR' in os.environ: 
 63          QBDIR = os.environ['QBDIR'] 
 64      else: 
 65          if os.name == 'posix': 
 66              if os.uname()[0] == 'Darwin': 
 67                  QBDIR = '/Applications/pfx/qube' 
 68              else: 
 69                  QBDIR = '/usr/local/pfx/qube' 
 70   
 71      sys.path.append('%s/api/python' % QBDIR) 
 72      import qb 
 73           
 74  import qb.frontend 
 75   
 76   
 77   
 78   
 79   
80 -class CmdRangeJob(qb.frontend.QubeJob):
81 - def __init__(self, cmdline, frameRange, padding=1, chunkSize=1, partitionCount=0, dev=False):
82 ''' 83 Function to initialize members of the class 84 85 @param dev: Set the 'dev' value for the Qube job, enables debug output on the farm. 86 @type dev: bool 87 ''' 88 assert frameRange 89 90 super(CmdRangeJob, self).__init__(dev=dev) 91 92 self['prototype'] = 'pyCmdrange' 93 94 pkg = { 95 'cmdline': cmdline, 96 'frameRange': frameRange, 97 'frameCount': len(qb.genframes(frameRange)), 98 'padding': padding, 99 'logParser': { 100 'className': 'CmdRangeChunkLogParser', 101 'modulePath': 'qb.backend.logParser' 102 } 103 } 104 self['package'].update(pkg) 105 106 self._buildAgenda(frameRange, chunkSize, partitionCount)
107
108 - def _buildAgenda(self, frameRange, chunkSize=1, partitionCount=0):
109 ''' 110 Generate a work agenda for the job. 111 112 If both chunkSize and partitionCount are non-default, chunkSize will be used. 113 114 @param frameRange: frame range, can be formatted like an combination of;'1,2,3,4,5','1-5x2', '1-5,10-50x5' 115 @type frameRange: C{str} 116 117 @param chunkSize: size of frame chunks to be build, don't care how many chunks result 118 @type chunkSize: C{int} 119 120 @param: partitionCount: number of chunks to build, don't care how large the resulting chunks are 121 @type partitionCount: C{int} 122 ''' 123 agenda = [] 124 125 if chunkSize > 1: 126 self['package']['rangeChunkSize'] = chunkSize 127 self['package']['rangeExecution'] = 'chunks:%s' % chunkSize 128 129 self['agenda'] = qb.genchunks(chunkSize, frameRange) 130 131 elif partitionCount > 0: 132 self['package']['rangePartitions'] = partitionCount 133 self['package']['rangeExecution'] = 'partitions:%s' % partitionCount 134 135 self['agenda'] = qb.genpartitions(partitionCount, frameRange) 136 137 else: 138 self['agenda'] = qb.genframes(frameRange)
139 140 141 142 if __name__ == '__main__': 143 ''' 144 A "hello world", meant to show how to build and submit a pyCmdrange job. 145 ''' 146 import sys, os 147 import qb 148 149 # It's only neccessary to setsupervisor() if you have multiple supervisors in your environment 150 #qb.setsupervisor('build01') 151 152 fRange1 = '1-15' 153 fRange2 = '101-150' 154 chunkSize = 25 155 156 jobList = [] 157 158 if '--dev' in sys.argv: 159 dev = True 160 else: 161 dev = False 162 163 #cmdLocDir = os.getcwd() 164 165 # test execution on remote Windows VM... 166 cmdLocDir = r'//psf/Home/Documents/dev/pfx/jburk_mbPro/rel-6.4/qube/src/types/pyCmdrange' 167 168 # test execution on remote linux VM... 169 #cmdLocDir = '/home/jburk/dev/pfx/jburk_mbPro/rel-6.4/qube/src/types/pyCmdrange' 170 171 cmd = '%s/test/printFrameNumber.py QB_FRAME_START QB_FRAME_END 1' % cmdLocDir 172 job = CmdRangeJob(cmd, fRange1, chunkSize=chunkSize, padding=2, dev=dev) 173 job['cpus'] = 2 174 job['env'] = {'FOO': 'bar'} 175 #job['cwd'] = r'C:\ ' 176 job['package']['regex_progress'] = 'Frame: (.*) of' 177 job['package']['regex_outputPaths'] = 'Frame: (.* of .*)' 178 job.setLogParser('CmdRangeChunkLogParser') 179 job['name'] = 'print frame numbers' 180 #jobList.append(job) 181 182 mayaBatchCmd = '__MAYA__ -s QB_FRAME_START -e QB_FRAME_END -b QB_FRAME_STEP -renderer "sw" -proj "/Users/jburk/Documents/maya/projects/default" "/Users/jburk/Documents/maya/projects/default/scenes/test2011.mb"' 183 job = CmdRangeJob(mayaBatchCmd, fRange1, chunkSize=1, padding=4, dev=dev) 184 job['cpus'] = 1 185 job['env'] = {'FOO': 'bar'} 186 #job['cwd'] = r'C:\ ' 187 job['package']['appVersion'] = (2013,) 188 job['package']['regex_progress'] = 'Frame: (.*) of' 189 job['package']['regex_outputPaths'] = 'Frame: (.* of .*)' 190 job.setLogParser('CmdRangeChunkLogParser') 191 job['name'] = 'mayaBatch test' 192 job['reservations'] = 'host.processors=1+' 193 jobList.append(job) 194 195 196 ''' 197 cmd = '%s/test/printProgressPercent.py 0.25' % cmdLocDir 198 job = CmdRangeJob(cmd, '1-8x2', padding=4, dev=dev) 199 job['cpus'] = 2 200 job['package']['regex_progress'] = 'Progress:(.*)%' 201 job.setLogParser('ProgressPercentageLogParser') 202 job['name'] = 'print progress percentage message' 203 jobList.append(job) 204 205 cmd = '%s/test/mimicAErender.py QB_FRAME_START QB_FRAME_END QB_FRAME_STEP 0.25' % cmdLocDir 206 job = CmdRangeJob(cmd, fRange1, chunkSize=chunkSize, dev=dev) 207 job['cpus'] = 2 208 job['package']['regex_progress'] = '^PROGRESS:.*\((\d+)\): \d+ Seconds' 209 job.setLogParser('AERenderLogParser', modulePath='qb.backend.afterEffectsLogParser') 210 job['name'] = 'AfterEffects sequence render messages' 211 jobList.append(job) 212 213 cmd = '%s/test/mimicAErender.py QB_FRAME_START QB_FRAME_END QB_FRAME_STEP 0.5' % cmdLocDir 214 job = CmdRangeJob(cmd, fRange1, partitionCount=1, dev=dev) 215 job['cpus'] = 2 216 job['package']['regex_progress'] = '^PROGRESS:.*\((\d+)\): \d+ Seconds' 217 job.setLogParser('AERenderLogParser', modulePath='qb.backend.afterEffectsLogParser') 218 job['name'] = 'AfterEffects movie render messages' 219 jobList.append(job) 220 221 cmd = '%s/test/mimicAErender.py QB_FRAME_START QB_FRAME_END QB_FRAME_STEP 0.25' % cmdLocDir 222 job = CmdRangeJob(cmd, fRange2, chunkSize=chunkSize, dev=dev) 223 job['cpus'] = 2 224 job['package']['regex_progress'] = '^PROGRESS:.*\((\d+)\): \d+ Seconds' 225 job.setLogParser('AERenderLogParser', modulePath='qb.backend.afterEffectsLogParser') 226 job['name'] = 'AfterEffects sequence render messages' 227 jobList.append(job) 228 229 cmd = '%s/test/mimicAErender.py QB_FRAME_START QB_FRAME_END QB_FRAME_STEP 0.5' % cmdLocDir 230 job = CmdRangeJob(cmd, '%sx3' % fRange2, partitionCount=1, dev=dev) 231 job['cpus'] = 2 232 job['package']['regex_progress'] = '^PROGRESS:.*\((\d+)\): \d+ Seconds' 233 job.setLogParser('AERenderLogParser', modulePath='qb.backend.afterEffectsLogParser') 234 job['name'] = 'AfterEffects movie render messages' 235 jobList.append(job) 236 237 238 cmd = '"/Applications/Adobe After Effects CS5.5/aerender" -project "/Users/jburk/Documents/AE_projects/Qube_Test.aep" -s QB_FRAME_START -e QB_FRAME_END -i QB_FRAME_STEP -output "/Users/jburk/Documents/AE_projects/images/QUBE_Test_0to279_[#####].tif" -rqindex 2' 239 240 job = CmdRangeJob(cmd, '1-400', chunkSize=90, partitionCount=1) 241 job['package']['regex_progress'] = '^PROGRESS:.*\((\d+)\): \d+ Seconds' 242 job.setLogParser('AERenderLogParser', modulePath='qb.backend.afterEffectsLogParser') 243 job['name'] = 'AE test' 244 job['package']['regex_errors'] = 'The name of an output module for this render item is already in use' 245 jobList.append(job) 246 ''' 247 248 if '--arc' in sys.argv: 249 arcFile = './job.qja' 250 arcSize = qb.archivejob(arcFile, job, qb.QB_API_BINARY) 251 print 'exported job. wrote: %s bytes to file: %s' % (arcSize, arcFile) 252 else: 253 for j in qb.submit(jobList): 254 print 'submitted: %(id)s' % j 255 256 sys.exit() 257