1
2
3
4
5
6
7
8
9
10
11
12
13
14 import sys
15 import os, os.path
16 import re
17 import time
18 import socket
19 import inspect
20 import traceback
21 import logging
22 import datetime
23
24 import qb
25 from qb.backend import QubeBackEndError
26
27 HOSTNAME = socket.gethostname()
28 try:
29 HOSTNAME = HOSTNAME.split('.')[0]
30 except IndexError:
31 pass
32
33 RGX_APP_TOKEN = re.compile('__([A-Z1-9]+)__')
34
35
37 '''
38 Add a path to the python module search path.
39
40 @param path: The file path to add.
41
42 @type path: C{str}
43
44 @param insert: Either insert the path at the head of sys.path, or append
45 to the end.
46
47 @type insert: C{bool} default: False
48 '''
49 if path:
50 fullPath = os.path.abspath(path)
51 if fullPath not in sys.path:
52 if insert:
53 idx = 0
54 else:
55 idx = len(sys.path) + 1
56 sys.path.insert(idx, fullPath)
57
59 '''
60 Work around the various ways that objects get cast to strings in the qube job object.
61
62 Add a package dict to the job if one is not found; this can occur when an empty dict
63 is used as a mock job during unittesting.
64
65 @param job: The job to run
66
67 @type job: C{dict} or qb.Job
68
69 @return: Return a boolean value of the value the job package's 'dev' key; this can be
70 either a string, int, or NoneType
71
72 @rtype: C{bool}
73 '''
74 dev = job.get('package', {}).get('dev', False)
75
76 if dev and isinstance(dev, str):
77 if dev.lower().startswith('f'):
78 dev = False
79 else:
80 try:
81 dev = bool(int(dev))
82 except ValueError:
83 dev = bool(dev)
84
85 return dev
86
87
89 """
90 Work around the various ways that objects get cast to strings in the qube job object.
91
92 Add a package dict to the job if one is not found; this can occur when an empty dict
93 is used as a mock job during unittesting.
94
95 @param job_attr: The job attribute to check
96 @type job_attr: C{str}
97
98 @return: Return a boolean value of the value; this can be either a string, int, or NoneType
99 @rtype: C{bool}
100 """
101 if job_attr is None or job_attr == 0 or job_attr == '':
102 job_attr = False
103 elif job_attr and isinstance(job_attr, str):
104 if job_attr.lower().startswith('f'):
105 job_attr = False
106 else:
107 try:
108 job_attr = bool(int(job_attr))
109 except ValueError:
110 job_attr = bool(job_attr)
111
112 return job_attr
113
114
116 """
117 Work around the various ways that objects get cast to strings in the qube job object.
118
119 @param val: the val to test for T/F
120 @type val: C{str} or C{int} or C{bool}
121
122 @return: Return a boolean value of the value
123 @rtype: C{bool}
124 """
125 if val and isinstance(val, str):
126 if val.lower().startswith('f'):
127 val = False
128 else:
129 try:
130 val = bool(int(val))
131 except ValueError:
132 val = bool(val)
133
134 return val
135
137 '''
138 Get the path to the module that called this instance
139
140 @return: The file path to the backend's class module
141
142 @rtype: C{str}
143 '''
144 modulePath = ''
145
146 frames = inspect.getouterframes(inspect.currentframe())
147 callingModulePath = frames[1][1]
148 modulePath = os.path.abspath(callingModulePath)
149
150 for frame in frames:
151 del frame
152 return modulePath
153
155 '''
156 Import an abritrary module, and return a class object (not an instance) from that module.
157
158 @param data: A dictionary describing the class object, must contain the following keys: libPath, modulePath, className
159 @type data: C{dict}
160 '''
161
162 assert data['modulePath']
163 assert data['className']
164
165 if 'libPath' in data:
166 addToSysPath(data['libPath'], insert=True)
167
168 moduleObj = __import__(str(data['modulePath']))
169
170 for pkg in data['modulePath'].split('.')[1:]:
171 moduleObj = getattr(moduleObj, pkg)
172
173 klass = getattr(moduleObj, data['className'])
174 return klass
175
177 return float('%s.%s' % (sys.version_info[0], sys.version_info[1]))
178
187
189 if not fhList:
190 fhList = [sys.stdout]
191 elif not isinstance(fhList, list):
192 fhList = [fhList]
193
194 for fh in fhList:
195 fh.flush()
196 fh.write('%s\n' % msg)
197 fh.flush()
198
200 """
201 Print a nicely-formatted message with a timestamp, easily readable in
202 a job's stdout/stderr logs
203
204 >>> bannerPrint( 'Finished work' )
205 ================================================================================
206 13:05:31 Finished work render023
207 ================================================================================
208
209 @param msg: The message to be printed
210 @type msg: string
211
212 @param timeFormat: a format string suitable for time.strftime()
213 @type timeFormat: string
214
215 @param fhList: A list of file objects to print to. Optionally supports
216 being passed a single file object; eg. 'fhList=sys.stderr'.
217
218 @type fhList: list
219 """
220
221 TIMEFMT = '%H:%M:%S'
222
223 if not timeFormat:
224 timeFormat = TIMEFMT
225
226 if not fhList:
227 fhList = [sys.stdout]
228
229 if isinstance(fhList, file):
230 fhList = [ fhList ]
231
232 LINELEN = 80
233 timestr = time.strftime(timeFormat, time.localtime())
234
235 for fh in fhList:
236 msgStr = ' %(time)s%(tab)s%(msg)s' % {'time':timestr, 'tab':' '*8, 'msg':msg}
237
238 fh.write('%s\n' % ('='*LINELEN,))
239 fh.write('%s %*s\n' % (msgStr, LINELEN-(len(msgStr)+2), HOSTNAME))
240 fh.write('%s\n' % ('='*LINELEN,))
241 fh.flush()
242
244 loggingLevel = logging.WARNING
245 loggingFormat = logging.BASIC_FORMAT
246
247 if level == 'debug':
248 loggingLevel = logging.DEBUG
249 loggingFormat = '%(name)20s : %(levelname)-8s : %(message)s (%(filename)s:%(lineno)d, %(threadName)s)'
250 elif level == 'warning':
251 loggingLevel = logging.INFO
252 loggingFormat = '%(levelname)s:%(name)s: %(message)s'
253
254 rootLogger = logging.getLogger()
255
256 logHandler = logging.StreamHandler()
257 logHandler.setFormatter(logging.Formatter(loggingFormat, None))
258 rootLogger.addHandler(logHandler)
259 rootLogger.setLevel(loggingLevel)
260
262 '''
263 scan a job.conf for a multi-line block the is bound by [...], with the closing bracket at the beginning of a new line.
264
265 valid OS names acting as keys in the job.conf are those returned by sys.platform
266
267 example:
268
269 HFS_PATHS = [
270 darwin:"/Library/NotFoundHere/Houdini.framework/Versions/0.0.0/Resources"
271 darwin: "/Library/Frameworks/Houdini.framework/Versions/0.0.0/Resources"
272 linux2:"/opt/hfs0.0.0"
273 win32: "C:/Program Files/Side Effect Software/Houdini 0.0.0"
274 ]
275
276 return a dictionary, keyed of sys.platform names, values are a list of paths
277 '''
278 paths = {}
279 osPathRGX = re.compile('^(\w+):\s*"(.*)"')
280
281 fh = open(confPath)
282 lines = fh.readlines()
283 fh.close()
284
285 blockFound = False
286 for i in range(len(lines)):
287
288 line = lines[i]
289
290 if blockFound is False and line.startswith(blockName):
291 blockFound = True
292 continue
293
294 if blockFound:
295 if line.startswith('['):
296
297 break
298
299 m = osPathRGX.search(line)
300 if m:
301 try:
302 (osName, path) = m.groups()
303 paths.setdefault(osName, []).append(path)
304 except ValueError:
305 flushPrint('ERROR: %s contains an invalid %s definition line at line %s' % (confPath, blockName, i), fhlist=[sys.stderr])
306
307 return paths
308
309 -def getJobLogPaths(redirectingStdErr=False):
310 jobStdoutLog = ''
311 jobStderrLog = ''
312
313 hostName = socket.gethostname()
314 workerCfg = qb.workerconfig(hostName)
315
316 logRoot = workerCfg.get('worker_logpath', '')
317 if logRoot == '':
318 logging.error('Unable to determine worker_logpath for host %s' % hostName)
319
320 try:
321 jobId = int(os.environ.get('QBJOBID'))
322 instanceId = int(os.environ.get('QBSUBID', 0))
323 except:
324 logging.error('Unable to determine job log paths, either QBJOBID or QBSUBID environment variables are not set.')
325 return ('','')
326
327 jobLogBucketDir = jobId - (jobId % 1000)
328
329 jobLogDir = '%s/job/%s/%s' % (logRoot, jobLogBucketDir, jobId)
330
331 jobStdoutLog = '%s/%s_%s.out' % (jobLogDir, jobId, instanceId)
332 if not os.path.exists(jobStdoutLog):
333 logging.warning('stdout job log not found: %s' % os.path.normpath(jobStdoutLog))
334 jobStdoutLog = ''
335
336 if not redirectingStdErr:
337 jobStderrLog = '%s/%s_%s.err' % (jobLogDir, jobId, instanceId)
338 if not os.path.exists(jobStderrLog):
339 logging.warning('stderr job log not found: %s' % os.path.normpath(jobStderrLog))
340 jobStderrLog = ''
341
342 return (jobStdoutLog, jobStderrLog)
343
345 appToken = None
346
347 m = RGX_APP_TOKEN.search(cmd)
348 if m:
349 try:
350 appToken = m.group(1)
351 except IndexError:
352 logging.error('IndexError: no appToken found')
353 pass
354 else:
355 logging.warning('translateAppPath: no appToken "%s" found, assuming explicit path to application executable' % RGX_APP_TOKEN.pattern)
356
357 if appToken:
358 import appDefaultPaths
359
360
361 appPath = appDefaultPaths.buildAppPath(appToken, appVersion)
362 cmd = re.sub('__%s__' % appToken, appPath, cmd)
363
364 return cmd
365
367 """
368 A convenience class containing some simple timer-type methods
369 """
371 self.startTime = 0
372 self.endTime = 0
373 self.timerIsRunning = False
374
376 self.startTime = datetime.datetime.now()
377 self.timerIsRunning = True
378
380 self.endTime = datetime.datetime.now()
381 self.timerIsRunning = False
382
384 if self.endTime == 0:
385 endTime = datetime.datetime.now()
386 else:
387 endTime = self.endTime
388 et = endTime - self.startTime
389 eTime = et.seconds + (et.microseconds/1e6)
390 return eTime
391