qb.utils

1 ''' 2 A package for convenience functions and classes 3 ''' 4 5 #======================================= 6 # $Revision: #13 $ 7 # $Change: 16215 $ 8 #======================================= 9 10 import logging 11 import sys 12 import os, os.path 13 import re 14 import time 15 import inspect 16 import traceback 17 import shutil 18 import platform 19 import tempfile 20 21 from qb import convertpath, workerpathmap 22 23 #======================================= 24 # utils sub-packages 25 #======================================= 26 import files 27 import flags 28 import logging 29 import qbTokens 30 import regex 31 from exceptions import * 32 from enums import ENUMS 33 34 35 #=============================================================================== 36 # misc helper functions 37 #===============================================================================

38 -def addToSysPath(path, append=False):

39 ''' 40 Add a path to the python module search path. 41 42 @param path: The file path to add. 43 44 @type path: C{str} 45 46 @param append: Either insert the path at the head of sys.path, or append 47 to the end. 48 49 @type append: C{bool} default: False 50 ''' 51 if path: 52 fullPath = os.path.abspath(path) 53 if fullPath not in sys.path: 54 if append: 55 idx = len(sys.path) + 1 56 else: 57 idx = 0 58 sys.path.insert(idx, fullPath)

59 60

61 -def getModulePath():

62 ''' 63 Get the path to the module that called this instance 64 65 @return: The file path to the backend's class module 66 67 @rtype: C{str} 68 ''' 69 modulePath = '' 70 71 frames = inspect.getouterframes(inspect.currentframe()) 72 callingModulePath = frames[1][1] 73 modulePath = os.path.abspath(callingModulePath) 74 75 for frame in frames: 76 del frame 77 78 return modulePath

79 80

81 -def pyVerAsFloat():

82 return float('%s.%s' % (sys.version_info[0], sys.version_info[1]))

83 84

85 -def formatExc(limit=5):

86 excAsStr = '' 87 if pyVerAsFloat() > 2.3: 88 excAsStr = traceback.format_exc(limit) 89 else: 90 excAsStr = traceback.print_exc() 91 92 return excAsStr

93 94

95 -def translateQbConvertPathStrings(cmd):

96 """ 97 @param cmd: a python command to perform path translation on, according to the worker's 98 worker_path_map. The cmd is scanned for any string wrapped in C{QB_CONVERT_PATH()} 99 @type cmd: C{str} 100 101 @return: the cmd with any paths translated as necessary 102 @rtype: C{str} 103 """ 104 preTranslationCmd = cmd 105 while cmd.count('QB_CONVERT_PATH'): 106 # extract the path inside the first QB_CONVERT_PATH() 107 unconvertedPath = regex.RGX_CONVERT_PATH_TOKEN.search(cmd).group(1).strip() 108 109 # pass the path to qb.convertpath(), apply the worker_path_map 110 convertedPath = convertpath(unconvertedPath) 111 112 # trailing single backslashes cause the re module to throw a fit... 113 if convertedPath.endswith('\\'): 114 convertedPath += '\\' 115 116 # double-quote the path if it contains spaces and is not already quoted 117 if convertedPath.count(' ') and not re.search('^".*"$', convertedPath): 118 convertedPath = '"%s"' % convertedPath 119 120 # replace the original path with the converted one 121 # encode the converted path so it makes it through re.sub() intact 122 cmd = regex.RGX_CONVERT_PATH_TOKEN.sub(convertedPath.encode('string_escape'), cmd, 1) 123 124 if cmd != preTranslationCmd: 125 logging.info('Paths in the command have been translated as per this worker\'s worker_path_map') 126 logging.info('%s %s' % (' '*4, preTranslationCmd)) 127 logging.info('%s -> %s' % (' '*1, cmd)) 128 129 return cmd

130 131 132 #=============================================================================== 133 # file read/write utility functions 134 #===============================================================================

135 -def sudoCopy(src, dst):

136 ''' 137 Attempt to perform an authenticated copy. An existing destination file will be overwritten. 138 139 Prompt for authentication on OS X, otherwise just return an error message on other OS's upon failure. 140 141 @param src: copy source file 142 @type src: C{str} 143 144 @param dst: copy destination file 145 @type src: C{str} 146 147 @return: Return an error message; a null-string indicates success 148 @rtype: C{str} 149 ''' 150 errMsg = '' 151 152 # capture original timestamp of destination file, to test for success 153 # also save file mode, so we can set new file to same mode 154 try: 155 dstStat = os.stat(dst) 156 origMode = dstStat.st_mode 157 origTimeStamp = dstStat.st_mtime 158 except OSError: 159 # dstFile doesn't exist 160 origMode = None 161 origTimeStamp = 0 162 163 if not os.path.exists(src): 164 # bail if src file specified, but does not exist 165 errMsg = 'Source file %s does not exist.' % src 166 logging.error(errMsg) 167 168 if os.path.exists(dst): 169 logging.log(logging.WARNING+5, 'Destination file exists, will be over-written: %s' % dst) 170 else: 171 logging.log(logging.INFO+5, 'Destination file does not exist, will be created: %s' % dst) 172 173 if platform.system() == 'Darwin': 174 # use 'authopen' on OS X 175 result = os.system('cat %s | /usr/libexec/authopen -w -c %s' % (src, dst)) 176 if result != 0: 177 errMsg = 'Unable to write to "%s". Permission denied.' % dst 178 logging.error(errMsg) 179 else: 180 if os.path.isfile(dst) and not os.access(dst, os.W_OK): 181 # attempt to make the destination file writeable 182 try: 183 origMode = os.stat(dst).st_mode 184 os.chmod(dst, origMode|stat.S_IWRITE|stat.S_IWGRP|stat.S_IWOTH) 185 except OSError: 186 errMsg = 'Unable to make existing destination file "%s" overwritable.' % dst 187 logging.error(errMsg) 188 189 if not errMsg: 190 try: 191 #================================= 192 # finally - do the file copy... 193 #================================= 194 shutil.copy(src, dst) 195 logging.info('Copied %s --> %s' % (src, dst)) 196 except IOError: 197 errMsg = 'Unable to copy %s --> %s. Permission denied.' % (src, dst) 198 logging.error(errMsg) 199 200 if not errMsg: 201 try: 202 os.chmod(dst, origMode) 203 pass 204 except: 205 logging.warning('Failed to re-set file mode for %s' % dst) 206 pass 207 208 if origTimeStamp >= os.stat(dst).st_mtime: 209 errMsg = 'File copy failed somehow, destination timestamp is not later than source file timestamp' 210 logging.error(errMsg) 211 212 return errMsg

213 214

215 -def sudoWrite(data, dst, concat=False):

216 ''' 217 Attempt to perform a file write that is expected to require authentication. 218 219 @param data: data to write to the destination file 220 @type data: C{str} 221 222 @param dst: destination file 223 @type dst: C{str} 224 225 @param concat: if true, concat the data to the destination, otherwise overwrite destination 226 @type concat: C{boolean} 227 228 @return: Return an error message; a null-string indicates success 229 @rtype: C{str} 230 ''' 231 errMsg = '' 232 233 if not data: 234 errMsg = 'No data supplied to write to file %s.' % dst 235 236 if not errMsg: 237 if not os.path.isfile(dst) and platform.system() != 'Darwin': 238 #================================================================ 239 # test ability to create destination file if it does not exist 240 #================================================================ 241 logging.log(logging.WARNING+5, 'Destination file %s does not exist, will be created.' % dst) 242 try: 243 fh = open(dst, 'w') 244 fh.write('') 245 fh.close() 246 except IOError, e: 247 errMsg = 'Unable to create file: %s' % dst 248 logging.error(errMsg) 249 250 elif os.path.isfile(dst): 251 #================================================================================= 252 # backup file exists and will not be concatenated to, save a backup copy before 253 # overwriting it 254 #================================================================================= 255 logging.log(logging.WARNING+5, 'Destination file %s will be overwritten, creating a backup copy.' % dst) 256 257 backupFname = getTimestampedFileName(dst) 258 errMsg = sudoCopy(dst, backupFname) 259 if not errMsg: 260 logging.info('Backup file created: %s' % backupFname) 261 else: 262 logging.error(errMsg) 263 logging.warning('Unable to create a backup file: %s' % backupFname) 264 265 if not errMsg: 266 #============================================================== 267 # destination file setup went OK, and we have data to write 268 #============================================================== 269 if platform.system() == 'Darwin': 270 271 (fd, tmpFile) = tempfile.mkstemp() 272 fh = open(tmpFile, 'w') 273 fh.writelines(data) 274 fh.close() 275 276 if concat and os.path.isfile(dst): 277 result = os.system('cat %(dst)s %(src)s | /usr/libexec/authopen -w -c %(dst)s' % {'src':tmpFile, 'dst':dst}) 278 else: 279 result = os.system('cat %s | /usr/libexec/authopen -w -c %s' % (tmpFile, dst)) 280 281 os.unlink(tmpFile) 282 283 if result != 0: 284 errMsg = 'Unable to write to destination file: %s' % dst 285 logging.error(errMsg) 286 287 else: 288 if concat: 289 fOpenMode = 'w+' 290 else: 291 fOpenMode = 'w' 292 293 try: 294 fh = open(dst, fOpenMode) 295 fh.writelines(data) 296 fh.close() 297 except IOError, e: 298 errMsg = e 299 logging.error(errMsg) 300 301 302 return errMsg

303 304

305 -def getTimestampedFileName(fName):

306 ''' 307 Generate a time-stamped backup file name: 308 qb.conf -> qb.20121231_235959.conf 309 310 @param fName: A file name 311 @type fName: C{str} 312 313 @return: A filename with a timestamp embedded in front of the file extension 314 @rtype: C{str} 315 ''' 316 timeFmt = '%Y_%m%d_%H%M%S' 317 timestamp = time.strftime(timeFmt, time.localtime()) 318 319 (fRoot, fExt) = os.path.splitext(fName) 320 newName = '%s.%s' % (fRoot, timestamp) 321 if fExt: 322 newName += '%s' % fExt 323 324 return newName

325

Source Code for Package qb.utils