qb.utils.files

16 """ 17 A class to read and write the qbwrk.conf file on the supervisor; this file contains blocks 18 herein referred to as "stanzas" for each template or host definition. 19 20 An example qbwrk.conf that has 3 stanzas looks like: 21 [default] 22 worker_cpus = 0 23 24 [test]: 25 worker_description = 'this is a test host' 26 27 [some-host-name] : test 28 worker_resources = host.foo=4 29 worker_path_map = { 30 "/Users/sam" = "S:" 31 "/Users/test" = "T:" 32 } 33 34 @ivar cfg: a dict of dicts to hold all the stanzas 35 36 The top-level key is the stanza name, the string that appears between the square brackets. 37 38 For each stanza dict, there will be a key for each parameter in the stanza, as well as the 39 following reserved-word keys: 40 - inherits: the list of templates that occurs after the ":" in the stanza definition 41 line 42 43 - index: the zero-based index which indicates the ordering of the stanzas in the cfgFile 44 45 Any of the *_map type multi-line hash-like values are stored as lists of tuples, since the 46 values are always C{src = target} 47 48 """ 49 RGX_STANZA_DEF = re.compile('^\[([a-zA-Z0-9_\.\[\]-]+)\]\s*:?\s*([^#]*)#?.*$') # match [template] : inherit inherit... (template name can contain a '.' or a h[1-2] range) 50 RGX_BLANK_LINE = re.compile('^\s*$') 51 RGX_PARAM = re.compile('^(\w+)\s*=\s*(.*)#?.*$') # match foo = bar 52 RGX_MAPPING_PARAM = re.compile('^(\w+map)\s*=\s*{(.*)}?$') # match *_map = { 53 RGX_PATH_MAPPING_DEF = re.compile('^\s*(.*)\s*=\s*(.*)\s*$') # match X: = /foo/bar or /foo/bar = Y: 54 RGX_DRIVE_MAPPING_DEF = re.compile('^\s*([A-Z]{1}:|.*)\s*$') # match X: = /foo/bar or /foo/bar = Y: 55

56 - def __init__(self, cfgFile=None):

57 ''' 58 @param cfgFile: file path to the central worker config file 59 @type cfgFile: C{str} 60 ''' 61 self.logging = logging.getLogger('%s' % self.__class__.__name__) 62 self.cfgFileExists = True 63 64 if cfgFile == None: 65 cfgFile = qb.supervisorconfig().get('supervisor_worker_configfile', qb.QB_SUPERVISOR_CONFIG_DEFAULT_WORKER_CONFIGFILE) 66 67 self.cfgFile = cfgFile 68 if not os.path.isfile(self.cfgFile): 69 errMsg = 'Worker config file not found: %s, will be created.' % self.cfgFile 70 self.logging.warning(errMsg) 71 self.cfgFileExists = False 72 73 # a dict to hold all the different "stanzas" in the qbwrk.conf 74 self.cfg = {} 75 76 # a list of keys in each stanza dictionary that should be culled when building the contents 77 # of the config file for writing 78 self.reservedKeyWords = ['index', 'inherits', 'delete']

79

80 - def read(self):

81 ''' 82 Parse the cfgFile, turn it into a dictionary of dictionaries 83 ''' 84 self.cfg = {} # re-init the config just in case this is called multiple times 85 86 # bail if the cfgFile does not yet exist, we've already initialized the cfg dictionary 87 if not self.cfgFileExists: 88 return 89 90 stanzaName = '' 91 mapName = '' # some worker_*_map parameters span multiple lines 92 93 inMappingBlock = False 94 commentWarningRaised = False 95 96 for line in open(self.cfgFile).readlines(): 97 98 line.strip() 99 if self.RGX_BLANK_LINE.search(line): 100 continue 101 102 if line.startswith('#'): 103 if not commentWarningRaised: 104 self.logging.warning('Worker config file contains comments, these will be stripped out when the new file is written') 105 commentWarningRaised = True 106 continue 107 108 if line.startswith('}'): 109 inMappingBlock = False 110 continue 111 112 stanzaMatch = self.RGX_STANZA_DEF.search(line) 113 if stanzaMatch: 114 # init a new stanza dict, set the stanzaName for the next iteration through the 115 # readlines loop 116 try: 117 stanzaOrder = len(self.cfg) 118 (stanzaName, inherits) = stanzaMatch.groups() 119 120 if stanzaName.count('['): 121 self.logging.warning('Worker config file utilizes a host range: %s. It will be retained but not modified' % stanzaName) 122 123 self.cfg[stanzaName] = { 124 'index': stanzaOrder, 125 'inherits': inherits.strip(), 126 'delete': [] 127 } 128 129 except Exception, e: 130 print e 131 continue 132 133 if inMappingBlock: 134 pathMapMatch = self.RGX_PATH_MAPPING_DEF.search(line) 135 if pathMapMatch: 136 try: 137 # clean up whitespace and get rid of double-quotes 138 (src, target) = [y.replace('"', '') for y in [x.strip() for x in pathMapMatch.groups()] ] 139 self.cfg[stanzaName].get(mapName, []).append('"%s" = "%s"' % (src, target)) 140 141 except Exception, e: 142 print e 143 continue 144 else: 145 driveMapMatch = self.RGX_DRIVE_MAPPING_DEF.search(line) 146 if driveMapMatch: 147 try: 148 self.cfg[stanzaName].get(mapName, []).append( driveMapMatch.group(1) ) 149 except Exception, e: 150 print e 151 continue 152 153 mappingParamMatch = self.RGX_MAPPING_PARAM.search(line) 154 if mappingParamMatch: 155 inMappingBlock = True 156 try: 157 (mapName, mapVal) = mappingParamMatch.groups() 158 self.cfg[stanzaName][mapName] = [] 159 # only one mapping can defined in the first line, so split the value into 160 # a tuple 161 if mapVal.count('='): 162 mapVal = tuple( [x.strip() for x in mapVal.split('=')] ) 163 164 # don't add blank lines 165 if mapVal: 166 self.cfg[stanzaName][mapName].append(mapVal) 167 168 if mapVal.count('}'): 169 # the map block is closed on the same line 170 inMappingBlock = False 171 172 except Exception, e: 173 print e 174 continue 175 176 paramMatch = self.RGX_PARAM.search(line) 177 if paramMatch: 178 try: 179 (paramName, paramVal) = paramMatch.groups() 180 self.cfg[stanzaName][paramName] = paramVal 181 except Exception, e: 182 print e 183 continue 184 185 # now convert any of the mapping parameters from a list to string 186 # and cull any empty lists 187 for stanzaName in self.cfg.keys(): 188 stanza = self.cfg[stanzaName] 189 for paramName in [x for x in stanza.keys() if x.endswith('_map')]: 190 if stanza[paramName]: 191 stanza[paramName] = '\n'.join(stanza[paramName]) 192 else: 193 del stanza[paramName]

194

195 - def mergeConfigs(self, newCfg):

196 ''' 197 @param newCfg: a dictionary containing all the config file parameters to update, keyed by hostname 198 @type newCfg: C{dict} 199 ''' 200 for stanzaName in newCfg: 201 if stanzaName not in self.cfg: 202 self.cfg[stanzaName] = { 203 'index': len(self.cfg), 204 'inherits': '', 205 'delete': [] 206 } 207 208 self.cfg[stanzaName].update(newCfg[stanzaName])

209

210 - def write(self):

211 ''' 212 @raise: raise QubeUtilityGeneralError on any failure to successfully update the worker config file 213 ''' 214 data = [] 215 # write out the stanzas by the original order in which they appeared in the config file 216 for (stanzaName, stanza) in sorted(self.cfg.iteritems(), key=lambda (k, v): v['index']): 217 templateLine = '[%s]' % stanzaName 218 if stanza['inherits']: 219 templateLine += ' : %s' % stanza['inherits'] 220 data.append(templateLine) 221 222 for paramName in sorted([x for x in stanza if x not in self.reservedKeyWords]): 223 if paramName not in stanza['delete']: 224 225 if paramName.endswith('_map'): 226 stanza[paramName] = stanza[paramName].strip() 227 if paramName.count('path_map'): 228 data.append('%s = {' % paramName) 229 for pathMapping in stanza[paramName].split('\n'): 230 data.append(' %s' % pathMapping.strip()) 231 else: 232 # worker_drive_map needs the first mapping on the first line 233 mappings = stanza[paramName].split('\n') 234 data.append('%s = {%s' % (paramName, mappings[0])) 235 for driveMapping in mappings[1:]: 236 data.append('%s' % driveMapping.strip()) 237 data.append('}') 238 else: 239 data.append('%s = %s' % (paramName, stanza[paramName])) 240 241 data.append('') 242 243 # blank line on the end 244 data.append('') 245 246 errMsg = qb.utils.sudoWrite('\n'.join(data), self.cfgFile) 247 if errMsg: 248 raise QubeUtilityGeneralError(errMsg)

Source Code for Module qb.utils.files