Pierrick Martin / ctools

Commit c60f05be3c63bf9cf733aa351aca79585ec80190

Authored by Michael Mayer
1 parent 28009928

Bug fixes, logging and code style improvements in some cscripts

Inline Side-by-side
Showing 7 changed files with 435 additions and 373 deletions
cscripts/csfindobs.py
  View file @c60f05b
... ... @@ -39,15 +39,15 @@ class csfindobs(ctools.cscript):
39 39 Constructor.
40 40 """
41 41 # Set name
42   - self._name = "csfindobs"
43   - self._version = "1.1.0"
44   - self._datapath = os.getenv("VHEFITS","")
45   - self._prodname = ""
  42 + self._name = 'csfindobs'
  43 + self._version = '1.1.0'
  44 + self._datapath = os.getenv('VHEFITS','')
  45 + self._prodname = ''
46 46 self._select_radec = True
47 47 self._radius = 0.0
48 48 self._ra = 0.0
49 49 self._dec = 0.0
50   - self._obs_index = ""
  50 + self._obs_index = ''
51 51 self._runs = []
52 52  
53 53 # Initialise application by calling the appropriate class
... ... @@ -64,17 +64,17 @@ class csfindobs(ctools.cscript):
64 64 Get parameters from parfile and setup the observation.
65 65 """
66 66 # Get parameters
67   - if self._datapath == "":
68   - self._datapath = self["datapath"].string()
  67 + if self._datapath == '':
  68 + self._datapath = self['datapath'].string()
69 69  
70 70 # Expand environment
71 71 self._datapath = gammalib.expand_env(self._datapath)
72 72  
73 73 # Get production name
74   - self._prodname = self["prodname"].string()
  74 + self._prodname = self['prodname'].string()
75 75  
76 76 # Master index file name
77   - master_indx = self["master_indx"].string()
  77 + master_indx = self['master_indx'].string()
78 78  
79 79 # Intiialise flag if spatial selection is required
80 80 self._select_radec = True
... ... @@ -83,14 +83,14 @@ class csfindobs(ctools.cscript):
83 83 self._radius = 0.0
84 84  
85 85 # Check for vailidity of spatial parameters
86   - if (self["ra"].is_valid() and
87   - self["dec"].is_valid() and
88   - self["rad"].is_valid()):
  86 + if (self['ra'].is_valid() and
  87 + self['dec'].is_valid() and
  88 + self['rad'].is_valid()):
89 89  
90 90 # Read spatial parameters
91   - self._ra = self["ra"].real()
92   - self._dec = self["dec"].real()
93   - self._radius = self["rad"].real()
  91 + self._ra = self['ra'].real()
  92 + self._dec = self['dec'].real()
  93 + self._radius = self['rad'].real()
94 94  
95 95 else:
96 96 self._select_radec = False
... ... @@ -100,12 +100,12 @@ class csfindobs(ctools.cscript):
100 100 self._select_radec = False
101 101  
102 102 # Query other parameters
103   - self["min_qual"].integer()
104   - self["expression"].string()
  103 + self['min_qual'].integer()
  104 + self['expression'].string()
105 105  
106 106 # Read ahead output parameters
107 107 if self._read_ahead():
108   - self["outfile"].filename()
  108 + self['outfile'].filename()
109 109  
110 110 # Open master index file and look for prodname
111 111 master_file = os.path.join(self._datapath, master_indx)
... ... @@ -119,33 +119,33 @@ class csfindobs(ctools.cscript):
119 119 # Open and load JSON file
120 120 json_data = open(master_file).read()
121 121 data = json.loads(json_data)
122   - if not "datasets" in data:
  122 + if not 'datasets' in data:
123 123 raise RuntimeError('Key "datasets" not available in '+
124 124 'master index file.')
125 125  
126 126 # Get configurations
127   - configs = data["datasets"]
  127 + configs = data['datasets']
128 128  
129 129 # Initialise obs index file
130   - self._obs_index = ""
  130 + self._obs_index = ''
131 131  
132 132 # Get HDUs
133 133 for config in configs:
134   - if self._prodname == config["name"]:
  134 + if self._prodname == config['name']:
135 135 self._obs_index = str(os.path.join(self._datapath,
136   - config["obsindx"]))
  136 + config['obsindx']))
137 137 break
138 138  
139 139 # Check HDUs
140   - if self._obs_index == "":
  140 + if self._obs_index == '':
141 141 raise RuntimeError('FITS data store "'+self._prodname+
142 142 '" not available. Run csiactdata to get '+
143 143 'a list of available storage names')
144   - filename = gammalib.GFilename(self._obs_index+"[OBS_INDEX]")
  144 + filename = gammalib.GFilename(self._obs_index+'[OBS_INDEX]')
145 145 if not filename.is_fits():
146 146 raise RuntimeError('Observation index file "'+
147 147 self._obs_index+
148   - '[OBS_INDEX]" for FITS data store "'+
  148 + '"[OBS_INDEX]" for FITS data store "'+
149 149 self._prodname+
150 150 '" not available. Check your master index '+
151 151 'file or run csiactdata to get a list of '+
... ... @@ -154,7 +154,7 @@ class csfindobs(ctools.cscript):
154 154 # Write input parameters into logger
155 155 if self._logTerse():
156 156 self._log_parameters()
157   - self._log("\n")
  157 + self._log('\n')
158 158  
159 159 # Return
160 160 return
... ... @@ -176,55 +176,62 @@ class csfindobs(ctools.cscript):
176 176 self._runs = []
177 177  
178 178 # Initialise selction expression
179   - expr = ""
  179 + expr = ''
180 180  
181 181 # Add spatial expression if possible
182 182 if self._select_radec:
183   - expr += "ANGSEP("+str(self._ra)+","+str(self._dec)+ \
184   - ",RA_PNT,DEC_PNT)<="+str(self._radius)
  183 + expr += 'ANGSEP('+str(self._ra)+','+str(self._dec)+ \
  184 + ',RA_PNT,DEC_PNT)<='+str(self._radius)
185 185  
186 186 # Add connector if expression is empty
187 187 if len(expr):
188   - expr += "&&"
  188 + expr += '&&'
189 189  
190 190 # Add quality expression
191   - expr += "QUALITY<="+str(self["min_qual"].integer())
  191 + expr += 'QUALITY<='+str(self['min_qual'].integer())
192 192  
193 193 # Add user expression
194   - expression = self["expression"].string()
195   - if (expression != "NONE" and expression != "INDEF" and
  194 + expression = self['expression'].string()
  195 + if (expression != 'NONE' and expression != 'INDEF' and
196 196 len(expression) > 0):
197   - expr += "&&" + expression
  197 + expr += '&&' + expression
198 198  
199 199 # Initialise filename
200   - openfile = self._obs_index+"[OBS_INDEX]["+expr+"]"
  200 + openfile = self._obs_index+'[OBS_INDEX]['+expr+']'
201 201  
202 202 # Open file
203 203 fits = gammalib.GFits(openfile)
204 204  
205 205 # Check for extistence of rows
206   - if fits["OBS_INDEX"].nrows() > 0:
  206 + if fits['OBS_INDEX'].nrows() > 0:
207 207  
208 208 # Get HDU
209   - obs_indx = fits["OBS_INDEX"]
  209 + obs_indx = fits['OBS_INDEX']
210 210  
211 211 # Read observation IDs into array
212 212 for i in range(obs_indx.nrows()):
213   - self._runs.append(obs_indx["OBS_ID"][i])
  213 + self._runs.append(obs_indx['OBS_ID'][i])
214 214  
215 215 # Dump expression
216 216 if self._logNormal():
217   - self._log("\n")
218   - self._log.header3("Expression")
  217 + self._log('\n')
  218 + self._log.header3('Expression')
219 219 self._log(expr)
220   - self._log("\n")
  220 + self._log('\n')
221 221  
222 222 # Dump number of observations
223 223 if self._logTerse():
224   - self._log("\n")
225   - self._log.header3("Observations")
226   - self._log("Found "+str(len(self._runs))+" observations")
227   - self._log("\n")
  224 + self._log('\n')
  225 + self._log.header3('Observations')
  226 + self._log('Found '+str(len(self._runs))+' observations')
  227 + self._log('\n')
  228 +
  229 + # Dump number of observations
  230 + if self._logVerbose():
  231 + self._log('\n')
  232 + self._log.header3('Observation IDs')
  233 + for run in self._runs:
  234 + self._log(str(run)+'\n')
228 235  
229 236 # Close file
230 237 fits.close()
... ... @@ -261,26 +268,26 @@ class csfindobs(ctools.cscript):
261 268 self._log.header1('Save runlist')
262 269  
263 270 # Get filename
264   - outfile = self["outfile"].filename()
  271 + outfile = self['outfile'].filename()
265 272  
266 273 # Check for clobber
267 274 if outfile.exists() and not self._clobber():
268 275 if self._logTerse():
269   - self._log('File "+outfile+" already exists. ')
  276 + self._log('File '+outfile+' already exists. ')
270 277 self._log('Set clobber=yes to overwrite file.')
271 278 self._log('\n')
272 279 else:
273 280  
274 281 # Log file name
275 282 if self._logTerse():
276   - self._log(gammalib.parformat("Runlist file"))
  283 + self._log(gammalib.parformat('Runlist file'))
277 284 self._log(outfile.url())
278   - self._log("\n")
  285 + self._log('\n')
279 286  
280 287 # Write runs to file
281   - f = open(outfile.url(),"w")
  288 + f = open(outfile.url(),'w')
282 289 for run in self._runs:
283   - f.write(str(run)+" \n")
  290 + f.write(str(run)+' \n')
284 291 f.close()
285 292  
286 293 # Return
... ...
cscripts/csiactcopy.py
  View file @c60f05b
... ... @@ -43,13 +43,13 @@ class csiactcopy(ctools.cscript):
43 43 Constructor.
44 44 """
45 45 # Set name
46   - self._name = "csiactcopy"
47   - self._version = "1.1.0"
48   - self._datapath = os.getenv("VHEFITS","")
  46 + self._name = 'csiactcopy'
  47 + self._version = '1.1.0'
  48 + self._datapath = os.getenv('VHEFITS','')
49 49 self._remote_master = gammalib.GFilename()
50   - self._remote_base = ""
51   - self._prodname = ""
52   - self._outpath = ""
  50 + self._remote_base = ''
  51 + self._prodname = ''
  52 + self._outpath = ''
53 53 self._runlist = gammalib.GFilename()
54 54 self._runs = []
55 55  
... ... @@ -68,7 +68,7 @@ class csiactcopy(ctools.cscript):
68 68 """
69 69  
70 70 # Get Parameters
71   - self._remote_master = self["remote_master"].filename()
  71 + self._remote_master = self['remote_master'].filename()
72 72 if not self._remote_master.exists():
73 73 raise RuntimeError('*** ERROR: FITS data not available. No '
74 74 'master index file found in "'+
... ... @@ -76,14 +76,14 @@ class csiactcopy(ctools.cscript):
76 76 'file system is properly mounted.')
77 77  
78 78 # Get parameters
79   - self._prodname = self["prodname"].string()
80   - self._outpath = gammalib.expand_env(self["outpath"].string())
81   - self._runlist = self["runlist"].filename()
  79 + self._prodname = self['prodname'].string()
  80 + self._outpath = gammalib.expand_env(self['outpath'].string())
  81 + self._runlist = self['runlist'].filename()
82 82  
83 83 # Write input parameters into logger
84 84 if self._logTerse():
85 85 self._log_parameters()
86   - self._log("\n")
  86 + self._log('\n')
87 87  
88 88 # Return
89 89 return
... ... @@ -101,21 +101,21 @@ class csiactcopy(ctools.cscript):
101 101  
102 102 # Logging
103 103 if self._logVerbose():
104   - self._log("\n")
105   - self._log.header3("Copy file")
106   - self._log.parformat("Source")
  104 + self._log('\n')
  105 + self._log.header3('Copy file')
  106 + self._log.parformat('Source')
107 107 self._log(str(source))
108   - self._log("\n")
109   - self._log.parformat("Destination")
  108 + self._log('\n')
  109 + self._log.parformat('Destination')
110 110 self._log(str(destination))
111   - self._log("\n")
112   - self._log.parformat("Already exists")
  111 + self._log('\n')
  112 + self._log.parformat('Already exists')
113 113 self._log(str(os.path.isfile(destination)))
114   - self._log("\n")
115   - self._log.parformat("Overwrite")
  114 + self._log('\n')
  115 + self._log.parformat('Overwrite')
116 116 self._log(str(clobber))
117   - self._log("\n")
118   - self._log.parformat("Copying")
  117 + self._log('\n')
  118 + self._log.parformat('Copying')
119 119  
120 120 # Flag if file destination is already available
121 121 is_file = os.path.isfile(destination)
... ... @@ -123,14 +123,14 @@ class csiactcopy(ctools.cscript):
123 123 # check if file could be skipped because clobber=no
124 124 if is_file and clobber == False:
125 125 if self._logVerbose():
126   - self._log("Skip (clobber=no)")
127   - self._log("\n")
  126 + self._log('Skip (clobber=no)')
  127 + self._log('\n')
128 128  
129 129 # check if file could be skipped because it is the same file
130 130 elif is_file and os.path.samefile(destination, source):
131 131 if self._logVerbose():
132   - self._log("Skip (same file)")
133   - self._log("\n")
  132 + self._log('Skip (same file)')
  133 + self._log('\n')
134 134  
135 135 else:
136 136 # Create directory if not existent
... ... @@ -146,8 +146,8 @@ class csiactcopy(ctools.cscript):
146 146  
147 147 # Logging
148 148 if self._logVerbose():
149   - self._log("Done!")
150   - self._log("\n")
  149 + self._log('Done!')
  150 + self._log('\n')
151 151  
152 152 # Return
153 153 return filesize
... ... @@ -159,21 +159,21 @@ class csiactcopy(ctools.cscript):
159 159 """
160 160  
161 161 if self._logTerse():
162   - self._log.parformat("Local file")
  162 + self._log.parformat('Local file')
163 163 self._log(str(localfits))
164   - self._log("\n")
165   - self._log.parformat("Remote file")
  164 + self._log('\n')
  165 + self._log.parformat('Remote file')
166 166 self._log(str(remotefits))
167   - self._log("\n")
168   - self._log.parformat("HDU name")
  167 + self._log('\n')
  168 + self._log.parformat('HDU name')
169 169 self._log(str(hduname))
170   - self._log("\n")
171   - self._log.parformat("Conflict preference")
  170 + self._log('\n')
  171 + self._log.parformat('Conflict preference')
172 172 if clobber:
173   - self._log("Remote")
  173 + self._log('Remote')
174 174 else:
175   - self._log("Local")
176   - self._log("\n")
  175 + self._log('Local')
  176 + self._log('\n')
177 177  
178 178 # Check if local fits file is available
179 179 if os.path.isfile(localfits):
... ... @@ -187,40 +187,70 @@ class csiactcopy(ctools.cscript):
187 187  
188 188 # find local obs_id
189 189 local_obs = []
190   - lobs_col = local_hdu["OBS_ID"]
  190 + lobs_col = local_hdu['OBS_ID']
191 191 for obs_id in lobs_col:
192 192 local_obs.append(obs_id)
193   -
194   - # Create selection expression for opening remote fits file
195   - selection = ""
196   - if len(self._runs):
197   - selection += "["+hduname+"]["
198   - for run in self._runs:
199   - selection += "OBS_ID=="+str(run)
200   - selection += "||"
201   - selection = selection[:-2]
202   - selection += "]"
203   -
204   - # Create file name
205   - remotefile = remotefits + selection
206   -
207   - # open remote fits file
208   - remote = gammalib.GFits(str(remotefile))
209   - remote_hdu = remote[hduname]
210   -
  193 +
  194 + # Initialise list of runlists
  195 + # This is kluge: to circumvent a too long file name, we split the
  196 + # runlists into chunks of 50 observations maximum
  197 + runlists = []
  198 + runs_per_chunk = 50
  199 + for i in range(len(self._runs) / runs_per_chunk + 1):
  200 + chunk = self._runs[i * runs_per_chunk : (i + 1) * runs_per_chunk]
  201 + runlists.append(chunk)
  202 +
  203 + # Loop over runlist chunks
  204 + for i, runlist in enumerate(runlists):
  205 +
  206 + # Create selection expression for opening remote fits file
  207 + selection = ''
  208 + if len(runlist):
  209 + selection += '['+hduname+']['
  210 + for run in runlist:
  211 + selection += 'OBS_ID=='+str(run)
  212 + selection += '||'
  213 + selection = selection[:-2]
  214 + selection += ']'
  215 +
  216 + # Create file name
  217 + remotefile = remotefits + selection
  218 +
  219 + # open remote fits file
  220 + remote = gammalib.GFits(str(remotefile))
  221 +
  222 + # If first iteration, create clone of FITS column for later use
  223 + if i == 0:
  224 + remote_hdu = remote[hduname].clone()
  225 + else:
  226 +
  227 + # get temporary hdu to be added to current HDU
  228 + tmp_hdu = remote[hduname]
  229 +
  230 + # Get old size of remote HDU
  231 + size = remote_hdu.size()
  232 +
  233 + # Extend remote HDU
  234 + remote_hdu.append_rows(tmp_hdu.nrows())
  235 +
  236 + # Loop over rows and columns to copy over values exteding remote HDU
  237 + for col in tmp_hdu:
  238 + for row in range(col.length()):
  239 + remote_hdu[col.name()][size + row] = col[row]
  240 +
211 241 # Get remote obs_id
212 242 remote_obs = []
213   - robs_col = remote_hdu["OBS_ID"]
  243 + robs_col = remote_hdu['OBS_ID']
214 244 for obs_id in robs_col:
215 245 remote_obs.append(obs_id)
216 246  
217 247 if self._logTerse():
218   - self._log.parformat("Remote entries")
  248 + self._log.parformat('Remote entries')
219 249 self._log(str(len(remote_obs)))
220   - self._log("\n")
221   - self._log.parformat("Local entries")
  250 + self._log('\n')
  251 + self._log.parformat('Local entries')
222 252 self._log(str(len(local_obs)))
223   - self._log("\n")
  253 + self._log('\n')
224 254  
225 255 # initialise counter for logging
226 256 removed_rows = 0
... ... @@ -239,13 +269,13 @@ class csiactcopy(ctools.cscript):
239 269 table_has_obsid = True
240 270 while table_has_obsid:
241 271 for i in range(local_hdu.nrows()):
242   - if remote_obs_id == local_hdu["OBS_ID"][i]:
  272 + if remote_obs_id == local_hdu['OBS_ID'][i]:
243 273 local_hdu.remove_rows(i,1)
244 274 removed_rows += 1
245 275 break
246 276 table_has_obsid = False
247 277 for i in range(local_hdu.nrows()):
248   - if remote_obs_id == local_hdu["OBS_ID"][i]:
  278 + if remote_obs_id == local_hdu['OBS_ID'][i]:
249 279 table_has_obsid = True
250 280 break
251 281  
... ... @@ -262,13 +292,13 @@ class csiactcopy(ctools.cscript):
262 292 table_has_obsid = True
263 293 while table_has_obsid:
264 294 for i in range(remote_hdu.nrows()):
265   - if local_obs_id == remote_hdu["OBS_ID"][i]:
  295 + if local_obs_id == remote_hdu['OBS_ID'][i]:
266 296 remote_hdu.remove_rows(i,1)
267 297 removed_rows += 1
268 298 break
269 299 table_has_obsid = False
270 300 for i in range(remote_hdu.nrows()):
271   - if local_obs_id == remote_hdu["OBS_ID"][i]:
  301 + if local_obs_id == remote_hdu['OBS_ID'][i]:
272 302 table_has_obsid = True
273 303 break
274 304  
... ... @@ -280,11 +310,11 @@ class csiactcopy(ctools.cscript):
280 310  
281 311 if self._logTerse():
282 312 if clobber:
283   - self._log.parformat("Removed local rows")
  313 + self._log.parformat('Removed local rows')
284 314 else:
285   - self._log.parformat("Skipped remote rows")
  315 + self._log.parformat('Skipped remote rows')
286 316 self._log(str(removed_rows))
287   - self._log("\n")
  317 + self._log('\n')
288 318  
289 319 # Loop over columns
290 320 for i in range(local_hdu.ncols()):
... ... @@ -322,8 +352,8 @@ class csiactcopy(ctools.cscript):
322 352  
323 353 # Log output
324 354 if self._logTerse():
325   - self._log("\n")
326   - self._log.header2("Runlist information")
  355 + self._log('\n')
  356 + self._log.header2('Runlist information')
327 357  
328 358 # Initialise runlist array
329 359 self._runs = []
... ... @@ -342,15 +372,15 @@ class csiactcopy(ctools.cscript):
342 372  
343 373 # Logging
344 374 if self._logTerse():
345   - self._log.parformat("Number of observations")
  375 + self._log.parformat('Number of observations')
346 376 self._log(str(len(self._runs)))
347   - self._log("\n")
  377 + self._log('\n')
348 378  
349   - # Copy all data if runlist file is "NONE"
350   - elif self._runlist == "NONE":
  379 + # Copy all data if runlist file is 'NONE'
  380 + elif self._runlist == 'NONE':
351 381 if self._logTerse():
352   - self._log("Copy all available data")
353   - self._log("\n")
  382 + self._log('Copy all available data')
  383 + self._log('\n')
354 384  
355 385 # Raise exception if file not valid
356 386 else:
... ... @@ -365,13 +395,13 @@ class csiactcopy(ctools.cscript):
365 395 # Retrieve json data from remote master
366 396 json_data = open(self._remote_master.url()).read()
367 397 data = json.loads(json_data)
368   - if not "datasets" in data:
  398 + if not 'datasets' in data:
369 399 raise RuntimeError('*** ERROR: Key "datasets" not available '
370 400 'in remote master index file "'+
371 401 self._remote_master+'".')
372 402  
373 403 # Get array of configurations
374   - configs = data["datasets"]
  404 + configs = data['datasets']
375 405  
376 406 # Get remote paths
377 407 self._remote_base = self._remote_master.path()
... ... @@ -384,25 +414,25 @@ class csiactcopy(ctools.cscript):
384 414  
385 415 # Logging
386 416 if self._logTerse():
387   - self._log("\n")
388   - self._log.header2("Loop over remote configs")
  417 + self._log('\n')
  418 + self._log.header2('Loop over remote configs')
389 419  
390 420 # Loop over configs
391 421 for config in configs:
392 422  
393 423 # Logging
394 424 if self._logVerbose():
395   - self._log.header2(str(config["name"]))
  425 + self._log.header2(str(config['name']))
396 426  
397 427 # Check if prodname was found
398   - if config["name"] == self._prodname:
  428 + if config['name'] == self._prodname:
399 429  
400 430 # Inidicate that prodname has been found
401 431 has_prod = True
402 432  
403 433 # Build path of index files
404   - remote_hdu = os.path.join(self._remote_base, config["hduindx"])
405   - remote_obs = os.path.join(self._remote_base, config["obsindx"])
  434 + remote_hdu = os.path.join(self._remote_base, config['hduindx'])
  435 + remote_obs = os.path.join(self._remote_base, config['obsindx'])
406 436  
407 437 # Log information
408 438 if self._logTerse():
... ... @@ -416,10 +446,10 @@ class csiactcopy(ctools.cscript):
416 446  
417 447 # Open remote HDU index file
418 448 fits = gammalib.GFits(str(remote_hdu))
419   - table = fits["HDU_INDEX"]
  449 + table = fits['HDU_INDEX']
420 450  
421 451 # Initialise flag if SIZE column is present
422   - has_size = table.contains("SIZE")
  452 + has_size = table.contains('SIZE')
423 453  
424 454 if has_size:
425 455 # Initialise file size
... ... @@ -428,7 +458,7 @@ class csiactcopy(ctools.cscript):
428 458 # Initialise remote observation IDs
429 459 remote_ids = set()
430 460 for row in range(table.nrows()):
431   - remote_ids.add(table["OBS_ID"][row])
  461 + remote_ids.add(table['OBS_ID'][row])
432 462  
433 463 # Log runs that are not available remotely
434 464 for run in self._runs:
... ... @@ -442,12 +472,12 @@ class csiactcopy(ctools.cscript):
442 472 for row in range(table.nrows()):
443 473  
444 474 # Get observation ID
445   - obs_id = table["OBS_ID"][row]
446   - file_dir = table["FILE_DIR"][row]
447   - file_name = table["FILE_NAME"][row]
  475 + obs_id = table['OBS_ID'][row]
  476 + file_dir = table['FILE_DIR'][row]
  477 + file_name = table['FILE_NAME'][row]
448 478  
449 479 # Skip if filename is empty
450   - if file_name == "":
  480 + if file_name == '':
451 481 continue
452 482  
453 483 # Check if we need to consider an input runlist
... ... @@ -467,7 +497,7 @@ class csiactcopy(ctools.cscript):
467 497  
468 498 # Add file size
469 499 if has_size and newlen > oldlen:
470   - cp_size += table["SIZE"][row]
  500 + cp_size += table['SIZE'][row]
471 501  
472 502 # Otherwise add every file
473 503 else:
... ... @@ -482,24 +512,24 @@ class csiactcopy(ctools.cscript):
482 512  
483 513 # Add file size
484 514 if has_size and newlen > oldlen:
485   - cp_size += table["SIZE"][row]
  515 + cp_size += table['SIZE'][row]
486 516  
487 517 # Log file information
488 518 if self._logTerse():
489   - self._log("\n")
490   - self._log.header2("File information")
491   - self._log.parformat("Number of files")
  519 + self._log('\n')
  520 + self._log.header2('File information')
  521 + self._log.parformat('Number of files')
492 522 self._log(str(len(files)))
493   - self._log("\n")
  523 + self._log('\n')
494 524 if has_size:
495   - self._log.parformat("Size")
496   - self._log("%.2f"%(float(cp_size)*1e-6)+" MB")
497   - self._log("\n")
  525 + self._log.parformat('Size')
  526 + self._log('%.2f'%(float(cp_size)*1e-6)+' MB')
  527 + self._log('\n')
498 528 if self._logVerbose():
499   - self._log("\n")
500   - self._log.header3("File names")
  529 + self._log('\n')
  530 + self._log.header3('File names')
501 531 for filename in files:
502   - self._log(str(filename)+"\n")
  532 + self._log(str(filename)+'\n')
503 533  
504 534 # Close HDU index file
505 535 fits.close()
... ... @@ -507,23 +537,23 @@ class csiactcopy(ctools.cscript):
507 537 # If prodname is not found just log that we skip the config
508 538 else:
509 539 if self._logExplicit():
510   - self._log("\n")
  540 + self._log('\n')
511 541 self._log.header3('Skipping config "'+
512   - str(config["name"])+'"')
513   - self._log("\n")
  542 + str(config['name'])+'"')
  543 + self._log('\n')
514 544  
515 545 # Raise Exception if prodname was not found
516 546 if not has_prod:
517   - msg = '*** ERROR: FITS production "'+self._prodname+\
518   - '" not available. Available productions are:\n'
  547 + msg = '*** ERROR: FITS production "'+self._prodname+'" not '
  548 + msg += 'available. Available productions are:\n'
519 549 for config in configs:
520   - msg += " - "+config["name"]+"\n"
  550 + msg += ' - '+config['name']+'\n'
521 551 raise RuntimeError(msg)
522 552  
523 553 # Logging
524 554 if self._logNormal():
525   - self._log("\n")
526   - self._log.header1("Copying files")
  555 + self._log('\n')
  556 + self._log.header1('Copying files')
527 557  
528 558 # intialise counter
529 559 k = 0
... ... @@ -549,9 +579,9 @@ class csiactcopy(ctools.cscript):
549 579 fraction = float(k) / float(len(files)) * 100.0
550 580 while fraction > last_fraction:
551 581 if self._logNormal() and not self._logVerbose():
552   - self._log.parformat("Status")
553   - self._log(str(int(last_fraction))+"%")
554   - self._log("\n")
  582 + self._log.parformat('Status')
  583 + self._log(str(int(last_fraction))+'%')
  584 + self._log('\n')
555 585 last_fraction += fraction_increment
556 586  
557 587 # Copy file
... ... @@ -566,12 +596,12 @@ class csiactcopy(ctools.cscript):
566 596  
567 597 # Logging
568 598 if self._logNormal() and not self._logVerbose():
569   - self._log.parformat("Status")
570   - self._log("Finished")
571   - self._log("\n")
  599 + self._log.parformat('Status')
  600 + self._log('Finished')
  601 + self._log('\n')
572 602 if self._logTerse():
573   - self._log("\n")
574   - self._log.header1("Updating index files")
  603 + self._log('\n')
  604 + self._log.header1('Updating index files')
575 605  
576 606 # Build local index file names
577 607 local_hdu = os.path.join(self._outpath,
... ... @@ -586,18 +616,18 @@ class csiactcopy(ctools.cscript):
586 616  
587 617 # Logging
588 618 if self._logTerse():
589   - self._log("\n")
590   - self._log.header3("HDU index")
  619 + self._log('\n')
  620 + self._log.header3('HDU index')
591 621  
592 622 # Merge remote index files with local files
593   - self._merge(local_hdu, remote_hdu, "HDU_INDEX", self._clobber())
  623 + self._merge(local_hdu, remote_hdu, 'HDU_INDEX', self._clobber())
594 624  
595 625 if self._logTerse():
596   - self._log("\n")
597   - self._log.header3("OBS index")
  626 + self._log('\n')
  627 + self._log.header3('OBS index')
598 628  
599 629 # Merge remote index files with local files
600   - self._merge(local_obs, remote_obs, "OBS_INDEX", self._clobber())
  630 + self._merge(local_obs, remote_obs, 'OBS_INDEX', self._clobber())
601 631  
602 632 else:
603 633 # If all files were copied, just copy index files too
... ... @@ -605,11 +635,11 @@ class csiactcopy(ctools.cscript):
605 635 self._copy(remote_obs, self._clobber())
606 636  
607 637 if self._logTerse():
608   - self._log("\n")
609   - self._log.header3("Master index file")
  638 + self._log('\n')
  639 + self._log.header3('Master index file')
610 640  
611 641 # Adding prodname to local master
612   - localmaster = os.path.join(self._outpath, "master.json")
  642 + localmaster = os.path.join(self._outpath, 'master.json')
613 643  
614 644 # If no local master is found, copy master over first
615 645 if not os.path.isfile(localmaster):
... ... @@ -618,7 +648,7 @@ class csiactcopy(ctools.cscript):
618 648 # Load local master
619 649 json_data = open(localmaster).read()
620 650 data = json.loads(json_data)
621   - configs = data["datasets"]
  651 + configs = data['datasets']
622 652  
623 653 # Initialise flag indicating if we already have prodname in master
624 654 # file
... ... @@ -631,66 +661,66 @@ class csiactcopy(ctools.cscript):
631 661 for config in configs:
632 662  
633 663 # Get hdu and obs index files
634   - hdu = os.path.join(self._outpath, config["hduindx"])
635   - obs = os.path.join(self._outpath, config["obsindx"])
  664 + hdu = os.path.join(self._outpath, config['hduindx'])
  665 + obs = os.path.join(self._outpath, config['obsindx'])
636 666  
637 667 # Check if index files are available
638 668 if not (gammalib.GFilename(str(hdu)).is_fits() and
639 669 gammalib.GFilename(str(obs)).is_fits()):
640 670 if self._logTerse():
641   - self._log('Removing "'+str(config["name"])+'" ')
  671 + self._log('Removing "'+str(config['name'])+'" ')
642 672 self._log('(not available)')
643 673 self._log('\n')
644 674 else:
645 675 # Append config if valid
646 676 newconfigs.append(config)
647 677 if self._logTerse():
648   - self._log('Keeping "'+str(config["name"])+'"')
  678 + self._log('Keeping "'+str(config['name'])+'"')
649 679 self._log('\n')
650 680  
651 681 # Signals that downloaded config is available
652   - if config["name"] == self._prodname:
  682 + if config['name'] == self._prodname:
653 683 has_config = True
654 684  
655 685 # Create new entry if config was not available
656 686 if not has_config:
657   - newdict = dict.fromkeys(["name","hduindx","obsindx"])
658   - newdict["name"] = self._prodname
659   - newdict["hduindx"] = os.path.relpath(local_hdu, self._outpath)
660   - newdict["obsindx"] = os.path.relpath(local_obs, self._outpath)
  687 + newdict = dict.fromkeys(['name','hduindx','obsindx'])
  688 + newdict['name'] = self._prodname
  689 + newdict['hduindx'] = os.path.relpath(local_hdu, self._outpath)
  690 + newdict['obsindx'] = os.path.relpath(local_obs, self._outpath)
661 691 newconfigs.append(newdict)
662 692 if self._logTerse():
663   - self._log('Adding "'+str(newdict["name"])+'"')
  693 + self._log('Adding "'+str(newdict['name'])+'"')
664 694 self._log('\n')
665 695  
666 696 # Write new json master file
667   - f = open(localmaster, "w")
668   - data["datasets"] = newconfigs
  697 + f = open(localmaster, 'w')
  698 + data['datasets'] = newconfigs
669 699 json.dump(data, f, indent=2)
670 700 f.close()
671 701  
672 702 # Log summary
673 703 if self._logNormal():
674   - self._log("\n")
675   - self._log.header1("Summary")
676   - self._log.parformat("Data files found")
  704 + self._log('\n')
  705 + self._log.header1('Summary')
  706 + self._log.parformat('Data files found')
677 707 self._log(str(len(files)))
678   - self._log("\n")
679   - self._log.parformat("Data files copied")
  708 + self._log('\n')
  709 + self._log.parformat('Data files copied')
680 710 self._log(str(n_copied))
681   - self._log("\n")
682   - self._log.parformat("Size")
683   - self._log("%.2f"%(float(total_size)*1e-6)+" MB")
684   - self._log("\n")
685   - self._log.parformat("Local configs")
  711 + self._log('\n')
  712 + self._log.parformat('Size')
  713 + self._log('%.2f'%(float(total_size)*1e-6)+' MB')
  714 + self._log('\n')
  715 + self._log.parformat('Local configs')
686 716 self._log(str(len(newconfigs)))
687   - self._log("\n")
  717 + self._log('\n')
688 718 if self._logTerse():
689   - self._log("\n")
690   - self._log.header3("Content of master index")
  719 + self._log('\n')
  720 + self._log.header3('Content of master index')
691 721 for config in newconfigs:
692   - self._log(str(config["name"]))
693   - self._log("\n")
  722 + self._log(str(config['name']))
  723 + self._log('\n')
694 724  
695 725 # Return
696 726 return
... ...
cscripts/csiactobs.par
  View file @c60f05b
... ... @@ -18,7 +18,7 @@
18 18 # Input and output parameters
19 19 #============================
20 20 datapath, s, a, $VHEFITS,,, "Path were data is located"
21   -prodname, s, a, ,,, "Data storage name"
  21 +prodname, s, a, fits-prod-name,,, "Data storage name"
22 22 infile, f, a, runlist.lis,,, "Input runlist file"
23 23 inmodel, f, h, NONE,,, "Input model definition XML file (optional)"
24 24 master_indx, s, h, master.json,,, "Input master index file"
... ...
cscripts/csiactobs.py
  View file @c60f05b
... ... @@ -46,14 +46,14 @@ class csiactobs(ctools.cscript):
46 46 Constructor.
47 47 """
48 48 # Set name and version
49   - self._name = "csiactobs"
50   - self._version = "1.1.0"
  49 + self._name = 'csiactobs'
  50 + self._version = '1.1.0'
51 51  
52 52 # Initialise some members
53 53 self._ebounds = gammalib.GEbounds()
54   - self._datapath = os.getenv("VHEFITS","")
  54 + self._datapath = os.getenv('VHEFITS','')
55 55 self._inmodels = gammalib.GModels()
56   - self._prodname = ""
  56 + self._prodname = ''
57 57 self._xml = gammalib.GXml()
58 58 self._models = gammalib.GModels()
59 59 self._runlist = []
... ... @@ -61,23 +61,23 @@ class csiactobs(ctools.cscript):
61 61 self._bkgpars = 0
62 62 self._outmodel = gammalib.GFilename()
63 63 self._outobs = gammalib.GFilename()
64   - self._master_indx = ""
  64 + self._master_indx = ''
65 65 self._use_bkg_scale = False
66   - self._ev_hiera = [""]
67   - self._aeff_hiera = [""]
68   - self._psf_hiera = [""]
69   - self._bkg_hiera = [""]
70   - self._edisp_hiera = [""]
71   - self._bkg_mod_hiera = [""]
  66 + self._ev_hiera = ['']
  67 + self._aeff_hiera = ['']
  68 + self._psf_hiera = ['']
  69 + self._bkg_hiera = ['']
  70 + self._edisp_hiera = ['']
  71 + self._bkg_mod_hiera = ['']
72 72 self._bkg_gauss_norm = 1.0
73 73 self._bkg_gauss_index = 0.0
74 74 self._bkg_gauss_sigma = 1.0
75 75 self._bkg_aeff_index = 0.0
76 76 self._bkg_aeff_norm = 1.0
77 77 self._bkg_range_factor = 1.0
78   - self._hdu_index = ""
79   - self._obs_index = ""
80   - self._subdir = ""
  78 + self._hdu_index = ''
  79 + self._obs_index = ''
  80 + self._subdir = ''
81 81 self._debug = False
82 82  
83 83 # Initialise application by calling the appropriate class
... ... @@ -94,24 +94,24 @@ class csiactobs(ctools.cscript):
94 94 Get parameters from parfile and setup the observation.
95 95 """
96 96 # Set data path
97   - if self._datapath == "":
98   - self._datapath = self["datapath"].string()
  97 + if self._datapath == '':
  98 + self._datapath = self['datapath'].string()
99 99  
100 100 # Check for input models
101   - if not self["inmodel"].filename() == "NONE":
102   - self._inmodels = gammalib.GModels(self["inmodel"].filename())
  101 + if not self['inmodel'].filename() == 'NONE':
  102 + self._inmodels = gammalib.GModels(self['inmodel'].filename())
103 103  
104 104 # Expand environment
105 105 self._datapath = gammalib.expand_env(self._datapath)
106 106  
107 107 # Read FITS production
108   - self._prodname = self["prodname"].string()
  108 + self._prodname = self['prodname'].string()
109 109  
110 110 # Read runlist file if list not already filled
111 111 if len(self._runlist) == 0:
112 112  
113 113 # Get file name
114   - self._runlistfile = self["infile"].filename()
  114 + self._runlistfile = self['infile'].filename()
115 115  
116 116 # Read runlist from file
117 117 runfile = open(self._runlistfile.url())
... ... @@ -125,73 +125,73 @@ class csiactobs(ctools.cscript):
125 125 runfile.close()
126 126  
127 127 # Read number of background parameters
128   - self._bkgpars = self["bkgpars"].integer()
  128 + self._bkgpars = self['bkgpars'].integer()
129 129  
130 130 # Output model file
131   - self._outmodel = self["outmodel"].filename()
  131 + self._outmodel = self['outmodel'].filename()
132 132  
133 133 # Observation outfile
134   - self._outobs = self["outobs"].filename()
  134 + self._outobs = self['outobs'].filename()
135 135  
136 136 # Master index file name
137   - self._master_indx = self["master_indx"].string()
  137 + self._master_indx = self['master_indx'].string()
138 138  
139 139 # Read flag for background scaling factor
140   - self._use_bkg_scale = self["bkg_scale"].boolean()
  140 + self._use_bkg_scale = self['bkg_scale'].boolean()
141 141  
142 142 # Read hierarchy of file loading
143   - self._ev_hiera = self["ev_hiera"].string().split("|")
144   - self._aeff_hiera = self["aeff_hiera"].string().split("|")
145   - self._psf_hiera = self["psf_hiera"].string().split("|")
146   - self._bkg_hiera = self["bkg_hiera"].string().split("|")
147   - self._edisp_hiera = self["edisp_hiera"].string().split("|")
148   - self._bkg_mod_hiera = self["bkg_mod_hiera"].string().split("|")
  143 + self._ev_hiera = self['ev_hiera'].string().split('|')
  144 + self._aeff_hiera = self['aeff_hiera'].string().split('|')
  145 + self._psf_hiera = self['psf_hiera'].string().split('|')
  146 + self._bkg_hiera = self['bkg_hiera'].string().split('|')
  147 + self._edisp_hiera = self['edisp_hiera'].string().split('|')
  148 + self._bkg_mod_hiera = self['bkg_mod_hiera'].string().split('|')
149 149  
150 150 # Read hidden background parameters
151   - self._bkg_gauss_norm = self["bkg_gauss_norm"].real()
152   - self._bkg_gauss_index = self["bkg_gauss_index"].real()
153   - self._bkg_gauss_sigma = self["bkg_gauss_sigma"].real()
154   - self._bkg_aeff_index = self["bkg_aeff_index"].real()
155   - self._bkg_aeff_norm = self["bkg_aeff_norm"].real()
156   - self._bkg_range_factor = self["bkg_range_factor"].real()
  151 + self._bkg_gauss_norm = self['bkg_gauss_norm'].real()
  152 + self._bkg_gauss_index = self['bkg_gauss_index'].real()
  153 + self._bkg_gauss_sigma = self['bkg_gauss_sigma'].real()
  154 + self._bkg_aeff_index = self['bkg_aeff_index'].real()
  155 + self._bkg_aeff_norm = self['bkg_aeff_norm'].real()
  156 + self._bkg_range_factor = self['bkg_range_factor'].real()
157 157  
158 158 # Open master index file and look for prodname
159 159 master_file = os.path.join(self._datapath, self._master_indx)
160 160 if not os.path.isfile(master_file):
161   - raise RuntimeError("FITS data store not available. No master index file found. Make sure the file is copied from the server and your datapath is set correctly.")
  161 + raise RuntimeError('FITS data store not available. No master index file found. Make sure the file is copied from the server and your datapath is set correctly.')
162 162  
163 163 # Open and load JSON file
164 164 json_data = open(master_file).read()
165 165 data = json.loads(json_data)
166   - if not "datasets" in data:
167   - raise RuntimeError("Key \"datasets\" not available in master index file.")
  166 + if not 'datasets' in data:
  167 + raise RuntimeError('Key "datasets" not available in master index file.')
168 168  
169 169 # Get configurations
170   - configs = data["datasets"]
  170 + configs = data['datasets']
171 171  
172 172 # Initialise HDUs
173   - self._hdu_index = self._obs_index = ""
  173 + self._hdu_index = self._obs_index = ''
174 174  
175 175 # Get HDUs
176 176 for config in configs:
177   - if self._prodname == config["name"]:
178   - self._hdu_index = str(os.path.join(self._datapath, config["hduindx"]))
179   - self._obs_index = str(os.path.join(self._datapath, config["obsindx"]))
  177 + if self._prodname == config['name']:
  178 + self._hdu_index = str(os.path.join(self._datapath, config['hduindx']))
  179 + self._obs_index = str(os.path.join(self._datapath, config['obsindx']))
180 180 break
181 181  
182 182 # Check HDUs
183   - if self._hdu_index == "" or self._obs_index == "":
184   - raise RuntimeError("*** ERROR: FITS data store \""+self._prodname+"\" not available. Run csiactdata to get a list of available storage names")
185   - filename = gammalib.GFilename(self._hdu_index+"[HDU_INDEX]")
  183 + if self._hdu_index == '' or self._obs_index == '':
  184 + raise RuntimeError('*** ERROR: FITS data store "'+self._prodname+'" not available. Run csiactdata to get a list of available storage names')
  185 + filename = gammalib.GFilename(self._hdu_index+'[HDU_INDEX]')
186 186 if not filename.is_fits():
187   - raise RuntimeError("*** ERROR: HDU index file \""+self._hdu_index+"[HDU_INDEX]\" for FITS data store \""+self._prodname+"\" not available. Check your master index file or run csiactdata to get a list of available storage names.")
  187 + raise RuntimeError('*** ERROR: HDU index file "'+self._hdu_index+'[HDU_INDEX]" for FITS data store "'+self._prodname+'" not available. Check your master index file or run csiactdata to get a list of available storage names.')
188 188  
189   - # Check for existence of "BKG_SCALE" in the observation index file if required
  189 + # Check for existence of 'BKG_SCALE' in the observation index file if required
190 190 if self._use_bkg_scale:
191   - filename = gammalib.GFilename(self._obs_index+"[OBS_INDEX]")
  191 + filename = gammalib.GFilename(self._obs_index+'[OBS_INDEX]')
192 192 if filename.is_fits():
193 193 fits = gammalib.GFits(self._obs_index)
194   - if not fits["OBS_INDEX"].contains("BKG_SCALE"):
  194 + if not fits['OBS_INDEX'].contains('BKG_SCALE'):
195 195 self._use_bkg_scale = False
196 196 fits.close()
197 197 else:
... ... @@ -204,7 +204,7 @@ class csiactobs(ctools.cscript):
204 204 # Write input parameters into logger
205 205 if self._logTerse():
206 206 self._log_parameters()
207   - self._log("\n")
  207 + self._log('\n')
208 208  
209 209 # Return
210 210 return
... ... @@ -223,7 +223,7 @@ class csiactobs(ctools.cscript):
223 223 except:
224 224  
225 225 # raise error if wrong type detected
226   - raise RuntimeError("Argument of csiactobs.runlist() must be of type 'list'")
  226 + raise RuntimeError('Argument of csiactobs.runlist() must be of type "list"')
227 227  
228 228 # Return
229 229 return
... ... @@ -233,24 +233,24 @@ class csiactobs(ctools.cscript):
233 233 # Handle constant spectral model
234 234 if index == 0.0 and self._bkgpars <= 1:
235 235 spec = gammalib.GModelSpectralConst()
236   - spec["Value"].min(prefactor / self._bkg_range_factor)
237   - spec["Value"].max(prefactor * self._bkg_range_factor)
238   - spec["Value"].value(prefactor)
  236 + spec['Value'].min(prefactor / self._bkg_range_factor)
  237 + spec['Value'].max(prefactor * self._bkg_range_factor)
  238 + spec['Value'].value(prefactor)
239 239 if self._bkgpars == 0:
240   - spec["Value"].fix()
  240 + spec['Value'].fix()
241 241 else:
242   - spec["Value"].free()
  242 + spec['Value'].free()
243 243  
244 244 else:
245 245  
246 246 # Create power law model
247 247 if self._bkgpars <= 2:
248   - e = gammalib.GEnergy(1.0,"TeV")
  248 + e = gammalib.GEnergy(1.0,'TeV')
249 249 spec = gammalib.GModelSpectralPlaw(prefactor, index, e)
250 250  
251 251 # Set parameter ranges
252   - spec[0].min(0.01 * prefactor)
253   - spec[0].max(100.0 * prefactor)
  252 + spec[0].min(prefactor / self._bkg_range_factor)
  253 + spec[0].max(prefactor * self._bkg_range_factor)
254 254 spec[1].scale(1)
255 255 spec[1].min(-5.0)
256 256 spec[1].max(5.0)
... ... @@ -269,20 +269,22 @@ class csiactobs(ctools.cscript):
269 269 else:
270 270  
271 271 # Create reference powerlaw
272   - plaw = gammalib.GModelSpectralPlaw(prefactor, index, gammalib.GEnergy(1.0,"TeV"))
  272 + plaw = gammalib.GModelSpectralPlaw(prefactor, index, gammalib.GEnergy(1.0,'TeV'))
273 273  
274 274 # Create spectral model and energy values
275 275 spec = gammalib.GModelSpectralNodes()
276   - bounds = gammalib.GEbounds(self._bkgpars,gammalib.GEnergy(emin,"TeV"),gammalib.GEnergy(emax,"TeV"), True)
  276 + bounds = gammalib.GEbounds(self._bkgpars,gammalib.GEnergy(emin,'TeV'),gammalib.GEnergy(emax,'TeV'), True)
277 277 for i in range(bounds.size()):
278 278 energy = bounds.elogmean(i)
279 279 value = plaw.eval(energy, gammalib.GTime())
280 280 spec.append(energy, value)
281 281 for par in spec:
282 282  
283   - if "Energy" in par.name():
  283 + if 'Energy' in par.name():
284 284 par.fix()
285   - elif "Intensity" in par.name():
  285 + elif 'Intensity' in par.name():
  286 + value = par.value()
  287 + par.scale(value)
286 288 par.min(value / self._bkg_range_factor)
287 289 par.max(value * self._bkg_range_factor)
288 290  
... ... @@ -292,7 +294,7 @@ class csiactobs(ctools.cscript):
292 294 def _iact_background(self, telescope, obs_id, bkg_scale, bkgtype, emin=0.01, emax=100):
293 295  
294 296 # handle IrfBackground
295   - if bkgtype == "irf":
  297 + if bkgtype == 'irf':
296 298 prefactor = 1.0
297 299 index = 0.0
298 300 prefactor *= bkg_scale
... ... @@ -302,7 +304,7 @@ class csiactobs(ctools.cscript):
302 304 bck = gammalib.GCTAModelIrfBackground(spec)
303 305  
304 306 # Set AeffBackground
305   - elif bkgtype == "aeff":
  307 + elif bkgtype == 'aeff':
306 308 prefactor = bkg_scale * self._bkg_aeff_norm
307 309 spec = self._background_spectrum(obs_id, prefactor, self._bkg_aeff_index, emin, emax)
308 310  
... ... @@ -310,14 +312,14 @@ class csiactobs(ctools.cscript):
310 312 bck = gammalib.GCTAModelAeffBackground(spec)
311 313  
312 314 # Set Gaussian Background
313   - elif bkgtype == "gauss":
  315 + elif bkgtype == 'gauss':
314 316 prefactor = bkg_scale * self._bkg_gauss_norm
315 317 spec = self._background_spectrum(obs_id, prefactor, self._bkg_gauss_index, emin, emax)
316 318 radial = gammalib.GCTAModelRadialGauss(self._bkg_gauss_sigma)
317 319 bck = gammalib.GCTAModelRadialAcceptance(radial, spec)
318 320  
319 321 else:
320   - sys.exit("Background type \""+self._bkgtype+"\" unsupported")
  322 + sys.exit('Background type "'+self._bkgtype+'" unsupported')
321 323  
322 324 # Copy model
323 325 model = bck.clone()
... ... @@ -329,7 +331,7 @@ class csiactobs(ctools.cscript):
329 331 model.instruments(telescope)
330 332  
331 333 # Set name (arbitrary)
332   - model.name("bkg_"+str(obs_id))
  334 + model.name('bkg_'+str(obs_id))
333 335  
334 336 # Turn off TS calculation for background model
335 337 model.tscalc(False)
... ... @@ -355,36 +357,36 @@ class csiactobs(ctools.cscript):
355 357  
356 358 # Clear xml file and append an observation list
357 359 self._xml.clear()
358   - self._xml.append(gammalib.GXmlElement("observation_list title=\"observation list\""))
359   - lib = self._xml.element("observation_list", 0)
  360 + self._xml.append(gammalib.GXmlElement('observation_list title="observation list"'))
  361 + lib = self._xml.element('observation_list', 0)
360 362  
361 363 # Log output
362 364 if self._logTerse():
363   - self._log("\n")
364   - self._log.header1("Looping over "+str(len(self._runlist))+" runs")
  365 + self._log('\n')
  366 + self._log.header1('Looping over '+str(len(self._runlist))+' runs')
365 367  
366 368 # Initialise energy range values for logging
367   - self._ebounds.clear()
  369 + self._ebounds.clear()
368 370  
369 371 # Loop over runs
370 372 for obs_id in self._runlist:
371 373  
372 374 # Create selection string
373   - obs_selection = "[OBS_ID=="+str(obs_id)+"]"
  375 + obs_selection = '[OBS_ID=='+str(obs_id)+']'
374 376  
375 377 # Open HDU index file
376   - hduindx = gammalib.GFits(self._hdu_index+"[HDU_INDEX]"+obs_selection)
377   - hduindx_hdu = hduindx["HDU_INDEX"]
  378 + hduindx = gammalib.GFits(self._hdu_index+'[HDU_INDEX]'+obs_selection)
  379 + hduindx_hdu = hduindx['HDU_INDEX']
378 380  
379 381 # Initialise files and hdu names
380   - eventfile = aefffile = psffile = edispfile = bkgfile = ""
381   - eventhdu = aeffhdu = ""
  382 + eventfile = aefffile = psffile = edispfile = bkgfile = ''
  383 + eventhdu = aeffhdu = ''
382 384  
383 385 types = []
384 386 formats = []
385 387 for i in range(hduindx_hdu.nrows()):
386   - types.append(hduindx_hdu["HDU_TYPE"][i])
387   - formats.append(hduindx_hdu["HDU_CLASS"][i])
  388 + types.append(hduindx_hdu['HDU_TYPE'][i])
  389 + formats.append(hduindx_hdu['HDU_CLASS'][i])
388 390  
389 391 # Handle events
390 392 index = -1
... ... @@ -394,12 +396,12 @@ class csiactobs(ctools.cscript):
394 396 index = formats.index(version)
395 397 break
396 398 if index >= 0:
397   - eventfile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index])
398   - eventhdu = hduindx_hdu["HDU_NAME"][index]
399   - eventfile += "["+eventhdu+"]"
  399 + eventfile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index])
  400 + eventhdu = hduindx_hdu['HDU_NAME'][index]
  401 + eventfile += '['+eventhdu+']'
400 402 if not gammalib.GFilename(eventfile).is_fits():
401 403 if self._logTerse():
402   - self._log("Skipping observation "+str(obs_id)+": eventfile \""+eventfile+"\" not found\n")
  404 + self._log('Skipping observation '+str(obs_id)+': eventfile "'+eventfile+'" not found\n')
403 405 continue
404 406  
405 407 # Handle Aeff
... ... @@ -410,12 +412,12 @@ class csiactobs(ctools.cscript):
410 412 index = formats.index(version)
411 413 break
412 414 if index >= 0:
413   - aefffile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index])
414   - aeffhdu = hduindx_hdu["HDU_NAME"][index]
415   - aefffile += "["+aeffhdu+"]"
  415 + aefffile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index])
  416 + aeffhdu = hduindx_hdu['HDU_NAME'][index]
  417 + aefffile += '['+aeffhdu+']'
416 418 if not gammalib.GFilename(aefffile).is_fits():
417 419 if self._logTerse():
418   - self._log("Skipping observation "+str(obs_id)+": effective area \""+aefffile+"\" not found\n")
  420 + self._log('Skipping observation '+str(obs_id)+': effective area "'+aefffile+'" not found\n')
419 421 continue
420 422  
421 423 # Handle psf
... ... @@ -426,11 +428,11 @@ class csiactobs(ctools.cscript):
426 428 index = formats.index(version)
427 429 break
428 430 if index >= 0:
429   - psffile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index])
430   - psffile += "["+hduindx_hdu["HDU_NAME"][index]+"]"
  431 + psffile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index])
  432 + psffile += '['+hduindx_hdu['HDU_NAME'][index]+']'
431 433 if not gammalib.GFilename(psffile).is_fits():
432 434 if self._logTerse():
433   - self._log("Skipping observation "+str(obs_id)+": point spread function \""+psffile+"\" not found\n")
  435 + self._log('Skipping observation '+str(obs_id)+': point spread function "'+psffile+'" not found\n')
434 436 continue
435 437  
436 438 # Handle edisp
... ... @@ -441,12 +443,12 @@ class csiactobs(ctools.cscript):
441 443 index = formats.index(version)
442 444 break
443 445 if index >= 0:
444   - edispfile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index])
445   - edispfile += "["+hduindx_hdu["HDU_NAME"][index]+"]"
  446 + edispfile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index])
  447 + edispfile += '['+hduindx_hdu['HDU_NAME'][index]+']'
446 448 if not gammalib.GFilename(edispfile).is_fits():
447 449 if self._logTerse():
448   - self._log("Warning: observation "+str(obs_id)+" has no energy dispersion \""+edispfile+"\" information\n")
449   - edispfile = ""
  450 + self._log('Warning: observation '+str(obs_id)+' has no energy dispersion "'+edispfile+'" information\n')
  451 + edispfile = ''
450 452  
451 453 # Handle background
452 454 index = -1
... ... @@ -457,14 +459,20 @@ class csiactobs(ctools.cscript):
457 459 index = formats.index(version)
458 460 break
459 461 if index >= 0:
460   - bkgfile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index])
461   - bkgfile += "["+hduindx_hdu["HDU_NAME"][index]+"]"
462   - if not gammalib.GFilename(bkgfile).is_fits():
  462 + bkgfile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index])
  463 + bkgfile += '['+hduindx_hdu['HDU_NAME'][index]+']'
  464 + if 'irf' in bkg_mod_hierarchy and not gammalib.GFilename(bkgfile).is_fits():
  465 + bkg_mod_hierarchy.remove('irf')
463 466 if self._logTerse():
464   - self._log("Warning: observation "+str(obs_id)+" has no background information (file=\""+bkgfile+"\"). IRF background cannot be used\n")
465   - bkgfile = ""
466   - if "irf" in bkg_mod_hierarchy:
467   - bkg_mod_hierarchy.remove("irf")
  467 + self._log('Warning: observation '+str(obs_id)+' has no background information (file="'+bkgfile+'"). IRF background cannot be used\n')
  468 + bkgfile = ''
  469 + if len(bkg_mod_hierarchy) == 0:
  470 + if self._logTerse():
  471 + self._log('Skipping observation '+str(obs_id)+': No background can be used\n')
  472 + continue
  473 + else:
  474 + if self._logTerse():
  475 + self._log('Observation '+str(obs_id)+': Falling back to background "'+bkg_mod_hierarchy[0]+'"\n')
468 476  
469 477 # Close hdu index file
470 478 hduindx.close()
... ... @@ -472,15 +480,15 @@ class csiactobs(ctools.cscript):
472 480 # Handle background scale information if available
473 481 bkg_scale = 1.0
474 482 if self._use_bkg_scale:
475   - obsindx = gammalib.GFits(self._obs_index+"[OBS_INDEX]"+obs_selection)
476   - bkg_scale = obsindx["OBS_INDEX"]["BKG_SCALE"][0]
  483 + obsindx = gammalib.GFits(self._obs_index+'[OBS_INDEX]'+obs_selection)
  484 + bkg_scale = obsindx['OBS_INDEX']['BKG_SCALE'][0]
477 485 obsindx.close()
478 486  
479 487 # Open fits file to determine the observation name
480 488 fits = gammalib.GFits(eventfile)
481 489 events = fits[eventhdu]
482   - object_name = events.string("OBJECT")
483   - telescope = events.string("TELESCOP")
  490 + object_name = events.string('OBJECT')
  491 + telescope = events.string('TELESCOP')
484 492  
485 493 # Close FITS file
486 494 fits.close()
... ... @@ -490,19 +498,19 @@ class csiactobs(ctools.cscript):
490 498 aeff_table = aeff_fits[aeffhdu]
491 499  
492 500 # Set energy range from header keyword if present
493   - if aeff_table.has_card("LO_THRES") and aeff_table.has_card("HI_THRES"):
  501 + if aeff_table.has_card('LO_THRES') and aeff_table.has_card('HI_THRES'):
494 502  
495 503 # Get safe energy range
496   - run_emin = gammalib.GEnergy(aeff_table.real("LO_THRES"), "TeV")
497   - run_emax = gammalib.GEnergy(aeff_table.real("HI_THRES"), "TeV")
  504 + run_emin = gammalib.GEnergy(aeff_table.real('LO_THRES'), 'TeV')
  505 + run_emax = gammalib.GEnergy(aeff_table.real('HI_THRES'), 'TeV')
498 506  
499 507 # Append to ebounds
500 508 self._ebounds.append(run_emin, run_emax)
501 509  
502 510 else:
503 511 # Set default values for energy range
504   - run_emin = gammalib.GEnergy(10,"GeV")
505   - run_emax = gammalib.GEnergy(100,"TeV")
  512 + run_emin = gammalib.GEnergy(10,'GeV')
  513 + run_emax = gammalib.GEnergy(100,'TeV')
506 514  
507 515 # Close Aeff fits file
508 516 aeff_fits.close()
... ... @@ -512,43 +520,43 @@ class csiactobs(ctools.cscript):
512 520  
513 521 # Logging
514 522 if self._logTerse():
515   - self._log("Adding observation "+str(obs_id)+" (\""+object_name+"\")\n")
  523 + self._log('Adding observation '+str(obs_id)+' ("'+object_name+'")\n')
516 524  
517 525 if self._logExplicit():
518   - self._log(" Event file: "+eventfile+"\n")
519   - self._log(" Effective area: "+aefffile+"\n")
520   - self._log(" Point spread function: "+psffile+"\n")
521   - self._log(" Energy dispersion: "+edispfile+"\n")
522   - self._log(" Background file: "+bkgfile+"\n")
  526 + self._log(' Event file: '+eventfile+'\n')
  527 + self._log(' Effective area: '+aefffile+'\n')
  528 + self._log(' Point spread function: '+psffile+'\n')
  529 + self._log(' Energy dispersion: '+edispfile+'\n')
  530 + self._log(' Background file: '+bkgfile+'\n')
523 531 if self._use_bkg_scale:
524   - self._log(" Background scale: "+str(bkg_scale)+"\n")
525   - self._log("\n")
  532 + self._log(' Background scale: '+str(bkg_scale)+'\n')
  533 + self._log('\n')
526 534  
527 535 # Append observation to XML and set attributes
528   - obs = lib.append("observation")
529   - obs.attribute("name", object_name)
530   - obs.attribute("id", str(obs_id))
531   - obs.attribute("instrument", telescope)
  536 + obs = lib.append('observation')
  537 + obs.attribute('name', object_name)
  538 + obs.attribute('id', str(obs_id))
  539 + obs.attribute('instrument', telescope)
532 540  
533 541 # Append event file
534   - ev = gammalib.GXmlElement("parameter name=\"EventList\"")
535   - ev.attribute("file", eventfile)
  542 + ev = gammalib.GXmlElement('parameter name="EventList"')
  543 + ev.attribute('file', eventfile)
536 544  
537 545 # Append effective area
538   - aeff = gammalib.GXmlElement("parameter name=\"EffectiveArea\"")
539   - aeff.attribute("file",aefffile)
  546 + aeff = gammalib.GXmlElement('parameter name="EffectiveArea"')
  547 + aeff.attribute('file',aefffile)
540 548  
541 549 # Append PSF
542   - psf = gammalib.GXmlElement("parameter name=\"PointSpreadFunction\"")
543   - psf.attribute("file",psffile)
  550 + psf = gammalib.GXmlElement('parameter name="PointSpreadFunction"')
  551 + psf.attribute('file',psffile)
544 552  
545 553 # Append energy dispersion
546   - edisp = gammalib.GXmlElement("parameter name=\"EnergyDispersion\"")
547   - edisp.attribute("file",edispfile)
  554 + edisp = gammalib.GXmlElement('parameter name="EnergyDispersion"')
  555 + edisp.attribute('file',edispfile)
548 556  
549 557 # Append background
550   - bck = gammalib.GXmlElement("parameter name=\"Background\"")
551   - bck.attribute("file",bkgfile)
  558 + bck = gammalib.GXmlElement('parameter name="Background"')
  559 + bck.attribute('file',bkgfile)
552 560  
553 561 # assign events and IRFs to observation
554 562 obs.append(ev)
... ... @@ -562,35 +570,35 @@ class csiactobs(ctools.cscript):
562 570  
563 571 # Log header of energy range
564 572 if self._logTerse():
565   - self._log("\n")
566   - self._log.header3("Energy range of obervation list")
  573 + self._log('\n')
  574 + self._log.header3('Energy range of obervation list')
567 575  
568 576 # Logging if energy range is available
569 577 if self._ebounds.size() > 0:
570 578  
571 579 # Write energy range into log file
572   - self._log(str(self._ebounds.emin())+" - "+str(self._ebounds.emax()))
573   - self._log("\n")
  580 + self._log(str(self._ebounds.emin())+' - '+str(self._ebounds.emax()))
  581 + self._log('\n')
574 582  
575 583 else:
576 584  
577 585 # Write 'not available' into log file
578   - self._log("not available")
579   - self._log("\n")
  586 + self._log('not available')
  587 + self._log('\n')
580 588  
581 589 else:
582   - self._log.header2("WARNING: No observation from given runlist available")
  590 + self._log.header2('WARNING: No observation from given runlist available')
583 591  
584 592 # Append models provided by 'inmodels' if necessary
585 593 if not self._inmodels == None:
586 594 if self._logTerse():
587   - self._log("\n")
588   - self._log.header1("Appending models")
  595 + self._log('\n')
  596 + self._log.header1('Appending models')
589 597  
590 598 # Loop over input models
591 599 for model in self._inmodels:
592 600 if self._logTerse():
593   - self._log.header3("Adding model \""+model.name()+"\"")
  601 + self._log.header3('Adding model "'+model.name()+'"')
594 602 self._models.append(model)
595 603  
596 604 # Return
... ...
cscripts/csmodelinfo.py
  View file @c60f05b
... ... @@ -430,10 +430,6 @@ class csmodelinfo(ctools.cscript):
430 430 """
431 431 Save models to ds9 region file if required
432 432 """
433   - # Write header
434   - if self._logTerse():
435   - self._log("\n")
436   - self._log.header1("Save models in DS9 file")
437 433  
438 434 # Get output filename in case it was not read ahead
439 435 self._ds9file = self["ds9file"].filename()
... ... @@ -441,6 +437,11 @@ class csmodelinfo(ctools.cscript):
441 437 # Check if DS9 file is valid
442 438 if self._ds9file != "NONE":
443 439  
  440 + # Write header
  441 + if self._logTerse():
  442 + self._log("\n")
  443 + self._log.header1("Save models in DS9 file")
  444 +
444 445 # Log filename
445 446 if self._logTerse():
446 447 self._log(gammalib.parformat("DS9 filename"))
... ...
cscripts/csmodelmerge.py
  View file @c60f05b
... ... @@ -90,6 +90,9 @@ class csmodelmerge(ctools.cscript):
90 90 # Read ahead output filename
91 91 if self._read_ahead():
92 92 self["outmodel"].filename()
  93 +
  94 + # Get clobber parameter
  95 + self._clobber = self["clobber"].boolean()
93 96  
94 97 # Write input parameters into logger
95 98 if self._logTerse():
... ... @@ -163,14 +166,26 @@ class csmodelmerge(ctools.cscript):
163 166 # Get output filename in case it was not read ahead
164 167 outmodel = self["outmodel"].filename()
165 168  
166   - # Log filename
167   - if self._logTerse():
168   - self._log(gammalib.parformat("Model definition XML file"))
169   - self._log(outmodel.url())
170   - self._log("\n")
171   -
172   - # Save models
173   - self._models.save(outmodel)
  169 + # Check if file exists and apply clobber if false
  170 + if outmodel.exists() and not self._clobber:
  171 +
  172 + # Set error message
  173 + msg = "Cannot save \""+outmodel.url()+"\": File already exists "
  174 + msg += "Use parameter clobber=yes to allow overwriting of files."
  175 +
  176 + # Throw exception since file cannnot be overwritten
  177 + raise RuntimeError(msg)
  178 +
  179 + else:
  180 +
  181 + # Log filename
  182 + if self._logTerse():
  183 + self._log(gammalib.parformat("Model definition XML file"))
  184 + self._log(outmodel.url())
  185 + self._log("\n")
  186 +
  187 + # Save models
  188 + self._models.save(outmodel)
174 189  
175 190 # Return
176 191 return
... ...
cscripts/csobsinfo.py
  View file @c60f05b
... ... @@ -364,16 +364,17 @@ class csobsinfo(ctools.cscript):
364 364 observation container into a DS9 region file. If "NONE" is
365 365 specified for the "ds9file" parameter the method does nothing.
366 366 """
367   - # Write header
368   - if self._logTerse():
369   - self._log("\n")
370   - self._log.header1("Save pointings in DS9 file")
371 367  
372 368 # Get output filename in case it was not read ahead
373 369 ds9file = self["ds9file"].filename()
374 370  
375 371 # Check if DS9 file is valid
376   - if ds9file.url() != "NONE":
  372 + if ds9file.url() != "NONE":
  373 +
  374 + # Write header
  375 + if self._logTerse():
  376 + self._log("\n")
  377 + self._log.header1("Save pointings in DS9 file")
377 378  
378 379 # Log filename
379 380 if self._logTerse():
... ...