Peter Deiml / ctools

Commit c60f05be3c63bf9cf733aa351aca79585ec80190

Authored by Michael Mayer
1 parent 28009928

Bug fixes, logging and code style improvements in some cscripts

Inline Side-by-side
Showing 7 changed files with 435 additions and 373 deletions
cscripts/csfindobs.py
  View file @c60f05b
@@ -39,15 +39,15 @@ class csfindobs(ctools.cscript): @@ -39,15 +39,15 @@ class csfindobs(ctools.cscript):
39 Constructor. 39 Constructor.
40 """ 40 """
41 # Set name 41 # Set name
42 - self._name = "csfindobs"  
43 - self._version = "1.1.0"  
44 - self._datapath = os.getenv("VHEFITS","")  
45 - self._prodname = "" 42 + self._name = 'csfindobs'
  43 + self._version = '1.1.0'
  44 + self._datapath = os.getenv('VHEFITS','')
  45 + self._prodname = ''
46 self._select_radec = True 46 self._select_radec = True
47 self._radius = 0.0 47 self._radius = 0.0
48 self._ra = 0.0 48 self._ra = 0.0
49 self._dec = 0.0 49 self._dec = 0.0
50 - self._obs_index = "" 50 + self._obs_index = ''
51 self._runs = [] 51 self._runs = []
52 52
53 # Initialise application by calling the appropriate class 53 # Initialise application by calling the appropriate class
@@ -64,17 +64,17 @@ class csfindobs(ctools.cscript): @@ -64,17 +64,17 @@ class csfindobs(ctools.cscript):
64 Get parameters from parfile and setup the observation. 64 Get parameters from parfile and setup the observation.
65 """ 65 """
66 # Get parameters 66 # Get parameters
67 - if self._datapath == "":  
68 - self._datapath = self["datapath"].string() 67 + if self._datapath == '':
  68 + self._datapath = self['datapath'].string()
69 69
70 # Expand environment 70 # Expand environment
71 self._datapath = gammalib.expand_env(self._datapath) 71 self._datapath = gammalib.expand_env(self._datapath)
72 72
73 # Get production name 73 # Get production name
74 - self._prodname = self["prodname"].string() 74 + self._prodname = self['prodname'].string()
75 75
76 # Master index file name 76 # Master index file name
77 - master_indx = self["master_indx"].string() 77 + master_indx = self['master_indx'].string()
78 78
79 # Intiialise flag if spatial selection is required 79 # Intiialise flag if spatial selection is required
80 self._select_radec = True 80 self._select_radec = True
@@ -83,14 +83,14 @@ class csfindobs(ctools.cscript): @@ -83,14 +83,14 @@ class csfindobs(ctools.cscript):
83 self._radius = 0.0 83 self._radius = 0.0
84 84
85 # Check for vailidity of spatial parameters 85 # Check for vailidity of spatial parameters
86 - if (self["ra"].is_valid() and  
87 - self["dec"].is_valid() and  
88 - self["rad"].is_valid()): 86 + if (self['ra'].is_valid() and
  87 + self['dec'].is_valid() and
  88 + self['rad'].is_valid()):
89 89
90 # Read spatial parameters 90 # Read spatial parameters
91 - self._ra = self["ra"].real()  
92 - self._dec = self["dec"].real()  
93 - self._radius = self["rad"].real() 91 + self._ra = self['ra'].real()
  92 + self._dec = self['dec'].real()
  93 + self._radius = self['rad'].real()
94 94
95 else: 95 else:
96 self._select_radec = False 96 self._select_radec = False
@@ -100,12 +100,12 @@ class csfindobs(ctools.cscript): @@ -100,12 +100,12 @@ class csfindobs(ctools.cscript):
100 self._select_radec = False 100 self._select_radec = False
101 101
102 # Query other parameters 102 # Query other parameters
103 - self["min_qual"].integer()  
104 - self["expression"].string() 103 + self['min_qual'].integer()
  104 + self['expression'].string()
105 105
106 # Read ahead output parameters 106 # Read ahead output parameters
107 if self._read_ahead(): 107 if self._read_ahead():
108 - self["outfile"].filename() 108 + self['outfile'].filename()
109 109
110 # Open master index file and look for prodname 110 # Open master index file and look for prodname
111 master_file = os.path.join(self._datapath, master_indx) 111 master_file = os.path.join(self._datapath, master_indx)
@@ -119,33 +119,33 @@ class csfindobs(ctools.cscript): @@ -119,33 +119,33 @@ class csfindobs(ctools.cscript):
119 # Open and load JSON file 119 # Open and load JSON file
120 json_data = open(master_file).read() 120 json_data = open(master_file).read()
121 data = json.loads(json_data) 121 data = json.loads(json_data)
122 - if not "datasets" in data: 122 + if not 'datasets' in data:
123 raise RuntimeError('Key "datasets" not available in '+ 123 raise RuntimeError('Key "datasets" not available in '+
124 'master index file.') 124 'master index file.')
125 125
126 # Get configurations 126 # Get configurations
127 - configs = data["datasets"] 127 + configs = data['datasets']
128 128
129 # Initialise obs index file 129 # Initialise obs index file
130 - self._obs_index = "" 130 + self._obs_index = ''
131 131
132 # Get HDUs 132 # Get HDUs
133 for config in configs: 133 for config in configs:
134 - if self._prodname == config["name"]: 134 + if self._prodname == config['name']:
135 self._obs_index = str(os.path.join(self._datapath, 135 self._obs_index = str(os.path.join(self._datapath,
136 - config["obsindx"])) 136 + config['obsindx']))
137 break 137 break
138 138
139 # Check HDUs 139 # Check HDUs
140 - if self._obs_index == "": 140 + if self._obs_index == '':
141 raise RuntimeError('FITS data store "'+self._prodname+ 141 raise RuntimeError('FITS data store "'+self._prodname+
142 '" not available. Run csiactdata to get '+ 142 '" not available. Run csiactdata to get '+
143 'a list of available storage names') 143 'a list of available storage names')
144 - filename = gammalib.GFilename(self._obs_index+"[OBS_INDEX]") 144 + filename = gammalib.GFilename(self._obs_index+'[OBS_INDEX]')
145 if not filename.is_fits(): 145 if not filename.is_fits():
146 raise RuntimeError('Observation index file "'+ 146 raise RuntimeError('Observation index file "'+
147 self._obs_index+ 147 self._obs_index+
148 - '[OBS_INDEX]" for FITS data store "'+ 148 + '"[OBS_INDEX]" for FITS data store "'+
149 self._prodname+ 149 self._prodname+
150 '" not available. Check your master index '+ 150 '" not available. Check your master index '+
151 'file or run csiactdata to get a list of '+ 151 'file or run csiactdata to get a list of '+
@@ -154,7 +154,7 @@ class csfindobs(ctools.cscript): @@ -154,7 +154,7 @@ class csfindobs(ctools.cscript):
154 # Write input parameters into logger 154 # Write input parameters into logger
155 if self._logTerse(): 155 if self._logTerse():
156 self._log_parameters() 156 self._log_parameters()
157 - self._log("\n") 157 + self._log('\n')
158 158
159 # Return 159 # Return
160 return 160 return
@@ -176,55 +176,62 @@ class csfindobs(ctools.cscript): @@ -176,55 +176,62 @@ class csfindobs(ctools.cscript):
176 self._runs = [] 176 self._runs = []
177 177
178 # Initialise selction expression 178 # Initialise selction expression
179 - expr = "" 179 + expr = ''
180 180
181 # Add spatial expression if possible 181 # Add spatial expression if possible
182 if self._select_radec: 182 if self._select_radec:
183 - expr += "ANGSEP("+str(self._ra)+","+str(self._dec)+ \  
184 - ",RA_PNT,DEC_PNT)<="+str(self._radius) 183 + expr += 'ANGSEP('+str(self._ra)+','+str(self._dec)+ \
  184 + ',RA_PNT,DEC_PNT)<='+str(self._radius)
185 185
186 # Add connector if expression is empty 186 # Add connector if expression is empty
187 if len(expr): 187 if len(expr):
188 - expr += "&&" 188 + expr += '&&'
189 189
190 # Add quality expression 190 # Add quality expression
191 - expr += "QUALITY<="+str(self["min_qual"].integer()) 191 + expr += 'QUALITY<='+str(self['min_qual'].integer())
192 192
193 # Add user expression 193 # Add user expression
194 - expression = self["expression"].string()  
195 - if (expression != "NONE" and expression != "INDEF" and 194 + expression = self['expression'].string()
  195 + if (expression != 'NONE' and expression != 'INDEF' and
196 len(expression) > 0): 196 len(expression) > 0):
197 - expr += "&&" + expression 197 + expr += '&&' + expression
198 198
199 # Initialise filename 199 # Initialise filename
200 - openfile = self._obs_index+"[OBS_INDEX]["+expr+"]" 200 + openfile = self._obs_index+'[OBS_INDEX]['+expr+']'
201 201
202 # Open file 202 # Open file
203 fits = gammalib.GFits(openfile) 203 fits = gammalib.GFits(openfile)
204 204
205 # Check for extistence of rows 205 # Check for extistence of rows
206 - if fits["OBS_INDEX"].nrows() > 0: 206 + if fits['OBS_INDEX'].nrows() > 0:
207 207
208 # Get HDU 208 # Get HDU
209 - obs_indx = fits["OBS_INDEX"] 209 + obs_indx = fits['OBS_INDEX']
210 210
211 # Read observation IDs into array 211 # Read observation IDs into array
212 for i in range(obs_indx.nrows()): 212 for i in range(obs_indx.nrows()):
213 - self._runs.append(obs_indx["OBS_ID"][i]) 213 + self._runs.append(obs_indx['OBS_ID'][i])
214 214
215 # Dump expression 215 # Dump expression
216 if self._logNormal(): 216 if self._logNormal():
217 - self._log("\n")  
218 - self._log.header3("Expression") 217 + self._log('\n')
  218 + self._log.header3('Expression')
219 self._log(expr) 219 self._log(expr)
220 - self._log("\n") 220 + self._log('\n')
221 221
222 # Dump number of observations 222 # Dump number of observations
223 if self._logTerse(): 223 if self._logTerse():
224 - self._log("\n")  
225 - self._log.header3("Observations")  
226 - self._log("Found "+str(len(self._runs))+" observations")  
227 - self._log("\n") 224 + self._log('\n')
  225 + self._log.header3('Observations')
  226 + self._log('Found '+str(len(self._runs))+' observations')
  227 + self._log('\n')
  228 +
  229 + # Dump number of observations
  230 + if self._logVerbose():
  231 + self._log('\n')
  232 + self._log.header3('Observation IDs')
  233 + for run in self._runs:
  234 + self._log(str(run)+'\n')
228 235
229 # Close file 236 # Close file
230 fits.close() 237 fits.close()
@@ -261,26 +268,26 @@ class csfindobs(ctools.cscript): @@ -261,26 +268,26 @@ class csfindobs(ctools.cscript):
261 self._log.header1('Save runlist') 268 self._log.header1('Save runlist')
262 269
263 # Get filename 270 # Get filename
264 - outfile = self["outfile"].filename() 271 + outfile = self['outfile'].filename()
265 272
266 # Check for clobber 273 # Check for clobber
267 if outfile.exists() and not self._clobber(): 274 if outfile.exists() and not self._clobber():
268 if self._logTerse(): 275 if self._logTerse():
269 - self._log('File "+outfile+" already exists. ') 276 + self._log('File '+outfile+' already exists. ')
270 self._log('Set clobber=yes to overwrite file.') 277 self._log('Set clobber=yes to overwrite file.')
271 self._log('\n') 278 self._log('\n')
272 else: 279 else:
273 280
274 # Log file name 281 # Log file name
275 if self._logTerse(): 282 if self._logTerse():
276 - self._log(gammalib.parformat("Runlist file")) 283 + self._log(gammalib.parformat('Runlist file'))
277 self._log(outfile.url()) 284 self._log(outfile.url())
278 - self._log("\n") 285 + self._log('\n')
279 286
280 # Write runs to file 287 # Write runs to file
281 - f = open(outfile.url(),"w") 288 + f = open(outfile.url(),'w')
282 for run in self._runs: 289 for run in self._runs:
283 - f.write(str(run)+" \n") 290 + f.write(str(run)+' \n')
284 f.close() 291 f.close()
285 292
286 # Return 293 # Return
cscripts/csiactcopy.py
  View file @c60f05b
@@ -43,13 +43,13 @@ class csiactcopy(ctools.cscript): @@ -43,13 +43,13 @@ class csiactcopy(ctools.cscript):
43 Constructor. 43 Constructor.
44 """ 44 """
45 # Set name 45 # Set name
46 - self._name = "csiactcopy"  
47 - self._version = "1.1.0"  
48 - self._datapath = os.getenv("VHEFITS","") 46 + self._name = 'csiactcopy'
  47 + self._version = '1.1.0'
  48 + self._datapath = os.getenv('VHEFITS','')
49 self._remote_master = gammalib.GFilename() 49 self._remote_master = gammalib.GFilename()
50 - self._remote_base = ""  
51 - self._prodname = ""  
52 - self._outpath = "" 50 + self._remote_base = ''
  51 + self._prodname = ''
  52 + self._outpath = ''
53 self._runlist = gammalib.GFilename() 53 self._runlist = gammalib.GFilename()
54 self._runs = [] 54 self._runs = []
55 55
@@ -68,7 +68,7 @@ class csiactcopy(ctools.cscript): @@ -68,7 +68,7 @@ class csiactcopy(ctools.cscript):
68 """ 68 """
69 69
70 # Get Parameters 70 # Get Parameters
71 - self._remote_master = self["remote_master"].filename() 71 + self._remote_master = self['remote_master'].filename()
72 if not self._remote_master.exists(): 72 if not self._remote_master.exists():
73 raise RuntimeError('*** ERROR: FITS data not available. No ' 73 raise RuntimeError('*** ERROR: FITS data not available. No '
74 'master index file found in "'+ 74 'master index file found in "'+
@@ -76,14 +76,14 @@ class csiactcopy(ctools.cscript): @@ -76,14 +76,14 @@ class csiactcopy(ctools.cscript):
76 'file system is properly mounted.') 76 'file system is properly mounted.')
77 77
78 # Get parameters 78 # Get parameters
79 - self._prodname = self["prodname"].string()  
80 - self._outpath = gammalib.expand_env(self["outpath"].string())  
81 - self._runlist = self["runlist"].filename() 79 + self._prodname = self['prodname'].string()
  80 + self._outpath = gammalib.expand_env(self['outpath'].string())
  81 + self._runlist = self['runlist'].filename()
82 82
83 # Write input parameters into logger 83 # Write input parameters into logger
84 if self._logTerse(): 84 if self._logTerse():
85 self._log_parameters() 85 self._log_parameters()
86 - self._log("\n") 86 + self._log('\n')
87 87
88 # Return 88 # Return
89 return 89 return
@@ -101,21 +101,21 @@ class csiactcopy(ctools.cscript): @@ -101,21 +101,21 @@ class csiactcopy(ctools.cscript):
101 101
102 # Logging 102 # Logging
103 if self._logVerbose(): 103 if self._logVerbose():
104 - self._log("\n")  
105 - self._log.header3("Copy file")  
106 - self._log.parformat("Source") 104 + self._log('\n')
  105 + self._log.header3('Copy file')
  106 + self._log.parformat('Source')
107 self._log(str(source)) 107 self._log(str(source))
108 - self._log("\n")  
109 - self._log.parformat("Destination") 108 + self._log('\n')
  109 + self._log.parformat('Destination')
110 self._log(str(destination)) 110 self._log(str(destination))
111 - self._log("\n")  
112 - self._log.parformat("Already exists") 111 + self._log('\n')
  112 + self._log.parformat('Already exists')
113 self._log(str(os.path.isfile(destination))) 113 self._log(str(os.path.isfile(destination)))
114 - self._log("\n")  
115 - self._log.parformat("Overwrite") 114 + self._log('\n')
  115 + self._log.parformat('Overwrite')
116 self._log(str(clobber)) 116 self._log(str(clobber))
117 - self._log("\n")  
118 - self._log.parformat("Copying") 117 + self._log('\n')
  118 + self._log.parformat('Copying')
119 119
120 # Flag if file destination is already available 120 # Flag if file destination is already available
121 is_file = os.path.isfile(destination) 121 is_file = os.path.isfile(destination)
@@ -123,14 +123,14 @@ class csiactcopy(ctools.cscript): @@ -123,14 +123,14 @@ class csiactcopy(ctools.cscript):
123 # check if file could be skipped because clobber=no 123 # check if file could be skipped because clobber=no
124 if is_file and clobber == False: 124 if is_file and clobber == False:
125 if self._logVerbose(): 125 if self._logVerbose():
126 - self._log("Skip (clobber=no)")  
127 - self._log("\n") 126 + self._log('Skip (clobber=no)')
  127 + self._log('\n')
128 128
129 # check if file could be skipped because it is the same file 129 # check if file could be skipped because it is the same file
130 elif is_file and os.path.samefile(destination, source): 130 elif is_file and os.path.samefile(destination, source):
131 if self._logVerbose(): 131 if self._logVerbose():
132 - self._log("Skip (same file)")  
133 - self._log("\n") 132 + self._log('Skip (same file)')
  133 + self._log('\n')
134 134
135 else: 135 else:
136 # Create directory if not existent 136 # Create directory if not existent
@@ -146,8 +146,8 @@ class csiactcopy(ctools.cscript): @@ -146,8 +146,8 @@ class csiactcopy(ctools.cscript):
146 146
147 # Logging 147 # Logging
148 if self._logVerbose(): 148 if self._logVerbose():
149 - self._log("Done!")  
150 - self._log("\n") 149 + self._log('Done!')
  150 + self._log('\n')
151 151
152 # Return 152 # Return
153 return filesize 153 return filesize
@@ -159,21 +159,21 @@ class csiactcopy(ctools.cscript): @@ -159,21 +159,21 @@ class csiactcopy(ctools.cscript):
159 """ 159 """
160 160
161 if self._logTerse(): 161 if self._logTerse():
162 - self._log.parformat("Local file") 162 + self._log.parformat('Local file')
163 self._log(str(localfits)) 163 self._log(str(localfits))
164 - self._log("\n")  
165 - self._log.parformat("Remote file") 164 + self._log('\n')
  165 + self._log.parformat('Remote file')
166 self._log(str(remotefits)) 166 self._log(str(remotefits))
167 - self._log("\n")  
168 - self._log.parformat("HDU name") 167 + self._log('\n')
  168 + self._log.parformat('HDU name')
169 self._log(str(hduname)) 169 self._log(str(hduname))
170 - self._log("\n")  
171 - self._log.parformat("Conflict preference") 170 + self._log('\n')
  171 + self._log.parformat('Conflict preference')
172 if clobber: 172 if clobber:
173 - self._log("Remote") 173 + self._log('Remote')
174 else: 174 else:
175 - self._log("Local")  
176 - self._log("\n") 175 + self._log('Local')
  176 + self._log('\n')
177 177
178 # Check if local fits file is available 178 # Check if local fits file is available
179 if os.path.isfile(localfits): 179 if os.path.isfile(localfits):
@@ -187,40 +187,70 @@ class csiactcopy(ctools.cscript): @@ -187,40 +187,70 @@ class csiactcopy(ctools.cscript):
187 187
188 # find local obs_id 188 # find local obs_id
189 local_obs = [] 189 local_obs = []
190 - lobs_col = local_hdu["OBS_ID"] 190 + lobs_col = local_hdu['OBS_ID']
191 for obs_id in lobs_col: 191 for obs_id in lobs_col:
192 local_obs.append(obs_id) 192 local_obs.append(obs_id)
193 -  
194 - # Create selection expression for opening remote fits file  
195 - selection = ""  
196 - if len(self._runs):  
197 - selection += "["+hduname+"]["  
198 - for run in self._runs:  
199 - selection += "OBS_ID=="+str(run)  
200 - selection += "||"  
201 - selection = selection[:-2]  
202 - selection += "]"  
203 -  
204 - # Create file name  
205 - remotefile = remotefits + selection  
206 -  
207 - # open remote fits file  
208 - remote = gammalib.GFits(str(remotefile))  
209 - remote_hdu = remote[hduname]  
210 - 193 +
  194 + # Initialise list of runlists
  195 + # This is kluge: to circumvent a too long file name, we split the
  196 + # runlists into chunks of 50 observations maximum
  197 + runlists = []
  198 + runs_per_chunk = 50
  199 + for i in range(len(self._runs) / runs_per_chunk + 1):
  200 + chunk = self._runs[i * runs_per_chunk : (i + 1) * runs_per_chunk]
  201 + runlists.append(chunk)
  202 +
  203 + # Loop over runlist chunks
  204 + for i, runlist in enumerate(runlists):
  205 +
  206 + # Create selection expression for opening remote fits file
  207 + selection = ''
  208 + if len(runlist):
  209 + selection += '['+hduname+']['
  210 + for run in runlist:
  211 + selection += 'OBS_ID=='+str(run)
  212 + selection += '||'
  213 + selection = selection[:-2]
  214 + selection += ']'
  215 +
  216 + # Create file name
  217 + remotefile = remotefits + selection
  218 +
  219 + # open remote fits file
  220 + remote = gammalib.GFits(str(remotefile))
  221 +
  222 + # If first iteration, create clone of FITS column for later use
  223 + if i == 0:
  224 + remote_hdu = remote[hduname].clone()
  225 + else:
  226 +
  227 + # get temporary hdu to be added to current HDU
  228 + tmp_hdu = remote[hduname]
  229 +
  230 + # Get old size of remote HDU
  231 + size = remote_hdu.size()
  232 +
  233 + # Extend remote HDU
  234 + remote_hdu.append_rows(tmp_hdu.nrows())
  235 +
  236 + # Loop over rows and columns to copy over values exteding remote HDU
  237 + for col in tmp_hdu:
  238 + for row in range(col.length()):
  239 + remote_hdu[col.name()][size + row] = col[row]
  240 +
211 # Get remote obs_id 241 # Get remote obs_id
212 remote_obs = [] 242 remote_obs = []
213 - robs_col = remote_hdu["OBS_ID"] 243 + robs_col = remote_hdu['OBS_ID']
214 for obs_id in robs_col: 244 for obs_id in robs_col:
215 remote_obs.append(obs_id) 245 remote_obs.append(obs_id)
216 246
217 if self._logTerse(): 247 if self._logTerse():
218 - self._log.parformat("Remote entries") 248 + self._log.parformat('Remote entries')
219 self._log(str(len(remote_obs))) 249 self._log(str(len(remote_obs)))
220 - self._log("\n")  
221 - self._log.parformat("Local entries") 250 + self._log('\n')
  251 + self._log.parformat('Local entries')
222 self._log(str(len(local_obs))) 252 self._log(str(len(local_obs)))
223 - self._log("\n") 253 + self._log('\n')
224 254
225 # initialise counter for logging 255 # initialise counter for logging
226 removed_rows = 0 256 removed_rows = 0
@@ -239,13 +269,13 @@ class csiactcopy(ctools.cscript): @@ -239,13 +269,13 @@ class csiactcopy(ctools.cscript):
239 table_has_obsid = True 269 table_has_obsid = True
240 while table_has_obsid: 270 while table_has_obsid:
241 for i in range(local_hdu.nrows()): 271 for i in range(local_hdu.nrows()):
242 - if remote_obs_id == local_hdu["OBS_ID"][i]: 272 + if remote_obs_id == local_hdu['OBS_ID'][i]:
243 local_hdu.remove_rows(i,1) 273 local_hdu.remove_rows(i,1)
244 removed_rows += 1 274 removed_rows += 1
245 break 275 break
246 table_has_obsid = False 276 table_has_obsid = False
247 for i in range(local_hdu.nrows()): 277 for i in range(local_hdu.nrows()):
248 - if remote_obs_id == local_hdu["OBS_ID"][i]: 278 + if remote_obs_id == local_hdu['OBS_ID'][i]:
249 table_has_obsid = True 279 table_has_obsid = True
250 break 280 break
251 281
@@ -262,13 +292,13 @@ class csiactcopy(ctools.cscript): @@ -262,13 +292,13 @@ class csiactcopy(ctools.cscript):
262 table_has_obsid = True 292 table_has_obsid = True
263 while table_has_obsid: 293 while table_has_obsid:
264 for i in range(remote_hdu.nrows()): 294 for i in range(remote_hdu.nrows()):
265 - if local_obs_id == remote_hdu["OBS_ID"][i]: 295 + if local_obs_id == remote_hdu['OBS_ID'][i]:
266 remote_hdu.remove_rows(i,1) 296 remote_hdu.remove_rows(i,1)
267 removed_rows += 1 297 removed_rows += 1
268 break 298 break
269 table_has_obsid = False 299 table_has_obsid = False
270 for i in range(remote_hdu.nrows()): 300 for i in range(remote_hdu.nrows()):
271 - if local_obs_id == remote_hdu["OBS_ID"][i]: 301 + if local_obs_id == remote_hdu['OBS_ID'][i]:
272 table_has_obsid = True 302 table_has_obsid = True
273 break 303 break
274 304
@@ -280,11 +310,11 @@ class csiactcopy(ctools.cscript): @@ -280,11 +310,11 @@ class csiactcopy(ctools.cscript):
280 310
281 if self._logTerse(): 311 if self._logTerse():
282 if clobber: 312 if clobber:
283 - self._log.parformat("Removed local rows") 313 + self._log.parformat('Removed local rows')
284 else: 314 else:
285 - self._log.parformat("Skipped remote rows") 315 + self._log.parformat('Skipped remote rows')
286 self._log(str(removed_rows)) 316 self._log(str(removed_rows))
287 - self._log("\n") 317 + self._log('\n')
288 318
289 # Loop over columns 319 # Loop over columns
290 for i in range(local_hdu.ncols()): 320 for i in range(local_hdu.ncols()):
@@ -322,8 +352,8 @@ class csiactcopy(ctools.cscript): @@ -322,8 +352,8 @@ class csiactcopy(ctools.cscript):
322 352
323 # Log output 353 # Log output
324 if self._logTerse(): 354 if self._logTerse():
325 - self._log("\n")  
326 - self._log.header2("Runlist information") 355 + self._log('\n')
  356 + self._log.header2('Runlist information')
327 357
328 # Initialise runlist array 358 # Initialise runlist array
329 self._runs = [] 359 self._runs = []
@@ -342,15 +372,15 @@ class csiactcopy(ctools.cscript): @@ -342,15 +372,15 @@ class csiactcopy(ctools.cscript):
342 372
343 # Logging 373 # Logging
344 if self._logTerse(): 374 if self._logTerse():
345 - self._log.parformat("Number of observations") 375 + self._log.parformat('Number of observations')
346 self._log(str(len(self._runs))) 376 self._log(str(len(self._runs)))
347 - self._log("\n") 377 + self._log('\n')
348 378
349 - # Copy all data if runlist file is "NONE"  
350 - elif self._runlist == "NONE": 379 + # Copy all data if runlist file is 'NONE'
  380 + elif self._runlist == 'NONE':
351 if self._logTerse(): 381 if self._logTerse():
352 - self._log("Copy all available data")  
353 - self._log("\n") 382 + self._log('Copy all available data')
  383 + self._log('\n')
354 384
355 # Raise exception if file not valid 385 # Raise exception if file not valid
356 else: 386 else:
@@ -365,13 +395,13 @@ class csiactcopy(ctools.cscript): @@ -365,13 +395,13 @@ class csiactcopy(ctools.cscript):
365 # Retrieve json data from remote master 395 # Retrieve json data from remote master
366 json_data = open(self._remote_master.url()).read() 396 json_data = open(self._remote_master.url()).read()
367 data = json.loads(json_data) 397 data = json.loads(json_data)
368 - if not "datasets" in data: 398 + if not 'datasets' in data:
369 raise RuntimeError('*** ERROR: Key "datasets" not available ' 399 raise RuntimeError('*** ERROR: Key "datasets" not available '
370 'in remote master index file "'+ 400 'in remote master index file "'+
371 self._remote_master+'".') 401 self._remote_master+'".')
372 402
373 # Get array of configurations 403 # Get array of configurations
374 - configs = data["datasets"] 404 + configs = data['datasets']
375 405
376 # Get remote paths 406 # Get remote paths
377 self._remote_base = self._remote_master.path() 407 self._remote_base = self._remote_master.path()
@@ -384,25 +414,25 @@ class csiactcopy(ctools.cscript): @@ -384,25 +414,25 @@ class csiactcopy(ctools.cscript):
384 414
385 # Logging 415 # Logging
386 if self._logTerse(): 416 if self._logTerse():
387 - self._log("\n")  
388 - self._log.header2("Loop over remote configs") 417 + self._log('\n')
  418 + self._log.header2('Loop over remote configs')
389 419
390 # Loop over configs 420 # Loop over configs
391 for config in configs: 421 for config in configs:
392 422
393 # Logging 423 # Logging
394 if self._logVerbose(): 424 if self._logVerbose():
395 - self._log.header2(str(config["name"])) 425 + self._log.header2(str(config['name']))
396 426
397 # Check if prodname was found 427 # Check if prodname was found
398 - if config["name"] == self._prodname: 428 + if config['name'] == self._prodname:
399 429
400 # Inidicate that prodname has been found 430 # Inidicate that prodname has been found
401 has_prod = True 431 has_prod = True
402 432
403 # Build path of index files 433 # Build path of index files
404 - remote_hdu = os.path.join(self._remote_base, config["hduindx"])  
405 - remote_obs = os.path.join(self._remote_base, config["obsindx"]) 434 + remote_hdu = os.path.join(self._remote_base, config['hduindx'])
  435 + remote_obs = os.path.join(self._remote_base, config['obsindx'])
406 436
407 # Log information 437 # Log information
408 if self._logTerse(): 438 if self._logTerse():
@@ -416,10 +446,10 @@ class csiactcopy(ctools.cscript): @@ -416,10 +446,10 @@ class csiactcopy(ctools.cscript):
416 446
417 # Open remote HDU index file 447 # Open remote HDU index file
418 fits = gammalib.GFits(str(remote_hdu)) 448 fits = gammalib.GFits(str(remote_hdu))
419 - table = fits["HDU_INDEX"] 449 + table = fits['HDU_INDEX']
420 450
421 # Initialise flag if SIZE column is present 451 # Initialise flag if SIZE column is present
422 - has_size = table.contains("SIZE") 452 + has_size = table.contains('SIZE')
423 453
424 if has_size: 454 if has_size:
425 # Initialise file size 455 # Initialise file size
@@ -428,7 +458,7 @@ class csiactcopy(ctools.cscript): @@ -428,7 +458,7 @@ class csiactcopy(ctools.cscript):
428 # Initialise remote observation IDs 458 # Initialise remote observation IDs
429 remote_ids = set() 459 remote_ids = set()
430 for row in range(table.nrows()): 460 for row in range(table.nrows()):
431 - remote_ids.add(table["OBS_ID"][row]) 461 + remote_ids.add(table['OBS_ID'][row])
432 462
433 # Log runs that are not available remotely 463 # Log runs that are not available remotely
434 for run in self._runs: 464 for run in self._runs:
@@ -442,12 +472,12 @@ class csiactcopy(ctools.cscript): @@ -442,12 +472,12 @@ class csiactcopy(ctools.cscript):
442 for row in range(table.nrows()): 472 for row in range(table.nrows()):
443 473
444 # Get observation ID 474 # Get observation ID
445 - obs_id = table["OBS_ID"][row]  
446 - file_dir = table["FILE_DIR"][row]  
447 - file_name = table["FILE_NAME"][row] 475 + obs_id = table['OBS_ID'][row]
  476 + file_dir = table['FILE_DIR'][row]
  477 + file_name = table['FILE_NAME'][row]
448 478
449 # Skip if filename is empty 479 # Skip if filename is empty
450 - if file_name == "": 480 + if file_name == '':
451 continue 481 continue
452 482
453 # Check if we need to consider an input runlist 483 # Check if we need to consider an input runlist
@@ -467,7 +497,7 @@ class csiactcopy(ctools.cscript): @@ -467,7 +497,7 @@ class csiactcopy(ctools.cscript):
467 497
468 # Add file size 498 # Add file size
469 if has_size and newlen > oldlen: 499 if has_size and newlen > oldlen:
470 - cp_size += table["SIZE"][row] 500 + cp_size += table['SIZE'][row]
471 501
472 # Otherwise add every file 502 # Otherwise add every file
473 else: 503 else:
@@ -482,24 +512,24 @@ class csiactcopy(ctools.cscript): @@ -482,24 +512,24 @@ class csiactcopy(ctools.cscript):
482 512
483 # Add file size 513 # Add file size
484 if has_size and newlen > oldlen: 514 if has_size and newlen > oldlen:
485 - cp_size += table["SIZE"][row] 515 + cp_size += table['SIZE'][row]
486 516
487 # Log file information 517 # Log file information
488 if self._logTerse(): 518 if self._logTerse():
489 - self._log("\n")  
490 - self._log.header2("File information")  
491 - self._log.parformat("Number of files") 519 + self._log('\n')
  520 + self._log.header2('File information')
  521 + self._log.parformat('Number of files')
492 self._log(str(len(files))) 522 self._log(str(len(files)))
493 - self._log("\n") 523 + self._log('\n')
494 if has_size: 524 if has_size:
495 - self._log.parformat("Size")  
496 - self._log("%.2f"%(float(cp_size)*1e-6)+" MB")  
497 - self._log("\n") 525 + self._log.parformat('Size')
  526 + self._log('%.2f'%(float(cp_size)*1e-6)+' MB')
  527 + self._log('\n')
498 if self._logVerbose(): 528 if self._logVerbose():
499 - self._log("\n")  
500 - self._log.header3("File names") 529 + self._log('\n')
  530 + self._log.header3('File names')
501 for filename in files: 531 for filename in files:
502 - self._log(str(filename)+"\n") 532 + self._log(str(filename)+'\n')
503 533
504 # Close HDU index file 534 # Close HDU index file
505 fits.close() 535 fits.close()
@@ -507,23 +537,23 @@ class csiactcopy(ctools.cscript): @@ -507,23 +537,23 @@ class csiactcopy(ctools.cscript):
507 # If prodname is not found just log that we skip the config 537 # If prodname is not found just log that we skip the config
508 else: 538 else:
509 if self._logExplicit(): 539 if self._logExplicit():
510 - self._log("\n") 540 + self._log('\n')
511 self._log.header3('Skipping config "'+ 541 self._log.header3('Skipping config "'+
512 - str(config["name"])+'"')  
513 - self._log("\n") 542 + str(config['name'])+'"')
  543 + self._log('\n')
514 544
515 # Raise Exception if prodname was not found 545 # Raise Exception if prodname was not found
516 if not has_prod: 546 if not has_prod:
517 - msg = '*** ERROR: FITS production "'+self._prodname+\  
518 - '" not available. Available productions are:\n' 547 + msg = '*** ERROR: FITS production "'+self._prodname+'" not '
  548 + msg += 'available. Available productions are:\n'
519 for config in configs: 549 for config in configs:
520 - msg += " - "+config["name"]+"\n" 550 + msg += ' - '+config['name']+'\n'
521 raise RuntimeError(msg) 551 raise RuntimeError(msg)
522 552
523 # Logging 553 # Logging
524 if self._logNormal(): 554 if self._logNormal():
525 - self._log("\n")  
526 - self._log.header1("Copying files") 555 + self._log('\n')
  556 + self._log.header1('Copying files')
527 557
528 # intialise counter 558 # intialise counter
529 k = 0 559 k = 0
@@ -549,9 +579,9 @@ class csiactcopy(ctools.cscript): @@ -549,9 +579,9 @@ class csiactcopy(ctools.cscript):
549 fraction = float(k) / float(len(files)) * 100.0 579 fraction = float(k) / float(len(files)) * 100.0
550 while fraction > last_fraction: 580 while fraction > last_fraction:
551 if self._logNormal() and not self._logVerbose(): 581 if self._logNormal() and not self._logVerbose():
552 - self._log.parformat("Status")  
553 - self._log(str(int(last_fraction))+"%")  
554 - self._log("\n") 582 + self._log.parformat('Status')
  583 + self._log(str(int(last_fraction))+'%')
  584 + self._log('\n')
555 last_fraction += fraction_increment 585 last_fraction += fraction_increment
556 586
557 # Copy file 587 # Copy file
@@ -566,12 +596,12 @@ class csiactcopy(ctools.cscript): @@ -566,12 +596,12 @@ class csiactcopy(ctools.cscript):
566 596
567 # Logging 597 # Logging
568 if self._logNormal() and not self._logVerbose(): 598 if self._logNormal() and not self._logVerbose():
569 - self._log.parformat("Status")  
570 - self._log("Finished")  
571 - self._log("\n") 599 + self._log.parformat('Status')
  600 + self._log('Finished')
  601 + self._log('\n')
572 if self._logTerse(): 602 if self._logTerse():
573 - self._log("\n")  
574 - self._log.header1("Updating index files") 603 + self._log('\n')
  604 + self._log.header1('Updating index files')
575 605
576 # Build local index file names 606 # Build local index file names
577 local_hdu = os.path.join(self._outpath, 607 local_hdu = os.path.join(self._outpath,
@@ -586,18 +616,18 @@ class csiactcopy(ctools.cscript): @@ -586,18 +616,18 @@ class csiactcopy(ctools.cscript):
586 616
587 # Logging 617 # Logging
588 if self._logTerse(): 618 if self._logTerse():
589 - self._log("\n")  
590 - self._log.header3("HDU index") 619 + self._log('\n')
  620 + self._log.header3('HDU index')
591 621
592 # Merge remote index files with local files 622 # Merge remote index files with local files
593 - self._merge(local_hdu, remote_hdu, "HDU_INDEX", self._clobber()) 623 + self._merge(local_hdu, remote_hdu, 'HDU_INDEX', self._clobber())
594 624
595 if self._logTerse(): 625 if self._logTerse():
596 - self._log("\n")  
597 - self._log.header3("OBS index") 626 + self._log('\n')
  627 + self._log.header3('OBS index')
598 628
599 # Merge remote index files with local files 629 # Merge remote index files with local files
600 - self._merge(local_obs, remote_obs, "OBS_INDEX", self._clobber()) 630 + self._merge(local_obs, remote_obs, 'OBS_INDEX', self._clobber())
601 631
602 else: 632 else:
603 # If all files were copied, just copy index files too 633 # If all files were copied, just copy index files too
@@ -605,11 +635,11 @@ class csiactcopy(ctools.cscript): @@ -605,11 +635,11 @@ class csiactcopy(ctools.cscript):
605 self._copy(remote_obs, self._clobber()) 635 self._copy(remote_obs, self._clobber())
606 636
607 if self._logTerse(): 637 if self._logTerse():
608 - self._log("\n")  
609 - self._log.header3("Master index file") 638 + self._log('\n')
  639 + self._log.header3('Master index file')
610 640
611 # Adding prodname to local master 641 # Adding prodname to local master
612 - localmaster = os.path.join(self._outpath, "master.json") 642 + localmaster = os.path.join(self._outpath, 'master.json')
613 643
614 # If no local master is found, copy master over first 644 # If no local master is found, copy master over first
615 if not os.path.isfile(localmaster): 645 if not os.path.isfile(localmaster):
@@ -618,7 +648,7 @@ class csiactcopy(ctools.cscript): @@ -618,7 +648,7 @@ class csiactcopy(ctools.cscript):
618 # Load local master 648 # Load local master
619 json_data = open(localmaster).read() 649 json_data = open(localmaster).read()
620 data = json.loads(json_data) 650 data = json.loads(json_data)
621 - configs = data["datasets"] 651 + configs = data['datasets']
622 652
623 # Initialise flag indicating if we already have prodname in master 653 # Initialise flag indicating if we already have prodname in master
624 # file 654 # file
@@ -631,66 +661,66 @@ class csiactcopy(ctools.cscript): @@ -631,66 +661,66 @@ class csiactcopy(ctools.cscript):
631 for config in configs: 661 for config in configs:
632 662
633 # Get hdu and obs index files 663 # Get hdu and obs index files
634 - hdu = os.path.join(self._outpath, config["hduindx"])  
635 - obs = os.path.join(self._outpath, config["obsindx"]) 664 + hdu = os.path.join(self._outpath, config['hduindx'])
  665 + obs = os.path.join(self._outpath, config['obsindx'])
636 666
637 # Check if index files are available 667 # Check if index files are available
638 if not (gammalib.GFilename(str(hdu)).is_fits() and 668 if not (gammalib.GFilename(str(hdu)).is_fits() and
639 gammalib.GFilename(str(obs)).is_fits()): 669 gammalib.GFilename(str(obs)).is_fits()):
640 if self._logTerse(): 670 if self._logTerse():
641 - self._log('Removing "'+str(config["name"])+'" ') 671 + self._log('Removing "'+str(config['name'])+'" ')
642 self._log('(not available)') 672 self._log('(not available)')
643 self._log('\n') 673 self._log('\n')
644 else: 674 else:
645 # Append config if valid 675 # Append config if valid
646 newconfigs.append(config) 676 newconfigs.append(config)
647 if self._logTerse(): 677 if self._logTerse():
648 - self._log('Keeping "'+str(config["name"])+'"') 678 + self._log('Keeping "'+str(config['name'])+'"')
649 self._log('\n') 679 self._log('\n')
650 680
651 # Signals that downloaded config is available 681 # Signals that downloaded config is available
652 - if config["name"] == self._prodname: 682 + if config['name'] == self._prodname:
653 has_config = True 683 has_config = True
654 684
655 # Create new entry if config was not available 685 # Create new entry if config was not available
656 if not has_config: 686 if not has_config:
657 - newdict = dict.fromkeys(["name","hduindx","obsindx"])  
658 - newdict["name"] = self._prodname  
659 - newdict["hduindx"] = os.path.relpath(local_hdu, self._outpath)  
660 - newdict["obsindx"] = os.path.relpath(local_obs, self._outpath) 687 + newdict = dict.fromkeys(['name','hduindx','obsindx'])
  688 + newdict['name'] = self._prodname
  689 + newdict['hduindx'] = os.path.relpath(local_hdu, self._outpath)
  690 + newdict['obsindx'] = os.path.relpath(local_obs, self._outpath)
661 newconfigs.append(newdict) 691 newconfigs.append(newdict)
662 if self._logTerse(): 692 if self._logTerse():
663 - self._log('Adding "'+str(newdict["name"])+'"') 693 + self._log('Adding "'+str(newdict['name'])+'"')
664 self._log('\n') 694 self._log('\n')
665 695
666 # Write new json master file 696 # Write new json master file
667 - f = open(localmaster, "w")  
668 - data["datasets"] = newconfigs 697 + f = open(localmaster, 'w')
  698 + data['datasets'] = newconfigs
669 json.dump(data, f, indent=2) 699 json.dump(data, f, indent=2)
670 f.close() 700 f.close()
671 701
672 # Log summary 702 # Log summary
673 if self._logNormal(): 703 if self._logNormal():
674 - self._log("\n")  
675 - self._log.header1("Summary")  
676 - self._log.parformat("Data files found") 704 + self._log('\n')
  705 + self._log.header1('Summary')
  706 + self._log.parformat('Data files found')
677 self._log(str(len(files))) 707 self._log(str(len(files)))
678 - self._log("\n")  
679 - self._log.parformat("Data files copied") 708 + self._log('\n')
  709 + self._log.parformat('Data files copied')
680 self._log(str(n_copied)) 710 self._log(str(n_copied))
681 - self._log("\n")  
682 - self._log.parformat("Size")  
683 - self._log("%.2f"%(float(total_size)*1e-6)+" MB")  
684 - self._log("\n")  
685 - self._log.parformat("Local configs") 711 + self._log('\n')
  712 + self._log.parformat('Size')
  713 + self._log('%.2f'%(float(total_size)*1e-6)+' MB')
  714 + self._log('\n')
  715 + self._log.parformat('Local configs')
686 self._log(str(len(newconfigs))) 716 self._log(str(len(newconfigs)))
687 - self._log("\n") 717 + self._log('\n')
688 if self._logTerse(): 718 if self._logTerse():
689 - self._log("\n")  
690 - self._log.header3("Content of master index") 719 + self._log('\n')
  720 + self._log.header3('Content of master index')
691 for config in newconfigs: 721 for config in newconfigs:
692 - self._log(str(config["name"]))  
693 - self._log("\n") 722 + self._log(str(config['name']))
  723 + self._log('\n')
694 724
695 # Return 725 # Return
696 return 726 return
cscripts/csiactobs.par
  View file @c60f05b
@@ -18,7 +18,7 @@ @@ -18,7 +18,7 @@
18 # Input and output parameters 18 # Input and output parameters
19 #============================ 19 #============================
20 datapath, s, a, $VHEFITS,,, "Path were data is located" 20 datapath, s, a, $VHEFITS,,, "Path were data is located"
21 -prodname, s, a, ,,, "Data storage name" 21 +prodname, s, a, fits-prod-name,,, "Data storage name"
22 infile, f, a, runlist.lis,,, "Input runlist file" 22 infile, f, a, runlist.lis,,, "Input runlist file"
23 inmodel, f, h, NONE,,, "Input model definition XML file (optional)" 23 inmodel, f, h, NONE,,, "Input model definition XML file (optional)"
24 master_indx, s, h, master.json,,, "Input master index file" 24 master_indx, s, h, master.json,,, "Input master index file"
cscripts/csiactobs.py
  View file @c60f05b
@@ -46,14 +46,14 @@ class csiactobs(ctools.cscript): @@ -46,14 +46,14 @@ class csiactobs(ctools.cscript):
46 Constructor. 46 Constructor.
47 """ 47 """
48 # Set name and version 48 # Set name and version
49 - self._name = "csiactobs"  
50 - self._version = "1.1.0" 49 + self._name = 'csiactobs'
  50 + self._version = '1.1.0'
51 51
52 # Initialise some members 52 # Initialise some members
53 self._ebounds = gammalib.GEbounds() 53 self._ebounds = gammalib.GEbounds()
54 - self._datapath = os.getenv("VHEFITS","") 54 + self._datapath = os.getenv('VHEFITS','')
55 self._inmodels = gammalib.GModels() 55 self._inmodels = gammalib.GModels()
56 - self._prodname = "" 56 + self._prodname = ''
57 self._xml = gammalib.GXml() 57 self._xml = gammalib.GXml()
58 self._models = gammalib.GModels() 58 self._models = gammalib.GModels()
59 self._runlist = [] 59 self._runlist = []
@@ -61,23 +61,23 @@ class csiactobs(ctools.cscript): @@ -61,23 +61,23 @@ class csiactobs(ctools.cscript):
61 self._bkgpars = 0 61 self._bkgpars = 0
62 self._outmodel = gammalib.GFilename() 62 self._outmodel = gammalib.GFilename()
63 self._outobs = gammalib.GFilename() 63 self._outobs = gammalib.GFilename()
64 - self._master_indx = "" 64 + self._master_indx = ''
65 self._use_bkg_scale = False 65 self._use_bkg_scale = False
66 - self._ev_hiera = [""]  
67 - self._aeff_hiera = [""]  
68 - self._psf_hiera = [""]  
69 - self._bkg_hiera = [""]  
70 - self._edisp_hiera = [""]  
71 - self._bkg_mod_hiera = [""] 66 + self._ev_hiera = ['']
  67 + self._aeff_hiera = ['']
  68 + self._psf_hiera = ['']
  69 + self._bkg_hiera = ['']
  70 + self._edisp_hiera = ['']
  71 + self._bkg_mod_hiera = ['']
72 self._bkg_gauss_norm = 1.0 72 self._bkg_gauss_norm = 1.0
73 self._bkg_gauss_index = 0.0 73 self._bkg_gauss_index = 0.0
74 self._bkg_gauss_sigma = 1.0 74 self._bkg_gauss_sigma = 1.0
75 self._bkg_aeff_index = 0.0 75 self._bkg_aeff_index = 0.0
76 self._bkg_aeff_norm = 1.0 76 self._bkg_aeff_norm = 1.0
77 self._bkg_range_factor = 1.0 77 self._bkg_range_factor = 1.0
78 - self._hdu_index = ""  
79 - self._obs_index = ""  
80 - self._subdir = "" 78 + self._hdu_index = ''
  79 + self._obs_index = ''
  80 + self._subdir = ''
81 self._debug = False 81 self._debug = False
82 82
83 # Initialise application by calling the appropriate class 83 # Initialise application by calling the appropriate class
@@ -94,24 +94,24 @@ class csiactobs(ctools.cscript): @@ -94,24 +94,24 @@ class csiactobs(ctools.cscript):
94 Get parameters from parfile and setup the observation. 94 Get parameters from parfile and setup the observation.
95 """ 95 """
96 # Set data path 96 # Set data path
97 - if self._datapath == "":  
98 - self._datapath = self["datapath"].string() 97 + if self._datapath == '':
  98 + self._datapath = self['datapath'].string()
99 99
100 # Check for input models 100 # Check for input models
101 - if not self["inmodel"].filename() == "NONE":  
102 - self._inmodels = gammalib.GModels(self["inmodel"].filename()) 101 + if not self['inmodel'].filename() == 'NONE':
  102 + self._inmodels = gammalib.GModels(self['inmodel'].filename())
103 103
104 # Expand environment 104 # Expand environment
105 self._datapath = gammalib.expand_env(self._datapath) 105 self._datapath = gammalib.expand_env(self._datapath)
106 106
107 # Read FITS production 107 # Read FITS production
108 - self._prodname = self["prodname"].string() 108 + self._prodname = self['prodname'].string()
109 109
110 # Read runlist file if list not already filled 110 # Read runlist file if list not already filled
111 if len(self._runlist) == 0: 111 if len(self._runlist) == 0:
112 112
113 # Get file name 113 # Get file name
114 - self._runlistfile = self["infile"].filename() 114 + self._runlistfile = self['infile'].filename()
115 115
116 # Read runlist from file 116 # Read runlist from file
117 runfile = open(self._runlistfile.url()) 117 runfile = open(self._runlistfile.url())
@@ -125,73 +125,73 @@ class csiactobs(ctools.cscript): @@ -125,73 +125,73 @@ class csiactobs(ctools.cscript):
125 runfile.close() 125 runfile.close()
126 126
127 # Read number of background parameters 127 # Read number of background parameters
128 - self._bkgpars = self["bkgpars"].integer() 128 + self._bkgpars = self['bkgpars'].integer()
129 129
130 # Output model file 130 # Output model file
131 - self._outmodel = self["outmodel"].filename() 131 + self._outmodel = self['outmodel'].filename()
132 132
133 # Observation outfile 133 # Observation outfile
134 - self._outobs = self["outobs"].filename() 134 + self._outobs = self['outobs'].filename()
135 135
136 # Master index file name 136 # Master index file name
137 - self._master_indx = self["master_indx"].string() 137 + self._master_indx = self['master_indx'].string()
138 138
139 # Read flag for background scaling factor 139 # Read flag for background scaling factor
140 - self._use_bkg_scale = self["bkg_scale"].boolean() 140 + self._use_bkg_scale = self['bkg_scale'].boolean()
141 141
142 # Read hierarchy of file loading 142 # Read hierarchy of file loading
143 - self._ev_hiera = self["ev_hiera"].string().split("|")  
144 - self._aeff_hiera = self["aeff_hiera"].string().split("|")  
145 - self._psf_hiera = self["psf_hiera"].string().split("|")  
146 - self._bkg_hiera = self["bkg_hiera"].string().split("|")  
147 - self._edisp_hiera = self["edisp_hiera"].string().split("|")  
148 - self._bkg_mod_hiera = self["bkg_mod_hiera"].string().split("|") 143 + self._ev_hiera = self['ev_hiera'].string().split('|')
  144 + self._aeff_hiera = self['aeff_hiera'].string().split('|')
  145 + self._psf_hiera = self['psf_hiera'].string().split('|')
  146 + self._bkg_hiera = self['bkg_hiera'].string().split('|')
  147 + self._edisp_hiera = self['edisp_hiera'].string().split('|')
  148 + self._bkg_mod_hiera = self['bkg_mod_hiera'].string().split('|')
149 149
150 # Read hidden background parameters 150 # Read hidden background parameters
151 - self._bkg_gauss_norm = self["bkg_gauss_norm"].real()  
152 - self._bkg_gauss_index = self["bkg_gauss_index"].real()  
153 - self._bkg_gauss_sigma = self["bkg_gauss_sigma"].real()  
154 - self._bkg_aeff_index = self["bkg_aeff_index"].real()  
155 - self._bkg_aeff_norm = self["bkg_aeff_norm"].real()  
156 - self._bkg_range_factor = self["bkg_range_factor"].real() 151 + self._bkg_gauss_norm = self['bkg_gauss_norm'].real()
  152 + self._bkg_gauss_index = self['bkg_gauss_index'].real()
  153 + self._bkg_gauss_sigma = self['bkg_gauss_sigma'].real()
  154 + self._bkg_aeff_index = self['bkg_aeff_index'].real()
  155 + self._bkg_aeff_norm = self['bkg_aeff_norm'].real()
  156 + self._bkg_range_factor = self['bkg_range_factor'].real()
157 157
158 # Open master index file and look for prodname 158 # Open master index file and look for prodname
159 master_file = os.path.join(self._datapath, self._master_indx) 159 master_file = os.path.join(self._datapath, self._master_indx)
160 if not os.path.isfile(master_file): 160 if not os.path.isfile(master_file):
161 - raise RuntimeError("FITS data store not available. No master index file found. Make sure the file is copied from the server and your datapath is set correctly.") 161 + raise RuntimeError('FITS data store not available. No master index file found. Make sure the file is copied from the server and your datapath is set correctly.')
162 162
163 # Open and load JSON file 163 # Open and load JSON file
164 json_data = open(master_file).read() 164 json_data = open(master_file).read()
165 data = json.loads(json_data) 165 data = json.loads(json_data)
166 - if not "datasets" in data:  
167 - raise RuntimeError("Key \"datasets\" not available in master index file.") 166 + if not 'datasets' in data:
  167 + raise RuntimeError('Key "datasets" not available in master index file.')
168 168
169 # Get configurations 169 # Get configurations
170 - configs = data["datasets"] 170 + configs = data['datasets']
171 171
172 # Initialise HDUs 172 # Initialise HDUs
173 - self._hdu_index = self._obs_index = "" 173 + self._hdu_index = self._obs_index = ''
174 174
175 # Get HDUs 175 # Get HDUs
176 for config in configs: 176 for config in configs:
177 - if self._prodname == config["name"]:  
178 - self._hdu_index = str(os.path.join(self._datapath, config["hduindx"]))  
179 - self._obs_index = str(os.path.join(self._datapath, config["obsindx"])) 177 + if self._prodname == config['name']:
  178 + self._hdu_index = str(os.path.join(self._datapath, config['hduindx']))
  179 + self._obs_index = str(os.path.join(self._datapath, config['obsindx']))
180 break 180 break
181 181
182 # Check HDUs 182 # Check HDUs
183 - if self._hdu_index == "" or self._obs_index == "":  
184 - raise RuntimeError("*** ERROR: FITS data store \""+self._prodname+"\" not available. Run csiactdata to get a list of available storage names")  
185 - filename = gammalib.GFilename(self._hdu_index+"[HDU_INDEX]") 183 + if self._hdu_index == '' or self._obs_index == '':
  184 + raise RuntimeError('*** ERROR: FITS data store "'+self._prodname+'" not available. Run csiactdata to get a list of available storage names')
  185 + filename = gammalib.GFilename(self._hdu_index+'[HDU_INDEX]')
186 if not filename.is_fits(): 186 if not filename.is_fits():
187 - raise RuntimeError("*** ERROR: HDU index file \""+self._hdu_index+"[HDU_INDEX]\" for FITS data store \""+self._prodname+"\" not available. Check your master index file or run csiactdata to get a list of available storage names.") 187 + raise RuntimeError('*** ERROR: HDU index file "'+self._hdu_index+'[HDU_INDEX]" for FITS data store "'+self._prodname+'" not available. Check your master index file or run csiactdata to get a list of available storage names.')
188 188
189 - # Check for existence of "BKG_SCALE" in the observation index file if required 189 + # Check for existence of 'BKG_SCALE' in the observation index file if required
190 if self._use_bkg_scale: 190 if self._use_bkg_scale:
191 - filename = gammalib.GFilename(self._obs_index+"[OBS_INDEX]") 191 + filename = gammalib.GFilename(self._obs_index+'[OBS_INDEX]')
192 if filename.is_fits(): 192 if filename.is_fits():
193 fits = gammalib.GFits(self._obs_index) 193 fits = gammalib.GFits(self._obs_index)
194 - if not fits["OBS_INDEX"].contains("BKG_SCALE"): 194 + if not fits['OBS_INDEX'].contains('BKG_SCALE'):
195 self._use_bkg_scale = False 195 self._use_bkg_scale = False
196 fits.close() 196 fits.close()
197 else: 197 else:
@@ -204,7 +204,7 @@ class csiactobs(ctools.cscript): @@ -204,7 +204,7 @@ class csiactobs(ctools.cscript):
204 # Write input parameters into logger 204 # Write input parameters into logger
205 if self._logTerse(): 205 if self._logTerse():
206 self._log_parameters() 206 self._log_parameters()
207 - self._log("\n") 207 + self._log('\n')
208 208
209 # Return 209 # Return
210 return 210 return
@@ -223,7 +223,7 @@ class csiactobs(ctools.cscript): @@ -223,7 +223,7 @@ class csiactobs(ctools.cscript):
223 except: 223 except:
224 224
225 # raise error if wrong type detected 225 # raise error if wrong type detected
226 - raise RuntimeError("Argument of csiactobs.runlist() must be of type 'list'") 226 + raise RuntimeError('Argument of csiactobs.runlist() must be of type "list"')
227 227
228 # Return 228 # Return
229 return 229 return
@@ -233,24 +233,24 @@ class csiactobs(ctools.cscript): @@ -233,24 +233,24 @@ class csiactobs(ctools.cscript):
233 # Handle constant spectral model 233 # Handle constant spectral model
234 if index == 0.0 and self._bkgpars <= 1: 234 if index == 0.0 and self._bkgpars <= 1:
235 spec = gammalib.GModelSpectralConst() 235 spec = gammalib.GModelSpectralConst()
236 - spec["Value"].min(prefactor / self._bkg_range_factor)  
237 - spec["Value"].max(prefactor * self._bkg_range_factor)  
238 - spec["Value"].value(prefactor) 236 + spec['Value'].min(prefactor / self._bkg_range_factor)
  237 + spec['Value'].max(prefactor * self._bkg_range_factor)
  238 + spec['Value'].value(prefactor)
239 if self._bkgpars == 0: 239 if self._bkgpars == 0:
240 - spec["Value"].fix() 240 + spec['Value'].fix()
241 else: 241 else:
242 - spec["Value"].free() 242 + spec['Value'].free()
243 243
244 else: 244 else:
245 245
246 # Create power law model 246 # Create power law model
247 if self._bkgpars <= 2: 247 if self._bkgpars <= 2:
248 - e = gammalib.GEnergy(1.0,"TeV") 248 + e = gammalib.GEnergy(1.0,'TeV')
249 spec = gammalib.GModelSpectralPlaw(prefactor, index, e) 249 spec = gammalib.GModelSpectralPlaw(prefactor, index, e)
250 250
251 # Set parameter ranges 251 # Set parameter ranges
252 - spec[0].min(0.01 * prefactor)  
253 - spec[0].max(100.0 * prefactor) 252 + spec[0].min(prefactor / self._bkg_range_factor)
  253 + spec[0].max(prefactor * self._bkg_range_factor)
254 spec[1].scale(1) 254 spec[1].scale(1)
255 spec[1].min(-5.0) 255 spec[1].min(-5.0)
256 spec[1].max(5.0) 256 spec[1].max(5.0)
@@ -269,20 +269,22 @@ class csiactobs(ctools.cscript): @@ -269,20 +269,22 @@ class csiactobs(ctools.cscript):
269 else: 269 else:
270 270
271 # Create reference powerlaw 271 # Create reference powerlaw
272 - plaw = gammalib.GModelSpectralPlaw(prefactor, index, gammalib.GEnergy(1.0,"TeV")) 272 + plaw = gammalib.GModelSpectralPlaw(prefactor, index, gammalib.GEnergy(1.0,'TeV'))
273 273
274 # Create spectral model and energy values 274 # Create spectral model and energy values
275 spec = gammalib.GModelSpectralNodes() 275 spec = gammalib.GModelSpectralNodes()
276 - bounds = gammalib.GEbounds(self._bkgpars,gammalib.GEnergy(emin,"TeV"),gammalib.GEnergy(emax,"TeV"), True) 276 + bounds = gammalib.GEbounds(self._bkgpars,gammalib.GEnergy(emin,'TeV'),gammalib.GEnergy(emax,'TeV'), True)
277 for i in range(bounds.size()): 277 for i in range(bounds.size()):
278 energy = bounds.elogmean(i) 278 energy = bounds.elogmean(i)
279 value = plaw.eval(energy, gammalib.GTime()) 279 value = plaw.eval(energy, gammalib.GTime())
280 spec.append(energy, value) 280 spec.append(energy, value)
281 for par in spec: 281 for par in spec:
282 282
283 - if "Energy" in par.name(): 283 + if 'Energy' in par.name():
284 par.fix() 284 par.fix()
285 - elif "Intensity" in par.name(): 285 + elif 'Intensity' in par.name():
  286 + value = par.value()
  287 + par.scale(value)
286 par.min(value / self._bkg_range_factor) 288 par.min(value / self._bkg_range_factor)
287 par.max(value * self._bkg_range_factor) 289 par.max(value * self._bkg_range_factor)
288 290
@@ -292,7 +294,7 @@ class csiactobs(ctools.cscript): @@ -292,7 +294,7 @@ class csiactobs(ctools.cscript):
292 def _iact_background(self, telescope, obs_id, bkg_scale, bkgtype, emin=0.01, emax=100): 294 def _iact_background(self, telescope, obs_id, bkg_scale, bkgtype, emin=0.01, emax=100):
293 295
294 # handle IrfBackground 296 # handle IrfBackground
295 - if bkgtype == "irf": 297 + if bkgtype == 'irf':
296 prefactor = 1.0 298 prefactor = 1.0
297 index = 0.0 299 index = 0.0
298 prefactor *= bkg_scale 300 prefactor *= bkg_scale
@@ -302,7 +304,7 @@ class csiactobs(ctools.cscript): @@ -302,7 +304,7 @@ class csiactobs(ctools.cscript):
302 bck = gammalib.GCTAModelIrfBackground(spec) 304 bck = gammalib.GCTAModelIrfBackground(spec)
303 305
304 # Set AeffBackground 306 # Set AeffBackground
305 - elif bkgtype == "aeff": 307 + elif bkgtype == 'aeff':
306 prefactor = bkg_scale * self._bkg_aeff_norm 308 prefactor = bkg_scale * self._bkg_aeff_norm
307 spec = self._background_spectrum(obs_id, prefactor, self._bkg_aeff_index, emin, emax) 309 spec = self._background_spectrum(obs_id, prefactor, self._bkg_aeff_index, emin, emax)
308 310
@@ -310,14 +312,14 @@ class csiactobs(ctools.cscript): @@ -310,14 +312,14 @@ class csiactobs(ctools.cscript):
310 bck = gammalib.GCTAModelAeffBackground(spec) 312 bck = gammalib.GCTAModelAeffBackground(spec)
311 313
312 # Set Gaussian Background 314 # Set Gaussian Background
313 - elif bkgtype == "gauss": 315 + elif bkgtype == 'gauss':
314 prefactor = bkg_scale * self._bkg_gauss_norm 316 prefactor = bkg_scale * self._bkg_gauss_norm
315 spec = self._background_spectrum(obs_id, prefactor, self._bkg_gauss_index, emin, emax) 317 spec = self._background_spectrum(obs_id, prefactor, self._bkg_gauss_index, emin, emax)
316 radial = gammalib.GCTAModelRadialGauss(self._bkg_gauss_sigma) 318 radial = gammalib.GCTAModelRadialGauss(self._bkg_gauss_sigma)
317 bck = gammalib.GCTAModelRadialAcceptance(radial, spec) 319 bck = gammalib.GCTAModelRadialAcceptance(radial, spec)
318 320
319 else: 321 else:
320 - sys.exit("Background type \""+self._bkgtype+"\" unsupported") 322 + sys.exit('Background type "'+self._bkgtype+'" unsupported')
321 323
322 # Copy model 324 # Copy model
323 model = bck.clone() 325 model = bck.clone()
@@ -329,7 +331,7 @@ class csiactobs(ctools.cscript): @@ -329,7 +331,7 @@ class csiactobs(ctools.cscript):
329 model.instruments(telescope) 331 model.instruments(telescope)
330 332
331 # Set name (arbitrary) 333 # Set name (arbitrary)
332 - model.name("bkg_"+str(obs_id)) 334 + model.name('bkg_'+str(obs_id))
333 335
334 # Turn off TS calculation for background model 336 # Turn off TS calculation for background model
335 model.tscalc(False) 337 model.tscalc(False)
@@ -355,36 +357,36 @@ class csiactobs(ctools.cscript): @@ -355,36 +357,36 @@ class csiactobs(ctools.cscript):
355 357
356 # Clear xml file and append an observation list 358 # Clear xml file and append an observation list
357 self._xml.clear() 359 self._xml.clear()
358 - self._xml.append(gammalib.GXmlElement("observation_list title=\"observation list\""))  
359 - lib = self._xml.element("observation_list", 0) 360 + self._xml.append(gammalib.GXmlElement('observation_list title="observation list"'))
  361 + lib = self._xml.element('observation_list', 0)
360 362
361 # Log output 363 # Log output
362 if self._logTerse(): 364 if self._logTerse():
363 - self._log("\n")  
364 - self._log.header1("Looping over "+str(len(self._runlist))+" runs") 365 + self._log('\n')
  366 + self._log.header1('Looping over '+str(len(self._runlist))+' runs')
365 367
366 # Initialise energy range values for logging 368 # Initialise energy range values for logging
367 - self._ebounds.clear() 369 + self._ebounds.clear()
368 370
369 # Loop over runs 371 # Loop over runs
370 for obs_id in self._runlist: 372 for obs_id in self._runlist:
371 373
372 # Create selection string 374 # Create selection string
373 - obs_selection = "[OBS_ID=="+str(obs_id)+"]" 375 + obs_selection = '[OBS_ID=='+str(obs_id)+']'
374 376
375 # Open HDU index file 377 # Open HDU index file
376 - hduindx = gammalib.GFits(self._hdu_index+"[HDU_INDEX]"+obs_selection)  
377 - hduindx_hdu = hduindx["HDU_INDEX"] 378 + hduindx = gammalib.GFits(self._hdu_index+'[HDU_INDEX]'+obs_selection)
  379 + hduindx_hdu = hduindx['HDU_INDEX']
378 380
379 # Initialise files and hdu names 381 # Initialise files and hdu names
380 - eventfile = aefffile = psffile = edispfile = bkgfile = ""  
381 - eventhdu = aeffhdu = "" 382 + eventfile = aefffile = psffile = edispfile = bkgfile = ''
  383 + eventhdu = aeffhdu = ''
382 384
383 types = [] 385 types = []
384 formats = [] 386 formats = []
385 for i in range(hduindx_hdu.nrows()): 387 for i in range(hduindx_hdu.nrows()):
386 - types.append(hduindx_hdu["HDU_TYPE"][i])  
387 - formats.append(hduindx_hdu["HDU_CLASS"][i]) 388 + types.append(hduindx_hdu['HDU_TYPE'][i])
  389 + formats.append(hduindx_hdu['HDU_CLASS'][i])
388 390
389 # Handle events 391 # Handle events
390 index = -1 392 index = -1
@@ -394,12 +396,12 @@ class csiactobs(ctools.cscript): @@ -394,12 +396,12 @@ class csiactobs(ctools.cscript):
394 index = formats.index(version) 396 index = formats.index(version)
395 break 397 break
396 if index >= 0: 398 if index >= 0:
397 - eventfile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index])  
398 - eventhdu = hduindx_hdu["HDU_NAME"][index]  
399 - eventfile += "["+eventhdu+"]" 399 + eventfile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index])
  400 + eventhdu = hduindx_hdu['HDU_NAME'][index]
  401 + eventfile += '['+eventhdu+']'
400 if not gammalib.GFilename(eventfile).is_fits(): 402 if not gammalib.GFilename(eventfile).is_fits():
401 if self._logTerse(): 403 if self._logTerse():
402 - self._log("Skipping observation "+str(obs_id)+": eventfile \""+eventfile+"\" not found\n") 404 + self._log('Skipping observation '+str(obs_id)+': eventfile "'+eventfile+'" not found\n')
403 continue 405 continue
404 406
405 # Handle Aeff 407 # Handle Aeff
@@ -410,12 +412,12 @@ class csiactobs(ctools.cscript): @@ -410,12 +412,12 @@ class csiactobs(ctools.cscript):
410 index = formats.index(version) 412 index = formats.index(version)
411 break 413 break
412 if index >= 0: 414 if index >= 0:
413 - aefffile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index])  
414 - aeffhdu = hduindx_hdu["HDU_NAME"][index]  
415 - aefffile += "["+aeffhdu+"]" 415 + aefffile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index])
  416 + aeffhdu = hduindx_hdu['HDU_NAME'][index]
  417 + aefffile += '['+aeffhdu+']'
416 if not gammalib.GFilename(aefffile).is_fits(): 418 if not gammalib.GFilename(aefffile).is_fits():
417 if self._logTerse(): 419 if self._logTerse():
418 - self._log("Skipping observation "+str(obs_id)+": effective area \""+aefffile+"\" not found\n") 420 + self._log('Skipping observation '+str(obs_id)+': effective area "'+aefffile+'" not found\n')
419 continue 421 continue
420 422
421 # Handle psf 423 # Handle psf
@@ -426,11 +428,11 @@ class csiactobs(ctools.cscript): @@ -426,11 +428,11 @@ class csiactobs(ctools.cscript):
426 index = formats.index(version) 428 index = formats.index(version)
427 break 429 break
428 if index >= 0: 430 if index >= 0:
429 - psffile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index])  
430 - psffile += "["+hduindx_hdu["HDU_NAME"][index]+"]" 431 + psffile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index])
  432 + psffile += '['+hduindx_hdu['HDU_NAME'][index]+']'
431 if not gammalib.GFilename(psffile).is_fits(): 433 if not gammalib.GFilename(psffile).is_fits():
432 if self._logTerse(): 434 if self._logTerse():
433 - self._log("Skipping observation "+str(obs_id)+": point spread function \""+psffile+"\" not found\n") 435 + self._log('Skipping observation '+str(obs_id)+': point spread function "'+psffile+'" not found\n')
434 continue 436 continue
435 437
436 # Handle edisp 438 # Handle edisp
@@ -441,12 +443,12 @@ class csiactobs(ctools.cscript): @@ -441,12 +443,12 @@ class csiactobs(ctools.cscript):
441 index = formats.index(version) 443 index = formats.index(version)
442 break 444 break
443 if index >= 0: 445 if index >= 0:
444 - edispfile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index])  
445 - edispfile += "["+hduindx_hdu["HDU_NAME"][index]+"]" 446 + edispfile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index])
  447 + edispfile += '['+hduindx_hdu['HDU_NAME'][index]+']'
446 if not gammalib.GFilename(edispfile).is_fits(): 448 if not gammalib.GFilename(edispfile).is_fits():
447 if self._logTerse(): 449 if self._logTerse():
448 - self._log("Warning: observation "+str(obs_id)+" has no energy dispersion \""+edispfile+"\" information\n")  
449 - edispfile = "" 450 + self._log('Warning: observation '+str(obs_id)+' has no energy dispersion "'+edispfile+'" information\n')
  451 + edispfile = ''
450 452
451 # Handle background 453 # Handle background
452 index = -1 454 index = -1
@@ -457,14 +459,20 @@ class csiactobs(ctools.cscript): @@ -457,14 +459,20 @@ class csiactobs(ctools.cscript):
457 index = formats.index(version) 459 index = formats.index(version)
458 break 460 break
459 if index >= 0: 461 if index >= 0:
460 - bkgfile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index])  
461 - bkgfile += "["+hduindx_hdu["HDU_NAME"][index]+"]"  
462 - if not gammalib.GFilename(bkgfile).is_fits(): 462 + bkgfile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index])
  463 + bkgfile += '['+hduindx_hdu['HDU_NAME'][index]+']'
  464 + if 'irf' in bkg_mod_hierarchy and not gammalib.GFilename(bkgfile).is_fits():
  465 + bkg_mod_hierarchy.remove('irf')
463 if self._logTerse(): 466 if self._logTerse():
464 - self._log("Warning: observation "+str(obs_id)+" has no background information (file=\""+bkgfile+"\"). IRF background cannot be used\n")  
465 - bkgfile = ""  
466 - if "irf" in bkg_mod_hierarchy:  
467 - bkg_mod_hierarchy.remove("irf") 467 + self._log('Warning: observation '+str(obs_id)+' has no background information (file="'+bkgfile+'"). IRF background cannot be used\n')
  468 + bkgfile = ''
  469 + if len(bkg_mod_hierarchy) == 0:
  470 + if self._logTerse():
  471 + self._log('Skipping observation '+str(obs_id)+': No background can be used\n')
  472 + continue
  473 + else:
  474 + if self._logTerse():
  475 + self._log('Observation '+str(obs_id)+': Falling back to background "'+bkg_mod_hierarchy[0]+'"\n')
468 476
469 # Close hdu index file 477 # Close hdu index file
470 hduindx.close() 478 hduindx.close()
@@ -472,15 +480,15 @@ class csiactobs(ctools.cscript): @@ -472,15 +480,15 @@ class csiactobs(ctools.cscript):
472 # Handle background scale information if available 480 # Handle background scale information if available
473 bkg_scale = 1.0 481 bkg_scale = 1.0
474 if self._use_bkg_scale: 482 if self._use_bkg_scale:
475 - obsindx = gammalib.GFits(self._obs_index+"[OBS_INDEX]"+obs_selection)  
476 - bkg_scale = obsindx["OBS_INDEX"]["BKG_SCALE"][0] 483 + obsindx = gammalib.GFits(self._obs_index+'[OBS_INDEX]'+obs_selection)
  484 + bkg_scale = obsindx['OBS_INDEX']['BKG_SCALE'][0]
477 obsindx.close() 485 obsindx.close()
478 486
479 # Open fits file to determine the observation name 487 # Open fits file to determine the observation name
480 fits = gammalib.GFits(eventfile) 488 fits = gammalib.GFits(eventfile)
481 events = fits[eventhdu] 489 events = fits[eventhdu]
482 - object_name = events.string("OBJECT")  
483 - telescope = events.string("TELESCOP") 490 + object_name = events.string('OBJECT')
  491 + telescope = events.string('TELESCOP')
484 492
485 # Close FITS file 493 # Close FITS file
486 fits.close() 494 fits.close()
@@ -490,19 +498,19 @@ class csiactobs(ctools.cscript): @@ -490,19 +498,19 @@ class csiactobs(ctools.cscript):
490 aeff_table = aeff_fits[aeffhdu] 498 aeff_table = aeff_fits[aeffhdu]
491 499
492 # Set energy range from header keyword if present 500 # Set energy range from header keyword if present
493 - if aeff_table.has_card("LO_THRES") and aeff_table.has_card("HI_THRES"): 501 + if aeff_table.has_card('LO_THRES') and aeff_table.has_card('HI_THRES'):
494 502
495 # Get safe energy range 503 # Get safe energy range
496 - run_emin = gammalib.GEnergy(aeff_table.real("LO_THRES"), "TeV")  
497 - run_emax = gammalib.GEnergy(aeff_table.real("HI_THRES"), "TeV") 504 + run_emin = gammalib.GEnergy(aeff_table.real('LO_THRES'), 'TeV')
  505 + run_emax = gammalib.GEnergy(aeff_table.real('HI_THRES'), 'TeV')
498 506
499 # Append to ebounds 507 # Append to ebounds
500 self._ebounds.append(run_emin, run_emax) 508 self._ebounds.append(run_emin, run_emax)
501 509
502 else: 510 else:
503 # Set default values for energy range 511 # Set default values for energy range
504 - run_emin = gammalib.GEnergy(10,"GeV")  
505 - run_emax = gammalib.GEnergy(100,"TeV") 512 + run_emin = gammalib.GEnergy(10,'GeV')
  513 + run_emax = gammalib.GEnergy(100,'TeV')
506 514
507 # Close Aeff fits file 515 # Close Aeff fits file
508 aeff_fits.close() 516 aeff_fits.close()
@@ -512,43 +520,43 @@ class csiactobs(ctools.cscript): @@ -512,43 +520,43 @@ class csiactobs(ctools.cscript):
512 520
513 # Logging 521 # Logging
514 if self._logTerse(): 522 if self._logTerse():
515 - self._log("Adding observation "+str(obs_id)+" (\""+object_name+"\")\n") 523 + self._log('Adding observation '+str(obs_id)+' ("'+object_name+'")\n')
516 524
517 if self._logExplicit(): 525 if self._logExplicit():
518 - self._log(" Event file: "+eventfile+"\n")  
519 - self._log(" Effective area: "+aefffile+"\n")  
520 - self._log(" Point spread function: "+psffile+"\n")  
521 - self._log(" Energy dispersion: "+edispfile+"\n")  
522 - self._log(" Background file: "+bkgfile+"\n") 526 + self._log(' Event file: '+eventfile+'\n')
  527 + self._log(' Effective area: '+aefffile+'\n')
  528 + self._log(' Point spread function: '+psffile+'\n')
  529 + self._log(' Energy dispersion: '+edispfile+'\n')
  530 + self._log(' Background file: '+bkgfile+'\n')
523 if self._use_bkg_scale: 531 if self._use_bkg_scale:
524 - self._log(" Background scale: "+str(bkg_scale)+"\n")  
525 - self._log("\n") 532 + self._log(' Background scale: '+str(bkg_scale)+'\n')
  533 + self._log('\n')
526 534
527 # Append observation to XML and set attributes 535 # Append observation to XML and set attributes
528 - obs = lib.append("observation")  
529 - obs.attribute("name", object_name)  
530 - obs.attribute("id", str(obs_id))  
531 - obs.attribute("instrument", telescope) 536 + obs = lib.append('observation')
  537 + obs.attribute('name', object_name)
  538 + obs.attribute('id', str(obs_id))
  539 + obs.attribute('instrument', telescope)
532 540
533 # Append event file 541 # Append event file
534 - ev = gammalib.GXmlElement("parameter name=\"EventList\"")  
535 - ev.attribute("file", eventfile) 542 + ev = gammalib.GXmlElement('parameter name="EventList"')
  543 + ev.attribute('file', eventfile)
536 544
537 # Append effective area 545 # Append effective area
538 - aeff = gammalib.GXmlElement("parameter name=\"EffectiveArea\"")  
539 - aeff.attribute("file",aefffile) 546 + aeff = gammalib.GXmlElement('parameter name="EffectiveArea"')
  547 + aeff.attribute('file',aefffile)
540 548
541 # Append PSF 549 # Append PSF
542 - psf = gammalib.GXmlElement("parameter name=\"PointSpreadFunction\"")  
543 - psf.attribute("file",psffile) 550 + psf = gammalib.GXmlElement('parameter name="PointSpreadFunction"')
  551 + psf.attribute('file',psffile)
544 552
545 # Append energy dispersion 553 # Append energy dispersion
546 - edisp = gammalib.GXmlElement("parameter name=\"EnergyDispersion\"")  
547 - edisp.attribute("file",edispfile) 554 + edisp = gammalib.GXmlElement('parameter name="EnergyDispersion"')
  555 + edisp.attribute('file',edispfile)
548 556
549 # Append background 557 # Append background
550 - bck = gammalib.GXmlElement("parameter name=\"Background\"")  
551 - bck.attribute("file",bkgfile) 558 + bck = gammalib.GXmlElement('parameter name="Background"')
  559 + bck.attribute('file',bkgfile)
552 560
553 # assign events and IRFs to observation 561 # assign events and IRFs to observation
554 obs.append(ev) 562 obs.append(ev)
@@ -562,35 +570,35 @@ class csiactobs(ctools.cscript): @@ -562,35 +570,35 @@ class csiactobs(ctools.cscript):
562 570
563 # Log header of energy range 571 # Log header of energy range
564 if self._logTerse(): 572 if self._logTerse():
565 - self._log("\n")  
566 - self._log.header3("Energy range of obervation list") 573 + self._log('\n')
  574 + self._log.header3('Energy range of obervation list')
567 575
568 # Logging if energy range is available 576 # Logging if energy range is available
569 if self._ebounds.size() > 0: 577 if self._ebounds.size() > 0:
570 578
571 # Write energy range into log file 579 # Write energy range into log file
572 - self._log(str(self._ebounds.emin())+" - "+str(self._ebounds.emax()))  
573 - self._log("\n") 580 + self._log(str(self._ebounds.emin())+' - '+str(self._ebounds.emax()))
  581 + self._log('\n')
574 582
575 else: 583 else:
576 584
577 # Write 'not available' into log file 585 # Write 'not available' into log file
578 - self._log("not available")  
579 - self._log("\n") 586 + self._log('not available')
  587 + self._log('\n')
580 588
581 else: 589 else:
582 - self._log.header2("WARNING: No observation from given runlist available") 590 + self._log.header2('WARNING: No observation from given runlist available')
583 591
584 # Append models provided by 'inmodels' if necessary 592 # Append models provided by 'inmodels' if necessary
585 if not self._inmodels == None: 593 if not self._inmodels == None:
586 if self._logTerse(): 594 if self._logTerse():
587 - self._log("\n")  
588 - self._log.header1("Appending models") 595 + self._log('\n')
  596 + self._log.header1('Appending models')
589 597
590 # Loop over input models 598 # Loop over input models
591 for model in self._inmodels: 599 for model in self._inmodels:
592 if self._logTerse(): 600 if self._logTerse():
593 - self._log.header3("Adding model \""+model.name()+"\"") 601 + self._log.header3('Adding model "'+model.name()+'"')
594 self._models.append(model) 602 self._models.append(model)
595 603
596 # Return 604 # Return
cscripts/csmodelinfo.py
  View file @c60f05b
@@ -430,10 +430,6 @@ class csmodelinfo(ctools.cscript): @@ -430,10 +430,6 @@ class csmodelinfo(ctools.cscript):
430 """ 430 """
431 Save models to ds9 region file if required 431 Save models to ds9 region file if required
432 """ 432 """
433 - # Write header  
434 - if self._logTerse():  
435 - self._log("\n")  
436 - self._log.header1("Save models in DS9 file")  
437 433
438 # Get output filename in case it was not read ahead 434 # Get output filename in case it was not read ahead
439 self._ds9file = self["ds9file"].filename() 435 self._ds9file = self["ds9file"].filename()
@@ -441,6 +437,11 @@ class csmodelinfo(ctools.cscript): @@ -441,6 +437,11 @@ class csmodelinfo(ctools.cscript):
441 # Check if DS9 file is valid 437 # Check if DS9 file is valid
442 if self._ds9file != "NONE": 438 if self._ds9file != "NONE":
443 439
  440 + # Write header
  441 + if self._logTerse():
  442 + self._log("\n")
  443 + self._log.header1("Save models in DS9 file")
  444 +
444 # Log filename 445 # Log filename
445 if self._logTerse(): 446 if self._logTerse():
446 self._log(gammalib.parformat("DS9 filename")) 447 self._log(gammalib.parformat("DS9 filename"))
cscripts/csmodelmerge.py
  View file @c60f05b
@@ -90,6 +90,9 @@ class csmodelmerge(ctools.cscript): @@ -90,6 +90,9 @@ class csmodelmerge(ctools.cscript):
90 # Read ahead output filename 90 # Read ahead output filename
91 if self._read_ahead(): 91 if self._read_ahead():
92 self["outmodel"].filename() 92 self["outmodel"].filename()
  93 +
  94 + # Get clobber parameter
  95 + self._clobber = self["clobber"].boolean()
93 96
94 # Write input parameters into logger 97 # Write input parameters into logger
95 if self._logTerse(): 98 if self._logTerse():
@@ -163,14 +166,26 @@ class csmodelmerge(ctools.cscript): @@ -163,14 +166,26 @@ class csmodelmerge(ctools.cscript):
163 # Get output filename in case it was not read ahead 166 # Get output filename in case it was not read ahead
164 outmodel = self["outmodel"].filename() 167 outmodel = self["outmodel"].filename()
165 168
166 - # Log filename  
167 - if self._logTerse():  
168 - self._log(gammalib.parformat("Model definition XML file"))  
169 - self._log(outmodel.url())  
170 - self._log("\n")  
171 -  
172 - # Save models  
173 - self._models.save(outmodel) 169 + # Check if file exists and apply clobber if false
  170 + if outmodel.exists() and not self._clobber:
  171 +
  172 + # Set error message
  173 + msg = "Cannot save \""+outmodel.url()+"\": File already exists "
  174 + msg += "Use parameter clobber=yes to allow overwriting of files."
  175 +
  176 + # Throw exception since file cannnot be overwritten
  177 + raise RuntimeError(msg)
  178 +
  179 + else:
  180 +
  181 + # Log filename
  182 + if self._logTerse():
  183 + self._log(gammalib.parformat("Model definition XML file"))
  184 + self._log(outmodel.url())
  185 + self._log("\n")
  186 +
  187 + # Save models
  188 + self._models.save(outmodel)
174 189
175 # Return 190 # Return
176 return 191 return
cscripts/csobsinfo.py
  View file @c60f05b
@@ -364,16 +364,17 @@ class csobsinfo(ctools.cscript): @@ -364,16 +364,17 @@ class csobsinfo(ctools.cscript):
364 observation container into a DS9 region file. If "NONE" is 364 observation container into a DS9 region file. If "NONE" is
365 specified for the "ds9file" parameter the method does nothing. 365 specified for the "ds9file" parameter the method does nothing.
366 """ 366 """
367 - # Write header  
368 - if self._logTerse():  
369 - self._log("\n")  
370 - self._log.header1("Save pointings in DS9 file")  
371 367
372 # Get output filename in case it was not read ahead 368 # Get output filename in case it was not read ahead
373 ds9file = self["ds9file"].filename() 369 ds9file = self["ds9file"].filename()
374 370
375 # Check if DS9 file is valid 371 # Check if DS9 file is valid
376 - if ds9file.url() != "NONE": 372 + if ds9file.url() != "NONE":
  373 +
  374 + # Write header
  375 + if self._logTerse():
  376 + self._log("\n")
  377 + self._log.header1("Save pointings in DS9 file")
377 378
378 # Log filename 379 # Log filename
379 if self._logTerse(): 380 if self._logTerse():