Simon Bonnefoy / ctools

Commit a90f5fd4df81054e3949cc31923128833b1a4daf

Authored by Jürgen Knödlseder
1 parent 63661f50

Increase branch coverage of csfindobs (#1828)

Inline Side-by-side
Showing 2 changed files with 152 additions and 95 deletions
cscripts/csfindobs.py
  View file @a90f5fd
@@ -30,13 +30,13 @@ import ctools @@ -30,13 +30,13 @@ import ctools
30 # =============== # 30 # =============== #
31 class csfindobs(ctools.cscript): 31 class csfindobs(ctools.cscript):
32 """ 32 """
33 - Find observations from an IACT data store. 33 + Find observations from an IACT data store
34 """ 34 """
35 35
36 # Constructor 36 # Constructor
37 def __init__(self, *argv): 37 def __init__(self, *argv):
38 """ 38 """
39 - Constructor. 39 + Constructor
40 """ 40 """
41 # Set name 41 # Set name
42 self._name = 'csfindobs' 42 self._name = 'csfindobs'
@@ -50,8 +50,7 @@ class csfindobs(ctools.cscript): @@ -50,8 +50,7 @@ class csfindobs(ctools.cscript):
50 self._obs_index = '' 50 self._obs_index = ''
51 self._runs = [] 51 self._runs = []
52 52
53 - # Initialise application by calling the appropriate class  
54 - # constructor. 53 + # Initialise application by calling the appropriate class constructor
55 self._init_cscript(argv) 54 self._init_cscript(argv)
56 55
57 # Return 56 # Return
@@ -76,13 +75,13 @@ class csfindobs(ctools.cscript): @@ -76,13 +75,13 @@ class csfindobs(ctools.cscript):
76 # Master index file name 75 # Master index file name
77 master_indx = self['master_indx'].string() 76 master_indx = self['master_indx'].string()
78 77
79 - # Intiialise flag if spatial selection is required 78 + # Initialise flag if spatial selection is required
80 self._select_radec = True 79 self._select_radec = True
81 80
82 # Initialise invalid radius 81 # Initialise invalid radius
83 self._radius = 0.0 82 self._radius = 0.0
84 83
85 - # Check for vailidity of spatial parameters 84 + # Check for validity of spatial parameters
86 if (self['ra'].is_valid() and 85 if (self['ra'].is_valid() and
87 self['dec'].is_valid() and 86 self['dec'].is_valid() and
88 self['rad'].is_valid()): 87 self['rad'].is_valid()):
@@ -91,7 +90,9 @@ class csfindobs(ctools.cscript): @@ -91,7 +90,9 @@ class csfindobs(ctools.cscript):
91 self._ra = self['ra'].real() 90 self._ra = self['ra'].real()
92 self._dec = self['dec'].real() 91 self._dec = self['dec'].real()
93 self._radius = self['rad'].real() 92 self._radius = self['rad'].real()
94 - 93 +
  94 + # ... otherwise signal that there are no spatial parameters for
  95 + # selection
95 else: 96 else:
96 self._select_radec = False 97 self._select_radec = False
97 98
@@ -107,49 +108,51 @@ class csfindobs(ctools.cscript): @@ -107,49 +108,51 @@ class csfindobs(ctools.cscript):
107 if self._read_ahead(): 108 if self._read_ahead():
108 self['outfile'].filename() 109 self['outfile'].filename()
109 110
110 - # Open master index file and look for prodname 111 + # Set filename of JSON master file and raise an exception if the file
  112 + # does not exist
111 master_file = os.path.join(self._datapath, master_indx) 113 master_file = os.path.join(self._datapath, master_indx)
112 if not os.path.isfile(master_file): 114 if not os.path.isfile(master_file):
113 - raise RuntimeError('FITS data store not available. '+  
114 - 'No master index file found at "'+  
115 - master_file+'". Make sure the file '+  
116 - 'is copied from the server and your '+  
117 - 'datapath is set correctly.') 115 + msg = ('FITS data store not available. No master index file found '
  116 + 'at "%s". Make sure the file is copied from the server and '
  117 + 'your datapath is set correctly.' % master_file)
  118 + raise RuntimeError(msg)
118 119
119 - # Open and load JSON file 120 + # Open and load JSON master file. If the "dataset" key is not available
  121 + # then raise an exception
120 json_data = open(master_file).read() 122 json_data = open(master_file).read()
121 - data = json.loads(json_data) 123 + data = json.loads(json_data)
122 if not 'datasets' in data: 124 if not 'datasets' in data:
123 - raise RuntimeError('Key "datasets" not available in '+  
124 - 'master index file.') 125 + msg = ('Key "datasets" not available in master index file.')
  126 + raise RuntimeError(msg)
125 127
126 - # Get configurations 128 + # Get configurations from JSON master file
127 configs = data['datasets'] 129 configs = data['datasets']
128 130
129 # Initialise obs index file 131 # Initialise obs index file
130 self._obs_index = '' 132 self._obs_index = ''
131 133
132 - # Get HDUs 134 + # Get name of observation index file
133 for config in configs: 135 for config in configs:
134 if self._prodname == config['name']: 136 if self._prodname == config['name']:
135 self._obs_index = str(os.path.join(self._datapath, 137 self._obs_index = str(os.path.join(self._datapath,
136 - config['obsindx'])) 138 + config['obsindx']))
137 break 139 break
138 140
139 - # Check HDUs 141 + # If the observation index file name is empty then raise an exception
140 if self._obs_index == '': 142 if self._obs_index == '':
141 - raise RuntimeError('FITS data store "'+self._prodname+  
142 - '" not available. Run csiactdata to get '+  
143 - 'a list of available storage names') 143 + msg = ('FITS data store "%s" not available. Run csiactdata to get '
  144 + 'a list of available storage names.' % self._prodname)
  145 + raise RuntimeError(msg)
  146 +
  147 + # If the observation index file is not a FITS file then raise an
  148 + # exception
144 filename = gammalib.GFilename(self._obs_index+'[OBS_INDEX]') 149 filename = gammalib.GFilename(self._obs_index+'[OBS_INDEX]')
145 if not filename.is_fits(): 150 if not filename.is_fits():
146 - raise RuntimeError('Observation index file "'+  
147 - self._obs_index+  
148 - '[OBS_INDEX]" for FITS data store "'+  
149 - self._prodname+  
150 - '" not available. Check your master index '+  
151 - 'file or run csiactdata to get a list of '+  
152 - 'available storage names.') 151 + msg = ('Observation index file "%s[OBS_INDEX]" for FITS data store '
  152 + '"%s" not available. Check your master index file or run '
  153 + 'csiactdata to get a list of available storage names.' %
  154 + (self._obs_index, self._prodname))
  155 + raise RuntimeError(msg)
153 156
154 # Write input parameters into logger 157 # Write input parameters into logger
155 if self._logTerse(): 158 if self._logTerse():
@@ -163,7 +166,7 @@ class csfindobs(ctools.cscript): @@ -163,7 +166,7 @@ class csfindobs(ctools.cscript):
163 # Public methods 166 # Public methods
164 def run(self): 167 def run(self):
165 """ 168 """
166 - Run the script. 169 + Run the script
167 """ 170 """
168 # Switch screen logging on in debug mode 171 # Switch screen logging on in debug mode
169 if self._logDebug(): 172 if self._logDebug():
@@ -171,69 +174,68 @@ class csfindobs(ctools.cscript): @@ -171,69 +174,68 @@ class csfindobs(ctools.cscript):
171 174
172 # Get parameters 175 # Get parameters
173 self._get_parameters() 176 self._get_parameters()
174 - 177 +
  178 + # Write header into logger
  179 + if self._logTerse():
  180 + self._log('\n')
  181 + self._log.header1('Find observations')
  182 +
175 # Initialise run list 183 # Initialise run list
176 self._runs = [] 184 self._runs = []
177 185
178 - # Initialise selction expression 186 + # Initialise selection expression
179 expr = '' 187 expr = ''
180 188
181 - # Add spatial expression if possible 189 + # If spatial selection is requested then add an angular separation
  190 + # expression to the expression string
182 if self._select_radec: 191 if self._select_radec:
183 expr += 'ANGSEP('+str(self._ra)+','+str(self._dec)+ \ 192 expr += 'ANGSEP('+str(self._ra)+','+str(self._dec)+ \
184 ',RA_PNT,DEC_PNT)<='+str(self._radius) 193 ',RA_PNT,DEC_PNT)<='+str(self._radius)
185 194
186 - # Add connector if expression is empty 195 + # Add '&&' connector if expression is not empty
187 if len(expr): 196 if len(expr):
188 expr += '&&' 197 expr += '&&'
189 198
190 - # Add quality expression 199 + # Add always quality expression
191 expr += 'QUALITY<='+str(self['min_qual'].integer()) 200 expr += 'QUALITY<='+str(self['min_qual'].integer())
192 201
193 - # Add user expression 202 + # Add user expression is one has been specified
194 expression = self['expression'].string() 203 expression = self['expression'].string()
195 if (expression != 'NONE' and expression != 'INDEF' and 204 if (expression != 'NONE' and expression != 'INDEF' and
196 len(expression) > 0): 205 len(expression) > 0):
197 expr += '&&' + expression 206 expr += '&&' + expression
198 -  
199 - # Initialise filename  
200 - openfile = self._obs_index+'[OBS_INDEX]['+expr+']'  
201 -  
202 - # Open file  
203 - fits = gammalib.GFits(openfile)  
204 -  
205 - # Check for extistence of rows  
206 - if fits['OBS_INDEX'].nrows() > 0:  
207 -  
208 - # Get HDU  
209 - obs_indx = fits['OBS_INDEX']  
210 -  
211 - # Read observation IDs into array  
212 - for i in range(obs_indx.nrows()):  
213 - self._runs.append(obs_indx['OBS_ID'][i])  
214 207
215 - # Dump expression 208 + # Write the expression into the logger
216 if self._logNormal(): 209 if self._logNormal():
217 - self._log('\n')  
218 - self._log.header3('Expression')  
219 - self._log(expr)  
220 - self._log('\n') 210 + self._log(gammalib.parformat('Expression'))
  211 + self._log(expr+'\n')
  212 +
  213 + # Set filename including the selection expression
  214 + filename = self._obs_index+'[OBS_INDEX]['+expr+']'
  215 +
  216 + # Open observation index FITS file
  217 + fits = gammalib.GFits(filename)
  218 +
  219 + # If there are entries in the observation index table then append the
  220 + # observations identifiers to the run list
  221 + obs_index = fits['OBS_INDEX']
  222 + if obs_index.nrows() > 0:
  223 + for i in range(obs_index.nrows()):
  224 + self._runs.append(obs_index['OBS_ID'][i])
221 225
222 - # Dump number of observations 226 + # Write the number of observations into the logger
223 if self._logTerse(): 227 if self._logTerse():
  228 + self._log(gammalib.parformat('Observations'))
  229 + self._log(len(self._runs))
224 self._log('\n') 230 self._log('\n')
225 - self._log.header3('Observations')  
226 - self._log('Found '+str(len(self._runs))+' observations')  
227 - self._log('\n')  
228 -  
229 - # Dump number of observations  
230 - if self._logVerbose():  
231 - self._log('\n')  
232 - self._log.header3('Observation IDs')  
233 - for run in self._runs: 231 +
  232 + # Write the observation identifiers into the logger
  233 + if self._logNormal() and len(self._runs) > 0:
  234 + for i, run in enumerate(self._runs):
  235 + self._log(gammalib.parformat('Observation %d' % (i+1)))
234 self._log(str(run)+'\n') 236 self._log(str(run)+'\n')
235 -  
236 - # Close file 237 +
  238 + # Close FITS file
237 fits.close() 239 fits.close()
238 240
239 # Return 241 # Return
@@ -241,7 +243,7 @@ class csfindobs(ctools.cscript): @@ -241,7 +243,7 @@ class csfindobs(ctools.cscript):
241 243
242 def execute(self): 244 def execute(self):
243 """ 245 """
244 - Execute the script. 246 + Execute the script
245 """ 247 """
246 # Open logfile 248 # Open logfile
247 self.logFileOpen() 249 self.logFileOpen()
@@ -260,7 +262,7 @@ class csfindobs(ctools.cscript): @@ -260,7 +262,7 @@ class csfindobs(ctools.cscript):
260 262
261 def save(self): 263 def save(self):
262 """ 264 """
263 - Save runlist. 265 + Save runlist
264 """ 266 """
265 # Write header 267 # Write header
266 if self._logTerse(): 268 if self._logTerse():
@@ -270,12 +272,14 @@ class csfindobs(ctools.cscript): @@ -270,12 +272,14 @@ class csfindobs(ctools.cscript):
270 # Get filename 272 # Get filename
271 outfile = self['outfile'].filename() 273 outfile = self['outfile'].filename()
272 274
273 - # Check for clobber 275 + # If the runlist file exists but the clobber flag is not set then signal
  276 + # that runlist file exists already
274 if outfile.exists() and not self._clobber(): 277 if outfile.exists() and not self._clobber():
275 if self._logTerse(): 278 if self._logTerse():
276 - self._log('File '+outfile+' already exists. ')  
277 - self._log('Set clobber=yes to overwrite file.')  
278 - self._log('\n') 279 + self._log('File "'+outfile+'" exists already, runlist not ')
  280 + self._log('saved. Set clobber=yes to overwrite the file.\n')
  281 +
  282 + # ... otherwise save the runlist into a file
279 else: 283 else:
280 284
281 # Log file name 285 # Log file name
@@ -284,7 +288,7 @@ class csfindobs(ctools.cscript): @@ -284,7 +288,7 @@ class csfindobs(ctools.cscript):
284 self._log(outfile.url()) 288 self._log(outfile.url())
285 self._log('\n') 289 self._log('\n')
286 290
287 - # Write runs to file 291 + # Write all runs to file
288 f = open(outfile.url(),'w') 292 f = open(outfile.url(),'w')
289 for run in self._runs: 293 for run in self._runs:
290 f.write(str(run)+' \n') 294 f.write(str(run)+' \n')
@@ -295,12 +299,13 @@ class csfindobs(ctools.cscript): @@ -295,12 +299,13 @@ class csfindobs(ctools.cscript):
295 299
296 def runs(self): 300 def runs(self):
297 """ 301 """
298 - Return run list. 302 + Return run list
299 303
300 - Returns:  
301 - A list of runs. 304 + Returns
  305 + -------
  306 + runs : list of str
  307 + A list of runs
302 """ 308 """
303 -  
304 # Return 309 # Return
305 return self._runs 310 return self._runs
306 311
test/test_csfindobs.py
  View file @a90f5fd
@@ -51,7 +51,7 @@ class Test(test): @@ -51,7 +51,7 @@ class Test(test):
51 # Set test functions 51 # Set test functions
52 def set(self): 52 def set(self):
53 """ 53 """
54 - Set all test functions. 54 + Set all test functions
55 """ 55 """
56 # Set test name 56 # Set test name
57 self.name('csfindobs') 57 self.name('csfindobs')
@@ -66,7 +66,7 @@ class Test(test): @@ -66,7 +66,7 @@ class Test(test):
66 # Test csfindobs on command line 66 # Test csfindobs on command line
67 def _test_cmd(self): 67 def _test_cmd(self):
68 """ 68 """
69 - Test csfindobs on the command line. 69 + Test csfindobs on the command line
70 """ 70 """
71 # Set script name 71 # Set script name
72 csfindobs = self._script('csfindobs') 72 csfindobs = self._script('csfindobs')
@@ -106,45 +106,97 @@ class Test(test): @@ -106,45 +106,97 @@ class Test(test):
106 # Test csfindobs from Python 106 # Test csfindobs from Python
107 def _test_python(self): 107 def _test_python(self):
108 """ 108 """
109 - Test csfindobs from Python. 109 + Test csfindobs from Python
110 """ 110 """
111 # Set-up csfindobs 111 # Set-up csfindobs
112 findobs = cscripts.csfindobs() 112 findobs = cscripts.csfindobs()
113 findobs['datapath'] = self._datapath 113 findobs['datapath'] = self._datapath
114 findobs['prodname'] = 'unit-test' 114 findobs['prodname'] = 'unit-test'
115 - findobs['outfile'] = 'runlist_py1.dat'  
116 findobs['ra'] = 83.63 115 findobs['ra'] = 83.63
117 findobs['dec'] = 22.01 116 findobs['dec'] = 22.01
118 findobs['rad'] = 1.0 117 findobs['rad'] = 1.0
  118 + findobs['outfile'] = 'runlist_py1.dat'
119 findobs['logfile'] = 'csfindobs_py1.log' 119 findobs['logfile'] = 'csfindobs_py1.log'
120 findobs['chatter'] = 2 120 findobs['chatter'] = 2
121 121
122 - # Run csfindobs script and save run list 122 + # Run csfindobs script
123 findobs.logFileOpen() # Make sure we get a log file 123 findobs.logFileOpen() # Make sure we get a log file
124 findobs.run() 124 findobs.run()
  125 +
  126 + # Check list of runs
  127 + runs = findobs.runs()
  128 + self.test_value(len(runs), 1,
  129 + 'Check for number of runs in runlist')
  130 + self.test_value(str(runs[0]), '0',
  131 + 'Check run ID in runlist')
  132 +
  133 + # Save run list
125 findobs.save() 134 findobs.save()
126 135
127 # Check run list 136 # Check run list
128 self._check_runlist('runlist_py1.dat') 137 self._check_runlist('runlist_py1.dat')
129 138
  139 + # Set-up csfindobs for undefined spatial parameters
  140 + findobs = cscripts.csfindobs()
  141 + findobs['datapath'] = self._datapath
  142 + findobs['prodname'] = 'unit-test'
  143 + findobs['ra'] = 'UNDEFINED'
  144 + findobs['dec'] = 'UNDEFINED'
  145 + findobs['rad'] = 'UNDEFINED'
  146 + findobs['outfile'] = 'runlist_py2.dat'
  147 + findobs['logfile'] = 'csfindobs_py2.log'
  148 + findobs['chatter'] = 3
  149 +
  150 + # Run csfindobs script and save run list
  151 + findobs.execute()
  152 +
  153 + # Check run list
  154 + self._check_runlist('runlist_py2.dat')
  155 +
  156 + # Set-up csfindobs for user expression
  157 + findobs = cscripts.csfindobs()
  158 + findobs['datapath'] = self._datapath
  159 + findobs['prodname'] = 'unit-test'
  160 + findobs['ra'] = 83.63
  161 + findobs['dec'] = 22.01
  162 + findobs['rad'] = 0.0
  163 + findobs['expression'] = 'ONTIME<10.0'
  164 + findobs['outfile'] = 'runlist_py3.dat'
  165 + findobs['logfile'] = 'csfindobs_py3.log'
  166 + findobs['chatter'] = 4
  167 +
  168 + # Run csfindobs script and save run list
  169 + findobs.execute()
  170 +
  171 + # Check run list
  172 + self._check_runlist('runlist_py3.dat', runs=0)
  173 +
130 # Return 174 # Return
131 return 175 return
132 176
133 # Check run list result file 177 # Check run list result file
134 - def _check_runlist(self, filename): 178 + def _check_runlist(self, filename, runs=1):
135 """ 179 """
136 - Check run list file. 180 + Check run list file
  181 +
  182 + Parameters
  183 + ----------
  184 + filename : str
  185 + Run list file name
  186 + runs : int, optional
  187 + Expected number of runs in runlist file
137 """ 188 """
138 # Open run list file as CSV file 189 # Open run list file as CSV file
139 runlist = gammalib.GCsv(filename) 190 runlist = gammalib.GCsv(filename)
140 191
141 # Check dimensions 192 # Check dimensions
142 - self.test_value(runlist.ncols(), 1,  
143 - 'Check for single column in runlist file')  
144 - self.test_value(runlist.nrows(), 1,  
145 - 'Check for single row in runlist file')  
146 - self.test_assert(runlist[0,0] == '0',  
147 - 'Check for value "0" in runlist file') 193 + self.test_value(runlist.nrows(), runs,
  194 + 'Check for number of runs in runlist file')
  195 + if runs > 0:
  196 + self.test_value(runlist.ncols(), 1,
  197 + 'Check for single column in runlist file')
  198 + self.test_value(runlist[0,0], '0',
  199 + 'Check for run ID in runlist file')
148 200
149 # Return 201 # Return
150 return 202 return