Johan Bregeon / ctools

Commit a90f5fd4df81054e3949cc31923128833b1a4daf

Authored by Jürgen Knödlseder
1 parent 63661f50

Increase branch coverage of csfindobs (#1828)

Inline Side-by-side
Showing 2 changed files with 152 additions and 95 deletions
cscripts/csfindobs.py
  View file @a90f5fd
... ... @@ -30,13 +30,13 @@ import ctools
30 30 # =============== #
31 31 class csfindobs(ctools.cscript):
32 32 """
33   - Find observations from an IACT data store.
  33 + Find observations from an IACT data store
34 34 """
35 35  
36 36 # Constructor
37 37 def __init__(self, *argv):
38 38 """
39   - Constructor.
  39 + Constructor
40 40 """
41 41 # Set name
42 42 self._name = 'csfindobs'
... ... @@ -50,8 +50,7 @@ class csfindobs(ctools.cscript):
50 50 self._obs_index = ''
51 51 self._runs = []
52 52  
53   - # Initialise application by calling the appropriate class
54   - # constructor.
  53 + # Initialise application by calling the appropriate class constructor
55 54 self._init_cscript(argv)
56 55  
57 56 # Return
... ... @@ -76,13 +75,13 @@ class csfindobs(ctools.cscript):
76 75 # Master index file name
77 76 master_indx = self['master_indx'].string()
78 77  
79   - # Intiialise flag if spatial selection is required
  78 + # Initialise flag if spatial selection is required
80 79 self._select_radec = True
81 80  
82 81 # Initialise invalid radius
83 82 self._radius = 0.0
84 83  
85   - # Check for vailidity of spatial parameters
  84 + # Check for validity of spatial parameters
86 85 if (self['ra'].is_valid() and
87 86 self['dec'].is_valid() and
88 87 self['rad'].is_valid()):
... ... @@ -91,7 +90,9 @@ class csfindobs(ctools.cscript):
91 90 self._ra = self['ra'].real()
92 91 self._dec = self['dec'].real()
93 92 self._radius = self['rad'].real()
94   -
  93 +
  94 + # ... otherwise signal that there are no spatial parameters for
  95 + # selection
95 96 else:
96 97 self._select_radec = False
97 98  
... ... @@ -107,49 +108,51 @@ class csfindobs(ctools.cscript):
107 108 if self._read_ahead():
108 109 self['outfile'].filename()
109 110  
110   - # Open master index file and look for prodname
  111 + # Set filename of JSON master file and raise an exception if the file
  112 + # does not exist
111 113 master_file = os.path.join(self._datapath, master_indx)
112 114 if not os.path.isfile(master_file):
113   - raise RuntimeError('FITS data store not available. '+
114   - 'No master index file found at "'+
115   - master_file+'". Make sure the file '+
116   - 'is copied from the server and your '+
117   - 'datapath is set correctly.')
  115 + msg = ('FITS data store not available. No master index file found '
  116 + 'at "%s". Make sure the file is copied from the server and '
  117 + 'your datapath is set correctly.' % master_file)
  118 + raise RuntimeError(msg)
118 119  
119   - # Open and load JSON file
  120 + # Open and load JSON master file. If the "dataset" key is not available
  121 + # then raise an exception
120 122 json_data = open(master_file).read()
121   - data = json.loads(json_data)
  123 + data = json.loads(json_data)
122 124 if not 'datasets' in data:
123   - raise RuntimeError('Key "datasets" not available in '+
124   - 'master index file.')
  125 + msg = ('Key "datasets" not available in master index file.')
  126 + raise RuntimeError(msg)
125 127  
126   - # Get configurations
  128 + # Get configurations from JSON master file
127 129 configs = data['datasets']
128 130  
129 131 # Initialise obs index file
130 132 self._obs_index = ''
131 133  
132   - # Get HDUs
  134 + # Get name of observation index file
133 135 for config in configs:
134 136 if self._prodname == config['name']:
135 137 self._obs_index = str(os.path.join(self._datapath,
136   - config['obsindx']))
  138 + config['obsindx']))
137 139 break
138 140  
139   - # Check HDUs
  141 + # If the observation index file name is empty then raise an exception
140 142 if self._obs_index == '':
141   - raise RuntimeError('FITS data store "'+self._prodname+
142   - '" not available. Run csiactdata to get '+
143   - 'a list of available storage names')
  143 + msg = ('FITS data store "%s" not available. Run csiactdata to get '
  144 + 'a list of available storage names.' % self._prodname)
  145 + raise RuntimeError(msg)
  146 +
  147 + # If the observation index file is not a FITS file then raise an
  148 + # exception
144 149 filename = gammalib.GFilename(self._obs_index+'[OBS_INDEX]')
145 150 if not filename.is_fits():
146   - raise RuntimeError('Observation index file "'+
147   - self._obs_index+
148   - '[OBS_INDEX]" for FITS data store "'+
149   - self._prodname+
150   - '" not available. Check your master index '+
151   - 'file or run csiactdata to get a list of '+
152   - 'available storage names.')
  151 + msg = ('Observation index file "%s[OBS_INDEX]" for FITS data store '
  152 + '"%s" not available. Check your master index file or run '
  153 + 'csiactdata to get a list of available storage names.' %
  154 + (self._obs_index, self._prodname))
  155 + raise RuntimeError(msg)
153 156  
154 157 # Write input parameters into logger
155 158 if self._logTerse():
... ... @@ -163,7 +166,7 @@ class csfindobs(ctools.cscript):
163 166 # Public methods
164 167 def run(self):
165 168 """
166   - Run the script.
  169 + Run the script
167 170 """
168 171 # Switch screen logging on in debug mode
169 172 if self._logDebug():
... ... @@ -171,69 +174,68 @@ class csfindobs(ctools.cscript):
171 174  
172 175 # Get parameters
173 176 self._get_parameters()
174   -
  177 +
  178 + # Write header into logger
  179 + if self._logTerse():
  180 + self._log('\n')
  181 + self._log.header1('Find observations')
  182 +
175 183 # Initialise run list
176 184 self._runs = []
177 185  
178   - # Initialise selction expression
  186 + # Initialise selection expression
179 187 expr = ''
180 188  
181   - # Add spatial expression if possible
  189 + # If spatial selection is requested then add an angular separation
  190 + # expression to the expression string
182 191 if self._select_radec:
183 192 expr += 'ANGSEP('+str(self._ra)+','+str(self._dec)+ \
184 193 ',RA_PNT,DEC_PNT)<='+str(self._radius)
185 194  
186   - # Add connector if expression is empty
  195 + # Add '&&' connector if expression is not empty
187 196 if len(expr):
188 197 expr += '&&'
189 198  
190   - # Add quality expression
  199 + # Add always quality expression
191 200 expr += 'QUALITY<='+str(self['min_qual'].integer())
192 201  
193   - # Add user expression
  202 + # Add user expression is one has been specified
194 203 expression = self['expression'].string()
195 204 if (expression != 'NONE' and expression != 'INDEF' and
196 205 len(expression) > 0):
197 206 expr += '&&' + expression
198   -
199   - # Initialise filename
200   - openfile = self._obs_index+'[OBS_INDEX]['+expr+']'
201   -
202   - # Open file
203   - fits = gammalib.GFits(openfile)
204   -
205   - # Check for extistence of rows
206   - if fits['OBS_INDEX'].nrows() > 0:
207   -
208   - # Get HDU
209   - obs_indx = fits['OBS_INDEX']
210   -
211   - # Read observation IDs into array
212   - for i in range(obs_indx.nrows()):
213   - self._runs.append(obs_indx['OBS_ID'][i])
214 207  
215   - # Dump expression
  208 + # Write the expression into the logger
216 209 if self._logNormal():
217   - self._log('\n')
218   - self._log.header3('Expression')
219   - self._log(expr)
220   - self._log('\n')
  210 + self._log(gammalib.parformat('Expression'))
  211 + self._log(expr+'\n')
  212 +
  213 + # Set filename including the selection expression
  214 + filename = self._obs_index+'[OBS_INDEX]['+expr+']'
  215 +
  216 + # Open observation index FITS file
  217 + fits = gammalib.GFits(filename)
  218 +
  219 + # If there are entries in the observation index table then append the
  220 + # observations identifiers to the run list
  221 + obs_index = fits['OBS_INDEX']
  222 + if obs_index.nrows() > 0:
  223 + for i in range(obs_index.nrows()):
  224 + self._runs.append(obs_index['OBS_ID'][i])
221 225  
222   - # Dump number of observations
  226 + # Write the number of observations into the logger
223 227 if self._logTerse():
  228 + self._log(gammalib.parformat('Observations'))
  229 + self._log(len(self._runs))
224 230 self._log('\n')
225   - self._log.header3('Observations')
226   - self._log('Found '+str(len(self._runs))+' observations')
227   - self._log('\n')
228   -
229   - # Dump number of observations
230   - if self._logVerbose():
231   - self._log('\n')
232   - self._log.header3('Observation IDs')
233   - for run in self._runs:
  231 +
  232 + # Write the observation identifiers into the logger
  233 + if self._logNormal() and len(self._runs) > 0:
  234 + for i, run in enumerate(self._runs):
  235 + self._log(gammalib.parformat('Observation %d' % (i+1)))
234 236 self._log(str(run)+'\n')
235   -
236   - # Close file
  237 +
  238 + # Close FITS file
237 239 fits.close()
238 240  
239 241 # Return
... ... @@ -241,7 +243,7 @@ class csfindobs(ctools.cscript):
241 243  
242 244 def execute(self):
243 245 """
244   - Execute the script.
  246 + Execute the script
245 247 """
246 248 # Open logfile
247 249 self.logFileOpen()
... ... @@ -260,7 +262,7 @@ class csfindobs(ctools.cscript):
260 262  
261 263 def save(self):
262 264 """
263   - Save runlist.
  265 + Save runlist
264 266 """
265 267 # Write header
266 268 if self._logTerse():
... ... @@ -270,12 +272,14 @@ class csfindobs(ctools.cscript):
270 272 # Get filename
271 273 outfile = self['outfile'].filename()
272 274  
273   - # Check for clobber
  275 + # If the runlist file exists but the clobber flag is not set then signal
  276 + # that runlist file exists already
274 277 if outfile.exists() and not self._clobber():
275 278 if self._logTerse():
276   - self._log('File '+outfile+' already exists. ')
277   - self._log('Set clobber=yes to overwrite file.')
278   - self._log('\n')
  279 + self._log('File "'+outfile+'" exists already, runlist not ')
  280 + self._log('saved. Set clobber=yes to overwrite the file.\n')
  281 +
  282 + # ... otherwise save the runlist into a file
279 283 else:
280 284  
281 285 # Log file name
... ... @@ -284,7 +288,7 @@ class csfindobs(ctools.cscript):
284 288 self._log(outfile.url())
285 289 self._log('\n')
286 290  
287   - # Write runs to file
  291 + # Write all runs to file
288 292 f = open(outfile.url(),'w')
289 293 for run in self._runs:
290 294 f.write(str(run)+' \n')
... ... @@ -295,12 +299,13 @@ class csfindobs(ctools.cscript):
295 299  
296 300 def runs(self):
297 301 """
298   - Return run list.
  302 + Return run list
299 303  
300   - Returns:
301   - A list of runs.
  304 + Returns
  305 + -------
  306 + runs : list of str
  307 + A list of runs
302 308 """
303   -
304 309 # Return
305 310 return self._runs
306 311  
... ...
test/test_csfindobs.py
  View file @a90f5fd
... ... @@ -51,7 +51,7 @@ class Test(test):
51 51 # Set test functions
52 52 def set(self):
53 53 """
54   - Set all test functions.
  54 + Set all test functions
55 55 """
56 56 # Set test name
57 57 self.name('csfindobs')
... ... @@ -66,7 +66,7 @@ class Test(test):
66 66 # Test csfindobs on command line
67 67 def _test_cmd(self):
68 68 """
69   - Test csfindobs on the command line.
  69 + Test csfindobs on the command line
70 70 """
71 71 # Set script name
72 72 csfindobs = self._script('csfindobs')
... ... @@ -106,45 +106,97 @@ class Test(test):
106 106 # Test csfindobs from Python
107 107 def _test_python(self):
108 108 """
109   - Test csfindobs from Python.
  109 + Test csfindobs from Python
110 110 """
111 111 # Set-up csfindobs
112 112 findobs = cscripts.csfindobs()
113 113 findobs['datapath'] = self._datapath
114 114 findobs['prodname'] = 'unit-test'
115   - findobs['outfile'] = 'runlist_py1.dat'
116 115 findobs['ra'] = 83.63
117 116 findobs['dec'] = 22.01
118 117 findobs['rad'] = 1.0
  118 + findobs['outfile'] = 'runlist_py1.dat'
119 119 findobs['logfile'] = 'csfindobs_py1.log'
120 120 findobs['chatter'] = 2
121 121  
122   - # Run csfindobs script and save run list
  122 + # Run csfindobs script
123 123 findobs.logFileOpen() # Make sure we get a log file
124 124 findobs.run()
  125 +
  126 + # Check list of runs
  127 + runs = findobs.runs()
  128 + self.test_value(len(runs), 1,
  129 + 'Check for number of runs in runlist')
  130 + self.test_value(str(runs[0]), '0',
  131 + 'Check run ID in runlist')
  132 +
  133 + # Save run list
125 134 findobs.save()
126 135  
127 136 # Check run list
128 137 self._check_runlist('runlist_py1.dat')
129 138  
  139 + # Set-up csfindobs for undefined spatial parameters
  140 + findobs = cscripts.csfindobs()
  141 + findobs['datapath'] = self._datapath
  142 + findobs['prodname'] = 'unit-test'
  143 + findobs['ra'] = 'UNDEFINED'
  144 + findobs['dec'] = 'UNDEFINED'
  145 + findobs['rad'] = 'UNDEFINED'
  146 + findobs['outfile'] = 'runlist_py2.dat'
  147 + findobs['logfile'] = 'csfindobs_py2.log'
  148 + findobs['chatter'] = 3
  149 +
  150 + # Run csfindobs script and save run list
  151 + findobs.execute()
  152 +
  153 + # Check run list
  154 + self._check_runlist('runlist_py2.dat')
  155 +
  156 + # Set-up csfindobs for user expression
  157 + findobs = cscripts.csfindobs()
  158 + findobs['datapath'] = self._datapath
  159 + findobs['prodname'] = 'unit-test'
  160 + findobs['ra'] = 83.63
  161 + findobs['dec'] = 22.01
  162 + findobs['rad'] = 0.0
  163 + findobs['expression'] = 'ONTIME<10.0'
  164 + findobs['outfile'] = 'runlist_py3.dat'
  165 + findobs['logfile'] = 'csfindobs_py3.log'
  166 + findobs['chatter'] = 4
  167 +
  168 + # Run csfindobs script and save run list
  169 + findobs.execute()
  170 +
  171 + # Check run list
  172 + self._check_runlist('runlist_py3.dat', runs=0)
  173 +
130 174 # Return
131 175 return
132 176  
133 177 # Check run list result file
134   - def _check_runlist(self, filename):
  178 + def _check_runlist(self, filename, runs=1):
135 179 """
136   - Check run list file.
  180 + Check run list file
  181 +
  182 + Parameters
  183 + ----------
  184 + filename : str
  185 + Run list file name
  186 + runs : int, optional
  187 + Expected number of runs in runlist file
137 188 """
138 189 # Open run list file as CSV file
139 190 runlist = gammalib.GCsv(filename)
140 191  
141 192 # Check dimensions
142   - self.test_value(runlist.ncols(), 1,
143   - 'Check for single column in runlist file')
144   - self.test_value(runlist.nrows(), 1,
145   - 'Check for single row in runlist file')
146   - self.test_assert(runlist[0,0] == '0',
147   - 'Check for value "0" in runlist file')
  193 + self.test_value(runlist.nrows(), runs,
  194 + 'Check for number of runs in runlist file')
  195 + if runs > 0:
  196 + self.test_value(runlist.ncols(), 1,
  197 + 'Check for single column in runlist file')
  198 + self.test_value(runlist[0,0], '0',
  199 + 'Check for run ID in runlist file')
148 200  
149 201 # Return
150 202 return
... ...