Commit 14d95915490f72a0c500c926cab163c7979683b1

Authored by Michael Mayer
1 parent 2b3d392a

Reducing sonar issues by adding more comments and cover more code with unit tests

cscripts/csiactcopy.py
@@ -91,7 +91,18 @@ class csiactcopy(ctools.cscript): @@ -91,7 +91,18 @@ class csiactcopy(ctools.cscript):
91 def _copy(self, source, clobber): 91 def _copy(self, source, clobber):
92 """ 92 """
93 Copy file to outpath 93 Copy file to outpath
  94 +
  95 + Parameters
  96 + ----------
  97 + source : string
  98 + Path of file to be copied
  99 +
  100 + Returns
  101 + -------
  102 + filesize : float
  103 + Size of the file that was copied
94 """ 104 """
  105 +
95 # Get file destination 106 # Get file destination
96 destination = os.path.join(self._outpath, 107 destination = os.path.join(self._outpath,
97 os.path.relpath(source, self._remote_base)) 108 os.path.relpath(source, self._remote_base))
@@ -155,10 +166,21 @@ class csiactcopy(ctools.cscript): @@ -155,10 +166,21 @@ class csiactcopy(ctools.cscript):
155 def _merge(self, localfits, remotefits, hduname, clobber): 166 def _merge(self, localfits, remotefits, hduname, clobber):
156 """ 167 """
157 Merge remote and local fits files 168 Merge remote and local fits files
158 -  
159 If the local fits file is not present, a new one is created. 169 If the local fits file is not present, a new one is created.
  170 +
  171 + Parameters
  172 + ----------
  173 + localfits : string
  174 + Path of local index FITS file
  175 + remotefits : string
  176 + Path of remote index FITS file
  177 + hduname : string
  178 + Name of HDU extension to be merged
  179 + clobber : bool
  180 + Flag if remote content should overwrite local content
160 """ 181 """
161 182
  183 + # Logging
162 if self._logTerse(): 184 if self._logTerse():
163 self._log.parformat('Local file') 185 self._log.parformat('Local file')
164 self._log(str(localfits)) 186 self._log(str(localfits))
@@ -229,7 +251,7 @@ class csiactcopy(ctools.cscript): @@ -229,7 +251,7 @@ class csiactcopy(ctools.cscript):
229 tmp_hdu = remote[hduname] 251 tmp_hdu = remote[hduname]
230 252
231 # Get old size of remote HDU 253 # Get old size of remote HDU
232 - size = remote_hdu.size() 254 + size = remote_hdu.nrows()
233 255
234 # Extend remote HDU 256 # Extend remote HDU
235 remote_hdu.append_rows(tmp_hdu.nrows()) 257 remote_hdu.append_rows(tmp_hdu.nrows())
@@ -244,7 +266,8 @@ class csiactcopy(ctools.cscript): @@ -244,7 +266,8 @@ class csiactcopy(ctools.cscript):
244 robs_col = remote_hdu['OBS_ID'] 266 robs_col = remote_hdu['OBS_ID']
245 for obs_id in robs_col: 267 for obs_id in robs_col:
246 remote_obs.append(obs_id) 268 remote_obs.append(obs_id)
247 - 269 +
  270 + # Log entries of index files
248 if self._logTerse(): 271 if self._logTerse():
249 self._log.parformat('Remote entries') 272 self._log.parformat('Remote entries')
250 self._log(str(len(remote_obs))) 273 self._log(str(len(remote_obs)))
@@ -268,12 +291,18 @@ class csiactcopy(ctools.cscript): @@ -268,12 +291,18 @@ class csiactcopy(ctools.cscript):
268 291
269 # Remove local obs_id entries 292 # Remove local obs_id entries
270 table_has_obsid = True 293 table_has_obsid = True
  294 +
  295 + # Loop over obs ids and overwrite with remote content
271 while table_has_obsid: 296 while table_has_obsid:
  297 +
  298 + # First remove old rows
272 for i in range(local_hdu.nrows()): 299 for i in range(local_hdu.nrows()):
273 if remote_obs_id == local_hdu['OBS_ID'][i]: 300 if remote_obs_id == local_hdu['OBS_ID'][i]:
274 local_hdu.remove_rows(i,1) 301 local_hdu.remove_rows(i,1)
275 removed_rows += 1 302 removed_rows += 1
276 break 303 break
  304 +
  305 + # Replace with remote content
277 table_has_obsid = False 306 table_has_obsid = False
278 for i in range(local_hdu.nrows()): 307 for i in range(local_hdu.nrows()):
279 if remote_obs_id == local_hdu['OBS_ID'][i]: 308 if remote_obs_id == local_hdu['OBS_ID'][i]:
@@ -309,6 +338,7 @@ class csiactcopy(ctools.cscript): @@ -309,6 +338,7 @@ class csiactcopy(ctools.cscript):
309 # Add rmeote HDUs 338 # Add rmeote HDUs
310 local_hdu.insert_rows(old_local_rows, remote_hdu.nrows()) 339 local_hdu.insert_rows(old_local_rows, remote_hdu.nrows())
311 340
  341 + # Log actions
312 if self._logTerse(): 342 if self._logTerse():
313 if clobber: 343 if clobber:
314 self._log.parformat('Removed local rows') 344 self._log.parformat('Removed local rows')
@@ -362,6 +392,8 @@ class csiactcopy(ctools.cscript): @@ -362,6 +392,8 @@ class csiactcopy(ctools.cscript):
362 # Check if runlist file is available 392 # Check if runlist file is available
363 if self._runlist.exists(): 393 if self._runlist.exists():
364 runfile = open(self._runlist.url()) 394 runfile = open(self._runlist.url())
  395 +
  396 + # Read runlist
365 for line in runfile.readlines(): 397 for line in runfile.readlines():
366 if len(line) == 0: 398 if len(line) == 0:
367 continue 399 continue
@@ -369,6 +401,8 @@ class csiactcopy(ctools.cscript): @@ -369,6 +401,8 @@ class csiactcopy(ctools.cscript):
369 continue 401 continue
370 if len(line.split()) > 0: 402 if len(line.split()) > 0:
371 self._runs.append(int(line.split()[0])) 403 self._runs.append(int(line.split()[0]))
  404 +
  405 + # Close runlist file
372 runfile.close() 406 runfile.close()
373 407
374 # Logging 408 # Logging
@@ -452,8 +486,8 @@ class csiactcopy(ctools.cscript): @@ -452,8 +486,8 @@ class csiactcopy(ctools.cscript):
452 # Initialise flag if SIZE column is present 486 # Initialise flag if SIZE column is present
453 has_size = table.contains('SIZE') 487 has_size = table.contains('SIZE')
454 488
  489 + # Initialise file size
455 if has_size: 490 if has_size:
456 - # Initialise file size  
457 cp_size = 0 491 cp_size = 0
458 492
459 # Initialise remote observation IDs 493 # Initialise remote observation IDs
@@ -463,6 +497,8 @@ class csiactcopy(ctools.cscript): @@ -463,6 +497,8 @@ class csiactcopy(ctools.cscript):
463 497
464 # Log runs that are not available remotely 498 # Log runs that are not available remotely
465 for run in self._runs: 499 for run in self._runs:
  500 +
  501 + # Check for run not in remote data store
466 if not run in remote_ids: 502 if not run in remote_ids:
467 if self._logNormal(): 503 if self._logNormal():
468 self._log('Skip observation '+str(run)+': ') 504 self._log('Skip observation '+str(run)+': ')
@@ -526,6 +562,8 @@ class csiactcopy(ctools.cscript): @@ -526,6 +562,8 @@ class csiactcopy(ctools.cscript):
526 self._log.parformat('Size') 562 self._log.parformat('Size')
527 self._log('%.2f'%(float(cp_size)*1e-6)+' MB') 563 self._log('%.2f'%(float(cp_size)*1e-6)+' MB')
528 self._log('\n') 564 self._log('\n')
  565 +
  566 + # Log more information if requested
529 if self._logVerbose(): 567 if self._logVerbose():
530 self._log('\n') 568 self._log('\n')
531 self._log.header3('File names') 569 self._log.header3('File names')
@@ -556,20 +594,26 @@ class csiactcopy(ctools.cscript): @@ -556,20 +594,26 @@ class csiactcopy(ctools.cscript):
556 self._log('\n') 594 self._log('\n')
557 self._log.header1('Copying files') 595 self._log.header1('Copying files')
558 596
559 - # intialise counter 597 + # Intialise counter
560 k = 0 598 k = 0
561 599
562 - # initialise values for logging 600 + # Initialise values for logging
563 last_fraction = 0.0 601 last_fraction = 0.0
564 fraction_increment = 20.0 602 fraction_increment = 20.0
  603 +
  604 + # Use 10% step increase
565 if self._logNormal(): 605 if self._logNormal():
566 fraction_increment = 10.0 606 fraction_increment = 10.0
  607 +
  608 + # Use 5% step increase
567 if self._logTerse(): 609 if self._logTerse():
568 fraction_increment = 5.0 610 fraction_increment = 5.0
  611 +
  612 + # Use 2% step increase
569 if self._logExplicit(): 613 if self._logExplicit():
570 fraction_increment = 2.0 614 fraction_increment = 2.0
571 615
572 - # initialise logging properties 616 + # Initialise logging properties
573 n_copied = 0 617 n_copied = 0
574 total_size = 0.0 618 total_size = 0.0
575 619
@@ -579,20 +623,25 @@ class csiactcopy(ctools.cscript): @@ -579,20 +623,25 @@ class csiactcopy(ctools.cscript):
579 # Log progress 623 # Log progress
580 fraction = float(k) / float(len(files)) * 100.0 624 fraction = float(k) / float(len(files)) * 100.0
581 while fraction > last_fraction: 625 while fraction > last_fraction:
  626 +
  627 + # Print status of copying procedure
582 if self._logNormal() and not self._logVerbose(): 628 if self._logNormal() and not self._logVerbose():
583 self._log.parformat('Status') 629 self._log.parformat('Status')
584 self._log(str(int(last_fraction))+'%') 630 self._log(str(int(last_fraction))+'%')
585 self._log('\n') 631 self._log('\n')
  632 +
  633 + # Increment fraction for next logging step
586 last_fraction += fraction_increment 634 last_fraction += fraction_increment
587 635
588 # Copy file 636 # Copy file
589 filesize = self._copy(filename, self._clobber()) 637 filesize = self._copy(filename, self._clobber())
590 638
  639 + # Increment counter if file size copied
591 if filesize > 0.0: 640 if filesize > 0.0:
592 total_size += filesize 641 total_size += filesize
593 n_copied += 1 642 n_copied += 1
594 643
595 - # Increment counter 644 + # Increment file counter
596 k += 1 645 k += 1
597 646
598 # Logging 647 # Logging
@@ -600,14 +649,17 @@ class csiactcopy(ctools.cscript): @@ -600,14 +649,17 @@ class csiactcopy(ctools.cscript):
600 self._log.parformat('Status') 649 self._log.parformat('Status')
601 self._log('Finished') 650 self._log('Finished')
602 self._log('\n') 651 self._log('\n')
  652 +
  653 + # Logging about index files
603 if self._logTerse(): 654 if self._logTerse():
604 self._log('\n') 655 self._log('\n')
605 self._log.header1('Updating index files') 656 self._log.header1('Updating index files')
606 657
607 - # Build local index file names 658 + # Build local hdu index file name
608 local_hdu = os.path.join(self._outpath, 659 local_hdu = os.path.join(self._outpath,
609 os.path.relpath(remote_hdu, 660 os.path.relpath(remote_hdu,
610 self._remote_base)) 661 self._remote_base))
  662 + # Build local obs index file name
611 local_obs = os.path.join(self._outpath, 663 local_obs = os.path.join(self._outpath,
612 os.path.relpath(remote_obs, 664 os.path.relpath(remote_obs,
613 self._remote_base)) 665 self._remote_base))
@@ -623,6 +675,7 @@ class csiactcopy(ctools.cscript): @@ -623,6 +675,7 @@ class csiactcopy(ctools.cscript):
623 # Merge remote index files with local files 675 # Merge remote index files with local files
624 self._merge(local_hdu, remote_hdu, 'HDU_INDEX', self._clobber()) 676 self._merge(local_hdu, remote_hdu, 'HDU_INDEX', self._clobber())
625 677
  678 + # Logging
626 if self._logTerse(): 679 if self._logTerse():
627 self._log('\n') 680 self._log('\n')
628 self._log.header3('OBS index') 681 self._log.header3('OBS index')
@@ -634,7 +687,8 @@ class csiactcopy(ctools.cscript): @@ -634,7 +687,8 @@ class csiactcopy(ctools.cscript):
634 # If all files were copied, just copy index files too 687 # If all files were copied, just copy index files too
635 self._copy(remote_hdu, self._clobber()) 688 self._copy(remote_hdu, self._clobber())
636 self._copy(remote_obs, self._clobber()) 689 self._copy(remote_obs, self._clobber())
637 - 690 +
  691 + # Logging
638 if self._logTerse(): 692 if self._logTerse():
639 self._log('\n') 693 self._log('\n')
640 self._log.header3('Master index file') 694 self._log.header3('Master index file')
cscripts/csiactdata.py
@@ -95,6 +95,12 @@ class csiactdata(ctools.cscript): @@ -95,6 +95,12 @@ class csiactdata(ctools.cscript):
95 def _log_configuration(self, config): 95 def _log_configuration(self, config):
96 """ 96 """
97 Log configuration 97 Log configuration
  98 +
  99 + Parameters
  100 + ----------
  101 + config : dict
  102 + Dictionary containing data configuration
  103 +
98 """ 104 """
99 # Log information 105 # Log information
100 if self._logTerse(): 106 if self._logTerse():
@@ -111,8 +117,12 @@ class csiactdata(ctools.cscript): @@ -111,8 +117,12 @@ class csiactdata(ctools.cscript):
111 117
112 # Print additional information 118 # Print additional information
113 for key in config: 119 for key in config:
  120 +
  121 + # Skip keys that were treated above
114 if key in ['name','hduindx','obsindx']: 122 if key in ['name','hduindx','obsindx']:
115 continue 123 continue
  124 +
  125 + # Log residual keys
116 self._log.parformat(str(key)) 126 self._log.parformat(str(key))
117 self._log(str(config[key])) 127 self._log(str(config[key]))
118 self._log('\n') 128 self._log('\n')
cscripts/csiactobs.py
@@ -174,27 +174,45 @@ class csiactobs(ctools.cscript): @@ -174,27 +174,45 @@ class csiactobs(ctools.cscript):
174 174
175 # Get HDUs 175 # Get HDUs
176 for config in configs: 176 for config in configs:
  177 +
  178 + # Check if prodname is present
177 if self._prodname == config['name']: 179 if self._prodname == config['name']:
178 self._hdu_index = str(os.path.join(self._datapath, config['hduindx'])) 180 self._hdu_index = str(os.path.join(self._datapath, config['hduindx']))
179 self._obs_index = str(os.path.join(self._datapath, config['obsindx'])) 181 self._obs_index = str(os.path.join(self._datapath, config['obsindx']))
  182 +
  183 + # Leave loop if index file names were found
180 break 184 break
181 185
182 - # Check HDUs 186 + # Check index files
183 if self._hdu_index == '' or self._obs_index == '': 187 if self._hdu_index == '' or self._obs_index == '':
184 raise RuntimeError('*** ERROR: FITS data store "'+self._prodname+'" not available. Run csiactdata to get a list of available storage names') 188 raise RuntimeError('*** ERROR: FITS data store "'+self._prodname+'" not available. Run csiactdata to get a list of available storage names')
  189 +
  190 + # Check HDU names
185 filename = gammalib.GFilename(self._hdu_index+'[HDU_INDEX]') 191 filename = gammalib.GFilename(self._hdu_index+'[HDU_INDEX]')
186 if not filename.is_fits(): 192 if not filename.is_fits():
187 raise RuntimeError('*** ERROR: HDU index file "'+self._hdu_index+'[HDU_INDEX]" for FITS data store "'+self._prodname+'" not available. Check your master index file or run csiactdata to get a list of available storage names.') 193 raise RuntimeError('*** ERROR: HDU index file "'+self._hdu_index+'[HDU_INDEX]" for FITS data store "'+self._prodname+'" not available. Check your master index file or run csiactdata to get a list of available storage names.')
188 194
189 # Check for existence of 'BKG_SCALE' in the observation index file if required 195 # Check for existence of 'BKG_SCALE' in the observation index file if required
190 if self._use_bkg_scale: 196 if self._use_bkg_scale:
  197 +
  198 + # Create filename
191 filename = gammalib.GFilename(self._obs_index+'[OBS_INDEX]') 199 filename = gammalib.GFilename(self._obs_index+'[OBS_INDEX]')
  200 +
  201 + # Check if it is a FITS file
192 if filename.is_fits(): 202 if filename.is_fits():
  203 +
  204 + # Open FITS file
193 fits = gammalib.GFits(self._obs_index) 205 fits = gammalib.GFits(self._obs_index)
  206 +
  207 + # Check if column "BKG_SCALE" is found and signal its possible usage
194 if not fits['OBS_INDEX'].contains('BKG_SCALE'): 208 if not fits['OBS_INDEX'].contains('BKG_SCALE'):
195 self._use_bkg_scale = False 209 self._use_bkg_scale = False
  210 +
  211 + # Close FITS file
196 fits.close() 212 fits.close()
  213 +
197 else: 214 else:
  215 + # Signal that there is no background scale
198 self._use_bkg_scale = False 216 self._use_bkg_scale = False
199 217
200 # Create base data directory from hdu index file location 218 # Create base data directory from hdu index file location
@@ -212,6 +230,11 @@ class csiactobs(ctools.cscript): @@ -212,6 +230,11 @@ class csiactobs(ctools.cscript):
212 def runlist(self, runlist): 230 def runlist(self, runlist):
213 """ 231 """
214 Set observation list 232 Set observation list
  233 +
  234 + Parameters
  235 + ----------
  236 + runlist : list
  237 + List of observation IDs
215 """ 238 """
216 239
217 # Check if convertable to a python list 240 # Check if convertable to a python list
@@ -228,14 +251,37 @@ class csiactobs(ctools.cscript): @@ -228,14 +251,37 @@ class csiactobs(ctools.cscript):
228 # Return 251 # Return
229 return 252 return
230 253
231 - def _background_spectrum(self, run, prefactor, index, emin = 0.01, emax = 100.0): 254 + def _background_spectrum(self, prefactor, index, emin = 0.01, emax = 100.0):
  255 + """
  256 + Create a background spectrum model dependent on user parameters
  257 +
  258 + Parameters
  259 + ----------
  260 + prefactor : float
  261 + Prefactor parameter of spectral model
  262 + index : float
  263 + Index parameter of spectral model
  264 + emin : float
  265 + Minimum energy (in case a spectral node function is required)
  266 + emax : float
  267 + Maximum energy (in case a spectral node function is required)
  268 +
  269 + Returns
  270 + -------
  271 + spec : `~gammalib.GModelSpectral()`
  272 + Spectral model for the background shape
  273 + """
232 274
233 # Handle constant spectral model 275 # Handle constant spectral model
234 if index == 0.0 and self._bkgpars <= 1: 276 if index == 0.0 and self._bkgpars <= 1:
235 spec = gammalib.GModelSpectralConst() 277 spec = gammalib.GModelSpectralConst()
  278 +
  279 + # Set parameter range
236 spec['Normalization'].min(prefactor / self._bkg_range_factor) 280 spec['Normalization'].min(prefactor / self._bkg_range_factor)
237 spec['Normalization'].max(prefactor * self._bkg_range_factor) 281 spec['Normalization'].max(prefactor * self._bkg_range_factor)
238 spec['Normalization'].value(prefactor) 282 spec['Normalization'].value(prefactor)
  283 +
  284 + # Freeze or release normalisation parameter
239 if self._bkgpars == 0: 285 if self._bkgpars == 0:
240 spec['Normalization'].fix() 286 spec['Normalization'].fix()
241 else: 287 else:
@@ -245,8 +291,10 @@ class csiactobs(ctools.cscript): @@ -245,8 +291,10 @@ class csiactobs(ctools.cscript):
245 291
246 # Create power law model 292 # Create power law model
247 if self._bkgpars <= 2: 293 if self._bkgpars <= 2:
248 - e = gammalib.GEnergy(1.0,'TeV')  
249 - spec = gammalib.GModelSpectralPlaw(prefactor, index, e) 294 +
  295 + # Set Power Law model with arbitrary pivot energy
  296 + pivot = gammalib.GEnergy(1.0,'TeV')
  297 + spec = gammalib.GModelSpectralPlaw(prefactor, index, pivot)
250 298
251 # Set parameter ranges 299 # Set parameter ranges
252 spec[0].min(prefactor / self._bkg_range_factor) 300 spec[0].min(prefactor / self._bkg_range_factor)
@@ -269,19 +317,31 @@ class csiactobs(ctools.cscript): @@ -269,19 +317,31 @@ class csiactobs(ctools.cscript):
269 else: 317 else:
270 318
271 # Create reference powerlaw 319 # Create reference powerlaw
272 - plaw = gammalib.GModelSpectralPlaw(prefactor, index, gammalib.GEnergy(1.0,'TeV')) 320 + pivot = gammalib.GEnergy(1.0,'TeV')
  321 + plaw = gammalib.GModelSpectralPlaw(prefactor, index, pivot)
273 322
274 # Create spectral model and energy values 323 # Create spectral model and energy values
275 spec = gammalib.GModelSpectralNodes() 324 spec = gammalib.GModelSpectralNodes()
  325 +
  326 + # Create logarithmic energy nodes
276 bounds = gammalib.GEbounds(self._bkgpars,gammalib.GEnergy(emin,'TeV'),gammalib.GEnergy(emax,'TeV'), True) 327 bounds = gammalib.GEbounds(self._bkgpars,gammalib.GEnergy(emin,'TeV'),gammalib.GEnergy(emax,'TeV'), True)
  328 +
  329 + # Loop over bounds and set intensity value
277 for i in range(bounds.size()): 330 for i in range(bounds.size()):
278 energy = bounds.elogmean(i) 331 energy = bounds.elogmean(i)
279 - value = plaw.eval(energy, gammalib.GTime()) 332 + value = plaw.eval(energy)
  333 +
  334 + # Append energy, value - tuple to node function
280 spec.append(energy, value) 335 spec.append(energy, value)
  336 +
  337 + # Loop over parameters
281 for par in spec: 338 for par in spec:
282 339
  340 + # Fix energy nodes
283 if 'Energy' in par.name(): 341 if 'Energy' in par.name():
284 par.fix() 342 par.fix()
  343 +
  344 + # Release intensity nodes
285 elif 'Intensity' in par.name(): 345 elif 'Intensity' in par.name():
286 value = par.value() 346 value = par.value()
287 par.scale(value) 347 par.scale(value)
@@ -292,13 +352,36 @@ class csiactobs(ctools.cscript): @@ -292,13 +352,36 @@ class csiactobs(ctools.cscript):
292 return spec 352 return spec
293 353
294 def _iact_background(self, telescope, obs_id, bkg_scale, bkgtype, emin=0.01, emax=100): 354 def _iact_background(self, telescope, obs_id, bkg_scale, bkgtype, emin=0.01, emax=100):
  355 + """
  356 + Create an IACT background model
  357 +
  358 + Parameters
  359 + ----------
  360 + telescope : string
  361 + Name of telescope
  362 + obs_id : string
  363 + Observation ID
  364 + bkg_scale : float
  365 + Background scaling factor
  366 + bkgtype : string
  367 + Type of background (irf,aeff, or gauss)
295 368
  369 + Returns
  370 + -------
  371 + model : `~gammalib.GModelData()`
  372 + Background model for IACT observation
  373 + """
  374 +
296 # handle IrfBackground 375 # handle IrfBackground
297 if bkgtype == 'irf': 376 if bkgtype == 'irf':
  377 +
  378 + # Set parameters to have a constant spectral model
298 prefactor = 1.0 379 prefactor = 1.0
299 index = 0.0 380 index = 0.0
300 prefactor *= bkg_scale 381 prefactor *= bkg_scale
301 - spec = self._background_spectrum(obs_id, prefactor, index, emin, emax) 382 +
  383 + # Create background spectrum
  384 + spec = self._background_spectrum(prefactor, index, emin, emax)
302 385
303 # Create background model instance 386 # Create background model instance
304 bck = gammalib.GCTAModelIrfBackground(spec) 387 bck = gammalib.GCTAModelIrfBackground(spec)
@@ -306,7 +389,7 @@ class csiactobs(ctools.cscript): @@ -306,7 +389,7 @@ class csiactobs(ctools.cscript):
306 # Set AeffBackground 389 # Set AeffBackground
307 elif bkgtype == 'aeff': 390 elif bkgtype == 'aeff':
308 prefactor = bkg_scale * self._bkg_aeff_norm 391 prefactor = bkg_scale * self._bkg_aeff_norm
309 - spec = self._background_spectrum(obs_id, prefactor, self._bkg_aeff_index, emin, emax) 392 + spec = self._background_spectrum(prefactor, self._bkg_aeff_index, emin, emax)
310 393
311 # Create background model instance 394 # Create background model instance
312 bck = gammalib.GCTAModelAeffBackground(spec) 395 bck = gammalib.GCTAModelAeffBackground(spec)
@@ -314,12 +397,14 @@ class csiactobs(ctools.cscript): @@ -314,12 +397,14 @@ class csiactobs(ctools.cscript):
314 # Set Gaussian Background 397 # Set Gaussian Background
315 elif bkgtype == 'gauss': 398 elif bkgtype == 'gauss':
316 prefactor = bkg_scale * self._bkg_gauss_norm 399 prefactor = bkg_scale * self._bkg_gauss_norm
317 - spec = self._background_spectrum(obs_id, prefactor, self._bkg_gauss_index, emin, emax) 400 +
  401 + # Create background spectrum
  402 + spec = self._background_spectrum(prefactor, self._bkg_gauss_index, emin, emax)
318 radial = gammalib.GCTAModelRadialGauss(self._bkg_gauss_sigma) 403 radial = gammalib.GCTAModelRadialGauss(self._bkg_gauss_sigma)
319 bck = gammalib.GCTAModelRadialAcceptance(radial, spec) 404 bck = gammalib.GCTAModelRadialAcceptance(radial, spec)
320 405
321 else: 406 else:
322 - msg = 'Background type "'+self._bkgtype+'" unsupported' 407 + msg = 'Background type "'+bkgtype+'" unsupported'
323 raise RuntimeError(msg) 408 raise RuntimeError(msg)
324 409
325 # Copy model 410 # Copy model
@@ -383,6 +468,7 @@ class csiactobs(ctools.cscript): @@ -383,6 +468,7 @@ class csiactobs(ctools.cscript):
383 eventfile = aefffile = psffile = edispfile = bkgfile = '' 468 eventfile = aefffile = psffile = edispfile = bkgfile = ''
384 eventhdu = aeffhdu = '' 469 eventhdu = aeffhdu = ''
385 470
  471 + # Initialise types and formats
386 types = [] 472 types = []
387 formats = [] 473 formats = []
388 for i in range(hduindx_hdu.nrows()): 474 for i in range(hduindx_hdu.nrows()):
@@ -402,7 +488,11 @@ class csiactobs(ctools.cscript): @@ -402,7 +488,11 @@ class csiactobs(ctools.cscript):
402 hduindx_hdu['FILE_NAME'][index]) 488 hduindx_hdu['FILE_NAME'][index])
403 eventhdu = hduindx_hdu['HDU_NAME'][index] 489 eventhdu = hduindx_hdu['HDU_NAME'][index]
404 eventfile += '['+eventhdu+']' 490 eventfile += '['+eventhdu+']'
  491 +
  492 + # Check if psf file is available
405 if not gammalib.GFilename(eventfile).is_fits(): 493 if not gammalib.GFilename(eventfile).is_fits():
  494 +
  495 + # Print warning that observation will be skipped since no events are present
406 if self._logTerse(): 496 if self._logTerse():
407 self._log('Skipping observation '+str(obs_id)+ 497 self._log('Skipping observation '+str(obs_id)+
408 ': eventfile "'+eventfile+'" not found\n') 498 ': eventfile "'+eventfile+'" not found\n')
@@ -421,7 +511,11 @@ class csiactobs(ctools.cscript): @@ -421,7 +511,11 @@ class csiactobs(ctools.cscript):
421 hduindx_hdu['FILE_NAME'][index]) 511 hduindx_hdu['FILE_NAME'][index])
422 aeffhdu = hduindx_hdu['HDU_NAME'][index] 512 aeffhdu = hduindx_hdu['HDU_NAME'][index]
423 aefffile += '['+aeffhdu+']' 513 aefffile += '['+aeffhdu+']'
  514 +
  515 + # Check if psf file is available
424 if not gammalib.GFilename(aefffile).is_fits(): 516 if not gammalib.GFilename(aefffile).is_fits():
  517 +
  518 + # Print warning that observation will be skipped since no effective area is present
425 if self._logTerse(): 519 if self._logTerse():
426 self._log('Skipping observation '+str(obs_id)+ 520 self._log('Skipping observation '+str(obs_id)+
427 ': effective area "'+aefffile+'" not found\n') 521 ': effective area "'+aefffile+'" not found\n')
@@ -439,7 +533,11 @@ class csiactobs(ctools.cscript): @@ -439,7 +533,11 @@ class csiactobs(ctools.cscript):
439 hduindx_hdu['FILE_DIR'][index], 533 hduindx_hdu['FILE_DIR'][index],
440 hduindx_hdu['FILE_NAME'][index]) 534 hduindx_hdu['FILE_NAME'][index])
441 psffile += '['+hduindx_hdu['HDU_NAME'][index]+']' 535 psffile += '['+hduindx_hdu['HDU_NAME'][index]+']'
  536 +
  537 + # Check if psf file is available
442 if not gammalib.GFilename(psffile).is_fits(): 538 if not gammalib.GFilename(psffile).is_fits():
  539 +
  540 + # Print warning that observation will be skipped since no PSF is present
443 if self._logTerse(): 541 if self._logTerse():
444 self._log('Skipping observation '+str(obs_id)+ 542 self._log('Skipping observation '+str(obs_id)+
445 ': point spread function "'+psffile+ 543 ': point spread function "'+psffile+
@@ -458,7 +556,11 @@ class csiactobs(ctools.cscript): @@ -458,7 +556,11 @@ class csiactobs(ctools.cscript):
458 hduindx_hdu['FILE_DIR'][index], 556 hduindx_hdu['FILE_DIR'][index],
459 hduindx_hdu['FILE_NAME'][index]) 557 hduindx_hdu['FILE_NAME'][index])
460 edispfile += '['+hduindx_hdu['HDU_NAME'][index]+']' 558 edispfile += '['+hduindx_hdu['HDU_NAME'][index]+']'
  559 +
  560 + # Check if edisp file is available
461 if not gammalib.GFilename(edispfile).is_fits(): 561 if not gammalib.GFilename(edispfile).is_fits():
  562 +
  563 + # Print warning that edisp cannot be used for this observation
462 if self._logTerse(): 564 if self._logTerse():
463 self._log('Warning: observation '+str(obs_id)+ 565 self._log('Warning: observation '+str(obs_id)+
464 ' has no energy dispersion "'+edispfile+ 566 ' has no energy dispersion "'+edispfile+
@@ -480,17 +582,22 @@ class csiactobs(ctools.cscript): @@ -480,17 +582,22 @@ class csiactobs(ctools.cscript):
480 bkgfile += '['+hduindx_hdu['HDU_NAME'][index]+']' 582 bkgfile += '['+hduindx_hdu['HDU_NAME'][index]+']'
481 if 'irf' in bkg_mod_hierarchy and not gammalib.GFilename(bkgfile).is_fits(): 583 if 'irf' in bkg_mod_hierarchy and not gammalib.GFilename(bkgfile).is_fits():
482 bkg_mod_hierarchy.remove('irf') 584 bkg_mod_hierarchy.remove('irf')
  585 +
  586 + # Throw warning if IRF background cannot be used
483 if self._logTerse(): 587 if self._logTerse():
484 self._log('Warning: observation '+str(obs_id)+ 588 self._log('Warning: observation '+str(obs_id)+
485 ' has no background information (file="'+ 589 ' has no background information (file="'+
486 bkgfile+'"). IRF background cannot be used\n') 590 bkgfile+'"). IRF background cannot be used\n')
487 bkgfile = '' 591 bkgfile = ''
488 if len(bkg_mod_hierarchy) == 0: 592 if len(bkg_mod_hierarchy) == 0:
  593 +
  594 + # Skip observation if no background can be used
489 if self._logTerse(): 595 if self._logTerse():
490 self._log('Skipping observation '+str(obs_id)+ 596 self._log('Skipping observation '+str(obs_id)+
491 ': No background can be used\n') 597 ': No background can be used\n')
492 continue 598 continue
493 else: 599 else:
  600 + # Log if we fall back to next background approach
494 if self._logTerse(): 601 if self._logTerse():
495 self._log('Observation '+str(obs_id)+ 602 self._log('Observation '+str(obs_id)+
496 ': Falling back to background "'+ 603 ': Falling back to background "'+
@@ -499,15 +606,21 @@ class csiactobs(ctools.cscript): @@ -499,15 +606,21 @@ class csiactobs(ctools.cscript):
499 # Close hdu index file 606 # Close hdu index file
500 hduindx.close() 607 hduindx.close()
501 608
502 - # Handle background scale information if available 609 + # Initialise background scale
503 bkg_scale = 1.0 610 bkg_scale = 1.0
  611 +
  612 + # Check if background scale should be used
504 if self._use_bkg_scale: 613 if self._use_bkg_scale:
  614 +
  615 + # Read background scale from fits file
505 obsindx = gammalib.GFits(self._obs_index+'[OBS_INDEX]'+obs_selection) 616 obsindx = gammalib.GFits(self._obs_index+'[OBS_INDEX]'+obs_selection)
506 bkg_scale = obsindx['OBS_INDEX']['BKG_SCALE'][0] 617 bkg_scale = obsindx['OBS_INDEX']['BKG_SCALE'][0]
507 obsindx.close() 618 obsindx.close()
508 619
509 - # Open fits file to determine the observation name 620 + # Open event FITS file
510 fits = gammalib.GFits(eventfile) 621 fits = gammalib.GFits(eventfile)
  622 +
  623 + # Get object and telescope strings from event hdu
511 events = fits[eventhdu] 624 events = fits[eventhdu]
512 object_name = events.string('OBJECT') 625 object_name = events.string('OBJECT')
513 telescope = events.string('TELESCOP') 626 telescope = events.string('TELESCOP')
@@ -550,6 +663,7 @@ class csiactobs(ctools.cscript): @@ -550,6 +663,7 @@ class csiactobs(ctools.cscript):
550 self._log('Adding observation '+str(obs_id)+ 663 self._log('Adding observation '+str(obs_id)+
551 ' ("'+object_name+'")\n') 664 ' ("'+object_name+'")\n')
552 665
  666 + # log more details
553 if self._logExplicit(): 667 if self._logExplicit():
554 self._log(' Event file: '+eventfile+'\n') 668 self._log(' Event file: '+eventfile+'\n')
555 self._log(' Effective area: '+aefffile+'\n') 669 self._log(' Effective area: '+aefffile+'\n')
@@ -615,10 +729,13 @@ class csiactobs(ctools.cscript): @@ -615,10 +729,13 @@ class csiactobs(ctools.cscript):
615 self._log('\n') 729 self._log('\n')
616 730
617 else: 731 else:
  732 + # Print warning if no observation could be be used from runlist
618 self._log.header2('WARNING: No observation from given runlist available') 733 self._log.header2('WARNING: No observation from given runlist available')
619 734
620 # Append models provided by 'inmodels' if necessary 735 # Append models provided by 'inmodels' if necessary
621 - if not self._inmodels == None: 736 + if self._inmodels.size() > 0:
  737 +
  738 + # Logging
622 if self._logTerse(): 739 if self._logTerse():
623 self._log('\n') 740 self._log('\n')
624 self._log.header1('Appending models') 741 self._log.header1('Appending models')
test/data/iactdata/master.json
@@ -3,7 +3,14 @@ @@ -3,7 +3,14 @@
3 { 3 {
4 "name": "unit-test", 4 "name": "unit-test",
5 "obsindx": "obs-index.fits.gz", 5 "obsindx": "obs-index.fits.gz",
6 - "hduindx": "hdu-index.fits.gz" 6 + "hduindx": "hdu-index.fits.gz",
  7 + "comment": "this is a comment"
  8 + },
  9 + {
  10 + "name": "unit-test-that-doesnt-exist",
  11 + "obsindx": "badfile.fits.gz",
  12 + "hduindx": "unavailable_file.fits.gz",
  13 + "comment": "this is a comment"
7 } 14 }
8 ] 15 ]
9 } 16 }
test/test_csiactdata.py
@@ -108,6 +108,13 @@ class Test(test): @@ -108,6 +108,13 @@ class Test(test):
108 # Run csiactdata script and save run list 108 # Run csiactdata script and save run list
109 iactdata.logFileOpen() # Make sure we get a log file 109 iactdata.logFileOpen() # Make sure we get a log file
110 iactdata.run() 110 iactdata.run()
  111 +
  112 + # Get available configs
  113 + names = iactdata.names()
  114 +
  115 + # Check if there is at least one config available
  116 + self.test_assert(len(names) > 0,
  117 + 'Check successful execution from Python')
111 118
112 # Return 119 # Return
113 return 120 return
test/test_csiactobs.py
@@ -77,7 +77,7 @@ class Test(test): @@ -77,7 +77,7 @@ class Test(test):
77 ' infile="runlist_cmd1.dat"'+ \ 77 ' infile="runlist_cmd1.dat"'+ \
78 ' bkgpars=1'+\ 78 ' bkgpars=1'+\
79 ' outobs="obs_iactobs_cmd1.xml"'+ \ 79 ' outobs="obs_iactobs_cmd1.xml"'+ \
80 - ' outmodel="bgd_iactobs_cmd1.xml"'+ \ 80 + ' outmodel="bkg_iactobs_cmd1.xml"'+ \
81 ' logfile="csiactobs_cmd1.log" chatter=1' 81 ' logfile="csiactobs_cmd1.log" chatter=1'
82 82
83 # Check if execution of wrong command fails 83 # Check if execution of wrong command fails
@@ -92,7 +92,7 @@ class Test(test): @@ -92,7 +92,7 @@ class Test(test):
92 self._check_obsdef('obs_iactobs_cmd1.xml') 92 self._check_obsdef('obs_iactobs_cmd1.xml')
93 93
94 # Check model definition XML file 94 # Check model definition XML file
95 - self._check_moddef('bgd_iactobs_cmd1.xml') 95 + self._check_moddef('bkg_iactobs_cmd1.xml')
96 96
97 # Setup csiactobs command 97 # Setup csiactobs command
98 cmd = csiactobs+' datapath="data_path_that_does_not_exist"'+ \ 98 cmd = csiactobs+' datapath="data_path_that_does_not_exist"'+ \
@@ -100,12 +100,25 @@ class Test(test): @@ -100,12 +100,25 @@ class Test(test):
100 ' infile="runlist_cmd1.dat"'+ \ 100 ' infile="runlist_cmd1.dat"'+ \
101 ' bkgpars=1'+\ 101 ' bkgpars=1'+\
102 ' outobs="obs_iactobs_cmd2.xml"'+ \ 102 ' outobs="obs_iactobs_cmd2.xml"'+ \
103 - ' outmodel="bgd_iactobs_cmd2.xml"'+ \ 103 + ' outmodel="bkg_iactobs_cmd2.xml"'+ \
104 ' logfile="csiactobs_cmd2.log" chatter=1' 104 ' logfile="csiactobs_cmd2.log" chatter=1'
105 105
106 # Check if execution failed 106 # Check if execution failed
107 self.test_assert(self._execute(cmd) != 0, 107 self.test_assert(self._execute(cmd) != 0,
108 - 'Check invalid input file when executed from command line') 108 + 'Check invalid input datapath when executed from command line')
  109 +
  110 + # Setup csiactobs command
  111 + cmd = csiactobs+' datapath="'+self._datapath+'"'+ \
  112 + ' prodname="unit-test-doesnt-exist"'+ \
  113 + ' infile="runlist_cmd1.dat"'+ \
  114 + ' bkgpars=1'+\
  115 + ' outobs="obs_iactobs_cmd3.xml"'+ \
  116 + ' outmodel="bkg_iactobs_cmd3.xml"'+ \
  117 + ' logfile="csiactobs_cmd3.log" chatter=1'
  118 +
  119 + # Check if execution failed
  120 + self.test_assert(self._execute(cmd) != 0,
  121 + 'Check invalid input prodname when executed from command line')
109 122
110 # Return 123 # Return
111 return 124 return
@@ -122,7 +135,7 @@ class Test(test): @@ -122,7 +135,7 @@ class Test(test):
122 iactobs['infile'] = 'runlist_py1.dat' 135 iactobs['infile'] = 'runlist_py1.dat'
123 iactobs['bkgpars'] = 1 136 iactobs['bkgpars'] = 1
124 iactobs['outobs'] = 'obs_iactobs_py1.xml' 137 iactobs['outobs'] = 'obs_iactobs_py1.xml'
125 - iactobs['outmodel'] = 'bgd_iactobs_py1.xml' 138 + iactobs['outmodel'] = 'bkg_iactobs_py1.xml'
126 iactobs['logfile'] = 'csiactobs_py1.log' 139 iactobs['logfile'] = 'csiactobs_py1.log'
127 iactobs['chatter'] = 2 140 iactobs['chatter'] = 2
128 141
@@ -135,7 +148,93 @@ class Test(test): @@ -135,7 +148,93 @@ class Test(test):
135 self._check_obsdef('obs_iactobs_py1.xml') 148 self._check_obsdef('obs_iactobs_py1.xml')
136 149
137 # Check model definition XML file 150 # Check model definition XML file
138 - self._check_moddef('bgd_iactobs_py1.xml') 151 + self._check_moddef('bkg_iactobs_py1.xml')
  152 +
  153 + # Create test runlist
  154 + runlist = ['0']
  155 +
  156 + # Set-up csiactobs - part 2 (set runlist via function, 2 background pars)
  157 + iactobs = cscripts.csiactobs()
  158 + iactobs['datapath'] = self._datapath
  159 + iactobs['prodname'] = 'unit-test'
  160 + iactobs['bkgpars'] = 2
  161 + iactobs['outobs'] = 'obs_iactobs_py2.xml'
  162 + iactobs['outmodel'] = 'bkg_iactobs_py2.xml'
  163 + iactobs['logfile'] = 'csiactobs_py2.log'
  164 + iactobs['chatter'] = 3
  165 + iactobs.runlist(runlist)
  166 +
  167 + # Run csiactobs script and save run list
  168 + iactobs.logFileOpen() # Make sure we get a log file
  169 + iactobs.run()
  170 + iactobs.save()
  171 +
  172 + # Test return functions
  173 + obs = iactobs.obs()
  174 + ebounds = iactobs.ebounds()
  175 +
  176 + self.test_assert(obs.size() == 1,
  177 + 'Check output observation container')
  178 + self.test_assert(ebounds.size() == 0,
  179 + 'Check output energy boundaries container')
  180 +
  181 + # Check observation definition XML file
  182 + self._check_obsdef('obs_iactobs_py2.xml')
  183 +
  184 + # Check model definition XML file
  185 + self._check_moddef('bkg_iactobs_py2.xml')
  186 +
  187 + # Set-up csiactobs - part 3 (use a large number of free parameters, aeff background)
  188 + iactobs = cscripts.csiactobs()
  189 + iactobs['datapath'] = self._datapath
  190 + iactobs['prodname'] = 'unit-test'
  191 + iactobs['infile'] = 'runlist_py1.dat'
  192 + iactobs['bkgpars'] = 8
  193 + iactobs['bkg_mod_hiera'] = 'aeff'
  194 + iactobs['outobs'] = 'obs_iactobs_py3.xml'
  195 + iactobs['outmodel'] = 'bkg_iactobs_py3.xml'
  196 + iactobs['logfile'] = 'csiactobs_py3.log'
  197 + iactobs['chatter'] = 4
  198 +
  199 + # Run csiactobs script and save run list
  200 + iactobs.logFileOpen() # Make sure we get a log file
  201 + iactobs.run()
  202 + iactobs.save()
  203 +
  204 + # Check observation definition XML file
  205 + self._check_obsdef('obs_iactobs_py3.xml')
  206 +
  207 + # Check model definition XML file
  208 + self._check_moddef('bkg_iactobs_py3.xml')
  209 +
  210 + # Set-up csiactobs - part 4 (use a Gauss background and inmodel parameter)
  211 + iactobs = cscripts.csiactobs()
  212 + iactobs['datapath'] = self._datapath
  213 + iactobs['inmodel'] = 'results.xml'
  214 + iactobs['prodname'] = 'unit-test'
  215 + iactobs['infile'] = 'runlist_py1.dat'
  216 + iactobs['bkgpars'] = 1
  217 + iactobs['bkg_mod_hiera'] = 'gauss'
  218 + iactobs['outobs'] = 'NONE'
  219 + iactobs['outmodel'] = 'NONE'
  220 + iactobs['logfile'] = 'csiactobs_py4.log'
  221 + iactobs['chatter'] = 2
  222 +
  223 + # Run csiactobs script
  224 + iactobs.logFileOpen() # Make sure we get a log file
  225 + iactobs.run()
  226 +
  227 + # Get observations and models
  228 + observations = iactobs.obs()
  229 + models = observations.models()
  230 +
  231 + # Check number of models
  232 + self.test_assert(models.size() == 3,
  233 + 'Check for three models in container')
  234 +
  235 + # Check number of observations
  236 + self.test_assert(observations.size() == 1,
  237 + 'Check for one observation in container')
139 238
140 # Return 239 # Return
141 return 240 return