Fabio Iocco / ctools

Commit 77e1bab8b048604f0a7a96952ac7fd3a26524d17

Authored by Michael Mayer
1 parent ff0d52ee

Added a more realistic and larger IACT test database and added various IACT test… 

… cases, restructured csiactobs to avoid code duplication and added csobsdef::obs() function to retrieve observation container
Inline Side-by-side
Showing 24 changed files with 427 additions and 224 deletions
cscripts/Makefile.am
  View file @77e1bab
@@ -155,14 +155,24 @@ testdata = $(top_srcdir)/test/data/crab.xml \ @@ -155,14 +155,24 @@ testdata = $(top_srcdir)/test/data/crab.xml \
155 $(top_srcdir)/test/data/model_filefunction.txt \ 155 $(top_srcdir)/test/data/model_filefunction.txt \
156 $(top_srcdir)/test/data/model_map.fits.gz \ 156 $(top_srcdir)/test/data/model_map.fits.gz \
157 $(top_srcdir)/test/data/model_mapcube.fits.gz \ 157 $(top_srcdir)/test/data/model_mapcube.fits.gz \
158 - $(top_srcdir)/test/data/iact_runlist.dat 158 + $(top_srcdir)/test/data/iact_runlist.dat \
  159 + $(top_srcdir)/test/data/iact_runlist_1.dat \
  160 + $(top_srcdir)/test/data/iact_runlist_1.dat
159 161
160 # Test data to be distributed in $(pyexecdir)/cscripts/tests/data/iactdata 162 # Test data to be distributed in $(pyexecdir)/cscripts/tests/data/iactdata
161 testiactdata = $(top_srcdir)/test/data/iactdata/hdu-index.fits.gz \ 163 testiactdata = $(top_srcdir)/test/data/iactdata/hdu-index.fits.gz \
162 $(top_srcdir)/test/data/iactdata/master.json \ 164 $(top_srcdir)/test/data/iactdata/master.json \
163 $(top_srcdir)/test/data/iactdata/obs-index.fits.gz \ 165 $(top_srcdir)/test/data/iactdata/obs-index.fits.gz \
164 - $(top_srcdir)/test/data/iactdata/irf_file.fits.gz \  
165 - $(top_srcdir)/test/data/iactdata/events.fits.gz 166 + $(top_srcdir)/test/data/iactdata/events/irf_file.fits.gz \
  167 + $(top_srcdir)/test/data/iactdata/events/events_0.fits.gz \
  168 + $(top_srcdir)/test/data/iactdata/events/events_1.fits.gz \
  169 + $(top_srcdir)/test/data/iactdata/events/events_2.fits.gz \
  170 + $(top_srcdir)/test/data/iactdata/events/events_3.fits.gz \
  171 + $(top_srcdir)/test/data/iactdata/events/events_4.fits.gz \
  172 + $(top_srcdir)/test/data/iactdata/events/events_5.fits.gz \
  173 + $(top_srcdir)/test/data/iactdata/events/events_6.fits.gz \
  174 + $(top_srcdir)/test/data/iactdata/events/events_7.fits.gz \
  175 + $(top_srcdir)/test/data/iactdata/events/events_8.fits.gz
166 176
167 # Install scripts, parfiles, test scripts and test data. Note that we don't 177 # Install scripts, parfiles, test scripts and test data. Note that we don't
168 # need to prepend $(DESTDIR) here (not fully clear why, but that's the way 178 # need to prepend $(DESTDIR) here (not fully clear why, but that's the way
cscripts/csiactcopy.py
  View file @77e1bab
@@ -377,9 +377,7 @@ class csiactcopy(ctools.cscript): @@ -377,9 +377,7 @@ class csiactcopy(ctools.cscript):
377 377
378 # Copy all data if runlist file is 'NONE' 378 # Copy all data if runlist file is 'NONE'
379 elif self._runlist == 'NONE': 379 elif self._runlist == 'NONE':
380 - if self._logTerse():  
381 - self._log('Copy all available data')  
382 - self._log('\n') 380 + self._log_string(gammalib.TERSE, 'Copy all available data')
383 381
384 # Raise exception if file not valid 382 # Raise exception if file not valid
385 else: 383 else:
cscripts/csiactobs.py
  View file @77e1bab
@@ -438,6 +438,94 @@ class csiactobs(ctools.cscript): @@ -438,6 +438,94 @@ class csiactobs(ctools.cscript):
438 # Return 438 # Return
439 return 439 return
440 440
  441 + def _get_filename(self, hdu, hierarchy, formats):
  442 + """
  443 + Retrieves a filename from a hdu index
  444 +
  445 + Parameters
  446 + ----------
  447 + hdu : `gammalib.GFitsTable`
  448 + HDU FITS table
  449 + hierarchy : list of strings
  450 + List of strings containing the hierarchy how to look for files
  451 + formats : list of strings
  452 + File formats available for this observation
  453 +
  454 + Returns
  455 + -------
  456 + filename : string
  457 + Filename of requested file
  458 + """
  459 +
  460 + # Initialise filename
  461 + filename = ''
  462 +
  463 + # Handle hierarchies
  464 + index = -1
  465 +
  466 + # Loop over possible formats
  467 + for version in hierarchy:
  468 +
  469 + # Count number of same available versions
  470 + n = formats.count(version)
  471 +
  472 + # Leave loop and store index of first occurence
  473 + if n > 0:
  474 + index = formats.index(version)
  475 + break
  476 +
  477 + # Build filename from information stored in hduindx_hdu
  478 + if index >= 0:
  479 + filename = os.path.join(self._subdir,
  480 + hdu['FILE_DIR'][index],
  481 + hdu['FILE_NAME'][index])
  482 + filehdu = hdu['HDU_NAME'][index]
  483 + filename += '['+filehdu+']'
  484 +
  485 + # Return filename
  486 + return filename
  487 +
  488 +
  489 + def _is_present(self, filename, obs_id, filetype):
  490 + """
  491 + Checks if a filename is present
  492 +
  493 + Parameters
  494 + ----------
  495 + filename : string
  496 + Filename
  497 + obs_id : int
  498 + Observation ID
  499 + filetype : string
  500 + Type of file
  501 +
  502 + Returns
  503 + -------
  504 + present : bool
  505 + Flag signaling if file is present
  506 + """
  507 +
  508 + # Initialise return value
  509 + present = True
  510 +
  511 + # Create filename instance
  512 + fname = gammalib.GFilename(filename)
  513 +
  514 + # Check if file is empty
  515 + if fname.is_empty():
  516 + self._log_string(gammalib.TERSE, 'Skipping observation '+str(obs_id)+
  517 + ': No '+filetype+' found\n')
  518 + present = False
  519 +
  520 + # Check if file exists
  521 + elif not fname.exists():
  522 + self._log_string(gammalib.TERSE, 'Skipping observation '+str(obs_id)+
  523 + ': '+filetype+' "'+filename+'" does not exist\n')
  524 + present = False
  525 +
  526 + # Return boolean
  527 + return present
  528 +
441 529
442 # Public methods 530 # Public methods
443 def run(self): 531 def run(self):
@@ -477,149 +565,77 @@ class csiactobs(ctools.cscript): @@ -477,149 +565,77 @@ class csiactobs(ctools.cscript):
477 hduindx = gammalib.GFits(self._hdu_index+'[HDU_INDEX]'+obs_selection) 565 hduindx = gammalib.GFits(self._hdu_index+'[HDU_INDEX]'+obs_selection)
478 hduindx_hdu = hduindx['HDU_INDEX'] 566 hduindx_hdu = hduindx['HDU_INDEX']
479 567
480 - # Initialise files and hdu names  
481 - eventfile = aefffile = psffile = edispfile = bkgfile = ''  
482 - eventhdu = aeffhdu = ''  
483 -  
484 # Initialise types and formats 568 # Initialise types and formats
485 - types = []  
486 formats = [] 569 formats = []
487 for i in range(hduindx_hdu.nrows()): 570 for i in range(hduindx_hdu.nrows()):
488 - types.append(hduindx_hdu['HDU_TYPE'][i])  
489 formats.append(hduindx_hdu['HDU_CLASS'][i]) 571 formats.append(hduindx_hdu['HDU_CLASS'][i])
490 572
491 - # Handle events  
492 - index = -1  
493 - for version in self._ev_hiera:  
494 - n = formats.count(version)  
495 - if n > 0:  
496 - index = formats.index(version)  
497 - break  
498 - if index >= 0:  
499 - eventfile = os.path.join(self._subdir,  
500 - hduindx_hdu['FILE_DIR'][index],  
501 - hduindx_hdu['FILE_NAME'][index])  
502 - eventhdu = hduindx_hdu['HDU_NAME'][index]  
503 - eventfile += '['+eventhdu+']'  
504 -  
505 - # Check if psf file is available  
506 - if not gammalib.GFilename(eventfile).is_fits():  
507 -  
508 - # Print warning that observation will be skipped since no  
509 - # events are present  
510 - if self._logTerse():  
511 - self._log('Skipping observation '+str(obs_id)+  
512 - ': eventfile "'+eventfile+'" not found\n') 573 + # Retrieve filenames
  574 + eventfile = self._get_filename(hduindx_hdu, self._ev_hiera, formats)
  575 + aefffile = self._get_filename(hduindx_hdu, self._aeff_hiera, formats)
  576 + psffile = self._get_filename(hduindx_hdu, self._psf_hiera, formats)
  577 + edispfile = self._get_filename(hduindx_hdu, self._edisp_hiera, formats)
  578 + bkgfile = self._get_filename(hduindx_hdu, self._bkg_hiera, formats)
  579 +
  580 + # Check for presence of required event file
  581 + if not self._is_present(eventfile, obs_id, "event file"):
513 continue 582 continue
514 583
515 - # Handle Aeff  
516 - index = -1  
517 - for version in self._aeff_hiera:  
518 - n = formats.count(version)  
519 - if n > 0:  
520 - index = formats.index(version)  
521 - break  
522 - if index >= 0:  
523 - aefffile = os.path.join(self._subdir,  
524 - hduindx_hdu['FILE_DIR'][index],  
525 - hduindx_hdu['FILE_NAME'][index])  
526 - aeffhdu = hduindx_hdu['HDU_NAME'][index]  
527 - aefffile += '['+aeffhdu+']'  
528 -  
529 - # Check if psf file is available  
530 - if not gammalib.GFilename(aefffile).is_fits():  
531 -  
532 - # Print warning that observation will be skipped since no effective area is present  
533 - if self._logTerse():  
534 - self._log('Skipping observation '+str(obs_id)+  
535 - ': effective area "'+aefffile+'" not found\n') 584 + # Check for presence of required effective area file
  585 + if not self._is_present(aefffile, obs_id, "effective area file"):
536 continue 586 continue
537 -  
538 - # Handle psf  
539 - index = -1  
540 - for version in self._psf_hiera:  
541 - n = formats.count(version)  
542 - if n > 0:  
543 - index = formats.index(version)  
544 - break  
545 - if index >= 0:  
546 - psffile = os.path.join(self._subdir,  
547 - hduindx_hdu['FILE_DIR'][index],  
548 - hduindx_hdu['FILE_NAME'][index])  
549 - psffile += '['+hduindx_hdu['HDU_NAME'][index]+']'  
550 -  
551 - # Check if psf file is available  
552 - if not gammalib.GFilename(psffile).is_fits():  
553 -  
554 - # Print warning that observation will be skipped since no PSF is present  
555 - if self._logTerse():  
556 - self._log('Skipping observation '+str(obs_id)+  
557 - ': point spread function "'+psffile+  
558 - '" not found\n') 587 +
  588 + # Check for presence of required psf file
  589 + if not self._is_present(psffile, obs_id, "PSF file"):
559 continue 590 continue
560 -  
561 - # Handle edisp  
562 - index = -1  
563 - for version in self._edisp_hiera:  
564 - n = formats.count(version)  
565 - if n > 0:  
566 - index = formats.index(version)  
567 - break  
568 - if index >= 0:  
569 - edispfile = os.path.join(self._subdir,  
570 - hduindx_hdu['FILE_DIR'][index],  
571 - hduindx_hdu['FILE_NAME'][index])  
572 - edispfile += '['+hduindx_hdu['HDU_NAME'][index]+']'  
573 -  
574 - # Check if edisp file is available  
575 - if not gammalib.GFilename(edispfile).is_fits(): 591 +
  592 + # Check for optional energy dispersion file
  593 + if not gammalib.GFilename(edispfile).exists():
576 594
577 # Print warning that edisp cannot be used for this observation 595 # Print warning that edisp cannot be used for this observation
578 - if self._logTerse():  
579 - self._log('Warning: observation '+str(obs_id)+  
580 - ' has no energy dispersion "'+edispfile+  
581 - '" information\n')  
582 - edispfile = '' 596 + self._log_string(gammalib.TERSE, 'Warning: observation '+str(obs_id)+
  597 + ' has no energy dispersion information\n')
583 598
584 - # Handle background  
585 - index = -1  
586 - bkg_mod_hierarchy = list(self._bkg_mod_hiera)  
587 - for version in self._bkg_hiera:  
588 - n = formats.count(version)  
589 - if n > 0:  
590 - index = formats.index(version)  
591 - break  
592 - if index >= 0:  
593 - bkgfile = os.path.join(self._subdir,  
594 - hduindx_hdu['FILE_DIR'][index],  
595 - hduindx_hdu['FILE_NAME'][index])  
596 - bkgfile += '['+hduindx_hdu['HDU_NAME'][index]+']'  
597 - if 'irf' in bkg_mod_hierarchy and not gammalib.GFilename(bkgfile).is_fits():  
598 - bkg_mod_hierarchy.remove('irf') 599 + # Set energy dispersion file as empty
  600 + edispfile = ''
  601 +
  602 + # Create a copy of background model hierarchy for this run
  603 + run_bkg_mod_hierarchy = list(self._bkg_mod_hiera)
  604 +
  605 + # Check if background file is available
  606 + # Remove IRF background from hierarchy if file doesnt exist
  607 + if not gammalib.GFilename(bkgfile).exists():
599 608
600 - # Throw warning if IRF background cannot be used  
601 - if self._logTerse():  
602 - self._log('Warning: observation '+str(obs_id)+  
603 - ' has no background information (file="'+  
604 - bkgfile+'"). IRF background cannot be used\n')  
605 - bkgfile = ''  
606 - if len(bkg_mod_hierarchy) == 0:  
607 -  
608 - # Skip observation if no background can be used  
609 - if self._logTerse():  
610 - self._log('Skipping observation '+str(obs_id)+  
611 - ': No background can be used\n')  
612 - continue  
613 - else:  
614 - # Log if we fall back to next background approach  
615 - if self._logTerse():  
616 - self._log('Observation '+str(obs_id)+  
617 - ': Falling back to background "'+  
618 - bkg_mod_hierarchy[0]+'"\n') 609 + # Set background file as empty
  610 + bkgfile = ''
  611 +
  612 + # Check for IRF background in background model hierarchy
  613 + if 'irf' in run_bkg_mod_hierarchy:
  614 +
  615 + # Remove IRF background if file doesnt exist
  616 + run_bkg_mod_hierarchy.remove('irf')
  617 +
  618 + # Print warning that edisp cannot be used for this observation
  619 + self._log_string(gammalib.TERSE, 'Warning: observation '+str(obs_id)+
  620 + ' has no background information (IRF background cannot be used)\n')
  621 +
  622 + # Check if background model hierarchy is empty
  623 + if len(run_bkg_mod_hierarchy) == 0:
  624 +
  625 + # Skip observation if no background can be used
  626 + self._log_string(gammalib.TERSE, 'Skipping observation '+str(obs_id)+
  627 + ': No background can be used\n')
  628 + continue
  629 +
  630 + else:
  631 + # Log if we fall back to next background approach
  632 + self._log_string(gammalib.TERSE, 'Observation '+str(obs_id)+
  633 + ': Falling back to background "'+
  634 + run_bkg_mod_hierarchy[0]+'"\n')
619 635
620 # Close hdu index file 636 # Close hdu index file
621 hduindx.close() 637 hduindx.close()
622 - 638 +
623 # Initialise background scale 639 # Initialise background scale
624 bkg_scale = 1.0 640 bkg_scale = 1.0
625 641
@@ -635,7 +651,7 @@ class csiactobs(ctools.cscript): @@ -635,7 +651,7 @@ class csiactobs(ctools.cscript):
635 fits = gammalib.GFits(eventfile) 651 fits = gammalib.GFits(eventfile)
636 652
637 # Get object and telescope strings from event hdu 653 # Get object and telescope strings from event hdu
638 - events = fits[eventhdu] 654 + events = fits[gammalib.GFilename(eventfile).extname()]
639 object_name = events.string('OBJECT') 655 object_name = events.string('OBJECT')
640 telescope = events.string('TELESCOP') 656 telescope = events.string('TELESCOP')
641 657
@@ -644,7 +660,7 @@ class csiactobs(ctools.cscript): @@ -644,7 +660,7 @@ class csiactobs(ctools.cscript):
644 660
645 # Open effective area file to look for threshold 661 # Open effective area file to look for threshold
646 aeff_fits = gammalib.GFits(aefffile) 662 aeff_fits = gammalib.GFits(aefffile)
647 - aeff_table = aeff_fits[aeffhdu] 663 + aeff_table = aeff_fits[gammalib.GFilename(aefffile).extname()]
648 664
649 # Set energy range from header keyword if present 665 # Set energy range from header keyword if present
650 if aeff_table.has_card('LO_THRES') and aeff_table.has_card('HI_THRES'): 666 if aeff_table.has_card('LO_THRES') and aeff_table.has_card('HI_THRES'):
@@ -668,7 +684,7 @@ class csiactobs(ctools.cscript): @@ -668,7 +684,7 @@ class csiactobs(ctools.cscript):
668 self._models.append(self._iact_background(telescope, 684 self._models.append(self._iact_background(telescope,
669 obs_id, 685 obs_id,
670 bkg_scale, 686 bkg_scale,
671 - bkg_mod_hierarchy[0], 687 + run_bkg_mod_hierarchy[0],
672 run_emin.TeV(), 688 run_emin.TeV(),
673 run_emax.TeV())) 689 run_emax.TeV()))
674 690
@@ -684,19 +700,19 @@ class csiactobs(ctools.cscript): @@ -684,19 +700,19 @@ class csiactobs(ctools.cscript):
684 700
685 # Append effective area 701 # Append effective area
686 aeff = gammalib.GXmlElement('parameter name="EffectiveArea"') 702 aeff = gammalib.GXmlElement('parameter name="EffectiveArea"')
687 - aeff.attribute('file',aefffile) 703 + aeff.attribute('file', aefffile)
688 704
689 # Append PSF 705 # Append PSF
690 psf = gammalib.GXmlElement('parameter name="PointSpreadFunction"') 706 psf = gammalib.GXmlElement('parameter name="PointSpreadFunction"')
691 - psf.attribute('file',psffile) 707 + psf.attribute('file', psffile)
692 708
693 # Append energy dispersion 709 # Append energy dispersion
694 edisp = gammalib.GXmlElement('parameter name="EnergyDispersion"') 710 edisp = gammalib.GXmlElement('parameter name="EnergyDispersion"')
695 - edisp.attribute('file',edispfile) 711 + edisp.attribute('file', edispfile)
696 712
697 # Append background 713 # Append background
698 bck = gammalib.GXmlElement('parameter name="Background"') 714 bck = gammalib.GXmlElement('parameter name="Background"')
699 - bck.attribute('file',bkgfile) 715 + bck.attribute('file', bkgfile)
700 716
701 # Assign events and IRFs to observation 717 # Assign events and IRFs to observation
702 obs.append(ev) 718 obs.append(ev)
@@ -726,8 +742,8 @@ class csiactobs(ctools.cscript): @@ -726,8 +742,8 @@ class csiactobs(ctools.cscript):
726 742
727 # Write warning in no observation could be used from runlist 743 # Write warning in no observation could be used from runlist
728 if lib.size() == 0: 744 if lib.size() == 0:
729 - self._log('WARNING: No observation was appended from specified '  
730 - 'runlist') 745 + self._log_string(gammalib.TERSE, 'WARNING: No observation was appended from specified '
  746 + 'runlist\n')
731 747
732 # Return 748 # Return
733 return 749 return
cscripts/csobsdef.py
  View file @77e1bab
@@ -386,6 +386,19 @@ class csobsdef(ctools.cscript): @@ -386,6 +386,19 @@ class csobsdef(ctools.cscript):
386 # Return 386 # Return
387 return 387 return
388 388
  389 + def obs(self):
  390 + """
  391 + Returns observation container
  392 + -----
  393 + Returns
  394 + obs : `~gammalib.GObservations`
  395 + Observation container
  396 + """
  397 +
  398 + # Return container
  399 + return self._obs
  400 +
  401 +
389 def pntdef(self, csv): 402 def pntdef(self, csv):
390 """ 403 """
391 Set pointing definition from a CSV table 404 Set pointing definition from a CSV table
test/data/iact_runlist.dat
  View file @77e1bab
1 -0 1 +15000
  2 +15001
  3 +15002
  4 +15003
  5 +15004
  6 +28000
  7 +28001
  8 +28002
  9 +28003
test/data/iact_runlist_1.dat 0 → 100644
  View file @77e1bab
  1 +15000
  2 +15001
  3 +15002
  4 +15003
  5 +15004
test/data/iact_runlist_2.dat 0 → 100644
  View file @77e1bab
  1 +28000
  2 +28001
  3 +28002
  4 +28003
test/data/iactdata/events.fits.gz deleted
View file @ff0d52e
No preview for this file type
test/data/iactdata/events/events_0.fits.gz 0 → 100644
View file @77e1bab
No preview for this file type
test/data/iactdata/events/events_1.fits.gz 0 → 100644
View file @77e1bab
No preview for this file type
test/data/iactdata/events/events_2.fits.gz 0 → 100644
View file @77e1bab
No preview for this file type
test/data/iactdata/events/events_3.fits.gz 0 → 100644
View file @77e1bab
No preview for this file type
test/data/iactdata/events/events_4.fits.gz 0 → 100644
View file @77e1bab
No preview for this file type
test/data/iactdata/events/events_5.fits.gz 0 → 100644
View file @77e1bab
No preview for this file type
test/data/iactdata/events/events_6.fits.gz 0 → 100644
View file @77e1bab
No preview for this file type
test/data/iactdata/events/events_7.fits.gz 0 → 100644
View file @77e1bab
No preview for this file type
test/data/iactdata/events/events_8.fits.gz 0 → 100644
View file @77e1bab
No preview for this file type
test/data/iactdata/irf_file.fits.gz renamed to test/data/iactdata/events/irf_file.fits.gz
View file @77e1bab
No preview for this file type
test/data/iactdata/hdu-index.fits.gz
View file @77e1bab
No preview for this file type
test/data/iactdata/master.json
  View file @77e1bab
1 { 1 {
2 - "datasets": [  
3 - {  
4 - "name": "unit-test",  
5 - "obsindx": "obs-index.fits.gz",  
6 - "hduindx": "hdu-index.fits.gz",  
7 - "comment": "this is a comment"  
8 - },  
9 - {  
10 - "name": "unit-test-that-doesnt-exist",  
11 - "obsindx": "badfile.fits.gz",  
12 - "hduindx": "unavailable_file.fits.gz",  
13 - "comment": "this is a comment"  
14 - }  
15 - ]  
16 -} 2 + "datasets": [
  3 + {
  4 + "name": "unit-test",
  5 + "hduindx": "hdu-index.fits.gz",
  6 + "obsindx": "obs-index.fits.gz",
  7 + "Total observations": 9,
  8 + "Total time": "0.07 hours",
  9 + "Total events": 533,
  10 + "Disk usage": "0.43 MB",
  11 + "Date created": "2016-08-03"
  12 + }
  13 + ]
  14 +}
17 \ No newline at end of file 15 \ No newline at end of file
test/data/iactdata/obs-index.fits.gz
View file @77e1bab
No preview for this file type
test/test_csfindobs.py
  View file @77e1bab
@@ -74,7 +74,7 @@ class Test(test): @@ -74,7 +74,7 @@ class Test(test):
74 # Setup csfindobs command 74 # Setup csfindobs command
75 cmd = csfindobs+' datapath="'+self._datapath+'"'+ \ 75 cmd = csfindobs+' datapath="'+self._datapath+'"'+ \
76 ' prodname="unit-test"'+ \ 76 ' prodname="unit-test"'+ \
77 - ' ra=83.63 dec=22.01 rad=1.0'+ \ 77 + ' ra=83.63 dec=22.01 rad=1.5'+ \
78 ' outfile="runlist_cmd1.dat"'+ \ 78 ' outfile="runlist_cmd1.dat"'+ \
79 ' logfile="csfindobs_cmd1.log" chatter=1' 79 ' logfile="csfindobs_cmd1.log" chatter=1'
80 80
@@ -87,12 +87,12 @@ class Test(test): @@ -87,12 +87,12 @@ class Test(test):
87 'Check successful execution from command line') 87 'Check successful execution from command line')
88 88
89 # Check run list 89 # Check run list
90 - self._check_runlist('runlist_cmd1.dat') 90 + self._check_runlist('runlist_cmd1.dat',5)
91 91
92 # Setup csfindobs command 92 # Setup csfindobs command
93 cmd = csfindobs+' datapath="path_that_does_not_exist"'+ \ 93 cmd = csfindobs+' datapath="path_that_does_not_exist"'+ \
94 ' prodname="unit-test"'+ \ 94 ' prodname="unit-test"'+ \
95 - ' ra=83.63 dec=22.01 rad=1.0'+ \ 95 + ' ra=83.63 dec=22.01 rad=1.5'+ \
96 ' outfile="runlist_cmd2.dat"'+ \ 96 ' outfile="runlist_cmd2.dat"'+ \
97 ' logfile="csfindobs_cmd2.log"' 97 ' logfile="csfindobs_cmd2.log"'
98 98
@@ -114,10 +114,10 @@ class Test(test): @@ -114,10 +114,10 @@ class Test(test):
114 findobs['prodname'] = 'unit-test' 114 findobs['prodname'] = 'unit-test'
115 findobs['ra'] = 83.63 115 findobs['ra'] = 83.63
116 findobs['dec'] = 22.01 116 findobs['dec'] = 22.01
117 - findobs['rad'] = 1.0 117 + findobs['rad'] = 1.5
118 findobs['outfile'] = 'runlist_py1.dat' 118 findobs['outfile'] = 'runlist_py1.dat'
119 findobs['logfile'] = 'csfindobs_py1.log' 119 findobs['logfile'] = 'csfindobs_py1.log'
120 - findobs['chatter'] = 2 120 + findobs['chatter'] = 4
121 121
122 # Run csfindobs script 122 # Run csfindobs script
123 findobs.logFileOpen() # Make sure we get a log file 123 findobs.logFileOpen() # Make sure we get a log file
@@ -125,16 +125,16 @@ class Test(test): @@ -125,16 +125,16 @@ class Test(test):
125 125
126 # Check list of runs 126 # Check list of runs
127 runs = findobs.runs() 127 runs = findobs.runs()
128 - self.test_value(len(runs), 1, 128 + self.test_value(len(runs), 5,
129 'Check for number of runs in runlist') 129 'Check for number of runs in runlist')
130 - self.test_value(str(runs[0]), '0', 130 + self.test_value(str(runs[0]), '15000',
131 'Check run ID in runlist') 131 'Check run ID in runlist')
132 132
133 # Save run list 133 # Save run list
134 findobs.save() 134 findobs.save()
135 135
136 # Check run list 136 # Check run list
137 - self._check_runlist('runlist_py1.dat') 137 + self._check_runlist('runlist_py1.dat', runs=5)
138 138
139 # Set-up csfindobs for undefined spatial parameters 139 # Set-up csfindobs for undefined spatial parameters
140 findobs = cscripts.csfindobs() 140 findobs = cscripts.csfindobs()
@@ -151,16 +151,16 @@ class Test(test): @@ -151,16 +151,16 @@ class Test(test):
151 findobs.execute() 151 findobs.execute()
152 152
153 # Check run list 153 # Check run list
154 - self._check_runlist('runlist_py2.dat') 154 + self._check_runlist('runlist_py2.dat', runs=9)
155 155
156 # Set-up csfindobs for user expression 156 # Set-up csfindobs for user expression
157 findobs = cscripts.csfindobs() 157 findobs = cscripts.csfindobs()
158 findobs['datapath'] = self._datapath 158 findobs['datapath'] = self._datapath
159 findobs['prodname'] = 'unit-test' 159 findobs['prodname'] = 'unit-test'
160 - findobs['ra'] = 83.63  
161 - findobs['dec'] = 22.01  
162 - findobs['rad'] = 0.0  
163 - findobs['expression'] = 'ONTIME<10.0' 160 + findobs['ra'] = 329.71
  161 + findobs['dec'] = -30.2339
  162 + findobs['rad'] = 2.0
  163 + findobs['expression'] = 'ONTIME<5.0'
164 findobs['outfile'] = 'runlist_py3.dat' 164 findobs['outfile'] = 'runlist_py3.dat'
165 findobs['logfile'] = 'csfindobs_py3.log' 165 findobs['logfile'] = 'csfindobs_py3.log'
166 findobs['chatter'] = 4 166 findobs['chatter'] = 4
@@ -195,7 +195,7 @@ class Test(test): @@ -195,7 +195,7 @@ class Test(test):
195 if runs > 0: 195 if runs > 0:
196 self.test_value(runlist.ncols(), 1, 196 self.test_value(runlist.ncols(), 1,
197 'Check for single column in runlist file') 197 'Check for single column in runlist file')
198 - self.test_value(runlist[0,0], '0', 198 + self.test_value(runlist[0,0], '15000',
199 'Check for run ID in runlist file') 199 'Check for run ID in runlist file')
200 200
201 # Return 201 # Return
test/test_csiactcopy.py
  View file @77e1bab
@@ -18,6 +18,7 @@ @@ -18,6 +18,7 @@
18 # along with this program. If not, see <http://www.gnu.org/licenses/>. 18 # along with this program. If not, see <http://www.gnu.org/licenses/>.
19 # 19 #
20 # ========================================================================== 20 # ==========================================================================
  21 +import os
21 import gammalib 22 import gammalib
22 import cscripts 23 import cscripts
23 from testing import test 24 from testing import test
@@ -43,7 +44,9 @@ class Test(test): @@ -43,7 +44,9 @@ class Test(test):
43 test.__init__(self) 44 test.__init__(self)
44 45
45 # Set data members 46 # Set data members
46 - self._remote = self._datadir + '/iactdata/master.json' 47 + self._remote = self._datadir + '/iactdata/master.json'
  48 + self._runlist1 = self._datadir + '/iact_runlist_1.dat'
  49 + self._runlist2 = self._datadir + '/iact_runlist_2.dat'
47 50
48 # Return 51 # Return
49 return 52 return
@@ -107,20 +110,115 @@ class Test(test): @@ -107,20 +110,115 @@ class Test(test):
107 Test csiactcopy from Python 110 Test csiactcopy from Python
108 """ 111 """
109 # Set-up csiactcopy 112 # Set-up csiactcopy
110 - findobs = cscripts.csiactcopy()  
111 - findobs['remote_master'] = self._remote  
112 - findobs['prodname'] = 'unit-test'  
113 - findobs['outpath'] = 'iactdata_py1'  
114 - findobs['logfile'] = 'csiactcopy_py1.log'  
115 - findobs['chatter'] = 2 113 + iactcopy = cscripts.csiactcopy()
  114 + iactcopy['remote_master'] = self._remote
  115 + iactcopy['prodname'] = 'unit-test'
  116 + iactcopy['outpath'] = 'iactdata_py1'
  117 + iactcopy['logfile'] = 'csiactcopy_py1.log'
  118 + iactcopy['chatter'] = 4
116 119
117 # Run csiactcopy script and save run list 120 # Run csiactcopy script and save run list
118 - findobs.logFileOpen() # Make sure we get a log file  
119 - findobs.run()  
120 - findobs.save() 121 + iactcopy.logFileOpen() # Make sure we get a log file
  122 + iactcopy.run()
  123 + iactcopy.save()
  124 +
  125 + # Check IACT data store
  126 + self._check_copy('iactdata_py1')
  127 + self._check_n_obs('iactdata_py1', 9)
  128 +
  129 + # Set-up csiactcopy
  130 + iactcopy = cscripts.csiactcopy()
  131 + iactcopy['remote_master'] = self._remote
  132 + iactcopy['prodname'] = 'unit-test'
  133 + iactcopy['outpath'] = 'iactdata_py1'
  134 + iactcopy['logfile'] = 'csiactcopy_py2.log'
  135 + iactcopy['chatter'] = 4
  136 + iactcopy['clobber'] = True
121 137
  138 + # Run csiactcopy script and save run list
  139 + iactcopy.logFileOpen() # Make sure we get a log file
  140 + iactcopy.run()
  141 + iactcopy.save()
  142 +
122 # Check copy 143 # Check copy
123 self._check_copy('iactdata_py1') 144 self._check_copy('iactdata_py1')
  145 + self._check_n_obs('iactdata_py1', 9)
  146 +
  147 + # Set-up csiactcopy
  148 + iactcopy = cscripts.csiactcopy()
  149 + iactcopy['remote_master'] = self._remote
  150 + iactcopy['prodname'] = 'unit-test'
  151 + iactcopy['outpath'] = 'iactdata_py1'
  152 + iactcopy['logfile'] = 'csiactcopy_py3.log'
  153 + iactcopy['chatter'] = 4
  154 + iactcopy['clobber'] = False
  155 +
  156 + # Run csiactcopy script and save run list
  157 + iactcopy.logFileOpen() # Make sure we get a log file
  158 + iactcopy.run()
  159 + iactcopy.save()
  160 +
  161 + # Check copy
  162 + self._check_copy('iactdata_py1')
  163 + self._check_n_obs('iactdata_py1', 9)
  164 +
  165 + if os.path.isdir('iactdata_py2'):
  166 + os.system("rm -r iactdata_py2")
  167 +
  168 + # Set-up csiactcopy
  169 + iactcopy = cscripts.csiactcopy()
  170 + iactcopy['runlist'] = self._runlist1
  171 + iactcopy['remote_master'] = self._remote
  172 + iactcopy['prodname'] = 'unit-test'
  173 + iactcopy['outpath'] = 'iactdata_py2'
  174 + iactcopy['logfile'] = 'csiactcopy_py4.log'
  175 + iactcopy['chatter'] = 3
  176 +
  177 + # Run csiactcopy script and save run list
  178 + iactcopy.logFileOpen() # Make sure we get a log file
  179 + iactcopy.run()
  180 + iactcopy.save()
  181 +
  182 + # Check copy
  183 + self._check_copy('iactdata_py2')
  184 + self._check_n_obs('iactdata_py2', 5)
  185 +
  186 + # Set-up csiactcopy
  187 + iactcopy = cscripts.csiactcopy()
  188 + iactcopy['runlist'] = self._runlist2
  189 + iactcopy['remote_master'] = self._remote
  190 + iactcopy['prodname'] = 'unit-test'
  191 + iactcopy['outpath'] = 'iactdata_py2'
  192 + iactcopy['logfile'] = 'csiactcopy_py5.log'
  193 + iactcopy['chatter'] = 3
  194 +
  195 + # Run csiactcopy script and save run list
  196 + iactcopy.logFileOpen() # Make sure we get a log file
  197 + iactcopy.run()
  198 + iactcopy.save()
  199 +
  200 + # Check copy
  201 + self._check_copy('iactdata_py2')
  202 + self._check_n_obs('iactdata_py2', 9)
  203 +
  204 + # Set-up csiactcopy
  205 + iactcopy = cscripts.csiactcopy()
  206 + iactcopy['runlist'] = self._runlist2
  207 + iactcopy['remote_master'] = self._remote
  208 + iactcopy['prodname'] = 'unit-test'
  209 + iactcopy['outpath'] = 'iactdata_py2'
  210 + iactcopy['logfile'] = 'csiactcopy_py6.log'
  211 + iactcopy['chatter'] = 3
  212 + iactcopy['clobber'] = True
  213 +
  214 + # Run csiactcopy script and save run list
  215 + iactcopy.logFileOpen() # Make sure we get a log file
  216 + iactcopy.run()
  217 + iactcopy.save()
  218 +
  219 + # Check copy
  220 + self._check_copy('iactdata_py2')
  221 + self._check_n_obs('iactdata_py2',9)
124 222
125 # Return 223 # Return
126 return 224 return
@@ -151,3 +249,27 @@ class Test(test): @@ -151,3 +249,27 @@ class Test(test):
151 249
152 # Return 250 # Return
153 return 251 return
  252 +
  253 + # Check copy
  254 + def _check_n_obs(self, pathname, n_expected):
  255 + """
  256 + Check number of available observations.
  257 + """
  258 + # Set file name
  259 + obs_index_name = gammalib.GFilename(pathname+'/obs-index.fits[OBS_INDEX]')
  260 +
  261 + # Open index file
  262 + fits = gammalib.GFits(obs_index_name)
  263 +
  264 + # Get number of observations
  265 + n_obs = fits[obs_index_name.extname()].nrows()
  266 +
  267 + # Close FITS file
  268 + fits.close()
  269 +
  270 + # Check for existence of observations
  271 + self.test_assert(n_obs == n_expected,
  272 + 'Check for '+str(n_expected)+' observations.')
  273 +
  274 + # Return
  275 + return
154 \ No newline at end of file 276 \ No newline at end of file
test/test_csiactobs.py
  View file @77e1bab
@@ -90,10 +90,10 @@ class Test(test): @@ -90,10 +90,10 @@ class Test(test):
90 'Check successful execution from command line') 90 'Check successful execution from command line')
91 91
92 # Check observation definition XML file 92 # Check observation definition XML file
93 - self._check_obsdef('csiactobs_obs_cmd1.xml') 93 + self._check_obsdef('csiactobs_obs_cmd1.xml', 6)
94 94
95 # Check model definition XML file 95 # Check model definition XML file
96 - self._check_moddef('csiactobs_bgd_cmd1.xml') 96 + self._check_moddef('csiactobs_bgd_cmd1.xml', 6)
97 97
98 # Setup csiactobs command 98 # Setup csiactobs command
99 cmd = csiactobs+' datapath="data_path_that_does_not_exist"'+ \ 99 cmd = csiactobs+' datapath="data_path_that_does_not_exist"'+ \
@@ -146,13 +146,13 @@ class Test(test): @@ -146,13 +146,13 @@ class Test(test):
146 iactobs.save() 146 iactobs.save()
147 147
148 # Check observation definition XML file 148 # Check observation definition XML file
149 - self._check_obsdef('csiactobs_obs_py1.xml') 149 + self._check_obsdef('csiactobs_obs_py1.xml', 6)
150 150
151 # Check model definition XML file 151 # Check model definition XML file
152 - self._check_moddef('csiactobs_bgd_py1.xml') 152 + self._check_moddef('csiactobs_bgd_py1.xml', 6)
153 153
154 # Create test runlist 154 # Create test runlist
155 - runlist = ['0'] 155 + runlist = ['15000','15001']
156 156
157 # Set-up csiactobs using a runlist with 2 background parameters 157 # Set-up csiactobs using a runlist with 2 background parameters
158 iactobs = cscripts.csiactobs() 158 iactobs = cscripts.csiactobs()
@@ -171,16 +171,16 @@ class Test(test): @@ -171,16 +171,16 @@ class Test(test):
171 iactobs.save() 171 iactobs.save()
172 172
173 # Test return functions 173 # Test return functions
174 - self.test_value(iactobs.obs().size(), 1, 174 + self.test_value(iactobs.obs().size(), 2,
175 'Check number of observations in container') 175 'Check number of observations in container')
176 self.test_value(iactobs.ebounds().size(), 0, 176 self.test_value(iactobs.ebounds().size(), 0,
177 'Check number of energy boundaries') 177 'Check number of energy boundaries')
178 178
179 # Check observation definition XML file 179 # Check observation definition XML file
180 - self._check_obsdef('csiactobs_obs_py2.xml') 180 + self._check_obsdef('csiactobs_obs_py2.xml',2)
181 181
182 # Check model definition XML file 182 # Check model definition XML file
183 - self._check_moddef('csiactobs_bgd_py2.xml') 183 + self._check_moddef('csiactobs_bgd_py2.xml',2)
184 184
185 # Set-up csiactobs with a large number of free parameters and "aeff" 185 # Set-up csiactobs with a large number of free parameters and "aeff"
186 # background 186 # background
@@ -199,10 +199,10 @@ class Test(test): @@ -199,10 +199,10 @@ class Test(test):
199 iactobs.execute() 199 iactobs.execute()
200 200
201 # Check observation definition XML file 201 # Check observation definition XML file
202 - self._check_obsdef('csiactobs_obs_py3.xml') 202 + self._check_obsdef('csiactobs_obs_py3.xml',6)
203 203
204 # Check model definition XML file 204 # Check model definition XML file
205 - self._check_moddef('csiactobs_bgd_py3.xml') 205 + self._check_moddef('csiactobs_bgd_py3.xml',6)
206 206
207 # Set-up csiactobs with a "gauss" background and "inmodel" parameter 207 # Set-up csiactobs with a "gauss" background and "inmodel" parameter
208 iactobs = cscripts.csiactobs() 208 iactobs = cscripts.csiactobs()
@@ -222,56 +222,85 @@ class Test(test): @@ -222,56 +222,85 @@ class Test(test):
222 iactobs.run() 222 iactobs.run()
223 223
224 # Check number of observations 224 # Check number of observations
225 - self.test_value(iactobs.obs().size(), 1, 225 + self.test_value(iactobs.obs().size(), 6,
226 'Check number of observations in container') 226 'Check number of observations in container')
227 227
228 # Check number of models 228 # Check number of models
229 - self.test_value(iactobs.obs().models().size(), 3,  
230 - 'Check number of models in container') 229 + self.test_value(iactobs.obs().models().size(), 8,
  230 + 'Check number of models in container')
  231 +
  232 + # Set-up csiactobs with a "gauss" background and "inmodel" parameter
  233 + iactobs = cscripts.csiactobs()
  234 + iactobs['datapath'] = self._datapath
  235 + iactobs['inmodel'] = self._model
  236 + iactobs['prodname'] = 'unit-test'
  237 + iactobs['infile'] = self._runlist
  238 + iactobs['bkgpars'] = 1
  239 + iactobs['bkg_mod_hiera'] = 'irf'
  240 + iactobs['outobs'] = 'NONE'
  241 + iactobs['outmodel'] = 'NONE'
  242 + iactobs['logfile'] = 'csiactobs_py4.log'
  243 + iactobs['chatter'] = 4
  244 +
  245 + # Run csiactobs script
  246 + iactobs.logFileOpen() # Make sure we get a log file
  247 + iactobs.run()
  248 +
  249 + # Check number of observations
  250 + self.test_value(iactobs.obs().size(), 5,
  251 + 'Check number of observations in container')
  252 +
  253 + # Check number of models
  254 + self.test_value(iactobs.obs().models().size(), 7,
  255 + 'Check number of models in container')
231 256
232 # Return 257 # Return
233 return 258 return
234 259
235 # Check observation definition XML file 260 # Check observation definition XML file
236 - def _check_obsdef(self, filename): 261 + def _check_obsdef(self, filename, obs_expected):
237 """ 262 """
238 Check observation definition XML file 263 Check observation definition XML file
239 """ 264 """
240 # Load observation definition XML file 265 # Load observation definition XML file
241 obs = gammalib.GObservations(filename) 266 obs = gammalib.GObservations(filename)
242 267
243 - # Get response  
244 - rsp = obs[0].response()  
245 -  
246 # Check number of observations 268 # Check number of observations
247 - self.test_value(obs.size(), 1,  
248 - 'Check for single observation in XML file')  
249 - self.test_value(obs[0].eventfile().file(), 'events.fits.gz',  
250 - 'Check event file name')  
251 - self.test_value(obs[0].eventfile().extname(), 'EVENTS',  
252 - 'Check event extension name')  
253 - self.test_value(rsp.aeff().filename().file(), 'irf_file.fits.gz',  
254 - 'Check effective area file name')  
255 - self.test_value(rsp.aeff().filename().extname(), 'EFFECTIVE AREA',  
256 - 'Check effective area extension name')  
257 - self.test_value(rsp.psf().filename().file(), 'irf_file.fits.gz',  
258 - 'Check point spread function file name')  
259 - self.test_value(rsp.psf().filename().extname(), 'POINT SPREAD FUNCTION',  
260 - 'Check point spread function extension name')  
261 - self.test_value(rsp.edisp().filename().file(), 'irf_file.fits.gz',  
262 - 'Check energy dispersion file name')  
263 - self.test_value(rsp.edisp().filename().extname(), 'ENERGY DISPERSION',  
264 - 'Check energy dispersion extension name')  
265 - self.test_value(rsp.background().filename().file(), 'irf_file.fits.gz',  
266 - 'Check background file name')  
267 - self.test_value(rsp.background().filename().extname(), 'BACKGROUND',  
268 - 'Check background extension name') 269 + self.test_value(obs.size(), obs_expected,
  270 + 'Check for '+str(obs_expected)+' observations in XML file')
  271 +
  272 + if obs_expected > 0:
  273 +
  274 + # Get response
  275 + rsp = obs[0].response()
  276 +
  277 + # Test response
  278 + self.test_value(obs[0].eventfile().file(), 'events_0.fits.gz',
  279 + 'Check event file name')
  280 + self.test_value(obs[0].eventfile().extname(), 'EVENTS',
  281 + 'Check event extension name')
  282 + self.test_value(rsp.aeff().filename().file(), 'irf_file.fits.gz',
  283 + 'Check effective area file name')
  284 + self.test_value(rsp.aeff().filename().extname(), 'EFFECTIVE AREA',
  285 + 'Check effective area extension name')
  286 + self.test_value(rsp.psf().filename().file(), 'irf_file.fits.gz',
  287 + 'Check point spread function file name')
  288 + self.test_value(rsp.psf().filename().extname(), 'POINT SPREAD FUNCTION',
  289 + 'Check point spread function extension name')
  290 + self.test_value(rsp.edisp().filename().file(), 'irf_file.fits.gz',
  291 + 'Check energy dispersion file name')
  292 + self.test_value(rsp.edisp().filename().extname(), 'ENERGY DISPERSION',
  293 + 'Check energy dispersion extension name')
  294 + self.test_value(rsp.background().filename().file(), 'irf_file.fits.gz',
  295 + 'Check background file name')
  296 + self.test_value(rsp.background().filename().extname(), 'BACKGROUND',
  297 + 'Check background extension name')
269 298
270 # Return 299 # Return
271 return 300 return
272 301
273 # Check model XML file 302 # Check model XML file
274 - def _check_moddef(self, filename): 303 + def _check_moddef(self, filename, models_expected):
275 """ 304 """
276 Check model definition XML file 305 Check model definition XML file
277 """ 306 """
@@ -279,8 +308,8 @@ class Test(test): @@ -279,8 +308,8 @@ class Test(test):
279 models = gammalib.GModels(filename) 308 models = gammalib.GModels(filename)
280 309
281 # Check number of models 310 # Check number of models
282 - self.test_value(models.size(), 1,  
283 - 'Check for single model in XML file') 311 + self.test_value(models.size(), models_expected,
  312 + 'Check for '+str(models_expected)+' models in XML file')
284 313
285 # Return 314 # Return
286 return 315 return