From 8c6691f42bf5b81fc36f24647bbad841077046a4 Mon Sep 17 00:00:00 2001 From: Jürgen Knödlseder Date: Thu, 30 Sep 2021 11:52:18 +0200 Subject: [PATCH] Add csadd2caldb script --- .gitignore | 1 + ChangeLog | 3 ++- NEWS | 8 +++++++- README.md | 2 +- cscripts/Makefile.am | 15 ++++++++++++--- cscripts/__init__.py.in | 4 +++- cscripts/csadd2caldb.par | 29 +++++++++++++++++++++++++++++ cscripts/csadd2caldb.py | 399 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ cscripts/mputils.py | 0 doc/source/admin/release_history/2.0.rst | 5 +++-- doc/source/users/reference_manual/csadd2caldb.rst | 55 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ doc/source/users/reference_manual/index.rst | 1 + doc/source/users/user_manual/csadd2caldb.py | 421 ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- doc/source/users/user_manual/irf_cta.rst | 7 +------ test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz | Bin 0 -> 457697 bytes test/test_csadd2caldb.py | 171 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ test/test_python_cscripts.py | 4 +++- 17 files changed, 688 insertions(+), 437 deletions(-) create mode 100755 cscripts/csadd2caldb.par create mode 100755 cscripts/csadd2caldb.py mode change 100644 => 100755 cscripts/mputils.py create mode 100755 doc/source/users/reference_manual/csadd2caldb.rst delete mode 100755 doc/source/users/user_manual/csadd2caldb.py create mode 100644 test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz create mode 100755 test/test_csadd2caldb.py diff --git a/.gitignore b/.gitignore index 6d2b4b3..761ffc2 100755 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ src/template/ctool_obs test/pfiles/ test/caldb/ test/prod3 +test/csadd2caldb_* test/csroot2caldb_* test/csiactcopy_* test/comobsbin_* diff --git a/ChangeLog b/ChangeLog index bb7244f..5c1d0d2 100755 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,9 @@ -2021-06-25 +2021-09-30 * Version 2.0.0 released ======================== + Add csadd2caldb script Add restart of bisection in case of bad convergence in ctulimit Add COMPTEL analysis scripts (#3738) Add source counts constraints to cssens (#3668) diff --git a/NEWS b/NEWS index e951c5e..01586d2 100755 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ New Features and Important Changes in ctools 2.0.0 -25 June 2021 +30 September 2021 Introduction @@ -167,6 +167,12 @@ support - Support classes Add stamping to ctool::save_event_list() method (#3662). +csadd2caldb - Adds CTA response function to calibration database +---------------------------------------------------------------- +New script that was added to support the installation of prod5-v0.1 CTA response +functions into the calibration database. + + csbkgmodel - Generate background model for 3D analysis ------------------------------------------------------ None diff --git a/README.md b/README.md index 75826b3..911128a 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ctools information ================== -* Version: 2.0.0.dev (25 June 2021) +* Version: 2.0.0.dev (30 September 2021) * GammaLib dependency: 2.0.0.dev [![Build Status](https://cta-jenkins.irap.omp.eu/buildStatus/icon?job=ctools-integrate-os)](https://cta-jenkins.irap.omp.eu/job/ctools-integrate-os/) diff --git a/cscripts/Makefile.am b/cscripts/Makefile.am index df4a0ec..f030612 100755 --- a/cscripts/Makefile.am +++ b/cscripts/Makefile.am @@ -21,6 +21,7 @@ # Scripts to be distributed in $(pyexecdir)/cscripts pythonscripts = $(top_builddir)/cscripts/__init__.py \ + $(srcdir)/csadd2caldb.py \ $(srcdir)/csbkgmodel.py \ $(srcdir)/cscaldb.py \ $(srcdir)/csebins.py \ @@ -60,7 +61,8 @@ pythonscripts = $(top_builddir)/cscripts/__init__.py \ $(srcdir)/mputils.py # Parfiles to be distributed in $(prefix)/syspfiles -parfiles = $(srcdir)/csbkgmodel.par \ +parfiles = $(srcdir)/csadd2caldb.par \ + $(srcdir)/csbkgmodel.par \ $(srcdir)/cscaldb.par \ $(srcdir)/csebins.par \ $(srcdir)/cslightcrv.par \ @@ -93,7 +95,8 @@ parfiles = $(srcdir)/csbkgmodel.par \ $(srcdir)/csiactobs.par # Scripts for which symbolic links will be created in bindir -linkscripts = csbkgmodel \ +linkscripts = csadd2caldb \ + csbkgmodel \ cscaldb \ csebins \ csinfo \ @@ -132,6 +135,7 @@ testscripts = $(srcdir)/tests/__init__.py \ $(top_srcdir)/test/test_python_cscripts.py \ $(top_srcdir)/test/test_cscript.py \ $(top_srcdir)/test/test_cscaldb.py \ + $(top_srcdir)/test/test_csadd2caldb.py \ $(top_srcdir)/test/test_csbkgmodel.py \ $(top_srcdir)/test/test_csebins.py \ $(top_srcdir)/test/test_csfindobs.py \ @@ -266,6 +270,9 @@ testiactevents = $(top_srcdir)/test/data/iactdata/events/irf_file.fits.gz \ $(top_srcdir)/test/data/iactdata/events/events_7.fits.gz \ $(top_srcdir)/test/data/iactdata/events/events_8.fits.gz +# Test data to be distributed in $(pyexecdir)/cscripts/tests/data/prod5-v0.1 +testprod501data = $(top_srcdir)/test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz + # Install scripts, parfiles, test scripts and test data. Note that we don't # need to prepend $(DESTDIR) here (not fully clear why, but that's the way # it is) @@ -283,11 +290,13 @@ testiactdatadir = $(pyexecdir)/cscripts/tests/data/iactdata testiactdata_DATA = $(testiactdata) testiacteventsdir = $(pyexecdir)/cscripts/tests/data/iactdata/events testiactevents_DATA = $(testiactevents) +testprod501dir = $(pyexecdir)/cscripts/tests/data/prod5-v0.1 +testprod501_DATA = $(testprod501data) # Files to be distributed in the tar file EXTRA_DIST = $(binscripts) $(pythonscripts) $(parfiles) \ $(testscripts) $(testdata) $(testiactdata) \ - $(testiactevents) + $(testiactevents) $(testprod501data) # Add install hook rule to create symbolic links to cscripts without .py in the bindir install-data-hook: diff --git a/cscripts/__init__.py.in b/cscripts/__init__.py.in index b428d80..6d28d1d 100755 --- a/cscripts/__init__.py.in +++ b/cscripts/__init__.py.in @@ -1,7 +1,7 @@ # ========================================================================== # cscripts Python module # -# Copyright (C) 2015-2020 Juergen Knoedlseder +# Copyright (C) 2015-2021 Juergen Knoedlseder # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,6 +24,7 @@ cur_version = sys.version_info # List all scripts in this module __all__ = [ + "csadd2caldb", "csbkgmodel", "cscaldb", "csebins", @@ -65,6 +66,7 @@ if cur_version > req_version: "csiactobs"]) # Import +from cscripts.csadd2caldb import csadd2caldb from cscripts.csbkgmodel import csbkgmodel from cscripts.cscaldb import cscaldb from cscripts.csebins import csebins diff --git a/cscripts/csadd2caldb.par b/cscripts/csadd2caldb.par new file mode 100755 index 0000000..5f13cdc --- /dev/null +++ b/cscripts/csadd2caldb.par @@ -0,0 +1,29 @@ +############################################################################# +# # +# CTA Science Analysis Tools # +# # +# ------------------------------------------------------------------------- # +# # +# File: csadd2caldb.par # +# # +# Author: Juergen Knoedlseder # +# Jurgen.Knodlseder@irap.omp.eu # +# IRAP # +# # +# Purpose: Parameter file for the csadd2caldb script # +# # +############################################################################# +# +# Input and output parameters +#============================ +indir, f, a, fits,,, "Input IRF folder" +outdir, f, a, $CALDB,,, "Output caldb folder" + +# +# Standard parameters +#==================== +chatter, i, h, 2,0,4, "Chattiness of output" +clobber, b, h, yes,,, "Overwrite existing output files with new output files?" +debug, b, h, no,,, "Activate debugging mode?" +mode, s, h, ql,,, "Mode of automatic parameters" +logfile, f, h, csadd2caldb.log,,, "Log filename" diff --git a/cscripts/csadd2caldb.py b/cscripts/csadd2caldb.py new file mode 100755 index 0000000..e70b101 --- /dev/null +++ b/cscripts/csadd2caldb.py @@ -0,0 +1,399 @@ +#! /usr/bin/env python +# ========================================================================== +# Add IRFs to CALDB +# +# Copyright (C) 2021 Juergen Knoedlseder +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ========================================================================== +import os +import sys +import glob +import tarfile +import gammalib +import ctools + + +# ================= # +# csadd2caldb class # +# ================= # +class csadd2caldb(ctools.cscript): + """ + Add IRFs to CALDB + """ + + # Constructor + def __init__(self, *argv): + """ + Constructor + """ + # Initialise application by calling the base class constructor + self._init_cscript(self.__class__.__name__, ctools.__version__, argv) + + # Return + return + + + # Private methods + def _get_parameters(self): + """ + Get parameters from parfile + """ + # Query parameters + self['indir'].filename() + self['outdir'].filename() + + # Write input parameters into logger + self._log_parameters(gammalib.TERSE) + + # Return + return + + def _collect_tarfiles(self, indir): + """ + Collect tarfiles + + Parameters + ---------- + indir : string + Input directory + + Returns + ------- + tarfiles : list of str + List of tarfiles + """ + # Initialise list of tarfiles + tarfiles = [] + + # Search for tarfiles in current directory + files = glob.glob('%s/*.FITS.tar.gz' % indir.url()) + tarfiles.extend(files) + + # Search for tarfiles in 'fits' subdirectory + files = glob.glob('%s/fits/*.FITS.tar.gz' % indir.url()) + tarfiles.extend(files) + + # Return tarfiles + return tarfiles + + def _extract_irfs(self, file, outdir): + """ + Collect IRFs from tarfile + + Parameters + ---------- + file : str + Tarfile with IRFs + outdir : str + Output directory + + Returns + ------- + irfs : list + List of IRF dictionaries + """ + # Initialise list of extracted IRFs + irfs = [] + + # Open tarfile + tar = tarfile.open(file) + + # Loop over all members in tarfile + for member in tar.getmembers(): + + # Get subarray + subarrays = '' + nsubarrays = 0 + if 'LST' in member.name: + subarrays += '_LST' + nsubarrays += 1 + if 'MST' in member.name: + subarrays += '_MST' + nsubarrays += 1 + if 'SST' in member.name: + subarrays += '_SST' + nsubarrays += 1 + if nsubarrays > 1: + subarrays = '' + + # Extract Instrument + if 'South-' in member.name: + site = 'South' + else: + site = 'North' + if '180000s' in member.name: + duration = '50h' + elif '18000s' in member.name: + duration = '5h' + elif '1800s' in member.name: + duration = '0.5h' + if 'SouthAz' in member.name: + orientation = '_S' + azimuth = 180.0 + elif 'NorthAz' in member.name: + orientation = '_N' + azimuth = 0.0 + else: + orientation = '' + azimuth = 90.0 + + elements = member.name.split('-') + zenith = [e for e in elements if 'deg' in e][0].strip('deg') + instrument = '%s_z%s%s_%s%s' % (site, zenith, orientation, duration, subarrays) + + # Build output directory + path = '%s/%s' % (outdir, instrument) + + # Write header + self._log_header3(gammalib.TERSE, 'Extract IRFs from "%s"' % member.name) + self._log_value(gammalib.NORMAL, 'Instrument', instrument) + + # Extract file + tar.extract(member, path='%s/%s' % (self['outdir'].filename().url(), path)) + + # Create IRF record + record = {'path': path, 'instrument': instrument, 'file': member.name, + 'site': site, 'duration': duration, 'zenith': zenith, 'azimuth': azimuth, + 'orientation': orientation} + + # Add record + irfs.append(record) + + # Close tarfile + tar.close() + + # Return + return irfs + + def _add_irf_to_cif(self, prod, hdu, irf): + """ + Add IRF to calibration index file HDU + + Parameters + ---------- + prod : str + Production + hdu : ~gammalib.GFitsBinTable + CIF HDU table + irf : dict + IRF dictionary + """ + # Set list of IRF component names + #names = ['EA', 'PSF', 'EDISP', 'BKG'] + components = [{'name': 'EA', 'CNAM': 'EFF_AREA', 'DESC': 'CTA effective area'}, + {'name': 'PSF', 'CNAM': 'RPSF', 'DESC': 'CTA point spread function'}, + {'name': 'EDISP', 'CNAM': 'EDISP', 'DESC': 'CTA energy dispersion'}, + {'name': 'BKG', 'CNAM': 'BKG', 'DESC': 'CTA background'}] + + # Initialise CIF row index + row = hdu.nrows() + + # Append rows for all components to CIF extension + hdu.append_rows(len(components)) + + # Add information for all components + for component in components: + + # Set calibration information + cal_name = 'NAME(%s)' % irf['instrument'] + cal_version = 'VERSION(%s)' % prod + cal_cut = 'CLASS(BEST)' + cal_analysis = 'ANALYSIS(CTA)' + cal_zenith = 'ZENITH(%.3f)deg' % float(irf['zenith']) + cal_azimuth = 'AZIMUTH(%.3f)deg' % float(irf['azimuth']) + cal_bounds = [cal_name, cal_version, cal_cut, cal_analysis, \ + cal_zenith, cal_azimuth] + + # Set generic information + hdu['TELESCOP'][row] = 'CTA' + hdu['INSTRUME'][row] = gammalib.toupper(prod) + hdu['DETNAM'][row] = 'NONE' + hdu['FILTER'][row] = 'NONE' + hdu['CAL_DEV'][row] = 'ONLINE' + hdu['CAL_CLAS'][row] = 'BCF' + hdu['CAL_DTYP'][row] = 'DATA' + hdu['CAL_VSD'][row] = '2014-01-30' + hdu['CAL_VST'][row] = '00:00:00' + hdu['REF_TIME'][row] = 51544.0 + hdu['CAL_QUAL'][row] = 0 # 0=good, 1=bad, 2=dubious, ... + hdu['CAL_DATE'][row] = '14/01/30' + + # Set component specific information + hdu['CAL_DIR'][row] = irf['path'] + hdu['CAL_FILE'][row] = irf['file'] + hdu['CAL_CNAM'][row] = component['CNAM'] + hdu['CAL_DESC'][row] = component['DESC'] + hdu['CAL_XNO'][row] = 1 + for i in range(9): + if i >= len(cal_bounds): + hdu['CAL_CBD'][row,i] = 'NONE' + else: + hdu['CAL_CBD'][row,i] = cal_bounds[i] + + # Increment row index + row += 1 + + # Return + return + + def _create_cif_table(self): + """ + Create Calibration Database Index File binary table + + Returns + ------- + table : ~gammalib.GFitsBinTable + Calibration Database Index File binary table + """ + # Create binary table + table = gammalib.GFitsBinTable() + + # Append columns. Reference: CAL/GEN/92-008 + table.append(gammalib.GFitsTableStringCol('TELESCOP', 0, 10)) + table.append(gammalib.GFitsTableStringCol('INSTRUME', 0, 10)) + table.append(gammalib.GFitsTableStringCol('DETNAM', 0, 20)) + table.append(gammalib.GFitsTableStringCol('FILTER', 0, 10)) + table.append(gammalib.GFitsTableStringCol('CAL_DEV', 0, 20)) + table.append(gammalib.GFitsTableStringCol('CAL_DIR', 0, 70)) + table.append(gammalib.GFitsTableStringCol('CAL_FILE', 0, 70)) # Extend beyond standard + table.append(gammalib.GFitsTableStringCol('CAL_CLAS', 0, 3)) + table.append(gammalib.GFitsTableStringCol('CAL_DTYP', 0, 4)) + table.append(gammalib.GFitsTableStringCol('CAL_CNAM', 0, 20)) + table.append(gammalib.GFitsTableStringCol('CAL_CBD', 0, 70, 9)) + table.append(gammalib.GFitsTableShortCol('CAL_XNO', 0)) + table.append(gammalib.GFitsTableStringCol('CAL_VSD', 0, 10)) + table.append(gammalib.GFitsTableStringCol('CAL_VST', 0, 8)) + table.append(gammalib.GFitsTableDoubleCol('REF_TIME', 0)) + table.append(gammalib.GFitsTableShortCol('CAL_QUAL', 0)) + table.append(gammalib.GFitsTableStringCol('CAL_DATE', 0, 8)) + table.append(gammalib.GFitsTableStringCol('CAL_DESC', 0, 70)) + + # Set keywords. Reference: CAL/GEN/92-008 + table.extname('CIF') + table.card('CIFVERSN', '1992a', 'Version of CIF format') + + # Return table + return table + + def _add_irfs_to_cif(self, prod, irfs): + """ + Add IRFs to calibration index file + + Parameters + ---------- + prod : str + Production + irfs : list + List of IRFs + """ + # Build calibration database index name + fname = '%s/data/cta/%s/caldb.indx' % (self['outdir'].filename().url(), prod) + + # Open calibration database index file + cif = gammalib.GFits(fname, True) + + # If file has no CIF extension than create it now + if not cif.contains('CIF'): + cif.append(self._create_cif_table()) + + # Get calibration database index HDU + cif_hdu = cif.table('CIF') + + # Loop over all IRFs and add information to CIF + for irf in irfs: + self._add_irf_to_cif(prod, cif_hdu, irf) + + # Save and close CIF + cif.save(True) + cif.close() + + # Return + return + + def _add_irfs(self): + """ + Add IRFs to calibration database + """ + # Collect tarfiles + tarfiles = self._collect_tarfiles(self['indir'].filename()) + + # Raise an exception if no tarfiles were found + if len(tarfiles) == 0: + msg = ('No tarfiles found in folder "'+self['indir'].filename().url()+'. ' + 'Please specify a folder that contains IRF tarfiles.') + raise RuntimeError(msg) + + # Add IRFs in tarfiles + for file in tarfiles: + + # Extract Production + fname = os.path.basename(file) + elements = fname.split('-') + prod = [e for e in elements if 'prod' in e][0] + version = [e for e in elements if 'v' in e][0] + prod = '%s-%s' % (prod, version) + + # Build output directory + outdir = 'data/cta/%s/bcf' % (prod) + + # Write header and log parameters + self._log_header2(gammalib.TERSE, 'Extract IRFs') + self._log_value(gammalib.NORMAL, 'Tarfile', file) + self._log_value(gammalib.NORMAL, 'Production', prod) + self._log_value(gammalib.NORMAL, 'Target directory', outdir) + + # Extract IRFs from tarfile + irfs = self._extract_irfs(file, outdir) + + # Add IRFs to calibration index database + self._add_irfs_to_cif(prod, irfs) + + # Return + return + + + # Public methods + def run(self): + """ + Run the script + """ + # Switch screen logging on in debug mode + if self._logDebug(): + self._log.cout(True) + + # Get parameters + self._get_parameters() + + # Write header + self._log_header1(gammalib.TERSE, 'Add IRFs to CALDB') + + # Add IRFs to CALDB + self._add_irfs() + + # Return + return + + +# ======================== # +# Main routine entry point # +# ======================== # +if __name__ == '__main__': + + # Create instance of application + app = csadd2caldb(sys.argv) + + # Execute application + app.execute() diff --git a/cscripts/mputils.py b/cscripts/mputils.py old mode 100644 new mode 100755 index 722c011..722c011 --- a/cscripts/mputils.py +++ b/cscripts/mputils.py diff --git a/doc/source/admin/release_history/2.0.rst b/doc/source/admin/release_history/2.0.rst index 7050be9..6635a95 100644 --- a/doc/source/admin/release_history/2.0.rst +++ b/doc/source/admin/release_history/2.0.rst @@ -1,7 +1,7 @@ .. _2.0: -ctools 2.0 release (25 June 2021) -================================= +ctools 2.0 release (30 September 2021) +====================================== Overview -------- @@ -58,6 +58,7 @@ New features * [`3738 `_] - Add COMPTEL analysis scripts +* Add :ref:`csadd2caldb` script Clean up diff --git a/doc/source/users/reference_manual/csadd2caldb.rst b/doc/source/users/reference_manual/csadd2caldb.rst new file mode 100755 index 0000000..e991272 --- /dev/null +++ b/doc/source/users/reference_manual/csadd2caldb.rst @@ -0,0 +1,55 @@ +.. _csadd2caldb: + +csadd2caldb +=========== + +Adds CTA response function to calibration database. + + +Synopsis +-------- + +This script adds a CTA response function to the calibration database. The +CTA response functions can be downloaded from CTAO Zenodo directory. + + +General parameters +------------------ + +``indir [file]`` + Input IRF folder. + +``outdir [file]`` + Output caldb folder. + + +Standard parameters +------------------- + +``(chatter = 2) [integer]`` + Verbosity of the executable: + ``chatter = 0``: no information will be logged + + ``chatter = 1``: only errors will be logged + + ``chatter = 2``: errors and actions will be logged + + ``chatter = 3``: report about the task execution + + ``chatter = 4``: detailed report about the task execution + +``(clobber = yes) [boolean]`` + Specifies whether an existing energy boundaries output file should be overwritten. + +``(debug = no) [boolean]`` + Enables debug mode. In debug mode the executable will dump any log file output to the console. + +``(mode = ql) [string]`` + Mode of automatic parameters (default is ``ql``, i.e. "query and learn"). + +``(logfile = csadd2caldb.log) [filename]`` + Log filename. + + +Related tools or scripts +------------------------ diff --git a/doc/source/users/reference_manual/index.rst b/doc/source/users/reference_manual/index.rst index 8f7d4f2..af2921a 100755 --- a/doc/source/users/reference_manual/index.rst +++ b/doc/source/users/reference_manual/index.rst @@ -64,6 +64,7 @@ CTA and IACT analysis tools and scripts cspull --- Generates pull distribution csresmap --- Generates residual map csresspec --- Generates residual spectrum + csadd2caldb --- Adds CTA response function to calibration database csroot2caldb --- Creates a caldb entry from a ROOT file csscs --- Performs spectral component separation cssens --- Computes CTA sensitivity diff --git a/doc/source/users/user_manual/csadd2caldb.py b/doc/source/users/user_manual/csadd2caldb.py deleted file mode 100755 index d194878..0000000 --- a/doc/source/users/user_manual/csadd2caldb.py +++ /dev/null @@ -1,421 +0,0 @@ -#! /usr/bin/env python -# ========================================================================== -# Add IRFs to CALDB -# -# Copyright (C) 2021 Juergen Knoedlseder -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# -# ========================================================================== -import os -import sys -import glob -import tarfile -import gammalib -import ctools - - -# ================= # -# csadd2caldb class # -# ================= # -class csadd2caldb(ctools.cscript): - """ - Add IRFs to CALDB - """ - - # Constructor - def __init__(self, *argv): - """ - Constructor - """ - # Make sure that parfile exists - self._parfile() - - # Initialise application by calling the base class constructor - self._init_cscript(self.__class__.__name__, ctools.__version__, argv) - - # Return - return - - - # Private methods - def _parfile(self): - """ - Check if parfile exists. If parfile does not exist then create a - default parfile. This kluge avoids shipping the cscript with a parfile. - """ - # Set parfile name - parfile = self.__class__.__name__+'.par' - - # Load parfile or create - try: - pars = gammalib.GApplicationPars(parfile) - except: - # Signal that parfile was not found - print('Parfile '+parfile+' not found. Create default parfile.') - - # Create default parfile - pars = gammalib.GApplicationPars() - pars.append(gammalib.GApplicationPar('indir','f','a','fits','','','Input IRF folder')) - pars.append(gammalib.GApplicationPar('outdir','f','a','$CALDB','','','Output caldb folder')) - pars.append_standard() - pars.save(parfile) - - # Return - return - - def _get_parameters(self): - """ - Get parameters from parfile - """ - # Query parameters - self['indir'].filename() - self['outdir'].filename() - - # Write input parameters into logger - self._log_parameters(gammalib.TERSE) - - # Return - return - - def _collect_tarfiles(self, indir): - """ - Collect tarfiles - - Parameters - ---------- - indir : string - Input directory - - Returns - ------- - tarfiles : list of str - List of tarfiles - """ - # Initialise list of tarfiles - tarfiles = [] - - # Search for tarfiles in current directory - files = glob.glob('%s/*.FITS.tar.gz' % indir.url()) - tarfiles.extend(files) - - # Search for tarfiles in 'fits' subdirectory - files = glob.glob('%s/fits/*.FITS.tar.gz' % indir.url()) - tarfiles.extend(files) - - # Return tarfiles - return tarfiles - - def _extract_irfs(self, file, outdir): - """ - Collect IRFs from tarfile - - Parameters - ---------- - file : str - Tarfile with IRFs - outdir : str - Output directory - - Returns - ------- - irfs : list - List of IRF dictionaries - """ - # Initialise list of extracted IRFs - irfs = [] - - # Open tarfile - tar = tarfile.open(file) - - # Loop over all members in tarfile - for member in tar.getmembers(): - - # Get subarray - subarrays = '' - nsubarrays = 0 - if 'LST' in member.name: - subarrays += '_LST' - nsubarrays += 1 - if 'MST' in member.name: - subarrays += '_MST' - nsubarrays += 1 - if 'SST' in member.name: - subarrays += '_SST' - nsubarrays += 1 - if nsubarrays > 1: - subarrays = '' - - # Extract Instrument - if 'South-' in member.name: - site = 'South' - else: - site = 'North' - if '180000s' in member.name: - duration = '50h' - elif '18000s' in member.name: - duration = '5h' - elif '1800s' in member.name: - duration = '0.5h' - if 'SouthAz' in member.name: - orientation = '_S' - azimuth = 180.0 - elif 'NorthAz' in member.name: - orientation = '_N' - azimuth = 0.0 - else: - orientation = '' - azimuth = 90.0 - - elements = member.name.split('-') - zenith = [e for e in elements if 'deg' in e][0].strip('deg') - instrument = '%s_z%s%s_%s%s' % (site, zenith, orientation, duration, subarrays) - - # Build output directory - path = '%s/%s' % (outdir, instrument) - - # Write header - self._log_header3(gammalib.TERSE, 'Extract IRFs from "%s"' % member.name) - self._log_value(gammalib.NORMAL, 'Instrument', instrument) - - # Extract file - tar.extract(member, path='%s/%s' % (self['outdir'].filename().url(), path)) - - # Create IRF record - record = {'path': path, 'instrument': instrument, 'file': member.name, - 'site': site, 'duration': duration, 'zenith': zenith, 'azimuth': azimuth, - 'orientation': orientation} - - # Add record - irfs.append(record) - - # Close tarfile - tar.close() - - # Return - return irfs - - def _add_irf_to_cif(self, prod, hdu, irf): - """ - Add IRF to calibration index file HDU - - Parameters - ---------- - prod : str - Production - hdu : ~gammalib.GFitsBinTable - CIF HDU table - irf : dict - IRF dictionary - """ - # Set list of IRF component names - #names = ['EA', 'PSF', 'EDISP', 'BKG'] - components = [{'name': 'EA', 'CNAM': 'EFF_AREA', 'DESC': 'CTA effective area'}, - {'name': 'PSF', 'CNAM': 'RPSF', 'DESC': 'CTA point spread function'}, - {'name': 'EDISP', 'CNAM': 'EDISP', 'DESC': 'CTA energy dispersion'}, - {'name': 'BKG', 'CNAM': 'BKG', 'DESC': 'CTA background'}] - - # Initialise CIF row index - row = hdu.nrows() - - # Append rows for all components to CIF extension - hdu.append_rows(len(components)) - - # Add information for all components - for component in components: - - # Set calibration information - cal_name = 'NAME(%s)' % irf['instrument'] - cal_version = 'VERSION(%s)' % prod - cal_cut = 'CLASS(BEST)' - cal_analysis = 'ANALYSIS(CTA)' - cal_zenith = 'ZENITH(%.3f)deg' % float(irf['zenith']) - cal_azimuth = 'AZIMUTH(%.3f)deg' % float(irf['azimuth']) - cal_bounds = [cal_name, cal_version, cal_cut, cal_analysis, \ - cal_zenith, cal_azimuth] - - # Set generic information - hdu['TELESCOP'][row] = 'CTA' - hdu['INSTRUME'][row] = gammalib.toupper(prod) - hdu['DETNAM'][row] = 'NONE' - hdu['FILTER'][row] = 'NONE' - hdu['CAL_DEV'][row] = 'ONLINE' - hdu['CAL_CLAS'][row] = 'BCF' - hdu['CAL_DTYP'][row] = 'DATA' - hdu['CAL_VSD'][row] = '2014-01-30' - hdu['CAL_VST'][row] = '00:00:00' - hdu['REF_TIME'][row] = 51544.0 - hdu['CAL_QUAL'][row] = 0 # 0=good, 1=bad, 2=dubious, ... - hdu['CAL_DATE'][row] = '14/01/30' - - # Set component specific information - hdu['CAL_DIR'][row] = irf['path'] - hdu['CAL_FILE'][row] = irf['file'] - hdu['CAL_CNAM'][row] = component['CNAM'] - hdu['CAL_DESC'][row] = component['DESC'] - hdu['CAL_XNO'][row] = 1 - for i in range(9): - if i >= len(cal_bounds): - hdu['CAL_CBD'][row,i] = 'NONE' - else: - hdu['CAL_CBD'][row,i] = cal_bounds[i] - - # Increment row index - row += 1 - - # Return - return - - def _create_cif_table(self): - """ - Create Calibration Database Index File binary table - - Returns - ------- - table : ~gammalib.GFitsBinTable - Calibration Database Index File binary table - """ - # Create binary table - table = gammalib.GFitsBinTable() - - # Append columns. Reference: CAL/GEN/92-008 - table.append(gammalib.GFitsTableStringCol('TELESCOP', 0, 10)) - table.append(gammalib.GFitsTableStringCol('INSTRUME', 0, 10)) - table.append(gammalib.GFitsTableStringCol('DETNAM', 0, 20)) - table.append(gammalib.GFitsTableStringCol('FILTER', 0, 10)) - table.append(gammalib.GFitsTableStringCol('CAL_DEV', 0, 20)) - table.append(gammalib.GFitsTableStringCol('CAL_DIR', 0, 70)) - table.append(gammalib.GFitsTableStringCol('CAL_FILE', 0, 70)) # Extend beyond standard - table.append(gammalib.GFitsTableStringCol('CAL_CLAS', 0, 3)) - table.append(gammalib.GFitsTableStringCol('CAL_DTYP', 0, 4)) - table.append(gammalib.GFitsTableStringCol('CAL_CNAM', 0, 20)) - table.append(gammalib.GFitsTableStringCol('CAL_CBD', 0, 70, 9)) - table.append(gammalib.GFitsTableShortCol('CAL_XNO', 0)) - table.append(gammalib.GFitsTableStringCol('CAL_VSD', 0, 10)) - table.append(gammalib.GFitsTableStringCol('CAL_VST', 0, 8)) - table.append(gammalib.GFitsTableDoubleCol('REF_TIME', 0)) - table.append(gammalib.GFitsTableShortCol('CAL_QUAL', 0)) - table.append(gammalib.GFitsTableStringCol('CAL_DATE', 0, 8)) - table.append(gammalib.GFitsTableStringCol('CAL_DESC', 0, 70)) - - # Set keywords. Reference: CAL/GEN/92-008 - table.extname('CIF') - table.card('CIFVERSN', '1992a', 'Version of CIF format') - - # Return table - return table - - def _add_irfs_to_cif(self, prod, irfs): - """ - Add IRFs to calibration index file - - Parameters - ---------- - prod : str - Production - irfs : list - List of IRFs - """ - # Build calibration database index name - fname = '%s/data/cta/%s/caldb.indx' % (self['outdir'].filename().url(), prod) - - # Open calibration database index file - cif = gammalib.GFits(fname, True) - - # If file has no CIF extension than create it now - if not cif.contains('CIF'): - cif.append(self._create_cif_table()) - - # Get calibration database index HDU - cif_hdu = cif.table('CIF') - - # Loop over all IRFs and add information to CIF - for irf in irfs: - self._add_irf_to_cif(prod, cif_hdu, irf) - - # Save and close CIF - cif.save(True) - cif.close() - - # Return - return - - def _add_irfs(self): - """ - Add IRFs to calibration database - """ - # Collect tarfiles - tarfiles = self._collect_tarfiles(self['indir'].filename()) - - # Add IRFs in tarfiles - for file in tarfiles: - - # Extract Production - fname = os.path.basename(file) - elements = fname.split('-') - prod = [e for e in elements if 'prod' in e][0] - version = [e for e in elements if 'v' in e][0] - prod = '%s-%s' % (prod, version) - - # Build output directory - outdir = 'data/cta/%s/bcf' % (prod) - - # Write header and log parameters - self._log_header2(gammalib.TERSE, 'Extract IRFs') - self._log_value(gammalib.NORMAL, 'Tarfile', file) - self._log_value(gammalib.NORMAL, 'Production', prod) - self._log_value(gammalib.NORMAL, 'Target directory', outdir) - - # Extract IRFs from tarfile - irfs = self._extract_irfs(file, outdir) - - # Add IRFs to calibration index database - self._add_irfs_to_cif(prod, irfs) - - # Return - return - - - # Public methods - def run(self): - """ - Run the script - """ - # Switch screen logging on in debug mode - if self._logDebug(): - self._log.cout(True) - - # Get parameters - self._get_parameters() - - # Write header - self._log_header1(gammalib.TERSE, 'Add IRFs to CALDB') - - # Add IRFs to CALDB - self._add_irfs() - - # Return - return - - -# ======================== # -# Main routine entry point # -# ======================== # -if __name__ == '__main__': - - # Create instance of application - app = csadd2caldb(sys.argv) - - # Execute application - app.execute() diff --git a/doc/source/users/user_manual/irf_cta.rst b/doc/source/users/user_manual/irf_cta.rst index d21b151..9a99651 100755 --- a/doc/source/users/user_manual/irf_cta.rst +++ b/doc/source/users/user_manual/irf_cta.rst @@ -83,10 +83,6 @@ In total, the following six instrument response functions are available: The latest CTA reponse functions (``prod5-v0.1``) can be downloaded from `this link `_. -Addition of CTA response functions to the calibration database can be done -using the script :download:`csadd2caldb.py ` that is available -for download. - To download and install the ``prod5-v0.1`` response functions you should execute the following commands in a terminal: @@ -96,8 +92,7 @@ the following commands in a terminal: $ cd temp $ wget https://zenodo.org/record/5499840/files/cta-prod5-zenodo-fitsonly-v0.1.zip $ unzip cta-prod5-zenodo-fitsonly-v0.1.zip - $ ./csadd2caldb.py - Parfile csadd2caldb.par not found. Create default parfile. + $ csadd2caldb Input IRF folder [fits] Output caldb folder [$CALDB] diff --git a/test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz b/test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz new file mode 100644 index 0000000..052f7f3 Binary files /dev/null and b/test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz differ diff --git a/test/test_csadd2caldb.py b/test/test_csadd2caldb.py new file mode 100755 index 0000000..bb47881 --- /dev/null +++ b/test/test_csadd2caldb.py @@ -0,0 +1,171 @@ +#! /usr/bin/env python +# ========================================================================== +# This scripts performs unit tests for the csadd2caldb script. +# +# Copyright (C) 2021 Juergen Knoedlseder +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ========================================================================== +import os +import gammalib +import cscripts +from testing import test + + +# ================================= # +# Test class for csadd2caldb script # +# ================================= # +class Test(test): + """ + Test class for csadd2caldb script + + This test class makes unit tests for the csadd2caldb script by using it + from the command line and from Python. + """ + + # Constructor + def __init__(self): + """ + Constructor + """ + # Call base class constructor + test.__init__(self) + + # Set test datasets and parameters + self._prod501 = self._datadir + '/prod5-v0.1' + + # Clear results + os.system('rm -rf csadd2caldb_cmd1') + os.system('rm -rf csadd2caldb_py1') + os.system('rm -rf csadd2caldb_py1_pickle') + + # Return + return + + # Set test functions + def set(self): + """ + Set all test functions. + """ + # Set test name + self.name('csadd2caldb') + + # Append tests + self.append(self._test_cmd, 'Test csadd2caldb on command line') + self.append(self._test_python, 'Test csadd2caldb from Python') + self.append(self._test_pickeling, 'Test csadd2caldb pickeling') + + # Return + return + + # Test csadd2caldb on command line + def _test_cmd(self): + """ + Test csadd2caldb on the command line. + """ + # Set script name + csadd2caldb = self._script('csadd2caldb') + + # Setup csadd2caldb command + cmd = csadd2caldb+' indir="'+self._prod501+'"'+ \ + ' outdir="csadd2caldb_cmd1"'+ \ + ' logfile="csadd2caldb_cmd1.log" chatter=4' + + # Check if execution was successful + self.test_assert(self._execute(cmd) == 0, + 'Check successful execution from command line') + + # Check IRF + self._check_irf('csadd2caldb_cmd1') + + # Setup csadd2caldb command + cmd = csadd2caldb+' indir="folder_that_does_not_exist"'+ \ + ' outdir="csadd2caldb_cmd2"'+ \ + ' logfile="csadd2caldb_cmd2.log" chatter=1' + + # Check if execution failed + self.test_assert(self._execute(cmd, success=False) != 0, + 'Check invalid input file when executed from command line') + + # Check csadd2caldb --help + self._check_help(csadd2caldb) + + # Return + return + + # Test csadd2caldb from Python + def _test_python(self): + """ + Test csadd2caldb from Python + """ + # Set-up csadd2caldb + addirf = cscripts.csadd2caldb() + addirf['indir'] = self._prod501 + addirf['outdir'] = 'csadd2caldb_py1' + addirf['logfile'] = 'csadd2caldb_py1.log' + + # Run csadd2caldb script and save background model + addirf.logFileOpen() # Make sure we get a log file + addirf.run() + addirf.save() + + # Check IRF + self._check_irf('csadd2caldb_py1') + + # Return + return + + # Test csadd2caldb pickeling + def _test_pickeling(self): + """ + Test csadd2caldb pickeling + """ + # Perform pickeling tests of empty class + self._pickeling(cscripts.csadd2caldb()) + + # Set-up csadd2caldb + addirf = cscripts.csadd2caldb() + addirf['indir'] = self._prod501 + addirf['outdir'] = 'csadd2caldb_py1_pickle' + addirf['logfile'] = 'csadd2caldb_py1_pickle.log' + + # Perform pickeling tests of filled class + obj = self._pickeling(addirf) + + # Run csadd2caldb script and save background model + obj.logFileOpen() # Make sure we get a log file + obj.run() + obj.save() + + # Check IRF + self._check_irf('csadd2caldb_py1_pickle') + + # Return + return + + # Check IRF + def _check_irf(self, filename, prod='prod5-v0.1', nirfs=8): + """ + Check IRF + """ + # Open CALDB + caldb = gammalib.GCaldb(filename) + caldb.open('CTA', prod) + + # Check CALDB + self.test_value(caldb.size(), nirfs, 'Check for %d entries in CALDB' % nirfs) + + # Return + return diff --git a/test/test_python_cscripts.py b/test/test_python_cscripts.py index 4e82909..1cb4aac 100755 --- a/test/test_python_cscripts.py +++ b/test/test_python_cscripts.py @@ -2,7 +2,7 @@ # ========================================================================== # This scripts performs unit tests for cscripts # -# Copyright (C) 2016-2020 Juergen Knoedlseder +# Copyright (C) 2016-2021 Juergen Knoedlseder # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,6 +25,7 @@ import sys sys.path.append(os.path.dirname(os.path.abspath(__file__))) import test_cscript import test_cscaldb +import test_csadd2caldb import test_csbkgmodel import test_csebins import test_csfindobs @@ -117,6 +118,7 @@ def test(installed=False, debug=False): # Define list of test suites tests = [test_cscript.Test(), + test_csadd2caldb.Test(), test_csbkgmodel.Test(), test_cscaldb.Test(), test_csebins.Test(), -- libgit2 0.21.4