Add csadd2caldb script

Jürgen Knödlseder
1 parent c057d371
Showing 16 changed files with 295 additions and 44 deletions
.gitignore
ChangeLog
NEWS
README.md
cscripts/Makefile.am
cscripts/__init__.py.in
cscripts/csadd2caldb.par
doc/source/users/user_manual/csadd2caldb.py → cscripts/csadd2caldb.py
cscripts/mputils.py
doc/source/admin/release_history/2.0.rst
doc/source/users/reference_manual/csadd2caldb.rst
doc/source/users/reference_manual/index.rst
doc/source/users/user_manual/irf_cta.rst
test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz
test/test_csadd2caldb.py
test/test_python_cscripts.py
@@ -83,6 +83,7 @@ src/template/ctool_obs
 test/pfiles/
 test/caldb/
 test/prod3
+test/csadd2caldb_*
 test/csroot2caldb_*
 test/csiactcopy_*
 test/comobsbin_*
-2021-06-25
+2021-09-30
  
         * Version 2.0.0 released
         ========================
  
+        Add csadd2caldb script
         Add restart of bisection in case of bad convergence in ctulimit
         Add COMPTEL analysis scripts (#3738)
         Add source counts constraints to cssens (#3668)
 New Features and Important Changes in ctools 2.0.0
  
-25 June 2021
+30 September 2021
  
  
 Introduction
@@ -167,6 +167,12 @@ support - Support classes
 Add stamping to ctool::save_event_list() method (#3662).
  
  
+csadd2caldb - Adds CTA response function to calibration database
+----------------------------------------------------------------
+New script that was added to support the installation of prod5-v0.1 CTA response
+functions into the calibration database.
+
+
 csbkgmodel - Generate background model for 3D analysis
 ------------------------------------------------------
 None
 ctools information
 ==================
-* Version:             2.0.0.dev (25 June 2021)
+* Version:             2.0.0.dev (30 September 2021)
 * GammaLib dependency: 2.0.0.dev
  
 [![Build Status](https://cta-jenkins.irap.omp.eu/buildStatus/icon?job=ctools-integrate-os)](https://cta-jenkins.irap.omp.eu/job/ctools-integrate-os/)
@@ -21,6 +21,7 @@
  
 # Scripts to be distributed in $(pyexecdir)/cscripts
 pythonscripts = $(top_builddir)/cscripts/__init__.py \
+                $(srcdir)/csadd2caldb.py \
                 $(srcdir)/csbkgmodel.py \
                 $(srcdir)/cscaldb.py \
                 $(srcdir)/csebins.py \
@@ -60,7 +61,8 @@ pythonscripts = $(top_builddir)/cscripts/__init__.py \
                 $(srcdir)/mputils.py 	
  
 # Parfiles to be distributed in $(prefix)/syspfiles
-parfiles = $(srcdir)/csbkgmodel.par \
+parfiles = $(srcdir)/csadd2caldb.par \
+           $(srcdir)/csbkgmodel.par \
            $(srcdir)/cscaldb.par \
            $(srcdir)/csebins.par \
            $(srcdir)/cslightcrv.par \
@@ -93,7 +95,8 @@ parfiles = $(srcdir)/csbkgmodel.par \
            $(srcdir)/csiactobs.par
  
 # Scripts for which symbolic links will be created in bindir
-linkscripts = csbkgmodel \
+linkscripts = csadd2caldb \
+              csbkgmodel \
               cscaldb \
               csebins \
               csinfo \
@@ -132,6 +135,7 @@ testscripts = $(srcdir)/tests/__init__.py \
               $(top_srcdir)/test/test_python_cscripts.py \
               $(top_srcdir)/test/test_cscript.py \
               $(top_srcdir)/test/test_cscaldb.py \
+              $(top_srcdir)/test/test_csadd2caldb.py \
               $(top_srcdir)/test/test_csbkgmodel.py \
               $(top_srcdir)/test/test_csebins.py \
               $(top_srcdir)/test/test_csfindobs.py \
@@ -266,6 +270,9 @@ testiactevents = $(top_srcdir)/test/data/iactdata/events/irf_file.fits.gz \
                  $(top_srcdir)/test/data/iactdata/events/events_7.fits.gz \
                  $(top_srcdir)/test/data/iactdata/events/events_8.fits.gz
  
+# Test data to be distributed in $(pyexecdir)/cscripts/tests/data/prod5-v0.1
+testprod501data = $(top_srcdir)/test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz
+
 # Install scripts, parfiles, test scripts and test data. Note that we don't
 # need to prepend $(DESTDIR) here (not fully clear why, but that's the way
 # it is)
@@ -283,11 +290,13 @@ testiactdatadir     = $(pyexecdir)/cscripts/tests/data/iactdata
 testiactdata_DATA   = $(testiactdata)
 testiacteventsdir   = $(pyexecdir)/cscripts/tests/data/iactdata/events
 testiactevents_DATA = $(testiactevents)
+testprod501dir      = $(pyexecdir)/cscripts/tests/data/prod5-v0.1
+testprod501_DATA    = $(testprod501data)
  
 # Files to be distributed in the tar file
 EXTRA_DIST = $(binscripts) $(pythonscripts) $(parfiles) \
              $(testscripts) $(testdata) $(testiactdata) \
-             $(testiactevents)
+             $(testiactevents) $(testprod501data)
  
 # Add install hook rule to create symbolic links to cscripts without .py in the bindir
 install-data-hook:
 # ==========================================================================
 # cscripts Python module
 #
-# Copyright (C) 2015-2020 Juergen Knoedlseder
+# Copyright (C) 2015-2021 Juergen Knoedlseder
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -24,6 +24,7 @@ cur_version = sys.version_info
  
 # List all scripts in this module
 __all__ = [
+    "csadd2caldb",
     "csbkgmodel",
     "cscaldb",
     "csebins",
@@ -65,6 +66,7 @@ if cur_version &gt; req_version:
                     "csiactobs"])
  
 # Import
+from cscripts.csadd2caldb   import csadd2caldb
 from cscripts.csbkgmodel    import csbkgmodel
 from cscripts.cscaldb       import cscaldb
 from cscripts.csebins       import csebins
+#############################################################################
+#                                                                           #
+#                       CTA Science Analysis Tools                          #
+#                                                                           #
+# ------------------------------------------------------------------------- #
+#                                                                           #
+#  File:      csadd2caldb.par                                               #
+#                                                                           #
+#  Author:    Juergen Knoedlseder                                           #
+#             Jurgen.Knodlseder@irap.omp.eu                                 #
+#             IRAP                                                          #
+#                                                                           #
+#  Purpose:   Parameter file for the csadd2caldb script                     #
+#                                                                           #
+#############################################################################
+#
+# Input and output parameters
+#============================
+indir,  f, a, fits,,, "Input IRF folder"
+outdir, f, a, $CALDB,,, "Output caldb folder"
+
+#
+# Standard parameters
+#====================
+chatter, i, h, 2,0,4, "Chattiness of output"
+clobber, b, h, yes,,, "Overwrite existing output files with new output files?"
+debug,   b, h, no,,, "Activate debugging mode?"
+mode,    s, h, ql,,, "Mode of automatic parameters"
+logfile, f, h, csadd2caldb.log,,, "Log filename"
@@ -39,9 +39,6 @@ class csadd2caldb(ctools.cscript):
         """
         Constructor
         """
-        # Make sure that parfile exists
-        self._parfile()
-
         # Initialise application by calling the base class constructor
         self._init_cscript(self.__class__.__name__, ctools.__version__, argv)
  
@@ -50,31 +47,6 @@ class csadd2caldb(ctools.cscript):
  
  
     # Private methods
-    def _parfile(self):
-        """
-        Check if parfile exists. If parfile does not exist then create a
-        default parfile. This kluge avoids shipping the cscript with a parfile.
-        """
-        # Set parfile name
-        parfile = self.__class__.__name__+'.par'
-
-        # Load parfile or create
-        try:
-            pars = gammalib.GApplicationPars(parfile)
-        except:
-            # Signal that parfile was not found
-            print('Parfile '+parfile+' not found. Create default parfile.')
-
-            # Create default parfile
-            pars = gammalib.GApplicationPars()
-            pars.append(gammalib.GApplicationPar('indir','f','a','fits','','','Input IRF folder'))
-            pars.append(gammalib.GApplicationPar('outdir','f','a','$CALDB','','','Output caldb folder'))
-            pars.append_standard()
-            pars.save(parfile)
-
-        # Return
-        return
-
     def _get_parameters(self):
         """
         Get parameters from parfile
@@ -358,6 +330,12 @@ class csadd2caldb(ctools.cscript):
         # Collect tarfiles
         tarfiles = self._collect_tarfiles(self['indir'].filename())
  
+        # Raise an exception if no tarfiles were found
+        if len(tarfiles) == 0:
+            msg = ('No tarfiles found in folder "'+self['indir'].filename().url()+'. '
+                   'Please specify a folder that contains IRF tarfiles.')
+            raise RuntimeError(msg)
+
         # Add IRFs in tarfiles
         for file in tarfiles:
  
 .. _2.0:
  
-ctools 2.0 release (25 June 2021)
-=================================
+ctools 2.0 release (30 September 2021)
+======================================
  
 Overview
 --------
@@ -58,6 +58,7 @@ New features
  
 * [`3738 <https://cta-redmine.irap.omp.eu/issues/3738>`_] -
   Add COMPTEL analysis scripts
+* Add :ref:`csadd2caldb` script
  
  
 Clean up
+.. _csadd2caldb:
+
+csadd2caldb
+===========
+
+Adds CTA response function to calibration database.
+
+
+Synopsis
+--------
+
+This script adds a CTA response function to the calibration database. The
+CTA response functions can be downloaded from CTAO Zenodo directory.
+
+
+General parameters
+------------------
+
+``indir [file]``
+    Input IRF folder.
+
+``outdir [file]``
+    Output caldb folder.
+
+
+Standard parameters
+-------------------
+
+``(chatter = 2) [integer]``
+    Verbosity of the executable:
+     ``chatter = 0``: no information will be logged
+
+     ``chatter = 1``: only errors will be logged
+
+     ``chatter = 2``: errors and actions will be logged
+
+     ``chatter = 3``: report about the task execution
+
+     ``chatter = 4``: detailed report about the task execution
+
+``(clobber = yes) [boolean]``
+    Specifies whether an existing energy boundaries output file should be overwritten.
+
+``(debug = no) [boolean]``
+    Enables debug mode. In debug mode the executable will dump any log file output to the console.
+
+``(mode = ql) [string]``
+    Mode of automatic parameters (default is ``ql``, i.e. "query and learn").
+
+``(logfile = csadd2caldb.log) [filename]``
+    Log filename.
+
+
+Related tools or scripts
+------------------------
@@ -64,6 +64,7 @@ CTA and IACT analysis tools and scripts
    cspull --- Generates pull distribution <cspull>
    csresmap --- Generates residual map <csresmap>
    csresspec --- Generates residual spectrum <csresspec>
+   csadd2caldb --- Adds CTA response function to calibration database <csadd2caldb>
    csroot2caldb --- Creates a caldb entry from a ROOT file <csroot2caldb>
    csscs --- Performs spectral component separation <csscs>
    cssens --- Computes CTA sensitivity <cssens>
@@ -83,10 +83,6 @@ In total, the following six instrument response functions are available:
 The latest CTA reponse functions (``prod5-v0.1``) can be downloaded from
 `this link <https://zenodo.org/record/5499840#.YVV1di0itZo>`_.
  
-Addition of CTA response functions to the calibration database can be done
-using the script :download:`csadd2caldb.py <csadd2caldb.py>` that is available
-for download.
-
 To download and install the ``prod5-v0.1`` response functions you should execute
 the following commands in a terminal:
  
@@ -96,8 +92,7 @@ the following commands in a terminal:
    $ cd temp
    $ wget https://zenodo.org/record/5499840/files/cta-prod5-zenodo-fitsonly-v0.1.zip
    $ unzip cta-prod5-zenodo-fitsonly-v0.1.zip
-   $ ./csadd2caldb.py
-   Parfile csadd2caldb.par not found. Create default parfile.
+   $ csadd2caldb
    Input IRF folder [fits]
    Output caldb folder [$CALDB]
  
+#! /usr/bin/env python
+# ==========================================================================
+# This scripts performs unit tests for the csadd2caldb script.
+#
+# Copyright (C) 2021 Juergen Knoedlseder
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# ==========================================================================
+import os
+import gammalib
+import cscripts
+from testing import test
+
+
+# ================================= #
+# Test class for csadd2caldb script #
+# ================================= #
+class Test(test):
+    """
+    Test class for csadd2caldb script
+
+    This test class makes unit tests for the csadd2caldb script by using it
+    from the command line and from Python.
+    """
+
+    # Constructor
+    def __init__(self):
+        """
+        Constructor
+        """
+        # Call base class constructor
+        test.__init__(self)
+
+        # Set test datasets and parameters
+        self._prod501 = self._datadir + '/prod5-v0.1'
+
+        # Clear results
+        os.system('rm -rf csadd2caldb_cmd1')
+        os.system('rm -rf csadd2caldb_py1')
+        os.system('rm -rf csadd2caldb_py1_pickle')
+
+        # Return
+        return
+
+    # Set test functions
+    def set(self):
+        """
+        Set all test functions.
+        """
+        # Set test name
+        self.name('csadd2caldb')
+
+        # Append tests
+        self.append(self._test_cmd, 'Test csadd2caldb on command line')
+        self.append(self._test_python, 'Test csadd2caldb from Python')
+        self.append(self._test_pickeling, 'Test csadd2caldb pickeling')
+
+        # Return
+        return
+
+    # Test csadd2caldb on command line
+    def _test_cmd(self):
+        """
+        Test csadd2caldb on the command line.
+        """
+        # Set script name
+        csadd2caldb = self._script('csadd2caldb')
+
+        # Setup csadd2caldb command
+        cmd = csadd2caldb+' indir="'+self._prod501+'"'+ \
+                          ' outdir="csadd2caldb_cmd1"'+ \
+                          ' logfile="csadd2caldb_cmd1.log" chatter=4'
+
+        # Check if execution was successful
+        self.test_assert(self._execute(cmd) == 0,
+             'Check successful execution from command line')
+
+        # Check IRF
+        self._check_irf('csadd2caldb_cmd1')
+
+        # Setup csadd2caldb command
+        cmd = csadd2caldb+' indir="folder_that_does_not_exist"'+ \
+                          ' outdir="csadd2caldb_cmd2"'+ \
+                          ' logfile="csadd2caldb_cmd2.log" chatter=1'
+
+        # Check if execution failed
+        self.test_assert(self._execute(cmd, success=False) != 0,
+             'Check invalid input file when executed from command line')
+
+        # Check csadd2caldb --help
+        self._check_help(csadd2caldb)
+
+        # Return
+        return
+
+    # Test csadd2caldb from Python
+    def _test_python(self):
+        """
+        Test csadd2caldb from Python
+        """
+        # Set-up csadd2caldb
+        addirf = cscripts.csadd2caldb()
+        addirf['indir']   = self._prod501
+        addirf['outdir']  = 'csadd2caldb_py1'
+        addirf['logfile'] = 'csadd2caldb_py1.log'
+
+        # Run csadd2caldb script and save background model
+        addirf.logFileOpen()   # Make sure we get a log file
+        addirf.run()
+        addirf.save()
+
+        # Check IRF
+        self._check_irf('csadd2caldb_py1')
+
+        # Return
+        return
+
+    # Test csadd2caldb pickeling
+    def _test_pickeling(self):
+        """
+        Test csadd2caldb pickeling
+        """
+        # Perform pickeling tests of empty class
+        self._pickeling(cscripts.csadd2caldb())
+
+        # Set-up csadd2caldb
+        addirf = cscripts.csadd2caldb()
+        addirf['indir']   = self._prod501
+        addirf['outdir']  = 'csadd2caldb_py1_pickle'
+        addirf['logfile'] = 'csadd2caldb_py1_pickle.log'
+
+        # Perform pickeling tests of filled class
+        obj = self._pickeling(addirf)
+
+        # Run csadd2caldb script and save background model
+        obj.logFileOpen()   # Make sure we get a log file
+        obj.run()
+        obj.save()
+
+        # Check IRF
+        self._check_irf('csadd2caldb_py1_pickle')
+
+        # Return
+        return
+
+    # Check IRF
+    def _check_irf(self, filename, prod='prod5-v0.1', nirfs=8):
+        """
+        Check IRF
+        """
+        # Open CALDB
+        caldb = gammalib.GCaldb(filename)
+        caldb.open('CTA', prod)
+
+        # Check CALDB
+        self.test_value(caldb.size(), nirfs, 'Check for %d entries in CALDB' % nirfs)
+
+        # Return
+        return
@@ -2,7 +2,7 @@
 # ==========================================================================
 # This scripts performs unit tests for cscripts
 #
-# Copyright (C) 2016-2020 Juergen Knoedlseder
+# Copyright (C) 2016-2021 Juergen Knoedlseder
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -25,6 +25,7 @@ import sys
 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
 import test_cscript
 import test_cscaldb
+import test_csadd2caldb
 import test_csbkgmodel
 import test_csebins
 import test_csfindobs
@@ -117,6 +118,7 @@ def test(installed=False, debug=False):
  
     # Define list of test suites
     tests = [test_cscript.Test(),
+             test_csadd2caldb.Test(),
              test_csbkgmodel.Test(),
              test_cscaldb.Test(),
              test_csebins.Test(),