Refactor cscaldb script and add unit test.

Jürgen Knödlseder
1 parent d726da04
Showing 7 changed files with 244 additions and 76 deletions
cscripts/Makefile.am
cscripts/cscaldb.par
cscripts/cscaldb.py
pyext/Makefile.am
test/test_cscaldb.py
test/test_cscripts.sh.in
test/test_python.py
@@ -9,6 +9,9 @@ pythonscripts = __init__.py \
                 csfindobs.py csiactcopy.py csiactdata.py csiactobs.py \
                 obsutils.py
  
+# Parfiles to be distributed in $(prefix)/syspfiles
+parfiles = cscaldb.par
+
 # Scripts for which symbolic links will be created in bindir
 linkscripts = cspull cssens cstsdist cscaldb csresmap \
               csobsdef csspec cslightcrv csobsinfo \
@@ -16,12 +19,14 @@ linkscripts = cspull cssens cstsdist cscaldb csresmap \
               csobs2caldb \
               csfindobs csiactcopy csiactdata csiactobs
  
-# Distribute scripts
+# Distribute scripts and parfiles
 EXTRA_DIST         = $(binscripts) $(pythonscripts)
 binscriptsdir      = $(bindir)
 binscripts_DATA    = $(binscripts)
 pythonscriptsdir   = $(pyexecdir)/cscripts
 pythonscripts_DATA = $(pythonscripts)
+pfilesdir          = $(prefix)/syspfiles
+pfiles_DATA        = $(parfiles)
  
 # Add hook rule to create symbolic links for cscripts
 install-data-hook:
+#############################################################################
+#                                                                           #
+#                       CTA Science Analysis Tools                          #
+#                                                                           #
+# ------------------------------------------------------------------------- #
+#                                                                           #
+#  File:      cscaldb.par                                                   #
+#  Version:   1.1.0                                                         #
+#                                                                           #
+#  Author:    Jurgen Knodlseder                                             #
+#             Jurgen.Knodlseder@irap.omp.eu                                 #
+#             IRAP                                                          #
+#                                                                           #
+#  Purpose:   Parameter file for the cscaldb script                         #
+#                                                                           #
+#############################################################################
+#
+# Standard parameters
+#====================
+chatter, i, h, 2,0,4, "Chattiness of output"
+clobber, b, h, yes,,, "Overwrite existing output files with new output files?"
+debug,   b, h, no,,, "Debugging mode activated"
+mode,    s, h, "ql",,, "Mode of automatic parameters"
+logfile, f, h, "cscaldb.log",,, "Log filename"
@@ -2,7 +2,7 @@
 # ==========================================================================
 # This script dumps all available calibrations into the console.
 #
-# Copyright (C) 2014-2015 Juergen Knoedlseder
+# Copyright (C) 2014-2016 Juergen Knoedlseder
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -25,32 +25,28 @@ import glob
 import os
  
  
-# ============ #
-# cspull class #
-# ============ #
+# ============= #
+# cscaldb class #
+# ============= #
 class cscaldb(ctools.cscript):
     """
-    This class dumps the content of the ctools calibration database.
-    It derives from the ctools.cscript class which provides support for
-    parameter files, command line arguments, and logging. In that way
-    the Python script behaves just as a regular ctool.
+    The cscaldb class logs the content of the ctools calibration database
+    into the log file. Optionally, if the "debug" parameter is set to "yes"
+    the calibration database is also logged into the console.
     """
     def __init__(self, *argv):
         """
         Constructor.
         """
         # Set name
-        self.name    = "cscaldb"
-        self.version = "1.0.0"
-
-        # Make sure that parfile exists
-        file = self.parfile()
+        self._name    = "cscaldb"
+        self._version = "1.0.0"
  
         # Initialise application
         if len(argv) == 0:
-            ctools.cscript.__init__(self, self.name, self.version)
+            ctools.cscript.__init__(self, self._name, self._version)
         elif len(argv) ==1:
-            ctools.cscript.__init__(self, self.name, self.version, *argv)
+            ctools.cscript.__init__(self, self._name, self._version, *argv)
         else:
             raise TypeError("Invalid number of arguments given.")
  
@@ -68,34 +64,97 @@ class cscaldb(ctools.cscript):
         # Return
         return
  
-    def parfile(self):
+    def _get_parameters(self):
         """
-        Check if parfile exists. If parfile does not exist then create a
-        default parfile. This kluge avoids shipping the cscript with a parfile.
+        Get parameters from parfile.
         """
-        # Set parfile name
-        parfile = self.name+".par"
+        # Return
+        return
  
-        try:
-            pars = gammalib.GApplicationPars(parfile)
-        except:
-            # Signal if parfile was not found
-            sys.stdout.write("Parfile "+parfile+" not found. Create default parfile.\n")
+    def _get_missions(self, caldb):
+        """
+        Extract mission names from a calibration database.
  
-            # Create default parfile
-            pars = gammalib.GApplicationPars()
-            pars.append_standard()
-            pars.save(parfile)
+        Args:
+            caldb: Calibration database.
  
-        # Return
-        return
+        Returns:
+            A list of mission names.
+        """
+        # Initialise mission list
+        missions = []
+
+        # Extract missions
+        paths = glob.glob(caldb.rootdir()+"/data/*")
+        paths.sort()
+        for path in paths:
+            missions.append(os.path.basename(path))
  
-    def get_parameters(self):
+        # Sort missions
+        missions.sort()
+
+        # Return missions
+        return missions
+
+    def _get_instruments(self, caldb, mission):
         """
-        Get parameters from parfile.
+        Extract instrument names from a calibration database.
+
+        Args:
+            caldb:   Calibration database.
+            mission: Mission name.
+
+        Returns:
+            A list of instrument names.
         """
-        # Return
-        return
+        # Initialise instrument list
+        instruments = []
+
+        # Extract instruments
+        paths       = glob.glob(caldb.rootdir()+"/data/"+mission+"/*")
+        paths.sort()
+        for path in paths:
+            instruments.append(os.path.basename(path))
+
+        # Sort instruments
+        instruments.sort()
+
+        # Return instruments
+        return instruments
+
+    def _get_response_names(self, calibrations):
+        """
+        Extract response names from a calibrations FITS table and return
+        them in form of a list.
+
+        Args:
+            calibrations: FITS table containing the calibrations.
+
+        Returns:
+            A list of response names.
+        """
+        # Initialise response name list
+        names = []
+
+        # Extract response names from calibrations and append them to the
+        # response name list
+        nrows = calibrations.length()
+        ncols = calibrations.number()
+        for row in range(nrows):
+            for col in range(ncols):
+                cal    = calibrations.string(row, col)
+                istart = cal.find("NAME(")
+                if istart != -1:
+                    istop = cal.find(")")
+                    name  = cal[5:istop]
+                    if names.count(name) == 0:
+                        names.append(name)
+        
+        # Sort response names
+        names.sort()
+
+        # Return response names
+        return names
  
     def execute(self):
         """
@@ -116,23 +175,18 @@ class cscaldb(ctools.cscript):
             self.log.cout(True)
  
         # Get parameters
-        self.get_parameters()
+        self._get_parameters()
  
         #  Write input parameters into logger
         if self.logTerse():
             self.log_parameters()
             self.log("\n")
  
-        # Get the calibration database path
+        # Get the calibration database
         caldb = gammalib.GCaldb()
  
-        # Gather missions
-        missions = []
-        paths    = glob.glob(caldb.rootdir()+"/data/*")
-        paths.sort()
-        for path in paths:
-            head, tail = os.path.split(path)
-            missions.append(tail)
+        # Extract mission names from the calibration database
+        missions = self._get_missions(caldb)
  
         # Loop over missions
         for mission in missions:
@@ -146,41 +200,26 @@ class cscaldb(ctools.cscript):
                 self.log("\n")
                 self.log.header1("Mission: "+mission)
  
-            # Gather instruments
-            instruments = []
-            paths       = glob.glob(caldb.rootdir()+"/data/"+mission+"/*")
-            paths.sort()
-            for path in paths:
-                head, tail = os.path.split(path)
-                instruments.append(tail)
+            # Extract instruments
+            instruments = self._get_instruments(caldb, mission)
  
             # Loop over instruments
             for instrument in instruments:
  
                 # Write mission into logger
                 if self.logTerse():
-                    self.log.header3("Response functions in database \""+instrument+"\"")
+                    self.log.header3("Response functions in database \""+
+                                     instrument+"\"")
  
-                # Open calibration index file
-                cifname = caldb.rootdir()+"/data/"+mission+"/"+instrument+"/caldb.indx"
-                fits = gammalib.GFits(cifname)
-                cif  = fits["CIF"]
-                cals = cif["CAL_CBD"]
+                # Open calibration index file and retrieve calibrations
+                filename = "/data/"+mission+"/"+instrument+"/caldb.indx"
+                cifname  = caldb.rootdir() + filename
+                fits     = gammalib.GFits(cifname)
+                cif      = fits["CIF"]
+                cals     = cif["CAL_CBD"]
  
                 # Extract response names
-                names = []
-                nrows = cals.length()
-                ncols = cals.number()
-                for row in range(nrows):
-                    for col in range(ncols):
-                        cal    = cals.string(row, col)
-                        istart = cal.find("NAME(")
-                        if istart != -1:
-                            istop = cal.find(")")
-                            name  = cal[5:istop]
-                            if names.count(name) == 0:
-                                names.append(name)
-                names.sort()
+                names = self._get_response_names(cals)
  
                 # Print response name
                 if self.logTerse():
@@ -197,7 +236,7 @@ class cscaldb(ctools.cscript):
 # ======================== #
 if __name__ == '__main__':
     """
-    Dumps calibration database into log file.
+    Logs calibration database into the log file.
     """
     # Create instance of application
     app = cscaldb(sys.argv)
 #############################################################################
 # Makefile for ctools Python extension module                               #
 # ------------------------------------------------------------------------- #
-# Copyright (C) 2011-2015 Juergen Knoedlseder                               #
+# Copyright (C) 2011-2016 Juergen Knoedlseder                               #
 # ------------------------------------------------------------------------- #
 #                                                                           #
 #  This program is free software: you can redistribute it and/or modify     #
@@ -46,6 +46,7 @@ testscripts = ctools/tests/__init__.py \
               ../test/test_ctbutterfly.py \
               ../test/test_ctulimit.py \
               ../test/test_cterror.py \
+              ../test/test_cscaldb.py \
               ../test/test_pipelines.py
 testdata    = ../test/data/crab.xml \
               ../test/data/crab_events.fits.gz \
+#! /usr/bin/env python
+# ==========================================================================
+# This scripts performs unit tests for the cscaldb script.
+#
+# Copyright (C) 2016 Juergen Knoedlseder
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# ==========================================================================
+import gammalib
+import cscripts
+
+
+# ============================= #
+# Test class for cscaldb script #
+# ============================= #
+class Test(gammalib.GPythonTestSuite):
+    """
+    Test class for cscaldb script.
+    """
+    # Constructor
+    def __init__(self):
+        """
+        Constructor.
+        """
+        # Call base class constructor
+        gammalib.GPythonTestSuite.__init__(self)
+
+        # Return
+        return
+
+    # Set test functions
+    def set(self):
+        """
+        Set all test functions.
+        """
+        # Set test name
+        self.name("cscaldb")
+
+        # Append tests
+        self.append(self._test_functional, "Test cscaldb functionality")
+
+        # Return
+        return
+
+    # Test cscaldb functionality
+    def _test_functional(self):
+        """
+        Test cscaldb functionality.
+        """
+        # Set-up cscaldb
+        caldb = cscripts.cscaldb()
+
+        # Run script
+        self.test_try("Run cscaldb")
+        try:
+            caldb.run()
+            self.test_try_success()
+        except:
+            msg = "Exception occured in cscaldb."
+            self.test_try_failure(msg)
+
+        # Return
+        return
@@ -47,11 +47,30 @@ export CALDB=$PWD/caldb
 rm -rf $PFILES
 mkdir -p $PFILES
 cp -r ../src/*/*.par pfiles/
+cp -r ../cscripts/*.par pfiles/
 rm -rf $CALDB
 mkdir -p $CALDB
  
  
 #
+# Test cscaldb
+# ============
+$ECHO -n "Test cscaldb: "
+chmod +x @abs_top_srcdir@/cscripts/cscaldb.py
+@PYTHON_COVERAGE@@abs_top_srcdir@/cscripts/cscaldb.py \
+  chatter=4 > cscaldb.log
+$ECHO -n "."
+if [ -s "cscaldb.log" ]
+then
+  $ECHO -n "."
+else
+  $ECHO " cscaldb.log file is not valid"
+  exit 1
+fi
+$ECHO " ok"
+
+
+#
 # Test csobs2caldb
 # ================
 $ECHO -n "Test csobs2caldb: "
@@ -37,6 +37,7 @@ import test_ctcubemask
 import test_ctbutterfly
 import test_ctulimit
 import test_cterror
+import test_cscaldb
 import test_pipelines
  
  
@@ -65,6 +66,7 @@ def test(installed=False):
     suite_ctbutterfly = test_ctbutterfly.Test()
     suite_ctulimit    = test_ctulimit.Test()
     suite_cterror     = test_cterror.Test()
+    suite_cscaldb     = test_cscaldb.Test()
     suite_pipelines   = test_pipelines.Test()
  
     # Setup unit tests
@@ -82,6 +84,7 @@ def test(installed=False):
     suite_ctbutterfly.set()
     suite_ctulimit.set()
     suite_cterror.set()
+    suite_cscaldb.set()
     suite_pipelines.set()
  
     # Append tests to container
@@ -99,6 +102,7 @@ def test(installed=False):
     suites.append(suite_ctbutterfly)
     suites.append(suite_ctulimit)
     suites.append(suite_cterror)
+    suites.append(suite_cscaldb)
     suites.append(suite_pipelines)
  
     # If we have an installed version then create a temporary
@@ -113,11 +117,11 @@ def test(installed=False):
         # Get test directory
         import inspect
         testdir = inspect.getfile(ctools.tests)
-        head, tail = os.path.split(testdir)
+        dirname = os.path.dirname(testdir)
  
         # Copy over test data and irf
-        os.system("cp -r %s %s" % (head+"/data", "data"))
-        os.system("cp -r %s %s" % (head+"/irf",  "irf"))
+        os.system("cp -r %s %s" % (dirname+"/data", "data"))
+        os.system("cp -r %s %s" % (dirname+"/irf",  "irf"))
  
     # Set PFILES environment variable
     try:
@@ -129,6 +133,7 @@ def test(installed=False):
     # Copy over pfiles
     if not installed:
         os.system("cp -r ../src/*/*.par pfiles/")
+        os.system("cp -r ../cscripts/*.par pfiles/")
     else:
         os.system("cp -r %s/syspfiles/*.par pfiles/" % (os.environ['CTOOLS']))