From efb7eaae22c3d28829648e143c7e029b8956c0f3 Mon Sep 17 00:00:00 2001 From: Jürgen Knödlseder Date: Fri, 11 Mar 2016 22:05:02 +0100 Subject: [PATCH] Refactor cscaldb script and add unit test. --- cscripts/Makefile.am | 7 ++++++- cscripts/cscaldb.par | 24 ++++++++++++++++++++++++ cscripts/cscaldb.py | 181 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----------------------------------------------------------------------- pyext/Makefile.am | 3 ++- test/test_cscaldb.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ test/test_cscripts.sh.in | 19 +++++++++++++++++++ test/test_python.py | 11 ++++++++--- 7 files changed, 244 insertions(+), 76 deletions(-) create mode 100644 cscripts/cscaldb.par create mode 100755 test/test_cscaldb.py diff --git a/cscripts/Makefile.am b/cscripts/Makefile.am index 85aefd0..a025cd4 100644 --- a/cscripts/Makefile.am +++ b/cscripts/Makefile.am @@ -9,6 +9,9 @@ pythonscripts = __init__.py \ csfindobs.py csiactcopy.py csiactdata.py csiactobs.py \ obsutils.py +# Parfiles to be distributed in $(prefix)/syspfiles +parfiles = cscaldb.par + # Scripts for which symbolic links will be created in bindir linkscripts = cspull cssens cstsdist cscaldb csresmap \ csobsdef csspec cslightcrv csobsinfo \ @@ -16,12 +19,14 @@ linkscripts = cspull cssens cstsdist cscaldb csresmap \ csobs2caldb \ csfindobs csiactcopy csiactdata csiactobs -# Distribute scripts +# Distribute scripts and parfiles EXTRA_DIST = $(binscripts) $(pythonscripts) binscriptsdir = $(bindir) binscripts_DATA = $(binscripts) pythonscriptsdir = $(pyexecdir)/cscripts pythonscripts_DATA = $(pythonscripts) +pfilesdir = $(prefix)/syspfiles +pfiles_DATA = $(parfiles) # Add hook rule to create symbolic links for cscripts install-data-hook: diff --git a/cscripts/cscaldb.par b/cscripts/cscaldb.par new file mode 100644 index 0000000..efc4a71 --- /dev/null +++ b/cscripts/cscaldb.par @@ -0,0 +1,24 @@ +############################################################################# +# # +# CTA Science Analysis Tools # +# # +# ------------------------------------------------------------------------- # +# # +# File: cscaldb.par # +# Version: 1.1.0 # +# # +# Author: Jurgen Knodlseder # +# Jurgen.Knodlseder@irap.omp.eu # +# IRAP # +# # +# Purpose: Parameter file for the cscaldb script # +# # +############################################################################# +# +# Standard parameters +#==================== +chatter, i, h, 2,0,4, "Chattiness of output" +clobber, b, h, yes,,, "Overwrite existing output files with new output files?" +debug, b, h, no,,, "Debugging mode activated" +mode, s, h, "ql",,, "Mode of automatic parameters" +logfile, f, h, "cscaldb.log",,, "Log filename" diff --git a/cscripts/cscaldb.py b/cscripts/cscaldb.py index dbdfeb6..589c6dd 100755 --- a/cscripts/cscaldb.py +++ b/cscripts/cscaldb.py @@ -2,7 +2,7 @@ # ========================================================================== # This script dumps all available calibrations into the console. # -# Copyright (C) 2014-2015 Juergen Knoedlseder +# Copyright (C) 2014-2016 Juergen Knoedlseder # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,32 +25,28 @@ import glob import os -# ============ # -# cspull class # -# ============ # +# ============= # +# cscaldb class # +# ============= # class cscaldb(ctools.cscript): """ - This class dumps the content of the ctools calibration database. - It derives from the ctools.cscript class which provides support for - parameter files, command line arguments, and logging. In that way - the Python script behaves just as a regular ctool. + The cscaldb class logs the content of the ctools calibration database + into the log file. Optionally, if the "debug" parameter is set to "yes" + the calibration database is also logged into the console. """ def __init__(self, *argv): """ Constructor. """ # Set name - self.name = "cscaldb" - self.version = "1.0.0" - - # Make sure that parfile exists - file = self.parfile() + self._name = "cscaldb" + self._version = "1.0.0" # Initialise application if len(argv) == 0: - ctools.cscript.__init__(self, self.name, self.version) + ctools.cscript.__init__(self, self._name, self._version) elif len(argv) ==1: - ctools.cscript.__init__(self, self.name, self.version, *argv) + ctools.cscript.__init__(self, self._name, self._version, *argv) else: raise TypeError("Invalid number of arguments given.") @@ -68,34 +64,97 @@ class cscaldb(ctools.cscript): # Return return - def parfile(self): + def _get_parameters(self): """ - Check if parfile exists. If parfile does not exist then create a - default parfile. This kluge avoids shipping the cscript with a parfile. + Get parameters from parfile. """ - # Set parfile name - parfile = self.name+".par" + # Return + return - try: - pars = gammalib.GApplicationPars(parfile) - except: - # Signal if parfile was not found - sys.stdout.write("Parfile "+parfile+" not found. Create default parfile.\n") + def _get_missions(self, caldb): + """ + Extract mission names from a calibration database. - # Create default parfile - pars = gammalib.GApplicationPars() - pars.append_standard() - pars.save(parfile) + Args: + caldb: Calibration database. - # Return - return + Returns: + A list of mission names. + """ + # Initialise mission list + missions = [] + + # Extract missions + paths = glob.glob(caldb.rootdir()+"/data/*") + paths.sort() + for path in paths: + missions.append(os.path.basename(path)) - def get_parameters(self): + # Sort missions + missions.sort() + + # Return missions + return missions + + def _get_instruments(self, caldb, mission): """ - Get parameters from parfile. + Extract instrument names from a calibration database. + + Args: + caldb: Calibration database. + mission: Mission name. + + Returns: + A list of instrument names. """ - # Return - return + # Initialise instrument list + instruments = [] + + # Extract instruments + paths = glob.glob(caldb.rootdir()+"/data/"+mission+"/*") + paths.sort() + for path in paths: + instruments.append(os.path.basename(path)) + + # Sort instruments + instruments.sort() + + # Return instruments + return instruments + + def _get_response_names(self, calibrations): + """ + Extract response names from a calibrations FITS table and return + them in form of a list. + + Args: + calibrations: FITS table containing the calibrations. + + Returns: + A list of response names. + """ + # Initialise response name list + names = [] + + # Extract response names from calibrations and append them to the + # response name list + nrows = calibrations.length() + ncols = calibrations.number() + for row in range(nrows): + for col in range(ncols): + cal = calibrations.string(row, col) + istart = cal.find("NAME(") + if istart != -1: + istop = cal.find(")") + name = cal[5:istop] + if names.count(name) == 0: + names.append(name) + + # Sort response names + names.sort() + + # Return response names + return names def execute(self): """ @@ -116,23 +175,18 @@ class cscaldb(ctools.cscript): self.log.cout(True) # Get parameters - self.get_parameters() + self._get_parameters() # Write input parameters into logger if self.logTerse(): self.log_parameters() self.log("\n") - # Get the calibration database path + # Get the calibration database caldb = gammalib.GCaldb() - # Gather missions - missions = [] - paths = glob.glob(caldb.rootdir()+"/data/*") - paths.sort() - for path in paths: - head, tail = os.path.split(path) - missions.append(tail) + # Extract mission names from the calibration database + missions = self._get_missions(caldb) # Loop over missions for mission in missions: @@ -146,41 +200,26 @@ class cscaldb(ctools.cscript): self.log("\n") self.log.header1("Mission: "+mission) - # Gather instruments - instruments = [] - paths = glob.glob(caldb.rootdir()+"/data/"+mission+"/*") - paths.sort() - for path in paths: - head, tail = os.path.split(path) - instruments.append(tail) + # Extract instruments + instruments = self._get_instruments(caldb, mission) # Loop over instruments for instrument in instruments: # Write mission into logger if self.logTerse(): - self.log.header3("Response functions in database \""+instrument+"\"") + self.log.header3("Response functions in database \""+ + instrument+"\"") - # Open calibration index file - cifname = caldb.rootdir()+"/data/"+mission+"/"+instrument+"/caldb.indx" - fits = gammalib.GFits(cifname) - cif = fits["CIF"] - cals = cif["CAL_CBD"] + # Open calibration index file and retrieve calibrations + filename = "/data/"+mission+"/"+instrument+"/caldb.indx" + cifname = caldb.rootdir() + filename + fits = gammalib.GFits(cifname) + cif = fits["CIF"] + cals = cif["CAL_CBD"] # Extract response names - names = [] - nrows = cals.length() - ncols = cals.number() - for row in range(nrows): - for col in range(ncols): - cal = cals.string(row, col) - istart = cal.find("NAME(") - if istart != -1: - istop = cal.find(")") - name = cal[5:istop] - if names.count(name) == 0: - names.append(name) - names.sort() + names = self._get_response_names(cals) # Print response name if self.logTerse(): @@ -197,7 +236,7 @@ class cscaldb(ctools.cscript): # ======================== # if __name__ == '__main__': """ - Dumps calibration database into log file. + Logs calibration database into the log file. """ # Create instance of application app = cscaldb(sys.argv) diff --git a/pyext/Makefile.am b/pyext/Makefile.am index dcde606..7be169a 100755 --- a/pyext/Makefile.am +++ b/pyext/Makefile.am @@ -1,7 +1,7 @@ ############################################################################# # Makefile for ctools Python extension module # # ------------------------------------------------------------------------- # -# Copyright (C) 2011-2015 Juergen Knoedlseder # +# Copyright (C) 2011-2016 Juergen Knoedlseder # # ------------------------------------------------------------------------- # # # # This program is free software: you can redistribute it and/or modify # @@ -46,6 +46,7 @@ testscripts = ctools/tests/__init__.py \ ../test/test_ctbutterfly.py \ ../test/test_ctulimit.py \ ../test/test_cterror.py \ + ../test/test_cscaldb.py \ ../test/test_pipelines.py testdata = ../test/data/crab.xml \ ../test/data/crab_events.fits.gz \ diff --git a/test/test_cscaldb.py b/test/test_cscaldb.py new file mode 100755 index 0000000..c9de3d9 --- /dev/null +++ b/test/test_cscaldb.py @@ -0,0 +1,75 @@ +#! /usr/bin/env python +# ========================================================================== +# This scripts performs unit tests for the cscaldb script. +# +# Copyright (C) 2016 Juergen Knoedlseder +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ========================================================================== +import gammalib +import cscripts + + +# ============================= # +# Test class for cscaldb script # +# ============================= # +class Test(gammalib.GPythonTestSuite): + """ + Test class for cscaldb script. + """ + # Constructor + def __init__(self): + """ + Constructor. + """ + # Call base class constructor + gammalib.GPythonTestSuite.__init__(self) + + # Return + return + + # Set test functions + def set(self): + """ + Set all test functions. + """ + # Set test name + self.name("cscaldb") + + # Append tests + self.append(self._test_functional, "Test cscaldb functionality") + + # Return + return + + # Test cscaldb functionality + def _test_functional(self): + """ + Test cscaldb functionality. + """ + # Set-up cscaldb + caldb = cscripts.cscaldb() + + # Run script + self.test_try("Run cscaldb") + try: + caldb.run() + self.test_try_success() + except: + msg = "Exception occured in cscaldb." + self.test_try_failure(msg) + + # Return + return diff --git a/test/test_cscripts.sh.in b/test/test_cscripts.sh.in index 9ca0a90..cbd8fe1 100755 --- a/test/test_cscripts.sh.in +++ b/test/test_cscripts.sh.in @@ -47,11 +47,30 @@ export CALDB=$PWD/caldb rm -rf $PFILES mkdir -p $PFILES cp -r ../src/*/*.par pfiles/ +cp -r ../cscripts/*.par pfiles/ rm -rf $CALDB mkdir -p $CALDB # +# Test cscaldb +# ============ +$ECHO -n "Test cscaldb: " +chmod +x @abs_top_srcdir@/cscripts/cscaldb.py +@PYTHON_COVERAGE@@abs_top_srcdir@/cscripts/cscaldb.py \ + chatter=4 > cscaldb.log +$ECHO -n "." +if [ -s "cscaldb.log" ] +then + $ECHO -n "." +else + $ECHO " cscaldb.log file is not valid" + exit 1 +fi +$ECHO " ok" + + +# # Test csobs2caldb # ================ $ECHO -n "Test csobs2caldb: " diff --git a/test/test_python.py b/test/test_python.py index 828deac..ccb5b99 100755 --- a/test/test_python.py +++ b/test/test_python.py @@ -37,6 +37,7 @@ import test_ctcubemask import test_ctbutterfly import test_ctulimit import test_cterror +import test_cscaldb import test_pipelines @@ -65,6 +66,7 @@ def test(installed=False): suite_ctbutterfly = test_ctbutterfly.Test() suite_ctulimit = test_ctulimit.Test() suite_cterror = test_cterror.Test() + suite_cscaldb = test_cscaldb.Test() suite_pipelines = test_pipelines.Test() # Setup unit tests @@ -82,6 +84,7 @@ def test(installed=False): suite_ctbutterfly.set() suite_ctulimit.set() suite_cterror.set() + suite_cscaldb.set() suite_pipelines.set() # Append tests to container @@ -99,6 +102,7 @@ def test(installed=False): suites.append(suite_ctbutterfly) suites.append(suite_ctulimit) suites.append(suite_cterror) + suites.append(suite_cscaldb) suites.append(suite_pipelines) # If we have an installed version then create a temporary @@ -113,11 +117,11 @@ def test(installed=False): # Get test directory import inspect testdir = inspect.getfile(ctools.tests) - head, tail = os.path.split(testdir) + dirname = os.path.dirname(testdir) # Copy over test data and irf - os.system("cp -r %s %s" % (head+"/data", "data")) - os.system("cp -r %s %s" % (head+"/irf", "irf")) + os.system("cp -r %s %s" % (dirname+"/data", "data")) + os.system("cp -r %s %s" % (dirname+"/irf", "irf")) # Set PFILES environment variable try: @@ -129,6 +133,7 @@ def test(installed=False): # Copy over pfiles if not installed: os.system("cp -r ../src/*/*.par pfiles/") + os.system("cp -r ../cscripts/*.par pfiles/") else: os.system("cp -r %s/syspfiles/*.par pfiles/" % (os.environ['CTOOLS'])) -- libgit2 0.21.4