diff --git a/.gitignore b/.gitignore index 6d2b4b3..761ffc2 100755 --- a/.gitignore +++ b/.gitignore @@ -83,6 +83,7 @@ src/template/ctool_obs test/pfiles/ test/caldb/ test/prod3 +test/csadd2caldb_* test/csroot2caldb_* test/csiactcopy_* test/comobsbin_* diff --git a/ChangeLog b/ChangeLog index bb7244f..5c1d0d2 100755 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,9 @@ -2021-06-25 +2021-09-30 * Version 2.0.0 released ======================== + Add csadd2caldb script Add restart of bisection in case of bad convergence in ctulimit Add COMPTEL analysis scripts (#3738) Add source counts constraints to cssens (#3668) diff --git a/NEWS b/NEWS index e951c5e..01586d2 100755 --- a/NEWS +++ b/NEWS @@ -1,6 +1,6 @@ New Features and Important Changes in ctools 2.0.0 -25 June 2021 +30 September 2021 Introduction @@ -167,6 +167,12 @@ support - Support classes Add stamping to ctool::save_event_list() method (#3662). +csadd2caldb - Adds CTA response function to calibration database +---------------------------------------------------------------- +New script that was added to support the installation of prod5-v0.1 CTA response +functions into the calibration database. + + csbkgmodel - Generate background model for 3D analysis ------------------------------------------------------ None diff --git a/README.md b/README.md index 75826b3..911128a 100755 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ ctools information ================== -* Version: 2.0.0.dev (25 June 2021) +* Version: 2.0.0.dev (30 September 2021) * GammaLib dependency: 2.0.0.dev [![Build Status](https://cta-jenkins.irap.omp.eu/buildStatus/icon?job=ctools-integrate-os)](https://cta-jenkins.irap.omp.eu/job/ctools-integrate-os/) diff --git a/cscripts/Makefile.am b/cscripts/Makefile.am index df4a0ec..f030612 100755 --- a/cscripts/Makefile.am +++ b/cscripts/Makefile.am @@ -21,6 +21,7 @@ # Scripts to be distributed in $(pyexecdir)/cscripts pythonscripts = $(top_builddir)/cscripts/__init__.py \ + $(srcdir)/csadd2caldb.py \ $(srcdir)/csbkgmodel.py \ $(srcdir)/cscaldb.py \ $(srcdir)/csebins.py \ @@ -60,7 +61,8 @@ pythonscripts = $(top_builddir)/cscripts/__init__.py \ $(srcdir)/mputils.py # Parfiles to be distributed in $(prefix)/syspfiles -parfiles = $(srcdir)/csbkgmodel.par \ +parfiles = $(srcdir)/csadd2caldb.par \ + $(srcdir)/csbkgmodel.par \ $(srcdir)/cscaldb.par \ $(srcdir)/csebins.par \ $(srcdir)/cslightcrv.par \ @@ -93,7 +95,8 @@ parfiles = $(srcdir)/csbkgmodel.par \ $(srcdir)/csiactobs.par # Scripts for which symbolic links will be created in bindir -linkscripts = csbkgmodel \ +linkscripts = csadd2caldb \ + csbkgmodel \ cscaldb \ csebins \ csinfo \ @@ -132,6 +135,7 @@ testscripts = $(srcdir)/tests/__init__.py \ $(top_srcdir)/test/test_python_cscripts.py \ $(top_srcdir)/test/test_cscript.py \ $(top_srcdir)/test/test_cscaldb.py \ + $(top_srcdir)/test/test_csadd2caldb.py \ $(top_srcdir)/test/test_csbkgmodel.py \ $(top_srcdir)/test/test_csebins.py \ $(top_srcdir)/test/test_csfindobs.py \ @@ -266,6 +270,9 @@ testiactevents = $(top_srcdir)/test/data/iactdata/events/irf_file.fits.gz \ $(top_srcdir)/test/data/iactdata/events/events_7.fits.gz \ $(top_srcdir)/test/data/iactdata/events/events_8.fits.gz +# Test data to be distributed in $(pyexecdir)/cscripts/tests/data/prod5-v0.1 +testprod501data = $(top_srcdir)/test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz + # Install scripts, parfiles, test scripts and test data. Note that we don't # need to prepend $(DESTDIR) here (not fully clear why, but that's the way # it is) @@ -283,11 +290,13 @@ testiactdatadir = $(pyexecdir)/cscripts/tests/data/iactdata testiactdata_DATA = $(testiactdata) testiacteventsdir = $(pyexecdir)/cscripts/tests/data/iactdata/events testiactevents_DATA = $(testiactevents) +testprod501dir = $(pyexecdir)/cscripts/tests/data/prod5-v0.1 +testprod501_DATA = $(testprod501data) # Files to be distributed in the tar file EXTRA_DIST = $(binscripts) $(pythonscripts) $(parfiles) \ $(testscripts) $(testdata) $(testiactdata) \ - $(testiactevents) + $(testiactevents) $(testprod501data) # Add install hook rule to create symbolic links to cscripts without .py in the bindir install-data-hook: diff --git a/cscripts/__init__.py.in b/cscripts/__init__.py.in index b428d80..6d28d1d 100755 --- a/cscripts/__init__.py.in +++ b/cscripts/__init__.py.in @@ -1,7 +1,7 @@ # ========================================================================== # cscripts Python module # -# Copyright (C) 2015-2020 Juergen Knoedlseder +# Copyright (C) 2015-2021 Juergen Knoedlseder # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,6 +24,7 @@ cur_version = sys.version_info # List all scripts in this module __all__ = [ + "csadd2caldb", "csbkgmodel", "cscaldb", "csebins", @@ -65,6 +66,7 @@ if cur_version > req_version: "csiactobs"]) # Import +from cscripts.csadd2caldb import csadd2caldb from cscripts.csbkgmodel import csbkgmodel from cscripts.cscaldb import cscaldb from cscripts.csebins import csebins diff --git a/cscripts/csadd2caldb.par b/cscripts/csadd2caldb.par new file mode 100755 index 0000000..5f13cdc --- /dev/null +++ b/cscripts/csadd2caldb.par @@ -0,0 +1,29 @@ +############################################################################# +# # +# CTA Science Analysis Tools # +# # +# ------------------------------------------------------------------------- # +# # +# File: csadd2caldb.par # +# # +# Author: Juergen Knoedlseder # +# Jurgen.Knodlseder@irap.omp.eu # +# IRAP # +# # +# Purpose: Parameter file for the csadd2caldb script # +# # +############################################################################# +# +# Input and output parameters +#============================ +indir, f, a, fits,,, "Input IRF folder" +outdir, f, a, $CALDB,,, "Output caldb folder" + +# +# Standard parameters +#==================== +chatter, i, h, 2,0,4, "Chattiness of output" +clobber, b, h, yes,,, "Overwrite existing output files with new output files?" +debug, b, h, no,,, "Activate debugging mode?" +mode, s, h, ql,,, "Mode of automatic parameters" +logfile, f, h, csadd2caldb.log,,, "Log filename" diff --git a/doc/source/users/user_manual/csadd2caldb.py b/cscripts/csadd2caldb.py index d194878..e70b101 100755 --- a/doc/source/users/user_manual/csadd2caldb.py +++ b/cscripts/csadd2caldb.py @@ -39,9 +39,6 @@ class csadd2caldb(ctools.cscript): """ Constructor """ - # Make sure that parfile exists - self._parfile() - # Initialise application by calling the base class constructor self._init_cscript(self.__class__.__name__, ctools.__version__, argv) @@ -50,31 +47,6 @@ class csadd2caldb(ctools.cscript): # Private methods - def _parfile(self): - """ - Check if parfile exists. If parfile does not exist then create a - default parfile. This kluge avoids shipping the cscript with a parfile. - """ - # Set parfile name - parfile = self.__class__.__name__+'.par' - - # Load parfile or create - try: - pars = gammalib.GApplicationPars(parfile) - except: - # Signal that parfile was not found - print('Parfile '+parfile+' not found. Create default parfile.') - - # Create default parfile - pars = gammalib.GApplicationPars() - pars.append(gammalib.GApplicationPar('indir','f','a','fits','','','Input IRF folder')) - pars.append(gammalib.GApplicationPar('outdir','f','a','$CALDB','','','Output caldb folder')) - pars.append_standard() - pars.save(parfile) - - # Return - return - def _get_parameters(self): """ Get parameters from parfile @@ -358,6 +330,12 @@ class csadd2caldb(ctools.cscript): # Collect tarfiles tarfiles = self._collect_tarfiles(self['indir'].filename()) + # Raise an exception if no tarfiles were found + if len(tarfiles) == 0: + msg = ('No tarfiles found in folder "'+self['indir'].filename().url()+'. ' + 'Please specify a folder that contains IRF tarfiles.') + raise RuntimeError(msg) + # Add IRFs in tarfiles for file in tarfiles: diff --git a/cscripts/mputils.py b/cscripts/mputils.py old mode 100644 new mode 100755 index 722c011..722c011 --- a/cscripts/mputils.py +++ b/cscripts/mputils.py diff --git a/doc/source/admin/release_history/2.0.rst b/doc/source/admin/release_history/2.0.rst index 7050be9..6635a95 100644 --- a/doc/source/admin/release_history/2.0.rst +++ b/doc/source/admin/release_history/2.0.rst @@ -1,7 +1,7 @@ .. _2.0: -ctools 2.0 release (25 June 2021) -================================= +ctools 2.0 release (30 September 2021) +====================================== Overview -------- @@ -58,6 +58,7 @@ New features * [`3738 `_] - Add COMPTEL analysis scripts +* Add :ref:`csadd2caldb` script Clean up diff --git a/doc/source/users/reference_manual/csadd2caldb.rst b/doc/source/users/reference_manual/csadd2caldb.rst new file mode 100755 index 0000000..e991272 --- /dev/null +++ b/doc/source/users/reference_manual/csadd2caldb.rst @@ -0,0 +1,55 @@ +.. _csadd2caldb: + +csadd2caldb +=========== + +Adds CTA response function to calibration database. + + +Synopsis +-------- + +This script adds a CTA response function to the calibration database. The +CTA response functions can be downloaded from CTAO Zenodo directory. + + +General parameters +------------------ + +``indir [file]`` + Input IRF folder. + +``outdir [file]`` + Output caldb folder. + + +Standard parameters +------------------- + +``(chatter = 2) [integer]`` + Verbosity of the executable: + ``chatter = 0``: no information will be logged + + ``chatter = 1``: only errors will be logged + + ``chatter = 2``: errors and actions will be logged + + ``chatter = 3``: report about the task execution + + ``chatter = 4``: detailed report about the task execution + +``(clobber = yes) [boolean]`` + Specifies whether an existing energy boundaries output file should be overwritten. + +``(debug = no) [boolean]`` + Enables debug mode. In debug mode the executable will dump any log file output to the console. + +``(mode = ql) [string]`` + Mode of automatic parameters (default is ``ql``, i.e. "query and learn"). + +``(logfile = csadd2caldb.log) [filename]`` + Log filename. + + +Related tools or scripts +------------------------ diff --git a/doc/source/users/reference_manual/index.rst b/doc/source/users/reference_manual/index.rst index 8f7d4f2..af2921a 100755 --- a/doc/source/users/reference_manual/index.rst +++ b/doc/source/users/reference_manual/index.rst @@ -64,6 +64,7 @@ CTA and IACT analysis tools and scripts cspull --- Generates pull distribution csresmap --- Generates residual map csresspec --- Generates residual spectrum + csadd2caldb --- Adds CTA response function to calibration database csroot2caldb --- Creates a caldb entry from a ROOT file csscs --- Performs spectral component separation cssens --- Computes CTA sensitivity diff --git a/doc/source/users/user_manual/irf_cta.rst b/doc/source/users/user_manual/irf_cta.rst index d21b151..9a99651 100755 --- a/doc/source/users/user_manual/irf_cta.rst +++ b/doc/source/users/user_manual/irf_cta.rst @@ -83,10 +83,6 @@ In total, the following six instrument response functions are available: The latest CTA reponse functions (``prod5-v0.1``) can be downloaded from `this link `_. -Addition of CTA response functions to the calibration database can be done -using the script :download:`csadd2caldb.py ` that is available -for download. - To download and install the ``prod5-v0.1`` response functions you should execute the following commands in a terminal: @@ -96,8 +92,7 @@ the following commands in a terminal: $ cd temp $ wget https://zenodo.org/record/5499840/files/cta-prod5-zenodo-fitsonly-v0.1.zip $ unzip cta-prod5-zenodo-fitsonly-v0.1.zip - $ ./csadd2caldb.py - Parfile csadd2caldb.par not found. Create default parfile. + $ csadd2caldb Input IRF folder [fits] Output caldb folder [$CALDB] diff --git a/test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz b/test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz new file mode 100644 index 0000000..052f7f3 --- /dev/null +++ b/test/data/prod5-v0.1/CTA-Performance-prod5-v0.1-South-20deg.FITS.tar.gz diff --git a/test/test_csadd2caldb.py b/test/test_csadd2caldb.py new file mode 100755 index 0000000..bb47881 --- /dev/null +++ b/test/test_csadd2caldb.py @@ -0,0 +1,171 @@ +#! /usr/bin/env python +# ========================================================================== +# This scripts performs unit tests for the csadd2caldb script. +# +# Copyright (C) 2021 Juergen Knoedlseder +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +# ========================================================================== +import os +import gammalib +import cscripts +from testing import test + + +# ================================= # +# Test class for csadd2caldb script # +# ================================= # +class Test(test): + """ + Test class for csadd2caldb script + + This test class makes unit tests for the csadd2caldb script by using it + from the command line and from Python. + """ + + # Constructor + def __init__(self): + """ + Constructor + """ + # Call base class constructor + test.__init__(self) + + # Set test datasets and parameters + self._prod501 = self._datadir + '/prod5-v0.1' + + # Clear results + os.system('rm -rf csadd2caldb_cmd1') + os.system('rm -rf csadd2caldb_py1') + os.system('rm -rf csadd2caldb_py1_pickle') + + # Return + return + + # Set test functions + def set(self): + """ + Set all test functions. + """ + # Set test name + self.name('csadd2caldb') + + # Append tests + self.append(self._test_cmd, 'Test csadd2caldb on command line') + self.append(self._test_python, 'Test csadd2caldb from Python') + self.append(self._test_pickeling, 'Test csadd2caldb pickeling') + + # Return + return + + # Test csadd2caldb on command line + def _test_cmd(self): + """ + Test csadd2caldb on the command line. + """ + # Set script name + csadd2caldb = self._script('csadd2caldb') + + # Setup csadd2caldb command + cmd = csadd2caldb+' indir="'+self._prod501+'"'+ \ + ' outdir="csadd2caldb_cmd1"'+ \ + ' logfile="csadd2caldb_cmd1.log" chatter=4' + + # Check if execution was successful + self.test_assert(self._execute(cmd) == 0, + 'Check successful execution from command line') + + # Check IRF + self._check_irf('csadd2caldb_cmd1') + + # Setup csadd2caldb command + cmd = csadd2caldb+' indir="folder_that_does_not_exist"'+ \ + ' outdir="csadd2caldb_cmd2"'+ \ + ' logfile="csadd2caldb_cmd2.log" chatter=1' + + # Check if execution failed + self.test_assert(self._execute(cmd, success=False) != 0, + 'Check invalid input file when executed from command line') + + # Check csadd2caldb --help + self._check_help(csadd2caldb) + + # Return + return + + # Test csadd2caldb from Python + def _test_python(self): + """ + Test csadd2caldb from Python + """ + # Set-up csadd2caldb + addirf = cscripts.csadd2caldb() + addirf['indir'] = self._prod501 + addirf['outdir'] = 'csadd2caldb_py1' + addirf['logfile'] = 'csadd2caldb_py1.log' + + # Run csadd2caldb script and save background model + addirf.logFileOpen() # Make sure we get a log file + addirf.run() + addirf.save() + + # Check IRF + self._check_irf('csadd2caldb_py1') + + # Return + return + + # Test csadd2caldb pickeling + def _test_pickeling(self): + """ + Test csadd2caldb pickeling + """ + # Perform pickeling tests of empty class + self._pickeling(cscripts.csadd2caldb()) + + # Set-up csadd2caldb + addirf = cscripts.csadd2caldb() + addirf['indir'] = self._prod501 + addirf['outdir'] = 'csadd2caldb_py1_pickle' + addirf['logfile'] = 'csadd2caldb_py1_pickle.log' + + # Perform pickeling tests of filled class + obj = self._pickeling(addirf) + + # Run csadd2caldb script and save background model + obj.logFileOpen() # Make sure we get a log file + obj.run() + obj.save() + + # Check IRF + self._check_irf('csadd2caldb_py1_pickle') + + # Return + return + + # Check IRF + def _check_irf(self, filename, prod='prod5-v0.1', nirfs=8): + """ + Check IRF + """ + # Open CALDB + caldb = gammalib.GCaldb(filename) + caldb.open('CTA', prod) + + # Check CALDB + self.test_value(caldb.size(), nirfs, 'Check for %d entries in CALDB' % nirfs) + + # Return + return diff --git a/test/test_python_cscripts.py b/test/test_python_cscripts.py index 4e82909..1cb4aac 100755 --- a/test/test_python_cscripts.py +++ b/test/test_python_cscripts.py @@ -2,7 +2,7 @@ # ========================================================================== # This scripts performs unit tests for cscripts # -# Copyright (C) 2016-2020 Juergen Knoedlseder +# Copyright (C) 2016-2021 Juergen Knoedlseder # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,6 +25,7 @@ import sys sys.path.append(os.path.dirname(os.path.abspath(__file__))) import test_cscript import test_cscaldb +import test_csadd2caldb import test_csbkgmodel import test_csebins import test_csfindobs @@ -117,6 +118,7 @@ def test(installed=False, debug=False): # Define list of test suites tests = [test_cscript.Test(), + test_csadd2caldb.Test(), test_csbkgmodel.Test(), test_cscaldb.Test(), test_csebins.Test(),