Konstancja Satalecka / ctools

Commit 3af8c3124e5d15df3a2dcd1b9a1942d0e3b72d21

Authored by Jürgen Knödlseder
2 parents 0a31b7c0 2ced8069

Merge branch 'integration' into devel

Inline Side-by-side
Showing 11 changed files with 759 additions and 152 deletions
cscripts/Makefile.am
  View file @3af8c31
... ... @@ -5,14 +5,14 @@ pythonscripts = __init__.py \
5 5 cspull.py cssens.py cstsdist.py cscaldb.py csresmap.py \
6 6 csiactobs.py csiactdata.py csobsdef.py csspec.py cslightcrv.py csobsinfo.py \
7 7 csinfo.py csmodelinfo.py csmodelmerge.py cstsmapmerge.py \
8   - csobs2caldb.py \
  8 + csobs2caldb.py csfindobs.py \
9 9 obsutils.py
10 10  
11 11 # Scripts for which symbolic links will be created in bindir
12 12 linkscripts = cspull cssens cstsdist cscaldb csresmap \
13 13 csiactobs csiactdata csobsdef csspec cslightcrv csobsinfo \
14 14 csinfo csmodelinfo csmodelmerge cstsmapmerge \
15   - csobs2caldb
  15 + csobs2caldb csfindobs
16 16  
17 17 # Distribute scripts
18 18 EXTRA_DIST = $(binscripts) $(pythonscripts)
... ...
cscripts/csfindobs.py 0 → 100755
  View file @3af8c31
  1 +#!/usr/bin/env python
  2 +# ==========================================================================
  3 +# Find observations from an IACT data store
  4 +#
  5 +# Copyright (C) 2016 Michael Mayer
  6 +#
  7 +# This program is free software: you can redistribute it and/or modify
  8 +# it under the terms of the GNU General Public License as published by
  9 +# the Free Software Foundation, either version 3 of the License, or
  10 +# (at your option) any later version.
  11 +#
  12 +# This program is distributed in the hope that it will be useful,
  13 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15 +# GNU General Public License for more details.
  16 +#
  17 +# You should have received a copy of the GNU General Public License
  18 +# along with this program. If not, see <http://www.gnu.org/licenses/>.
  19 +#
  20 +# ==========================================================================
  21 +import gammalib
  22 +import ctools
  23 +import sys
  24 +import os
  25 +import json
  26 +
  27 +
  28 +# =============== #
  29 +# csfindobs class #
  30 +# =============== #
  31 +class csfindobs(ctools.cscript):
  32 + """
  33 + This class can be used to find observations from an IACT data store.
  34 + """
  35 + def __init__(self, *argv):
  36 + """
  37 + Constructor.
  38 + """
  39 + # Set name
  40 + self.name = "csfindobs"
  41 + self.version = "1.0.0"
  42 + self.verbose = False
  43 + self.datapath = os.getenv("VHEFITS","")
  44 +
  45 + # Make sure that parfile exists
  46 + file = self.parfile()
  47 +
  48 + # Initialise application
  49 + if len(argv) == 0:
  50 + ctools.cscript.__init__(self, self.name, self.version)
  51 + elif len(argv) ==1:
  52 + ctools.cscript.__init__(self, self.name, self.version, *argv)
  53 + else:
  54 + raise TypeError("Invalid number of arguments given.")
  55 +
  56 + # Set logger properties
  57 + self.log_header()
  58 + self.log.date(True)
  59 +
  60 + # Return
  61 + return
  62 +
  63 + def __del__(self):
  64 + """
  65 + Destructor.
  66 + """
  67 + # Write separator into logger
  68 + if self.logTerse():
  69 + self.log("\n")
  70 +
  71 + # Return
  72 + return
  73 +
  74 + def parfile(self):
  75 + """
  76 + Check if parfile exists. If parfile does not exist then create a
  77 + default parfile. This kluge avoids shipping the cscript with a parfile.
  78 + """
  79 + # Set parfile name
  80 + parfile = self.name+".par"
  81 +
  82 + try:
  83 + pars = gammalib.GApplicationPars(parfile)
  84 + except:
  85 + # Signal if parfile was not found
  86 + sys.stdout.write("Parfile "+parfile+" not found. Create default parfile.\n")
  87 +
  88 + # Create default parfile
  89 + pars = gammalib.GApplicationPars()
  90 + pars.append(gammalib.GApplicationPar("datapath","s","a",self.datapath,"","","Path were data are located"))
  91 + pars.append(gammalib.GApplicationPar("prodname","s","a","prod-name","","","Name of FITS production (Run csiactdata to view your options)"))
  92 + pars.append(gammalib.GApplicationPar("outfile","f","a","runlist.lis","","","Runlist outfile"))
  93 + pars.append(gammalib.GApplicationPar("ra","r","a","83.6331","","","Right ascension"))
  94 + pars.append(gammalib.GApplicationPar("dec","r","a","22.01","","","Declination"))
  95 + pars.append(gammalib.GApplicationPar("rad","r","a","2.5","","","Search radius"))
  96 + pars.append(gammalib.GApplicationPar("min_qual","i","h","0","0|1|2","","Minimum data quality (0=perfect, 1=ok, 2=bad)"))
  97 + pars.append(gammalib.GApplicationPar("expression","s","h","NONE","","","Additional expression"))
  98 + pars.append(gammalib.GApplicationPar("master_indx","s","h","master.json","","","Name of master index file"))
  99 + pars.append_standard()
  100 + pars.append(gammalib.GApplicationPar("logfile","f","h","csfindobs.log","","","Log filename"))
  101 + pars.save(parfile)
  102 +
  103 + # Return
  104 + return
  105 +
  106 + def get_parameters(self):
  107 + """
  108 + Get parameters from parfile and setup the observation.
  109 + """
  110 + # Get parameters
  111 + if self.datapath == "":
  112 + self.datapath = self["datapath"].string()
  113 +
  114 + # Expand environment
  115 + self.datapath = gammalib.expand_env(self.datapath)
  116 +
  117 + # Get production name
  118 + self.m_prodname = self["prodname"].string()
  119 +
  120 + # Master index file name
  121 + self.m_master_indx = self["master_indx"].string()
  122 +
  123 + # Intiialise flag if spatial selection is required
  124 + self.select_radec = True
  125 +
  126 + # Check for vailidity of spatial parameters
  127 + if self["ra"].is_valid() and self["dec"].is_valid() and self["rad"].is_valid():
  128 +
  129 + # Read spatial parameters
  130 + self.m_ra = self["ra"].real()
  131 + self.m_dec = self["dec"].real()
  132 + self.m_radius = self["rad"].real()
  133 +
  134 + else:
  135 + self.select_radec = False
  136 +
  137 + # Check Radius for validity
  138 + if self.m_radius <= 0.0:
  139 + self.select_radec = False
  140 +
  141 + # Read other parameters
  142 + self.m_quality = self["min_qual"].integer()
  143 + self.m_expression = self["expression"].string()
  144 +
  145 + # Check for validity of expression
  146 + if self.m_expression == "NONE" or self.m_expression == "INDEF":
  147 + self.m_expression = ""
  148 +
  149 + # Read outfile
  150 + self.m_outfile = self["outfile"].filename()
  151 +
  152 + # Open master index file and look for prodname
  153 + master_file = os.path.join(self.datapath, self.m_master_indx)
  154 + if not os.path.isfile(master_file):
  155 + raise RuntimeError("FITS data store not available. No master index file found at \""+master_file+"\". Make sure the file is copied from the server and your datapath is set correctly.")
  156 +
  157 + # Open and load JSON file
  158 + json_data = open(master_file).read()
  159 + data = json.loads(json_data)
  160 + if not "datasets" in data:
  161 + raise RuntimeError("Key \"datasets\" not available in master index file.")
  162 +
  163 + # Get configurations
  164 + configs = data["datasets"]
  165 +
  166 + # Initialise obs index file
  167 + self.m_obs_index = ""
  168 +
  169 + # Get HDUs
  170 + for config in configs:
  171 + if self.m_prodname == config["name"]:
  172 + self.m_obs_index = str(os.path.join(self.datapath, config["obsindx"]))
  173 + break
  174 +
  175 + # Check HDUs
  176 + if self.m_obs_index == "":
  177 + raise RuntimeError("FITS data store \""+self.m_prodname+"\" not available. Run csiactdata to get a list of available storage names")
  178 + if not gammalib.is_fits(self.m_obs_index+"[OBS_INDEX]"):
  179 + raise RuntimeError("Observation index file \""+self.m_obs_index+"[OBS_INDEX]\" for FITS data store \""+self.m_prodname+"\" not available. Check your master index file or run csdsinfo to get a list of available storage names.")
  180 +
  181 + # Set other members
  182 + self.m_log = False # Logging in client tools
  183 + self.m_debug = False # Debugging in client tools
  184 + self.m_clobber = self["clobber"].boolean()
  185 +
  186 + # Return
  187 + return
  188 +
  189 + def obs_ids(self):
  190 + """
  191 + Return OBS IDs
  192 + """
  193 +
  194 + # Return
  195 + return self.runs
  196 +
  197 + def execute(self):
  198 + """
  199 + Execute the script.
  200 + """
  201 + # Run the script
  202 + self.run()
  203 +
  204 + # Save residual map
  205 + self.save(self.m_outfile)
  206 +
  207 + # Return
  208 + return
  209 +
  210 + def run(self):
  211 + """
  212 + Run the script.
  213 + """
  214 + # Switch screen logging on in debug mode
  215 + if self.logDebug():
  216 + self.log.cout(True)
  217 +
  218 + # Get parameters
  219 + self.get_parameters()
  220 +
  221 + # Write input parameters into logger
  222 + if self.logTerse():
  223 + self.log_parameters()
  224 + self.log("\n")
  225 +
  226 + # Initialise run list
  227 + self.runs = []
  228 +
  229 + # Initialise selction expression
  230 + expr = ""
  231 +
  232 + # Add spatial expression if possible
  233 + if self.select_radec:
  234 + expr += "ANGSEP("+str(self.m_ra)+","+str(self.m_dec)+",RA_PNT,DEC_PNT)<="+str(self.m_radius)
  235 +
  236 + # Add connector if expression is empty
  237 + if len(expr):
  238 + expr += "&&"
  239 +
  240 + # Add quality expression
  241 + expr += "QUALITY<="+str(self.m_quality)
  242 +
  243 + # Add user expression
  244 + if self.m_expression == "NONE" or len(self.m_expression) > 0:
  245 + expr += "&&"+self.m_expression
  246 +
  247 + # Initialise filename
  248 + openfile = self.m_obs_index+"[OBS_INDEX]["+expr+"]"
  249 +
  250 + # Open file
  251 + fits = gammalib.GFits(openfile)
  252 +
  253 + # Check for extistence of rows
  254 + if fits["OBS_INDEX"].nrows() > 0:
  255 +
  256 + # Get HDU
  257 + obs_indx = fits["OBS_INDEX"]
  258 +
  259 + # Read observation IDs into array
  260 + for i in range(obs_indx.nrows()):
  261 + self.runs.append(obs_indx["OBS_ID"][i])
  262 +
  263 + # Dump expression
  264 + if self.logNormal():
  265 + self.log("\n")
  266 + self.log.header3("Expression")
  267 + self.log(expr)
  268 + self.log("\n")
  269 +
  270 + # Dump number of observations
  271 + if self.logTerse():
  272 + self.log("\n")
  273 + self.log.header3("Observations")
  274 + self.log("Found "+str(len(self.runs))+" observations")
  275 + self.log("\n")
  276 +
  277 + # Close file
  278 + fits.close()
  279 +
  280 + # Return
  281 + return
  282 +
  283 + def save(self, outfile):
  284 +
  285 + # Check for clobber
  286 + if os.path.isfile(outfile) and not self.clobber():
  287 + self.log("File "+outfile+" already exists. Set clobber=yes to overwrite file")
  288 + self.log("\n")
  289 + else:
  290 + # Write runs to file
  291 + f = open(outfile,"w")
  292 + for run in self.runs:
  293 + f.write(str(run)+" \n")
  294 + f.close()
  295 +
  296 + # Return
  297 + return
  298 +
  299 +# ======================== #
  300 +# Main routine entry point #
  301 +# ======================== #
  302 +if __name__ == '__main__':
  303 + """
  304 + Find IACT observations.
  305 + """
  306 + # Create instance of application
  307 + app = csfindobs(sys.argv)
  308 +
  309 + # Open logfile
  310 + app.logFileOpen()
  311 +
  312 + # Execute application
  313 + app.execute()
  314 +
... ...
cscripts/csiactdata.py
  View file @3af8c31
... ... @@ -40,8 +40,8 @@ class csiactdata(ctools.cscript):
40 40 Constructor.
41 41 """
42 42 # Set name
43   - self.name = "csiactdata"
44   - self.version = "1.0.0"
  43 + self.name = "csiactdata"
  44 + self.version = "1.0.0"
45 45 self.datapath = os.getenv("VHEFITS","")
46 46  
47 47 # Make sure that parfile exists
... ... @@ -110,7 +110,11 @@ class csiactdata(ctools.cscript):
110 110 # Get filename of master index file
111 111 self.m_master_indx = self["master_indx"].string()
112 112 self.m_master_file = os.path.join(self.datapath, self.m_master_indx)
113   -
  113 +
  114 + # Check for presence of master index file
  115 + if not os.path.isfile(self.m_master_file):
  116 + raise RuntimeError("Master index file \""+self.m_master_file+"\" not found. Use hidden parameter \"master_indx\" to specifiy a different filename.")
  117 +
114 118 # Return
115 119 return
116 120  
... ...
cscripts/csiactobs.py
  View file @3af8c31
... ... @@ -93,19 +93,20 @@ class csiactobs(ctools.cscript):
93 93 # Create default parfile
94 94 pars = gammalib.GApplicationPars()
95 95 pars.append(gammalib.GApplicationPar("datapath","s","a",self.datapath,"","","Path were data is located"))
96   - pars.append(gammalib.GApplicationPar("prodname","s","a","","","","Data storage name (Run csdsinfo to view your options)"))
97   - pars.append(gammalib.GApplicationPar("runlistfile","f","a","runlist.lis","","","Runlist file"))
98   - pars.append(gammalib.GApplicationPar("bkgpars","i","a","1","","","Number of free parameters per background model"))
  96 + pars.append(gammalib.GApplicationPar("prodname","s","a","","","","Data storage name (Run csiactdata to view your options)"))
  97 + pars.append(gammalib.GApplicationPar("infile","f","a","runlist.lis","","","Runlist file"))
  98 + pars.append(gammalib.GApplicationPar("inmodel","f","h","NONE","","","Input model XML file (optional)"))
99 99 pars.append(gammalib.GApplicationPar("outobs","f","a","obs.xml","","","Observation XML outfile"))
100 100 pars.append(gammalib.GApplicationPar("outmodel","f","a","bgmodels.xml","","","Output model XML file"))
101   - pars.append(gammalib.GApplicationPar("inmodel","f","h","NONE","","","Input model XML file (optional)"))
  101 + pars.append(gammalib.GApplicationPar("bkgpars","i","a","1","","","Number of free parameters per background model"))
102 102 pars.append(gammalib.GApplicationPar("master_indx","s","h","master.json","","","Name of master index file"))
  103 + pars.append(gammalib.GApplicationPar("bkg_scale","b","h","yes","","","Specifies whether the background scaling factor from the observation index file should be applied if available. "))
103 104 pars.append(gammalib.GApplicationPar("ev_hiera","s","h","events","","","Hierarchy of event formats"))
104 105 pars.append(gammalib.GApplicationPar("aeff_hiera","s","h","aeff_2d","","","Hierarchy of effective area formats"))
105 106 pars.append(gammalib.GApplicationPar("psf_hiera","s","h","psf_king|psf_3gauss","","","Hierarchy of psf formats"))
106 107 pars.append(gammalib.GApplicationPar("edisp_hiera","s","h","edisp_2d","","","Hierarchy of energy dispersion formats"))
107 108 pars.append(gammalib.GApplicationPar("bkg_hiera","s","h","bkg_3d","","","Hierarchy of background formats"))
108   - pars.append(gammalib.GApplicationPar("bkg_mod_hiera","s","h","irf","irf|aeff|gauss","","Hierarchy of background models"))
  109 + pars.append(gammalib.GApplicationPar("bkg_mod_hiera","s","h","irf|aeff|gauss","","","Hierarchy of background models"))
109 110 pars.append(gammalib.GApplicationPar("bkg_gauss_norm","r","h","1e-8","","","Input normalisation for Gaussian background"))
110 111 pars.append(gammalib.GApplicationPar("bkg_gauss_index","r","h","-2.0","","","Input spectral index for Gaussian background"))
111 112 pars.append(gammalib.GApplicationPar("bkg_gauss_sigma","r","h","2.5","","","Input sigma for Gaussian background"))
... ... @@ -132,7 +133,7 @@ class csiactobs(ctools.cscript):
132 133  
133 134 # Read user parameters
134 135 self.m_prodname = self["prodname"].string()
135   - self.m_runlistfile = gammalib.expand_env(self["runlistfile"].filename())
  136 + self.m_runlistfile = gammalib.expand_env(self["infile"].filename())
136 137 self.m_bkgpars = self["bkgpars"].integer()
137 138  
138 139 # Output model file
... ... @@ -144,6 +145,9 @@ class csiactobs(ctools.cscript):
144 145 # Master index file name
145 146 self.m_master_indx = self["master_indx"].string()
146 147  
  148 + # Read flag for background scaling factor
  149 + self.m_use_bkg_scale = self["bkg_scale"].boolean()
  150 +
147 151 # Read hierarchy of file loading
148 152 self.m_ev_hiera = self["ev_hiera"].string().split("|")
149 153 self.m_aeff_hiera = self["aeff_hiera"].string().split("|")
... ... @@ -186,17 +190,19 @@ class csiactobs(ctools.cscript):
186 190  
187 191 # Check HDUs
188 192 if self.m_hdu_index == "" or self.m_obs_index == "":
189   - raise RuntimeError("FITS data store \""+self.m_prodname+"\" not available. Run csdsinfo to get a list of available storage names")
  193 + raise RuntimeError("FITS data store \""+self.m_prodname+"\" not available. Run csiactdata to get a list of available storage names")
190 194 if not gammalib.is_fits(self.m_hdu_index+"[HDU_INDEX]"):
191   - raise RuntimeError("HDU index file \""+self.m_hdu_index+"[HDU_INDEX]\" for FITS data store \""+self.m_prodname+"\" not available. Check your master index file or run csdsinfo to get a list of available storage names.")
192   -
193   - # Load index
194   - if gammalib.is_fits(self.m_obs_index+"[OBS_INDEX]"):
195   - fits = gammalib.GFits(self.m_obs_index)
196   - self.has_bkg_scale = fits["OBS_INDEX"].contains("BKG_SCALE")
197   - fits.close()
198   - else:
199   - self.has_bkg_scale = False
  195 + raise RuntimeError("HDU index file \""+self.m_hdu_index+"[HDU_INDEX]\" for FITS data store \""+self.m_prodname+"\" not available. Check your master index file or run csiactdata to get a list of available storage names.")
  196 +
  197 + # Check for existence of "BKG_SCALE" in the observation index file if required
  198 + if self.m_use_bkg_scale:
  199 + if gammalib.is_fits(self.m_obs_index+"[OBS_INDEX]"):
  200 + fits = gammalib.GFits(self.m_obs_index)
  201 + if not fits["OBS_INDEX"].contains("BKG_SCALE"):
  202 + self.m_use_bkg_scale = False
  203 + fits.close()
  204 + else:
  205 + self.m_use_bkg_scale = False
200 206  
201 207 # Create base data directory from hdu index file location
202 208 self.subdir = os.path.dirname(self.m_hdu_index)
... ... @@ -481,7 +487,7 @@ class csiactobs(ctools.cscript):
481 487  
482 488 # Handle background scale information if available
483 489 bkg_scale = 1.0
484   - if self.has_bkg_scale:
  490 + if self.m_use_bkg_scale:
485 491 obsindx = gammalib.GFits(self.m_obs_index+"[OBS_INDEX]"+obs_selection)
486 492 bkg_scale = obsindx["OBS_INDEX"]["BKG_SCALE"][0]
487 493 obsindx.close()
... ... @@ -527,7 +533,7 @@ class csiactobs(ctools.cscript):
527 533 self.log(" Point spread function: "+psffile+"\n")
528 534 self.log(" Energy dispersion: "+edispfile+"\n")
529 535 self.log(" Background file: "+bkgfile+"\n")
530   - if self.has_bkg_scale:
  536 + if self.m_use_bkg_scale:
531 537 self.log(" Background scale: "+str(bkg_scale)+"\n")
532 538 self.log("\n")
533 539  
... ...
doc/source/reference_manual/csfindobs.rst 0 → 100644
  View file @3af8c31
  1 +.. _csfindobs:
  2 +
  3 +csfindobs
  4 +=========
  5 +
  6 +Creates a list of observation IDs that fulfils certain criteria, e.g.
  7 +pointing close to specific source.
  8 +
  9 +
  10 +Synopsis
  11 +--------
  12 +
  13 +This script can be used to create lists of observation IDs that share certain
  14 +properties. For instance, all observations pointing around a specific sky
  15 +position can be selected. In addition, the script is flexible to use expressions
  16 +to further constrain criteria to select observations. The user can e.g. select
  17 +all observations within a certain zenith angle range by specifying the hidden
  18 +parameter expression. For example
  19 +
  20 +``csfindobs expression="ZEN_PNT<50&&ZEN_PNT>40"``
  21 +
  22 +is used to select observations with zenith angle range between 40 and 50 degrees.
  23 +
  24 +The parameter ``datapath`` is only queried if the environment variable $VHEFITS
  25 +is not set.
  26 +
  27 +The output of this script is an ASCII file containing observation IDs that
  28 +fulfil criteria specified by the user. This file can be used as input for
  29 +:doc:`csiactobs`. The script operates on an observation index file that needs
  30 +to be provided along with IACT data. `See here <http://gamma-astro-data-formats.readthedocs.org/en/latest/index.html>`__
  31 +how IACT data should be structured. To investigate additional parameters which
  32 +are available for selection, have a look `here <http://gamma-astro-data-formats.readthedocs.org/en/latest/data_storage/obs_index/index.html>`__.
  33 +
  34 +
  35 +General parameters
  36 +------------------
  37 +
  38 +``datapath [string]``
  39 + Path were data are located.
  40 +
  41 +``prodname [string]``
  42 + Name of FITS production (Run :doc:`csiactdata` to view your options).
  43 +
  44 +``outfile [file]``
  45 + Output runlist ASCII file.
  46 +
  47 +``ra [real]``
  48 + Right ascension of search region (invoke "INDEF" to avoid spatial selection).
  49 +
  50 +``dec [real]``
  51 + Declination of search region (invoke "INDEF" to avoid spatial selection).
  52 +
  53 +``rad [real]``
  54 + Radius of search region (deg) (invoke "INDEF" or negative values to avoid
  55 + spatial selection)
  56 +
  57 +``(min_qual = 0) [integer]``
  58 + Minimum data quality (0=best, 1=medium, 2=bad).
  59 +
  60 +``(expression = NONE) [string]``
  61 + Additional expression.
  62 +
  63 +``(master_index = master.json) [string]``
  64 + Name of master index file.
  65 +
  66 +
  67 +Standard parameters
  68 +-------------------
  69 +
  70 +``(chatter = 2) [integer]``
  71 + Verbosity of the executable:
  72 + chatter = 0: no information will be logged
  73 +
  74 + chatter = 1: only errors will be logged
  75 +
  76 + chatter = 2: errors and actions will be logged
  77 +
  78 + chatter = 3: report about the task execution
  79 +
  80 + chatter = 4: detailed report about the task execution
  81 +
  82 +``(clobber = yes) [boolean]``
  83 + Specifies whether an existing output counts cube should be overwritten.
  84 +
  85 +``(debug = no) [boolean]``
  86 + Enables debug mode. In debug mode the executable will dump any log file output to the console.
  87 +
  88 +``(mode = ql) [string]``
  89 + Mode of automatic parameters (default is "ql", i.e. "query and learn").
  90 +
  91 +``(logfile = csfindobs.log) [filename]``
  92 + Log filename.
  93 +
  94 +
  95 +Related tools or scripts
  96 +------------------------
  97 +
  98 +:doc:`csiactobs`
0 99 \ No newline at end of file
... ...
doc/source/reference_manual/csiactdata.rst 0 → 100644
  View file @3af8c31
  1 +.. _csiactdata:
  2 +
  3 +csiactdata
  4 +===========
  5 +
  6 +Dumps information about the available FITS productions of IACT data on the
  7 +screen.
  8 +
  9 +
  10 +Synopsis
  11 +--------
  12 +
  13 +This script inspects the local IACT FITS data storage and prints the names of
  14 +available and valid data productions on the screen. The parameter ``datapath``
  15 +is only queried if the environment variable $VHEFITS is not set. The path should
  16 +be set to the location of the `master index file <http://gamma-astro-data-formats.readthedocs.org/en/latest/data_storage/super_index/index.html>`__.
  17 +The output of this script can be used as input for further tools that use IACT
  18 +data.
  19 +
  20 +
  21 +General parameters
  22 +------------------
  23 +
  24 +``datapath [string]``
  25 + Path were data are located.
  26 +
  27 +``(master_index = master.json) [string]``
  28 + Name of master index file.
  29 +
  30 +
  31 +Standard parameters
  32 +-------------------
  33 +
  34 +``(chatter = 2) [integer]``
  35 + Verbosity of the executable:
  36 + chatter = 0: no information will be logged
  37 +
  38 + chatter = 1: only errors will be logged
  39 +
  40 + chatter = 2: errors and actions will be logged
  41 +
  42 + chatter = 3: report about the task execution
  43 +
  44 + chatter = 4: detailed report about the task execution
  45 +
  46 +``(clobber = yes) [boolean]``
  47 + Specifies whether an existing output counts cube should be overwritten.
  48 +
  49 +``(debug = no) [boolean]``
  50 + Enables debug mode. In debug mode the executable will dump any log file output to the console.
  51 +
  52 +``(mode = ql) [string]``
  53 + Mode of automatic parameters (default is "ql", i.e. "query and learn").
  54 +
  55 +``(logfile = csiactdata.log) [filename]``
  56 + Log filename.
  57 +
  58 +
  59 +Related tools or scripts
  60 +------------------------
  61 +
  62 +:doc:`csfindobs`
0 63 \ No newline at end of file
... ...
doc/source/reference_manual/csiactobs.rst 0 → 100644
  View file @3af8c31
  1 +.. _csiactobs:
  2 +
  3 +csiactobs
  4 +===========
  5 +
  6 +Creates an observation XML file for IACTs using a list of observation IDs as
  7 +input.
  8 +
  9 +
  10 +Synopsis
  11 +--------
  12 +
  13 +This script creates an observation XML file from a list of observation IDs
  14 +provided in an ASCII file. Such an ASCII file can for instance be created with
  15 +:doc:`csfindobs`. The script operates on IACT data that needs to be present on
  16 +the user machine. `See here <http://gamma-astro-data-formats.readthedocs.org/en/latest/index.html>`__
  17 +how IACT data should be structured.
  18 +
  19 +The parameter ``datapath`` is only queried if the environment variable $VHEFITS
  20 +is not set.
  21 +
  22 +Besides creating an observation XML file, this script also stores a model XML
  23 +file that contains background models for each observation. The number of free
  24 +parameters is steered via the ``bkgpars`` parameter.
  25 +
  26 +The user can also specify as ``inmodel`` parameter an XML model file with e.g.
  27 +sky models. The output model will then contain these models as well.
  28 +
  29 +The script further allows to specify hierarchies for the data formats and
  30 +background models. The script will pick the first available option and fall back
  31 +to the next one if the first choice is unavailable. If e.g. the IRF background
  32 +model is not available, the script will assign the next background model in the
  33 +hierarchy ``bkg_mod_hiera``. If the user wants to use a Gaussian background model
  34 +as first choice, the parameter ``bkg_mod_hiera`` should be specified to "gauss".
  35 +In addition, the user can specify start parameters for the background parameters
  36 +that will be in the output XML model.
  37 +
  38 +If an observation that given in the input ASCII file is not available on the user
  39 +machine, the script dumps a warning into the logfile and the obsservation is
  40 +skipped.
  41 +
  42 +
  43 +General parameters
  44 +------------------
  45 +
  46 +``datapath [string]``
  47 + Path were data are located.
  48 +
  49 +``prodname [string]``
  50 + Name of FITS production (Run :doc:`csiactdata` to view your options).
  51 +
  52 +``infile [file]``
  53 + Input runlist ASCII file.
  54 +
  55 +``(inmodel = NONE) [file]``
  56 + Input model XML file.
  57 +
  58 +``outobs [file]``
  59 + Output Observation Definition XML file.
  60 +
  61 +``outmodel [file]``
  62 + Output model XML file.
  63 +
  64 +``bkgpars [integer]``
  65 + Number of free parameters for each background model.
  66 +
  67 +``(master_indx = master.json) [file]``
  68 + Name of master index file.
  69 +
  70 +``(bkg_scale = yes) [boolean]
  71 + Specifies whether the background scaling factor from the observation index
  72 + file should be applied if available.
  73 +
  74 +``(ev_hiera = events) [string]``
  75 + Hierarchy of event file formats.
  76 +
  77 +``(aeff_hiera = aeff_2d) [string]``
  78 + Hierarchy of effective area formats.
  79 +
  80 +``(psf_hiera = psf_king|psf_3gauss) [string]``
  81 + Hierarchy of point spread function formats.
  82 +
  83 +``(edisp_hiera = edisp_2d) [string]``
  84 + Hierarchy of energy disperson formats.
  85 +
  86 +``(bkg_hiera = bkg_3d) [string]``
  87 + Hierarchy of background formats.
  88 +
  89 +``(bkg_mod_hiera = irf|aeff|gauss) [string]``
  90 + Hierarchy of background models.
  91 +
  92 +``(bkg_gauss_norm = 1e-8) [real]``
  93 + Input normalisation for Gaussian background.
  94 +
  95 +``(bkg_gauss_index = -2.0) [real]``
  96 + Input spectral index for Gaussian background.
  97 +
  98 +``(bkg_gauss_sigma = 1e-8) [real]``
  99 + Input sigma for Gaussian background.
  100 +
  101 +``(bkg_aeff_norm = 1e-14) [real]``
  102 + Input normalisation for effective area background.
  103 +
  104 +``(bkg_aeff_index = -2.0) [real]``
  105 + Input spectral index for effective area background.
  106 +
  107 +``(bkg_range_factor = 100.0) [real]``
  108 + Factor to determine range of background normalisation.
  109 +
  110 +
  111 +Standard parameters
  112 +-------------------
  113 +
  114 +``(chatter = 2) [integer]``
  115 + Verbosity of the executable:
  116 + chatter = 0: no information will be logged
  117 +
  118 + chatter = 1: only errors will be logged
  119 +
  120 + chatter = 2: errors and actions will be logged
  121 +
  122 + chatter = 3: report about the task execution
  123 +
  124 + chatter = 4: detailed report about the task execution
  125 +
  126 +``(clobber = yes) [boolean]``
  127 + Specifies whether an existing output counts cube should be overwritten.
  128 +
  129 +``(debug = no) [boolean]``
  130 + Enables debug mode. In debug mode the executable will dump any log file output to the console.
  131 +
  132 +``(mode = ql) [string]``
  133 + Mode of automatic parameters (default is "ql", i.e. "query and learn").
  134 +
  135 +``(logfile = csiactobs.log) [filename]``
  136 + Log filename.
  137 +
  138 +
  139 +Related tools or scripts
  140 +------------------------
  141 +
  142 +:doc:`csiactobs`
0 143 \ No newline at end of file
... ...
doc/source/reference_manual/cstsmapmerge.rst
  View file @3af8c31
... ... @@ -69,4 +69,4 @@ Standard parameters
69 69 Related tools or scripts
70 70 ------------------------
71 71  
72   -:ref:`cstsmap`
  72 +:ref:`cttsmap`
... ...
doc/source/reference_manual/index.rst
  View file @3af8c31
... ... @@ -31,22 +31,33 @@ ctools
31 31 cterror --- Calculates likelihood profile errors <cterror>
32 32  
33 33  
34   -cscripts
35   ---------
  34 +General cscripts
  35 +----------------
36 36  
37 37 .. toctree::
38 38 :maxdepth: 1
39 39  
40 40 cscaldb --- Lists available instrument response functions <cscaldb>
41   - csobsdef --- Generates observation definition file <csobsdef>
42 41 cslightcrv --- Computes lightcurve <cslightcrv>
  42 + csmodelinfo --- Shows model container content <csmodelinfo>
  43 + csmodelmerge --- Merges several model containers into one file <csmodelmerge>
  44 + csobsdef --- Generates observation definition file <csobsdef>
  45 + csobsinfo --- Shows observation container content <csobsinfo>
43 46 cspull --- Generates pull distribution <cspull>
  47 + csresmap --- Generates residual map <csresmap>
44 48 cssens --- Computes CTA sensitivity <cssens>
45 49 csspec --- Computes spectral points <csspec>
46   - csresmap --- Generates residual map <csresmap>
47 50 cstsdist --- Generates TS distribution <cstsdist>
48   - csobsinfo --- Shows observation container content <csobsinfo>
49   - csmodelinfo --- Shows model container content <csmodelinfo>
50   - csmodelmerge --- Merges several model containers into one file <csmodelmerge>
51 51 cstsmapmerge --- Merges slices from ts map computations <cstsmapmerge>
  52 +
  53 +
  54 +IACT cscripts
  55 +-------------
  56 +
  57 +.. toctree::
  58 + :maxdepth: 1
  59 +
52 60 csobs2caldb --- Creates a caldb entry from an input observation <csobs2caldb>
  61 + csiactdata --- Shows information about IACT data available on the user machine <csiactdata>
  62 + csiactobs --- Generates observation definition file for IACT data from observation IDs <csiactobs>
  63 + csfindobs --- Generates a list of IACT observation IDs <csfindobs>
... ...
test/iactdata/runlist.txt deleted
View file @0a31b7c
1   -0
test/test_cscripts.sh.in
  View file @3af8c31
... ... @@ -2,7 +2,7 @@
2 2 # ==========================================================================
3 3 # This script tests all cscripts that are shipped with ctools
4 4 #
5   -# Copyright (C) 2011-2015 Juergen Knoedlseder
  5 +# Copyright (C) 2011-2016 Juergen Knoedlseder
6 6 #
7 7 # This program is free software: you can redistribute it and/or modify
8 8 # it under the terms of the GNU General Public License as published by
... ... @@ -52,6 +52,26 @@ mkdir -p $CALDB
52 52  
53 53  
54 54 #
  55 +# Test csobs2caldb
  56 +# ================
  57 +$ECHO -n "Test csobs2caldb: "
  58 +chmod +x @abs_top_srcdir@/cscripts/csobs2caldb.py
  59 +@abs_top_srcdir@/cscripts/csobs2caldb.py \
  60 + inobs="data/obs_unbinned.xml" \
  61 + rspname="test_irfs" \
  62 + chatter=4 > csobs2caldb.log
  63 +$ECHO -n "."
  64 +if [ -s "$CALDB/data/cta/cta/bcf/test_irfs/irf_file.fits" ]
  65 +then
  66 + $ECHO -n "."
  67 +else
  68 + $ECHO " $CALDB/data/cta/cta/bcf/test_irfs/irf_file.fits file is not valid"
  69 + exit 1
  70 +fi
  71 +$ECHO " ok"
  72 +
  73 +
  74 +#
55 75 # Test csobsdef
56 76 # =============
57 77 $ECHO -n "Test csobsdef: "
... ... @@ -77,26 +97,6 @@ $ECHO &quot; ok&quot;
77 97  
78 98  
79 99 #
80   -# Test csobs2caldb
81   -# ================
82   -$ECHO -n "Test csobs2caldb: "
83   -chmod +x @abs_top_srcdir@/cscripts/csobs2caldb.py
84   -@abs_top_srcdir@/cscripts/csobs2caldb.py \
85   - inobs="data/obs_unbinned.xml" \
86   - rspname="test_irfs" \
87   - chatter=4 #> csobs2caldb.log
88   -$ECHO -n "."
89   -if [ -s "$CALDB/data/cta/cta/bcf/test_irfs/irf_file.fits" ]
90   -then
91   - $ECHO -n "."
92   -else
93   - $ECHO " $CALDB/data/cta/cta/bcf/test_irfs/irf_file.fits file is not valid"
94   - exit 1
95   -fi
96   -$ECHO " ok"
97   -
98   -
99   -#
100 100 # Test csobsinfo
101 101 # ==============
102 102 $ECHO -n "Test csobsinfo: "
... ... @@ -119,29 +119,6 @@ $ECHO &quot; ok&quot;
119 119  
120 120  
121 121 #
122   -# Test csiactobs
123   -# ==============
124   -$ECHO -n "Test csiactobs: "
125   -chmod +x @abs_top_srcdir@/cscripts/csiactobs.py
126   -@abs_top_srcdir@/cscripts/csiactobs.py \
127   - datapath="iactdata/" \
128   - prodname="unit-test" \
129   - runlistfile="iactdata/runlist.txt" \
130   - bkgpars="1" \
131   - outobs="obs2.xml" \
132   - outmodel="bkgmodels.xml" > csiactobs.log
133   -$ECHO -n "."
134   -if [ -s "obs2.xml" ]
135   -then
136   - $ECHO -n "."
137   -else
138   - $ECHO " obs.xml file is not valid"
139   - exit 1
140   -fi
141   -$ECHO " ok"
142   -
143   -
144   -#
145 122 # Test csmodelinfo
146 123 # ================
147 124 $ECHO -n "Test csmodelinfo: "
... ... @@ -161,9 +138,10 @@ $ECHO &quot; ok&quot;
161 138  
162 139  
163 140 #
164   -# Test csmodelmerge (space-separated)
165   -# ===================================
166   -$ECHO -n "Test csmodelmerge (space-separated): "
  141 +# Test csmodelmerge
  142 +# =================
  143 +$ECHO -n "Test csmodelmerge: "
  144 +# === space-separated ===
167 145 chmod +x @abs_top_srcdir@/cscripts/csmodelmerge.py
168 146 @abs_top_srcdir@/cscripts/csmodelmerge.py \
169 147 inmodels="data/crab.xml data/source1.xml" \
... ... @@ -176,14 +154,7 @@ else
176 154 $ECHO " merged_models.xml file is not valid"
177 155 exit 1
178 156 fi
179   -$ECHO " ok"
180   -
181   -
182   -#
183   -# Test csmodelmerge (comma-separated)
184   -# ===================================
185   -$ECHO -n "Test csmodelmerge (comma-separated): "
186   -chmod +x @abs_top_srcdir@/cscripts/csmodelmerge.py
  157 +# === comma-separated ===
187 158 @abs_top_srcdir@/cscripts/csmodelmerge.py \
188 159 inmodels="data/crab.xml,data/source1.xml" \
189 160 outmodel="merged_models2.xml" > csmodelmerge.log
... ... @@ -195,14 +166,7 @@ else
195 166 $ECHO " merged_models2.xml file is not valid"
196 167 exit 1
197 168 fi
198   -$ECHO " ok"
199   -
200   -
201   -#
202   -# Test csmodelmerge (wildcard string)
203   -# ===================================
204   -$ECHO -n "Test csmodelmerge (wildcard string): "
205   -chmod +x @abs_top_srcdir@/cscripts/csmodelmerge.py
  169 +# === wildcard string ===
206 170 @abs_top_srcdir@/cscripts/csmodelmerge.py \
207 171 inmodels="data/model_cube_background*.xml" \
208 172 outmodel="merged_models3.xml" > csmodelmerge.log
... ... @@ -214,15 +178,8 @@ else
214 178 $ECHO " merged_models3.xml file is not valid"
215 179 exit 1
216 180 fi
217   -$ECHO " ok"
218   -
219   -
220   -#
221   -# Test csmodelmerge (ascii-file)
222   -# ==============================
  181 +# === ascii-file ===
223 182 ls data/model_cube_background* > models.txt
224   -$ECHO -n "Test csmodelmerge (ascii-file): "
225   -chmod +x @abs_top_srcdir@/cscripts/csmodelmerge.py
226 183 @abs_top_srcdir@/cscripts/csmodelmerge.py \
227 184 inmodels="@models.txt" \
228 185 outmodel="merged_models4.xml" > csmodelmerge.log
... ... @@ -239,9 +196,10 @@ rm models.txt
239 196  
240 197  
241 198 #
242   -# Test cstsmapmerge (comma-separated)
243   -# ===================================
244   -$ECHO -n "Test cstsmapmerge (comma-separated): "
  199 +# Test cstsmapmerge
  200 +# =================
  201 +$ECHO -n "Test cstsmapmerge: "
  202 +# === comma-separated ===
245 203 chmod +x @abs_top_srcdir@/cscripts/cstsmapmerge.py
246 204 @abs_top_srcdir@/cscripts/cstsmapmerge.py \
247 205 inmaps="data/tsmap_0.fits,data/tsmap_1.fits" \
... ... @@ -254,14 +212,7 @@ else
254 212 $ECHO " merged_tsmap1.fits file is not valid"
255 213 exit 1
256 214 fi
257   -$ECHO " ok"
258   -
259   -
260   -#
261   -# Test cstsmapmerge (space-separated)
262   -# ===================================
263   -$ECHO -n "Test cstsmapmerge (space-separated): "
264   -chmod +x @abs_top_srcdir@/cscripts/cstsmapmerge.py
  215 +# === space-separated ===
265 216 @abs_top_srcdir@/cscripts/cstsmapmerge.py \
266 217 inmaps="data/tsmap_0.fits data/tsmap_1.fits" \
267 218 outmap="merged_tsmap2.fits" > cstsmapmerge.log
... ... @@ -273,14 +224,7 @@ else
273 224 $ECHO " merged_tsmap2.fits file is not valid"
274 225 exit 1
275 226 fi
276   -$ECHO " ok"
277   -
278   -
279   -#
280   -# Test cstsmapmerge (wildcard string)
281   -# ===================================
282   -$ECHO -n "Test cstsmapmerge (wildcard string): "
283   -chmod +x @abs_top_srcdir@/cscripts/cstsmapmerge.py
  227 +# === wildcard string ===
284 228 @abs_top_srcdir@/cscripts/cstsmapmerge.py \
285 229 inmaps="data/tsmap_*.fits" \
286 230 outmap="merged_tsmap3.fits" > cstsmapmerge.log
... ... @@ -292,15 +236,8 @@ else
292 236 $ECHO " merged_tsmap3.fits file is not valid"
293 237 exit 1
294 238 fi
295   -$ECHO " ok"
296   -
297   -
298   -#
299   -# Test cstsmapmerge (ascii-file)
300   -# ==============================
  239 +# === ascii-file ===
301 240 ls data/tsmap_*.fits > tsmaps.txt
302   -$ECHO -n "Test cstsmapmerge (ascii-file): "
303   -chmod +x @abs_top_srcdir@/cscripts/cstsmapmerge.py
304 241 @abs_top_srcdir@/cscripts/cstsmapmerge.py \
305 242 inmaps="@tsmaps.txt" \
306 243 outmap="merged_tsmap4.fits" > cstsmapmerge.log
... ... @@ -439,9 +376,10 @@ $ECHO &quot; ok&quot;
439 376  
440 377  
441 378 #
442   -# Test csspec (unbinned)
443   -# =====================
444   -$ECHO -n "Test csspec (unbinned): "
  379 +# Test csspec
  380 +# ===========
  381 +$ECHO -n "Test csspec: "
  382 +# === Unbinned ===
445 383 chmod +x @abs_top_srcdir@/cscripts/csspec.py
446 384 @abs_top_srcdir@/cscripts/csspec.py \
447 385 inobs="data/crab_events.fits" \
... ... @@ -462,14 +400,7 @@ else
462 400 $ECHO " spectrum.fits file is not valid"
463 401 exit 1
464 402 fi
465   -$ECHO " ok"
466   -
467   -
468   -#
469   -# Test csspec (binned)
470   -# ====================
471   -$ECHO -n "Test csspec (binned): "
472   -chmod +x @abs_top_srcdir@/cscripts/csspec.py
  403 +# === Binned ===
473 404 @abs_top_srcdir@/cscripts/csspec.py \
474 405 inobs="data/crab_cntmap_small.fits.gz" \
475 406 expcube="data/crab_expcube.fits.gz" \
... ... @@ -494,9 +425,10 @@ $ECHO &quot; ok&quot;
494 425  
495 426  
496 427 #
497   -# Test cslightcrv (unbinned)
498   -# ==========================
499   -$ECHO -n "Test cslightcrv (unbinned): "
  428 +# Test cslightcrv
  429 +# ===============
  430 +$ECHO -n "Test cslightcrv: "
  431 +# === unbinned ===
500 432 chmod +x @abs_top_srcdir@/cscripts/cslightcrv.py
501 433 @abs_top_srcdir@/cscripts/cslightcrv.py \
502 434 inobs="data/crab_events.fits" \
... ... @@ -520,14 +452,7 @@ else
520 452 $ECHO " lightcurve.fits file is not valid"
521 453 exit 1
522 454 fi
523   -$ECHO " ok"
524   -
525   -
526   -#
527   -# Test cslightcrv (binned)
528   -# ========================
529   -$ECHO -n "Test cslightcrv (binned): "
530   -chmod +x @abs_top_srcdir@/cscripts/cslightcrv.py
  455 +# === binned ===
531 456 @abs_top_srcdir@/cscripts/cslightcrv.py \
532 457 inobs="data/crab_events.fits" \
533 458 inmodel="data/crab.xml" \
... ... @@ -555,3 +480,49 @@ else
555 480 exit 1
556 481 fi
557 482 $ECHO " ok"
  483 +
  484 +
  485 +#
  486 +# Test csfindobs
  487 +# ==============
  488 +$ECHO -n "Test csfindobs: "
  489 +chmod +x @abs_top_srcdir@/cscripts/csfindobs.py
  490 +@abs_top_srcdir@/cscripts/csfindobs.py \
  491 + datapath="iactdata/" \
  492 + prodname="unit-test" \
  493 + ra="83.6331" \
  494 + dec="22.01" \
  495 + rad="1.0" \
  496 + outfile="iactrunlist.dat" > csfindobs.log
  497 +$ECHO -n "."
  498 +if [ -s "iactrunlist.dat" ]
  499 +then
  500 + $ECHO -n "."
  501 +else
  502 + $ECHO " iactdata/runlist.txt file is not valid"
  503 + exit 1
  504 +fi
  505 +$ECHO " ok"
  506 +
  507 +
  508 +#
  509 +# Test csiactobs
  510 +# ==============
  511 +$ECHO -n "Test csiactobs: "
  512 +chmod +x @abs_top_srcdir@/cscripts/csiactobs.py
  513 +@abs_top_srcdir@/cscripts/csiactobs.py \
  514 + datapath="iactdata/" \
  515 + prodname="unit-test" \
  516 + infile="iactrunlist.dat" \
  517 + bkgpars="1" \
  518 + outobs="obs2.xml" \
  519 + outmodel="bkgmodels.xml" > csiactobs.log
  520 +$ECHO -n "."
  521 +if [ -s "obs2.xml" ]
  522 +then
  523 + $ECHO -n "."
  524 +else
  525 + $ECHO " obs.xml file is not valid"
  526 + exit 1
  527 +fi
  528 +$ECHO " ok"
... ...