Commit 0c3247471ea4bf1a5ac93e1278bbd6feeb4c6c15
1 parent
53ca257e
Added csfindobs including docs and unit test, also added documentation for csiactobs and csiactdata
Showing
10 changed files
with
625 additions
and
18 deletions
cscripts/Makefile.am
| @@ -5,14 +5,14 @@ pythonscripts = __init__.py \ | @@ -5,14 +5,14 @@ pythonscripts = __init__.py \ | ||
| 5 | cspull.py cssens.py cstsdist.py cscaldb.py csresmap.py \ | 5 | cspull.py cssens.py cstsdist.py cscaldb.py csresmap.py \ |
| 6 | csiactobs.py csiactdata.py csobsdef.py csspec.py cslightcrv.py csobsinfo.py \ | 6 | csiactobs.py csiactdata.py csobsdef.py csspec.py cslightcrv.py csobsinfo.py \ |
| 7 | csinfo.py csmodelinfo.py csmodelmerge.py cstsmapmerge.py \ | 7 | csinfo.py csmodelinfo.py csmodelmerge.py cstsmapmerge.py \ |
| 8 | - csobs2caldb.py \ | 8 | + csobs2caldb.py csfindobs.py \ |
| 9 | obsutils.py | 9 | obsutils.py |
| 10 | 10 | ||
| 11 | # Scripts for which symbolic links will be created in bindir | 11 | # Scripts for which symbolic links will be created in bindir |
| 12 | linkscripts = cspull cssens cstsdist cscaldb csresmap \ | 12 | linkscripts = cspull cssens cstsdist cscaldb csresmap \ |
| 13 | csiactobs csiactdata csobsdef csspec cslightcrv csobsinfo \ | 13 | csiactobs csiactdata csobsdef csspec cslightcrv csobsinfo \ |
| 14 | csinfo csmodelinfo csmodelmerge cstsmapmerge \ | 14 | csinfo csmodelinfo csmodelmerge cstsmapmerge \ |
| 15 | - csobs2caldb | 15 | + csobs2caldb csfindobs |
| 16 | 16 | ||
| 17 | # Distribute scripts | 17 | # Distribute scripts |
| 18 | EXTRA_DIST = $(binscripts) $(pythonscripts) | 18 | EXTRA_DIST = $(binscripts) $(pythonscripts) |
cscripts/csfindobs.py
0 → 100755
| 1 | +#!/usr/bin/env python | ||
| 2 | +import gammalib | ||
| 3 | +import ctools | ||
| 4 | +import sys | ||
| 5 | +import os | ||
| 6 | +import json | ||
| 7 | + | ||
| 8 | +# ============== # | ||
| 9 | +# csrunfinder class # | ||
| 10 | +# ============== # | ||
| 11 | +class csfindobs(ctools.cscript): | ||
| 12 | + """ | ||
| 13 | + This class can be used to find observations from an IACT data store. It derives | ||
| 14 | + from the GammaLib::GApplication class which provides support for parameter | ||
| 15 | + files, command line arguments, and logging. In that way the Python | ||
| 16 | + script behaves just as a regular ctool. | ||
| 17 | + """ | ||
| 18 | + def __init__(self, *argv): | ||
| 19 | + """ | ||
| 20 | + Constructor. | ||
| 21 | + """ | ||
| 22 | + # Set name | ||
| 23 | + self.name = "csfindobs" | ||
| 24 | + self.version = "1.0.0" | ||
| 25 | + self.verbose = False | ||
| 26 | + self.datapath = os.getenv("VHEFITS","") | ||
| 27 | + | ||
| 28 | + # Make sure that parfile exists | ||
| 29 | + file = self.parfile() | ||
| 30 | + | ||
| 31 | + # Initialise application | ||
| 32 | + if len(argv) == 0: | ||
| 33 | + ctools.cscript.__init__(self, self.name, self.version) | ||
| 34 | + elif len(argv) ==1: | ||
| 35 | + ctools.cscript.__init__(self, self.name, self.version, *argv) | ||
| 36 | + else: | ||
| 37 | + raise TypeError("Invalid number of arguments given.") | ||
| 38 | + | ||
| 39 | + # Set logger properties | ||
| 40 | + self.log_header() | ||
| 41 | + self.log.date(True) | ||
| 42 | + | ||
| 43 | + # Return | ||
| 44 | + return | ||
| 45 | + | ||
| 46 | + def __del__(self): | ||
| 47 | + """ | ||
| 48 | + Destructor. | ||
| 49 | + """ | ||
| 50 | + # Write separator into logger | ||
| 51 | + if self.logTerse(): | ||
| 52 | + self.log("\n") | ||
| 53 | + | ||
| 54 | + # Return | ||
| 55 | + return | ||
| 56 | + | ||
| 57 | + def parfile(self): | ||
| 58 | + """ | ||
| 59 | + Check if parfile exists. If parfile does not exist then create a | ||
| 60 | + default parfile. This kluge avoids shipping the cscript with a parfile. | ||
| 61 | + """ | ||
| 62 | + # Set parfile name | ||
| 63 | + parfile = self.name+".par" | ||
| 64 | + | ||
| 65 | + try: | ||
| 66 | + pars = gammalib.GApplicationPars(parfile) | ||
| 67 | + except: | ||
| 68 | + # Signal if parfile was not found | ||
| 69 | + sys.stdout.write("Parfile "+parfile+" not found. Create default parfile.\n") | ||
| 70 | + | ||
| 71 | + # Create default parfile | ||
| 72 | + pars = gammalib.GApplicationPars() | ||
| 73 | + pars.append(gammalib.GApplicationPar("datapath","s","a",self.datapath,"","","Path were data is located")) | ||
| 74 | + pars.append(gammalib.GApplicationPar("prodname","s","a","prod-name","","","Name of FITS production (Run csiactdata to view your options)")) | ||
| 75 | + pars.append(gammalib.GApplicationPar("ra","r","a","83.6331","","","Right ascension")) | ||
| 76 | + pars.append(gammalib.GApplicationPar("dec","r","a","22.01","","","Declination")) | ||
| 77 | + pars.append(gammalib.GApplicationPar("rad","r","a","2.5","","","Search radius")) | ||
| 78 | + pars.append(gammalib.GApplicationPar("min_qual","i","h","0","0|1|2","","Minimum data quality (0=perfect, 1=ok, 2=bad)")) | ||
| 79 | + pars.append(gammalib.GApplicationPar("expression","s","h","NONE","","","Additional expression")) | ||
| 80 | + pars.append(gammalib.GApplicationPar("master_indx","s","h","master.json","","","Name of master index file")) | ||
| 81 | + pars.append(gammalib.GApplicationPar("outfile","f","a","runlist.lis","","","Runlist outfile")) | ||
| 82 | + pars.append_standard() | ||
| 83 | + pars.save(parfile) | ||
| 84 | + | ||
| 85 | + # Return | ||
| 86 | + return | ||
| 87 | + | ||
| 88 | + def get_parameters(self): | ||
| 89 | + """ | ||
| 90 | + Get parameters from parfile and setup the observation. | ||
| 91 | + """ | ||
| 92 | + # Get parameters | ||
| 93 | + if self.datapath == "": | ||
| 94 | + self.datapath = self["datapath"].string() | ||
| 95 | + | ||
| 96 | + # Expand environment | ||
| 97 | + self.datapath = gammalib.expand_env(self.datapath) | ||
| 98 | + | ||
| 99 | + # Get production name | ||
| 100 | + self.m_prodname = self["prodname"].string() | ||
| 101 | + | ||
| 102 | + # Master index file name | ||
| 103 | + self.m_master_indx = self["master_indx"].string() | ||
| 104 | + | ||
| 105 | + # Intiialise flag if spatial selection is required | ||
| 106 | + self.select_radec = True | ||
| 107 | + | ||
| 108 | + # Check for vailidity of spatial parameters | ||
| 109 | + if self["ra"].is_valid() and self["dec"].is_valid() and self["rad"].is_valid(): | ||
| 110 | + | ||
| 111 | + # Read spatial parameters | ||
| 112 | + self.m_ra = self["ra"].real() | ||
| 113 | + self.m_dec = self["dec"].real() | ||
| 114 | + self.m_radius = self["rad"].real() | ||
| 115 | + | ||
| 116 | + else: | ||
| 117 | + self.select_radec = False | ||
| 118 | + | ||
| 119 | + # Check Radius for validity | ||
| 120 | + if self.m_radius <= 0.0: | ||
| 121 | + self.select_radec = False | ||
| 122 | + | ||
| 123 | + # Read other parameters | ||
| 124 | + self.m_quality = self["min_qual"].integer() | ||
| 125 | + self.m_expression = self["expression"].string() | ||
| 126 | + | ||
| 127 | + # Check for validity of expression | ||
| 128 | + if self.m_expression == "NONE" or self.m_expression == "INDEF": | ||
| 129 | + self.m_expression = "" | ||
| 130 | + | ||
| 131 | + # Read outfile | ||
| 132 | + self.m_outfile = self["outfile"].filename() | ||
| 133 | + | ||
| 134 | + # Open master index file and look for prodname | ||
| 135 | + master_file = os.path.join(self.datapath, self.m_master_indx) | ||
| 136 | + if not os.path.isfile(master_file): | ||
| 137 | + raise RuntimeError("FITS data store not available. No master index file found at \""+master_file+"\". Make sure the file is copied from the server and your datapath is set correctly.") | ||
| 138 | + | ||
| 139 | + # Open and load JSON file | ||
| 140 | + json_data = open(master_file).read() | ||
| 141 | + data = json.loads(json_data) | ||
| 142 | + if not "datasets" in data: | ||
| 143 | + raise RuntimeError("Key \"datasets\" not available in master index file.") | ||
| 144 | + | ||
| 145 | + # Get configurations | ||
| 146 | + configs = data["datasets"] | ||
| 147 | + | ||
| 148 | + # Initialise obs index file | ||
| 149 | + self.m_obs_index = "" | ||
| 150 | + | ||
| 151 | + # Get HDUs | ||
| 152 | + for config in configs: | ||
| 153 | + if self.m_prodname == config["name"]: | ||
| 154 | + self.m_obs_index = str(os.path.join(self.datapath, config["obsindx"])) | ||
| 155 | + break | ||
| 156 | + | ||
| 157 | + # Check HDUs | ||
| 158 | + if self.m_obs_index == "": | ||
| 159 | + raise RuntimeError("FITS data store \""+self.m_prodname+"\" not available. Run csiactdata to get a list of available storage names") | ||
| 160 | + if not gammalib.is_fits(self.m_obs_index+"[OBS_INDEX]"): | ||
| 161 | + raise RuntimeError("Observation index file \""+self.m_obs_index+"[OBS_INDEX]\" for FITS data store \""+self.m_prodname+"\" not available. Check your master index file or run csdsinfo to get a list of available storage names.") | ||
| 162 | + | ||
| 163 | + self.m_log = False # Logging in client tools | ||
| 164 | + self.m_debug = False # Debugging in client tools | ||
| 165 | + self.m_clobber = self["clobber"].boolean() | ||
| 166 | + | ||
| 167 | + # Return | ||
| 168 | + return | ||
| 169 | + | ||
| 170 | + def obs_ids(self): | ||
| 171 | + """ | ||
| 172 | + Return OBS IDs | ||
| 173 | + """ | ||
| 174 | + | ||
| 175 | + # Return | ||
| 176 | + return self.runs | ||
| 177 | + | ||
| 178 | + def execute(self): | ||
| 179 | + """ | ||
| 180 | + Execute the script. | ||
| 181 | + """ | ||
| 182 | + # Run the script | ||
| 183 | + self.run() | ||
| 184 | + | ||
| 185 | + # Save residual map | ||
| 186 | + self.save(self.m_outfile) | ||
| 187 | + | ||
| 188 | + # Return | ||
| 189 | + return | ||
| 190 | + | ||
| 191 | + | ||
| 192 | + def run(self): | ||
| 193 | + """ | ||
| 194 | + Run the script. | ||
| 195 | + """ | ||
| 196 | + # Switch screen logging on in debug mode | ||
| 197 | + if self.logDebug(): | ||
| 198 | + self.log.cout(True) | ||
| 199 | + | ||
| 200 | + # Get parameters | ||
| 201 | + self.get_parameters() | ||
| 202 | + | ||
| 203 | + # Write input parameters into logger | ||
| 204 | + if self.logTerse(): | ||
| 205 | + self.log_parameters() | ||
| 206 | + self.log("\n") | ||
| 207 | + | ||
| 208 | + # Initialise run list | ||
| 209 | + self.runs = [] | ||
| 210 | + | ||
| 211 | + # Initialise selction expression | ||
| 212 | + expr = "" | ||
| 213 | + | ||
| 214 | + # Add spatial expression if possible | ||
| 215 | + if self.select_radec: | ||
| 216 | + expr += "ANGSEP("+str(self.m_ra)+","+str(self.m_dec)+",RA_PNT,DEC_PNT)<="+str(self.m_radius) | ||
| 217 | + | ||
| 218 | + # Add connector if expression is empty | ||
| 219 | + if len(expr): | ||
| 220 | + expr += "&&" | ||
| 221 | + | ||
| 222 | + # Add quality expression | ||
| 223 | + expr += "QUALITY<="+str(self.m_quality) | ||
| 224 | + | ||
| 225 | + # Add user expression | ||
| 226 | + if self.m_expression == "NONE" or len(self.m_expression) > 0: | ||
| 227 | + expr += "&&"+self.m_expression | ||
| 228 | + | ||
| 229 | + # Initialise filename | ||
| 230 | + openfile = self.m_obs_index+"[OBS_INDEX]["+expr+"]" | ||
| 231 | + | ||
| 232 | + # Open file | ||
| 233 | + fits = gammalib.GFits(openfile) | ||
| 234 | + | ||
| 235 | + # Check for extistence of rows | ||
| 236 | + if fits["OBS_INDEX"].nrows() > 0: | ||
| 237 | + | ||
| 238 | + # Get HDU | ||
| 239 | + obs_indx = fits["OBS_INDEX"] | ||
| 240 | + | ||
| 241 | + # Read observation IDs into array | ||
| 242 | + for i in range(obs_indx.nrows()): | ||
| 243 | + self.runs.append(obs_indx["OBS_ID"][i]) | ||
| 244 | + | ||
| 245 | + if self.logNormal(): | ||
| 246 | + self.log("\n") | ||
| 247 | + self.log.header3("Expression") | ||
| 248 | + self.log(expr) | ||
| 249 | + self.log("\n") | ||
| 250 | + | ||
| 251 | + | ||
| 252 | + if self.logTerse(): | ||
| 253 | + self.log("\n") | ||
| 254 | + self.log.header3("Observations") | ||
| 255 | + self.log("Found "+str(len(self.runs))+" observations") | ||
| 256 | + self.log("\n") | ||
| 257 | + | ||
| 258 | + # Close file | ||
| 259 | + fits.close() | ||
| 260 | + | ||
| 261 | + # Return | ||
| 262 | + return | ||
| 263 | + | ||
| 264 | + def save(self, outfile): | ||
| 265 | + | ||
| 266 | + # Check for clobber | ||
| 267 | + if os.path.isfile(outfile) and not self.clobber(): | ||
| 268 | + self.log("File "+outfile+" already exists. Set clobber=yes to overwrite file") | ||
| 269 | + self.log("\n") | ||
| 270 | + else: | ||
| 271 | + # Write runs to file | ||
| 272 | + f = open(outfile,"w") | ||
| 273 | + for run in self.runs: | ||
| 274 | + f.write(str(run)+" \n") | ||
| 275 | + f.close() | ||
| 276 | + | ||
| 277 | + # Return | ||
| 278 | + return | ||
| 279 | + | ||
| 280 | +# ======================== # | ||
| 281 | +# Main routine entry point # | ||
| 282 | +# ======================== # | ||
| 283 | +if __name__ == '__main__': | ||
| 284 | + """ | ||
| 285 | + Generates residual count map. | ||
| 286 | + """ | ||
| 287 | + # Create instance of application | ||
| 288 | + app = csfindobs(sys.argv) | ||
| 289 | + | ||
| 290 | + # Open logfile | ||
| 291 | + app.logFileOpen() | ||
| 292 | + | ||
| 293 | + # Execute application | ||
| 294 | + app.execute() | ||
| 295 | + |
cscripts/csiactdata.py
| @@ -110,7 +110,11 @@ class csiactdata(ctools.cscript): | @@ -110,7 +110,11 @@ class csiactdata(ctools.cscript): | ||
| 110 | # Get filename of master index file | 110 | # Get filename of master index file |
| 111 | self.m_master_indx = self["master_indx"].string() | 111 | self.m_master_indx = self["master_indx"].string() |
| 112 | self.m_master_file = os.path.join(self.datapath, self.m_master_indx) | 112 | self.m_master_file = os.path.join(self.datapath, self.m_master_indx) |
| 113 | - | 113 | + |
| 114 | + # Check for presence of master index file | ||
| 115 | + if not os.path.isfile(self.m_master_file): | ||
| 116 | + raise RuntimeError("Master index file \""+self.m_master_file+"\" not found. Use hidden parameter \"master_indx\" to specifiy a different filename.") | ||
| 117 | + | ||
| 114 | # Return | 118 | # Return |
| 115 | return | 119 | return |
| 116 | 120 |
cscripts/csiactobs.py
| @@ -93,19 +93,20 @@ class csiactobs(ctools.cscript): | @@ -93,19 +93,20 @@ class csiactobs(ctools.cscript): | ||
| 93 | # Create default parfile | 93 | # Create default parfile |
| 94 | pars = gammalib.GApplicationPars() | 94 | pars = gammalib.GApplicationPars() |
| 95 | pars.append(gammalib.GApplicationPar("datapath","s","a",self.datapath,"","","Path were data is located")) | 95 | pars.append(gammalib.GApplicationPar("datapath","s","a",self.datapath,"","","Path were data is located")) |
| 96 | - pars.append(gammalib.GApplicationPar("prodname","s","a","","","","Data storage name (Run csdsinfo to view your options)")) | 96 | + pars.append(gammalib.GApplicationPar("prodname","s","a","","","","Data storage name (Run csiactdata to view your options)")) |
| 97 | pars.append(gammalib.GApplicationPar("runlistfile","f","a","runlist.lis","","","Runlist file")) | 97 | pars.append(gammalib.GApplicationPar("runlistfile","f","a","runlist.lis","","","Runlist file")) |
| 98 | pars.append(gammalib.GApplicationPar("bkgpars","i","a","1","","","Number of free parameters per background model")) | 98 | pars.append(gammalib.GApplicationPar("bkgpars","i","a","1","","","Number of free parameters per background model")) |
| 99 | pars.append(gammalib.GApplicationPar("outobs","f","a","obs.xml","","","Observation XML outfile")) | 99 | pars.append(gammalib.GApplicationPar("outobs","f","a","obs.xml","","","Observation XML outfile")) |
| 100 | pars.append(gammalib.GApplicationPar("outmodel","f","a","bgmodels.xml","","","Output model XML file")) | 100 | pars.append(gammalib.GApplicationPar("outmodel","f","a","bgmodels.xml","","","Output model XML file")) |
| 101 | pars.append(gammalib.GApplicationPar("inmodel","f","h","NONE","","","Input model XML file (optional)")) | 101 | pars.append(gammalib.GApplicationPar("inmodel","f","h","NONE","","","Input model XML file (optional)")) |
| 102 | pars.append(gammalib.GApplicationPar("master_indx","s","h","master.json","","","Name of master index file")) | 102 | pars.append(gammalib.GApplicationPar("master_indx","s","h","master.json","","","Name of master index file")) |
| 103 | + pars.append(gammalib.GApplicationPar("bkg_scale","b","h","yes","","","Specifies whether the background scaling factor from the observation index file should be applied if available. ")) | ||
| 103 | pars.append(gammalib.GApplicationPar("ev_hiera","s","h","events","","","Hierarchy of event formats")) | 104 | pars.append(gammalib.GApplicationPar("ev_hiera","s","h","events","","","Hierarchy of event formats")) |
| 104 | pars.append(gammalib.GApplicationPar("aeff_hiera","s","h","aeff_2d","","","Hierarchy of effective area formats")) | 105 | pars.append(gammalib.GApplicationPar("aeff_hiera","s","h","aeff_2d","","","Hierarchy of effective area formats")) |
| 105 | pars.append(gammalib.GApplicationPar("psf_hiera","s","h","psf_king|psf_3gauss","","","Hierarchy of psf formats")) | 106 | pars.append(gammalib.GApplicationPar("psf_hiera","s","h","psf_king|psf_3gauss","","","Hierarchy of psf formats")) |
| 106 | pars.append(gammalib.GApplicationPar("edisp_hiera","s","h","edisp_2d","","","Hierarchy of energy dispersion formats")) | 107 | pars.append(gammalib.GApplicationPar("edisp_hiera","s","h","edisp_2d","","","Hierarchy of energy dispersion formats")) |
| 107 | pars.append(gammalib.GApplicationPar("bkg_hiera","s","h","bkg_3d","","","Hierarchy of background formats")) | 108 | pars.append(gammalib.GApplicationPar("bkg_hiera","s","h","bkg_3d","","","Hierarchy of background formats")) |
| 108 | - pars.append(gammalib.GApplicationPar("bkg_mod_hiera","s","h","irf","irf|aeff|gauss","","Hierarchy of background models")) | 109 | + pars.append(gammalib.GApplicationPar("bkg_mod_hiera","s","h","irf|aeff|gauss","","","Hierarchy of background models")) |
| 109 | pars.append(gammalib.GApplicationPar("bkg_gauss_norm","r","h","1e-8","","","Input normalisation for Gaussian background")) | 110 | pars.append(gammalib.GApplicationPar("bkg_gauss_norm","r","h","1e-8","","","Input normalisation for Gaussian background")) |
| 110 | pars.append(gammalib.GApplicationPar("bkg_gauss_index","r","h","-2.0","","","Input spectral index for Gaussian background")) | 111 | pars.append(gammalib.GApplicationPar("bkg_gauss_index","r","h","-2.0","","","Input spectral index for Gaussian background")) |
| 111 | pars.append(gammalib.GApplicationPar("bkg_gauss_sigma","r","h","2.5","","","Input sigma for Gaussian background")) | 112 | pars.append(gammalib.GApplicationPar("bkg_gauss_sigma","r","h","2.5","","","Input sigma for Gaussian background")) |
| @@ -144,6 +145,9 @@ class csiactobs(ctools.cscript): | @@ -144,6 +145,9 @@ class csiactobs(ctools.cscript): | ||
| 144 | # Master index file name | 145 | # Master index file name |
| 145 | self.m_master_indx = self["master_indx"].string() | 146 | self.m_master_indx = self["master_indx"].string() |
| 146 | 147 | ||
| 148 | + # Read flag for background scaling factor | ||
| 149 | + self.m_use_bkg_scale = self["bkg_scale"].boolean() | ||
| 150 | + | ||
| 147 | # Read hierarchy of file loading | 151 | # Read hierarchy of file loading |
| 148 | self.m_ev_hiera = self["ev_hiera"].string().split("|") | 152 | self.m_ev_hiera = self["ev_hiera"].string().split("|") |
| 149 | self.m_aeff_hiera = self["aeff_hiera"].string().split("|") | 153 | self.m_aeff_hiera = self["aeff_hiera"].string().split("|") |
| @@ -186,17 +190,19 @@ class csiactobs(ctools.cscript): | @@ -186,17 +190,19 @@ class csiactobs(ctools.cscript): | ||
| 186 | 190 | ||
| 187 | # Check HDUs | 191 | # Check HDUs |
| 188 | if self.m_hdu_index == "" or self.m_obs_index == "": | 192 | if self.m_hdu_index == "" or self.m_obs_index == "": |
| 189 | - raise RuntimeError("FITS data store \""+self.m_prodname+"\" not available. Run csdsinfo to get a list of available storage names") | 193 | + raise RuntimeError("FITS data store \""+self.m_prodname+"\" not available. Run csiactdata to get a list of available storage names") |
| 190 | if not gammalib.is_fits(self.m_hdu_index+"[HDU_INDEX]"): | 194 | if not gammalib.is_fits(self.m_hdu_index+"[HDU_INDEX]"): |
| 191 | - raise RuntimeError("HDU index file \""+self.m_hdu_index+"[HDU_INDEX]\" for FITS data store \""+self.m_prodname+"\" not available. Check your master index file or run csdsinfo to get a list of available storage names.") | ||
| 192 | - | ||
| 193 | - # Load index | ||
| 194 | - if gammalib.is_fits(self.m_obs_index+"[OBS_INDEX]"): | ||
| 195 | - fits = gammalib.GFits(self.m_obs_index) | ||
| 196 | - self.has_bkg_scale = fits["OBS_INDEX"].contains("BKG_SCALE") | ||
| 197 | - fits.close() | ||
| 198 | - else: | ||
| 199 | - self.has_bkg_scale = False | 195 | + raise RuntimeError("HDU index file \""+self.m_hdu_index+"[HDU_INDEX]\" for FITS data store \""+self.m_prodname+"\" not available. Check your master index file or run csiactdata to get a list of available storage names.") |
| 196 | + | ||
| 197 | + # Check for existence of "BKG_SCALE" in the observation index file if required | ||
| 198 | + if self.m_use_bkg_scale: | ||
| 199 | + if gammalib.is_fits(self.m_obs_index+"[OBS_INDEX]"): | ||
| 200 | + fits = gammalib.GFits(self.m_obs_index) | ||
| 201 | + if not fits["OBS_INDEX"].contains("BKG_SCALE"): | ||
| 202 | + self.m_use_bkg_scale = False | ||
| 203 | + fits.close() | ||
| 204 | + else: | ||
| 205 | + self.m_use_bkg_scale = False | ||
| 200 | 206 | ||
| 201 | # Create base data directory from hdu index file location | 207 | # Create base data directory from hdu index file location |
| 202 | self.subdir = os.path.dirname(self.m_hdu_index) | 208 | self.subdir = os.path.dirname(self.m_hdu_index) |
| @@ -481,7 +487,7 @@ class csiactobs(ctools.cscript): | @@ -481,7 +487,7 @@ class csiactobs(ctools.cscript): | ||
| 481 | 487 | ||
| 482 | # Handle background scale information if available | 488 | # Handle background scale information if available |
| 483 | bkg_scale = 1.0 | 489 | bkg_scale = 1.0 |
| 484 | - if self.has_bkg_scale: | 490 | + if self.m_use_bkg_scale: |
| 485 | obsindx = gammalib.GFits(self.m_obs_index+"[OBS_INDEX]"+obs_selection) | 491 | obsindx = gammalib.GFits(self.m_obs_index+"[OBS_INDEX]"+obs_selection) |
| 486 | bkg_scale = obsindx["OBS_INDEX"]["BKG_SCALE"][0] | 492 | bkg_scale = obsindx["OBS_INDEX"]["BKG_SCALE"][0] |
| 487 | obsindx.close() | 493 | obsindx.close() |
| @@ -527,7 +533,7 @@ class csiactobs(ctools.cscript): | @@ -527,7 +533,7 @@ class csiactobs(ctools.cscript): | ||
| 527 | self.log(" Point spread function: "+psffile+"\n") | 533 | self.log(" Point spread function: "+psffile+"\n") |
| 528 | self.log(" Energy dispersion: "+edispfile+"\n") | 534 | self.log(" Energy dispersion: "+edispfile+"\n") |
| 529 | self.log(" Background file: "+bkgfile+"\n") | 535 | self.log(" Background file: "+bkgfile+"\n") |
| 530 | - if self.has_bkg_scale: | 536 | + if self.m_use_bkg_scale: |
| 531 | self.log(" Background scale: "+str(bkg_scale)+"\n") | 537 | self.log(" Background scale: "+str(bkg_scale)+"\n") |
| 532 | self.log("\n") | 538 | self.log("\n") |
| 533 | 539 |
doc/source/reference_manual/csfindobs.rst
0 → 100644
| 1 | +.. _csfindobs: | ||
| 2 | + | ||
| 3 | +csfindobs | ||
| 4 | +========= | ||
| 5 | + | ||
| 6 | +Creates a list of observation IDs that fulfil certain criteria, e.g. pointing close to specific source | ||
| 7 | + | ||
| 8 | + | ||
| 9 | +Synopsis | ||
| 10 | +-------- | ||
| 11 | + | ||
| 12 | +This script can be used to create lists of observation IDs that share certain properties. For | ||
| 13 | +instance, all observations pointing around a specific sky position can be selected. In addition, | ||
| 14 | +the script is flexible to use expressions to further constrain criteria to select observations. | ||
| 15 | +The user can e.g. select all observations within a certain zenith angle range by specifying the hidden | ||
| 16 | +parameter expression: e.g. use ``csfindobs expression="ZEN_PNT<50&&ZEN_PNT>40"`` to constrain the zenith angle range | ||
| 17 | +from 40 to 50 degrees. The output of this script is an ASCII file containing observation IDs that fulfil criteria | ||
| 18 | +specified by the user. This file can be used as input for :doc:`csiactobs`. The script operates | ||
| 19 | +on an observation index file that needs to be provided along with IACT data. `See here <http://gamma-astro-data-formats.readthedocs.org/en/latest/index.html>`__ how IACT data | ||
| 20 | +should be structured. To investigate additional parameters which are available for selection, | ||
| 21 | +have a look `here <http://gamma-astro-data-formats.readthedocs.org/en/latest/data_storage/obs_index/index.html>`__. | ||
| 22 | + | ||
| 23 | + | ||
| 24 | +General parameters | ||
| 25 | +------------------ | ||
| 26 | + | ||
| 27 | +``datapath [string]`` | ||
| 28 | + Path were data is located | ||
| 29 | + | ||
| 30 | +``prodname [string]`` | ||
| 31 | + Name of FITS production (Run :doc:`csiactdata` to view your options) | ||
| 32 | + | ||
| 33 | +``ra [real]`` | ||
| 34 | + Right ascension of search region (invoke "INDEF" to avoid spatial selection) | ||
| 35 | + | ||
| 36 | +``dec [real]`` | ||
| 37 | + Declination of search region (invoke "INDEF" to avoid spatial selection) | ||
| 38 | + | ||
| 39 | +``rad [real]`` | ||
| 40 | + Radius of search region (deg) (invoke "INDEF" or negative values to avoid spatial selection) | ||
| 41 | + | ||
| 42 | +``outfile [file]`` | ||
| 43 | + Runlist outfile | ||
| 44 | + | ||
| 45 | +``(min_qual = 0) [integer]`` | ||
| 46 | + Minimum data quality (0=best, 1=medium, 2=bad) | ||
| 47 | + | ||
| 48 | +``(expression = NONE) [string]`` | ||
| 49 | + Additional expression | ||
| 50 | + | ||
| 51 | +``(master_index = master.json) [string]`` | ||
| 52 | + Name of master index file | ||
| 53 | + | ||
| 54 | + | ||
| 55 | + | ||
| 56 | +Standard parameters | ||
| 57 | +------------------- | ||
| 58 | + | ||
| 59 | +``(chatter = 2) [integer]`` | ||
| 60 | + Verbosity of the executable: | ||
| 61 | + chatter = 0: no information will be logged | ||
| 62 | + | ||
| 63 | + chatter = 1: only errors will be logged | ||
| 64 | + | ||
| 65 | + chatter = 2: errors and actions will be logged | ||
| 66 | + | ||
| 67 | + chatter = 3: report about the task execution | ||
| 68 | + | ||
| 69 | + chatter = 4: detailed report about the task execution | ||
| 70 | + | ||
| 71 | +``(clobber = yes) [boolean]`` | ||
| 72 | + Specifies whether an existing output counts cube should be overwritten. | ||
| 73 | + | ||
| 74 | +``(debug = no) [boolean]`` | ||
| 75 | + Enables debug mode. In debug mode the executable will dump any log file output to the console. | ||
| 76 | + | ||
| 77 | +``(mode = ql) [string]`` | ||
| 78 | + Mode of automatic parameters (default is "ql", i.e. "query and learn"). | ||
| 79 | + | ||
| 80 | +``(logfile = csfindobs.log) [filename]`` | ||
| 81 | + Log filename. | ||
| 82 | + | ||
| 83 | + | ||
| 84 | +Related tools or scripts | ||
| 85 | +------------------------ | ||
| 86 | + | ||
| 87 | +None | ||
| 0 | \ No newline at end of file | 88 | \ No newline at end of file |
doc/source/reference_manual/csiactdata.rst
0 → 100644
| 1 | +.. _csiactdata: | ||
| 2 | + | ||
| 3 | +csiactdata | ||
| 4 | +=========== | ||
| 5 | + | ||
| 6 | +Dumps information about the available FITS productions of IACT data on the screen | ||
| 7 | + | ||
| 8 | + | ||
| 9 | +Synopsis | ||
| 10 | +-------- | ||
| 11 | + | ||
| 12 | +This script inspects the local IACT FITS data storage and prints the names of | ||
| 13 | +available and valid data productions on the screen. The first parameter ``datapath`` | ||
| 14 | +is only queried if the environment variable $VHEFITS is not set. The path should be set | ||
| 15 | +to the location of the `master index file <http://gamma-astro-data-formats.readthedocs.org/en/latest/data_storage/super_index/index.html>`__. | ||
| 16 | +The output of this script can be used as input for further tools that use IACT data. | ||
| 17 | + | ||
| 18 | +General parameters | ||
| 19 | +------------------ | ||
| 20 | + | ||
| 21 | +``datapath [string]`` | ||
| 22 | + Path were data is located | ||
| 23 | + | ||
| 24 | +``(master_index = master.json) [string]`` | ||
| 25 | + Name of master index file | ||
| 26 | + | ||
| 27 | + | ||
| 28 | +Standard parameters | ||
| 29 | +------------------- | ||
| 30 | + | ||
| 31 | +``(chatter = 2) [integer]`` | ||
| 32 | + Verbosity of the executable: | ||
| 33 | + chatter = 0: no information will be logged | ||
| 34 | + | ||
| 35 | + chatter = 1: only errors will be logged | ||
| 36 | + | ||
| 37 | + chatter = 2: errors and actions will be logged | ||
| 38 | + | ||
| 39 | + chatter = 3: report about the task execution | ||
| 40 | + | ||
| 41 | + chatter = 4: detailed report about the task execution | ||
| 42 | + | ||
| 43 | +``(clobber = yes) [boolean]`` | ||
| 44 | + Specifies whether an existing output counts cube should be overwritten. | ||
| 45 | + | ||
| 46 | +``(debug = no) [boolean]`` | ||
| 47 | + Enables debug mode. In debug mode the executable will dump any log file output to the console. | ||
| 48 | + | ||
| 49 | +``(mode = ql) [string]`` | ||
| 50 | + Mode of automatic parameters (default is "ql", i.e. "query and learn"). | ||
| 51 | + | ||
| 52 | +``(logfile = csiactdata.log) [filename]`` | ||
| 53 | + Log filename. | ||
| 54 | + | ||
| 55 | + | ||
| 56 | +Related tools or scripts | ||
| 57 | +------------------------ | ||
| 58 | + | ||
| 59 | +None | ||
| 0 | \ No newline at end of file | 60 | \ No newline at end of file |
doc/source/reference_manual/csiactobs.rst
0 → 100644
| 1 | +.. _csiactobs: | ||
| 2 | + | ||
| 3 | +csiactobs | ||
| 4 | +=========== | ||
| 5 | + | ||
| 6 | +Creates an observation XML file for IACTs using a list of observation IDs as input | ||
| 7 | + | ||
| 8 | + | ||
| 9 | +Synopsis | ||
| 10 | +-------- | ||
| 11 | + | ||
| 12 | +This script creates an observation XML file from a list of observation IDs provided in an ASCII file. | ||
| 13 | +Such an ASCII file can for instance be created with :doc:`csfindobs`. The script operates on IACT data | ||
| 14 | +that needs to be present on the user machine. `See here <http://gamma-astro-data-formats.readthedocs.org/en/latest/index.html>`__ how IACT data | ||
| 15 | +should be structured. | ||
| 16 | + | ||
| 17 | +Besides creating an observation XML file, this script also stores a model XML file that contains background models for each | ||
| 18 | +observation. The number of free parameters is steered via the ``bkgpars`` parameter. | ||
| 19 | + | ||
| 20 | +The user can also specify as ``inmodel`` parameter an XML model file with e.g. sky models. The output model | ||
| 21 | +will then contain these models as well. | ||
| 22 | + | ||
| 23 | +The script further allows to specify hierarchies for the data formats and background models. The program will | ||
| 24 | +pick the first available option and fall back to the next one if the first choice is unavailable. | ||
| 25 | +If e.g. the IRF background model is not available, the script will assign the next background model in the hierarchy ``bkg_mod_hiera``. | ||
| 26 | +If the user wants to use a Gaussian background model as first choice, the parameter ``bkg_mod_hiera`` should be specified to "gauss". | ||
| 27 | +In addition, the user can specify start parameters for the background parameters that will be in the output XML model. | ||
| 28 | + | ||
| 29 | +If an observation that given in the input ASCII file is not available on the user machine, the script dumps a warning into the logfile | ||
| 30 | +and the obsservation is skipped. | ||
| 31 | + | ||
| 32 | + | ||
| 33 | +General parameters | ||
| 34 | +------------------ | ||
| 35 | + | ||
| 36 | +``datapath [string]`` | ||
| 37 | + Path were data is located | ||
| 38 | + | ||
| 39 | +``prodname [string]`` | ||
| 40 | + Name of FITS production (Run :doc:`csiactdata` to view your options) | ||
| 41 | + | ||
| 42 | +``runlistfile [file]`` | ||
| 43 | + Input list of observation IDs | ||
| 44 | + | ||
| 45 | +``bkgpars [integer]`` | ||
| 46 | + Number of free parameters for each background model | ||
| 47 | + | ||
| 48 | +``outobs [file]`` | ||
| 49 | + Observation XML outfile | ||
| 50 | + | ||
| 51 | +``outmodel [file]`` | ||
| 52 | + Output model XML file | ||
| 53 | + | ||
| 54 | +``(inmodel = NONE) [file]`` | ||
| 55 | + Input model XML file (optional) | ||
| 56 | + | ||
| 57 | +``(master_indx = master.json) [file]`` | ||
| 58 | + Name of master index file | ||
| 59 | + | ||
| 60 | +``(bkg_scale = yes) [boolean] | ||
| 61 | + Specifies whether the background scaling factor from the observation index file should be applied if available. | ||
| 62 | + | ||
| 63 | +``(ev_hiera = events) [string]`` | ||
| 64 | + Hierarchy of event file formats | ||
| 65 | + | ||
| 66 | +``(aeff_hiera = aeff_2d) [string]`` | ||
| 67 | + Hierarchy of effective area formats | ||
| 68 | + | ||
| 69 | +``(psf_hiera = psf_king|psf_3gauss) [string]`` | ||
| 70 | + Hierarchy of psf formats | ||
| 71 | + | ||
| 72 | +``(edisp_hiera = edisp_2d) [string]`` | ||
| 73 | + Hierarchy of energy disperson formats | ||
| 74 | + | ||
| 75 | +``(bkg_hiera = bkg_3d) [string]`` | ||
| 76 | + Hierarchy of background formats. | ||
| 77 | + | ||
| 78 | +``(bkg_mod_hiera = irf|aeff|gauss) [string]`` | ||
| 79 | + Hierarchy of background models | ||
| 80 | + | ||
| 81 | +``(bkg_gauss_norm = 1e-8) [real]`` | ||
| 82 | + Input normalisation for Gaussian background | ||
| 83 | + | ||
| 84 | +``(bkg_gauss_index = -2.0) [real]`` | ||
| 85 | + Input spectral index for Gaussian background | ||
| 86 | + | ||
| 87 | +``(bkg_gauss_sigma = 1e-8) [real]`` | ||
| 88 | + Input sigma for Gaussian background | ||
| 89 | + | ||
| 90 | +``(bkg_aeff_norm = 1e-14) [real]`` | ||
| 91 | + Input normalisation for effective area background | ||
| 92 | + | ||
| 93 | +``(bkg_aeff_index = -2.0) [real]`` | ||
| 94 | + Input spectral index for effective area background | ||
| 95 | + | ||
| 96 | +``(bkg_range_factor = 100.0) [real]`` | ||
| 97 | + Factor to determine range of background normalisation | ||
| 98 | + | ||
| 99 | + | ||
| 100 | +Standard parameters | ||
| 101 | +------------------- | ||
| 102 | + | ||
| 103 | +``(chatter = 2) [integer]`` | ||
| 104 | + Verbosity of the executable: | ||
| 105 | + chatter = 0: no information will be logged | ||
| 106 | + | ||
| 107 | + chatter = 1: only errors will be logged | ||
| 108 | + | ||
| 109 | + chatter = 2: errors and actions will be logged | ||
| 110 | + | ||
| 111 | + chatter = 3: report about the task execution | ||
| 112 | + | ||
| 113 | + chatter = 4: detailed report about the task execution | ||
| 114 | + | ||
| 115 | +``(clobber = yes) [boolean]`` | ||
| 116 | + Specifies whether an existing output counts cube should be overwritten. | ||
| 117 | + | ||
| 118 | +``(debug = no) [boolean]`` | ||
| 119 | + Enables debug mode. In debug mode the executable will dump any log file output to the console. | ||
| 120 | + | ||
| 121 | +``(mode = ql) [string]`` | ||
| 122 | + Mode of automatic parameters (default is "ql", i.e. "query and learn"). | ||
| 123 | + | ||
| 124 | +``(logfile = csiactobs.log) [filename]`` | ||
| 125 | + Log filename. | ||
| 126 | + | ||
| 127 | + | ||
| 128 | +Related tools or scripts | ||
| 129 | +------------------------ | ||
| 130 | + | ||
| 131 | +None | ||
| 0 | \ No newline at end of file | 132 | \ No newline at end of file |
doc/source/reference_manual/index.rst
| @@ -50,3 +50,6 @@ cscripts | @@ -50,3 +50,6 @@ cscripts | ||
| 50 | csmodelmerge --- Merges several model containers into one file <csmodelmerge> | 50 | csmodelmerge --- Merges several model containers into one file <csmodelmerge> |
| 51 | cstsmapmerge --- Merges slices from ts map computations <cstsmapmerge> | 51 | cstsmapmerge --- Merges slices from ts map computations <cstsmapmerge> |
| 52 | csobs2caldb --- Creates a caldb entry from an input observation <csobs2caldb> | 52 | csobs2caldb --- Creates a caldb entry from an input observation <csobs2caldb> |
| 53 | + csiactdata --- Dumps information about the IACT data available on the user machine <csiactdata> | ||
| 54 | + csiactobs --- Creates an observation XML file for IACTs using a list of observation IDs as input <csiactobs> | ||
| 55 | + csfindobs --- Creates a list of observation IDs <csfindobs> |
test/iactdata/runlist.txt deleted
| 1 | -0 |
test/test_cscripts.sh.in
| @@ -119,6 +119,29 @@ $ECHO " ok" | @@ -119,6 +119,29 @@ $ECHO " ok" | ||
| 119 | 119 | ||
| 120 | 120 | ||
| 121 | # | 121 | # |
| 122 | +# Test csfindobs | ||
| 123 | +# ============== | ||
| 124 | +$ECHO -n "Test csfindobs: " | ||
| 125 | +chmod +x @abs_top_srcdir@/cscripts/csfindobs.py | ||
| 126 | +@abs_top_srcdir@/cscripts/csfindobs.py \ | ||
| 127 | + datapath="iactdata/" \ | ||
| 128 | + prodname="unit-test" \ | ||
| 129 | + ra="83.6331" \ | ||
| 130 | + dec="22.01" \ | ||
| 131 | + rad="1.0" \ | ||
| 132 | + outfile="iactdata/runlist.txt" > csfindobs.log | ||
| 133 | +$ECHO -n "." | ||
| 134 | +if [ -s "iactdata/runlist.txt" ] | ||
| 135 | +then | ||
| 136 | + $ECHO -n "." | ||
| 137 | +else | ||
| 138 | + $ECHO " iactdata/runlist.txt file is not valid" | ||
| 139 | + exit 1 | ||
| 140 | +fi | ||
| 141 | +$ECHO " ok" | ||
| 142 | + | ||
| 143 | + | ||
| 144 | +# | ||
| 122 | # Test csiactobs | 145 | # Test csiactobs |
| 123 | # ============== | 146 | # ============== |
| 124 | $ECHO -n "Test csiactobs: " | 147 | $ECHO -n "Test csiactobs: " |