Mathieu Servillat / ctools

Commit 920b233bfc92ee7fb84b337a771f5385c6f8d7f3

Authored by Jürgen Knödlseder
2 parents 6c97c6e0 13d11843

Merge branch 'integration' into devel 

# Conflicts:
#	cscripts/cslightcrv.py
Inline Side-by-side
Showing 4 changed files with 169 additions and 126 deletions
cscripts/csfindobs.py
  View file @920b233
... ... @@ -39,10 +39,15 @@ class csfindobs(ctools.cscript):
39 39 Constructor.
40 40 """
41 41 # Set name
42   - self._name = "csfindobs"
43   - self._version = "1.1.0"
44   - self._verbose = False
45   - self._datapath = os.getenv("VHEFITS","")
  42 + self._name = "csfindobs"
  43 + self._version = "1.1.0"
  44 + self._datapath = os.getenv("VHEFITS","")
  45 + self._prodname = ""
  46 + self._select_radec = True
  47 + self._radius = 0.0
  48 + self._ra = 0.0
  49 + self._dec = 0.0
  50 + self._obs_index = ""
46 51  
47 52 # Initialise application by calling the appropriate class
48 53 # constructor.
... ... @@ -68,7 +73,7 @@ class csfindobs(ctools.cscript):
68 73 self._prodname = self["prodname"].string()
69 74  
70 75 # Master index file name
71   - self._master_indx = self["master_indx"].string()
  76 + master_indx = self["master_indx"].string()
72 77  
73 78 # Intiialise flag if spatial selection is required
74 79 self._select_radec = True
... ... @@ -93,32 +98,29 @@ class csfindobs(ctools.cscript):
93 98 if self._radius <= 0.0:
94 99 self._select_radec = False
95 100  
96   - # Read other parameters
97   - self._quality = self["min_qual"].integer()
98   - self._expression = self["expression"].string()
99   -
100   - # Check for validity of expression
101   - if self._expression == "NONE" or self._expression == "INDEF":
102   - self._expression = ""
103   -
104   - # Read outfile
105   - self._outfile = self["outfile"].filename()
  101 + # Query other parameters
  102 + self["min_qual"].integer()
  103 + self["expression"].string()
  104 +
  105 + # Read ahead output parameters
  106 + if self._read_ahead():
  107 + self["outfile"].filename()
106 108  
107 109 # Open master index file and look for prodname
108   - master_file = os.path.join(self._datapath, self._master_indx)
  110 + master_file = os.path.join(self._datapath, master_indx)
109 111 if not os.path.isfile(master_file):
110   - raise RuntimeError("FITS data store not available. "+
111   - "No master index file found at \""+
112   - master_file+"\". Make sure the file "+
113   - "is copied from the server and your "+
114   - "datapath is set correctly.")
  112 + raise RuntimeError('FITS data store not available. '+
  113 + 'No master index file found at "'+
  114 + master_file+'". Make sure the file '+
  115 + 'is copied from the server and your '+
  116 + 'datapath is set correctly.')
115 117  
116 118 # Open and load JSON file
117 119 json_data = open(master_file).read()
118 120 data = json.loads(json_data)
119 121 if not "datasets" in data:
120   - raise RuntimeError("Key \"datasets\" not available in "+
121   - "master index file.")
  122 + raise RuntimeError('Key "datasets" not available in '+
  123 + 'master index file.')
122 124  
123 125 # Get configurations
124 126 configs = data["datasets"]
... ... @@ -135,27 +137,23 @@ class csfindobs(ctools.cscript):
135 137  
136 138 # Check HDUs
137 139 if self._obs_index == "":
138   - raise RuntimeError("FITS data store \""+self._prodname+
139   - "\" not available. Run csiactdata to get "+
140   - "a list of available storage names")
  140 + raise RuntimeError('FITS data store "'+self._prodname+
  141 + '" not available. Run csiactdata to get '+
  142 + 'a list of available storage names')
141 143 filename = gammalib.GFilename(self._obs_index+"[OBS_INDEX]")
142 144 if not filename.is_fits():
143   - raise RuntimeError("Observation index file \""+
  145 + raise RuntimeError('Observation index file "'+
144 146 self._obs_index+
145   - "[OBS_INDEX]\" for FITS data store \""+
  147 + '[OBS_INDEX]" for FITS data store "'+
146 148 self._prodname+
147   - "\" not available. "+
148   - "Check your master index file or run "+
149   - "csiactdata to get a list of available "+
150   - "storage names.")
  149 + '" not available. Check your master index '+
  150 + 'file or run csiactdata to get a list of '+
  151 + 'available storage names.')
151 152  
152   - # Set other members
153   - self._debug = False # Debugging in client tools
154   - self._clobber = self["clobber"].boolean()
155   -
156 153 # Return
157 154 return
158 155  
  156 +
159 157 # Public methods
160 158 def run(self):
161 159 """
... ... @@ -189,11 +187,13 @@ class csfindobs(ctools.cscript):
189 187 expr += "&&"
190 188  
191 189 # Add quality expression
192   - expr += "QUALITY<="+str(self._quality)
  190 + expr += "QUALITY<="+str(self["min_qual"].integer())
193 191  
194   - # Add user expression
195   - if self._expression == "NONE" or len(self._expression) > 0:
196   - expr += "&&"+self._expression
  192 + # Add user expression
  193 + expression = self["expression"].string()
  194 + if (expression != "NONE" and expression != "INDEF" and
  195 + len(expression) > 0):
  196 + expr += "&&" + expression
197 197  
198 198 # Initialise filename
199 199 openfile = self._obs_index+"[OBS_INDEX]["+expr+"]"
... ... @@ -238,27 +238,46 @@ class csfindobs(ctools.cscript):
238 238 # Open logfile
239 239 self.logFileOpen()
240 240  
  241 + # Read ahead output parameters
  242 + self._read_ahead(True)
  243 +
241 244 # Run the script
242 245 self.run()
243 246  
244 247 # Save residual map
245   - self.save(self._outfile)
  248 + self.save()
246 249  
247 250 # Return
248 251 return
249 252  
250   - def save(self, outfile):
  253 + def save(self):
251 254 """
252   - Save.
  255 + Save runlist.
253 256 """
  257 + # Write header
  258 + if self._logTerse():
  259 + self._log('\n')
  260 + self._log.header1('Save runlist')
  261 +
  262 + # Get filename
  263 + outfile = self["outfile"].filename()
  264 +
254 265 # Check for clobber
255 266 if outfile.exists() and not self._clobber():
256   - self._log("File \""+outfile+"\" already exists. ")
257   - self._log("Set clobber=yes to overwrite file.")
258   - self._log("\n")
  267 + if self._logTerse():
  268 + self._log('File "+outfile+" already exists. ')
  269 + self._log('Set clobber=yes to overwrite file.')
  270 + self._log('\n')
259 271 else:
  272 +
  273 + # Log file name
  274 + if self._logTerse():
  275 + self._log(gammalib.parformat("Runlist file"))
  276 + self._log(outfile.url())
  277 + self._log("\n")
  278 +
260 279 # Write runs to file
261   - f = open(repr(outfile),"w")
  280 + f = open(outfile.url(),"w")
262 281 for run in self._runs:
263 282 f.write(str(run)+" \n")
264 283 f.close()
... ... @@ -266,9 +285,12 @@ class csfindobs(ctools.cscript):
266 285 # Return
267 286 return
268 287  
269   - def obs_ids(self):
  288 + def runs(self):
270 289 """
271   - Return OBS IDs
  290 + Return run list.
  291 +
  292 + Returns:
  293 + A list of runs.
272 294 """
273 295  
274 296 # Return
... ...
cscripts/csiactcopy.py
  View file @920b233
1 1 #!/usr/bin/env python
2 2 # ==========================================================================
3   -# Download IACT data from remote machine
  3 +# Copies IACT data from remote machine
4 4 #
5 5 # Copyright (C) 2016 Michael Mayer
6 6 #
... ... @@ -30,8 +30,10 @@ import shutil
30 30 # ================ #
31 31 class csiactcopy(ctools.cscript):
32 32 """
33   - This script copies IACT data from one location to another.
34   - It can take a list of observation IDs to allow the download specific
  33 + Copies IACT data from remote machine.
  34 +
  35 + This script copies IACT data from one location to another. It can
  36 + take a list of observation IDs to allow the download specific
35 37 observations. Index files get merged and updated accordingly.
36 38 """
37 39  
... ... @@ -41,9 +43,15 @@ class csiactcopy(ctools.cscript):
41 43 Constructor.
42 44 """
43 45 # Set name
44   - self._name = "csiactcopy"
45   - self._version = "1.1.0"
46   - self._datapath = os.getenv("VHEFITS","")
  46 + self._name = "csiactcopy"
  47 + self._version = "1.1.0"
  48 + self._datapath = os.getenv("VHEFITS","")
  49 + self._remote_master = gammalib.GFilename()
  50 + self._remote_base = ""
  51 + self._prodname = ""
  52 + self._outpath = ""
  53 + self._runlist = gammalib.GFilename()
  54 + self._runs = []
47 55  
48 56 # Initialise application by calling the appropriate class
49 57 # constructor.
... ... @@ -62,10 +70,10 @@ class csiactcopy(ctools.cscript):
62 70 # Get Parameters
63 71 self._remote_master = self["remote_master"].filename()
64 72 if not self._remote_master.exists():
65   - raise RuntimeError("*** ERROR: FITS data not available. No "
66   - "master index file found in \""+
67   - self._remote_master+"\". Make sure remote "
68   - "file system is properly mounted.")
  73 + raise RuntimeError('*** ERROR: FITS data not available. No '
  74 + 'master index file found in "'+
  75 + self._remote_master+'". Make sure remote '
  76 + 'file system is properly mounted.')
69 77  
70 78 # Get parameters
71 79 self._prodname = self["prodname"].string()
... ... @@ -80,9 +88,9 @@ class csiactcopy(ctools.cscript):
80 88 # Return
81 89 return
82 90  
83   - def copy(self, source, clobber):
  91 + def _copy(self, source, clobber):
84 92 """
85   - copy file to _outpath
  93 + Copy file to outpath
86 94 """
87 95 # Get file destination
88 96 destination = os.path.join(self._outpath,
... ... @@ -143,9 +151,8 @@ class csiactcopy(ctools.cscript):
143 151  
144 152 # Return
145 153 return filesize
146   -
147   -
148   - def merge(self, localfits, remotefits, hduname, clobber):
  154 +
  155 + def _merge(self, localfits, remotefits, hduname, clobber):
149 156 """
150 157 merge remote and local fits file
151 158 If local fits file not present, a new one is created
... ... @@ -180,7 +187,7 @@ class csiactcopy(ctools.cscript):
180 187  
181 188 # find local obs_id
182 189 local_obs = []
183   - lobs_col = local_hdu["OBS_ID"]
  190 + lobs_col = local_hdu["OBS_ID"]
184 191 for obs_id in lobs_col:
185 192 local_obs.append(obs_id)
186 193  
... ... @@ -198,12 +205,12 @@ class csiactcopy(ctools.cscript):
198 205 remotefile = remotefits + selection
199 206  
200 207 # open remote fits file
201   - remote = gammalib.GFits(str(remotefile))
  208 + remote = gammalib.GFits(str(remotefile))
202 209 remote_hdu = remote[hduname]
203 210  
204 211 # Get remote obs_id
205 212 remote_obs = []
206   - robs_col = remote_hdu["OBS_ID"]
  213 + robs_col = remote_hdu["OBS_ID"]
207 214 for obs_id in robs_col:
208 215 remote_obs.append(obs_id)
209 216  
... ... @@ -346,27 +353,26 @@ class csiactcopy(ctools.cscript):
346 353  
347 354 # Raise exception if file not valid
348 355 else:
349   - raise RuntimeError("*** ERROR: Runlist file \""+
350   - self._runlist+"\" not available.")
  356 + raise RuntimeError('*** ERROR: Runlist file "'+
  357 + self._runlist+'" not available.')
351 358  
352 359 # Check for availability of remote master file
353 360 if not self._remote_master.exists():
354   - raise RuntimeError("*** ERROR: Remote master file \""+
355   - self._remote_master+"\" does not exist.")
  361 + raise RuntimeError('*** ERROR: Remote master file "'+
  362 + self._remote_master+'" does not exist.')
356 363  
357 364 # Retrieve json data from remote master
358 365 json_data = open(self._remote_master.url()).read()
359 366 data = json.loads(json_data)
360 367 if not "datasets" in data:
361   - raise RuntimeError("*** ERROR: Key \"datasets\" not available "
362   - "in remote master index file \""+
363   - self._remote_master+"\".")
  368 + raise RuntimeError('*** ERROR: Key "datasets" not available '
  369 + 'in remote master index file "'+
  370 + self._remote_master+'".')
364 371  
365 372 # Get array of configurations
366 373 configs = data["datasets"]
367 374  
368 375 # Get remote paths
369   - #self._remote_base = os.path.dirname(self._remote_master.url())
370 376 self._remote_base = self._remote_master.path()
371 377  
372 378 # Initialise flag if prod has been found
... ... @@ -399,13 +405,13 @@ class csiactcopy(ctools.cscript):
399 405  
400 406 # Log information
401 407 if self._logTerse():
402   - self._log.header3("Remote config \""+self._prodname+"\"")
403   - self._log.parformat("HDU index")
  408 + self._log.header3('Remote config "'+self._prodname+'"')
  409 + self._log.parformat('HDU index')
404 410 self._log(str(remote_hdu))
405   - self._log("\n")
406   - self._log.parformat("Observation index")
  411 + self._log('\n')
  412 + self._log.parformat('Observation index')
407 413 self._log(str(remote_obs))
408   - self._log("\n")
  414 + self._log('\n')
409 415  
410 416 # Open remote HDU index file
411 417 fits = gammalib.GFits(str(remote_hdu))
... ... @@ -427,9 +433,9 @@ class csiactcopy(ctools.cscript):
427 433 for run in self._runs:
428 434 if not run in remote_ids:
429 435 if self._logNormal():
430   - self._log("Skip observation "+str(run)+": ")
431   - self._log("ID not available remotely")
432   - self._log("\n")
  436 + self._log('Skip observation '+str(run)+': ')
  437 + self._log('ID not available remotely')
  438 + self._log('\n')
433 439  
434 440 # Loop over remote HDU index file
435 441 for row in range(table.nrows()):
... ... @@ -450,7 +456,8 @@ class csiactcopy(ctools.cscript):
450 456 if obs_id in self._runs:
451 457  
452 458 # Get filename
453   - fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name)
  459 + fname = os.path.join(os.path.dirname(remote_hdu),
  460 + file_dir, file_name)
454 461  
455 462 # Add file
456 463 oldlen = len(files)
... ... @@ -464,7 +471,8 @@ class csiactcopy(ctools.cscript):
464 471 # Otherwise add every file
465 472 else:
466 473 # Get filename
467   - fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name)
  474 + fname = os.path.join(os.path.dirname(remote_hdu),
  475 + file_dir, file_name)
468 476  
469 477 # Add file
470 478 oldlen = len(files)
... ... @@ -499,13 +507,14 @@ class csiactcopy(ctools.cscript):
499 507 else:
500 508 if self._logExplicit():
501 509 self._log("\n")
502   - self._log.header3("Skipping config \""+str(config["name"])+"\"")
  510 + self._log.header3('Skipping config "'+
  511 + str(config["name"])+'"')
503 512 self._log("\n")
504 513  
505 514 # Raise Exception if prodname was not found
506 515 if not has_prod:
507   - msg = "*** ERROR: FITS production \""+self._prodname+\
508   - "\" not available. Available productions are:\n"
  516 + msg = '*** ERROR: FITS production "'+self._prodname+\
  517 + '" not available. Available productions are:\n'
509 518 for config in configs:
510 519 msg += " - "+config["name"]+"\n"
511 520 raise RuntimeError(msg)
... ... @@ -545,7 +554,7 @@ class csiactcopy(ctools.cscript):
545 554 last_fraction += fraction_increment
546 555  
547 556 # Copy file
548   - filesize = self.copy(filename, self._clobber())
  557 + filesize = self._copy(filename, self._clobber())
549 558  
550 559 if filesize > 0.0:
551 560 total_size += filesize
... ... @@ -564,8 +573,12 @@ class csiactcopy(ctools.cscript):
564 573 self._log.header1("Updating index files")
565 574  
566 575 # Build local index file names
567   - local_hdu = os.path.join(self._outpath, os.path.relpath(remote_hdu, self._remote_base))
568   - local_obs = os.path.join(self._outpath, os.path.relpath(remote_obs, self._remote_base))
  576 + local_hdu = os.path.join(self._outpath,
  577 + os.path.relpath(remote_hdu,
  578 + self._remote_base))
  579 + local_obs = os.path.join(self._outpath,
  580 + os.path.relpath(remote_obs,
  581 + self._remote_base))
569 582  
570 583 # If we have a runlist then merge index file
571 584 if len(self._runs):
... ... @@ -576,19 +589,19 @@ class csiactcopy(ctools.cscript):
576 589 self._log.header3("HDU index")
577 590  
578 591 # Merge remote index files with local files
579   - self.merge(local_hdu, remote_hdu, "HDU_INDEX", self._clobber())
  592 + self._merge(local_hdu, remote_hdu, "HDU_INDEX", self._clobber())
580 593  
581 594 if self._logTerse():
582 595 self._log("\n")
583 596 self._log.header3("OBS index")
584 597  
585 598 # Merge remote index files with local files
586   - self.merge(local_obs, remote_obs, "OBS_INDEX", self._clobber())
  599 + self._merge(local_obs, remote_obs, "OBS_INDEX", self._clobber())
587 600  
588 601 else:
589 602 # If all files were copied, just copy index files too
590   - self.copy(remote_hdu, self._clobber())
591   - self.copy(remote_obs, self._clobber())
  603 + self._copy(remote_hdu, self._clobber())
  604 + self._copy(remote_obs, self._clobber())
592 605  
593 606 if self._logTerse():
594 607 self._log("\n")
... ... @@ -599,7 +612,7 @@ class csiactcopy(ctools.cscript):
599 612  
600 613 # If no local master is found, copy master over first
601 614 if not os.path.isfile(localmaster):
602   - self.copy(self._remote_master.url(), self._clobber())
  615 + self._copy(self._remote_master.url(), self._clobber())
603 616  
604 617 # Load local master
605 618 json_data = open(localmaster).read()
... ... @@ -624,15 +637,15 @@ class csiactcopy(ctools.cscript):
624 637 if not (gammalib.GFilename(str(hdu)).is_fits() and
625 638 gammalib.GFilename(str(obs)).is_fits()):
626 639 if self._logTerse():
627   - self._log("Removing \""+str(config["name"])+"\" ")
628   - self._log("(not available)")
629   - self._log("\n")
  640 + self._log('Removing "'+str(config["name"])+'" ')
  641 + self._log('(not available)')
  642 + self._log('\n')
630 643 else:
631 644 # Append config if valid
632 645 newconfigs.append(config)
633 646 if self._logTerse():
634   - self._log("Keeping \""+str(config["name"])+"\"")
635   - self._log("\n")
  647 + self._log('Keeping "'+str(config["name"])+'"')
  648 + self._log('\n')
636 649  
637 650 # Signals that downloaded config is available
638 651 if config["name"] == self._prodname:
... ... @@ -646,8 +659,8 @@ class csiactcopy(ctools.cscript):
646 659 newdict["obsindx"] = os.path.relpath(local_obs, self._outpath)
647 660 newconfigs.append(newdict)
648 661 if self._logTerse():
649   - self._log("Adding \""+str(newdict["name"])+"\"")
650   - self._log("\n")
  662 + self._log('Adding "'+str(newdict["name"])+'"')
  663 + self._log('\n')
651 664  
652 665 # Write new json master file
653 666 f = open(localmaster, "w")
... ...
cscripts/csiactdata.py
  View file @920b233
1 1 #!/usr/bin/env python
2 2 # ==========================================================================
3   -# Inspection of IACT data storage
  3 +# Inspect an IACT data storage
4 4 #
5 5 # Copyright (C) 2015-2016 Michael Mayer
6 6 #
... ... @@ -30,6 +30,8 @@ import json
30 30 # ================ #
31 31 class csiactdata(ctools.cscript):
32 32 """
  33 + Inspect an IACT data storage.
  34 +
33 35 This script inspects the available FITS data storage on the user machine
34 36 and writes information about the available FITS configurations into a log
35 37 file or prints it on the screen. These information can be used as input
... ... @@ -42,10 +44,12 @@ class csiactdata(ctools.cscript):
42 44 Constructor.
43 45 """
44 46 # Set name
45   - self._name = "csiactdata"
46   - self._version = "1.1.0"
47   - self._datapath = os.getenv("VHEFITS","")
48   - self._prodnames = []
  47 + self._name = "csiactdata"
  48 + self._version = "1.1.0"
  49 + self._datapath = os.getenv("VHEFITS","")
  50 + self._prodnames = []
  51 + self._master_indx = ""
  52 + self._master_file = ""
49 53  
50 54 # Initialise application by calling the appropriate class
51 55 # constructor.
... ... @@ -74,7 +78,10 @@ class csiactdata(ctools.cscript):
74 78  
75 79 # Check for presence of master index file
76 80 if not os.path.isfile(self._master_file):
77   - raise RuntimeError("Master index file \""+self._master_file+"\" not found. Use hidden parameter \"master_indx\" to specifiy a different filename.")
  81 + raise RuntimeError('Master index file "'+self._master_file+
  82 + '" not found. Use hidden parameter '+
  83 + '"master_indx" to specifiy a different '+
  84 + 'filename.')
78 85  
79 86 # Return
80 87 return
... ...
cscripts/cslightcrv.py
  View file @920b233
... ... @@ -74,6 +74,7 @@ class cslightcrv(ctools.cscript):
74 74 self._srcname = ""
75 75 self._tbins = gammalib.GGti()
76 76 self._stacked = False
  77 + self._fits = gammalib.GFits()
77 78  
78 79 # Initialise observation container from constructor arguments.
79 80 self._obs, argv = self._set_input_obs(argv)
... ... @@ -397,20 +398,20 @@ class cslightcrv(ctools.cscript):
397 398 self._log.header3("Binning events")
398 399  
399 400 # Bin events
400   - bin = ctools.ctbin(obs)
401   - bin["usepnt"] = False
402   - bin["ebinalg"] = "LOG"
403   - bin["xref"] = self["xref"].real()
404   - bin["yref"] = self["yref"].real()
405   - bin["binsz"] = self["binsz"].real()
406   - bin["nxpix"] = self["nxpix"].integer()
407   - bin["nypix"] = self["nypix"].integer()
408   - bin["enumbins"] = self["enumbins"].integer()
409   - bin["emin"] = self["emin"].real()
410   - bin["emax"] = self["emax"].real()
411   - bin["coordsys"] = self["coordsys"].string()
412   - bin["proj"] = self["proj"].string()
413   - bin.run()
  401 + cntcube = ctools.ctbin(obs)
  402 + cntcube["usepnt"] = False
  403 + cntcube["ebinalg"] = "LOG"
  404 + cntcube["xref"] = self["xref"].real()
  405 + cntcube["yref"] = self["yref"].real()
  406 + cntcube["binsz"] = self["binsz"].real()
  407 + cntcube["nxpix"] = self["nxpix"].integer()
  408 + cntcube["nypix"] = self["nypix"].integer()
  409 + cntcube["enumbins"] = self["enumbins"].integer()
  410 + cntcube["emin"] = self["emin"].real()
  411 + cntcube["emax"] = self["emax"].real()
  412 + cntcube["coordsys"] = self["coordsys"].string()
  413 + cntcube["proj"] = self["proj"].string()
  414 + cntcube.run()
414 415  
415 416 # Header
416 417 if self._logExplicit():
... ... @@ -510,7 +511,7 @@ class cslightcrv(ctools.cscript):
510 511 bkgcube.run()
511 512  
512 513 # Retrieve a new oberservation container
513   - new_obs = bin.obs().copy()
  514 + new_obs = cntcube.obs().copy()
514 515  
515 516 # Get new models
516 517 models = bkgcube.models()
... ... @@ -606,8 +607,8 @@ class cslightcrv(ctools.cscript):
606 607 select = ctools.ctselect(self._obs)
607 608 select["emin"] = self["emin"].real()
608 609 select["emax"] = self["emax"].real()
609   - select["tmin"] = tmin.convert(select._time_reference())
610   - select["tmax"] = tmax.convert(select._time_reference())
  610 + select["tmin"] = tmin.convert(self._time_reference())
  611 + select["tmax"] = tmax.convert(self._time_reference())
611 612 select["rad"] = "UNDEFINED"
612 613 select["ra"] = "UNDEFINED"
613 614 select["dec"] = "UNDEFINED"
... ...