diff --git a/cscripts/csfindobs.py b/cscripts/csfindobs.py index 3acb0f0..8b02761 100755 --- a/cscripts/csfindobs.py +++ b/cscripts/csfindobs.py @@ -39,10 +39,15 @@ class csfindobs(ctools.cscript): Constructor. """ # Set name - self._name = "csfindobs" - self._version = "1.1.0" - self._verbose = False - self._datapath = os.getenv("VHEFITS","") + self._name = "csfindobs" + self._version = "1.1.0" + self._datapath = os.getenv("VHEFITS","") + self._prodname = "" + self._select_radec = True + self._radius = 0.0 + self._ra = 0.0 + self._dec = 0.0 + self._obs_index = "" # Initialise application by calling the appropriate class # constructor. @@ -68,7 +73,7 @@ class csfindobs(ctools.cscript): self._prodname = self["prodname"].string() # Master index file name - self._master_indx = self["master_indx"].string() + master_indx = self["master_indx"].string() # Intiialise flag if spatial selection is required self._select_radec = True @@ -93,32 +98,29 @@ class csfindobs(ctools.cscript): if self._radius <= 0.0: self._select_radec = False - # Read other parameters - self._quality = self["min_qual"].integer() - self._expression = self["expression"].string() - - # Check for validity of expression - if self._expression == "NONE" or self._expression == "INDEF": - self._expression = "" - - # Read outfile - self._outfile = self["outfile"].filename() + # Query other parameters + self["min_qual"].integer() + self["expression"].string() + + # Read ahead output parameters + if self._read_ahead(): + self["outfile"].filename() # Open master index file and look for prodname - master_file = os.path.join(self._datapath, self._master_indx) + master_file = os.path.join(self._datapath, master_indx) if not os.path.isfile(master_file): - raise RuntimeError("FITS data store not available. "+ - "No master index file found at \""+ - master_file+"\". Make sure the file "+ - "is copied from the server and your "+ - "datapath is set correctly.") + raise RuntimeError('FITS data store not available. '+ + 'No master index file found at "'+ + master_file+'". Make sure the file '+ + 'is copied from the server and your '+ + 'datapath is set correctly.') # Open and load JSON file json_data = open(master_file).read() data = json.loads(json_data) if not "datasets" in data: - raise RuntimeError("Key \"datasets\" not available in "+ - "master index file.") + raise RuntimeError('Key "datasets" not available in '+ + 'master index file.') # Get configurations configs = data["datasets"] @@ -135,27 +137,23 @@ class csfindobs(ctools.cscript): # Check HDUs if self._obs_index == "": - raise RuntimeError("FITS data store \""+self._prodname+ - "\" not available. Run csiactdata to get "+ - "a list of available storage names") + raise RuntimeError('FITS data store "'+self._prodname+ + '" not available. Run csiactdata to get '+ + 'a list of available storage names') filename = gammalib.GFilename(self._obs_index+"[OBS_INDEX]") if not filename.is_fits(): - raise RuntimeError("Observation index file \""+ + raise RuntimeError('Observation index file "'+ self._obs_index+ - "[OBS_INDEX]\" for FITS data store \""+ + '[OBS_INDEX]" for FITS data store "'+ self._prodname+ - "\" not available. "+ - "Check your master index file or run "+ - "csiactdata to get a list of available "+ - "storage names.") + '" not available. Check your master index '+ + 'file or run csiactdata to get a list of '+ + 'available storage names.') - # Set other members - self._debug = False # Debugging in client tools - self._clobber = self["clobber"].boolean() - # Return return + # Public methods def run(self): """ @@ -189,11 +187,13 @@ class csfindobs(ctools.cscript): expr += "&&" # Add quality expression - expr += "QUALITY<="+str(self._quality) + expr += "QUALITY<="+str(self["min_qual"].integer()) - # Add user expression - if self._expression == "NONE" or len(self._expression) > 0: - expr += "&&"+self._expression + # Add user expression + expression = self["expression"].string() + if (expression != "NONE" and expression != "INDEF" and + len(expression) > 0): + expr += "&&" + expression # Initialise filename openfile = self._obs_index+"[OBS_INDEX]["+expr+"]" @@ -238,27 +238,46 @@ class csfindobs(ctools.cscript): # Open logfile self.logFileOpen() + # Read ahead output parameters + self._read_ahead(True) + # Run the script self.run() # Save residual map - self.save(self._outfile) + self.save() # Return return - def save(self, outfile): + def save(self): """ - Save. + Save runlist. """ + # Write header + if self._logTerse(): + self._log('\n') + self._log.header1('Save runlist') + + # Get filename + outfile = self["outfile"].filename() + # Check for clobber if outfile.exists() and not self._clobber(): - self._log("File \""+outfile+"\" already exists. ") - self._log("Set clobber=yes to overwrite file.") - self._log("\n") + if self._logTerse(): + self._log('File "+outfile+" already exists. ') + self._log('Set clobber=yes to overwrite file.') + self._log('\n') else: + + # Log file name + if self._logTerse(): + self._log(gammalib.parformat("Runlist file")) + self._log(outfile.url()) + self._log("\n") + # Write runs to file - f = open(repr(outfile),"w") + f = open(outfile.url(),"w") for run in self._runs: f.write(str(run)+" \n") f.close() @@ -266,9 +285,12 @@ class csfindobs(ctools.cscript): # Return return - def obs_ids(self): + def runs(self): """ - Return OBS IDs + Return run list. + + Returns: + A list of runs. """ # Return diff --git a/cscripts/csiactcopy.py b/cscripts/csiactcopy.py index 35ddc9d..bee4c50 100755 --- a/cscripts/csiactcopy.py +++ b/cscripts/csiactcopy.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # ========================================================================== -# Download IACT data from remote machine +# Copies IACT data from remote machine # # Copyright (C) 2016 Michael Mayer # @@ -30,8 +30,10 @@ import shutil # ================ # class csiactcopy(ctools.cscript): """ - This script copies IACT data from one location to another. - It can take a list of observation IDs to allow the download specific + Copies IACT data from remote machine. + + This script copies IACT data from one location to another. It can + take a list of observation IDs to allow the download specific observations. Index files get merged and updated accordingly. """ @@ -41,9 +43,15 @@ class csiactcopy(ctools.cscript): Constructor. """ # Set name - self._name = "csiactcopy" - self._version = "1.1.0" - self._datapath = os.getenv("VHEFITS","") + self._name = "csiactcopy" + self._version = "1.1.0" + self._datapath = os.getenv("VHEFITS","") + self._remote_master = gammalib.GFilename() + self._remote_base = "" + self._prodname = "" + self._outpath = "" + self._runlist = gammalib.GFilename() + self._runs = [] # Initialise application by calling the appropriate class # constructor. @@ -62,10 +70,10 @@ class csiactcopy(ctools.cscript): # Get Parameters self._remote_master = self["remote_master"].filename() if not self._remote_master.exists(): - raise RuntimeError("*** ERROR: FITS data not available. No " - "master index file found in \""+ - self._remote_master+"\". Make sure remote " - "file system is properly mounted.") + raise RuntimeError('*** ERROR: FITS data not available. No ' + 'master index file found in "'+ + self._remote_master+'". Make sure remote ' + 'file system is properly mounted.') # Get parameters self._prodname = self["prodname"].string() @@ -80,9 +88,9 @@ class csiactcopy(ctools.cscript): # Return return - def copy(self, source, clobber): + def _copy(self, source, clobber): """ - copy file to _outpath + Copy file to outpath """ # Get file destination destination = os.path.join(self._outpath, @@ -143,9 +151,8 @@ class csiactcopy(ctools.cscript): # Return return filesize - - - def merge(self, localfits, remotefits, hduname, clobber): + + def _merge(self, localfits, remotefits, hduname, clobber): """ merge remote and local fits file If local fits file not present, a new one is created @@ -180,7 +187,7 @@ class csiactcopy(ctools.cscript): # find local obs_id local_obs = [] - lobs_col = local_hdu["OBS_ID"] + lobs_col = local_hdu["OBS_ID"] for obs_id in lobs_col: local_obs.append(obs_id) @@ -198,12 +205,12 @@ class csiactcopy(ctools.cscript): remotefile = remotefits + selection # open remote fits file - remote = gammalib.GFits(str(remotefile)) + remote = gammalib.GFits(str(remotefile)) remote_hdu = remote[hduname] # Get remote obs_id remote_obs = [] - robs_col = remote_hdu["OBS_ID"] + robs_col = remote_hdu["OBS_ID"] for obs_id in robs_col: remote_obs.append(obs_id) @@ -346,27 +353,26 @@ class csiactcopy(ctools.cscript): # Raise exception if file not valid else: - raise RuntimeError("*** ERROR: Runlist file \""+ - self._runlist+"\" not available.") + raise RuntimeError('*** ERROR: Runlist file "'+ + self._runlist+'" not available.') # Check for availability of remote master file if not self._remote_master.exists(): - raise RuntimeError("*** ERROR: Remote master file \""+ - self._remote_master+"\" does not exist.") + raise RuntimeError('*** ERROR: Remote master file "'+ + self._remote_master+'" does not exist.') # Retrieve json data from remote master json_data = open(self._remote_master.url()).read() data = json.loads(json_data) if not "datasets" in data: - raise RuntimeError("*** ERROR: Key \"datasets\" not available " - "in remote master index file \""+ - self._remote_master+"\".") + raise RuntimeError('*** ERROR: Key "datasets" not available ' + 'in remote master index file "'+ + self._remote_master+'".') # Get array of configurations configs = data["datasets"] # Get remote paths - #self._remote_base = os.path.dirname(self._remote_master.url()) self._remote_base = self._remote_master.path() # Initialise flag if prod has been found @@ -399,13 +405,13 @@ class csiactcopy(ctools.cscript): # Log information if self._logTerse(): - self._log.header3("Remote config \""+self._prodname+"\"") - self._log.parformat("HDU index") + self._log.header3('Remote config "'+self._prodname+'"') + self._log.parformat('HDU index') self._log(str(remote_hdu)) - self._log("\n") - self._log.parformat("Observation index") + self._log('\n') + self._log.parformat('Observation index') self._log(str(remote_obs)) - self._log("\n") + self._log('\n') # Open remote HDU index file fits = gammalib.GFits(str(remote_hdu)) @@ -427,9 +433,9 @@ class csiactcopy(ctools.cscript): for run in self._runs: if not run in remote_ids: if self._logNormal(): - self._log("Skip observation "+str(run)+": ") - self._log("ID not available remotely") - self._log("\n") + self._log('Skip observation '+str(run)+': ') + self._log('ID not available remotely') + self._log('\n') # Loop over remote HDU index file for row in range(table.nrows()): @@ -450,7 +456,8 @@ class csiactcopy(ctools.cscript): if obs_id in self._runs: # Get filename - fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name) + fname = os.path.join(os.path.dirname(remote_hdu), + file_dir, file_name) # Add file oldlen = len(files) @@ -464,7 +471,8 @@ class csiactcopy(ctools.cscript): # Otherwise add every file else: # Get filename - fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name) + fname = os.path.join(os.path.dirname(remote_hdu), + file_dir, file_name) # Add file oldlen = len(files) @@ -499,13 +507,14 @@ class csiactcopy(ctools.cscript): else: if self._logExplicit(): self._log("\n") - self._log.header3("Skipping config \""+str(config["name"])+"\"") + self._log.header3('Skipping config "'+ + str(config["name"])+'"') self._log("\n") # Raise Exception if prodname was not found if not has_prod: - msg = "*** ERROR: FITS production \""+self._prodname+\ - "\" not available. Available productions are:\n" + msg = '*** ERROR: FITS production "'+self._prodname+\ + '" not available. Available productions are:\n' for config in configs: msg += " - "+config["name"]+"\n" raise RuntimeError(msg) @@ -545,7 +554,7 @@ class csiactcopy(ctools.cscript): last_fraction += fraction_increment # Copy file - filesize = self.copy(filename, self._clobber()) + filesize = self._copy(filename, self._clobber()) if filesize > 0.0: total_size += filesize @@ -564,8 +573,12 @@ class csiactcopy(ctools.cscript): self._log.header1("Updating index files") # Build local index file names - local_hdu = os.path.join(self._outpath, os.path.relpath(remote_hdu, self._remote_base)) - local_obs = os.path.join(self._outpath, os.path.relpath(remote_obs, self._remote_base)) + local_hdu = os.path.join(self._outpath, + os.path.relpath(remote_hdu, + self._remote_base)) + local_obs = os.path.join(self._outpath, + os.path.relpath(remote_obs, + self._remote_base)) # If we have a runlist then merge index file if len(self._runs): @@ -576,19 +589,19 @@ class csiactcopy(ctools.cscript): self._log.header3("HDU index") # Merge remote index files with local files - self.merge(local_hdu, remote_hdu, "HDU_INDEX", self._clobber()) + self._merge(local_hdu, remote_hdu, "HDU_INDEX", self._clobber()) if self._logTerse(): self._log("\n") self._log.header3("OBS index") # Merge remote index files with local files - self.merge(local_obs, remote_obs, "OBS_INDEX", self._clobber()) + self._merge(local_obs, remote_obs, "OBS_INDEX", self._clobber()) else: # If all files were copied, just copy index files too - self.copy(remote_hdu, self._clobber()) - self.copy(remote_obs, self._clobber()) + self._copy(remote_hdu, self._clobber()) + self._copy(remote_obs, self._clobber()) if self._logTerse(): self._log("\n") @@ -599,7 +612,7 @@ class csiactcopy(ctools.cscript): # If no local master is found, copy master over first if not os.path.isfile(localmaster): - self.copy(self._remote_master.url(), self._clobber()) + self._copy(self._remote_master.url(), self._clobber()) # Load local master json_data = open(localmaster).read() @@ -624,15 +637,15 @@ class csiactcopy(ctools.cscript): if not (gammalib.GFilename(str(hdu)).is_fits() and gammalib.GFilename(str(obs)).is_fits()): if self._logTerse(): - self._log("Removing \""+str(config["name"])+"\" ") - self._log("(not available)") - self._log("\n") + self._log('Removing "'+str(config["name"])+'" ') + self._log('(not available)') + self._log('\n') else: # Append config if valid newconfigs.append(config) if self._logTerse(): - self._log("Keeping \""+str(config["name"])+"\"") - self._log("\n") + self._log('Keeping "'+str(config["name"])+'"') + self._log('\n') # Signals that downloaded config is available if config["name"] == self._prodname: @@ -646,8 +659,8 @@ class csiactcopy(ctools.cscript): newdict["obsindx"] = os.path.relpath(local_obs, self._outpath) newconfigs.append(newdict) if self._logTerse(): - self._log("Adding \""+str(newdict["name"])+"\"") - self._log("\n") + self._log('Adding "'+str(newdict["name"])+'"') + self._log('\n') # Write new json master file f = open(localmaster, "w") diff --git a/cscripts/csiactdata.py b/cscripts/csiactdata.py index ea864b7..1c81d8d 100755 --- a/cscripts/csiactdata.py +++ b/cscripts/csiactdata.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # ========================================================================== -# Inspection of IACT data storage +# Inspect an IACT data storage # # Copyright (C) 2015-2016 Michael Mayer # @@ -30,6 +30,8 @@ import json # ================ # class csiactdata(ctools.cscript): """ + Inspect an IACT data storage. + This script inspects the available FITS data storage on the user machine and writes information about the available FITS configurations into a log file or prints it on the screen. These information can be used as input @@ -42,10 +44,12 @@ class csiactdata(ctools.cscript): Constructor. """ # Set name - self._name = "csiactdata" - self._version = "1.1.0" - self._datapath = os.getenv("VHEFITS","") - self._prodnames = [] + self._name = "csiactdata" + self._version = "1.1.0" + self._datapath = os.getenv("VHEFITS","") + self._prodnames = [] + self._master_indx = "" + self._master_file = "" # Initialise application by calling the appropriate class # constructor. @@ -74,7 +78,10 @@ class csiactdata(ctools.cscript): # Check for presence of master index file if not os.path.isfile(self._master_file): - raise RuntimeError("Master index file \""+self._master_file+"\" not found. Use hidden parameter \"master_indx\" to specifiy a different filename.") + raise RuntimeError('Master index file "'+self._master_file+ + '" not found. Use hidden parameter '+ + '"master_indx" to specifiy a different '+ + 'filename.') # Return return diff --git a/cscripts/cslightcrv.py b/cscripts/cslightcrv.py index 73b295d..6ca1063 100755 --- a/cscripts/cslightcrv.py +++ b/cscripts/cslightcrv.py @@ -74,6 +74,7 @@ class cslightcrv(ctools.cscript): self._srcname = "" self._tbins = gammalib.GGti() self._stacked = False + self._fits = gammalib.GFits() # Initialise observation container from constructor arguments. self._obs, argv = self._set_input_obs(argv) @@ -397,20 +398,20 @@ class cslightcrv(ctools.cscript): self._log.header3("Binning events") # Bin events - bin = ctools.ctbin(obs) - bin["usepnt"] = False - bin["ebinalg"] = "LOG" - bin["xref"] = self["xref"].real() - bin["yref"] = self["yref"].real() - bin["binsz"] = self["binsz"].real() - bin["nxpix"] = self["nxpix"].integer() - bin["nypix"] = self["nypix"].integer() - bin["enumbins"] = self["enumbins"].integer() - bin["emin"] = self["emin"].real() - bin["emax"] = self["emax"].real() - bin["coordsys"] = self["coordsys"].string() - bin["proj"] = self["proj"].string() - bin.run() + cntcube = ctools.ctbin(obs) + cntcube["usepnt"] = False + cntcube["ebinalg"] = "LOG" + cntcube["xref"] = self["xref"].real() + cntcube["yref"] = self["yref"].real() + cntcube["binsz"] = self["binsz"].real() + cntcube["nxpix"] = self["nxpix"].integer() + cntcube["nypix"] = self["nypix"].integer() + cntcube["enumbins"] = self["enumbins"].integer() + cntcube["emin"] = self["emin"].real() + cntcube["emax"] = self["emax"].real() + cntcube["coordsys"] = self["coordsys"].string() + cntcube["proj"] = self["proj"].string() + cntcube.run() # Header if self._logExplicit(): @@ -439,9 +440,9 @@ class cslightcrv(ctools.cscript): # Compute spatial binning for point spread function and # energy dispersion cubes - binsz = 0.1 * self["binsz"].real() - nxpix = self["nxpix"].integer() / 10 - nypix = self["nypix"].integer() / 10 + binsz = 10 * self["binsz"].real() + nxpix = self["nxpix"].integer() // 10 + nypix = self["nypix"].integer() // 10 if nxpix < 2: nxpix = 2 if nypix < 2: @@ -510,7 +511,7 @@ class cslightcrv(ctools.cscript): bkgcube.run() # Retrieve a new oberservation container - new_obs = bin.obs().copy() + new_obs = cntcube.obs().copy() # Get new models models = bkgcube.models() @@ -606,8 +607,8 @@ class cslightcrv(ctools.cscript): select = ctools.ctselect(self._obs) select["emin"] = self["emin"].real() select["emax"] = self["emax"].real() - select["tmin"] = tmin.convert(select._time_reference()) - select["tmax"] = tmax.convert(select._time_reference()) + select["tmin"] = tmin.convert(self._time_reference()) + select["tmax"] = tmax.convert(self._time_reference()) select["rad"] = "UNDEFINED" select["ra"] = "UNDEFINED" select["dec"] = "UNDEFINED"