diff --git a/cscripts/csfindobs.py b/cscripts/csfindobs.py index 977069f..e5090a0 100755 --- a/cscripts/csfindobs.py +++ b/cscripts/csfindobs.py @@ -39,15 +39,15 @@ class csfindobs(ctools.cscript): Constructor. """ # Set name - self._name = "csfindobs" - self._version = "1.1.0" - self._datapath = os.getenv("VHEFITS","") - self._prodname = "" + self._name = 'csfindobs' + self._version = '1.1.0' + self._datapath = os.getenv('VHEFITS','') + self._prodname = '' self._select_radec = True self._radius = 0.0 self._ra = 0.0 self._dec = 0.0 - self._obs_index = "" + self._obs_index = '' self._runs = [] # Initialise application by calling the appropriate class @@ -64,17 +64,17 @@ class csfindobs(ctools.cscript): Get parameters from parfile and setup the observation. """ # Get parameters - if self._datapath == "": - self._datapath = self["datapath"].string() + if self._datapath == '': + self._datapath = self['datapath'].string() # Expand environment self._datapath = gammalib.expand_env(self._datapath) # Get production name - self._prodname = self["prodname"].string() + self._prodname = self['prodname'].string() # Master index file name - master_indx = self["master_indx"].string() + master_indx = self['master_indx'].string() # Intiialise flag if spatial selection is required self._select_radec = True @@ -83,14 +83,14 @@ class csfindobs(ctools.cscript): self._radius = 0.0 # Check for vailidity of spatial parameters - if (self["ra"].is_valid() and - self["dec"].is_valid() and - self["rad"].is_valid()): + if (self['ra'].is_valid() and + self['dec'].is_valid() and + self['rad'].is_valid()): # Read spatial parameters - self._ra = self["ra"].real() - self._dec = self["dec"].real() - self._radius = self["rad"].real() + self._ra = self['ra'].real() + self._dec = self['dec'].real() + self._radius = self['rad'].real() else: self._select_radec = False @@ -100,12 +100,12 @@ class csfindobs(ctools.cscript): self._select_radec = False # Query other parameters - self["min_qual"].integer() - self["expression"].string() + self['min_qual'].integer() + self['expression'].string() # Read ahead output parameters if self._read_ahead(): - self["outfile"].filename() + self['outfile'].filename() # Open master index file and look for prodname master_file = os.path.join(self._datapath, master_indx) @@ -119,33 +119,33 @@ class csfindobs(ctools.cscript): # Open and load JSON file json_data = open(master_file).read() data = json.loads(json_data) - if not "datasets" in data: + if not 'datasets' in data: raise RuntimeError('Key "datasets" not available in '+ 'master index file.') # Get configurations - configs = data["datasets"] + configs = data['datasets'] # Initialise obs index file - self._obs_index = "" + self._obs_index = '' # Get HDUs for config in configs: - if self._prodname == config["name"]: + if self._prodname == config['name']: self._obs_index = str(os.path.join(self._datapath, - config["obsindx"])) + config['obsindx'])) break # Check HDUs - if self._obs_index == "": + if self._obs_index == '': raise RuntimeError('FITS data store "'+self._prodname+ '" not available. Run csiactdata to get '+ 'a list of available storage names') - filename = gammalib.GFilename(self._obs_index+"[OBS_INDEX]") + filename = gammalib.GFilename(self._obs_index+'[OBS_INDEX]') if not filename.is_fits(): raise RuntimeError('Observation index file "'+ self._obs_index+ - '[OBS_INDEX]" for FITS data store "'+ + '"[OBS_INDEX]" for FITS data store "'+ self._prodname+ '" not available. Check your master index '+ 'file or run csiactdata to get a list of '+ @@ -154,7 +154,7 @@ class csfindobs(ctools.cscript): # Write input parameters into logger if self._logTerse(): self._log_parameters() - self._log("\n") + self._log('\n') # Return return @@ -176,55 +176,62 @@ class csfindobs(ctools.cscript): self._runs = [] # Initialise selction expression - expr = "" + expr = '' # Add spatial expression if possible if self._select_radec: - expr += "ANGSEP("+str(self._ra)+","+str(self._dec)+ \ - ",RA_PNT,DEC_PNT)<="+str(self._radius) + expr += 'ANGSEP('+str(self._ra)+','+str(self._dec)+ \ + ',RA_PNT,DEC_PNT)<='+str(self._radius) # Add connector if expression is empty if len(expr): - expr += "&&" + expr += '&&' # Add quality expression - expr += "QUALITY<="+str(self["min_qual"].integer()) + expr += 'QUALITY<='+str(self['min_qual'].integer()) # Add user expression - expression = self["expression"].string() - if (expression != "NONE" and expression != "INDEF" and + expression = self['expression'].string() + if (expression != 'NONE' and expression != 'INDEF' and len(expression) > 0): - expr += "&&" + expression + expr += '&&' + expression # Initialise filename - openfile = self._obs_index+"[OBS_INDEX]["+expr+"]" + openfile = self._obs_index+'[OBS_INDEX]['+expr+']' # Open file fits = gammalib.GFits(openfile) # Check for extistence of rows - if fits["OBS_INDEX"].nrows() > 0: + if fits['OBS_INDEX'].nrows() > 0: # Get HDU - obs_indx = fits["OBS_INDEX"] + obs_indx = fits['OBS_INDEX'] # Read observation IDs into array for i in range(obs_indx.nrows()): - self._runs.append(obs_indx["OBS_ID"][i]) + self._runs.append(obs_indx['OBS_ID'][i]) # Dump expression if self._logNormal(): - self._log("\n") - self._log.header3("Expression") + self._log('\n') + self._log.header3('Expression') self._log(expr) - self._log("\n") + self._log('\n') # Dump number of observations if self._logTerse(): - self._log("\n") - self._log.header3("Observations") - self._log("Found "+str(len(self._runs))+" observations") - self._log("\n") + self._log('\n') + self._log.header3('Observations') + self._log('Found '+str(len(self._runs))+' observations') + self._log('\n') + + # Dump number of observations + if self._logVerbose(): + self._log('\n') + self._log.header3('Observation IDs') + for run in self._runs: + self._log(str(run)+'\n') # Close file fits.close() @@ -261,26 +268,26 @@ class csfindobs(ctools.cscript): self._log.header1('Save runlist') # Get filename - outfile = self["outfile"].filename() + outfile = self['outfile'].filename() # Check for clobber if outfile.exists() and not self._clobber(): if self._logTerse(): - self._log('File "+outfile+" already exists. ') + self._log('File '+outfile+' already exists. ') self._log('Set clobber=yes to overwrite file.') self._log('\n') else: # Log file name if self._logTerse(): - self._log(gammalib.parformat("Runlist file")) + self._log(gammalib.parformat('Runlist file')) self._log(outfile.url()) - self._log("\n") + self._log('\n') # Write runs to file - f = open(outfile.url(),"w") + f = open(outfile.url(),'w') for run in self._runs: - f.write(str(run)+" \n") + f.write(str(run)+' \n') f.close() # Return diff --git a/cscripts/csiactcopy.py b/cscripts/csiactcopy.py index d0495d2..bd7b859 100755 --- a/cscripts/csiactcopy.py +++ b/cscripts/csiactcopy.py @@ -43,13 +43,13 @@ class csiactcopy(ctools.cscript): Constructor. """ # Set name - self._name = "csiactcopy" - self._version = "1.1.0" - self._datapath = os.getenv("VHEFITS","") + self._name = 'csiactcopy' + self._version = '1.1.0' + self._datapath = os.getenv('VHEFITS','') self._remote_master = gammalib.GFilename() - self._remote_base = "" - self._prodname = "" - self._outpath = "" + self._remote_base = '' + self._prodname = '' + self._outpath = '' self._runlist = gammalib.GFilename() self._runs = [] @@ -68,7 +68,7 @@ class csiactcopy(ctools.cscript): """ # Get Parameters - self._remote_master = self["remote_master"].filename() + self._remote_master = self['remote_master'].filename() if not self._remote_master.exists(): raise RuntimeError('*** ERROR: FITS data not available. No ' 'master index file found in "'+ @@ -76,14 +76,14 @@ class csiactcopy(ctools.cscript): 'file system is properly mounted.') # Get parameters - self._prodname = self["prodname"].string() - self._outpath = gammalib.expand_env(self["outpath"].string()) - self._runlist = self["runlist"].filename() + self._prodname = self['prodname'].string() + self._outpath = gammalib.expand_env(self['outpath'].string()) + self._runlist = self['runlist'].filename() # Write input parameters into logger if self._logTerse(): self._log_parameters() - self._log("\n") + self._log('\n') # Return return @@ -101,21 +101,21 @@ class csiactcopy(ctools.cscript): # Logging if self._logVerbose(): - self._log("\n") - self._log.header3("Copy file") - self._log.parformat("Source") + self._log('\n') + self._log.header3('Copy file') + self._log.parformat('Source') self._log(str(source)) - self._log("\n") - self._log.parformat("Destination") + self._log('\n') + self._log.parformat('Destination') self._log(str(destination)) - self._log("\n") - self._log.parformat("Already exists") + self._log('\n') + self._log.parformat('Already exists') self._log(str(os.path.isfile(destination))) - self._log("\n") - self._log.parformat("Overwrite") + self._log('\n') + self._log.parformat('Overwrite') self._log(str(clobber)) - self._log("\n") - self._log.parformat("Copying") + self._log('\n') + self._log.parformat('Copying') # Flag if file destination is already available is_file = os.path.isfile(destination) @@ -123,14 +123,14 @@ class csiactcopy(ctools.cscript): # check if file could be skipped because clobber=no if is_file and clobber == False: if self._logVerbose(): - self._log("Skip (clobber=no)") - self._log("\n") + self._log('Skip (clobber=no)') + self._log('\n') # check if file could be skipped because it is the same file elif is_file and os.path.samefile(destination, source): if self._logVerbose(): - self._log("Skip (same file)") - self._log("\n") + self._log('Skip (same file)') + self._log('\n') else: # Create directory if not existent @@ -146,8 +146,8 @@ class csiactcopy(ctools.cscript): # Logging if self._logVerbose(): - self._log("Done!") - self._log("\n") + self._log('Done!') + self._log('\n') # Return return filesize @@ -159,21 +159,21 @@ class csiactcopy(ctools.cscript): """ if self._logTerse(): - self._log.parformat("Local file") + self._log.parformat('Local file') self._log(str(localfits)) - self._log("\n") - self._log.parformat("Remote file") + self._log('\n') + self._log.parformat('Remote file') self._log(str(remotefits)) - self._log("\n") - self._log.parformat("HDU name") + self._log('\n') + self._log.parformat('HDU name') self._log(str(hduname)) - self._log("\n") - self._log.parformat("Conflict preference") + self._log('\n') + self._log.parformat('Conflict preference') if clobber: - self._log("Remote") + self._log('Remote') else: - self._log("Local") - self._log("\n") + self._log('Local') + self._log('\n') # Check if local fits file is available if os.path.isfile(localfits): @@ -187,40 +187,70 @@ class csiactcopy(ctools.cscript): # find local obs_id local_obs = [] - lobs_col = local_hdu["OBS_ID"] + lobs_col = local_hdu['OBS_ID'] for obs_id in lobs_col: local_obs.append(obs_id) - - # Create selection expression for opening remote fits file - selection = "" - if len(self._runs): - selection += "["+hduname+"][" - for run in self._runs: - selection += "OBS_ID=="+str(run) - selection += "||" - selection = selection[:-2] - selection += "]" - - # Create file name - remotefile = remotefits + selection - - # open remote fits file - remote = gammalib.GFits(str(remotefile)) - remote_hdu = remote[hduname] - + + # Initialise list of runlists + # This is kluge: to circumvent a too long file name, we split the + # runlists into chunks of 50 observations maximum + runlists = [] + runs_per_chunk = 50 + for i in range(len(self._runs) / runs_per_chunk + 1): + chunk = self._runs[i * runs_per_chunk : (i + 1) * runs_per_chunk] + runlists.append(chunk) + + # Loop over runlist chunks + for i, runlist in enumerate(runlists): + + # Create selection expression for opening remote fits file + selection = '' + if len(runlist): + selection += '['+hduname+'][' + for run in runlist: + selection += 'OBS_ID=='+str(run) + selection += '||' + selection = selection[:-2] + selection += ']' + + # Create file name + remotefile = remotefits + selection + + # open remote fits file + remote = gammalib.GFits(str(remotefile)) + + # If first iteration, create clone of FITS column for later use + if i == 0: + remote_hdu = remote[hduname].clone() + else: + + # get temporary hdu to be added to current HDU + tmp_hdu = remote[hduname] + + # Get old size of remote HDU + size = remote_hdu.size() + + # Extend remote HDU + remote_hdu.append_rows(tmp_hdu.nrows()) + + # Loop over rows and columns to copy over values exteding remote HDU + for col in tmp_hdu: + for row in range(col.length()): + remote_hdu[col.name()][size + row] = col[row] + # Get remote obs_id remote_obs = [] - robs_col = remote_hdu["OBS_ID"] + robs_col = remote_hdu['OBS_ID'] for obs_id in robs_col: remote_obs.append(obs_id) if self._logTerse(): - self._log.parformat("Remote entries") + self._log.parformat('Remote entries') self._log(str(len(remote_obs))) - self._log("\n") - self._log.parformat("Local entries") + self._log('\n') + self._log.parformat('Local entries') self._log(str(len(local_obs))) - self._log("\n") + self._log('\n') # initialise counter for logging removed_rows = 0 @@ -239,13 +269,13 @@ class csiactcopy(ctools.cscript): table_has_obsid = True while table_has_obsid: for i in range(local_hdu.nrows()): - if remote_obs_id == local_hdu["OBS_ID"][i]: + if remote_obs_id == local_hdu['OBS_ID'][i]: local_hdu.remove_rows(i,1) removed_rows += 1 break table_has_obsid = False for i in range(local_hdu.nrows()): - if remote_obs_id == local_hdu["OBS_ID"][i]: + if remote_obs_id == local_hdu['OBS_ID'][i]: table_has_obsid = True break @@ -262,13 +292,13 @@ class csiactcopy(ctools.cscript): table_has_obsid = True while table_has_obsid: for i in range(remote_hdu.nrows()): - if local_obs_id == remote_hdu["OBS_ID"][i]: + if local_obs_id == remote_hdu['OBS_ID'][i]: remote_hdu.remove_rows(i,1) removed_rows += 1 break table_has_obsid = False for i in range(remote_hdu.nrows()): - if local_obs_id == remote_hdu["OBS_ID"][i]: + if local_obs_id == remote_hdu['OBS_ID'][i]: table_has_obsid = True break @@ -280,11 +310,11 @@ class csiactcopy(ctools.cscript): if self._logTerse(): if clobber: - self._log.parformat("Removed local rows") + self._log.parformat('Removed local rows') else: - self._log.parformat("Skipped remote rows") + self._log.parformat('Skipped remote rows') self._log(str(removed_rows)) - self._log("\n") + self._log('\n') # Loop over columns for i in range(local_hdu.ncols()): @@ -322,8 +352,8 @@ class csiactcopy(ctools.cscript): # Log output if self._logTerse(): - self._log("\n") - self._log.header2("Runlist information") + self._log('\n') + self._log.header2('Runlist information') # Initialise runlist array self._runs = [] @@ -342,15 +372,15 @@ class csiactcopy(ctools.cscript): # Logging if self._logTerse(): - self._log.parformat("Number of observations") + self._log.parformat('Number of observations') self._log(str(len(self._runs))) - self._log("\n") + self._log('\n') - # Copy all data if runlist file is "NONE" - elif self._runlist == "NONE": + # Copy all data if runlist file is 'NONE' + elif self._runlist == 'NONE': if self._logTerse(): - self._log("Copy all available data") - self._log("\n") + self._log('Copy all available data') + self._log('\n') # Raise exception if file not valid else: @@ -365,13 +395,13 @@ class csiactcopy(ctools.cscript): # Retrieve json data from remote master json_data = open(self._remote_master.url()).read() data = json.loads(json_data) - if not "datasets" in data: + if not 'datasets' in data: raise RuntimeError('*** ERROR: Key "datasets" not available ' 'in remote master index file "'+ self._remote_master+'".') # Get array of configurations - configs = data["datasets"] + configs = data['datasets'] # Get remote paths self._remote_base = self._remote_master.path() @@ -384,25 +414,25 @@ class csiactcopy(ctools.cscript): # Logging if self._logTerse(): - self._log("\n") - self._log.header2("Loop over remote configs") + self._log('\n') + self._log.header2('Loop over remote configs') # Loop over configs for config in configs: # Logging if self._logVerbose(): - self._log.header2(str(config["name"])) + self._log.header2(str(config['name'])) # Check if prodname was found - if config["name"] == self._prodname: + if config['name'] == self._prodname: # Inidicate that prodname has been found has_prod = True # Build path of index files - remote_hdu = os.path.join(self._remote_base, config["hduindx"]) - remote_obs = os.path.join(self._remote_base, config["obsindx"]) + remote_hdu = os.path.join(self._remote_base, config['hduindx']) + remote_obs = os.path.join(self._remote_base, config['obsindx']) # Log information if self._logTerse(): @@ -416,10 +446,10 @@ class csiactcopy(ctools.cscript): # Open remote HDU index file fits = gammalib.GFits(str(remote_hdu)) - table = fits["HDU_INDEX"] + table = fits['HDU_INDEX'] # Initialise flag if SIZE column is present - has_size = table.contains("SIZE") + has_size = table.contains('SIZE') if has_size: # Initialise file size @@ -428,7 +458,7 @@ class csiactcopy(ctools.cscript): # Initialise remote observation IDs remote_ids = set() for row in range(table.nrows()): - remote_ids.add(table["OBS_ID"][row]) + remote_ids.add(table['OBS_ID'][row]) # Log runs that are not available remotely for run in self._runs: @@ -442,12 +472,12 @@ class csiactcopy(ctools.cscript): for row in range(table.nrows()): # Get observation ID - obs_id = table["OBS_ID"][row] - file_dir = table["FILE_DIR"][row] - file_name = table["FILE_NAME"][row] + obs_id = table['OBS_ID'][row] + file_dir = table['FILE_DIR'][row] + file_name = table['FILE_NAME'][row] # Skip if filename is empty - if file_name == "": + if file_name == '': continue # Check if we need to consider an input runlist @@ -467,7 +497,7 @@ class csiactcopy(ctools.cscript): # Add file size if has_size and newlen > oldlen: - cp_size += table["SIZE"][row] + cp_size += table['SIZE'][row] # Otherwise add every file else: @@ -482,24 +512,24 @@ class csiactcopy(ctools.cscript): # Add file size if has_size and newlen > oldlen: - cp_size += table["SIZE"][row] + cp_size += table['SIZE'][row] # Log file information if self._logTerse(): - self._log("\n") - self._log.header2("File information") - self._log.parformat("Number of files") + self._log('\n') + self._log.header2('File information') + self._log.parformat('Number of files') self._log(str(len(files))) - self._log("\n") + self._log('\n') if has_size: - self._log.parformat("Size") - self._log("%.2f"%(float(cp_size)*1e-6)+" MB") - self._log("\n") + self._log.parformat('Size') + self._log('%.2f'%(float(cp_size)*1e-6)+' MB') + self._log('\n') if self._logVerbose(): - self._log("\n") - self._log.header3("File names") + self._log('\n') + self._log.header3('File names') for filename in files: - self._log(str(filename)+"\n") + self._log(str(filename)+'\n') # Close HDU index file fits.close() @@ -507,23 +537,23 @@ class csiactcopy(ctools.cscript): # If prodname is not found just log that we skip the config else: if self._logExplicit(): - self._log("\n") + self._log('\n') self._log.header3('Skipping config "'+ - str(config["name"])+'"') - self._log("\n") + str(config['name'])+'"') + self._log('\n') # Raise Exception if prodname was not found if not has_prod: - msg = '*** ERROR: FITS production "'+self._prodname+\ - '" not available. Available productions are:\n' + msg = '*** ERROR: FITS production "'+self._prodname+'" not ' + msg += 'available. Available productions are:\n' for config in configs: - msg += " - "+config["name"]+"\n" + msg += ' - '+config['name']+'\n' raise RuntimeError(msg) # Logging if self._logNormal(): - self._log("\n") - self._log.header1("Copying files") + self._log('\n') + self._log.header1('Copying files') # intialise counter k = 0 @@ -549,9 +579,9 @@ class csiactcopy(ctools.cscript): fraction = float(k) / float(len(files)) * 100.0 while fraction > last_fraction: if self._logNormal() and not self._logVerbose(): - self._log.parformat("Status") - self._log(str(int(last_fraction))+"%") - self._log("\n") + self._log.parformat('Status') + self._log(str(int(last_fraction))+'%') + self._log('\n') last_fraction += fraction_increment # Copy file @@ -566,12 +596,12 @@ class csiactcopy(ctools.cscript): # Logging if self._logNormal() and not self._logVerbose(): - self._log.parformat("Status") - self._log("Finished") - self._log("\n") + self._log.parformat('Status') + self._log('Finished') + self._log('\n') if self._logTerse(): - self._log("\n") - self._log.header1("Updating index files") + self._log('\n') + self._log.header1('Updating index files') # Build local index file names local_hdu = os.path.join(self._outpath, @@ -586,18 +616,18 @@ class csiactcopy(ctools.cscript): # Logging if self._logTerse(): - self._log("\n") - self._log.header3("HDU index") + self._log('\n') + self._log.header3('HDU index') # Merge remote index files with local files - self._merge(local_hdu, remote_hdu, "HDU_INDEX", self._clobber()) + self._merge(local_hdu, remote_hdu, 'HDU_INDEX', self._clobber()) if self._logTerse(): - self._log("\n") - self._log.header3("OBS index") + self._log('\n') + self._log.header3('OBS index') # Merge remote index files with local files - self._merge(local_obs, remote_obs, "OBS_INDEX", self._clobber()) + self._merge(local_obs, remote_obs, 'OBS_INDEX', self._clobber()) else: # If all files were copied, just copy index files too @@ -605,11 +635,11 @@ class csiactcopy(ctools.cscript): self._copy(remote_obs, self._clobber()) if self._logTerse(): - self._log("\n") - self._log.header3("Master index file") + self._log('\n') + self._log.header3('Master index file') # Adding prodname to local master - localmaster = os.path.join(self._outpath, "master.json") + localmaster = os.path.join(self._outpath, 'master.json') # If no local master is found, copy master over first if not os.path.isfile(localmaster): @@ -618,7 +648,7 @@ class csiactcopy(ctools.cscript): # Load local master json_data = open(localmaster).read() data = json.loads(json_data) - configs = data["datasets"] + configs = data['datasets'] # Initialise flag indicating if we already have prodname in master # file @@ -631,66 +661,66 @@ class csiactcopy(ctools.cscript): for config in configs: # Get hdu and obs index files - hdu = os.path.join(self._outpath, config["hduindx"]) - obs = os.path.join(self._outpath, config["obsindx"]) + hdu = os.path.join(self._outpath, config['hduindx']) + obs = os.path.join(self._outpath, config['obsindx']) # Check if index files are available if not (gammalib.GFilename(str(hdu)).is_fits() and gammalib.GFilename(str(obs)).is_fits()): if self._logTerse(): - self._log('Removing "'+str(config["name"])+'" ') + self._log('Removing "'+str(config['name'])+'" ') self._log('(not available)') self._log('\n') else: # Append config if valid newconfigs.append(config) if self._logTerse(): - self._log('Keeping "'+str(config["name"])+'"') + self._log('Keeping "'+str(config['name'])+'"') self._log('\n') # Signals that downloaded config is available - if config["name"] == self._prodname: + if config['name'] == self._prodname: has_config = True # Create new entry if config was not available if not has_config: - newdict = dict.fromkeys(["name","hduindx","obsindx"]) - newdict["name"] = self._prodname - newdict["hduindx"] = os.path.relpath(local_hdu, self._outpath) - newdict["obsindx"] = os.path.relpath(local_obs, self._outpath) + newdict = dict.fromkeys(['name','hduindx','obsindx']) + newdict['name'] = self._prodname + newdict['hduindx'] = os.path.relpath(local_hdu, self._outpath) + newdict['obsindx'] = os.path.relpath(local_obs, self._outpath) newconfigs.append(newdict) if self._logTerse(): - self._log('Adding "'+str(newdict["name"])+'"') + self._log('Adding "'+str(newdict['name'])+'"') self._log('\n') # Write new json master file - f = open(localmaster, "w") - data["datasets"] = newconfigs + f = open(localmaster, 'w') + data['datasets'] = newconfigs json.dump(data, f, indent=2) f.close() # Log summary if self._logNormal(): - self._log("\n") - self._log.header1("Summary") - self._log.parformat("Data files found") + self._log('\n') + self._log.header1('Summary') + self._log.parformat('Data files found') self._log(str(len(files))) - self._log("\n") - self._log.parformat("Data files copied") + self._log('\n') + self._log.parformat('Data files copied') self._log(str(n_copied)) - self._log("\n") - self._log.parformat("Size") - self._log("%.2f"%(float(total_size)*1e-6)+" MB") - self._log("\n") - self._log.parformat("Local configs") + self._log('\n') + self._log.parformat('Size') + self._log('%.2f'%(float(total_size)*1e-6)+' MB') + self._log('\n') + self._log.parformat('Local configs') self._log(str(len(newconfigs))) - self._log("\n") + self._log('\n') if self._logTerse(): - self._log("\n") - self._log.header3("Content of master index") + self._log('\n') + self._log.header3('Content of master index') for config in newconfigs: - self._log(str(config["name"])) - self._log("\n") + self._log(str(config['name'])) + self._log('\n') # Return return diff --git a/cscripts/csiactobs.par b/cscripts/csiactobs.par index d6e2aa5..ab0c4d5 100644 --- a/cscripts/csiactobs.par +++ b/cscripts/csiactobs.par @@ -18,7 +18,7 @@ # Input and output parameters #============================ datapath, s, a, $VHEFITS,,, "Path were data is located" -prodname, s, a, ,,, "Data storage name" +prodname, s, a, fits-prod-name,,, "Data storage name" infile, f, a, runlist.lis,,, "Input runlist file" inmodel, f, h, NONE,,, "Input model definition XML file (optional)" master_indx, s, h, master.json,,, "Input master index file" diff --git a/cscripts/csiactobs.py b/cscripts/csiactobs.py index c63e5ff..c508768 100755 --- a/cscripts/csiactobs.py +++ b/cscripts/csiactobs.py @@ -46,14 +46,14 @@ class csiactobs(ctools.cscript): Constructor. """ # Set name and version - self._name = "csiactobs" - self._version = "1.1.0" + self._name = 'csiactobs' + self._version = '1.1.0' # Initialise some members self._ebounds = gammalib.GEbounds() - self._datapath = os.getenv("VHEFITS","") + self._datapath = os.getenv('VHEFITS','') self._inmodels = gammalib.GModels() - self._prodname = "" + self._prodname = '' self._xml = gammalib.GXml() self._models = gammalib.GModels() self._runlist = [] @@ -61,23 +61,23 @@ class csiactobs(ctools.cscript): self._bkgpars = 0 self._outmodel = gammalib.GFilename() self._outobs = gammalib.GFilename() - self._master_indx = "" + self._master_indx = '' self._use_bkg_scale = False - self._ev_hiera = [""] - self._aeff_hiera = [""] - self._psf_hiera = [""] - self._bkg_hiera = [""] - self._edisp_hiera = [""] - self._bkg_mod_hiera = [""] + self._ev_hiera = [''] + self._aeff_hiera = [''] + self._psf_hiera = [''] + self._bkg_hiera = [''] + self._edisp_hiera = [''] + self._bkg_mod_hiera = [''] self._bkg_gauss_norm = 1.0 self._bkg_gauss_index = 0.0 self._bkg_gauss_sigma = 1.0 self._bkg_aeff_index = 0.0 self._bkg_aeff_norm = 1.0 self._bkg_range_factor = 1.0 - self._hdu_index = "" - self._obs_index = "" - self._subdir = "" + self._hdu_index = '' + self._obs_index = '' + self._subdir = '' self._debug = False # Initialise application by calling the appropriate class @@ -94,24 +94,24 @@ class csiactobs(ctools.cscript): Get parameters from parfile and setup the observation. """ # Set data path - if self._datapath == "": - self._datapath = self["datapath"].string() + if self._datapath == '': + self._datapath = self['datapath'].string() # Check for input models - if not self["inmodel"].filename() == "NONE": - self._inmodels = gammalib.GModels(self["inmodel"].filename()) + if not self['inmodel'].filename() == 'NONE': + self._inmodels = gammalib.GModels(self['inmodel'].filename()) # Expand environment self._datapath = gammalib.expand_env(self._datapath) # Read FITS production - self._prodname = self["prodname"].string() + self._prodname = self['prodname'].string() # Read runlist file if list not already filled if len(self._runlist) == 0: # Get file name - self._runlistfile = self["infile"].filename() + self._runlistfile = self['infile'].filename() # Read runlist from file runfile = open(self._runlistfile.url()) @@ -125,73 +125,73 @@ class csiactobs(ctools.cscript): runfile.close() # Read number of background parameters - self._bkgpars = self["bkgpars"].integer() + self._bkgpars = self['bkgpars'].integer() # Output model file - self._outmodel = self["outmodel"].filename() + self._outmodel = self['outmodel'].filename() # Observation outfile - self._outobs = self["outobs"].filename() + self._outobs = self['outobs'].filename() # Master index file name - self._master_indx = self["master_indx"].string() + self._master_indx = self['master_indx'].string() # Read flag for background scaling factor - self._use_bkg_scale = self["bkg_scale"].boolean() + self._use_bkg_scale = self['bkg_scale'].boolean() # Read hierarchy of file loading - self._ev_hiera = self["ev_hiera"].string().split("|") - self._aeff_hiera = self["aeff_hiera"].string().split("|") - self._psf_hiera = self["psf_hiera"].string().split("|") - self._bkg_hiera = self["bkg_hiera"].string().split("|") - self._edisp_hiera = self["edisp_hiera"].string().split("|") - self._bkg_mod_hiera = self["bkg_mod_hiera"].string().split("|") + self._ev_hiera = self['ev_hiera'].string().split('|') + self._aeff_hiera = self['aeff_hiera'].string().split('|') + self._psf_hiera = self['psf_hiera'].string().split('|') + self._bkg_hiera = self['bkg_hiera'].string().split('|') + self._edisp_hiera = self['edisp_hiera'].string().split('|') + self._bkg_mod_hiera = self['bkg_mod_hiera'].string().split('|') # Read hidden background parameters - self._bkg_gauss_norm = self["bkg_gauss_norm"].real() - self._bkg_gauss_index = self["bkg_gauss_index"].real() - self._bkg_gauss_sigma = self["bkg_gauss_sigma"].real() - self._bkg_aeff_index = self["bkg_aeff_index"].real() - self._bkg_aeff_norm = self["bkg_aeff_norm"].real() - self._bkg_range_factor = self["bkg_range_factor"].real() + self._bkg_gauss_norm = self['bkg_gauss_norm'].real() + self._bkg_gauss_index = self['bkg_gauss_index'].real() + self._bkg_gauss_sigma = self['bkg_gauss_sigma'].real() + self._bkg_aeff_index = self['bkg_aeff_index'].real() + self._bkg_aeff_norm = self['bkg_aeff_norm'].real() + self._bkg_range_factor = self['bkg_range_factor'].real() # Open master index file and look for prodname master_file = os.path.join(self._datapath, self._master_indx) if not os.path.isfile(master_file): - raise RuntimeError("FITS data store not available. No master index file found. Make sure the file is copied from the server and your datapath is set correctly.") + raise RuntimeError('FITS data store not available. No master index file found. Make sure the file is copied from the server and your datapath is set correctly.') # Open and load JSON file json_data = open(master_file).read() data = json.loads(json_data) - if not "datasets" in data: - raise RuntimeError("Key \"datasets\" not available in master index file.") + if not 'datasets' in data: + raise RuntimeError('Key "datasets" not available in master index file.') # Get configurations - configs = data["datasets"] + configs = data['datasets'] # Initialise HDUs - self._hdu_index = self._obs_index = "" + self._hdu_index = self._obs_index = '' # Get HDUs for config in configs: - if self._prodname == config["name"]: - self._hdu_index = str(os.path.join(self._datapath, config["hduindx"])) - self._obs_index = str(os.path.join(self._datapath, config["obsindx"])) + if self._prodname == config['name']: + self._hdu_index = str(os.path.join(self._datapath, config['hduindx'])) + self._obs_index = str(os.path.join(self._datapath, config['obsindx'])) break # Check HDUs - if self._hdu_index == "" or self._obs_index == "": - raise RuntimeError("*** ERROR: FITS data store \""+self._prodname+"\" not available. Run csiactdata to get a list of available storage names") - filename = gammalib.GFilename(self._hdu_index+"[HDU_INDEX]") + if self._hdu_index == '' or self._obs_index == '': + raise RuntimeError('*** ERROR: FITS data store "'+self._prodname+'" not available. Run csiactdata to get a list of available storage names') + filename = gammalib.GFilename(self._hdu_index+'[HDU_INDEX]') if not filename.is_fits(): - raise RuntimeError("*** ERROR: HDU index file \""+self._hdu_index+"[HDU_INDEX]\" for FITS data store \""+self._prodname+"\" not available. Check your master index file or run csiactdata to get a list of available storage names.") + raise RuntimeError('*** ERROR: HDU index file "'+self._hdu_index+'[HDU_INDEX]" for FITS data store "'+self._prodname+'" not available. Check your master index file or run csiactdata to get a list of available storage names.') - # Check for existence of "BKG_SCALE" in the observation index file if required + # Check for existence of 'BKG_SCALE' in the observation index file if required if self._use_bkg_scale: - filename = gammalib.GFilename(self._obs_index+"[OBS_INDEX]") + filename = gammalib.GFilename(self._obs_index+'[OBS_INDEX]') if filename.is_fits(): fits = gammalib.GFits(self._obs_index) - if not fits["OBS_INDEX"].contains("BKG_SCALE"): + if not fits['OBS_INDEX'].contains('BKG_SCALE'): self._use_bkg_scale = False fits.close() else: @@ -204,7 +204,7 @@ class csiactobs(ctools.cscript): # Write input parameters into logger if self._logTerse(): self._log_parameters() - self._log("\n") + self._log('\n') # Return return @@ -223,7 +223,7 @@ class csiactobs(ctools.cscript): except: # raise error if wrong type detected - raise RuntimeError("Argument of csiactobs.runlist() must be of type 'list'") + raise RuntimeError('Argument of csiactobs.runlist() must be of type "list"') # Return return @@ -233,24 +233,24 @@ class csiactobs(ctools.cscript): # Handle constant spectral model if index == 0.0 and self._bkgpars <= 1: spec = gammalib.GModelSpectralConst() - spec["Value"].min(prefactor / self._bkg_range_factor) - spec["Value"].max(prefactor * self._bkg_range_factor) - spec["Value"].value(prefactor) + spec['Value'].min(prefactor / self._bkg_range_factor) + spec['Value'].max(prefactor * self._bkg_range_factor) + spec['Value'].value(prefactor) if self._bkgpars == 0: - spec["Value"].fix() + spec['Value'].fix() else: - spec["Value"].free() + spec['Value'].free() else: # Create power law model if self._bkgpars <= 2: - e = gammalib.GEnergy(1.0,"TeV") + e = gammalib.GEnergy(1.0,'TeV') spec = gammalib.GModelSpectralPlaw(prefactor, index, e) # Set parameter ranges - spec[0].min(0.01 * prefactor) - spec[0].max(100.0 * prefactor) + spec[0].min(prefactor / self._bkg_range_factor) + spec[0].max(prefactor * self._bkg_range_factor) spec[1].scale(1) spec[1].min(-5.0) spec[1].max(5.0) @@ -269,20 +269,22 @@ class csiactobs(ctools.cscript): else: # Create reference powerlaw - plaw = gammalib.GModelSpectralPlaw(prefactor, index, gammalib.GEnergy(1.0,"TeV")) + plaw = gammalib.GModelSpectralPlaw(prefactor, index, gammalib.GEnergy(1.0,'TeV')) # Create spectral model and energy values spec = gammalib.GModelSpectralNodes() - bounds = gammalib.GEbounds(self._bkgpars,gammalib.GEnergy(emin,"TeV"),gammalib.GEnergy(emax,"TeV"), True) + bounds = gammalib.GEbounds(self._bkgpars,gammalib.GEnergy(emin,'TeV'),gammalib.GEnergy(emax,'TeV'), True) for i in range(bounds.size()): energy = bounds.elogmean(i) value = plaw.eval(energy, gammalib.GTime()) spec.append(energy, value) for par in spec: - if "Energy" in par.name(): + if 'Energy' in par.name(): par.fix() - elif "Intensity" in par.name(): + elif 'Intensity' in par.name(): + value = par.value() + par.scale(value) par.min(value / self._bkg_range_factor) par.max(value * self._bkg_range_factor) @@ -292,7 +294,7 @@ class csiactobs(ctools.cscript): def _iact_background(self, telescope, obs_id, bkg_scale, bkgtype, emin=0.01, emax=100): # handle IrfBackground - if bkgtype == "irf": + if bkgtype == 'irf': prefactor = 1.0 index = 0.0 prefactor *= bkg_scale @@ -302,7 +304,7 @@ class csiactobs(ctools.cscript): bck = gammalib.GCTAModelIrfBackground(spec) # Set AeffBackground - elif bkgtype == "aeff": + elif bkgtype == 'aeff': prefactor = bkg_scale * self._bkg_aeff_norm spec = self._background_spectrum(obs_id, prefactor, self._bkg_aeff_index, emin, emax) @@ -310,14 +312,14 @@ class csiactobs(ctools.cscript): bck = gammalib.GCTAModelAeffBackground(spec) # Set Gaussian Background - elif bkgtype == "gauss": + elif bkgtype == 'gauss': prefactor = bkg_scale * self._bkg_gauss_norm spec = self._background_spectrum(obs_id, prefactor, self._bkg_gauss_index, emin, emax) radial = gammalib.GCTAModelRadialGauss(self._bkg_gauss_sigma) bck = gammalib.GCTAModelRadialAcceptance(radial, spec) else: - sys.exit("Background type \""+self._bkgtype+"\" unsupported") + sys.exit('Background type "'+self._bkgtype+'" unsupported') # Copy model model = bck.clone() @@ -329,7 +331,7 @@ class csiactobs(ctools.cscript): model.instruments(telescope) # Set name (arbitrary) - model.name("bkg_"+str(obs_id)) + model.name('bkg_'+str(obs_id)) # Turn off TS calculation for background model model.tscalc(False) @@ -355,36 +357,36 @@ class csiactobs(ctools.cscript): # Clear xml file and append an observation list self._xml.clear() - self._xml.append(gammalib.GXmlElement("observation_list title=\"observation list\"")) - lib = self._xml.element("observation_list", 0) + self._xml.append(gammalib.GXmlElement('observation_list title="observation list"')) + lib = self._xml.element('observation_list', 0) # Log output if self._logTerse(): - self._log("\n") - self._log.header1("Looping over "+str(len(self._runlist))+" runs") + self._log('\n') + self._log.header1('Looping over '+str(len(self._runlist))+' runs') # Initialise energy range values for logging - self._ebounds.clear() + self._ebounds.clear() # Loop over runs for obs_id in self._runlist: # Create selection string - obs_selection = "[OBS_ID=="+str(obs_id)+"]" + obs_selection = '[OBS_ID=='+str(obs_id)+']' # Open HDU index file - hduindx = gammalib.GFits(self._hdu_index+"[HDU_INDEX]"+obs_selection) - hduindx_hdu = hduindx["HDU_INDEX"] + hduindx = gammalib.GFits(self._hdu_index+'[HDU_INDEX]'+obs_selection) + hduindx_hdu = hduindx['HDU_INDEX'] # Initialise files and hdu names - eventfile = aefffile = psffile = edispfile = bkgfile = "" - eventhdu = aeffhdu = "" + eventfile = aefffile = psffile = edispfile = bkgfile = '' + eventhdu = aeffhdu = '' types = [] formats = [] for i in range(hduindx_hdu.nrows()): - types.append(hduindx_hdu["HDU_TYPE"][i]) - formats.append(hduindx_hdu["HDU_CLASS"][i]) + types.append(hduindx_hdu['HDU_TYPE'][i]) + formats.append(hduindx_hdu['HDU_CLASS'][i]) # Handle events index = -1 @@ -394,12 +396,12 @@ class csiactobs(ctools.cscript): index = formats.index(version) break if index >= 0: - eventfile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index]) - eventhdu = hduindx_hdu["HDU_NAME"][index] - eventfile += "["+eventhdu+"]" + eventfile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index]) + eventhdu = hduindx_hdu['HDU_NAME'][index] + eventfile += '['+eventhdu+']' if not gammalib.GFilename(eventfile).is_fits(): if self._logTerse(): - self._log("Skipping observation "+str(obs_id)+": eventfile \""+eventfile+"\" not found\n") + self._log('Skipping observation '+str(obs_id)+': eventfile "'+eventfile+'" not found\n') continue # Handle Aeff @@ -410,12 +412,12 @@ class csiactobs(ctools.cscript): index = formats.index(version) break if index >= 0: - aefffile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index]) - aeffhdu = hduindx_hdu["HDU_NAME"][index] - aefffile += "["+aeffhdu+"]" + aefffile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index]) + aeffhdu = hduindx_hdu['HDU_NAME'][index] + aefffile += '['+aeffhdu+']' if not gammalib.GFilename(aefffile).is_fits(): if self._logTerse(): - self._log("Skipping observation "+str(obs_id)+": effective area \""+aefffile+"\" not found\n") + self._log('Skipping observation '+str(obs_id)+': effective area "'+aefffile+'" not found\n') continue # Handle psf @@ -426,11 +428,11 @@ class csiactobs(ctools.cscript): index = formats.index(version) break if index >= 0: - psffile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index]) - psffile += "["+hduindx_hdu["HDU_NAME"][index]+"]" + psffile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index]) + psffile += '['+hduindx_hdu['HDU_NAME'][index]+']' if not gammalib.GFilename(psffile).is_fits(): if self._logTerse(): - self._log("Skipping observation "+str(obs_id)+": point spread function \""+psffile+"\" not found\n") + self._log('Skipping observation '+str(obs_id)+': point spread function "'+psffile+'" not found\n') continue # Handle edisp @@ -441,12 +443,12 @@ class csiactobs(ctools.cscript): index = formats.index(version) break if index >= 0: - edispfile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index]) - edispfile += "["+hduindx_hdu["HDU_NAME"][index]+"]" + edispfile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index]) + edispfile += '['+hduindx_hdu['HDU_NAME'][index]+']' if not gammalib.GFilename(edispfile).is_fits(): if self._logTerse(): - self._log("Warning: observation "+str(obs_id)+" has no energy dispersion \""+edispfile+"\" information\n") - edispfile = "" + self._log('Warning: observation '+str(obs_id)+' has no energy dispersion "'+edispfile+'" information\n') + edispfile = '' # Handle background index = -1 @@ -457,14 +459,20 @@ class csiactobs(ctools.cscript): index = formats.index(version) break if index >= 0: - bkgfile = os.path.join(self._subdir, hduindx_hdu["FILE_DIR"][index], hduindx_hdu["FILE_NAME"][index]) - bkgfile += "["+hduindx_hdu["HDU_NAME"][index]+"]" - if not gammalib.GFilename(bkgfile).is_fits(): + bkgfile = os.path.join(self._subdir, hduindx_hdu['FILE_DIR'][index], hduindx_hdu['FILE_NAME'][index]) + bkgfile += '['+hduindx_hdu['HDU_NAME'][index]+']' + if 'irf' in bkg_mod_hierarchy and not gammalib.GFilename(bkgfile).is_fits(): + bkg_mod_hierarchy.remove('irf') if self._logTerse(): - self._log("Warning: observation "+str(obs_id)+" has no background information (file=\""+bkgfile+"\"). IRF background cannot be used\n") - bkgfile = "" - if "irf" in bkg_mod_hierarchy: - bkg_mod_hierarchy.remove("irf") + self._log('Warning: observation '+str(obs_id)+' has no background information (file="'+bkgfile+'"). IRF background cannot be used\n') + bkgfile = '' + if len(bkg_mod_hierarchy) == 0: + if self._logTerse(): + self._log('Skipping observation '+str(obs_id)+': No background can be used\n') + continue + else: + if self._logTerse(): + self._log('Observation '+str(obs_id)+': Falling back to background "'+bkg_mod_hierarchy[0]+'"\n') # Close hdu index file hduindx.close() @@ -472,15 +480,15 @@ class csiactobs(ctools.cscript): # Handle background scale information if available bkg_scale = 1.0 if self._use_bkg_scale: - obsindx = gammalib.GFits(self._obs_index+"[OBS_INDEX]"+obs_selection) - bkg_scale = obsindx["OBS_INDEX"]["BKG_SCALE"][0] + obsindx = gammalib.GFits(self._obs_index+'[OBS_INDEX]'+obs_selection) + bkg_scale = obsindx['OBS_INDEX']['BKG_SCALE'][0] obsindx.close() # Open fits file to determine the observation name fits = gammalib.GFits(eventfile) events = fits[eventhdu] - object_name = events.string("OBJECT") - telescope = events.string("TELESCOP") + object_name = events.string('OBJECT') + telescope = events.string('TELESCOP') # Close FITS file fits.close() @@ -490,19 +498,19 @@ class csiactobs(ctools.cscript): aeff_table = aeff_fits[aeffhdu] # Set energy range from header keyword if present - if aeff_table.has_card("LO_THRES") and aeff_table.has_card("HI_THRES"): + if aeff_table.has_card('LO_THRES') and aeff_table.has_card('HI_THRES'): # Get safe energy range - run_emin = gammalib.GEnergy(aeff_table.real("LO_THRES"), "TeV") - run_emax = gammalib.GEnergy(aeff_table.real("HI_THRES"), "TeV") + run_emin = gammalib.GEnergy(aeff_table.real('LO_THRES'), 'TeV') + run_emax = gammalib.GEnergy(aeff_table.real('HI_THRES'), 'TeV') # Append to ebounds self._ebounds.append(run_emin, run_emax) else: # Set default values for energy range - run_emin = gammalib.GEnergy(10,"GeV") - run_emax = gammalib.GEnergy(100,"TeV") + run_emin = gammalib.GEnergy(10,'GeV') + run_emax = gammalib.GEnergy(100,'TeV') # Close Aeff fits file aeff_fits.close() @@ -512,43 +520,43 @@ class csiactobs(ctools.cscript): # Logging if self._logTerse(): - self._log("Adding observation "+str(obs_id)+" (\""+object_name+"\")\n") + self._log('Adding observation '+str(obs_id)+' ("'+object_name+'")\n') if self._logExplicit(): - self._log(" Event file: "+eventfile+"\n") - self._log(" Effective area: "+aefffile+"\n") - self._log(" Point spread function: "+psffile+"\n") - self._log(" Energy dispersion: "+edispfile+"\n") - self._log(" Background file: "+bkgfile+"\n") + self._log(' Event file: '+eventfile+'\n') + self._log(' Effective area: '+aefffile+'\n') + self._log(' Point spread function: '+psffile+'\n') + self._log(' Energy dispersion: '+edispfile+'\n') + self._log(' Background file: '+bkgfile+'\n') if self._use_bkg_scale: - self._log(" Background scale: "+str(bkg_scale)+"\n") - self._log("\n") + self._log(' Background scale: '+str(bkg_scale)+'\n') + self._log('\n') # Append observation to XML and set attributes - obs = lib.append("observation") - obs.attribute("name", object_name) - obs.attribute("id", str(obs_id)) - obs.attribute("instrument", telescope) + obs = lib.append('observation') + obs.attribute('name', object_name) + obs.attribute('id', str(obs_id)) + obs.attribute('instrument', telescope) # Append event file - ev = gammalib.GXmlElement("parameter name=\"EventList\"") - ev.attribute("file", eventfile) + ev = gammalib.GXmlElement('parameter name="EventList"') + ev.attribute('file', eventfile) # Append effective area - aeff = gammalib.GXmlElement("parameter name=\"EffectiveArea\"") - aeff.attribute("file",aefffile) + aeff = gammalib.GXmlElement('parameter name="EffectiveArea"') + aeff.attribute('file',aefffile) # Append PSF - psf = gammalib.GXmlElement("parameter name=\"PointSpreadFunction\"") - psf.attribute("file",psffile) + psf = gammalib.GXmlElement('parameter name="PointSpreadFunction"') + psf.attribute('file',psffile) # Append energy dispersion - edisp = gammalib.GXmlElement("parameter name=\"EnergyDispersion\"") - edisp.attribute("file",edispfile) + edisp = gammalib.GXmlElement('parameter name="EnergyDispersion"') + edisp.attribute('file',edispfile) # Append background - bck = gammalib.GXmlElement("parameter name=\"Background\"") - bck.attribute("file",bkgfile) + bck = gammalib.GXmlElement('parameter name="Background"') + bck.attribute('file',bkgfile) # assign events and IRFs to observation obs.append(ev) @@ -562,35 +570,35 @@ class csiactobs(ctools.cscript): # Log header of energy range if self._logTerse(): - self._log("\n") - self._log.header3("Energy range of obervation list") + self._log('\n') + self._log.header3('Energy range of obervation list') # Logging if energy range is available if self._ebounds.size() > 0: # Write energy range into log file - self._log(str(self._ebounds.emin())+" - "+str(self._ebounds.emax())) - self._log("\n") + self._log(str(self._ebounds.emin())+' - '+str(self._ebounds.emax())) + self._log('\n') else: # Write 'not available' into log file - self._log("not available") - self._log("\n") + self._log('not available') + self._log('\n') else: - self._log.header2("WARNING: No observation from given runlist available") + self._log.header2('WARNING: No observation from given runlist available') # Append models provided by 'inmodels' if necessary if not self._inmodels == None: if self._logTerse(): - self._log("\n") - self._log.header1("Appending models") + self._log('\n') + self._log.header1('Appending models') # Loop over input models for model in self._inmodels: if self._logTerse(): - self._log.header3("Adding model \""+model.name()+"\"") + self._log.header3('Adding model "'+model.name()+'"') self._models.append(model) # Return diff --git a/cscripts/csmodelinfo.py b/cscripts/csmodelinfo.py index d93ae89..a431aaf 100755 --- a/cscripts/csmodelinfo.py +++ b/cscripts/csmodelinfo.py @@ -430,10 +430,6 @@ class csmodelinfo(ctools.cscript): """ Save models to ds9 region file if required """ - # Write header - if self._logTerse(): - self._log("\n") - self._log.header1("Save models in DS9 file") # Get output filename in case it was not read ahead self._ds9file = self["ds9file"].filename() @@ -441,6 +437,11 @@ class csmodelinfo(ctools.cscript): # Check if DS9 file is valid if self._ds9file != "NONE": + # Write header + if self._logTerse(): + self._log("\n") + self._log.header1("Save models in DS9 file") + # Log filename if self._logTerse(): self._log(gammalib.parformat("DS9 filename")) diff --git a/cscripts/csmodelmerge.py b/cscripts/csmodelmerge.py index cf18fbd..f2e8ef5 100755 --- a/cscripts/csmodelmerge.py +++ b/cscripts/csmodelmerge.py @@ -90,6 +90,9 @@ class csmodelmerge(ctools.cscript): # Read ahead output filename if self._read_ahead(): self["outmodel"].filename() + + # Get clobber parameter + self._clobber = self["clobber"].boolean() # Write input parameters into logger if self._logTerse(): @@ -163,14 +166,26 @@ class csmodelmerge(ctools.cscript): # Get output filename in case it was not read ahead outmodel = self["outmodel"].filename() - # Log filename - if self._logTerse(): - self._log(gammalib.parformat("Model definition XML file")) - self._log(outmodel.url()) - self._log("\n") - - # Save models - self._models.save(outmodel) + # Check if file exists and apply clobber if false + if outmodel.exists() and not self._clobber: + + # Set error message + msg = "Cannot save \""+outmodel.url()+"\": File already exists " + msg += "Use parameter clobber=yes to allow overwriting of files." + + # Throw exception since file cannnot be overwritten + raise RuntimeError(msg) + + else: + + # Log filename + if self._logTerse(): + self._log(gammalib.parformat("Model definition XML file")) + self._log(outmodel.url()) + self._log("\n") + + # Save models + self._models.save(outmodel) # Return return diff --git a/cscripts/csobsinfo.py b/cscripts/csobsinfo.py index b69afda..45e9f24 100755 --- a/cscripts/csobsinfo.py +++ b/cscripts/csobsinfo.py @@ -364,16 +364,17 @@ class csobsinfo(ctools.cscript): observation container into a DS9 region file. If "NONE" is specified for the "ds9file" parameter the method does nothing. """ - # Write header - if self._logTerse(): - self._log("\n") - self._log.header1("Save pointings in DS9 file") # Get output filename in case it was not read ahead ds9file = self["ds9file"].filename() # Check if DS9 file is valid - if ds9file.url() != "NONE": + if ds9file.url() != "NONE": + + # Write header + if self._logTerse(): + self._log("\n") + self._log.header1("Save pointings in DS9 file") # Log filename if self._logTerse():