From 0746e91989a80ef482768bb042a587cfd65f922f Mon Sep 17 00:00:00 2001 From: Jürgen Knödlseder Date: Fri, 5 Aug 2016 11:03:24 +0200 Subject: [PATCH] Fix Python 3 problem with csiactcopy.py script --- cscripts/csiactcopy.py | 223 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------------------------------------------------------ cscripts/csiactobs.py | 12 ++++++------ test/test_csiactcopy.py | 36 ++++++++++++++++++++++++------------ 3 files changed, 151 insertions(+), 120 deletions(-) diff --git a/cscripts/csiactcopy.py b/cscripts/csiactcopy.py index 6e1e97a..5f0273e 100755 --- a/cscripts/csiactcopy.py +++ b/cscripts/csiactcopy.py @@ -40,11 +40,11 @@ class csiactcopy(ctools.cscript): # Constructor def __init__(self, *argv): """ - Constructor. + Constructor """ # Set name self._name = 'csiactcopy' - self._version = '1.1.0' + self._version = '1.2.0' self._datapath = os.getenv('VHEFITS','') self._remote_master = gammalib.GFilename() self._remote_base = '' @@ -66,14 +66,13 @@ class csiactcopy(ctools.cscript): """ Get parameters from parfile and setup the observation """ - # Get Parameters self._remote_master = self['remote_master'].filename() if not self._remote_master.exists(): - raise RuntimeError('*** ERROR: FITS data not available. No ' - 'master index file found in "'+ - self._remote_master+'". Make sure remote ' - 'file system is properly mounted.') + msg = ('*** ERROR: FITS data not available. No master index file ' + 'found in "%s". Make sure remote file system is properly ' + 'mounted.' % self._remote_master) + raise RuntimeError(msg) # Get parameters self._prodname = self['prodname'].string() @@ -88,11 +87,11 @@ class csiactcopy(ctools.cscript): def _copy(self, source, clobber): """ - Copy file to outpath + Copy file to self._outpath directory Parameters ---------- - source : string + source : str Path of file to be copied Returns @@ -147,39 +146,39 @@ class csiactcopy(ctools.cscript): def _merge(self, localfits, remotefits, hduname, clobber): """ Merge remote and local fits files + If the local fits file is not present, a new one is created. Parameters ---------- - localfits : string + localfits : str Path of local index FITS file - remotefits : string + remotefits : str Path of remote index FITS file - hduname : string + hduname : str Name of HDU extension to be merged clobber : bool Flag if remote content should overwrite local content - """ - - # Logging - self._log_value(gammalib.TERSE, 'Local file', localfits) - self._log_value(gammalib.TERSE, 'Remote file', remotefits) - self._log_value(gammalib.TERSE, 'HDU name', hduname) + """ + # Write input parameters into logger + self._log_value(gammalib.NORMAL, 'Local file', localfits) + self._log_value(gammalib.NORMAL, 'Remote file', remotefits) + self._log_value(gammalib.NORMAL, 'HDU name', hduname) if clobber: preference = 'Remote' else: preference = 'Local' - self._log_value(gammalib.TERSE, 'Conflict preference', preference) + self._log_value(gammalib.NORMAL, 'Conflict preference', preference) # Check if local fits file is available if os.path.isfile(localfits): - local = gammalib.GFits(str(localfits)) + local = gammalib.GFits(localfits) local_hdu = local[hduname] else: # Otherwise load remote fits file and delete rows - local = gammalib.GFits(str(remotefits)) + local = gammalib.GFits(remotefits) local_hdu = local[hduname] - local_hdu.remove_rows(0,local_hdu.nrows()) + local_hdu.remove_rows(0, local_hdu.nrows()) # find local obs_id local_obs = [] @@ -187,13 +186,14 @@ class csiactcopy(ctools.cscript): for obs_id in lobs_col: local_obs.append(obs_id) - # Initialise list of runlists - # This is kluge: to circumvent a too long file name, we split the - # runlists into chunks of 50 observations maximum - runlists = [] + # Initialise run list. A kluge has been implemented to circumvent a + # too long file name: the run list is split into chunks of 50 + # observations maximum. Note also the "//" division operator that + # makes sure that "i" is an integer on Python 3. + runlists = [] runs_per_chunk = 50 - for i in range(len(self._runs) / runs_per_chunk + 1): - chunk = self._runs[i * runs_per_chunk : (i + 1) * runs_per_chunk] + for i in range(len(self._runs) // runs_per_chunk + 1): + chunk = self._runs[i * runs_per_chunk : (i+1) * runs_per_chunk] runlists.append(chunk) # Loop over runlist chunks @@ -241,8 +241,8 @@ class csiactcopy(ctools.cscript): remote_obs.append(obs_id) # Log entries of index files - self._log_value(gammalib.TERSE, 'Remote entries', len(remote_obs)) - self._log_value(gammalib.TERSE, 'Local entries', len(local_obs)) + self._log_value(gammalib.NORMAL, 'Remote entries', len(remote_obs)) + self._log_value(gammalib.NORMAL, 'Local entries', len(local_obs)) # initialise counter for logging removed_rows = 0 @@ -311,13 +311,13 @@ class csiactcopy(ctools.cscript): what = 'Removed local rows' else: what = 'Skipped remote rows' - self._log_value(gammalib.TERSE, what, removed_rows) + self._log_value(gammalib.NORMAL, what, removed_rows) # Loop over columns for i in range(local_hdu.ncols()): # Get local and remote columns - local_col = local_hdu[i] + local_col = local_hdu[i] remote_col = remote_hdu[i] # Loop over entries and merge @@ -325,77 +325,103 @@ class csiactcopy(ctools.cscript): local_col[j+old_local_rows] = remote_col[j] # Save local fits file - local.saveto(str(localfits), True) + local.saveto(localfits, True) # Return return - - # Public methods - def run(self): - """ - Run the script + def _set_runs(self, filename): """ - # Switch screen logging on in debug mode - if self._logDebug(): - self._log.cout(True) + Set the run list - # Get parameters - self._get_parameters() + Parameters + ---------- + filename : `~gammalib.GFilename` + Run list file name - # Make destination directory if not available - if not os.path.isdir(self._outpath): - os.makedirs(self._outpath) - - # Log output - if self._logTerse(): - self._log('\n') - self._log.header2('Runlist information') + Returns + ------- + runs : list of int + Run list + """ + # Write header into logger + self._log_header1(gammalib.TERSE, 'Set runlist') - # Initialise runlist array - self._runs = [] + # Initialise runlist + runs = [] - # Check if runlist file is available - if self._runlist.exists(): - runfile = open(self._runlist.url()) + # If runlist file exists then open file and extract the run + # information into the run list + if filename.exists(): + + # Open the runlist file + runfile = open(filename.url()) - # Read runlist + # Read runlist file and append all run ID as integers to the + # run list. Skip any blank or comment lines for line in runfile.readlines(): if len(line) == 0: continue if line[0] == '#': continue if len(line.split()) > 0: - self._runs.append(int(line.split()[0])) + run = int(line.split()[0]) + runs.append(run) + self._log_value(gammalib.EXPLICIT, 'Run %d' % len(runs), run) # Close runlist file runfile.close() # Logging - self._log_value(gammalib.TERSE, 'Number of observations', - len(self._runs)) + self._log_value(gammalib.NORMAL, 'Number of observations', + len(runs)) - # Copy all data if runlist file is 'NONE' - elif self._runlist == 'NONE': - self._log_string(gammalib.TERSE, 'Copy all available data') + # ... otherwise, if the runlist file is 'NONE' then leave the + # run list empty. This implies copying all available data. + elif filename == 'NONE': + self._log_string(gammalib.NORMAL, 'Copy all available data') - # Raise exception if file not valid + # ... otherwise raise an exception else: - raise RuntimeError('*** ERROR: Runlist file "'+ - self._runlist+'" not available.') + msg = '*** ERROR: Runlist file "%s" not available' % filename + raise RuntimeError(msg) + # Return run list + return runs + + + # Public methods + def run(self): + """ + Run the script + """ + # Switch screen logging on in debug mode + if self._logDebug(): + self._log.cout(True) + + # Get parameters + self._get_parameters() + + # Make destination directory if not available + if not os.path.isdir(self._outpath): + os.makedirs(self._outpath) + + # Set run list + self._runs = self._set_runs(self._runlist) + # Check for availability of remote master file if not self._remote_master.exists(): - raise RuntimeError('*** ERROR: Remote master file "'+ - self._remote_master+'" does not exist.') + msg = ('*** ERROR: Remote master file "%s" does not exist' % + self._remote_master) + raise RuntimeError(msg) # Retrieve json data from remote master json_data = open(self._remote_master.url()).read() data = json.loads(json_data) if not 'datasets' in data: - raise RuntimeError('*** ERROR: Key "datasets" not available ' - 'in remote master index file "'+ - self._remote_master+'".') + msg = ('*** ERROR: Key "datasets" not available in remote master ' + 'index file "%s"' % self._remote_master) + raise RuntimeError(msg) # Get array of configurations configs = data['datasets'] @@ -409,17 +435,16 @@ class csiactcopy(ctools.cscript): # Initialise file names to be copied files = set() - # Logging - if self._logTerse(): - self._log('\n') - self._log.header2('Loop over remote configs') + # Write header into logger + self._log_header2(gammalib.TERSE, 'Loop over remote configs') # Loop over configs - for config in configs: - - # Logging - if self._logVerbose(): - self._log.header2(str(config['name'])) + for config in configs: + + # Write header into logger. The str() is needed since + # "config['name']" is , and under Python 2 a + # string is expected. + self._log_header2(gammalib.VERBOSE, str(config['name'])) # Check if prodname was found if config['name'] == self._prodname: @@ -432,13 +457,13 @@ class csiactcopy(ctools.cscript): remote_obs = os.path.join(self._remote_base, config['obsindx']) # Log information - self._log_header3(gammalib.TERSE, 'Remote config "'+ + self._log_header3(gammalib.NORMAL, 'Remote config "'+ self._prodname+'"') - self._log_value(gammalib.TERSE, 'HDU index', remote_hdu) - self._log_value(gammalib.TERSE, 'Observation index', remote_obs) + self._log_value(gammalib.NORMAL, 'HDU index', remote_hdu) + self._log_value(gammalib.NORMAL, 'Observation index', remote_obs) # Open remote HDU index file - fits = gammalib.GFits(str(remote_hdu)) + fits = gammalib.GFits(remote_hdu) table = fits['HDU_INDEX'] # Initialise flag if SIZE column is present @@ -457,11 +482,10 @@ class csiactcopy(ctools.cscript): for run in self._runs: # Check for run not in remote data store - if not run in remote_ids: - if self._logNormal(): - self._log('Skip observation '+str(run)+': ') - self._log('ID not available remotely') - self._log('\n') + if not run in remote_ids: + msg = ('Skip observation "%s": ID not available ' + 'remotely' % str(run)) + self._log_string(msg) # Loop over remote HDU index file for row in range(table.nrows()): @@ -510,11 +534,11 @@ class csiactcopy(ctools.cscript): cp_size += table['SIZE'][row] # Log file information - self._log_header2(gammalib.TERSE, 'File information') - self._log_value(gammalib.TERSE, 'Number of files', len(files)) + self._log_header2(gammalib.NORMAL, 'File information') + self._log_value(gammalib.NORMAL, 'Number of files', len(files)) if has_size: size = float(cp_size) * 1.0e-6 - self._log_value(gammalib.TERSE, 'Size', '%.2f MB' % size) + self._log_value(gammalib.NORMAL, 'Size', '%.2f MB' % size) self._log_header3(gammalib.VERBOSE, 'File names') for filename in files: self._log_string(gammalib.VERBOSE, str(filename)+'\n') @@ -524,8 +548,8 @@ class csiactcopy(ctools.cscript): # If prodname is not found just log that we skip the config else: - self._log_header3(gammalib.EXPLICIT, 'Skipping config "'+ - str(config['name'])+'"') + msg = 'Skipping config "%s"' % config['name'] + self._log_header3(gammalib.EXPLICIT, msg) # Raise Exception if prodname was not found if not has_prod: @@ -586,15 +610,10 @@ class csiactcopy(ctools.cscript): k += 1 # Logging - if self._logNormal() and not self._logVerbose(): - self._log.parformat('Status') - self._log('Finished') - self._log('\n') + self._log_value(gammalib.NORMAL, 'Status', 'Finished') - # Logging about index files - if self._logTerse(): - self._log('\n') - self._log.header1('Updating index files') + # Write header into logger + self._log_header1(gammalib.TERSE, 'Updating index files') # Build local hdu index file name local_hdu = os.path.join(self._outpath, diff --git a/cscripts/csiactobs.py b/cscripts/csiactobs.py index 581c3cc..1feb9b1 100755 --- a/cscripts/csiactobs.py +++ b/cscripts/csiactobs.py @@ -43,7 +43,7 @@ class csiactobs(ctools.cscript): # Constructor def __init__(self, *argv): """ - Constructor. + Constructor """ # Set name and version self._name = 'csiactobs' @@ -229,9 +229,9 @@ class csiactobs(ctools.cscript): Parameters ---------- prefactor : float - Prefactor parameter of spectral model + Power law prefactor of spectral model index : float - Index parameter of spectral model + Power law index of spectral model emin : float Minimum energy (in case a spectral node function is required) emax : float @@ -330,13 +330,13 @@ class csiactobs(ctools.cscript): Parameters ---------- - telescope : string + telescope : str Name of telescope - obs_id : string + obs_id : str Observation ID bkg_scale : float Background scaling factor - bkgtype : string + bkgtype : str Type of background (irf,aeff, or gauss) Returns diff --git a/test/test_csiactcopy.py b/test/test_csiactcopy.py index fc83f0a..ae8a3cb 100755 --- a/test/test_csiactcopy.py +++ b/test/test_csiactcopy.py @@ -162,8 +162,9 @@ class Test(test): self._check_copy('iactdata_py1') self._check_n_obs('iactdata_py1', 9) + # Remove the copied IACT data store before continuing if os.path.isdir('iactdata_py2'): - os.system("rm -r iactdata_py2") + os.system('rm -r iactdata_py2') # Set-up csiactcopy iactcopy = cscripts.csiactcopy() @@ -172,7 +173,7 @@ class Test(test): iactcopy['prodname'] = 'unit-test' iactcopy['outpath'] = 'iactdata_py2' iactcopy['logfile'] = 'csiactcopy_py4.log' - iactcopy['chatter'] = 3 + iactcopy['chatter'] = 3 # Run csiactcopy script and save run list iactcopy.logFileOpen() # Make sure we get a log file @@ -223,10 +224,15 @@ class Test(test): # Return return - # Check copy + # Check copy of IACT data store def _check_copy(self, pathname): """ - Check copy. + Check copy of IACT data store + + Parameters + ---------- + pathname : str + Path to copied IACT data store """ # Set file names hdu_index_name = gammalib.GFilename(pathname+'/hdu-index.fits') @@ -235,17 +241,17 @@ class Test(test): # Check for existence of files self.test_assert(hdu_index_name.exists(), - 'Check if file "hdu-index.fits" exists.') + 'Check if file "hdu-index.fits" exists') self.test_assert(obs_index_name.exists(), - 'Check if file "obs-index.fits" exists.') + 'Check if file "obs-index.fits" exists') self.test_assert(master_name.exists(), - 'Check if file "master.json" exists.') + 'Check if file "master.json" exists') # Check if index files are FITS file self.test_assert(hdu_index_name.is_fits(), - 'Check if file "hdu-index.fits" is FITS file.') + 'Check if file "hdu-index.fits" is FITS file') self.test_assert(obs_index_name.is_fits(), - 'Check if file "obs-index.fits" is FITS file.') + 'Check if file "obs-index.fits" is FITS file') # Return return @@ -253,7 +259,14 @@ class Test(test): # Check copy def _check_n_obs(self, pathname, n_expected): """ - Check number of available observations. + Check number of available observations + + Parameters + ---------- + pathname : str + Path to copied IACT data store + n_expected : int + Expected number of observation in IACT data store """ # Set file name obs_index_name = gammalib.GFilename(pathname+'/obs-index.fits[OBS_INDEX]') @@ -268,8 +281,7 @@ class Test(test): fits.close() # Check for existence of observations - self.test_assert(n_obs == n_expected, - 'Check for '+str(n_expected)+' observations.') + self.test_value(n_obs, n_expected, 'Check for number of observations') # Return return \ No newline at end of file -- libgit2 0.21.4