From 673a1331336dd9d563c3f7543feab02c9aaaf8eb Mon Sep 17 00:00:00 2001 From: Jürgen Knödlseder Date: Tue, 9 Aug 2016 00:50:17 +0200 Subject: [PATCH] Put back former version of csiactcopy.py --- cscripts/csiactcopy.py | 223 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------------------------------------------------------------------------- 1 file changed, 102 insertions(+), 121 deletions(-) diff --git a/cscripts/csiactcopy.py b/cscripts/csiactcopy.py index 5f0273e..6e1e97a 100755 --- a/cscripts/csiactcopy.py +++ b/cscripts/csiactcopy.py @@ -40,11 +40,11 @@ class csiactcopy(ctools.cscript): # Constructor def __init__(self, *argv): """ - Constructor + Constructor. """ # Set name self._name = 'csiactcopy' - self._version = '1.2.0' + self._version = '1.1.0' self._datapath = os.getenv('VHEFITS','') self._remote_master = gammalib.GFilename() self._remote_base = '' @@ -66,13 +66,14 @@ class csiactcopy(ctools.cscript): """ Get parameters from parfile and setup the observation """ + # Get Parameters self._remote_master = self['remote_master'].filename() if not self._remote_master.exists(): - msg = ('*** ERROR: FITS data not available. No master index file ' - 'found in "%s". Make sure remote file system is properly ' - 'mounted.' % self._remote_master) - raise RuntimeError(msg) + raise RuntimeError('*** ERROR: FITS data not available. No ' + 'master index file found in "'+ + self._remote_master+'". Make sure remote ' + 'file system is properly mounted.') # Get parameters self._prodname = self['prodname'].string() @@ -87,11 +88,11 @@ class csiactcopy(ctools.cscript): def _copy(self, source, clobber): """ - Copy file to self._outpath directory + Copy file to outpath Parameters ---------- - source : str + source : string Path of file to be copied Returns @@ -146,39 +147,39 @@ class csiactcopy(ctools.cscript): def _merge(self, localfits, remotefits, hduname, clobber): """ Merge remote and local fits files - If the local fits file is not present, a new one is created. Parameters ---------- - localfits : str + localfits : string Path of local index FITS file - remotefits : str + remotefits : string Path of remote index FITS file - hduname : str + hduname : string Name of HDU extension to be merged clobber : bool Flag if remote content should overwrite local content - """ - # Write input parameters into logger - self._log_value(gammalib.NORMAL, 'Local file', localfits) - self._log_value(gammalib.NORMAL, 'Remote file', remotefits) - self._log_value(gammalib.NORMAL, 'HDU name', hduname) + """ + + # Logging + self._log_value(gammalib.TERSE, 'Local file', localfits) + self._log_value(gammalib.TERSE, 'Remote file', remotefits) + self._log_value(gammalib.TERSE, 'HDU name', hduname) if clobber: preference = 'Remote' else: preference = 'Local' - self._log_value(gammalib.NORMAL, 'Conflict preference', preference) + self._log_value(gammalib.TERSE, 'Conflict preference', preference) # Check if local fits file is available if os.path.isfile(localfits): - local = gammalib.GFits(localfits) + local = gammalib.GFits(str(localfits)) local_hdu = local[hduname] else: # Otherwise load remote fits file and delete rows - local = gammalib.GFits(remotefits) + local = gammalib.GFits(str(remotefits)) local_hdu = local[hduname] - local_hdu.remove_rows(0, local_hdu.nrows()) + local_hdu.remove_rows(0,local_hdu.nrows()) # find local obs_id local_obs = [] @@ -186,14 +187,13 @@ class csiactcopy(ctools.cscript): for obs_id in lobs_col: local_obs.append(obs_id) - # Initialise run list. A kluge has been implemented to circumvent a - # too long file name: the run list is split into chunks of 50 - # observations maximum. Note also the "//" division operator that - # makes sure that "i" is an integer on Python 3. - runlists = [] + # Initialise list of runlists + # This is kluge: to circumvent a too long file name, we split the + # runlists into chunks of 50 observations maximum + runlists = [] runs_per_chunk = 50 - for i in range(len(self._runs) // runs_per_chunk + 1): - chunk = self._runs[i * runs_per_chunk : (i+1) * runs_per_chunk] + for i in range(len(self._runs) / runs_per_chunk + 1): + chunk = self._runs[i * runs_per_chunk : (i + 1) * runs_per_chunk] runlists.append(chunk) # Loop over runlist chunks @@ -241,8 +241,8 @@ class csiactcopy(ctools.cscript): remote_obs.append(obs_id) # Log entries of index files - self._log_value(gammalib.NORMAL, 'Remote entries', len(remote_obs)) - self._log_value(gammalib.NORMAL, 'Local entries', len(local_obs)) + self._log_value(gammalib.TERSE, 'Remote entries', len(remote_obs)) + self._log_value(gammalib.TERSE, 'Local entries', len(local_obs)) # initialise counter for logging removed_rows = 0 @@ -311,13 +311,13 @@ class csiactcopy(ctools.cscript): what = 'Removed local rows' else: what = 'Skipped remote rows' - self._log_value(gammalib.NORMAL, what, removed_rows) + self._log_value(gammalib.TERSE, what, removed_rows) # Loop over columns for i in range(local_hdu.ncols()): # Get local and remote columns - local_col = local_hdu[i] + local_col = local_hdu[i] remote_col = remote_hdu[i] # Loop over entries and merge @@ -325,103 +325,77 @@ class csiactcopy(ctools.cscript): local_col[j+old_local_rows] = remote_col[j] # Save local fits file - local.saveto(localfits, True) + local.saveto(str(localfits), True) # Return return - def _set_runs(self, filename): + + # Public methods + def run(self): + """ + Run the script """ - Set the run list + # Switch screen logging on in debug mode + if self._logDebug(): + self._log.cout(True) - Parameters - ---------- - filename : `~gammalib.GFilename` - Run list file name + # Get parameters + self._get_parameters() - Returns - ------- - runs : list of int - Run list - """ - # Write header into logger - self._log_header1(gammalib.TERSE, 'Set runlist') + # Make destination directory if not available + if not os.path.isdir(self._outpath): + os.makedirs(self._outpath) - # Initialise runlist - runs = [] + # Log output + if self._logTerse(): + self._log('\n') + self._log.header2('Runlist information') - # If runlist file exists then open file and extract the run - # information into the run list - if filename.exists(): - - # Open the runlist file - runfile = open(filename.url()) + # Initialise runlist array + self._runs = [] + + # Check if runlist file is available + if self._runlist.exists(): + runfile = open(self._runlist.url()) - # Read runlist file and append all run ID as integers to the - # run list. Skip any blank or comment lines + # Read runlist for line in runfile.readlines(): if len(line) == 0: continue if line[0] == '#': continue if len(line.split()) > 0: - run = int(line.split()[0]) - runs.append(run) - self._log_value(gammalib.EXPLICIT, 'Run %d' % len(runs), run) + self._runs.append(int(line.split()[0])) # Close runlist file runfile.close() # Logging - self._log_value(gammalib.NORMAL, 'Number of observations', - len(runs)) + self._log_value(gammalib.TERSE, 'Number of observations', + len(self._runs)) - # ... otherwise, if the runlist file is 'NONE' then leave the - # run list empty. This implies copying all available data. - elif filename == 'NONE': - self._log_string(gammalib.NORMAL, 'Copy all available data') + # Copy all data if runlist file is 'NONE' + elif self._runlist == 'NONE': + self._log_string(gammalib.TERSE, 'Copy all available data') - # ... otherwise raise an exception + # Raise exception if file not valid else: - msg = '*** ERROR: Runlist file "%s" not available' % filename - raise RuntimeError(msg) + raise RuntimeError('*** ERROR: Runlist file "'+ + self._runlist+'" not available.') - # Return run list - return runs - - - # Public methods - def run(self): - """ - Run the script - """ - # Switch screen logging on in debug mode - if self._logDebug(): - self._log.cout(True) - - # Get parameters - self._get_parameters() - - # Make destination directory if not available - if not os.path.isdir(self._outpath): - os.makedirs(self._outpath) - - # Set run list - self._runs = self._set_runs(self._runlist) - # Check for availability of remote master file if not self._remote_master.exists(): - msg = ('*** ERROR: Remote master file "%s" does not exist' % - self._remote_master) - raise RuntimeError(msg) + raise RuntimeError('*** ERROR: Remote master file "'+ + self._remote_master+'" does not exist.') # Retrieve json data from remote master json_data = open(self._remote_master.url()).read() data = json.loads(json_data) if not 'datasets' in data: - msg = ('*** ERROR: Key "datasets" not available in remote master ' - 'index file "%s"' % self._remote_master) - raise RuntimeError(msg) + raise RuntimeError('*** ERROR: Key "datasets" not available ' + 'in remote master index file "'+ + self._remote_master+'".') # Get array of configurations configs = data['datasets'] @@ -435,16 +409,17 @@ class csiactcopy(ctools.cscript): # Initialise file names to be copied files = set() - # Write header into logger - self._log_header2(gammalib.TERSE, 'Loop over remote configs') + # Logging + if self._logTerse(): + self._log('\n') + self._log.header2('Loop over remote configs') # Loop over configs - for config in configs: - - # Write header into logger. The str() is needed since - # "config['name']" is , and under Python 2 a - # string is expected. - self._log_header2(gammalib.VERBOSE, str(config['name'])) + for config in configs: + + # Logging + if self._logVerbose(): + self._log.header2(str(config['name'])) # Check if prodname was found if config['name'] == self._prodname: @@ -457,13 +432,13 @@ class csiactcopy(ctools.cscript): remote_obs = os.path.join(self._remote_base, config['obsindx']) # Log information - self._log_header3(gammalib.NORMAL, 'Remote config "'+ + self._log_header3(gammalib.TERSE, 'Remote config "'+ self._prodname+'"') - self._log_value(gammalib.NORMAL, 'HDU index', remote_hdu) - self._log_value(gammalib.NORMAL, 'Observation index', remote_obs) + self._log_value(gammalib.TERSE, 'HDU index', remote_hdu) + self._log_value(gammalib.TERSE, 'Observation index', remote_obs) # Open remote HDU index file - fits = gammalib.GFits(remote_hdu) + fits = gammalib.GFits(str(remote_hdu)) table = fits['HDU_INDEX'] # Initialise flag if SIZE column is present @@ -482,10 +457,11 @@ class csiactcopy(ctools.cscript): for run in self._runs: # Check for run not in remote data store - if not run in remote_ids: - msg = ('Skip observation "%s": ID not available ' - 'remotely' % str(run)) - self._log_string(msg) + if not run in remote_ids: + if self._logNormal(): + self._log('Skip observation '+str(run)+': ') + self._log('ID not available remotely') + self._log('\n') # Loop over remote HDU index file for row in range(table.nrows()): @@ -534,11 +510,11 @@ class csiactcopy(ctools.cscript): cp_size += table['SIZE'][row] # Log file information - self._log_header2(gammalib.NORMAL, 'File information') - self._log_value(gammalib.NORMAL, 'Number of files', len(files)) + self._log_header2(gammalib.TERSE, 'File information') + self._log_value(gammalib.TERSE, 'Number of files', len(files)) if has_size: size = float(cp_size) * 1.0e-6 - self._log_value(gammalib.NORMAL, 'Size', '%.2f MB' % size) + self._log_value(gammalib.TERSE, 'Size', '%.2f MB' % size) self._log_header3(gammalib.VERBOSE, 'File names') for filename in files: self._log_string(gammalib.VERBOSE, str(filename)+'\n') @@ -548,8 +524,8 @@ class csiactcopy(ctools.cscript): # If prodname is not found just log that we skip the config else: - msg = 'Skipping config "%s"' % config['name'] - self._log_header3(gammalib.EXPLICIT, msg) + self._log_header3(gammalib.EXPLICIT, 'Skipping config "'+ + str(config['name'])+'"') # Raise Exception if prodname was not found if not has_prod: @@ -610,10 +586,15 @@ class csiactcopy(ctools.cscript): k += 1 # Logging - self._log_value(gammalib.NORMAL, 'Status', 'Finished') + if self._logNormal() and not self._logVerbose(): + self._log.parformat('Status') + self._log('Finished') + self._log('\n') - # Write header into logger - self._log_header1(gammalib.TERSE, 'Updating index files') + # Logging about index files + if self._logTerse(): + self._log('\n') + self._log.header1('Updating index files') # Build local hdu index file name local_hdu = os.path.join(self._outpath, -- libgit2 0.21.4