Fix Python 3 problem with csiactcopy.py script

Jürgen Knödlseder
1 parent 45a80b60
Showing 3 changed files with 151 additions and 120 deletions
cscripts/csiactcopy.py
cscripts/csiactobs.py
test/test_csiactcopy.py
@@ -40,11 +40,11 @@ class csiactcopy(ctools.cscript):
     # Constructor
     def __init__(self, *argv):
         """
-        Constructor.
+        Constructor
         """
         # Set name
         self._name          = 'csiactcopy'
-        self._version       = '1.1.0'
+        self._version       = '1.2.0'
         self._datapath      = os.getenv('VHEFITS','')
         self._remote_master = gammalib.GFilename()
         self._remote_base   = ''
@@ -66,14 +66,13 @@ class csiactcopy(ctools.cscript):
         """
         Get parameters from parfile and setup the observation
         """
-        
         # Get Parameters
         self._remote_master = self['remote_master'].filename()
         if not self._remote_master.exists():
-            raise RuntimeError('*** ERROR: FITS data not available. No '
-                               'master index file found in "'+
-                               self._remote_master+'". Make sure remote '
-                               'file system is properly mounted.')
+            msg = ('*** ERROR: FITS data not available. No master index file '
+                   'found in "%s". Make sure remote file system is properly '
+                   'mounted.' % self._remote_master)
+            raise RuntimeError(msg)
         # Get parameters
         self._prodname = self['prodname'].string()
@@ -88,11 +87,11 @@ class csiactcopy(ctools.cscript):
     def _copy(self, source, clobber):     
         """
-        Copy file to outpath
+        Copy file to self._outpath directory
         Parameters
         ----------
-        source : string
+        source : str
             Path of file to be copied
         Returns
@@ -147,39 +146,39 @@ class csiactcopy(ctools.cscript):
     def _merge(self, localfits, remotefits, hduname, clobber):
         """
         Merge remote and local fits files
+
         If the local fits file is not present, a new one is created.
         Parameters
         ----------
-        localfits : string
+        localfits : str
             Path of local index FITS file         
-        remotefits : string
+        remotefits : str
             Path of remote index FITS file         
-        hduname : string
+        hduname : str
             Name of HDU extension to be merged       
         clobber : bool
             Flag if remote content should overwrite local content        
-        """    
-        
-        # Logging
-        self._log_value(gammalib.TERSE, 'Local file', localfits)
-        self._log_value(gammalib.TERSE, 'Remote file', remotefits)
-        self._log_value(gammalib.TERSE, 'HDU name', hduname)
+        """
+        # Write input parameters into logger
+        self._log_value(gammalib.NORMAL, 'Local file', localfits)
+        self._log_value(gammalib.NORMAL, 'Remote file', remotefits)
+        self._log_value(gammalib.NORMAL, 'HDU name', hduname)
         if clobber:
             preference = 'Remote'
         else:
             preference = 'Local'
-        self._log_value(gammalib.TERSE, 'Conflict preference', preference)
+        self._log_value(gammalib.NORMAL, 'Conflict preference', preference)
         # Check if local fits file is available
         if os.path.isfile(localfits):    
-            local = gammalib.GFits(str(localfits))
+            local = gammalib.GFits(localfits)
             local_hdu = local[hduname]
         else:
             # Otherwise load remote fits file and delete rows
-            local = gammalib.GFits(str(remotefits))
+            local = gammalib.GFits(remotefits)
             local_hdu = local[hduname]
-            local_hdu.remove_rows(0,local_hdu.nrows())
+            local_hdu.remove_rows(0, local_hdu.nrows())
         # find local obs_id 
         local_obs = []
@@ -187,13 +186,14 @@ class csiactcopy(ctools.cscript):
         for obs_id in lobs_col:
             local_obs.append(obs_id)      
-        # Initialise list of runlists
-        # This is kluge: to circumvent a too long file name, we split the 
-        # runlists into chunks of 50 observations maximum
-        runlists = []
+        # Initialise run list. A kluge has been implemented to circumvent a
+        # too long file name: the run list is split into chunks of 50
+        # observations maximum. Note also the "//" division operator that
+        # makes sure that "i" is an integer on Python 3.
+        runlists       = []
         runs_per_chunk = 50
-        for i in range(len(self._runs) / runs_per_chunk + 1):
-            chunk = self._runs[i * runs_per_chunk : (i + 1) * runs_per_chunk]
+        for i in range(len(self._runs) // runs_per_chunk + 1):
+            chunk = self._runs[i * runs_per_chunk : (i+1) * runs_per_chunk]
             runlists.append(chunk)
         # Loop over runlist chunks
@@ -241,8 +241,8 @@ class csiactcopy(ctools.cscript):
             remote_obs.append(obs_id)
         # Log entries of index files   
-        self._log_value(gammalib.TERSE, 'Remote entries', len(remote_obs))
-        self._log_value(gammalib.TERSE, 'Local entries', len(local_obs))
+        self._log_value(gammalib.NORMAL, 'Remote entries', len(remote_obs))
+        self._log_value(gammalib.NORMAL, 'Local entries', len(local_obs))
         # initialise counter for logging
         removed_rows = 0
@@ -311,13 +311,13 @@ class csiactcopy(ctools.cscript):
             what = 'Removed local rows'
         else:
             what = 'Skipped remote rows'
-        self._log_value(gammalib.TERSE, what, removed_rows)
+        self._log_value(gammalib.NORMAL, what, removed_rows)
         # Loop over columns 
         for i in range(local_hdu.ncols()):
             # Get local and remote columns
-            local_col = local_hdu[i]
+            local_col  = local_hdu[i]
             remote_col = remote_hdu[i]    
             # Loop over entries and merge
@@ -325,77 +325,103 @@ class csiactcopy(ctools.cscript):
                 local_col[j+old_local_rows] = remote_col[j]
         # Save local fits file
-        local.saveto(str(localfits), True)
+        local.saveto(localfits, True)
         # Return
         return
-
-    # Public methods
-    def run(self):
-        """
-        Run the script
+    def _set_runs(self, filename):
         """
-        # Switch screen logging on in debug mode
-        if self._logDebug():
-            self._log.cout(True)
+        Set the run list
-        # Get parameters
-        self._get_parameters()
+        Parameters
+        ----------
+        filename : `~gammalib.GFilename`
+            Run list file name
-        # Make destination directory if not available
-        if not os.path.isdir(self._outpath):            
-            os.makedirs(self._outpath)   
-        
-        # Log output
-        if self._logTerse():
-            self._log('\n')
-            self._log.header2('Runlist information')
+        Returns
+        -------
+        runs : list of int
+            Run list
+        """
+        # Write header into logger
+        self._log_header1(gammalib.TERSE, 'Set runlist')
-        # Initialise runlist array   
-        self._runs = []
+        # Initialise runlist
+        runs = []
-        # Check if runlist file is available
-        if self._runlist.exists():
-            runfile = open(self._runlist.url())
+        # If runlist file exists then open file and extract the run
+        # information into the run list
+        if filename.exists():
+
+            # Open the runlist file
+            runfile = open(filename.url())
-            # Read runlist
+            # Read runlist file and append all run ID as integers to the
+            # run list. Skip any blank or comment lines
             for line in runfile.readlines():
                 if len(line) == 0:
                     continue
                 if line[0] == '#':
                     continue
                 if len(line.split()) > 0:
-                    self._runs.append(int(line.split()[0]))
+                    run = int(line.split()[0])
+                    runs.append(run)
+                    self._log_value(gammalib.EXPLICIT, 'Run %d' % len(runs), run)
             # Close runlist file
             runfile.close()   
             # Logging
-            self._log_value(gammalib.TERSE, 'Number of observations',
-                            len(self._runs))
+            self._log_value(gammalib.NORMAL, 'Number of observations',
+                            len(runs))
-        # Copy all data if runlist file is 'NONE'
-        elif self._runlist == 'NONE':
-            self._log_string(gammalib.TERSE, 'Copy all available data')
+        # ... otherwise, if the runlist file is 'NONE' then leave the
+        # run list empty. This implies copying all available data.
+        elif filename == 'NONE':
+            self._log_string(gammalib.NORMAL, 'Copy all available data')
-        # Raise exception if file not valid
+        # ... otherwise raise an exception
         else:
-            raise RuntimeError('*** ERROR: Runlist file "'+
-                               self._runlist+'" not available.')
+            msg = '*** ERROR: Runlist file "%s" not available' % filename
+            raise RuntimeError(msg)
+        # Return run list
+        return runs
+
+
+    # Public methods
+    def run(self):
+        """
+        Run the script
+        """
+        # Switch screen logging on in debug mode
+        if self._logDebug():
+            self._log.cout(True)
+
+        # Get parameters
+        self._get_parameters()
+
+        # Make destination directory if not available
+        if not os.path.isdir(self._outpath):            
+            os.makedirs(self._outpath)   
+
+        # Set run list
+        self._runs = self._set_runs(self._runlist)
+
         # Check for availability of remote master file
         if not self._remote_master.exists():
-            raise RuntimeError('*** ERROR: Remote master file "'+
-                               self._remote_master+'" does not exist.')
+            msg = ('*** ERROR: Remote master file "%s" does not exist' %
+                   self._remote_master)
+            raise RuntimeError(msg)
         # Retrieve json data from remote master
         json_data = open(self._remote_master.url()).read()
         data      = json.loads(json_data) 
         if not 'datasets' in data:
-            raise RuntimeError('*** ERROR: Key "datasets" not available '
-                               'in remote master index file "'+
-                               self._remote_master+'".')
+            msg = ('*** ERROR: Key "datasets" not available in remote master '
+                   'index file "%s"' % self._remote_master)
+            raise RuntimeError(msg)
         # Get array of configurations
         configs = data['datasets']
@@ -409,17 +435,16 @@ class csiactcopy(ctools.cscript):
         # Initialise file names to be copied
         files = set()
-        # Logging
-        if self._logTerse():
-            self._log('\n')
-            self._log.header2('Loop over remote configs')
+        # Write header into logger
+        self._log_header2(gammalib.TERSE, 'Loop over remote configs')
         # Loop over configs
-        for config in configs: 
-            
-            # Logging          
-            if self._logVerbose():
-                self._log.header2(str(config['name']))
+        for config in configs:
+
+            # Write header into logger. The str() is needed since
+            # "config['name']" is <type 'unicode'>, and under Python 2 a
+            # string is expected.
+            self._log_header2(gammalib.VERBOSE, str(config['name']))
             # Check if prodname was found
             if config['name'] == self._prodname:
@@ -432,13 +457,13 @@ class csiactcopy(ctools.cscript):
                 remote_obs = os.path.join(self._remote_base, config['obsindx'])
                 # Log information
-                self._log_header3(gammalib.TERSE, 'Remote config "'+
+                self._log_header3(gammalib.NORMAL, 'Remote config "'+
                                   self._prodname+'"')
-                self._log_value(gammalib.TERSE, 'HDU index', remote_hdu)
-                self._log_value(gammalib.TERSE, 'Observation index', remote_obs)
+                self._log_value(gammalib.NORMAL, 'HDU index', remote_hdu)
+                self._log_value(gammalib.NORMAL, 'Observation index', remote_obs)
                 # Open remote HDU index file
-                fits = gammalib.GFits(str(remote_hdu))
+                fits = gammalib.GFits(remote_hdu)
                 table = fits['HDU_INDEX']
                 # Initialise flag if SIZE column is present
@@ -457,11 +482,10 @@ class csiactcopy(ctools.cscript):
                 for run in self._runs:
                     # Check for run not in remote data store
-                    if not run in remote_ids:  
-                        if self._logNormal():
-                            self._log('Skip observation '+str(run)+': ')
-                            self._log('ID not available remotely')
-                            self._log('\n')
+                    if not run in remote_ids:
+                        msg = ('Skip observation "%s": ID not available '
+                               'remotely' % str(run))
+                        self._log_string(msg)
                 # Loop over remote HDU index file
                 for row in range(table.nrows()):
@@ -510,11 +534,11 @@ class csiactcopy(ctools.cscript):
                             cp_size += table['SIZE'][row]
                 # Log file information
-                self._log_header2(gammalib.TERSE, 'File information')
-                self._log_value(gammalib.TERSE, 'Number of files', len(files))
+                self._log_header2(gammalib.NORMAL, 'File information')
+                self._log_value(gammalib.NORMAL, 'Number of files', len(files))
                 if has_size:
                     size = float(cp_size) * 1.0e-6
-                    self._log_value(gammalib.TERSE, 'Size', '%.2f MB' % size)
+                    self._log_value(gammalib.NORMAL, 'Size', '%.2f MB' % size)
                 self._log_header3(gammalib.VERBOSE, 'File names')
                 for filename in files:
                     self._log_string(gammalib.VERBOSE, str(filename)+'\n')
@@ -524,8 +548,8 @@ class csiactcopy(ctools.cscript):
             # If prodname is not found just log that we skip the config
             else:
-                self._log_header3(gammalib.EXPLICIT, 'Skipping config "'+
-                                  str(config['name'])+'"')
+                msg = 'Skipping config "%s"' % config['name']
+                self._log_header3(gammalib.EXPLICIT, msg)
         # Raise Exception if prodname was not found
         if not has_prod: 
@@ -586,15 +610,10 @@ class csiactcopy(ctools.cscript):
             k += 1
         # Logging
-        if self._logNormal() and not self._logVerbose():
-            self._log.parformat('Status')
-            self._log('Finished')
-            self._log('\n')
+        self._log_value(gammalib.NORMAL, 'Status', 'Finished')
-        # Logging about index files
-        if self._logTerse():
-            self._log('\n')
-            self._log.header1('Updating index files')
+        # Write header into logger
+        self._log_header1(gammalib.TERSE, 'Updating index files')
         # Build local hdu index file name
         local_hdu = os.path.join(self._outpath,
@@ -43,7 +43,7 @@ class csiactobs(ctools.cscript):
     # Constructor
     def __init__(self, *argv):
         """
-        Constructor.
+        Constructor
         """
         # Set name and version
         self._name    = 'csiactobs'
@@ -229,9 +229,9 @@ class csiactobs(ctools.cscript):
         Parameters
         ----------
         prefactor : float
-            Prefactor parameter of spectral model
+            Power law prefactor of spectral model
         index : float
-            Index parameter of spectral model
+            Power law index of spectral model
         emin : float
             Minimum energy (in case a spectral node function is required)
         emax : float
@@ -330,13 +330,13 @@ class csiactobs(ctools.cscript):
         Parameters
         ----------
-        telescope : string
+        telescope : str
             Name of telescope
-        obs_id : string
+        obs_id : str
             Observation ID
         bkg_scale : float
             Background scaling factor
-        bkgtype : string
+        bkgtype : str
             Type of background (irf,aeff, or gauss)
         Returns
@@ -162,8 +162,9 @@ class Test(test):
         self._check_copy('iactdata_py1')
         self._check_n_obs('iactdata_py1', 9)
+        # Remove the copied IACT data store before continuing
         if os.path.isdir('iactdata_py2'):
-            os.system("rm -r iactdata_py2")
+            os.system('rm -r iactdata_py2')
         # Set-up csiactcopy
         iactcopy = cscripts.csiactcopy()
@@ -172,7 +173,7 @@ class Test(test):
         iactcopy['prodname']      = 'unit-test'
         iactcopy['outpath']       = 'iactdata_py2'
         iactcopy['logfile']       = 'csiactcopy_py4.log'
-        iactcopy['chatter']       = 3       
+        iactcopy['chatter']       = 3
         # Run csiactcopy script and save run list
         iactcopy.logFileOpen()   # Make sure we get a log file
@@ -223,10 +224,15 @@ class Test(test):
         # Return
         return
-    # Check copy
+    # Check copy of IACT data store
     def _check_copy(self, pathname):
         """
-        Check copy.
+        Check copy of IACT data store
+
+        Parameters
+        ----------
+        pathname : str
+            Path to copied IACT data store
         """
         # Set file names
         hdu_index_name = gammalib.GFilename(pathname+'/hdu-index.fits')
@@ -235,17 +241,17 @@ class Test(test):
         # Check for existence of files
         self.test_assert(hdu_index_name.exists(),
-                         'Check if file "hdu-index.fits" exists.')
+                         'Check if file "hdu-index.fits" exists')
         self.test_assert(obs_index_name.exists(),
-                         'Check if file "obs-index.fits" exists.')
+                         'Check if file "obs-index.fits" exists')
         self.test_assert(master_name.exists(),
-                         'Check if file "master.json" exists.')
+                         'Check if file "master.json" exists')
         # Check if index files are FITS file
         self.test_assert(hdu_index_name.is_fits(),
-                         'Check if file "hdu-index.fits" is FITS file.')
+                         'Check if file "hdu-index.fits" is FITS file')
         self.test_assert(obs_index_name.is_fits(),
-                         'Check if file "obs-index.fits" is FITS file.')
+                         'Check if file "obs-index.fits" is FITS file')
         # Return
         return
@@ -253,7 +259,14 @@ class Test(test):
     # Check copy
     def _check_n_obs(self, pathname, n_expected):
         """
-        Check number of available observations.
+        Check number of available observations
+
+        Parameters
+        ----------
+        pathname : str
+            Path to copied IACT data store
+        n_expected : int
+            Expected number of observation in IACT data store
         """
         # Set file name
         obs_index_name = gammalib.GFilename(pathname+'/obs-index.fits[OBS_INDEX]')
@@ -268,8 +281,7 @@ class Test(test):
         fits.close()
         # Check for existence of observations
-        self.test_assert(n_obs == n_expected,
-                         'Check for '+str(n_expected)+' observations.')
+        self.test_value(n_obs, n_expected, 'Check for number of observations')
         # Return
         return
 \ No newline at end of file