Iftach Sadeh / ctools

Commit b73a6e3a477c0af236b0cce6df0105be80e7d7d4

Authored by Jürgen Knödlseder
1 parent 85772e55

Re-correct integer division in csiactopy.py for Python 3 compliance

Inline Side-by-side
Showing 1 changed file with 110 additions and 92 deletions
cscripts/csiactcopy.py
  View file @b73a6e3
... ... @@ -40,11 +40,11 @@ class csiactcopy(ctools.cscript):
40 40 # Constructor
41 41 def __init__(self, *argv):
42 42 """
43   - Constructor.
  43 + Constructor
44 44 """
45 45 # Set name
46 46 self._name = 'csiactcopy'
47   - self._version = '1.1.0'
  47 + self._version = '1.2.0'
48 48 self._datapath = os.getenv('VHEFITS','')
49 49 self._remote_master = gammalib.GFilename()
50 50 self._remote_base = ''
... ... @@ -66,7 +66,6 @@ class csiactcopy(ctools.cscript):
66 66 """
67 67 Get parameters from parfile and setup the observation
68 68 """
69   -
70 69 # Get Parameters
71 70 self._remote_master = self['remote_master'].filename()
72 71 if not self._remote_master.exists():
... ... @@ -88,11 +87,11 @@ class csiactcopy(ctools.cscript):
88 87  
89 88 def _copy(self, source, clobber):
90 89 """
91   - Copy file to outpath
  90 + Copy file to self._outpath directory
92 91  
93 92 Parameters
94 93 ----------
95   - source : string
  94 + source : str
96 95 Path of file to be copied
97 96  
98 97 Returns
... ... @@ -100,7 +99,6 @@ class csiactcopy(ctools.cscript):
100 99 filesize : float
101 100 Size of the file that was copied
102 101 """
103   -
104 102 # Get file destination
105 103 destination = os.path.join(self._outpath,
106 104 os.path.relpath(source, self._remote_base))
... ... @@ -147,29 +145,29 @@ class csiactcopy(ctools.cscript):
147 145 def _merge(self, localfits, remotefits, hduname, clobber):
148 146 """
149 147 Merge remote and local fits files
  148 +
150 149 If the local fits file is not present, a new one is created.
151 150  
152 151 Parameters
153 152 ----------
154   - localfits : string
  153 + localfits : str
155 154 Path of local index FITS file
156   - remotefits : string
  155 + remotefits : str
157 156 Path of remote index FITS file
158   - hduname : string
  157 + hduname : str
159 158 Name of HDU extension to be merged
160 159 clobber : bool
161 160 Flag if remote content should overwrite local content
162 161 """
163   -
164   - # Logging
165   - self._log_value(gammalib.TERSE, 'Local file', localfits)
166   - self._log_value(gammalib.TERSE, 'Remote file', remotefits)
167   - self._log_value(gammalib.TERSE, 'HDU name', hduname)
  162 + # Write input parameters into logger
  163 + self._log_value(gammalib.NORMAL, 'Local file', localfits)
  164 + self._log_value(gammalib.NORMAL, 'Remote file', remotefits)
  165 + self._log_value(gammalib.NORMAL, 'HDU name', hduname)
168 166 if clobber:
169 167 preference = 'Remote'
170 168 else:
171 169 preference = 'Local'
172   - self._log_value(gammalib.TERSE, 'Conflict preference', preference)
  170 + self._log_value(gammalib.NORMAL, 'Conflict preference', preference)
173 171  
174 172 # Check if local fits file is available
175 173 if os.path.isfile(localfits):
... ... @@ -187,12 +185,13 @@ class csiactcopy(ctools.cscript):
187 185 for obs_id in lobs_col:
188 186 local_obs.append(obs_id)
189 187  
190   - # Initialise list of runlists
191   - # This is kluge: to circumvent a too long file name, we split the
192   - # runlists into chunks of 50 observations maximum
193   - runlists = []
  188 + # Initialise run list. A kluge has been implemented to circumvent a
  189 + # too long file name: the run list is split into chunks of 50
  190 + # observations maximum. Note also the "//" division operator that
  191 + # makes sure that "i" is an integer on Python 3.
  192 + runlists = []
194 193 runs_per_chunk = 50
195   - for i in range(len(self._runs) / runs_per_chunk + 1):
  194 + for i in range(len(self._runs) // runs_per_chunk + 1):
196 195 chunk = self._runs[i * runs_per_chunk : (i + 1) * runs_per_chunk]
197 196 runlists.append(chunk)
198 197  
... ... @@ -241,8 +240,8 @@ class csiactcopy(ctools.cscript):
241 240 remote_obs.append(obs_id)
242 241  
243 242 # Log entries of index files
244   - self._log_value(gammalib.TERSE, 'Remote entries', len(remote_obs))
245   - self._log_value(gammalib.TERSE, 'Local entries', len(local_obs))
  243 + self._log_value(gammalib.NORMAL, 'Remote entries', len(remote_obs))
  244 + self._log_value(gammalib.NORMAL, 'Local entries', len(local_obs))
246 245  
247 246 # initialise counter for logging
248 247 removed_rows = 0
... ... @@ -311,7 +310,7 @@ class csiactcopy(ctools.cscript):
311 310 what = 'Removed local rows'
312 311 else:
313 312 what = 'Skipped remote rows'
314   - self._log_value(gammalib.TERSE, what, removed_rows)
  313 + self._log_value(gammalib.NORMAL, what, removed_rows)
315 314  
316 315 # Loop over columns
317 316 for i in range(local_hdu.ncols()):
... ... @@ -330,60 +329,85 @@ class csiactcopy(ctools.cscript):
330 329 # Return
331 330 return
332 331  
333   -
334   - # Public methods
335   - def run(self):
336   - """
337   - Run the script
  332 + def _set_runs(self, filename):
338 333 """
339   - # Switch screen logging on in debug mode
340   - if self._logDebug():
341   - self._log.cout(True)
  334 + Set the run list
342 335  
343   - # Get parameters
344   - self._get_parameters()
  336 + Parameters
  337 + ----------
  338 + filename : `~gammalib.GFilename`
  339 + Run list file name
345 340  
346   - # Make destination directory if not available
347   - if not os.path.isdir(self._outpath):
348   - os.makedirs(self._outpath)
349   -
350   - # Log output
351   - if self._logTerse():
352   - self._log('\n')
353   - self._log.header2('Runlist information')
  341 + Returns
  342 + -------
  343 + runs : list of int
  344 + Run list
  345 + """
  346 + # Write header into logger
  347 + self._log_header1(gammalib.TERSE, 'Set runlist')
354 348  
355   - # Initialise runlist array
356   - self._runs = []
  349 + # Initialise runlist
  350 + runs = []
357 351  
358   - # Check if runlist file is available
359   - if self._runlist.exists():
360   - runfile = open(self._runlist.url())
  352 + # If runlist file exists then open file and extract the run
  353 + # information into the run list
  354 + if filename.exists():
  355 +
  356 + # Open the runlist file
  357 + runfile = open(filename.url())
361 358  
362   - # Read runlist
  359 + # Read runlist file and append all run ID as integers to the
  360 + # run list. Skip any blank or comment lines
363 361 for line in runfile.readlines():
364 362 if len(line) == 0:
365 363 continue
366 364 if line[0] == '#':
367 365 continue
368 366 if len(line.split()) > 0:
369   - self._runs.append(int(line.split()[0]))
  367 + run = int(line.split()[0])
  368 + runs.append(run)
  369 + self._log_value(gammalib.EXPLICIT, 'Run %d' % len(runs), run)
370 370  
371 371 # Close runlist file
372 372 runfile.close()
373 373  
374 374 # Logging
375   - self._log_value(gammalib.TERSE, 'Number of observations',
376   - len(self._runs))
  375 + self._log_value(gammalib.NORMAL, 'Number of observations',
  376 + len(runs))
377 377  
378   - # Copy all data if runlist file is 'NONE'
379   - elif self._runlist == 'NONE':
380   - self._log_string(gammalib.TERSE, 'Copy all available data')
  378 + # ... otherwise, if the runlist file is 'NONE' then leave the
  379 + # run list empty. This implies copying all available data.
  380 + elif filename == 'NONE':
  381 + self._log_string(gammalib.NORMAL, 'Copy all available data')
381 382  
382   - # Raise exception if file not valid
  383 + # ... otherwise raise an exception
383 384 else:
384   - raise RuntimeError('*** ERROR: Runlist file "'+
385   - self._runlist+'" not available.')
  385 + msg = '*** ERROR: Runlist file "%s" not available' % filename
  386 + raise RuntimeError(msg)
386 387  
  388 + # Return run list
  389 + return runs
  390 +
  391 +
  392 + # Public methods
  393 + def run(self):
  394 + """
  395 + Run the script
  396 + """
  397 + # Switch screen logging on in debug mode
  398 + if self._logDebug():
  399 + self._log.cout(True)
  400 +
  401 + # Get parameters
  402 + self._get_parameters()
  403 +
  404 + # Make destination directory if not available
  405 + if not os.path.isdir(self._outpath):
  406 + os.makedirs(self._outpath)
  407 +
  408 + # Set run list
  409 + self._runs = self._set_runs(self._runlist)
  410 +
387 411 # Check for availability of remote master file
388 412 if not self._remote_master.exists():
389 413 raise RuntimeError('*** ERROR: Remote master file "'+
... ... @@ -409,17 +433,16 @@ class csiactcopy(ctools.cscript):
409 433 # Initialise file names to be copied
410 434 files = set()
411 435  
412   - # Logging
413   - if self._logTerse():
414   - self._log('\n')
415   - self._log.header2('Loop over remote configs')
  436 + # Write header into logger
  437 + self._log_header2(gammalib.TERSE, 'Loop over remote configs')
416 438  
417 439 # Loop over configs
418 440 for config in configs:
419 441  
420   - # Logging
421   - if self._logVerbose():
422   - self._log.header2(str(config['name']))
  442 + # Write header into logger. The str() is needed since
  443 + # "config['name']" is <type 'unicode'>, and under Python 2 a
  444 + # string is expected.
  445 + self._log_header2(gammalib.VERBOSE, str(config['name']))
423 446  
424 447 # Check if prodname was found
425 448 if config['name'] == self._prodname:
... ... @@ -428,14 +451,16 @@ class csiactcopy(ctools.cscript):
428 451 has_prod = True
429 452  
430 453 # Build path of index files
431   - remote_hdu = os.path.join(self._remote_base, config['hduindx'])
432   - remote_obs = os.path.join(self._remote_base, config['obsindx'])
  454 + remote_hdu = str(os.path.join(self._remote_base,
  455 + config['hduindx']))
  456 + remote_obs = str(os.path.join(self._remote_base,
  457 + config['obsindx']))
433 458  
434 459 # Log information
435   - self._log_header3(gammalib.TERSE, 'Remote config "'+
  460 + self._log_header3(gammalib.NORMAL, 'Remote config "'+
436 461 self._prodname+'"')
437   - self._log_value(gammalib.TERSE, 'HDU index', remote_hdu)
438   - self._log_value(gammalib.TERSE, 'Observation index', remote_obs)
  462 + self._log_value(gammalib.NORMAL, 'HDU index', remote_hdu)
  463 + self._log_value(gammalib.NORMAL, 'Observation index', remote_obs)
439 464  
440 465 # Open remote HDU index file
441 466 fits = gammalib.GFits(str(remote_hdu))
... ... @@ -457,11 +482,10 @@ class csiactcopy(ctools.cscript):
457 482 for run in self._runs:
458 483  
459 484 # Check for run not in remote data store
460   - if not run in remote_ids:
461   - if self._logNormal():
462   - self._log('Skip observation '+str(run)+': ')
463   - self._log('ID not available remotely')
464   - self._log('\n')
  485 + if not run in remote_ids:
  486 + msg = ('Skip observation "%s": ID not available '
  487 + 'remotely' % str(run))
  488 + self._log_string(msg)
465 489  
466 490 # Loop over remote HDU index file
467 491 for row in range(table.nrows()):
... ... @@ -510,11 +534,11 @@ class csiactcopy(ctools.cscript):
510 534 cp_size += table['SIZE'][row]
511 535  
512 536 # Log file information
513   - self._log_header2(gammalib.TERSE, 'File information')
514   - self._log_value(gammalib.TERSE, 'Number of files', len(files))
  537 + self._log_header2(gammalib.NORMAL, 'File information')
  538 + self._log_value(gammalib.NORMAL, 'Number of files', len(files))
515 539 if has_size:
516 540 size = float(cp_size) * 1.0e-6
517   - self._log_value(gammalib.TERSE, 'Size', '%.2f MB' % size)
  541 + self._log_value(gammalib.NORMAL, 'Size', '%.2f MB' % size)
518 542 self._log_header3(gammalib.VERBOSE, 'File names')
519 543 for filename in files:
520 544 self._log_string(gammalib.VERBOSE, str(filename)+'\n')
... ... @@ -586,16 +610,11 @@ class csiactcopy(ctools.cscript):
586 610 k += 1
587 611  
588 612 # Logging
589   - if self._logNormal() and not self._logVerbose():
590   - self._log.parformat('Status')
591   - self._log('Finished')
592   - self._log('\n')
593   -
  613 + self._log_value(gammalib.NORMAL, 'Status', 'Finished')
  614 +
594 615 # Logging about index files
595   - if self._logTerse():
596   - self._log('\n')
597   - self._log.header1('Updating index files')
598   -
  616 + self._log_header1(gammalib.TERSE, 'Updating index files')
  617 +
599 618 # Build local hdu index file name
600 619 local_hdu = os.path.join(self._outpath,
601 620 os.path.relpath(remote_hdu,
... ... @@ -657,16 +676,15 @@ class csiactcopy(ctools.cscript):
657 676 # Check if index files are available
658 677 if not (gammalib.GFilename(str(hdu)).is_fits() and
659 678 gammalib.GFilename(str(obs)).is_fits()):
660   - if self._logTerse():
661   - self._log('Removing "'+str(config['name'])+'" ')
662   - self._log('(not available)')
663   - self._log('\n')
  679 + self._log_value(gammalib.NORMAL,
  680 + 'Removing "'+str(config['name']),
  681 + 'Not available')
664 682 else:
665 683 # Append config if valid
666 684 newconfigs.append(config)
667   - if self._logTerse():
668   - self._log('Keeping "'+str(config['name'])+'"')
669   - self._log('\n')
  685 + self._log_value(gammalib.NORMAL,
  686 + 'Keeping "'+str(config['name']),
  687 + 'Available')
670 688  
671 689 # Signals that downloaded config is available
672 690 if config['name'] == self._prodname:
... ...