Johan Bregeon / ctools

Commit 673a1331336dd9d563c3f7543feab02c9aaaf8eb

Authored by Jürgen Knödlseder
1 parent 0622a2cb

Put back former version of csiactcopy.py

Inline Side-by-side
Showing 1 changed file with 102 additions and 121 deletions
cscripts/csiactcopy.py
  View file @673a133
... ... @@ -40,11 +40,11 @@ class csiactcopy(ctools.cscript):
40 40 # Constructor
41 41 def __init__(self, *argv):
42 42 """
43   - Constructor
  43 + Constructor.
44 44 """
45 45 # Set name
46 46 self._name = 'csiactcopy'
47   - self._version = '1.2.0'
  47 + self._version = '1.1.0'
48 48 self._datapath = os.getenv('VHEFITS','')
49 49 self._remote_master = gammalib.GFilename()
50 50 self._remote_base = ''
... ... @@ -66,13 +66,14 @@ class csiactcopy(ctools.cscript):
66 66 """
67 67 Get parameters from parfile and setup the observation
68 68 """
  69 +
69 70 # Get Parameters
70 71 self._remote_master = self['remote_master'].filename()
71 72 if not self._remote_master.exists():
72   - msg = ('*** ERROR: FITS data not available. No master index file '
73   - 'found in "%s". Make sure remote file system is properly '
74   - 'mounted.' % self._remote_master)
75   - raise RuntimeError(msg)
  73 + raise RuntimeError('*** ERROR: FITS data not available. No '
  74 + 'master index file found in "'+
  75 + self._remote_master+'". Make sure remote '
  76 + 'file system is properly mounted.')
76 77  
77 78 # Get parameters
78 79 self._prodname = self['prodname'].string()
... ... @@ -87,11 +88,11 @@ class csiactcopy(ctools.cscript):
87 88  
88 89 def _copy(self, source, clobber):
89 90 """
90   - Copy file to self._outpath directory
  91 + Copy file to outpath
91 92  
92 93 Parameters
93 94 ----------
94   - source : str
  95 + source : string
95 96 Path of file to be copied
96 97  
97 98 Returns
... ... @@ -146,39 +147,39 @@ class csiactcopy(ctools.cscript):
146 147 def _merge(self, localfits, remotefits, hduname, clobber):
147 148 """
148 149 Merge remote and local fits files
149   -
150 150 If the local fits file is not present, a new one is created.
151 151  
152 152 Parameters
153 153 ----------
154   - localfits : str
  154 + localfits : string
155 155 Path of local index FITS file
156   - remotefits : str
  156 + remotefits : string
157 157 Path of remote index FITS file
158   - hduname : str
  158 + hduname : string
159 159 Name of HDU extension to be merged
160 160 clobber : bool
161 161 Flag if remote content should overwrite local content
162   - """
163   - # Write input parameters into logger
164   - self._log_value(gammalib.NORMAL, 'Local file', localfits)
165   - self._log_value(gammalib.NORMAL, 'Remote file', remotefits)
166   - self._log_value(gammalib.NORMAL, 'HDU name', hduname)
  162 + """
  163 +
  164 + # Logging
  165 + self._log_value(gammalib.TERSE, 'Local file', localfits)
  166 + self._log_value(gammalib.TERSE, 'Remote file', remotefits)
  167 + self._log_value(gammalib.TERSE, 'HDU name', hduname)
167 168 if clobber:
168 169 preference = 'Remote'
169 170 else:
170 171 preference = 'Local'
171   - self._log_value(gammalib.NORMAL, 'Conflict preference', preference)
  172 + self._log_value(gammalib.TERSE, 'Conflict preference', preference)
172 173  
173 174 # Check if local fits file is available
174 175 if os.path.isfile(localfits):
175   - local = gammalib.GFits(localfits)
  176 + local = gammalib.GFits(str(localfits))
176 177 local_hdu = local[hduname]
177 178 else:
178 179 # Otherwise load remote fits file and delete rows
179   - local = gammalib.GFits(remotefits)
  180 + local = gammalib.GFits(str(remotefits))
180 181 local_hdu = local[hduname]
181   - local_hdu.remove_rows(0, local_hdu.nrows())
  182 + local_hdu.remove_rows(0,local_hdu.nrows())
182 183  
183 184 # find local obs_id
184 185 local_obs = []
... ... @@ -186,14 +187,13 @@ class csiactcopy(ctools.cscript):
186 187 for obs_id in lobs_col:
187 188 local_obs.append(obs_id)
188 189  
189   - # Initialise run list. A kluge has been implemented to circumvent a
190   - # too long file name: the run list is split into chunks of 50
191   - # observations maximum. Note also the "//" division operator that
192   - # makes sure that "i" is an integer on Python 3.
193   - runlists = []
  190 + # Initialise list of runlists
  191 + # This is kluge: to circumvent a too long file name, we split the
  192 + # runlists into chunks of 50 observations maximum
  193 + runlists = []
194 194 runs_per_chunk = 50
195   - for i in range(len(self._runs) // runs_per_chunk + 1):
196   - chunk = self._runs[i * runs_per_chunk : (i+1) * runs_per_chunk]
  195 + for i in range(len(self._runs) / runs_per_chunk + 1):
  196 + chunk = self._runs[i * runs_per_chunk : (i + 1) * runs_per_chunk]
197 197 runlists.append(chunk)
198 198  
199 199 # Loop over runlist chunks
... ... @@ -241,8 +241,8 @@ class csiactcopy(ctools.cscript):
241 241 remote_obs.append(obs_id)
242 242  
243 243 # Log entries of index files
244   - self._log_value(gammalib.NORMAL, 'Remote entries', len(remote_obs))
245   - self._log_value(gammalib.NORMAL, 'Local entries', len(local_obs))
  244 + self._log_value(gammalib.TERSE, 'Remote entries', len(remote_obs))
  245 + self._log_value(gammalib.TERSE, 'Local entries', len(local_obs))
246 246  
247 247 # initialise counter for logging
248 248 removed_rows = 0
... ... @@ -311,13 +311,13 @@ class csiactcopy(ctools.cscript):
311 311 what = 'Removed local rows'
312 312 else:
313 313 what = 'Skipped remote rows'
314   - self._log_value(gammalib.NORMAL, what, removed_rows)
  314 + self._log_value(gammalib.TERSE, what, removed_rows)
315 315  
316 316 # Loop over columns
317 317 for i in range(local_hdu.ncols()):
318 318  
319 319 # Get local and remote columns
320   - local_col = local_hdu[i]
  320 + local_col = local_hdu[i]
321 321 remote_col = remote_hdu[i]
322 322  
323 323 # Loop over entries and merge
... ... @@ -325,103 +325,77 @@ class csiactcopy(ctools.cscript):
325 325 local_col[j+old_local_rows] = remote_col[j]
326 326  
327 327 # Save local fits file
328   - local.saveto(localfits, True)
  328 + local.saveto(str(localfits), True)
329 329  
330 330 # Return
331 331 return
332 332  
333   - def _set_runs(self, filename):
  333 +
  334 + # Public methods
  335 + def run(self):
  336 + """
  337 + Run the script
334 338 """
335   - Set the run list
  339 + # Switch screen logging on in debug mode
  340 + if self._logDebug():
  341 + self._log.cout(True)
336 342  
337   - Parameters
338   - ----------
339   - filename : `~gammalib.GFilename`
340   - Run list file name
  343 + # Get parameters
  344 + self._get_parameters()
341 345  
342   - Returns
343   - -------
344   - runs : list of int
345   - Run list
346   - """
347   - # Write header into logger
348   - self._log_header1(gammalib.TERSE, 'Set runlist')
  346 + # Make destination directory if not available
  347 + if not os.path.isdir(self._outpath):
  348 + os.makedirs(self._outpath)
349 349  
350   - # Initialise runlist
351   - runs = []
  350 + # Log output
  351 + if self._logTerse():
  352 + self._log('\n')
  353 + self._log.header2('Runlist information')
352 354  
353   - # If runlist file exists then open file and extract the run
354   - # information into the run list
355   - if filename.exists():
356   -
357   - # Open the runlist file
358   - runfile = open(filename.url())
  355 + # Initialise runlist array
  356 + self._runs = []
  357 +
  358 + # Check if runlist file is available
  359 + if self._runlist.exists():
  360 + runfile = open(self._runlist.url())
359 361  
360   - # Read runlist file and append all run ID as integers to the
361   - # run list. Skip any blank or comment lines
  362 + # Read runlist
362 363 for line in runfile.readlines():
363 364 if len(line) == 0:
364 365 continue
365 366 if line[0] == '#':
366 367 continue
367 368 if len(line.split()) > 0:
368   - run = int(line.split()[0])
369   - runs.append(run)
370   - self._log_value(gammalib.EXPLICIT, 'Run %d' % len(runs), run)
  369 + self._runs.append(int(line.split()[0]))
371 370  
372 371 # Close runlist file
373 372 runfile.close()
374 373  
375 374 # Logging
376   - self._log_value(gammalib.NORMAL, 'Number of observations',
377   - len(runs))
  375 + self._log_value(gammalib.TERSE, 'Number of observations',
  376 + len(self._runs))
378 377  
379   - # ... otherwise, if the runlist file is 'NONE' then leave the
380   - # run list empty. This implies copying all available data.
381   - elif filename == 'NONE':
382   - self._log_string(gammalib.NORMAL, 'Copy all available data')
  378 + # Copy all data if runlist file is 'NONE'
  379 + elif self._runlist == 'NONE':
  380 + self._log_string(gammalib.TERSE, 'Copy all available data')
383 381  
384   - # ... otherwise raise an exception
  382 + # Raise exception if file not valid
385 383 else:
386   - msg = '*** ERROR: Runlist file "%s" not available' % filename
387   - raise RuntimeError(msg)
  384 + raise RuntimeError('*** ERROR: Runlist file "'+
  385 + self._runlist+'" not available.')
388 386  
389   - # Return run list
390   - return runs
391   -
392   -
393   - # Public methods
394   - def run(self):
395   - """
396   - Run the script
397   - """
398   - # Switch screen logging on in debug mode
399   - if self._logDebug():
400   - self._log.cout(True)
401   -
402   - # Get parameters
403   - self._get_parameters()
404   -
405   - # Make destination directory if not available
406   - if not os.path.isdir(self._outpath):
407   - os.makedirs(self._outpath)
408   -
409   - # Set run list
410   - self._runs = self._set_runs(self._runlist)
411   -
412 387 # Check for availability of remote master file
413 388 if not self._remote_master.exists():
414   - msg = ('*** ERROR: Remote master file "%s" does not exist' %
415   - self._remote_master)
416   - raise RuntimeError(msg)
  389 + raise RuntimeError('*** ERROR: Remote master file "'+
  390 + self._remote_master+'" does not exist.')
417 391  
418 392 # Retrieve json data from remote master
419 393 json_data = open(self._remote_master.url()).read()
420 394 data = json.loads(json_data)
421 395 if not 'datasets' in data:
422   - msg = ('*** ERROR: Key "datasets" not available in remote master '
423   - 'index file "%s"' % self._remote_master)
424   - raise RuntimeError(msg)
  396 + raise RuntimeError('*** ERROR: Key "datasets" not available '
  397 + 'in remote master index file "'+
  398 + self._remote_master+'".')
425 399  
426 400 # Get array of configurations
427 401 configs = data['datasets']
... ... @@ -435,16 +409,17 @@ class csiactcopy(ctools.cscript):
435 409 # Initialise file names to be copied
436 410 files = set()
437 411  
438   - # Write header into logger
439   - self._log_header2(gammalib.TERSE, 'Loop over remote configs')
  412 + # Logging
  413 + if self._logTerse():
  414 + self._log('\n')
  415 + self._log.header2('Loop over remote configs')
440 416  
441 417 # Loop over configs
442   - for config in configs:
443   -
444   - # Write header into logger. The str() is needed since
445   - # "config['name']" is <type 'unicode'>, and under Python 2 a
446   - # string is expected.
447   - self._log_header2(gammalib.VERBOSE, str(config['name']))
  418 + for config in configs:
  419 +
  420 + # Logging
  421 + if self._logVerbose():
  422 + self._log.header2(str(config['name']))
448 423  
449 424 # Check if prodname was found
450 425 if config['name'] == self._prodname:
... ... @@ -457,13 +432,13 @@ class csiactcopy(ctools.cscript):
457 432 remote_obs = os.path.join(self._remote_base, config['obsindx'])
458 433  
459 434 # Log information
460   - self._log_header3(gammalib.NORMAL, 'Remote config "'+
  435 + self._log_header3(gammalib.TERSE, 'Remote config "'+
461 436 self._prodname+'"')
462   - self._log_value(gammalib.NORMAL, 'HDU index', remote_hdu)
463   - self._log_value(gammalib.NORMAL, 'Observation index', remote_obs)
  437 + self._log_value(gammalib.TERSE, 'HDU index', remote_hdu)
  438 + self._log_value(gammalib.TERSE, 'Observation index', remote_obs)
464 439  
465 440 # Open remote HDU index file
466   - fits = gammalib.GFits(remote_hdu)
  441 + fits = gammalib.GFits(str(remote_hdu))
467 442 table = fits['HDU_INDEX']
468 443  
469 444 # Initialise flag if SIZE column is present
... ... @@ -482,10 +457,11 @@ class csiactcopy(ctools.cscript):
482 457 for run in self._runs:
483 458  
484 459 # Check for run not in remote data store
485   - if not run in remote_ids:
486   - msg = ('Skip observation "%s": ID not available '
487   - 'remotely' % str(run))
488   - self._log_string(msg)
  460 + if not run in remote_ids:
  461 + if self._logNormal():
  462 + self._log('Skip observation '+str(run)+': ')
  463 + self._log('ID not available remotely')
  464 + self._log('\n')
489 465  
490 466 # Loop over remote HDU index file
491 467 for row in range(table.nrows()):
... ... @@ -534,11 +510,11 @@ class csiactcopy(ctools.cscript):
534 510 cp_size += table['SIZE'][row]
535 511  
536 512 # Log file information
537   - self._log_header2(gammalib.NORMAL, 'File information')
538   - self._log_value(gammalib.NORMAL, 'Number of files', len(files))
  513 + self._log_header2(gammalib.TERSE, 'File information')
  514 + self._log_value(gammalib.TERSE, 'Number of files', len(files))
539 515 if has_size:
540 516 size = float(cp_size) * 1.0e-6
541   - self._log_value(gammalib.NORMAL, 'Size', '%.2f MB' % size)
  517 + self._log_value(gammalib.TERSE, 'Size', '%.2f MB' % size)
542 518 self._log_header3(gammalib.VERBOSE, 'File names')
543 519 for filename in files:
544 520 self._log_string(gammalib.VERBOSE, str(filename)+'\n')
... ... @@ -548,8 +524,8 @@ class csiactcopy(ctools.cscript):
548 524  
549 525 # If prodname is not found just log that we skip the config
550 526 else:
551   - msg = 'Skipping config "%s"' % config['name']
552   - self._log_header3(gammalib.EXPLICIT, msg)
  527 + self._log_header3(gammalib.EXPLICIT, 'Skipping config "'+
  528 + str(config['name'])+'"')
553 529  
554 530 # Raise Exception if prodname was not found
555 531 if not has_prod:
... ... @@ -610,10 +586,15 @@ class csiactcopy(ctools.cscript):
610 586 k += 1
611 587  
612 588 # Logging
613   - self._log_value(gammalib.NORMAL, 'Status', 'Finished')
  589 + if self._logNormal() and not self._logVerbose():
  590 + self._log.parformat('Status')
  591 + self._log('Finished')
  592 + self._log('\n')
614 593  
615   - # Write header into logger
616   - self._log_header1(gammalib.TERSE, 'Updating index files')
  594 + # Logging about index files
  595 + if self._logTerse():
  596 + self._log('\n')
  597 + self._log.header1('Updating index files')
617 598  
618 599 # Build local hdu index file name
619 600 local_hdu = os.path.join(self._outpath,
... ...