Johan Bregeon / ctools

Commit 673a1331336dd9d563c3f7543feab02c9aaaf8eb

Authored by Jürgen Knödlseder
1 parent 0622a2cb

Put back former version of csiactcopy.py

Inline Side-by-side
Showing 1 changed file with 102 additions and 121 deletions
cscripts/csiactcopy.py
  View file @673a133
@@ -40,11 +40,11 @@ class csiactcopy(ctools.cscript): @@ -40,11 +40,11 @@ class csiactcopy(ctools.cscript):
40 # Constructor 40 # Constructor
41 def __init__(self, *argv): 41 def __init__(self, *argv):
42 """ 42 """
43 - Constructor 43 + Constructor.
44 """ 44 """
45 # Set name 45 # Set name
46 self._name = 'csiactcopy' 46 self._name = 'csiactcopy'
47 - self._version = '1.2.0' 47 + self._version = '1.1.0'
48 self._datapath = os.getenv('VHEFITS','') 48 self._datapath = os.getenv('VHEFITS','')
49 self._remote_master = gammalib.GFilename() 49 self._remote_master = gammalib.GFilename()
50 self._remote_base = '' 50 self._remote_base = ''
@@ -66,13 +66,14 @@ class csiactcopy(ctools.cscript): @@ -66,13 +66,14 @@ class csiactcopy(ctools.cscript):
66 """ 66 """
67 Get parameters from parfile and setup the observation 67 Get parameters from parfile and setup the observation
68 """ 68 """
  69 +
69 # Get Parameters 70 # Get Parameters
70 self._remote_master = self['remote_master'].filename() 71 self._remote_master = self['remote_master'].filename()
71 if not self._remote_master.exists(): 72 if not self._remote_master.exists():
72 - msg = ('*** ERROR: FITS data not available. No master index file '  
73 - 'found in "%s". Make sure remote file system is properly '  
74 - 'mounted.' % self._remote_master)  
75 - raise RuntimeError(msg) 73 + raise RuntimeError('*** ERROR: FITS data not available. No '
  74 + 'master index file found in "'+
  75 + self._remote_master+'". Make sure remote '
  76 + 'file system is properly mounted.')
76 77
77 # Get parameters 78 # Get parameters
78 self._prodname = self['prodname'].string() 79 self._prodname = self['prodname'].string()
@@ -87,11 +88,11 @@ class csiactcopy(ctools.cscript): @@ -87,11 +88,11 @@ class csiactcopy(ctools.cscript):
87 88
88 def _copy(self, source, clobber): 89 def _copy(self, source, clobber):
89 """ 90 """
90 - Copy file to self._outpath directory 91 + Copy file to outpath
91 92
92 Parameters 93 Parameters
93 ---------- 94 ----------
94 - source : str 95 + source : string
95 Path of file to be copied 96 Path of file to be copied
96 97
97 Returns 98 Returns
@@ -146,39 +147,39 @@ class csiactcopy(ctools.cscript): @@ -146,39 +147,39 @@ class csiactcopy(ctools.cscript):
146 def _merge(self, localfits, remotefits, hduname, clobber): 147 def _merge(self, localfits, remotefits, hduname, clobber):
147 """ 148 """
148 Merge remote and local fits files 149 Merge remote and local fits files
149 -  
150 If the local fits file is not present, a new one is created. 150 If the local fits file is not present, a new one is created.
151 151
152 Parameters 152 Parameters
153 ---------- 153 ----------
154 - localfits : str 154 + localfits : string
155 Path of local index FITS file 155 Path of local index FITS file
156 - remotefits : str 156 + remotefits : string
157 Path of remote index FITS file 157 Path of remote index FITS file
158 - hduname : str 158 + hduname : string
159 Name of HDU extension to be merged 159 Name of HDU extension to be merged
160 clobber : bool 160 clobber : bool
161 Flag if remote content should overwrite local content 161 Flag if remote content should overwrite local content
162 - """  
163 - # Write input parameters into logger  
164 - self._log_value(gammalib.NORMAL, 'Local file', localfits)  
165 - self._log_value(gammalib.NORMAL, 'Remote file', remotefits)  
166 - self._log_value(gammalib.NORMAL, 'HDU name', hduname) 162 + """
  163 +
  164 + # Logging
  165 + self._log_value(gammalib.TERSE, 'Local file', localfits)
  166 + self._log_value(gammalib.TERSE, 'Remote file', remotefits)
  167 + self._log_value(gammalib.TERSE, 'HDU name', hduname)
167 if clobber: 168 if clobber:
168 preference = 'Remote' 169 preference = 'Remote'
169 else: 170 else:
170 preference = 'Local' 171 preference = 'Local'
171 - self._log_value(gammalib.NORMAL, 'Conflict preference', preference) 172 + self._log_value(gammalib.TERSE, 'Conflict preference', preference)
172 173
173 # Check if local fits file is available 174 # Check if local fits file is available
174 if os.path.isfile(localfits): 175 if os.path.isfile(localfits):
175 - local = gammalib.GFits(localfits) 176 + local = gammalib.GFits(str(localfits))
176 local_hdu = local[hduname] 177 local_hdu = local[hduname]
177 else: 178 else:
178 # Otherwise load remote fits file and delete rows 179 # Otherwise load remote fits file and delete rows
179 - local = gammalib.GFits(remotefits) 180 + local = gammalib.GFits(str(remotefits))
180 local_hdu = local[hduname] 181 local_hdu = local[hduname]
181 - local_hdu.remove_rows(0, local_hdu.nrows()) 182 + local_hdu.remove_rows(0,local_hdu.nrows())
182 183
183 # find local obs_id 184 # find local obs_id
184 local_obs = [] 185 local_obs = []
@@ -186,14 +187,13 @@ class csiactcopy(ctools.cscript): @@ -186,14 +187,13 @@ class csiactcopy(ctools.cscript):
186 for obs_id in lobs_col: 187 for obs_id in lobs_col:
187 local_obs.append(obs_id) 188 local_obs.append(obs_id)
188 189
189 - # Initialise run list. A kluge has been implemented to circumvent a  
190 - # too long file name: the run list is split into chunks of 50  
191 - # observations maximum. Note also the "//" division operator that  
192 - # makes sure that "i" is an integer on Python 3.  
193 - runlists = [] 190 + # Initialise list of runlists
  191 + # This is kluge: to circumvent a too long file name, we split the
  192 + # runlists into chunks of 50 observations maximum
  193 + runlists = []
194 runs_per_chunk = 50 194 runs_per_chunk = 50
195 - for i in range(len(self._runs) // runs_per_chunk + 1):  
196 - chunk = self._runs[i * runs_per_chunk : (i+1) * runs_per_chunk] 195 + for i in range(len(self._runs) / runs_per_chunk + 1):
  196 + chunk = self._runs[i * runs_per_chunk : (i + 1) * runs_per_chunk]
197 runlists.append(chunk) 197 runlists.append(chunk)
198 198
199 # Loop over runlist chunks 199 # Loop over runlist chunks
@@ -241,8 +241,8 @@ class csiactcopy(ctools.cscript): @@ -241,8 +241,8 @@ class csiactcopy(ctools.cscript):
241 remote_obs.append(obs_id) 241 remote_obs.append(obs_id)
242 242
243 # Log entries of index files 243 # Log entries of index files
244 - self._log_value(gammalib.NORMAL, 'Remote entries', len(remote_obs))  
245 - self._log_value(gammalib.NORMAL, 'Local entries', len(local_obs)) 244 + self._log_value(gammalib.TERSE, 'Remote entries', len(remote_obs))
  245 + self._log_value(gammalib.TERSE, 'Local entries', len(local_obs))
246 246
247 # initialise counter for logging 247 # initialise counter for logging
248 removed_rows = 0 248 removed_rows = 0
@@ -311,13 +311,13 @@ class csiactcopy(ctools.cscript): @@ -311,13 +311,13 @@ class csiactcopy(ctools.cscript):
311 what = 'Removed local rows' 311 what = 'Removed local rows'
312 else: 312 else:
313 what = 'Skipped remote rows' 313 what = 'Skipped remote rows'
314 - self._log_value(gammalib.NORMAL, what, removed_rows) 314 + self._log_value(gammalib.TERSE, what, removed_rows)
315 315
316 # Loop over columns 316 # Loop over columns
317 for i in range(local_hdu.ncols()): 317 for i in range(local_hdu.ncols()):
318 318
319 # Get local and remote columns 319 # Get local and remote columns
320 - local_col = local_hdu[i] 320 + local_col = local_hdu[i]
321 remote_col = remote_hdu[i] 321 remote_col = remote_hdu[i]
322 322
323 # Loop over entries and merge 323 # Loop over entries and merge
@@ -325,103 +325,77 @@ class csiactcopy(ctools.cscript): @@ -325,103 +325,77 @@ class csiactcopy(ctools.cscript):
325 local_col[j+old_local_rows] = remote_col[j] 325 local_col[j+old_local_rows] = remote_col[j]
326 326
327 # Save local fits file 327 # Save local fits file
328 - local.saveto(localfits, True) 328 + local.saveto(str(localfits), True)
329 329
330 # Return 330 # Return
331 return 331 return
332 332
333 - def _set_runs(self, filename): 333 +
  334 + # Public methods
  335 + def run(self):
  336 + """
  337 + Run the script
334 """ 338 """
335 - Set the run list 339 + # Switch screen logging on in debug mode
  340 + if self._logDebug():
  341 + self._log.cout(True)
336 342
337 - Parameters  
338 - ----------  
339 - filename : `~gammalib.GFilename`  
340 - Run list file name 343 + # Get parameters
  344 + self._get_parameters()
341 345
342 - Returns  
343 - -------  
344 - runs : list of int  
345 - Run list  
346 - """  
347 - # Write header into logger  
348 - self._log_header1(gammalib.TERSE, 'Set runlist') 346 + # Make destination directory if not available
  347 + if not os.path.isdir(self._outpath):
  348 + os.makedirs(self._outpath)
349 349
350 - # Initialise runlist  
351 - runs = [] 350 + # Log output
  351 + if self._logTerse():
  352 + self._log('\n')
  353 + self._log.header2('Runlist information')
352 354
353 - # If runlist file exists then open file and extract the run  
354 - # information into the run list  
355 - if filename.exists():  
356 -  
357 - # Open the runlist file  
358 - runfile = open(filename.url()) 355 + # Initialise runlist array
  356 + self._runs = []
  357 +
  358 + # Check if runlist file is available
  359 + if self._runlist.exists():
  360 + runfile = open(self._runlist.url())
359 361
360 - # Read runlist file and append all run ID as integers to the  
361 - # run list. Skip any blank or comment lines 362 + # Read runlist
362 for line in runfile.readlines(): 363 for line in runfile.readlines():
363 if len(line) == 0: 364 if len(line) == 0:
364 continue 365 continue
365 if line[0] == '#': 366 if line[0] == '#':
366 continue 367 continue
367 if len(line.split()) > 0: 368 if len(line.split()) > 0:
368 - run = int(line.split()[0])  
369 - runs.append(run)  
370 - self._log_value(gammalib.EXPLICIT, 'Run %d' % len(runs), run) 369 + self._runs.append(int(line.split()[0]))
371 370
372 # Close runlist file 371 # Close runlist file
373 runfile.close() 372 runfile.close()
374 373
375 # Logging 374 # Logging
376 - self._log_value(gammalib.NORMAL, 'Number of observations',  
377 - len(runs)) 375 + self._log_value(gammalib.TERSE, 'Number of observations',
  376 + len(self._runs))
378 377
379 - # ... otherwise, if the runlist file is 'NONE' then leave the  
380 - # run list empty. This implies copying all available data.  
381 - elif filename == 'NONE':  
382 - self._log_string(gammalib.NORMAL, 'Copy all available data') 378 + # Copy all data if runlist file is 'NONE'
  379 + elif self._runlist == 'NONE':
  380 + self._log_string(gammalib.TERSE, 'Copy all available data')
383 381
384 - # ... otherwise raise an exception 382 + # Raise exception if file not valid
385 else: 383 else:
386 - msg = '*** ERROR: Runlist file "%s" not available' % filename  
387 - raise RuntimeError(msg) 384 + raise RuntimeError('*** ERROR: Runlist file "'+
  385 + self._runlist+'" not available.')
388 386
389 - # Return run list  
390 - return runs  
391 -  
392 -  
393 - # Public methods  
394 - def run(self):  
395 - """  
396 - Run the script  
397 - """  
398 - # Switch screen logging on in debug mode  
399 - if self._logDebug():  
400 - self._log.cout(True)  
401 -  
402 - # Get parameters  
403 - self._get_parameters()  
404 -  
405 - # Make destination directory if not available  
406 - if not os.path.isdir(self._outpath):  
407 - os.makedirs(self._outpath)  
408 -  
409 - # Set run list  
410 - self._runs = self._set_runs(self._runlist)  
411 -  
412 # Check for availability of remote master file 387 # Check for availability of remote master file
413 if not self._remote_master.exists(): 388 if not self._remote_master.exists():
414 - msg = ('*** ERROR: Remote master file "%s" does not exist' %  
415 - self._remote_master)  
416 - raise RuntimeError(msg) 389 + raise RuntimeError('*** ERROR: Remote master file "'+
  390 + self._remote_master+'" does not exist.')
417 391
418 # Retrieve json data from remote master 392 # Retrieve json data from remote master
419 json_data = open(self._remote_master.url()).read() 393 json_data = open(self._remote_master.url()).read()
420 data = json.loads(json_data) 394 data = json.loads(json_data)
421 if not 'datasets' in data: 395 if not 'datasets' in data:
422 - msg = ('*** ERROR: Key "datasets" not available in remote master '  
423 - 'index file "%s"' % self._remote_master)  
424 - raise RuntimeError(msg) 396 + raise RuntimeError('*** ERROR: Key "datasets" not available '
  397 + 'in remote master index file "'+
  398 + self._remote_master+'".')
425 399
426 # Get array of configurations 400 # Get array of configurations
427 configs = data['datasets'] 401 configs = data['datasets']
@@ -435,16 +409,17 @@ class csiactcopy(ctools.cscript): @@ -435,16 +409,17 @@ class csiactcopy(ctools.cscript):
435 # Initialise file names to be copied 409 # Initialise file names to be copied
436 files = set() 410 files = set()
437 411
438 - # Write header into logger  
439 - self._log_header2(gammalib.TERSE, 'Loop over remote configs') 412 + # Logging
  413 + if self._logTerse():
  414 + self._log('\n')
  415 + self._log.header2('Loop over remote configs')
440 416
441 # Loop over configs 417 # Loop over configs
442 - for config in configs:  
443 -  
444 - # Write header into logger. The str() is needed since  
445 - # "config['name']" is <type 'unicode'>, and under Python 2 a  
446 - # string is expected.  
447 - self._log_header2(gammalib.VERBOSE, str(config['name'])) 418 + for config in configs:
  419 +
  420 + # Logging
  421 + if self._logVerbose():
  422 + self._log.header2(str(config['name']))
448 423
449 # Check if prodname was found 424 # Check if prodname was found
450 if config['name'] == self._prodname: 425 if config['name'] == self._prodname:
@@ -457,13 +432,13 @@ class csiactcopy(ctools.cscript): @@ -457,13 +432,13 @@ class csiactcopy(ctools.cscript):
457 remote_obs = os.path.join(self._remote_base, config['obsindx']) 432 remote_obs = os.path.join(self._remote_base, config['obsindx'])
458 433
459 # Log information 434 # Log information
460 - self._log_header3(gammalib.NORMAL, 'Remote config "'+ 435 + self._log_header3(gammalib.TERSE, 'Remote config "'+
461 self._prodname+'"') 436 self._prodname+'"')
462 - self._log_value(gammalib.NORMAL, 'HDU index', remote_hdu)  
463 - self._log_value(gammalib.NORMAL, 'Observation index', remote_obs) 437 + self._log_value(gammalib.TERSE, 'HDU index', remote_hdu)
  438 + self._log_value(gammalib.TERSE, 'Observation index', remote_obs)
464 439
465 # Open remote HDU index file 440 # Open remote HDU index file
466 - fits = gammalib.GFits(remote_hdu) 441 + fits = gammalib.GFits(str(remote_hdu))
467 table = fits['HDU_INDEX'] 442 table = fits['HDU_INDEX']
468 443
469 # Initialise flag if SIZE column is present 444 # Initialise flag if SIZE column is present
@@ -482,10 +457,11 @@ class csiactcopy(ctools.cscript): @@ -482,10 +457,11 @@ class csiactcopy(ctools.cscript):
482 for run in self._runs: 457 for run in self._runs:
483 458
484 # Check for run not in remote data store 459 # Check for run not in remote data store
485 - if not run in remote_ids:  
486 - msg = ('Skip observation "%s": ID not available '  
487 - 'remotely' % str(run))  
488 - self._log_string(msg) 460 + if not run in remote_ids:
  461 + if self._logNormal():
  462 + self._log('Skip observation '+str(run)+': ')
  463 + self._log('ID not available remotely')
  464 + self._log('\n')
489 465
490 # Loop over remote HDU index file 466 # Loop over remote HDU index file
491 for row in range(table.nrows()): 467 for row in range(table.nrows()):
@@ -534,11 +510,11 @@ class csiactcopy(ctools.cscript): @@ -534,11 +510,11 @@ class csiactcopy(ctools.cscript):
534 cp_size += table['SIZE'][row] 510 cp_size += table['SIZE'][row]
535 511
536 # Log file information 512 # Log file information
537 - self._log_header2(gammalib.NORMAL, 'File information')  
538 - self._log_value(gammalib.NORMAL, 'Number of files', len(files)) 513 + self._log_header2(gammalib.TERSE, 'File information')
  514 + self._log_value(gammalib.TERSE, 'Number of files', len(files))
539 if has_size: 515 if has_size:
540 size = float(cp_size) * 1.0e-6 516 size = float(cp_size) * 1.0e-6
541 - self._log_value(gammalib.NORMAL, 'Size', '%.2f MB' % size) 517 + self._log_value(gammalib.TERSE, 'Size', '%.2f MB' % size)
542 self._log_header3(gammalib.VERBOSE, 'File names') 518 self._log_header3(gammalib.VERBOSE, 'File names')
543 for filename in files: 519 for filename in files:
544 self._log_string(gammalib.VERBOSE, str(filename)+'\n') 520 self._log_string(gammalib.VERBOSE, str(filename)+'\n')
@@ -548,8 +524,8 @@ class csiactcopy(ctools.cscript): @@ -548,8 +524,8 @@ class csiactcopy(ctools.cscript):
548 524
549 # If prodname is not found just log that we skip the config 525 # If prodname is not found just log that we skip the config
550 else: 526 else:
551 - msg = 'Skipping config "%s"' % config['name']  
552 - self._log_header3(gammalib.EXPLICIT, msg) 527 + self._log_header3(gammalib.EXPLICIT, 'Skipping config "'+
  528 + str(config['name'])+'"')
553 529
554 # Raise Exception if prodname was not found 530 # Raise Exception if prodname was not found
555 if not has_prod: 531 if not has_prod:
@@ -610,10 +586,15 @@ class csiactcopy(ctools.cscript): @@ -610,10 +586,15 @@ class csiactcopy(ctools.cscript):
610 k += 1 586 k += 1
611 587
612 # Logging 588 # Logging
613 - self._log_value(gammalib.NORMAL, 'Status', 'Finished') 589 + if self._logNormal() and not self._logVerbose():
  590 + self._log.parformat('Status')
  591 + self._log('Finished')
  592 + self._log('\n')
614 593
615 - # Write header into logger  
616 - self._log_header1(gammalib.TERSE, 'Updating index files') 594 + # Logging about index files
  595 + if self._logTerse():
  596 + self._log('\n')
  597 + self._log.header1('Updating index files')
617 598
618 # Build local hdu index file name 599 # Build local hdu index file name
619 local_hdu = os.path.join(self._outpath, 600 local_hdu = os.path.join(self._outpath,