Michael Mayer / ctools

Commit 0746e91989a80ef482768bb042a587cfd65f922f

Authored by Jürgen Knödlseder
1 parent 45a80b60

Fix Python 3 problem with csiactcopy.py script

Inline Side-by-side
Showing 3 changed files with 151 additions and 120 deletions
cscripts/csiactcopy.py
  View file @0746e91
... ... @@ -40,11 +40,11 @@ class csiactcopy(ctools.cscript):
40 40 # Constructor
41 41 def __init__(self, *argv):
42 42 """
43   - Constructor.
  43 + Constructor
44 44 """
45 45 # Set name
46 46 self._name = 'csiactcopy'
47   - self._version = '1.1.0'
  47 + self._version = '1.2.0'
48 48 self._datapath = os.getenv('VHEFITS','')
49 49 self._remote_master = gammalib.GFilename()
50 50 self._remote_base = ''
... ... @@ -66,14 +66,13 @@ class csiactcopy(ctools.cscript):
66 66 """
67 67 Get parameters from parfile and setup the observation
68 68 """
69   -
70 69 # Get Parameters
71 70 self._remote_master = self['remote_master'].filename()
72 71 if not self._remote_master.exists():
73   - raise RuntimeError('*** ERROR: FITS data not available. No '
74   - 'master index file found in "'+
75   - self._remote_master+'". Make sure remote '
76   - 'file system is properly mounted.')
  72 + msg = ('*** ERROR: FITS data not available. No master index file '
  73 + 'found in "%s". Make sure remote file system is properly '
  74 + 'mounted.' % self._remote_master)
  75 + raise RuntimeError(msg)
77 76  
78 77 # Get parameters
79 78 self._prodname = self['prodname'].string()
... ... @@ -88,11 +87,11 @@ class csiactcopy(ctools.cscript):
88 87  
89 88 def _copy(self, source, clobber):
90 89 """
91   - Copy file to outpath
  90 + Copy file to self._outpath directory
92 91  
93 92 Parameters
94 93 ----------
95   - source : string
  94 + source : str
96 95 Path of file to be copied
97 96  
98 97 Returns
... ... @@ -147,39 +146,39 @@ class csiactcopy(ctools.cscript):
147 146 def _merge(self, localfits, remotefits, hduname, clobber):
148 147 """
149 148 Merge remote and local fits files
  149 +
150 150 If the local fits file is not present, a new one is created.
151 151  
152 152 Parameters
153 153 ----------
154   - localfits : string
  154 + localfits : str
155 155 Path of local index FITS file
156   - remotefits : string
  156 + remotefits : str
157 157 Path of remote index FITS file
158   - hduname : string
  158 + hduname : str
159 159 Name of HDU extension to be merged
160 160 clobber : bool
161 161 Flag if remote content should overwrite local content
162   - """
163   -
164   - # Logging
165   - self._log_value(gammalib.TERSE, 'Local file', localfits)
166   - self._log_value(gammalib.TERSE, 'Remote file', remotefits)
167   - self._log_value(gammalib.TERSE, 'HDU name', hduname)
  162 + """
  163 + # Write input parameters into logger
  164 + self._log_value(gammalib.NORMAL, 'Local file', localfits)
  165 + self._log_value(gammalib.NORMAL, 'Remote file', remotefits)
  166 + self._log_value(gammalib.NORMAL, 'HDU name', hduname)
168 167 if clobber:
169 168 preference = 'Remote'
170 169 else:
171 170 preference = 'Local'
172   - self._log_value(gammalib.TERSE, 'Conflict preference', preference)
  171 + self._log_value(gammalib.NORMAL, 'Conflict preference', preference)
173 172  
174 173 # Check if local fits file is available
175 174 if os.path.isfile(localfits):
176   - local = gammalib.GFits(str(localfits))
  175 + local = gammalib.GFits(localfits)
177 176 local_hdu = local[hduname]
178 177 else:
179 178 # Otherwise load remote fits file and delete rows
180   - local = gammalib.GFits(str(remotefits))
  179 + local = gammalib.GFits(remotefits)
181 180 local_hdu = local[hduname]
182   - local_hdu.remove_rows(0,local_hdu.nrows())
  181 + local_hdu.remove_rows(0, local_hdu.nrows())
183 182  
184 183 # find local obs_id
185 184 local_obs = []
... ... @@ -187,13 +186,14 @@ class csiactcopy(ctools.cscript):
187 186 for obs_id in lobs_col:
188 187 local_obs.append(obs_id)
189 188  
190   - # Initialise list of runlists
191   - # This is kluge: to circumvent a too long file name, we split the
192   - # runlists into chunks of 50 observations maximum
193   - runlists = []
  189 + # Initialise run list. A kluge has been implemented to circumvent a
  190 + # too long file name: the run list is split into chunks of 50
  191 + # observations maximum. Note also the "//" division operator that
  192 + # makes sure that "i" is an integer on Python 3.
  193 + runlists = []
194 194 runs_per_chunk = 50
195   - for i in range(len(self._runs) / runs_per_chunk + 1):
196   - chunk = self._runs[i * runs_per_chunk : (i + 1) * runs_per_chunk]
  195 + for i in range(len(self._runs) // runs_per_chunk + 1):
  196 + chunk = self._runs[i * runs_per_chunk : (i+1) * runs_per_chunk]
197 197 runlists.append(chunk)
198 198  
199 199 # Loop over runlist chunks
... ... @@ -241,8 +241,8 @@ class csiactcopy(ctools.cscript):
241 241 remote_obs.append(obs_id)
242 242  
243 243 # Log entries of index files
244   - self._log_value(gammalib.TERSE, 'Remote entries', len(remote_obs))
245   - self._log_value(gammalib.TERSE, 'Local entries', len(local_obs))
  244 + self._log_value(gammalib.NORMAL, 'Remote entries', len(remote_obs))
  245 + self._log_value(gammalib.NORMAL, 'Local entries', len(local_obs))
246 246  
247 247 # initialise counter for logging
248 248 removed_rows = 0
... ... @@ -311,13 +311,13 @@ class csiactcopy(ctools.cscript):
311 311 what = 'Removed local rows'
312 312 else:
313 313 what = 'Skipped remote rows'
314   - self._log_value(gammalib.TERSE, what, removed_rows)
  314 + self._log_value(gammalib.NORMAL, what, removed_rows)
315 315  
316 316 # Loop over columns
317 317 for i in range(local_hdu.ncols()):
318 318  
319 319 # Get local and remote columns
320   - local_col = local_hdu[i]
  320 + local_col = local_hdu[i]
321 321 remote_col = remote_hdu[i]
322 322  
323 323 # Loop over entries and merge
... ... @@ -325,77 +325,103 @@ class csiactcopy(ctools.cscript):
325 325 local_col[j+old_local_rows] = remote_col[j]
326 326  
327 327 # Save local fits file
328   - local.saveto(str(localfits), True)
  328 + local.saveto(localfits, True)
329 329  
330 330 # Return
331 331 return
332 332  
333   -
334   - # Public methods
335   - def run(self):
336   - """
337   - Run the script
  333 + def _set_runs(self, filename):
338 334 """
339   - # Switch screen logging on in debug mode
340   - if self._logDebug():
341   - self._log.cout(True)
  335 + Set the run list
342 336  
343   - # Get parameters
344   - self._get_parameters()
  337 + Parameters
  338 + ----------
  339 + filename : `~gammalib.GFilename`
  340 + Run list file name
345 341  
346   - # Make destination directory if not available
347   - if not os.path.isdir(self._outpath):
348   - os.makedirs(self._outpath)
349   -
350   - # Log output
351   - if self._logTerse():
352   - self._log('\n')
353   - self._log.header2('Runlist information')
  342 + Returns
  343 + -------
  344 + runs : list of int
  345 + Run list
  346 + """
  347 + # Write header into logger
  348 + self._log_header1(gammalib.TERSE, 'Set runlist')
354 349  
355   - # Initialise runlist array
356   - self._runs = []
  350 + # Initialise runlist
  351 + runs = []
357 352  
358   - # Check if runlist file is available
359   - if self._runlist.exists():
360   - runfile = open(self._runlist.url())
  353 + # If runlist file exists then open file and extract the run
  354 + # information into the run list
  355 + if filename.exists():
  356 +
  357 + # Open the runlist file
  358 + runfile = open(filename.url())
361 359  
362   - # Read runlist
  360 + # Read runlist file and append all run ID as integers to the
  361 + # run list. Skip any blank or comment lines
363 362 for line in runfile.readlines():
364 363 if len(line) == 0:
365 364 continue
366 365 if line[0] == '#':
367 366 continue
368 367 if len(line.split()) > 0:
369   - self._runs.append(int(line.split()[0]))
  368 + run = int(line.split()[0])
  369 + runs.append(run)
  370 + self._log_value(gammalib.EXPLICIT, 'Run %d' % len(runs), run)
370 371  
371 372 # Close runlist file
372 373 runfile.close()
373 374  
374 375 # Logging
375   - self._log_value(gammalib.TERSE, 'Number of observations',
376   - len(self._runs))
  376 + self._log_value(gammalib.NORMAL, 'Number of observations',
  377 + len(runs))
377 378  
378   - # Copy all data if runlist file is 'NONE'
379   - elif self._runlist == 'NONE':
380   - self._log_string(gammalib.TERSE, 'Copy all available data')
  379 + # ... otherwise, if the runlist file is 'NONE' then leave the
  380 + # run list empty. This implies copying all available data.
  381 + elif filename == 'NONE':
  382 + self._log_string(gammalib.NORMAL, 'Copy all available data')
381 383  
382   - # Raise exception if file not valid
  384 + # ... otherwise raise an exception
383 385 else:
384   - raise RuntimeError('*** ERROR: Runlist file "'+
385   - self._runlist+'" not available.')
  386 + msg = '*** ERROR: Runlist file "%s" not available' % filename
  387 + raise RuntimeError(msg)
386 388  
  389 + # Return run list
  390 + return runs
  391 +
  392 +
  393 + # Public methods
  394 + def run(self):
  395 + """
  396 + Run the script
  397 + """
  398 + # Switch screen logging on in debug mode
  399 + if self._logDebug():
  400 + self._log.cout(True)
  401 +
  402 + # Get parameters
  403 + self._get_parameters()
  404 +
  405 + # Make destination directory if not available
  406 + if not os.path.isdir(self._outpath):
  407 + os.makedirs(self._outpath)
  408 +
  409 + # Set run list
  410 + self._runs = self._set_runs(self._runlist)
  411 +
387 412 # Check for availability of remote master file
388 413 if not self._remote_master.exists():
389   - raise RuntimeError('*** ERROR: Remote master file "'+
390   - self._remote_master+'" does not exist.')
  414 + msg = ('*** ERROR: Remote master file "%s" does not exist' %
  415 + self._remote_master)
  416 + raise RuntimeError(msg)
391 417  
392 418 # Retrieve json data from remote master
393 419 json_data = open(self._remote_master.url()).read()
394 420 data = json.loads(json_data)
395 421 if not 'datasets' in data:
396   - raise RuntimeError('*** ERROR: Key "datasets" not available '
397   - 'in remote master index file "'+
398   - self._remote_master+'".')
  422 + msg = ('*** ERROR: Key "datasets" not available in remote master '
  423 + 'index file "%s"' % self._remote_master)
  424 + raise RuntimeError(msg)
399 425  
400 426 # Get array of configurations
401 427 configs = data['datasets']
... ... @@ -409,17 +435,16 @@ class csiactcopy(ctools.cscript):
409 435 # Initialise file names to be copied
410 436 files = set()
411 437  
412   - # Logging
413   - if self._logTerse():
414   - self._log('\n')
415   - self._log.header2('Loop over remote configs')
  438 + # Write header into logger
  439 + self._log_header2(gammalib.TERSE, 'Loop over remote configs')
416 440  
417 441 # Loop over configs
418   - for config in configs:
419   -
420   - # Logging
421   - if self._logVerbose():
422   - self._log.header2(str(config['name']))
  442 + for config in configs:
  443 +
  444 + # Write header into logger. The str() is needed since
  445 + # "config['name']" is <type 'unicode'>, and under Python 2 a
  446 + # string is expected.
  447 + self._log_header2(gammalib.VERBOSE, str(config['name']))
423 448  
424 449 # Check if prodname was found
425 450 if config['name'] == self._prodname:
... ... @@ -432,13 +457,13 @@ class csiactcopy(ctools.cscript):
432 457 remote_obs = os.path.join(self._remote_base, config['obsindx'])
433 458  
434 459 # Log information
435   - self._log_header3(gammalib.TERSE, 'Remote config "'+
  460 + self._log_header3(gammalib.NORMAL, 'Remote config "'+
436 461 self._prodname+'"')
437   - self._log_value(gammalib.TERSE, 'HDU index', remote_hdu)
438   - self._log_value(gammalib.TERSE, 'Observation index', remote_obs)
  462 + self._log_value(gammalib.NORMAL, 'HDU index', remote_hdu)
  463 + self._log_value(gammalib.NORMAL, 'Observation index', remote_obs)
439 464  
440 465 # Open remote HDU index file
441   - fits = gammalib.GFits(str(remote_hdu))
  466 + fits = gammalib.GFits(remote_hdu)
442 467 table = fits['HDU_INDEX']
443 468  
444 469 # Initialise flag if SIZE column is present
... ... @@ -457,11 +482,10 @@ class csiactcopy(ctools.cscript):
457 482 for run in self._runs:
458 483  
459 484 # Check for run not in remote data store
460   - if not run in remote_ids:
461   - if self._logNormal():
462   - self._log('Skip observation '+str(run)+': ')
463   - self._log('ID not available remotely')
464   - self._log('\n')
  485 + if not run in remote_ids:
  486 + msg = ('Skip observation "%s": ID not available '
  487 + 'remotely' % str(run))
  488 + self._log_string(msg)
465 489  
466 490 # Loop over remote HDU index file
467 491 for row in range(table.nrows()):
... ... @@ -510,11 +534,11 @@ class csiactcopy(ctools.cscript):
510 534 cp_size += table['SIZE'][row]
511 535  
512 536 # Log file information
513   - self._log_header2(gammalib.TERSE, 'File information')
514   - self._log_value(gammalib.TERSE, 'Number of files', len(files))
  537 + self._log_header2(gammalib.NORMAL, 'File information')
  538 + self._log_value(gammalib.NORMAL, 'Number of files', len(files))
515 539 if has_size:
516 540 size = float(cp_size) * 1.0e-6
517   - self._log_value(gammalib.TERSE, 'Size', '%.2f MB' % size)
  541 + self._log_value(gammalib.NORMAL, 'Size', '%.2f MB' % size)
518 542 self._log_header3(gammalib.VERBOSE, 'File names')
519 543 for filename in files:
520 544 self._log_string(gammalib.VERBOSE, str(filename)+'\n')
... ... @@ -524,8 +548,8 @@ class csiactcopy(ctools.cscript):
524 548  
525 549 # If prodname is not found just log that we skip the config
526 550 else:
527   - self._log_header3(gammalib.EXPLICIT, 'Skipping config "'+
528   - str(config['name'])+'"')
  551 + msg = 'Skipping config "%s"' % config['name']
  552 + self._log_header3(gammalib.EXPLICIT, msg)
529 553  
530 554 # Raise Exception if prodname was not found
531 555 if not has_prod:
... ... @@ -586,15 +610,10 @@ class csiactcopy(ctools.cscript):
586 610 k += 1
587 611  
588 612 # Logging
589   - if self._logNormal() and not self._logVerbose():
590   - self._log.parformat('Status')
591   - self._log('Finished')
592   - self._log('\n')
  613 + self._log_value(gammalib.NORMAL, 'Status', 'Finished')
593 614  
594   - # Logging about index files
595   - if self._logTerse():
596   - self._log('\n')
597   - self._log.header1('Updating index files')
  615 + # Write header into logger
  616 + self._log_header1(gammalib.TERSE, 'Updating index files')
598 617  
599 618 # Build local hdu index file name
600 619 local_hdu = os.path.join(self._outpath,
... ...
cscripts/csiactobs.py
  View file @0746e91
... ... @@ -43,7 +43,7 @@ class csiactobs(ctools.cscript):
43 43 # Constructor
44 44 def __init__(self, *argv):
45 45 """
46   - Constructor.
  46 + Constructor
47 47 """
48 48 # Set name and version
49 49 self._name = 'csiactobs'
... ... @@ -229,9 +229,9 @@ class csiactobs(ctools.cscript):
229 229 Parameters
230 230 ----------
231 231 prefactor : float
232   - Prefactor parameter of spectral model
  232 + Power law prefactor of spectral model
233 233 index : float
234   - Index parameter of spectral model
  234 + Power law index of spectral model
235 235 emin : float
236 236 Minimum energy (in case a spectral node function is required)
237 237 emax : float
... ... @@ -330,13 +330,13 @@ class csiactobs(ctools.cscript):
330 330  
331 331 Parameters
332 332 ----------
333   - telescope : string
  333 + telescope : str
334 334 Name of telescope
335   - obs_id : string
  335 + obs_id : str
336 336 Observation ID
337 337 bkg_scale : float
338 338 Background scaling factor
339   - bkgtype : string
  339 + bkgtype : str
340 340 Type of background (irf,aeff, or gauss)
341 341  
342 342 Returns
... ...
test/test_csiactcopy.py
  View file @0746e91
... ... @@ -162,8 +162,9 @@ class Test(test):
162 162 self._check_copy('iactdata_py1')
163 163 self._check_n_obs('iactdata_py1', 9)
164 164  
  165 + # Remove the copied IACT data store before continuing
165 166 if os.path.isdir('iactdata_py2'):
166   - os.system("rm -r iactdata_py2")
  167 + os.system('rm -r iactdata_py2')
167 168  
168 169 # Set-up csiactcopy
169 170 iactcopy = cscripts.csiactcopy()
... ... @@ -172,7 +173,7 @@ class Test(test):
172 173 iactcopy['prodname'] = 'unit-test'
173 174 iactcopy['outpath'] = 'iactdata_py2'
174 175 iactcopy['logfile'] = 'csiactcopy_py4.log'
175   - iactcopy['chatter'] = 3
  176 + iactcopy['chatter'] = 3
176 177  
177 178 # Run csiactcopy script and save run list
178 179 iactcopy.logFileOpen() # Make sure we get a log file
... ... @@ -223,10 +224,15 @@ class Test(test):
223 224 # Return
224 225 return
225 226  
226   - # Check copy
  227 + # Check copy of IACT data store
227 228 def _check_copy(self, pathname):
228 229 """
229   - Check copy.
  230 + Check copy of IACT data store
  231 +
  232 + Parameters
  233 + ----------
  234 + pathname : str
  235 + Path to copied IACT data store
230 236 """
231 237 # Set file names
232 238 hdu_index_name = gammalib.GFilename(pathname+'/hdu-index.fits')
... ... @@ -235,17 +241,17 @@ class Test(test):
235 241  
236 242 # Check for existence of files
237 243 self.test_assert(hdu_index_name.exists(),
238   - 'Check if file "hdu-index.fits" exists.')
  244 + 'Check if file "hdu-index.fits" exists')
239 245 self.test_assert(obs_index_name.exists(),
240   - 'Check if file "obs-index.fits" exists.')
  246 + 'Check if file "obs-index.fits" exists')
241 247 self.test_assert(master_name.exists(),
242   - 'Check if file "master.json" exists.')
  248 + 'Check if file "master.json" exists')
243 249  
244 250 # Check if index files are FITS file
245 251 self.test_assert(hdu_index_name.is_fits(),
246   - 'Check if file "hdu-index.fits" is FITS file.')
  252 + 'Check if file "hdu-index.fits" is FITS file')
247 253 self.test_assert(obs_index_name.is_fits(),
248   - 'Check if file "obs-index.fits" is FITS file.')
  254 + 'Check if file "obs-index.fits" is FITS file')
249 255  
250 256 # Return
251 257 return
... ... @@ -253,7 +259,14 @@ class Test(test):
253 259 # Check copy
254 260 def _check_n_obs(self, pathname, n_expected):
255 261 """
256   - Check number of available observations.
  262 + Check number of available observations
  263 +
  264 + Parameters
  265 + ----------
  266 + pathname : str
  267 + Path to copied IACT data store
  268 + n_expected : int
  269 + Expected number of observation in IACT data store
257 270 """
258 271 # Set file name
259 272 obs_index_name = gammalib.GFilename(pathname+'/obs-index.fits[OBS_INDEX]')
... ... @@ -268,8 +281,7 @@ class Test(test):
268 281 fits.close()
269 282  
270 283 # Check for existence of observations
271   - self.test_assert(n_obs == n_expected,
272   - 'Check for '+str(n_expected)+' observations.')
  284 + self.test_value(n_obs, n_expected, 'Check for number of observations')
273 285  
274 286 # Return
275 287 return
276 288 \ No newline at end of file
... ...