Michael Mayer / ctools

Commit 0b84af8566e85b328608bd54c601750678f88a80

Authored by Jürgen Knödlseder
2 parents 0c9ae016 0bd0ca1a

Merge branch '1661-csiactdload' into integration

Inline Side-by-side
Showing 10 changed files with 877 additions and 18 deletions
.gitignore
  View file @0b84af8
... ... @@ -66,6 +66,7 @@ src/ctulimit/ctulimit
66 66 src/cterror/cterror
67 67 test/pfiles/
68 68 test/caldb/
  69 +test/iactdata_test
69 70 test/*.fits
70 71 test/*.dat
71 72 test/*.xml
... ...
ChangeLog
  View file @0b84af8
1   -2016-01-20 Juergen Knoedlseder <jurgen.knodlseder@irap.omp.eu>
  1 +2016-02-11 Juergen Knoedlseder <jurgen.knodlseder@irap.omp.eu>
2 2  
3 3 * Version 1.1.0 released
4 4 ========================
5 5  
  6 + Add csiactcopy script (#1661)
6 7 ctselect allows usage of non-default events and GTI extensions (#1598)
7 8  
8 9  
... ...
... ... @@ -2,7 +2,7 @@ New Features and Important Changes in ctools 1.1.0
2 2  
3 3 Author(s) : Juergen Knoedlseder
4 4  
5   -20 January 2016
  5 +11 February 2016
6 6  
7 7  
8 8 1. Introduction
... ... @@ -127,7 +127,8 @@ None
127 127  
128 128 19. cscripts
129 129 ------------
130   -None
  130 +Added the csiactcopy script that copies IACT data from a remote machine to
  131 +the user's machine (#1661).
131 132  
132 133  
133 134 20. Examples
... ...
cscripts/Makefile.am
  View file @0b84af8
... ... @@ -5,14 +5,14 @@ pythonscripts = __init__.py \
5 5 cspull.py cssens.py cstsdist.py cscaldb.py csresmap.py \
6 6 csiactobs.py csiactdata.py csobsdef.py csspec.py cslightcrv.py csobsinfo.py \
7 7 csinfo.py csmodelinfo.py csmodelmerge.py cstsmapmerge.py \
8   - csobs2caldb.py csfindobs.py \
  8 + csobs2caldb.py csfindobs.py csiactcopy.py \
9 9 obsutils.py
10 10  
11 11 # Scripts for which symbolic links will be created in bindir
12 12 linkscripts = cspull cssens cstsdist cscaldb csresmap \
13 13 csiactobs csiactdata csobsdef csspec cslightcrv csobsinfo \
14 14 csinfo csmodelinfo csmodelmerge cstsmapmerge \
15   - csobs2caldb csfindobs
  15 + csobs2caldb csfindobs csiactcopy
16 16  
17 17 # Distribute scripts
18 18 EXTRA_DIST = $(binscripts) $(pythonscripts)
... ...
cscripts/__init__.py.in
  View file @0b84af8
... ... @@ -39,6 +39,7 @@ __all__ = [
39 39 "csfindobs",
40 40 "csiactobs",
41 41 "csiactdata",
  42 + "csiactcopy",
42 43 "obsutils",
43 44 "test",
44 45 ]
... ... @@ -63,6 +64,7 @@ from .cstsmapmerge import cstsmapmerge
63 64 from .csfindobs import csfindobs
64 65 from .csiactobs import csiactobs
65 66 from .csiactdata import csiactdata
  67 +from .csiactcopy import csiactcopy
66 68 from . import obsutils
67 69  
68 70 # Add test function
... ...
cscripts/csiactcopy.py 0 → 100755
  View file @0b84af8
  1 +#!/usr/bin/env python
  2 +# ==========================================================================
  3 +# Download IACT data from remote machine
  4 +#
  5 +# Copyright (C) 2016 Michael Mayer
  6 +#
  7 +# This program is free software: you can redistribute it and/or modify
  8 +# it under the terms of the GNU General Public License as published by
  9 +# the Free Software Foundation, either version 3 of the License, or
  10 +# (at your option) any later version.
  11 +#
  12 +# This program is distributed in the hope that it will be useful,
  13 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15 +# GNU General Public License for more details.
  16 +#
  17 +# You should have received a copy of the GNU General Public License
  18 +# along with this program. If not, see <http://www.gnu.org/licenses/>.
  19 +#
  20 +# ==========================================================================
  21 +import gammalib
  22 +import ctools
  23 +import sys
  24 +import os
  25 +import json
  26 +import shutil
  27 +
  28 +# ================ #
  29 +# csiactdata class #
  30 +# ================ #
  31 +class csiactcopy(ctools.cscript):
  32 + """
  33 + This script copies IACT data from one location to another.
  34 + It can take a list of observation IDs to allow the download specific
  35 + observations. Index files get merged and updated accordingly.
  36 + """
  37 + def __init__(self, *argv):
  38 + """
  39 + Constructor.
  40 + """
  41 + # Set name
  42 + self.name = "csiactcopy"
  43 + self.version = "1.1.0"
  44 + self.datapath = os.getenv("VHEFITS","")
  45 +
  46 + # Make sure that parfile exists
  47 + file = self.parfile()
  48 +
  49 + # Initialise application
  50 + if len(argv) == 0:
  51 + ctools.cscript.__init__(self, self.name, self.version)
  52 + elif len(argv) == 1:
  53 + ctools.cscript.__init__(self, self.name, self.version, *argv)
  54 + else:
  55 + raise TypeError("Invalid number of arguments given.")
  56 +
  57 + # Set logger properties
  58 + self.log_header()
  59 + self.log.date(True)
  60 +
  61 + # Return
  62 + return
  63 +
  64 + def __del__(self):
  65 + """
  66 + Destructor.
  67 + """
  68 +
  69 + # Return
  70 + return
  71 +
  72 + def parfile(self):
  73 + """
  74 + Check if parfile exists. If parfile does not exist then create a
  75 + default parfile. This kluge avoids shipping the cscript with a
  76 + parfile.
  77 + """
  78 + # Set parfile name
  79 + parfile = self.name+".par"
  80 +
  81 + try:
  82 + pars = gammalib.GApplicationPars(parfile)
  83 + except:
  84 + # Signal if parfile was not found
  85 + sys.stdout.write("Parfile "+parfile+" not found. Create default parfile.\n")
  86 +
  87 + # Create default parfile
  88 + pars = gammalib.GApplicationPars()
  89 + pars.append(gammalib.GApplicationPar("remote_master","f","a","/Volumes/mountpoint/master.json","","","Location of remote master file"))
  90 + pars.append(gammalib.GApplicationPar("prodname","s","a","iact-fits","","","Name of FITS production to download"))
  91 + pars.append(gammalib.GApplicationPar("outpath","s","a","/path/to/fits","","","Destination path of FITS data"))
  92 + pars.append(gammalib.GApplicationPar("runlist","f","h","NONE","","","List of observation IDs"))
  93 + pars.append_standard()
  94 + pars.append(gammalib.GApplicationPar("logfile","f","h","csiactcopy.log","","","Log filename"))
  95 + pars.save(parfile)
  96 +
  97 + # Return
  98 + return
  99 +
  100 + def get_parameters(self):
  101 + """
  102 + Get parameters from parfile and setup the observation.
  103 + """
  104 +
  105 + # Get Parameters
  106 + self.m_remote_master = gammalib.expand_env(self["remote_master"].filename())
  107 + if not os.path.isfile(self.m_remote_master):
  108 + raise RuntimeError("*** ERROR: FITS data not available. No master index file found in \""+self.m_remote_master+"\". Make sure remote file system is properly mounted")
  109 +
  110 + # Get parameters
  111 + self.m_prodname = self["prodname"].string()
  112 + self.m_outpath = gammalib.expand_env(self["outpath"].string())
  113 + self.m_runlist = self["runlist"].filename()
  114 +
  115 + # Return
  116 + return
  117 +
  118 + def execute(self):
  119 + """
  120 + Execute the script.
  121 + """
  122 + # Run the script
  123 + self.run()
  124 +
  125 + # Return
  126 + return
  127 +
  128 + def run(self):
  129 + """
  130 + Run the script.
  131 + """
  132 + # Switch screen logging on
  133 + self.log.cout(True)
  134 +
  135 + # Get parameters
  136 + self.get_parameters()
  137 +
  138 + # Write input parameters into logger
  139 + if self.logTerse():
  140 + self.log_parameters()
  141 + self.log("\n")
  142 +
  143 + # Make destination directory if not available
  144 + if not os.path.isdir(self.m_outpath):
  145 + os.makedirs(self.m_outpath)
  146 +
  147 + # Log output
  148 + if self.logTerse():
  149 + self.log("\n")
  150 + self.log.header2("Runlist information")
  151 +
  152 + # Initialise runlist array
  153 + self.runlist = []
  154 +
  155 + # Check if runlist file is available
  156 + if os.path.isfile(self.m_runlist):
  157 + runfile = open(self.m_runlist)
  158 + for line in runfile.readlines():
  159 + if len(line) == 0:
  160 + continue
  161 + if line[0] == '#':
  162 + continue
  163 + if len(line.split()) > 0:
  164 + self.runlist.append(int(line.split()[0]))
  165 + runfile.close()
  166 +
  167 + # Logging
  168 + if self.logTerse():
  169 + self.log.parformat("Number of observations")
  170 + self.log(str(len(self.runlist)))
  171 + self.log("\n")
  172 +
  173 + # Copy all data if runlist file is "NONE"
  174 + elif self.m_runlist == "NONE":
  175 + if self.logTerse():
  176 + self.log("Copy all available data")
  177 + self.log("\n")
  178 +
  179 + # Raise exception if file not valid
  180 + else:
  181 + raise RuntimeError("*** ERROR: Runlist file \""+self.m_runlist+"\" not available")
  182 +
  183 + # Check for availability of remote master file
  184 + if not os.path.isfile(self.m_remote_master):
  185 + raise RuntimeError("*** ERROR: Remote master file \""+self.m_remote_master+"\" does not exist")
  186 +
  187 + # Retrieve json data from remote master
  188 + json_data = open(self.m_remote_master).read()
  189 + data = json.loads(json_data)
  190 + if not "datasets" in data:
  191 + raise RuntimeError("*** ERROR: Key \"datasets\" not available in remote master index file \""+self.m_remote_master+"\".")
  192 +
  193 + # Get array of configurations
  194 + configs = data["datasets"]
  195 +
  196 + # Get remote paths
  197 + self.remote_base = os.path.dirname(self.m_remote_master)
  198 +
  199 + # Initialise flag if prod has been found
  200 + has_prod = False
  201 +
  202 + # Initialise file names to be copied
  203 + files = set()
  204 +
  205 + # Logging
  206 + if self.logTerse():
  207 + self.log("\n")
  208 + self.log.header2("Loop over remote configs")
  209 +
  210 + # Loop over configs
  211 + for config in configs:
  212 +
  213 + # Logging
  214 + if self.logVerbose():
  215 + self.log.header2(str(config["name"]))
  216 +
  217 + # Check if prodname was found
  218 + if config["name"] == self.m_prodname:
  219 +
  220 + # Inidicate that prodname has been found
  221 + has_prod = True
  222 +
  223 + # Build path of index files
  224 + remote_hdu = os.path.join(self.remote_base, config["hduindx"])
  225 + remote_obs = os.path.join(self.remote_base, config["obsindx"])
  226 +
  227 + # Log information
  228 + if self.logTerse():
  229 + self.log.header3("Remote config \""+self.m_prodname+"\"")
  230 + self.log.parformat("HDU index")
  231 + self.log(str(remote_hdu))
  232 + self.log("\n")
  233 + self.log.parformat("Observation index")
  234 + self.log(str(remote_obs))
  235 + self.log("\n")
  236 +
  237 + # Open remote HDU index file
  238 + fits = gammalib.GFits(str(remote_hdu))
  239 + table = fits["HDU_INDEX"]
  240 +
  241 + # Initialise flag if SIZE column is present
  242 + has_size = table.contains("SIZE")
  243 +
  244 + if has_size:
  245 + # Initialise file size
  246 + cp_size = 0
  247 +
  248 + # Initialise remote observation IDs
  249 + remote_ids = set()
  250 + for row in range(table.nrows()):
  251 + remote_ids.add(table["OBS_ID"][row])
  252 +
  253 + # Log runs that are not available remotely
  254 + for run in self.runlist:
  255 + if not run in remote_ids:
  256 + if self.logNormal():
  257 + self.log("Skip observation "+str(run)+": ID not available remotely")
  258 + self.log("\n")
  259 +
  260 +
  261 + # Loop over remote HDU index file
  262 + for row in range(table.nrows()):
  263 +
  264 + # Get observation ID
  265 + obs_id = table["OBS_ID"][row]
  266 + file_dir = table["FILE_DIR"][row]
  267 + file_name = table["FILE_NAME"][row]
  268 +
  269 + # Skip if filename is empty
  270 + if file_name == "":
  271 + continue
  272 +
  273 + # Check if we need to consider an input runlist
  274 + if len(self.runlist):
  275 +
  276 + # Check if obs id is in runlist
  277 + if obs_id in self.runlist:
  278 +
  279 + # Get filename
  280 + fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name)
  281 +
  282 + # Add file
  283 + oldlen = len(files)
  284 + files.add(fname)
  285 + newlen = len(files)
  286 +
  287 + # Add file size
  288 + if has_size and newlen > oldlen:
  289 + cp_size += table["SIZE"][row]
  290 +
  291 + # Otherwise add every file
  292 + else:
  293 + # Get filename
  294 + fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name)
  295 +
  296 + # Add file
  297 + oldlen = len(files)
  298 + files.add(fname)
  299 + newlen = len(files)
  300 +
  301 + # Add file size
  302 + if has_size and newlen > oldlen:
  303 + cp_size += table["SIZE"][row]
  304 +
  305 + # Log file information
  306 + if self.logTerse():
  307 + self.log("\n")
  308 + self.log.header2("File information")
  309 + self.log.parformat("Number of files")
  310 + self.log(str(len(files)))
  311 + self.log("\n")
  312 + if has_size:
  313 + self.log.parformat("Size")
  314 + self.log("%.2f"%(float(cp_size)*1e-6)+" MB")
  315 + self.log("\n")
  316 + if self.logVerbose():
  317 + self.log("\n")
  318 + self.log.header3("File names")
  319 + for filename in files:
  320 + self.log(str(filename)+"\n")
  321 +
  322 + # Close HDU index file
  323 + fits.close()
  324 +
  325 + # If prodname is not found just log that we skip the config
  326 + else:
  327 + if self.logExplicit():
  328 + self.log("\n")
  329 + self.log.header3("Skipping config \""+str(config["name"])+"\"")
  330 + self.log("\n")
  331 +
  332 + # Raise Exception if prodname was not found
  333 + if not has_prod:
  334 + msg = "*** ERROR: FITS production \""+self.m_prodname+"\" not available. "
  335 + msg += "Available productions are: \n"
  336 + for config in configs:
  337 + msg += " - "+config["name"] + "\n"
  338 + raise RuntimeError(msg)
  339 +
  340 + # Logging
  341 + if self.logNormal():
  342 + self.log("\n")
  343 + self.log.header1("Copying files")
  344 +
  345 + # intialise counter
  346 + k = 0
  347 +
  348 + # initialise values for logging
  349 + last_fraction = 0.0
  350 + fraction_increment = 20.0
  351 + if self.logNormal():
  352 + fraction_increment = 10.0
  353 + if self.logTerse():
  354 + fraction_increment = 5.0
  355 + if self.logExplicit():
  356 + fraction_increment = 2.0
  357 +
  358 + # initialise logging properties
  359 + n_copied = 0
  360 + total_size = 0.0
  361 +
  362 + # Loop over files and copy
  363 + for filename in files:
  364 +
  365 + # Log progress
  366 + fraction = float(k) / float(len(files)) * 100.0
  367 + while fraction > last_fraction:
  368 + if self.logNormal() and not self.logVerbose():
  369 + self.log.parformat("Status")
  370 + self.log(str(int(last_fraction))+"%")
  371 + self.log("\n")
  372 + last_fraction += fraction_increment
  373 +
  374 + # Copy file
  375 + filesize = self.copy(filename, self.clobber())
  376 +
  377 + if filesize > 0.0:
  378 + total_size += filesize
  379 + n_copied += 1
  380 +
  381 + # Increment counter
  382 + k += 1
  383 +
  384 + # Logging
  385 + if self.logNormal() and not self.logVerbose():
  386 + self.log.parformat("Status")
  387 + self.log("Finished")
  388 + self.log("\n")
  389 + if self.logTerse():
  390 + self.log("\n")
  391 + self.log.header1("Updating index files")
  392 +
  393 + # Build local index file names
  394 + local_hdu = os.path.join(self.m_outpath, os.path.relpath(remote_hdu, self.remote_base))
  395 + local_obs = os.path.join(self.m_outpath, os.path.relpath(remote_obs, self.remote_base))
  396 +
  397 + # If we have a runlist then merge index file
  398 + if len(self.runlist):
  399 +
  400 + # Logging
  401 + if self.logTerse():
  402 + self.log("\n")
  403 + self.log.header3("HDU index")
  404 +
  405 + # Merge remote index files with local files
  406 + self.merge(local_hdu, remote_hdu, "HDU_INDEX", self.clobber())
  407 +
  408 + if self.logTerse():
  409 + self.log("\n")
  410 + self.log.header3("OBS index")
  411 +
  412 + # Merge remote index files with local files
  413 + self.merge(local_obs, remote_obs, "OBS_INDEX", self.clobber())
  414 +
  415 + else:
  416 + # If all files were copied, just copy index files too
  417 + self.copy(remote_hdu, self.clobber())
  418 + self.copy(remote_obs, self.clobber())
  419 +
  420 + if self.logTerse():
  421 + self.log("\n")
  422 + self.log.header3("Master index file")
  423 +
  424 + # Adding prodname to local master
  425 + localmaster = os.path.join(self.m_outpath, "master.json")
  426 +
  427 + # If no local master is found, copy master over first
  428 + if not os.path.isfile(localmaster):
  429 + self.copy(self.m_remote_master, self.clobber())
  430 +
  431 + # Load local master
  432 + json_data = open(localmaster).read()
  433 + data = json.loads(json_data)
  434 + configs = data["datasets"]
  435 +
  436 + # Initialise flag indicating if we already have prodname in master file
  437 + has_config = False
  438 +
  439 + # Initialise new configs array
  440 + newconfigs = []
  441 +
  442 + # Loop over configs in master index file
  443 + for config in configs:
  444 +
  445 + # Get hdu and obs index files
  446 + hdu = os.path.join(self.m_outpath, config["hduindx"])
  447 + obs = os.path.join(self.m_outpath, config["obsindx"])
  448 +
  449 + # Check if index files are available
  450 + if not (gammalib.is_fits(str(hdu)) and gammalib.is_fits(str(obs))):
  451 + if self.logTerse():
  452 + self.log("Removing \""+str(config["name"])+"\" (not available)")
  453 + self.log("\n")
  454 + else:
  455 + # Append config if valid
  456 + newconfigs.append(config)
  457 + if self.logTerse():
  458 + self.log("Keeping \""+str(config["name"])+"\"")
  459 + self.log("\n")
  460 +
  461 + # Signals that downloaded config is available
  462 + if config["name"] == self.m_prodname:
  463 + has_config = True
  464 +
  465 + # Create new entry if config was not available
  466 + if not has_config:
  467 + newdict = dict.fromkeys(["name","hduindx","obsindx"])
  468 + newdict["name"] = self.m_prodname
  469 + newdict["hduindx"] = os.path.relpath(local_hdu, self.m_outpath)
  470 + newdict["obsindx"] = os.path.relpath(local_obs, self.m_outpath)
  471 + newconfigs.append(newdict)
  472 + if self.logTerse():
  473 + self.log("Adding \""+str(newdict["name"])+"\"")
  474 + self.log("\n")
  475 +
  476 + # Write new json master file
  477 + f = open(localmaster, "w")
  478 + data["datasets"] = newconfigs
  479 + json.dump(data, f, indent=2)
  480 + f.close()
  481 +
  482 + # Log summary
  483 + if self.logNormal():
  484 + self.log("\n")
  485 + self.log.header1("Summary")
  486 + self.log.parformat("Data files found")
  487 + self.log(str(len(files)))
  488 + self.log("\n")
  489 + self.log.parformat("Data files copied")
  490 + self.log(str(n_copied))
  491 + self.log("\n")
  492 + self.log.parformat("Size")
  493 + self.log("%.2f"%(float(total_size)*1e-6)+" MB")
  494 + self.log("\n")
  495 + self.log.parformat("Local configs")
  496 + self.log(str(len(newconfigs)))
  497 + self.log("\n")
  498 + if self.logTerse():
  499 + self.log("\n")
  500 + self.log.header3("Content of master index")
  501 + for config in newconfigs:
  502 + self.log(str(config["name"]))
  503 + self.log("\n")
  504 +
  505 + # Return
  506 + return
  507 +
  508 + def copy(self, source, clobber):
  509 + """
  510 + copy file to m_outpath
  511 + """
  512 + # Get file destination
  513 + destination = os.path.join(self.m_outpath,os.path.relpath(source, self.remote_base))
  514 +
  515 + # Initialise return value
  516 + filesize = 0.0
  517 +
  518 + # Logging
  519 + if self.logVerbose():
  520 + self.log("\n")
  521 + self.log.header3("Copy file")
  522 + self.log.parformat("Source")
  523 + self.log(str(source))
  524 + self.log("\n")
  525 + self.log.parformat("Destination")
  526 + self.log(str(destination))
  527 + self.log("\n")
  528 + self.log.parformat("Already exists")
  529 + self.log(str(os.path.isfile(destination)))
  530 + self.log("\n")
  531 + self.log.parformat("Overwrite")
  532 + self.log(str(clobber))
  533 + self.log("\n")
  534 + self.log.parformat("Copying")
  535 +
  536 + # Flag if file destination is already available
  537 + is_file = os.path.isfile(destination)
  538 +
  539 + # check if file could be skipped because clobber=no
  540 + if is_file and clobber == False:
  541 + if self.logVerbose():
  542 + self.log("Skip (clobber=no)")
  543 + self.log("\n")
  544 +
  545 + # check if file could be skipped because it is the same file
  546 + elif is_file and os.path.samefile(destination, source):
  547 + if self.logVerbose():
  548 + self.log("Skip (same file)")
  549 + self.log("\n")
  550 +
  551 + else:
  552 + # Create directory if not existent
  553 + dest_dir = os.path.dirname(destination)
  554 + if not os.path.isdir(dest_dir):
  555 + os.makedirs(dest_dir)
  556 +
  557 + # Copy file (if source not identical to destination)
  558 + shutil.copy2(source, destination)
  559 +
  560 + # Get filesize
  561 + filesize = os.stat(destination).st_size
  562 +
  563 + # Logging
  564 + if self.logVerbose():
  565 + self.log("Done!")
  566 + self.log("\n")
  567 +
  568 + # Return
  569 + return filesize
  570 +
  571 +
  572 + def merge(self, localfits, remotefits, hduname, clobber):
  573 + """
  574 + merge remote and local fits file
  575 + If local fits file not present, a new one is created
  576 + """
  577 +
  578 + if self.logTerse():
  579 + self.log.parformat("Local file")
  580 + self.log(str(localfits))
  581 + self.log("\n")
  582 + self.log.parformat("Remote file")
  583 + self.log(str(remotefits))
  584 + self.log("\n")
  585 + self.log.parformat("HDU name")
  586 + self.log(str(hduname))
  587 + self.log("\n")
  588 + self.log.parformat("Conflict preference")
  589 + if clobber:
  590 + self.log("Remote")
  591 + else:
  592 + self.log("Local")
  593 + self.log("\n")
  594 +
  595 + # Check if local fits file is available
  596 + if os.path.isfile(localfits):
  597 + local = gammalib.GFits(str(localfits))
  598 + local_hdu = local[hduname]
  599 + else:
  600 + # Otherwise load remote fits file and delete rows
  601 + local = gammalib.GFits(str(remotefits))
  602 + local_hdu = local[hduname]
  603 + local_hdu.remove_rows(0,local_hdu.nrows())
  604 +
  605 + # find local obs_id
  606 + local_obs = []
  607 + lobs_col = local_hdu["OBS_ID"]
  608 + for obs_id in lobs_col:
  609 + local_obs.append(obs_id)
  610 +
  611 + # Create selection expression for opening remote fits file
  612 + selection = ""
  613 + if len(self.runlist):
  614 + selection += "["+hduname+"]["
  615 + for run in self.runlist:
  616 + selection += "OBS_ID=="+str(run)
  617 + selection += "||"
  618 + selection = selection[:-2]
  619 + selection += "]"
  620 +
  621 + # Create file name
  622 + remotefile = remotefits + selection
  623 +
  624 + # open remote fits file
  625 + remote = gammalib.GFits(str(remotefile))
  626 + remote_hdu = remote[hduname]
  627 +
  628 + # Get remote obs_id
  629 + remote_obs = []
  630 + robs_col = remote_hdu["OBS_ID"]
  631 + for obs_id in robs_col:
  632 + remote_obs.append(obs_id)
  633 +
  634 + if self.logTerse():
  635 + self.log.parformat("Remote entries")
  636 + self.log(str(len(remote_obs)))
  637 + self.log("\n")
  638 + self.log.parformat("Local entries")
  639 + self.log(str(len(local_obs)))
  640 + self.log("\n")
  641 +
  642 + # initialise counter for logging
  643 + removed_rows = 0
  644 +
  645 + # If clobber=yes, we use the remote content to overwrite corresponding local content
  646 + if clobber:
  647 +
  648 + # Loop over remote obs_ids
  649 + for remote_obs_id in remote_obs:
  650 +
  651 + # Check if remote obs_id is present locally
  652 + if remote_obs_id in local_obs:
  653 +
  654 + # Remove local obs_id entries
  655 + table_has_obsid = True
  656 + while table_has_obsid:
  657 + for i in range(local_hdu.nrows()):
  658 + if remote_obs_id == local_hdu["OBS_ID"][i]:
  659 + local_hdu.remove_rows(i,1)
  660 + removed_rows += 1
  661 + break
  662 + table_has_obsid = False
  663 + for i in range(local_hdu.nrows()):
  664 + if remote_obs_id == local_hdu["OBS_ID"][i]:
  665 + table_has_obsid = True
  666 + break
  667 +
  668 + # If clobber=false, we keep the local content unchanged and just expand by remote content
  669 + else:
  670 + # Loop over remote obs_ids
  671 + for local_obs_id in local_obs:
  672 +
  673 + # Check if local obs_id is present remotely
  674 + if local_obs_id in remote_obs:
  675 +
  676 + # Remove remote obs_id entries
  677 + table_has_obsid = True
  678 + while table_has_obsid:
  679 + for i in range(remote_hdu.nrows()):
  680 + if local_obs_id == remote_hdu["OBS_ID"][i]:
  681 + remote_hdu.remove_rows(i,1)
  682 + removed_rows += 1
  683 + break
  684 + table_has_obsid = False
  685 + for i in range(remote_hdu.nrows()):
  686 + if local_obs_id == remote_hdu["OBS_ID"][i]:
  687 + table_has_obsid = True
  688 + break
  689 +
  690 + # Store number of local rows
  691 + old_local_rows = local_hdu.nrows()
  692 +
  693 + # Add rmeote HDUs
  694 + local_hdu.insert_rows(old_local_rows, remote_hdu.nrows())
  695 +
  696 + if self.logTerse():
  697 + if clobber:
  698 + self.log.parformat("Removed local rows")
  699 + else:
  700 + self.log.parformat("Skipped remote rows")
  701 + self.log(str(removed_rows))
  702 + self.log("\n")
  703 +
  704 + # Loop over columns
  705 + for i in range(local_hdu.ncols()):
  706 +
  707 + # Get local and remote columns
  708 + local_col = local_hdu[i]
  709 + remote_col = remote_hdu[i]
  710 +
  711 + # Loop over entries and merge
  712 + for j in range(remote_col.length()):
  713 + local_col[j+old_local_rows] = remote_col[j]
  714 +
  715 + # Save local fits file
  716 + local.saveto(str(localfits), True)
  717 +
  718 + # Return
  719 + return
  720 +
  721 +# ======================== #
  722 +# Main routine entry point #
  723 +# ======================== #
  724 +if __name__ == '__main__':
  725 + """
  726 + Copy IACT data
  727 + """
  728 + # Create instance of application
  729 + app = csiactcopy(sys.argv)
  730 +
  731 + # Open logfile
  732 + app.logFileOpen()
  733 +
  734 + # Execute application
  735 + app.execute()
... ...
cscripts/csiactobs.py
  View file @0b84af8
... ... @@ -47,9 +47,10 @@ class csiactobs(ctools.cscript):
47 47  
48 48 # Initialise some members
49 49 self.m_ebounds = gammalib.GEbounds()
50   - self.datapath = os.getenv("VHEFITS","")
51   - self.inmodels = None
52   - self.outobs = "obs.xml"
  50 + self.datapath = os.getenv("VHEFITS","")
  51 + self.inmodels = None
  52 + self.xml = gammalib.GXml()
  53 + self.models = gammalib.GModels()
53 54  
54 55 # Make sure that parfile exists
55 56 file = self.parfile()
... ... @@ -141,10 +142,10 @@ class csiactobs(ctools.cscript):
141 142 self.m_bkgpars = self["bkgpars"].integer()
142 143  
143 144 # Output model file
144   - self.outmodel = self["outmodel"].filename()
  145 + self.m_outmodel = self["outmodel"].filename()
145 146  
146 147 # Observation outfile
147   - self.outobs = self["outobs"].filename()
  148 + self.m_outobs = self["outobs"].filename()
148 149  
149 150 # Master index file name
150 151 self.m_master_indx = self["master_indx"].string()
... ... @@ -332,6 +333,24 @@ class csiactobs(ctools.cscript):
332 333 Returns runlist energy range
333 334 """
334 335 return self.m_ebounds
  336 +
  337 + def obs(self):
  338 + """
  339 + Returns GObservations object
  340 + """
  341 + # Initialise observations
  342 + obs = gammalib.GObservations()
  343 +
  344 + # Read observations if XML is filled
  345 + if self.xml.size():
  346 + obs.read(self.xml)
  347 +
  348 + # Assign models
  349 + obs.models(self.models)
  350 +
  351 + # Return observations
  352 + return obs
  353 +
335 354  
336 355 def execute(self):
337 356 """
... ... @@ -341,7 +360,7 @@ class csiactobs(ctools.cscript):
341 360 self.run()
342 361  
343 362 # Save residual map
344   - self.save(self.outobs, self.m_clobber)
  363 + self.save(self.m_outobs, self.m_clobber)
345 364  
346 365 # Return
347 366 return
... ... @@ -362,11 +381,11 @@ class csiactobs(ctools.cscript):
362 381 self.log_parameters()
363 382 self.log("\n")
364 383  
365   - # Initialise empty models
366   - self.models = gammalib.GModels()
  384 + # Clear models
  385 + self.models.clear()
367 386  
368   - # Initialise empty xml file and append an observation list
369   - self.xml = gammalib.GXml()
  387 + # Clear xml file and append an observation list
  388 + self.xml.clear()
370 389 self.xml.append(gammalib.GXmlElement("observation_list title=\"observation list\""))
371 390 lib = self.xml.element("observation_list", 0)
372 391  
... ... @@ -476,7 +495,7 @@ class csiactobs(ctools.cscript):
476 495  
477 496 # Handle background
478 497 index = -1
479   - bkg_mod_hierarchy = self.m_bkg_mod_hiera
  498 + bkg_mod_hierarchy = list(self.m_bkg_mod_hiera)
480 499 for version in self.m_bkg_hiera:
481 500 n = formats.count(version)
482 501 if n > 0:
... ... @@ -489,7 +508,8 @@ class csiactobs(ctools.cscript):
489 508 if self.logTerse():
490 509 self.log("Warning: observation "+str(obs_id)+" has no background information (file=\""+bkgfile+"\"). IRF background cannot be used\n")
491 510 bkgfile = ""
492   - bkg_mod_hierarchy.remove("irf")
  511 + if "irf" in bkg_mod_hierarchy:
  512 + bkg_mod_hierarchy.remove("irf")
493 513  
494 514 # Close hdu index file
495 515 hduindx.close()
... ... @@ -627,7 +647,7 @@ class csiactobs(ctools.cscript):
627 647 self.xml.save(outfile)
628 648  
629 649 # Save model XML file
630   - self.models.save(self.outmodel)
  650 + self.models.save(self.m_outmodel)
631 651  
632 652 # Return
633 653 return
... ...
doc/source/reference_manual/csiactcopy.rst 0 → 100644
  View file @0b84af8
  1 +.. _csiactcopy:
  2 +
  3 +csiactcopy
  4 +===========
  5 +
  6 +Copies IACT data from one location to another
  7 +
  8 +
  9 +Synopsis
  10 +--------
  11 +
  12 +This script allows to download IACT data from a remote machine. The remote file system
  13 +has to be mounted before running the script. For mounting, e.g. sshfs can be used:
  14 +
  15 +.. code-block:: bash
  16 +
  17 + $ sshfs user@remote.server.com:/path/to/remote/fits /path/to/mountpoint/
  18 +
  19 +The script can optionally take an ASCII file with a list of observation IDs as input.
  20 +In this way the user can copy only a specific subset of the data. Index files and data
  21 +structure is updated accordingly by the script.
  22 +
  23 +In case the copying procedure failed due to e.g. a broken connection, the script can be executed
  24 +again using the parameter ``clobber=no``. Thus files that were already copied don't get
  25 +recopied and overwritten which saves time.
  26 +
  27 +In order to monitor the progress on the screen, the script can be executed with the
  28 +hidden parameter ``debug=yes``
  29 +
  30 +
  31 +General parameters
  32 +------------------
  33 +
  34 +``remote_master [file]``
  35 + Location of remote master file
  36 +
  37 +``prodname [string]``
  38 + Name of FITS production to download
  39 +
  40 +``outpath [file]``
  41 + Destination path of FITS data
  42 +
  43 +``(runlist = NONE) [file]``
  44 + List of observation IDs
  45 +
  46 +
  47 +Standard parameters
  48 +-------------------
  49 +
  50 +``(chatter = 2) [integer]``
  51 + Verbosity of the executable:
  52 + chatter = 0: no information will be logged
  53 +
  54 + chatter = 1: only errors will be logged
  55 +
  56 + chatter = 2: errors and actions will be logged
  57 +
  58 + chatter = 3: report about the task execution
  59 +
  60 + chatter = 4: detailed report about the task execution
  61 +
  62 +``(clobber = yes) [boolean]``
  63 + Specifies whether an existing output counts cube should be overwritten.
  64 +
  65 +``(debug = no) [boolean]``
  66 + Enables debug mode. In debug mode the executable will dump any log file output to the console.
  67 +
  68 +``(mode = ql) [string]``
  69 + Mode of automatic parameters (default is "ql", i.e. "query and learn").
  70 +
  71 +``(logfile = csiactdload.log) [filename]``
  72 + Log filename.
  73 +
  74 +
  75 +Related tools or scripts
  76 +------------------------
  77 +
  78 +:doc:`csiactdata`
0 79 \ No newline at end of file
... ...
doc/source/reference_manual/index.rst
  View file @0b84af8
... ... @@ -61,3 +61,4 @@ IACT cscripts
61 61 csiactdata --- Shows information about IACT data available on the user machine <csiactdata>
62 62 csiactobs --- Generates observation definition file for IACT data from observation IDs <csiactobs>
63 63 csfindobs --- Generates a list of IACT observation IDs <csfindobs>
  64 + csiactcopy --- Copies IACT data from one location to another <csiactcopy>
... ...
test/test_cscripts.sh.in
  View file @0b84af8
... ... @@ -483,6 +483,26 @@ $ECHO &quot; ok&quot;
483 483  
484 484  
485 485 #
  486 +# Test csiactcopy
  487 +# ===============
  488 +$ECHO -n "Test csiactcopy: "
  489 +chmod +x @abs_top_srcdir@/cscripts/csiactcopy.py
  490 +@abs_top_srcdir@/cscripts/csiactcopy.py \
  491 + remote_master="iactdata/master.json" \
  492 + outpath="iactdata_test" \
  493 + prodname="unit-test" > csiactcopy.log
  494 +$ECHO -n "."
  495 +if [ -s "iactdata_test/master.json" ]
  496 +then
  497 + $ECHO -n "."
  498 +else
  499 + $ECHO " iactdata_test/master.json file is not valid"
  500 + exit 1
  501 +fi
  502 +$ECHO " ok"
  503 +
  504 +
  505 +#
486 506 # Test csfindobs
487 507 # ==============
488 508 $ECHO -n "Test csfindobs: "
... ...