Mathieu Servillat / ctools

Commit 0c61eac86780587876341a82f6411689214a63c8

Authored by Michael Mayer
1 parent cfa7a20b

Added csiactdload (#1661) including documentation and unit test, added a minor addition to csiactobs

Inline Side-by-side
Showing 7 changed files with 860 additions and 11 deletions
cscripts/Makefile.am
  View file @0c61eac
... ... @@ -5,14 +5,14 @@ pythonscripts = __init__.py \
5 5 cspull.py cssens.py cstsdist.py cscaldb.py csresmap.py \
6 6 csiactobs.py csiactdata.py csobsdef.py csspec.py cslightcrv.py csobsinfo.py \
7 7 csinfo.py csmodelinfo.py csmodelmerge.py cstsmapmerge.py \
8   - csobs2caldb.py csfindobs.py \
  8 + csobs2caldb.py csfindobs.py csiactdload.py \
9 9 obsutils.py
10 10  
11 11 # Scripts for which symbolic links will be created in bindir
12 12 linkscripts = cspull cssens cstsdist cscaldb csresmap \
13 13 csiactobs csiactdata csobsdef csspec cslightcrv csobsinfo \
14 14 csinfo csmodelinfo csmodelmerge cstsmapmerge \
15   - csobs2caldb csfindobs
  15 + csobs2caldb csfindobs csiactdload
16 16  
17 17 # Distribute scripts
18 18 EXTRA_DIST = $(binscripts) $(pythonscripts)
... ...
cscripts/__init__.py.in
  View file @0c61eac
... ... @@ -39,6 +39,7 @@ __all__ = [
39 39 "csfindobs",
40 40 "csiactobs",
41 41 "csiactdata",
  42 + "csiactdload",
42 43 "obsutils",
43 44 "test",
44 45 ]
... ... @@ -63,6 +64,7 @@ from .cstsmapmerge import cstsmapmerge
63 64 from .csfindobs import csfindobs
64 65 from .csiactobs import csiactobs
65 66 from .csiactdata import csiactdata
  67 +from .csiactdload import csiactdload
66 68 from . import obsutils
67 69  
68 70 # Add test function
... ...
cscripts/csiactdload.py 0 → 100755
  View file @0c61eac
  1 +#!/usr/bin/env python
  2 +# ==========================================================================
  3 +# Download IACT data from remote machine
  4 +#
  5 +# Copyright (C) 2015 Michael Mayer
  6 +#
  7 +# This program is free software: you can redistribute it and/or modify
  8 +# it under the terms of the GNU General Public License as published by
  9 +# the Free Software Foundation, either version 3 of the License, or
  10 +# (at your option) any later version.
  11 +#
  12 +# This program is distributed in the hope that it will be useful,
  13 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15 +# GNU General Public License for more details.
  16 +#
  17 +# You should have received a copy of the GNU General Public License
  18 +# along with this program. If not, see <http://www.gnu.org/licenses/>.
  19 +#
  20 +# ==========================================================================
  21 +import gammalib
  22 +import ctools
  23 +import sys
  24 +import os
  25 +import json
  26 +import shutil
  27 +
  28 +# ================ #
  29 +# csiactdata class #
  30 +# ================ #
  31 +class csiactdload(ctools.cscript):
  32 + """
  33 + This script copies IACT data from one location to another.
  34 + It can take a list of observation IDs to allow the download specific
  35 + observations. Index files get merged and updated accordingly.
  36 + """
  37 + def __init__(self, *argv):
  38 + """
  39 + Constructor.
  40 + """
  41 + # Set name
  42 + self.name = "csiactdload"
  43 + self.version = "1.1.0"
  44 + self.datapath = os.getenv("VHEFITS","")
  45 +
  46 + # Make sure that parfile exists
  47 + file = self.parfile()
  48 +
  49 + # Initialise application
  50 + if len(argv) == 0:
  51 + ctools.cscript.__init__(self, self.name, self.version)
  52 + elif len(argv) == 1:
  53 + ctools.cscript.__init__(self, self.name, self.version, *argv)
  54 + else:
  55 + raise TypeError("Invalid number of arguments given.")
  56 +
  57 + # Set logger properties
  58 + self.log_header()
  59 + self.log.date(True)
  60 +
  61 + # Return
  62 + return
  63 +
  64 + def __del__(self):
  65 + """
  66 + Destructor.
  67 + """
  68 +
  69 + # Return
  70 + return
  71 +
  72 + def parfile(self):
  73 + """
  74 + Check if parfile exists. If parfile does not exist then create a
  75 + default parfile. This kluge avoids shipping the cscript with a
  76 + parfile.
  77 + """
  78 + # Set parfile name
  79 + parfile = self.name+".par"
  80 +
  81 + try:
  82 + pars = gammalib.GApplicationPars(parfile)
  83 + except:
  84 + # Signal if parfile was not found
  85 + sys.stdout.write("Parfile "+parfile+" not found. Create default parfile.\n")
  86 +
  87 + # Create default parfile
  88 + pars = gammalib.GApplicationPars()
  89 + pars.append(gammalib.GApplicationPar("remote_master","f","a","/Volumes/mountpoint/master.json","","","Location of remote master file"))
  90 + pars.append(gammalib.GApplicationPar("prodname","s","a","iact-fits","","","Name of FITS production to download"))
  91 + pars.append(gammalib.GApplicationPar("outpath","s","a","/path/to/fits","","","Destination path of FITS data"))
  92 + pars.append(gammalib.GApplicationPar("runlist","f","h","NONE","","","List of observation IDs"))
  93 + pars.append_standard()
  94 + pars.append(gammalib.GApplicationPar("logfile","f","h","csiactdload.log","","","Log filename"))
  95 + pars.save(parfile)
  96 +
  97 + # Return
  98 + return
  99 +
  100 + def get_parameters(self):
  101 + """
  102 + Get parameters from parfile and setup the observation.
  103 + """
  104 +
  105 + # Get Parameters
  106 + self.m_remote_master = gammalib.expand_env(self["remote_master"].filename())
  107 + if not os.path.isfile(self.m_remote_master):
  108 + raise RuntimeError("*** ERROR: FITS data not available. No master index file found in \""+self.m_remote_master+"\". Make sure remote file system is properly mounted")
  109 +
  110 + # Get parameters
  111 + self.m_prodname = self["prodname"].string()
  112 + self.m_outpath = gammalib.expand_env(self["outpath"].string())
  113 + self.m_runlist = self["runlist"].filename()
  114 +
  115 + # Return
  116 + return
  117 +
  118 + def execute(self):
  119 + """
  120 + Execute the script.
  121 + """
  122 + # Run the script
  123 + self.run()
  124 +
  125 + # Return
  126 + return
  127 +
  128 + def run(self):
  129 + """
  130 + Run the script.
  131 + """
  132 + # Switch screen logging on
  133 + self.log.cout(True)
  134 +
  135 + # Get parameters
  136 + self.get_parameters()
  137 +
  138 + # Write input parameters into logger
  139 + if self.logTerse():
  140 + self.log_parameters()
  141 + self.log("\n")
  142 +
  143 + # Make destination directory if not available
  144 + if not os.path.isdir(self.m_outpath):
  145 + os.makedirs(self.m_outpath)
  146 +
  147 + # Log output
  148 + if self.logTerse():
  149 + self.log("\n")
  150 + self.log.header2("Runlist information")
  151 +
  152 + # Initialise runlist array
  153 + self.runlist = []
  154 +
  155 + # Check if runlist file is available
  156 + if os.path.isfile(self.m_runlist):
  157 + runfile = open(self.m_runlist)
  158 + for line in runfile.readlines():
  159 + if len(line) == 0:
  160 + continue
  161 + if line[0] == '#':
  162 + continue
  163 + if len(line.split()) > 0:
  164 + self.runlist.append(int(line.split()[0]))
  165 + runfile.close()
  166 +
  167 + # Logging
  168 + if self.logTerse():
  169 + self.log.parformat("Number of observations")
  170 + self.log(str(len(self.runlist)))
  171 + self.log("\n")
  172 + # Copy all data if runlist file is "NONE"
  173 + elif self.m_runlist == "NONE":
  174 + if self.logTerse():
  175 + self.log("Copy all available data")
  176 + self.log("\n")
  177 + # Raise exception if file not valid
  178 + else:
  179 + raise RuntimeError("*** ERROR: Runlist file \""+self.m_runlist+"\" not available")
  180 +
  181 + # Check for availability of remote master file
  182 + if not os.path.isfile(self.m_remote_master):
  183 + raise RuntimeError("*** ERROR: Remote master file \""+self.m_remote_master+"\" does not exist")
  184 +
  185 + # Retrieve json data from remote master
  186 + json_data = open(self.m_remote_master).read()
  187 + data = json.loads(json_data)
  188 + if not "datasets" in data:
  189 + raise RuntimeError("*** ERROR: Key \"datasets\" not available in remote master index file \""+self.m_remote_master+"\".")
  190 +
  191 + # Get array of configurations
  192 + configs = data["datasets"]
  193 +
  194 + # Get remote paths
  195 + self.remote_base = os.path.dirname(self.m_remote_master)
  196 +
  197 + # Initialise flag if prod has been found
  198 + has_prod = False
  199 +
  200 + # Initialise file names to be copied
  201 + files = set()
  202 +
  203 + # Logging
  204 + if self.logTerse():
  205 + self.log("\n")
  206 + self.log.header2("Loop over remote configs")
  207 +
  208 + # Loop over configs
  209 + for config in configs:
  210 +
  211 + # Logging
  212 + if self.logVerbose():
  213 + self.log.header2(str(config["name"]))
  214 +
  215 + # Check if prodname was found
  216 + if config["name"] == self.m_prodname:
  217 +
  218 + # Inidicate that prodname has been found
  219 + has_prod = True
  220 +
  221 + # Build path of index files
  222 + remote_hdu = os.path.join(self.remote_base, config["hduindx"])
  223 + remote_obs = os.path.join(self.remote_base, config["obsindx"])
  224 +
  225 + # Log information
  226 + if self.logTerse():
  227 + self.log.header3("Remote config \""+self.m_prodname+"\"")
  228 + self.log.parformat("HDU index")
  229 + self.log(str(remote_hdu))
  230 + self.log("\n")
  231 + self.log.parformat("Observation index")
  232 + self.log(str(remote_obs))
  233 + self.log("\n")
  234 +
  235 + # Open remote HDU index file
  236 + fits = gammalib.GFits(str(remote_hdu))
  237 + table = fits["HDU_INDEX"]
  238 +
  239 + # Initialise flag if SIZE column is present
  240 + has_size = table.contains("SIZE")
  241 +
  242 + if has_size:
  243 + # Initialise file size
  244 + cp_size = 0
  245 +
  246 + # Initialise remote observation IDs
  247 + remote_ids = set()
  248 + for row in range(table.nrows()):
  249 + remote_ids.add(table["OBS_ID"][row])
  250 +
  251 + # Log runs that are not available remotely
  252 + for run in self.runlist:
  253 + if not run in remote_ids:
  254 + if self.logNormal():
  255 + self.log("Skip observation "+str(run)+": ID not available remotely")
  256 + self.log("\n")
  257 +
  258 +
  259 + # Loop over remote HDU index file
  260 + for row in range(table.nrows()):
  261 +
  262 + # Get observation ID
  263 + obs_id = table["OBS_ID"][row]
  264 + file_dir = table["FILE_DIR"][row]
  265 + file_name = table["FILE_NAME"][row]
  266 +
  267 + # Skip if filename is empty
  268 + if file_name == "":
  269 + continue
  270 +
  271 + # Check if we need to consider an input runlist
  272 + if len(self.runlist):
  273 +
  274 + # Check if obs id is in runlist
  275 + if obs_id in self.runlist:
  276 +
  277 + # Get filename
  278 + fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name)
  279 +
  280 + # Add file
  281 + oldlen = len(files)
  282 + files.add(fname)
  283 + newlen = len(files)
  284 +
  285 + # Add file size
  286 + if has_size and newlen > oldlen:
  287 + cp_size += table["SIZE"][row]
  288 +
  289 + # Otherwise add every file
  290 + else:
  291 + # Get filename
  292 + fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name)
  293 +
  294 + # Add file
  295 + oldlen = len(files)
  296 + files.add(fname)
  297 + newlen = len(files)
  298 +
  299 + # Add file size
  300 + if has_size and newlen > oldlen:
  301 + cp_size += table["SIZE"][row]
  302 +
  303 + # Log file information
  304 + if self.logTerse():
  305 + self.log("\n")
  306 + self.log.header2("File information")
  307 + self.log.parformat("Number of files")
  308 + self.log(str(len(files)))
  309 + self.log("\n")
  310 + if has_size:
  311 + self.log.parformat("Size")
  312 + self.log("%.2f"%(float(cp_size)*1e-6)+" MB")
  313 + self.log("\n")
  314 + if self.logVerbose():
  315 + self.log("\n")
  316 + self.log.header3("File names")
  317 + for filename in files:
  318 + self.log(str(filename)+"\n")
  319 +
  320 + # Close HDU index file
  321 + fits.close()
  322 +
  323 + # If prodname is not found just log that we skip the config
  324 + else:
  325 + if self.logExplicit():
  326 + self.log("\n")
  327 + self.log.header3("Skipping config \""+str(config["name"])+"\"")
  328 + self.log("\n")
  329 +
  330 + # Raise Exception if prodname was not found
  331 + if not has_prod:
  332 + msg = "*** ERROR: FITS production \""+self.m_prodname+"\" not available. "
  333 + msg += "Available productions are: \n"
  334 + for config in configs:
  335 + msg += " - "+config["name"] + "\n"
  336 + raise RuntimeError(msg)
  337 +
  338 + # Logging
  339 + if self.logNormal():
  340 + self.log("\n")
  341 + self.log.header1("Copying files")
  342 +
  343 + # intialise counter
  344 + k = 0
  345 +
  346 + # initialise values for logging
  347 + last_fraction = 0.0
  348 + fraction_increment = 20.0
  349 + if self.logNormal():
  350 + fraction_increment = 10.0
  351 + if self.logTerse():
  352 + fraction_increment = 5.0
  353 + if self.logExplicit():
  354 + fraction_increment = 2.0
  355 +
  356 + # initialise logging properties
  357 + n_copied = 0
  358 + total_size = 0.0
  359 +
  360 + # Loop over files and copy
  361 + for filename in files:
  362 +
  363 + # Log progress
  364 + fraction = float(k) / float(len(files)) * 100.0
  365 + if fraction > last_fraction:
  366 + if self.logNormal() and not self.logVerbose():
  367 + self.log.parformat("Status")
  368 + self.log(str(int(last_fraction))+"%")
  369 + self.log("\n")
  370 + last_fraction += fraction_increment
  371 +
  372 + # Copy file
  373 + filesize = self.copy(filename, self.clobber())
  374 +
  375 + if filesize > 0.0:
  376 + total_size += filesize
  377 + n_copied += 1
  378 +
  379 + # Increment counter
  380 + k += 1
  381 +
  382 + # Logging
  383 + if self.logNormal() and not self.logVerbose():
  384 + self.log.parformat("Status")
  385 + self.log("Finished")
  386 + self.log("\n")
  387 + if self.logTerse():
  388 + self.log("\n")
  389 + self.log.header1("Updating index files")
  390 +
  391 + # If we have a runlist then merge index file
  392 + if len(self.runlist):
  393 +
  394 + # Check if there are local index files
  395 + local_hdu = os.path.join(self.m_outpath,os.path.relpath(remote_hdu, self.remote_base))
  396 + local_obs = os.path.join(self.m_outpath,os.path.relpath(remote_obs, self.remote_base))
  397 +
  398 + if self.logTerse():
  399 + self.log("\n")
  400 + self.log.header3("HDU index")
  401 +
  402 + # Merge remote index files with local files
  403 + self.merge(local_hdu, remote_hdu, "HDU_INDEX", self.clobber())
  404 +
  405 + if self.logTerse():
  406 + self.log("\n")
  407 + self.log.header3("OBS index")
  408 +
  409 + # Merge remote index files with local files
  410 + self.merge(local_obs, remote_obs, "OBS_INDEX", self.clobber())
  411 +
  412 + else:
  413 + # If all files were copied, just copy index files too
  414 + self.copy(remote_hdu, self.clobber())
  415 + self.copy(remote_obs, self.clobber())
  416 +
  417 + if self.logTerse():
  418 + self.log("\n")
  419 + self.log.header3("Master index file")
  420 +
  421 + # Adding prodname to local master
  422 + localmaster = os.path.join(self.m_outpath, "master.json")
  423 +
  424 + # If no local master is found, copy master over first
  425 + if not os.path.isfile(localmaster):
  426 + self.copy(self.m_remote_master, self.clobber())
  427 +
  428 + # Load local master
  429 + json_data = open(localmaster).read()
  430 + data = json.loads(json_data)
  431 + configs = data["datasets"]
  432 +
  433 + # Initialise flag indicating if we already have prodname in master file
  434 + has_config = False
  435 +
  436 + # Initialise new configs array
  437 + newconfigs = []
  438 +
  439 + # Loop over configs in master index file
  440 + for config in configs:
  441 +
  442 + # Get hdu and obs index files
  443 + hdu = os.path.join(self.m_outpath, config["hduindx"])
  444 + obs = os.path.join(self.m_outpath, config["obsindx"])
  445 +
  446 + # Check if index files are available
  447 + if not (gammalib.is_fits(str(hdu)) and gammalib.is_fits(str(obs))):
  448 + if self.logTerse():
  449 + self.log("Removing \""+str(config["name"])+"\" (not available)")
  450 + self.log("\n")
  451 + else:
  452 + # Append config if valid
  453 + newconfigs.append(config)
  454 + if self.logTerse():
  455 + self.log("Keeping \""+str(config["name"])+"\"")
  456 + self.log("\n")
  457 +
  458 + # Signals that downloaded config is available
  459 + if config["name"] == self.m_prodname:
  460 + has_config = True
  461 +
  462 + # Create new entry if config was not available
  463 + if not has_config:
  464 + newdict = dict.fromkeys(["name","hduindx","obsindx"])
  465 + newdict["name"] = self.m_prodname
  466 + newdict["hduindx"] = os.path.relpath(local_hdu, self.m_outpath)
  467 + newdict["obsindx"] = os.path.relpath(local_obs, self.m_outpath)
  468 + newconfigs.append(newdict)
  469 +
  470 + # Write new json master file
  471 + f = open(localmaster, "w")
  472 + data["datasets"] = newconfigs
  473 + json.dump(data, f, indent=2)
  474 + f.close()
  475 +
  476 + # Log summary
  477 + if self.logNormal():
  478 + self.log("\n")
  479 + self.log.header1("Summary")
  480 + self.log.parformat("Data files found")
  481 + self.log(str(len(files)))
  482 + self.log("\n")
  483 + self.log.parformat("Data files copied")
  484 + self.log(str(n_copied))
  485 + self.log("\n")
  486 + self.log.parformat("Size")
  487 + self.log("%.2f"%(float(total_size)*1e-6)+" MB")
  488 + self.log("\n")
  489 + self.log.parformat("Local configs")
  490 + self.log(str(len(newconfigs)))
  491 + self.log("\n")
  492 + if self.logTerse():
  493 + self.log("\n")
  494 + self.log.header3("Content of master index")
  495 + for config in newconfigs:
  496 + self.log(str(config["name"]))
  497 + self.log("\n")
  498 +
  499 + # Return
  500 + return
  501 +
  502 + def copy(self, source, clobber):
  503 + """
  504 + copy file to m_outpath
  505 + """
  506 + # Get file destination
  507 + destination = os.path.join(self.m_outpath,os.path.relpath(source, self.remote_base))
  508 +
  509 +
  510 + # Initialise return value
  511 + filesize = 0.0
  512 +
  513 + # Logging
  514 + if self.logVerbose():
  515 + self.log("\n")
  516 + self.log.header3("Copy file")
  517 + self.log.parformat("Source")
  518 + self.log(str(source))
  519 + self.log("\n")
  520 + self.log.parformat("Destination")
  521 + self.log(str(destination))
  522 + self.log("\n")
  523 + self.log.parformat("Already exists")
  524 + self.log(str(os.path.isfile(destination)))
  525 + self.log("\n")
  526 + self.log.parformat("Overwrite")
  527 + self.log(str(clobber))
  528 + self.log("\n")
  529 + self.log.parformat("Copying")
  530 +
  531 + # Flag if file destination is already available
  532 + is_file = os.path.isfile(destination)
  533 +
  534 + # check if file could be skipped because clobber=no
  535 + if is_file and clobber == False:
  536 + if self.logVerbose():
  537 + self.log("Skip (clobber=no)")
  538 + self.log("\n")
  539 +
  540 + # check if file could be skipped because it is the same file
  541 + elif is_file and os.path.samefile(destination, source):
  542 + if self.logVerbose():
  543 + self.log("Skip (same file)")
  544 + self.log("\n")
  545 +
  546 + else:
  547 + # Create directory if not existent
  548 + dest_dir = os.path.dirname(destination)
  549 + if not os.path.isdir(dest_dir):
  550 + os.makedirs(dest_dir)
  551 +
  552 + # Copy file (if source not identical to destination)
  553 + shutil.copy2(source, destination)
  554 +
  555 + # Get filesize
  556 + filesize = os.stat(destination).st_size
  557 +
  558 + # Logging
  559 + if self.logVerbose():
  560 + self.log("Done!")
  561 + self.log("\n")
  562 +
  563 + # Return
  564 + return filesize
  565 +
  566 +
  567 + def merge(self, localfits, remotefits, hduname, clobber):
  568 + """
  569 + merge remote and local fits file
  570 + """
  571 +
  572 + if self.logTerse():
  573 + self.log.parformat("Local file")
  574 + self.log(str(localfits))
  575 + self.log("\n")
  576 + self.log.parformat("Remote file")
  577 + self.log(str(remotefits))
  578 + self.log("\n")
  579 + self.log.parformat("HDU name")
  580 + self.log(str(hduname))
  581 + self.log("\n")
  582 + self.log.parformat("Conflict preference")
  583 + if clobber:
  584 + self.log("Remote")
  585 + else:
  586 + self.log("Local")
  587 + self.log("\n")
  588 +
  589 + # Check if local fits file is available
  590 + if os.path.isfile(localfits):
  591 + local = gammalib.GFits(str(localfits))
  592 + local_hdu = local[hduname]
  593 + else:
  594 + # Otherwise load remote fits file and delete rows
  595 + local = gammalib.GFits(str(remotefits))
  596 + local_hdu = local[hduname]
  597 + local_hdu.remove_rows(0,local_hdu.nrows())
  598 +
  599 + # find local obs_id
  600 + local_obs = []
  601 + lobs_col = local_hdu["OBS_ID"]
  602 + for obs_id in lobs_col:
  603 + local_obs.append(obs_id)
  604 +
  605 + # Create selection expression for opening remote fits file
  606 + selection = ""
  607 + if len(self.runlist):
  608 + selection += "["+hduname+"]["
  609 + for run in self.runlist:
  610 + selection += "OBS_ID=="+str(run)
  611 + selection += "||"
  612 + selection = selection[:-2]
  613 + selection += "]"
  614 +
  615 + # Create file name
  616 + remotefile = remotefits + selection
  617 +
  618 + # open remote fits file
  619 + remote = gammalib.GFits(str(remotefile))
  620 + remote_hdu = remote[hduname]
  621 +
  622 + # Get remote obs_id
  623 + remote_obs = []
  624 + robs_col = remote_hdu["OBS_ID"]
  625 + for obs_id in robs_col:
  626 + remote_obs.append(obs_id)
  627 +
  628 + if self.logTerse():
  629 + self.log.parformat("Remote entries")
  630 + self.log(str(len(remote_obs)))
  631 + self.log("\n")
  632 + self.log.parformat("Local entries")
  633 + self.log(str(len(local_obs)))
  634 + self.log("\n")
  635 +
  636 + # initialise counter for logging
  637 + removed_rows = 0
  638 +
  639 + # If clobber=yes, we use the remote content to overwrite corresponding local content
  640 + if clobber:
  641 +
  642 + # Loop over remote obs_ids
  643 + for remote_obs_id in remote_obs:
  644 +
  645 + # Check if remote obs_id is present locally
  646 + if remote_obs_id in local_obs:
  647 +
  648 + # Remove local obs_id entries
  649 + table_has_obsid = True
  650 + while table_has_obsid:
  651 + for i in range(local_hdu.nrows()):
  652 + if remote_obs_id == local_hdu["OBS_ID"][i]:
  653 + local_hdu.remove_rows(i,1)
  654 + removed_rows += 1
  655 + break
  656 + table_has_obsid = False
  657 + for i in range(local_hdu.nrows()):
  658 + if remote_obs_id == local_hdu["OBS_ID"][i]:
  659 + table_has_obsid = True
  660 + break
  661 +
  662 + # If clobber=false, we keep the local content unchanged and just expand by remote content
  663 + else:
  664 + # Loop over remote obs_ids
  665 + for local_obs_id in local_obs:
  666 +
  667 + # Check if local obs_id is present remotely
  668 + if local_obs_id in remote_obs:
  669 +
  670 + # Remove remote obs_id entries
  671 + table_has_obsid = True
  672 + while table_has_obsid:
  673 + for i in range(remote_hdu.nrows()):
  674 + if local_obs_id == remote_hdu["OBS_ID"][i]:
  675 + remote_hdu.remove_rows(i,1)
  676 + removed_rows += 1
  677 + break
  678 + table_has_obsid = False
  679 + for i in range(remote_hdu.nrows()):
  680 + if local_obs_id == remote_hdu["OBS_ID"][i]:
  681 + table_has_obsid = True
  682 + break
  683 +
  684 + # Store number of local rows
  685 + old_local_rows = local_hdu.nrows()
  686 +
  687 + # Add rmeote HDUs
  688 + local_hdu.insert_rows(old_local_rows, remote_hdu.nrows())
  689 +
  690 + if self.logTerse():
  691 + if clobber:
  692 + self.log.parformat("Removed local rows")
  693 + else:
  694 + self.log.parformat("Skipped remote rows")
  695 + self.log(str(removed_rows))
  696 + self.log("\n")
  697 +
  698 + # Loop over columns
  699 + for i in range(local_hdu.ncols()):
  700 +
  701 + # Get local and remote columns
  702 + local_col = local_hdu[i]
  703 + remote_col = remote_hdu[i]
  704 +
  705 + # Loop over entries and merge
  706 + for j in range(remote_col.length()):
  707 + local_col[j+old_local_rows] = remote_col[j]
  708 +
  709 + # Save local fits file
  710 + local.saveto(str(localfits), True)
  711 +
  712 + # Return
  713 + return
  714 +
  715 +# ======================== #
  716 +# Main routine entry point #
  717 +# ======================== #
  718 +if __name__ == '__main__':
  719 + """
  720 + Copy IACT data
  721 + """
  722 + # Create instance of application
  723 + app = csiactdload(sys.argv)
  724 +
  725 + # Open logfile
  726 + app.logFileOpen()
  727 +
  728 + # Execute application
  729 + app.execute()
... ...
cscripts/csiactobs.py
  View file @0c61eac
... ... @@ -49,7 +49,8 @@ class csiactobs(ctools.cscript):
49 49 self.m_ebounds = gammalib.GEbounds()
50 50 self.datapath = os.getenv("VHEFITS","")
51 51 self.inmodels = None
52   - self.outobs = "obs.xml"
  52 + self.xml = gammalib.GXml()
  53 + self.models = gammalib.GModels()
53 54  
54 55 # Make sure that parfile exists
55 56 file = self.parfile()
... ... @@ -141,10 +142,10 @@ class csiactobs(ctools.cscript):
141 142 self.m_bkgpars = self["bkgpars"].integer()
142 143  
143 144 # Output model file
144   - self.outmodel = self["outmodel"].filename()
  145 + self.m_outmodel = self["outmodel"].filename()
145 146  
146 147 # Observation outfile
147   - self.outobs = self["outobs"].filename()
  148 + self.m_outobs = self["outobs"].filename()
148 149  
149 150 # Master index file name
150 151 self.m_master_indx = self["master_indx"].string()
... ... @@ -332,6 +333,24 @@ class csiactobs(ctools.cscript):
332 333 Returns runlist energy range
333 334 """
334 335 return self.m_ebounds
  336 +
  337 + def obs(self):
  338 + """
  339 + Returns GObservations object
  340 + """
  341 + # Initialise observations
  342 + obs = gammalib.GObservations()
  343 +
  344 + # Read observations if XML is filled
  345 + if self.xml.size():
  346 + obs.read(self.xml)
  347 +
  348 + # Assign models
  349 + obs.models(self.models)
  350 +
  351 + # Return observations
  352 + return obs
  353 +
335 354  
336 355 def execute(self):
337 356 """
... ... @@ -341,7 +360,7 @@ class csiactobs(ctools.cscript):
341 360 self.run()
342 361  
343 362 # Save residual map
344   - self.save(self.outobs, self.m_clobber)
  363 + self.save(self.m_outobs, self.m_clobber)
345 364  
346 365 # Return
347 366 return
... ... @@ -362,11 +381,11 @@ class csiactobs(ctools.cscript):
362 381 self.log_parameters()
363 382 self.log("\n")
364 383  
365   - # Initialise empty models
366   - self.models = gammalib.GModels()
  384 + # Clear models
  385 + self.models.clear()
367 386  
368   - # Initialise empty xml file and append an observation list
369   - self.xml = gammalib.GXml()
  387 + # Clear xml file and append an observation list
  388 + self.xml.clear()
370 389 self.xml.append(gammalib.GXmlElement("observation_list title=\"observation list\""))
371 390 lib = self.xml.element("observation_list", 0)
372 391  
... ... @@ -627,7 +646,7 @@ class csiactobs(ctools.cscript):
627 646 self.xml.save(outfile)
628 647  
629 648 # Save model XML file
630   - self.models.save(self.outmodel)
  649 + self.models.save(self.m_outmodel)
631 650  
632 651 # Return
633 652 return
... ...
doc/source/reference_manual/csiactdload.rst 0 → 100644
  View file @0c61eac
  1 +.. _csiactdload:
  2 +
  3 +csiactdload
  4 +===========
  5 +
  6 +Copies IACT data from one location to another
  7 +
  8 +
  9 +Synopsis
  10 +--------
  11 +
  12 +This script allows to download IACT data from a remote machine. The remote file system
  13 +has to be mounted before running the script. For mounting, e.g. sshfs can be used:
  14 +
  15 +.. code-block:: bash
  16 +
  17 + $ sshfs user@remote.server.com:/path/to/remote/fits /path/to/mountpoint/
  18 +
  19 +The script can optionally take an ASCII file with a list of observation IDs as input.
  20 +In this way the user can copy only a specific subset of the data. Index files and data
  21 +structure is updated accordingly by the script.
  22 +
  23 +In case the copying procedure failed due to e.g. a broken connection, the script can be executed
  24 +again using the parameter ``clobber=no``. Thus files that were already copied don't get
  25 +recopied and overwritten which saves time.
  26 +
  27 +In order to monitor the progress on the screen, the script can be executed with the
  28 +hidden parameter ``debug=yes``
  29 +
  30 +
  31 +General parameters
  32 +------------------
  33 +
  34 +``remote_master [file]``
  35 + Location of remote master file
  36 +
  37 +``prodname [string]``
  38 + Name of FITS production to download
  39 +
  40 +``outpath [file]``
  41 + Destination path of FITS data
  42 +
  43 +``(runlist = NONE) [file]``
  44 + List of observation IDs
  45 +
  46 +
  47 +Standard parameters
  48 +-------------------
  49 +
  50 +``(chatter = 2) [integer]``
  51 + Verbosity of the executable:
  52 + chatter = 0: no information will be logged
  53 +
  54 + chatter = 1: only errors will be logged
  55 +
  56 + chatter = 2: errors and actions will be logged
  57 +
  58 + chatter = 3: report about the task execution
  59 +
  60 + chatter = 4: detailed report about the task execution
  61 +
  62 +``(clobber = yes) [boolean]``
  63 + Specifies whether an existing output counts cube should be overwritten.
  64 +
  65 +``(debug = no) [boolean]``
  66 + Enables debug mode. In debug mode the executable will dump any log file output to the console.
  67 +
  68 +``(mode = ql) [string]``
  69 + Mode of automatic parameters (default is "ql", i.e. "query and learn").
  70 +
  71 +``(logfile = csiactdload.log) [filename]``
  72 + Log filename.
  73 +
  74 +
  75 +Related tools or scripts
  76 +------------------------
  77 +
  78 +:doc:`csiactdata`
0 79 \ No newline at end of file
... ...
doc/source/reference_manual/index.rst
  View file @0c61eac
... ... @@ -61,3 +61,4 @@ IACT cscripts
61 61 csiactdata --- Shows information about IACT data available on the user machine <csiactdata>
62 62 csiactobs --- Generates observation definition file for IACT data from observation IDs <csiactobs>
63 63 csfindobs --- Generates a list of IACT observation IDs <csfindobs>
  64 + csiactdload --- Copies IACT data from one location to another <csiactdload>
... ...
test/test_cscripts.sh.in
  View file @0c61eac
... ... @@ -483,6 +483,26 @@ $ECHO &quot; ok&quot;
483 483  
484 484  
485 485 #
  486 +# Test csiactdload
  487 +# ==============
  488 +$ECHO -n "Test csiactdload: "
  489 +chmod +x @abs_top_srcdir@/cscripts/csiactdload.py
  490 +@abs_top_srcdir@/cscripts/csiactdload.py \
  491 + remote_master="iactdata/master.json" \
  492 + outpath="iactdata_test" \
  493 + prodname="unit-test" > csiactdload.log
  494 +$ECHO -n "."
  495 +if [ -s "iactdata_test/master.json" ]
  496 +then
  497 + $ECHO -n "."
  498 +else
  499 + $ECHO " iactdata_test/master.json file is not valid"
  500 + exit 1
  501 +fi
  502 +$ECHO " ok"
  503 +
  504 +
  505 +#
486 506 # Test csfindobs
487 507 # ==============
488 508 $ECHO -n "Test csfindobs: "
... ...