diff --git a/cscripts/Makefile.am b/cscripts/Makefile.am
index efd24db..55fa0ed 100644
--- a/cscripts/Makefile.am
+++ b/cscripts/Makefile.am
@@ -5,14 +5,14 @@ pythonscripts = __init__.py \
cspull.py cssens.py cstsdist.py cscaldb.py csresmap.py \
csiactobs.py csiactdata.py csobsdef.py csspec.py cslightcrv.py csobsinfo.py \
csinfo.py csmodelinfo.py csmodelmerge.py cstsmapmerge.py \
- csobs2caldb.py csfindobs.py \
+ csobs2caldb.py csfindobs.py csiactdload.py \
obsutils.py
# Scripts for which symbolic links will be created in bindir
linkscripts = cspull cssens cstsdist cscaldb csresmap \
csiactobs csiactdata csobsdef csspec cslightcrv csobsinfo \
csinfo csmodelinfo csmodelmerge cstsmapmerge \
- csobs2caldb csfindobs
+ csobs2caldb csfindobs csiactdload
# Distribute scripts
EXTRA_DIST = $(binscripts) $(pythonscripts)
diff --git a/cscripts/__init__.py.in b/cscripts/__init__.py.in
index e9fa16c..2e3bfa2 100644
--- a/cscripts/__init__.py.in
+++ b/cscripts/__init__.py.in
@@ -39,6 +39,7 @@ __all__ = [
"csfindobs",
"csiactobs",
"csiactdata",
+ "csiactdload",
"obsutils",
"test",
]
@@ -63,6 +64,7 @@ from .cstsmapmerge import cstsmapmerge
from .csfindobs import csfindobs
from .csiactobs import csiactobs
from .csiactdata import csiactdata
+from .csiactdload import csiactdload
from . import obsutils
# Add test function
diff --git a/cscripts/csiactdload.py b/cscripts/csiactdload.py
new file mode 100755
index 0000000..23d57fb
--- /dev/null
+++ b/cscripts/csiactdload.py
@@ -0,0 +1,729 @@
+#!/usr/bin/env python
+# ==========================================================================
+# Download IACT data from remote machine
+#
+# Copyright (C) 2015 Michael Mayer
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see .
+#
+# ==========================================================================
+import gammalib
+import ctools
+import sys
+import os
+import json
+import shutil
+
+# ================ #
+# csiactdata class #
+# ================ #
+class csiactdload(ctools.cscript):
+ """
+ This script copies IACT data from one location to another.
+ It can take a list of observation IDs to allow the download specific
+ observations. Index files get merged and updated accordingly.
+ """
+ def __init__(self, *argv):
+ """
+ Constructor.
+ """
+ # Set name
+ self.name = "csiactdload"
+ self.version = "1.1.0"
+ self.datapath = os.getenv("VHEFITS","")
+
+ # Make sure that parfile exists
+ file = self.parfile()
+
+ # Initialise application
+ if len(argv) == 0:
+ ctools.cscript.__init__(self, self.name, self.version)
+ elif len(argv) == 1:
+ ctools.cscript.__init__(self, self.name, self.version, *argv)
+ else:
+ raise TypeError("Invalid number of arguments given.")
+
+ # Set logger properties
+ self.log_header()
+ self.log.date(True)
+
+ # Return
+ return
+
+ def __del__(self):
+ """
+ Destructor.
+ """
+
+ # Return
+ return
+
+ def parfile(self):
+ """
+ Check if parfile exists. If parfile does not exist then create a
+ default parfile. This kluge avoids shipping the cscript with a
+ parfile.
+ """
+ # Set parfile name
+ parfile = self.name+".par"
+
+ try:
+ pars = gammalib.GApplicationPars(parfile)
+ except:
+ # Signal if parfile was not found
+ sys.stdout.write("Parfile "+parfile+" not found. Create default parfile.\n")
+
+ # Create default parfile
+ pars = gammalib.GApplicationPars()
+ pars.append(gammalib.GApplicationPar("remote_master","f","a","/Volumes/mountpoint/master.json","","","Location of remote master file"))
+ pars.append(gammalib.GApplicationPar("prodname","s","a","iact-fits","","","Name of FITS production to download"))
+ pars.append(gammalib.GApplicationPar("outpath","s","a","/path/to/fits","","","Destination path of FITS data"))
+ pars.append(gammalib.GApplicationPar("runlist","f","h","NONE","","","List of observation IDs"))
+ pars.append_standard()
+ pars.append(gammalib.GApplicationPar("logfile","f","h","csiactdload.log","","","Log filename"))
+ pars.save(parfile)
+
+ # Return
+ return
+
+ def get_parameters(self):
+ """
+ Get parameters from parfile and setup the observation.
+ """
+
+ # Get Parameters
+ self.m_remote_master = gammalib.expand_env(self["remote_master"].filename())
+ if not os.path.isfile(self.m_remote_master):
+ raise RuntimeError("*** ERROR: FITS data not available. No master index file found in \""+self.m_remote_master+"\". Make sure remote file system is properly mounted")
+
+ # Get parameters
+ self.m_prodname = self["prodname"].string()
+ self.m_outpath = gammalib.expand_env(self["outpath"].string())
+ self.m_runlist = self["runlist"].filename()
+
+ # Return
+ return
+
+ def execute(self):
+ """
+ Execute the script.
+ """
+ # Run the script
+ self.run()
+
+ # Return
+ return
+
+ def run(self):
+ """
+ Run the script.
+ """
+ # Switch screen logging on
+ self.log.cout(True)
+
+ # Get parameters
+ self.get_parameters()
+
+ # Write input parameters into logger
+ if self.logTerse():
+ self.log_parameters()
+ self.log("\n")
+
+ # Make destination directory if not available
+ if not os.path.isdir(self.m_outpath):
+ os.makedirs(self.m_outpath)
+
+ # Log output
+ if self.logTerse():
+ self.log("\n")
+ self.log.header2("Runlist information")
+
+ # Initialise runlist array
+ self.runlist = []
+
+ # Check if runlist file is available
+ if os.path.isfile(self.m_runlist):
+ runfile = open(self.m_runlist)
+ for line in runfile.readlines():
+ if len(line) == 0:
+ continue
+ if line[0] == '#':
+ continue
+ if len(line.split()) > 0:
+ self.runlist.append(int(line.split()[0]))
+ runfile.close()
+
+ # Logging
+ if self.logTerse():
+ self.log.parformat("Number of observations")
+ self.log(str(len(self.runlist)))
+ self.log("\n")
+ # Copy all data if runlist file is "NONE"
+ elif self.m_runlist == "NONE":
+ if self.logTerse():
+ self.log("Copy all available data")
+ self.log("\n")
+ # Raise exception if file not valid
+ else:
+ raise RuntimeError("*** ERROR: Runlist file \""+self.m_runlist+"\" not available")
+
+ # Check for availability of remote master file
+ if not os.path.isfile(self.m_remote_master):
+ raise RuntimeError("*** ERROR: Remote master file \""+self.m_remote_master+"\" does not exist")
+
+ # Retrieve json data from remote master
+ json_data = open(self.m_remote_master).read()
+ data = json.loads(json_data)
+ if not "datasets" in data:
+ raise RuntimeError("*** ERROR: Key \"datasets\" not available in remote master index file \""+self.m_remote_master+"\".")
+
+ # Get array of configurations
+ configs = data["datasets"]
+
+ # Get remote paths
+ self.remote_base = os.path.dirname(self.m_remote_master)
+
+ # Initialise flag if prod has been found
+ has_prod = False
+
+ # Initialise file names to be copied
+ files = set()
+
+ # Logging
+ if self.logTerse():
+ self.log("\n")
+ self.log.header2("Loop over remote configs")
+
+ # Loop over configs
+ for config in configs:
+
+ # Logging
+ if self.logVerbose():
+ self.log.header2(str(config["name"]))
+
+ # Check if prodname was found
+ if config["name"] == self.m_prodname:
+
+ # Inidicate that prodname has been found
+ has_prod = True
+
+ # Build path of index files
+ remote_hdu = os.path.join(self.remote_base, config["hduindx"])
+ remote_obs = os.path.join(self.remote_base, config["obsindx"])
+
+ # Log information
+ if self.logTerse():
+ self.log.header3("Remote config \""+self.m_prodname+"\"")
+ self.log.parformat("HDU index")
+ self.log(str(remote_hdu))
+ self.log("\n")
+ self.log.parformat("Observation index")
+ self.log(str(remote_obs))
+ self.log("\n")
+
+ # Open remote HDU index file
+ fits = gammalib.GFits(str(remote_hdu))
+ table = fits["HDU_INDEX"]
+
+ # Initialise flag if SIZE column is present
+ has_size = table.contains("SIZE")
+
+ if has_size:
+ # Initialise file size
+ cp_size = 0
+
+ # Initialise remote observation IDs
+ remote_ids = set()
+ for row in range(table.nrows()):
+ remote_ids.add(table["OBS_ID"][row])
+
+ # Log runs that are not available remotely
+ for run in self.runlist:
+ if not run in remote_ids:
+ if self.logNormal():
+ self.log("Skip observation "+str(run)+": ID not available remotely")
+ self.log("\n")
+
+
+ # Loop over remote HDU index file
+ for row in range(table.nrows()):
+
+ # Get observation ID
+ obs_id = table["OBS_ID"][row]
+ file_dir = table["FILE_DIR"][row]
+ file_name = table["FILE_NAME"][row]
+
+ # Skip if filename is empty
+ if file_name == "":
+ continue
+
+ # Check if we need to consider an input runlist
+ if len(self.runlist):
+
+ # Check if obs id is in runlist
+ if obs_id in self.runlist:
+
+ # Get filename
+ fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name)
+
+ # Add file
+ oldlen = len(files)
+ files.add(fname)
+ newlen = len(files)
+
+ # Add file size
+ if has_size and newlen > oldlen:
+ cp_size += table["SIZE"][row]
+
+ # Otherwise add every file
+ else:
+ # Get filename
+ fname = os.path.join(os.path.dirname(remote_hdu), file_dir, file_name)
+
+ # Add file
+ oldlen = len(files)
+ files.add(fname)
+ newlen = len(files)
+
+ # Add file size
+ if has_size and newlen > oldlen:
+ cp_size += table["SIZE"][row]
+
+ # Log file information
+ if self.logTerse():
+ self.log("\n")
+ self.log.header2("File information")
+ self.log.parformat("Number of files")
+ self.log(str(len(files)))
+ self.log("\n")
+ if has_size:
+ self.log.parformat("Size")
+ self.log("%.2f"%(float(cp_size)*1e-6)+" MB")
+ self.log("\n")
+ if self.logVerbose():
+ self.log("\n")
+ self.log.header3("File names")
+ for filename in files:
+ self.log(str(filename)+"\n")
+
+ # Close HDU index file
+ fits.close()
+
+ # If prodname is not found just log that we skip the config
+ else:
+ if self.logExplicit():
+ self.log("\n")
+ self.log.header3("Skipping config \""+str(config["name"])+"\"")
+ self.log("\n")
+
+ # Raise Exception if prodname was not found
+ if not has_prod:
+ msg = "*** ERROR: FITS production \""+self.m_prodname+"\" not available. "
+ msg += "Available productions are: \n"
+ for config in configs:
+ msg += " - "+config["name"] + "\n"
+ raise RuntimeError(msg)
+
+ # Logging
+ if self.logNormal():
+ self.log("\n")
+ self.log.header1("Copying files")
+
+ # intialise counter
+ k = 0
+
+ # initialise values for logging
+ last_fraction = 0.0
+ fraction_increment = 20.0
+ if self.logNormal():
+ fraction_increment = 10.0
+ if self.logTerse():
+ fraction_increment = 5.0
+ if self.logExplicit():
+ fraction_increment = 2.0
+
+ # initialise logging properties
+ n_copied = 0
+ total_size = 0.0
+
+ # Loop over files and copy
+ for filename in files:
+
+ # Log progress
+ fraction = float(k) / float(len(files)) * 100.0
+ if fraction > last_fraction:
+ if self.logNormal() and not self.logVerbose():
+ self.log.parformat("Status")
+ self.log(str(int(last_fraction))+"%")
+ self.log("\n")
+ last_fraction += fraction_increment
+
+ # Copy file
+ filesize = self.copy(filename, self.clobber())
+
+ if filesize > 0.0:
+ total_size += filesize
+ n_copied += 1
+
+ # Increment counter
+ k += 1
+
+ # Logging
+ if self.logNormal() and not self.logVerbose():
+ self.log.parformat("Status")
+ self.log("Finished")
+ self.log("\n")
+ if self.logTerse():
+ self.log("\n")
+ self.log.header1("Updating index files")
+
+ # If we have a runlist then merge index file
+ if len(self.runlist):
+
+ # Check if there are local index files
+ local_hdu = os.path.join(self.m_outpath,os.path.relpath(remote_hdu, self.remote_base))
+ local_obs = os.path.join(self.m_outpath,os.path.relpath(remote_obs, self.remote_base))
+
+ if self.logTerse():
+ self.log("\n")
+ self.log.header3("HDU index")
+
+ # Merge remote index files with local files
+ self.merge(local_hdu, remote_hdu, "HDU_INDEX", self.clobber())
+
+ if self.logTerse():
+ self.log("\n")
+ self.log.header3("OBS index")
+
+ # Merge remote index files with local files
+ self.merge(local_obs, remote_obs, "OBS_INDEX", self.clobber())
+
+ else:
+ # If all files were copied, just copy index files too
+ self.copy(remote_hdu, self.clobber())
+ self.copy(remote_obs, self.clobber())
+
+ if self.logTerse():
+ self.log("\n")
+ self.log.header3("Master index file")
+
+ # Adding prodname to local master
+ localmaster = os.path.join(self.m_outpath, "master.json")
+
+ # If no local master is found, copy master over first
+ if not os.path.isfile(localmaster):
+ self.copy(self.m_remote_master, self.clobber())
+
+ # Load local master
+ json_data = open(localmaster).read()
+ data = json.loads(json_data)
+ configs = data["datasets"]
+
+ # Initialise flag indicating if we already have prodname in master file
+ has_config = False
+
+ # Initialise new configs array
+ newconfigs = []
+
+ # Loop over configs in master index file
+ for config in configs:
+
+ # Get hdu and obs index files
+ hdu = os.path.join(self.m_outpath, config["hduindx"])
+ obs = os.path.join(self.m_outpath, config["obsindx"])
+
+ # Check if index files are available
+ if not (gammalib.is_fits(str(hdu)) and gammalib.is_fits(str(obs))):
+ if self.logTerse():
+ self.log("Removing \""+str(config["name"])+"\" (not available)")
+ self.log("\n")
+ else:
+ # Append config if valid
+ newconfigs.append(config)
+ if self.logTerse():
+ self.log("Keeping \""+str(config["name"])+"\"")
+ self.log("\n")
+
+ # Signals that downloaded config is available
+ if config["name"] == self.m_prodname:
+ has_config = True
+
+ # Create new entry if config was not available
+ if not has_config:
+ newdict = dict.fromkeys(["name","hduindx","obsindx"])
+ newdict["name"] = self.m_prodname
+ newdict["hduindx"] = os.path.relpath(local_hdu, self.m_outpath)
+ newdict["obsindx"] = os.path.relpath(local_obs, self.m_outpath)
+ newconfigs.append(newdict)
+
+ # Write new json master file
+ f = open(localmaster, "w")
+ data["datasets"] = newconfigs
+ json.dump(data, f, indent=2)
+ f.close()
+
+ # Log summary
+ if self.logNormal():
+ self.log("\n")
+ self.log.header1("Summary")
+ self.log.parformat("Data files found")
+ self.log(str(len(files)))
+ self.log("\n")
+ self.log.parformat("Data files copied")
+ self.log(str(n_copied))
+ self.log("\n")
+ self.log.parformat("Size")
+ self.log("%.2f"%(float(total_size)*1e-6)+" MB")
+ self.log("\n")
+ self.log.parformat("Local configs")
+ self.log(str(len(newconfigs)))
+ self.log("\n")
+ if self.logTerse():
+ self.log("\n")
+ self.log.header3("Content of master index")
+ for config in newconfigs:
+ self.log(str(config["name"]))
+ self.log("\n")
+
+ # Return
+ return
+
+ def copy(self, source, clobber):
+ """
+ copy file to m_outpath
+ """
+ # Get file destination
+ destination = os.path.join(self.m_outpath,os.path.relpath(source, self.remote_base))
+
+
+ # Initialise return value
+ filesize = 0.0
+
+ # Logging
+ if self.logVerbose():
+ self.log("\n")
+ self.log.header3("Copy file")
+ self.log.parformat("Source")
+ self.log(str(source))
+ self.log("\n")
+ self.log.parformat("Destination")
+ self.log(str(destination))
+ self.log("\n")
+ self.log.parformat("Already exists")
+ self.log(str(os.path.isfile(destination)))
+ self.log("\n")
+ self.log.parformat("Overwrite")
+ self.log(str(clobber))
+ self.log("\n")
+ self.log.parformat("Copying")
+
+ # Flag if file destination is already available
+ is_file = os.path.isfile(destination)
+
+ # check if file could be skipped because clobber=no
+ if is_file and clobber == False:
+ if self.logVerbose():
+ self.log("Skip (clobber=no)")
+ self.log("\n")
+
+ # check if file could be skipped because it is the same file
+ elif is_file and os.path.samefile(destination, source):
+ if self.logVerbose():
+ self.log("Skip (same file)")
+ self.log("\n")
+
+ else:
+ # Create directory if not existent
+ dest_dir = os.path.dirname(destination)
+ if not os.path.isdir(dest_dir):
+ os.makedirs(dest_dir)
+
+ # Copy file (if source not identical to destination)
+ shutil.copy2(source, destination)
+
+ # Get filesize
+ filesize = os.stat(destination).st_size
+
+ # Logging
+ if self.logVerbose():
+ self.log("Done!")
+ self.log("\n")
+
+ # Return
+ return filesize
+
+
+ def merge(self, localfits, remotefits, hduname, clobber):
+ """
+ merge remote and local fits file
+ """
+
+ if self.logTerse():
+ self.log.parformat("Local file")
+ self.log(str(localfits))
+ self.log("\n")
+ self.log.parformat("Remote file")
+ self.log(str(remotefits))
+ self.log("\n")
+ self.log.parformat("HDU name")
+ self.log(str(hduname))
+ self.log("\n")
+ self.log.parformat("Conflict preference")
+ if clobber:
+ self.log("Remote")
+ else:
+ self.log("Local")
+ self.log("\n")
+
+ # Check if local fits file is available
+ if os.path.isfile(localfits):
+ local = gammalib.GFits(str(localfits))
+ local_hdu = local[hduname]
+ else:
+ # Otherwise load remote fits file and delete rows
+ local = gammalib.GFits(str(remotefits))
+ local_hdu = local[hduname]
+ local_hdu.remove_rows(0,local_hdu.nrows())
+
+ # find local obs_id
+ local_obs = []
+ lobs_col = local_hdu["OBS_ID"]
+ for obs_id in lobs_col:
+ local_obs.append(obs_id)
+
+ # Create selection expression for opening remote fits file
+ selection = ""
+ if len(self.runlist):
+ selection += "["+hduname+"]["
+ for run in self.runlist:
+ selection += "OBS_ID=="+str(run)
+ selection += "||"
+ selection = selection[:-2]
+ selection += "]"
+
+ # Create file name
+ remotefile = remotefits + selection
+
+ # open remote fits file
+ remote = gammalib.GFits(str(remotefile))
+ remote_hdu = remote[hduname]
+
+ # Get remote obs_id
+ remote_obs = []
+ robs_col = remote_hdu["OBS_ID"]
+ for obs_id in robs_col:
+ remote_obs.append(obs_id)
+
+ if self.logTerse():
+ self.log.parformat("Remote entries")
+ self.log(str(len(remote_obs)))
+ self.log("\n")
+ self.log.parformat("Local entries")
+ self.log(str(len(local_obs)))
+ self.log("\n")
+
+ # initialise counter for logging
+ removed_rows = 0
+
+ # If clobber=yes, we use the remote content to overwrite corresponding local content
+ if clobber:
+
+ # Loop over remote obs_ids
+ for remote_obs_id in remote_obs:
+
+ # Check if remote obs_id is present locally
+ if remote_obs_id in local_obs:
+
+ # Remove local obs_id entries
+ table_has_obsid = True
+ while table_has_obsid:
+ for i in range(local_hdu.nrows()):
+ if remote_obs_id == local_hdu["OBS_ID"][i]:
+ local_hdu.remove_rows(i,1)
+ removed_rows += 1
+ break
+ table_has_obsid = False
+ for i in range(local_hdu.nrows()):
+ if remote_obs_id == local_hdu["OBS_ID"][i]:
+ table_has_obsid = True
+ break
+
+ # If clobber=false, we keep the local content unchanged and just expand by remote content
+ else:
+ # Loop over remote obs_ids
+ for local_obs_id in local_obs:
+
+ # Check if local obs_id is present remotely
+ if local_obs_id in remote_obs:
+
+ # Remove remote obs_id entries
+ table_has_obsid = True
+ while table_has_obsid:
+ for i in range(remote_hdu.nrows()):
+ if local_obs_id == remote_hdu["OBS_ID"][i]:
+ remote_hdu.remove_rows(i,1)
+ removed_rows += 1
+ break
+ table_has_obsid = False
+ for i in range(remote_hdu.nrows()):
+ if local_obs_id == remote_hdu["OBS_ID"][i]:
+ table_has_obsid = True
+ break
+
+ # Store number of local rows
+ old_local_rows = local_hdu.nrows()
+
+ # Add rmeote HDUs
+ local_hdu.insert_rows(old_local_rows, remote_hdu.nrows())
+
+ if self.logTerse():
+ if clobber:
+ self.log.parformat("Removed local rows")
+ else:
+ self.log.parformat("Skipped remote rows")
+ self.log(str(removed_rows))
+ self.log("\n")
+
+ # Loop over columns
+ for i in range(local_hdu.ncols()):
+
+ # Get local and remote columns
+ local_col = local_hdu[i]
+ remote_col = remote_hdu[i]
+
+ # Loop over entries and merge
+ for j in range(remote_col.length()):
+ local_col[j+old_local_rows] = remote_col[j]
+
+ # Save local fits file
+ local.saveto(str(localfits), True)
+
+ # Return
+ return
+
+# ======================== #
+# Main routine entry point #
+# ======================== #
+if __name__ == '__main__':
+ """
+ Copy IACT data
+ """
+ # Create instance of application
+ app = csiactdload(sys.argv)
+
+ # Open logfile
+ app.logFileOpen()
+
+ # Execute application
+ app.execute()
diff --git a/cscripts/csiactobs.py b/cscripts/csiactobs.py
index e42f704..26c6967 100755
--- a/cscripts/csiactobs.py
+++ b/cscripts/csiactobs.py
@@ -49,7 +49,8 @@ class csiactobs(ctools.cscript):
self.m_ebounds = gammalib.GEbounds()
self.datapath = os.getenv("VHEFITS","")
self.inmodels = None
- self.outobs = "obs.xml"
+ self.xml = gammalib.GXml()
+ self.models = gammalib.GModels()
# Make sure that parfile exists
file = self.parfile()
@@ -141,10 +142,10 @@ class csiactobs(ctools.cscript):
self.m_bkgpars = self["bkgpars"].integer()
# Output model file
- self.outmodel = self["outmodel"].filename()
+ self.m_outmodel = self["outmodel"].filename()
# Observation outfile
- self.outobs = self["outobs"].filename()
+ self.m_outobs = self["outobs"].filename()
# Master index file name
self.m_master_indx = self["master_indx"].string()
@@ -332,6 +333,24 @@ class csiactobs(ctools.cscript):
Returns runlist energy range
"""
return self.m_ebounds
+
+ def obs(self):
+ """
+ Returns GObservations object
+ """
+ # Initialise observations
+ obs = gammalib.GObservations()
+
+ # Read observations if XML is filled
+ if self.xml.size():
+ obs.read(self.xml)
+
+ # Assign models
+ obs.models(self.models)
+
+ # Return observations
+ return obs
+
def execute(self):
"""
@@ -341,7 +360,7 @@ class csiactobs(ctools.cscript):
self.run()
# Save residual map
- self.save(self.outobs, self.m_clobber)
+ self.save(self.m_outobs, self.m_clobber)
# Return
return
@@ -362,11 +381,11 @@ class csiactobs(ctools.cscript):
self.log_parameters()
self.log("\n")
- # Initialise empty models
- self.models = gammalib.GModels()
+ # Clear models
+ self.models.clear()
- # Initialise empty xml file and append an observation list
- self.xml = gammalib.GXml()
+ # Clear xml file and append an observation list
+ self.xml.clear()
self.xml.append(gammalib.GXmlElement("observation_list title=\"observation list\""))
lib = self.xml.element("observation_list", 0)
@@ -627,7 +646,7 @@ class csiactobs(ctools.cscript):
self.xml.save(outfile)
# Save model XML file
- self.models.save(self.outmodel)
+ self.models.save(self.m_outmodel)
# Return
return
diff --git a/doc/source/reference_manual/csiactdload.rst b/doc/source/reference_manual/csiactdload.rst
new file mode 100644
index 0000000..b9dafae
--- /dev/null
+++ b/doc/source/reference_manual/csiactdload.rst
@@ -0,0 +1,78 @@
+.. _csiactdload:
+
+csiactdload
+===========
+
+Copies IACT data from one location to another
+
+
+Synopsis
+--------
+
+This script allows to download IACT data from a remote machine. The remote file system
+has to be mounted before running the script. For mounting, e.g. sshfs can be used:
+
+.. code-block:: bash
+
+ $ sshfs user@remote.server.com:/path/to/remote/fits /path/to/mountpoint/
+
+The script can optionally take an ASCII file with a list of observation IDs as input.
+In this way the user can copy only a specific subset of the data. Index files and data
+structure is updated accordingly by the script.
+
+In case the copying procedure failed due to e.g. a broken connection, the script can be executed
+again using the parameter ``clobber=no``. Thus files that were already copied don't get
+recopied and overwritten which saves time.
+
+In order to monitor the progress on the screen, the script can be executed with the
+hidden parameter ``debug=yes``
+
+
+General parameters
+------------------
+
+``remote_master [file]``
+ Location of remote master file
+
+``prodname [string]``
+ Name of FITS production to download
+
+``outpath [file]``
+ Destination path of FITS data
+
+``(runlist = NONE) [file]``
+ List of observation IDs
+
+
+Standard parameters
+-------------------
+
+``(chatter = 2) [integer]``
+ Verbosity of the executable:
+ chatter = 0: no information will be logged
+
+ chatter = 1: only errors will be logged
+
+ chatter = 2: errors and actions will be logged
+
+ chatter = 3: report about the task execution
+
+ chatter = 4: detailed report about the task execution
+
+``(clobber = yes) [boolean]``
+ Specifies whether an existing output counts cube should be overwritten.
+
+``(debug = no) [boolean]``
+ Enables debug mode. In debug mode the executable will dump any log file output to the console.
+
+``(mode = ql) [string]``
+ Mode of automatic parameters (default is "ql", i.e. "query and learn").
+
+``(logfile = csiactdload.log) [filename]``
+ Log filename.
+
+
+Related tools or scripts
+------------------------
+
+:doc:`csiactdata`
\ No newline at end of file
diff --git a/doc/source/reference_manual/index.rst b/doc/source/reference_manual/index.rst
index f9a54c7..c8f548b 100644
--- a/doc/source/reference_manual/index.rst
+++ b/doc/source/reference_manual/index.rst
@@ -61,3 +61,4 @@ IACT cscripts
csiactdata --- Shows information about IACT data available on the user machine
csiactobs --- Generates observation definition file for IACT data from observation IDs
csfindobs --- Generates a list of IACT observation IDs
+ csiactdload --- Copies IACT data from one location to another
diff --git a/test/test_cscripts.sh.in b/test/test_cscripts.sh.in
index 237b7fa..ddd6847 100755
--- a/test/test_cscripts.sh.in
+++ b/test/test_cscripts.sh.in
@@ -483,6 +483,26 @@ $ECHO " ok"
#
+# Test csiactdload
+# ==============
+$ECHO -n "Test csiactdload: "
+chmod +x @abs_top_srcdir@/cscripts/csiactdload.py
+@abs_top_srcdir@/cscripts/csiactdload.py \
+ remote_master="iactdata/master.json" \
+ outpath="iactdata_test" \
+ prodname="unit-test" > csiactdload.log
+$ECHO -n "."
+if [ -s "iactdata_test/master.json" ]
+then
+ $ECHO -n "."
+else
+ $ECHO " iactdata_test/master.json file is not valid"
+ exit 1
+fi
+$ECHO " ok"
+
+
+#
# Test csfindobs
# ==============
$ECHO -n "Test csfindobs: "