/***************************************************************************
* ctselect - CTA data selection tool *
* ----------------------------------------------------------------------- *
* copyright (C) 2010-2014 by Juergen Knoedlseder *
* ----------------------------------------------------------------------- *
* *
* This program is free software: you can redistribute it and/or modify *
* it under the terms of the GNU General Public License as published by *
* the Free Software Foundation, either version 3 of the License, or *
* (at your option) any later version. *
* *
* This program is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* GNU General Public License for more details. *
* *
* You should have received a copy of the GNU General Public License *
* along with this program. If not, see . *
* *
***************************************************************************/
/**
* @file ctselect.cpp
* @brief CTA data selection tool implementation
* @author Juergen Knoedlseder
*/
/* __ Includes ___________________________________________________________ */
#ifdef HAVE_CONFIG_H
#include
#endif
#include
#include "ctselect.hpp"
#include "GTools.hpp"
/* __ Method name definitions ____________________________________________ */
#define G_RUN "ctselect::run()"
#define G_SELECT_EVENTS "ctselect::select_events(GCTAObservation*, "\
"std::string&)"
/* __ Debug definitions __________________________________________________ */
/* __ Coding definitions _________________________________________________ */
//#define G_USE_MKSTEMP //!< Use mkstemp for temporary filename
/*==========================================================================
= =
= Constructors/destructors =
= =
==========================================================================*/
/***********************************************************************//**
* @brief Void constructor
***************************************************************************/
ctselect::ctselect(void) : GApplication(CTSELECT_NAME, CTSELECT_VERSION)
{
// Initialise members
init_members();
// Write header into logger
log_header();
// Return
return;
}
/***********************************************************************//**
* @brief Observations constructor
*
* param[in] obs Observation container.
*
* This constructor creates an instance of the class that is initialised from
* an observation container.
***************************************************************************/
ctselect::ctselect(const GObservations& obs) :
GApplication(CTSELECT_NAME, CTSELECT_VERSION)
{
// Initialise members
init_members();
// Set observations
m_obs = obs;
// Write header into logger
log_header();
// Return
return;
}
/***********************************************************************//**
* @brief Command line constructor
*
* @param[in] argc Number of arguments in command line.
* @param[in] argv Array of command line arguments.
***************************************************************************/
ctselect::ctselect(int argc, char *argv[]) :
GApplication(CTSELECT_NAME, CTSELECT_VERSION, argc, argv)
{
// Initialise members
init_members();
// Write header into logger
log_header();
// Return
return;
}
/***********************************************************************//**
* @brief Copy constructor
*
* @param[in] app Application.
***************************************************************************/
ctselect::ctselect(const ctselect& app) : GApplication(app)
{
// Initialise members
init_members();
// Copy members
copy_members(app);
// Return
return;
}
/***********************************************************************//**
* @brief Destructor
***************************************************************************/
ctselect::~ctselect(void)
{
// Free members
free_members();
// Return
return;
}
/*==========================================================================
= =
= Operators =
= =
==========================================================================*/
/***********************************************************************//**
* @brief Assignment operator
*
* @param[in] app Application.
***************************************************************************/
ctselect& ctselect::operator= (const ctselect& app)
{
// Execute only if object is not identical
if (this != &app) {
// Copy base class members
this->GApplication::operator=(app);
// Free members
free_members();
// Initialise members
init_members();
// Copy members
copy_members(app);
} // endif: object was not identical
// Return this object
return *this;
}
/*==========================================================================
= =
= Public methods =
= =
==========================================================================*/
/***********************************************************************//**
* @brief Clear instance
***************************************************************************/
void ctselect::clear(void)
{
// Free members
free_members();
this->GApplication::free_members();
// Initialise members
this->GApplication::init_members();
init_members();
// Return
return;
}
/***********************************************************************//**
* @brief Execute application
*
* This method performs the event selection and saves the result
***************************************************************************/
void ctselect::execute(void)
{
// Signal that some parameters should be read ahead
m_read_ahead = true;
// Perform event selection
run();
// Save results
save();
// Return
return;
}
/***********************************************************************//**
* @brief Select event data
*
* This method reads in the application parameters and loops over all
* observations that were found to perform an event selection. Event
* selection is done by writing each observation to a temporary file and
* re-opening the temporary file using the cfitsio event filter syntax.
* The temporary file is deleted after this action so that no disk overflow
* will occur.
***************************************************************************/
void ctselect::run(void)
{
// Switch screen logging on in debug mode
if (logDebug()) {
log.cout(true);
}
// Get parameters
get_parameters();
// Write parameters into logger
if (logTerse()) {
log_parameters();
log << std::endl;
}
// Write observation(s) into logger
if (logTerse()) {
log << std::endl;
log.header1("Observations before selection");
log << m_obs << std::endl;
}
// Write header
if (logTerse()) {
log << std::endl;
log.header1("Event selection");
}
// Initialise counters
int n_observations = 0;
// Loop over all observation in the container
for (int i = 0; i < m_obs.size(); ++i) {
// Initialise event input and output filenames
m_infiles.push_back("");
// Get CTA observation
GCTAObservation* obs = dynamic_cast(m_obs[i]);
// Continue only if observation is a CTA observation
if (obs != NULL) {
// Write header for observation
if (logTerse()) {
if (obs->name().length() > 1) {
log.header3("Observation "+obs->name());
}
else {
log.header3("Observation");
}
}
// Increment counter
n_observations++;
// Save event file name (for possible saving)
m_infiles[i] = obs->eventfile();
// Get temporary file name
#if G_USE_MKSTEMP
char tpl[] = "ctselectXXXXXX";
int fileid = mkstemp(tpl);
std::string filename(tpl);
#else
std::string filename = std::tmpnam(NULL);
#endif
// Save observation in temporary file
obs->save(filename, true);
// Log saved FITS file
if (logExplicit()) {
GFits tmpfile(filename);
log << std::endl;
log.header1("FITS file content of temporary file");
log << tmpfile << std::endl;
tmpfile.close();
}
// Check temporary file
std::string message = check_infile(filename);
if (message.length() > 0) {
throw GException::app_error(G_RUN, message);
}
// Load observation from temporary file, including event selection
select_events(obs, filename);
// Close temporary file
#if G_USE_MKSTEMP
close(fileid);
#endif
// Remove temporary file
std::remove(filename.c_str());
} // endif: had a CTA observation
} // endfor: looped over all observations
// If more than a single observation has been handled then make sure that
// an XML file will be used for storage
if (n_observations > 1) {
m_use_xml = true;
}
// Write observation(s) into logger
if (logTerse()) {
log << std::endl;
log.header1("Observations after selection");
log << m_obs << std::endl;
}
// Return
return;
}
/***********************************************************************//**
* @brief Save the selected event list(s)
*
* This method saves the selected event list(s) into FITS file(s). There are
* two modes, depending on the m_use_xml flag.
*
* If m_use_xml is true, all selected event list(s) will be saved into FITS
* files, where the output filenames are constructued from the input
* filenames by prepending the m_prefix string to name. Any path information
* will be stripped form the input name, hence event files will be written
* into the local working directory (unless some path information is present
* in the prefix). In addition, an XML file will be created that gathers
* the filename information for the selected event list(s). If an XML file
* was present on input, all metadata information will be copied from this
* input file.
*
* If m_use_xml is false, the selected event list will be saved into a FITS
* file.
***************************************************************************/
void ctselect::save(void)
{
// Write header
if (logTerse()) {
log << std::endl;
if (m_obs.size() > 1) {
log.header1("Save observations");
}
else {
log.header1("Save observation");
}
}
// Case A: Save event file(s) and XML metadata information
if (m_use_xml) {
save_xml();
}
// Case B: Save event file as FITS file
else {
save_fits();
}
// Return
return;
}
/***********************************************************************//**
* @brief Get application parameters
*
* Get all task parameters from parameter file or (if required) by querying
* the user. Times are assumed to be in the native CTA MJD format.
*
* This method also loads observations if no observations are yet allocated.
* Observations are either loaded from a single CTA even list, or from a
* XML file using the metadata information that is stored in that file.
***************************************************************************/
void ctselect::get_parameters(void)
{
// If there are no observations in container then add a single CTA
// observation using the parameters from the parameter file
if (m_obs.size() == 0) {
// Get CTA event list file name
m_infile = (*this)["infile"].filename();
// Allocate CTA observation
GCTAObservation obs;
// Try first to open as FITS file
try {
// Load event list in CTA observation
obs.load(m_infile);
// Append CTA observation to container
m_obs.append(obs);
// Signal that no XML file should be used for storage
m_use_xml = false;
}
// ... otherwise try to open as XML file
catch (GException::fits_open_error &e) {
// Load observations from XML file
m_obs.load(m_infile);
// Signal that XML file should be used for storage
m_use_xml = true;
}
} // endif: there was no observation in the container
// Get parameters
m_usepnt = (*this)["usepnt"].boolean();
if (!m_usepnt) {
m_ra = (*this)["ra"].real();
m_dec = (*this)["dec"].real();
}
m_rad = (*this)["rad"].real();
m_tmin = (*this)["tmin"].real();
m_tmax = (*this)["tmax"].real();
m_emin = (*this)["emin"].real();
m_emax = (*this)["emax"].real();
m_expr = (*this)["expr"].string();
// Optionally read ahead parameters so that they get correctly
// dumped into the log file
if (m_read_ahead) {
m_outfile = (*this)["outfile"].filename();
m_prefix = (*this)["prefix"].string();
}
// Set time interval with input times given in CTA reference
// time (in seconds)
m_timemin.set(m_tmin, m_cta_ref);
m_timemax.set(m_tmax, m_cta_ref);
// Return
return;
}
/***********************************************************************//**
* @brief Select events
*
* @param[in] obs CTA observation.
* @param[in] filename File name.
*
* Select events from a FITS file by making use of the selection possibility
* of the cfitsio library on loading a file. A selection string is created
* from the specified criteria that is appended to the filename so that
* cfitsio will automatically filter the event data. This selection string
* is then applied when opening the FITS file. The opened FITS file is then
* saved into a temporary file which is the loaded into the actual CTA
* observation, overwriting the old CTA observation. The ROI, GTI and EBounds
* of the CTA event list are then set accordingly to the specified selection.
* Finally, the temporary file created during this process is removed.
*
* Good Time Intervals of the observation will be limited to the time
* interval [m_tmin, m_tmax]. If m_tmin=m_tmax=0, no time selection is
* performed.
*
* @todo Use INDEF instead of 0.0 for pointing as RA/DEC selection
***************************************************************************/
void ctselect::select_events(GCTAObservation* obs, const std::string& filename)
{
// Allocate selection string
std::string selection;
char cmin[80];
char cmax[80];
char cra[80];
char cdec[80];
char crad[80];
// Set requested selections
bool select_time = (m_tmin != 0.0 || m_tmax != 0.0);
bool select_roi = ((m_dec != -1.0 && m_ra != -1.0 && m_rad != -1.0) || m_usepnt);
// Set RA/DEC selection
double ra = m_ra;
double dec = m_dec;
if (m_usepnt) {
const GCTAPointing &pnt = obs->pointing();
ra = pnt.dir().ra_deg();
dec = pnt.dir().dec_deg();
}
// Set time selection interval. We make sure here that the time selection
// interval cannot be wider than the GTIs covering the data. This is done
// using GGti's reduce() method.
if (select_time) {
// Reduce GTIs to specified time interval. The complicated cast is
// necessary here because the gti() method is declared const, so
// we're not officially allowed to modify the GTIs.
((GGti*)(&obs->events()->gti()))->reduce(m_timemin, m_timemax);
} // endif: time selection was required
// Save GTI for later usage
GGti gti = obs->events()->gti();
// Make time selection
if (select_time) {
// Extract effective time interval in CTA reference time. We need
// this reference for filtering.
double tmin = gti.tstart().convert(m_cta_ref);
double tmax = gti.tstop().convert(m_cta_ref);
// Format time with sufficient accuracy and add to selection string
sprintf(cmin, "%.8f", tmin);
sprintf(cmax, "%.8f", tmax);
selection = "TIME >= "+std::string(cmin)+" && TIME <= "+std::string(cmax);
if (logTerse()) {
log << gammalib::parformat("Time range");
log << tmin << " - " << tmax << " s" << std::endl;
}
if (selection.length() > 0) {
selection += " && ";
}
}
// Make energy selection
sprintf(cmin, "%.8f", m_emin);
sprintf(cmax, "%.8f", m_emax);
selection += "ENERGY >= "+std::string(cmin)+" && ENERGY <= "+std::string(cmax);
if (logTerse()) {
log << gammalib::parformat("Energy range");
log << m_emin << " - " << m_emax << " TeV" << std::endl;
}
// Make ROI selection
if (select_roi){
if (selection.length() > 0) {
selection += " && ";
}
sprintf(cra, "%.6f", ra);
sprintf(cdec, "%.6f", dec);
sprintf(crad, "%.6f", m_rad);
selection += "ANGSEP("+std::string(cra)+"," +
std::string(cdec)+",RA,DEC) <= " +
std::string(crad);
if (logTerse()) {
log << gammalib::parformat("Acceptance cone centre");
log << "RA=" << ra << ", DEC=" << dec << " deg" << std::endl;
log << gammalib::parformat("Acceptance cone radius");
log << m_rad << " deg" << std::endl;
}
if (logTerse()) {
log << gammalib::parformat("cfitsio selection");
log << selection << std::endl;
}
}
// Add additional expression
if (gammalib::strip_whitespace(m_expr).length() > 0) {
if (selection.length() > 0) {
selection += " && ";
}
selection += "("+gammalib::strip_whitespace(m_expr)+")";
}
// Build input filename including selection expression
std::string expression = filename;
if (selection.length() > 0)
expression += "[EVENTS]["+selection+"]";
if (logTerse()) {
log << gammalib::parformat("FITS filename");
log << expression << std::endl;
}
// Open FITS file
GFits file(expression);
// Log selected FITS file
if (logExplicit()) {
log << std::endl;
log.header1("FITS file content after selection");
log << file << std::endl;
}
// Check if we have an EVENTS HDU
if (!file.contains("EVENTS")) {
std::string message = "No \"EVENTS\" extension found in FITS file "+
expression+".";
throw GException::app_error(G_SELECT_EVENTS, message);
}
// Determine number of events in EVENTS HDU
int nevents = file.table("EVENTS")->nrows();
// If the selected event list is empty then append an empty event list
// to the observation. Otherwise load the data from the temporary file.
if (nevents < 1) {
// Create empty event list
GCTAEventList eventlist;
// Append list to observation
obs->events(eventlist);
}
else {
// Re-read observation from FITS object
obs->read(file);
}
// Get CTA event list pointer
GCTAEventList* list =
static_cast(const_cast(obs->events()));
if (select_roi){
// Set ROI
GCTAInstDir instdir;
instdir.dir().radec_deg(ra, dec);
list->roi(GCTARoi(instdir, m_rad));
}
// Set GTI
list->gti(gti);
// Set energy boundaries
GEbounds ebounds;
GEnergy emin;
GEnergy emax;
emin.TeV(m_emin);
emax.TeV(m_emax);
ebounds.append(emin, emax);
list->ebounds(ebounds);
// Recompute ontime and livetime.
GTime meantime = 0.5 * (gti.tstart() + gti.tstop());
obs->ontime(gti.ontime());
obs->livetime(gti.ontime() * obs->deadc(meantime));
// Return
return;
}
/*==========================================================================
= =
= Private methods =
= =
==========================================================================*/
/***********************************************************************//**
* @brief Initialise class members
***************************************************************************/
void ctselect::init_members(void)
{
// Initialise parameters
m_infile.clear();
m_outfile.clear();
m_prefix.clear();
m_usepnt = false;
m_ra = -1.0;
m_dec = -1.0;
m_rad = -1.0;
m_tmin = 0.0;
m_tmax = 0.0;
m_emin = 0.0;
m_emax = 0.0;
m_expr.clear();
// Initialise protected members
m_obs.clear();
m_infiles.clear();
m_timemin.clear();
m_timemax.clear();
m_use_xml = false;
m_read_ahead = false;
// Set CTA time reference. G_CTA_MJDREF is the CTA reference MJD,
// which is defined in GCTALib.hpp. This is somehow a kluge. We need
// a better mechanism to implement the CTA reference MJD.
m_cta_ref.set(G_CTA_MJDREF, "s", "TT", "LOCAL");
// Set logger properties
log.date(true);
// Return
return;
}
/***********************************************************************//**
* @brief Copy class members
*
* @param[in] app Application.
***************************************************************************/
void ctselect::copy_members(const ctselect& app)
{
// Copy parameters
m_infile = app.m_infile;
m_outfile = app.m_outfile;
m_prefix = app.m_prefix;
m_usepnt = app.m_usepnt;
m_ra = app.m_ra;
m_dec = app.m_dec;
m_rad = app.m_rad;
m_tmin = app.m_tmin;
m_tmax = app.m_tmax;
m_emin = app.m_emin;
m_emax = app.m_emax;
m_expr = app.m_expr;
// Copy protected members
m_obs = app.m_obs;
m_infiles = app.m_infiles;
m_timemin = app.m_timemin;
m_timemax = app.m_timemax;
m_use_xml = app.m_use_xml;
m_read_ahead = app.m_read_ahead;
m_cta_ref = app.m_cta_ref;
// Return
return;
}
/***********************************************************************//**
* @brief Delete class members
***************************************************************************/
void ctselect::free_members(void)
{
// Write separator into logger
if (logTerse()) {
log << std::endl;
}
// Return
return;
}
/***********************************************************************//**
* @brief Check input filename
*
* @param[in] filename File name.
*
* This method checks if the input FITS file is correct.
***************************************************************************/
std::string ctselect::check_infile(const std::string& filename) const
{
// Initialise message string
std::string message = "";
// Open FITS file
GFits file(filename);
// Check for EVENTS HDU
GFitsTable* table = NULL;
try {
// Get pointer to FITS table
table = file.table("EVENTS");
// Initialise list of missing columns
std::vector missing;
// Check for existence of TIME column
if (!table->contains("TIME")) {
missing.push_back("TIME");
}
// Check for existence of ENERGY column
if (!table->contains("ENERGY")) {
missing.push_back("ENERGY");
}
// Check for existence of RA column
if (!table->contains("RA")) {
missing.push_back("RA");
}
// Check for existence of DEC column
if (!table->contains("DEC")) {
missing.push_back("DEC");
}
// Set error message for missing columns
if (missing.size() > 0) {
message = "The following columns are missing in the"
" \"EVENTS\" extension of input file \""
+ m_outfile + "\": ";
for (int i = 0; i < missing.size(); ++i) {
message += "\"" + missing[i] + "\"";
if (i < missing.size()-1) {
message += ", ";
}
}
}
}
catch (GException::fits_hdu_not_found& e) {
message = "No \"EVENTS\" extension found in input file \""
+ m_outfile + "\".";
}
// Return
return message;
}
/***********************************************************************//**
* @brief Set output file name.
*
* @param[in] filename Input file name.
*
* Converts an input filename into an output filename by prepending the
* prefix stored in the member m_prefix to the input filename. Any path will
* be stripped from the input filename. Also a trailing ".gz" will be
* stripped.
***************************************************************************/
std::string ctselect::set_outfile_name(const std::string& filename) const
{
// Split input filename into path elements
std::vector elements = gammalib::split(filename, "/");
// The last path element is the filename
std::string outname = m_prefix + elements[elements.size()-1];
// Strip any ".gz"
outname = gammalib::strip_chars(outname, ".gz");
// Return output filename
return outname;
}
/***********************************************************************//**
* @brief Save event list in FITS format.
*
* Save the event list as a FITS file. The filename of the FITS file is
* specified by the outfile parameter.
***************************************************************************/
void ctselect::save_fits(void)
{
// Get output filename
m_outfile = (*this)["outfile"].filename();
// Get CTA observation from observation container
GCTAObservation* obs = dynamic_cast(m_obs[0]);
// Save event list
save_event_list(obs, m_infiles[0], m_outfile);
// Return
return;
}
/***********************************************************************//**
* @brief Save event list(s) in XML format.
*
* Save the event list(s) into FITS files and write the file path information
* into a XML file. The filename of the XML file is specified by the outfile
* parameter, the filename(s) of the event lists are built by prepending a
* prefix to the input event list filenames. Any path present in the input
* filename will be stripped, i.e. the event list(s) will be written in the
* local working directory (unless a path is specified in the prefix).
***************************************************************************/
void ctselect::save_xml(void)
{
// Get output filename and prefix
m_outfile = (*this)["outfile"].filename();
m_prefix = (*this)["prefix"].string();
// Issue warning if output filename has no .xml suffix
std::string suffix = gammalib::tolower(m_outfile.substr(m_outfile.length()-4,4));
if (suffix != ".xml") {
log << "*** WARNING: Name of observation definition output file \""+
m_outfile+"\"" << std::endl;
log << "*** WARNING: does not terminate with \".xml\"." << std::endl;
log << "*** WARNING: This is not an error, but might be misleading."
" It is recommended" << std::endl;
log << "*** WARNING: to use the suffix \".xml\" for observation"
" definition files." << std::endl;
}
// Loop over all observation in the container
for (int i = 0; i < m_obs.size(); ++i) {
// Get CTA observation
GCTAObservation* obs = dynamic_cast(m_obs[i]);
// Handle only CTA observations
if (obs != NULL) {
// Set event output file name
std::string outfile = set_outfile_name(m_infiles[i]);
// Store output file name in observation
obs->eventfile(outfile);
// Save event list
save_event_list(obs, m_infiles[i], outfile);
} // endif: observation was a CTA observations
} // endfor: looped over observations
// Save observations in XML file
m_obs.save(m_outfile);
// Return
return;
}
/***********************************************************************//**
* @brief Save event list into FITS file
*
* @param[in] obs Pointer to CTA observation.
* @param[in] infile Input file name.
* @param[in] outfile Output file name.
*
* Saves an event list into a FITS file and copy all others extensions from
* the input file to the output file.
***************************************************************************/
void ctselect::save_event_list(const GCTAObservation* obs,
const std::string& infile,
const std::string& outfile) const
{
// Save only if observation is valid
if (obs != NULL) {
// Save observation into FITS file
obs->save(outfile, clobber());
// Copy all extensions other than EVENTS and GTI from the input to
// the output event list. The EVENTS and GTI extensions are written
// by the save method, all others that may eventually be present
// have to be copied by hand.
GFits infits(infile);
GFits outfits(outfile);
for (int extno = 1; extno < infits.size(); ++extno) {
GFitsHDU* hdu = infits.at(extno);
if (hdu->extname() != "EVENTS" && hdu->extname() != "GTI") {
outfits.append(*hdu);
}
}
// Close input file
infits.close();
// Save file to disk and close it (we need both operations)
outfits.save(true);
outfits.close();
} // endif: observation was valid
// Return
return;
}