Iftach Sadeh / ctools

Commit d294f6196a3e55ad3e0f33be978cce540756a86e

Authored by Jürgen Knödlseder
1 parent c20e2f08

Add csworkflow script as prove of principle for a XML file-based analysis workflow.

Inline Side-by-side
Showing 2 changed files with 549 additions and 0 deletions
cscripts/csworkflow.py 0 → 100755
  View file @d294f61
  1 +#! /usr/bin/env python
  2 +# ==========================================================================
  3 +# This script executes a workflow defined in an XML file.
  4 +#
  5 +# Copyright (C) 2016 Juergen Knoedlseder
  6 +#
  7 +# This program is free software: you can redistribute it and/or modify
  8 +# it under the terms of the GNU General Public License as published by
  9 +# the Free Software Foundation, either version 3 of the License, or
  10 +# (at your option) any later version.
  11 +#
  12 +# This program is distributed in the hope that it will be useful,
  13 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15 +# GNU General Public License for more details.
  16 +#
  17 +# You should have received a copy of the GNU General Public License
  18 +# along with this program. If not, see <http://www.gnu.org/licenses/>.
  19 +#
  20 +# ==========================================================================
  21 +import gammalib
  22 +import ctools
  23 +import cscripts
  24 +import sys
  25 +
  26 +
  27 +# ================ #
  28 +# csworkflow class #
  29 +# ================ #
  30 +class csworkflow(ctools.cscript):
  31 + """
  32 + """
  33 + def __init__(self, *argv):
  34 + """
  35 + Constructor.
  36 + """
  37 + # Set name and version
  38 + self.name = "csworkflow"
  39 + self.version = "1.0.0"
  40 +
  41 + # Set members
  42 + self.workflow = gammalib.GXml()
  43 + self.actors = []
  44 +
  45 + # Make sure that parfile exists
  46 + self.parfile()
  47 +
  48 + # Initialise application
  49 + if len(argv) == 0:
  50 + ctools.cscript.__init__(self, self.name, self.version)
  51 + elif len(argv) ==1:
  52 + ctools.cscript.__init__(self, self.name, self.version, *argv)
  53 + else:
  54 + raise TypeError("Invalid number of arguments given.")
  55 +
  56 + # Set logger properties
  57 + self.log_header()
  58 + self.log.date(True)
  59 +
  60 + # Return
  61 + return
  62 +
  63 + def __del__(self):
  64 + """
  65 + Destructor.
  66 + """
  67 + # Return
  68 + return
  69 +
  70 + def parfile(self):
  71 + """
  72 + Check if parfile exists. If parfile does not exist then create a
  73 + default parfile. This kluge avoids shipping the cscript with a parfile.
  74 + """
  75 + # Set parfile name
  76 + parfile = self.name+".par"
  77 +
  78 + # Load parfile or create one
  79 + try:
  80 + pars = gammalib.GApplicationPars(parfile)
  81 + except:
  82 + # Signal if parfile was not found
  83 + print("Parfile "+parfile+" not found. Create default parfile.")
  84 +
  85 + # Create default parfile
  86 + pars = gammalib.GApplicationPars()
  87 + pars.append(gammalib.GApplicationPar("inflow","f","h","test/data/workflow.xml","","","Input workflow XML file"))
  88 + pars.append_standard()
  89 + pars.append(gammalib.GApplicationPar("logfile","f","h","csworkflow.log","","","Log filename"))
  90 + pars.save(parfile)
  91 +
  92 + # Return
  93 + return
  94 +
  95 + def get_parameters(self):
  96 + """
  97 + Get parameters from parfile.
  98 + """
  99 + # Load workflow XML file
  100 + xmlfile = self["inflow"].filename()
  101 + self.workflow = gammalib.GXml(xmlfile.url())
  102 +
  103 + # Return
  104 + return
  105 +
  106 + def execute(self):
  107 + """
  108 + Execute the script.
  109 + """
  110 + # Run the script
  111 + self.run()
  112 +
  113 + # Return
  114 + return
  115 +
  116 + def run(self):
  117 + """
  118 + Run the script.
  119 + """
  120 + # Switch screen logging on in debug mode
  121 + if self.logDebug():
  122 + self.log.cout(True)
  123 +
  124 + # Get parameters
  125 + self.get_parameters()
  126 +
  127 + # Write input parameters into logger
  128 + if self.logTerse():
  129 + self.log_parameters()
  130 + self.log("\n")
  131 +
  132 + # Parse XML file
  133 + if self.logTerse():
  134 + self.log("\n")
  135 + self.log.header1("Parse workflow XML file")
  136 + self.parse_workflow()
  137 +
  138 + # Execute workflow
  139 + if self.logTerse():
  140 + self.log("\n")
  141 + self.log.header1("Execute workflow")
  142 + self.execute_workflow()
  143 +
  144 + # Return
  145 + return
  146 +
  147 + def parse_workflow(self):
  148 + """
  149 + """
  150 + # Get workflow
  151 + workflow = self.workflow.element("workflow")
  152 +
  153 + # Get number of actors
  154 + num_actors = workflow.elements("actor")
  155 +
  156 + # Initialise actors
  157 + self.actors = []
  158 +
  159 + # Loop over all actors
  160 + for i in range(num_actors):
  161 +
  162 + # Get actor
  163 + actor = workflow.element("actor", i)
  164 +
  165 + # Initialise parameters and input actors
  166 + input_parameters = []
  167 + output_parameters = []
  168 + input_actors = []
  169 + output_actors = []
  170 +
  171 + # Get actor attributes
  172 + name = actor.attribute("name")
  173 + tool = actor.attribute("tool")
  174 +
  175 + # Get actor input parameters
  176 + if actor.elements("input") > 0:
  177 + actor_inputs = actor.element("input")
  178 + num_inputs = actor_inputs.elements("parameter")
  179 + for k in range(num_inputs):
  180 + input = actor_inputs.element("parameter", k)
  181 + input_name = input.attribute("name")
  182 + input_value = input.attribute("value")
  183 + input_actor = input.attribute("actor")
  184 + parameter = {'name': input_name, \
  185 + 'value': input_value, \
  186 + 'actor': input_actor}
  187 + input_parameters.append(parameter)
  188 + if input_actor != "":
  189 + if input_actor not in input_actors:
  190 + input_actors.append(input_actor)
  191 +
  192 + # Get actor output parameters
  193 + if actor.elements("output") > 0:
  194 + actor_output = actor.element("output")
  195 + num_outputs = actor_output.elements("parameter")
  196 + for k in range(num_outputs):
  197 + output = actor_output.element("parameter", k)
  198 + output_name = output.attribute("name")
  199 + output_value = output.attribute("value")
  200 + output_actor = output.attribute("actor")
  201 + parameter = {'name': output_name, \
  202 + 'value': output_value, \
  203 + 'actor': output_actor}
  204 + output_parameters.append(parameter)
  205 + if output_actor != "":
  206 + if output_actor not in output_actors:
  207 + output_actors.append(output_actor)
  208 +
  209 + # Determine number of dependencies
  210 + num_inputs = len(input_actors)
  211 +
  212 + # Set actor status
  213 + if num_inputs > 0:
  214 + status = 'waiting for input'
  215 + else:
  216 + status = 'ready'
  217 +
  218 + # Create actor entry
  219 + entry = {'name': name,
  220 + 'tool': tool,
  221 + 'input_parameters': input_parameters,
  222 + 'input_actors': input_actors,
  223 + 'output_parameters': output_parameters,
  224 + 'output_actors': output_actors,
  225 + 'status': status}
  226 +
  227 + # Append entry
  228 + self.actors.append(entry)
  229 +
  230 + # Log information about actors
  231 + if self.logNormal():
  232 + self.log.parformat("Actor \""+name+"\"")
  233 + self.log(tool)
  234 + self.log("\n")
  235 + if num_inputs == 0:
  236 + self.log.parformat(" Predecessor")
  237 + self.log("none\n")
  238 + else:
  239 + if num_inputs == 1:
  240 + self.log.parformat(" Predecessor")
  241 + else:
  242 + self.log.parformat(" Predecessors")
  243 + for k in range(num_inputs):
  244 + if k > 0:
  245 + self.log(", ")
  246 + self.log("\""+input_actors[k]+"\"")
  247 + self.log("\n")
  248 +
  249 + # Return
  250 + return
  251 +
  252 + def execute_workflow(self):
  253 + """
  254 + """
  255 + # Continue while there are actors
  256 + while len(self.actors) > 0:
  257 +
  258 + # Find actors which are ready
  259 + num_executed = 0
  260 + for actor in self.actors:
  261 +
  262 + # If actor is ready then execute it
  263 + if actor['status'] == 'ready':
  264 +
  265 + # Log execution start
  266 + if self.logNormal():
  267 + self.log.parformat("Execute actor")
  268 + self.log("\""+actor['name']+"\"\n")
  269 +
  270 + # Execute actor
  271 + self.execute_actor(actor)
  272 +
  273 + # Log execution finish
  274 + if self.logNormal():
  275 + self.log.parformat("Finished actor execution")
  276 + self.log("\""+actor['name']+"\"\n")
  277 +
  278 + # Set actor status to finished
  279 + actor['status'] = 'finished'
  280 +
  281 + # Increment number of executed actors
  282 + num_executed += 1
  283 +
  284 + # Break if no actors have been executed
  285 + if num_executed < 1:
  286 + break
  287 +
  288 + # Update actor status
  289 + for actor in self.actors:
  290 + if actor['status'] != 'finished':
  291 + input_actors = actor['input_actors']
  292 + ready = True
  293 + for input_actor in input_actors:
  294 + a = self.get_actor(input_actor)
  295 + if a['status'] != 'finished':
  296 + ready = False
  297 + break
  298 + if ready:
  299 + actor['status'] = 'ready'
  300 +
  301 + # Return
  302 + return
  303 +
  304 + def execute_actor(self, actor):
  305 + """
  306 + Excute an actor.
  307 + """
  308 + # Log input parameters
  309 + if self.logNormal():
  310 + pars = actor['input_parameters']
  311 + for par in pars:
  312 + self.log.parformat(" Input parameter")
  313 + self.log(par['name'])
  314 + self.log("=")
  315 + self.log(self.get_parameter_value(par))
  316 + self.log("\n")
  317 +
  318 + # Log output parameters
  319 + if self.logNormal():
  320 + pars = actor['output_parameters']
  321 + for par in pars:
  322 + self.log.parformat(" Output parameter")
  323 + self.log(par['name'])
  324 + self.log("=")
  325 + self.log(self.get_parameter_value(par))
  326 + self.log("\n")
  327 +
  328 + # Set actor tool
  329 + if 'tool' in actor:
  330 + tool = actor['tool']
  331 + if tool[:2] == 'ct':
  332 + tool_eval = 'ctools.'+tool+'()'
  333 + elif tool[:2] == 'cs':
  334 + tool_eval = 'cscripts.'+tool+'()'
  335 + else:
  336 + tool_eval = ""
  337 + if tool_eval == "":
  338 + return
  339 +
  340 + # Create actor tool object
  341 + object = eval(tool_eval)
  342 +
  343 + # Set actor input parameters
  344 + pars = actor['input_parameters']
  345 + for par in pars:
  346 + app_par = object[par['name']]
  347 + app_par.value(self.get_parameter_value(par))
  348 + object[par['name']] = app_par
  349 +
  350 + # Set actor output parameters
  351 + pars = actor['output_parameters']
  352 + for par in pars:
  353 + app_par = object[par['name']]
  354 + app_par.value(self.get_parameter_value(par))
  355 + object[par['name']] = app_par
  356 +
  357 + # Execute actor
  358 + object.logFileOpen()
  359 + object.execute()
  360 +
  361 + # Return
  362 + return
  363 +
  364 + def get_parameter_value(self, par):
  365 + """
  366 + Get the parameter value from the respective actor.
  367 + """
  368 + # If the actor is empty then simply return the value
  369 + if (par['actor'] == ""):
  370 + value = par['value']
  371 +
  372 + # ... otherwise get the value from the actor
  373 + else:
  374 + found = False
  375 + actor = self.get_actor(par['actor'])
  376 + for output in actor['output_parameters']:
  377 + if output['name'] == par['value']:
  378 + value = output['value']
  379 + found = True
  380 + break
  381 + if not found:
  382 + msg = "Parameter \""+par['value']+"\" is not an output "\
  383 + "parameter of actor \""+par['actor']+"\"."
  384 + raise RuntimeError(msg)
  385 +
  386 + # Return value
  387 + return value
  388 +
  389 + def get_actor(self, name):
  390 + """
  391 + Get the actor with the specified name.
  392 + """
  393 + # Return actor with specified name
  394 + for actor in self.actors:
  395 + if actor['name'] == name:
  396 + return actor
  397 +
  398 + # Throw an exception
  399 + msg = "Actor \""+name+"\" not found."
  400 + raise RuntimeError(msg)
  401 +
  402 + # Return
  403 + return
  404 +
  405 +
  406 +# ======================== #
  407 +# Main routine entry point #
  408 +# ======================== #
  409 +if __name__ == '__main__':
  410 + """
  411 + Executes workflow.
  412 + """
  413 + # Create instance of application
  414 + app = csworkflow(sys.argv)
  415 +
  416 + # Open logfile
  417 + app.logFileOpen()
  418 +
  419 + # Execute application
  420 + app.execute()
test/data/workflow.xml 0 → 100644
  View file @d294f61
  1 +<?xml version="1.0" encoding="UTF-8" standalone="no"?>
  2 +<workflow>
  3 + <actor name="input">
  4 + <output>
  5 + <parameter name="inmodel" value="$CTOOLS/share/models/crab.xml" />
  6 + <parameter name="caldb" value="prod2" />
  7 + <parameter name="irf" value="South_50h" />
  8 + <parameter name="edisp" value="no" />
  9 + <parameter name="deadc" value="0.95" />
  10 + <parameter name="ra" value="83.63" />
  11 + <parameter name="dec" value="22.01" />
  12 + <parameter name="target" value="Crab" />
  13 + <parameter name="emin" value="0.1" />
  14 + <parameter name="emax" value="100.0" />
  15 + <parameter name="tmin" value="0.0" />
  16 + <parameter name="tmax" value="1800.0" />
  17 + <parameter name="chatter" value="2" />
  18 + <parameter name="clobber" value="yes" />
  19 + <parameter name="debug" value="no" />
  20 + <parameter name="mode" value="ql" />
  21 + </output>
  22 + </actor>
  23 + <actor name="ctobssim" tool="ctobssim">
  24 + <input>
  25 + <parameter name="inobs" value="NONE" />
  26 + <parameter name="inmodel" value="inmodel" actor="input" />
  27 + <parameter name="caldb" value="caldb" actor="input" />
  28 + <parameter name="irf" value="irf" actor="input" />
  29 + <parameter name="edisp" value="edisp" actor="input" />
  30 + <parameter name="prefix" value="sim_events_" />
  31 + <parameter name="seed" value="1" />
  32 + <parameter name="ra" value="ra" actor="input" />
  33 + <parameter name="dec" value="dec" actor="input" />
  34 + <parameter name="rad" value="5.0" />
  35 + <parameter name="tmin" value="tmin" actor="input" />
  36 + <parameter name="tmax" value="tmax" actor="input" />
  37 + <parameter name="emin" value="emin" actor="input" />
  38 + <parameter name="emax" value="emax" actor="input" />
  39 + <parameter name="deadc" value="deadc" actor="input" />
  40 + <parameter name="maxrate" value="1.0e6" />
  41 + <parameter name="eslices" value="10" />
  42 + <parameter name="chatter" value="chatter" actor="input" />
  43 + <parameter name="clobber" value="clobber" actor="input" />
  44 + <parameter name="debug" value="debug" actor="input" />
  45 + <parameter name="mode" value="mode" actor="input" />
  46 + <parameter name="logfile" value="ctobssim.log" />
  47 + </input>
  48 + <output>
  49 + <parameter name="outevents" value="events.fits" />
  50 + </output>
  51 + </actor>
  52 + <actor name="ctselect" tool="ctselect">
  53 + <input>
  54 + <parameter name="inobs" value="outevents" actor="ctobssim" />
  55 + <parameter name="prefix" value="selected_" />
  56 + <parameter name="usepnt" value="no" />
  57 + <parameter name="ra" value="ra" actor="input" />
  58 + <parameter name="dec" value="dec" actor="input" />
  59 + <parameter name="rad" value="3.0" />
  60 + <parameter name="tmin" value="tmin" actor="input" />
  61 + <parameter name="tmax" value="tmax" actor="input" />
  62 + <parameter name="emin" value="emin" actor="input" />
  63 + <parameter name="emax" value="emax" actor="input" />
  64 + <parameter name="expr" value="" />
  65 + <parameter name="usethres" value="NONE" />
  66 + <parameter name="chatter" value="chatter" actor="input" />
  67 + <parameter name="clobber" value="clobber" actor="input" />
  68 + <parameter name="debug" value="debug" actor="input" />
  69 + <parameter name="mode" value="mode" actor="input" />
  70 + <parameter name="logfile" value="ctselect.log" />
  71 + </input>
  72 + <output>
  73 + <parameter name="outobs" value="selected_events.fits" />
  74 + </output>
  75 + </actor>
  76 + <actor name="ctlike" tool="ctlike">
  77 + <input>
  78 + <parameter name="inobs" value="outobs" actor="ctselect" />
  79 + <parameter name="inmodel" value="inmodel" actor="input" />
  80 + <parameter name="expcube" value="NONE" />
  81 + <parameter name="psfcube" value="NONE" />
  82 + <parameter name="bkgcube" value="NONE" />
  83 + <parameter name="caldb" value="caldb" actor="input" />
  84 + <parameter name="irf" value="irf" actor="input" />
  85 + <parameter name="edisp" value="edisp" actor="input" />
  86 + <parameter name="stat" value="POISSON" />
  87 + <parameter name="edisp" value="no" />
  88 + <parameter name="refit" value="no" />
  89 + <parameter name="chatter" value="chatter" actor="input" />
  90 + <parameter name="clobber" value="clobber" actor="input" />
  91 + <parameter name="debug" value="debug" actor="input" />
  92 + <parameter name="mode" value="mode" actor="input" />
  93 + <parameter name="logfile" value="ctlike.log" />
  94 + </input>
  95 + <output>
  96 + <parameter name="outmodel" value="crab_results.xml" />
  97 + </output>
  98 + </actor>
  99 + <actor name="csspec" tool="csspec">
  100 + <input>
  101 + <parameter name="inobs" value="outobs" actor="ctselect" />
  102 + <parameter name="inmodel" value="outmodel" actor="ctlike" />
  103 + <parameter name="srcname" value="target" actor="input" />
  104 + <parameter name="expcube" value="NONE" />
  105 + <parameter name="psfcube" value="NONE" />
  106 + <parameter name="bkgcube" value="NONE" />
  107 + <parameter name="caldb" value="caldb" actor="input" />
  108 + <parameter name="irf" value="irf" actor="input" />
  109 + <parameter name="edisp" value="edisp" actor="input" />
  110 + <parameter name="emin" value="emin" actor="input" />
  111 + <parameter name="emax" value="emax" actor="input" />
  112 + <parameter name="enumbins" value="10" />
  113 + <parameter name="ebinalg" value="LOG" />
  114 + <parameter name="ebinfile" value="NONE" />
  115 + <parameter name="calc_ts" value="yes" />
  116 + <parameter name="calc_ulim" value="yes" />
  117 + <parameter name="fix_srcs" value="yes" />
  118 + <parameter name="fix_bkg" value="no" />
  119 + <parameter name="chatter" value="chatter" actor="input" />
  120 + <parameter name="clobber" value="clobber" actor="input" />
  121 + <parameter name="debug" value="debug" actor="input" />
  122 + <parameter name="mode" value="mode" actor="input" />
  123 + <parameter name="logfile" value="csspec.log" />
  124 + </input>
  125 + <output>
  126 + <parameter name="outfile" value="spectrum.fits" />
  127 + </output>
  128 + </actor>
  129 +</workflow>