Karim Sabri / ctools

Commit 698bf587ea4dff6858d72e8620ea93926380c286

Authored by Jürgen Knödlseder
1 parent a6fc9fa3

Integrate csworkflow script and add unit test.

Inline Side-by-side
Showing 8 changed files with 407 additions and 246 deletions
cscripts/Makefile.am
  View file @698bf58
@@ -2,26 +2,70 @@ @@ -2,26 +2,70 @@
2 2
3 # Scripts to be distributed in $(pyexecdir)/cscripts 3 # Scripts to be distributed in $(pyexecdir)/cscripts
4 pythonscripts = __init__.py \ 4 pythonscripts = __init__.py \
5 - cspull.py cssens.py cstsdist.py cscaldb.py csresmap.py \  
6 - csobsdef.py csspec.py cslightcrv.py csobsinfo.py \  
7 - csinfo.py csmodelinfo.py csmodelmerge.py cstsmapmerge.py \  
8 - csroot2caldb.py csobs2caldb.py \  
9 - csfindobs.py csiactcopy.py csiactdata.py csiactobs.py \ 5 + cscaldb.py \
  6 + csinfo.py \
  7 + cslightcrv.py \
  8 + csmodelinfo.py \
  9 + csmodelmerge.py \
  10 + csobs2caldb.py \
  11 + csobsdef.py \
  12 + csobsinfo.py \
  13 + cspull.py \
  14 + csresmap.py \
  15 + csroot2caldb.py \
  16 + cssens.py \
  17 + csspec.py \
  18 + cstsdist.py \
  19 + cstsmapmerge.py \
  20 + csworkflow.py \
  21 + csfindobs.py \
  22 + csiactcopy.py \
  23 + csiactdata.py \
  24 + csiactobs.py \
10 obsutils.py 25 obsutils.py
11 26
12 # Parfiles to be distributed in $(prefix)/syspfiles 27 # Parfiles to be distributed in $(prefix)/syspfiles
13 -parfiles = cspull.par cssens.par cstsdist.par cscaldb.par csresmap.par \  
14 - csobsdef.par csspec.par cslightcrv.par csobsinfo.par \  
15 - csmodelinfo.par csmodelmerge.par cstsmapmerge.par \  
16 - csroot2caldb.par csobs2caldb.par \  
17 - csfindobs.par csiactcopy.par csiactdata.par csiactobs.par 28 +parfiles = cscaldb.par \
  29 + cslightcrv.par \
  30 + csmodelinfo.par \
  31 + csmodelmerge.par \
  32 + csobs2caldb.par \
  33 + csobsdef.par \
  34 + csobsinfo.par \
  35 + cspull.par \
  36 + csresmap.par \
  37 + csroot2caldb.par \
  38 + cssens.par \
  39 + csspec.par \
  40 + cstsdist.par \
  41 + cstsmapmerge.par \
  42 + csworkflow.par \
  43 + csfindobs.par \
  44 + csiactcopy.par \
  45 + csiactdata.par \
  46 + csiactobs.par
18 47
19 # Scripts for which symbolic links will be created in bindir 48 # Scripts for which symbolic links will be created in bindir
20 -linkscripts = cspull cssens cstsdist cscaldb csresmap \  
21 - csobsdef csspec cslightcrv csobsinfo \  
22 - csinfo csmodelinfo csmodelmerge cstsmapmerge \  
23 - csroot2caldb csobs2caldb \  
24 - csfindobs csiactcopy csiactdata csiactobs 49 +linkscripts = cscaldb \
  50 + csinfo \
  51 + cslightcrv \
  52 + csmodelinfo \
  53 + csmodelmerge \
  54 + csobs2caldb \
  55 + csobsdef \
  56 + csobsinfo \
  57 + cspull \
  58 + csresmap \
  59 + csroot2caldb \
  60 + cssens \
  61 + csspec \
  62 + cstsdist \
  63 + cstsmapmerge \
  64 + csworkflow \
  65 + csfindobs \
  66 + csiactcopy \
  67 + csiactdata \
  68 + csiactobs
25 69
26 # Test scripts to be distributed in $(pyexecdir)/cscripts/tests 70 # Test scripts to be distributed in $(pyexecdir)/cscripts/tests
27 testscripts = tests/__init__.py \ 71 testscripts = tests/__init__.py \
@@ -35,6 +79,7 @@ testscripts = tests/__init__.py \ @@ -35,6 +79,7 @@ testscripts = tests/__init__.py \
35 ../test/test_csobsinfo.py \ 79 ../test/test_csobsinfo.py \
36 ../test/test_cspull.py \ 80 ../test/test_cspull.py \
37 ../test/test_csroot2caldb.py \ 81 ../test/test_csroot2caldb.py \
  82 + ../test/test_csworkflow.py \
38 ../test/test_csfindobs.py \ 83 ../test/test_csfindobs.py \
39 ../test/test_csiactcopy.py \ 84 ../test/test_csiactcopy.py \
40 ../test/test_csiactdata.py \ 85 ../test/test_csiactdata.py \
@@ -55,7 +100,8 @@ testdata = ../test/data/crab.xml \ @@ -55,7 +100,8 @@ testdata = ../test/data/crab.xml \
55 ../test/data/model_cube_background.xml \ 100 ../test/data/model_cube_background.xml \
56 ../test/data/model_cube_background2.xml \ 101 ../test/data/model_cube_background2.xml \
57 ../test/data/crab_bkgcube.xml \ 102 ../test/data/crab_bkgcube.xml \
58 - ../test/data/irf.root 103 + ../test/data/irf.root \
  104 + ../test/data/workflow.xml
59 105
60 # Test data to be distributed in $(pyexecdir)/cscripts/tests/iactdata 106 # Test data to be distributed in $(pyexecdir)/cscripts/tests/iactdata
61 testiactdata = ../test/iactdata/hdu-index.fits.gz \ 107 testiactdata = ../test/iactdata/hdu-index.fits.gz \
cscripts/__init__.py.in
  View file @698bf58
@@ -39,7 +39,7 @@ __all__ = [ @@ -39,7 +39,7 @@ __all__ = [
39 "csspec", 39 "csspec",
40 "cstsdist", 40 "cstsdist",
41 "cstsmapmerge", 41 "cstsmapmerge",
42 - "csfindobs", 42 + "csworkflow",
43 "obsutils" 43 "obsutils"
44 ] 44 ]
45 if cur_version > req_version: 45 if cur_version > req_version:
@@ -64,6 +64,7 @@ from cscripts.cssens import cssens @@ -64,6 +64,7 @@ from cscripts.cssens import cssens
64 from cscripts.csspec import csspec 64 from cscripts.csspec import csspec
65 from cscripts.cstsdist import cstsdist 65 from cscripts.cstsdist import cstsdist
66 from cscripts.cstsmapmerge import cstsmapmerge 66 from cscripts.cstsmapmerge import cstsmapmerge
  67 +from cscripts.csworkflow import csworkflow
67 from cscripts import obsutils 68 from cscripts import obsutils
68 if cur_version > req_version: 69 if cur_version > req_version:
69 from cscripts.csfindobs import csfindobs 70 from cscripts.csfindobs import csfindobs
cscripts/cscaldb.py
  View file @698bf58
@@ -206,16 +206,6 @@ class cscaldb(ctools.cscript): @@ -206,16 +206,6 @@ class cscaldb(ctools.cscript):
206 """ 206 """
207 Execute the script. 207 Execute the script.
208 """ 208 """
209 - # Run the script  
210 - self.run()  
211 -  
212 - # Return  
213 - return  
214 -  
215 - def execute(self):  
216 - """  
217 - Execute the script.  
218 - """  
219 # Open logfile 209 # Open logfile
220 self.logFileOpen() 210 self.logFileOpen()
221 211
cscripts/csworkflow.par 0 → 100644
  View file @698bf58
  1 +#############################################################################
  2 +# #
  3 +# CTA Science Analysis Tools #
  4 +# #
  5 +# ------------------------------------------------------------------------- #
  6 +# #
  7 +# File: csworkflow.par #
  8 +# Version: 1.0.0 #
  9 +# #
  10 +# Author: Juergen Knoedlseder #
  11 +# Jurgen.Knodlseder@irap.omp.eu #
  12 +# IRAP #
  13 +# #
  14 +# Purpose: Parameter file for the csworkflow script #
  15 +# #
  16 +#############################################################################
  17 +#
  18 +# Input parameters
  19 +#=================
  20 +inflow, f, h, workflow.xml,,, "Input workflow XML file"
  21 +
  22 +#
  23 +# Standard parameters
  24 +#====================
  25 +chatter, i, h, 2,0,4, "Chattiness of output"
  26 +clobber, b, h, yes,,, "Overwrite existing output files with new output files?"
  27 +debug, b, h, no,,, "Debugging mode activated"
  28 +mode, s, h, ql,,, "Mode of automatic parameters"
  29 +logfile, f, h, csworkflow.log,,, "Log filename"
cscripts/csworkflow.py
  View file @698bf58
1 #! /usr/bin/env python 1 #! /usr/bin/env python
2 # ========================================================================== 2 # ==========================================================================
3 -# This script executes a workflow defined in an XML file. 3 +# Executes analysis workflow
4 # 4 #
5 # Copyright (C) 2016 Juergen Knoedlseder 5 # Copyright (C) 2016 Juergen Knoedlseder
6 # 6 #
@@ -29,138 +29,77 @@ import sys @@ -29,138 +29,77 @@ import sys
29 # ================ # 29 # ================ #
30 class csworkflow(ctools.cscript): 30 class csworkflow(ctools.cscript):
31 """ 31 """
  32 + Executes an analysis workflow.
  33 +
  34 + The ``csworkflow`` script executes an analysis workflow defined in an
  35 + XML file.
32 """ 36 """
  37 +
  38 + # Constructor
33 def __init__(self, *argv): 39 def __init__(self, *argv):
34 """ 40 """
35 Constructor. 41 Constructor.
  42 +
  43 + Parameters
  44 + ----------
  45 + argv : list of str
  46 + List of IRAF command line parameter strings of the form
  47 + ``parameter=3``.
  48 +
  49 + Raises
  50 + ------
  51 + TypeError
  52 + An invalid number of command line arguments was provided.
36 """ 53 """
37 # Set name and version 54 # Set name and version
38 - self.name = "csworkflow"  
39 - self.version = "1.0.0" 55 + self._name = 'csworkflow'
  56 + self._version = '1.0.0'
40 57
41 # Set members 58 # Set members
42 - self.workflow = gammalib.GXml()  
43 - self.actors = []  
44 -  
45 - # Make sure that parfile exists  
46 - self.parfile()  
47 -  
48 - # Initialise application  
49 - if len(argv) == 0:  
50 - ctools.cscript.__init__(self, self.name, self.version)  
51 - elif len(argv) ==1:  
52 - ctools.cscript.__init__(self, self.name, self.version, *argv)  
53 - else:  
54 - raise TypeError("Invalid number of arguments given.") 59 + self._workflow = gammalib.GXml()
  60 + self._actors = []
55 61
56 - # Set logger properties  
57 - self.log_header()  
58 - self.log.date(True) 62 + # Initialise application by calling the appropriate class constructor
  63 + self._init_cscript(argv)
59 64
60 # Return 65 # Return
61 return 66 return
62 67
63 - def __del__(self):  
64 - """  
65 - Destructor.  
66 - """  
67 - # Return  
68 - return  
69 68
70 - def parfile(self):  
71 - """  
72 - Check if parfile exists. If parfile does not exist then create a  
73 - default parfile. This kluge avoids shipping the cscript with a parfile.  
74 - """  
75 - # Set parfile name  
76 - parfile = self.name+".par"  
77 -  
78 - # Load parfile or create one  
79 - try:  
80 - pars = gammalib.GApplicationPars(parfile)  
81 - except:  
82 - # Signal if parfile was not found  
83 - print("Parfile "+parfile+" not found. Create default parfile.")  
84 -  
85 - # Create default parfile  
86 - pars = gammalib.GApplicationPars()  
87 - pars.append(gammalib.GApplicationPar("inflow","f","h","test/data/workflow.xml","","","Input workflow XML file"))  
88 - pars.append_standard()  
89 - pars.append(gammalib.GApplicationPar("logfile","f","h","csworkflow.log","","","Log filename"))  
90 - pars.save(parfile)  
91 -  
92 - # Return  
93 - return 69 + # Private methods
  70 + #
  71 + # Query all user parameters
  72 + def _get_parameters(self):
94 73
95 - def get_parameters(self):  
96 - """  
97 - Get parameters from parfile.  
98 - """  
99 # Load workflow XML file 74 # Load workflow XML file
100 - xmlfile = self["inflow"].filename()  
101 - self.workflow = gammalib.GXml(xmlfile.url())  
102 -  
103 - # Return  
104 - return  
105 -  
106 - def execute(self):  
107 - """  
108 - Execute the script.  
109 - """  
110 - # Run the script  
111 - self.run()  
112 -  
113 - # Return  
114 - return  
115 -  
116 - def run(self):  
117 - """  
118 - Run the script.  
119 - """  
120 - # Switch screen logging on in debug mode  
121 - if self.logDebug():  
122 - self.log.cout(True)  
123 -  
124 - # Get parameters  
125 - self.get_parameters() 75 + xmlfile = self['inflow'].filename()
  76 + self._workflow = gammalib.GXml(xmlfile.url())
126 77
127 # Write input parameters into logger 78 # Write input parameters into logger
128 - if self.logTerse():  
129 - self.log_parameters()  
130 - self.log("\n")  
131 -  
132 - # Parse XML file  
133 - if self.logTerse():  
134 - self.log("\n")  
135 - self.log.header1("Parse workflow XML file")  
136 - self.parse_workflow()  
137 -  
138 - # Execute workflow  
139 - if self.logTerse():  
140 - self.log("\n")  
141 - self.log.header1("Execute workflow")  
142 - self.execute_workflow() 79 + if self._logTerse():
  80 + self._log_parameters()
  81 + self._log('\n')
143 82
144 # Return 83 # Return
145 return 84 return
146 85
147 - def parse_workflow(self):  
148 - """  
149 - """ 86 + # Parse the workflow XML file
  87 + def _parse_workflow(self):
  88 +
150 # Get workflow 89 # Get workflow
151 - workflow = self.workflow.element("workflow") 90 + workflow = self._workflow.element('workflow')
152 91
153 # Get number of actors 92 # Get number of actors
154 - num_actors = workflow.elements("actor") 93 + num_actors = workflow.elements('actor')
155 94
156 # Initialise actors 95 # Initialise actors
157 - self.actors = [] 96 + self._actors = []
158 97
159 # Loop over all actors 98 # Loop over all actors
160 for i in range(num_actors): 99 for i in range(num_actors):
161 100
162 # Get actor 101 # Get actor
163 - actor = workflow.element("actor", i) 102 + actor = workflow.element('actor', i)
164 103
165 # Initialise parameters and input actors 104 # Initialise parameters and input actors
166 input_parameters = [] 105 input_parameters = []
@@ -169,40 +108,40 @@ class csworkflow(ctools.cscript): @@ -169,40 +108,40 @@ class csworkflow(ctools.cscript):
169 output_actors = [] 108 output_actors = []
170 109
171 # Get actor attributes 110 # Get actor attributes
172 - name = actor.attribute("name")  
173 - tool = actor.attribute("tool") 111 + name = actor.attribute('name')
  112 + tool = actor.attribute('tool')
174 113
175 # Get actor input parameters 114 # Get actor input parameters
176 - if actor.elements("input") > 0:  
177 - actor_inputs = actor.element("input")  
178 - num_inputs = actor_inputs.elements("parameter") 115 + if actor.elements('input') > 0:
  116 + actor_inputs = actor.element('input')
  117 + num_inputs = actor_inputs.elements('parameter')
179 for k in range(num_inputs): 118 for k in range(num_inputs):
180 - input = actor_inputs.element("parameter", k)  
181 - input_name = input.attribute("name")  
182 - input_value = input.attribute("value")  
183 - input_actor = input.attribute("actor") 119 + input = actor_inputs.element('parameter', k)
  120 + input_name = input.attribute('name')
  121 + input_value = input.attribute('value')
  122 + input_actor = input.attribute('actor')
184 parameter = {'name': input_name, \ 123 parameter = {'name': input_name, \
185 'value': input_value, \ 124 'value': input_value, \
186 'actor': input_actor} 125 'actor': input_actor}
187 input_parameters.append(parameter) 126 input_parameters.append(parameter)
188 - if input_actor != "": 127 + if input_actor != '':
189 if input_actor not in input_actors: 128 if input_actor not in input_actors:
190 input_actors.append(input_actor) 129 input_actors.append(input_actor)
191 130
192 # Get actor output parameters 131 # Get actor output parameters
193 - if actor.elements("output") > 0:  
194 - actor_output = actor.element("output")  
195 - num_outputs = actor_output.elements("parameter") 132 + if actor.elements('output') > 0:
  133 + actor_output = actor.element('output')
  134 + num_outputs = actor_output.elements('parameter')
196 for k in range(num_outputs): 135 for k in range(num_outputs):
197 - output = actor_output.element("parameter", k)  
198 - output_name = output.attribute("name")  
199 - output_value = output.attribute("value")  
200 - output_actor = output.attribute("actor") 136 + output = actor_output.element('parameter', k)
  137 + output_name = output.attribute('name')
  138 + output_value = output.attribute('value')
  139 + output_actor = output.attribute('actor')
201 parameter = {'name': output_name, \ 140 parameter = {'name': output_name, \
202 'value': output_value, \ 141 'value': output_value, \
203 'actor': output_actor} 142 'actor': output_actor}
204 output_parameters.append(parameter) 143 output_parameters.append(parameter)
205 - if output_actor != "": 144 + if output_actor != '':
206 if output_actor not in output_actors: 145 if output_actor not in output_actors:
207 output_actors.append(output_actor) 146 output_actors.append(output_actor)
208 147
@@ -225,55 +164,55 @@ class csworkflow(ctools.cscript): @@ -225,55 +164,55 @@ class csworkflow(ctools.cscript):
225 'status': status} 164 'status': status}
226 165
227 # Append entry 166 # Append entry
228 - self.actors.append(entry) 167 + self._actors.append(entry)
229 168
230 # Log information about actors 169 # Log information about actors
231 - if self.logNormal():  
232 - self.log.parformat("Actor \""+name+"\"")  
233 - self.log(tool)  
234 - self.log("\n") 170 + if self._logNormal():
  171 + self._log.parformat('Actor "'+name+'"')
  172 + self._log(tool)
  173 + self._log('\n')
235 if num_inputs == 0: 174 if num_inputs == 0:
236 - self.log.parformat(" Predecessor")  
237 - self.log("none\n") 175 + self._log.parformat(' Predecessor')
  176 + self._log('none\n')
238 else: 177 else:
239 if num_inputs == 1: 178 if num_inputs == 1:
240 - self.log.parformat(" Predecessor") 179 + self._log.parformat(' Predecessor')
241 else: 180 else:
242 - self.log.parformat(" Predecessors") 181 + self._log.parformat(' Predecessors')
243 for k in range(num_inputs): 182 for k in range(num_inputs):
244 if k > 0: 183 if k > 0:
245 - self.log(", ")  
246 - self.log("\""+input_actors[k]+"\"")  
247 - self.log("\n") 184 + self._log(', ')
  185 + self._log('"'+input_actors[k]+'"')
  186 + self._log('\n')
248 187
249 # Return 188 # Return
250 return 189 return
251 190
252 - def execute_workflow(self):  
253 - """  
254 - """ 191 + # Execute a workflow
  192 + def _execute_workflow(self):
  193 +
255 # Continue while there are actors 194 # Continue while there are actors
256 - while len(self.actors) > 0: 195 + while len(self._actors) > 0:
257 196
258 # Find actors which are ready 197 # Find actors which are ready
259 num_executed = 0 198 num_executed = 0
260 - for actor in self.actors: 199 + for actor in self._actors:
261 200
262 # If actor is ready then execute it 201 # If actor is ready then execute it
263 if actor['status'] == 'ready': 202 if actor['status'] == 'ready':
264 203
265 # Log execution start 204 # Log execution start
266 - if self.logNormal():  
267 - self.log.parformat("Execute actor")  
268 - self.log("\""+actor['name']+"\"\n") 205 + if self._logNormal():
  206 + self._log.parformat('Execute actor')
  207 + self._log('"'+actor['name']+'"\n')
269 208
270 # Execute actor 209 # Execute actor
271 - self.execute_actor(actor) 210 + self._execute_actor(actor)
272 211
273 # Log execution finish 212 # Log execution finish
274 - if self.logNormal():  
275 - self.log.parformat("Finished actor execution")  
276 - self.log("\""+actor['name']+"\"\n") 213 + if self._logNormal():
  214 + self._log.parformat('Finished actor execution')
  215 + self._log('"'+actor['name']+'"\n')
277 216
278 # Set actor status to finished 217 # Set actor status to finished
279 actor['status'] = 'finished' 218 actor['status'] = 'finished'
@@ -286,12 +225,12 @@ class csworkflow(ctools.cscript): @@ -286,12 +225,12 @@ class csworkflow(ctools.cscript):
286 break 225 break
287 226
288 # Update actor status 227 # Update actor status
289 - for actor in self.actors: 228 + for actor in self._actors:
290 if actor['status'] != 'finished': 229 if actor['status'] != 'finished':
291 input_actors = actor['input_actors'] 230 input_actors = actor['input_actors']
292 ready = True 231 ready = True
293 for input_actor in input_actors: 232 for input_actor in input_actors:
294 - a = self.get_actor(input_actor) 233 + a = self._get_actor(input_actor)
295 if a['status'] != 'finished': 234 if a['status'] != 'finished':
296 ready = False 235 ready = False
297 break 236 break
@@ -301,29 +240,28 @@ class csworkflow(ctools.cscript): @@ -301,29 +240,28 @@ class csworkflow(ctools.cscript):
301 # Return 240 # Return
302 return 241 return
303 242
304 - def execute_actor(self, actor):  
305 - """  
306 - Excute an actor.  
307 - """ 243 + # Execute an actor
  244 + def _execute_actor(self, actor):
  245 +
308 # Log input parameters 246 # Log input parameters
309 - if self.logNormal(): 247 + if self._logNormal():
310 pars = actor['input_parameters'] 248 pars = actor['input_parameters']
311 for par in pars: 249 for par in pars:
312 - self.log.parformat(" Input parameter")  
313 - self.log(par['name'])  
314 - self.log("=")  
315 - self.log(self.get_parameter_value(par))  
316 - self.log("\n") 250 + self._log.parformat(' Input parameter')
  251 + self._log(par['name'])
  252 + self._log('=')
  253 + self._log(self._get_parameter_value(par))
  254 + self._log('\n')
317 255
318 # Log output parameters 256 # Log output parameters
319 - if self.logNormal(): 257 + if self._logNormal():
320 pars = actor['output_parameters'] 258 pars = actor['output_parameters']
321 for par in pars: 259 for par in pars:
322 - self.log.parformat(" Output parameter")  
323 - self.log(par['name'])  
324 - self.log("=")  
325 - self.log(self.get_parameter_value(par))  
326 - self.log("\n") 260 + self._log.parformat(' Output parameter')
  261 + self._log(par['name'])
  262 + self._log('=')
  263 + self._log(self._get_parameter_value(par))
  264 + self._log('\n')
327 265
328 # Set actor tool 266 # Set actor tool
329 if 'tool' in actor: 267 if 'tool' in actor:
@@ -333,8 +271,8 @@ class csworkflow(ctools.cscript): @@ -333,8 +271,8 @@ class csworkflow(ctools.cscript):
333 elif tool[:2] == 'cs': 271 elif tool[:2] == 'cs':
334 tool_eval = 'cscripts.'+tool+'()' 272 tool_eval = 'cscripts.'+tool+'()'
335 else: 273 else:
336 - tool_eval = ""  
337 - if tool_eval == "": 274 + tool_eval = ''
  275 + if tool_eval == '':
338 return 276 return
339 277
340 # Create actor tool object 278 # Create actor tool object
@@ -344,14 +282,14 @@ class csworkflow(ctools.cscript): @@ -344,14 +282,14 @@ class csworkflow(ctools.cscript):
344 pars = actor['input_parameters'] 282 pars = actor['input_parameters']
345 for par in pars: 283 for par in pars:
346 app_par = object[par['name']] 284 app_par = object[par['name']]
347 - app_par.value(self.get_parameter_value(par)) 285 + app_par.value(self._get_parameter_value(par))
348 object[par['name']] = app_par 286 object[par['name']] = app_par
349 287
350 # Set actor output parameters 288 # Set actor output parameters
351 pars = actor['output_parameters'] 289 pars = actor['output_parameters']
352 for par in pars: 290 for par in pars:
353 app_par = object[par['name']] 291 app_par = object[par['name']]
354 - app_par.value(self.get_parameter_value(par)) 292 + app_par.value(self._get_parameter_value(par))
355 object[par['name']] = app_par 293 object[par['name']] = app_par
356 294
357 # Execute actor 295 # Execute actor
@@ -361,60 +299,94 @@ class csworkflow(ctools.cscript): @@ -361,60 +299,94 @@ class csworkflow(ctools.cscript):
361 # Return 299 # Return
362 return 300 return
363 301
364 - def get_parameter_value(self, par):  
365 - """  
366 - Get the parameter value from the respective actor.  
367 - """ 302 + # Get the parameter value from the respective actor
  303 + def _get_parameter_value(self, par):
  304 +
368 # If the actor is empty then simply return the value 305 # If the actor is empty then simply return the value
369 - if (par['actor'] == ""): 306 + if (par['actor'] == ''):
370 value = par['value'] 307 value = par['value']
371 308
372 # ... otherwise get the value from the actor 309 # ... otherwise get the value from the actor
373 else: 310 else:
374 found = False 311 found = False
375 - actor = self.get_actor(par['actor']) 312 + actor = self._get_actor(par['actor'])
376 for output in actor['output_parameters']: 313 for output in actor['output_parameters']:
377 if output['name'] == par['value']: 314 if output['name'] == par['value']:
378 value = output['value'] 315 value = output['value']
379 found = True 316 found = True
380 break 317 break
381 if not found: 318 if not found:
382 - msg = "Parameter \""+par['value']+"\" is not an output "\  
383 - "parameter of actor \""+par['actor']+"\"." 319 + msg = 'Parameter "'+par['value']+'" is not an output '+ \
  320 + 'parameter of actor "'+par['actor']+'".'
384 raise RuntimeError(msg) 321 raise RuntimeError(msg)
385 322
386 # Return value 323 # Return value
387 return value 324 return value
388 325
389 - def get_actor(self, name):  
390 - """  
391 - Get the actor with the specified name.  
392 - """ 326 + # Get the actor with the specified name
  327 + def _get_actor(self, name):
  328 +
393 # Return actor with specified name 329 # Return actor with specified name
394 - for actor in self.actors: 330 + for actor in self._actors:
395 if actor['name'] == name: 331 if actor['name'] == name:
396 return actor 332 return actor
397 333
398 # Throw an exception 334 # Throw an exception
399 - msg = "Actor \""+name+"\" not found." 335 + msg = 'Actor "'+name+'" not found.'
400 raise RuntimeError(msg) 336 raise RuntimeError(msg)
401 337
402 # Return 338 # Return
403 return 339 return
404 340
405 341
  342 + # Public methods
  343 + def run(self):
  344 + """
  345 + Run the script.
  346 + """
  347 + # Switch screen logging on in debug mode
  348 + if self._logDebug():
  349 + self._log.cout(True)
  350 +
  351 + # Get parameters
  352 + self._get_parameters()
  353 +
  354 + # Parse XML file
  355 + if self._logTerse():
  356 + self._log('\n')
  357 + self._log.header1('Parse workflow XML file')
  358 + self._parse_workflow()
  359 +
  360 + # Execute workflow
  361 + if self._logTerse():
  362 + self._log('\n')
  363 + self._log.header1('Execute workflow')
  364 + self._execute_workflow()
  365 +
  366 + # Return
  367 + return
  368 +
  369 + def execute(self):
  370 + """
  371 + Execute the script.
  372 + """
  373 + # Open logfile
  374 + self.logFileOpen()
  375 +
  376 + # Run the script
  377 + self.run()
  378 +
  379 + # Return
  380 + return
  381 +
  382 +
406 # ======================== # 383 # ======================== #
407 # Main routine entry point # 384 # Main routine entry point #
408 # ======================== # 385 # ======================== #
409 if __name__ == '__main__': 386 if __name__ == '__main__':
410 - """  
411 - Executes workflow.  
412 - """ 387 +
413 # Create instance of application 388 # Create instance of application
414 app = csworkflow(sys.argv) 389 app = csworkflow(sys.argv)
415 390
416 - # Open logfile  
417 - app.logFileOpen()  
418 -  
419 # Execute application 391 # Execute application
420 app.execute() 392 app.execute()
test/data/workflow.xml
  View file @698bf58
@@ -46,7 +46,7 @@ @@ -46,7 +46,7 @@
46 <parameter name="logfile" value="ctobssim.log" /> 46 <parameter name="logfile" value="ctobssim.log" />
47 </input> 47 </input>
48 <output> 48 <output>
49 - <parameter name="outevents" value="events.fits" /> 49 + <parameter name="outevents" value="wf_events.fits" />
50 </output> 50 </output>
51 </actor> 51 </actor>
52 <actor name="ctselect" tool="ctselect"> 52 <actor name="ctselect" tool="ctselect">
@@ -70,7 +70,7 @@ @@ -70,7 +70,7 @@
70 <parameter name="logfile" value="ctselect.log" /> 70 <parameter name="logfile" value="ctselect.log" />
71 </input> 71 </input>
72 <output> 72 <output>
73 - <parameter name="outobs" value="selected_events.fits" /> 73 + <parameter name="outobs" value="wf_selected_events.fits" />
74 </output> 74 </output>
75 </actor> 75 </actor>
76 <actor name="ctlike" tool="ctlike"> 76 <actor name="ctlike" tool="ctlike">
@@ -93,37 +93,7 @@ @@ -93,37 +93,7 @@
93 <parameter name="logfile" value="ctlike.log" /> 93 <parameter name="logfile" value="ctlike.log" />
94 </input> 94 </input>
95 <output> 95 <output>
96 - <parameter name="outmodel" value="crab_results.xml" />  
97 - </output>  
98 - </actor>  
99 - <actor name="csspec" tool="csspec">  
100 - <input>  
101 - <parameter name="inobs" value="outobs" actor="ctselect" />  
102 - <parameter name="inmodel" value="outmodel" actor="ctlike" />  
103 - <parameter name="srcname" value="target" actor="input" />  
104 - <parameter name="expcube" value="NONE" />  
105 - <parameter name="psfcube" value="NONE" />  
106 - <parameter name="bkgcube" value="NONE" />  
107 - <parameter name="caldb" value="caldb" actor="input" />  
108 - <parameter name="irf" value="irf" actor="input" />  
109 - <parameter name="edisp" value="edisp" actor="input" />  
110 - <parameter name="emin" value="emin" actor="input" />  
111 - <parameter name="emax" value="emax" actor="input" />  
112 - <parameter name="enumbins" value="10" />  
113 - <parameter name="ebinalg" value="LOG" />  
114 - <parameter name="ebinfile" value="NONE" />  
115 - <parameter name="calc_ts" value="yes" />  
116 - <parameter name="calc_ulim" value="yes" />  
117 - <parameter name="fix_srcs" value="yes" />  
118 - <parameter name="fix_bkg" value="no" />  
119 - <parameter name="chatter" value="chatter" actor="input" />  
120 - <parameter name="clobber" value="clobber" actor="input" />  
121 - <parameter name="debug" value="debug" actor="input" />  
122 - <parameter name="mode" value="mode" actor="input" />  
123 - <parameter name="logfile" value="csspec.log" />  
124 - </input>  
125 - <output>  
126 - <parameter name="outfile" value="spectrum.fits" /> 96 + <parameter name="outmodel" value="wf_crab_results.xml" />
127 </output> 97 </output>
128 </actor> 98 </actor>
129 </workflow> 99 </workflow>
test/test_csworkflow.py 0 → 100755
  View file @698bf58
  1 +#! /usr/bin/env python
  2 +# ==========================================================================
  3 +# This scripts performs unit tests for the csworkflow script.
  4 +#
  5 +# Copyright (C) 2016 Juergen Knoedlseder
  6 +#
  7 +# This program is free software: you can redistribute it and/or modify
  8 +# it under the terms of the GNU General Public License as published by
  9 +# the Free Software Foundation, either version 3 of the License, or
  10 +# (at your option) any later version.
  11 +#
  12 +# This program is distributed in the hope that it will be useful,
  13 +# but WITHOUT ANY WARRANTY; without even the implied warranty of
  14 +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  15 +# GNU General Public License for more details.
  16 +#
  17 +# You should have received a copy of the GNU General Public License
  18 +# along with this program. If not, see <http://www.gnu.org/licenses/>.
  19 +#
  20 +# ==========================================================================
  21 +import os
  22 +import gammalib
  23 +import cscripts
  24 +
  25 +
  26 +# ================================ #
  27 +# Test class for csworkflow script #
  28 +# ================================ #
  29 +class Test(gammalib.GPythonTestSuite):
  30 + """
  31 + Test class for csworkflow script.
  32 +
  33 + This test class makes unit tests for the cslightcrv script by using it
  34 + from the command line and from Python.
  35 + """
  36 +
  37 + # Constructor
  38 + def __init__(self):
  39 + """
  40 + Constructor.
  41 + """
  42 + # Call base class constructor
  43 + gammalib.GPythonTestSuite.__init__(self)
  44 +
  45 + # Return
  46 + return
  47 +
  48 + # Set test functions
  49 + def set(self):
  50 + """
  51 + Set all test functions.
  52 + """
  53 + # Set test name
  54 + self.name('csworkflow')
  55 +
  56 + # Append tests
  57 + self.append(self._test_cmd, 'Test csworkflow on command line')
  58 + self.append(self._test_python, 'Test csworkflow from Python')
  59 +
  60 + # Return
  61 + return
  62 +
  63 + # Test csworkflow on command line
  64 + def _test_cmd(self):
  65 +
  66 + # Kluge to set the command (installed version has no README file)
  67 + if os.path.isfile('README'):
  68 + csworkflow = '../cscripts/csworkflow.py'
  69 + else:
  70 + csworkflow = 'csworkflow'
  71 +
  72 + # Remove result file
  73 + gammalib.GFilename('wf_crab_results.xml').remove()
  74 +
  75 + # Setup csworkflow command
  76 + cmd = csworkflow+' inflow="data/workflow.xml"'+ \
  77 + ' logfile="csworkflow_cmd1.log" chatter=1'
  78 +
  79 + # Execute csworkflow, make sure we catch any exception
  80 + try:
  81 + rc = os.system(cmd+" >/dev/null 2>&1")
  82 + except:
  83 + pass
  84 +
  85 + # Check if execution was successful
  86 + self.test_assert(rc == 0,
  87 + 'Successful csworkflow execution on command line')
  88 +
  89 + # Check fit result
  90 + self._check_fit_result('wf_crab_results.xml')
  91 +
  92 + # Return
  93 + return
  94 +
  95 + # Test csworkflow from Python
  96 + def _test_python(self):
  97 +
  98 + # Remove result file
  99 + gammalib.GFilename('wf_crab_results.xml').remove()
  100 +
  101 + # Set-up csworkflow
  102 + workflow = cscripts.csworkflow()
  103 + workflow['inflow'] = 'data/workflow.xml'
  104 + workflow['logfile'] = 'csworkflow_py1.log'
  105 + workflow['chatter'] = 2
  106 +
  107 + # Run script
  108 + workflow.logFileOpen() # Make sure we get a log file
  109 + workflow.run()
  110 +
  111 + # Check fit result
  112 + self._check_fit_result('wf_crab_results.xml')
  113 +
  114 + # Remove result file
  115 + gammalib.GFilename('wf_crab_results.xml').remove()
  116 +
  117 + # Set-up csworkflow
  118 + workflow = cscripts.csworkflow()
  119 + workflow['inflow'] = 'data/workflow.xml'
  120 + workflow['logfile'] = 'csworkflow_py2.log'
  121 + workflow['chatter'] = 3
  122 +
  123 + # Execute script
  124 + workflow.execute()
  125 +
  126 + # Check fit result
  127 + self._check_fit_result('wf_crab_results.xml')
  128 +
  129 + # Return
  130 + return
  131 +
  132 + # Check fit result XML values
  133 + def _check_fit_result(self, filename):
  134 +
  135 + # Load fit results
  136 + models = gammalib.GModels(filename)
  137 +
  138 + # Check fit result values
  139 + self.test_value(models['Crab'][2].value(),
  140 + 5.73787e-16, 1.0e-5, 'Crab prefactor')
  141 + self.test_value(models['Crab'][3].value(),
  142 + -2.47386, 1.0e-5, 'Crab index')
  143 + self.test_value(models['CTABackgroundModel'][0].value(),
  144 + 1.01414, 1.0e-5, 'Background model prefactor')
  145 + self.test_value(models['CTABackgroundModel'][1].value(),
  146 + 0.00242827, 1.0e-5, 'Background model index')
  147 +
  148 + # Return
  149 + return
test/test_python_cscripts.py
  View file @698bf58
@@ -24,6 +24,7 @@ import gammalib @@ -24,6 +24,7 @@ import gammalib
24 import cscripts 24 import cscripts
25 sys.path.append(os.path.dirname(os.path.abspath(__file__))) 25 sys.path.append(os.path.dirname(os.path.abspath(__file__)))
26 import test_cscaldb 26 import test_cscaldb
  27 +import test_csfindobs
27 import test_cslightcrv 28 import test_cslightcrv
28 import test_csmodelinfo 29 import test_csmodelinfo
29 import test_csmodelmerge 30 import test_csmodelmerge
@@ -32,10 +33,10 @@ import test_csobsdef @@ -32,10 +33,10 @@ import test_csobsdef
32 import test_csobsinfo 33 import test_csobsinfo
33 import test_cspull 34 import test_cspull
34 import test_csroot2caldb 35 import test_csroot2caldb
35 -import test_csfindobs  
36 import test_csiactcopy 36 import test_csiactcopy
37 import test_csiactdata 37 import test_csiactdata
38 import test_csiactobs 38 import test_csiactobs
  39 +import test_csworkflow
39 40
40 41
41 # ================== # 42 # ================== #
@@ -94,6 +95,7 @@ def test(installed=False, debug=False): @@ -94,6 +95,7 @@ def test(installed=False, debug=False):
94 suite_csobsinfo = test_csobsinfo.Test() 95 suite_csobsinfo = test_csobsinfo.Test()
95 suite_cspull = test_cspull.Test() 96 suite_cspull = test_cspull.Test()
96 suite_csroot2caldb = test_csroot2caldb.Test() 97 suite_csroot2caldb = test_csroot2caldb.Test()
  98 + suite_csworkflow = test_csworkflow.Test()
97 99
98 # Setup unit tests 100 # Setup unit tests
99 suite_cscaldb.set() 101 suite_cscaldb.set()
@@ -105,6 +107,7 @@ def test(installed=False, debug=False): @@ -105,6 +107,7 @@ def test(installed=False, debug=False):
105 suite_csobsinfo.set() 107 suite_csobsinfo.set()
106 suite_cspull.set() 108 suite_cspull.set()
107 suite_csroot2caldb.set() 109 suite_csroot2caldb.set()
  110 + suite_csworkflow.set()
108 111
109 # Append tests to container 112 # Append tests to container
110 suites.append(suite_cscaldb) 113 suites.append(suite_cscaldb)
@@ -116,6 +119,7 @@ def test(installed=False, debug=False): @@ -116,6 +119,7 @@ def test(installed=False, debug=False):
116 suites.append(suite_csobsinfo) 119 suites.append(suite_csobsinfo)
117 suites.append(suite_cspull) 120 suites.append(suite_cspull)
118 suites.append(suite_csroot2caldb) 121 suites.append(suite_csroot2caldb)
  122 + suites.append(suite_csworkflow)
119 123
120 # Append tests for Python 2.6+ (the IACT cscripts depend on the json 124 # Append tests for Python 2.6+ (the IACT cscripts depend on the json
121 # module which is only available since Python 2.6+ 125 # module which is only available since Python 2.6+