1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20 import qm
21 from qm.test.context import ContextException
22 import sys, os
23 import types
24 import cgi
25
26
27
28
29
31 """A 'Result' describes the outcome of a test.
32
33 A 'Result' contains two pieces of data: an outcome and a set
34 of annotations. The outcome indicates whether the test passed
35 or failed. More specifically, the outcome may be one of the
36 following constants:
37
38 'Result.PASS' -- The test passed.
39
40 'Result.FAIL' -- The test failed.
41
42 'Result.ERROR' -- Something went wrong in the process of trying to
43 execute the test. For example, if the Python code implementing
44 the 'Run' method in the test class raised an exception, the
45 outcome would be 'Result.ERROR'.
46
47 'Result.UNTESTED' -- QMTest did not even try to run the test.
48 For example, if a prerequiste was not satisfied, then this outcome
49 will be used.'
50
51 The annotations are a dictionary, mapping strings to strings.
52
53 The indices should be of the form 'class.name' where 'class' is
54 the name of the test class that created the annotation. Any
55 annotations created by QMTest, as opposed to the test class, will
56 have indices of the form 'qmtest.name'.
57
58 The annotation values are HTML. When displayed in the GUI, the
59 HTML is inserted directly into the result page; when the
60 command-line interface is used the HTML is converted to plain
61 text.
62
63 Currently, QMTest recognizes the following built-in annotations:
64
65 'Result.CAUSE' -- For results whose outcome is not 'FAIL', this
66 annotation gives a brief description of why the test failed. The
67 preferred form of this message is a phrase like "Incorrect
68 output." or "Exception thrown." The message should begin with a
69 capital letter and end with a period. Most results formatters
70 will display this information prominently.
71
72 'Result.EXCEPTION' -- If an exeption was thrown during the
73 test execution, a brief description of the exception.
74
75 'Result.TARGET' -- This annotation indicates on which target the
76 test was executed.
77
78 'Result.TRACEBACK' -- If an exeption was thrown during the test
79 execution, a representation of the traceback indicating where
80 the exception was thrown.
81
82 A 'Result' object has methods that allow it to act as a dictionary
83 from annotation names to annotation values. You can directly add
84 an annotation to a 'Result' by writing code of the form
85 'result[CAUSE] = "Exception thrown."'.
86
87 A 'Result' object is also used to describe the outcome of
88 executing either setup or cleanup phase of a 'Resource'."""
89
90
91
92 RESOURCE_SETUP = "resource_setup"
93 RESOURCE_CLEANUP = "resource_cleanup"
94 TEST = "test"
95
96
97
98 FAIL = "FAIL"
99 ERROR = "ERROR"
100 UNTESTED = "UNTESTED"
101 PASS = "PASS"
102
103
104
105 CAUSE = "qmtest.cause"
106 EXCEPTION = "qmtest.exception"
107 RESOURCE = "qmtest.resource"
108 TARGET = "qmtest.target"
109 TRACEBACK = "qmtest.traceback"
110 START_TIME = "qmtest.start_time"
111 END_TIME = "qmtest.end_time"
112
113
114
115 kinds = [ RESOURCE_SETUP, RESOURCE_CLEANUP, TEST ]
116 """A list of the possible kinds."""
117
118 outcomes = [ ERROR, FAIL, UNTESTED, PASS ]
119 """A list of the possible outcomes.
120
121 The order of the 'outcomes' is significant; they are ordered from
122 most interesting to least interesting from the point of view of
123 someone browsing results."""
124
125 - def __init__(self, kind, id, outcome=PASS, annotations={}):
126 """Construct a new 'Result'.
127
128 'kind' -- The kind of result. The value must be one of the
129 'Result.kinds'.
130
131 'id' -- The label for the test or resource to which this
132 result corresponds.
133
134 'outcome' -- The outcome associated with the test. The value
135 must be one of the 'Result.outcomes'.
136
137 'annotations' -- The annotations associated with the test."""
138
139 assert kind in Result.kinds
140 assert outcome in Result.outcomes
141
142 self.__kind = kind
143 self.__id = id
144 self.__outcome = outcome
145 self.__annotations = annotations.copy()
146
147
149 """Return a representation of this result for pickling.
150
151 By using an explicit tuple representation of 'Result's when
152 storing them in a pickle file, we decouple our storage format
153 from internal implementation details (e.g., the names of private
154 variables)."""
155
156
157
158
159
160 return (self.__kind,
161 self.__id,
162 self.__outcome,
163 self.__annotations)
164
165
167 """Construct a 'Result' from its pickled form."""
168
169 if isinstance(pickled_state, dict):
170
171
172
173 self.__kind = pickled_state["_Result__kind"]
174 self.__id = pickled_state["_Result__id"]
175 self.__outcome = pickled_state["_Result__outcome"]
176 self.__annotations = pickled_state["_Result__annotations"]
177
178
179 else:
180 assert isinstance(pickled_state, tuple) \
181 and len(pickled_state) == 4
182
183
184
185 (self.__kind,
186 self.__id,
187 self.__outcome,
188 self.__annotations) = pickled_state
189
190
192 """Return the kind of result this is.
193
194 returns -- The kind of entity (one of the 'kinds') to which
195 this result corresponds."""
196
197 return self.__kind
198
199
201 """Return the outcome associated with the test.
202
203 returns -- The outcome associated with the test. This value
204 will be one of the 'Result.outcomes'."""
205
206 return self.__outcome
207
208
210 """Set the outcome associated with the test.
211
212 'outcome' -- One of the 'Result.outcomes'.
213
214 'cause' -- If not 'None', this value becomes the value of the
215 'Result.CAUSE' annotation.
216
217 'annotations' -- The annotations are added to the current set
218 of annotations."""
219
220 assert outcome in Result.outcomes
221 self.__outcome = outcome
222 if cause:
223 self.SetCause(cause)
224 self.Annotate(annotations)
225
226
228 """Add 'annotations' to the current set of annotations."""
229 self.__annotations.update(annotations)
230
231
232 - def Fail(self, cause = None, annotations = {}):
233 """Mark the test as failing.
234
235 'cause' -- If not 'None', this value becomes the value of the
236 'Result.CAUSE' annotation.
237
238 'annotations' -- The annotations are added to the current set
239 of annotations."""
240
241 self.SetOutcome(Result.FAIL, cause, annotations)
242
243
245 """Return the label for the test or resource.
246
247 returns -- A label indicating indicating to which test or
248 resource this result corresponds."""
249
250 return self.__id
251
252
254 """Return the cause of failure, if the test failed.
255
256 returns -- If the test failed, return the cause of the
257 failure, if available."""
258
259 if self.has_key(Result.CAUSE):
260 return self[Result.CAUSE]
261 else:
262 return ""
263
264
266 """Set the cause of failure.
267
268 'cause' -- A string indicating the cause of failure. Like all
269 annotations, 'cause' will be interested as HTML."""
270
271 self[Result.CAUSE] = cause
272
273
274 - def Quote(self, string):
275 """Return a version of string suitable for an annotation value.
276
277 Performs appropriate quoting for a string that should be taken
278 verbatim; this includes HTML entity escaping, and addition of
279 <pre> tags.
280
281 'string' -- The verbatim string to be quoted.
282
283 returns -- The quoted string."""
284
285 return "<pre>%s</pre>" % cgi.escape(string)
286
287
292 """Note that an exception occurred during execution.
293
294 'exc_info' -- A triple, in the same form as that returned
295 from 'sys.exc_info'. If 'None', the value of 'sys.exc_info()'
296 is used instead.
297
298 'cause' -- The value of the 'Result.CAUSE' annotation. If
299 'None', a default message is used.
300
301 'outcome' -- The outcome of the test, now that the exception
302 has occurred.
303
304 A test class can call this method if an exception occurs while
305 the test is being run."""
306
307 if not exc_info:
308 exc_info = sys.exc_info()
309
310 exception_type = exc_info[0]
311
312
313 if not cause:
314 if exception_type is ContextException:
315 cause = str(exc_info[1])
316 else:
317 cause = "An exception occurred."
318
319
320
321 if exception_type is ContextException:
322 self["qmtest.context_variable"] = exc_info[1].key
323
324 self.SetOutcome(outcome, cause)
325 self[Result.EXCEPTION] \
326 = self.Quote("%s: %s" % exc_info[:2])
327 self[Result.TRACEBACK] \
328 = self.Quote(qm.format_traceback(exc_info))
329
330
332 """Check the exit status from a command.
333
334 'prefix' -- The prefix that should be used when creating
335 result annotations.
336
337 'desc' -- A description of the executing program.
338
339 'status' -- The exit status, as returned by 'waitpid'.
340
341 'non_zero_exit_ok' -- True if a non-zero exit code is not
342 considered failure.
343
344 returns -- False if the test failed, true otherwise."""
345
346 if sys.platform == "win32" or os.WIFEXITED(status):
347
348 if sys.platform == "win32":
349 exit_code = status
350 else:
351 exit_code = os.WEXITSTATUS(status)
352
353 if exit_code != 0 and not non_zero_exit_ok:
354 self.Fail("%s failed with exit code %d." % (desc, exit_code))
355
356 self[prefix + "exit_code"] = str(exit_code)
357 return False
358
359 elif os.WIFSIGNALED(status):
360
361 signal = os.WTERMSIG(status)
362
363 self.Fail("%s received fatal signal %d." % (desc, signal))
364 self[prefix + "signal"] = str(signal)
365 return False
366 else:
367
368
369 assert None
370
371 return True
372
373
375 """Generate a DOM element node for this result.
376
377 Note that the context is not represented in the DOM node.
378
379 'document' -- The containing DOM document.
380
381 returns -- The element created."""
382
383
384 element = document.createElement("result")
385 element.setAttribute("id", self.GetId())
386 element.setAttribute("kind", self.GetKind())
387 element.setAttribute("outcome", str(self.GetOutcome()))
388
389 keys = self.keys()
390 keys.sort()
391 for key in keys:
392 value = self[key]
393 annotation_element = document.createElement("annotation")
394
395 annotation_element.setAttribute("name", str(key))
396
397
398
399 node = document.createTextNode('"' + str(value) + '"')
400 annotation_element.appendChild(node)
401
402 element.appendChild(annotation_element)
403
404 return element
405
406
407
408
410 assert type(key) in types.StringTypes
411 return self.__annotations[key]
412
413
415 assert type(key) in types.StringTypes
416 assert type(value) in types.StringTypes
417 self.__annotations[key] = value
418
419
421 assert type(key) in types.StringTypes
422 del self.__annotations[key]
423
424
426 assert type(key) in types.StringTypes
427 return self.__annotations.get(key, default)
428
429
431 assert type(key) in types.StringTypes
432 return self.__annotations.has_key(key)
433
434
436 return self.__annotations.keys()
437
438
440 return self.__annotations.items()
441
442
443
444
445
446
447 __all__ = ["Result"]
448