Package qm :: Package test :: Module result
[hide private]
[frames] | no frames]

Source Code for Module qm.test.result

  1  ######################################################################## 
  2  # 
  3  # File:   result.py 
  4  # Author: Mark Mitchell 
  5  # Date:   2001-10-10 
  6  # 
  7  # Contents: 
  8  #   QMTest Result class. 
  9  # 
 10  # Copyright (c) 2001, 2002, 2003 by CodeSourcery, LLC.  All rights reserved.  
 11  # 
 12  # For license terms see the file COPYING. 
 13  # 
 14  ######################################################################## 
 15   
 16  ######################################################################## 
 17  # Imports 
 18  ######################################################################## 
 19   
 20  import qm 
 21  from   qm.test.context import ContextException 
 22  import sys, os 
 23  import types 
 24  import cgi 
 25   
 26  ######################################################################## 
 27  # Classes 
 28  ######################################################################## 
 29   
30 -class Result:
31 """A 'Result' describes the outcome of a test. 32 33 A 'Result' contains two pieces of data: an outcome and a set 34 of annotations. The outcome indicates whether the test passed 35 or failed. More specifically, the outcome may be one of the 36 following constants: 37 38 'Result.PASS' -- The test passed. 39 40 'Result.FAIL' -- The test failed. 41 42 'Result.ERROR' -- Something went wrong in the process of trying to 43 execute the test. For example, if the Python code implementing 44 the 'Run' method in the test class raised an exception, the 45 outcome would be 'Result.ERROR'. 46 47 'Result.UNTESTED' -- QMTest did not even try to run the test. 48 For example, if a prerequiste was not satisfied, then this outcome 49 will be used.' 50 51 The annotations are a dictionary, mapping strings to strings. 52 53 The indices should be of the form 'class.name' where 'class' is 54 the name of the test class that created the annotation. Any 55 annotations created by QMTest, as opposed to the test class, will 56 have indices of the form 'qmtest.name'. 57 58 The annotation values are HTML. When displayed in the GUI, the 59 HTML is inserted directly into the result page; when the 60 command-line interface is used the HTML is converted to plain 61 text. 62 63 Currently, QMTest recognizes the following built-in annotations: 64 65 'Result.CAUSE' -- For results whose outcome is not 'FAIL', this 66 annotation gives a brief description of why the test failed. The 67 preferred form of this message is a phrase like "Incorrect 68 output." or "Exception thrown." The message should begin with a 69 capital letter and end with a period. Most results formatters 70 will display this information prominently. 71 72 'Result.EXCEPTION' -- If an exeption was thrown during the 73 test execution, a brief description of the exception. 74 75 'Result.TARGET' -- This annotation indicates on which target the 76 test was executed. 77 78 'Result.TRACEBACK' -- If an exeption was thrown during the test 79 execution, a representation of the traceback indicating where 80 the exception was thrown. 81 82 A 'Result' object has methods that allow it to act as a dictionary 83 from annotation names to annotation values. You can directly add 84 an annotation to a 'Result' by writing code of the form 85 'result[CAUSE] = "Exception thrown."'. 86 87 A 'Result' object is also used to describe the outcome of 88 executing either setup or cleanup phase of a 'Resource'.""" 89 90 # Constants for result kinds. 91 92 RESOURCE_SETUP = "resource_setup" 93 RESOURCE_CLEANUP = "resource_cleanup" 94 TEST = "test" 95 96 # Constants for outcomes. 97 98 FAIL = "FAIL" 99 ERROR = "ERROR" 100 UNTESTED = "UNTESTED" 101 PASS = "PASS" 102 103 # Constants for predefined annotations. 104 105 CAUSE = "qmtest.cause" 106 EXCEPTION = "qmtest.exception" 107 RESOURCE = "qmtest.resource" 108 TARGET = "qmtest.target" 109 TRACEBACK = "qmtest.traceback" 110 START_TIME = "qmtest.start_time" 111 END_TIME = "qmtest.end_time" 112 113 # Other class variables. 114 115 kinds = [ RESOURCE_SETUP, RESOURCE_CLEANUP, TEST ] 116 """A list of the possible kinds.""" 117 118 outcomes = [ ERROR, FAIL, UNTESTED, PASS ] 119 """A list of the possible outcomes. 120 121 The order of the 'outcomes' is significant; they are ordered from 122 most interesting to least interesting from the point of view of 123 someone browsing results.""" 124
125 - def __init__(self, kind, id, outcome=PASS, annotations={}):
126 """Construct a new 'Result'. 127 128 'kind' -- The kind of result. The value must be one of the 129 'Result.kinds'. 130 131 'id' -- The label for the test or resource to which this 132 result corresponds. 133 134 'outcome' -- The outcome associated with the test. The value 135 must be one of the 'Result.outcomes'. 136 137 'annotations' -- The annotations associated with the test.""" 138 139 assert kind in Result.kinds 140 assert outcome in Result.outcomes 141 142 self.__kind = kind 143 self.__id = id 144 self.__outcome = outcome 145 self.__annotations = annotations.copy()
146 147
148 - def __getstate__(self):
149 """Return a representation of this result for pickling. 150 151 By using an explicit tuple representation of 'Result's when 152 storing them in a pickle file, we decouple our storage format 153 from internal implementation details (e.g., the names of private 154 variables).""" 155 156 # A tuple containing the data needed to reconstruct a 'Result'. 157 # No part of this structure should ever be a user-defined type, 158 # because that will introduce interdependencies that we want to 159 # avoid. 160 return (self.__kind, 161 self.__id, 162 self.__outcome, 163 self.__annotations)
164 165
166 - def __setstate__(self, pickled_state):
167 """Construct a 'Result' from its pickled form.""" 168 169 if isinstance(pickled_state, dict): 170 # Old style pickle, from before we defined '__getstate__'. 171 # (Notionally, this is version "0".) The state is a 172 # dictionary containing the variables we used to have. 173 self.__kind = pickled_state["_Result__kind"] 174 self.__id = pickled_state["_Result__id"] 175 self.__outcome = pickled_state["_Result__outcome"] 176 self.__annotations = pickled_state["_Result__annotations"] 177 # Also has a key "_Result__context" containing a (probably 178 # invalid) context object, but we discard it. 179 else: 180 assert isinstance(pickled_state, tuple) \ 181 and len(pickled_state) == 4 182 # New style pickle, from after we defined '__getstate__'. 183 # (Notionally, this is version "1".) The state is a tuple 184 # containing the values of the variables we care about. 185 (self.__kind, 186 self.__id, 187 self.__outcome, 188 self.__annotations) = pickled_state
189 190
191 - def GetKind(self):
192 """Return the kind of result this is. 193 194 returns -- The kind of entity (one of the 'kinds') to which 195 this result corresponds.""" 196 197 return self.__kind
198 199
200 - def GetOutcome(self):
201 """Return the outcome associated with the test. 202 203 returns -- The outcome associated with the test. This value 204 will be one of the 'Result.outcomes'.""" 205 206 return self.__outcome
207 208
209 - def SetOutcome(self, outcome, cause = None, annotations = {}):
210 """Set the outcome associated with the test. 211 212 'outcome' -- One of the 'Result.outcomes'. 213 214 'cause' -- If not 'None', this value becomes the value of the 215 'Result.CAUSE' annotation. 216 217 'annotations' -- The annotations are added to the current set 218 of annotations.""" 219 220 assert outcome in Result.outcomes 221 self.__outcome = outcome 222 if cause: 223 self.SetCause(cause) 224 self.Annotate(annotations)
225 226
227 - def Annotate(self, annotations):
228 """Add 'annotations' to the current set of annotations.""" 229 self.__annotations.update(annotations)
230 231
232 - def Fail(self, cause = None, annotations = {}):
233 """Mark the test as failing. 234 235 'cause' -- If not 'None', this value becomes the value of the 236 'Result.CAUSE' annotation. 237 238 'annotations' -- The annotations are added to the current set 239 of annotations.""" 240 241 self.SetOutcome(Result.FAIL, cause, annotations)
242 243
244 - def GetId(self):
245 """Return the label for the test or resource. 246 247 returns -- A label indicating indicating to which test or 248 resource this result corresponds.""" 249 250 return self.__id
251 252
253 - def GetCause(self):
254 """Return the cause of failure, if the test failed. 255 256 returns -- If the test failed, return the cause of the 257 failure, if available.""" 258 259 if self.has_key(Result.CAUSE): 260 return self[Result.CAUSE] 261 else: 262 return ""
263 264
265 - def SetCause(self, cause):
266 """Set the cause of failure. 267 268 'cause' -- A string indicating the cause of failure. Like all 269 annotations, 'cause' will be interested as HTML.""" 270 271 self[Result.CAUSE] = cause
272 273
274 - def Quote(self, string):
275 """Return a version of string suitable for an annotation value. 276 277 Performs appropriate quoting for a string that should be taken 278 verbatim; this includes HTML entity escaping, and addition of 279 <pre> tags. 280 281 'string' -- The verbatim string to be quoted. 282 283 returns -- The quoted string.""" 284 285 return "<pre>%s</pre>" % cgi.escape(string)
286 287
288 - def NoteException(self, 289 exc_info = None, 290 cause = None, 291 outcome = ERROR):
292 """Note that an exception occurred during execution. 293 294 'exc_info' -- A triple, in the same form as that returned 295 from 'sys.exc_info'. If 'None', the value of 'sys.exc_info()' 296 is used instead. 297 298 'cause' -- The value of the 'Result.CAUSE' annotation. If 299 'None', a default message is used. 300 301 'outcome' -- The outcome of the test, now that the exception 302 has occurred. 303 304 A test class can call this method if an exception occurs while 305 the test is being run.""" 306 307 if not exc_info: 308 exc_info = sys.exc_info() 309 310 exception_type = exc_info[0] 311 312 # If no cause was specified, use an appropriate message. 313 if not cause: 314 if exception_type is ContextException: 315 cause = str(exc_info[1]) 316 else: 317 cause = "An exception occurred." 318 319 # For a 'ContextException', indicate which context variable 320 # was invalid. 321 if exception_type is ContextException: 322 self["qmtest.context_variable"] = exc_info[1].key 323 324 self.SetOutcome(outcome, cause) 325 self[Result.EXCEPTION] \ 326 = self.Quote("%s: %s" % exc_info[:2]) 327 self[Result.TRACEBACK] \ 328 = self.Quote(qm.format_traceback(exc_info))
329 330
331 - def CheckExitStatus(self, prefix, desc, status, non_zero_exit_ok = 0):
332 """Check the exit status from a command. 333 334 'prefix' -- The prefix that should be used when creating 335 result annotations. 336 337 'desc' -- A description of the executing program. 338 339 'status' -- The exit status, as returned by 'waitpid'. 340 341 'non_zero_exit_ok' -- True if a non-zero exit code is not 342 considered failure. 343 344 returns -- False if the test failed, true otherwise.""" 345 346 if sys.platform == "win32" or os.WIFEXITED(status): 347 # Obtain the exit code. 348 if sys.platform == "win32": 349 exit_code = status 350 else: 351 exit_code = os.WEXITSTATUS(status) 352 # If the exit code is non-zero, the test fails. 353 if exit_code != 0 and not non_zero_exit_ok: 354 self.Fail("%s failed with exit code %d." % (desc, exit_code)) 355 # Record the exit code in the result. 356 self[prefix + "exit_code"] = str(exit_code) 357 return False 358 359 elif os.WIFSIGNALED(status): 360 # Obtain the signal number. 361 signal = os.WTERMSIG(status) 362 # If the program gets a fatal signal, the test fails . 363 self.Fail("%s received fatal signal %d." % (desc, signal)) 364 self[prefix + "signal"] = str(signal) 365 return False 366 else: 367 # A process should only be able to stop by exiting, or 368 # by being terminated with a signal. 369 assert None 370 371 return True
372 373
374 - def MakeDomNode(self, document):
375 """Generate a DOM element node for this result. 376 377 Note that the context is not represented in the DOM node. 378 379 'document' -- The containing DOM document. 380 381 returns -- The element created.""" 382 383 # The node is a result element. 384 element = document.createElement("result") 385 element.setAttribute("id", self.GetId()) 386 element.setAttribute("kind", self.GetKind()) 387 element.setAttribute("outcome", str(self.GetOutcome())) 388 # Add an annotation element for each annotation. 389 keys = self.keys() 390 keys.sort() 391 for key in keys: 392 value = self[key] 393 annotation_element = document.createElement("annotation") 394 # The annotation name is an attribute. 395 annotation_element.setAttribute("name", str(key)) 396 # The annotation value is contained in a text node. The 397 # data is enclosed in quotes for robustness if the 398 # document is pretty-printed. 399 node = document.createTextNode('"' + str(value) + '"') 400 annotation_element.appendChild(node) 401 # Add the annotation element to the result node. 402 element.appendChild(annotation_element) 403 404 return element
405 406 # These methods allow 'Result' to act like a dictionary of 407 # annotations. 408
409 - def __getitem__(self, key):
410 assert type(key) in types.StringTypes 411 return self.__annotations[key]
412 413
414 - def __setitem__(self, key, value):
415 assert type(key) in types.StringTypes 416 assert type(value) in types.StringTypes 417 self.__annotations[key] = value
418 419
420 - def __delitem__(self, key):
421 assert type(key) in types.StringTypes 422 del self.__annotations[key]
423 424
425 - def get(self, key, default=None):
426 assert type(key) in types.StringTypes 427 return self.__annotations.get(key, default)
428 429
430 - def has_key(self, key):
431 assert type(key) in types.StringTypes 432 return self.__annotations.has_key(key)
433 434
435 - def keys(self):
436 return self.__annotations.keys()
437 438
439 - def items(self):
440 return self.__annotations.items()
441 442 443 ######################################################################## 444 # Variables 445 ######################################################################## 446 447 __all__ = ["Result"] 448