Package Gnumed :: Package importers :: Module gmLDTimporter
[frames] | no frames]

Source Code for Module Gnumed.importers.gmLDTimporter

  1  # -*- coding: utf-8 -*- 
  2  """GNUmed LDT importer. 
  3   
  4  This script automatically imports German pathology result 
  5  files in LDT format. 
  6   
  7  It relies on patient-to-request-ID mappings to be present 
  8  in the GNUmed database. It will only import those request 
  9  that have a mapping. 
 10   
 11  The general theory of operation of automatic import at 
 12  Hilbert office is as follows: 
 13   
 14  - automatically retrieve LDT files from labs 
 15  - archive them 
 16  - make them available in a GNUmed private directory 
 17  - run importer every hour 
 18  - import those records that have a mapping 
 19  - make those records available to TurboMed 
 20  - retain unmapped records until next time around 
 21   
 22  copyright: authors 
 23   
 24  FIXME: check status on save_payload()s 
 25  """ 
 26  #=============================================================== 
 27  __version__ = "$Revision: 1.34 $" 
 28  __author__ = "Karsten Hilbert <Karsten.Hilbert@gmx.net>" 
 29  __license__ = "GPL, details at http://www.gnu.org" 
 30   
 31  # stdlib 
 32  import glob, os.path, sys, tempfile, fileinput, time, copy, random, shutil 
 33   
 34   
 35  from Gnumed.pycommon import gmCfg, gmLoginInfo, gmExceptions, gmI18N 
 36  from Gnumed.business import gmPathLab, gmXdtMappings, gmPerson, gmPersonSearch 
 37   
 38  import mx.DateTime as mxDT 
 39   
 40  _log = gmLog.gmDefLog 
 41  _cfg = gmCfg.gmDefCfgFile 
 42  #=============================================================== 
43 -class cLDTImporter:
44 45 _chunk_starters = ['8000', '8410'] 46 _map_820xline2req_field = { 47 '8311': 'lab_request_id', 48 '8301': 'lab_rxd_when', 49 '8401': 'request_status', 50 '8405': 'progress_note' 51 } 52
53 - def __init__(self, cfg=None):
54 self._cfg = cfg 55 # verify db connectivity 56 # FIXME: port to gmPG2 57 pool = gmPG.ConnectionPool() 58 conn = pool.GetConnection('historica') 59 if conn is None: 60 _log.Log(gmLog.lErr, 'cannot connect to database') 61 raise gmExceptions.ConstructorError, 'cannot connect to database' 62 else: 63 pool.ReleaseConnection('historica')
64 #-----------------------------------------------------------
65 - def import_file(self, filename=None):
66 # verify ldt file 67 if not os.access(filename, os.R_OK): 68 _log.Log(gmLog.lErr, 'cannot access LDT file [%s] for reads' % filename) 69 return False 70 self.ldt_filename = filename 71 72 # verify header of LDT file 73 if not self.__verify_file_header(self.ldt_filename): 74 _log.Log(gmLog.lInfo, 'cannot verify file header on [%s]' % self.ldt_filename) 75 return False 76 77 # verify base working directory 78 self.work_base = self._cfg.get('import', 'work dir base') 79 if self.work_base is None: 80 self.work_base = os.path.dirname(self.ldt_filename) 81 self.work_base = os.path.abspath(os.path.expanduser(self.work_base)) 82 if not os.access(self.work_base, os.W_OK): 83 _log.Log(gmLog.lErr, 'cannot write to work directory [%s]' % self.work_base) 84 return False 85 86 # create scratch directory 87 tempfile.tempdir = self.work_base 88 self.work_dir = tempfile.mktemp() 89 os.mkdir(self.work_dir, 0700) 90 91 # split into parts 92 source_files = self.__split_file(self.ldt_filename) 93 if source_files is None: 94 _log.Log(gmLog.lErr, 'cannot split LDT file [%s]' % self.ldt_filename) 95 return False 96 97 # import requested results 98 if len(source_files['data']) > 0: 99 tmp_src_files = copy.copy(source_files['data']) 100 for request_file in tmp_src_files: 101 _log.Log(gmLog.lInfo, 'importing request file [%s]' % request_file) 102 if self.__import_request_file(request_file): 103 source_files['data'].remove(request_file) 104 _log.Log(gmLog.lErr, 'success importing request file\n') 105 else: 106 _log.Log(gmLog.lErr, 'cannot import request file [%s]\n' % request_file) 107 else: 108 _log.Log(gmLog.lData, 'skipping empty LDT file [%s]' % self.ldt_filename) 109 110 # cleanup work area 111 try: 112 shutil.rmtree(path=self.work_dir, ignore_errors=True) 113 except Exception: 114 _log.LogException('cannot cleanup work dir [%s]' % self.work_dir, sys.exc_info(), verbose=0) 115 116 # anything left ? 117 if len(source_files['data']) > 0: 118 # keep source file for next time 119 return False 120 return True
121 #----------------------------------------------------------- 122 # internal helpers 123 #-----------------------------------------------------------
124 - def _verify_8300(self, a_line, field_data):
125 # FIXME: internal_name is no more 126 cmd = "select exists(select pk from test_org where internal_name=%s)" 127 status = gmPG.run_ro_query('historica', cmd, None, field_data) 128 if status is None: 129 _log.Log(gmLog.lErr, 'cannot check for lab existance on [%s]' % field_data) 130 return False 131 if not status[0][0]: 132 _log.Log(gmLog.lErr, 'Unbekanntes Labor [%s]' % field_data) 133 prob = 'Labor unbekannt. Import abgebrochen.' % field_data 134 sol = 'Labor ergänzen oder vorhandenes Labor anpassen (test_org.internal_name).' 135 ctxt = 'LDT-Datei [%s], Labor [%s]' % (self.ldt_filename, field_data) 136 add_todo(problem=prob, solution=sol, context=txt) 137 return False 138 self.__lab_name = field_data 139 return True
140 #----------------------------------------------------------- 141 # def _verify_9211(self, a_line, field_data): 142 # _log.Log(gmLog.lInfo, 'LDT version [%s]' % field_data) 143 # return True 144 #----------------------------------------------------------- 145 _map_field2verifier = { 146 '8300': _verify_8300 147 # '9211': _verify_9211 148 } 149 #-----------------------------------------------------------
150 - def __verify_file_header(self, filename):
151 """Verify that header is suitable for import. 152 153 This does not verify whether the header is conforming 154 to the LDT specs but rather that it is fit for import. 155 """ 156 verified_lines = 0 157 in_header = False 158 for line in fileinput.input(filename): 159 tmp = line.replace('\r','') 160 tmp = tmp.replace('\n','') 161 line_type = tmp[3:7] 162 line_data = tmp[7:] 163 # found start of record 164 if line_type == '8000': 165 # header found ? 166 if line_data == '8220': 167 in_header = True 168 continue 169 # skip prepended junk 170 if not in_header: 171 continue 172 # found record following header 173 if line_data != '8220': 174 fileinput.close() 175 if verified_lines == len(cLDTImporter._map_field2verifier): 176 return True 177 _log.Log(gmLog.lErr, 'zuwenige verifizierbare Zeilen im LDT-Datei-Header') 178 return False 179 if not in_header: 180 continue 181 try: 182 verify_line = cLDTImporter._map_field2verifier[line_type] 183 except KeyError: 184 _log.Log(gmLog.lData, 'kein Handler für Zeilentyp [%s] in LDT-Datei-Header' % line_type) 185 continue 186 if verify_line(self, line, line_data): 187 verified_lines += 1 188 else: 189 _log.Log(gmLog.lErr, 'cannot handle LDT file [%s]' % filename) 190 fileinput.close() 191 return False 192 193 _log.Log(gmLog.lErr, 'LDT file [%s] contains nothing but a header' % filename) 194 fileinput.close() 195 return False
196 #-----------------------------------------------------------
197 - def __split_file(self, filename):
198 """Split LDT file. 199 200 Splits LDT files into header (record type 8220), data 201 records (8202, etc) and trailer (8221). 202 """ 203 tempfile.tempdir = self.work_dir 204 source_files = {} 205 source_files['data'] = [] 206 outname = None 207 in_header = False 208 in_trailer = False 209 for line in fileinput.input(filename): 210 tmp = line.replace('\r','') 211 tmp = tmp.replace('\n','') 212 line_type = tmp[3:7] 213 line_data = tmp[7:] 214 if line_type == '8000': 215 # start headers == start of file 216 if line_data in ['0020', '8220']: 217 if not in_header: 218 header = os.path.join(self.work_dir, 'header.txt') 219 source_files['header'] = header 220 outfile = open(header, 'w+b') 221 in_header = True 222 # start of trailer 223 elif line_data in ['8221', '0021']: 224 in_header = False 225 # did we have any data records ? 226 if outname is not None: 227 # yes, so append them 228 source_files['data'].append(outname) 229 if not in_trailer: 230 outfile.close() 231 trailer = os.path.join(self.work_dir, 'trailer.txt') 232 source_files['trailer'] = trailer 233 outfile = open(trailer, 'w+b') 234 in_trailer = True 235 # start of data record 236 else: 237 in_header = False 238 in_trailer = False 239 outfile.close() 240 # first record ? 241 if outname is not None: 242 # no, so append record name 243 source_files['data'].append(outname) 244 outname = os.path.join(self.work_dir, tempfile.mktemp(suffix='.txt')) 245 outfile = open(outname, 'w+b') 246 if line_data not in ['8202', '8201']: 247 _log.Log(gmLog.lWarn, 'unbekannter Satztyp [%s]' % line_data) 248 # keep line 249 outfile.write(line) 250 251 # end of file 252 outfile.close() 253 fileinput.close() 254 return source_files
255 #----------------------------------------------------------- 256 #-----------------------------------------------------------
257 - def __xform_8311(self, request_data):
258 return request_data['8311'][0].strip()
259 #-----------------------------------------------------------
260 - def __xform_8301(self, request_data):
261 return gmXdtMappings.xdt_8date2iso(request_data['8301'][0].strip())
262 #-----------------------------------------------------------
263 - def __xform_8302(self, request_data):
264 """8302: Berichtsdatum""" 265 if self.__request['results_reported_when'] is None: 266 self.__request['results_reported_when'] = mxDT.now() 267 self.__request['results_reported_when'] = mxDT.strptime( 268 request_data['8302'][0].strip(), 269 '%d%m%Y', 270 self.__request['results_reported_when'] 271 ) 272 return None
273 #-----------------------------------------------------------
274 - def __xform_8303(self, request_data):
275 """8303: Berichtszeit""" 276 if self.__request['results_reported_when'] is None: 277 self.__request['results_reported_when'] = mxDT.now() 278 self.__request['results_reported_when'] = mxDT.strptime( 279 request_data['8303'][0].strip(), 280 '%H%M', 281 self.__request['results_reported_when'] 282 ) 283 return None
284 #-----------------------------------------------------------
285 - def __xform_3103(self, request_data):
286 tmp = gmXdtMappings.xdt_8date2iso(request_data['3103'][0]) 287 # keep for result records to store 288 self.__ref_group_str = ' / '.join([self.__ref_group_str, ("geb: %s" % tmp)]) 289 # - sanity check patient dob 290 cmd = """ 291 select exists( 292 select 1 293 from clin.v_pat_items vpi, clin.lab_request lr, dem.identity i 294 where 295 vpi.pk_patient=i.pk 296 and 297 vpi.pk_item=%s 298 and 299 date_trunc('day', i.dob)=%s::timestamp 300 )""" 301 data = gmPG.run_ro_query('personalia', cmd, None, self.__request['pk_item'], tmp) 302 if data is None: 303 _log.Log(gmLog.lErr, 'cannot sanity check patient dob [%s]' % tmp) 304 if not data[0]: 305 _log.Log(gmLog.lErr, 'lab used [%s] as reference dob while patient has different dob !' % tmp) 306 self.__ref_group_str = "!!!*** Im Labor wurde vermutlich ein falsches Referenzalter verwendet. ***!!!\n%s" % self.__ref_group_str 307 return None
308 #-----------------------------------------------------------
309 - def __xform_8401(self, request_data):
310 try: 311 req_stat = gmXdtMappings.map_Befundstatus_xdt2gm[request_data['8401'][0].strip()] 312 except KeyError: 313 _log.Log(gmLog.lErr, 'unbekannter Befundstatus [%s] (Feld 8401, Regel 135)' % request_data['8401'][0]) 314 req_stat = 'preliminary' 315 # sanity check 316 if (not self.__request['is_pending']) and (req_stat != 'final'): 317 prob = 'kein Befund für [%s] mehr erwartet, aber Befund mit Status [%s] erhalten)' % (self.__request['request_id'], req_stat) 318 sol = 'Befund wird trotzdem importiert. Bitte Befunde auf Duplikate überprüfen.' 319 ctxt = 'Patient: %s, Labor [%s], LDT-Datei [%s], Probe [%s], (Feld 8401, Regel 135)' % (self.__request.get_patient(), self.__lab_name, self.ldt_filename, self.__request['request_id']) 320 add_todo(problem=prob, solution=sol, context=ctxt) 321 _log.Log(gmLog.lWarn, prob) 322 _log.Log(gmLog.lWarn, ctxt) 323 # don't force is_pending back to true just because 324 # a wrong status was received from the lab 325 req_stat = 'final' 326 if req_stat == 'final': 327 self.__request['is_pending'] = 'false' 328 else: 329 self.__request['is_pending'] = 'true' 330 return req_stat
331 #-----------------------------------------------------------
332 - def __xform_8405(self, request_data):
333 new_comment = '\n'.join(request_data['8405']) 334 old_narrative = self.__request['progress_note'] 335 if old_narrative is None: 336 return new_comment 337 if old_narrative.find(new_comment) == -1: 338 old_narrative = old_narrative + '\n' + new_comment 339 return old_narrative
340 #-----------------------------------------------------------
341 - def __xform_8407(self, request_data):
342 tmp = request_data['8407'][0] 343 # keep this so test results can store it 344 self.__ref_group_str = ' / '.join([gmXdtMappings.map_8407_2str[tmp], self.__ref_group_str]) 345 # - sanity check patient gender/age 346 # I discussed the age cutoff with a pediatrician and 347 # we came to the conclusion that a useful value for 348 # child/adult age separation in terms of lab results 349 # would be 12 years. 350 # - get patient gender/age 351 cmd = """ 352 select 353 i.gender, 354 case when i.dob < (now() - '12 years'::interval) 355 then false 356 else true 357 end as is_child 358 from v_pat_items vpi, lab_request lr, identity i 359 where 360 vpi.id_patient=i.pk 361 and 362 vpi.pk_item=%s""" 363 data = gmPG.run_ro_query('personalia', cmd, None, self.__request['pk_item']) 364 if data is None: 365 _log.Log(gmLog.lErr, 'cannot sanity check patient ref group [%s]' % gmXdtMappings.map_8407_2str[tmp]) 366 elif len(data) == 0: 367 _log.Log(gmLog.lErr, 'cannot sanity check patient ref group [%s]' % gmXdtMappings.map_8407_2str[tmp]) 368 else: 369 gender = data[0] 370 is_child = data[1] 371 if tmp in [0,6]: 372 self.__ref_group_str = "%s\n* Kann Referenzgruppe nicht mit Patient abgleichen. *" % self.__ref_group_str 373 else: 374 # Kind 375 if (((tmp in [3,4,5]) and (not is_child)) or 376 # männlich 377 ((tmp in [1,4]) and (gender != 'm')) or 378 # weiblich 379 ((tmp in [2,5]) and (gender != 'f'))): 380 _log.Log(gmLog.lErr, 'lab thinks patient is [%s] but patient is [%s:child->%s]' % (gmXdtMappings.map_8407_2str[tmp], gender, is_child)) 381 self.__ref_group_str = "!!!*** Im Labor wurde vermutlich eine falsche Referenzgruppe verwendet (Alter/Geschlecht). ***!!!\n%s" % self.__ref_group_str 382 return None
383 #----------------------------------------------------------- 384 __820xline_handler = { 385 '0020': None, 386 '9105': None, 387 '8000': None, 388 '8100': None, 389 '8310': None, 390 '8311': __xform_8311, 391 '8301': __xform_8301, 392 '8302': __xform_8302, 393 '8303': __xform_8303, 394 '3100': None, 395 '3101': None, 396 '3102': None, 397 '3103': __xform_3103, 398 '3104': None, 399 '8401': __xform_8401, 400 '8403': None, # Gebührenordnung 401 '8405': __xform_8405, 402 '8407': __xform_8407, 403 '8609': None # Gebührenordnung 404 } 405 #-----------------------------------------------------------
406 - def __handle_8202(self, request_data):
407 # essential fields 408 try: 409 reqid = request_data['8310'][0] 410 except KeyError, IndexError: 411 # FIXME: todo item 412 _log.Log(gmLog.lErr, 'Satz vom Typ [8000:%s] enthält keine Probennummer' % request_data['8000'][0]) 413 return False 414 # get lab request 415 try: 416 self.__request = gmPathLab.cLabRequest(req_id=reqid, lab=self.__lab_name) 417 except gmExceptions.ConstructorError: 418 prob = 'Kann keine Patientenzuordnung der Probe finden.' 419 sol = 'Zuordnung der Probe zu einem Patienten prüfen. Falls doch vorhanden, Systembetreuer verständigen.' 420 ctxt = 'Labor [%s], Probe [%s], LDT-Datei [%s]' % (self.__lab_name, reqid, self.ldt_filename) 421 add_todo(problem=prob, solution=sol, context=ctxt) 422 _log.LogException('cannot get lab request', sys.exc_info(), verbose=0) 423 return False 424 # update fields in request from request_data 425 for line_type in request_data.keys(): 426 # get handler 427 try: 428 handle_line = cLDTImporter.__820xline_handler[line_type] 429 except KeyError: 430 _log.LogException('no handler for line [%s:%s] in [8000:8201] record' % (line_type, request_data[line_type]), sys.exc_info(), verbose=0) 431 continue 432 # ignore line 433 if handle_line is None: 434 try: 435 name = gmXdtMappings.xdt_id_map[line_type] 436 except KeyError: 437 name = '?' 438 _log.Log(gmLog.lData, 'skipping [%s] (%s)' % (line_type, name)) 439 continue 440 # handle line 441 line_data = handle_line(self, request_data) 442 if line_data is False: 443 # FIXME: todo item 444 _log.Log(gmLog.lErr, 'handling line [%s] failed' % line_type) 445 return False 446 try: 447 self.__request[cLDTImporter._map_820xline2req_field[line_type]] = line_data 448 except KeyError: 449 pass 450 self.__request.save_payload() 451 return True
452 #----------------------------------------------------------- 453 #-----------------------------------------------------------
454 - def __get_request_from_8201(self, request_data):
455 request = None 456 try: 457 pat_ldt = { 458 'lastnames': request_data['3101'][0], 459 'firstnames': request_data['3102'][0], 460 'dob': gmXdtMappings.xdt_8date2iso(request_data['3103'][0]) 461 } 462 except KeyError, IndexError: 463 pat_ldt = None 464 # either get lab request from request id 465 if '8310' in request_data: 466 reqid = request_data['8310'][0] 467 try: 468 request = gmPathLab.cLabRequest(req_id=reqid, lab=self.__lab_name) 469 except gmExceptions.ConstructorError: 470 _log.LogException('cannot get lab request', sys.exc_info(), verbose=0) 471 # try to verify patient 472 if request is not None: 473 if pat_ldt is not None: 474 pat_db = request.get_patient() 475 if ((pat_ldt['lastnames'] != pat_db[3]) or 476 (pat_ldt['firstnames'] != pat_db[2]) or 477 (pat_ldt['dob'] != pat_db[4].strftime('%Y-%m-%d'))): 478 _log.Log(gmLog.lErr, 'patient mismatch LDT-Datei <-> Datenbank') 479 _log.Log(gmLog.lData, 'Datei: %s' % pat_ldt) 480 _log.Log(gmLog.lData, 'DB: %s' % pat_db) 481 return None 482 483 # or create one from name/dob 484 # FIXME: we may have to use field 3100, too 485 if request is None: 486 # check essential fields 487 if pat_ldt is None: 488 _log.Log(gmLog.lErr, 'Satz vom Typ [8000:%s] enthaelt nicht alle Felder [3101, 3102, 3103]' % request_data['8000'][0]) 489 _log.Log(gmLog.lErr, 'Kann lab_request nicht automatisch erzeugen.') 490 return None 491 # find patient 492 searcher = gmPersonSearch.cPatientSearcher_SQL() 493 pat_ids = searcher.get_patient_ids(search_dict=pat_ldt) 494 print "must use dto, not search_dict" 495 if len(pat_ids) == 0: 496 _log.Log(gmLog.lErr, 'Kann in der Datenbank keinen Patienten fuer %s finden.' % str(pat_ldt)) 497 return None 498 if len(pat_ids) > 1: 499 _log.Log(gmLog.lErr, 'Mehrere Patienten fuer %s gefunden: %s' % (str(pat_ldt), str(pat_ids))) 500 return None 501 # create lab request 502 try: 503 pat = gmPerson.gmPerson(aPKey=pat_ids[0]) 504 except gmExceptions.ConstructorError: 505 _log.LogException('patient error', sys.exc_info()) 506 return None 507 emr = pat.emr 508 if '8310' in request_data: 509 reqid = request_data['8310'][0] 510 elif '8311' in request_data: 511 reqid = request_data['8311'][0] 512 else: 513 reqid = str(random.randrange(sys.maxsize)) 514 request = emr.add_lab_request(lab=self.__lab_name, req_id=reqid) 515 pat.cleanup() 516 if request is None: 517 _log.Log(gmLog.lErr, 'cannot auto-create lab request with [%s:%s]' % (self.__lab_name, reqid)) 518 return None 519 # eventually we got one 520 return request
521 #-----------------------------------------------------------
522 - def __handle_8201(self, request_data):
523 self.__request = self.__get_request_from_8201(request_data) 524 if self.__request is None: 525 prob = 'Kann Labordaten keiner Anforderung zuordnen.' 526 sol = 'Zuordnungen ueberpruefen. Systembetreuer verstaendigen. Details im Log.' 527 ctxt = 'Labor [%s], LDT-Datei [%s]' % (self.__lab_name, self.ldt_filename) 528 add_todo(problem=prob, solution=sol, context=ctxt) 529 _log.Log(gmLog.lErr, 'cannot find lab request matching data derived from 8201 record') 530 return False 531 532 # update fields in request from request_data 533 for line_type in request_data.keys(): 534 # get handler 535 try: 536 handle_line = cLDTImporter.__820xline_handler[line_type] 537 except KeyError: 538 _log.LogException('no handler for line [%s:%s] in [8000:8201] record' % (line_type, request_data[line_type]), sys.exc_info(), verbose=0) 539 continue 540 # ignore line 541 if handle_line is None: 542 try: 543 name = gmXdtMappings.xdt_id_map[line_type] 544 except KeyError: 545 name = '?' 546 _log.Log(gmLog.lData, 'skipping [%s] (%s)' % (line_type, name)) 547 continue 548 # handle line 549 line_data = handle_line(self, request_data) 550 if line_data is False: 551 # FIXME: todo item 552 _log.Log(gmLog.lErr, 'failed to handle [%s] line' % line_type) 553 return False 554 try: 555 self.__request[cLDTImporter._map_820xline2req_field[line_type]] = line_data 556 except KeyError: 557 pass 558 self.__request.save_payload() 559 return True
560 #----------------------------------------------------------- 561 #----------------------------------------------------------- 562 __chunk8000_handler = { 563 # skip file header chunk 564 '8220': None, 565 '8201': __handle_8201, 566 '8202': __handle_8202 567 } 568 #-----------------------------------------------------------
569 - def __handle_8000(self, chunk):
570 try: 571 handler = cLDTImporter.__chunk8000_handler[chunk['8000'][0]] 572 except KeyError: 573 _log.Log(gmLog.lErr, 'kein Handler fuer Satztyp [8000:%s]' % chunk['8000'][0]) 574 return False 575 if handler is None: 576 return True 577 if not handler(self, chunk): 578 _log.Log(gmLog.lErr, 'kann Satz vom Typ [8000:%s] nicht importieren' % chunk['8000'][0]) 579 return False 580 return True
581 #----------------------------------------------------------- 582 #-----------------------------------------------------------
583 - def __xform_8418(self, result_data):
584 try: 585 tmp = gmXdtMappings.xdt_Teststatus_map[result_data['8418'][0].strip()] 586 except KeyError: 587 tmp = _('unknown test status [%s]') % result_data['8418'][0].strip() 588 if self.__lab_result['note_provider'] is None: 589 self.__lab_result['note_provider'] = tmp 590 else: 591 self.__lab_result['note_provider'] = "%s\n* %s *" % (self.__lab_result['note_provider'], tmp) 592 return True
593 #-----------------------------------------------------------
594 - def __xform_8428(self, result_data):
595 tmp = "[ID:%s]" % result_data['8428'][0].strip() 596 if self.__lab_result['material_detail'] is None: 597 self.__lab_result['material_detail'] = tmp 598 else: 599 self.__lab_result['material_detail'] = "%s %s" % (self.__lab_result['material_detail'], tmp) 600 return True
601 #-----------------------------------------------------------
602 - def __xform_8429(self, result_data):
603 tmp = "[IDX:%s]" % result_data['8429'][0].strip() 604 if self.__lab_result['material_detail'] is None: 605 self.__lab_result['material_detail'] = tmp 606 else: 607 self.__lab_result['material_detail'] = "%s %s" % (self.__lab_result['material_detail'], tmp) 608 return True
609 #-----------------------------------------------------------
610 - def __xform_8430(self, result_data):
611 self.__lab_result['material'] = result_data['8430'][0].strip() 612 return True
613 #-----------------------------------------------------------
614 - def __xform_8431(self, result_data):
615 tmp = '\n'.join(result_data['8431']) 616 if self.__lab_result['material_detail'] is None: 617 self.__lab_result['material_detail'] = tmp 618 else: 619 self.__lab_result['material_detail'] = "%s\n%s" % (self.__lab_result['material_detail'], tmp) 620 return True
621 #-----------------------------------------------------------
622 - def __xform_8432(self, result_data):
623 self.__request['sampled_when'] = mxDT.strptime( 624 result_data['8432'][0], 625 '%d%m%Y', 626 self.__request['sampled_when'] 627 ) 628 self.__request.save_payload()
629 #-----------------------------------------------------------
630 - def __xform_8433(self, result_data):
631 self.__request['sampled_when'] = mxDT.strptime( 632 result_data['8433'][0], 633 '%H%M', 634 self.__request['sampled_when'] 635 ) 636 self.__request.save_payload()
637 #-----------------------------------------------------------
638 - def __xform_8420(self, result_data):
639 tmp = result_data['8420'][0].strip() 640 try: 641 self.__lab_result['val_num'] = float(tmp) 642 except ValueError: 643 _log.Log(gmLog.lErr, 'angeblich numerisches Ergebnis [%s] ist nicht-numerisch, speichere als alphanumerisch' % str(result_data['8420'])) 644 if self.__lab_result['val_alpha'] is None: 645 self.__lab_result['val_alpha'] = tmp 646 else: 647 self.__lab_result['val_alpha'] = "%s\n* %s *" % (self.__lab_result['val_alpha'], tmp) 648 return True
649 #-----------------------------------------------------------
650 - def __xform_8421(self, result_data):
651 self.__lab_result['val_unit'] = result_data['8421'][0].strip() 652 return True
653 #-----------------------------------------------------------
654 - def __xform_8480(self, result_data):
655 tmp = '\n'.join(result_data['8480']) 656 if self.__lab_result['val_alpha'] is None: 657 self.__lab_result['val_alpha'] = tmp 658 else: 659 self.__lab_result['val_alpha'] = "%s\n%s" % (tmp, self.__lab_result['val_alpha']) 660 return True
661 #-----------------------------------------------------------
662 - def __xform_8470(self, result_data):
663 tmp = '\n'.join(result_data['8470']) 664 if self.__lab_result['note_provider'] is None: 665 self.__lab_result['note_provider'] = tmp 666 else: 667 self.__lab_result['note_provider'] = "%s\n%s" % (tmp, self.__lab_result['note_provider']) 668 return True
669 #-----------------------------------------------------------
670 - def __xform_8460(self, result_data):
671 self.__lab_result['val_normal_range'] = '\n'.join(result_data['8460']) 672 return True
673 #-----------------------------------------------------------
674 - def __xform_8461(self, result_data):
675 self.__lab_result['val_normal_min'] = result_data['8461'][0] 676 return True
677 #-----------------------------------------------------------
678 - def __xform_8462(self, result_data):
679 self.__lab_result['val_normal_max'] = result_data['8462'][0] 680 return True
681 #-----------------------------------------------------------
682 - def __xform_8422(self, result_data):
683 self.__lab_result['abnormal'] = result_data['8422'][0].strip() 684 return True
685 #-----------------------------------------------------------
686 - def __xform_8490(self, result_data):
687 new_comment = '\n'.join(result_data['8490']) 688 old_narrative = self.__request['progress_note'] 689 if old_narrative is None: 690 self.__request['progress_note'] = new_comment 691 self.__request.save_payload() 692 return True 693 if old_narrative.find(new_comment) == -1: 694 self.__request['progress_note'] = old_narrative + '\n' + new_comment 695 self.__request.save_payload() 696 return True
697 #----------------------------------------------------------- 698 __8410line_handler = { 699 '5001': None, 700 '8404': None, 701 '8406': None, # Kosten Cent 702 '8410': None, # don't update code 703 '8411': None, # don't update name 704 '8412': None, # Abrechnungskennung 705 '8418': __xform_8418, 706 '8428': __xform_8428, 707 '8429': __xform_8429, 708 '8430': __xform_8430, 709 '8431': __xform_8431, 710 '8432': __xform_8432, 711 '8433': __xform_8433, 712 '8420': __xform_8420, 713 '8421': __xform_8421, 714 '8480': __xform_8480, 715 '8470': __xform_8470, 716 '8460': __xform_8460, 717 '8461': __xform_8461, 718 '8462': __xform_8462, 719 '8422': __xform_8422, 720 '8490': __xform_8490 721 } 722 #-----------------------------------------------------------
723 - def __handle_8410(self, result_data):
724 if self.__request is None: 725 _log.Log(gmLog.lErr, 'Kann Labordaten nicht ohne Zuordnung importieren.') 726 return False 727 # skip pseudo results 728 if len(result_data) == 3: 729 if ('8410' in result_data and # code 730 '8411' in result_data and # name 731 '8412' in result_data): # Abrechnungskennung 732 _log.Log(gmLog.lInfo, 'skipping billing-only record') 733 return True 734 # verify essential fields 735 # - numeric result 736 try: 737 vnum = '\n'.join(result_data['8420']) 738 except KeyError: 739 vnum = None 740 # - alphanumeric result 741 try: 742 valpha = '\n'.join(result_data['8480']) 743 except KeyError: 744 valpha = None 745 if (valpha is None) and (vnum is None): 746 valpha = '' 747 # - code 748 try: 749 vcode = result_data['8410'][0] 750 except KeyError: 751 _log.Log(gmLog.lWarn, 'adding default test type code') 752 # - name 753 try: 754 vname = result_data['8411'][0] 755 except KeyError: 756 _log.Log(gmLog.lWarn, 'adding default test type name') 757 # - unit 758 try: 759 vunit = result_data['8421'][0] 760 except KeyError: 761 vunit = '' 762 # verify/create test type 763 status, ttype = gmPathLab.create_measurement_type ( 764 lab=self.__lab_name, 765 code=vcode, # must be abbrev 766 name=vname, 767 unit=vunit 768 ) 769 if status in [False, None]: 770 _log.Log(gmLog.lErr, 'cannot create/retrieve test type') 771 return False 772 if ttype['comment'] in [None, '']: 773 ttype['comment'] = 'created [%s] by [$RCSfile: gmLDTimporter.py,v $ $Revision: 1.34 $] from [%s]' % (time.strftime('%Y-%m-%d %H:%M'), self.ldt_filename) 774 ttype.save_payload() 775 # try to create test result row 776 whenfield = 'lab_rxd_when' # FIXME: make this configurable 777 status, self.__lab_result = gmPathLab.create_lab_result( 778 patient_id = self.__request.get_patient()[0], 779 when_field = whenfield, 780 when = self.__request[whenfield], 781 test_type = ttype['pk'], 782 val_num = vnum, 783 val_alpha = valpha, 784 unit = vunit, 785 request = self.__request 786 ) 787 if status is False: 788 _log.Log(gmLog.lErr, 'cannot create result record') 789 _log.Log(gmLog.lInfo, str(result_data)[:500]) 790 return False 791 # FIXME: make this configurable (whether skipping or duplicating) 792 if status is None: 793 _log.Log(gmLog.lData, 'skipping duplicate lab result on import') 794 _log.Log(gmLog.lData, 'in file: %s' % str(result_data)) 795 return True 796 # update result record from dict 797 # - force these to None so we can add in the proper values 798 self.__lab_result['val_num'] = None 799 self.__lab_result['val_alpha'] = None 800 # - preset some fields 801 self.__lab_result['reviewed'] = 'false' 802 if self.__ref_group_str != '': 803 self.__lab_result['ref_group'] = self.__ref_group_str 804 # - process lines 805 for line_type in result_data.keys(): 806 # get handler 807 try: 808 handle_line = cLDTImporter.__8410line_handler[line_type] 809 except KeyError: 810 _log.LogException('no handler for line [%s] in [8410] record' % line_type, sys.exc_info(), verbose=0) 811 return False 812 # ignore line 813 if handle_line is None: 814 continue 815 if handle_line(self, result_data) is False: 816 # FIXME: todo item 817 _log.Log(gmLog.lErr, 'cannot handle [%s] line' % line_type) 818 return False 819 if (self.__lab_result['val_alpha'] is None) and (self.__lab_result['val_num'] is None): 820 _log.Log(gmLog.lWarn, 'both result fields empty, setting alphanumeric default') 821 self.__lab_result['val_alpha'] = '' 822 saved, msg = self.__lab_result.save_payload() 823 del self.__lab_result 824 if not saved: 825 _log.Log(gmLog.lErr, 'kann Laborwert (8410) nicht importieren') 826 return False 827 _log.Log(gmLog.lInfo, 'Laborwert (8410) erfolgreich importiert') 828 return True
829 #----------------------------------------------------------- 830 #----------------------------------------------------------- 831 __chunk_handler = { 832 '8000': __handle_8000, 833 '8410': __handle_8410 834 } 835 #-----------------------------------------------------------
836 - def __import_request_file(self, filename):
837 # throw away up to 4 bits (plus the randint() cost) 838 random.getrandbits(random.randint(1,4)) 839 self.__ref_group_str = '' 840 self.__request = None 841 chunk = {} 842 for line in fileinput.input(filename): 843 tmp = line.replace('\r','') 844 tmp = tmp.replace('\n','') 845 line_type = tmp[3:7] 846 line_data = tmp[7:] 847 if line_type in cLDTImporter._chunk_starters: 848 # process any previous data 849 if len(chunk) != 0: 850 # get handler 851 try: 852 handle_chunk = cLDTImporter.__chunk_handler[chunk_type] 853 except KeyError: 854 fileinput.close() 855 if self.__request is not None: 856 self.__request['request_status'] = 'partial' 857 self.__request['is_pending'] = 'true' 858 self.__request.save_payload() 859 _log.Log(gmLog.lErr, 'kein Handler fuer Satztyp [%s] verfuegbar' % chunk_type) 860 return False 861 # handle chunk 862 if not handle_chunk(self, chunk): 863 if self.__request is not None: 864 self.__request['request_status'] = 'partial' 865 self.__request['is_pending'] = 'true' 866 self.__request.save_payload() 867 fileinput.close() 868 _log.Log(gmLog.lErr, 'cannot handle [%s] chunk' % chunk_type) 869 return False 870 # start new chunk 871 chunk = {} 872 chunk_type = line_type 873 # FIXME: max line count 874 if line_type not in chunk: 875 chunk[line_type] = [] 876 chunk[line_type].append(line_data) 877 fileinput.close() 878 return True
879 #=============================================================== 880 #--------------------------------------------------------------- 881 # LDT functions 882 #--------------------------------------------------------------- 883 #def _verify_9221(a_line, field_data): 884 # FIXME: check minor version, too 885 # return True 886 #--------------------------------------------------------------- 887 #def _verify_0203(a_line, field_data): 888 # FIXME: validate against database 889 # return True 890 #---------------------------------------------------------------
891 -def run_import():
892 # make sure files can moved to the target repository 893 target_dir = _cfg.get('target', 'repository') 894 if target_dir is None: 895 return False 896 target_dir = os.path.abspath(os.path.expanduser(target_dir)) 897 if not (os.access(target_dir, os.W_OK) and os.path.isdir(target_dir)): 898 _log.Log(gmLog.lErr, 'cannot write to target repository [%s]\n' % target_dir) 899 return False 900 # get import files 901 import_dir = _cfg.get('import', 'repository') 902 if import_dir is None: 903 _log.Log(gmLog.lErr, 'no import dir found\n') 904 return False 905 import_dir = os.path.abspath(os.path.expanduser(import_dir)) 906 filename_pattern = _cfg.get('import', 'file pattern') 907 if filename_pattern is None: 908 _log.Log(gmLog.lErr, 'no import file name pattern found\n') 909 return False 910 import_file_pattern = os.path.join(import_dir, filename_pattern) 911 files2import = glob.glob(import_file_pattern) 912 # now import 913 importer = cLDTImporter(cfg=_cfg) 914 for ldt_file in files2import: 915 _log.Log(gmLog.lInfo, 'importing LDT file [%s]' % ldt_file) 916 if not importer.import_file(ldt_file): 917 _log.Log(gmLog.lErr, 'cannot import LDT file\n') 918 else: 919 _log.Log(gmLog.lInfo, 'success importing LDT file\n') 920 try: 921 shutil.copy(ldt_file, target_dir) 922 os.remove(ldt_file) 923 except Exception: 924 _log.LogException('cannot move [%s] to [%s]\n' % (ldt_file, target_dir)) 925 return True
926 #---------------------------------------------------------------
927 -def add_todo(problem, solution, context):
928 cat = 'lab' 929 by = '$RCSfile: gmLDTimporter.py,v $ $Revision: 1.34 $' 930 rcvr = 'user' 931 gmPG.add_housekeeping_todo(reporter=by, receiver=rcvr, problem=problem, solution=solution, context=context, category=cat)
932 #=============================================================== 933 # main 934 #--------------------------------------------------------------- 935 if __name__ == '__main__': 936 if _cfg is None: 937 _log.Log(gmLog.lErr, 'need config file to run') 938 sys.exit(1) 939 # set encoding 940 # setup login defs 941 auth_data = gmLoginInfo.LoginInfo( 942 user = _cfg.get('database', 'user'), 943 password = _cfg.get('database', 'password'), 944 host = _cfg.get('database', 'host'), 945 port = _cfg.get('database', 'port'), 946 database = _cfg.get('database', 'database') 947 ) 948 backend = gmPG.ConnectionPool(login = auth_data) 949 # actually run the import 950 try: 951 import profile 952 profile.run('run_import()', './profile.log') 953 except Exception: 954 _log.LogException('unhandled exception caught', sys.exc_info(), verbose=1) 955 backend.StopListeners() 956 sys.exit('aborting') 957 backend.StopListeners() 958 sys.exit(0) 959 960 #=============================================================== 961