Package Gnumed :: Package business :: Module gmHL7
[frames] | no frames]

Source Code for Module Gnumed.business.gmHL7

   1  # -*- coding: utf-8 -*- 
   2  """Some HL7 handling.""" 
   3  #============================================================ 
   4  __author__ = "K.Hilbert <Karsten.Hilbert@gmx.net>" 
   5  __license__ = "GPL v2 or later" 
   6   
   7   
   8  import sys 
   9  import os 
  10  import io 
  11  import logging 
  12  import time 
  13  import shutil 
  14  import datetime as pyDT 
  15  import hl7 as pyhl7 
  16  from xml.etree import ElementTree as pyxml 
  17   
  18   
  19  if __name__ == '__main__': 
  20          sys.path.insert(0, '../../') 
  21   
  22  from Gnumed.pycommon import gmI18N 
  23  if __name__ == '__main__': 
  24          gmI18N.activate_locale() 
  25          gmI18N.install_domain() 
  26  from Gnumed.pycommon import gmTools 
  27  from Gnumed.pycommon import gmBusinessDBObject 
  28  from Gnumed.pycommon import gmPG2 
  29  from Gnumed.pycommon import gmDateTime 
  30   
  31  from Gnumed.business import gmIncomingData 
  32  from Gnumed.business import gmPathLab 
  33  from Gnumed.business import gmPerson 
  34  from Gnumed.business import gmPraxis 
  35  from Gnumed.business import gmStaff 
  36   
  37   
  38  _log = logging.getLogger('gm.hl7') 
  39   
  40  # constants 
  41  HL7_EOL = '\r' 
  42  HL7_BRK = '\.br\\' 
  43   
  44  HL7_SEGMENTS = 'FHS BHS MSH PID PV1 OBX NTE ORC OBR'.split() 
  45   
  46  HL7_segment2field_count = { 
  47          'FHS': 12, 
  48          'BHS': 12, 
  49          'MSH': 19, 
  50          'PID': 30, 
  51          'PV1': 52, 
  52          'OBR': 43, 
  53          'OBX': 17, 
  54          'NTE': 3, 
  55          'ORC': 19 
  56  } 
  57   
  58  MSH_field__sending_lab = 3 
  59   
  60  PID_field__name = 5 
  61  PID_field__dob = 7 
  62  PID_field__gender = 8 
  63  PID_component__lastname = 1 
  64  PID_component__firstname = 2 
  65  PID_component__middlename = 3 
  66   
  67  OBR_field__service_name = 4 
  68  OBR_field__ts_requested = 6 
  69  OBR_field__ts_started = 7 
  70  OBR_field__ts_ended = 8 
  71  OBR_field__ts_specimen_received = 14 
  72   
  73  OBX_field__set_id = 1 
  74  OBX_field__datatype = 2 
  75  OBX_field__type = 3 
  76  # components of 3rd field: 
  77  OBX_component__loinc = 1 
  78  OBX_component__name = 2 
  79  OBX_field__subid = 4 
  80  OBX_field__value = 5 
  81  OBX_field__unit = 6 
  82  OBX_field__range = 7 
  83  OBX_field__abnormal_flag = 8 
  84  OBX_field__status = 11 
  85  OBX_field__timestamp = 14 
  86   
  87  NET_field__set_id = 1 
  88  NET_field__src = 2 
  89  NET_field__note = 3 
  90   
  91  HL7_field_labels = { 
  92          'MSH': { 
  93                  0: 'Segment Type', 
  94                  1: 'Field Separator', 
  95                  2: 'Encoding Characters', 
  96                  3: 'Sending Application', 
  97                  4: 'Sending Facility', 
  98                  5: 'Receiving Application', 
  99                  6: 'Receiving Facility', 
 100                  7: 'Date/Time of Message', 
 101                  8: 'Security', 
 102                  9: 'Message Type', 
 103                  10: 'ID: Message Control', 
 104                  11: 'ID: Processing', 
 105                  12: 'ID: Version', 
 106                  14: 'Continuation Pointer', 
 107                  15: 'Accept Acknowledgement Type', 
 108                  16: 'Application Acknowledgement Type' 
 109          }, 
 110          'PID': { 
 111                  0: 'Segment Type', 
 112                  1: '<PID> Set ID', 
 113                  2: 'Patient ID (external)', 
 114                  3: 'Patient ID (internal)', 
 115                  4: 'Patient ID (alternate)', 
 116                  5: 'Patient Name', 
 117                  7: 'Date/Time of birth', 
 118                  8: 'Administrative Gender', 
 119                  11: 'Patient Address', 
 120                  13: 'Patient Phone Number - Home' 
 121          }, 
 122          'OBR': { 
 123                  0: 'Segment Type', 
 124                  1: 'ID: Set', 
 125                  3: 'Filler Order Number (= ORC-3)', 
 126                  4: 'ID: Universal Service', 
 127                  5: 'Priority', 
 128                  6: 'Date/Time requested', 
 129                  7: 'Date/Time Observation started', 
 130                  14: 'Date/Time Specimen received', 
 131                  16: 'Ordering Provider', 
 132                  18: 'Placer Field 1', 
 133                  20: 'Filler Field 1', 
 134                  21: 'Filler Field 2', 
 135                  22: 'Date/Time Results reported/Status changed', 
 136                  24: 'ID: Diagnostic Service Section', 
 137                  25: 'Result Status', 
 138                  27: 'Quantity/Timing', 
 139                  28: 'Result Copies To' 
 140          }, 
 141          'ORC': { 
 142                  0: 'Segment Type', 
 143                  1: 'Order Control', 
 144                  3: 'Filler Order Number', 
 145                  12: 'Ordering Provider' 
 146          }, 
 147          'OBX': { 
 148                  0: 'Segment Type', 
 149                  1: 'Set ID', 
 150                  2: 'Value Type', 
 151                  3: 'Identifier (LOINC)', 
 152                  4: 'Observation Sub-ID', 
 153                  5: 'Value', 
 154                  6: 'Units', 
 155                  7: 'References Range (Low - High)', 
 156                  8: 'Abnormal Flags', 
 157                  11: 'Result Status', 
 158                  14: 'Date/Time of Observation' 
 159          }, 
 160          'NTE': { 
 161                  0: 'Segment Type', 
 162                  3: 'Comment' 
 163          } 
 164  } 
 165   
 166  HL7_GENDERS = { 
 167          'F': 'f', 
 168          'M': 'm', 
 169          'O': None, 
 170          'U': None, 
 171          None: None 
 172  } 
 173   
 174  #============================================================ 
 175  # public API 
 176  #============================================================ 
177 -def extract_HL7_from_XML_CDATA(filename, xml_path, target_dir=None):
178 179 _log.debug('extracting HL7 from CDATA of <%s> nodes in XML file [%s]', xml_path, filename) 180 181 # sanity checks/setup 182 try: 183 open(filename).close() 184 orig_dir = os.path.split(filename)[0] 185 work_filename = gmTools.get_unique_filename(prefix = 'gm-x2h-%s-' % gmTools.fname_stem(filename), suffix = '.hl7') 186 if target_dir is None: 187 target_dir = os.path.join(orig_dir, 'HL7') 188 done_dir = os.path.join(orig_dir, 'done') 189 else: 190 done_dir = os.path.join(target_dir, 'done') 191 _log.debug('target dir: %s', target_dir) 192 gmTools.mkdir(target_dir) 193 gmTools.mkdir(done_dir) 194 except Exception: 195 _log.exception('cannot setup unwrapping environment') 196 return None 197 198 hl7_xml = pyxml.ElementTree() 199 try: 200 hl7_xml.parse(filename) 201 except pyxml.ParseError: 202 _log.exception('cannot parse [%s]' % filename) 203 return None 204 nodes = hl7_xml.findall(xml_path) 205 if len(nodes) == 0: 206 _log.debug('no nodes found for data extraction') 207 return None 208 209 _log.debug('unwrapping HL7 from XML into [%s]', work_filename) 210 hl7_file = io.open(work_filename, mode = 'wt', encoding = 'utf8', newline = '') # universal newlines acceptance but no translation on output 211 for node in nodes: 212 # hl7_file.write(node.text.rstrip() + HL7_EOL) 213 hl7_file.write(node.text + '') # trick to make node.text unicode 214 hl7_file.close() 215 216 target_fname = os.path.join(target_dir, os.path.split(work_filename)[1]) 217 shutil.copy(work_filename, target_dir) 218 shutil.move(filename, done_dir) 219 220 return target_fname
221 222 #------------------------------------------------------------
223 -def split_hl7_file(filename, target_dir=None, encoding='utf8'):
224 """Multi-step processing of HL7 files. 225 226 - input can be multi-MSH / multi-PID / partially malformed HL7 227 - tries to fix oddities 228 - splits by MSH 229 - splits by PID into <target_dir> 230 231 - needs write permissions in dir_of(filename) 232 - moves HL7 files which were successfully split up into dir_of(filename)/done/ 233 234 - returns (True|False, list_of_PID_files) 235 """ 236 local_log_name = gmTools.get_unique_filename ( 237 prefix = gmTools.fname_stem(filename) + '-', 238 suffix = '.split.log' 239 ) 240 local_logger = logging.FileHandler(local_log_name) 241 local_logger.setLevel(logging.DEBUG) 242 root_logger = logging.getLogger('') 243 root_logger.addHandler(local_logger) 244 _log.info('splitting HL7 file: %s', filename) 245 _log.debug('log file: %s', local_log_name) 246 247 # sanity checks/setup 248 try: 249 open(filename).close() 250 orig_dir = os.path.split(filename)[0] 251 done_dir = os.path.join(orig_dir, 'done') 252 gmTools.mkdir(done_dir) 253 error_dir = os.path.join(orig_dir, 'failed') 254 gmTools.mkdir(error_dir) 255 work_filename = gmTools.get_unique_filename(prefix = gmTools.fname_stem(filename) + '-', suffix = '.hl7') 256 if target_dir is None: 257 target_dir = os.path.join(orig_dir, 'PID') 258 _log.debug('target dir: %s', target_dir) 259 gmTools.mkdir(target_dir) 260 except Exception: 261 _log.exception('cannot setup splitting environment') 262 root_logger.removeHandler(local_logger) 263 return False, None 264 265 # split 266 target_names = [] 267 try: 268 shutil.copy(filename, work_filename) 269 fixed_filename = __fix_malformed_hl7_file(work_filename, encoding = encoding) 270 MSH_fnames = __split_hl7_file_by_MSH(fixed_filename, encoding) 271 PID_fnames = [] 272 for MSH_fname in MSH_fnames: 273 PID_fnames.extend(__split_MSH_by_PID(MSH_fname)) 274 for PID_fname in PID_fnames: 275 shutil.move(PID_fname, target_dir) 276 target_names.append(os.path.join(target_dir, os.path.split(PID_fname)[1])) 277 except Exception: 278 _log.exception('cannot split HL7 file') 279 for target_name in target_names: 280 try: os.remove(target_name) 281 except Exception: pass 282 root_logger.removeHandler(local_logger) 283 shutil.move(local_log_name, error_dir) 284 return False, None 285 286 _log.info('successfully split') 287 root_logger.removeHandler(local_logger) 288 try: 289 shutil.move(filename, done_dir) 290 shutil.move(local_log_name, done_dir) 291 except shutil.Error: 292 _log.exception('cannot move hl7 file or log file to holding area') 293 return True, target_names
294 295 #------------------------------------------------------------
296 -def format_hl7_message(message=None, skip_empty_fields=True, eol='\n ', source=None):
297 # a segment is a line starting with a type 298 msg = pyhl7.parse(message) 299 300 output = [] 301 if source is not None: 302 output.append([_('HL7 Source'), '%s' % source]) 303 output.append([_('HL7 data size'), _('%s bytes') % len(message)]) 304 output.append([_('HL7 Message'), _(' %s segments (lines)%s') % (len(msg), gmTools.bool2subst(skip_empty_fields, _(', skipping empty fields'), ''))]) 305 306 max_len = 0 307 for seg_idx in range(len(msg)): 308 seg = msg[seg_idx] 309 seg_type = seg[0][0] 310 311 output.append([_('Segment #%s <%s>') % (seg_idx, seg_type), _('%s fields') % len(seg)]) 312 313 for field_idx in range(len(seg)): 314 field = seg[field_idx] 315 try: 316 label = HL7_field_labels[seg_type][field_idx] 317 except KeyError: 318 label = _('HL7 %s field') % seg_type 319 320 max_len = max(max_len, len(label)) 321 322 if len(field) == 0: 323 if not skip_empty_fields: 324 output.append(['%2s - %s' % (field_idx, label), _('<EMTPY>')]) 325 continue 326 if (len(field) == 1) and (('%s' % field[0]).strip() == ''): 327 if not skip_empty_fields: 328 output.append(['%2s - %s' % (field_idx, label), _('<EMTPY>')]) 329 continue 330 331 content_lines = ('%s' % field).split(HL7_BRK) 332 output.append(['%2s - %s' % (field_idx, label), content_lines[0]]) 333 for line in content_lines[1:]: 334 output.append(['', line]) 335 #output.append([u'%2s - %s' % (field_idx, label), u'%s' % field]) 336 337 if eol is None: 338 return output 339 340 max_len += 7 341 return eol.join([ '%s: %s' % ((o[0] + (' ' * max_len))[:max_len], o[1]) for o in output ])
342 343 #------------------------------------------------------------
344 -def format_hl7_file(filename, skip_empty_fields=True, eol='\n ', return_filename=False, fix_hl7=True):
345 if fix_hl7: 346 fixed_name = __fix_malformed_hl7_file(filename) 347 hl7_file = io.open(fixed_name, mode = 'rt', encoding = 'utf8', newline = '') # read universal but pass on untranslated 348 source = '%s (<- %s)' % (fixed_name, filename) 349 else: 350 hl7_file = io.open(filename, mode = 'rt', encoding = 'utf8', newline = '') # read universal but pass on untranslated 351 source = filename 352 output = format_hl7_message ( 353 message = hl7_file.read(1024 * 1024 * 5), # 5 MB max 354 skip_empty_fields = skip_empty_fields, 355 eol = eol, 356 source = source 357 ) 358 hl7_file.close() 359 360 if not return_filename: 361 return output 362 363 if eol is None: 364 output = '\n '.join([ '%s: %s' % ((o[0] + (' ' * max_len))[:max_len], o[1]) for o in output ]) 365 366 out_name = gmTools.get_unique_filename(prefix = 'gm-formatted_hl7-', suffix = '.hl7') 367 out_file = io.open(out_name, mode = 'wt', encoding = 'utf8') 368 out_file.write(output) 369 out_file.close() 370 371 return out_name
372 373 #------------------------------------------------------------ 374 # this is used in the main code:
375 -def stage_single_PID_hl7_file(filename, source=None, encoding='utf8'):
376 """Multi-step processing of HL7 files. 377 378 - input must be single-MSH / single-PID / normalized HL7 379 380 - imports into clin.incoming_data_unmatched 381 382 - needs write permissions in dir_of(filename) 383 - moves PID files which were successfully staged into dir_of(filename)/done/PID/ 384 """ 385 local_log_name = gmTools.get_unique_filename ( 386 prefix = gmTools.fname_stem(filename) + '-', 387 suffix = '.stage.log' 388 ) 389 local_logger = logging.FileHandler(local_log_name) 390 local_logger.setLevel(logging.DEBUG) 391 root_logger = logging.getLogger('') 392 root_logger.addHandler(local_logger) 393 _log.info('staging [%s] as unmatched incoming HL7%s', filename, gmTools.coalesce(source, '', ' (%s)')) 394 _log.debug('log file: %s', local_log_name) 395 396 # sanity checks/setup 397 try: 398 open(filename).close() 399 orig_dir = os.path.split(filename)[0] 400 done_dir = os.path.join(orig_dir, 'done') 401 gmTools.mkdir(done_dir) 402 error_dir = os.path.join(orig_dir, 'failed') 403 gmTools.mkdir(error_dir) 404 except Exception: 405 _log.exception('cannot setup staging environment') 406 root_logger.removeHandler(local_logger) 407 return False 408 409 # stage 410 try: 411 incoming = gmIncomingData.create_incoming_data('HL7%s' % gmTools.coalesce(source, '', ' (%s)'), filename) 412 if incoming is None: 413 _log.error('cannot stage PID file: %s', filename) 414 root_logger.removeHandler(local_logger) 415 shutil.move(filename, error_dir) 416 shutil.move(local_log_name, error_dir) 417 return False 418 incoming.update_data_from_file(fname = filename) 419 except Exception: 420 _log.exception('error staging PID file') 421 root_logger.removeHandler(local_logger) 422 shutil.move(filename, error_dir) 423 shutil.move(local_log_name, error_dir) 424 return False 425 426 # set additional data 427 MSH_file = io.open(filename, mode = 'rt', encoding = 'utf8', newline = '') 428 raw_hl7 = MSH_file.read(1024 * 1024 * 5) # 5 MB max 429 MSH_file.close() 430 shutil.move(filename, done_dir) 431 incoming['comment'] = format_hl7_message ( 432 message = raw_hl7, 433 skip_empty_fields = True, 434 eol = '\n' 435 ) 436 HL7 = pyhl7.parse(raw_hl7) 437 del raw_hl7 438 incoming['comment'] += '\n' 439 incoming['comment'] += ('-' * 80) 440 incoming['comment'] += '\n\n' 441 log = io.open(local_log_name, mode = 'rt', encoding = 'utf8') 442 incoming['comment'] += log.read() 443 log.close() 444 try: 445 incoming['lastnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname) 446 incoming['firstnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__firstname) 447 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__middlename) 448 if val is not None: 449 incoming['firstnames'] += ' ' 450 incoming['firstnames'] += val 451 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__dob) 452 if val is not None: 453 tmp = time.strptime(val, '%Y%m%d') 454 incoming['dob'] = pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone) 455 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__gender) 456 if val is not None: 457 incoming['gender'] = val 458 incoming['external_data_id'] = filename 459 #u'fk_patient_candidates', 460 # u'request_id', # request ID as found in <data> 461 # u'postcode', 462 # u'other_info', # other identifying info in .data 463 # u'requestor', # Requestor of data (e.g. who ordered test results) if available in source data. 464 # u'fk_identity_disambiguated', 465 # u'comment', # a free text comment on this row, eg. why is it here, error logs etc 466 # u'fk_provider_disambiguated' # The provider the data is relevant to. 467 except Exception: 468 _log.exception('cannot add more data') 469 incoming.save() 470 471 _log.info('successfully staged') 472 root_logger.removeHandler(local_logger) 473 shutil.move(local_log_name, done_dir) 474 return True
475 476 #------------------------------------------------------------
477 -def process_staged_single_PID_hl7_file(staged_item):
478 479 log_name = gmTools.get_unique_filename ( 480 prefix = 'gm-staged_hl7_import-', 481 suffix = '.log' 482 ) 483 import_logger = logging.FileHandler(log_name) 484 import_logger.setLevel(logging.DEBUG) 485 root_logger = logging.getLogger('') 486 root_logger.addHandler(import_logger) 487 _log.debug('log file: %s', log_name) 488 489 if not staged_item.lock(): 490 _log.error('cannot lock staged data for HL7 import') 491 root_logger.removeHandler(import_logger) 492 return False, log_name 493 494 _log.debug('reference ID of staged HL7 data: %s', staged_item['external_data_id']) 495 496 filename = staged_item.save_to_file() 497 _log.debug('unstaged HL7 data into: %s', filename) 498 499 if staged_item['pk_identity_disambiguated'] is None: 500 emr = None 501 else: 502 emr = gmPerson.cPatient(staged_item['pk_identity_disambiguated']).emr 503 504 success = False 505 try: 506 success = __import_single_PID_hl7_file(filename, emr = emr) 507 if success: 508 gmIncomingData.delete_incoming_data(pk_incoming_data = staged_item['pk_incoming_data_unmatched']) 509 staged_item.unlock() 510 root_logger.removeHandler(import_logger) 511 return True, log_name 512 _log.error('error when importing single-PID/single-MSH file') 513 except Exception: 514 _log.exception('error when importing single-PID/single-MSH file') 515 516 if not success: 517 staged_item['comment'] = _('failed import: %s\n') % gmDateTime.pydt_strftime(gmDateTime.pydt_now_here()) 518 staged_item['comment'] += '\n' 519 staged_item['comment'] += ('-' * 80) 520 staged_item['comment'] += '\n\n' 521 log = io.open(log_name, mode = 'rt', encoding = 'utf8') 522 staged_item['comment'] += log.read() 523 log.close() 524 staged_item['comment'] += '\n' 525 staged_item['comment'] += ('-' * 80) 526 staged_item['comment'] += '\n\n' 527 staged_item['comment'] += format_hl7_file ( 528 filename, 529 skip_empty_fields = True, 530 eol = '\n ', 531 return_filename = False 532 ) 533 staged_item.save() 534 535 staged_item.unlock() 536 root_logger.removeHandler(import_logger) 537 return success, log_name
538 539 #------------------------------------------------------------
540 -def import_single_PID_hl7_file(filename):
541 542 log_name = '%s.import.log' % filename 543 import_logger = logging.FileHandler(log_name) 544 import_logger.setLevel(logging.DEBUG) 545 root_logger = logging.getLogger('') 546 root_logger.addHandler(import_logger) 547 _log.debug('log file: %s', log_name) 548 549 success = True 550 try: 551 success = __import_single_PID_hl7_file(filename) 552 if not success: 553 _log.error('error when importing single-PID/single-MSH file') 554 except Exception: 555 _log.exception('error when importing single-PID/single-MSH file') 556 557 root_logger.removeHandler(import_logger) 558 return success, log_name
559 560 #============================================================ 561 # internal helpers 562 #============================================================
563 -def __fix_malformed_hl7_file(filename, encoding='utf8'):
564 565 _log.debug('fixing HL7 file [%s]', filename) 566 567 # first pass: 568 # - remove empty lines 569 # - normalize line endings 570 # - unwrap wrapped segments (based on the assumption that segments are wrapped until a line starts with a known segment marker) 571 out1_fname = gmTools.get_unique_filename ( 572 prefix = 'gm_fix1-%s-' % gmTools.fname_stem(filename), 573 suffix = '.hl7' 574 ) 575 hl7_in = io.open(filename, mode = 'rt', encoding = encoding) # universal newlines: translate any type of EOL to \n 576 hl7_out = io.open(out1_fname, mode = 'wt', encoding = 'utf8', newline = '') # newline='' -> no translation of EOL at all 577 is_first_line = True 578 for line in hl7_in: 579 # skip empty line 580 if line.strip() == '': 581 continue 582 # starts with known segment ? 583 segment = line[:3] 584 if (segment in HL7_SEGMENTS) and (line[3] == '|'): 585 if not is_first_line: 586 hl7_out.write(HL7_EOL) 587 else: 588 is_first_line = False 589 else: 590 hl7_out.write(' ') 591 hl7_out.write(line.rstrip()) 592 hl7_out.write(HL7_EOL) 593 hl7_out.close() 594 hl7_in.close() 595 596 # second pass: 597 # - normalize # of fields per line 598 # - remove '\.br.\'-only fields ;-) 599 out2_fname = gmTools.get_unique_filename ( 600 prefix = 'gm_fix2-%s-' % gmTools.fname_stem(filename), 601 suffix = '.hl7' 602 ) 603 # we can now _expect_ lines to end in HL7_EOL, anything else is an error 604 hl7_in = io.open(out1_fname, mode = 'rt', encoding = 'utf8', newline = HL7_EOL) 605 hl7_out = io.open(out2_fname, mode = 'wt', encoding = 'utf8', newline = '') 606 for line in hl7_in: 607 line = line.strip() 608 seg_type = line[:3] # assumption: field separator = '|' 609 field_count = line.count('|') + 1 # assumption: no '|' in data ... 610 try: 611 required_fields = HL7_segment2field_count[seg_type] 612 except KeyError: 613 required_fields = field_count 614 missing_fields_count = required_fields - field_count 615 if missing_fields_count > 0: 616 line += ('|' * missing_fields_count) 617 cleaned_fields = [] 618 for field in line.split('|'): 619 if field.replace(HL7_BRK, '').strip() == '': 620 cleaned_fields.append('') 621 continue 622 cleaned = gmTools.strip_prefix(field, HL7_BRK, remove_repeats = True, remove_whitespace = True) 623 cleaned = gmTools.strip_suffix(cleaned, HL7_BRK, remove_repeats = True, remove_whitespace = True) 624 cleaned_fields.append(cleaned) 625 hl7_out.write('|'.join(cleaned_fields) + HL7_EOL) 626 hl7_out.close() 627 hl7_in.close() 628 629 # third pass: 630 # - unsplit same-name, same-time, text-type OBX segments 631 out3_fname = gmTools.get_unique_filename ( 632 prefix = 'gm_fix3-%s-' % gmTools.fname_stem(filename), 633 suffix = '.hl7' 634 ) 635 # we can now _expect_ lines to end in HL7_EOL, anything else is an error 636 hl7_in = io.open(out2_fname, mode = 'rt', encoding = 'utf8', newline = HL7_EOL) 637 hl7_out = io.open(out3_fname, mode = 'wt', encoding = 'utf8', newline = '') 638 prev_identity = None 639 prev_fields = None 640 for line in hl7_in: 641 line = line.strip() 642 if not line.startswith('OBX|'): 643 if prev_fields is not None: 644 hl7_out.write('|'.join(prev_fields) + HL7_EOL) 645 hl7_out.write(line + HL7_EOL) 646 prev_identity = None 647 prev_fields = None 648 curr_fields = None 649 continue 650 # first OBX 651 curr_fields = line.split('|') 652 if curr_fields[OBX_field__datatype] != 'FT': 653 hl7_out.write(line + HL7_EOL) 654 prev_identity = None 655 prev_fields = None 656 curr_fields = None 657 continue 658 # first FT type OBX 659 if prev_fields is None: 660 prev_fields = line.split('|') 661 prev_identity = line.split('|') 662 prev_identity[OBX_field__set_id] = '' 663 prev_identity[OBX_field__subid] = '' 664 prev_identity[OBX_field__value] = '' 665 prev_identity = '|'.join(prev_identity) 666 continue 667 # non-first FT type OBX 668 curr_identity = line.split('|') 669 curr_identity[OBX_field__set_id] = '' 670 curr_identity[OBX_field__subid] = '' 671 curr_identity[OBX_field__value] = '' 672 curr_identity = '|'.join(curr_identity) 673 if curr_identity != prev_identity: 674 # write out previous line 675 hl7_out.write('|'.join(prev_fields) + HL7_EOL) 676 # keep current fields, since it may start a "repeat FT type OBX block" 677 prev_fields = curr_fields 678 prev_identity = curr_identity 679 continue 680 if prev_fields[OBX_field__value].endswith(HL7_BRK): 681 prev_fields[OBX_field__value] += curr_fields[OBX_field__value] 682 else: 683 if curr_fields[OBX_field__value].startswith(HL7_BRK): 684 prev_fields[OBX_field__value] += curr_fields[OBX_field__value] 685 else: 686 prev_fields[OBX_field__value] += HL7_BRK 687 prev_fields[OBX_field__value] += curr_fields[OBX_field__value] 688 if prev_fields is not None: 689 hl7_out.write('|'.join(prev_fields) + HL7_EOL) 690 hl7_out.close() 691 hl7_in.close() 692 693 return out3_fname
694 695 #------------------------------------------------------------
696 -def __split_hl7_file_by_MSH(filename, encoding='utf8'):
697 698 _log.debug('splitting [%s] into single-MSH files', filename) 699 700 hl7_in = io.open(filename, mode = 'rt', encoding = encoding) 701 702 idx = 0 703 first_line = True 704 MSH_file = None 705 MSH_fnames = [] 706 for line in hl7_in: 707 line = line.strip() 708 # first line must be MSH 709 if first_line: 710 # ignore empty / FHS / BHS lines 711 if line == '': 712 continue 713 if line.startswith('FHS|'): 714 _log.debug('ignoring FHS') 715 continue 716 if line.startswith('BHS|'): 717 _log.debug('ignoring BHS') 718 continue 719 if not line.startswith('MSH|'): 720 raise ValueError('HL7 file <%s> does not start with "MSH" line' % filename) 721 first_line = False 722 # start new file 723 if line.startswith('MSH|'): 724 if MSH_file is not None: 725 MSH_file.close() 726 idx += 1 727 out_fname = gmTools.get_unique_filename(prefix = '%s-MSH_%s-' % (gmTools.fname_stem(filename), idx), suffix = 'hl7') 728 _log.debug('writing message %s to [%s]', idx, out_fname) 729 MSH_fnames.append(out_fname) 730 MSH_file = io.open(out_fname, mode = 'wt', encoding = 'utf8', newline = '') 731 # ignore BTS / FTS lines 732 if line.startswith('BTS|'): 733 _log.debug('ignoring BTS') 734 continue 735 if line.startswith('FTS|'): 736 _log.debug('ignoring FTS') 737 continue 738 # else write line to new file 739 MSH_file.write(line + HL7_EOL) 740 741 if MSH_file is not None: 742 MSH_file.close() 743 hl7_in.close() 744 745 return MSH_fnames
746 747 #------------------------------------------------------------
748 -def __split_MSH_by_PID(filename):
749 """Assumes: 750 - ONE MSH per file 751 - utf8 encoding 752 - first non-empty line must be MSH line 753 - next line must be PID line 754 755 IOW, what's created by __split_hl7_file_by_MSH() 756 """ 757 _log.debug('splitting single-MSH file [%s] into single-PID files', filename) 758 759 MSH_in = io.open(filename, mode = 'rt', encoding = 'utf8') 760 761 looking_for_MSH = True 762 MSH_line = None 763 looking_for_first_PID = True 764 PID_file = None 765 PID_fnames = [] 766 idx = 0 767 for line in MSH_in: 768 line = line.strip() 769 # ignore empty 770 if line == '': 771 continue 772 773 # first non-empty line must be MSH 774 if looking_for_MSH: 775 if line.startswith('MSH|'): 776 looking_for_MSH = False 777 MSH_line = line + HL7_EOL 778 continue 779 raise ValueError('HL7 MSH file <%s> does not start with "MSH" line' % filename) 780 else: 781 if line.startswith('MSH|'): 782 raise ValueError('HL7 single-MSH file <%s> contains more than one MSH line' % filename) 783 784 # first non-empty line after MSH must be PID 785 if looking_for_first_PID: 786 if not line.startswith('PID|'): 787 raise ValueError('HL7 MSH file <%s> does not have "PID" line follow "MSH" line' % filename) 788 looking_for_first_PID = False 789 790 # start new file if line is PID 791 if line.startswith('PID|'): 792 if PID_file is not None: 793 PID_file.close() 794 idx += 1 795 out_fname = gmTools.get_unique_filename(prefix = '%s-PID_%s-' % (gmTools.fname_stem(filename), idx), suffix = 'hl7') 796 _log.debug('writing message for PID %s to [%s]', idx, out_fname) 797 PID_fnames.append(out_fname) 798 PID_file = io.open(out_fname, mode = 'wt', encoding = 'utf8', newline = '') 799 PID_file.write(MSH_line) 800 # else write line to new file 801 PID_file.write(line + HL7_EOL) 802 803 if PID_file is not None: 804 PID_file.close() 805 MSH_in.close() 806 807 return PID_fnames
808 809 #------------------------------------------------------------
810 -def __find_or_create_lab(hl7_lab, link_obj=None):
811 comment_tag = '[HL7 name::%s]' % hl7_lab 812 for gm_lab in gmPathLab.get_test_orgs(): 813 if comment_tag in gmTools.coalesce(gm_lab['comment'], ''): 814 _log.debug('found lab [%s] from HL7 file in GNUmed database:', hl7_lab) 815 _log.debug(gm_lab) 816 return gm_lab 817 _log.debug('lab not found: %s', hl7_lab) 818 gm_lab = gmPathLab.create_test_org(link_obj = link_obj, name = hl7_lab, comment = comment_tag) 819 if gm_lab is None: 820 raise ValueError('cannot create lab [%s] in GNUmed' % hl7_lab) 821 _log.debug('created lab: %s', gm_lab) 822 return gm_lab
823 824 #------------------------------------------------------------
825 -def __find_or_create_test_type(loinc=None, name=None, pk_lab=None, unit=None, link_obj=None, abbrev=None):
826 827 tt = gmPathLab.find_measurement_type(link_obj = link_obj, lab = pk_lab, name = name) 828 if tt is None: 829 _log.debug('test type [%s::%s::%s] not found for lab #%s, creating', name, unit, loinc, pk_lab) 830 tt = gmPathLab.create_measurement_type(link_obj = link_obj, lab = pk_lab, abbrev = gmTools.coalesce(abbrev, name), unit = unit, name = name) 831 _log.debug('created as: %s', tt) 832 833 if loinc is None: 834 return tt 835 if loinc.strip() == '': 836 return tt 837 if tt['loinc'] is None: 838 tt['loinc'] = loinc 839 tt.save(conn = link_obj) 840 return tt 841 if tt['loinc'] != loinc: 842 # raise ValueError('LOINC code mismatch between GM (%s) and HL7 (%s) for result type [%s]' % (tt['loinc'], loinc, name)) 843 _log.error('LOINC code mismatch between GM (%s) and HL7 (%s) for result type [%s]', tt['loinc'], loinc, name) 844 845 return tt
846 847 #------------------------------------------------------------
848 -def __ensure_hl7_test_types_exist_in_gnumed(link_obj=None, hl7_data=None, pk_test_org=None):
849 try: 850 OBX_count = len(hl7_data.segments('OBX')) 851 except KeyError: 852 _log.error("HL7 does not contain OBX segments, nothing to do") 853 return 854 for OBX_idx in range(OBX_count): 855 unit = hl7_data.extract_field(segment = 'OBX', segment_num = OBX_idx, field_num = OBX_field__unit) 856 if unit == '': 857 unit = None 858 LOINC = hl7_data.extract_field(segment = 'OBX', segment_num = OBX_idx, field_num = OBX_field__type, component_num = OBX_component__loinc) 859 tname = hl7_data.extract_field(segment = 'OBX', segment_num = OBX_idx, field_num = OBX_field__type, component_num = OBX_component__name) 860 tt = __find_or_create_test_type ( 861 loinc = LOINC, 862 name = tname, 863 pk_lab = pk_test_org, 864 unit = unit, 865 link_obj = link_obj 866 )
867 868 #------------------------------------------------------------
869 -def __PID2dto(HL7=None):
870 pat_lname = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname) 871 pat_fname = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__firstname) 872 pat_mname = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__middlename) 873 if pat_mname is not None: 874 pat_fname += ' ' 875 pat_fname += pat_mname 876 _log.debug('patient data from PID segment: first=%s (middle=%s) last=%s', pat_fname, pat_mname, pat_lname) 877 878 dto = gmPerson.cDTO_person() 879 dto.firstnames = pat_fname 880 dto.lastnames = pat_lname 881 dto.gender = HL7_GENDERS[HL7.extract_field('PID', segment_num = 1, field_num = PID_field__gender)] 882 hl7_dob = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__dob) 883 if hl7_dob is not None: 884 tmp = time.strptime(hl7_dob, '%Y%m%d') 885 dto.dob = pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone) 886 887 idents = dto.get_candidate_identities() 888 if len(idents) == 0: 889 _log.warning('no match candidate, not auto-importing') 890 _log.debug(dto) 891 return [] 892 if len(idents) > 1: 893 _log.warning('more than one match candidate, not auto-importing') 894 _log.debug(dto) 895 return idents 896 return [gmPerson.cPatient(idents[0].ID)]
897 898 #------------------------------------------------------------
899 -def __hl7dt2pydt(hl7dt):
900 if hl7dt == '': 901 return None 902 903 if len(hl7dt) == 8: 904 tmp = time.strptime(hl7dt, '%Y%m%d') 905 return pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone) 906 907 if len(hl7dt) == 12: 908 tmp = time.strptime(hl7dt, '%Y%m%d%H%M') 909 return pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tmp.tm_hour, tmp.tm_min, tzinfo = gmDateTime.gmCurrentLocalTimezone) 910 911 if len(hl7dt) == 14: 912 tmp = time.strptime(hl7dt, '%Y%m%d%H%M%S') 913 return pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tmp.tm_hour, tmp.tm_min, tmp.tm_sec, tzinfo = gmDateTime.gmCurrentLocalTimezone) 914 915 raise ValueError('Observation timestamp not parseable: [%s]', hl7dt)
916 917 #------------------------------------------------------------
918 -def __import_single_PID_hl7_file(filename, emr=None):
919 """Assumes single-PID/single-MSH HL7 file.""" 920 921 _log.debug('importing single-PID single-MSH HL7 data from [%s]', filename) 922 923 # read the file 924 MSH_file = io.open(filename, mode = 'rt', encoding = 'utf8', newline = '') 925 HL7 = pyhl7.parse(MSH_file.read(1024 * 1024 * 5)) # 5 MB max 926 MSH_file.close() 927 928 # sanity checks 929 if len(HL7.segments('MSH')) != 1: 930 _log.error('more than one MSH segment') 931 return False 932 if len(HL7.segments('PID')) != 1: 933 _log.error('more than one PID segment') 934 return False 935 936 # ensure lab is in database 937 hl7_lab = HL7.extract_field('MSH', field_num = MSH_field__sending_lab) 938 gm_lab = __find_or_create_lab(hl7_lab) 939 940 # ensure test types exist 941 conn = gmPG2.get_connection(readonly = False) 942 __ensure_hl7_test_types_exist_in_gnumed(link_obj = conn, hl7_data = HL7, pk_test_org = gm_lab['pk_test_org']) 943 944 # find patient 945 if emr is None: 946 #PID = HL7.segment('PID') 947 pats = __PID2dto(HL7 = HL7) 948 if len(pats) == 0: 949 conn.rollback() 950 return False 951 if len(pats) > 1: 952 conn.rollback() 953 return False 954 emr = pats[0].emr 955 956 # import values: loop over segments 957 when_list = {} 958 current_result = None 959 previous_segment = None 960 had_errors = False 961 msh_seen = False 962 pid_seen = False 963 last_obr = None 964 obr = {} 965 for seg_idx in range(len(HL7)): 966 seg = HL7[seg_idx] 967 seg_type = seg[0][0] 968 969 _log.debug('processing line #%s = segment of type <%s>', seg_idx, seg_type) 970 971 if seg_type == 'MSH': 972 msh_seen = True 973 974 if seg_type == 'PID': 975 if not msh_seen: 976 conn.rollback() 977 _log.error('PID segment before MSH segment') 978 return False 979 pid_seen = True 980 981 if seg_type in ['MSH', 'PID']: 982 _log.info('segment already handled') 983 previous_segment = seg_type 984 obr = {} 985 current_result = None 986 continue 987 988 if seg_type in ['ORC']: 989 _log.info('currently ignoring %s segments', seg_type) 990 previous_segment = seg_type 991 obr = {} 992 current_result = None 993 continue 994 995 if seg_type == 'OBR': 996 previous_segment = seg_type 997 last_obr = seg 998 current_result = None 999 obr['abbrev'] = ('%s' % seg[OBR_field__service_name][0]).strip() 1000 try: 1001 obr['name'] = ('%s' % seg[OBR_field__service_name][1]).strip() 1002 except IndexError: 1003 obr['name'] = obr['abbrev'] 1004 for field_name in [OBR_field__ts_ended, OBR_field__ts_started, OBR_field__ts_specimen_received, OBR_field__ts_requested]: 1005 obr['clin_when'] = seg[field_name][0].strip() 1006 if obr['clin_when'] != '': 1007 break 1008 continue 1009 1010 if seg_type == 'OBX': 1011 current_result = None 1012 # determine value 1013 val_alpha = seg[OBX_field__value][0].strip() 1014 is_num, val_num = gmTools.input2decimal(initial = val_alpha) 1015 if is_num: 1016 val_alpha = None 1017 else: 1018 val_num = None 1019 val_alpha = val_alpha.replace('\.br\\', '\n') 1020 # determine test type 1021 unit = seg[OBX_field__unit][0].strip() 1022 if unit == '': 1023 if is_num: 1024 unit = '1/1' 1025 else: 1026 unit = None 1027 test_type = __find_or_create_test_type ( 1028 loinc = '%s' % seg[OBX_field__type][0][OBX_component__loinc-1], 1029 name = '%s' % seg[OBX_field__type][0][OBX_component__name-1], 1030 pk_lab = gm_lab['pk_test_org'], 1031 unit = unit, 1032 link_obj = conn 1033 ) 1034 # eventually, episode should be read from lab_request 1035 epi = emr.add_episode ( 1036 link_obj = conn, 1037 episode_name = 'administrative', 1038 is_open = False, 1039 allow_dupes = False 1040 ) 1041 current_result = emr.add_test_result ( 1042 link_obj = conn, 1043 episode = epi['pk_episode'], 1044 type = test_type['pk_test_type'], 1045 intended_reviewer = gmStaff.gmCurrentProvider()['pk_staff'], 1046 val_num = val_num, 1047 val_alpha = val_alpha, 1048 unit = unit 1049 ) 1050 # handle range information et al 1051 ref_range = seg[OBX_field__range][0].strip() 1052 if ref_range != '': 1053 current_result.reference_range = ref_range 1054 flag = seg[OBX_field__abnormal_flag][0].strip() 1055 if flag != '': 1056 current_result['abnormality_indicator'] = flag 1057 current_result['status'] = seg[OBX_field__status][0].strip() 1058 current_result['val_grouping'] = seg[OBX_field__subid][0].strip() 1059 current_result['source_data'] = '' 1060 if last_obr is not None: 1061 current_result['source_data'] += str(last_obr) 1062 current_result['source_data'] += '\n' 1063 current_result['source_data'] += str(seg) 1064 clin_when = seg[OBX_field__timestamp][0].strip() 1065 if clin_when == '': 1066 _log.warning('no <Observation timestamp> in OBX, trying OBR timestamp') 1067 clin_when = obr['clin_when'] 1068 try: 1069 clin_when = __hl7dt2pydt(clin_when) 1070 except ValueError: 1071 _log.exception('clin_when from OBX or OBR not useable, assuming <today>') 1072 if clin_when is not None: 1073 current_result['clin_when'] = clin_when 1074 current_result.save(conn = conn) 1075 when_list[gmDateTime.pydt_strftime(current_result['clin_when'], '%Y %b %d')] = 1 1076 previous_segment = seg_type 1077 continue 1078 1079 if seg_type == 'NTE': 1080 note = seg[NET_field__note][0].strip().replace('\.br\\', '\n') 1081 if note == '': 1082 _log.debug('empty NTE segment') 1083 previous_segment = seg_type # maybe not ? (HL7 providers happen to use empty NTE segments to "structure" raw HL7 |-) 1084 continue 1085 1086 # if this is an NTE following an OBR (IOW an order-related 1087 # comment): make this a test result all of its own :-) 1088 if previous_segment == 'OBR': 1089 _log.debug('NTE following OBR: general note, using OBR timestamp [%s]', obr['clin_when']) 1090 current_result = None 1091 name = obr['name'] 1092 if name == '': 1093 name = _('Comment') 1094 # FIXME: please suggest a LOINC for "order comment" 1095 test_type = __find_or_create_test_type(name = name, pk_lab = gm_lab['pk_test_org'], abbrev = obr['abbrev'], link_obj = conn) 1096 # eventually, episode should be read from lab_request 1097 epi = emr.add_episode ( 1098 link_obj = conn, 1099 episode_name = 'administrative', 1100 is_open = False, 1101 allow_dupes = False 1102 ) 1103 nte_result = emr.add_test_result ( 1104 link_obj = conn, 1105 episode = epi['pk_episode'], 1106 type = test_type['pk_test_type'], 1107 intended_reviewer = gmStaff.gmCurrentProvider()['pk_staff'], 1108 val_alpha = note 1109 ) 1110 #nte_result['val_grouping'] = seg[OBX_field__subid][0].strip() 1111 nte_result['source_data'] = str(seg) 1112 try: 1113 nte_result['clin_when'] = __hl7dt2pydt(obr['clin_when']) 1114 except ValueError: 1115 _log.exception('no .clin_when from OBR for NTE pseudo-OBX available') 1116 nte_result.save(conn = conn) 1117 continue 1118 1119 if (previous_segment == 'OBX') and (current_result is not None): 1120 current_result['source_data'] += '\n' 1121 current_result['source_data'] += str(seg) 1122 current_result['note_test_org'] = gmTools.coalesce ( 1123 current_result['note_test_org'], 1124 note, 1125 '%%s\n%s' % note 1126 ) 1127 current_result.save(conn = conn) 1128 previous_segment = seg_type 1129 continue 1130 1131 _log.error('unexpected NTE segment') 1132 had_errors = True 1133 break 1134 1135 _log.error('unknown segment, aborting') 1136 _log.debug('line: %s', seg) 1137 had_errors = True 1138 break 1139 1140 if had_errors: 1141 conn.rollback() 1142 return False 1143 1144 conn.commit() 1145 1146 # record import in chart 1147 try: 1148 no_results = len(HL7.segments('OBX')) 1149 except KeyError: 1150 no_results = '?' 1151 soap = _( 1152 'Imported HL7 file [%s]:\n' 1153 ' lab "%s" (%s@%s), %s results (%s)' 1154 ) % ( 1155 filename, 1156 hl7_lab, 1157 gm_lab['unit'], 1158 gm_lab['organization'], 1159 no_results, 1160 ' / '.join(when_list.keys()) 1161 ) 1162 epi = emr.add_episode ( 1163 episode_name = 'administrative', 1164 is_open = False, 1165 allow_dupes = False 1166 ) 1167 emr.add_clin_narrative ( 1168 note = soap, 1169 soap_cat = None, 1170 episode = epi 1171 ) 1172 1173 # keep copy of HL7 data in document archive 1174 folder = gmPerson.cPatient(emr.pk_patient).document_folder 1175 hl7_docs = folder.get_documents ( 1176 doc_type = 'HL7 data', 1177 pk_episodes = [epi['pk_episode']], 1178 order_by = 'ORDER BY clin_when DESC' 1179 ) 1180 if len(hl7_docs) > 0: 1181 # there should only ever be one unless the user manually creates more, 1182 # also, it should always be the latest since "ORDER BY clin_when DESC" 1183 hl7_doc = hl7_docs[0] 1184 else: 1185 hl7_doc = folder.add_document ( 1186 document_type = 'HL7 data', 1187 encounter = emr.active_encounter['pk_encounter'], 1188 episode = epi['pk_episode'] 1189 ) 1190 hl7_doc['comment'] = _('list of imported HL7 data files') 1191 hl7_doc['pk_org_unit'] = gmPraxis.gmCurrentPraxisBranch()['pk_org_unit'] 1192 hl7_doc['clin_when'] = gmDateTime.pydt_now_here() 1193 hl7_doc.save() 1194 part = hl7_doc.add_part(file = filename) 1195 part['obj_comment'] = _('Result dates: %s') % ' / '.join(when_list.keys()) 1196 part.save() 1197 hl7_doc.set_reviewed(technically_abnormal = False, clinically_relevant = False) 1198 1199 return True
1200 1201 #------------------------------------------------------------ 1202 # this is only used for testing here in this file
1203 -def __stage_MSH_as_incoming_data(filename, source=None, logfile=None):
1204 """Consumes single-MSH single-PID HL7 files.""" 1205 1206 _log.debug('staging [%s] as unmatched incoming HL7%s', gmTools.coalesce(source, '', ' (%s)'), filename) 1207 1208 # parse HL7 1209 MSH_file = io.open(filename, mode = 'rt', encoding = 'utf8', newline = '') 1210 raw_hl7 = MSH_file.read(1024 * 1024 * 5) # 5 MB max 1211 MSH_file.close() 1212 formatted_hl7 = format_hl7_message ( 1213 message = raw_hl7, 1214 skip_empty_fields = True, 1215 eol = '\n' 1216 ) 1217 HL7 = pyhl7.parse(raw_hl7) 1218 del raw_hl7 1219 1220 # import file 1221 incoming = gmIncomingData.create_incoming_data('HL7%s' % gmTools.coalesce(source, '', ' (%s)'), filename) 1222 if incoming is None: 1223 return None 1224 incoming.update_data_from_file(fname = filename) 1225 incoming['comment'] = formatted_hl7 1226 if logfile is not None: 1227 log = io.open(logfile, mode = 'rt', encoding = 'utf8') 1228 incoming['comment'] += '\n' 1229 incoming['comment'] += ('-' * 80) 1230 incoming['comment'] += '\n\n' 1231 incoming['comment'] += log.read() 1232 log.close() 1233 try: 1234 incoming['lastnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname) 1235 incoming['firstnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__firstname) 1236 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__middlename) 1237 if val is not None: 1238 incoming['firstnames'] += ' ' 1239 incoming['firstnames'] += val 1240 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__dob) 1241 if val is not None: 1242 tmp = time.strptime(val, '%Y%m%d') 1243 incoming['dob'] = pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone) 1244 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__gender) 1245 if val is not None: 1246 incoming['gender'] = val 1247 incoming['external_data_id'] = filename 1248 #u'fk_patient_candidates', 1249 # u'request_id', # request ID as found in <data> 1250 # u'postcode', 1251 # u'other_info', # other identifying info in .data 1252 # u'requestor', # Requestor of data (e.g. who ordered test results) if available in source data. 1253 # u'fk_identity_disambiguated', 1254 # u'comment', # a free text comment on this row, eg. why is it here, error logs etc 1255 # u'fk_provider_disambiguated' # The provider the data is relevant to. 1256 except KeyError: 1257 _log.exception('no PID segment, cannot add more data') 1258 incoming.save() 1259 1260 return incoming
1261 1262 #============================================================ 1263 # main 1264 #------------------------------------------------------------ 1265 if __name__ == "__main__": 1266 1267 if len(sys.argv) < 2: 1268 sys.exit() 1269 1270 if sys.argv[1] != 'test': 1271 sys.exit() 1272 1273 from Gnumed.pycommon import gmLog2 1274 1275 gmDateTime.init() 1276 gmTools.gmPaths() 1277 1278 #-------------------------------------------------------
1279 - def test_import_HL7(filename):
1280 # would normally be set by external configuration: 1281 from Gnumed.business import gmPraxis 1282 gmPraxis.gmCurrentPraxisBranch(branch = gmPraxis.get_praxis_branches()[0]) 1283 if not import_hl7_file(filename): 1284 print("error with", filename)
1285 #-------------------------------------------------------
1286 - def test_xml_extract():
1287 hl7 = extract_HL7_from_XML_CDATA(sys.argv[2], './/Message') 1288 print("HL7:", hl7)
1289 #result, PID_fnames = split_hl7_file(hl7) 1290 #print "result:", result 1291 #print "per-PID MSH files:" 1292 #for name in PID_fnames: 1293 # print " ", name 1294 #-------------------------------------------------------
1295 - def test_stage_hl7_from_xml():
1296 hl7 = extract_HL7_from_XML_CDATA(sys.argv[2], './/Message') 1297 print("HL7:", hl7) 1298 result, PID_fnames = split_hl7_file(hl7) 1299 print("result:", result) 1300 print("staging per-PID HL7 files:") 1301 for name in PID_fnames: 1302 print(" file:", name) 1303 __stage_MSH_as_incoming_data(name, source = 'Excelleris')
1304 #-------------------------------------------------------
1305 - def test_split_hl7_file():
1306 result, PID_fnames = split_hl7_file(sys.argv[2]) 1307 print("result:", result) 1308 print("per-PID HL7 files:") 1309 for name in PID_fnames: 1310 print(" file:", name)
1311 #-------------------------------------------------------
1312 - def test_stage_hl7():
1313 fixed = __fix_malformed_hl7_file(sys.argv[2]) 1314 print("fixed HL7:", fixed) 1315 PID_fnames = split_HL7_by_PID(fixed, encoding='utf8') 1316 print("staging per-PID HL7 files:") 1317 for name in PID_fnames: 1318 print(" file:", name)
1319 #print "", __stage_MSH_as_incoming_data(name, source = u'?') 1320 #-------------------------------------------------------
1321 - def test_format_hl7_message():
1322 tests = [ 1323 "OBR|1||03-1350023-LIP-0|LIP^Lipids||20031004073300|20031004073300|||||||20031004073300||22333^MEDIC^IAN^TEST||031350023||03-1350023|031350023|20031004131600||CHEM|F|||22333^MEDIC^IAN^TEST", 1324 "OBX|2|NM|22748-8^LDL Cholesterol||4.0|mmol/L|1.5 - 3.4|H|||F|||20031004073300" 1325 ] 1326 for test in tests: 1327 print(format_hl7_message ( 1328 # skip_empty_fields = True, 1329 message = test 1330 ))
1331 #-------------------------------------------------------
1332 - def test_format_hl7_file(filename):
1333 print(format_hl7_file ( 1334 filename, 1335 # skip_empty_fields = True 1336 return_filename = True 1337 ))
1338 #-------------------------------------------------------
1339 - def test___fix_malformed_hl7():
1340 print("fixed HL7:", __fix_malformed_hl7_file(sys.argv[2]))
1341 #-------------------------------------------------------
1342 - def test_parse_hl7():
1343 MSH_file = io.open(sys.argv[2], mode = 'rt', encoding = 'utf8', newline = '') 1344 raw_hl7 = MSH_file.read(1024 * 1024 * 5) # 5 MB max 1345 MSH_file.close() 1346 print(format_hl7_message ( 1347 message = raw_hl7, 1348 skip_empty_fields = True, 1349 eol = '\n' 1350 )) 1351 HL7 = pyhl7.parse(raw_hl7) 1352 del raw_hl7 1353 for seg in HL7.segments('MSH'): 1354 print(seg) 1355 print("PID:") 1356 print(HL7.extract_field('PID')) 1357 print(HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname)) 1358 print(HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname))
1359 1360 # incoming['firstnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__firstname) 1361 # val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__middlename) 1362 # if val is not None: 1363 # incoming['firstnames'] += u' ' 1364 # incoming['firstnames'] += val 1365 # val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__dob) 1366 # if val is not None: 1367 # tmp = time.strptime(val, '%Y%m%d') 1368 # incoming['dob'] = pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone) 1369 # val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__gender) 1370 # if val is not None: 1371 # incoming['gender'] = val 1372 # incoming['external_data_id'] = filename 1373 1374 #------------------------------------------------------- 1375 #test_import_HL7(sys.argv[2]) 1376 #test_xml_extract() 1377 #test_stage_hl7_from_xml() 1378 #test_stage_hl7() 1379 #test_format_hl7_message() 1380 #test_format_hl7_file(sys.argv[2]) 1381 #test___fix_malformed_hl7() 1382 #test_split_hl7_file() 1383 test_parse_hl7() 1384