1
2 """Some HL7 handling."""
3
4 __author__ = "K.Hilbert <Karsten.Hilbert@gmx.net>"
5 __license__ = "GPL v2 or later"
6
7
8 import sys
9 import os
10 import io
11 import logging
12 import time
13 import shutil
14 import datetime as pyDT
15 import hl7 as pyhl7
16 from xml.etree import ElementTree as pyxml
17
18
19 if __name__ == '__main__':
20 sys.path.insert(0, '../../')
21
22 from Gnumed.pycommon import gmI18N
23 if __name__ == '__main__':
24 gmI18N.activate_locale()
25 gmI18N.install_domain()
26 from Gnumed.pycommon import gmTools
27 from Gnumed.pycommon import gmBusinessDBObject
28 from Gnumed.pycommon import gmPG2
29 from Gnumed.pycommon import gmDateTime
30
31 from Gnumed.business import gmIncomingData
32 from Gnumed.business import gmPathLab
33 from Gnumed.business import gmPerson
34 from Gnumed.business import gmPraxis
35 from Gnumed.business import gmStaff
36
37
38 _log = logging.getLogger('gm.hl7')
39
40
41 HL7_EOL = '\r'
42 HL7_BRK = '\.br\\'
43
44 HL7_SEGMENTS = 'FHS BHS MSH PID PV1 OBX NTE ORC OBR'.split()
45
46 HL7_segment2field_count = {
47 'FHS': 12,
48 'BHS': 12,
49 'MSH': 19,
50 'PID': 30,
51 'PV1': 52,
52 'OBR': 43,
53 'OBX': 17,
54 'NTE': 3,
55 'ORC': 19
56 }
57
58 MSH_field__sending_lab = 3
59
60 PID_field__name = 5
61 PID_field__dob = 7
62 PID_field__gender = 8
63 PID_component__lastname = 1
64 PID_component__firstname = 2
65 PID_component__middlename = 3
66
67 OBR_field__service_name = 4
68 OBR_field__ts_requested = 6
69 OBR_field__ts_started = 7
70 OBR_field__ts_ended = 8
71 OBR_field__ts_specimen_received = 14
72
73 OBX_field__set_id = 1
74 OBX_field__datatype = 2
75 OBX_field__type = 3
76
77 OBX_component__loinc = 1
78 OBX_component__name = 2
79 OBX_field__subid = 4
80 OBX_field__value = 5
81 OBX_field__unit = 6
82 OBX_field__range = 7
83 OBX_field__abnormal_flag = 8
84 OBX_field__status = 11
85 OBX_field__timestamp = 14
86
87 NET_field__set_id = 1
88 NET_field__src = 2
89 NET_field__note = 3
90
91 HL7_field_labels = {
92 'MSH': {
93 0: 'Segment Type',
94 1: 'Field Separator',
95 2: 'Encoding Characters',
96 3: 'Sending Application',
97 4: 'Sending Facility',
98 5: 'Receiving Application',
99 6: 'Receiving Facility',
100 7: 'Date/Time of Message',
101 8: 'Security',
102 9: 'Message Type',
103 10: 'ID: Message Control',
104 11: 'ID: Processing',
105 12: 'ID: Version',
106 14: 'Continuation Pointer',
107 15: 'Accept Acknowledgement Type',
108 16: 'Application Acknowledgement Type'
109 },
110 'PID': {
111 0: 'Segment Type',
112 1: '<PID> Set ID',
113 2: 'Patient ID (external)',
114 3: 'Patient ID (internal)',
115 4: 'Patient ID (alternate)',
116 5: 'Patient Name',
117 7: 'Date/Time of birth',
118 8: 'Administrative Gender',
119 11: 'Patient Address',
120 13: 'Patient Phone Number - Home'
121 },
122 'OBR': {
123 0: 'Segment Type',
124 1: 'ID: Set',
125 3: 'Filler Order Number (= ORC-3)',
126 4: 'ID: Universal Service',
127 5: 'Priority',
128 6: 'Date/Time requested',
129 7: 'Date/Time Observation started',
130 14: 'Date/Time Specimen received',
131 16: 'Ordering Provider',
132 18: 'Placer Field 1',
133 20: 'Filler Field 1',
134 21: 'Filler Field 2',
135 22: 'Date/Time Results reported/Status changed',
136 24: 'ID: Diagnostic Service Section',
137 25: 'Result Status',
138 27: 'Quantity/Timing',
139 28: 'Result Copies To'
140 },
141 'ORC': {
142 0: 'Segment Type',
143 1: 'Order Control',
144 3: 'Filler Order Number',
145 12: 'Ordering Provider'
146 },
147 'OBX': {
148 0: 'Segment Type',
149 1: 'Set ID',
150 2: 'Value Type',
151 3: 'Identifier (LOINC)',
152 4: 'Observation Sub-ID',
153 5: 'Value',
154 6: 'Units',
155 7: 'References Range (Low - High)',
156 8: 'Abnormal Flags',
157 11: 'Result Status',
158 14: 'Date/Time of Observation'
159 },
160 'NTE': {
161 0: 'Segment Type',
162 3: 'Comment'
163 }
164 }
165
166 HL7_GENDERS = {
167 'F': 'f',
168 'M': 'm',
169 'O': None,
170 'U': None,
171 None: None
172 }
173
174
175
176
178
179 _log.debug('extracting HL7 from CDATA of <%s> nodes in XML file [%s]', xml_path, filename)
180
181
182 try:
183 open(filename).close()
184 orig_dir = os.path.split(filename)[0]
185 work_filename = gmTools.get_unique_filename(prefix = 'gm-x2h-%s-' % gmTools.fname_stem(filename), suffix = '.hl7')
186 if target_dir is None:
187 target_dir = os.path.join(orig_dir, 'HL7')
188 done_dir = os.path.join(orig_dir, 'done')
189 else:
190 done_dir = os.path.join(target_dir, 'done')
191 _log.debug('target dir: %s', target_dir)
192 gmTools.mkdir(target_dir)
193 gmTools.mkdir(done_dir)
194 except Exception:
195 _log.exception('cannot setup unwrapping environment')
196 return None
197
198 hl7_xml = pyxml.ElementTree()
199 try:
200 hl7_xml.parse(filename)
201 except pyxml.ParseError:
202 _log.exception('cannot parse [%s]' % filename)
203 return None
204 nodes = hl7_xml.findall(xml_path)
205 if len(nodes) == 0:
206 _log.debug('no nodes found for data extraction')
207 return None
208
209 _log.debug('unwrapping HL7 from XML into [%s]', work_filename)
210 hl7_file = io.open(work_filename, mode = 'wt', encoding = 'utf8', newline = '')
211 for node in nodes:
212
213 hl7_file.write(node.text + '')
214 hl7_file.close()
215
216 target_fname = os.path.join(target_dir, os.path.split(work_filename)[1])
217 shutil.copy(work_filename, target_dir)
218 shutil.move(filename, done_dir)
219
220 return target_fname
221
222
224 """Multi-step processing of HL7 files.
225
226 - input can be multi-MSH / multi-PID / partially malformed HL7
227 - tries to fix oddities
228 - splits by MSH
229 - splits by PID into <target_dir>
230
231 - needs write permissions in dir_of(filename)
232 - moves HL7 files which were successfully split up into dir_of(filename)/done/
233
234 - returns (True|False, list_of_PID_files)
235 """
236 local_log_name = gmTools.get_unique_filename (
237 prefix = gmTools.fname_stem(filename) + '-',
238 suffix = '.split.log'
239 )
240 local_logger = logging.FileHandler(local_log_name)
241 local_logger.setLevel(logging.DEBUG)
242 root_logger = logging.getLogger('')
243 root_logger.addHandler(local_logger)
244 _log.info('splitting HL7 file: %s', filename)
245 _log.debug('log file: %s', local_log_name)
246
247
248 try:
249 open(filename).close()
250 orig_dir = os.path.split(filename)[0]
251 done_dir = os.path.join(orig_dir, 'done')
252 gmTools.mkdir(done_dir)
253 error_dir = os.path.join(orig_dir, 'failed')
254 gmTools.mkdir(error_dir)
255 work_filename = gmTools.get_unique_filename(prefix = gmTools.fname_stem(filename) + '-', suffix = '.hl7')
256 if target_dir is None:
257 target_dir = os.path.join(orig_dir, 'PID')
258 _log.debug('target dir: %s', target_dir)
259 gmTools.mkdir(target_dir)
260 except Exception:
261 _log.exception('cannot setup splitting environment')
262 root_logger.removeHandler(local_logger)
263 return False, None
264
265
266 target_names = []
267 try:
268 shutil.copy(filename, work_filename)
269 fixed_filename = __fix_malformed_hl7_file(work_filename, encoding = encoding)
270 MSH_fnames = __split_hl7_file_by_MSH(fixed_filename, encoding)
271 PID_fnames = []
272 for MSH_fname in MSH_fnames:
273 PID_fnames.extend(__split_MSH_by_PID(MSH_fname))
274 for PID_fname in PID_fnames:
275 shutil.move(PID_fname, target_dir)
276 target_names.append(os.path.join(target_dir, os.path.split(PID_fname)[1]))
277 except Exception:
278 _log.exception('cannot split HL7 file')
279 for target_name in target_names:
280 try: os.remove(target_name)
281 except Exception: pass
282 root_logger.removeHandler(local_logger)
283 shutil.move(local_log_name, error_dir)
284 return False, None
285
286 _log.info('successfully split')
287 root_logger.removeHandler(local_logger)
288 try:
289 shutil.move(filename, done_dir)
290 shutil.move(local_log_name, done_dir)
291 except shutil.Error:
292 _log.exception('cannot move hl7 file or log file to holding area')
293 return True, target_names
294
295
342
343
372
373
374
376 """Multi-step processing of HL7 files.
377
378 - input must be single-MSH / single-PID / normalized HL7
379
380 - imports into clin.incoming_data_unmatched
381
382 - needs write permissions in dir_of(filename)
383 - moves PID files which were successfully staged into dir_of(filename)/done/PID/
384 """
385 local_log_name = gmTools.get_unique_filename (
386 prefix = gmTools.fname_stem(filename) + '-',
387 suffix = '.stage.log'
388 )
389 local_logger = logging.FileHandler(local_log_name)
390 local_logger.setLevel(logging.DEBUG)
391 root_logger = logging.getLogger('')
392 root_logger.addHandler(local_logger)
393 _log.info('staging [%s] as unmatched incoming HL7%s', filename, gmTools.coalesce(source, '', ' (%s)'))
394 _log.debug('log file: %s', local_log_name)
395
396
397 try:
398 open(filename).close()
399 orig_dir = os.path.split(filename)[0]
400 done_dir = os.path.join(orig_dir, 'done')
401 gmTools.mkdir(done_dir)
402 error_dir = os.path.join(orig_dir, 'failed')
403 gmTools.mkdir(error_dir)
404 except Exception:
405 _log.exception('cannot setup staging environment')
406 root_logger.removeHandler(local_logger)
407 return False
408
409
410 try:
411 incoming = gmIncomingData.create_incoming_data('HL7%s' % gmTools.coalesce(source, '', ' (%s)'), filename)
412 if incoming is None:
413 _log.error('cannot stage PID file: %s', filename)
414 root_logger.removeHandler(local_logger)
415 shutil.move(filename, error_dir)
416 shutil.move(local_log_name, error_dir)
417 return False
418 incoming.update_data_from_file(fname = filename)
419 except Exception:
420 _log.exception('error staging PID file')
421 root_logger.removeHandler(local_logger)
422 shutil.move(filename, error_dir)
423 shutil.move(local_log_name, error_dir)
424 return False
425
426
427 MSH_file = io.open(filename, mode = 'rt', encoding = 'utf8', newline = '')
428 raw_hl7 = MSH_file.read(1024 * 1024 * 5)
429 MSH_file.close()
430 shutil.move(filename, done_dir)
431 incoming['comment'] = format_hl7_message (
432 message = raw_hl7,
433 skip_empty_fields = True,
434 eol = '\n'
435 )
436 HL7 = pyhl7.parse(raw_hl7)
437 del raw_hl7
438 incoming['comment'] += '\n'
439 incoming['comment'] += ('-' * 80)
440 incoming['comment'] += '\n\n'
441 log = io.open(local_log_name, mode = 'rt', encoding = 'utf8')
442 incoming['comment'] += log.read()
443 log.close()
444 try:
445 incoming['lastnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname)
446 incoming['firstnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__firstname)
447 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__middlename)
448 if val is not None:
449 incoming['firstnames'] += ' '
450 incoming['firstnames'] += val
451 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__dob)
452 if val is not None:
453 tmp = time.strptime(val, '%Y%m%d')
454 incoming['dob'] = pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone)
455 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__gender)
456 if val is not None:
457 incoming['gender'] = val
458 incoming['external_data_id'] = filename
459
460
461
462
463
464
465
466
467 except Exception:
468 _log.exception('cannot add more data')
469 incoming.save()
470
471 _log.info('successfully staged')
472 root_logger.removeHandler(local_logger)
473 shutil.move(local_log_name, done_dir)
474 return True
475
476
478
479 log_name = gmTools.get_unique_filename (
480 prefix = 'gm-staged_hl7_import-',
481 suffix = '.log'
482 )
483 import_logger = logging.FileHandler(log_name)
484 import_logger.setLevel(logging.DEBUG)
485 root_logger = logging.getLogger('')
486 root_logger.addHandler(import_logger)
487 _log.debug('log file: %s', log_name)
488
489 if not staged_item.lock():
490 _log.error('cannot lock staged data for HL7 import')
491 root_logger.removeHandler(import_logger)
492 return False, log_name
493
494 _log.debug('reference ID of staged HL7 data: %s', staged_item['external_data_id'])
495
496 filename = staged_item.save_to_file()
497 _log.debug('unstaged HL7 data into: %s', filename)
498
499 if staged_item['pk_identity_disambiguated'] is None:
500 emr = None
501 else:
502 emr = gmPerson.cPatient(staged_item['pk_identity_disambiguated']).emr
503
504 success = False
505 try:
506 success = __import_single_PID_hl7_file(filename, emr = emr)
507 if success:
508 gmIncomingData.delete_incoming_data(pk_incoming_data = staged_item['pk_incoming_data_unmatched'])
509 staged_item.unlock()
510 root_logger.removeHandler(import_logger)
511 return True, log_name
512 _log.error('error when importing single-PID/single-MSH file')
513 except Exception:
514 _log.exception('error when importing single-PID/single-MSH file')
515
516 if not success:
517 staged_item['comment'] = _('failed import: %s\n') % gmDateTime.pydt_strftime(gmDateTime.pydt_now_here())
518 staged_item['comment'] += '\n'
519 staged_item['comment'] += ('-' * 80)
520 staged_item['comment'] += '\n\n'
521 log = io.open(log_name, mode = 'rt', encoding = 'utf8')
522 staged_item['comment'] += log.read()
523 log.close()
524 staged_item['comment'] += '\n'
525 staged_item['comment'] += ('-' * 80)
526 staged_item['comment'] += '\n\n'
527 staged_item['comment'] += format_hl7_file (
528 filename,
529 skip_empty_fields = True,
530 eol = '\n ',
531 return_filename = False
532 )
533 staged_item.save()
534
535 staged_item.unlock()
536 root_logger.removeHandler(import_logger)
537 return success, log_name
538
539
541
542 log_name = '%s.import.log' % filename
543 import_logger = logging.FileHandler(log_name)
544 import_logger.setLevel(logging.DEBUG)
545 root_logger = logging.getLogger('')
546 root_logger.addHandler(import_logger)
547 _log.debug('log file: %s', log_name)
548
549 success = True
550 try:
551 success = __import_single_PID_hl7_file(filename)
552 if not success:
553 _log.error('error when importing single-PID/single-MSH file')
554 except Exception:
555 _log.exception('error when importing single-PID/single-MSH file')
556
557 root_logger.removeHandler(import_logger)
558 return success, log_name
559
560
561
562
694
695
697
698 _log.debug('splitting [%s] into single-MSH files', filename)
699
700 hl7_in = io.open(filename, mode = 'rt', encoding = encoding)
701
702 idx = 0
703 first_line = True
704 MSH_file = None
705 MSH_fnames = []
706 for line in hl7_in:
707 line = line.strip()
708
709 if first_line:
710
711 if line == '':
712 continue
713 if line.startswith('FHS|'):
714 _log.debug('ignoring FHS')
715 continue
716 if line.startswith('BHS|'):
717 _log.debug('ignoring BHS')
718 continue
719 if not line.startswith('MSH|'):
720 raise ValueError('HL7 file <%s> does not start with "MSH" line' % filename)
721 first_line = False
722
723 if line.startswith('MSH|'):
724 if MSH_file is not None:
725 MSH_file.close()
726 idx += 1
727 out_fname = gmTools.get_unique_filename(prefix = '%s-MSH_%s-' % (gmTools.fname_stem(filename), idx), suffix = 'hl7')
728 _log.debug('writing message %s to [%s]', idx, out_fname)
729 MSH_fnames.append(out_fname)
730 MSH_file = io.open(out_fname, mode = 'wt', encoding = 'utf8', newline = '')
731
732 if line.startswith('BTS|'):
733 _log.debug('ignoring BTS')
734 continue
735 if line.startswith('FTS|'):
736 _log.debug('ignoring FTS')
737 continue
738
739 MSH_file.write(line + HL7_EOL)
740
741 if MSH_file is not None:
742 MSH_file.close()
743 hl7_in.close()
744
745 return MSH_fnames
746
747
749 """Assumes:
750 - ONE MSH per file
751 - utf8 encoding
752 - first non-empty line must be MSH line
753 - next line must be PID line
754
755 IOW, what's created by __split_hl7_file_by_MSH()
756 """
757 _log.debug('splitting single-MSH file [%s] into single-PID files', filename)
758
759 MSH_in = io.open(filename, mode = 'rt', encoding = 'utf8')
760
761 looking_for_MSH = True
762 MSH_line = None
763 looking_for_first_PID = True
764 PID_file = None
765 PID_fnames = []
766 idx = 0
767 for line in MSH_in:
768 line = line.strip()
769
770 if line == '':
771 continue
772
773
774 if looking_for_MSH:
775 if line.startswith('MSH|'):
776 looking_for_MSH = False
777 MSH_line = line + HL7_EOL
778 continue
779 raise ValueError('HL7 MSH file <%s> does not start with "MSH" line' % filename)
780 else:
781 if line.startswith('MSH|'):
782 raise ValueError('HL7 single-MSH file <%s> contains more than one MSH line' % filename)
783
784
785 if looking_for_first_PID:
786 if not line.startswith('PID|'):
787 raise ValueError('HL7 MSH file <%s> does not have "PID" line follow "MSH" line' % filename)
788 looking_for_first_PID = False
789
790
791 if line.startswith('PID|'):
792 if PID_file is not None:
793 PID_file.close()
794 idx += 1
795 out_fname = gmTools.get_unique_filename(prefix = '%s-PID_%s-' % (gmTools.fname_stem(filename), idx), suffix = 'hl7')
796 _log.debug('writing message for PID %s to [%s]', idx, out_fname)
797 PID_fnames.append(out_fname)
798 PID_file = io.open(out_fname, mode = 'wt', encoding = 'utf8', newline = '')
799 PID_file.write(MSH_line)
800
801 PID_file.write(line + HL7_EOL)
802
803 if PID_file is not None:
804 PID_file.close()
805 MSH_in.close()
806
807 return PID_fnames
808
809
811 comment_tag = '[HL7 name::%s]' % hl7_lab
812 for gm_lab in gmPathLab.get_test_orgs():
813 if comment_tag in gmTools.coalesce(gm_lab['comment'], ''):
814 _log.debug('found lab [%s] from HL7 file in GNUmed database:', hl7_lab)
815 _log.debug(gm_lab)
816 return gm_lab
817 _log.debug('lab not found: %s', hl7_lab)
818 gm_lab = gmPathLab.create_test_org(link_obj = link_obj, name = hl7_lab, comment = comment_tag)
819 if gm_lab is None:
820 raise ValueError('cannot create lab [%s] in GNUmed' % hl7_lab)
821 _log.debug('created lab: %s', gm_lab)
822 return gm_lab
823
824
826
827 tt = gmPathLab.find_measurement_type(link_obj = link_obj, lab = pk_lab, name = name)
828 if tt is None:
829 _log.debug('test type [%s::%s::%s] not found for lab #%s, creating', name, unit, loinc, pk_lab)
830 tt = gmPathLab.create_measurement_type(link_obj = link_obj, lab = pk_lab, abbrev = gmTools.coalesce(abbrev, name), unit = unit, name = name)
831 _log.debug('created as: %s', tt)
832
833 if loinc is None:
834 return tt
835 if loinc.strip() == '':
836 return tt
837 if tt['loinc'] is None:
838 tt['loinc'] = loinc
839 tt.save(conn = link_obj)
840 return tt
841 if tt['loinc'] != loinc:
842
843 _log.error('LOINC code mismatch between GM (%s) and HL7 (%s) for result type [%s]', tt['loinc'], loinc, name)
844
845 return tt
846
847
849 try:
850 OBX_count = len(hl7_data.segments('OBX'))
851 except KeyError:
852 _log.error("HL7 does not contain OBX segments, nothing to do")
853 return
854 for OBX_idx in range(OBX_count):
855 unit = hl7_data.extract_field(segment = 'OBX', segment_num = OBX_idx, field_num = OBX_field__unit)
856 if unit == '':
857 unit = None
858 LOINC = hl7_data.extract_field(segment = 'OBX', segment_num = OBX_idx, field_num = OBX_field__type, component_num = OBX_component__loinc)
859 tname = hl7_data.extract_field(segment = 'OBX', segment_num = OBX_idx, field_num = OBX_field__type, component_num = OBX_component__name)
860 tt = __find_or_create_test_type (
861 loinc = LOINC,
862 name = tname,
863 pk_lab = pk_test_org,
864 unit = unit,
865 link_obj = link_obj
866 )
867
868
870 pat_lname = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname)
871 pat_fname = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__firstname)
872 pat_mname = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__middlename)
873 if pat_mname is not None:
874 pat_fname += ' '
875 pat_fname += pat_mname
876 _log.debug('patient data from PID segment: first=%s (middle=%s) last=%s', pat_fname, pat_mname, pat_lname)
877
878 dto = gmPerson.cDTO_person()
879 dto.firstnames = pat_fname
880 dto.lastnames = pat_lname
881 dto.gender = HL7_GENDERS[HL7.extract_field('PID', segment_num = 1, field_num = PID_field__gender)]
882 hl7_dob = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__dob)
883 if hl7_dob is not None:
884 tmp = time.strptime(hl7_dob, '%Y%m%d')
885 dto.dob = pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone)
886
887 idents = dto.get_candidate_identities()
888 if len(idents) == 0:
889 _log.warning('no match candidate, not auto-importing')
890 _log.debug(dto)
891 return []
892 if len(idents) > 1:
893 _log.warning('more than one match candidate, not auto-importing')
894 _log.debug(dto)
895 return idents
896 return [gmPerson.cPatient(idents[0].ID)]
897
898
900 if hl7dt == '':
901 return None
902
903 if len(hl7dt) == 8:
904 tmp = time.strptime(hl7dt, '%Y%m%d')
905 return pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone)
906
907 if len(hl7dt) == 12:
908 tmp = time.strptime(hl7dt, '%Y%m%d%H%M')
909 return pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tmp.tm_hour, tmp.tm_min, tzinfo = gmDateTime.gmCurrentLocalTimezone)
910
911 if len(hl7dt) == 14:
912 tmp = time.strptime(hl7dt, '%Y%m%d%H%M%S')
913 return pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tmp.tm_hour, tmp.tm_min, tmp.tm_sec, tzinfo = gmDateTime.gmCurrentLocalTimezone)
914
915 raise ValueError('Observation timestamp not parseable: [%s]', hl7dt)
916
917
919 """Assumes single-PID/single-MSH HL7 file."""
920
921 _log.debug('importing single-PID single-MSH HL7 data from [%s]', filename)
922
923
924 MSH_file = io.open(filename, mode = 'rt', encoding = 'utf8', newline = '')
925 HL7 = pyhl7.parse(MSH_file.read(1024 * 1024 * 5))
926 MSH_file.close()
927
928
929 if len(HL7.segments('MSH')) != 1:
930 _log.error('more than one MSH segment')
931 return False
932 if len(HL7.segments('PID')) != 1:
933 _log.error('more than one PID segment')
934 return False
935
936
937 hl7_lab = HL7.extract_field('MSH', field_num = MSH_field__sending_lab)
938 gm_lab = __find_or_create_lab(hl7_lab)
939
940
941 conn = gmPG2.get_connection(readonly = False)
942 __ensure_hl7_test_types_exist_in_gnumed(link_obj = conn, hl7_data = HL7, pk_test_org = gm_lab['pk_test_org'])
943
944
945 if emr is None:
946
947 pats = __PID2dto(HL7 = HL7)
948 if len(pats) == 0:
949 conn.rollback()
950 return False
951 if len(pats) > 1:
952 conn.rollback()
953 return False
954 emr = pats[0].emr
955
956
957 when_list = {}
958 current_result = None
959 previous_segment = None
960 had_errors = False
961 msh_seen = False
962 pid_seen = False
963 last_obr = None
964 obr = {}
965 for seg_idx in range(len(HL7)):
966 seg = HL7[seg_idx]
967 seg_type = seg[0][0]
968
969 _log.debug('processing line #%s = segment of type <%s>', seg_idx, seg_type)
970
971 if seg_type == 'MSH':
972 msh_seen = True
973
974 if seg_type == 'PID':
975 if not msh_seen:
976 conn.rollback()
977 _log.error('PID segment before MSH segment')
978 return False
979 pid_seen = True
980
981 if seg_type in ['MSH', 'PID']:
982 _log.info('segment already handled')
983 previous_segment = seg_type
984 obr = {}
985 current_result = None
986 continue
987
988 if seg_type in ['ORC']:
989 _log.info('currently ignoring %s segments', seg_type)
990 previous_segment = seg_type
991 obr = {}
992 current_result = None
993 continue
994
995 if seg_type == 'OBR':
996 previous_segment = seg_type
997 last_obr = seg
998 current_result = None
999 obr['abbrev'] = ('%s' % seg[OBR_field__service_name][0]).strip()
1000 try:
1001 obr['name'] = ('%s' % seg[OBR_field__service_name][1]).strip()
1002 except IndexError:
1003 obr['name'] = obr['abbrev']
1004 for field_name in [OBR_field__ts_ended, OBR_field__ts_started, OBR_field__ts_specimen_received, OBR_field__ts_requested]:
1005 obr['clin_when'] = seg[field_name][0].strip()
1006 if obr['clin_when'] != '':
1007 break
1008 continue
1009
1010 if seg_type == 'OBX':
1011 current_result = None
1012
1013 val_alpha = seg[OBX_field__value][0].strip()
1014 is_num, val_num = gmTools.input2decimal(initial = val_alpha)
1015 if is_num:
1016 val_alpha = None
1017 else:
1018 val_num = None
1019 val_alpha = val_alpha.replace('\.br\\', '\n')
1020
1021 unit = seg[OBX_field__unit][0].strip()
1022 if unit == '':
1023 if is_num:
1024 unit = '1/1'
1025 else:
1026 unit = None
1027 test_type = __find_or_create_test_type (
1028 loinc = '%s' % seg[OBX_field__type][0][OBX_component__loinc-1],
1029 name = '%s' % seg[OBX_field__type][0][OBX_component__name-1],
1030 pk_lab = gm_lab['pk_test_org'],
1031 unit = unit,
1032 link_obj = conn
1033 )
1034
1035 epi = emr.add_episode (
1036 link_obj = conn,
1037 episode_name = 'administrative',
1038 is_open = False,
1039 allow_dupes = False
1040 )
1041 current_result = emr.add_test_result (
1042 link_obj = conn,
1043 episode = epi['pk_episode'],
1044 type = test_type['pk_test_type'],
1045 intended_reviewer = gmStaff.gmCurrentProvider()['pk_staff'],
1046 val_num = val_num,
1047 val_alpha = val_alpha,
1048 unit = unit
1049 )
1050
1051 ref_range = seg[OBX_field__range][0].strip()
1052 if ref_range != '':
1053 current_result.reference_range = ref_range
1054 flag = seg[OBX_field__abnormal_flag][0].strip()
1055 if flag != '':
1056 current_result['abnormality_indicator'] = flag
1057 current_result['status'] = seg[OBX_field__status][0].strip()
1058 current_result['val_grouping'] = seg[OBX_field__subid][0].strip()
1059 current_result['source_data'] = ''
1060 if last_obr is not None:
1061 current_result['source_data'] += str(last_obr)
1062 current_result['source_data'] += '\n'
1063 current_result['source_data'] += str(seg)
1064 clin_when = seg[OBX_field__timestamp][0].strip()
1065 if clin_when == '':
1066 _log.warning('no <Observation timestamp> in OBX, trying OBR timestamp')
1067 clin_when = obr['clin_when']
1068 try:
1069 clin_when = __hl7dt2pydt(clin_when)
1070 except ValueError:
1071 _log.exception('clin_when from OBX or OBR not useable, assuming <today>')
1072 if clin_when is not None:
1073 current_result['clin_when'] = clin_when
1074 current_result.save(conn = conn)
1075 when_list[gmDateTime.pydt_strftime(current_result['clin_when'], '%Y %b %d')] = 1
1076 previous_segment = seg_type
1077 continue
1078
1079 if seg_type == 'NTE':
1080 note = seg[NET_field__note][0].strip().replace('\.br\\', '\n')
1081 if note == '':
1082 _log.debug('empty NTE segment')
1083 previous_segment = seg_type
1084 continue
1085
1086
1087
1088 if previous_segment == 'OBR':
1089 _log.debug('NTE following OBR: general note, using OBR timestamp [%s]', obr['clin_when'])
1090 current_result = None
1091 name = obr['name']
1092 if name == '':
1093 name = _('Comment')
1094
1095 test_type = __find_or_create_test_type(name = name, pk_lab = gm_lab['pk_test_org'], abbrev = obr['abbrev'], link_obj = conn)
1096
1097 epi = emr.add_episode (
1098 link_obj = conn,
1099 episode_name = 'administrative',
1100 is_open = False,
1101 allow_dupes = False
1102 )
1103 nte_result = emr.add_test_result (
1104 link_obj = conn,
1105 episode = epi['pk_episode'],
1106 type = test_type['pk_test_type'],
1107 intended_reviewer = gmStaff.gmCurrentProvider()['pk_staff'],
1108 val_alpha = note
1109 )
1110
1111 nte_result['source_data'] = str(seg)
1112 try:
1113 nte_result['clin_when'] = __hl7dt2pydt(obr['clin_when'])
1114 except ValueError:
1115 _log.exception('no .clin_when from OBR for NTE pseudo-OBX available')
1116 nte_result.save(conn = conn)
1117 continue
1118
1119 if (previous_segment == 'OBX') and (current_result is not None):
1120 current_result['source_data'] += '\n'
1121 current_result['source_data'] += str(seg)
1122 current_result['note_test_org'] = gmTools.coalesce (
1123 current_result['note_test_org'],
1124 note,
1125 '%%s\n%s' % note
1126 )
1127 current_result.save(conn = conn)
1128 previous_segment = seg_type
1129 continue
1130
1131 _log.error('unexpected NTE segment')
1132 had_errors = True
1133 break
1134
1135 _log.error('unknown segment, aborting')
1136 _log.debug('line: %s', seg)
1137 had_errors = True
1138 break
1139
1140 if had_errors:
1141 conn.rollback()
1142 return False
1143
1144 conn.commit()
1145
1146
1147 try:
1148 no_results = len(HL7.segments('OBX'))
1149 except KeyError:
1150 no_results = '?'
1151 soap = _(
1152 'Imported HL7 file [%s]:\n'
1153 ' lab "%s" (%s@%s), %s results (%s)'
1154 ) % (
1155 filename,
1156 hl7_lab,
1157 gm_lab['unit'],
1158 gm_lab['organization'],
1159 no_results,
1160 ' / '.join(list(when_list))
1161 )
1162 epi = emr.add_episode (
1163 episode_name = 'administrative',
1164 is_open = False,
1165 allow_dupes = False
1166 )
1167 emr.add_clin_narrative (
1168 note = soap,
1169 soap_cat = None,
1170 episode = epi
1171 )
1172
1173
1174 folder = gmPerson.cPatient(emr.pk_patient).document_folder
1175 hl7_docs = folder.get_documents (
1176 doc_type = 'HL7 data',
1177 pk_episodes = [epi['pk_episode']],
1178 order_by = 'ORDER BY clin_when DESC'
1179 )
1180 if len(hl7_docs) > 0:
1181
1182
1183 hl7_doc = hl7_docs[0]
1184 else:
1185 hl7_doc = folder.add_document (
1186 document_type = 'HL7 data',
1187 encounter = emr.active_encounter['pk_encounter'],
1188 episode = epi['pk_episode']
1189 )
1190 hl7_doc['comment'] = _('list of imported HL7 data files')
1191 hl7_doc['pk_org_unit'] = gmPraxis.gmCurrentPraxisBranch()['pk_org_unit']
1192 hl7_doc['clin_when'] = gmDateTime.pydt_now_here()
1193 hl7_doc.save()
1194 part = hl7_doc.add_part(file = filename)
1195 part['obj_comment'] = _('Result dates: %s') % ' / '.join(list(when_list))
1196 part.save()
1197 hl7_doc.set_reviewed(technically_abnormal = False, clinically_relevant = False)
1198
1199 return True
1200
1201
1202
1204 """Consumes single-MSH single-PID HL7 files."""
1205
1206 _log.debug('staging [%s] as unmatched incoming HL7%s', gmTools.coalesce(source, '', ' (%s)'), filename)
1207
1208
1209 MSH_file = io.open(filename, mode = 'rt', encoding = 'utf8', newline = '')
1210 raw_hl7 = MSH_file.read(1024 * 1024 * 5)
1211 MSH_file.close()
1212 formatted_hl7 = format_hl7_message (
1213 message = raw_hl7,
1214 skip_empty_fields = True,
1215 eol = '\n'
1216 )
1217 HL7 = pyhl7.parse(raw_hl7)
1218 del raw_hl7
1219
1220
1221 incoming = gmIncomingData.create_incoming_data('HL7%s' % gmTools.coalesce(source, '', ' (%s)'), filename)
1222 if incoming is None:
1223 return None
1224 incoming.update_data_from_file(fname = filename)
1225 incoming['comment'] = formatted_hl7
1226 if logfile is not None:
1227 log = io.open(logfile, mode = 'rt', encoding = 'utf8')
1228 incoming['comment'] += '\n'
1229 incoming['comment'] += ('-' * 80)
1230 incoming['comment'] += '\n\n'
1231 incoming['comment'] += log.read()
1232 log.close()
1233 try:
1234 incoming['lastnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname)
1235 incoming['firstnames'] = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__firstname)
1236 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__middlename)
1237 if val is not None:
1238 incoming['firstnames'] += ' '
1239 incoming['firstnames'] += val
1240 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__dob)
1241 if val is not None:
1242 tmp = time.strptime(val, '%Y%m%d')
1243 incoming['dob'] = pyDT.datetime(tmp.tm_year, tmp.tm_mon, tmp.tm_mday, tzinfo = gmDateTime.gmCurrentLocalTimezone)
1244 val = HL7.extract_field('PID', segment_num = 1, field_num = PID_field__gender)
1245 if val is not None:
1246 incoming['gender'] = val
1247 incoming['external_data_id'] = filename
1248
1249
1250
1251
1252
1253
1254
1255
1256 except KeyError:
1257 _log.exception('no PID segment, cannot add more data')
1258 incoming.save()
1259
1260 return incoming
1261
1262
1263
1264
1265 if __name__ == "__main__":
1266
1267 if len(sys.argv) < 2:
1268 sys.exit()
1269
1270 if sys.argv[1] != 'test':
1271 sys.exit()
1272
1273 from Gnumed.pycommon import gmLog2
1274
1275 gmDateTime.init()
1276 gmTools.gmPaths()
1277
1278
1285
1289
1290
1291
1292
1293
1294
1296 hl7 = extract_HL7_from_XML_CDATA(sys.argv[2], './/Message')
1297 print("HL7:", hl7)
1298 result, PID_fnames = split_hl7_file(hl7)
1299 print("result:", result)
1300 print("staging per-PID HL7 files:")
1301 for name in PID_fnames:
1302 print(" file:", name)
1303 __stage_MSH_as_incoming_data(name, source = 'Excelleris')
1304
1306 result, PID_fnames = split_hl7_file(sys.argv[2])
1307 print("result:", result)
1308 print("per-PID HL7 files:")
1309 for name in PID_fnames:
1310 print(" file:", name)
1311
1313 fixed = __fix_malformed_hl7_file(sys.argv[2])
1314 print("fixed HL7:", fixed)
1315 PID_fnames = split_HL7_by_PID(fixed, encoding='utf8')
1316 print("staging per-PID HL7 files:")
1317 for name in PID_fnames:
1318 print(" file:", name)
1319
1320
1331
1338
1341
1343 MSH_file = io.open(sys.argv[2], mode = 'rt', encoding = 'utf8', newline = '')
1344 raw_hl7 = MSH_file.read(1024 * 1024 * 5)
1345 MSH_file.close()
1346 print(format_hl7_message (
1347 message = raw_hl7,
1348 skip_empty_fields = True,
1349 eol = '\n'
1350 ))
1351 HL7 = pyhl7.parse(raw_hl7)
1352 del raw_hl7
1353 for seg in HL7.segments('MSH'):
1354 print(seg)
1355 print("PID:")
1356 print(HL7.extract_field('PID'))
1357 print(HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname))
1358 print(HL7.extract_field('PID', segment_num = 1, field_num = PID_field__name, component_num = PID_component__lastname))
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383 test_parse_hl7()
1384