Package Gnumed :: Package importers :: Module gmDrugsFromCSV
[frames] | no frames]

Source Code for Module Gnumed.importers.gmDrugsFromCSV

  1  # creates SQL to imports a list of mono-component drugs from a CSV file 
  2   
  3  import sys 
  4  import io 
  5   
  6   
  7  # GNUmed 
  8  if __name__ == '__main__': 
  9          sys.path.insert(0, '../../') 
 10  from Gnumed.pycommon import gmLog2 
 11  from Gnumed.pycommon import gmTools 
 12   
 13   
 14  field_names = ['substance', 'product', 'form', 'company', 'strength_1', 'strength_2', 'strength_3', 'always_empty', 'unit'] 
 15  non_empty_fields = ['substance', 'product', 'form', 'company', 'strength_1', 'unit'] 
 16  numeric_fields = ['strength_1', 'strength_2', 'strength_3'] 
 17   
 18   
 19  SQL_start = """-- --------------------------------------------------------- 
 20  -- data pack install script example 
 21  -- 
 22  -- add a description here: Mono-substance drugs as available in India 
 23  -- license: GPL v2 or later, manually transferred 3rd party data 
 24  -- provided by Vaibhav Banait 
 25  -- 
 26  -- http://wiki.gnumed.de/bin/view/Gnumed/GmManualReferenceData 
 27  -- --------------------------------------------------------- 
 28  -- enable this if running locally via 
 29  -- psql -d gnumed_vXX -U gm-dbo -f install-data-pack.sql 
 30  --SET default_transaction_read_only TO OFF; 
 31   
 32  -- --------------------------------------------------------- 
 33  -- drop staging tables if needed 
 34  \\unset ON_ERROR_STOP 
 35  DROP TABLE staging.india_drugs CASCADE; 
 36  \set ON_ERROR_STOP 1 
 37   
 38  -- --------------------------------------------------------- 
 39  -- run everything else in one transaction 
 40  BEGIN; 
 41   
 42  -- create staging tables as needed -- 
 43  CREATE TABLE staging.india_drugs ( 
 44          brand_name text, 
 45          substance text, 
 46          form text, 
 47          strength numeric, 
 48          unit text 
 49  ); 
 50   
 51  -- --------------------------------------------------------- 
 52  -- insert data in staging table 
 53  """ 
 54   
 55   
 56  SQL_stage_drug = """INSERT INTO staging.india_drugs (brand_name, substance, form, strength, unit) SELECT 
 57          '%(brand_name)s', 
 58          '%(substance)s', 
 59          '%(form)s', 
 60          gm.nullify_empty_string('%(strength)s')::numeric, 
 61          '%(unit)s' 
 62  WHERE NOT EXISTS ( 
 63          SELECT 1 FROM staging.india_drugs WHERE brand_name = '%(brand_name)s' 
 64  ); 
 65  """ 
 66   
 67   
 68  SQL_end = """-- --------------------------------------------------------- 
 69  -- transfer data to real tables 
 70   
 71  -- substances 
 72  INSERT INTO ref.consumable_substance (description, amount, unit) SELECT 
 73          DISTINCT ON (s_id.substance, s_id.strength, s_id.unit) 
 74          s_id.substance, s_id.strength, s_id.unit 
 75  FROM  
 76          staging.india_drugs s_id 
 77  WHERE 
 78          s_id.strength IS NOT NULL 
 79                  AND 
 80          NOT EXISTS ( 
 81                  SELECT 1 FROM ref.consumable_substance r_cs WHERE 
 82                          r_cs.description = s_id.substance 
 83                                  AND 
 84                          r_cs.amount = s_id.strength 
 85                                  AND 
 86                          r_cs.unit = s_id.unit 
 87          ) 
 88  ; 
 89   
 90  -- drug products 
 91  INSERT INTO ref.drug_product (description, preparation) SELECT 
 92          s_id.brand_name, s_id.form 
 93  FROM 
 94          staging.india_drugs s_id 
 95  WHERE NOT EXISTS ( 
 96          SELECT 1 FROM ref.drug_product r_bd WHERE 
 97                  r_bd.description = s_id.brand_name 
 98                          AND 
 99                  r_bd.preparation = s_id.form 
100  ); 
101   
102  -- link components 
103  INSERT INTO ref.lnk_substance2brand (fk_drug_product, fk_substance) SELECT 
104          (SELECT pk FROM ref.drug_product r_bd 
105           WHERE 
106                  r_bd.description = s_id.brand_name 
107                          AND 
108                  r_bd.preparation = s_id.form 
109          ), 
110          (SELECT pk FROM ref.consumable_substance r_cs WHERE 
111                  r_cs.description = s_id.substance 
112                          AND 
113                  r_cs.amount = s_id.strength 
114                          AND 
115                  r_cs.unit = s_id.unit 
116          ) 
117  FROM 
118          staging.india_drugs s_id 
119  WHERE NOT EXISTS ( 
120          SELECT 1 FROM ref.lnk_substance2brand WHERE 
121                  fk_drug_product = ( 
122                          SELECT pk FROM ref.drug_product WHERE 
123                                  description = s_id.brand_name 
124                                          AND 
125                                  preparation = s_id.form 
126                  ) 
127                          AND 
128                  fk_substance = ( 
129                          SELECT pk FROM ref.consumable_substance r_cs WHERE 
130                                  r_cs.description = s_id.substance 
131                                          AND 
132                                  r_cs.amount = s_id.strength 
133                                          AND 
134                                  r_cs.unit = s_id.unit 
135                  ) 
136  ); 
137   
138  -- --------------------------------------------------------- 
139  -- drop staging tables again, if needed -- 
140  \\unset ON_ERROR_STOP 
141  DROP TABLE staging.india_drugs CASCADE; 
142  \set ON_ERROR_STOP 1 
143   
144  -- --------------------------------------------------------- 
145   
146  -- finalize transaction -- 
147  -- uncomment this once you are satisfied your script works: 
148  COMMIT; 
149  -- --------------------------------------------------------- 
150  """ 
151   
152  #--------------------------------------------------------------------------------------------------- 
153 -def create_sql(filename):
154 155 csv_file = io.open(filename, mode = 'rt', encoding = 'utf8') 156 157 csv_lines = gmTools.unicode_csv_reader ( 158 csv_file, 159 fieldnames = field_names, 160 delimiter = ';', 161 quotechar = '"', 162 dict = True 163 ) 164 165 print SQL_start 166 167 line_idx = 0 168 skip_line = False 169 for line in csv_lines: 170 line_idx += 1 171 print "-- line #%s" % line_idx 172 # normalize field content 173 for field in field_names: 174 try: 175 line[field] = line[field].strip().strip(';,').strip().replace("'", "''") 176 except AttributeError: # trailing fields are a list 177 pass 178 # verify required fields 179 for field in non_empty_fields: 180 if line[field] == '': 181 print "-- ignoring line: empty field [%s]" % field 182 print "--", line 183 print "" 184 skip_line = True 185 break 186 if skip_line: 187 skip_line = False 188 continue 189 # verify numeric fields 190 for field in numeric_fields: 191 if line[field] == '': 192 continue 193 success, num_val = gmTools.input2decimal(initial = line[field]) 194 if not success: 195 print "-- ignoring line: field [%s] not numeric: >>>%s<<<" % (field, line[field]) 196 print "--", line 197 print "" 198 skip_line = True 199 break 200 line[field] = num_val 201 if skip_line: 202 skip_line = False 203 continue 204 205 # actually create SQL 206 # loop over strengths 207 for field in numeric_fields: 208 if line[field] == '': 209 continue 210 line['brand_name'] = ('%%(product)s %%(%s)s (%%(company)s)' % field) % line 211 line['strength'] = line[field] 212 print SQL_stage_drug % line 213 214 print SQL_end
215 216 #--------------------------------------------------------------------------------------------------- 217 218 create_sql(sys.argv[1]) 219