1 """GNUmed PostgreSQL connection handling.
2
3 TODO: iterator/generator batch fetching:
4 - http://groups-beta.google.com/group/comp.lang.python/msg/7ff516d7d9387dad
5 - search Google for "Geneator/Iterator Nesting Problem - Any Ideas? 2.4"
6
7 winner:
8 def resultset_functional_batchgenerator(cursor, size=100):
9 for results in iter(lambda: cursor.fetchmany(size), []):
10 for rec in results:
11 yield rec
12 """
13
14 __author__ = "K.Hilbert <Karsten.Hilbert@gmx.net>"
15 __license__ = 'GPL v2 or later (details at http://www.gnu.org)'
16
17
18 import time
19 import sys
20 import os
21 import stat
22 import io
23 import codecs
24 import logging
25 import datetime as pydt
26 import re as regex
27 import threading
28 import hashlib
29 import shutil
30
31
32
33 if __name__ == '__main__':
34 sys.path.insert(0, '../../')
35 from Gnumed.pycommon import gmLoginInfo
36 from Gnumed.pycommon import gmExceptions
37 from Gnumed.pycommon import gmDateTime
38 from Gnumed.pycommon import gmBorg
39 from Gnumed.pycommon import gmI18N
40 from Gnumed.pycommon import gmLog2
41 from Gnumed.pycommon import gmTools
42 from Gnumed.pycommon.gmTools import prompted_input, u_replacement_character, format_dict_like
43
44 _log = logging.getLogger('gm.db')
45
46
47
48 try:
49 import psycopg2 as dbapi
50 except ImportError:
51 _log.exception("Python database adapter psycopg2 not found.")
52 print("CRITICAL ERROR: Cannot find module psycopg2 for connecting to the database server.")
53 raise
54
55
56 _log.info('psycopg2 version: %s' % dbapi.__version__)
57 _log.info('PostgreSQL via DB-API module "%s": API level %s, thread safety %s, parameter style "%s"' % (dbapi, dbapi.apilevel, dbapi.threadsafety, dbapi.paramstyle))
58 if not (float(dbapi.apilevel) >= 2.0):
59 raise ImportError('gmPG2: supported DB-API level too low')
60 if not (dbapi.threadsafety > 0):
61 raise ImportError('gmPG2: lacking minimum thread safety in psycopg2')
62 if not (dbapi.paramstyle == 'pyformat'):
63 raise ImportError('gmPG2: lacking pyformat (%%(<name>)s style) placeholder support in psycopg2')
64 try:
65 dbapi.__version__.index('dt')
66 except ValueError:
67 raise ImportError('gmPG2: lacking datetime support in psycopg2')
68 try:
69 dbapi.__version__.index('ext')
70 except ValueError:
71 raise ImportError('gmPG2: lacking extensions support in psycopg2')
72 try:
73 dbapi.__version__.index('pq3')
74 except ValueError:
75 raise ImportError('gmPG2: lacking v3 backend protocol support in psycopg2')
76
77 import psycopg2.extras
78 import psycopg2.extensions
79 import psycopg2.pool
80 import psycopg2.errorcodes as sql_error_codes
81
82 PG_ERROR_EXCEPTION = dbapi.Error
83
84
85 _default_client_encoding = 'UTF8'
86 _log.info('assuming default client encoding of [%s]' % _default_client_encoding)
87
88
89 _default_client_timezone = None
90 _sql_set_timezone = None
91 _timestamp_template = "cast('%s' as timestamp with time zone)"
92 FixedOffsetTimezone = dbapi.tz.FixedOffsetTimezone
93
94 _default_dsn = None
95 _default_login = None
96
97 default_database = 'gnumed_v22'
98
99 postgresql_version_string = None
100 postgresql_version = None
101
102 __ro_conn_pool = None
103
104 auto_request_login_params = True
105
106
107
108
109
110 known_schema_hashes = {
111 0: 'not released, testing only',
112 2: 'b09d50d7ed3f91ddf4c4ddb8ea507720',
113 3: 'e73718eaf230d8f1d2d01afa8462e176',
114 4: '4428ccf2e54c289136819e701bb095ea',
115 5: '7e7b093af57aea48c288e76632a382e5',
116 6: '90e2026ac2efd236da9c8608b8685b2d',
117 7: '6c9f6d3981483f8e9433df99d1947b27',
118 8: '89b13a7af83337c3aad153b717e52360',
119 9: '641a9b2be3c378ffc2bb2f0b1c9f051d',
120 10: '7ef42a8fb2bd929a2cdd0c63864b4e8a',
121 11: '03042ae24f3f92877d986fb0a6184d76',
122 12: '06183a6616db62257e22814007a8ed07',
123 13: 'fab7c1ae408a6530c47f9b5111a0841e',
124 14: 'e170d543f067d1ea60bfe9076b1560cf',
125 15: '70012ff960b77ecdff4981c94b5b55b6',
126 16: '0bcf44ca22c479b52976e5eda1de8161',
127 17: '161428ee97a00e3bf56168c3a15b7b50',
128 18: 'a0f9efcabdecfb4ddb6d8c0b69c02092',
129
130
131 19: '57f009a159f55f77525cc0291e0c8b60',
132 20: 'baed1901ed4c2f272b56c8cb2c6d88e8',
133 21: 'e6a51a89dd22b75b61ead8f7083f251f',
134 22: 'bf45f01327fb5feb2f5d3c06ba4a6792'
135 }
136
137 map_schema_hash2version = {
138 'b09d50d7ed3f91ddf4c4ddb8ea507720': 2,
139 'e73718eaf230d8f1d2d01afa8462e176': 3,
140 '4428ccf2e54c289136819e701bb095ea': 4,
141 '7e7b093af57aea48c288e76632a382e5': 5,
142 '90e2026ac2efd236da9c8608b8685b2d': 6,
143 '6c9f6d3981483f8e9433df99d1947b27': 7,
144 '89b13a7af83337c3aad153b717e52360': 8,
145 '641a9b2be3c378ffc2bb2f0b1c9f051d': 9,
146 '7ef42a8fb2bd929a2cdd0c63864b4e8a': 10,
147 '03042ae24f3f92877d986fb0a6184d76': 11,
148 '06183a6616db62257e22814007a8ed07': 12,
149 'fab7c1ae408a6530c47f9b5111a0841e': 13,
150 'e170d543f067d1ea60bfe9076b1560cf': 14,
151 '70012ff960b77ecdff4981c94b5b55b6': 15,
152 '0bcf44ca22c479b52976e5eda1de8161': 16,
153 '161428ee97a00e3bf56168c3a15b7b50': 17,
154 'a0f9efcabdecfb4ddb6d8c0b69c02092': 18,
155
156
157 '57f009a159f55f77525cc0291e0c8b60': 19,
158 'baed1901ed4c2f272b56c8cb2c6d88e8': 20,
159 'e6a51a89dd22b75b61ead8f7083f251f': 21,
160 'bf45f01327fb5feb2f5d3c06ba4a6792': 22
161 }
162
163 map_client_branch2required_db_version = {
164 'GIT tree': 0,
165 '0.3': 9,
166 '0.4': 10,
167 '0.5': 11,
168 '0.6': 12,
169 '0.7': 13,
170 '0.8': 14,
171 '0.9': 15,
172 '1.0': 16,
173 '1.1': 16,
174 '1.2': 17,
175 '1.3': 18,
176 '1.4': 19,
177 '1.5': 20,
178 '1.6': 21,
179 '1.7': 22,
180 '1.8': 22
181 }
182
183 map_psyco_tx_status2str = [
184 'TRANSACTION_STATUS_IDLE',
185 'TRANSACTION_STATUS_ACTIVE',
186 'TRANSACTION_STATUS_INTRANS',
187 'TRANSACTION_STATUS_INERROR',
188 'TRANSACTION_STATUS_UNKNOWN'
189 ]
190
191 map_psyco_conn_status2str = [
192 '0 - ?',
193 'STATUS_READY',
194 'STATUS_BEGIN_ALIAS_IN_TRANSACTION',
195 'STATUS_PREPARED'
196 ]
197
198 map_psyco_iso_level2str = {
199 None: 'ISOLATION_LEVEL_DEFAULT (configured on server)',
200 0: 'ISOLATION_LEVEL_AUTOCOMMIT',
201 1: 'ISOLATION_LEVEL_READ_UNCOMMITTED',
202 2: 'ISOLATION_LEVEL_REPEATABLE_READ',
203 3: 'ISOLATION_LEVEL_SERIALIZABLE',
204 4: 'ISOLATION_LEVEL_READ_UNCOMMITTED'
205 }
206
207
208 query_table_col_defs = """select
209 cols.column_name,
210 cols.udt_name
211 from
212 information_schema.columns cols
213 where
214 cols.table_schema = %s
215 and
216 cols.table_name = %s
217 order by
218 cols.ordinal_position"""
219
220 query_table_attributes = """select
221 cols.column_name
222 from
223 information_schema.columns cols
224 where
225 cols.table_schema = %s
226 and
227 cols.table_name = %s
228 order by
229 cols.ordinal_position"""
230
231
232
233 SQL_foreign_key_name = """SELECT
234 fk_tbl.*,
235 (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = fk_tbl.connamespace) AS constraint_schema,
236 fk_tbl.conname AS constraint_name,
237 (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = (SELECT relnamespace FROM pg_class where oid = fk_tbl.conrelid)) AS source_schema,
238 (SELECT relname FROM pg_catalog.pg_class where oid = fk_tbl.conrelid) AS source_table,
239 (SELECT attname FROM pg_catalog.pg_attribute WHERE attnum = fk_tbl.conkey[1] AND attrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass) AS source_column,
240 (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = (SELECT relnamespace FROM pg_class where oid = fk_tbl.confrelid)) AS target_schema,
241 (SELECT relname FROM pg_catalog.pg_class where oid = fk_tbl.confrelid) AS target_table,
242 (SELECT attname FROM pg_catalog.pg_attribute WHERE attnum = fk_tbl.confkey[1] AND attrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass) AS target_column
243 FROM
244 pg_catalog.pg_constraint fk_tbl
245 WHERE
246 fk_tbl.contype = 'f'
247 AND
248 fk_tbl.conrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass
249 AND
250 fk_tbl.conkey[1] = (
251 SELECT
252 col_tbl1.attnum
253 FROM
254 pg_catalog.pg_attribute col_tbl1
255 WHERE
256 col_tbl1.attname = %(src_col)s
257 AND
258 col_tbl1.attrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass
259 )
260 AND
261 fk_tbl.confrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass
262 AND
263 fk_tbl.confkey[1] = (
264 SELECT
265 col_tbl2.attnum
266 FROM
267 pg_catalog.pg_attribute col_tbl2
268 WHERE
269 col_tbl2.attname = %(target_col)s
270 AND
271 col_tbl2.attrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass
272 )
273 """
274
275 SQL_get_index_name = """
276 SELECT
277 (SELECT nspname FROM pg_namespace WHERE pg_namespace.oid = pg_class.relnamespace)
278 AS index_schema,
279 pg_class.relname
280 AS index_name
281 FROM
282 pg_class
283 WHERE
284 pg_class.oid IN (
285 SELECT
286 indexrelid
287 FROM
288 pg_index
289 WHERE
290 pg_index.indrelid = %(idx_tbl)s::regclass
291 AND
292 pg_index.indnatts = 1 -- only one column in index
293 AND
294 pg_index.indkey[0] IN (
295 SELECT
296 pg_attribute.attnum
297 FROM
298 pg_attribute
299 WHERE
300 pg_attribute.attrelid = %(idx_tbl)s::regclass
301 AND
302 pg_attribute.attname = %(idx_col)s
303 )
304 )
305 """
306
307 SQL_get_pk_col_def = """
308 SELECT
309 pg_attribute.attname
310 AS pk_col,
311 format_type(pg_attribute.atttypid, pg_attribute.atttypmod)
312 AS pk_type
313 FROM pg_index, pg_class, pg_attribute, pg_namespace
314 WHERE
315 pg_class.oid = %(table)s::regclass
316 AND
317 indrelid = pg_class.oid
318 AND
319 -- nspname = %%(schema)s
320 -- AND
321 pg_class.relnamespace = pg_namespace.oid
322 AND
323 pg_attribute.attrelid = pg_class.oid
324 AND
325 pg_attribute.attnum = any(pg_index.indkey)
326 AND
327 indisprimary
328 """
329
330
331
332
334
335 if encoding not in psycopg2.extensions.encodings:
336 raise ValueError('psycopg2 does not know how to handle client (wire) encoding [%s]' % encoding)
337
338 py_enc = psycopg2.extensions.encodings[encoding]
339 try:
340 codecs.lookup(py_enc)
341 except LookupError:
342 _log.warning('<codecs> module can NOT handle encoding [psycopg2::<%s> -> Python::<%s>]' % (encoding, py_enc))
343 raise
344
345
346
347
348 global _default_client_encoding
349 _log.info('setting default client encoding from [%s] to [%s]' % (_default_client_encoding, encoding))
350 _default_client_encoding = encoding
351 return True
352
353
365
366
368
369 _log.debug('validating time zone [%s]', timezone)
370
371 cmd = 'set timezone to %(tz)s'
372 args = {'tz': timezone}
373
374 conn.commit()
375 curs = conn.cursor()
376 is_valid = False
377 try:
378 curs.execute(cmd, args)
379 _log.info('time zone [%s] is settable', timezone)
380
381 cmd = """select '1920-01-19 23:00:00+01'::timestamp with time zone"""
382 try:
383 curs.execute(cmd)
384 curs.fetchone()
385 _log.info('time zone [%s] is usable', timezone)
386 is_valid = True
387 except:
388 _log.error('error using time zone [%s]', timezone)
389 except dbapi.DataError:
390 _log.warning('time zone [%s] is not settable', timezone)
391 except:
392 _log.error('failed to set time zone to [%s]', timezone)
393 _log.exception('')
394
395 curs.close()
396 conn.rollback()
397
398 return is_valid
399
400
402 """some timezone defs are abbreviations so try to expand
403 them because "set time zone" doesn't take abbreviations"""
404
405 cmd = """
406 select distinct on (abbrev) name
407 from pg_timezone_names
408 where
409 abbrev = %(tz)s and
410 name ~ '^[^/]+/[^/]+$' and
411 name !~ '^Etc/'
412 """
413 args = {'tz': timezone}
414
415 conn.commit()
416 curs = conn.cursor()
417
418 result = timezone
419 try:
420 curs.execute(cmd, args)
421 rows = curs.fetchall()
422 if len(rows) > 0:
423 result = rows[0]['name']
424 _log.debug('[%s] maps to [%s]', timezone, result)
425 except:
426 _log.exception('cannot expand timezone abbreviation [%s]', timezone)
427
428 curs.close()
429 conn.rollback()
430
431 return result
432
433
479
480
481
482
484 """Text mode request of database login parameters"""
485 import getpass
486 login = gmLoginInfo.LoginInfo()
487
488 print("\nPlease enter the required login parameters:")
489 try:
490 login.host = prompted_input(prompt = "host ('' = non-TCP/IP)", default = '')
491 login.database = prompted_input(prompt = "database", default = default_database)
492 login.user = prompted_input(prompt = "user name", default = '')
493 tmp = 'password for "%s" (not shown): ' % login.user
494 login.password = getpass.getpass(tmp)
495 gmLog2.add_word2hide(login.password)
496 login.port = prompted_input(prompt = "port", default = 5432)
497 except KeyboardInterrupt:
498 _log.warning("user cancelled text mode login dialog")
499 print("user cancelled text mode login dialog")
500 raise gmExceptions.ConnectionError(_("Cannot connect to database without login information!"))
501
502 return login
503
504
506 """GUI (wx) input request for database login parameters.
507
508 Returns gmLoginInfo.LoginInfo object
509 """
510 import wx
511
512
513 if wx.GetApp() is None:
514 raise AssertionError(_("The wxPython GUI framework hasn't been initialized yet!"))
515
516
517
518 import gmAuthWidgets
519 dlg = gmAuthWidgets.cLoginDialog(None, -1)
520 dlg.ShowModal()
521 login = dlg.panel.GetLoginInfo()
522 dlg.DestroyLater()
523
524
525 if login is None:
526 raise gmExceptions.ConnectionError(_("Can't connect to database without login information!"))
527
528 gmLog2.add_word2hide(login.password)
529
530 return login
531
532
534 """Request login parameters for database connection."""
535
536 if not auto_request_login_params:
537 raise Exception('Cannot request login parameters.')
538
539
540
541 if 'DISPLAY' in os.environ:
542
543 try:
544 return __request_login_params_gui_wx()
545 except:
546 pass
547
548
549
550 return __request_login_params_tui()
551
552
553
554
555 -def make_psycopg2_dsn(database=None, host=None, port=5432, user=None, password=None):
556 dsn_parts = []
557
558 if (database is not None) and (database.strip() != ''):
559 dsn_parts.append('dbname=%s' % database)
560
561 if (host is not None) and (host.strip() != ''):
562 dsn_parts.append('host=%s' % host)
563
564 if (port is not None) and (str(port).strip() != ''):
565 dsn_parts.append('port=%s' % port)
566
567 if (user is not None) and (user.strip() != ''):
568 dsn_parts.append('user=%s' % user)
569
570 if (password is not None) and (password.strip() != ''):
571 dsn_parts.append('password=%s' % password)
572
573 dsn_parts.append('sslmode=prefer')
574 dsn_parts.append('fallback_application_name=GNUmed')
575
576 return ' '.join(dsn_parts)
577
578
583
584
594
595
597 if login is None:
598 return False
599
600 if login.host is not None:
601 if login.host.strip() == '':
602 login.host = None
603
604 global _default_login
605 _default_login = login
606 _log.info('setting default login from [%s] to [%s]' % (_default_login, login))
607
608 dsn = make_psycopg2_dsn(login.database, login.host, login.port, login.user, login.password)
609
610 global _default_dsn
611 if _default_dsn is None:
612 old_dsn = 'None'
613 else:
614 old_dsn = regex.sub(r'password=[^\s]+', 'password=%s' % u_replacement_character, _default_dsn)
615 _log.info ('setting default DSN from [%s] to [%s]',
616 old_dsn,
617 regex.sub(r'password=[^\s]+', 'password=%s' % u_replacement_character, dsn)
618 )
619 _default_dsn = dsn
620
621 return True
622
623
625 try:
626 pgpass_file = os.path.expanduser(os.path.join('~', '.pgpass'))
627 if os.path.exists(pgpass_file):
628 _log.debug('standard .pgpass (%s) exists', pgpass_file)
629 else:
630 _log.debug('standard .pgpass (%s) not found', pgpass_file)
631 pgpass_var = os.getenv('PGPASSFILE')
632 if pgpass_var is None:
633 _log.debug('$PGPASSFILE not set')
634 else:
635 if os.path.exists(pgpass_var):
636 _log.debug('$PGPASSFILE=%s exists', pgpass_var)
637 else:
638 _log.debug('$PGPASSFILE=%s not found')
639 except Exception:
640 _log.exception('cannot detect .pgpass and or $PGPASSFILE')
641
642
643
644
646 expected_hash = known_schema_hashes[version]
647 if version == 0:
648 args = {'ver': 9999}
649 else:
650 args = {'ver': version}
651 rows, idx = run_ro_queries (
652 link_obj = link_obj,
653 queries = [{
654 'cmd': 'select md5(gm.concat_table_structure(%(ver)s::integer)) as md5',
655 'args': args
656 }]
657 )
658 if rows[0]['md5'] != expected_hash:
659 _log.error('database schema version mismatch')
660 _log.error('expected: %s (%s)' % (version, expected_hash))
661 _log.error('detected: %s (%s)' % (get_schema_version(link_obj=link_obj), rows[0]['md5']))
662 if verbose:
663 _log.debug('schema dump follows:')
664 for line in get_schema_structure(link_obj = link_obj).split():
665 _log.debug(line)
666 _log.debug('schema revision history dump follows:')
667 for line in get_schema_revision_history(link_obj = link_obj):
668 _log.debug(' - '.join(line))
669 return False
670 _log.info('detected schema version [%s], hash [%s]' % (map_schema_hash2version[rows[0]['md5']], rows[0]['md5']))
671 return True
672
673
675 rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select md5(gm.concat_table_structure()) as md5'}])
676 try:
677 return map_schema_hash2version[rows[0]['md5']]
678 except KeyError:
679 return 'unknown database schema version, MD5 hash is [%s]' % rows[0]['md5']
680
681
683 rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select gm.concat_table_structure()'}])
684 return rows[0][0]
685
686
688 rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select md5(gm.concat_table_structure()) as md5'}])
689 return rows[0]['md5']
690
691
693
694 if table_exists(link_obj = link_obj, schema = 'gm', table = 'schema_revision'):
695 cmd = """
696 SELECT
697 imported::text,
698 version,
699 filename
700 FROM gm.schema_revision
701 ORDER BY imported"""
702 elif table_exists(link_obj = link_obj, schema = 'public', table = 'gm_schema_revision'):
703 cmd = """
704 SELECT
705 imported::text,
706 version,
707 filename
708 FROM public.gm_schema_revision
709 ORDER BY imported"""
710 else:
711 return []
712
713 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}])
714 return rows
715
717 rows, idx = run_ro_queries(queries = [{'cmd': 'select CURRENT_USER'}])
718 return rows[0][0]
719
720
722 """Get the foreign keys pointing to schema.table.column.
723
724 Does not properly work with multi-column FKs.
725 GNUmed doesn't use any, however.
726 """
727 args = {
728 'schema': schema,
729 'tbl': table,
730 'col': column
731 }
732 cmd = """
733 SELECT
734 %(schema)s AS referenced_schema,
735 %(tbl)s AS referenced_table,
736 %(col)s AS referenced_column,
737 pgc.confkey AS referenced_column_list,
738
739 pgc.conrelid::regclass AS referencing_table,
740 pgc.conkey AS referencing_column_list,
741 (select attname from pg_attribute where attnum = pgc.conkey[1] and attrelid = pgc.conrelid) AS referencing_column
742 FROM
743 pg_constraint pgc
744 WHERE
745 pgc.contype = 'f'
746 AND
747 pgc.confrelid = (
748 select oid from pg_class where relname = %(tbl)s and relnamespace = (
749 select oid from pg_namespace where nspname = %(schema)s
750 )
751 ) and
752 (
753 select attnum
754 from pg_attribute
755 where
756 attrelid = (select oid from pg_class where relname = %(tbl)s and relnamespace = (
757 select oid from pg_namespace where nspname = %(schema)s
758 ))
759 and
760 attname = %(col)s
761 ) = any(pgc.confkey)
762 """
763 rows, idx = run_ro_queries (
764 link_obj = link_obj,
765 queries = [
766 {'cmd': cmd, 'args': args}
767 ]
768 )
769
770 return rows
771
772
773 -def get_index_name(indexed_table=None, indexed_column=None, link_obj=None):
774
775 args = {
776 'idx_tbl': indexed_table,
777 'idx_col': indexed_column
778 }
779 rows, idx = run_ro_queries (
780 link_obj = link_obj,
781 queries = [{'cmd': SQL_get_index_name, 'args': args}],
782 get_col_idx = False
783 )
784
785 return rows
786
787
788 -def get_foreign_key_names(src_schema=None, src_table=None, src_column=None, target_schema=None, target_table=None, target_column=None, link_obj=None):
789
790 args = {
791 'src_schema': src_schema,
792 'src_tbl': src_table,
793 'src_col': src_column,
794 'target_schema': target_schema,
795 'target_tbl': target_table,
796 'target_col': target_column
797 }
798
799 rows, idx = run_ro_queries (
800 link_obj = link_obj,
801 queries = [{'cmd': SQL_foreign_key_name, 'args': args}],
802 get_col_idx = False
803 )
804
805 return rows
806
807
809 """Return child tables of <table>."""
810 cmd = """
811 select
812 pgn.nspname as namespace,
813 pgc.relname as table
814 from
815 pg_namespace pgn,
816 pg_class pgc
817 where
818 pgc.relnamespace = pgn.oid
819 and
820 pgc.oid in (
821 select inhrelid from pg_inherits where inhparent = (
822 select oid from pg_class where
823 relnamespace = (select oid from pg_namespace where nspname = %(schema)s) and
824 relname = %(table)s
825 )
826 )"""
827 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': {'schema': schema, 'table': table}}])
828 return rows
829
830
832 cmd = """SELECT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = %(schema)s)"""
833 args = {'schema': schema}
834 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}])
835 return rows[0][0]
836
837
839 """Returns false, true."""
840 cmd = """
841 select exists (
842 select 1 from information_schema.tables
843 where
844 table_schema = %s and
845 table_name = %s and
846 table_type = 'BASE TABLE'
847 )"""
848 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': (schema, table)}])
849 return rows[0][0]
850
851
853
854 cmd = """
855 SELECT EXISTS (
856 SELECT 1 FROM pg_proc
857 WHERE proname = %(func)s AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = %(schema)s)
858 )
859 """
860 args = {
861 'func': function,
862 'schema': schema
863 }
864 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}])
865 return rows[0][0]
866
867
869 if cursor.description is None:
870 _log.error('no result description available: unused cursor or last query did not select rows')
871 return None
872 col_indices = {}
873 col_index = 0
874 for col_desc in cursor.description:
875 col_name = col_desc[0]
876
877
878
879 if col_name in col_indices:
880 col_name = '%s_%s' % (col_name, col_index)
881 col_indices[col_name] = col_index
882 col_index += 1
883
884 return col_indices
885
886 -def get_col_defs(link_obj=None, schema='public', table=None):
887 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_col_defs, 'args': (schema, table)}])
888 col_names = []
889 col_type = {}
890 for row in rows:
891 col_names.append(row[0])
892
893 if row[1].startswith('_'):
894 col_type[row[0]] = row[1][1:] + '[]'
895 else:
896 col_type[row[0]] = row[1]
897 col_defs = []
898 col_defs.append(col_names)
899 col_defs.append(col_type)
900 return col_defs
901
903 """Return column attributes of table"""
904 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_attributes, 'args': (schema, table)}])
905 cols = []
906 for row in rows:
907 cols.append(row[0])
908 return cols
909
910
911
912
914 tx_file = io.open(filename, mode = 'wt', encoding = 'utf8')
915 tx_file.write('-- GNUmed database string translations exported %s\n' % gmDateTime.pydt_now_here().strftime('%Y-%m-%d %H:%M'))
916 tx_file.write('-- - contains translations for each of [%s]\n' % ', '.join(get_translation_languages()))
917 tx_file.write('-- - user database language is set to [%s]\n\n' % get_current_user_language())
918 tx_file.write('-- Please email this file to <gnumed-devel@gnu.org>.\n')
919 tx_file.write('-- ----------------------------------------------------------------------------------------------\n\n')
920 tx_file.write('set default_transaction_read_only to off;\n\n')
921 tx_file.write("set client_encoding to 'utf-8';\n\n")
922 tx_file.write('\\unset ON_ERROR_STOP\n\n')
923
924 cmd = 'SELECT lang, orig, trans FROM i18n.translations ORDER BY lang, orig'
925 rows, idx = run_ro_queries(queries = [{'cmd': cmd}], get_col_idx = False)
926 for row in rows:
927 line = "select i18n.upd_tx(E'%s', E'%s', E'%s');\n" % (
928 row['lang'].replace("'", "\\'"),
929 row['orig'].replace("'", "\\'"),
930 row['trans'].replace("'", "\\'")
931 )
932 tx_file.write(line)
933 tx_file.write('\n')
934
935 tx_file.write('\set ON_ERROR_STOP 1\n')
936 tx_file.close()
937
938 return True
939
940
942 cmd = 'DELETE FROM i18n.translations WHERE lang = %(lang)s AND orig = %(orig)s'
943 args = {'lang': language, 'orig': original}
944 run_rw_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}], return_data = False, end_tx = True)
945 return True
946
947
949 if language is None:
950 cmd = 'SELECT i18n.upd_tx(%(orig)s, %(trans)s)'
951 else:
952 cmd = 'SELECT i18n.upd_tx(%(lang)s, %(orig)s, %(trans)s)'
953 args = {'lang': language, 'orig': original, 'trans': translation}
954 run_rw_queries(queries = [{'cmd': cmd, 'args': args}], return_data = False, link_obj = link_obj)
955 return args
956
957
959 rows, idx = run_ro_queries (
960 queries = [{'cmd': 'select distinct lang from i18n.translations'}]
961 )
962 return [ r[0] for r in rows ]
963
964
966
967 args = {'lang': language}
968 _log.debug('language [%s]', language)
969
970 if order_by is None:
971 order_by = 'ORDER BY %s' % order_by
972 else:
973 order_by = 'ORDER BY lang, orig'
974
975 if language is None:
976 cmd = """
977 SELECT DISTINCT ON (orig, lang)
978 lang, orig, trans
979 FROM ((
980
981 -- strings stored as translation keys whether translated or not
982 SELECT
983 NULL as lang,
984 ik.orig,
985 NULL AS trans
986 FROM
987 i18n.keys ik
988
989 ) UNION ALL (
990
991 -- already translated strings
992 SELECT
993 it.lang,
994 it.orig,
995 it.trans
996 FROM
997 i18n.translations it
998
999 )) as translatable_strings
1000 %s""" % order_by
1001 else:
1002 cmd = """
1003 SELECT DISTINCT ON (orig, lang)
1004 lang, orig, trans
1005 FROM ((
1006
1007 -- strings stored as translation keys whether translated or not
1008 SELECT
1009 %%(lang)s as lang,
1010 ik.orig,
1011 i18n._(ik.orig, %%(lang)s) AS trans
1012 FROM
1013 i18n.keys ik
1014
1015 ) UNION ALL (
1016
1017 -- already translated strings
1018 SELECT
1019 %%(lang)s as lang,
1020 it.orig,
1021 i18n._(it.orig, %%(lang)s) AS trans
1022 FROM
1023 i18n.translations it
1024
1025 )) AS translatable_strings
1026 %s""" % order_by
1027
1028 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': args}], get_col_idx = False)
1029
1030 if rows is None:
1031 _log.error('no translatable strings found')
1032 else:
1033 _log.debug('%s translatable strings found', len(rows))
1034
1035 return rows
1036
1037
1039 cmd = 'select i18n.get_curr_lang()'
1040 rows, idx = run_ro_queries(queries = [{'cmd': cmd}])
1041 return rows[0][0]
1042
1043
1045 """Set the user language in the database.
1046
1047 user = None: current db user
1048 language = None: unset
1049 """
1050 _log.info('setting database language for user [%s] to [%s]', user, language)
1051
1052 args = {
1053 'usr': user,
1054 'lang': language
1055 }
1056
1057 if language is None:
1058 if user is None:
1059 queries = [{'cmd': 'select i18n.unset_curr_lang()'}]
1060 else:
1061 queries = [{'cmd': 'select i18n.unset_curr_lang(%(usr)s)', 'args': args}]
1062 queries.append({'cmd': 'select True'})
1063 else:
1064 if user is None:
1065 queries = [{'cmd': 'select i18n.set_curr_lang(%(lang)s)', 'args': args}]
1066 else:
1067 queries = [{'cmd': 'select i18n.set_curr_lang(%(lang)s, %(usr)s)', 'args': args}]
1068
1069 rows, idx = run_rw_queries(queries = queries, return_data = True)
1070
1071 if not rows[0][0]:
1072 _log.error('cannot set database language to [%s] for user [%s]', language, user)
1073
1074 return rows[0][0]
1075
1077 """Set the user language in the database.
1078
1079 - regardless of whether there is any translation available.
1080 - only for the current user
1081 """
1082 _log.info('forcing database language for current db user to [%s]', language)
1083
1084 run_rw_queries(queries = [{
1085 'cmd': 'select i18n.force_curr_lang(%(lang)s)',
1086 'args': {'lang': language}
1087 }])
1088
1089
1090
1091
1093 cmd = 'notify "db_maintenance_warning"'
1094 run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
1095
1096
1098 cmd = 'notify "db_maintenance_disconnect"'
1099 run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
1100
1101
1103 cmd = 'SELECT %(candidate)s::interval'
1104 try:
1105 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}])
1106 return True
1107 except:
1108 cmd = 'SELECT %(candidate)s::text::interval'
1109 try:
1110 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}])
1111 return True
1112 except:
1113 return False
1114
1115
1116 -def lock_row(link_obj=None, table=None, pk=None, exclusive=False):
1117 """Uses pg_advisory(_shared).
1118
1119 - locks stack upon each other and need one unlock per lock
1120 - same connection:
1121 - all locks succeed
1122 - different connections:
1123 - shared + shared succeed
1124 - shared + exclusive fail
1125 """
1126 _log.debug('locking row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
1127 if exclusive:
1128 cmd = """SELECT pg_try_advisory_lock('%s'::regclass::oid::int, %s)""" % (table, pk)
1129 else:
1130 cmd = """SELECT pg_try_advisory_lock_shared('%s'::regclass::oid::int, %s)""" % (table, pk)
1131 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}], get_col_idx = False)
1132 if rows[0][0]:
1133 return True
1134 _log.warning('cannot lock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
1135 return False
1136
1137
1138 -def unlock_row(link_obj=None, table=None, pk=None, exclusive=False):
1139 """Uses pg_advisory_unlock(_shared).
1140
1141 - each lock needs one unlock
1142 """
1143 _log.debug('trying to unlock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
1144 if exclusive:
1145 cmd = "SELECT pg_advisory_unlock('%s'::regclass::oid::int, %s)" % (table, pk)
1146 else:
1147 cmd = "SELECT pg_advisory_unlock_shared('%s'::regclass::oid::int, %s)" % (table, pk)
1148 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}], get_col_idx = False)
1149 if rows[0][0]:
1150 return True
1151 _log.warning('cannot unlock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive)
1152 return False
1153
1154
1156 """Looks at pk_locks
1157
1158 - does not take into account locks other than 'advisory', however
1159 """
1160 cmd = """SELECT EXISTS (
1161 SELECT 1 FROM pg_locks WHERE
1162 classid = '%s'::regclass::oid::int
1163 AND
1164 objid = %s
1165 AND
1166 locktype = 'advisory'
1167 )""" % (table, pk)
1168 rows, idx = run_ro_queries(queries = [{'cmd': cmd}], get_col_idx = False)
1169 if rows[0][0]:
1170 _log.debug('row is locked: [%s] [%s]', table, pk)
1171 return True
1172 _log.debug('row is NOT locked: [%s] [%s]', table, pk)
1173 return False
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1208 md5 = hashlib.md5()
1209 md5.update(('%s' % cache_key_data).encode('utf8'))
1210 return os.path.join(gmTools.gmPaths().bytea_cache_dir, md5.hexdigest())
1211
1212
1214 cached_name = __generate_cached_filename(cache_key_data)
1215 _log.debug('[%s] -> [%s] -> [%s]', filename, cache_key_data, cached_name)
1216 if not gmTools.remove_file(cached_name, log_error = True, force = True):
1217 _log.error('cannot remove existing file [%s] for key [%s] from cache', filename, cached_name)
1218 return None
1219
1220 PERMS_owner_only = 0o0660
1221 try:
1222 shutil.copyfile(filename, cached_name, follow_symlinks = True)
1223 os.chmod(cached_name, PERMS_owner_only)
1224 except shutil.SameFileError:
1225 _log.exception('file seems to exist in cache, despite having checked and possible removed it just before')
1226
1227
1228
1229
1230 return None
1231 except OSError:
1232 _log.exception('cannot copy file into cache: [%s] -> [%s]', filename, cached_name)
1233 return None
1234 except PermissionError:
1235 _log.exception('cannot set cache file [%s] permissions to [%s]', cached_name, stat.filemode(PERMS_owner_only))
1236 return None
1237
1238 return cached_name
1239
1240
1242 """Calculate and verify filename in cache given cache key details."""
1243 cached_name = __generate_cached_filename(cache_key_data)
1244 try:
1245 stat = os.stat(cached_name)
1246 except FileNotFoundError:
1247 return None
1248 _log.debug('cache hit: [%s] -> [%s] (%s)', cache_key_data, cached_name, stat)
1249 if os.path.islink(cached_name) or (not os.path.isfile(cached_name)):
1250 _log.error('object in cache is not a regular file: %s', cached_name)
1251 _log.error('possibly an attack, removing')
1252 if gmTools.remove_file(cached_name, log_error = True):
1253 return None
1254 raise Exception('cannot delete suspicious object in cache dir: %s', cached_name)
1255 if stat.st_size == data_size:
1256 return cached_name
1257 _log.debug('size in cache [%s] <> expected size [%s], removing cached file', stat.st_size, data_size)
1258 if gmTools.remove_file(cached_name, log_error = True):
1259 return None
1260 raise Exception('cannot remove suspicous object from cache dir: %s', cached_name)
1261
1262
1264 """Get file from cache if available."""
1265 cached_filename = __get_filename_in_cache(cache_key_data = cache_key_data, data_size = data_size)
1266 if cached_filename is None:
1267 return False
1268 if link2cached:
1269 try:
1270
1271
1272 os.link(cached_filename, filename)
1273 _log.debug('hardlinked [%s] as [%s]', cached_filename, filename)
1274 return True
1275 except Exception:
1276 pass
1277 _log.debug('cannot hardlink to cache, trying copy-from-cache')
1278
1279 try:
1280 shutil.copyfile(cached_filename, filename, follow_symlinks = True)
1281 return True
1282 except shutil.SameFileError:
1283
1284 pass
1285 except OSError:
1286 _log.exception('cannot copy cached file [%s] into [%s]', cached_filename, filename)
1287
1288 _log.debug('downloading new copy of file, despite found in cache')
1289 return False
1290
1291
1292 -def bytea2file(data_query=None, filename=None, chunk_size=0, data_size=None, data_size_query=None, conn=None, link2cached=True):
1293
1294 if data_size is None:
1295 rows, idx = run_ro_queries(link_obj = conn, queries = [data_size_query])
1296 data_size = rows[0][0]
1297 if data_size in [None, 0]:
1298 conn.rollback()
1299
1300 return True
1301
1302
1303 cache_key_data = '<%s>@%s:%s/%s::%s' % (
1304 _default_login.user,
1305 _default_login.host,
1306 _default_login.port,
1307 _default_login.database,
1308 data_query
1309 )
1310 found_in_cache = __get_file_from_cache(filename, cache_key_data = cache_key_data, data_size = data_size, link2cached = link2cached)
1311 if found_in_cache:
1312 return True
1313
1314 outfile = io.open(filename, 'wb')
1315 result = bytea2file_object (
1316 data_query = data_query,
1317 file_obj = outfile,
1318 chunk_size = chunk_size,
1319 data_size = data_size,
1320 data_size_query = data_size_query,
1321 conn = conn
1322 )
1323 outfile.close()
1324 __store_file_in_cache(filename, cache_key_data)
1325 return result
1326
1327
1328 -def bytea2file_object(data_query=None, file_obj=None, chunk_size=0, data_size=None, data_size_query=None, conn=None):
1329 """Store data from a bytea field into a file.
1330
1331 <data_query>
1332 - dict {'cmd': ..., 'args': ...}
1333 - 'cmd' must be a string containing "... substring(data from %(start)s for %(size)s) ..."
1334 - 'args' must be a dict
1335 - must return one row with one field of type bytea
1336 <file>
1337 - must be a file like Python object
1338 <data_size>
1339 - integer of the total size of the expected data or None
1340 <data_size_query>
1341 - dict {'cmd': ..., 'args': ...}
1342 - must return one row with one field with the octet_length() of the data field
1343 - used only when <data_size> is None
1344 """
1345 if data_size == 0:
1346 return True
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359 if conn is None:
1360 conn = get_raw_connection(readonly = True)
1361
1362 if data_size is None:
1363 rows, idx = run_ro_queries(link_obj = conn, queries = [data_size_query])
1364 data_size = rows[0][0]
1365 if data_size in [None, 0]:
1366 conn.rollback()
1367 return True
1368
1369 max_chunk_size = 1024 * 1024 * 20
1370 if chunk_size == 0:
1371 chunk_size = min(data_size, max_chunk_size)
1372
1373 _log.debug('expecting %s bytes of BYTEA data in chunks of %s bytes', data_size, chunk_size)
1374
1375
1376
1377
1378 needed_chunks, remainder = divmod(data_size, chunk_size)
1379 _log.debug('# of chunks: %s; remainder: %s bytes', needed_chunks, remainder)
1380
1381
1382
1383
1384
1385
1386
1387 for chunk_id in range(needed_chunks):
1388 chunk_start = (chunk_id * chunk_size) + 1
1389 data_query['args']['start'] = chunk_start
1390 data_query['args']['size'] = chunk_size
1391 try:
1392 rows, idx = run_ro_queries(link_obj=conn, queries=[data_query])
1393 except:
1394 _log.error('cannot retrieve chunk [%s/%s], size [%s], try decreasing chunk size' % (chunk_id+1, needed_chunks, chunk_size))
1395 conn.rollback()
1396 raise
1397
1398 file_obj.write(rows[0][0])
1399
1400
1401 if remainder > 0:
1402 chunk_start = (needed_chunks * chunk_size) + 1
1403 data_query['args']['start'] = chunk_start
1404 data_query['args']['size'] = remainder
1405 try:
1406 rows, idx = run_ro_queries(link_obj=conn, queries=[data_query])
1407 except:
1408 _log.error('cannot retrieve remaining [%s] bytes' % remainder)
1409 conn.rollback()
1410 raise
1411
1412 file_obj.write(rows[0][0])
1413
1414 conn.rollback()
1415 return True
1416
1417
1418 -def file2bytea(query=None, filename=None, args=None, conn=None, file_md5=None):
1419 """Store data from a file into a bytea field.
1420
1421 The query must:
1422 - be in unicode
1423 - contain a format spec identifying the row (eg a primary key)
1424 matching <args> if it is an UPDATE
1425 - contain a format spec " <field> = %(data)s::bytea"
1426
1427 The query CAN return the MD5 of the inserted data:
1428 RETURNING md5(<field>) AS md5
1429 in which case it will compare it to the md5
1430 of the file.
1431 """
1432
1433 infile = open(filename, "rb")
1434 data_as_byte_string = infile.read()
1435 infile.close()
1436 if args is None:
1437 args = {}
1438
1439 args['data'] = memoryview(data_as_byte_string)
1440 del(data_as_byte_string)
1441
1442
1443 if conn is None:
1444 conn = get_raw_connection(readonly = False)
1445 close_conn = True
1446 else:
1447 close_conn = False
1448
1449 rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = (file_md5 is not None))
1450
1451 success_status = True
1452 if file_md5 is None:
1453 conn.commit()
1454 else:
1455 db_md5 = rows[0]['md5']
1456 if file_md5 != db_md5:
1457 conn.rollback()
1458 success_status = False
1459 _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5)
1460 else:
1461 conn.commit()
1462 _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5)
1463
1464 if close_conn:
1465 conn.close()
1466
1467 return success_status
1468
1469
1470 -def file2lo(filename=None, conn=None, check_md5=False):
1471
1472 file_size = os.path.getsize(filename)
1473 if file_size > (1024 * 1024) * 1024:
1474 _log.debug('file size of [%s] > 1 GB, supposedly not supported by psycopg2 large objects (but seems to work anyway ?)', file_size)
1475
1476
1477 if conn is None:
1478 conn = get_raw_connection(readonly = False)
1479 close_conn = conn.close
1480 else:
1481 close_conn = __noop
1482 _log.debug('[%s] -> large object', filename)
1483
1484
1485 lo = conn.lobject(0, 'w', 0, filename)
1486 lo_oid = lo.oid
1487 lo.close()
1488 _log.debug('large object OID: %s', lo_oid)
1489
1490
1491 if file_md5 is None:
1492 conn.commit()
1493 close_conn()
1494 return lo_oid
1495 cmd = 'SELECT md5(lo_get(%(loid)s::oid))'
1496 args = {'loid': lo_oid}
1497 rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': cmd, 'args': args}])
1498 db_md5 = rows[0][0]
1499 if file_md5 == db_md5:
1500 conn.commit()
1501 close_conn()
1502 _log.debug('MD5 sums of data file and database large object match: [file::%s] = [DB::%s]', file_md5, db_md5)
1503 return lo_oid
1504 conn.rollback()
1505 close_conn()
1506 _log.error('MD5 sums of data file and database large object [%s] do not match: [file::%s] <> [DB::%s]', lo_oid, file_md5, db_md5)
1507 return -1
1508
1509
1511
1512 file_size = os.path.getsize(filename)
1513 if file_size > (1024 * 1024) * 1024:
1514 _log.debug('file size of [%s] > 1 GB, supposedly not supported by psycopg2 large objects (but seems to work anyway ?)', file_size)
1515
1516
1517 if conn is None:
1518 conn = get_raw_connection(readonly = False)
1519 close_conn = conn.close
1520 else:
1521 close_conn = __noop
1522 _log.debug('[%s] -> large object', filename)
1523
1524
1525 lo = conn.lobject(0, 'w', 0, filename)
1526 lo_oid = lo.oid
1527 lo.close()
1528 _log.debug('large object OID: %s', lo_oid)
1529
1530
1531 if file_md5 is None:
1532 conn.commit()
1533 close_conn()
1534 return lo_oid
1535 cmd = 'SELECT md5(lo_get(%(loid)s::oid))'
1536 args = {'loid': lo_oid}
1537 rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': cmd, 'args': args}])
1538 db_md5 = rows[0][0]
1539 if file_md5 == db_md5:
1540 conn.commit()
1541 close_conn()
1542 _log.debug('MD5 sums of data file and database large object match: [file::%s] = [DB::%s]', file_md5, db_md5)
1543 return lo_oid
1544 conn.rollback()
1545 close_conn()
1546 _log.error('MD5 sums of data file and database large object [%s] do not match: [file::%s] <> [DB::%s]', lo_oid, file_md5, db_md5)
1547 return -1
1548
1549
1550 -def file2bytea_copy_from(table=None, columns=None, filename=None, conn=None, md5_query=None, file_md5=None):
1551
1552
1553
1554
1555 chunk_size = 32 * (1024 * 1024)
1556 _log.debug('[%s] (%s bytes) --(%s bytes)-> %s(%s)', filename, os.path.getsize(filename), chunk_size, table, columns)
1557 if conn is None:
1558 conn = get_raw_connection(readonly = False)
1559 close_conn = True
1560 else:
1561 close_conn = False
1562 curs = conn.cursor()
1563
1564 infile = open(filename, "rb")
1565 curs.copy_from(infile, table, size = chunk_size, columns = columns)
1566 infile.close()
1567 curs.close()
1568 if None in [file_md5, md5_query]:
1569 conn.commit()
1570 close_conn()
1571 return True
1572
1573 rows, idx = run_ro_queries(link_obj = conn, queries = [md5_query])
1574 db_md5 = rows[0][0]
1575 if file_md5 == db_md5:
1576 conn.commit()
1577 close_conn()
1578 _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5)
1579 return True
1580 close_conn()
1581 _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5)
1582 return False
1583
1584
1585 -def file2bytea_overlay(query=None, args=None, filename=None, conn=None, md5_query=None, file_md5=None):
1586 """Store data from a file into a bytea field.
1587
1588 The query must:
1589 - 'cmd' must be in unicode
1590 - 'cmd' must contain a format spec identifying the row (eg
1591 a primary key) matching <args> if it is an UPDATE
1592 - 'cmd' must contain "... SET ... <some_bytea_field> = OVERLAY(some_bytea_field PLACING %(data)s::bytea FROM %(start)s FOR %(size)s) ..."
1593 - 'args' must be a dict matching 'cmd'
1594
1595 The query CAN return the MD5 of the inserted data:
1596 RETURNING md5(<field>) AS md5
1597 in which case it will compare it to the md5
1598 of the file.
1599
1600 UPDATE
1601 the_table
1602 SET
1603 bytea_field = OVERLAY (
1604 coalesce(bytea_field, '':bytea),
1605 PLACING
1606 %(data)s::bytea
1607 FROM
1608 %(start)s
1609 FOR
1610 %(size)s
1611 )
1612 WHERE
1613 primary_key = pk_value
1614
1615 SELECT md5(bytea_field) FROM the_table WHERE primary_key = pk_value
1616 """
1617 chunk_size = 32 * (1024 * 1024)
1618 file_size = os.path.getsize(filename)
1619 if file_size <= chunk_size:
1620 chunk_size = file_size
1621 needed_chunks, remainder = divmod(file_size, chunk_size)
1622 _log.debug('file data: %s bytes, chunks: %s, chunk size: %s bytes, remainder: %s bytes', file_size, needed_chunks, chunk_size, remainder)
1623
1624 if conn is None:
1625 conn = get_raw_connection(readonly = False)
1626 close_conn = conn.close
1627 else:
1628 close_conn = __noop
1629
1630 infile = open(filename, "rb")
1631
1632 for chunk_id in range(needed_chunks):
1633 chunk_start = (chunk_id * chunk_size) + 1
1634 args['start'] = chunk_start
1635 args['size'] = chunk_size
1636 data_as_byte_string = infile.read(chunk_size)
1637
1638 args['data'] = memoryview(data_as_byte_string)
1639 del(data_as_byte_string)
1640 try:
1641 rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = False)
1642 except Exception:
1643 _log.exception('cannot write chunk [%s/%s] of size [%s], try decreasing chunk size', chunk_id+1, needed_chunks, chunk_size)
1644 conn.rollback()
1645 close_conn()
1646 infile.close()
1647 raise
1648
1649 if remainder > 0:
1650 chunk_start = (needed_chunks * chunk_size) + 1
1651 args['start'] = chunk_start
1652 args['size'] = remainder
1653 data_as_byte_string = infile.read(remainder)
1654
1655 args['data'] = memoryview(data_as_byte_string)
1656 del(data_as_byte_string)
1657 try:
1658 rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = False)
1659 except Exception:
1660 _log.error('cannot retrieve remaining [%s] bytes' % remainder)
1661 conn.rollback()
1662 close_conn()
1663 infile.close()
1664 raise
1665 infile.close()
1666 if None in [file_md5, md5_query]:
1667 conn.commit()
1668 close_conn()
1669 return True
1670
1671 rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': md5_query, 'args': args}])
1672 db_md5 = rows[0][0]
1673 if file_md5 == db_md5:
1674 conn.commit()
1675 close_conn()
1676 _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5)
1677 return True
1678 close_conn()
1679 _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5)
1680 return False
1681
1682
1684
1685 if conn is None:
1686 conn = get_connection(readonly = False)
1687
1688 from Gnumed.pycommon import gmPsql
1689 psql = gmPsql.Psql(conn)
1690
1691 if psql.run(sql_script) == 0:
1692 query = {
1693 'cmd': 'select gm.log_script_insertion(%(name)s, %(ver)s)',
1694 'args': {'name': sql_script, 'ver': 'current'}
1695 }
1696 run_rw_queries(link_obj = conn, queries = [query])
1697 conn.commit()
1698 return True
1699
1700 _log.error('error running sql script: %s', sql_script)
1701 return False
1702
1703
1705 """Escape input for use in a PostgreSQL regular expression.
1706
1707 If a fragment comes from user input and is to be used
1708 as a regular expression we need to make sure it doesn't
1709 contain invalid regex patterns such as unbalanced ('s.
1710
1711 <escape_all>
1712 True: try to escape *all* metacharacters
1713 False: only escape those which render the regex invalid
1714 """
1715 return expression.replace (
1716 '(', '\('
1717 ).replace (
1718 ')', '\)'
1719 ).replace (
1720 '[', '\['
1721 ).replace (
1722 '+', '\+'
1723 ).replace (
1724 '.', '\.'
1725 ).replace (
1726 '*', '\*'
1727 )
1728
1729
1730
1732
1733 tx_status = conn.get_transaction_status()
1734 if tx_status in [ psycopg2.extensions.TRANSACTION_STATUS_INERROR, psycopg2.extensions.TRANSACTION_STATUS_UNKNOWN ]:
1735 isolation_level = '%s (tx aborted or unknown, cannot retrieve)' % conn.isolation_level
1736 else:
1737 isolation_level = '%s (%s)' % (conn.isolation_level, map_psyco_iso_level2str[conn.isolation_level])
1738 conn_status = '%s (%s)' % (conn.status, map_psyco_conn_status2str[conn.status])
1739 if conn.closed != 0:
1740 conn_status = 'undefined (%s)' % conn_status
1741 backend_pid = '<conn closed, cannot retrieve>'
1742 else:
1743 backend_pid = conn.get_backend_pid()
1744 try:
1745 conn_deferrable = conn.deferrable
1746 except AttributeError:
1747 conn_deferrable = '<unavailable>'
1748
1749 d = {
1750 'identity': id(conn),
1751 'backend PID': backend_pid,
1752 'protocol version': conn.protocol_version,
1753 'encoding': conn.encoding,
1754 'closed': conn.closed,
1755 'readonly': conn.readonly,
1756 'autocommit': conn.autocommit,
1757 'isolation level (psyco)': isolation_level,
1758 'async': conn.async_,
1759 'deferrable': conn_deferrable,
1760 'transaction status': '%s (%s)' % (tx_status, map_psyco_tx_status2str[tx_status]),
1761 'connection status': conn_status,
1762 'executing async op': conn.isexecuting(),
1763 'type': type(conn)
1764 }
1765 return '%s\n' % conn + format_dict_like (
1766 d,
1767 relevant_keys = [
1768 'type',
1769 'identity',
1770 'backend PID',
1771 'protocol version',
1772 'encoding',
1773 'isolation level (psyco)',
1774 'readonly',
1775 'autocommit',
1776 'closed',
1777 'connection status',
1778 'transaction status',
1779 'deferrable',
1780 'async',
1781 'executing async op'
1782 ],
1783 tabular = True,
1784 value_delimiters = None
1785 )
1786
1787
1789 conn = cursor.connection
1790
1791 tx_status = conn.get_transaction_status()
1792 if tx_status in [ psycopg2.extensions.TRANSACTION_STATUS_INERROR, psycopg2.extensions.TRANSACTION_STATUS_UNKNOWN ]:
1793 isolation_level = '<tx aborted or unknown, cannot retrieve>'
1794 else:
1795 isolation_level = conn.isolation_level
1796 try:
1797 conn_deferrable = conn.deferrable
1798 except AttributeError:
1799 conn_deferrable = '<unavailable>'
1800
1801 if cursor.query is None:
1802 query = '<no query>'
1803 else:
1804 query = cursor.query.decode(errors = 'replace')
1805
1806 if conn.closed != 0:
1807 backend_pid = '<conn closed, cannot retrieve>'
1808 else:
1809 backend_pid = conn.get_backend_pid()
1810
1811 txt = """Link state:
1812 Cursor
1813 identity: %s; name: %s
1814 closed: %s; scrollable: %s; with hold: %s; arraysize: %s; itersize: %s;
1815 last rowcount: %s; rownumber: %s; lastrowid (OID): %s;
1816 last description: %s
1817 statusmessage: %s
1818 Connection
1819 identity: %s; backend pid: %s; protocol version: %s;
1820 closed: %s; autocommit: %s; isolation level: %s; encoding: %s; async: %s; deferrable: %s; readonly: %s;
1821 TX status: %s; CX status: %s; executing async op: %s;
1822 Query
1823 %s
1824 """ % (
1825 id(cursor),
1826 cursor.name,
1827 cursor.closed,
1828 cursor.scrollable,
1829 cursor.withhold,
1830 cursor.arraysize,
1831 cursor.itersize,
1832 cursor.rowcount,
1833 cursor.rownumber,
1834 cursor.lastrowid,
1835 cursor.description,
1836 cursor.statusmessage,
1837
1838 id(conn),
1839 backend_pid,
1840 conn.protocol_version,
1841 conn.closed,
1842 conn.autocommit,
1843 isolation_level,
1844 conn.encoding,
1845 conn.async_,
1846 conn_deferrable,
1847 conn.readonly,
1848 map_psyco_tx_status2str[tx_status],
1849 map_psyco_conn_status2str[conn.status],
1850 conn.isexecuting(),
1851
1852 query
1853 )
1854 return txt
1855
1856
1857 -def run_ro_queries(link_obj=None, queries=None, verbose=False, return_data=True, get_col_idx=False):
1858 """Run read-only queries.
1859
1860 <queries> must be a list of dicts:
1861 [
1862 {'cmd': <string>, 'args': <dict> or <tuple>},
1863 {...},
1864 ...
1865 ]
1866 """
1867 if isinstance(link_obj, dbapi._psycopg.cursor):
1868 curs = link_obj
1869 curs_close = lambda :1
1870 tx_rollback = lambda :1
1871 readonly_rollback_just_in_case = lambda :1
1872 elif isinstance(link_obj, dbapi._psycopg.connection):
1873 curs = link_obj.cursor()
1874 curs_close = curs.close
1875 tx_rollback = link_obj.rollback
1876 if link_obj.autocommit is True:
1877 readonly_rollback_just_in_case = link_obj.rollback
1878 else:
1879
1880
1881
1882 readonly_rollback_just_in_case = lambda :1
1883 elif link_obj is None:
1884 conn = get_connection(readonly = True, verbose = verbose)
1885 curs = conn.cursor()
1886 curs_close = curs.close
1887 tx_rollback = conn.rollback
1888 readonly_rollback_just_in_case = conn.rollback
1889 else:
1890 raise ValueError('link_obj must be cursor, connection or None but not [%s]' % link_obj)
1891
1892 if verbose:
1893 _log.debug('cursor: %s', curs)
1894
1895 for query in queries:
1896 try:
1897 args = query['args']
1898 except KeyError:
1899 args = None
1900 try:
1901 curs.execute(query['cmd'], args)
1902 if verbose:
1903 _log.debug(capture_cursor_state(curs))
1904 except PG_ERROR_EXCEPTION as pg_exc:
1905 _log.error('query failed in RO connection')
1906 log_pg_exception_details(pg_exc)
1907 try:
1908 curs_close()
1909 except PG_ERROR_EXCEPTION as pg_exc2:
1910 _log.exception('cannot close cursor')
1911 log_pg_exception_details(pg_exc2)
1912 try:
1913 tx_rollback()
1914 except PG_ERROR_EXCEPTION as pg_exc2:
1915 _log.exception('cannot rollback transaction')
1916 log_pg_exception_details(pg_exc2)
1917 if pg_exc.pgcode == sql_error_codes.INSUFFICIENT_PRIVILEGE:
1918 details = 'Query: [%s]' % curs.query.strip().strip('\n').strip().strip('\n')
1919 if curs.statusmessage != '':
1920 details = 'Status: %s\n%s' % (
1921 curs.statusmessage.strip().strip('\n').strip().strip('\n'),
1922 details
1923 )
1924 if pg_exc.pgerror is None:
1925 msg = '[%s]' % pg_exc.pgcode
1926 else:
1927 msg = '[%s]: %s' % (pg_exc.pgcode, pg_exc.pgerror)
1928 raise gmExceptions.AccessDenied (
1929 msg,
1930 source = 'PostgreSQL',
1931 code = pg_exc.pgcode,
1932 details = details
1933 )
1934 raise
1935 except Exception:
1936 _log.exception('error during query run in RO connection')
1937 _log.error(capture_cursor_state(curs))
1938 try:
1939 curs_close()
1940 except PG_ERROR_EXCEPTION as pg_exc:
1941 _log.exception('cannot close cursor')
1942 log_pg_exception_details(pg_exc)
1943 try:
1944 tx_rollback()
1945 except PG_ERROR_EXCEPTION as pg_exc:
1946 _log.exception('cannot rollback transation')
1947 log_pg_exception_details(pg_exc)
1948 raise
1949
1950 data = None
1951 col_idx = None
1952 if return_data:
1953 data = curs.fetchall()
1954 if verbose:
1955 _log.debug('last query returned [%s (%s)] rows', curs.rowcount, len(data))
1956 _log.debug('cursor description: %s', curs.description)
1957 if get_col_idx:
1958 col_idx = get_col_indices(curs)
1959
1960 curs_close()
1961
1962
1963
1964 readonly_rollback_just_in_case()
1965 return (data, col_idx)
1966
1967
1968 -def run_rw_queries(link_obj=None, queries=None, end_tx=False, return_data=None, get_col_idx=False, verbose=False):
1969 """Convenience function for running a transaction
1970 that is supposed to get committed.
1971
1972 <link_obj>
1973 can be either:
1974 - a cursor
1975 - a connection
1976
1977 <queries>
1978 is a list of dicts [{'cmd': <string>, 'args': <dict> or <tuple>)
1979 to be executed as a single transaction, the last
1980 query may usefully return rows (such as a
1981 "SELECT currval('some_sequence')" statement)
1982
1983 <end_tx>
1984 - controls whether the transaction is finalized (eg.
1985 committed/rolled back) or not, this allows the
1986 call to run_rw_queries() to be part of a framing
1987 transaction
1988 - if link_obj is a connection then <end_tx> will
1989 default to False unless it is explicitly set to
1990 True which is taken to mean "yes, you do have full
1991 control over the transaction" in which case the
1992 transaction is properly finalized
1993 - if link_obj is a cursor we CANNOT finalize the
1994 transaction because we would need the connection for that
1995 - if link_obj is None <end_tx> will, of course, always be True
1996
1997 <return_data>
1998 - if true, the returned data will include the rows
1999 the last query selected
2000 - if false, it returns None instead
2001
2002 <get_col_idx>
2003 - if true, the returned data will include a dictionary
2004 mapping field names to column positions
2005 - if false, the returned data returns None instead
2006
2007 method result:
2008 - returns a tuple (data, idx)
2009 - <data>:
2010 * (None, None) if last query did not return rows
2011 * ("fetchall() result", <index>) if last query returned any rows
2012 * for <index> see <get_col_idx>
2013 """
2014 if isinstance(link_obj, dbapi._psycopg.cursor):
2015 conn_close = lambda :1
2016 conn_commit = lambda :1
2017 tx_rollback = lambda :1
2018 curs = link_obj
2019 curs_close = lambda :1
2020 notices_accessor = curs.connection
2021 elif isinstance(link_obj, dbapi._psycopg.connection):
2022 conn_close = lambda :1
2023 if end_tx:
2024 conn_commit = link_obj.commit
2025 tx_rollback = link_obj.rollback
2026 else:
2027 conn_commit = lambda :1
2028 tx_rollback = lambda :1
2029 curs = link_obj.cursor()
2030 curs_close = curs.close
2031 notices_accessor = link_obj
2032 elif link_obj is None:
2033 conn = get_connection(readonly=False)
2034 conn_close = conn.close
2035 conn_commit = conn.commit
2036 tx_rollback = conn.rollback
2037 curs = conn.cursor()
2038 curs_close = curs.close
2039 notices_accessor = conn
2040 else:
2041 raise ValueError('link_obj must be cursor, connection or None but not [%s]' % link_obj)
2042
2043 for query in queries:
2044 try:
2045 args = query['args']
2046 except KeyError:
2047 args = None
2048 try:
2049 curs.execute(query['cmd'], args)
2050 if verbose:
2051 _log.debug(capture_cursor_state(curs))
2052 for notice in notices_accessor.notices:
2053 _log.debug(notice.replace('\n', '/').replace('\n', '/'))
2054 del notices_accessor.notices[:]
2055
2056 except dbapi.Error as pg_exc:
2057 _log.error('query failed in RW connection')
2058 log_pg_exception_details(pg_exc)
2059 for notice in notices_accessor.notices:
2060 _log.debug(notice.replace('\n', '/').replace('\n', '/'))
2061 del notices_accessor.notices[:]
2062 try:
2063 curs_close()
2064 except PG_ERROR_EXCEPTION as pg_exc2:
2065 _log.exception('cannot close cursor')
2066 log_pg_exception_details(pg_exc2)
2067 try:
2068 tx_rollback()
2069 except PG_ERROR_EXCEPTION as pg_exc2:
2070 _log.exception('cannot rollback transaction')
2071 log_pg_exception_details(pg_exc2)
2072
2073 if pg_exc.pgcode == sql_error_codes.INSUFFICIENT_PRIVILEGE:
2074 details = 'Query: [%s]' % curs.query.strip().strip('\n').strip().strip('\n')
2075 if curs.statusmessage != '':
2076 details = 'Status: %s\n%s' % (
2077 curs.statusmessage.strip().strip('\n').strip().strip('\n'),
2078 details
2079 )
2080 if pg_exc.pgerror is None:
2081 msg = '[%s]' % pg_exc.pgcode
2082 else:
2083 msg = '[%s]: %s' % (pg_exc.pgcode, pg_exc.pgerror)
2084 try:
2085 curs_close()
2086 tx_rollback()
2087 conn_close()
2088 except dbapi.InterfaceError:
2089 _log.exception('cannot cleanup')
2090 raise gmExceptions.AccessDenied (
2091 msg,
2092 source = 'PostgreSQL',
2093 code = pg_exc.pgcode,
2094 details = details
2095 )
2096
2097 gmLog2.log_stack_trace()
2098 try:
2099 curs_close()
2100 tx_rollback()
2101 conn_close()
2102 except dbapi.InterfaceError:
2103 _log.exception('cannot cleanup')
2104 raise
2105
2106 except Exception:
2107 _log.exception('error running query in RW connection')
2108 _log.error(capture_cursor_state(curs))
2109 for notice in notices_accessor.notices:
2110 _log.debug(notice.replace('\n', '/').replace('\n', '/'))
2111 del notices_accessor.notices[:]
2112 gmLog2.log_stack_trace()
2113 try:
2114 curs_close()
2115 except PG_ERROR_EXCEPTION as pg_exc:
2116 _log.exception('cannot close cursor')
2117 log_pg_exception_details(pg_exc)
2118 try:
2119 tx_rollback()
2120 conn_close()
2121 except PG_ERROR_EXCEPTION as pg_exc:
2122 _log.exception('cannot rollback transation')
2123 log_pg_exception_details(pg_exc)
2124 raise
2125
2126 data = None
2127 col_idx = None
2128 if return_data:
2129 try:
2130 data = curs.fetchall()
2131 except Exception:
2132 _log.exception('error fetching data from RW query')
2133 gmLog2.log_stack_trace()
2134 try:
2135 curs_close()
2136 tx_rollback()
2137 conn_close()
2138 except dbapi.InterfaceError:
2139 _log.exception('cannot cleanup')
2140 raise
2141 raise
2142 if get_col_idx:
2143 col_idx = get_col_indices(curs)
2144
2145 curs_close()
2146 conn_commit()
2147 conn_close()
2148
2149 return (data, col_idx)
2150
2151
2152 -def run_insert(link_obj=None, schema=None, table=None, values=None, returning=None, end_tx=False, get_col_idx=False, verbose=False):
2153 """Generates SQL for an INSERT query.
2154
2155 values: dict of values keyed by field to insert them into
2156 """
2157 if schema is None:
2158 schema = 'public'
2159
2160 fields = values.keys()
2161 val_snippets = []
2162 for field in fields:
2163 val_snippets.append('%%(%s)s' % field)
2164
2165 if returning is None:
2166 returning = ''
2167 return_data = False
2168 else:
2169 returning = '\n\tRETURNING\n\t\t%s' % ', '.join(returning)
2170 return_data = True
2171
2172 cmd = """\nINSERT INTO %s.%s (
2173 %s
2174 ) VALUES (
2175 %s
2176 )%s""" % (
2177 schema,
2178 table,
2179 ',\n\t\t'.join(fields),
2180 ',\n\t\t'.join(val_snippets),
2181 returning
2182 )
2183
2184 _log.debug('running SQL: >>>%s<<<', cmd)
2185
2186 return run_rw_queries (
2187 link_obj = link_obj,
2188 queries = [{'cmd': cmd, 'args': values}],
2189 end_tx = end_tx,
2190 return_data = return_data,
2191 get_col_idx = get_col_idx,
2192 verbose = verbose
2193 )
2194
2195
2196
2197
2199 """GNUmed database connection pool.
2200
2201 Extends psycopg2's ThreadedConnectionPool with
2202 a custom _connect() function. Supports one connection
2203 per thread - which also ties it to one particular DSN."""
2204
2206 _log.debug('conn request with key [%s]', key)
2207 conn = get_raw_connection(dsn = self._kwargs['dsn'], verbose = self._kwargs['verbose'], readonly = True)
2208
2209 conn.original_close = conn.close
2210 conn.close = _raise_exception_on_ro_conn_close
2211 if key is not None:
2212 self._used[key] = conn
2213 self._rused[id(conn)] = key
2214 else:
2215 self._pool.append(conn)
2216 return conn
2217
2218
2220 if key is None:
2221 key = threading.current_thread().ident
2222 try:
2223 conn = self._used[key]
2224 except KeyError:
2225 _log.error('no such key in connection pool: %s', key)
2226 _log.debug('available keys: %s', self._used.keys())
2227 return
2228 del self._used[key]
2229 del self._rused[id(conn)]
2230 conn.original_close()
2231
2232
2234 for conn_key in self._used.keys():
2235 conn = self._used[conn_key]
2236 if conn.closed != 0:
2237 continue
2238 _log.debug('closing pooled database connection, pool key: %s, backend PID: %s', conn_key, self._used[conn_key].get_backend_pid())
2239 conn.original_close()
2240
2241
2242 -def get_raw_connection(dsn=None, verbose=False, readonly=True, connection_name=None, autocommit=False):
2243 """Get a raw, unadorned connection.
2244
2245 - this will not set any parameters such as encoding, timezone, datestyle
2246 - the only requirement is a valid DSN
2247 - hence it can be used for "service" connections
2248 for verifying encodings etc
2249 """
2250
2251 if dsn is None:
2252 dsn = get_default_dsn()
2253
2254 if 'host=salaam.homeunix' in dsn:
2255 raise ValueError('The public database is not hosted by <salaam.homeunix.com> anymore.\n\nPlease point your configuration files to <publicdb.gnumed.de>.')
2256
2257
2258
2259
2260 if ' client_encoding=' not in dsn:
2261 dsn += ' client_encoding=utf8'
2262
2263 if ' application_name' not in dsn:
2264 if connection_name is None:
2265 dsn += " application_name=GNUmed-[%s]" % threading.current_thread().name.replace(' ', '_')
2266 else:
2267 dsn += " application_name=%s" % connection_name
2268
2269 try:
2270
2271 conn = dbapi.connect(dsn = dsn, connection_factory = psycopg2.extras.DictConnection)
2272 except dbapi.OperationalError as e:
2273 t, v, tb = sys.exc_info()
2274 try:
2275 msg = e.args[0]
2276 except (AttributeError, IndexError, TypeError):
2277 raise
2278 if 'fe_sendauth' in msg:
2279 raise cAuthenticationError(dsn, msg).with_traceback(tb)
2280 if regex.search('user ".*" does not exist', msg) is not None:
2281 raise cAuthenticationError(dsn, msg).with_traceback(tb)
2282 if (( (regex.search('user ".*"', msg) is not None)
2283 or
2284 (regex.search('(R|r)ol{1,2}e', msg) is not None)
2285 )
2286 and ('exist' in msg)
2287 and (regex.search('n(o|ich)t', msg) is not None)
2288 ):
2289 raise cAuthenticationError(dsn, msg).with_traceback(tb)
2290 if regex.search('user ".*" does not exist', msg) is not None:
2291 raise cAuthenticationError(dsn, msg).with_traceback(tb)
2292 if 'uthenti' in msg:
2293 raise cAuthenticationError(dsn, msg).with_traceback(tb)
2294 raise
2295
2296 if connection_name is None:
2297 _log.debug('established anonymous database connection, backend PID: %s', conn.get_backend_pid())
2298 else:
2299 _log.debug('established database connection "%s", backend PID: %s', connection_name, conn.get_backend_pid())
2300
2301
2302
2303 global postgresql_version
2304 if postgresql_version is None:
2305 curs = conn.cursor()
2306 curs.execute("""
2307 SELECT
2308 substring(setting, E'^\\\\d{1,2}\\\\.\\\\d{1,2}')::numeric AS version
2309 FROM
2310 pg_settings
2311 WHERE
2312 name = 'server_version'
2313 """)
2314 postgresql_version = curs.fetchone()['version']
2315 _log.info('PostgreSQL version (numeric): %s' % postgresql_version)
2316 try:
2317 curs.execute("SELECT pg_size_pretty(pg_database_size(current_database()))")
2318 _log.info('database size: %s', curs.fetchone()[0])
2319 except Exception:
2320 _log.exception('cannot get database size')
2321 finally:
2322 curs.close()
2323 conn.commit()
2324 if verbose:
2325 curs = conn.cursor()
2326 _log_PG_settings(curs = curs)
2327 curs.close()
2328
2329 if _default_client_timezone is None:
2330 __detect_client_timezone(conn = conn)
2331
2332
2333 if readonly:
2334 _log.debug('readonly: forcing autocommit=True to avoid <IDLE IN TRANSACTION>')
2335 autocommit = True
2336 else:
2337 _log.debug('autocommit is desired to be: %s', autocommit)
2338
2339 conn.commit()
2340 conn.autocommit = autocommit
2341 conn.readonly = readonly
2342
2343
2344 if verbose:
2345 _log.debug('enabling <plpgsql.extra_warnings/_errors>')
2346 curs = conn.cursor()
2347 try:
2348 curs.execute("SET plpgsql.extra_warnings TO 'all'")
2349 curs.execute("SET plpgsql.extra_errors TO 'all'")
2350 except Exception:
2351 _log.exception('cannot enable <plpgsql.extra_warnings/_errors>')
2352 finally:
2353 curs.close()
2354 conn.commit()
2355
2356 conn.is_decorated = False
2357 return conn
2358
2359
2360 -def get_connection(dsn=None, readonly=True, encoding=None, verbose=False, pooled=True, connection_name=None, autocommit=False):
2361 """Get a new connection.
2362
2363 This assumes the locale system has been initialized
2364 unless an encoding is specified.
2365 """
2366
2367
2368 if pooled and readonly and (dsn is None):
2369 global __ro_conn_pool
2370 if __ro_conn_pool is None:
2371 log_ro_conn = True
2372 __ro_conn_pool = cConnectionPool (
2373 minconn = 1,
2374 maxconn = 2,
2375 dsn = dsn,
2376 verbose = verbose
2377 )
2378 else:
2379 log_ro_conn = False
2380 try:
2381 conn = __ro_conn_pool.getconn()
2382 except psycopg2.pool.PoolError:
2383 _log.exception('falling back to non-pooled connection')
2384 conn = get_raw_connection(dsn = dsn, verbose = verbose, readonly = readonly, connection_name = connection_name, autocommit = autocommit)
2385 log_ro_conn = True
2386 if log_ro_conn:
2387 [ _log.debug(line) for line in capture_conn_state(conn = conn).split('\n') ]
2388 else:
2389 conn = get_raw_connection(dsn = dsn, verbose = verbose, readonly = readonly, connection_name = connection_name, autocommit = autocommit)
2390
2391 if conn.is_decorated:
2392 return conn
2393
2394 if encoding is None:
2395 encoding = _default_client_encoding
2396 if encoding is None:
2397 encoding = gmI18N.get_encoding()
2398 _log.warning('client encoding not specified')
2399 _log.warning('the string encoding currently set in the active locale is used: [%s]' % encoding)
2400 _log.warning('for this to work properly the application MUST have called locale.setlocale() before')
2401
2402
2403
2404 try:
2405 conn.set_client_encoding(encoding)
2406 except dbapi.DataError:
2407 t, v, tb = sys.exc_info()
2408 if 'cannot set encoding to' in str(v):
2409 raise cEncodingError(encoding, v).with_traceback(tb)
2410 if 'invalid value for parameter "client_encoding"' in str(v):
2411 raise cEncodingError(encoding, v).with_traceback(tb)
2412 raise
2413
2414
2415 if readonly:
2416
2417 pass
2418 else:
2419 conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE)
2420
2421 _log.debug('client time zone [%s]', _default_client_timezone)
2422
2423
2424 curs = conn.cursor()
2425 curs.execute(_sql_set_timezone, [_default_client_timezone])
2426 curs.close()
2427 conn.commit()
2428
2429 conn.is_decorated = True
2430
2431 if verbose:
2432 [ _log.debug(line) for line in capture_conn_state(conn = conn).split('\n') ]
2433
2434 return conn
2435
2436
2441
2442
2447
2448
2449
2450
2453
2454
2456 raise TypeError('close() called on read-only connection')
2457
2458
2460 run_insert (
2461 schema = 'gm',
2462 table = 'access_log',
2463 values = {'user_action': action},
2464 end_tx = True
2465 )
2466
2467
2469 """Check server time and local time to be within
2470 the given tolerance of each other.
2471
2472 tolerance: seconds
2473 """
2474 _log.debug('maximum skew tolerance (seconds): %s', tolerance)
2475
2476 cmd = "SELECT now() at time zone 'UTC'"
2477 conn = get_raw_connection(readonly=True)
2478 curs = conn.cursor()
2479
2480 start = time.time()
2481 rows, idx = run_ro_queries(link_obj = curs, queries = [{'cmd': cmd}])
2482 end = time.time()
2483 client_now_as_utc = pydt.datetime.utcnow()
2484
2485 curs.close()
2486 conn.commit()
2487
2488 server_now_as_utc = rows[0][0]
2489 query_duration = end - start
2490 _log.info('server "now" (UTC): %s', server_now_as_utc)
2491 _log.info('client "now" (UTC): %s', client_now_as_utc)
2492 _log.debug('wire roundtrip (seconds): %s', query_duration)
2493
2494 if query_duration > tolerance:
2495 _log.error('useless to check client/server time skew, wire roundtrip > tolerance')
2496 return False
2497
2498 if server_now_as_utc > client_now_as_utc:
2499 real_skew = server_now_as_utc - client_now_as_utc
2500 else:
2501 real_skew = client_now_as_utc - server_now_as_utc
2502
2503 _log.debug('client/server time skew: %s', real_skew)
2504
2505 if real_skew > pydt.timedelta(seconds = tolerance):
2506 _log.error('client/server time skew > tolerance')
2507 return False
2508
2509 return True
2510
2511
2513 """Checks database settings.
2514
2515 returns (status, message)
2516 status:
2517 0: no problem
2518 1: non-fatal problem
2519 2: fatal problem
2520 """
2521 _log.debug('checking database settings')
2522
2523 conn = get_connection()
2524
2525
2526 global postgresql_version_string
2527 if postgresql_version_string is None:
2528 curs = conn.cursor()
2529 curs.execute('SELECT version()')
2530 postgresql_version_string = curs.fetchone()['version']
2531 curs.close()
2532 _log.info('PostgreSQL version (string): "%s"' % postgresql_version_string)
2533
2534 options2check = {
2535
2536 'allow_system_table_mods': [['off'], 'system breakage', False],
2537 'check_function_bodies': [['on'], 'suboptimal error detection', False],
2538 'datestyle': [['ISO'], 'faulty timestamp parsing', True],
2539 'default_transaction_isolation': [['read committed'], 'faulty database reads', True],
2540 'default_transaction_read_only': [['on'], 'accidental database writes', False],
2541 'fsync': [['on'], 'data loss/corruption', True],
2542 'full_page_writes': [['on'], 'data loss/corruption', False],
2543 'lc_messages': [['C'], 'suboptimal error detection', False],
2544 'password_encryption': [['on', 'md5', 'scram-sha-256'], 'breach of confidentiality', False],
2545
2546 'synchronous_commit': [['on'], 'data loss/corruption', False],
2547 'sql_inheritance': [['on'], 'query breakage, data loss/corruption', True],
2548 'ignore_checksum_failure': [['off'], 'data loss/corruption', False],
2549 'track_commit_timestamp': [['on'], 'suboptimal auditing', False]
2550 }
2551
2552 from Gnumed.pycommon import gmCfg2
2553 _cfg = gmCfg2.gmCfgData()
2554 if _cfg.get(option = 'hipaa'):
2555 options2check['log_connections'] = [['on'], 'non-compliance with HIPAA', True]
2556 options2check['log_disconnections'] = [['on'], 'non-compliance with HIPAA', True]
2557 else:
2558 options2check['log_connections'] = [['on'], 'non-compliance with HIPAA', None]
2559 options2check['log_disconnections'] = [['on'], 'non-compliance with HIPAA', None]
2560
2561 cmd = "SELECT name, setting from pg_settings where name in %(settings)s"
2562 rows, idx = run_ro_queries (
2563 link_obj = conn,
2564 queries = [{'cmd': cmd, 'args': {'settings': tuple(options2check.keys())}}],
2565 get_col_idx = False
2566 )
2567
2568 found_error = False
2569 found_problem = False
2570 msg = []
2571 for row in rows:
2572 option = row['name']
2573 value_found = row['setting']
2574 values_expected = options2check[option][0]
2575 risk = options2check[option][1]
2576 fatal_setting = options2check[option][2]
2577 if value_found not in values_expected:
2578 if fatal_setting is True:
2579 found_error = True
2580 elif fatal_setting is False:
2581 found_problem = True
2582 elif fatal_setting is None:
2583 pass
2584 else:
2585 _log.error(options2check[option])
2586 raise ValueError('invalid database configuration sanity check')
2587 msg.append(_(' option [%s]: %s') % (option, value_found))
2588 msg.append(_(' risk: %s') % risk)
2589 _log.warning('PG option [%s] set to [%s], expected %s, risk: <%s>' % (option, value_found, values_expected, risk))
2590
2591 if found_error:
2592 return 2, '\n'.join(msg)
2593
2594 if found_problem:
2595 return 1, '\n'.join(msg)
2596
2597 return 0, ''
2598
2599
2601
2602
2603 try:
2604 curs.execute('SELECT * FROM pg_settings')
2605 except dbapi.Error:
2606 _log.exception('cannot retrieve PG settings ("SELECT ... FROM pg_settings" failed)')
2607 return False
2608 settings = curs.fetchall()
2609 for setting in settings:
2610 if setting['unit'] is None:
2611 unit = ''
2612 else:
2613 unit = ' %s' % setting['unit']
2614 if setting['sourcefile'] is None:
2615 sfile = ''
2616 else:
2617 sfile = '// %s @ %s' % (setting['sourcefile'], setting['sourceline'])
2618 pending_restart = u''
2619 try:
2620 if setting['pending_restart']:
2621 pending_restart = u'// needs restart'
2622 except KeyError:
2623
2624 pass
2625 _log.debug('%s: %s%s (set from: [%s] // sess RESET will set to: [%s]%s%s)',
2626 setting['name'],
2627 setting['setting'],
2628 unit,
2629 setting['source'],
2630 setting['reset_val'],
2631 pending_restart,
2632 sfile
2633 )
2634 try:
2635 curs.execute('select pg_available_extensions()')
2636 except:
2637 _log.exception('cannot log available PG extensions')
2638 return False
2639 extensions = curs.fetchall()
2640 if extensions is None:
2641 _log.error('no PG extensions available')
2642 return False
2643 for ext in extensions:
2644 _log.debug('PG extension: %s', ext['pg_available_extensions'])
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657 return True
2658
2659
2661 if not isinstance(exc, dbapi.Error):
2662 return False
2663 try:
2664 args = exc.args
2665 for arg in args:
2666 _log.debug('exc.arg: %s', arg)
2667 except AttributeError:
2668 _log.debug('exception has no <.args>')
2669 _log.debug('pgerror: [%s]', exc.pgerror)
2670 if exc.pgcode is None:
2671 _log.debug('pgcode : %s', exc.pgcode)
2672 else:
2673 _log.debug('pgcode : %s (%s)', exc.pgcode, sql_error_codes.lookup(exc.pgcode))
2674 if exc.cursor is None:
2675 _log.debug('cursor: None')
2676 else:
2677 capture_cursor_state(cursor = exc.cursor)
2678 try:
2679 exc.diag
2680 for attr in dir(exc.diag):
2681 if attr.startswith('__'):
2682 continue
2683 val = getattr(exc.diag, attr)
2684 if val is None:
2685 continue
2686 _log.debug('%s: %s', attr, val)
2687 except AttributeError:
2688 _log.debug('diag: not available')
2689 return True
2690
2691
2693 if not isinstance(exc, dbapi.Error):
2694
2695 return False
2696 try:
2697 msg = '%s' % exc.args[0]
2698 except (AttributeError, IndexError, TypeError):
2699 _log.debug('cannot extract message from exception')
2700
2701 return False
2702 _log.debug('interpreting: %s', msg)
2703
2704 conn_lost = (
2705 ('erver' in msg)
2706 and
2707 (
2708 ('terminat' in msg)
2709 or
2710 ('abnorm' in msg)
2711 or
2712 ('end' in msg)
2713
2714
2715 )
2716 )
2717 if conn_lost:
2718 _log.debug('indicates connection loss')
2719 return True
2720
2721 conn_lost = (
2722 ('onnect' in msg)
2723 and
2724 (
2725 ('close' in msg)
2726 or
2727 ('end' in msg)
2728 )
2729 )
2730 if conn_lost:
2731 _log.debug('indicates connection loss')
2732 return conn_lost
2733
2734
2736
2737 - def __init__(self, dsn=None, prev_val=None):
2738 self.dsn = dsn
2739 self.prev_val = prev_val
2740
2742 return 'PostgreSQL: %sDSN: %s' % (self.prev_val, self.dsn)
2743
2744
2745
2746
2748
2749 - def __init__(self, encoding=None, prev_val=None):
2750 self.encoding = encoding
2751 self.prev_val = prev_val
2752
2754 return 'PostgreSQL: %s\nencoding: %s' % (self.prev_val, self.encoding)
2755
2756
2757
2758
2759
2761
2763 if dt.tzinfo is None:
2764 raise ValueError('datetime.datetime instance is lacking a time zone: [%s]' % _timestamp_template % dt.isoformat())
2765 self.__dt = dt
2766
2769
2770
2771
2772
2773
2774
2775 psycopg2.extensions.register_type(psycopg2.extensions.UNICODE)
2776 psycopg2.extensions.register_type(psycopg2._psycopg.UNICODEARRAY)
2777
2778
2779
2780 psycopg2.extensions.register_adapter(pydt.datetime, cAdapterPyDateTime)
2781
2782
2783
2784
2785
2786
2787
2788
2789 if __name__ == "__main__":
2790
2791 if len(sys.argv) < 2:
2792 sys.exit()
2793
2794 if sys.argv[1] != 'test':
2795 sys.exit()
2796
2797 from Gnumed.pycommon.gmTools import file2md5
2798
2799 logging.basicConfig(level=logging.DEBUG)
2800
2801
2803 run_rw_queries(queries = [
2804 {'cmd': 'drop table if exists test_bytea'},
2805 {'cmd': 'create table test_bytea (data bytea)'}
2806 ])
2807
2808 try:
2809 file2bytea(query = 'insert into test_bytea values (%(data)s::bytea) returning md5(data) as md5', filename = sys.argv[2], file_md5 = file2md5(sys.argv[2], True))
2810 except:
2811 _log.exception('error')
2812
2813 run_rw_queries(queries = [
2814 {'cmd': 'drop table test_bytea'}
2815 ])
2816
2817
2824
2825
2826
2827
2828
2829
2831
2832 run_rw_queries(queries = [
2833 {'cmd': 'drop table if exists test_bytea'},
2834 {'cmd': 'create table test_bytea (pk serial primary key, data bytea)'},
2835 {'cmd': "insert into test_bytea (data) values (NULL::bytea)"}
2836 ])
2837
2838 md5_query = {
2839 'cmd': 'select md5(data) AS md5 FROM test_bytea WHERE pk = %(pk)s',
2840 'args': {'pk': 1}
2841 }
2842
2843 file2bytea_copy_from (
2844 table = 'test_bytea',
2845 columns = ['data'],
2846 filename = sys.argv[2],
2847 md5_query = md5_query,
2848 file_md5 = file2md5(sys.argv[2], True)
2849 )
2850
2851 run_rw_queries(queries = [
2852 {'cmd': 'drop table if exists test_bytea'}
2853 ])
2854
2855
2857
2858 run_rw_queries(queries = [
2859 {'cmd': 'drop table if exists test_bytea'},
2860 {'cmd': 'create table test_bytea (pk serial primary key, data bytea)'},
2861 {'cmd': "insert into test_bytea (data) values (NULL::bytea)"}
2862 ])
2863
2864 cmd = """
2865 update test_bytea
2866 set data = overlay (
2867 coalesce(data, ''::bytea)
2868 placing %(data)s::bytea
2869 from %(start)s
2870 for %(size)s
2871 )
2872 where
2873 pk > %(pk)s
2874 """
2875 md5_cmd = 'select md5(data) from test_bytea'
2876 args = {'pk': 0}
2877 file2bytea_overlay (
2878 query = cmd,
2879 args = args,
2880 filename = sys.argv[2],
2881 conn = None,
2882 md5_query = md5_cmd,
2883 file_md5 = file2md5(sys.argv[2], True)
2884 )
2885
2886 run_rw_queries(queries = [
2887 {'cmd': 'drop table test_bytea'}
2888 ])
2889
2890
2892 print("testing get_connection()")
2893
2894 print('')
2895 dsn = 'foo'
2896 try:
2897 conn = get_connection(dsn=dsn)
2898 except dbapi.ProgrammingError as e:
2899 print("1) SUCCESS: get_connection(%s) failed as expected" % dsn)
2900 t, v = sys.exc_info()[:2]
2901 print (' ', t)
2902 print (' ', v)
2903
2904 print('')
2905 dsn = 'dbname=gnumed_v9'
2906 try:
2907 conn = get_connection(dsn=dsn)
2908 except cAuthenticationError:
2909 print("2) SUCCESS: get_connection(%s) failed as expected" % dsn)
2910 t, v = sys.exc_info()[:2]
2911 print(' ', t)
2912 print(' ', v)
2913
2914 print('')
2915 dsn = 'dbname=gnumed_v9 user=abc'
2916 try:
2917 conn = get_connection(dsn=dsn)
2918 except cAuthenticationError:
2919 print("3) SUCCESS: get_connection(%s) failed as expected" % dsn)
2920 t, v = sys.exc_info()[:2]
2921 print(' ', t)
2922 print(' ', v)
2923
2924 print('')
2925 dsn = 'dbname=gnumed_v9 user=any-doc password=abc'
2926 try:
2927 conn = get_connection(dsn=dsn)
2928 except cAuthenticationError:
2929 print("5) SUCCESS: get_connection(%s) failed as expected" % dsn)
2930 t, v = sys.exc_info()[:2]
2931 print(' ', t)
2932 print(' ', v)
2933
2934 print('')
2935 dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc'
2936 conn = get_connection(dsn=dsn, readonly=True)
2937
2938 dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc'
2939 conn = get_connection(dsn=dsn, readonly=False, verbose=True)
2940
2941 print('')
2942 dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc'
2943 encoding = 'foo'
2944 try:
2945 conn = get_connection(dsn=dsn, encoding=encoding)
2946 except cEncodingError:
2947 print("6) SUCCESS: get_connection(%s, %s) failed as expected" % (dsn, encoding))
2948 t, v = sys.exc_info()[:2]
2949 print(' ', t)
2950 print(' ', v)
2951
2952 print('')
2953 dsn = 'dbname=gnumed_v22 user=any-doc'
2954 try:
2955 conn = get_connection(dsn=dsn)
2956 print("6) SUCCESS:", dsn)
2957 print('pid:', conn.get_backend_pid())
2958 except cAuthenticationError:
2959 print("4) SUCCESS: get_connection(%s) failed" % dsn)
2960 t, v = sys.exc_info()[:2]
2961 print(' ', t)
2962 print(' ', v)
2963
2964 try:
2965 curs = conn.cursor()
2966 input('hit enter to run query')
2967 curs.execute('selec 1')
2968 except Exception as exc:
2969 print('ERROR')
2970 _log.exception('exception occurred')
2971 log_pg_exception_details(exc)
2972 if exception_is_connection_loss(exc):
2973 _log.error('lost connection')
2974
2975
2977 print("testing exceptions")
2978
2979 try:
2980 raise cAuthenticationError('no dsn', 'no previous exception')
2981 except cAuthenticationError:
2982 t, v, tb = sys.exc_info()
2983 print(t)
2984 print(v)
2985 print(tb)
2986
2987 try:
2988 raise cEncodingError('no dsn', 'no previous exception')
2989 except cEncodingError:
2990 t, v, tb = sys.exc_info()
2991 print(t)
2992 print(v)
2993 print(tb)
2994
2996 print("testing run_ro_queries()")
2997
2998 dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc'
2999 conn = get_connection(dsn, readonly=True)
3000
3001 data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': 'SELECT version()'}], return_data=True, get_col_idx=True, verbose=True)
3002 print(data)
3003 print(idx)
3004 data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': 'SELECT 1'}], return_data=True, get_col_idx=True)
3005 print(data)
3006 print(idx)
3007
3008 curs = conn.cursor()
3009
3010 data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'SELECT version()'}], return_data=True, get_col_idx=True, verbose=True)
3011 print(data)
3012 print(idx)
3013
3014 data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'SELECT 1'}], return_data=True, get_col_idx=True, verbose=True)
3015 print(data)
3016 print(idx)
3017
3018 try:
3019 data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'selec 1'}], return_data=True, get_col_idx=True, verbose=True)
3020 print(data)
3021 print(idx)
3022 except psycopg2.ProgrammingError:
3023 print('SUCCESS: run_ro_queries("selec 1") failed as expected')
3024 t, v = sys.exc_info()[:2]
3025 print(' ', t)
3026 print(' ', v)
3027
3028 curs.close()
3029
3030
3035
3037 print("testing set_default_client_encoding()")
3038
3039 enc = 'foo'
3040 try:
3041 set_default_client_encoding(enc)
3042 print("SUCCESS: encoding [%s] worked" % enc)
3043 except ValueError:
3044 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
3045 t, v = sys.exc_info()[:2]
3046 print(' ', t)
3047 print(' ', v)
3048
3049 enc = ''
3050 try:
3051 set_default_client_encoding(enc)
3052 print("SUCCESS: encoding [%s] worked" % enc)
3053 except ValueError:
3054 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
3055 t, v = sys.exc_info()[:2]
3056 print(' ', t)
3057 print(' ', v)
3058
3059 enc = 'latin1'
3060 try:
3061 set_default_client_encoding(enc)
3062 print("SUCCESS: encoding [%s] worked" % enc)
3063 except ValueError:
3064 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
3065 t, v = sys.exc_info()[:2]
3066 print(' ', t)
3067 print(' ', v)
3068
3069 enc = 'utf8'
3070 try:
3071 set_default_client_encoding(enc)
3072 print("SUCCESS: encoding [%s] worked" % enc)
3073 except ValueError:
3074 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
3075 t, v = sys.exc_info()[:2]
3076 print(' ', t)
3077 print(' ', v)
3078
3079 enc = 'unicode'
3080 try:
3081 set_default_client_encoding(enc)
3082 print("SUCCESS: encoding [%s] worked" % enc)
3083 except ValueError:
3084 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
3085 t, v = sys.exc_info()[:2]
3086 print(' ', t)
3087 print(' ', v)
3088
3089 enc = 'UNICODE'
3090 try:
3091 set_default_client_encoding(enc)
3092 print("SUCCESS: encoding [%s] worked" % enc)
3093 except ValueError:
3094 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc)
3095 t, v = sys.exc_info()[:2]
3096 print(' ', t)
3097 print(' ', v)
3098
3107
3109 dsn = get_default_dsn()
3110 conn = get_connection(dsn, readonly=True)
3111 curs = conn.cursor()
3112 curs.execute('SELECT * from clin.clin_narrative where narrative = %s', ['a'])
3113
3115 tests = [
3116 ['(', '\\(']
3117 , ['[', '\\[']
3118 , [')', '\\)']
3119 ]
3120 for test in tests:
3121 result = sanitize_pg_regex(test[0])
3122 if result != test[1]:
3123 print('ERROR: sanitize_pg_regex(%s) returned "%s", expected "%s"' % (test[0], result, test[1]))
3124
3126 status = True
3127 tests = [
3128 [None, True],
3129 [1, True],
3130 ['1', True],
3131 ['abc', False]
3132 ]
3133
3134 if not is_pg_interval():
3135 print('ERROR: is_pg_interval() returned "False", expected "True"')
3136 status = False
3137
3138 for test in tests:
3139 result = is_pg_interval(test[0])
3140 if result != test[1]:
3141 print('ERROR: is_pg_interval(%s) returned "%s", expected "%s"' % (test[0], result, test[1]))
3142 status = False
3143
3144 return status
3145
3148
3149
3151 print(get_foreign_key_names (
3152 src_schema = 'clin',
3153 src_table = 'vaccination',
3154 src_column = 'fk_episode',
3155 target_schema = 'clin',
3156 target_table = 'episode',
3157 target_column = 'pk'
3158 ))
3159
3160
3162 schema = 'clin'
3163 table = 'episode'
3164 col = 'pk'
3165 print('column %s.%s.%s is referenced by:' % (schema, table, col))
3166 for row in get_foreign_keys2column (
3167 schema = schema,
3168 table = table,
3169 column = col
3170 ):
3171 print(' <- %s.%s' % (
3172 row['referencing_table'],
3173 row['referencing_column']
3174 ))
3175
3176
3178
3179 tests = [
3180
3181 [None, 'de_DE', True],
3182 [None, 'lang_w/o_tx', False],
3183 [None, None, True],
3184
3185 ['any-doc', 'de_DE', True],
3186 ['any-doc', 'lang_w/o_tx', False],
3187 ['any-doc', None, True],
3188
3189 ['invalid user', 'de_DE', None],
3190 ['invalid user', 'lang_w/o_tx', False],
3191 ['invalid user', None, True]
3192 ]
3193 for test in tests:
3194 try:
3195 result = set_user_language(user = test[0], language = test[1])
3196 if result != test[2]:
3197 print("test:", test)
3198 print("result:", result, "expected:", test[2])
3199 except psycopg2.IntegrityError as e:
3200 if test[2] is None:
3201 continue
3202 print("test:", test)
3203 print("expected exception")
3204 print("result:", e)
3205
3206
3208 for line in get_schema_revision_history():
3209 print(' - '.join(line))
3210
3211
3213 gmDateTime.init()
3214 args = {'dt': gmDateTime.pydt_max_here()}
3215 cmd = "SELECT %(dt)s"
3216
3217
3218
3219 cmd = """
3220 SELECT to_timestamp (foofoo,'YYMMDD.HH24MI') FROM (
3221 SELECT REGEXP_REPLACE (
3222 't1.130729.0902.tif', -- string
3223 E'(.1)\.([0-9\.]+)(\.tif)', -- pattern
3224 E'\\\\2' -- replacement
3225 ) AS foofoo
3226 ) AS foo"""
3227 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': args}], get_col_idx = False)
3228 print(rows)
3229 print(rows[0])
3230 print(rows[0][0])
3231
3234
3236 row_is_locked(table = 'dem.identity', pk = 12)
3237
3238 print("1st connection:")
3239 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
3240 print(" 1st shared lock succeeded:", lock_row(table = 'dem.identity', pk = 12, exclusive = False))
3241 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
3242
3243 print(" 2nd shared lock should succeed:", lock_row(table = 'dem.identity', pk = 12, exclusive = False))
3244 print(" `-> unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = False))
3245 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
3246 print(" exclusive lock should succeed:", lock_row(table = 'dem.identity', pk = 12, exclusive = True))
3247 print(" `-> unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = True))
3248 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
3249
3250 print("2nd connection:")
3251 conn = get_raw_connection(readonly=True)
3252 print(" shared lock should succeed:", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
3253 print(" `-> unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
3254 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
3255 print(" exclusive lock succeeded ?", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = True), "(should fail)")
3256 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
3257
3258 print("1st connection:")
3259 print(" unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = False))
3260 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
3261
3262 print("2nd connection:")
3263 print(" exclusive lock should succeed", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = True))
3264 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
3265 print(" shared lock should succeed:", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
3266 print(" `-> unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
3267 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
3268 print(" unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False))
3269 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12))
3270
3271 conn.close()
3272
3273
3275 print(get_foreign_key_names (
3276 src_schema = 'dem',
3277 src_table = 'names',
3278 src_column = 'id_identity',
3279 target_schema = 'dem',
3280 target_table = 'identity',
3281 target_column = 'pk'
3282 ))
3283
3284
3286 print(get_index_name(indexed_table = 'clin.vaccination', indexed_column = 'fk_episode'))
3287
3288
3291
3292
3296
3297
3298
3299
3300
3301 test_ro_queries()
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326