Package Gnumed :: Package pycommon :: Module gmPG2
[frames] | no frames]

Source Code for Module Gnumed.pycommon.gmPG2

   1  """GNUmed PostgreSQL connection handling. 
   2   
   3  TODO: iterator/generator batch fetching: 
   4          - http://groups-beta.google.com/group/comp.lang.python/msg/7ff516d7d9387dad 
   5          - search Google for "Geneator/Iterator Nesting Problem - Any Ideas? 2.4" 
   6   
   7  winner: 
   8  def resultset_functional_batchgenerator(cursor, size=100): 
   9          for results in iter(lambda: cursor.fetchmany(size), []): 
  10                  for rec in results: 
  11                          yield rec 
  12  """ 
  13  # ======================================================================= 
  14  __author__  = "K.Hilbert <Karsten.Hilbert@gmx.net>" 
  15  __license__ = 'GPL v2 or later (details at http://www.gnu.org)' 
  16   
  17  # stdlib 
  18  import time 
  19  import sys 
  20  import os 
  21  import stat 
  22  import io 
  23  import codecs 
  24  import logging 
  25  import datetime as pydt 
  26  import re as regex 
  27  import threading 
  28  import hashlib 
  29  import shutil 
  30   
  31   
  32  # GNUmed 
  33  if __name__ == '__main__': 
  34          sys.path.insert(0, '../../') 
  35  from Gnumed.pycommon import gmLoginInfo 
  36  from Gnumed.pycommon import gmExceptions 
  37  from Gnumed.pycommon import gmDateTime 
  38  from Gnumed.pycommon import gmBorg 
  39  from Gnumed.pycommon import gmI18N 
  40  from Gnumed.pycommon import gmLog2 
  41  from Gnumed.pycommon import gmTools 
  42  from Gnumed.pycommon.gmTools import prompted_input, u_replacement_character, format_dict_like 
  43   
  44  _log = logging.getLogger('gm.db') 
  45   
  46   
  47  # 3rd party 
  48  try: 
  49          import psycopg2 as dbapi 
  50  except ImportError: 
  51          _log.exception("Python database adapter psycopg2 not found.") 
  52          print("CRITICAL ERROR: Cannot find module psycopg2 for connecting to the database server.") 
  53          raise 
  54   
  55   
  56  _log.info('psycopg2 version: %s' % dbapi.__version__) 
  57  _log.info('PostgreSQL via DB-API module "%s": API level %s, thread safety %s, parameter style "%s"' % (dbapi, dbapi.apilevel, dbapi.threadsafety, dbapi.paramstyle)) 
  58  if not (float(dbapi.apilevel) >= 2.0): 
  59          raise ImportError('gmPG2: supported DB-API level too low') 
  60  if not (dbapi.threadsafety > 0): 
  61          raise ImportError('gmPG2: lacking minimum thread safety in psycopg2') 
  62  if not (dbapi.paramstyle == 'pyformat'): 
  63          raise ImportError('gmPG2: lacking pyformat (%%(<name>)s style) placeholder support in psycopg2') 
  64  try: 
  65          dbapi.__version__.index('dt') 
  66  except ValueError: 
  67          raise ImportError('gmPG2: lacking datetime support in psycopg2') 
  68  try: 
  69          dbapi.__version__.index('ext') 
  70  except ValueError: 
  71          raise ImportError('gmPG2: lacking extensions support in psycopg2') 
  72  try: 
  73          dbapi.__version__.index('pq3') 
  74  except ValueError: 
  75          raise ImportError('gmPG2: lacking v3 backend protocol support in psycopg2') 
  76   
  77  import psycopg2.extras 
  78  import psycopg2.extensions 
  79  import psycopg2.pool 
  80  import psycopg2.errorcodes as sql_error_codes 
  81   
  82  PG_ERROR_EXCEPTION = dbapi.Error 
  83   
  84  # ======================================================================= 
  85  _default_client_encoding = 'UTF8' 
  86  _log.info('assuming default client encoding of [%s]' % _default_client_encoding) 
  87   
  88  # things timezone 
  89  _default_client_timezone = None                 # default time zone for connections 
  90  _sql_set_timezone = None 
  91  _timestamp_template = "cast('%s' as timestamp with time zone)"          # MUST NOT be uniocde or else getquoted will not work 
  92  FixedOffsetTimezone = dbapi.tz.FixedOffsetTimezone 
  93   
  94  _default_dsn = None 
  95  _default_login = None 
  96   
  97  default_database = 'gnumed_v22' 
  98   
  99  postgresql_version_string = None 
 100  postgresql_version = None                       # accuracy: major.minor 
 101   
 102  __ro_conn_pool = None 
 103   
 104  auto_request_login_params = True 
 105   
 106  # ======================================================================= 
 107  # global data 
 108  # ======================================================================= 
 109   
 110  known_schema_hashes = { 
 111          0: 'not released, testing only', 
 112          2: 'b09d50d7ed3f91ddf4c4ddb8ea507720', 
 113          3: 'e73718eaf230d8f1d2d01afa8462e176', 
 114          4: '4428ccf2e54c289136819e701bb095ea', 
 115          5: '7e7b093af57aea48c288e76632a382e5',  # ... old (v1) style hashes 
 116          6: '90e2026ac2efd236da9c8608b8685b2d',  # new (v2) style hashes ... 
 117          7: '6c9f6d3981483f8e9433df99d1947b27', 
 118          8: '89b13a7af83337c3aad153b717e52360', 
 119          9: '641a9b2be3c378ffc2bb2f0b1c9f051d', 
 120          10: '7ef42a8fb2bd929a2cdd0c63864b4e8a', 
 121          11: '03042ae24f3f92877d986fb0a6184d76', 
 122          12: '06183a6616db62257e22814007a8ed07', 
 123          13: 'fab7c1ae408a6530c47f9b5111a0841e', 
 124          14: 'e170d543f067d1ea60bfe9076b1560cf', 
 125          15: '70012ff960b77ecdff4981c94b5b55b6', 
 126          16: '0bcf44ca22c479b52976e5eda1de8161', 
 127          17: '161428ee97a00e3bf56168c3a15b7b50', 
 128          18: 'a0f9efcabdecfb4ddb6d8c0b69c02092', 
 129          #19: '419e5225259c53dd36ad80d82066ff02' # 19.0 only 
 130          #19: '9765373098b03fb208332498f34cd4b5' # until 19.11 
 131          19: '57f009a159f55f77525cc0291e0c8b60', # starting with 19.12 
 132          20: 'baed1901ed4c2f272b56c8cb2c6d88e8', 
 133          21: 'e6a51a89dd22b75b61ead8f7083f251f', 
 134          22: 'bf45f01327fb5feb2f5d3c06ba4a6792' 
 135  } 
 136   
 137  map_schema_hash2version = { 
 138          'b09d50d7ed3f91ddf4c4ddb8ea507720': 2, 
 139          'e73718eaf230d8f1d2d01afa8462e176': 3, 
 140          '4428ccf2e54c289136819e701bb095ea': 4, 
 141          '7e7b093af57aea48c288e76632a382e5': 5, 
 142          '90e2026ac2efd236da9c8608b8685b2d': 6, 
 143          '6c9f6d3981483f8e9433df99d1947b27': 7, 
 144          '89b13a7af83337c3aad153b717e52360': 8, 
 145          '641a9b2be3c378ffc2bb2f0b1c9f051d': 9, 
 146          '7ef42a8fb2bd929a2cdd0c63864b4e8a': 10, 
 147          '03042ae24f3f92877d986fb0a6184d76': 11, 
 148          '06183a6616db62257e22814007a8ed07': 12, 
 149          'fab7c1ae408a6530c47f9b5111a0841e': 13, 
 150          'e170d543f067d1ea60bfe9076b1560cf': 14, 
 151          '70012ff960b77ecdff4981c94b5b55b6': 15, 
 152          '0bcf44ca22c479b52976e5eda1de8161': 16, 
 153          '161428ee97a00e3bf56168c3a15b7b50': 17, 
 154          'a0f9efcabdecfb4ddb6d8c0b69c02092': 18, 
 155          #'419e5225259c53dd36ad80d82066ff02': 19 # 19.0 only 
 156          #'9765373098b03fb208332498f34cd4b5': 19 # until 19.11 
 157          '57f009a159f55f77525cc0291e0c8b60': 19, # starting with 19.12 
 158          'baed1901ed4c2f272b56c8cb2c6d88e8': 20, 
 159          'e6a51a89dd22b75b61ead8f7083f251f': 21, 
 160          'bf45f01327fb5feb2f5d3c06ba4a6792': 22 
 161  } 
 162   
 163  map_client_branch2required_db_version = { 
 164          'GIT tree': 0, 
 165          '0.3': 9, 
 166          '0.4': 10, 
 167          '0.5': 11, 
 168          '0.6': 12, 
 169          '0.7': 13, 
 170          '0.8': 14, 
 171          '0.9': 15, 
 172          '1.0': 16,              # intentional duplicate with 1.1 
 173          '1.1': 16, 
 174          '1.2': 17, 
 175          '1.3': 18, 
 176          '1.4': 19, 
 177          '1.5': 20, 
 178          '1.6': 21, 
 179          '1.7': 22, 
 180          '1.8': 22               # Yes, SAME as 1.7, no DB change. 
 181  } 
 182   
 183  map_psyco_tx_status2str = [ 
 184          'TRANSACTION_STATUS_IDLE', 
 185          'TRANSACTION_STATUS_ACTIVE', 
 186          'TRANSACTION_STATUS_INTRANS', 
 187          'TRANSACTION_STATUS_INERROR', 
 188          'TRANSACTION_STATUS_UNKNOWN' 
 189  ] 
 190   
 191  map_psyco_conn_status2str = [ 
 192          '0 - ?', 
 193          'STATUS_READY', 
 194          'STATUS_BEGIN_ALIAS_IN_TRANSACTION', 
 195          'STATUS_PREPARED' 
 196  ] 
 197   
 198  map_psyco_iso_level2str = { 
 199          None: 'ISOLATION_LEVEL_DEFAULT (configured on server)', 
 200          0: 'ISOLATION_LEVEL_AUTOCOMMIT', 
 201          1: 'ISOLATION_LEVEL_READ_UNCOMMITTED', 
 202          2: 'ISOLATION_LEVEL_REPEATABLE_READ', 
 203          3: 'ISOLATION_LEVEL_SERIALIZABLE', 
 204          4: 'ISOLATION_LEVEL_READ_UNCOMMITTED' 
 205  } 
 206   
 207  # get columns and data types for a given table 
 208  query_table_col_defs = """select 
 209          cols.column_name, 
 210          cols.udt_name 
 211  from 
 212          information_schema.columns cols 
 213  where 
 214          cols.table_schema = %s 
 215                  and 
 216          cols.table_name = %s 
 217  order by 
 218          cols.ordinal_position""" 
 219   
 220  query_table_attributes = """select 
 221          cols.column_name 
 222  from 
 223          information_schema.columns cols 
 224  where 
 225          cols.table_schema = %s 
 226                  and 
 227          cols.table_name = %s 
 228  order by 
 229          cols.ordinal_position""" 
 230   
 231  # only works for single-column FKs but that's fine 
 232  # needs gm-dbo, any-doc won't work 
 233  SQL_foreign_key_name = """SELECT 
 234          fk_tbl.*, 
 235          (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = fk_tbl.connamespace) AS constraint_schema, 
 236          fk_tbl.conname AS constraint_name, 
 237          (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = (SELECT relnamespace FROM pg_class where oid = fk_tbl.conrelid)) AS source_schema, 
 238          (SELECT relname FROM pg_catalog.pg_class where oid = fk_tbl.conrelid) AS source_table, 
 239          (SELECT attname FROM pg_catalog.pg_attribute WHERE attnum = fk_tbl.conkey[1] AND attrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass) AS source_column, 
 240          (SELECT nspname FROM pg_catalog.pg_namespace WHERE oid = (SELECT relnamespace FROM pg_class where oid = fk_tbl.confrelid)) AS target_schema, 
 241          (SELECT relname FROM pg_catalog.pg_class where oid = fk_tbl.confrelid) AS target_table, 
 242          (SELECT attname FROM pg_catalog.pg_attribute WHERE attnum = fk_tbl.confkey[1] AND attrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass) AS target_column 
 243  FROM 
 244          pg_catalog.pg_constraint fk_tbl 
 245  WHERE 
 246          fk_tbl.contype = 'f' 
 247                  AND 
 248          fk_tbl.conrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass 
 249                  AND 
 250          fk_tbl.conkey[1] = ( 
 251                  SELECT 
 252                          col_tbl1.attnum 
 253                  FROM 
 254                          pg_catalog.pg_attribute col_tbl1 
 255                  WHERE 
 256                          col_tbl1.attname = %(src_col)s 
 257                                  AND 
 258                          col_tbl1.attrelid = (%(src_schema)s || '.' || %(src_tbl)s)::regclass 
 259          ) 
 260                  AND 
 261          fk_tbl.confrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass 
 262                  AND 
 263          fk_tbl.confkey[1] = ( 
 264                  SELECT 
 265                          col_tbl2.attnum 
 266                  FROM 
 267                          pg_catalog.pg_attribute col_tbl2 
 268                  WHERE 
 269                          col_tbl2.attname = %(target_col)s 
 270                                  AND 
 271                          col_tbl2.attrelid = (%(target_schema)s || '.' || %(target_tbl)s)::regclass 
 272          ) 
 273  """ 
 274   
 275  SQL_get_index_name = """ 
 276  SELECT 
 277          (SELECT nspname FROM pg_namespace WHERE pg_namespace.oid = pg_class.relnamespace) 
 278                  AS index_schema, 
 279          pg_class.relname 
 280                  AS index_name 
 281  FROM 
 282          pg_class 
 283  WHERE 
 284          pg_class.oid IN ( 
 285                  SELECT 
 286                          indexrelid 
 287                  FROM 
 288                          pg_index 
 289                  WHERE 
 290                          pg_index.indrelid = %(idx_tbl)s::regclass 
 291                                  AND 
 292                          pg_index.indnatts = 1           -- only one column in index 
 293                                  AND 
 294                          pg_index.indkey[0] IN ( 
 295                                  SELECT 
 296                                          pg_attribute.attnum 
 297                                  FROM 
 298                                          pg_attribute 
 299                                  WHERE 
 300                                          pg_attribute.attrelid = %(idx_tbl)s::regclass 
 301                                                  AND 
 302                                          pg_attribute.attname = %(idx_col)s 
 303                                  ) 
 304          ) 
 305  """ 
 306   
 307  SQL_get_pk_col_def = """ 
 308  SELECT 
 309          pg_attribute.attname 
 310                  AS pk_col, 
 311          format_type(pg_attribute.atttypid, pg_attribute.atttypmod) 
 312                  AS pk_type 
 313  FROM pg_index, pg_class, pg_attribute, pg_namespace 
 314  WHERE 
 315          pg_class.oid = %(table)s::regclass 
 316                  AND 
 317          indrelid = pg_class.oid 
 318                  AND 
 319  --      nspname = %%(schema)s 
 320  --              AND 
 321          pg_class.relnamespace = pg_namespace.oid 
 322                  AND 
 323          pg_attribute.attrelid = pg_class.oid 
 324                  AND 
 325          pg_attribute.attnum = any(pg_index.indkey) 
 326                  AND 
 327          indisprimary 
 328  """ 
 329   
 330  # ======================================================================= 
 331  # module globals API 
 332  # ======================================================================= 
333 -def set_default_client_encoding(encoding = None):
334 # check whether psycopg2 can handle this encoding 335 if encoding not in psycopg2.extensions.encodings: 336 raise ValueError('psycopg2 does not know how to handle client (wire) encoding [%s]' % encoding) 337 # check whether Python can handle this encoding 338 py_enc = psycopg2.extensions.encodings[encoding] 339 try: 340 codecs.lookup(py_enc) 341 except LookupError: 342 _log.warning('<codecs> module can NOT handle encoding [psycopg2::<%s> -> Python::<%s>]' % (encoding, py_enc)) 343 raise 344 # FIXME: check encoding against the database 345 # FIXME: - but we may not yet have access 346 # FIXME: - psycopg2 will pull its encodings from the database eventually 347 # it seems save to set it 348 global _default_client_encoding 349 _log.info('setting default client encoding from [%s] to [%s]' % (_default_client_encoding, encoding)) 350 _default_client_encoding = encoding 351 return True
352 353 #---------------------------------------------------
354 -def set_default_client_timezone(timezone = None):
355 356 # FIXME: use __validate 357 global _default_client_timezone 358 _log.info('setting default client time zone from [%s] to [%s]' % (_default_client_timezone, timezone)) 359 _default_client_timezone = timezone 360 361 global _sql_set_timezone 362 _sql_set_timezone = 'set timezone to %s' 363 364 return True
365 366 #---------------------------------------------------
367 -def __validate_timezone(conn=None, timezone=None):
368 369 _log.debug('validating time zone [%s]', timezone) 370 371 cmd = 'set timezone to %(tz)s' 372 args = {'tz': timezone} 373 374 conn.commit() 375 curs = conn.cursor() 376 is_valid = False 377 try: 378 curs.execute(cmd, args) 379 _log.info('time zone [%s] is settable', timezone) 380 # can we actually use it, though ? 381 cmd = """select '1920-01-19 23:00:00+01'::timestamp with time zone""" 382 try: 383 curs.execute(cmd) 384 curs.fetchone() 385 _log.info('time zone [%s] is usable', timezone) 386 is_valid = True 387 except: 388 _log.error('error using time zone [%s]', timezone) 389 except dbapi.DataError: 390 _log.warning('time zone [%s] is not settable', timezone) 391 except: 392 _log.error('failed to set time zone to [%s]', timezone) 393 _log.exception('') 394 395 curs.close() 396 conn.rollback() 397 398 return is_valid
399 400 #---------------------------------------------------
401 -def __expand_timezone(conn=None, timezone=None):
402 """some timezone defs are abbreviations so try to expand 403 them because "set time zone" doesn't take abbreviations""" 404 405 cmd = """ 406 select distinct on (abbrev) name 407 from pg_timezone_names 408 where 409 abbrev = %(tz)s and 410 name ~ '^[^/]+/[^/]+$' and 411 name !~ '^Etc/' 412 """ 413 args = {'tz': timezone} 414 415 conn.commit() 416 curs = conn.cursor() 417 418 result = timezone 419 try: 420 curs.execute(cmd, args) 421 rows = curs.fetchall() 422 if len(rows) > 0: 423 result = rows[0]['name'] 424 _log.debug('[%s] maps to [%s]', timezone, result) 425 except: 426 _log.exception('cannot expand timezone abbreviation [%s]', timezone) 427 428 curs.close() 429 conn.rollback() 430 431 return result
432 433 #---------------------------------------------------
434 -def __detect_client_timezone(conn=None):
435 """This is run on the very first connection.""" 436 437 # FIXME: check whether server.timezone is the same 438 # FIXME: value as what we eventually detect 439 440 # we need gmDateTime to be initialized 441 if gmDateTime.current_local_iso_numeric_timezone_string is None: 442 gmDateTime.init() 443 444 _log.debug('trying to detect timezone from system') 445 446 tz_candidates = [] 447 try: 448 tz = os.environ['TZ'] 449 tz_candidates.append(tz) 450 expanded = __expand_timezone(conn = conn, timezone = tz) 451 if expanded != tz: 452 tz_candidates.append(expanded) 453 except KeyError: 454 pass 455 456 tz_candidates.append(gmDateTime.current_local_timezone_name) 457 expanded = __expand_timezone(conn = conn, timezone = gmDateTime.current_local_timezone_name) 458 if expanded != gmDateTime.current_local_timezone_name: 459 tz_candidates.append(expanded) 460 461 _log.debug('candidates: %s', tz_candidates) 462 463 # find best among candidates 464 global _default_client_timezone 465 global _sql_set_timezone 466 found = False 467 for tz in tz_candidates: 468 if __validate_timezone(conn = conn, timezone = tz): 469 _default_client_timezone = tz 470 _sql_set_timezone = 'set timezone to %s' 471 found = True 472 break 473 474 if not found: 475 _default_client_timezone = gmDateTime.current_local_iso_numeric_timezone_string 476 _sql_set_timezone = "set time zone interval %s hour to minute" 477 478 _log.info('client system time zone detected as equivalent to [%s]', _default_client_timezone)
479 480 # ======================================================================= 481 # login API 482 # =======================================================================
483 -def __request_login_params_tui():
484 """Text mode request of database login parameters""" 485 import getpass 486 login = gmLoginInfo.LoginInfo() 487 488 print("\nPlease enter the required login parameters:") 489 try: 490 login.host = prompted_input(prompt = "host ('' = non-TCP/IP)", default = '') 491 login.database = prompted_input(prompt = "database", default = default_database) 492 login.user = prompted_input(prompt = "user name", default = '') 493 tmp = 'password for "%s" (not shown): ' % login.user 494 login.password = getpass.getpass(tmp) 495 gmLog2.add_word2hide(login.password) 496 login.port = prompted_input(prompt = "port", default = 5432) 497 except KeyboardInterrupt: 498 _log.warning("user cancelled text mode login dialog") 499 print("user cancelled text mode login dialog") 500 raise gmExceptions.ConnectionError(_("Cannot connect to database without login information!")) 501 502 return login
503 504 #---------------------------------------------------
505 -def __request_login_params_gui_wx():
506 """GUI (wx) input request for database login parameters. 507 508 Returns gmLoginInfo.LoginInfo object 509 """ 510 import wx 511 # OK, wxPython was already loaded. But has the main Application instance 512 # been initialized yet ? if not, the exception will kick us out 513 if wx.GetApp() is None: 514 raise AssertionError(_("The wxPython GUI framework hasn't been initialized yet!")) 515 516 # Let's launch the login dialog 517 # if wx was not initialized /no main App loop, an exception should be raised anyway 518 import gmAuthWidgets 519 dlg = gmAuthWidgets.cLoginDialog(None, -1) 520 dlg.ShowModal() 521 login = dlg.panel.GetLoginInfo() 522 dlg.DestroyLater() 523 524 #if user cancelled or something else went wrong, raise an exception 525 if login is None: 526 raise gmExceptions.ConnectionError(_("Can't connect to database without login information!")) 527 528 gmLog2.add_word2hide(login.password) 529 530 return login
531 532 #---------------------------------------------------
533 -def request_login_params():
534 """Request login parameters for database connection.""" 535 # do we auto-request parameters at all ? 536 if not auto_request_login_params: 537 raise Exception('Cannot request login parameters.') 538 539 # are we inside X ? 540 # if we aren't wxGTK will crash hard at the C-level with "can't open Display" 541 if 'DISPLAY' in os.environ: 542 # try wxPython GUI 543 try: 544 return __request_login_params_gui_wx() 545 except: 546 pass 547 548 # well, either we are on the console or 549 # wxPython does not work, use text mode 550 return __request_login_params_tui()
551 552 # ======================================================================= 553 # DSN API 554 # -----------------------------------------------------------------------
555 -def make_psycopg2_dsn(database=None, host=None, port=5432, user=None, password=None):
556 dsn_parts = [] 557 558 if (database is not None) and (database.strip() != ''): 559 dsn_parts.append('dbname=%s' % database) 560 561 if (host is not None) and (host.strip() != ''): 562 dsn_parts.append('host=%s' % host) 563 564 if (port is not None) and (str(port).strip() != ''): 565 dsn_parts.append('port=%s' % port) 566 567 if (user is not None) and (user.strip() != ''): 568 dsn_parts.append('user=%s' % user) 569 570 if (password is not None) and (password.strip() != ''): 571 dsn_parts.append('password=%s' % password) 572 573 dsn_parts.append('sslmode=prefer') 574 dsn_parts.append('fallback_application_name=GNUmed') 575 576 return ' '.join(dsn_parts)
577 578 # ------------------------------------------------------
579 -def get_default_login():
580 # make sure we do have a login 581 get_default_dsn() 582 return _default_login
583 584 # ------------------------------------------------------
585 -def get_default_dsn():
586 global _default_dsn 587 if _default_dsn is not None: 588 return _default_dsn 589 590 login = request_login_params() 591 set_default_login(login=login) 592 593 return _default_dsn
594 595 # ------------------------------------------------------
596 -def set_default_login(login=None):
597 if login is None: 598 return False 599 600 if login.host is not None: 601 if login.host.strip() == '': 602 login.host = None 603 604 global _default_login 605 _default_login = login 606 _log.info('setting default login from [%s] to [%s]' % (_default_login, login)) 607 608 dsn = make_psycopg2_dsn(login.database, login.host, login.port, login.user, login.password) 609 610 global _default_dsn 611 if _default_dsn is None: 612 old_dsn = 'None' 613 else: 614 old_dsn = regex.sub(r'password=[^\s]+', 'password=%s' % u_replacement_character, _default_dsn) 615 _log.info ('setting default DSN from [%s] to [%s]', 616 old_dsn, 617 regex.sub(r'password=[^\s]+', 'password=%s' % u_replacement_character, dsn) 618 ) 619 _default_dsn = dsn 620 621 return True
622 623 #------------------------------------------------------------------------
624 -def log_auth_environment():
625 try: 626 pgpass_file = os.path.expanduser(os.path.join('~', '.pgpass')) 627 if os.path.exists(pgpass_file): 628 _log.debug('standard .pgpass (%s) exists', pgpass_file) 629 else: 630 _log.debug('standard .pgpass (%s) not found', pgpass_file) 631 pgpass_var = os.getenv('PGPASSFILE') 632 if pgpass_var is None: 633 _log.debug('$PGPASSFILE not set') 634 else: 635 if os.path.exists(pgpass_var): 636 _log.debug('$PGPASSFILE=%s exists', pgpass_var) 637 else: 638 _log.debug('$PGPASSFILE=%s not found') 639 except Exception: 640 _log.exception('cannot detect .pgpass and or $PGPASSFILE')
641 642 # ======================================================================= 643 # netadata API 644 # =======================================================================
645 -def database_schema_compatible(link_obj=None, version=None, verbose=True):
646 expected_hash = known_schema_hashes[version] 647 if version == 0: 648 args = {'ver': 9999} 649 else: 650 args = {'ver': version} 651 rows, idx = run_ro_queries ( 652 link_obj = link_obj, 653 queries = [{ 654 'cmd': 'select md5(gm.concat_table_structure(%(ver)s::integer)) as md5', 655 'args': args 656 }] 657 ) 658 if rows[0]['md5'] != expected_hash: 659 _log.error('database schema version mismatch') 660 _log.error('expected: %s (%s)' % (version, expected_hash)) 661 _log.error('detected: %s (%s)' % (get_schema_version(link_obj=link_obj), rows[0]['md5'])) 662 if verbose: 663 _log.debug('schema dump follows:') 664 for line in get_schema_structure(link_obj = link_obj).split(): 665 _log.debug(line) 666 _log.debug('schema revision history dump follows:') 667 for line in get_schema_revision_history(link_obj = link_obj): 668 _log.debug(' - '.join(line)) 669 return False 670 _log.info('detected schema version [%s], hash [%s]' % (map_schema_hash2version[rows[0]['md5']], rows[0]['md5'])) 671 return True
672 673 #------------------------------------------------------------------------
674 -def get_schema_version(link_obj=None):
675 rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select md5(gm.concat_table_structure()) as md5'}]) 676 try: 677 return map_schema_hash2version[rows[0]['md5']] 678 except KeyError: 679 return 'unknown database schema version, MD5 hash is [%s]' % rows[0]['md5']
680 681 #------------------------------------------------------------------------
682 -def get_schema_structure(link_obj=None):
683 rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select gm.concat_table_structure()'}]) 684 return rows[0][0]
685 686 #------------------------------------------------------------------------
687 -def get_schema_hash(link_obj=None):
688 rows, idx = run_ro_queries(link_obj=link_obj, queries = [{'cmd': 'select md5(gm.concat_table_structure()) as md5'}]) 689 return rows[0]['md5']
690 691 #------------------------------------------------------------------------
692 -def get_schema_revision_history(link_obj=None):
693 694 if table_exists(link_obj = link_obj, schema = 'gm', table = 'schema_revision'): 695 cmd = """ 696 SELECT 697 imported::text, 698 version, 699 filename 700 FROM gm.schema_revision 701 ORDER BY imported""" 702 elif table_exists(link_obj = link_obj, schema = 'public', table = 'gm_schema_revision'): 703 cmd = """ 704 SELECT 705 imported::text, 706 version, 707 filename 708 FROM public.gm_schema_revision 709 ORDER BY imported""" 710 else: 711 return [] 712 713 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}]) 714 return rows
715 #------------------------------------------------------------------------
716 -def get_current_user():
717 rows, idx = run_ro_queries(queries = [{'cmd': 'select CURRENT_USER'}]) 718 return rows[0][0]
719 720 #------------------------------------------------------------------------
721 -def get_foreign_keys2column(schema='public', table=None, column=None, link_obj=None):
722 """Get the foreign keys pointing to schema.table.column. 723 724 Does not properly work with multi-column FKs. 725 GNUmed doesn't use any, however. 726 """ 727 args = { 728 'schema': schema, 729 'tbl': table, 730 'col': column 731 } 732 cmd = """ 733 SELECT 734 %(schema)s AS referenced_schema, 735 %(tbl)s AS referenced_table, 736 %(col)s AS referenced_column, 737 pgc.confkey AS referenced_column_list, 738 739 pgc.conrelid::regclass AS referencing_table, 740 pgc.conkey AS referencing_column_list, 741 (select attname from pg_attribute where attnum = pgc.conkey[1] and attrelid = pgc.conrelid) AS referencing_column 742 FROM 743 pg_constraint pgc 744 WHERE 745 pgc.contype = 'f' 746 AND 747 pgc.confrelid = ( 748 select oid from pg_class where relname = %(tbl)s and relnamespace = ( 749 select oid from pg_namespace where nspname = %(schema)s 750 ) 751 ) and 752 ( 753 select attnum 754 from pg_attribute 755 where 756 attrelid = (select oid from pg_class where relname = %(tbl)s and relnamespace = ( 757 select oid from pg_namespace where nspname = %(schema)s 758 )) 759 and 760 attname = %(col)s 761 ) = any(pgc.confkey) 762 """ 763 rows, idx = run_ro_queries ( 764 link_obj = link_obj, 765 queries = [ 766 {'cmd': cmd, 'args': args} 767 ] 768 ) 769 770 return rows
771 772 #------------------------------------------------------------------------
773 -def get_index_name(indexed_table=None, indexed_column=None, link_obj=None):
774 775 args = { 776 'idx_tbl': indexed_table, 777 'idx_col': indexed_column 778 } 779 rows, idx = run_ro_queries ( 780 link_obj = link_obj, 781 queries = [{'cmd': SQL_get_index_name, 'args': args}], 782 get_col_idx = False 783 ) 784 785 return rows
786 787 #------------------------------------------------------------------------
788 -def get_foreign_key_names(src_schema=None, src_table=None, src_column=None, target_schema=None, target_table=None, target_column=None, link_obj=None):
789 790 args = { 791 'src_schema': src_schema, 792 'src_tbl': src_table, 793 'src_col': src_column, 794 'target_schema': target_schema, 795 'target_tbl': target_table, 796 'target_col': target_column 797 } 798 799 rows, idx = run_ro_queries ( 800 link_obj = link_obj, 801 queries = [{'cmd': SQL_foreign_key_name, 'args': args}], 802 get_col_idx = False 803 ) 804 805 return rows
806 807 #------------------------------------------------------------------------
808 -def get_child_tables(schema='public', table=None, link_obj=None):
809 """Return child tables of <table>.""" 810 cmd = """ 811 select 812 pgn.nspname as namespace, 813 pgc.relname as table 814 from 815 pg_namespace pgn, 816 pg_class pgc 817 where 818 pgc.relnamespace = pgn.oid 819 and 820 pgc.oid in ( 821 select inhrelid from pg_inherits where inhparent = ( 822 select oid from pg_class where 823 relnamespace = (select oid from pg_namespace where nspname = %(schema)s) and 824 relname = %(table)s 825 ) 826 )""" 827 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': {'schema': schema, 'table': table}}]) 828 return rows
829 830 #------------------------------------------------------------------------
831 -def schema_exists(link_obj=None, schema='gm'):
832 cmd = """SELECT EXISTS (SELECT 1 FROM pg_namespace WHERE nspname = %(schema)s)""" 833 args = {'schema': schema} 834 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}]) 835 return rows[0][0]
836 837 #------------------------------------------------------------------------
838 -def table_exists(link_obj=None, schema=None, table=None):
839 """Returns false, true.""" 840 cmd = """ 841 select exists ( 842 select 1 from information_schema.tables 843 where 844 table_schema = %s and 845 table_name = %s and 846 table_type = 'BASE TABLE' 847 )""" 848 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': (schema, table)}]) 849 return rows[0][0]
850 851 #------------------------------------------------------------------------
852 -def function_exists(link_obj=None, schema=None, function=None):
853 854 cmd = """ 855 SELECT EXISTS ( 856 SELECT 1 FROM pg_proc 857 WHERE proname = %(func)s AND pronamespace = (SELECT oid FROM pg_namespace WHERE nspname = %(schema)s) 858 ) 859 """ 860 args = { 861 'func': function, 862 'schema': schema 863 } 864 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}]) 865 return rows[0][0]
866 867 #------------------------------------------------------------------------
868 -def get_col_indices(cursor = None):
869 if cursor.description is None: 870 _log.error('no result description available: unused cursor or last query did not select rows') 871 return None 872 col_indices = {} 873 col_index = 0 874 for col_desc in cursor.description: 875 col_name = col_desc[0] 876 # a query like "select 1,2;" will return two columns of the same name ! 877 # hence adjust to that, note, however, that dict-style access won't work 878 # on results of such queries ... 879 if col_name in col_indices: 880 col_name = '%s_%s' % (col_name, col_index) 881 col_indices[col_name] = col_index 882 col_index += 1 883 884 return col_indices
885 #------------------------------------------------------------------------
886 -def get_col_defs(link_obj=None, schema='public', table=None):
887 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_col_defs, 'args': (schema, table)}]) 888 col_names = [] 889 col_type = {} 890 for row in rows: 891 col_names.append(row[0]) 892 # map array types 893 if row[1].startswith('_'): 894 col_type[row[0]] = row[1][1:] + '[]' 895 else: 896 col_type[row[0]] = row[1] 897 col_defs = [] 898 col_defs.append(col_names) 899 col_defs.append(col_type) 900 return col_defs
901 #------------------------------------------------------------------------
902 -def get_col_names(link_obj=None, schema='public', table=None):
903 """Return column attributes of table""" 904 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': query_table_attributes, 'args': (schema, table)}]) 905 cols = [] 906 for row in rows: 907 cols.append(row[0]) 908 return cols
909 910 #------------------------------------------------------------------------ 911 # i18n functions 912 #------------------------------------------------------------------------
913 -def export_translations_from_database(filename=None):
914 tx_file = io.open(filename, mode = 'wt', encoding = 'utf8') 915 tx_file.write('-- GNUmed database string translations exported %s\n' % gmDateTime.pydt_now_here().strftime('%Y-%m-%d %H:%M')) 916 tx_file.write('-- - contains translations for each of [%s]\n' % ', '.join(get_translation_languages())) 917 tx_file.write('-- - user database language is set to [%s]\n\n' % get_current_user_language()) 918 tx_file.write('-- Please email this file to <gnumed-devel@gnu.org>.\n') 919 tx_file.write('-- ----------------------------------------------------------------------------------------------\n\n') 920 tx_file.write('set default_transaction_read_only to off;\n\n') 921 tx_file.write("set client_encoding to 'utf-8';\n\n") 922 tx_file.write('\\unset ON_ERROR_STOP\n\n') 923 924 cmd = 'SELECT lang, orig, trans FROM i18n.translations ORDER BY lang, orig' 925 rows, idx = run_ro_queries(queries = [{'cmd': cmd}], get_col_idx = False) 926 for row in rows: 927 line = "select i18n.upd_tx(E'%s', E'%s', E'%s');\n" % ( 928 row['lang'].replace("'", "\\'"), 929 row['orig'].replace("'", "\\'"), 930 row['trans'].replace("'", "\\'") 931 ) 932 tx_file.write(line) 933 tx_file.write('\n') 934 935 tx_file.write('\set ON_ERROR_STOP 1\n') 936 tx_file.close() 937 938 return True
939 940 #------------------------------------------------------------------------
941 -def delete_translation_from_database(link_obj=None, language=None, original=None):
942 cmd = 'DELETE FROM i18n.translations WHERE lang = %(lang)s AND orig = %(orig)s' 943 args = {'lang': language, 'orig': original} 944 run_rw_queries(link_obj = link_obj, queries = [{'cmd': cmd, 'args': args}], return_data = False, end_tx = True) 945 return True
946 947 #------------------------------------------------------------------------
948 -def update_translation_in_database(language=None, original=None, translation=None, link_obj=None):
949 if language is None: 950 cmd = 'SELECT i18n.upd_tx(%(orig)s, %(trans)s)' 951 else: 952 cmd = 'SELECT i18n.upd_tx(%(lang)s, %(orig)s, %(trans)s)' 953 args = {'lang': language, 'orig': original, 'trans': translation} 954 run_rw_queries(queries = [{'cmd': cmd, 'args': args}], return_data = False, link_obj = link_obj) 955 return args
956 957 #------------------------------------------------------------------------
958 -def get_translation_languages():
959 rows, idx = run_ro_queries ( 960 queries = [{'cmd': 'select distinct lang from i18n.translations'}] 961 ) 962 return [ r[0] for r in rows ]
963 964 #------------------------------------------------------------------------
965 -def get_database_translations(language=None, order_by=None):
966 967 args = {'lang': language} 968 _log.debug('language [%s]', language) 969 970 if order_by is None: 971 order_by = 'ORDER BY %s' % order_by 972 else: 973 order_by = 'ORDER BY lang, orig' 974 975 if language is None: 976 cmd = """ 977 SELECT DISTINCT ON (orig, lang) 978 lang, orig, trans 979 FROM (( 980 981 -- strings stored as translation keys whether translated or not 982 SELECT 983 NULL as lang, 984 ik.orig, 985 NULL AS trans 986 FROM 987 i18n.keys ik 988 989 ) UNION ALL ( 990 991 -- already translated strings 992 SELECT 993 it.lang, 994 it.orig, 995 it.trans 996 FROM 997 i18n.translations it 998 999 )) as translatable_strings 1000 %s""" % order_by 1001 else: 1002 cmd = """ 1003 SELECT DISTINCT ON (orig, lang) 1004 lang, orig, trans 1005 FROM (( 1006 1007 -- strings stored as translation keys whether translated or not 1008 SELECT 1009 %%(lang)s as lang, 1010 ik.orig, 1011 i18n._(ik.orig, %%(lang)s) AS trans 1012 FROM 1013 i18n.keys ik 1014 1015 ) UNION ALL ( 1016 1017 -- already translated strings 1018 SELECT 1019 %%(lang)s as lang, 1020 it.orig, 1021 i18n._(it.orig, %%(lang)s) AS trans 1022 FROM 1023 i18n.translations it 1024 1025 )) AS translatable_strings 1026 %s""" % order_by 1027 1028 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': args}], get_col_idx = False) 1029 1030 if rows is None: 1031 _log.error('no translatable strings found') 1032 else: 1033 _log.debug('%s translatable strings found', len(rows)) 1034 1035 return rows
1036 1037 #------------------------------------------------------------------------
1038 -def get_current_user_language():
1039 cmd = 'select i18n.get_curr_lang()' 1040 rows, idx = run_ro_queries(queries = [{'cmd': cmd}]) 1041 return rows[0][0]
1042 1043 #------------------------------------------------------------------------
1044 -def set_user_language(user=None, language=None):
1045 """Set the user language in the database. 1046 1047 user = None: current db user 1048 language = None: unset 1049 """ 1050 _log.info('setting database language for user [%s] to [%s]', user, language) 1051 1052 args = { 1053 'usr': user, 1054 'lang': language 1055 } 1056 1057 if language is None: 1058 if user is None: 1059 queries = [{'cmd': 'select i18n.unset_curr_lang()'}] 1060 else: 1061 queries = [{'cmd': 'select i18n.unset_curr_lang(%(usr)s)', 'args': args}] 1062 queries.append({'cmd': 'select True'}) 1063 else: 1064 if user is None: 1065 queries = [{'cmd': 'select i18n.set_curr_lang(%(lang)s)', 'args': args}] 1066 else: 1067 queries = [{'cmd': 'select i18n.set_curr_lang(%(lang)s, %(usr)s)', 'args': args}] 1068 1069 rows, idx = run_rw_queries(queries = queries, return_data = True) 1070 1071 if not rows[0][0]: 1072 _log.error('cannot set database language to [%s] for user [%s]', language, user) 1073 1074 return rows[0][0]
1075 #------------------------------------------------------------------------
1076 -def force_user_language(language=None):
1077 """Set the user language in the database. 1078 1079 - regardless of whether there is any translation available. 1080 - only for the current user 1081 """ 1082 _log.info('forcing database language for current db user to [%s]', language) 1083 1084 run_rw_queries(queries = [{ 1085 'cmd': 'select i18n.force_curr_lang(%(lang)s)', 1086 'args': {'lang': language} 1087 }])
1088 1089 # ======================================================================= 1090 # query runners and helpers 1091 # =======================================================================
1092 -def send_maintenance_notification():
1093 cmd = 'notify "db_maintenance_warning"' 1094 run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
1095 1096 #------------------------------------------------------------------------
1097 -def send_maintenance_shutdown():
1098 cmd = 'notify "db_maintenance_disconnect"' 1099 run_rw_queries(queries = [{'cmd': cmd}], return_data = False)
1100 1101 #------------------------------------------------------------------------
1102 -def is_pg_interval(candidate=None):
1103 cmd = 'SELECT %(candidate)s::interval' 1104 try: 1105 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}]) 1106 return True 1107 except: 1108 cmd = 'SELECT %(candidate)s::text::interval' 1109 try: 1110 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': {'candidate': candidate}}]) 1111 return True 1112 except: 1113 return False
1114 1115 #------------------------------------------------------------------------
1116 -def lock_row(link_obj=None, table=None, pk=None, exclusive=False):
1117 """Uses pg_advisory(_shared). 1118 1119 - locks stack upon each other and need one unlock per lock 1120 - same connection: 1121 - all locks succeed 1122 - different connections: 1123 - shared + shared succeed 1124 - shared + exclusive fail 1125 """ 1126 _log.debug('locking row: [%s] [%s] (exclusive: %s)', table, pk, exclusive) 1127 if exclusive: 1128 cmd = """SELECT pg_try_advisory_lock('%s'::regclass::oid::int, %s)""" % (table, pk) 1129 else: 1130 cmd = """SELECT pg_try_advisory_lock_shared('%s'::regclass::oid::int, %s)""" % (table, pk) 1131 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}], get_col_idx = False) 1132 if rows[0][0]: 1133 return True 1134 _log.warning('cannot lock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive) 1135 return False
1136 1137 #------------------------------------------------------------------------
1138 -def unlock_row(link_obj=None, table=None, pk=None, exclusive=False):
1139 """Uses pg_advisory_unlock(_shared). 1140 1141 - each lock needs one unlock 1142 """ 1143 _log.debug('trying to unlock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive) 1144 if exclusive: 1145 cmd = "SELECT pg_advisory_unlock('%s'::regclass::oid::int, %s)" % (table, pk) 1146 else: 1147 cmd = "SELECT pg_advisory_unlock_shared('%s'::regclass::oid::int, %s)" % (table, pk) 1148 rows, idx = run_ro_queries(link_obj = link_obj, queries = [{'cmd': cmd}], get_col_idx = False) 1149 if rows[0][0]: 1150 return True 1151 _log.warning('cannot unlock row: [%s] [%s] (exclusive: %s)', table, pk, exclusive) 1152 return False
1153 1154 #------------------------------------------------------------------------
1155 -def row_is_locked(table=None, pk=None):
1156 """Looks at pk_locks 1157 1158 - does not take into account locks other than 'advisory', however 1159 """ 1160 cmd = """SELECT EXISTS ( 1161 SELECT 1 FROM pg_locks WHERE 1162 classid = '%s'::regclass::oid::int 1163 AND 1164 objid = %s 1165 AND 1166 locktype = 'advisory' 1167 )""" % (table, pk) 1168 rows, idx = run_ro_queries(queries = [{'cmd': cmd}], get_col_idx = False) 1169 if rows[0][0]: 1170 _log.debug('row is locked: [%s] [%s]', table, pk) 1171 return True 1172 _log.debug('row is NOT locked: [%s] [%s]', table, pk) 1173 return False
1174 1175 #------------------------------------------------------------------------ 1176 # BYTEA cache handling 1177 #------------------------------------------------------------------------ 1178 #_BYTEA_CACHE_SUBDIR = None 1179 # 1180 #def __get_bytea_cache_dir(): 1181 # """Get, and create if necessary, the BYTEA cache directory. 1182 # 1183 # The BYTEA cache needs to be tied to the database login 1184 # because different logins might have access to different 1185 # data even with the same query. 1186 # """ 1187 # global _BYTEA_CACHE_SUBDIR 1188 # if _BYTEA_CACHE_SUBDIR is not None: 1189 # return _BYTEA_CACHE_SUBDIR 1190 # 1191 # subdir_key = '%s::%s::%s::%s' % ( 1192 # _default_login.host, 1193 # _default_login.port, 1194 # _default_login.database, 1195 # _default_login.user 1196 # ) 1197 # md5 = hashlib.md5() 1198 # md5.update(subdir_key.encode('utf8')) 1199 # md5_sum = md5.hexdigest() 1200 # subdir = os.path.join(gmTools.gmPaths().bytea_cache_dir, md5_sum) 1201 # gmTools.mkdir(subdir, 0o700) 1202 # _log.info('BYTEA cache dir for [%s]: %s', subdir_key, subdir) 1203 # _BYTEA_CACHE_SUBDIR = subdir 1204 # return _BYTEA_CACHE_SUBDIR 1205 # 1206 #------------------------------------------------------------------------
1207 -def __generate_cached_filename(cache_key_data):
1208 md5 = hashlib.md5() 1209 md5.update(('%s' % cache_key_data).encode('utf8')) 1210 return os.path.join(gmTools.gmPaths().bytea_cache_dir, md5.hexdigest())
1211 1212 #------------------------------------------------------------------------
1213 -def __store_file_in_cache(filename, cache_key_data):
1214 cached_name = __generate_cached_filename(cache_key_data) 1215 _log.debug('[%s] -> [%s] -> [%s]', filename, cache_key_data, cached_name) 1216 if not gmTools.remove_file(cached_name, log_error = True, force = True): 1217 _log.error('cannot remove existing file [%s] for key [%s] from cache', filename, cached_name) 1218 return None 1219 1220 PERMS_owner_only = 0o0660 1221 try: 1222 shutil.copyfile(filename, cached_name, follow_symlinks = True) 1223 os.chmod(cached_name, PERMS_owner_only) 1224 except shutil.SameFileError: 1225 _log.exception('file seems to exist in cache, despite having checked and possible removed it just before') 1226 # don't use that file, it is unsafe, it might have come from 1227 # a race being exploited to make us use the wrong data, this 1228 # then constitutes a DOS attack against the cache but that's 1229 # far less problematic than using the wrong data for care 1230 return None 1231 except OSError: 1232 _log.exception('cannot copy file into cache: [%s] -> [%s]', filename, cached_name) 1233 return None 1234 except PermissionError: 1235 _log.exception('cannot set cache file [%s] permissions to [%s]', cached_name, stat.filemode(PERMS_owner_only)) 1236 return None 1237 1238 return cached_name
1239 1240 #------------------------------------------------------------------------
1241 -def __get_filename_in_cache(cache_key_data=None, data_size=None):
1242 """Calculate and verify filename in cache given cache key details.""" 1243 cached_name = __generate_cached_filename(cache_key_data) 1244 try: 1245 stat = os.stat(cached_name) 1246 except FileNotFoundError: 1247 return None 1248 _log.debug('cache hit: [%s] -> [%s] (%s)', cache_key_data, cached_name, stat) 1249 if os.path.islink(cached_name) or (not os.path.isfile(cached_name)): 1250 _log.error('object in cache is not a regular file: %s', cached_name) 1251 _log.error('possibly an attack, removing') 1252 if gmTools.remove_file(cached_name, log_error = True): 1253 return None 1254 raise Exception('cannot delete suspicious object in cache dir: %s', cached_name) 1255 if stat.st_size == data_size: 1256 return cached_name 1257 _log.debug('size in cache [%s] <> expected size [%s], removing cached file', stat.st_size, data_size) 1258 if gmTools.remove_file(cached_name, log_error = True): 1259 return None 1260 raise Exception('cannot remove suspicous object from cache dir: %s', cached_name)
1261 1262 #------------------------------------------------------------------------
1263 -def __get_file_from_cache(filename, cache_key_data=None, data_size=None, link2cached=True):
1264 """Get file from cache if available.""" 1265 cached_filename = __get_filename_in_cache(cache_key_data = cache_key_data, data_size = data_size) 1266 if cached_filename is None: 1267 return False 1268 if link2cached: 1269 try: 1270 # (hard)link as desired name, quite a few security 1271 # and archival tools refuse to handle softlinks 1272 os.link(cached_filename, filename) 1273 _log.debug('hardlinked [%s] as [%s]', cached_filename, filename) 1274 return True 1275 except Exception: 1276 pass 1277 _log.debug('cannot hardlink to cache, trying copy-from-cache') 1278 # copy from cache 1279 try: 1280 shutil.copyfile(cached_filename, filename, follow_symlinks = True) 1281 return True 1282 except shutil.SameFileError: 1283 # flaky - might be same name but different content 1284 pass 1285 except OSError: 1286 _log.exception('cannot copy cached file [%s] into [%s]', cached_filename, filename) 1287 # if cache fails entirely -> fall through to new file 1288 _log.debug('downloading new copy of file, despite found in cache') 1289 return False
1290 1291 #------------------------------------------------------------------------
1292 -def bytea2file(data_query=None, filename=None, chunk_size=0, data_size=None, data_size_query=None, conn=None, link2cached=True):
1293 1294 if data_size is None: 1295 rows, idx = run_ro_queries(link_obj = conn, queries = [data_size_query]) 1296 data_size = rows[0][0] 1297 if data_size in [None, 0]: 1298 conn.rollback() 1299 # should an empty file be created if size == 0 ? 1300 return True 1301 1302 # actually needs to get values from <conn> or "default conn" if <conn> is None 1303 cache_key_data = '<%s>@%s:%s/%s::%s' % ( 1304 _default_login.user, 1305 _default_login.host, 1306 _default_login.port, 1307 _default_login.database, 1308 data_query 1309 ) 1310 found_in_cache = __get_file_from_cache(filename, cache_key_data = cache_key_data, data_size = data_size, link2cached = link2cached) 1311 if found_in_cache: 1312 return True 1313 1314 outfile = io.open(filename, 'wb') 1315 result = bytea2file_object ( 1316 data_query = data_query, 1317 file_obj = outfile, 1318 chunk_size = chunk_size, 1319 data_size = data_size, 1320 data_size_query = data_size_query, 1321 conn = conn 1322 ) 1323 outfile.close() 1324 __store_file_in_cache(filename, cache_key_data) 1325 return result
1326 1327 #------------------------------------------------------------------------
1328 -def bytea2file_object(data_query=None, file_obj=None, chunk_size=0, data_size=None, data_size_query=None, conn=None):
1329 """Store data from a bytea field into a file. 1330 1331 <data_query> 1332 - dict {'cmd': ..., 'args': ...} 1333 - 'cmd' must be a string containing "... substring(data from %(start)s for %(size)s) ..." 1334 - 'args' must be a dict 1335 - must return one row with one field of type bytea 1336 <file> 1337 - must be a file like Python object 1338 <data_size> 1339 - integer of the total size of the expected data or None 1340 <data_size_query> 1341 - dict {'cmd': ..., 'args': ...} 1342 - must return one row with one field with the octet_length() of the data field 1343 - used only when <data_size> is None 1344 """ 1345 if data_size == 0: 1346 return True 1347 1348 # If the client sets an encoding other than the default we 1349 # will receive encoding-parsed data which isn't the binary 1350 # content we want. Hence we need to get our own connection. 1351 # It must be a read-write one so that we don't affect the 1352 # encoding for other users of the shared read-only 1353 # connections. 1354 # Actually, encodings shouldn't be applied to binary data 1355 # (eg. bytea types) in the first place but that is only 1356 # reported to be fixed > v7.4. 1357 # further tests reveal that at least on PG 8.0 this bug still 1358 # manifests itself 1359 if conn is None: 1360 conn = get_raw_connection(readonly = True) 1361 1362 if data_size is None: 1363 rows, idx = run_ro_queries(link_obj = conn, queries = [data_size_query]) 1364 data_size = rows[0][0] 1365 if data_size in [None, 0]: 1366 conn.rollback() 1367 return True 1368 1369 max_chunk_size = 1024 * 1024 * 20 # 20 MB, works for typical CR DICOMs 1370 if chunk_size == 0: 1371 chunk_size = min(data_size, max_chunk_size) 1372 1373 _log.debug('expecting %s bytes of BYTEA data in chunks of %s bytes', data_size, chunk_size) 1374 1375 # Windoze sucks: it can't transfer objects of arbitrary size, 1376 # anyways, we need to split the transfer, 1377 # however, only possible if postgres >= 7.2 1378 needed_chunks, remainder = divmod(data_size, chunk_size) 1379 _log.debug('# of chunks: %s; remainder: %s bytes', needed_chunks, remainder) 1380 1381 # retrieve chunks, skipped if data size < chunk size, 1382 # does this not carry the danger of cutting up multi-byte escape sequences ? 1383 # no, since bytea is binary, 1384 # yes, since in bytea there are *some* escaped values, still 1385 # no, since those are only escaped during *transfer*, not on-disk, hence 1386 # only complete escape sequences are put on the wire 1387 for chunk_id in range(needed_chunks): 1388 chunk_start = (chunk_id * chunk_size) + 1 1389 data_query['args']['start'] = chunk_start 1390 data_query['args']['size'] = chunk_size 1391 try: 1392 rows, idx = run_ro_queries(link_obj=conn, queries=[data_query]) 1393 except: 1394 _log.error('cannot retrieve chunk [%s/%s], size [%s], try decreasing chunk size' % (chunk_id+1, needed_chunks, chunk_size)) 1395 conn.rollback() 1396 raise 1397 # it would be a fatal error to see more than one result as ids are supposed to be unique 1398 file_obj.write(rows[0][0]) 1399 1400 # retrieve remainder 1401 if remainder > 0: 1402 chunk_start = (needed_chunks * chunk_size) + 1 1403 data_query['args']['start'] = chunk_start 1404 data_query['args']['size'] = remainder 1405 try: 1406 rows, idx = run_ro_queries(link_obj=conn, queries=[data_query]) 1407 except: 1408 _log.error('cannot retrieve remaining [%s] bytes' % remainder) 1409 conn.rollback() 1410 raise 1411 # it would be a fatal error to see more than one result as ids are supposed to be unique 1412 file_obj.write(rows[0][0]) 1413 1414 conn.rollback() 1415 return True
1416 1417 #------------------------------------------------------------------------
1418 -def file2bytea(query=None, filename=None, args=None, conn=None, file_md5=None):
1419 """Store data from a file into a bytea field. 1420 1421 The query must: 1422 - be in unicode 1423 - contain a format spec identifying the row (eg a primary key) 1424 matching <args> if it is an UPDATE 1425 - contain a format spec " <field> = %(data)s::bytea" 1426 1427 The query CAN return the MD5 of the inserted data: 1428 RETURNING md5(<field>) AS md5 1429 in which case it will compare it to the md5 1430 of the file. 1431 """ 1432 # read data from file 1433 infile = open(filename, "rb") 1434 data_as_byte_string = infile.read() 1435 infile.close() 1436 if args is None: 1437 args = {} 1438 # really still needed for BYTEA input ? 1439 args['data'] = memoryview(data_as_byte_string) 1440 del(data_as_byte_string) 1441 1442 # insert the data 1443 if conn is None: 1444 conn = get_raw_connection(readonly = False) 1445 close_conn = True 1446 else: 1447 close_conn = False 1448 1449 rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = (file_md5 is not None)) 1450 1451 success_status = True 1452 if file_md5 is None: 1453 conn.commit() 1454 else: 1455 db_md5 = rows[0]['md5'] 1456 if file_md5 != db_md5: 1457 conn.rollback() 1458 success_status = False 1459 _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5) 1460 else: 1461 conn.commit() 1462 _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5) 1463 1464 if close_conn: 1465 conn.close() 1466 1467 return success_status
1468 1469 #------------------------------------------------------------------------
1470 -def file2lo(filename=None, conn=None, check_md5=False):
1471 # 1 GB limit unless 64 bit Python build ... 1472 file_size = os.path.getsize(filename) 1473 if file_size > (1024 * 1024) * 1024: 1474 _log.debug('file size of [%s] > 1 GB, supposedly not supported by psycopg2 large objects (but seems to work anyway ?)', file_size) 1475 # return -1 1476 1477 if conn is None: 1478 conn = get_raw_connection(readonly = False) 1479 close_conn = conn.close 1480 else: 1481 close_conn = __noop 1482 _log.debug('[%s] -> large object', filename) 1483 1484 # insert the data 1485 lo = conn.lobject(0, 'w', 0, filename) 1486 lo_oid = lo.oid 1487 lo.close() 1488 _log.debug('large object OID: %s', lo_oid) 1489 1490 # verify 1491 if file_md5 is None: 1492 conn.commit() 1493 close_conn() 1494 return lo_oid 1495 cmd = 'SELECT md5(lo_get(%(loid)s::oid))' 1496 args = {'loid': lo_oid} 1497 rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': cmd, 'args': args}]) 1498 db_md5 = rows[0][0] 1499 if file_md5 == db_md5: 1500 conn.commit() 1501 close_conn() 1502 _log.debug('MD5 sums of data file and database large object match: [file::%s] = [DB::%s]', file_md5, db_md5) 1503 return lo_oid 1504 conn.rollback() 1505 close_conn() 1506 _log.error('MD5 sums of data file and database large object [%s] do not match: [file::%s] <> [DB::%s]', lo_oid, file_md5, db_md5) 1507 return -1
1508 1509 #------------------------------------------------------------------------
1510 -def file2bytea_lo(filename=None, conn=None, file_md5=None):
1511 # 1 GB limit unless 64 bit Python build ... 1512 file_size = os.path.getsize(filename) 1513 if file_size > (1024 * 1024) * 1024: 1514 _log.debug('file size of [%s] > 1 GB, supposedly not supported by psycopg2 large objects (but seems to work anyway ?)', file_size) 1515 # return -1 1516 1517 if conn is None: 1518 conn = get_raw_connection(readonly = False) 1519 close_conn = conn.close 1520 else: 1521 close_conn = __noop 1522 _log.debug('[%s] -> large object', filename) 1523 1524 # insert the data 1525 lo = conn.lobject(0, 'w', 0, filename) 1526 lo_oid = lo.oid 1527 lo.close() 1528 _log.debug('large object OID: %s', lo_oid) 1529 1530 # verify 1531 if file_md5 is None: 1532 conn.commit() 1533 close_conn() 1534 return lo_oid 1535 cmd = 'SELECT md5(lo_get(%(loid)s::oid))' 1536 args = {'loid': lo_oid} 1537 rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': cmd, 'args': args}]) 1538 db_md5 = rows[0][0] 1539 if file_md5 == db_md5: 1540 conn.commit() 1541 close_conn() 1542 _log.debug('MD5 sums of data file and database large object match: [file::%s] = [DB::%s]', file_md5, db_md5) 1543 return lo_oid 1544 conn.rollback() 1545 close_conn() 1546 _log.error('MD5 sums of data file and database large object [%s] do not match: [file::%s] <> [DB::%s]', lo_oid, file_md5, db_md5) 1547 return -1
1548 1549 #------------------------------------------------------------------------
1550 -def file2bytea_copy_from(table=None, columns=None, filename=None, conn=None, md5_query=None, file_md5=None):
1551 # md5_query: dict{'cmd': ..., 'args': ...} 1552 1553 # UNTESTED 1554 1555 chunk_size = 32 * (1024 * 1024) 1556 _log.debug('[%s] (%s bytes) --(%s bytes)-> %s(%s)', filename, os.path.getsize(filename), chunk_size, table, columns) 1557 if conn is None: 1558 conn = get_raw_connection(readonly = False) 1559 close_conn = True 1560 else: 1561 close_conn = False 1562 curs = conn.cursor() 1563 # write 1564 infile = open(filename, "rb") 1565 curs.copy_from(infile, table, size = chunk_size, columns = columns) 1566 infile.close() 1567 curs.close() 1568 if None in [file_md5, md5_query]: 1569 conn.commit() 1570 close_conn() 1571 return True 1572 # verify 1573 rows, idx = run_ro_queries(link_obj = conn, queries = [md5_query]) 1574 db_md5 = rows[0][0] 1575 if file_md5 == db_md5: 1576 conn.commit() 1577 close_conn() 1578 _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5) 1579 return True 1580 close_conn() 1581 _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5) 1582 return False
1583 1584 #------------------------------------------------------------------------
1585 -def file2bytea_overlay(query=None, args=None, filename=None, conn=None, md5_query=None, file_md5=None):
1586 """Store data from a file into a bytea field. 1587 1588 The query must: 1589 - 'cmd' must be in unicode 1590 - 'cmd' must contain a format spec identifying the row (eg 1591 a primary key) matching <args> if it is an UPDATE 1592 - 'cmd' must contain "... SET ... <some_bytea_field> = OVERLAY(some_bytea_field PLACING %(data)s::bytea FROM %(start)s FOR %(size)s) ..." 1593 - 'args' must be a dict matching 'cmd' 1594 1595 The query CAN return the MD5 of the inserted data: 1596 RETURNING md5(<field>) AS md5 1597 in which case it will compare it to the md5 1598 of the file. 1599 1600 UPDATE 1601 the_table 1602 SET 1603 bytea_field = OVERLAY ( 1604 coalesce(bytea_field, '':bytea), 1605 PLACING 1606 %(data)s::bytea 1607 FROM 1608 %(start)s 1609 FOR 1610 %(size)s 1611 ) 1612 WHERE 1613 primary_key = pk_value 1614 1615 SELECT md5(bytea_field) FROM the_table WHERE primary_key = pk_value 1616 """ 1617 chunk_size = 32 * (1024 * 1024) 1618 file_size = os.path.getsize(filename) 1619 if file_size <= chunk_size: 1620 chunk_size = file_size 1621 needed_chunks, remainder = divmod(file_size, chunk_size) 1622 _log.debug('file data: %s bytes, chunks: %s, chunk size: %s bytes, remainder: %s bytes', file_size, needed_chunks, chunk_size, remainder) 1623 1624 if conn is None: 1625 conn = get_raw_connection(readonly = False) 1626 close_conn = conn.close 1627 else: 1628 close_conn = __noop 1629 1630 infile = open(filename, "rb") 1631 # write chunks 1632 for chunk_id in range(needed_chunks): 1633 chunk_start = (chunk_id * chunk_size) + 1 1634 args['start'] = chunk_start 1635 args['size'] = chunk_size 1636 data_as_byte_string = infile.read(chunk_size) 1637 # really still needed for BYTEA input ? 1638 args['data'] = memoryview(data_as_byte_string) 1639 del(data_as_byte_string) 1640 try: 1641 rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = False) 1642 except Exception: 1643 _log.exception('cannot write chunk [%s/%s] of size [%s], try decreasing chunk size', chunk_id+1, needed_chunks, chunk_size) 1644 conn.rollback() 1645 close_conn() 1646 infile.close() 1647 raise 1648 # write remainder 1649 if remainder > 0: 1650 chunk_start = (needed_chunks * chunk_size) + 1 1651 args['start'] = chunk_start 1652 args['size'] = remainder 1653 data_as_byte_string = infile.read(remainder) 1654 # really still needed for BYTEA input ? 1655 args['data'] = memoryview(data_as_byte_string) 1656 del(data_as_byte_string) 1657 try: 1658 rows, idx = run_rw_queries(link_obj = conn, queries = [{'cmd': query, 'args': args}], end_tx = False, return_data = False) 1659 except Exception: 1660 _log.error('cannot retrieve remaining [%s] bytes' % remainder) 1661 conn.rollback() 1662 close_conn() 1663 infile.close() 1664 raise 1665 infile.close() 1666 if None in [file_md5, md5_query]: 1667 conn.commit() 1668 close_conn() 1669 return True 1670 # verify 1671 rows, idx = run_ro_queries(link_obj = conn, queries = [{'cmd': md5_query, 'args': args}]) 1672 db_md5 = rows[0][0] 1673 if file_md5 == db_md5: 1674 conn.commit() 1675 close_conn() 1676 _log.debug('MD5 sums of data file and database BYTEA field match: [file::%s] = [DB::%s]', file_md5, db_md5) 1677 return True 1678 close_conn() 1679 _log.error('MD5 sums of data file and database BYTEA field do not match: [file::%s] <> [DB::%s]', file_md5, db_md5) 1680 return False
1681 1682 #---------------------------------------------------------------------------
1683 -def run_sql_script(sql_script, conn=None):
1684 1685 if conn is None: 1686 conn = get_connection(readonly = False) 1687 1688 from Gnumed.pycommon import gmPsql 1689 psql = gmPsql.Psql(conn) 1690 1691 if psql.run(sql_script) == 0: 1692 query = { 1693 'cmd': 'select gm.log_script_insertion(%(name)s, %(ver)s)', 1694 'args': {'name': sql_script, 'ver': 'current'} 1695 } 1696 run_rw_queries(link_obj = conn, queries = [query]) 1697 conn.commit() 1698 return True 1699 1700 _log.error('error running sql script: %s', sql_script) 1701 return False
1702 1703 #------------------------------------------------------------------------
1704 -def sanitize_pg_regex(expression=None, escape_all=False):
1705 """Escape input for use in a PostgreSQL regular expression. 1706 1707 If a fragment comes from user input and is to be used 1708 as a regular expression we need to make sure it doesn't 1709 contain invalid regex patterns such as unbalanced ('s. 1710 1711 <escape_all> 1712 True: try to escape *all* metacharacters 1713 False: only escape those which render the regex invalid 1714 """ 1715 return expression.replace ( 1716 '(', '\(' 1717 ).replace ( 1718 ')', '\)' 1719 ).replace ( 1720 '[', '\[' 1721 ).replace ( 1722 '+', '\+' 1723 ).replace ( 1724 '.', '\.' 1725 ).replace ( 1726 '*', '\*' 1727 )
1728 #']', '\]', # not needed 1729 1730 #------------------------------------------------------------------------
1731 -def capture_conn_state(conn=None):
1732 1733 tx_status = conn.get_transaction_status() 1734 if tx_status in [ psycopg2.extensions.TRANSACTION_STATUS_INERROR, psycopg2.extensions.TRANSACTION_STATUS_UNKNOWN ]: 1735 isolation_level = '%s (tx aborted or unknown, cannot retrieve)' % conn.isolation_level 1736 else: 1737 isolation_level = '%s (%s)' % (conn.isolation_level, map_psyco_iso_level2str[conn.isolation_level]) 1738 conn_status = '%s (%s)' % (conn.status, map_psyco_conn_status2str[conn.status]) 1739 if conn.closed != 0: 1740 conn_status = 'undefined (%s)' % conn_status 1741 backend_pid = '<conn closed, cannot retrieve>' 1742 else: 1743 backend_pid = conn.get_backend_pid() 1744 try: 1745 conn_deferrable = conn.deferrable 1746 except AttributeError: 1747 conn_deferrable = '<unavailable>' 1748 1749 d = { 1750 'identity': id(conn), 1751 'backend PID': backend_pid, 1752 'protocol version': conn.protocol_version, 1753 'encoding': conn.encoding, 1754 'closed': conn.closed, 1755 'readonly': conn.readonly, 1756 'autocommit': conn.autocommit, 1757 'isolation level (psyco)': isolation_level, 1758 'async': conn.async_, 1759 'deferrable': conn_deferrable, 1760 'transaction status': '%s (%s)' % (tx_status, map_psyco_tx_status2str[tx_status]), 1761 'connection status': conn_status, 1762 'executing async op': conn.isexecuting(), 1763 'type': type(conn) 1764 } 1765 return '%s\n' % conn + format_dict_like ( 1766 d, 1767 relevant_keys = [ 1768 'type', 1769 'identity', 1770 'backend PID', 1771 'protocol version', 1772 'encoding', 1773 'isolation level (psyco)', 1774 'readonly', 1775 'autocommit', 1776 'closed', 1777 'connection status', 1778 'transaction status', 1779 'deferrable', 1780 'async', 1781 'executing async op' 1782 ], 1783 tabular = True, 1784 value_delimiters = None 1785 )
1786 1787 #------------------------------------------------------------------------
1788 -def capture_cursor_state(cursor=None):
1789 conn = cursor.connection 1790 1791 tx_status = conn.get_transaction_status() 1792 if tx_status in [ psycopg2.extensions.TRANSACTION_STATUS_INERROR, psycopg2.extensions.TRANSACTION_STATUS_UNKNOWN ]: 1793 isolation_level = '<tx aborted or unknown, cannot retrieve>' 1794 else: 1795 isolation_level = conn.isolation_level 1796 try: 1797 conn_deferrable = conn.deferrable 1798 except AttributeError: 1799 conn_deferrable = '<unavailable>' 1800 1801 if cursor.query is None: 1802 query = '<no query>' 1803 else: 1804 query = cursor.query.decode(errors = 'replace') 1805 1806 if conn.closed != 0: 1807 backend_pid = '<conn closed, cannot retrieve>' 1808 else: 1809 backend_pid = conn.get_backend_pid() 1810 1811 txt = """Link state: 1812 Cursor 1813 identity: %s; name: %s 1814 closed: %s; scrollable: %s; with hold: %s; arraysize: %s; itersize: %s; 1815 last rowcount: %s; rownumber: %s; lastrowid (OID): %s; 1816 last description: %s 1817 statusmessage: %s 1818 Connection 1819 identity: %s; backend pid: %s; protocol version: %s; 1820 closed: %s; autocommit: %s; isolation level: %s; encoding: %s; async: %s; deferrable: %s; readonly: %s; 1821 TX status: %s; CX status: %s; executing async op: %s; 1822 Query 1823 %s 1824 """ % ( 1825 id(cursor), 1826 cursor.name, 1827 cursor.closed, 1828 cursor.scrollable, 1829 cursor.withhold, 1830 cursor.arraysize, 1831 cursor.itersize, 1832 cursor.rowcount, 1833 cursor.rownumber, 1834 cursor.lastrowid, 1835 cursor.description, 1836 cursor.statusmessage, 1837 1838 id(conn), 1839 backend_pid, 1840 conn.protocol_version, 1841 conn.closed, 1842 conn.autocommit, 1843 isolation_level, 1844 conn.encoding, 1845 conn.async_, 1846 conn_deferrable, 1847 conn.readonly, 1848 map_psyco_tx_status2str[tx_status], 1849 map_psyco_conn_status2str[conn.status], 1850 conn.isexecuting(), 1851 1852 query 1853 ) 1854 return txt
1855 1856 #------------------------------------------------------------------------
1857 -def run_ro_queries(link_obj=None, queries=None, verbose=False, return_data=True, get_col_idx=False):
1858 """Run read-only queries. 1859 1860 <queries> must be a list of dicts: 1861 [ 1862 {'cmd': <string>, 'args': <dict> or <tuple>}, 1863 {...}, 1864 ... 1865 ] 1866 """ 1867 if isinstance(link_obj, dbapi._psycopg.cursor): 1868 curs = link_obj 1869 curs_close = lambda :1 1870 tx_rollback = lambda :1 1871 readonly_rollback_just_in_case = lambda :1 1872 elif isinstance(link_obj, dbapi._psycopg.connection): 1873 curs = link_obj.cursor() 1874 curs_close = curs.close 1875 tx_rollback = link_obj.rollback 1876 if link_obj.autocommit is True: # readonly connection ? 1877 readonly_rollback_just_in_case = link_obj.rollback 1878 else: 1879 # do not rollback readonly queries on passed-in readwrite 1880 # connections just in case because they may have already 1881 # seen fully legitimate write action which would get lost 1882 readonly_rollback_just_in_case = lambda :1 1883 elif link_obj is None: 1884 conn = get_connection(readonly = True, verbose = verbose) 1885 curs = conn.cursor() 1886 curs_close = curs.close 1887 tx_rollback = conn.rollback 1888 readonly_rollback_just_in_case = conn.rollback 1889 else: 1890 raise ValueError('link_obj must be cursor, connection or None but not [%s]' % link_obj) 1891 1892 if verbose: 1893 _log.debug('cursor: %s', curs) 1894 1895 for query in queries: 1896 try: 1897 args = query['args'] 1898 except KeyError: 1899 args = None 1900 try: 1901 curs.execute(query['cmd'], args) 1902 if verbose: 1903 _log.debug(capture_cursor_state(curs)) 1904 except PG_ERROR_EXCEPTION as pg_exc: 1905 _log.error('query failed in RO connection') 1906 log_pg_exception_details(pg_exc) 1907 try: 1908 curs_close() 1909 except PG_ERROR_EXCEPTION as pg_exc2: 1910 _log.exception('cannot close cursor') 1911 log_pg_exception_details(pg_exc2) 1912 try: 1913 tx_rollback() # need to rollback so ABORT state isn't preserved in pooled conns 1914 except PG_ERROR_EXCEPTION as pg_exc2: 1915 _log.exception('cannot rollback transaction') 1916 log_pg_exception_details(pg_exc2) 1917 if pg_exc.pgcode == sql_error_codes.INSUFFICIENT_PRIVILEGE: 1918 details = 'Query: [%s]' % curs.query.strip().strip('\n').strip().strip('\n') 1919 if curs.statusmessage != '': 1920 details = 'Status: %s\n%s' % ( 1921 curs.statusmessage.strip().strip('\n').strip().strip('\n'), 1922 details 1923 ) 1924 if pg_exc.pgerror is None: 1925 msg = '[%s]' % pg_exc.pgcode 1926 else: 1927 msg = '[%s]: %s' % (pg_exc.pgcode, pg_exc.pgerror) 1928 raise gmExceptions.AccessDenied ( 1929 msg, 1930 source = 'PostgreSQL', 1931 code = pg_exc.pgcode, 1932 details = details 1933 ) 1934 raise 1935 except Exception: 1936 _log.exception('error during query run in RO connection') 1937 _log.error(capture_cursor_state(curs)) 1938 try: 1939 curs_close() 1940 except PG_ERROR_EXCEPTION as pg_exc: 1941 _log.exception('cannot close cursor') 1942 log_pg_exception_details(pg_exc) 1943 try: 1944 tx_rollback() # need to rollback so ABORT state isn't preserved in pooled conns 1945 except PG_ERROR_EXCEPTION as pg_exc: 1946 _log.exception('cannot rollback transation') 1947 log_pg_exception_details(pg_exc) 1948 raise 1949 1950 data = None 1951 col_idx = None 1952 if return_data: 1953 data = curs.fetchall() 1954 if verbose: 1955 _log.debug('last query returned [%s (%s)] rows', curs.rowcount, len(data)) 1956 _log.debug('cursor description: %s', curs.description) 1957 if get_col_idx: 1958 col_idx = get_col_indices(curs) 1959 1960 curs_close() 1961 # so we can see data committed meanwhile if the 1962 # link object had been passed in and thusly might 1963 # be part of a long-running read-only transaction 1964 readonly_rollback_just_in_case() 1965 return (data, col_idx)
1966 1967 #------------------------------------------------------------------------
1968 -def run_rw_queries(link_obj=None, queries=None, end_tx=False, return_data=None, get_col_idx=False, verbose=False):
1969 """Convenience function for running a transaction 1970 that is supposed to get committed. 1971 1972 <link_obj> 1973 can be either: 1974 - a cursor 1975 - a connection 1976 1977 <queries> 1978 is a list of dicts [{'cmd': <string>, 'args': <dict> or <tuple>) 1979 to be executed as a single transaction, the last 1980 query may usefully return rows (such as a 1981 "SELECT currval('some_sequence')" statement) 1982 1983 <end_tx> 1984 - controls whether the transaction is finalized (eg. 1985 committed/rolled back) or not, this allows the 1986 call to run_rw_queries() to be part of a framing 1987 transaction 1988 - if link_obj is a connection then <end_tx> will 1989 default to False unless it is explicitly set to 1990 True which is taken to mean "yes, you do have full 1991 control over the transaction" in which case the 1992 transaction is properly finalized 1993 - if link_obj is a cursor we CANNOT finalize the 1994 transaction because we would need the connection for that 1995 - if link_obj is None <end_tx> will, of course, always be True 1996 1997 <return_data> 1998 - if true, the returned data will include the rows 1999 the last query selected 2000 - if false, it returns None instead 2001 2002 <get_col_idx> 2003 - if true, the returned data will include a dictionary 2004 mapping field names to column positions 2005 - if false, the returned data returns None instead 2006 2007 method result: 2008 - returns a tuple (data, idx) 2009 - <data>: 2010 * (None, None) if last query did not return rows 2011 * ("fetchall() result", <index>) if last query returned any rows 2012 * for <index> see <get_col_idx> 2013 """ 2014 if isinstance(link_obj, dbapi._psycopg.cursor): 2015 conn_close = lambda :1 2016 conn_commit = lambda :1 2017 tx_rollback = lambda :1 2018 curs = link_obj 2019 curs_close = lambda :1 2020 notices_accessor = curs.connection 2021 elif isinstance(link_obj, dbapi._psycopg.connection): 2022 conn_close = lambda :1 2023 if end_tx: 2024 conn_commit = link_obj.commit 2025 tx_rollback = link_obj.rollback 2026 else: 2027 conn_commit = lambda :1 2028 tx_rollback = lambda :1 2029 curs = link_obj.cursor() 2030 curs_close = curs.close 2031 notices_accessor = link_obj 2032 elif link_obj is None: 2033 conn = get_connection(readonly=False) 2034 conn_close = conn.close 2035 conn_commit = conn.commit 2036 tx_rollback = conn.rollback 2037 curs = conn.cursor() 2038 curs_close = curs.close 2039 notices_accessor = conn 2040 else: 2041 raise ValueError('link_obj must be cursor, connection or None but not [%s]' % link_obj) 2042 2043 for query in queries: 2044 try: 2045 args = query['args'] 2046 except KeyError: 2047 args = None 2048 try: 2049 curs.execute(query['cmd'], args) 2050 if verbose: 2051 _log.debug(capture_cursor_state(curs)) 2052 for notice in notices_accessor.notices: 2053 _log.debug(notice.replace('\n', '/').replace('\n', '/')) 2054 del notices_accessor.notices[:] 2055 # DB related exceptions 2056 except dbapi.Error as pg_exc: 2057 _log.error('query failed in RW connection') 2058 log_pg_exception_details(pg_exc) 2059 for notice in notices_accessor.notices: 2060 _log.debug(notice.replace('\n', '/').replace('\n', '/')) 2061 del notices_accessor.notices[:] 2062 try: 2063 curs_close() 2064 except PG_ERROR_EXCEPTION as pg_exc2: 2065 _log.exception('cannot close cursor') 2066 log_pg_exception_details(pg_exc2) 2067 try: 2068 tx_rollback() # need to rollback so ABORT state isn't preserved in pooled conns 2069 except PG_ERROR_EXCEPTION as pg_exc2: 2070 _log.exception('cannot rollback transaction') 2071 log_pg_exception_details(pg_exc2) 2072 # privilege problem 2073 if pg_exc.pgcode == sql_error_codes.INSUFFICIENT_PRIVILEGE: 2074 details = 'Query: [%s]' % curs.query.strip().strip('\n').strip().strip('\n') 2075 if curs.statusmessage != '': 2076 details = 'Status: %s\n%s' % ( 2077 curs.statusmessage.strip().strip('\n').strip().strip('\n'), 2078 details 2079 ) 2080 if pg_exc.pgerror is None: 2081 msg = '[%s]' % pg_exc.pgcode 2082 else: 2083 msg = '[%s]: %s' % (pg_exc.pgcode, pg_exc.pgerror) 2084 try: 2085 curs_close() 2086 tx_rollback() # just for good measure 2087 conn_close() 2088 except dbapi.InterfaceError: 2089 _log.exception('cannot cleanup') 2090 raise gmExceptions.AccessDenied ( 2091 msg, 2092 source = 'PostgreSQL', 2093 code = pg_exc.pgcode, 2094 details = details 2095 ) 2096 # other problem 2097 gmLog2.log_stack_trace() 2098 try: 2099 curs_close() 2100 tx_rollback() # just for good measure 2101 conn_close() 2102 except dbapi.InterfaceError: 2103 _log.exception('cannot cleanup') 2104 raise 2105 # other exception 2106 except Exception: 2107 _log.exception('error running query in RW connection') 2108 _log.error(capture_cursor_state(curs)) 2109 for notice in notices_accessor.notices: 2110 _log.debug(notice.replace('\n', '/').replace('\n', '/')) 2111 del notices_accessor.notices[:] 2112 gmLog2.log_stack_trace() 2113 try: 2114 curs_close() 2115 except PG_ERROR_EXCEPTION as pg_exc: 2116 _log.exception('cannot close cursor') 2117 log_pg_exception_details(pg_exc) 2118 try: 2119 tx_rollback() # need to rollback so ABORT state isn't preserved in pooled conns 2120 conn_close() 2121 except PG_ERROR_EXCEPTION as pg_exc: 2122 _log.exception('cannot rollback transation') 2123 log_pg_exception_details(pg_exc) 2124 raise 2125 2126 data = None 2127 col_idx = None 2128 if return_data: 2129 try: 2130 data = curs.fetchall() 2131 except Exception: 2132 _log.exception('error fetching data from RW query') 2133 gmLog2.log_stack_trace() 2134 try: 2135 curs_close() 2136 tx_rollback() 2137 conn_close() 2138 except dbapi.InterfaceError: 2139 _log.exception('cannot cleanup') 2140 raise 2141 raise 2142 if get_col_idx: 2143 col_idx = get_col_indices(curs) 2144 2145 curs_close() 2146 conn_commit() 2147 conn_close() 2148 2149 return (data, col_idx)
2150 2151 #------------------------------------------------------------------------
2152 -def run_insert(link_obj=None, schema=None, table=None, values=None, returning=None, end_tx=False, get_col_idx=False, verbose=False):
2153 """Generates SQL for an INSERT query. 2154 2155 values: dict of values keyed by field to insert them into 2156 """ 2157 if schema is None: 2158 schema = 'public' 2159 2160 fields = values.keys() # that way val_snippets and fields really should end up in the same order 2161 val_snippets = [] 2162 for field in fields: 2163 val_snippets.append('%%(%s)s' % field) 2164 2165 if returning is None: 2166 returning = '' 2167 return_data = False 2168 else: 2169 returning = '\n\tRETURNING\n\t\t%s' % ', '.join(returning) 2170 return_data = True 2171 2172 cmd = """\nINSERT INTO %s.%s ( 2173 %s 2174 ) VALUES ( 2175 %s 2176 )%s""" % ( 2177 schema, 2178 table, 2179 ',\n\t\t'.join(fields), 2180 ',\n\t\t'.join(val_snippets), 2181 returning 2182 ) 2183 2184 _log.debug('running SQL: >>>%s<<<', cmd) 2185 2186 return run_rw_queries ( 2187 link_obj = link_obj, 2188 queries = [{'cmd': cmd, 'args': values}], 2189 end_tx = end_tx, 2190 return_data = return_data, 2191 get_col_idx = get_col_idx, 2192 verbose = verbose 2193 )
2194 2195 # ======================================================================= 2196 # connection handling API 2197 # -----------------------------------------------------------------------
2198 -class cConnectionPool(psycopg2.pool.PersistentConnectionPool):
2199 """GNUmed database connection pool. 2200 2201 Extends psycopg2's ThreadedConnectionPool with 2202 a custom _connect() function. Supports one connection 2203 per thread - which also ties it to one particular DSN.""" 2204 #--------------------------------------------------
2205 - def _connect(self, key=None):
2206 _log.debug('conn request with key [%s]', key) 2207 conn = get_raw_connection(dsn = self._kwargs['dsn'], verbose = self._kwargs['verbose'], readonly = True) 2208 # monkey patching close() 2209 conn.original_close = conn.close 2210 conn.close = _raise_exception_on_ro_conn_close 2211 if key is not None: 2212 self._used[key] = conn 2213 self._rused[id(conn)] = key 2214 else: 2215 self._pool.append(conn) 2216 return conn
2217 2218 #--------------------------------------------------
2219 - def discard_connection(self, key=None):
2220 if key is None: 2221 key = threading.current_thread().ident 2222 try: 2223 conn = self._used[key] 2224 except KeyError: 2225 _log.error('no such key in connection pool: %s', key) 2226 _log.debug('available keys: %s', self._used.keys()) 2227 return 2228 del self._used[key] 2229 del self._rused[id(conn)] 2230 conn.original_close()
2231 2232 #--------------------------------------------------
2233 - def shutdown(self):
2234 for conn_key in self._used.keys(): 2235 conn = self._used[conn_key] 2236 if conn.closed != 0: 2237 continue 2238 _log.debug('closing pooled database connection, pool key: %s, backend PID: %s', conn_key, self._used[conn_key].get_backend_pid()) 2239 conn.original_close()
2240 2241 # -----------------------------------------------------------------------
2242 -def get_raw_connection(dsn=None, verbose=False, readonly=True, connection_name=None, autocommit=False):
2243 """Get a raw, unadorned connection. 2244 2245 - this will not set any parameters such as encoding, timezone, datestyle 2246 - the only requirement is a valid DSN 2247 - hence it can be used for "service" connections 2248 for verifying encodings etc 2249 """ 2250 # FIXME: support verbose 2251 if dsn is None: 2252 dsn = get_default_dsn() 2253 2254 if 'host=salaam.homeunix' in dsn: 2255 raise ValueError('The public database is not hosted by <salaam.homeunix.com> anymore.\n\nPlease point your configuration files to <publicdb.gnumed.de>.') 2256 2257 # try to enforce a useful encoding early on so that we 2258 # have a good chance of decoding authentication errors 2259 # containing foreign language characters 2260 if ' client_encoding=' not in dsn: 2261 dsn += ' client_encoding=utf8' 2262 2263 if ' application_name' not in dsn: 2264 if connection_name is None: 2265 dsn += " application_name=GNUmed-[%s]" % threading.current_thread().name.replace(' ', '_') 2266 else: 2267 dsn += " application_name=%s" % connection_name 2268 2269 try: 2270 # DictConnection now _is_ a real dictionary 2271 conn = dbapi.connect(dsn = dsn, connection_factory = psycopg2.extras.DictConnection) 2272 except dbapi.OperationalError as e: 2273 t, v, tb = sys.exc_info() 2274 try: 2275 msg = e.args[0] 2276 except (AttributeError, IndexError, TypeError): 2277 raise 2278 if 'fe_sendauth' in msg: 2279 raise cAuthenticationError(dsn, msg).with_traceback(tb) 2280 if regex.search('user ".*" does not exist', msg) is not None: 2281 raise cAuthenticationError(dsn, msg).with_traceback(tb) 2282 if (( (regex.search('user ".*"', msg) is not None) 2283 or 2284 (regex.search('(R|r)ol{1,2}e', msg) is not None) 2285 ) 2286 and ('exist' in msg) 2287 and (regex.search('n(o|ich)t', msg) is not None) 2288 ): 2289 raise cAuthenticationError(dsn, msg).with_traceback(tb) 2290 if regex.search('user ".*" does not exist', msg) is not None: 2291 raise cAuthenticationError(dsn, msg).with_traceback(tb) 2292 if 'uthenti' in msg: 2293 raise cAuthenticationError(dsn, msg).with_traceback(tb) 2294 raise 2295 2296 if connection_name is None: 2297 _log.debug('established anonymous database connection, backend PID: %s', conn.get_backend_pid()) 2298 else: 2299 _log.debug('established database connection "%s", backend PID: %s', connection_name, conn.get_backend_pid()) 2300 2301 # do first-connection-only stuff 2302 # - verify PG version 2303 global postgresql_version 2304 if postgresql_version is None: 2305 curs = conn.cursor() 2306 curs.execute(""" 2307 SELECT 2308 substring(setting, E'^\\\\d{1,2}\\\\.\\\\d{1,2}')::numeric AS version 2309 FROM 2310 pg_settings 2311 WHERE 2312 name = 'server_version' 2313 """) 2314 postgresql_version = curs.fetchone()['version'] 2315 _log.info('PostgreSQL version (numeric): %s' % postgresql_version) 2316 try: 2317 curs.execute("SELECT pg_size_pretty(pg_database_size(current_database()))") 2318 _log.info('database size: %s', curs.fetchone()[0]) 2319 except Exception: 2320 _log.exception('cannot get database size') 2321 finally: 2322 curs.close() 2323 conn.commit() 2324 if verbose: 2325 curs = conn.cursor() 2326 _log_PG_settings(curs = curs) 2327 curs.close() 2328 # - verify PG understands client time zone 2329 if _default_client_timezone is None: 2330 __detect_client_timezone(conn = conn) 2331 2332 # - set access mode 2333 if readonly: 2334 _log.debug('readonly: forcing autocommit=True to avoid <IDLE IN TRANSACTION>') 2335 autocommit = True 2336 else: 2337 _log.debug('autocommit is desired to be: %s', autocommit) 2338 2339 conn.commit() 2340 conn.autocommit = autocommit 2341 conn.readonly = readonly 2342 2343 # - assume verbose=True to mean we want debugging in the database, too 2344 if verbose: 2345 _log.debug('enabling <plpgsql.extra_warnings/_errors>') 2346 curs = conn.cursor() 2347 try: 2348 curs.execute("SET plpgsql.extra_warnings TO 'all'") 2349 curs.execute("SET plpgsql.extra_errors TO 'all'") 2350 except Exception: 2351 _log.exception('cannot enable <plpgsql.extra_warnings/_errors>') 2352 finally: 2353 curs.close() 2354 conn.commit() 2355 2356 conn.is_decorated = False 2357 return conn
2358 2359 # =======================================================================
2360 -def get_connection(dsn=None, readonly=True, encoding=None, verbose=False, pooled=True, connection_name=None, autocommit=False):
2361 """Get a new connection. 2362 2363 This assumes the locale system has been initialized 2364 unless an encoding is specified. 2365 """ 2366 # FIXME: support pooled on RW, too 2367 # FIXME: for now, support the default DSN only 2368 if pooled and readonly and (dsn is None): 2369 global __ro_conn_pool 2370 if __ro_conn_pool is None: 2371 log_ro_conn = True 2372 __ro_conn_pool = cConnectionPool ( 2373 minconn = 1, 2374 maxconn = 2, 2375 dsn = dsn, 2376 verbose = verbose 2377 ) 2378 else: 2379 log_ro_conn = False 2380 try: 2381 conn = __ro_conn_pool.getconn() 2382 except psycopg2.pool.PoolError: 2383 _log.exception('falling back to non-pooled connection') 2384 conn = get_raw_connection(dsn = dsn, verbose = verbose, readonly = readonly, connection_name = connection_name, autocommit = autocommit) 2385 log_ro_conn = True 2386 if log_ro_conn: 2387 [ _log.debug(line) for line in capture_conn_state(conn = conn).split('\n') ] 2388 else: 2389 conn = get_raw_connection(dsn = dsn, verbose = verbose, readonly = readonly, connection_name = connection_name, autocommit = autocommit) 2390 2391 if conn.is_decorated: 2392 return conn 2393 2394 if encoding is None: 2395 encoding = _default_client_encoding 2396 if encoding is None: 2397 encoding = gmI18N.get_encoding() 2398 _log.warning('client encoding not specified') 2399 _log.warning('the string encoding currently set in the active locale is used: [%s]' % encoding) 2400 _log.warning('for this to work properly the application MUST have called locale.setlocale() before') 2401 2402 # set connection properties 2403 # - client encoding 2404 try: 2405 conn.set_client_encoding(encoding) 2406 except dbapi.DataError: 2407 t, v, tb = sys.exc_info() 2408 if 'cannot set encoding to' in str(v): 2409 raise cEncodingError(encoding, v).with_traceback(tb) 2410 if 'invalid value for parameter "client_encoding"' in str(v): 2411 raise cEncodingError(encoding, v).with_traceback(tb) 2412 raise 2413 2414 # - transaction isolation level 2415 if readonly: 2416 # alter-database default, checked at connect, no need to set here 2417 pass 2418 else: 2419 conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_SERIALIZABLE) 2420 2421 _log.debug('client time zone [%s]', _default_client_timezone) 2422 2423 # - client time zone 2424 curs = conn.cursor() 2425 curs.execute(_sql_set_timezone, [_default_client_timezone]) 2426 curs.close() 2427 conn.commit() 2428 2429 conn.is_decorated = True 2430 2431 if verbose: 2432 [ _log.debug(line) for line in capture_conn_state(conn = conn).split('\n') ] 2433 2434 return conn
2435 2436 #-----------------------------------------------------------------------
2437 -def discard_pooled_connection(conn_key=None):
2438 if __ro_conn_pool is None: 2439 return 2440 __ro_conn_pool.discard_connection(key = conn_key)
2441 2442 #-----------------------------------------------------------------------
2443 -def shutdown():
2444 if __ro_conn_pool is None: 2445 return 2446 __ro_conn_pool.shutdown()
2447 2448 # ====================================================================== 2449 # internal helpers 2450 #-----------------------------------------------------------------------
2451 -def __noop():
2452 pass
2453 2454 #-----------------------------------------------------------------------
2455 -def _raise_exception_on_ro_conn_close():
2456 raise TypeError('close() called on read-only connection')
2457 2458 #-----------------------------------------------------------------------
2459 -def log_database_access(action=None):
2460 run_insert ( 2461 schema = 'gm', 2462 table = 'access_log', 2463 values = {'user_action': action}, 2464 end_tx = True 2465 )
2466 2467 #-----------------------------------------------------------------------
2468 -def sanity_check_time_skew(tolerance=60):
2469 """Check server time and local time to be within 2470 the given tolerance of each other. 2471 2472 tolerance: seconds 2473 """ 2474 _log.debug('maximum skew tolerance (seconds): %s', tolerance) 2475 2476 cmd = "SELECT now() at time zone 'UTC'" 2477 conn = get_raw_connection(readonly=True) 2478 curs = conn.cursor() 2479 2480 start = time.time() 2481 rows, idx = run_ro_queries(link_obj = curs, queries = [{'cmd': cmd}]) 2482 end = time.time() 2483 client_now_as_utc = pydt.datetime.utcnow() 2484 2485 curs.close() 2486 conn.commit() 2487 2488 server_now_as_utc = rows[0][0] 2489 query_duration = end - start 2490 _log.info('server "now" (UTC): %s', server_now_as_utc) 2491 _log.info('client "now" (UTC): %s', client_now_as_utc) 2492 _log.debug('wire roundtrip (seconds): %s', query_duration) 2493 2494 if query_duration > tolerance: 2495 _log.error('useless to check client/server time skew, wire roundtrip > tolerance') 2496 return False 2497 2498 if server_now_as_utc > client_now_as_utc: 2499 real_skew = server_now_as_utc - client_now_as_utc 2500 else: 2501 real_skew = client_now_as_utc - server_now_as_utc 2502 2503 _log.debug('client/server time skew: %s', real_skew) 2504 2505 if real_skew > pydt.timedelta(seconds = tolerance): 2506 _log.error('client/server time skew > tolerance') 2507 return False 2508 2509 return True
2510 2511 #-----------------------------------------------------------------------
2512 -def sanity_check_database_settings():
2513 """Checks database settings. 2514 2515 returns (status, message) 2516 status: 2517 0: no problem 2518 1: non-fatal problem 2519 2: fatal problem 2520 """ 2521 _log.debug('checking database settings') 2522 2523 conn = get_connection() 2524 2525 # - version string 2526 global postgresql_version_string 2527 if postgresql_version_string is None: 2528 curs = conn.cursor() 2529 curs.execute('SELECT version()') 2530 postgresql_version_string = curs.fetchone()['version'] 2531 curs.close() 2532 _log.info('PostgreSQL version (string): "%s"' % postgresql_version_string) 2533 2534 options2check = { 2535 # setting: [expected value, risk, fatal?] 2536 'allow_system_table_mods': [['off'], 'system breakage', False], 2537 'check_function_bodies': [['on'], 'suboptimal error detection', False], 2538 'datestyle': [['ISO'], 'faulty timestamp parsing', True], 2539 'default_transaction_isolation': [['read committed'], 'faulty database reads', True], 2540 'default_transaction_read_only': [['on'], 'accidental database writes', False], 2541 'fsync': [['on'], 'data loss/corruption', True], 2542 'full_page_writes': [['on'], 'data loss/corruption', False], 2543 'lc_messages': [['C'], 'suboptimal error detection', False], 2544 'password_encryption': [['on', 'md5', 'scram-sha-256'], 'breach of confidentiality', False], 2545 #u'regex_flavor': [[u'advanced'], u'query breakage', False], # 9.0 doesn't support this anymore, default now advanced anyway 2546 'synchronous_commit': [['on'], 'data loss/corruption', False], 2547 'sql_inheritance': [['on'], 'query breakage, data loss/corruption', True], # IF returned (<PG10): better be ON, if NOT returned (PG10): hardwired 2548 'ignore_checksum_failure': [['off'], 'data loss/corruption', False], # starting with PG 9.3 2549 'track_commit_timestamp': [['on'], 'suboptimal auditing', False] # starting with PG 9.3 2550 } 2551 2552 from Gnumed.pycommon import gmCfg2 2553 _cfg = gmCfg2.gmCfgData() 2554 if _cfg.get(option = 'hipaa'): 2555 options2check['log_connections'] = [['on'], 'non-compliance with HIPAA', True] 2556 options2check['log_disconnections'] = [['on'], 'non-compliance with HIPAA', True] 2557 else: 2558 options2check['log_connections'] = [['on'], 'non-compliance with HIPAA', None] 2559 options2check['log_disconnections'] = [['on'], 'non-compliance with HIPAA', None] 2560 2561 cmd = "SELECT name, setting from pg_settings where name in %(settings)s" 2562 rows, idx = run_ro_queries ( 2563 link_obj = conn, 2564 queries = [{'cmd': cmd, 'args': {'settings': tuple(options2check.keys())}}], 2565 get_col_idx = False 2566 ) 2567 2568 found_error = False 2569 found_problem = False 2570 msg = [] 2571 for row in rows: 2572 option = row['name'] 2573 value_found = row['setting'] 2574 values_expected = options2check[option][0] 2575 risk = options2check[option][1] 2576 fatal_setting = options2check[option][2] 2577 if value_found not in values_expected: 2578 if fatal_setting is True: 2579 found_error = True 2580 elif fatal_setting is False: 2581 found_problem = True 2582 elif fatal_setting is None: 2583 pass 2584 else: 2585 _log.error(options2check[option]) 2586 raise ValueError('invalid database configuration sanity check') 2587 msg.append(_(' option [%s]: %s') % (option, value_found)) 2588 msg.append(_(' risk: %s') % risk) 2589 _log.warning('PG option [%s] set to [%s], expected %s, risk: <%s>' % (option, value_found, values_expected, risk)) 2590 2591 if found_error: 2592 return 2, '\n'.join(msg) 2593 2594 if found_problem: 2595 return 1, '\n'.join(msg) 2596 2597 return 0, ''
2598 2599 #------------------------------------------------------------------------
2600 -def _log_PG_settings(curs=None):
2601 # don't use any of the run_*()s helper functions 2602 # since that might create a loop if we fail here 2603 try: 2604 curs.execute('SELECT * FROM pg_settings') 2605 except dbapi.Error: 2606 _log.exception('cannot retrieve PG settings ("SELECT ... FROM pg_settings" failed)') 2607 return False 2608 settings = curs.fetchall() 2609 for setting in settings: 2610 if setting['unit'] is None: 2611 unit = '' 2612 else: 2613 unit = ' %s' % setting['unit'] 2614 if setting['sourcefile'] is None: 2615 sfile = '' 2616 else: 2617 sfile = '// %s @ %s' % (setting['sourcefile'], setting['sourceline']) 2618 pending_restart = u'' 2619 try: 2620 if setting['pending_restart']: 2621 pending_restart = u'// needs restart' 2622 except KeyError: 2623 # 'pending_restart' does not exist in PG 9.4 yet 2624 pass 2625 _log.debug('%s: %s%s (set from: [%s] // sess RESET will set to: [%s]%s%s)', 2626 setting['name'], 2627 setting['setting'], 2628 unit, 2629 setting['source'], 2630 setting['reset_val'], 2631 pending_restart, 2632 sfile 2633 ) 2634 try: 2635 curs.execute('select pg_available_extensions()') 2636 except: 2637 _log.exception('cannot log available PG extensions') 2638 return False 2639 extensions = curs.fetchall() 2640 if extensions is None: 2641 _log.error('no PG extensions available') 2642 return False 2643 for ext in extensions: 2644 _log.debug('PG extension: %s', ext['pg_available_extensions']) 2645 2646 # not really that useful because: 2647 # - clusterwide 2648 # - not retained across server restart (fixed in 9.6.1 - really ?) 2649 # try: 2650 # curs.execute(u'SELECT pg_last_committed_xact()') 2651 # except: 2652 # _log.exception(u'cannot retrieve last committed xact') 2653 # xact = curs.fetchall() 2654 # if xact is not None: 2655 # _log.debug(u'last committed transaction in cluster: %s', xact[0]) 2656 2657 return True
2658 2659 #------------------------------------------------------------------------
2660 -def log_pg_exception_details(exc):
2661 if not isinstance(exc, dbapi.Error): 2662 return False 2663 try: 2664 args = exc.args 2665 for arg in args: 2666 _log.debug('exc.arg: %s', arg) 2667 except AttributeError: 2668 _log.debug('exception has no <.args>') 2669 _log.debug('pgerror: [%s]', exc.pgerror) 2670 if exc.pgcode is None: 2671 _log.debug('pgcode : %s', exc.pgcode) 2672 else: 2673 _log.debug('pgcode : %s (%s)', exc.pgcode, sql_error_codes.lookup(exc.pgcode)) 2674 if exc.cursor is None: 2675 _log.debug('cursor: None') 2676 else: 2677 capture_cursor_state(cursor = exc.cursor) 2678 try: 2679 exc.diag 2680 for attr in dir(exc.diag): 2681 if attr.startswith('__'): 2682 continue 2683 val = getattr(exc.diag, attr) 2684 if val is None: 2685 continue 2686 _log.debug('%s: %s', attr, val) 2687 except AttributeError: 2688 _log.debug('diag: not available') 2689 return True
2690 2691 #------------------------------------------------------------------------
2692 -def exception_is_connection_loss(exc):
2693 if not isinstance(exc, dbapi.Error): 2694 # not a PG exception 2695 return False 2696 try: 2697 msg = '%s' % exc.args[0] 2698 except (AttributeError, IndexError, TypeError): 2699 _log.debug('cannot extract message from exception') 2700 # cannot process message 2701 return False 2702 _log.debug('interpreting: %s', msg) 2703 # OperationalError 2704 conn_lost = ( 2705 ('erver' in msg) 2706 and 2707 ( 2708 ('terminat' in msg) 2709 or 2710 ('abnorm' in msg) 2711 or 2712 ('end' in msg) 2713 # or 2714 # ('oute' in msg) 2715 ) 2716 ) 2717 if conn_lost: 2718 _log.debug('indicates connection loss') 2719 return True 2720 # InterfaceError 2721 conn_lost = ( 2722 ('onnect' in msg) 2723 and 2724 ( 2725 ('close' in msg) 2726 or 2727 ('end' in msg) 2728 ) 2729 ) 2730 if conn_lost: 2731 _log.debug('indicates connection loss') 2732 return conn_lost
2733 2734 #========================================================================
2735 -class cAuthenticationError(dbapi.OperationalError):
2736
2737 - def __init__(self, dsn=None, prev_val=None):
2738 self.dsn = dsn 2739 self.prev_val = prev_val
2740
2741 - def __str__(self):
2742 return 'PostgreSQL: %sDSN: %s' % (self.prev_val, self.dsn)
2743 2744 #======================================================================== 2745 # custom psycopg2 extensions 2746 #========================================================================
2747 -class cEncodingError(dbapi.OperationalError):
2748
2749 - def __init__(self, encoding=None, prev_val=None):
2750 self.encoding = encoding 2751 self.prev_val = prev_val
2752
2753 - def __str__(self):
2754 return 'PostgreSQL: %s\nencoding: %s' % (self.prev_val, self.encoding)
2755 2756 # ----------------------------------------------------------------------- 2757 # Python -> PostgreSQL 2758 # ----------------------------------------------------------------------- 2759 # test when Squeeze (and thus psycopg2 2.2 becomes Stable
2760 -class cAdapterPyDateTime(object):
2761
2762 - def __init__(self, dt):
2763 if dt.tzinfo is None: 2764 raise ValueError('datetime.datetime instance is lacking a time zone: [%s]' % _timestamp_template % dt.isoformat()) 2765 self.__dt = dt
2766
2767 - def getquoted(self):
2768 return _timestamp_template % self.__dt.isoformat()
2769 2770 #======================================================================= 2771 # main 2772 #----------------------------------------------------------------------- 2773 2774 # make sure psycopg2 knows how to handle unicode ... 2775 psycopg2.extensions.register_type(psycopg2.extensions.UNICODE) 2776 psycopg2.extensions.register_type(psycopg2._psycopg.UNICODEARRAY) 2777 2778 # tell psycopg2 how to adapt datetime types with timestamps when locales are in use 2779 # check in 0.9: 2780 psycopg2.extensions.register_adapter(pydt.datetime, cAdapterPyDateTime) 2781 2782 # turn dict()s into JSON - only works > 9.2 2783 #psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) 2784 2785 # do NOT adapt *lists* to "... IN (*) ..." syntax because we want 2786 # them adapted to "... ARRAY[]..." so we can support PG arrays 2787 2788 #======================================================================= 2789 if __name__ == "__main__": 2790 2791 if len(sys.argv) < 2: 2792 sys.exit() 2793 2794 if sys.argv[1] != 'test': 2795 sys.exit() 2796 2797 from Gnumed.pycommon.gmTools import file2md5 2798 2799 logging.basicConfig(level=logging.DEBUG) 2800 2801 #--------------------------------------------------------------------
2802 - def test_file2bytea():
2803 run_rw_queries(queries = [ 2804 {'cmd': 'drop table if exists test_bytea'}, 2805 {'cmd': 'create table test_bytea (data bytea)'} 2806 ]) 2807 2808 try: 2809 file2bytea(query = 'insert into test_bytea values (%(data)s::bytea) returning md5(data) as md5', filename = sys.argv[2], file_md5 = file2md5(sys.argv[2], True)) 2810 except: 2811 _log.exception('error') 2812 2813 run_rw_queries(queries = [ 2814 {'cmd': 'drop table test_bytea'} 2815 ])
2816 2817 #--------------------------------------------------------------------
2818 - def test_file2bytea_lo():
2819 lo_oid = file2bytea_lo ( 2820 filename = sys.argv[2] 2821 #, file_md5 = file2md5(sys.argv[2], True) 2822 ) 2823 print(lo_oid)
2824 # if lo_oid != -1: 2825 # run_rw_queries(queries = [ 2826 # {'cmd': u'select lo_unlink(%(loid)s::oid)', 'args': {'loid': lo_oid}} 2827 # ]) 2828 2829 #--------------------------------------------------------------------
2830 - def test_file2bytea_copy_from():
2831 2832 run_rw_queries(queries = [ 2833 {'cmd': 'drop table if exists test_bytea'}, 2834 {'cmd': 'create table test_bytea (pk serial primary key, data bytea)'}, 2835 {'cmd': "insert into test_bytea (data) values (NULL::bytea)"} 2836 ]) 2837 2838 md5_query = { 2839 'cmd': 'select md5(data) AS md5 FROM test_bytea WHERE pk = %(pk)s', 2840 'args': {'pk': 1} 2841 } 2842 2843 file2bytea_copy_from ( 2844 table = 'test_bytea', 2845 columns = ['data'], 2846 filename = sys.argv[2], 2847 md5_query = md5_query, 2848 file_md5 = file2md5(sys.argv[2], True) 2849 ) 2850 2851 run_rw_queries(queries = [ 2852 {'cmd': 'drop table if exists test_bytea'} 2853 ])
2854 2855 #--------------------------------------------------------------------
2856 - def test_file2bytea_overlay():
2857 2858 run_rw_queries(queries = [ 2859 {'cmd': 'drop table if exists test_bytea'}, 2860 {'cmd': 'create table test_bytea (pk serial primary key, data bytea)'}, 2861 {'cmd': "insert into test_bytea (data) values (NULL::bytea)"} 2862 ]) 2863 2864 cmd = """ 2865 update test_bytea 2866 set data = overlay ( 2867 coalesce(data, ''::bytea) 2868 placing %(data)s::bytea 2869 from %(start)s 2870 for %(size)s 2871 ) 2872 where 2873 pk > %(pk)s 2874 """ 2875 md5_cmd = 'select md5(data) from test_bytea' 2876 args = {'pk': 0} 2877 file2bytea_overlay ( 2878 query = cmd, 2879 args = args, 2880 filename = sys.argv[2], 2881 conn = None, 2882 md5_query = md5_cmd, 2883 file_md5 = file2md5(sys.argv[2], True) 2884 ) 2885 2886 run_rw_queries(queries = [ 2887 {'cmd': 'drop table test_bytea'} 2888 ])
2889 2890 #--------------------------------------------------------------------
2891 - def test_get_connection():
2892 print("testing get_connection()") 2893 2894 print('') 2895 dsn = 'foo' 2896 try: 2897 conn = get_connection(dsn=dsn) 2898 except dbapi.ProgrammingError as e: 2899 print("1) SUCCESS: get_connection(%s) failed as expected" % dsn) 2900 t, v = sys.exc_info()[:2] 2901 print (' ', t) 2902 print (' ', v) 2903 2904 print('') 2905 dsn = 'dbname=gnumed_v9' 2906 try: 2907 conn = get_connection(dsn=dsn) 2908 except cAuthenticationError: 2909 print("2) SUCCESS: get_connection(%s) failed as expected" % dsn) 2910 t, v = sys.exc_info()[:2] 2911 print(' ', t) 2912 print(' ', v) 2913 2914 print('') 2915 dsn = 'dbname=gnumed_v9 user=abc' 2916 try: 2917 conn = get_connection(dsn=dsn) 2918 except cAuthenticationError: 2919 print("3) SUCCESS: get_connection(%s) failed as expected" % dsn) 2920 t, v = sys.exc_info()[:2] 2921 print(' ', t) 2922 print(' ', v) 2923 2924 print('') 2925 dsn = 'dbname=gnumed_v9 user=any-doc password=abc' 2926 try: 2927 conn = get_connection(dsn=dsn) 2928 except cAuthenticationError: 2929 print("5) SUCCESS: get_connection(%s) failed as expected" % dsn) 2930 t, v = sys.exc_info()[:2] 2931 print(' ', t) 2932 print(' ', v) 2933 2934 print('') 2935 dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc' 2936 conn = get_connection(dsn=dsn, readonly=True) 2937 2938 dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc' 2939 conn = get_connection(dsn=dsn, readonly=False, verbose=True) 2940 2941 print('') 2942 dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc' 2943 encoding = 'foo' 2944 try: 2945 conn = get_connection(dsn=dsn, encoding=encoding) 2946 except cEncodingError: 2947 print("6) SUCCESS: get_connection(%s, %s) failed as expected" % (dsn, encoding)) 2948 t, v = sys.exc_info()[:2] 2949 print(' ', t) 2950 print(' ', v) 2951 2952 print('') 2953 dsn = 'dbname=gnumed_v22 user=any-doc' 2954 try: 2955 conn = get_connection(dsn=dsn) 2956 print("6) SUCCESS:", dsn) 2957 print('pid:', conn.get_backend_pid()) 2958 except cAuthenticationError: 2959 print("4) SUCCESS: get_connection(%s) failed" % dsn) 2960 t, v = sys.exc_info()[:2] 2961 print(' ', t) 2962 print(' ', v) 2963 2964 try: 2965 curs = conn.cursor() 2966 input('hit enter to run query') 2967 curs.execute('selec 1') 2968 except Exception as exc: 2969 print('ERROR') 2970 _log.exception('exception occurred') 2971 log_pg_exception_details(exc) 2972 if exception_is_connection_loss(exc): 2973 _log.error('lost connection')
2974 2975 #--------------------------------------------------------------------
2976 - def test_exceptions():
2977 print("testing exceptions") 2978 2979 try: 2980 raise cAuthenticationError('no dsn', 'no previous exception') 2981 except cAuthenticationError: 2982 t, v, tb = sys.exc_info() 2983 print(t) 2984 print(v) 2985 print(tb) 2986 2987 try: 2988 raise cEncodingError('no dsn', 'no previous exception') 2989 except cEncodingError: 2990 t, v, tb = sys.exc_info() 2991 print(t) 2992 print(v) 2993 print(tb)
2994 #--------------------------------------------------------------------
2995 - def test_ro_queries():
2996 print("testing run_ro_queries()") 2997 2998 dsn = 'dbname=gnumed_v22 user=any-doc password=any-doc' 2999 conn = get_connection(dsn, readonly=True) 3000 3001 data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': 'SELECT version()'}], return_data=True, get_col_idx=True, verbose=True) 3002 print(data) 3003 print(idx) 3004 data, idx = run_ro_queries(link_obj=conn, queries=[{'cmd': 'SELECT 1'}], return_data=True, get_col_idx=True) 3005 print(data) 3006 print(idx) 3007 3008 curs = conn.cursor() 3009 3010 data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'SELECT version()'}], return_data=True, get_col_idx=True, verbose=True) 3011 print(data) 3012 print(idx) 3013 3014 data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'SELECT 1'}], return_data=True, get_col_idx=True, verbose=True) 3015 print(data) 3016 print(idx) 3017 3018 try: 3019 data, idx = run_ro_queries(link_obj=curs, queries=[{'cmd': 'selec 1'}], return_data=True, get_col_idx=True, verbose=True) 3020 print(data) 3021 print(idx) 3022 except psycopg2.ProgrammingError: 3023 print('SUCCESS: run_ro_queries("selec 1") failed as expected') 3024 t, v = sys.exc_info()[:2] 3025 print(' ', t) 3026 print(' ', v) 3027 3028 curs.close()
3029 3030 #--------------------------------------------------------------------
3031 - def test_request_dsn():
3032 conn = get_connection() 3033 print(conn) 3034 conn.close()
3035 #--------------------------------------------------------------------
3036 - def test_set_encoding():
3037 print("testing set_default_client_encoding()") 3038 3039 enc = 'foo' 3040 try: 3041 set_default_client_encoding(enc) 3042 print("SUCCESS: encoding [%s] worked" % enc) 3043 except ValueError: 3044 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc) 3045 t, v = sys.exc_info()[:2] 3046 print(' ', t) 3047 print(' ', v) 3048 3049 enc = '' 3050 try: 3051 set_default_client_encoding(enc) 3052 print("SUCCESS: encoding [%s] worked" % enc) 3053 except ValueError: 3054 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc) 3055 t, v = sys.exc_info()[:2] 3056 print(' ', t) 3057 print(' ', v) 3058 3059 enc = 'latin1' 3060 try: 3061 set_default_client_encoding(enc) 3062 print("SUCCESS: encoding [%s] worked" % enc) 3063 except ValueError: 3064 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc) 3065 t, v = sys.exc_info()[:2] 3066 print(' ', t) 3067 print(' ', v) 3068 3069 enc = 'utf8' 3070 try: 3071 set_default_client_encoding(enc) 3072 print("SUCCESS: encoding [%s] worked" % enc) 3073 except ValueError: 3074 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc) 3075 t, v = sys.exc_info()[:2] 3076 print(' ', t) 3077 print(' ', v) 3078 3079 enc = 'unicode' 3080 try: 3081 set_default_client_encoding(enc) 3082 print("SUCCESS: encoding [%s] worked" % enc) 3083 except ValueError: 3084 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc) 3085 t, v = sys.exc_info()[:2] 3086 print(' ', t) 3087 print(' ', v) 3088 3089 enc = 'UNICODE' 3090 try: 3091 set_default_client_encoding(enc) 3092 print("SUCCESS: encoding [%s] worked" % enc) 3093 except ValueError: 3094 print("SUCCESS: set_default_client_encoding(%s) failed as expected" % enc) 3095 t, v = sys.exc_info()[:2] 3096 print(' ', t) 3097 print(' ', v)
3098 #--------------------------------------------------------------------
3099 - def test_connection_pool():
3100 dsn = get_default_dsn() 3101 pool = cConnectionPool(minconn=1, maxconn=2, dsn=None, verbose=False) 3102 print(pool) 3103 print(pool.getconn()) 3104 print(pool.getconn()) 3105 print(pool.getconn()) 3106 print(type(pool.getconn()))
3107 #--------------------------------------------------------------------
3108 - def test_list_args():
3109 dsn = get_default_dsn() 3110 conn = get_connection(dsn, readonly=True) 3111 curs = conn.cursor() 3112 curs.execute('SELECT * from clin.clin_narrative where narrative = %s', ['a'])
3113 #--------------------------------------------------------------------
3114 - def test_sanitize_pg_regex():
3115 tests = [ 3116 ['(', '\\('] 3117 , ['[', '\\['] 3118 , [')', '\\)'] 3119 ] 3120 for test in tests: 3121 result = sanitize_pg_regex(test[0]) 3122 if result != test[1]: 3123 print('ERROR: sanitize_pg_regex(%s) returned "%s", expected "%s"' % (test[0], result, test[1]))
3124 #--------------------------------------------------------------------
3125 - def test_is_pg_interval():
3126 status = True 3127 tests = [ 3128 [None, True], # None == NULL == succeeds ! 3129 [1, True], 3130 ['1', True], 3131 ['abc', False] 3132 ] 3133 3134 if not is_pg_interval(): 3135 print('ERROR: is_pg_interval() returned "False", expected "True"') 3136 status = False 3137 3138 for test in tests: 3139 result = is_pg_interval(test[0]) 3140 if result != test[1]: 3141 print('ERROR: is_pg_interval(%s) returned "%s", expected "%s"' % (test[0], result, test[1])) 3142 status = False 3143 3144 return status
3145 #--------------------------------------------------------------------
3146 - def test_sanity_check_time_skew():
3147 sanity_check_time_skew()
3148 3149 #--------------------------------------------------------------------
3150 - def test_get_foreign_key_names():
3151 print(get_foreign_key_names ( 3152 src_schema = 'clin', 3153 src_table = 'vaccination', 3154 src_column = 'fk_episode', 3155 target_schema = 'clin', 3156 target_table = 'episode', 3157 target_column = 'pk' 3158 ))
3159 3160 #--------------------------------------------------------------------
3161 - def test_get_foreign_key_details():
3162 schema = 'clin' 3163 table = 'episode' 3164 col = 'pk' 3165 print('column %s.%s.%s is referenced by:' % (schema, table, col)) 3166 for row in get_foreign_keys2column ( 3167 schema = schema, 3168 table = table, 3169 column = col 3170 ): 3171 print(' <- %s.%s' % ( 3172 row['referencing_table'], 3173 row['referencing_column'] 3174 ))
3175 3176 #--------------------------------------------------------------------
3177 - def test_set_user_language():
3178 # (user, language, result, exception type) 3179 tests = [ 3180 # current user 3181 [None, 'de_DE', True], 3182 [None, 'lang_w/o_tx', False], 3183 [None, None, True], 3184 # valid user 3185 ['any-doc', 'de_DE', True], 3186 ['any-doc', 'lang_w/o_tx', False], 3187 ['any-doc', None, True], 3188 # invalid user 3189 ['invalid user', 'de_DE', None], 3190 ['invalid user', 'lang_w/o_tx', False], # lang checking happens before user checking 3191 ['invalid user', None, True] 3192 ] 3193 for test in tests: 3194 try: 3195 result = set_user_language(user = test[0], language = test[1]) 3196 if result != test[2]: 3197 print("test:", test) 3198 print("result:", result, "expected:", test[2]) 3199 except psycopg2.IntegrityError as e: 3200 if test[2] is None: 3201 continue 3202 print("test:", test) 3203 print("expected exception") 3204 print("result:", e)
3205 3206 #--------------------------------------------------------------------
3207 - def test_get_schema_revision_history():
3208 for line in get_schema_revision_history(): 3209 print(' - '.join(line))
3210 3211 #--------------------------------------------------------------------
3212 - def test_run_query():
3213 gmDateTime.init() 3214 args = {'dt': gmDateTime.pydt_max_here()} 3215 cmd = "SELECT %(dt)s" 3216 3217 #cmd = u"SELECT 'infinity'::timestamp with time zone" 3218 3219 cmd = """ 3220 SELECT to_timestamp (foofoo,'YYMMDD.HH24MI') FROM ( 3221 SELECT REGEXP_REPLACE ( 3222 't1.130729.0902.tif', -- string 3223 E'(.1)\.([0-9\.]+)(\.tif)', -- pattern 3224 E'\\\\2' -- replacement 3225 ) AS foofoo 3226 ) AS foo""" 3227 rows, idx = run_ro_queries(queries = [{'cmd': cmd, 'args': args}], get_col_idx = False) 3228 print(rows) 3229 print(rows[0]) 3230 print(rows[0][0])
3231 #--------------------------------------------------------------------
3232 - def test_schema_exists():
3233 print(schema_exists())
3234 #--------------------------------------------------------------------
3235 - def test_row_locks():
3236 row_is_locked(table = 'dem.identity', pk = 12) 3237 3238 print("1st connection:") 3239 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12)) 3240 print(" 1st shared lock succeeded:", lock_row(table = 'dem.identity', pk = 12, exclusive = False)) 3241 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12)) 3242 3243 print(" 2nd shared lock should succeed:", lock_row(table = 'dem.identity', pk = 12, exclusive = False)) 3244 print(" `-> unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = False)) 3245 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12)) 3246 print(" exclusive lock should succeed:", lock_row(table = 'dem.identity', pk = 12, exclusive = True)) 3247 print(" `-> unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = True)) 3248 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12)) 3249 3250 print("2nd connection:") 3251 conn = get_raw_connection(readonly=True) 3252 print(" shared lock should succeed:", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False)) 3253 print(" `-> unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False)) 3254 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12)) 3255 print(" exclusive lock succeeded ?", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = True), "(should fail)") 3256 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12)) 3257 3258 print("1st connection:") 3259 print(" unlock succeeded:", unlock_row(table = 'dem.identity', pk = 12, exclusive = False)) 3260 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12)) 3261 3262 print("2nd connection:") 3263 print(" exclusive lock should succeed", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = True)) 3264 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12)) 3265 print(" shared lock should succeed:", lock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False)) 3266 print(" `-> unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False)) 3267 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12)) 3268 print(" unlock succeeded:", unlock_row(link_obj = conn, table = 'dem.identity', pk = 12, exclusive = False)) 3269 print(" locked:", row_is_locked(table = 'dem.identity', pk = 12)) 3270 3271 conn.close()
3272 3273 #--------------------------------------------------------------------
3274 - def test_get_foreign_key_names():
3275 print(get_foreign_key_names ( 3276 src_schema = 'dem', 3277 src_table = 'names', 3278 src_column = 'id_identity', 3279 target_schema = 'dem', 3280 target_table = 'identity', 3281 target_column = 'pk' 3282 ))
3283 3284 #--------------------------------------------------------------------
3285 - def test_get_index_name():
3286 print(get_index_name(indexed_table = 'clin.vaccination', indexed_column = 'fk_episode'))
3287 3288 #--------------------------------------------------------------------
3289 - def test_faulty_SQL():
3290 run_rw_queries(queries = [{'cmd': 'SELEC 1'}])
3291 3292 #--------------------------------------------------------------------
3293 - def test_log_settings():
3294 conn = conn = get_connection() 3295 _log_PG_settings(curs = conn.cursor())
3296 3297 #-------------------------------------------------------------------- 3298 # run tests 3299 #test_get_connection() 3300 #test_exceptions() 3301 test_ro_queries() 3302 #test_request_dsn() 3303 #test_set_encoding() 3304 #test_connection_pool() 3305 #test_list_args() 3306 #test_sanitize_pg_regex() 3307 #test_is_pg_interval() 3308 #test_sanity_check_time_skew() 3309 #test_get_foreign_key_details() 3310 #test_get_foreign_key_names() 3311 #test_get_index_name() 3312 #test_set_user_language() 3313 #test_get_schema_revision_history() 3314 #test_run_query() 3315 #test_schema_exists() 3316 #test_get_foreign_key_names() 3317 #test_row_locks() 3318 #test_file2bytea() 3319 #test_file2bytea_overlay() 3320 #test_file2bytea_copy_from() 3321 #test_file2bytea_lo() 3322 #test_faulty_SQL() 3323 #test_log_settings() 3324 3325 # ====================================================================== 3326