Package Gnumed :: Package pycommon :: Module gmBackendListener
[frames] | no frames]

Source Code for Module Gnumed.pycommon.gmBackendListener

  1  __doc__ = """GNUmed database backend listener. 
  2   
  3  This module implements threaded listening for asynchronuous 
  4  notifications from the database backend. 
  5  """ 
  6  #===================================================================== 
  7  __author__ = "H. Herb <hherb@gnumed.net>, K.Hilbert <karsten.hilbert@gmx.net>" 
  8  __license__ = "GPL v2 or later" 
  9   
 10  import sys 
 11  import time 
 12  import threading 
 13  import select 
 14  import logging 
 15   
 16   
 17  if __name__ == '__main__': 
 18          sys.path.insert(0, '../../') 
 19  from Gnumed.pycommon import gmDispatcher 
 20  from Gnumed.pycommon import gmBorg 
 21   
 22   
 23  _log = logging.getLogger('gm.db') 
 24   
 25   
 26  signals2listen4 = [ 
 27          'db_maintenance_warning',               # warns of impending maintenance and asks for disconnect 
 28          'db_maintenance_disconnect',    # announces a forced disconnect and disconnects 
 29          'gm_table_mod'                                  # sent for any (registered) table modification, payload contains details 
 30  ] 
 31   
 32  #===================================================================== 
33 -class gmBackendListener(gmBorg.cBorg):
34
35 - def __init__(self, conn=None, poll_interval=3):
36 37 try: 38 self.already_inited 39 return 40 41 except AttributeError: 42 pass 43 44 self.debug = False 45 self.__notifications_received = 0 46 self.__messages_sent = 0 47 48 _log.info('starting backend notifications listener thread') 49 50 # the listener thread will regularly try to acquire 51 # this lock, when it succeeds it will quit 52 self._quit_lock = threading.Lock() 53 # take the lock now so it cannot be taken by the worker 54 # thread until it is released in shutdown() 55 if not self._quit_lock.acquire(0): 56 _log.error('cannot acquire thread-quit lock, aborting') 57 raise EnvironmentError("cannot acquire thread-quit lock") 58 59 self._conn = conn 60 self.backend_pid = self._conn.get_backend_pid() 61 _log.debug('notification listener connection has backend PID [%s]', self.backend_pid) 62 self._conn.set_isolation_level(0) # autocommit mode = psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT 63 self._cursor = self._conn.cursor() 64 try: 65 self._conn_fd = self._conn.fileno() 66 except AttributeError: 67 self._conn_fd = self._cursor.fileno() 68 self._conn_lock = threading.Lock() # lock for access to connection object 69 70 self.__register_interests() 71 72 # check for messages every 'poll_interval' seconds 73 self._poll_interval = poll_interval 74 self._listener_thread = None 75 self.__start_thread() 76 77 self.already_inited = True
78 79 #------------------------------- 80 # public API 81 #-------------------------------
82 - def shutdown(self):
83 _log.debug('received %s notifications', self.__notifications_received) 84 _log.debug('sent %s messages', self.__messages_sent) 85 86 if self._listener_thread is None: 87 self.__shutdown_connection() 88 return 89 90 _log.info('stopping backend notifications listener thread') 91 self._quit_lock.release() 92 try: 93 # give the worker thread time to terminate 94 self._listener_thread.join(self._poll_interval+2.0) 95 try: 96 if self._listener_thread.isAlive(): 97 _log.error('listener thread still alive after join()') 98 _log.debug('active threads: %s' % threading.enumerate()) 99 except Exception: 100 pass 101 except Exception: 102 print(sys.exc_info()) 103 104 self._listener_thread = None 105 106 try: 107 self.__unregister_unspecific_notifications() 108 except Exception: 109 _log.exception('unable to unregister unspecific notifications') 110 111 self.__shutdown_connection() 112 113 return
114 #------------------------------- 115 # event handlers 116 #------------------------------- 117 # internal helpers 118 #-------------------------------
119 - def __register_interests(self):
120 # determine unspecific notifications 121 self.unspecific_notifications = signals2listen4 122 _log.info('configured unspecific notifications:') 123 _log.info('%s' % self.unspecific_notifications) 124 gmDispatcher.known_signals.extend(self.unspecific_notifications) 125 126 # listen to unspecific notifications 127 self.__register_unspecific_notifications()
128 129 #-------------------------------
131 for sig in self.unspecific_notifications: 132 _log.info('starting to listen for [%s]' % sig) 133 cmd = 'LISTEN "%s"' % sig 134 self._conn_lock.acquire(1) 135 try: 136 self._cursor.execute(cmd) 137 finally: 138 self._conn_lock.release()
139 140 #-------------------------------
142 for sig in self.unspecific_notifications: 143 _log.info('stopping to listen for [%s]' % sig) 144 cmd = 'UNLISTEN "%s"' % sig 145 self._conn_lock.acquire(1) 146 try: 147 self._cursor.execute(cmd) 148 finally: 149 self._conn_lock.release()
150 151 #-------------------------------
152 - def __shutdown_connection(self):
153 _log.debug('shutting down connection with backend PID [%s]', self.backend_pid) 154 self._conn_lock.acquire(1) 155 try: 156 self._conn.rollback() 157 except Exception: 158 pass 159 finally: 160 self._conn_lock.release()
161 162 #-------------------------------
163 - def __start_thread(self):
164 if self._conn is None: 165 raise ValueError("no connection to backend available, useless to start thread") 166 167 self._listener_thread = threading.Thread ( 168 target = self._process_notifications, 169 name = self.__class__.__name__, 170 daemon = True 171 ) 172 _log.info('starting listener thread') 173 self._listener_thread.start()
174 175 #------------------------------- 176 # the actual thread code 177 #-------------------------------
178 - def _process_notifications(self):
179 180 # loop until quitting 181 _have_quit_lock = None 182 while not _have_quit_lock: 183 # quitting ? 184 if self._quit_lock.acquire(0): 185 break 186 187 # wait at most self._poll_interval for new data 188 self._conn_lock.acquire(1) 189 try: 190 ready_input_sockets = select.select([self._conn_fd], [], [], self._poll_interval)[0] 191 finally: 192 self._conn_lock.release() 193 # any input available ? 194 if len(ready_input_sockets) == 0: 195 # no, select.select() timed out 196 # give others a chance to grab the conn lock (eg listen/unlisten) 197 time.sleep(0.3) 198 continue 199 # data available, wait for it to fully arrive 200 self._conn_lock.acquire(1) 201 try: 202 self._conn.poll() 203 finally: 204 self._conn_lock.release() 205 # any notifications ? 206 while len(self._conn.notifies) > 0: 207 # if self._quit_lock can be acquired we may be in 208 # __del__ in which case gmDispatcher is not 209 # guaranteed to exist anymore 210 if self._quit_lock.acquire(0): 211 _have_quit_lock = 1 212 break 213 214 self._conn_lock.acquire(1) 215 try: 216 notification = self._conn.notifies.pop() 217 finally: 218 self._conn_lock.release() 219 self.__notifications_received += 1 220 if self.debug: 221 print(notification) 222 _log.debug('#%s: %s (first param is PID of sending backend)', self.__notifications_received, notification) 223 # decode payload 224 payload = notification.payload.split('::') 225 operation = None 226 table = None 227 pk_column_name = None 228 pk_of_row = None 229 pk_identity = None 230 for item in payload: 231 if item.startswith('operation='): 232 operation = item.split('=')[1] 233 if item.startswith('table='): 234 table = item.split('=')[1] 235 if item.startswith('PK name='): 236 pk_column_name = item.split('=')[1] 237 if item.startswith('row PK='): 238 pk_of_row = int(item.split('=')[1]) 239 if item.startswith('person PK='): 240 try: 241 pk_identity = int(item.split('=')[1]) 242 except ValueError: 243 _log.exception('error in change notification trigger') 244 pk_identity = -1 245 # try sending intra-client signals: 246 # 1) generic signal 247 self.__messages_sent += 1 248 try: 249 results = gmDispatcher.send ( 250 signal = notification.channel, 251 originated_in_database = True, 252 listener_pid = self.backend_pid, 253 sending_backend_pid = notification.pid, 254 pk_identity = pk_identity, 255 operation = operation, 256 table = table, 257 pk_column_name = pk_column_name, 258 pk_of_row = pk_of_row, 259 message_index = self.__messages_sent, 260 notification_index = self.__notifications_received 261 ) 262 except Exception: 263 print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (notification.channel, notification.pid)) 264 print(sys.exc_info()) 265 # 2) dynamically emulated old style table specific signals 266 if table is not None: 267 self.__messages_sent += 1 268 signal = '%s_mod_db' % table 269 _log.debug('emulating old-style table specific signal [%s]', signal) 270 try: 271 results = gmDispatcher.send ( 272 signal = signal, 273 originated_in_database = True, 274 listener_pid = self.backend_pid, 275 sending_backend_pid = notification.pid, 276 pk_identity = pk_identity, 277 operation = operation, 278 table = table, 279 pk_column_name = pk_column_name, 280 pk_of_row = pk_of_row, 281 message_index = self.__messages_sent, 282 notification_index = self.__notifications_received 283 ) 284 except Exception: 285 print("problem routing notification [%s] from backend [%s] to intra-client dispatcher" % (signal, notification.pid)) 286 print(sys.exc_info()) 287 288 # there *may* be more pending notifications but 289 # we don't care when quitting 290 if self._quit_lock.acquire(0): 291 _have_quit_lock = 1 292 break 293 294 # exit thread activity 295 return
296 297 #===================================================================== 298 # main 299 #===================================================================== 300 if __name__ == "__main__": 301 302 if len(sys.argv) < 2: 303 sys.exit() 304 305 if sys.argv[1] not in ['test', 'monitor']: 306 sys.exit() 307 308 309 notifies = 0 310 311 from Gnumed.pycommon import gmPG2, gmI18N 312 from Gnumed.business import gmPerson, gmPersonSearch 313 314 gmI18N.activate_locale() 315 gmI18N.install_domain(domain='gnumed') 316 #-------------------------------
317 - def run_test():
318 319 #------------------------------- 320 def dummy(n): 321 return float(n)*n/float(1+n)
322 #------------------------------- 323 def OnPatientModified(): 324 global notifies 325 notifies += 1 326 sys.stdout.flush() 327 print("\nBackend says: patient data has been modified (%s. notification)" % notifies) 328 #------------------------------- 329 try: 330 n = int(sys.argv[2]) 331 except Exception: 332 print("You can set the number of iterations\nwith the second command line argument") 333 n = 100000 334 335 # try loop without backend listener 336 print("Looping", n, "times through dummy function") 337 i = 0 338 t1 = time.time() 339 while i < n: 340 r = dummy(i) 341 i += 1 342 t2 = time.time() 343 t_nothreads = t2-t1 344 print("Without backend thread, it took", t_nothreads, "seconds") 345 346 listener = gmBackendListener(conn = gmPG2.get_raw_connection()) 347 348 # now try with listener to measure impact 349 print("Now in a new shell connect psql to the") 350 print("database <gnumed_v9> on localhost, return") 351 print("here and hit <enter> to continue.") 352 input('hit <enter> when done starting psql') 353 print("You now have about 30 seconds to go") 354 print("to the psql shell and type") 355 print(" notify patient_changed<enter>") 356 print("several times.") 357 print("This should trigger our backend listening callback.") 358 print("You can also try to stop the demo with Ctrl-C !") 359 360 listener.register_callback('patient_changed', OnPatientModified) 361 362 try: 363 counter = 0 364 while counter < 20: 365 counter += 1 366 time.sleep(1) 367 sys.stdout.flush() 368 print('.') 369 print("Looping",n,"times through dummy function") 370 i = 0 371 t1 = time.time() 372 while i < n: 373 r = dummy(i) 374 i += 1 375 t2 = time.time() 376 t_threaded = t2-t1 377 print("With backend thread, it took", t_threaded, "seconds") 378 print("Difference:", t_threaded-t_nothreads) 379 except KeyboardInterrupt: 380 print("cancelled by user") 381 382 listener.shutdown() 383 listener.unregister_callback('patient_changed', OnPatientModified) 384 #-------------------------------
385 - def run_monitor():
386 387 print("starting up backend notifications monitor") 388 389 def monitoring_callback(*args, **kwargs): 390 try: 391 kwargs['originated_in_database'] 392 print('==> got notification from database "%s":' % kwargs['signal']) 393 except KeyError: 394 print('==> received signal from client: "%s"' % kwargs['signal']) 395 del kwargs['signal'] 396 for key in kwargs.keys(): 397 print(' [%s]: %s' % (key, kwargs[key]))
398 399 gmDispatcher.connect(receiver = monitoring_callback) 400 401 listener = gmBackendListener(conn = gmPG2.get_raw_connection()) 402 print("listening for the following notifications:") 403 print("1) unspecific:") 404 for sig in listener.unspecific_notifications: 405 print(' - %s' % sig) 406 407 while True: 408 pat = gmPersonSearch.ask_for_patient() 409 if pat is None: 410 break 411 print("found patient", pat) 412 gmPerson.set_active_patient(patient=pat) 413 print("now waiting for notifications, hit <ENTER> to select another patient") 414 input() 415 416 print("cleanup") 417 listener.shutdown() 418 419 print("shutting down backend notifications monitor") 420 421 #------------------------------- 422 if sys.argv[1] == 'monitor': 423 run_monitor() 424 else: 425 run_test() 426 427 #===================================================================== 428