Package Gnumed :: Package pycommon :: Module gmTools
[frames] | no frames]

Source Code for Module Gnumed.pycommon.gmTools

   1  # -*- coding: utf-8 -*- 
   2   
   3   
   4  __doc__ = """GNUmed general tools.""" 
   5   
   6  #=========================================================================== 
   7  __author__ = "K. Hilbert <Karsten.Hilbert@gmx.net>" 
   8  __license__ = "GPL v2 or later (details at http://www.gnu.org)" 
   9   
  10  # std libs 
  11  import sys 
  12  import os 
  13  import os.path 
  14  import csv 
  15  import tempfile 
  16  import logging 
  17  import hashlib 
  18  import platform 
  19  import subprocess 
  20  import decimal 
  21  import getpass 
  22  import io 
  23  import functools 
  24  import json 
  25  import shutil 
  26  import zipfile 
  27  import datetime as pydt 
  28  import re as regex 
  29  import xml.sax.saxutils as xml_tools 
  30  # old: 
  31  import pickle, zlib 
  32  # docutils 
  33  du_core = None 
  34   
  35   
  36  # GNUmed libs 
  37  if __name__ == '__main__': 
  38          sys.path.insert(0, '../../') 
  39  from Gnumed.pycommon import gmBorg 
  40   
  41   
  42  _log = logging.getLogger('gm.tools') 
  43   
  44  # CAPitalization modes: 
  45  (       CAPS_NONE,                                      # don't touch it 
  46          CAPS_FIRST,                                     # CAP first char, leave rest as is 
  47          CAPS_ALLCAPS,                           # CAP all chars 
  48          CAPS_WORDS,                                     # CAP first char of every word 
  49          CAPS_NAMES,                                     # CAP in a way suitable for names (tries to be smart) 
  50          CAPS_FIRST_ONLY                         # CAP first char, lowercase the rest 
  51  ) = range(6) 
  52   
  53   
  54  u_currency_pound = '\u00A3'                             # Pound sign 
  55  u_currency_sign = '\u00A4'                                      # generic currency sign 
  56  u_currency_yen = '\u00A5'                                       # Yen sign 
  57  u_right_double_angle_quote = '\u00AB'           # << 
  58  u_registered_trademark = '\u00AE' 
  59  u_plus_minus = '\u00B1' 
  60  u_superscript_one = '\u00B9'                            # ^1 
  61  u_left_double_angle_quote = '\u00BB'            # >> 
  62  u_one_quarter = '\u00BC' 
  63  u_one_half = '\u00BD' 
  64  u_three_quarters = '\u00BE' 
  65  u_multiply = '\u00D7'                                           # x 
  66  u_greek_ALPHA = '\u0391' 
  67  u_greek_alpha = '\u03b1' 
  68  u_greek_OMEGA = '\u03A9' 
  69  u_greek_omega = '\u03c9' 
  70  u_dagger = '\u2020' 
  71  u_triangular_bullet = '\u2023'                                  # triangular bullet  (>) 
  72  u_ellipsis = '\u2026'                                                   # ... 
  73  u_euro = '\u20AC'                                                               # EURO sign 
  74  u_numero = '\u2116'                                                             # No. / # sign 
  75  u_down_left_arrow = '\u21B5'                                    # <-' 
  76  u_left_arrow = '\u2190'                                                 # <-- 
  77  u_up_arrow = '\u2191' 
  78  u_arrow2right = '\u2192'                                                # --> 
  79  u_down_arrow = '\u2193' 
  80  u_left_arrow_with_tail = '\u21a2'                               # <--< 
  81  u_arrow2right_from_bar = '\u21a6'                               # |-> 
  82  u_arrow2right_until_vertical_bar = '\u21e5'             # -->| 
  83  u_sum = '\u2211'                                                                # sigma 
  84  u_almost_equal_to = '\u2248'                                    # approximately / nearly / roughly 
  85  u_corresponds_to = '\u2258' 
  86  u_infinity = '\u221E' 
  87  u_arrow2right_until_vertical_bar2 = '\u2b72'    # -->| 
  88   
  89  u_diameter = '\u2300' 
  90  u_checkmark_crossed_out = '\u237B' 
  91  u_box_horiz_high = '\u23ba' 
  92  u_box_vert_left = '\u23b8' 
  93  u_box_vert_right = '\u23b9' 
  94   
  95  u_box_horiz_single = '\u2500'                           # - 
  96  u_box_vert_light = '\u2502' 
  97  u_box_horiz_light_3dashes = '\u2504'            # ... 
  98  u_box_vert_light_4dashes = '\u2506' 
  99  u_box_horiz_4dashes = '\u2508'                          # .... 
 100  u_box_T_right = '\u251c'                                        # |- 
 101  u_box_T_left = '\u2524'                                         # -| 
 102  u_box_T_down = '\u252c' 
 103  u_box_T_up = '\u2534' 
 104  u_box_plus = '\u253c' 
 105  u_box_top_double = '\u2550' 
 106  u_box_top_left_double_single = '\u2552' 
 107  u_box_top_right_double_single = '\u2555' 
 108  u_box_top_left_arc = '\u256d' 
 109  u_box_top_right_arc = '\u256e' 
 110  u_box_bottom_right_arc = '\u256f' 
 111  u_box_bottom_left_arc = '\u2570' 
 112  u_box_horiz_light_heavy = '\u257c' 
 113  u_box_horiz_heavy_light = '\u257e' 
 114   
 115  u_skull_and_crossbones = '\u2620' 
 116  u_caduceus = '\u2624' 
 117  u_frowning_face = '\u2639' 
 118  u_smiling_face = '\u263a' 
 119  u_black_heart = '\u2665' 
 120  u_female = '\u2640' 
 121  u_male = '\u2642' 
 122  u_male_female = '\u26a5' 
 123  u_chain = '\u26d3' 
 124   
 125  u_checkmark_thin = '\u2713' 
 126  u_checkmark_thick = '\u2714' 
 127  u_heavy_greek_cross = '\u271a' 
 128  u_arrow2right_thick = '\u2794' 
 129  u_writing_hand = '\u270d' 
 130  u_pencil_1 = '\u270e' 
 131  u_pencil_2 = '\u270f' 
 132  u_pencil_3 = '\u2710' 
 133  u_latin_cross = '\u271d' 
 134   
 135  u_arrow2right_until_black_diamond = '\u291e'    # ->* 
 136   
 137  u_kanji_yen = '\u5186'                                                  # Yen kanji 
 138  u_replacement_character = '\ufffd' 
 139  u_link_symbol = '\u1f517' 
 140   
 141  _kB = 1024 
 142  _MB = 1024 * _kB 
 143  _GB = 1024 * _MB 
 144  _TB = 1024 * _GB 
 145  _PB = 1024 * _TB 
 146   
 147  _GM_TITLE_PREFIX = 'GMd' 
 148   
 149  #=========================================================================== 
150 -def handle_uncaught_exception_console(t, v, tb):
151 152 print(".========================================================") 153 print("| Unhandled exception caught !") 154 print("| Type :", t) 155 print("| Value:", v) 156 print("`========================================================") 157 _log.critical('unhandled exception caught', exc_info = (t,v,tb)) 158 sys.__excepthook__(t,v,tb)
159 160 #=========================================================================== 161 # path level operations 162 #---------------------------------------------------------------------------
163 -def mkdir(directory=None, mode=None):
164 """Create directory. 165 166 - creates parent dirs if necessary 167 - does not fail if directory exists 168 <mode>: numeric, say 0o0700 for "-rwx------" 169 """ 170 if os.path.isdir(directory): 171 if mode is None: 172 return True 173 try: 174 old_umask = os.umask(0) 175 # does not WORK ! 176 #os.chmod(directory, mode, follow_symlinks = (os.chmod in os.supports_follow_symlinks)) # can't do better 177 os.chmod(directory, mode) 178 return True 179 except Exception: 180 _log.exception('cannot os.chmod(%s, %s)', oct(mode), directory) 181 raise 182 finally: 183 os.umask(old_umask) 184 return False 185 186 if mode is None: 187 os.makedirs(directory) 188 return True 189 190 try: 191 old_umask = os.umask(0) 192 os.makedirs(directory, mode) 193 return True 194 except Exception: 195 _log.exception('cannot os.makedirs(%s, %s)', oct(mode), directory) 196 raise 197 finally: 198 os.umask(old_umask) 199 return True
200 201 #---------------------------------------------------------------------------
202 -def rmdir(directory):
203 #------------------------------- 204 def _on_rm_error(func, path, exc): 205 _log.error('error while shutil.rmtree(%s)', path, exc_info=exc) 206 return True
207 #------------------------------- 208 error_count = 0 209 try: 210 shutil.rmtree(directory, False, _on_rm_error) 211 except Exception: 212 _log.exception('cannot shutil.rmtree(%s)', directory) 213 error_count += 1 214 return error_count 215 216 #---------------------------------------------------------------------------
217 -def rm_dir_content(directory):
218 _log.debug('cleaning out [%s]', directory) 219 try: 220 items = os.listdir(directory) 221 except OSError: 222 return False 223 for item in items: 224 # attempt file/link removal and ignore (but log) errors 225 full_item = os.path.join(directory, item) 226 try: 227 os.remove(full_item) 228 except OSError: # as per the docs, this is a directory 229 _log.debug('[%s] seems to be a subdirectory', full_item) 230 errors = rmdir(full_item) 231 if errors > 0: 232 return False 233 except Exception: 234 _log.exception('cannot os.remove(%s) [a file or a link]', full_item) 235 return False 236 237 return True
238 239 #---------------------------------------------------------------------------
240 -def mk_sandbox_dir(prefix=None, base_dir=None):
241 if base_dir is None: 242 base_dir = gmPaths().tmp_dir 243 else: 244 if not os.path.isdir(base_dir): 245 mkdir(base_dir, mode = 0o0700) # (invoking user only) 246 if prefix is None: 247 prefix = 'sndbx-' 248 return tempfile.mkdtemp(prefix = prefix, suffix = '', dir = base_dir)
249 250 #---------------------------------------------------------------------------
251 -def parent_dir(directory):
252 return os.path.abspath(os.path.join(directory, '..'))
253 254 #---------------------------------------------------------------------------
255 -def dirname_stem(directory):
256 # /home/user/dir/ -> dir 257 # /home/user/dir -> dir 258 return os.path.basename(os.path.normpath(directory)) # normpath removes trailing slashes if any
259 260 #---------------------------------------------------------------------------
261 -def dir_is_empty(directory=None):
262 try: 263 return len(os.listdir(directory)) == 0 264 except OSError as exc: 265 if exc.errno == 2: # no such file 266 return None 267 raise
268 269 #---------------------------------------------------------------------------
270 -def copy_tree_content(directory, target_directory):
271 """Copy the *content* of <directory> *into* <target_directory> 272 which is created if need be. 273 """ 274 assert (directory is not None), 'source <directory> should not be None' 275 _log.debug('copying content of [%s] into [%s]', directory, target_directory) 276 try: 277 items = os.listdir(directory) 278 except OSError: 279 return None 280 281 for item in items: 282 full_item = os.path.join(directory, item) 283 if os.path.isdir(full_item): 284 target_subdir = os.path.join(target_directory, item) 285 try: 286 shutil.copytree(full_item, target_subdir) 287 except Exception: 288 _log.exception('cannot copy subdir [%s]', full_item) 289 return None 290 else: 291 try: 292 shutil.copy2(full_item, target_directory) 293 except Exception: 294 _log.exception('cannot copy file [%s]', full_item) 295 return None 296 297 return target_directory
298 299 #--------------------------------------------------------------------------- 300 #---------------------------------------------------------------------------
301 -class gmPaths(gmBorg.cBorg):
302 """This class provides the following paths: 303 304 .home_dir user home 305 .local_base_dir script installation dir 306 .working_dir current dir 307 .user_config_dir 308 .system_config_dir 309 .system_app_data_dir (not writable) 310 .tmp_dir instance-local 311 .user_tmp_dir user-local (NOT per instance) 312 .bytea_cache_dir caches downloaded BYTEA data 313 """
314 - def __init__(self, app_name=None, wx=None):
315 """Setup pathes. 316 317 <app_name> will default to (name of the script - .py) 318 """ 319 try: 320 self.already_inited 321 return 322 except AttributeError: 323 pass 324 325 self.init_paths(app_name=app_name, wx=wx) 326 self.already_inited = True
327 328 #-------------------------------------- 329 # public API 330 #--------------------------------------
331 - def init_paths(self, app_name=None, wx=None):
332 333 if wx is None: 334 _log.debug('wxPython not available') 335 _log.debug('detecting paths directly') 336 337 if app_name is None: 338 app_name, ext = os.path.splitext(os.path.basename(sys.argv[0])) 339 _log.info('app name detected as [%s]', app_name) 340 else: 341 _log.info('app name passed in as [%s]', app_name) 342 343 # the user home, doesn't work in Wine so work around that 344 self.__home_dir = None 345 346 # where the main script (the "binary") is installed 347 if getattr(sys, 'frozen', False): 348 _log.info('frozen app, installed into temporary path') 349 # this would find the path of *THIS* file 350 #self.local_base_dir = os.path.dirname(__file__) 351 # while this is documented on the web, the ${_MEIPASS2} does not exist 352 #self.local_base_dir = os.environ.get('_MEIPASS2') 353 # this is what Martin Zibricky <mzibr.public@gmail.com> told us to use 354 # when asking about this on pyinstaller@googlegroups.com 355 #self.local_base_dir = sys._MEIPASS 356 # however, we are --onedir, so we should look at sys.executable 357 # as per the pyinstaller manual 358 self.local_base_dir = os.path.dirname(sys.executable) 359 else: 360 self.local_base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) 361 362 # the current working dir at the OS 363 self.working_dir = os.path.abspath(os.curdir) 364 365 # user-specific config dir, usually below the home dir 366 mkdir(os.path.join(self.home_dir, '.%s' % app_name)) 367 self.user_config_dir = os.path.join(self.home_dir, '.%s' % app_name) 368 369 # system-wide config dir, usually below /etc/ under UN*X 370 try: 371 self.system_config_dir = os.path.join('/etc', app_name) 372 except ValueError: 373 #self.system_config_dir = self.local_base_dir 374 self.system_config_dir = self.user_config_dir 375 376 # system-wide application data dir 377 try: 378 self.system_app_data_dir = os.path.join(sys.prefix, 'share', app_name) 379 except ValueError: 380 self.system_app_data_dir = self.local_base_dir 381 382 # temporary directory 383 try: 384 self.__tmp_dir_already_set 385 _log.debug('temp dir already set') 386 except AttributeError: 387 _log.info('temp file prefix: %s', tempfile.gettempprefix()) 388 _log.info('initial (user level) temp dir: %s', tempfile.gettempdir()) 389 bytes_free = shutil.disk_usage(tempfile.gettempdir()).free 390 _log.info('free disk space for temp dir: %s (%s bytes)', size2str(size = bytes_free), bytes_free) 391 # $TMP/gnumed-$USER/ 392 self.user_tmp_dir = os.path.join(tempfile.gettempdir(), app_name + '-' + getpass.getuser()) 393 mkdir(self.user_tmp_dir, 0o700) 394 tempfile.tempdir = self.user_tmp_dir 395 _log.info('intermediate (app level) temp dir: %s', tempfile.gettempdir()) 396 # $TMP/gnumed-$USER/g-$UNIQUE/ 397 self.tmp_dir = tempfile.mkdtemp(prefix = 'g-') 398 _log.info('final (app instance level) temp dir: %s', tempfile.gettempdir()) 399 400 # BYTEA cache dir 401 cache_dir = os.path.join(self.user_tmp_dir, '.bytea_cache') 402 try: 403 stat = os.stat(cache_dir) 404 _log.warning('reusing BYTEA cache dir: %s', cache_dir) 405 _log.debug(stat) 406 except FileNotFoundError: 407 mkdir(cache_dir, mode = 0o0700) 408 self.bytea_cache_dir = cache_dir 409 410 self.__log_paths() 411 if wx is None: 412 return True 413 414 # retry with wxPython 415 _log.debug('re-detecting paths with wxPython') 416 417 std_paths = wx.StandardPaths.Get() 418 _log.info('wxPython app name is [%s]', wx.GetApp().GetAppName()) 419 420 # user-specific config dir, usually below the home dir 421 mkdir(os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name)) 422 self.user_config_dir = os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name) 423 424 # system-wide config dir, usually below /etc/ under UN*X 425 try: 426 tmp = std_paths.GetConfigDir() 427 if not tmp.endswith(app_name): 428 tmp = os.path.join(tmp, app_name) 429 self.system_config_dir = tmp 430 except ValueError: 431 # leave it at what it was from direct detection 432 pass 433 434 # system-wide application data dir 435 # Robin attests that the following doesn't always 436 # give sane values on Windows, so IFDEF it 437 if 'wxMSW' in wx.PlatformInfo: 438 _log.warning('this platform (wxMSW) sometimes returns a broken value for the system-wide application data dir') 439 else: 440 try: 441 self.system_app_data_dir = std_paths.GetDataDir() 442 except ValueError: 443 pass 444 445 self.__log_paths() 446 return True
447 448 #--------------------------------------
449 - def __log_paths(self):
450 _log.debug('sys.argv[0]: %s', sys.argv[0]) 451 _log.debug('sys.executable: %s', sys.executable) 452 _log.debug('sys._MEIPASS: %s', getattr(sys, '_MEIPASS', '<not found>')) 453 _log.debug('os.environ["_MEIPASS2"]: %s', os.environ.get('_MEIPASS2', '<not found>')) 454 _log.debug('__file__ : %s', __file__) 455 _log.debug('local application base dir: %s', self.local_base_dir) 456 _log.debug('current working dir: %s', self.working_dir) 457 _log.debug('user home dir: %s', self.home_dir) 458 _log.debug('user-specific config dir: %s', self.user_config_dir) 459 _log.debug('system-wide config dir: %s', self.system_config_dir) 460 _log.debug('system-wide application data dir: %s', self.system_app_data_dir) 461 _log.debug('temporary dir (user): %s', self.user_tmp_dir) 462 _log.debug('temporary dir (instance): %s', self.tmp_dir) 463 _log.debug('BYTEA cache dir: %s', self.bytea_cache_dir)
464 465 #-------------------------------------- 466 # properties 467 #--------------------------------------
468 - def _set_user_config_dir(self, path):
469 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 470 msg = '[%s:user_config_dir]: invalid path [%s]' % (self.__class__.__name__, path) 471 _log.error(msg) 472 raise ValueError(msg) 473 self.__user_config_dir = path
474
475 - def _get_user_config_dir(self):
476 return self.__user_config_dir
477 478 user_config_dir = property(_get_user_config_dir, _set_user_config_dir) 479 #--------------------------------------
480 - def _set_system_config_dir(self, path):
481 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 482 msg = '[%s:system_config_dir]: invalid path [%s]' % (self.__class__.__name__, path) 483 _log.error(msg) 484 raise ValueError(msg) 485 self.__system_config_dir = path
486
487 - def _get_system_config_dir(self):
488 return self.__system_config_dir
489 490 system_config_dir = property(_get_system_config_dir, _set_system_config_dir) 491 #--------------------------------------
492 - def _set_system_app_data_dir(self, path):
493 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 494 msg = '[%s:system_app_data_dir]: invalid path [%s]' % (self.__class__.__name__, path) 495 _log.error(msg) 496 raise ValueError(msg) 497 self.__system_app_data_dir = path
498
499 - def _get_system_app_data_dir(self):
500 return self.__system_app_data_dir
501 502 system_app_data_dir = property(_get_system_app_data_dir, _set_system_app_data_dir) 503 #--------------------------------------
504 - def _set_home_dir(self, path):
505 raise ValueError('invalid to set home dir')
506
507 - def _get_home_dir(self):
508 if self.__home_dir is not None: 509 return self.__home_dir 510 511 tmp = os.path.expanduser('~') 512 if tmp == '~': 513 _log.error('this platform does not expand ~ properly') 514 try: 515 tmp = os.environ['USERPROFILE'] 516 except KeyError: 517 _log.error('cannot access $USERPROFILE in environment') 518 519 if not ( 520 os.access(tmp, os.R_OK) 521 and 522 os.access(tmp, os.X_OK) 523 and 524 os.access(tmp, os.W_OK) 525 ): 526 msg = '[%s:home_dir]: invalid path [%s]' % (self.__class__.__name__, tmp) 527 _log.error(msg) 528 raise ValueError(msg) 529 530 self.__home_dir = tmp 531 return self.__home_dir
532 533 home_dir = property(_get_home_dir, _set_home_dir) 534 535 #--------------------------------------
536 - def _set_tmp_dir(self, path):
537 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 538 msg = '[%s:tmp_dir]: invalid path [%s]' % (self.__class__.__name__, path) 539 _log.error(msg) 540 raise ValueError(msg) 541 _log.debug('previous temp dir: %s', tempfile.gettempdir()) 542 self.__tmp_dir = path 543 tempfile.tempdir = self.__tmp_dir 544 _log.debug('new temp dir: %s', tempfile.gettempdir()) 545 self.__tmp_dir_already_set = True
546
547 - def _get_tmp_dir(self):
548 return self.__tmp_dir
549 550 tmp_dir = property(_get_tmp_dir, _set_tmp_dir)
551 552 #=========================================================================== 553 # file related tools 554 #---------------------------------------------------------------------------
555 -def recode_file(source_file=None, target_file=None, source_encoding='utf8', target_encoding=None, base_dir=None, error_mode='replace'):
556 if target_encoding is None: 557 return source_file 558 if target_encoding == source_encoding: 559 return source_file 560 if target_file is None: 561 target_file = get_unique_filename ( 562 prefix = '%s-%s_%s-' % (fname_stem(source_file), source_encoding, target_encoding), 563 suffix = fname_extension(source_file, '.txt'), 564 tmp_dir = base_dir 565 ) 566 567 _log.debug('[%s] -> [%s] (%s -> %s)', source_encoding, target_encoding, source_file, target_file) 568 569 in_file = io.open(source_file, mode = 'rt', encoding = source_encoding) 570 out_file = io.open(target_file, mode = 'wt', encoding = target_encoding, errors = error_mode) 571 for line in in_file: 572 out_file.write(line) 573 out_file.close() 574 in_file.close() 575 576 return target_file
577 578 #---------------------------------------------------------------------------
579 -def unzip_archive(archive_name, target_dir=None, remove_archive=False):
580 _log.debug('unzipping [%s] -> [%s]', archive_name, target_dir) 581 success = False 582 try: 583 with zipfile.ZipFile(archive_name) as archive: 584 archive.extractall(target_dir) 585 success = True 586 except Exception: 587 _log.exception('cannot unzip') 588 return False 589 if remove_archive: 590 remove_file(archive_name) 591 return success
592 593 #---------------------------------------------------------------------------
594 -def remove_file(filename, log_error=True, force=False):
595 if not os.path.lexists(filename): 596 return True 597 598 # attempt file remove and ignore (but log) errors 599 try: 600 os.remove(filename) 601 return True 602 603 except Exception: 604 if log_error: 605 _log.exception('cannot os.remove(%s)', filename) 606 607 if force: 608 tmp_name = get_unique_filename(tmp_dir = fname_dir(filename)) 609 _log.debug('attempting os.replace(%s -> %s)', filename, tmp_name) 610 try: 611 os.replace(filename, tmp_name) 612 return True 613 614 except Exception: 615 if log_error: 616 _log.exception('cannot os.remove(%s)', filename) 617 618 return False
619 620 #---------------------------------------------------------------------------
621 -def file2md5(filename=None, return_hex=True):
622 blocksize = 2**10 * 128 # 128k, since md5 uses 128 byte blocks 623 _log.debug('md5(%s): <%s> byte blocks', filename, blocksize) 624 625 f = io.open(filename, mode = 'rb') 626 627 md5 = hashlib.md5() 628 while True: 629 data = f.read(blocksize) 630 if not data: 631 break 632 md5.update(data) 633 f.close() 634 635 _log.debug('md5(%s): %s', filename, md5.hexdigest()) 636 637 if return_hex: 638 return md5.hexdigest() 639 return md5.digest()
640 641 #---------------------------------------------------------------------------
642 -def file2chunked_md5(filename=None, chunk_size=500*_MB):
643 _log.debug('chunked_md5(%s, chunk_size=%s bytes)', filename, chunk_size) 644 md5_concat = '' 645 f = open(filename, 'rb') 646 while True: 647 md5 = hashlib.md5() 648 data = f.read(chunk_size) 649 if not data: 650 break 651 md5.update(data) 652 md5_concat += md5.hexdigest() 653 f.close() 654 md5 = hashlib.md5() 655 md5.update(md5_concat) 656 hex_digest = md5.hexdigest() 657 _log.debug('md5("%s"): %s', md5_concat, hex_digest) 658 return hex_digest
659 660 #--------------------------------------------------------------------------- 661 default_csv_reader_rest_key = 'list_of_values_of_unknown_fields' 662
663 -def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
664 try: 665 is_dict_reader = kwargs['dict'] 666 del kwargs['dict'] 667 except KeyError: 668 is_dict_reader = False 669 670 if is_dict_reader: 671 kwargs['restkey'] = default_csv_reader_rest_key 672 return csv.DictReader(unicode_csv_data, dialect=dialect, **kwargs) 673 return csv.reader(unicode_csv_data, dialect=dialect, **kwargs)
674 675 676 677
678 -def old_unicode2charset_encoder(unicode_csv_data, encoding='utf-8'):
679 for line in unicode_csv_data: 680 yield line.encode(encoding)
681 682 #def utf_8_encoder(unicode_csv_data): 683 # for line in unicode_csv_data: 684 # yield line.encode('utf-8') 685
686 -def old_unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
687 688 # csv.py doesn't do Unicode; encode temporarily as UTF-8: 689 try: 690 is_dict_reader = kwargs['dict'] 691 del kwargs['dict'] 692 if is_dict_reader is not True: 693 raise KeyError 694 kwargs['restkey'] = default_csv_reader_rest_key 695 csv_reader = csv.DictReader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs) 696 except KeyError: 697 is_dict_reader = False 698 csv_reader = csv.reader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs) 699 700 for row in csv_reader: 701 # decode ENCODING back to Unicode, cell by cell: 702 if is_dict_reader: 703 for key in row.keys(): 704 if key == default_csv_reader_rest_key: 705 old_data = row[key] 706 new_data = [] 707 for val in old_data: 708 new_data.append(str(val, encoding)) 709 row[key] = new_data 710 if default_csv_reader_rest_key not in csv_reader.fieldnames: 711 csv_reader.fieldnames.append(default_csv_reader_rest_key) 712 else: 713 row[key] = str(row[key], encoding) 714 yield row 715 else: 716 yield [ str(cell, encoding) for cell in row ]
717 #yield [str(cell, 'utf-8') for cell in row] 718 719 #---------------------------------------------------------------------------
720 -def fname_sanitize(filename):
721 """Normalizes unicode, removes non-alpha characters, converts spaces to underscores.""" 722 723 dir_part, name_part = os.path.split(filename) 724 if name_part == '': 725 return filename 726 727 import unicodedata 728 name_part = unicodedata.normalize('NFKD', name_part) 729 # remove everything not in group [] 730 name_part = regex.sub ( 731 '[^.\w\s[\]()%§+-]', 732 '', 733 name_part, 734 flags = regex.UNICODE 735 ).strip() 736 # translate whitespace to underscore 737 name_part = regex.sub ( 738 '\s+', 739 '_', 740 name_part, 741 flags = regex.UNICODE 742 ) 743 return os.path.join(dir_part, name_part)
744 745 #---------------------------------------------------------------------------
746 -def fname_stem(filename):
747 """/home/user/dir/filename.ext -> filename""" 748 return os.path.splitext(os.path.basename(filename))[0]
749 750 #---------------------------------------------------------------------------
751 -def fname_stem_with_path(filename):
752 """/home/user/dir/filename.ext -> /home/user/dir/filename""" 753 return os.path.splitext(filename)[0]
754 755 #---------------------------------------------------------------------------
756 -def fname_extension(filename=None, fallback=None):
757 """ /home/user/dir/filename.ext -> .ext 758 '' or '.' -> fallback if any else '' 759 """ 760 ext = os.path.splitext(filename)[1] 761 if ext.strip() not in ['.', '']: 762 return ext 763 if fallback is None: 764 return '' 765 return fallback
766 767 #---------------------------------------------------------------------------
768 -def fname_dir(filename):
769 # /home/user/dir/filename.ext -> /home/user/dir 770 return os.path.split(filename)[0]
771 772 #---------------------------------------------------------------------------
773 -def fname_from_path(filename):
774 # /home/user/dir/filename.ext -> filename.ext 775 return os.path.split(filename)[1]
776 777 #---------------------------------------------------------------------------
778 -def get_unique_filename(prefix=None, suffix=None, tmp_dir=None, include_timestamp=False):
779 """This function has a race condition between 780 its file.close() 781 and actually 782 using the filename in callers. 783 784 The file will NOT exist after calling this function. 785 """ 786 if tmp_dir is not None: 787 if ( 788 not os.access(tmp_dir, os.F_OK) 789 or 790 not os.access(tmp_dir, os.X_OK | os.W_OK) 791 ): 792 _log.warning('cannot os.access() temporary dir [%s], using system default', tmp_dir) 793 tmp_dir = None 794 795 if include_timestamp: 796 ts = pydt.datetime.now().strftime('%m%d-%H%M%S-') 797 else: 798 ts = '' 799 800 kwargs = { 801 'dir': tmp_dir, 802 # make sure file gets deleted as soon as 803 # .close()d so we can "safely" open it again 804 'delete': True 805 } 806 807 if prefix is None: 808 kwargs['prefix'] = 'gm-%s' % ts 809 else: 810 kwargs['prefix'] = prefix + ts 811 812 if suffix in [None, '']: 813 kwargs['suffix'] = '.tmp' 814 else: 815 if not suffix.startswith('.'): 816 suffix = '.' + suffix 817 kwargs['suffix'] = suffix 818 819 f = tempfile.NamedTemporaryFile(**kwargs) 820 filename = f.name 821 f.close() 822 823 return filename
824 825 #--------------------------------------------------------------------------- 842 843 #--------------------------------------------------------------------------- 867 868 #===========================================================================
869 -def import_module_from_directory(module_path=None, module_name=None, always_remove_path=False):
870 """Import a module from any location.""" 871 872 _log.debug('CWD: %s', os.getcwd()) 873 874 remove_path = always_remove_path or False 875 if module_path not in sys.path: 876 _log.info('appending to sys.path: [%s]' % module_path) 877 sys.path.append(module_path) 878 remove_path = True 879 880 _log.debug('will remove import path: %s', remove_path) 881 882 if module_name.endswith('.py'): 883 module_name = module_name[:-3] 884 885 try: 886 module = __import__(module_name) 887 except Exception: 888 _log.exception('cannot __import__() module [%s] from [%s]' % (module_name, module_path)) 889 while module_path in sys.path: 890 sys.path.remove(module_path) 891 raise 892 893 _log.info('imported module [%s] as [%s]' % (module_name, module)) 894 if remove_path: 895 while module_path in sys.path: 896 sys.path.remove(module_path) 897 898 return module
899 900 #=========================================================================== 901 # text related tools 902 #---------------------------------------------------------------------------
903 -def size2str(size=0, template='%s'):
904 if size == 1: 905 return template % _('1 Byte') 906 if size < 10 * _kB: 907 return template % _('%s Bytes') % size 908 if size < _MB: 909 return template % '%.1f kB' % (float(size) / _kB) 910 if size < _GB: 911 return template % '%.1f MB' % (float(size) / _MB) 912 if size < _TB: 913 return template % '%.1f GB' % (float(size) / _GB) 914 if size < _PB: 915 return template % '%.1f TB' % (float(size) / _TB) 916 return template % '%.1f PB' % (float(size) / _PB)
917 918 #---------------------------------------------------------------------------
919 -def bool2subst(boolean=None, true_return=True, false_return=False, none_return=None):
920 if boolean is None: 921 return none_return 922 if boolean: 923 return true_return 924 if not boolean: 925 return false_return 926 raise ValueError('bool2subst(): <boolean> arg must be either of True, False, None')
927 928 #---------------------------------------------------------------------------
929 -def bool2str(boolean=None, true_str='True', false_str='False'):
930 return bool2subst ( 931 boolean = bool(boolean), 932 true_return = true_str, 933 false_return = false_str 934 )
935 936 #---------------------------------------------------------------------------
937 -def none_if(value=None, none_equivalent=None, strip_string=False):
938 """Modelled after the SQL NULLIF function.""" 939 if value is None: 940 return None 941 if strip_string: 942 stripped = value.strip() 943 else: 944 stripped = value 945 if stripped == none_equivalent: 946 return None 947 return value
948 949 #---------------------------------------------------------------------------
950 -def coalesce(value2test=None, return_instead=None, template4value=None, template4instead=None, none_equivalents=None, function4value=None, value2return=None):
951 """Modelled after the SQL coalesce function. 952 953 To be used to simplify constructs like: 954 955 if value2test is None (or in none_equivalents): 956 value = (template4instead % return_instead) or return_instead 957 else: 958 value = (template4value % value2test) or value2test 959 print value 960 961 @param value2test: the value to be tested for <None> 962 @type value2test: any Python type, must have a __str__ method if template4value is not None 963 @param return_instead: the value to be returned if <initial> is None 964 @type return_instead: any Python type, must have a __str__ method if template4instead is not None 965 @param template4value: if <initial> is returned replace the value into this template, must contain one <%s> 966 @type template4value: string or None 967 @param template4instead: if <return_instead> is returned replace the value into this template, must contain one <%s> 968 @type template4instead: string or None 969 970 example: 971 function4value = ('strftime', '%Y-%m-%d') 972 973 Ideas: 974 - list of return_insteads: initial, [return_instead, template], [return_instead, template], [return_instead, template], template4value, ... 975 """ 976 if none_equivalents is None: 977 none_equivalents = [None] 978 979 if value2test in none_equivalents: 980 if template4instead is None: 981 return return_instead 982 return template4instead % return_instead 983 984 # at this point, value2test was not equivalent to None 985 986 # 1) explicit value to return supplied ? 987 if value2return is not None: 988 return value2return 989 990 value2return = value2test 991 # 2) function supplied to be applied to the value ? 992 if function4value is not None: 993 funcname, args = function4value 994 func = getattr(value2test, funcname) 995 value2return = func(args) 996 997 # 3) template supplied to be applied to the value ? 998 if template4value is None: 999 return value2return 1000 1001 try: 1002 return template4value % value2return 1003 except TypeError: 1004 # except (TypeError, ValueError): 1005 # this should go, actually, only needed because "old" calls 1006 # to coalesce will still abuse template4value as explicit value2return, 1007 # relying on the replacement to above to fail 1008 if hasattr(_log, 'log_stack_trace'): 1009 _log.log_stack_trace(message = 'deprecated use of <template4value> for <value2return>') 1010 else: 1011 _log.error('deprecated use of <template4value> for <value2return>') 1012 _log.error(locals()) 1013 return template4value
1014 1015 #---------------------------------------------------------------------------
1016 -def __cap_name(match_obj=None):
1017 val = match_obj.group(0).lower() 1018 if val in ['von', 'van', 'de', 'la', 'l', 'der', 'den']: # FIXME: this needs to expand, configurable ? 1019 return val 1020 buf = list(val) 1021 buf[0] = buf[0].upper() 1022 for part in ['mac', 'mc', 'de', 'la']: 1023 if len(val) > len(part) and val[:len(part)] == part: 1024 buf[len(part)] = buf[len(part)].upper() 1025 return ''.join(buf)
1026 1027 #---------------------------------------------------------------------------
1028 -def capitalize(text=None, mode=CAPS_NAMES):
1029 """Capitalize the first character but leave the rest alone. 1030 1031 Note that we must be careful about the locale, this may 1032 have issues ! However, for UTF strings it should just work. 1033 """ 1034 if (mode is None) or (mode == CAPS_NONE): 1035 return text 1036 1037 if len(text) == 0: 1038 return text 1039 1040 if mode == CAPS_FIRST: 1041 if len(text) == 1: 1042 return text[0].upper() 1043 return text[0].upper() + text[1:] 1044 1045 if mode == CAPS_ALLCAPS: 1046 return text.upper() 1047 1048 if mode == CAPS_FIRST_ONLY: 1049 # if len(text) == 1: 1050 # return text[0].upper() 1051 return text[0].upper() + text[1:].lower() 1052 1053 if mode == CAPS_WORDS: 1054 #return regex.sub(ur'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text) 1055 return regex.sub(r'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text) 1056 1057 if mode == CAPS_NAMES: 1058 #return regex.sub(r'\w+', __cap_name, text) 1059 return capitalize(text=text, mode=CAPS_FIRST) # until fixed 1060 1061 print("ERROR: invalid capitalization mode: [%s], leaving input as is" % mode) 1062 return text
1063 1064 #---------------------------------------------------------------------------
1065 -def input2decimal(initial=None):
1066 1067 if isinstance(initial, decimal.Decimal): 1068 return True, initial 1069 1070 val = initial 1071 1072 # float ? -> to string first 1073 if type(val) == type(float(1.4)): 1074 val = str(val) 1075 1076 # string ? -> "," to "." 1077 if isinstance(val, str): 1078 val = val.replace(',', '.', 1) 1079 val = val.strip() 1080 1081 try: 1082 d = decimal.Decimal(val) 1083 return True, d 1084 except (TypeError, decimal.InvalidOperation): 1085 return False, val
1086 1087 #---------------------------------------------------------------------------
1088 -def input2int(initial=None, minval=None, maxval=None):
1089 1090 val = initial 1091 1092 # string ? -> "," to "." 1093 if isinstance(val, str): 1094 val = val.replace(',', '.', 1) 1095 val = val.strip() 1096 1097 try: 1098 int_val = int(val) 1099 except (TypeError, ValueError): 1100 _log.exception('int(%s) failed', val) 1101 return False, initial 1102 1103 if minval is not None: 1104 if int_val < minval: 1105 _log.debug('%s < min (%s)', val, minval) 1106 return False, initial 1107 if maxval is not None: 1108 if int_val > maxval: 1109 _log.debug('%s > max (%s)', val, maxval) 1110 return False, initial 1111 1112 return True, int_val
1113 1114 #---------------------------------------------------------------------------
1115 -def strip_prefix(text, prefix, remove_repeats=False, remove_whitespace=False):
1116 if remove_whitespace: 1117 text = text.lstrip() 1118 if not text.startswith(prefix): 1119 return text 1120 1121 text = text.replace(prefix, '', 1) 1122 if not remove_repeats: 1123 if remove_whitespace: 1124 return text.lstrip() 1125 return text 1126 1127 return strip_prefix(text, prefix, remove_repeats = True, remove_whitespace = remove_whitespace)
1128 1129 #---------------------------------------------------------------------------
1130 -def decorate_window_title(title):
1131 if title.startswith(_GM_TITLE_PREFIX): 1132 return title 1133 return '%s: %s' % ( 1134 _GM_TITLE_PREFIX, 1135 title.lstrip() 1136 )
1137 1138 #---------------------------------------------------------------------------
1139 -def undecorate_window_title(title):
1140 return strip_prefix(title, _GM_TITLE_PREFIX + ':', remove_repeats = True, remove_whitespace = True)
1141 1142 #---------------------------------------------------------------------------
1143 -def strip_suffix(text, suffix, remove_repeats=False, remove_whitespace=False):
1144 suffix_len = len(suffix) 1145 if remove_repeats: 1146 if remove_whitespace: 1147 while text.rstrip().endswith(suffix): 1148 text = text.rstrip()[:-suffix_len].rstrip() 1149 return text 1150 while text.endswith(suffix): 1151 text = text[:-suffix_len] 1152 return text 1153 if remove_whitespace: 1154 return text.rstrip()[:-suffix_len].rstrip() 1155 return text[:-suffix_len]
1156 1157 #---------------------------------------------------------------------------
1158 -def strip_leading_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1159 if lines is None: 1160 lines = text.split(eol) 1161 1162 while True: 1163 if lines[0].strip(eol).strip() != '': 1164 break 1165 lines = lines[1:] 1166 1167 if return_list: 1168 return lines 1169 1170 return eol.join(lines)
1171 1172 #---------------------------------------------------------------------------
1173 -def strip_trailing_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1174 if lines is None: 1175 lines = text.split(eol) 1176 1177 while True: 1178 if lines[-1].strip(eol).strip() != '': 1179 break 1180 lines = lines[:-1] 1181 1182 if return_list: 1183 return lines 1184 1185 return eol.join(lines)
1186 1187 #---------------------------------------------------------------------------
1188 -def strip_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1189 return strip_trailing_empty_lines ( 1190 lines = strip_leading_empty_lines(lines = lines, text = text, eol = eol, return_list = True), 1191 text = None, 1192 eol = eol, 1193 return_list = return_list 1194 )
1195 1196 #---------------------------------------------------------------------------
1197 -def list2text(lines, initial_indent='', subsequent_indent='', eol='\n', strip_leading_empty_lines=True, strip_trailing_empty_lines=True, strip_trailing_whitespace=True, max_line_width=None):
1198 1199 if len(lines) == 0: 1200 return '' 1201 1202 if strip_leading_empty_lines: 1203 lines = strip_leading_empty_lines(lines = lines, eol = eol, return_list = True) 1204 1205 if strip_trailing_empty_lines: 1206 lines = strip_trailing_empty_lines(lines = lines, eol = eol, return_list = True) 1207 1208 if strip_trailing_whitespace: 1209 lines = [ l.rstrip() for l in lines ] 1210 1211 if max_line_width is not None: 1212 wrapped_lines = [] 1213 for l in lines: 1214 wrapped_lines.extend(wrap(l, max_line_width).split('\n')) 1215 lines = wrapped_lines 1216 1217 indented_lines = [initial_indent + lines[0]] 1218 indented_lines.extend([ subsequent_indent + l for l in lines[1:] ]) 1219 1220 return eol.join(indented_lines)
1221 1222 #---------------------------------------------------------------------------
1223 -def wrap(text=None, width=None, initial_indent='', subsequent_indent='', eol='\n'):
1224 """A word-wrap function that preserves existing line breaks 1225 and most spaces in the text. Expects that existing line 1226 breaks are posix newlines (\n). 1227 """ 1228 if width is None: 1229 return text 1230 1231 wrapped = initial_indent + functools.reduce ( 1232 lambda line, word, width=width: '%s%s%s' % ( 1233 line, 1234 ' \n'[(len(line) - line.rfind('\n') - 1 + len(word.split('\n',1)[0]) >= width)], 1235 word 1236 ), 1237 text.split(' ') 1238 ) 1239 if subsequent_indent != '': 1240 wrapped = ('\n%s' % subsequent_indent).join(wrapped.split('\n')) 1241 if eol != '\n': 1242 wrapped = wrapped.replace('\n', eol) 1243 return wrapped
1244 1245 #---------------------------------------------------------------------------
1246 -def unwrap(text=None, max_length=None, strip_whitespace=True, remove_empty_lines=True, line_separator = ' // '):
1247 1248 text = text.replace('\r', '') 1249 lines = text.split('\n') 1250 text = '' 1251 for line in lines: 1252 1253 if strip_whitespace: 1254 line = line.strip().strip('\t').strip() 1255 1256 if remove_empty_lines: 1257 if line == '': 1258 continue 1259 1260 text += ('%s%s' % (line, line_separator)) 1261 1262 text = text.rstrip(line_separator) 1263 1264 if max_length is not None: 1265 text = text[:max_length] 1266 1267 text = text.rstrip(line_separator) 1268 1269 return text
1270 1271 #---------------------------------------------------------------------------
1272 -def shorten_text(text=None, max_length=None):
1273 1274 if len(text) <= max_length: 1275 return text 1276 1277 return text[:max_length-1] + u_ellipsis
1278 1279 #---------------------------------------------------------------------------
1280 -def shorten_words_in_line(text=None, max_length=None, min_word_length=None, ignore_numbers=True, ellipsis=u_ellipsis):
1281 if text is None: 1282 return None 1283 if max_length is None: 1284 max_length = len(text) 1285 else: 1286 if len(text) <= max_length: 1287 return text 1288 old_words = regex.split('\s+', text, flags = regex.UNICODE) 1289 no_old_words = len(old_words) 1290 max_word_length = max(min_word_length, (max_length // no_old_words)) 1291 words = [] 1292 for word in old_words: 1293 if len(word) <= max_word_length: 1294 words.append(word) 1295 continue 1296 if ignore_numbers: 1297 tmp = word.replace('-', '').replace('+', '').replace('.', '').replace(',', '').replace('/', '').replace('&', '').replace('*', '') 1298 if tmp.isdigit(): 1299 words.append(word) 1300 continue 1301 words.append(word[:max_word_length] + ellipsis) 1302 return ' '.join(words)
1303 1304 #---------------------------------------------------------------------------
1305 -def xml_escape_string(text=None):
1306 """check for special XML characters and transform them""" 1307 return xml_tools.escape(text)
1308 1309 #---------------------------------------------------------------------------
1310 -def tex_escape_string(text=None, replace_known_unicode=True, replace_eol=False, keep_visual_eol=False):
1311 """Check for special TeX characters and transform them. 1312 1313 replace_eol: 1314 replaces "\n" with "\\newline" 1315 keep_visual_eol: 1316 replaces "\n" with "\\newline \n" such that 1317 both LaTeX will know to place a line break 1318 at this point as well as the visual formatting 1319 is preserved in the LaTeX source (think multi- 1320 row table cells) 1321 """ 1322 text = text.replace('\\', '\\textbackslash') # requires \usepackage{textcomp} in LaTeX source 1323 text = text.replace('^', '\\textasciicircum') 1324 text = text.replace('~', '\\textasciitilde') 1325 1326 text = text.replace('{', '\\{') 1327 text = text.replace('}', '\\}') 1328 text = text.replace('%', '\\%') 1329 text = text.replace('&', '\\&') 1330 text = text.replace('#', '\\#') 1331 text = text.replace('$', '\\$') 1332 text = text.replace('_', '\\_') 1333 if replace_eol: 1334 if keep_visual_eol: 1335 text = text.replace('\n', '\\newline \n') 1336 else: 1337 text = text.replace('\n', '\\newline ') 1338 1339 if replace_known_unicode: 1340 # this should NOT be replaced for Xe(La)Tex 1341 text = text.replace(u_euro, '\\EUR') # requires \usepackage{textcomp} in LaTeX source 1342 text = text.replace(u_sum, '$\\Sigma$') 1343 1344 return text
1345 1346 #---------------------------------------------------------------------------
1347 -def rst2latex_snippet(rst_text):
1348 global du_core 1349 if du_core is None: 1350 try: 1351 from docutils import core as du_core 1352 except ImportError: 1353 _log.warning('cannot turn ReST into LaTeX: docutils not installed') 1354 return tex_escape_string(text = rst_text) 1355 1356 parts = du_core.publish_parts ( 1357 source = rst_text.replace('\\', '\\\\'), 1358 source_path = '<internal>', 1359 writer_name = 'latex', 1360 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex', 1361 settings_overrides = { 1362 'input_encoding': 'unicode' # un-encoded unicode 1363 }, 1364 enable_exit_status = True # how to use ? 1365 ) 1366 return parts['body']
1367 1368 #---------------------------------------------------------------------------
1369 -def rst2html(rst_text, replace_eol=False, keep_visual_eol=False):
1370 global du_core 1371 if du_core is None: 1372 try: 1373 from docutils import core as du_core 1374 except ImportError: 1375 _log.warning('cannot turn ReST into HTML: docutils not installed') 1376 return html_escape_string(text = rst_text, replace_eol=False, keep_visual_eol=False) 1377 1378 parts = du_core.publish_parts ( 1379 source = rst_text.replace('\\', '\\\\'), 1380 source_path = '<internal>', 1381 writer_name = 'latex', 1382 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex', 1383 settings_overrides = { 1384 'input_encoding': 'unicode' # un-encoded unicode 1385 }, 1386 enable_exit_status = True # how to use ? 1387 ) 1388 return parts['body']
1389 1390 #---------------------------------------------------------------------------
1391 -def xetex_escape_string(text=None):
1392 # a web search did not reveal anything else for Xe(La)Tex 1393 # as opposed to LaTeX, except true unicode chars 1394 return tex_escape_string(text = text, replace_known_unicode = False)
1395 1396 #--------------------------------------------------------------------------- 1397 __html_escape_table = { 1398 "&": "&amp;", 1399 '"': "&quot;", 1400 "'": "&apos;", 1401 ">": "&gt;", 1402 "<": "&lt;", 1403 } 1404
1405 -def html_escape_string(text=None, replace_eol=False, keep_visual_eol=False):
1406 text = ''.join(__html_escape_table.get(char, char) for char in text) 1407 if replace_eol: 1408 if keep_visual_eol: 1409 text = text.replace('\n', '<br>\n') 1410 else: 1411 text = text.replace('\n', '<br>') 1412 return text
1413 1414 #---------------------------------------------------------------------------
1415 -def dict2json(obj):
1416 return json.dumps(obj, default = json_serialize)
1417 1418 #---------------------------------------------------------------------------
1419 -def json_serialize(obj):
1420 if isinstance(obj, pydt.datetime): 1421 return obj.isoformat() 1422 raise TypeError('cannot json_serialize(%s)' % type(obj))
1423 1424 #--------------------------------------------------------------------------- 1425 #---------------------------------------------------------------------------
1426 -def compare_dict_likes(d1, d2, title1=None, title2=None):
1427 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title1, '', '"%s" '), type(d1), coalesce(title2, '', '"%s" '), type(d2)) 1428 try: 1429 d1 = dict(d1) 1430 except TypeError: 1431 pass 1432 try: 1433 d2 = dict(d2) 1434 except TypeError: 1435 pass 1436 keys_d1 = frozenset(d1.keys()) 1437 keys_d2 = frozenset(d2.keys()) 1438 different = False 1439 if len(keys_d1) != len(keys_d2): 1440 _log.info('different number of keys: %s vs %s', len(keys_d1), len(keys_d2)) 1441 different = True 1442 for key in keys_d1: 1443 if key in keys_d2: 1444 if type(d1[key]) != type(d2[key]): 1445 _log.info('%25.25s: type(dict1) = %s = >>>%s<<<' % (key, type(d1[key]), d1[key])) 1446 _log.info('%25.25s type(dict2) = %s = >>>%s<<<' % ('', type(d2[key]), d2[key])) 1447 different = True 1448 continue 1449 if d1[key] == d2[key]: 1450 _log.info('%25.25s: both = >>>%s<<<' % (key, d1[key])) 1451 else: 1452 _log.info('%25.25s: dict1 = >>>%s<<<' % (key, d1[key])) 1453 _log.info('%25.25s dict2 = >>>%s<<<' % ('', d2[key])) 1454 different = True 1455 else: 1456 _log.info('%25.25s: %50.50s | <MISSING>' % (key, '>>>%s<<<' % d1[key])) 1457 different = True 1458 for key in keys_d2: 1459 if key in keys_d1: 1460 continue 1461 _log.info('%25.25s: %50.50s | %.50s' % (key, '<MISSING>', '>>>%s<<<' % d2[key])) 1462 different = True 1463 if different: 1464 _log.info('dict-likes appear to be different from each other') 1465 return False 1466 _log.info('dict-likes appear equal to each other') 1467 return True
1468 1469 #---------------------------------------------------------------------------
1470 -def format_dict_likes_comparison(d1, d2, title_left=None, title_right=None, left_margin=0, key_delim=' || ', data_delim=' | ', missing_string='=/=', difference_indicator='! ', ignore_diff_in_keys=None):
1471 1472 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title_left, '', '"%s" '), type(d1), coalesce(title_right, '', '"%s" '), type(d2)) 1473 append_type = False 1474 if None not in [title_left, title_right]: 1475 append_type = True 1476 type_left = type(d1) 1477 type_right = type(d2) 1478 if title_left is None: 1479 title_left = '%s' % type_left 1480 if title_right is None: 1481 title_right = '%s' % type_right 1482 1483 try: d1 = dict(d1) 1484 except TypeError: pass 1485 try: d2 = dict(d2) 1486 except TypeError: pass 1487 keys_d1 = d1.keys() 1488 keys_d2 = d2.keys() 1489 data = {} 1490 for key in keys_d1: 1491 data[key] = [d1[key], ' '] 1492 if key in d2: 1493 data[key][1] = d2[key] 1494 for key in keys_d2: 1495 if key in keys_d1: 1496 continue 1497 data[key] = [' ', d2[key]] 1498 max1 = max([ len('%s' % k) for k in keys_d1 ]) 1499 max2 = max([ len('%s' % k) for k in keys_d2 ]) 1500 max_len = max(max1, max2, len(_('<type>'))) 1501 max_key_len_str = '%' + '%s.%s' % (max_len, max_len) + 's' 1502 max1 = max([ len('%s' % d1[k]) for k in keys_d1 ]) 1503 max2 = max([ len('%s' % d2[k]) for k in keys_d2 ]) 1504 max_data_len = min(max(max1, max2), 100) 1505 max_data_len_str = '%' + '%s.%s' % (max_data_len, max_data_len) + 's' 1506 diff_indicator_len_str = '%' + '%s.%s' % (len(difference_indicator), len(difference_indicator)) + 's' 1507 line_template = (' ' * left_margin) + diff_indicator_len_str + max_key_len_str + key_delim + max_data_len_str + data_delim + '%s' 1508 1509 lines = [] 1510 # debugging: 1511 #lines.append(u' (40 regular spaces)') 1512 #lines.append((u' ' * 40) + u"(u' ' * 40)") 1513 #lines.append((u'%40.40s' % u'') + u"(u'%40.40s' % u'')") 1514 #lines.append((u'%40.40s' % u' ') + u"(u'%40.40s' % u' ')") 1515 #lines.append((u'%40.40s' % u'.') + u"(u'%40.40s' % u'.')") 1516 #lines.append(line_template) 1517 lines.append(line_template % ('', '', title_left, title_right)) 1518 if append_type: 1519 lines.append(line_template % ('', _('<type>'), type_left, type_right)) 1520 1521 if ignore_diff_in_keys is None: 1522 ignore_diff_in_keys = [] 1523 1524 for key in keys_d1: 1525 append_type = False 1526 txt_left_col = '%s' % d1[key] 1527 try: 1528 txt_right_col = '%s' % d2[key] 1529 if type(d1[key]) != type(d2[key]): 1530 append_type = True 1531 except KeyError: 1532 txt_right_col = missing_string 1533 lines.append(line_template % ( 1534 bool2subst ( 1535 ((txt_left_col == txt_right_col) or (key in ignore_diff_in_keys)), 1536 '', 1537 difference_indicator 1538 ), 1539 key, 1540 shorten_text(txt_left_col, max_data_len), 1541 shorten_text(txt_right_col, max_data_len) 1542 )) 1543 if append_type: 1544 lines.append(line_template % ( 1545 '', 1546 _('<type>'), 1547 shorten_text('%s' % type(d1[key]), max_data_len), 1548 shorten_text('%s' % type(d2[key]), max_data_len) 1549 )) 1550 1551 for key in keys_d2: 1552 if key in keys_d1: 1553 continue 1554 lines.append(line_template % ( 1555 bool2subst((key in ignore_diff_in_keys), '', difference_indicator), 1556 key, 1557 shorten_text(missing_string, max_data_len), 1558 shorten_text('%s' % d2[key], max_data_len) 1559 )) 1560 1561 return lines
1562 1563 #---------------------------------------------------------------------------
1564 -def format_dict_like(d, relevant_keys=None, template=None, missing_key_template='<[%(key)s] MISSING>', left_margin=0, tabular=False, value_delimiters=('>>>', '<<<'), eol='\n', values2ignore=None):
1565 if values2ignore is None: 1566 values2ignore = [] 1567 if template is not None: 1568 # all keys in template better exist in d 1569 try: 1570 return template % d 1571 except KeyError: 1572 # or else 1573 _log.exception('template contains %%()s key(s) which do not exist in data dict') 1574 # try to extend dict <d> to contain all required keys, 1575 # for that to work <relevant_keys> better list all 1576 # keys used in <template> 1577 if relevant_keys is not None: 1578 for key in relevant_keys: 1579 try: 1580 d[key] 1581 except KeyError: 1582 d[key] = missing_key_template % {'key': key} 1583 return template % d 1584 1585 if relevant_keys is None: 1586 relevant_keys = list(d.keys()) 1587 lines = [] 1588 if value_delimiters is None: 1589 delim_left = '' 1590 delim_right = '' 1591 else: 1592 delim_left, delim_right = value_delimiters 1593 if tabular: 1594 max_len = max([ len('%s' % k) for k in relevant_keys ]) 1595 max_len_str = '%s.%s' % (max_len, max_len) 1596 line_template = (' ' * left_margin) + '%' + max_len_str + ('s: %s%%s%s' % (delim_left, delim_right)) 1597 else: 1598 line_template = (' ' * left_margin) + '%%s: %s%%s%s' % (delim_left, delim_right) 1599 for key in relevant_keys: 1600 try: 1601 val = d[key] 1602 except KeyError: 1603 continue 1604 if val not in values2ignore: 1605 lines.append(line_template % (key, val)) 1606 if eol is None: 1607 return lines 1608 return eol.join(lines)
1609 1610 #---------------------------------------------------------------------------
1611 -def dicts2table(dict_list, left_margin=0, eol='\n', keys2ignore=None, headers=None, show_only_changes=False, equality_value='<=>', date_format=None): #, relevant_keys=None, template=None
1612 1613 keys2show = [] 1614 dict_max_size = {} 1615 max_row_label_size = 0 1616 if keys2ignore is None: 1617 keys2ignore = [] 1618 1619 # extract keys from all dicts 1620 for d in dict_list: 1621 dict_max_size[id(d)] = 0 1622 for key in d.keys(): 1623 if key in keys2ignore: 1624 continue 1625 dict_max_size[id(d)] = max(dict_max_size[id(d)], len('%s' % d[key])) 1626 if key in keys2show: 1627 continue 1628 keys2show.append(key) 1629 max_row_label_size = max(max_row_label_size, len('%s' % key)) 1630 1631 # pivot data into dict of lists per line 1632 lines = { k: [] for k in keys2show } 1633 prev_vals = {} 1634 for d in dict_list: 1635 if show_only_changes: 1636 max_size = max(dict_max_size[id(d)], len(equality_value)) 1637 else: 1638 max_size = dict_max_size[id(d)] 1639 max_len_str = '%s.%s' % (max_size, max_size) 1640 field_template = ' %' + max_len_str + 's' 1641 for key in keys2show: 1642 try: 1643 val = d[key] 1644 except KeyError: 1645 lines[key].append(field_template % _('<missing>')) 1646 continue 1647 if isinstance(val, pydt.datetime): 1648 if date_format is not None: 1649 val = val.strftime(date_format) 1650 lines[key].append(field_template % val) 1651 if show_only_changes: 1652 if key not in prev_vals: 1653 prev_vals[key] = '%s' % lines[key][-1] 1654 continue 1655 if lines[key][-1] != prev_vals[key]: 1656 prev_vals[key] = '%s' % lines[key][-1] 1657 continue 1658 lines[key][-1] = field_template % equality_value 1659 1660 # format data into table 1661 table_lines = [] 1662 max_len_str = '%s.%s' % (max_row_label_size, max_row_label_size) 1663 row_label_template = '%' + max_len_str + 's' 1664 for key in lines: 1665 line = (' ' * left_margin) + row_label_template % key + '|' 1666 line += '|'.join(lines[key]) 1667 table_lines.append(line) 1668 1669 # add header line if any 1670 if headers is not None: 1671 header_line1 = (' ' * left_margin) + row_label_template % '' 1672 header_line2 = (' ' * left_margin) + u_box_horiz_single * (max_row_label_size) 1673 header_sizes = [ max(dict_max_size[dict_id], len(equality_value)) for dict_id in dict_max_size ] 1674 for idx in range(len(headers)): 1675 max_len_str = '%s.%s' % (header_sizes[idx], header_sizes[idx]) 1676 header_template = '%' + max_len_str + 's' 1677 header_line1 += '| ' 1678 header_line1 += header_template % headers[idx] 1679 header_line2 += '%s%s' % (u_box_plus, u_box_horiz_single) 1680 header_line2 += u_box_horiz_single * header_sizes[idx] 1681 table_lines.insert(0, header_line2) 1682 table_lines.insert(0, header_line1) 1683 1684 if eol is None: 1685 return table_lines 1686 1687 return ('|' + eol).join(table_lines) + '|' + eol 1688 1689 #---------------------------------------------------------------------------
1690 -def normalize_dict_like(d, required_keys, missing_key_template='<[%(key)s] MISSING>'):
1691 for key in required_keys: 1692 try: 1693 d[key] 1694 except KeyError: 1695 if missing_key_template is None: 1696 d[key] = None 1697 else: 1698 d[key] = missing_key_template % {'key': key} 1699 return d
1700 1701 #---------------------------------------------------------------------------
1702 -def enumerate_removable_partitions():
1703 try: 1704 import pyudev 1705 import psutil 1706 except ImportError: 1707 _log.error('pyudev and/or psutil not installed') 1708 return {} 1709 1710 removable_partitions = {} 1711 ctxt = pyudev.Context() 1712 removable_devices = [ dev for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk') if dev.attributes.get('removable') == b'1' ] 1713 all_mounted_partitions = { part.device: part for part in psutil.disk_partitions() } 1714 for device in removable_devices: 1715 _log.debug('removable device: %s', device.properties['ID_MODEL']) 1716 partitions_on_removable_device = { 1717 part.device_node: { 1718 'type': device.properties['ID_TYPE'], 1719 'bus': device.properties['ID_BUS'], 1720 'device': device.properties['DEVNAME'], 1721 'partition': part.properties['DEVNAME'], 1722 'vendor': part.properties['ID_VENDOR'], 1723 'model': part.properties['ID_MODEL'], 1724 'fs_label': part.properties['ID_FS_LABEL'], 1725 'is_mounted': False, 1726 'mountpoint': None, 1727 'fs_type': None, 1728 'size_in_bytes': -1, 1729 'bytes_free': 0 1730 } for part in ctxt.list_devices(subsystem='block', DEVTYPE='partition', parent=device) 1731 } 1732 for part in partitions_on_removable_device: 1733 try: 1734 partitions_on_removable_device[part]['mountpoint'] = all_mounted_partitions[part].mountpoint 1735 partitions_on_removable_device[part]['is_mounted'] = True 1736 partitions_on_removable_device[part]['fs_type'] = all_mounted_partitions[part].fstype 1737 du = shutil.disk_usage(all_mounted_partitions[part].mountpoint) 1738 partitions_on_removable_device[part]['size_in_bytes'] = du.total 1739 partitions_on_removable_device[part]['bytes_free'] = du.free 1740 except KeyError: 1741 pass # not mounted 1742 removable_partitions.update(partitions_on_removable_device) 1743 return removable_partitions
1744 1745 # debugging: 1746 #ctxt = pyudev.Context() 1747 #for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk'):# if dev.attributes.get('removable') == b'1': 1748 # for a in dev.attributes.available_attributes: 1749 # print(a, dev.attributes.get(a)) 1750 # for key, value in dev.items(): 1751 # print('{key}={value}'.format(key=key, value=value)) 1752 # print('---------------------------') 1753 1754 #---------------------------------------------------------------------------
1755 -def enumerate_optical_writers():
1756 try: 1757 import pyudev 1758 except ImportError: 1759 _log.error('pyudev not installed') 1760 return [] 1761 1762 optical_writers = [] 1763 ctxt = pyudev.Context() 1764 for dev in [ dev for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk') if dev.properties.get('ID_CDROM_CD_RW', None) == '1' ]: 1765 optical_writers.append ({ 1766 'type': dev.properties['ID_TYPE'], 1767 'bus': dev.properties['ID_BUS'], 1768 'device': dev.properties['DEVNAME'], 1769 'model': dev.properties['ID_MODEL'] 1770 }) 1771 return optical_writers
1772 1773 #--------------------------------------------------------------------------- 1774 #---------------------------------------------------------------------------
1775 -def prompted_input(prompt=None, default=None):
1776 """Obtains entry from standard input. 1777 1778 prompt: Prompt text to display in standard output 1779 default: Default value (for user to press enter only) 1780 CTRL-C: aborts and returns None 1781 """ 1782 if prompt is None: 1783 msg = '(CTRL-C aborts)' 1784 else: 1785 msg = '%s (CTRL-C aborts)' % prompt 1786 1787 if default is None: 1788 msg = msg + ': ' 1789 else: 1790 msg = '%s [%s]: ' % (msg, default) 1791 1792 try: 1793 usr_input = input(msg) 1794 except KeyboardInterrupt: 1795 return None 1796 1797 if usr_input == '': 1798 return default 1799 1800 return usr_input
1801 1802 #=========================================================================== 1803 # image handling tools 1804 #--------------------------------------------------------------------------- 1805 # builtin (ugly but tried and true) fallback icon 1806 __icon_serpent = \ 1807 """x\xdae\x8f\xb1\x0e\x83 \x10\x86w\x9f\xe2\x92\x1blb\xf2\x07\x96\xeaH:0\xd6\ 1808 \xc1\x85\xd5\x98N5\xa5\xef?\xf5N\xd0\x8a\xdcA\xc2\xf7qw\x84\xdb\xfa\xb5\xcd\ 1809 \xd4\xda;\xc9\x1a\xc8\xb6\xcd<\xb5\xa0\x85\x1e\xeb\xbc\xbc7b!\xf6\xdeHl\x1c\ 1810 \x94\x073\xec<*\xf7\xbe\xf7\x99\x9d\xb21~\xe7.\xf5\x1f\x1c\xd3\xbdVlL\xc2\ 1811 \xcf\xf8ye\xd0\x00\x90\x0etH \x84\x80B\xaa\x8a\x88\x85\xc4(U\x9d$\xfeR;\xc5J\ 1812 \xa6\x01\xbbt9\xceR\xc8\x81e_$\x98\xb9\x9c\xa9\x8d,y\xa9t\xc8\xcf\x152\xe0x\ 1813 \xe9$\xf5\x07\x95\x0cD\x95t:\xb1\x92\xae\x9cI\xa8~\x84\x1f\xe0\xa3ec""" 1814
1815 -def get_icon(wx=None):
1816 1817 paths = gmPaths(app_name = 'gnumed', wx = wx) 1818 1819 candidates = [ 1820 os.path.join(paths.system_app_data_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'), 1821 os.path.join(paths.local_base_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'), 1822 os.path.join(paths.system_app_data_dir, 'bitmaps', 'serpent.png'), 1823 os.path.join(paths.local_base_dir, 'bitmaps', 'serpent.png') 1824 ] 1825 1826 found_as = None 1827 for candidate in candidates: 1828 try: 1829 open(candidate, 'r').close() 1830 found_as = candidate 1831 break 1832 except IOError: 1833 _log.debug('icon not found in [%s]', candidate) 1834 1835 if found_as is None: 1836 _log.warning('no icon file found, falling back to builtin (ugly) icon') 1837 icon_bmp_data = wx.BitmapFromXPMData(pickle.loads(zlib.decompress(__icon_serpent))) 1838 icon.CopyFromBitmap(icon_bmp_data) 1839 else: 1840 _log.debug('icon found in [%s]', found_as) 1841 icon = wx.Icon() 1842 try: 1843 icon.LoadFile(found_as, wx.BITMAP_TYPE_ANY) #_PNG 1844 except AttributeError: 1845 _log.exception("this platform doesn't support wx.Icon().LoadFile()") 1846 1847 return icon
1848 1849 #---------------------------------------------------------------------------
1850 -def create_qrcode(text=None, filename=None, qr_filename=None, verbose=False):
1851 assert (not ((text is None) and (filename is None))), 'either <text> or <filename> must be specified' 1852 1853 try: 1854 import pyqrcode 1855 except ImportError: 1856 _log.exception('cannot import <pyqrcode>') 1857 return None 1858 if text is None: 1859 with io.open(filename, mode = 'rt', encoding = 'utf8') as input_file: 1860 text = input_file.read() 1861 if qr_filename is None: 1862 if filename is None: 1863 qr_filename = get_unique_filename(prefix = 'gm-qr-', suffix = '.png') 1864 else: 1865 qr_filename = get_unique_filename ( 1866 prefix = fname_stem(filename) + '-', 1867 suffix = fname_extension(filename) + '.png' 1868 ) 1869 _log.debug('[%s] -> [%s]', filename, qr_filename) 1870 qr = pyqrcode.create(text, encoding = 'utf8') 1871 if verbose: 1872 print('input file:', filename) 1873 print('output file:', qr_filename) 1874 print('text to encode:', text) 1875 print(qr.terminal()) 1876 qr.png(qr_filename, quiet_zone = 1) 1877 return qr_filename
1878 1879 #=========================================================================== 1880 # main 1881 #--------------------------------------------------------------------------- 1882 if __name__ == '__main__': 1883 1884 if len(sys.argv) < 2: 1885 sys.exit() 1886 1887 if sys.argv[1] != 'test': 1888 sys.exit() 1889 1890 # for testing: 1891 logging.basicConfig(level = logging.DEBUG) 1892 from Gnumed.pycommon import gmI18N 1893 gmI18N.activate_locale() 1894 gmI18N.install_domain() 1895 1896 #-----------------------------------------------------------------------
1897 - def test_input2decimal():
1898 1899 tests = [ 1900 [None, False], 1901 1902 ['', False], 1903 [' 0 ', True, 0], 1904 1905 [0, True, 0], 1906 [0.0, True, 0], 1907 [.0, True, 0], 1908 ['0', True, 0], 1909 ['0.0', True, 0], 1910 ['0,0', True, 0], 1911 ['00.0', True, 0], 1912 ['.0', True, 0], 1913 [',0', True, 0], 1914 1915 [0.1, True, decimal.Decimal('0.1')], 1916 [.01, True, decimal.Decimal('0.01')], 1917 ['0.1', True, decimal.Decimal('0.1')], 1918 ['0,1', True, decimal.Decimal('0.1')], 1919 ['00.1', True, decimal.Decimal('0.1')], 1920 ['.1', True, decimal.Decimal('0.1')], 1921 [',1', True, decimal.Decimal('0.1')], 1922 1923 [1, True, 1], 1924 [1.0, True, 1], 1925 ['1', True, 1], 1926 ['1.', True, 1], 1927 ['1,', True, 1], 1928 ['1.0', True, 1], 1929 ['1,0', True, 1], 1930 ['01.0', True, 1], 1931 ['01,0', True, 1], 1932 [' 01, ', True, 1], 1933 1934 [decimal.Decimal('1.1'), True, decimal.Decimal('1.1')] 1935 ] 1936 for test in tests: 1937 conversion_worked, result = input2decimal(initial = test[0]) 1938 1939 expected2work = test[1] 1940 1941 if conversion_worked: 1942 if expected2work: 1943 if result == test[2]: 1944 continue 1945 else: 1946 print("ERROR (conversion result wrong): >%s<, expected >%s<, got >%s<" % (test[0], test[2], result)) 1947 else: 1948 print("ERROR (conversion worked but was expected to fail): >%s<, got >%s<" % (test[0], result)) 1949 else: 1950 if not expected2work: 1951 continue 1952 else: 1953 print("ERROR (conversion failed but was expected to work): >%s<, expected >%s<" % (test[0], test[2]))
1954 #-----------------------------------------------------------------------
1955 - def test_input2int():
1956 print(input2int(0)) 1957 print(input2int('0')) 1958 print(input2int('0', 0, 0))
1959 #-----------------------------------------------------------------------
1960 - def test_coalesce():
1961 1962 val = None 1963 print(val, coalesce(val, 'is None', 'is not None')) 1964 val = 1 1965 print(val, coalesce(val, 'is None', 'is not None')) 1966 return 1967 1968 import datetime as dt 1969 print(coalesce(value2test = dt.datetime.now(), template4value = '-- %s --', function4value = ('strftime', '%Y-%m-%d'))) 1970 1971 print('testing coalesce()') 1972 print("------------------") 1973 tests = [ 1974 [None, 'something other than <None>', None, None, 'something other than <None>'], 1975 ['Captain', 'Mr.', '%s.'[:4], 'Mr.', 'Capt.'], 1976 ['value to test', 'test 3 failed', 'template with "%s" included', None, 'template with "value to test" included'], 1977 ['value to test', 'test 4 failed', 'template with value not included', None, 'template with value not included'], 1978 [None, 'initial value was None', 'template4value: %s', None, 'initial value was None'], 1979 [None, 'initial value was None', 'template4value: %%(abc)s', None, 'initial value was None'] 1980 ] 1981 passed = True 1982 for test in tests: 1983 result = coalesce ( 1984 value2test = test[0], 1985 return_instead = test[1], 1986 template4value = test[2], 1987 template4instead = test[3] 1988 ) 1989 if result != test[4]: 1990 print("ERROR") 1991 print("coalesce: (%s, %s, %s, %s)" % (test[0], test[1], test[2], test[3])) 1992 print("expected:", test[4]) 1993 print("received:", result) 1994 passed = False 1995 1996 if passed: 1997 print("passed") 1998 else: 1999 print("failed") 2000 return passed
2001 #-----------------------------------------------------------------------
2002 - def test_capitalize():
2003 print('testing capitalize() ...') 2004 success = True 2005 pairs = [ 2006 # [original, expected result, CAPS mode] 2007 ['Boot', 'Boot', CAPS_FIRST_ONLY], 2008 ['boot', 'Boot', CAPS_FIRST_ONLY], 2009 ['booT', 'Boot', CAPS_FIRST_ONLY], 2010 ['BoOt', 'Boot', CAPS_FIRST_ONLY], 2011 ['boots-Schau', 'Boots-Schau', CAPS_WORDS], 2012 ['boots-sChau', 'Boots-Schau', CAPS_WORDS], 2013 ['boot camp', 'Boot Camp', CAPS_WORDS], 2014 ['fahrner-Kampe', 'Fahrner-Kampe', CAPS_NAMES], 2015 ['häkkönen', 'Häkkönen', CAPS_NAMES], 2016 ['McBurney', 'McBurney', CAPS_NAMES], 2017 ['mcBurney', 'McBurney', CAPS_NAMES], 2018 ['blumberg', 'Blumberg', CAPS_NAMES], 2019 ['roVsing', 'RoVsing', CAPS_NAMES], 2020 ['Özdemir', 'Özdemir', CAPS_NAMES], 2021 ['özdemir', 'Özdemir', CAPS_NAMES], 2022 ] 2023 for pair in pairs: 2024 result = capitalize(pair[0], pair[2]) 2025 if result != pair[1]: 2026 success = False 2027 print('ERROR (caps mode %s): "%s" -> "%s", expected "%s"' % (pair[2], pair[0], result, pair[1])) 2028 2029 if success: 2030 print("... SUCCESS") 2031 2032 return success
2033 #-----------------------------------------------------------------------
2034 - def test_import_module():
2035 print("testing import_module_from_directory()") 2036 path = sys.argv[1] 2037 name = sys.argv[2] 2038 try: 2039 mod = import_module_from_directory(module_path = path, module_name = name) 2040 except: 2041 print("module import failed, see log") 2042 return False 2043 2044 print("module import succeeded", mod) 2045 print(dir(mod)) 2046 return True
2047 #-----------------------------------------------------------------------
2048 - def test_mkdir():
2049 print("testing mkdir(%s)" % sys.argv[2]) 2050 mkdir(sys.argv[2], 0o0700)
2051 #-----------------------------------------------------------------------
2052 - def test_gmPaths():
2053 print("testing gmPaths()") 2054 print("-----------------") 2055 paths = gmPaths(wx=None, app_name='gnumed') 2056 print("user config dir:", paths.user_config_dir) 2057 print("system config dir:", paths.system_config_dir) 2058 print("local base dir:", paths.local_base_dir) 2059 print("system app data dir:", paths.system_app_data_dir) 2060 print("working directory :", paths.working_dir) 2061 print("temp directory :", paths.tmp_dir)
2062 #-----------------------------------------------------------------------
2063 - def test_none_if():
2064 print("testing none_if()") 2065 print("-----------------") 2066 tests = [ 2067 [None, None, None], 2068 ['a', 'a', None], 2069 ['a', 'b', 'a'], 2070 ['a', None, 'a'], 2071 [None, 'a', None], 2072 [1, 1, None], 2073 [1, 2, 1], 2074 [1, None, 1], 2075 [None, 1, None] 2076 ] 2077 2078 for test in tests: 2079 if none_if(value = test[0], none_equivalent = test[1]) != test[2]: 2080 print('ERROR: none_if(%s) returned [%s], expected [%s]' % (test[0], none_if(test[0], test[1]), test[2])) 2081 2082 return True
2083 #-----------------------------------------------------------------------
2084 - def test_bool2str():
2085 tests = [ 2086 [True, 'Yes', 'Yes', 'Yes'], 2087 [False, 'OK', 'not OK', 'not OK'] 2088 ] 2089 for test in tests: 2090 if bool2str(test[0], test[1], test[2]) != test[3]: 2091 print('ERROR: bool2str(%s, %s, %s) returned [%s], expected [%s]' % (test[0], test[1], test[2], bool2str(test[0], test[1], test[2]), test[3])) 2092 2093 return True
2094 #-----------------------------------------------------------------------
2095 - def test_bool2subst():
2096 2097 print(bool2subst(True, 'True', 'False', 'is None')) 2098 print(bool2subst(False, 'True', 'False', 'is None')) 2099 print(bool2subst(None, 'True', 'False', 'is None'))
2100 #-----------------------------------------------------------------------
2101 - def test_get_unique_filename():
2102 print(get_unique_filename()) 2103 print(get_unique_filename(prefix='test-')) 2104 print(get_unique_filename(suffix='tst')) 2105 print(get_unique_filename(prefix='test-', suffix='tst')) 2106 print(get_unique_filename(tmp_dir='/home/ncq/Archiv/'))
2107 #-----------------------------------------------------------------------
2108 - def test_size2str():
2109 print("testing size2str()") 2110 print("------------------") 2111 tests = [0, 1, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 100000000000, 1000000000000, 10000000000000] 2112 for test in tests: 2113 print(size2str(test))
2114 #-----------------------------------------------------------------------
2115 - def test_unwrap():
2116 2117 test = """ 2118 second line\n 2119 3rd starts with tab \n 2120 4th with a space \n 2121 2122 6th 2123 2124 """ 2125 print(unwrap(text = test, max_length = 25))
2126 #-----------------------------------------------------------------------
2127 - def test_wrap():
2128 test = 'line 1\nline 2\nline 3' 2129 2130 print("wrap 5-6-7 initial 0, subsequent 0") 2131 print(wrap(test, 5)) 2132 print() 2133 print(wrap(test, 6)) 2134 print() 2135 print(wrap(test, 7)) 2136 print("-------") 2137 input() 2138 print("wrap 5 initial 1-1-3, subsequent 1-3-1") 2139 print(wrap(test, 5, ' ', ' ')) 2140 print() 2141 print(wrap(test, 5, ' ', ' ')) 2142 print() 2143 print(wrap(test, 5, ' ', ' ')) 2144 print("-------") 2145 input() 2146 print("wrap 6 initial 1-1-3, subsequent 1-3-1") 2147 print(wrap(test, 6, ' ', ' ')) 2148 print() 2149 print(wrap(test, 6, ' ', ' ')) 2150 print() 2151 print(wrap(test, 6, ' ', ' ')) 2152 print("-------") 2153 input() 2154 print("wrap 7 initial 1-1-3, subsequent 1-3-1") 2155 print(wrap(test, 7, ' ', ' ')) 2156 print() 2157 print(wrap(test, 7, ' ', ' ')) 2158 print() 2159 print(wrap(test, 7, ' ', ' '))
2160 #-----------------------------------------------------------------------
2161 - def test_md5():
2162 print('md5 %s: %s' % (sys.argv[2], file2md5(sys.argv[2]))) 2163 print('chunked md5 %s: %s' % (sys.argv[2], file2chunked_md5(sys.argv[2])))
2164 #-----------------------------------------------------------------------
2165 - def test_unicode():
2166 print(u_link_symbol * 10)
2167 #-----------------------------------------------------------------------
2168 - def test_xml_escape():
2169 print(xml_escape_string('<')) 2170 print(xml_escape_string('>')) 2171 print(xml_escape_string('&'))
2172 #-----------------------------------------------------------------------
2173 - def test_tex_escape():
2174 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234'] 2175 tests.append(' '.join(tests)) 2176 for test in tests: 2177 print('%s:' % test, tex_escape_string(test))
2178 2179 #-----------------------------------------------------------------------
2180 - def test_rst2latex_snippet():
2181 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234'] 2182 tests.append(' '.join(tests)) 2183 tests.append('C:\Windows\Programme\System 32\lala.txt') 2184 tests.extend([ 2185 'should be identical', 2186 'text *some text* text', 2187 """A List 2188 ====== 2189 2190 1. 1 2191 2. 2 2192 2193 3. ist-list 2194 1. more 2195 2. noch was ü 2196 #. nummer x""" 2197 ]) 2198 for test in tests: 2199 print('==================================================') 2200 print('raw:') 2201 print(test) 2202 print('---------') 2203 print('ReST 2 LaTeX:') 2204 latex = rst2latex_snippet(test) 2205 print(latex) 2206 if latex.strip() == test.strip(): 2207 print('=> identical') 2208 print('---------') 2209 print('tex_escape_string:') 2210 print(tex_escape_string(test)) 2211 input()
2212 2213 #-----------------------------------------------------------------------
2214 - def test_strip_trailing_empty_lines():
2215 tests = [ 2216 'one line, no embedded line breaks ', 2217 'one line\nwith embedded\nline\nbreaks\n ' 2218 ] 2219 for test in tests: 2220 print('as list:') 2221 print(strip_trailing_empty_lines(text = test, eol='\n', return_list = True)) 2222 print('as string:') 2223 print('>>>%s<<<' % strip_trailing_empty_lines(text = test, eol='\n', return_list = False)) 2224 tests = [ 2225 ['list', 'without', 'empty', 'trailing', 'lines'], 2226 ['list', 'with', 'empty', 'trailing', 'lines', '', ' ', ''] 2227 ] 2228 for test in tests: 2229 print('as list:') 2230 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = True)) 2231 print('as string:') 2232 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = False))
2233 #-----------------------------------------------------------------------
2234 - def test_fname_stem():
2235 tests = [ 2236 r'abc.exe', 2237 r'\abc.exe', 2238 r'c:\abc.exe', 2239 r'c:\d\abc.exe', 2240 r'/home/ncq/tmp.txt', 2241 r'~/tmp.txt', 2242 r'./tmp.txt', 2243 r'./.././tmp.txt', 2244 r'tmp.txt' 2245 ] 2246 for t in tests: 2247 print("[%s] -> [%s]" % (t, fname_stem(t)))
2248 #-----------------------------------------------------------------------
2249 - def test_dir_is_empty():
2250 print(sys.argv[2], 'empty:', dir_is_empty(sys.argv[2]))
2251 2252 #-----------------------------------------------------------------------
2253 - def test_compare_dicts():
2254 d1 = {} 2255 d2 = {} 2256 d1[1] = 1 2257 d1[2] = 2 2258 d1[3] = 3 2259 # 4 2260 d1[5] = 5 2261 2262 d2[1] = 1 2263 d2[2] = None 2264 # 3 2265 d2[4] = 4 2266 2267 #compare_dict_likes(d1, d2) 2268 2269 d1 = {1: 1, 2: 2} 2270 d2 = {1: 1, 2: 2} 2271 2272 #compare_dict_likes(d1, d2, 'same1', 'same2') 2273 print(format_dict_like(d1, tabular = False)) 2274 print(format_dict_like(d1, tabular = True))
2275 #print(format_dict_like(d2)) 2276 2277 #-----------------------------------------------------------------------
2278 - def test_format_compare_dicts():
2279 d1 = {} 2280 d2 = {} 2281 d1[1] = 1 2282 d1[2] = 2 2283 d1[3] = 3 2284 # 4 2285 d1[5] = 5 2286 2287 d2[1] = 1 2288 d2[2] = None 2289 # 3 2290 d2[4] = 4 2291 2292 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2'))) 2293 2294 d1 = {1: 1, 2: 2} 2295 d2 = {1: 1, 2: 2} 2296 2297 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
2298 2299 #-----------------------------------------------------------------------
2300 - def test_rm_dir():
2301 rmdir('cx:\windows\system3__2xxxxxxxxxxxxx')
2302 2303 #-----------------------------------------------------------------------
2304 - def test_rm_dir_content():
2305 #print(rm_dir_content('cx:\windows\system3__2xxxxxxxxxxxxx')) 2306 print(rm_dir_content('/tmp/user/1000/tmp'))
2307 2308 #-----------------------------------------------------------------------
2309 - def test_strip_prefix():
2310 tests = [ 2311 ('', '', ''), 2312 ('a', 'a', ''), 2313 ('GMd: a window title', _GM_TITLE_PREFIX + ':', 'a window title'), 2314 ('\.br\MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\', '\.br\\', 'MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\') 2315 ] 2316 for test in tests: 2317 text, prefix, expect = test 2318 result = strip_prefix(text, prefix, remove_whitespace = True) 2319 if result == expect: 2320 continue 2321 print('test failed:', test) 2322 print('result:', result)
2323 2324 #-----------------------------------------------------------------------
2325 - def test_shorten_text():
2326 tst = [ 2327 ('123', 1), 2328 ('123', 2), 2329 ('123', 3), 2330 ('123', 4), 2331 ('', 1), 2332 ('1', 1), 2333 ('12', 1), 2334 ('', 2), 2335 ('1', 2), 2336 ('12', 2), 2337 ('123', 2) 2338 ] 2339 for txt, lng in tst: 2340 print('max', lng, 'of', txt, '=', shorten_text(txt, lng))
2341 #-----------------------------------------------------------------------
2342 - def test_fname_sanitize():
2343 tests = [ 2344 '/tmp/test.txt', 2345 '/tmp/ test.txt', 2346 '/tmp/ tes\\t.txt', 2347 'test' 2348 ] 2349 for test in tests: 2350 print (test, fname_sanitize(test))
2351 2352 #-----------------------------------------------------------------------
2353 - def test_create_qrcode():
2354 print(create_qrcode(text = sys.argv[2], filename=None, qr_filename=None, verbose = True))
2355 2356 #-----------------------------------------------------------------------
2357 - def test_enumerate_removable_partitions():
2358 parts = enumerate_removable_partitions() 2359 for part_name in parts: 2360 part = parts[part_name] 2361 print(part['device']) 2362 print(part['partition']) 2363 if part['is_mounted']: 2364 print('%s@%s: %s on %s by %s @ %s (FS=%s: %s free of %s total)' % ( 2365 part['type'], 2366 part['bus'], 2367 part['fs_label'], 2368 part['model'], 2369 part['vendor'], 2370 part['mountpoint'], 2371 part['fs_type'], 2372 part['bytes_free'], 2373 part['size_in_bytes'] 2374 )) 2375 else: 2376 print('%s@%s: %s on %s by %s (not mounted)' % ( 2377 part['type'], 2378 part['bus'], 2379 part['fs_label'], 2380 part['model'], 2381 part['vendor'] 2382 ))
2383 2384 #-----------------------------------------------------------------------
2385 - def test_enumerate_optical_writers():
2386 for writer in enumerate_optical_writers(): 2387 print('%s@%s: %s @ %s' % ( 2388 writer['type'], 2389 writer['bus'], 2390 writer['model'], 2391 writer['device'] 2392 ))
2393 2394 #-----------------------------------------------------------------------
2395 - def test_copy_tree_content():
2396 print(sys.argv[2], '->', sys.argv[3]) 2397 print(copy_tree_content(sys.argv[2], sys.argv[3]))
2398 2399 #-----------------------------------------------------------------------
2400 - def test_mk_sandbox_dir():
2401 print(mk_sandbox_dir(base_dir = '/tmp/abcd/efg/h'))
2402 2403 #-----------------------------------------------------------------------
2404 - def test_make_table_from_dicts():
2405 dicts = [ 2406 {'pkey': 1, 'value': 'a1'}, 2407 {'pkey': 2, 'value': 'b2'}, 2408 {'pkey': 3, 'value': 'c3'}, 2409 {'pkey': 4, 'value': 'd4'}, 2410 {'pkey': 5, 'value': 'd4'}, 2411 {'pkey': 5, 'value': 'c5'}, 2412 ] 2413 with open('x.txt', 'w', encoding = 'utf8') as f: 2414 f.write(dicts2table(dicts, left_margin=2, eol='\n', keys2ignore=None, show_only_changes=True, headers = ['d1', 'd2', 'd3', 'd4', 'd5', 'd6']))
2415 #print(dicts2table(dicts, left_margin=2, eol='\n', keys2ignore=None, show_only_changes=True, headers = ['d1', 'd2', 'd3', 'd4', 'd5', 'd6'])) 2416 2417 #-----------------------------------------------------------------------
2418 - def test_decorate_window_title():
2419 for txt in globals(): 2420 print(decorate_window_title(txt))
2421 2422 #----------------------------------------------------------------------- 2423 #test_coalesce() 2424 #test_capitalize() 2425 #test_import_module() 2426 #test_mkdir() 2427 #test_gmPaths() 2428 #test_none_if() 2429 #test_bool2str() 2430 #test_bool2subst() 2431 #test_get_unique_filename() 2432 #test_size2str() 2433 #test_wrap() 2434 #test_input2decimal() 2435 #test_input2int() 2436 #test_unwrap() 2437 #test_md5() 2438 #test_unicode() 2439 #test_xml_escape() 2440 #test_strip_trailing_empty_lines() 2441 #test_fname_stem() 2442 #test_tex_escape() 2443 #test_rst2latex_snippet() 2444 #test_dir_is_empty() 2445 #test_compare_dicts() 2446 #test_rm_dir() 2447 #test_rm_dir_content() 2448 #test_strip_prefix() 2449 #test_shorten_text() 2450 #test_format_compare_dicts() 2451 #test_fname_sanitize() 2452 #test_create_qrcode() 2453 #test_enumerate_removable_partitions() 2454 #test_enumerate_optical_writers() 2455 #test_copy_tree_content() 2456 #test_mk_sandbox_dir() 2457 #test_make_table_from_dicts() 2458 test_decorate_window_title() 2459 2460 #=========================================================================== 2461