Package Gnumed :: Package pycommon :: Module gmTools
[frames] | no frames]

Source Code for Module Gnumed.pycommon.gmTools

   1  # -*- coding: utf-8 -*- 
   2   
   3   
   4  __doc__ = """GNUmed general tools.""" 
   5   
   6  #=========================================================================== 
   7  __author__ = "K. Hilbert <Karsten.Hilbert@gmx.net>" 
   8  __license__ = "GPL v2 or later (details at http://www.gnu.org)" 
   9   
  10  # std libs 
  11  import sys 
  12  import os 
  13  import os.path 
  14  import csv 
  15  import tempfile 
  16  import logging 
  17  import hashlib 
  18  import platform 
  19  import subprocess 
  20  import decimal 
  21  import getpass 
  22  import io 
  23  import functools 
  24  import json 
  25  import shutil 
  26  import zipfile 
  27  import datetime as pydt 
  28  import re as regex 
  29  import xml.sax.saxutils as xml_tools 
  30  # old: 
  31  import pickle, zlib 
  32  # docutils 
  33  du_core = None 
  34   
  35   
  36  # GNUmed libs 
  37  if __name__ == '__main__': 
  38          sys.path.insert(0, '../../') 
  39  from Gnumed.pycommon import gmBorg 
  40   
  41   
  42  _log = logging.getLogger('gm.tools') 
  43   
  44  # CAPitalization modes: 
  45  (       CAPS_NONE,                                      # don't touch it 
  46          CAPS_FIRST,                                     # CAP first char, leave rest as is 
  47          CAPS_ALLCAPS,                           # CAP all chars 
  48          CAPS_WORDS,                                     # CAP first char of every word 
  49          CAPS_NAMES,                                     # CAP in a way suitable for names (tries to be smart) 
  50          CAPS_FIRST_ONLY                         # CAP first char, lowercase the rest 
  51  ) = range(6) 
  52   
  53   
  54  u_currency_pound = '\u00A3'                             # Pound sign 
  55  u_currency_sign = '\u00A4'                                      # generic currency sign 
  56  u_currency_yen = '\u00A5'                                       # Yen sign 
  57  u_right_double_angle_quote = '\u00AB'           # << 
  58  u_registered_trademark = '\u00AE' 
  59  u_plus_minus = '\u00B1' 
  60  u_superscript_one = '\u00B9'                            # ^1 
  61  u_left_double_angle_quote = '\u00BB'            # >> 
  62  u_one_quarter = '\u00BC' 
  63  u_one_half = '\u00BD' 
  64  u_three_quarters = '\u00BE' 
  65  u_multiply = '\u00D7'                                           # x 
  66  u_greek_ALPHA = '\u0391' 
  67  u_greek_alpha = '\u03b1' 
  68  u_greek_OMEGA = '\u03A9' 
  69  u_greek_omega = '\u03c9' 
  70  u_dagger = '\u2020' 
  71  u_triangular_bullet = '\u2023'                                  # triangular bullet  (>) 
  72  u_ellipsis = '\u2026'                                                   # ... 
  73  u_euro = '\u20AC'                                                               # EURO sign 
  74  u_numero = '\u2116'                                                             # No. / # sign 
  75  u_down_left_arrow = '\u21B5'                                    # <-' 
  76  u_left_arrow = '\u2190'                                                 # <-- 
  77  u_up_arrow = '\u2191' 
  78  u_arrow2right = '\u2192'                                                # --> 
  79  u_down_arrow = '\u2193' 
  80  u_left_arrow_with_tail = '\u21a2'                               # <--< 
  81  u_arrow2right_from_bar = '\u21a6'                               # |-> 
  82  u_arrow2right_until_vertical_bar = '\u21e5'             # -->| 
  83  u_sum = '\u2211'                                                                # sigma 
  84  u_almost_equal_to = '\u2248'                                    # approximately / nearly / roughly 
  85  u_corresponds_to = '\u2258' 
  86  u_infinity = '\u221E' 
  87  u_arrow2right_until_vertical_bar2 = '\u2b72'    # -->| 
  88  u_diameter = '\u2300' 
  89  u_checkmark_crossed_out = '\u237B' 
  90  u_box_horiz_high = '\u23ba' 
  91  u_box_vert_left = '\u23b8' 
  92  u_box_vert_right = '\u23b9' 
  93  u_box_horiz_single = '\u2500'                           # - 
  94  u_box_vert_light = '\u2502' 
  95  u_box_horiz_light_3dashes = '\u2504'            # ... 
  96  u_box_vert_light_4dashes = '\u2506' 
  97  u_box_horiz_4dashes = '\u2508'                          # .... 
  98  u_box_T_right = '\u251c'                                        # |- 
  99  u_box_T_left = '\u2524'                                         # -| 
 100  u_box_T_down = '\u252c' 
 101  u_box_T_up = '\u2534' 
 102  u_box_plus = '\u253c' 
 103  u_box_top_double = '\u2550' 
 104  u_box_top_left_double_single = '\u2552' 
 105  u_box_top_right_double_single = '\u2555' 
 106  u_box_top_left_arc = '\u256d' 
 107  u_box_top_right_arc = '\u256e' 
 108  u_box_bottom_right_arc = '\u256f' 
 109  u_box_bottom_left_arc = '\u2570' 
 110  u_box_horiz_light_heavy = '\u257c' 
 111  u_box_horiz_heavy_light = '\u257e' 
 112  u_skull_and_crossbones = '\u2620' 
 113  u_caduceus = '\u2624' 
 114  u_frowning_face = '\u2639' 
 115  u_smiling_face = '\u263a' 
 116  u_black_heart = '\u2665' 
 117  u_female = '\u2640' 
 118  u_male = '\u2642' 
 119  u_male_female = '\u26a5' 
 120  u_checkmark_thin = '\u2713' 
 121  u_checkmark_thick = '\u2714' 
 122  u_heavy_greek_cross = '\u271a' 
 123  u_arrow2right_thick = '\u2794' 
 124  u_writing_hand = '\u270d' 
 125  u_pencil_1 = '\u270e' 
 126  u_pencil_2 = '\u270f' 
 127  u_pencil_3 = '\u2710' 
 128  u_latin_cross = '\u271d' 
 129  u_arrow2right_until_black_diamond = '\u291e'    # ->* 
 130  u_kanji_yen = '\u5186'                                                  # Yen kanji 
 131  u_replacement_character = '\ufffd' 
 132  u_link_symbol = '\u1f517' 
 133   
 134  _kB = 1024 
 135  _MB = 1024 * _kB 
 136  _GB = 1024 * _MB 
 137  _TB = 1024 * _GB 
 138  _PB = 1024 * _TB 
 139   
 140  #=========================================================================== 
141 -def handle_uncaught_exception_console(t, v, tb):
142 143 print(".========================================================") 144 print("| Unhandled exception caught !") 145 print("| Type :", t) 146 print("| Value:", v) 147 print("`========================================================") 148 _log.critical('unhandled exception caught', exc_info = (t,v,tb)) 149 sys.__excepthook__(t,v,tb)
150 151 #=========================================================================== 152 # path level operations 153 #---------------------------------------------------------------------------
154 -def mkdir(directory=None, mode=None):
155 """Create directory. 156 157 - creates parent dirs if necessary 158 - does not fail if directory exists 159 <mode>: numeric, say 0o0700 for "-rwx------" 160 """ 161 if os.path.isdir(directory): 162 if mode is None: 163 return True 164 try: 165 old_umask = os.umask(0) 166 # does not WORK ! 167 #os.chmod(directory, mode, follow_symlinks = (os.chmod in os.supports_follow_symlinks)) # can't do better 168 os.chmod(directory, mode) 169 return True 170 except Exception: 171 _log.exception('cannot os.chmod(%s, %s)', oct(mode), directory) 172 raise 173 finally: 174 os.umask(old_umask) 175 return False 176 177 if mode is None: 178 os.makedirs(directory) 179 return True 180 181 try: 182 old_umask = os.umask(0) 183 os.makedirs(directory, mode) 184 return True 185 except Exception: 186 _log.exception('cannot os.makedirs(%s, %s)', oct(mode), directory) 187 raise 188 finally: 189 os.umask(old_umask) 190 return True
191 192 #---------------------------------------------------------------------------
193 -def rmdir(directory):
194 #------------------------------- 195 def _on_rm_error(func, path, exc): 196 _log.error('error while shutil.rmtree(%s)', path, exc_info=exc) 197 return True
198 #------------------------------- 199 error_count = 0 200 try: 201 shutil.rmtree(directory, False, _on_rm_error) 202 except Exception: 203 _log.exception('cannot shutil.rmtree(%s)', directory) 204 error_count += 1 205 return error_count 206 207 #---------------------------------------------------------------------------
208 -def rm_dir_content(directory):
209 _log.debug('cleaning out [%s]', directory) 210 try: 211 items = os.listdir(directory) 212 except OSError: 213 return False 214 for item in items: 215 # attempt file/link removal and ignore (but log) errors 216 full_item = os.path.join(directory, item) 217 try: 218 os.remove(full_item) 219 except OSError: # as per the docs, this is a directory 220 _log.debug('[%s] seems to be a subdirectory', full_item) 221 errors = rmdir(full_item) 222 if errors > 0: 223 return False 224 except Exception: 225 _log.exception('cannot os.remove(%s) [a file or a link]', full_item) 226 return False 227 228 return True
229 230 #---------------------------------------------------------------------------
231 -def mk_sandbox_dir(prefix=None, base_dir=None):
232 if base_dir is None: 233 base_dir = gmPaths().tmp_dir 234 else: 235 if not os.path.isdir(base_dir): 236 mkdir(base_dir, mode = 0o0700) # (invoking user only) 237 if prefix is None: 238 prefix = 'sndbx-' 239 return tempfile.mkdtemp(prefix = prefix, suffix = '', dir = base_dir)
240 241 #---------------------------------------------------------------------------
242 -def parent_dir(directory):
243 return os.path.abspath(os.path.join(directory, '..'))
244 245 #---------------------------------------------------------------------------
246 -def dirname_stem(directory):
247 # /home/user/dir/ -> dir 248 # /home/user/dir -> dir 249 return os.path.basename(os.path.normpath(directory)) # normpath removes trailing slashes if any
250 251 #---------------------------------------------------------------------------
252 -def dir_is_empty(directory=None):
253 try: 254 return len(os.listdir(directory)) == 0 255 except OSError as exc: 256 if exc.errno == 2: # no such file 257 return None 258 raise
259 260 #---------------------------------------------------------------------------
261 -def copy_tree_content(directory, target_directory):
262 """Copy the *content* of <directory> *into* <target_directory> 263 which is created if need be. 264 """ 265 assert (directory is not None), 'source <directory> should not be None' 266 _log.debug('copying content of [%s] into [%s]', directory, target_directory) 267 try: 268 items = os.listdir(directory) 269 except OSError: 270 return None 271 272 for item in items: 273 full_item = os.path.join(directory, item) 274 if os.path.isdir(full_item): 275 target_subdir = os.path.join(target_directory, item) 276 try: 277 shutil.copytree(full_item, target_subdir) 278 except Exception: 279 _log.exception('cannot copy subdir [%s]', full_item) 280 return None 281 else: 282 try: 283 shutil.copy2(full_item, target_directory) 284 except Exception: 285 _log.exception('cannot copy file [%s]', full_item) 286 return None 287 288 return target_directory
289 290 #--------------------------------------------------------------------------- 291 #---------------------------------------------------------------------------
292 -class gmPaths(gmBorg.cBorg):
293 """This class provides the following paths: 294 295 .home_dir user home 296 .local_base_dir script installation dir 297 .working_dir current dir 298 .user_config_dir 299 .system_config_dir 300 .system_app_data_dir (not writable) 301 .tmp_dir instance-local 302 .user_tmp_dir user-local (NOT per instance) 303 .bytea_cache_dir caches downloaded BYTEA data 304 """
305 - def __init__(self, app_name=None, wx=None):
306 """Setup pathes. 307 308 <app_name> will default to (name of the script - .py) 309 """ 310 try: 311 self.already_inited 312 return 313 except AttributeError: 314 pass 315 316 self.init_paths(app_name=app_name, wx=wx) 317 self.already_inited = True
318 319 #-------------------------------------- 320 # public API 321 #--------------------------------------
322 - def init_paths(self, app_name=None, wx=None):
323 324 if wx is None: 325 _log.debug('wxPython not available') 326 _log.debug('detecting paths directly') 327 328 if app_name is None: 329 app_name, ext = os.path.splitext(os.path.basename(sys.argv[0])) 330 _log.info('app name detected as [%s]', app_name) 331 else: 332 _log.info('app name passed in as [%s]', app_name) 333 334 # the user home, doesn't work in Wine so work around that 335 self.__home_dir = None 336 337 # where the main script (the "binary") is installed 338 if getattr(sys, 'frozen', False): 339 _log.info('frozen app, installed into temporary path') 340 # this would find the path of *THIS* file 341 #self.local_base_dir = os.path.dirname(__file__) 342 # while this is documented on the web, the ${_MEIPASS2} does not exist 343 #self.local_base_dir = os.environ.get('_MEIPASS2') 344 # this is what Martin Zibricky <mzibr.public@gmail.com> told us to use 345 # when asking about this on pyinstaller@googlegroups.com 346 #self.local_base_dir = sys._MEIPASS 347 # however, we are --onedir, so we should look at sys.executable 348 # as per the pyinstaller manual 349 self.local_base_dir = os.path.dirname(sys.executable) 350 else: 351 self.local_base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) 352 353 # the current working dir at the OS 354 self.working_dir = os.path.abspath(os.curdir) 355 356 # user-specific config dir, usually below the home dir 357 mkdir(os.path.join(self.home_dir, '.%s' % app_name)) 358 self.user_config_dir = os.path.join(self.home_dir, '.%s' % app_name) 359 360 # system-wide config dir, usually below /etc/ under UN*X 361 try: 362 self.system_config_dir = os.path.join('/etc', app_name) 363 except ValueError: 364 #self.system_config_dir = self.local_base_dir 365 self.system_config_dir = self.user_config_dir 366 367 # system-wide application data dir 368 try: 369 self.system_app_data_dir = os.path.join(sys.prefix, 'share', app_name) 370 except ValueError: 371 self.system_app_data_dir = self.local_base_dir 372 373 # temporary directory 374 try: 375 self.__tmp_dir_already_set 376 _log.debug('temp dir already set') 377 except AttributeError: 378 _log.info('temp file prefix: %s', tempfile.gettempprefix()) 379 _log.info('initial (user level) temp dir: %s', tempfile.gettempdir()) 380 bytes_free = shutil.disk_usage(tempfile.gettempdir()).free 381 _log.info('free disk space for temp dir: %s (%s bytes)', size2str(size = bytes_free), bytes_free) 382 # $TMP/gnumed-$USER/ 383 self.user_tmp_dir = os.path.join(tempfile.gettempdir(), app_name + '-' + getpass.getuser()) 384 mkdir(self.user_tmp_dir, 0o700) 385 tempfile.tempdir = self.user_tmp_dir 386 _log.info('intermediate (app level) temp dir: %s', tempfile.gettempdir()) 387 # $TMP/gnumed-$USER/g-$UNIQUE/ 388 self.tmp_dir = tempfile.mkdtemp(prefix = 'g-') 389 _log.info('final (app instance level) temp dir: %s', tempfile.gettempdir()) 390 391 # BYTEA cache dir 392 cache_dir = os.path.join(self.user_tmp_dir, '.bytea_cache') 393 try: 394 stat = os.stat(cache_dir) 395 _log.warning('reusing BYTEA cache dir: %s', cache_dir) 396 _log.debug(stat) 397 except FileNotFoundError: 398 mkdir(cache_dir, mode = 0o0700) 399 self.bytea_cache_dir = cache_dir 400 401 self.__log_paths() 402 if wx is None: 403 return True 404 405 # retry with wxPython 406 _log.debug('re-detecting paths with wxPython') 407 408 std_paths = wx.StandardPaths.Get() 409 _log.info('wxPython app name is [%s]', wx.GetApp().GetAppName()) 410 411 # user-specific config dir, usually below the home dir 412 mkdir(os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name)) 413 self.user_config_dir = os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name) 414 415 # system-wide config dir, usually below /etc/ under UN*X 416 try: 417 tmp = std_paths.GetConfigDir() 418 if not tmp.endswith(app_name): 419 tmp = os.path.join(tmp, app_name) 420 self.system_config_dir = tmp 421 except ValueError: 422 # leave it at what it was from direct detection 423 pass 424 425 # system-wide application data dir 426 # Robin attests that the following doesn't always 427 # give sane values on Windows, so IFDEF it 428 if 'wxMSW' in wx.PlatformInfo: 429 _log.warning('this platform (wxMSW) sometimes returns a broken value for the system-wide application data dir') 430 else: 431 try: 432 self.system_app_data_dir = std_paths.GetDataDir() 433 except ValueError: 434 pass 435 436 self.__log_paths() 437 return True
438 439 #--------------------------------------
440 - def __log_paths(self):
441 _log.debug('sys.argv[0]: %s', sys.argv[0]) 442 _log.debug('sys.executable: %s', sys.executable) 443 _log.debug('sys._MEIPASS: %s', getattr(sys, '_MEIPASS', '<not found>')) 444 _log.debug('os.environ["_MEIPASS2"]: %s', os.environ.get('_MEIPASS2', '<not found>')) 445 _log.debug('__file__ : %s', __file__) 446 _log.debug('local application base dir: %s', self.local_base_dir) 447 _log.debug('current working dir: %s', self.working_dir) 448 _log.debug('user home dir: %s', self.home_dir) 449 _log.debug('user-specific config dir: %s', self.user_config_dir) 450 _log.debug('system-wide config dir: %s', self.system_config_dir) 451 _log.debug('system-wide application data dir: %s', self.system_app_data_dir) 452 _log.debug('temporary dir (user): %s', self.user_tmp_dir) 453 _log.debug('temporary dir (instance): %s', self.tmp_dir) 454 _log.debug('BYTEA cache dir: %s', self.bytea_cache_dir)
455 456 #-------------------------------------- 457 # properties 458 #--------------------------------------
459 - def _set_user_config_dir(self, path):
460 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 461 msg = '[%s:user_config_dir]: invalid path [%s]' % (self.__class__.__name__, path) 462 _log.error(msg) 463 raise ValueError(msg) 464 self.__user_config_dir = path
465
466 - def _get_user_config_dir(self):
467 return self.__user_config_dir
468 469 user_config_dir = property(_get_user_config_dir, _set_user_config_dir) 470 #--------------------------------------
471 - def _set_system_config_dir(self, path):
472 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 473 msg = '[%s:system_config_dir]: invalid path [%s]' % (self.__class__.__name__, path) 474 _log.error(msg) 475 raise ValueError(msg) 476 self.__system_config_dir = path
477
478 - def _get_system_config_dir(self):
479 return self.__system_config_dir
480 481 system_config_dir = property(_get_system_config_dir, _set_system_config_dir) 482 #--------------------------------------
483 - def _set_system_app_data_dir(self, path):
484 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 485 msg = '[%s:system_app_data_dir]: invalid path [%s]' % (self.__class__.__name__, path) 486 _log.error(msg) 487 raise ValueError(msg) 488 self.__system_app_data_dir = path
489
490 - def _get_system_app_data_dir(self):
491 return self.__system_app_data_dir
492 493 system_app_data_dir = property(_get_system_app_data_dir, _set_system_app_data_dir) 494 #--------------------------------------
495 - def _set_home_dir(self, path):
496 raise ValueError('invalid to set home dir')
497
498 - def _get_home_dir(self):
499 if self.__home_dir is not None: 500 return self.__home_dir 501 502 tmp = os.path.expanduser('~') 503 if tmp == '~': 504 _log.error('this platform does not expand ~ properly') 505 try: 506 tmp = os.environ['USERPROFILE'] 507 except KeyError: 508 _log.error('cannot access $USERPROFILE in environment') 509 510 if not ( 511 os.access(tmp, os.R_OK) 512 and 513 os.access(tmp, os.X_OK) 514 and 515 os.access(tmp, os.W_OK) 516 ): 517 msg = '[%s:home_dir]: invalid path [%s]' % (self.__class__.__name__, tmp) 518 _log.error(msg) 519 raise ValueError(msg) 520 521 self.__home_dir = tmp 522 return self.__home_dir
523 524 home_dir = property(_get_home_dir, _set_home_dir) 525 526 #--------------------------------------
527 - def _set_tmp_dir(self, path):
528 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 529 msg = '[%s:tmp_dir]: invalid path [%s]' % (self.__class__.__name__, path) 530 _log.error(msg) 531 raise ValueError(msg) 532 _log.debug('previous temp dir: %s', tempfile.gettempdir()) 533 self.__tmp_dir = path 534 tempfile.tempdir = self.__tmp_dir 535 _log.debug('new temp dir: %s', tempfile.gettempdir()) 536 self.__tmp_dir_already_set = True
537
538 - def _get_tmp_dir(self):
539 return self.__tmp_dir
540 541 tmp_dir = property(_get_tmp_dir, _set_tmp_dir)
542 543 #=========================================================================== 544 # file related tools 545 #---------------------------------------------------------------------------
546 -def recode_file(source_file=None, target_file=None, source_encoding='utf8', target_encoding=None, base_dir=None, error_mode='replace'):
547 if target_encoding is None: 548 return source_file 549 if target_encoding == source_encoding: 550 return source_file 551 if target_file is None: 552 target_file = get_unique_filename ( 553 prefix = '%s-%s_%s-' % (fname_stem(source_file), source_encoding, target_encoding), 554 suffix = fname_extension(source_file, '.txt'), 555 tmp_dir = base_dir 556 ) 557 558 _log.debug('[%s] -> [%s] (%s -> %s)', source_encoding, target_encoding, source_file, target_file) 559 560 in_file = io.open(source_file, mode = 'rt', encoding = source_encoding) 561 out_file = io.open(target_file, mode = 'wt', encoding = target_encoding, errors = error_mode) 562 for line in in_file: 563 out_file.write(line) 564 out_file.close() 565 in_file.close() 566 567 return target_file
568 569 #---------------------------------------------------------------------------
570 -def unzip_archive(archive_name, target_dir=None, remove_archive=False):
571 _log.debug('unzipping [%s] -> [%s]', archive_name, target_dir) 572 success = False 573 try: 574 with zipfile.ZipFile(archive_name) as archive: 575 archive.extractall(target_dir) 576 success = True 577 except Exception: 578 _log.exception('cannot unzip') 579 return False 580 if remove_archive: 581 remove_file(archive_name) 582 return success
583 584 #---------------------------------------------------------------------------
585 -def remove_file(filename, log_error=True, force=False):
586 if not os.path.lexists(filename): 587 return True 588 589 # attempt file remove and ignore (but log) errors 590 try: 591 os.remove(filename) 592 return True 593 594 except Exception: 595 if log_error: 596 _log.exception('cannot os.remove(%s)', filename) 597 598 if force: 599 tmp_name = get_unique_filename(tmp_dir = fname_dir(filename)) 600 _log.debug('attempting os.replace(%s -> %s)', filename, tmp_name) 601 try: 602 os.replace(filename, tmp_name) 603 return True 604 605 except Exception: 606 if log_error: 607 _log.exception('cannot os.remove(%s)', filename) 608 609 return False
610 611 #---------------------------------------------------------------------------
612 -def file2md5(filename=None, return_hex=True):
613 blocksize = 2**10 * 128 # 128k, since md5 uses 128 byte blocks 614 _log.debug('md5(%s): <%s> byte blocks', filename, blocksize) 615 616 f = io.open(filename, mode = 'rb') 617 618 md5 = hashlib.md5() 619 while True: 620 data = f.read(blocksize) 621 if not data: 622 break 623 md5.update(data) 624 f.close() 625 626 _log.debug('md5(%s): %s', filename, md5.hexdigest()) 627 628 if return_hex: 629 return md5.hexdigest() 630 return md5.digest()
631 632 #---------------------------------------------------------------------------
633 -def file2chunked_md5(filename=None, chunk_size=500*_MB):
634 _log.debug('chunked_md5(%s, chunk_size=%s bytes)', filename, chunk_size) 635 md5_concat = '' 636 f = open(filename, 'rb') 637 while True: 638 md5 = hashlib.md5() 639 data = f.read(chunk_size) 640 if not data: 641 break 642 md5.update(data) 643 md5_concat += md5.hexdigest() 644 f.close() 645 md5 = hashlib.md5() 646 md5.update(md5_concat) 647 hex_digest = md5.hexdigest() 648 _log.debug('md5("%s"): %s', md5_concat, hex_digest) 649 return hex_digest
650 651 #--------------------------------------------------------------------------- 652 default_csv_reader_rest_key = 'list_of_values_of_unknown_fields' 653
654 -def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
655 try: 656 is_dict_reader = kwargs['dict'] 657 del kwargs['dict'] 658 except KeyError: 659 is_dict_reader = False 660 661 if is_dict_reader: 662 kwargs['restkey'] = default_csv_reader_rest_key 663 return csv.DictReader(unicode_csv_data, dialect=dialect, **kwargs) 664 return csv.reader(csv_data, dialect=dialect, **kwargs)
665 666 667 668
669 -def old_unicode2charset_encoder(unicode_csv_data, encoding='utf-8'):
670 for line in unicode_csv_data: 671 yield line.encode(encoding)
672 673 #def utf_8_encoder(unicode_csv_data): 674 # for line in unicode_csv_data: 675 # yield line.encode('utf-8') 676
677 -def old_unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
678 679 # csv.py doesn't do Unicode; encode temporarily as UTF-8: 680 try: 681 is_dict_reader = kwargs['dict'] 682 del kwargs['dict'] 683 if is_dict_reader is not True: 684 raise KeyError 685 kwargs['restkey'] = default_csv_reader_rest_key 686 csv_reader = csv.DictReader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs) 687 except KeyError: 688 is_dict_reader = False 689 csv_reader = csv.reader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs) 690 691 for row in csv_reader: 692 # decode ENCODING back to Unicode, cell by cell: 693 if is_dict_reader: 694 for key in row.keys(): 695 if key == default_csv_reader_rest_key: 696 old_data = row[key] 697 new_data = [] 698 for val in old_data: 699 new_data.append(str(val, encoding)) 700 row[key] = new_data 701 if default_csv_reader_rest_key not in csv_reader.fieldnames: 702 csv_reader.fieldnames.append(default_csv_reader_rest_key) 703 else: 704 row[key] = str(row[key], encoding) 705 yield row 706 else: 707 yield [ str(cell, encoding) for cell in row ]
708 #yield [str(cell, 'utf-8') for cell in row] 709 710 #---------------------------------------------------------------------------
711 -def fname_sanitize(filename):
712 """Normalizes unicode, removes non-alpha characters, converts spaces to underscores.""" 713 714 dir_part, name_part = os.path.split(filename) 715 if name_part == '': 716 return filename 717 718 import unicodedata 719 name_part = unicodedata.normalize('NFKD', name_part) 720 # remove everything not in group [] 721 name_part = regex.sub ( 722 '[^.\w\s[\]()%§+-]', 723 '', 724 name_part, 725 flags = regex.UNICODE 726 ).strip() 727 # translate whitespace to underscore 728 name_part = regex.sub ( 729 '\s+', 730 '_', 731 name_part, 732 flags = regex.UNICODE 733 ) 734 return os.path.join(dir_part, name_part)
735 736 #---------------------------------------------------------------------------
737 -def fname_stem(filename):
738 """/home/user/dir/filename.ext -> filename""" 739 return os.path.splitext(os.path.basename(filename))[0]
740 741 #---------------------------------------------------------------------------
742 -def fname_stem_with_path(filename):
743 """/home/user/dir/filename.ext -> /home/user/dir/filename""" 744 return os.path.splitext(filename)[0]
745 746 #---------------------------------------------------------------------------
747 -def fname_extension(filename=None, fallback=None):
748 """ /home/user/dir/filename.ext -> .ext 749 '' or '.' -> fallback if any else '' 750 """ 751 ext = os.path.splitext(filename)[1] 752 if ext.strip() not in ['.', '']: 753 return ext 754 if fallback is None: 755 return '' 756 return fallback
757 758 #---------------------------------------------------------------------------
759 -def fname_dir(filename):
760 # /home/user/dir/filename.ext -> /home/user/dir 761 return os.path.split(filename)[0]
762 763 #---------------------------------------------------------------------------
764 -def fname_from_path(filename):
765 # /home/user/dir/filename.ext -> filename.ext 766 return os.path.split(filename)[1]
767 768 #---------------------------------------------------------------------------
769 -def get_unique_filename(prefix=None, suffix=None, tmp_dir=None, include_timestamp=False):
770 """This function has a race condition between 771 its file.close() 772 and actually 773 using the filename in callers. 774 775 The file will NOT exist after calling this function. 776 """ 777 if tmp_dir is not None: 778 if ( 779 not os.access(tmp_dir, os.F_OK) 780 or 781 not os.access(tmp_dir, os.X_OK | os.W_OK) 782 ): 783 _log.warning('cannot os.access() temporary dir [%s], using system default', tmp_dir) 784 tmp_dir = None 785 786 if include_timestamp: 787 ts = pydt.datetime.now().strftime('%m%d-%H%M%S-') 788 else: 789 ts = '' 790 791 kwargs = { 792 'dir': tmp_dir, 793 # make sure file gets deleted as soon as 794 # .close()d so we can "safely" open it again 795 'delete': True 796 } 797 798 if prefix is None: 799 kwargs['prefix'] = 'gm-%s' % ts 800 else: 801 kwargs['prefix'] = prefix + ts 802 803 if suffix in [None, '']: 804 kwargs['suffix'] = '.tmp' 805 else: 806 if not suffix.startswith('.'): 807 suffix = '.' + suffix 808 kwargs['suffix'] = suffix 809 810 f = tempfile.NamedTemporaryFile(**kwargs) 811 filename = f.name 812 f.close() 813 814 return filename
815 816 #--------------------------------------------------------------------------- 833 834 #--------------------------------------------------------------------------- 858 859 #===========================================================================
860 -def import_module_from_directory(module_path=None, module_name=None, always_remove_path=False):
861 """Import a module from any location.""" 862 863 _log.debug('CWD: %s', os.getcwd()) 864 865 remove_path = always_remove_path or False 866 if module_path not in sys.path: 867 _log.info('appending to sys.path: [%s]' % module_path) 868 sys.path.append(module_path) 869 remove_path = True 870 871 _log.debug('will remove import path: %s', remove_path) 872 873 if module_name.endswith('.py'): 874 module_name = module_name[:-3] 875 876 try: 877 module = __import__(module_name) 878 except Exception: 879 _log.exception('cannot __import__() module [%s] from [%s]' % (module_name, module_path)) 880 while module_path in sys.path: 881 sys.path.remove(module_path) 882 raise 883 884 _log.info('imported module [%s] as [%s]' % (module_name, module)) 885 if remove_path: 886 while module_path in sys.path: 887 sys.path.remove(module_path) 888 889 return module
890 891 #=========================================================================== 892 # text related tools 893 #---------------------------------------------------------------------------
894 -def size2str(size=0, template='%s'):
895 if size == 1: 896 return template % _('1 Byte') 897 if size < 10 * _kB: 898 return template % _('%s Bytes') % size 899 if size < _MB: 900 return template % '%.1f kB' % (float(size) / _kB) 901 if size < _GB: 902 return template % '%.1f MB' % (float(size) / _MB) 903 if size < _TB: 904 return template % '%.1f GB' % (float(size) / _GB) 905 if size < _PB: 906 return template % '%.1f TB' % (float(size) / _TB) 907 return template % '%.1f PB' % (float(size) / _PB)
908 909 #---------------------------------------------------------------------------
910 -def bool2subst(boolean=None, true_return=True, false_return=False, none_return=None):
911 if boolean is None: 912 return none_return 913 if boolean: 914 return true_return 915 if not boolean: 916 return false_return 917 raise ValueError('bool2subst(): <boolean> arg must be either of True, False, None')
918 919 #---------------------------------------------------------------------------
920 -def bool2str(boolean=None, true_str='True', false_str='False'):
921 return bool2subst ( 922 boolean = bool(boolean), 923 true_return = true_str, 924 false_return = false_str 925 )
926 927 #---------------------------------------------------------------------------
928 -def none_if(value=None, none_equivalent=None, strip_string=False):
929 """Modelled after the SQL NULLIF function.""" 930 if value is None: 931 return None 932 if strip_string: 933 stripped = value.strip() 934 else: 935 stripped = value 936 if stripped == none_equivalent: 937 return None 938 return value
939 940 #---------------------------------------------------------------------------
941 -def coalesce(value2test=None, return_instead=None, template4value=None, template4instead=None, none_equivalents=None, function4value=None, value2return=None):
942 """Modelled after the SQL coalesce function. 943 944 To be used to simplify constructs like: 945 946 if value2test is None (or in none_equivalents): 947 value = (template4instead % return_instead) or return_instead 948 else: 949 value = (template4value % value2test) or value2test 950 print value 951 952 @param value2test: the value to be tested for <None> 953 @type value2test: any Python type, must have a __str__ method if template4value is not None 954 @param return_instead: the value to be returned if <initial> is None 955 @type return_instead: any Python type, must have a __str__ method if template4instead is not None 956 @param template4value: if <initial> is returned replace the value into this template, must contain one <%s> 957 @type template4value: string or None 958 @param template4instead: if <return_instead> is returned replace the value into this template, must contain one <%s> 959 @type template4instead: string or None 960 961 example: 962 function4value = ('strftime', '%Y-%m-%d') 963 964 Ideas: 965 - list of return_insteads: initial, [return_instead, template], [return_instead, template], [return_instead, template], template4value, ... 966 """ 967 if none_equivalents is None: 968 none_equivalents = [None] 969 970 if value2test in none_equivalents: 971 if template4instead is None: 972 return return_instead 973 return template4instead % return_instead 974 975 # at this point, value2test was not equivalent to None 976 977 # 1) explicit value to return supplied ? 978 if value2return is not None: 979 return value2return 980 981 value2return = value2test 982 # 2) function supplied to be applied to the value ? 983 if function4value is not None: 984 funcname, args = function4value 985 func = getattr(value2test, funcname) 986 value2return = func(args) 987 988 # 3) template supplied to be applied to the value ? 989 if template4value is None: 990 return value2return 991 992 try: 993 return template4value % value2return 994 except TypeError: 995 # except (TypeError, ValueError): 996 # this should go, actually, only needed because "old" calls 997 # to coalesce will still abuse template4value as explicit value2return, 998 # relying on the replacement to above to fail 999 if hasattr(_log, 'log_stack_trace'): 1000 _log.log_stack_trace(message = 'deprecated use of <template4value> for <value2return>') 1001 else: 1002 _log.error('deprecated use of <template4value> for <value2return>') 1003 _log.error(locals()) 1004 return template4value
1005 1006 #---------------------------------------------------------------------------
1007 -def __cap_name(match_obj=None):
1008 val = match_obj.group(0).lower() 1009 if val in ['von', 'van', 'de', 'la', 'l', 'der', 'den']: # FIXME: this needs to expand, configurable ? 1010 return val 1011 buf = list(val) 1012 buf[0] = buf[0].upper() 1013 for part in ['mac', 'mc', 'de', 'la']: 1014 if len(val) > len(part) and val[:len(part)] == part: 1015 buf[len(part)] = buf[len(part)].upper() 1016 return ''.join(buf)
1017 1018 #---------------------------------------------------------------------------
1019 -def capitalize(text=None, mode=CAPS_NAMES):
1020 """Capitalize the first character but leave the rest alone. 1021 1022 Note that we must be careful about the locale, this may 1023 have issues ! However, for UTF strings it should just work. 1024 """ 1025 if (mode is None) or (mode == CAPS_NONE): 1026 return text 1027 1028 if len(text) == 0: 1029 return text 1030 1031 if mode == CAPS_FIRST: 1032 if len(text) == 1: 1033 return text[0].upper() 1034 return text[0].upper() + text[1:] 1035 1036 if mode == CAPS_ALLCAPS: 1037 return text.upper() 1038 1039 if mode == CAPS_FIRST_ONLY: 1040 # if len(text) == 1: 1041 # return text[0].upper() 1042 return text[0].upper() + text[1:].lower() 1043 1044 if mode == CAPS_WORDS: 1045 #return regex.sub(ur'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text) 1046 return regex.sub(r'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text) 1047 1048 if mode == CAPS_NAMES: 1049 #return regex.sub(r'\w+', __cap_name, text) 1050 return capitalize(text=text, mode=CAPS_FIRST) # until fixed 1051 1052 print("ERROR: invalid capitalization mode: [%s], leaving input as is" % mode) 1053 return text
1054 1055 #---------------------------------------------------------------------------
1056 -def input2decimal(initial=None):
1057 1058 if isinstance(initial, decimal.Decimal): 1059 return True, initial 1060 1061 val = initial 1062 1063 # float ? -> to string first 1064 if type(val) == type(float(1.4)): 1065 val = str(val) 1066 1067 # string ? -> "," to "." 1068 if isinstance(val, str): 1069 val = val.replace(',', '.', 1) 1070 val = val.strip() 1071 1072 try: 1073 d = decimal.Decimal(val) 1074 return True, d 1075 except (TypeError, decimal.InvalidOperation): 1076 return False, val
1077 1078 #---------------------------------------------------------------------------
1079 -def input2int(initial=None, minval=None, maxval=None):
1080 1081 val = initial 1082 1083 # string ? -> "," to "." 1084 if isinstance(val, str): 1085 val = val.replace(',', '.', 1) 1086 val = val.strip() 1087 1088 try: 1089 int_val = int(val) 1090 except (TypeError, ValueError): 1091 _log.exception('int(%s) failed', val) 1092 return False, initial 1093 1094 if minval is not None: 1095 if int_val < minval: 1096 _log.debug('%s < min (%s)', val, minval) 1097 return False, initial 1098 if maxval is not None: 1099 if int_val > maxval: 1100 _log.debug('%s > max (%s)', val, maxval) 1101 return False, initial 1102 1103 return True, int_val
1104 1105 #---------------------------------------------------------------------------
1106 -def strip_prefix(text, prefix, remove_repeats=False, remove_whitespace=False):
1107 if remove_repeats: 1108 if remove_whitespace: 1109 while text.lstrip().startswith(prefix): 1110 text = text.lstrip().replace(prefix, '', 1).lstrip() 1111 return text 1112 while text.startswith(prefix): 1113 text = text.replace(prefix, '', 1) 1114 return text 1115 if remove_whitespace: 1116 return text.lstrip().replace(prefix, '', 1).lstrip() 1117 return text.replace(prefix, '', 1)
1118 1119 #---------------------------------------------------------------------------
1120 -def strip_suffix(text, suffix, remove_repeats=False, remove_whitespace=False):
1121 suffix_len = len(suffix) 1122 if remove_repeats: 1123 if remove_whitespace: 1124 while text.rstrip().endswith(suffix): 1125 text = text.rstrip()[:-suffix_len].rstrip() 1126 return text 1127 while text.endswith(suffix): 1128 text = text[:-suffix_len] 1129 return text 1130 if remove_whitespace: 1131 return text.rstrip()[:-suffix_len].rstrip() 1132 return text[:-suffix_len]
1133 1134 #---------------------------------------------------------------------------
1135 -def strip_leading_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1136 if lines is None: 1137 lines = text.split(eol) 1138 1139 while True: 1140 if lines[0].strip(eol).strip() != '': 1141 break 1142 lines = lines[1:] 1143 1144 if return_list: 1145 return lines 1146 1147 return eol.join(lines)
1148 1149 #---------------------------------------------------------------------------
1150 -def strip_trailing_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1151 if lines is None: 1152 lines = text.split(eol) 1153 1154 while True: 1155 if lines[-1].strip(eol).strip() != '': 1156 break 1157 lines = lines[:-1] 1158 1159 if return_list: 1160 return lines 1161 1162 return eol.join(lines)
1163 1164 #---------------------------------------------------------------------------
1165 -def strip_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1166 return strip_trailing_empty_lines ( 1167 lines = strip_leading_empty_lines(lines = lines, text = text, eol = eol, return_list = True), 1168 text = None, 1169 eol = eol, 1170 return_list = return_list 1171 )
1172 1173 #---------------------------------------------------------------------------
1174 -def list2text(lines, initial_indent='', subsequent_indent='', eol='\n', strip_leading_empty_lines=True, strip_trailing_empty_lines=True, strip_trailing_whitespace=True, max_line_width=None):
1175 1176 if len(lines) == 0: 1177 return '' 1178 1179 if strip_leading_empty_lines: 1180 lines = strip_leading_empty_lines(lines = lines, eol = eol, return_list = True) 1181 1182 if strip_trailing_empty_lines: 1183 lines = strip_trailing_empty_lines(lines = lines, eol = eol, return_list = True) 1184 1185 if strip_trailing_whitespace: 1186 lines = [ l.rstrip() for l in lines ] 1187 1188 if max_line_width is not None: 1189 wrapped_lines = [] 1190 for l in lines: 1191 wrapped_lines.extend(wrap(l, max_line_width).split('\n')) 1192 lines = wrapped_lines 1193 1194 indented_lines = [initial_indent + lines[0]] 1195 indented_lines.extend([ subsequent_indent + l for l in lines[1:] ]) 1196 1197 return eol.join(indented_lines)
1198 1199 #---------------------------------------------------------------------------
1200 -def wrap(text=None, width=None, initial_indent='', subsequent_indent='', eol='\n'):
1201 """A word-wrap function that preserves existing line breaks 1202 and most spaces in the text. Expects that existing line 1203 breaks are posix newlines (\n). 1204 """ 1205 if width is None: 1206 return text 1207 1208 wrapped = initial_indent + functools.reduce ( 1209 lambda line, word, width=width: '%s%s%s' % ( 1210 line, 1211 ' \n'[(len(line) - line.rfind('\n') - 1 + len(word.split('\n',1)[0]) >= width)], 1212 word 1213 ), 1214 text.split(' ') 1215 ) 1216 if subsequent_indent != '': 1217 wrapped = ('\n%s' % subsequent_indent).join(wrapped.split('\n')) 1218 if eol != '\n': 1219 wrapped = wrapped.replace('\n', eol) 1220 return wrapped
1221 1222 #---------------------------------------------------------------------------
1223 -def unwrap(text=None, max_length=None, strip_whitespace=True, remove_empty_lines=True, line_separator = ' // '):
1224 1225 text = text.replace('\r', '') 1226 lines = text.split('\n') 1227 text = '' 1228 for line in lines: 1229 1230 if strip_whitespace: 1231 line = line.strip().strip('\t').strip() 1232 1233 if remove_empty_lines: 1234 if line == '': 1235 continue 1236 1237 text += ('%s%s' % (line, line_separator)) 1238 1239 text = text.rstrip(line_separator) 1240 1241 if max_length is not None: 1242 text = text[:max_length] 1243 1244 text = text.rstrip(line_separator) 1245 1246 return text
1247 1248 #---------------------------------------------------------------------------
1249 -def shorten_text(text=None, max_length=None):
1250 1251 if len(text) <= max_length: 1252 return text 1253 1254 return text[:max_length-1] + u_ellipsis
1255 1256 #---------------------------------------------------------------------------
1257 -def shorten_words_in_line(text=None, max_length=None, min_word_length=None, ignore_numbers=True, ellipsis=u_ellipsis):
1258 if text is None: 1259 return None 1260 if max_length is None: 1261 max_length = len(text) 1262 else: 1263 if len(text) <= max_length: 1264 return text 1265 old_words = regex.split('\s+', text, flags = regex.UNICODE) 1266 no_old_words = len(old_words) 1267 max_word_length = max(min_word_length, (max_length // no_old_words)) 1268 words = [] 1269 for word in old_words: 1270 if len(word) <= max_word_length: 1271 words.append(word) 1272 continue 1273 if ignore_numbers: 1274 tmp = word.replace('-', '').replace('+', '').replace('.', '').replace(',', '').replace('/', '').replace('&', '').replace('*', '') 1275 if tmp.isdigit(): 1276 words.append(word) 1277 continue 1278 words.append(word[:max_word_length] + ellipsis) 1279 return ' '.join(words)
1280 1281 #---------------------------------------------------------------------------
1282 -def xml_escape_string(text=None):
1283 """check for special XML characters and transform them""" 1284 return xml_tools.escape(text)
1285 1286 #---------------------------------------------------------------------------
1287 -def tex_escape_string(text=None, replace_known_unicode=True, replace_eol=False, keep_visual_eol=False):
1288 """Check for special TeX characters and transform them. 1289 1290 replace_eol: 1291 replaces "\n" with "\\newline" 1292 keep_visual_eol: 1293 replaces "\n" with "\\newline \n" such that 1294 both LaTeX will know to place a line break 1295 at this point as well as the visual formatting 1296 is preserved in the LaTeX source (think multi- 1297 row table cells) 1298 """ 1299 text = text.replace('\\', '\\textbackslash') # requires \usepackage{textcomp} in LaTeX source 1300 text = text.replace('^', '\\textasciicircum') 1301 text = text.replace('~', '\\textasciitilde') 1302 1303 text = text.replace('{', '\\{') 1304 text = text.replace('}', '\\}') 1305 text = text.replace('%', '\\%') 1306 text = text.replace('&', '\\&') 1307 text = text.replace('#', '\\#') 1308 text = text.replace('$', '\\$') 1309 text = text.replace('_', '\\_') 1310 if replace_eol: 1311 if keep_visual_eol: 1312 text = text.replace('\n', '\\newline \n') 1313 else: 1314 text = text.replace('\n', '\\newline ') 1315 1316 if replace_known_unicode: 1317 # this should NOT be replaced for Xe(La)Tex 1318 text = text.replace(u_euro, '\\EUR') # requires \usepackage{textcomp} in LaTeX source 1319 text = text.replace(u_sum, '$\\Sigma$') 1320 1321 return text
1322 1323 #---------------------------------------------------------------------------
1324 -def rst2latex_snippet(rst_text):
1325 global du_core 1326 if du_core is None: 1327 try: 1328 from docutils import core as du_core 1329 except ImportError: 1330 _log.warning('cannot turn ReST into LaTeX: docutils not installed') 1331 return tex_escape_string(text = rst_text) 1332 1333 parts = du_core.publish_parts ( 1334 source = rst_text.replace('\\', '\\\\'), 1335 source_path = '<internal>', 1336 writer_name = 'latex', 1337 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex', 1338 settings_overrides = { 1339 'input_encoding': 'unicode' # un-encoded unicode 1340 }, 1341 enable_exit_status = True # how to use ? 1342 ) 1343 return parts['body']
1344 1345 #---------------------------------------------------------------------------
1346 -def rst2html(rst_text, replace_eol=False, keep_visual_eol=False):
1347 global du_core 1348 if du_core is None: 1349 try: 1350 from docutils import core as du_core 1351 except ImportError: 1352 _log.warning('cannot turn ReST into HTML: docutils not installed') 1353 return html_escape_string(text = rst_text, replace_eol=False, keep_visual_eol=False) 1354 1355 parts = du_core.publish_parts ( 1356 source = rst_text.replace('\\', '\\\\'), 1357 source_path = '<internal>', 1358 writer_name = 'latex', 1359 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex', 1360 settings_overrides = { 1361 'input_encoding': 'unicode' # un-encoded unicode 1362 }, 1363 enable_exit_status = True # how to use ? 1364 ) 1365 return parts['body']
1366 1367 #---------------------------------------------------------------------------
1368 -def xetex_escape_string(text=None):
1369 # a web search did not reveal anything else for Xe(La)Tex 1370 # as opposed to LaTeX, except true unicode chars 1371 return tex_escape_string(text = text, replace_known_unicode = False)
1372 1373 #--------------------------------------------------------------------------- 1374 __html_escape_table = { 1375 "&": "&amp;", 1376 '"': "&quot;", 1377 "'": "&apos;", 1378 ">": "&gt;", 1379 "<": "&lt;", 1380 } 1381
1382 -def html_escape_string(text=None, replace_eol=False, keep_visual_eol=False):
1383 text = ''.join(__html_escape_table.get(char, char) for char in text) 1384 if replace_eol: 1385 if keep_visual_eol: 1386 text = text.replace('\n', '<br>\n') 1387 else: 1388 text = text.replace('\n', '<br>') 1389 return text
1390 1391 #---------------------------------------------------------------------------
1392 -def dict2json(obj):
1393 return json.dumps(obj, default = json_serialize)
1394 1395 #---------------------------------------------------------------------------
1396 -def json_serialize(obj):
1397 if isinstance(obj, pydt.datetime): 1398 return obj.isoformat() 1399 raise TypeError('cannot json_serialize(%s)' % type(obj))
1400 1401 #--------------------------------------------------------------------------- 1402 #---------------------------------------------------------------------------
1403 -def compare_dict_likes(d1, d2, title1=None, title2=None):
1404 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title1, '', '"%s" '), type(d1), coalesce(title2, '', '"%s" '), type(d2)) 1405 try: 1406 d1 = dict(d1) 1407 except TypeError: 1408 pass 1409 try: 1410 d2 = dict(d2) 1411 except TypeError: 1412 pass 1413 keys_d1 = frozenset(d1.keys()) 1414 keys_d2 = frozenset(d2.keys()) 1415 different = False 1416 if len(keys_d1) != len(keys_d2): 1417 _log.info('different number of keys: %s vs %s', len(keys_d1), len(keys_d2)) 1418 different = True 1419 for key in keys_d1: 1420 if key in keys_d2: 1421 if type(d1[key]) != type(d2[key]): 1422 _log.info('%25.25s: type(dict1) = %s = >>>%s<<<' % (key, type(d1[key]), d1[key])) 1423 _log.info('%25.25s type(dict2) = %s = >>>%s<<<' % ('', type(d2[key]), d2[key])) 1424 different = True 1425 continue 1426 if d1[key] == d2[key]: 1427 _log.info('%25.25s: both = >>>%s<<<' % (key, d1[key])) 1428 else: 1429 _log.info('%25.25s: dict1 = >>>%s<<<' % (key, d1[key])) 1430 _log.info('%25.25s dict2 = >>>%s<<<' % ('', d2[key])) 1431 different = True 1432 else: 1433 _log.info('%25.25s: %50.50s | <MISSING>' % (key, '>>>%s<<<' % d1[key])) 1434 different = True 1435 for key in keys_d2: 1436 if key in keys_d1: 1437 continue 1438 _log.info('%25.25s: %50.50s | %.50s' % (key, '<MISSING>', '>>>%s<<<' % d2[key])) 1439 different = True 1440 if different: 1441 _log.info('dict-likes appear to be different from each other') 1442 return False 1443 _log.info('dict-likes appear equal to each other') 1444 return True
1445 1446 #---------------------------------------------------------------------------
1447 -def format_dict_likes_comparison(d1, d2, title_left=None, title_right=None, left_margin=0, key_delim=' || ', data_delim=' | ', missing_string='=/=', difference_indicator='! ', ignore_diff_in_keys=None):
1448 1449 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title_left, '', '"%s" '), type(d1), coalesce(title_right, '', '"%s" '), type(d2)) 1450 append_type = False 1451 if None not in [title_left, title_right]: 1452 append_type = True 1453 type_left = type(d1) 1454 type_right = type(d2) 1455 if title_left is None: 1456 title_left = '%s' % type_left 1457 if title_right is None: 1458 title_right = '%s' % type_right 1459 1460 try: d1 = dict(d1) 1461 except TypeError: pass 1462 try: d2 = dict(d2) 1463 except TypeError: pass 1464 keys_d1 = d1.keys() 1465 keys_d2 = d2.keys() 1466 data = {} 1467 for key in keys_d1: 1468 data[key] = [d1[key], ' '] 1469 if key in d2: 1470 data[key][1] = d2[key] 1471 for key in keys_d2: 1472 if key in keys_d1: 1473 continue 1474 data[key] = [' ', d2[key]] 1475 max1 = max([ len('%s' % k) for k in keys_d1 ]) 1476 max2 = max([ len('%s' % k) for k in keys_d2 ]) 1477 max_len = max(max1, max2, len(_('<type>'))) 1478 max_key_len_str = '%' + '%s.%s' % (max_len, max_len) + 's' 1479 max1 = max([ len('%s' % d1[k]) for k in keys_d1 ]) 1480 max2 = max([ len('%s' % d2[k]) for k in keys_d2 ]) 1481 max_data_len = min(max(max1, max2), 100) 1482 max_data_len_str = '%' + '%s.%s' % (max_data_len, max_data_len) + 's' 1483 diff_indicator_len_str = '%' + '%s.%s' % (len(difference_indicator), len(difference_indicator)) + 's' 1484 line_template = (' ' * left_margin) + diff_indicator_len_str + max_key_len_str + key_delim + max_data_len_str + data_delim + '%s' 1485 1486 lines = [] 1487 # debugging: 1488 #lines.append(u' (40 regular spaces)') 1489 #lines.append((u' ' * 40) + u"(u' ' * 40)") 1490 #lines.append((u'%40.40s' % u'') + u"(u'%40.40s' % u'')") 1491 #lines.append((u'%40.40s' % u' ') + u"(u'%40.40s' % u' ')") 1492 #lines.append((u'%40.40s' % u'.') + u"(u'%40.40s' % u'.')") 1493 #lines.append(line_template) 1494 lines.append(line_template % ('', '', title_left, title_right)) 1495 if append_type: 1496 lines.append(line_template % ('', _('<type>'), type_left, type_right)) 1497 1498 if ignore_diff_in_keys is None: 1499 ignore_diff_in_keys = [] 1500 1501 for key in keys_d1: 1502 append_type = False 1503 txt_left_col = '%s' % d1[key] 1504 try: 1505 txt_right_col = '%s' % d2[key] 1506 if type(d1[key]) != type(d2[key]): 1507 append_type = True 1508 except KeyError: 1509 txt_right_col = missing_string 1510 lines.append(line_template % ( 1511 bool2subst ( 1512 ((txt_left_col == txt_right_col) or (key in ignore_diff_in_keys)), 1513 '', 1514 difference_indicator 1515 ), 1516 key, 1517 shorten_text(txt_left_col, max_data_len), 1518 shorten_text(txt_right_col, max_data_len) 1519 )) 1520 if append_type: 1521 lines.append(line_template % ( 1522 '', 1523 _('<type>'), 1524 shorten_text('%s' % type(d1[key]), max_data_len), 1525 shorten_text('%s' % type(d2[key]), max_data_len) 1526 )) 1527 1528 for key in keys_d2: 1529 if key in keys_d1: 1530 continue 1531 lines.append(line_template % ( 1532 bool2subst((key in ignore_diff_in_keys), '', difference_indicator), 1533 key, 1534 shorten_text(missing_string, max_data_len), 1535 shorten_text('%s' % d2[key], max_data_len) 1536 )) 1537 1538 return lines
1539 1540 #---------------------------------------------------------------------------
1541 -def format_dict_like(d, relevant_keys=None, template=None, missing_key_template='<[%(key)s] MISSING>', left_margin=0, tabular=False, value_delimiters=('>>>', '<<<'), eol='\n', values2ignore=None):
1542 if values2ignore is None: 1543 values2ignore = [] 1544 if template is not None: 1545 # all keys in template better exist in d 1546 try: 1547 return template % d 1548 except KeyError: 1549 # or else 1550 _log.exception('template contains %%()s key(s) which do not exist in data dict') 1551 # try to extend dict <d> to contain all required keys, 1552 # for that to work <relevant_keys> better list all 1553 # keys used in <template> 1554 if relevant_keys is not None: 1555 for key in relevant_keys: 1556 try: 1557 d[key] 1558 except KeyError: 1559 d[key] = missing_key_template % {'key': key} 1560 return template % d 1561 1562 if relevant_keys is None: 1563 relevant_keys = list(d.keys()) 1564 lines = [] 1565 if value_delimiters is None: 1566 delim_left = '' 1567 delim_right = '' 1568 else: 1569 delim_left, delim_right = value_delimiters 1570 if tabular: 1571 max_len = max([ len('%s' % k) for k in relevant_keys ]) 1572 max_len_str = '%s.%s' % (max_len, max_len) 1573 line_template = (' ' * left_margin) + '%' + max_len_str + ('s: %s%%s%s' % (delim_left, delim_right)) 1574 else: 1575 line_template = (' ' * left_margin) + '%%s: %s%%s%s' % (delim_left, delim_right) 1576 for key in relevant_keys: 1577 try: 1578 val = d[key] 1579 except KeyError: 1580 continue 1581 if val not in values2ignore: 1582 lines.append(line_template % (key, val)) 1583 if eol is None: 1584 return lines 1585 return eol.join(lines)
1586 1587 #---------------------------------------------------------------------------
1588 -def dicts2table(dict_list, left_margin=0, eol='\n', keys2ignore=None, headers=None, show_only_changes=False, equality_value='<=>', date_format=None): #, relevant_keys=None, template=None
1589 1590 keys2show = [] 1591 dict_max_size = {} 1592 max_row_label_size = 0 1593 if keys2ignore is None: 1594 keys2ignore = [] 1595 1596 # extract keys from all dicts 1597 for d in dict_list: 1598 dict_max_size[id(d)] = 0 1599 for key in d.keys(): 1600 if key in keys2ignore: 1601 continue 1602 dict_max_size[id(d)] = max(dict_max_size[id(d)], len('%s' % d[key])) 1603 if key in keys2show: 1604 continue 1605 keys2show.append(key) 1606 max_row_label_size = max(max_row_label_size, len('%s' % key)) 1607 1608 # pivot data into dict of lists per line 1609 lines = { k: [] for k in keys2show } 1610 prev_vals = {} 1611 for d in dict_list: 1612 if show_only_changes: 1613 max_size = max(dict_max_size[id(d)], len(equality_value)) 1614 else: 1615 max_size = dict_max_size[id(d)] 1616 max_len_str = '%s.%s' % (max_size, max_size) 1617 field_template = ' %' + max_len_str + 's' 1618 for key in keys2show: 1619 try: 1620 val = d[key] 1621 except KeyError: 1622 lines[key].append(field_template % _('<missing>')) 1623 continue 1624 if isinstance(val, pydt.datetime): 1625 if date_format is not None: 1626 val = val.strftime(date_format) 1627 lines[key].append(field_template % val) 1628 if show_only_changes: 1629 if key not in prev_vals: 1630 prev_vals[key] = '%s' % lines[key][-1] 1631 continue 1632 if lines[key][-1] != prev_vals[key]: 1633 prev_vals[key] = '%s' % lines[key][-1] 1634 continue 1635 lines[key][-1] = field_template % equality_value 1636 1637 # format data into table 1638 table_lines = [] 1639 max_len_str = '%s.%s' % (max_row_label_size, max_row_label_size) 1640 row_label_template = '%' + max_len_str + 's' 1641 for key in lines: 1642 line = (' ' * left_margin) + row_label_template % key + '|' 1643 line += '|'.join(lines[key]) 1644 table_lines.append(line) 1645 1646 # add header line if any 1647 if headers is not None: 1648 header_line1 = (' ' * left_margin) + row_label_template % '' 1649 header_line2 = (' ' * left_margin) + u_box_horiz_single * (max_row_label_size) 1650 header_sizes = [ max(dict_max_size[dict_id], len(equality_value)) for dict_id in dict_max_size ] 1651 for idx in range(len(headers)): 1652 max_len_str = '%s.%s' % (header_sizes[idx], header_sizes[idx]) 1653 header_template = '%' + max_len_str + 's' 1654 header_line1 += '| ' 1655 header_line1 += header_template % headers[idx] 1656 header_line2 += '%s%s' % (u_box_plus, u_box_horiz_single) 1657 header_line2 += u_box_horiz_single * header_sizes[idx] 1658 table_lines.insert(0, header_line2) 1659 table_lines.insert(0, header_line1) 1660 1661 if eol is None: 1662 return table_lines 1663 1664 return ('|' + eol).join(table_lines) + '|' + eol 1665 1666 #---------------------------------------------------------------------------
1667 -def normalize_dict_like(d, required_keys, missing_key_template='<[%(key)s] MISSING>'):
1668 for key in required_keys: 1669 try: 1670 d[key] 1671 except KeyError: 1672 if missing_key_template is None: 1673 d[key] = None 1674 else: 1675 d[key] = missing_key_template % {'key': key} 1676 return d
1677 1678 #---------------------------------------------------------------------------
1679 -def enumerate_removable_partitions():
1680 try: 1681 import pyudev 1682 import psutil 1683 except ImportError: 1684 _log.error('pyudev and/or psutil not installed') 1685 return {} 1686 1687 removable_partitions = {} 1688 ctxt = pyudev.Context() 1689 removable_devices = [ dev for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk') if dev.attributes.get('removable') == b'1' ] 1690 all_mounted_partitions = { part.device: part for part in psutil.disk_partitions() } 1691 for device in removable_devices: 1692 _log.debug('removable device: %s', device.properties['ID_MODEL']) 1693 partitions_on_removable_device = { 1694 part.device_node: { 1695 'type': device.properties['ID_TYPE'], 1696 'bus': device.properties['ID_BUS'], 1697 'device': device.properties['DEVNAME'], 1698 'partition': part.properties['DEVNAME'], 1699 'vendor': part.properties['ID_VENDOR'], 1700 'model': part.properties['ID_MODEL'], 1701 'fs_label': part.properties['ID_FS_LABEL'], 1702 'is_mounted': False, 1703 'mountpoint': None, 1704 'fs_type': None, 1705 'size_in_bytes': -1, 1706 'bytes_free': 0 1707 } for part in ctxt.list_devices(subsystem='block', DEVTYPE='partition', parent=device) 1708 } 1709 for part in partitions_on_removable_device: 1710 try: 1711 partitions_on_removable_device[part]['mountpoint'] = all_mounted_partitions[part].mountpoint 1712 partitions_on_removable_device[part]['is_mounted'] = True 1713 partitions_on_removable_device[part]['fs_type'] = all_mounted_partitions[part].fstype 1714 du = shutil.disk_usage(all_mounted_partitions[part].mountpoint) 1715 partitions_on_removable_device[part]['size_in_bytes'] = du.total 1716 partitions_on_removable_device[part]['bytes_free'] = du.free 1717 except KeyError: 1718 pass # not mounted 1719 removable_partitions.update(partitions_on_removable_device) 1720 return removable_partitions
1721 1722 # debugging: 1723 #ctxt = pyudev.Context() 1724 #for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk'):# if dev.attributes.get('removable') == b'1': 1725 # for a in dev.attributes.available_attributes: 1726 # print(a, dev.attributes.get(a)) 1727 # for key, value in dev.items(): 1728 # print('{key}={value}'.format(key=key, value=value)) 1729 # print('---------------------------') 1730 1731 #---------------------------------------------------------------------------
1732 -def enumerate_optical_writers():
1733 try: 1734 import pyudev 1735 except ImportError: 1736 _log.error('pyudev not installed') 1737 return [] 1738 1739 optical_writers = [] 1740 ctxt = pyudev.Context() 1741 for dev in [ dev for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk') if dev.properties.get('ID_CDROM_CD_RW', None) == '1' ]: 1742 optical_writers.append ({ 1743 'type': dev.properties['ID_TYPE'], 1744 'bus': dev.properties['ID_BUS'], 1745 'device': dev.properties['DEVNAME'], 1746 'model': dev.properties['ID_MODEL'] 1747 }) 1748 return optical_writers
1749 1750 #--------------------------------------------------------------------------- 1751 #---------------------------------------------------------------------------
1752 -def prompted_input(prompt=None, default=None):
1753 """Obtains entry from standard input. 1754 1755 prompt: Prompt text to display in standard output 1756 default: Default value (for user to press enter only) 1757 CTRL-C: aborts and returns None 1758 """ 1759 if prompt is None: 1760 msg = '(CTRL-C aborts)' 1761 else: 1762 msg = '%s (CTRL-C aborts)' % prompt 1763 1764 if default is None: 1765 msg = msg + ': ' 1766 else: 1767 msg = '%s [%s]: ' % (msg, default) 1768 1769 try: 1770 usr_input = input(msg) 1771 except KeyboardInterrupt: 1772 return None 1773 1774 if usr_input == '': 1775 return default 1776 1777 return usr_input
1778 1779 #=========================================================================== 1780 # image handling tools 1781 #--------------------------------------------------------------------------- 1782 # builtin (ugly but tried and true) fallback icon 1783 __icon_serpent = \ 1784 """x\xdae\x8f\xb1\x0e\x83 \x10\x86w\x9f\xe2\x92\x1blb\xf2\x07\x96\xeaH:0\xd6\ 1785 \xc1\x85\xd5\x98N5\xa5\xef?\xf5N\xd0\x8a\xdcA\xc2\xf7qw\x84\xdb\xfa\xb5\xcd\ 1786 \xd4\xda;\xc9\x1a\xc8\xb6\xcd<\xb5\xa0\x85\x1e\xeb\xbc\xbc7b!\xf6\xdeHl\x1c\ 1787 \x94\x073\xec<*\xf7\xbe\xf7\x99\x9d\xb21~\xe7.\xf5\x1f\x1c\xd3\xbdVlL\xc2\ 1788 \xcf\xf8ye\xd0\x00\x90\x0etH \x84\x80B\xaa\x8a\x88\x85\xc4(U\x9d$\xfeR;\xc5J\ 1789 \xa6\x01\xbbt9\xceR\xc8\x81e_$\x98\xb9\x9c\xa9\x8d,y\xa9t\xc8\xcf\x152\xe0x\ 1790 \xe9$\xf5\x07\x95\x0cD\x95t:\xb1\x92\xae\x9cI\xa8~\x84\x1f\xe0\xa3ec""" 1791
1792 -def get_icon(wx=None):
1793 1794 paths = gmPaths(app_name = 'gnumed', wx = wx) 1795 1796 candidates = [ 1797 os.path.join(paths.system_app_data_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'), 1798 os.path.join(paths.local_base_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'), 1799 os.path.join(paths.system_app_data_dir, 'bitmaps', 'serpent.png'), 1800 os.path.join(paths.local_base_dir, 'bitmaps', 'serpent.png') 1801 ] 1802 1803 found_as = None 1804 for candidate in candidates: 1805 try: 1806 open(candidate, 'r').close() 1807 found_as = candidate 1808 break 1809 except IOError: 1810 _log.debug('icon not found in [%s]', candidate) 1811 1812 if found_as is None: 1813 _log.warning('no icon file found, falling back to builtin (ugly) icon') 1814 icon_bmp_data = wx.BitmapFromXPMData(pickle.loads(zlib.decompress(__icon_serpent))) 1815 icon.CopyFromBitmap(icon_bmp_data) 1816 else: 1817 _log.debug('icon found in [%s]', found_as) 1818 icon = wx.Icon() 1819 try: 1820 icon.LoadFile(found_as, wx.BITMAP_TYPE_ANY) #_PNG 1821 except AttributeError: 1822 _log.exception("this platform doesn't support wx.Icon().LoadFile()") 1823 1824 return icon
1825 1826 #---------------------------------------------------------------------------
1827 -def create_qrcode(text=None, filename=None, qr_filename=None, verbose=False):
1828 assert (not ((text is None) and (filename is None))), 'either <text> or <filename> must be specified' 1829 1830 try: 1831 import pyqrcode 1832 except ImportError: 1833 _log.exception('cannot import <pyqrcode>') 1834 return None 1835 if text is None: 1836 with io.open(filename, mode = 'rt', encoding = 'utf8') as input_file: 1837 text = input_file.read() 1838 if qr_filename is None: 1839 if filename is None: 1840 qr_filename = get_unique_filename(prefix = 'gm-qr-', suffix = '.png') 1841 else: 1842 qr_filename = get_unique_filename ( 1843 prefix = fname_stem(filename) + '-', 1844 suffix = fname_extension(filename) + '.png' 1845 ) 1846 _log.debug('[%s] -> [%s]', filename, qr_filename) 1847 qr = pyqrcode.create(text, encoding = 'utf8') 1848 if verbose: 1849 print('input file:', filename) 1850 print('output file:', qr_filename) 1851 print('text to encode:', text) 1852 print(qr.terminal()) 1853 qr.png(qr_filename, quiet_zone = 1) 1854 return qr_filename
1855 1856 #=========================================================================== 1857 # main 1858 #--------------------------------------------------------------------------- 1859 if __name__ == '__main__': 1860 1861 if len(sys.argv) < 2: 1862 sys.exit() 1863 1864 if sys.argv[1] != 'test': 1865 sys.exit() 1866 1867 # for testing: 1868 logging.basicConfig(level = logging.DEBUG) 1869 from Gnumed.pycommon import gmI18N 1870 gmI18N.activate_locale() 1871 gmI18N.install_domain() 1872 1873 #-----------------------------------------------------------------------
1874 - def test_input2decimal():
1875 1876 tests = [ 1877 [None, False], 1878 1879 ['', False], 1880 [' 0 ', True, 0], 1881 1882 [0, True, 0], 1883 [0.0, True, 0], 1884 [.0, True, 0], 1885 ['0', True, 0], 1886 ['0.0', True, 0], 1887 ['0,0', True, 0], 1888 ['00.0', True, 0], 1889 ['.0', True, 0], 1890 [',0', True, 0], 1891 1892 [0.1, True, decimal.Decimal('0.1')], 1893 [.01, True, decimal.Decimal('0.01')], 1894 ['0.1', True, decimal.Decimal('0.1')], 1895 ['0,1', True, decimal.Decimal('0.1')], 1896 ['00.1', True, decimal.Decimal('0.1')], 1897 ['.1', True, decimal.Decimal('0.1')], 1898 [',1', True, decimal.Decimal('0.1')], 1899 1900 [1, True, 1], 1901 [1.0, True, 1], 1902 ['1', True, 1], 1903 ['1.', True, 1], 1904 ['1,', True, 1], 1905 ['1.0', True, 1], 1906 ['1,0', True, 1], 1907 ['01.0', True, 1], 1908 ['01,0', True, 1], 1909 [' 01, ', True, 1], 1910 1911 [decimal.Decimal('1.1'), True, decimal.Decimal('1.1')] 1912 ] 1913 for test in tests: 1914 conversion_worked, result = input2decimal(initial = test[0]) 1915 1916 expected2work = test[1] 1917 1918 if conversion_worked: 1919 if expected2work: 1920 if result == test[2]: 1921 continue 1922 else: 1923 print("ERROR (conversion result wrong): >%s<, expected >%s<, got >%s<" % (test[0], test[2], result)) 1924 else: 1925 print("ERROR (conversion worked but was expected to fail): >%s<, got >%s<" % (test[0], result)) 1926 else: 1927 if not expected2work: 1928 continue 1929 else: 1930 print("ERROR (conversion failed but was expected to work): >%s<, expected >%s<" % (test[0], test[2]))
1931 #-----------------------------------------------------------------------
1932 - def test_input2int():
1933 print(input2int(0)) 1934 print(input2int('0')) 1935 print(input2int('0', 0, 0))
1936 #-----------------------------------------------------------------------
1937 - def test_coalesce():
1938 1939 val = None 1940 print(val, coalesce(val, 'is None', 'is not None')) 1941 val = 1 1942 print(val, coalesce(val, 'is None', 'is not None')) 1943 return 1944 1945 import datetime as dt 1946 print(coalesce(value2test = dt.datetime.now(), template4value = '-- %s --', function4value = ('strftime', '%Y-%m-%d'))) 1947 1948 print('testing coalesce()') 1949 print("------------------") 1950 tests = [ 1951 [None, 'something other than <None>', None, None, 'something other than <None>'], 1952 ['Captain', 'Mr.', '%s.'[:4], 'Mr.', 'Capt.'], 1953 ['value to test', 'test 3 failed', 'template with "%s" included', None, 'template with "value to test" included'], 1954 ['value to test', 'test 4 failed', 'template with value not included', None, 'template with value not included'], 1955 [None, 'initial value was None', 'template4value: %s', None, 'initial value was None'], 1956 [None, 'initial value was None', 'template4value: %%(abc)s', None, 'initial value was None'] 1957 ] 1958 passed = True 1959 for test in tests: 1960 result = coalesce ( 1961 value2test = test[0], 1962 return_instead = test[1], 1963 template4value = test[2], 1964 template4instead = test[3] 1965 ) 1966 if result != test[4]: 1967 print("ERROR") 1968 print("coalesce: (%s, %s, %s, %s)" % (test[0], test[1], test[2], test[3])) 1969 print("expected:", test[4]) 1970 print("received:", result) 1971 passed = False 1972 1973 if passed: 1974 print("passed") 1975 else: 1976 print("failed") 1977 return passed
1978 #-----------------------------------------------------------------------
1979 - def test_capitalize():
1980 print('testing capitalize() ...') 1981 success = True 1982 pairs = [ 1983 # [original, expected result, CAPS mode] 1984 ['Boot', 'Boot', CAPS_FIRST_ONLY], 1985 ['boot', 'Boot', CAPS_FIRST_ONLY], 1986 ['booT', 'Boot', CAPS_FIRST_ONLY], 1987 ['BoOt', 'Boot', CAPS_FIRST_ONLY], 1988 ['boots-Schau', 'Boots-Schau', CAPS_WORDS], 1989 ['boots-sChau', 'Boots-Schau', CAPS_WORDS], 1990 ['boot camp', 'Boot Camp', CAPS_WORDS], 1991 ['fahrner-Kampe', 'Fahrner-Kampe', CAPS_NAMES], 1992 ['häkkönen', 'Häkkönen', CAPS_NAMES], 1993 ['McBurney', 'McBurney', CAPS_NAMES], 1994 ['mcBurney', 'McBurney', CAPS_NAMES], 1995 ['blumberg', 'Blumberg', CAPS_NAMES], 1996 ['roVsing', 'RoVsing', CAPS_NAMES], 1997 ['Özdemir', 'Özdemir', CAPS_NAMES], 1998 ['özdemir', 'Özdemir', CAPS_NAMES], 1999 ] 2000 for pair in pairs: 2001 result = capitalize(pair[0], pair[2]) 2002 if result != pair[1]: 2003 success = False 2004 print('ERROR (caps mode %s): "%s" -> "%s", expected "%s"' % (pair[2], pair[0], result, pair[1])) 2005 2006 if success: 2007 print("... SUCCESS") 2008 2009 return success
2010 #-----------------------------------------------------------------------
2011 - def test_import_module():
2012 print("testing import_module_from_directory()") 2013 path = sys.argv[1] 2014 name = sys.argv[2] 2015 try: 2016 mod = import_module_from_directory(module_path = path, module_name = name) 2017 except: 2018 print("module import failed, see log") 2019 return False 2020 2021 print("module import succeeded", mod) 2022 print(dir(mod)) 2023 return True
2024 #-----------------------------------------------------------------------
2025 - def test_mkdir():
2026 print("testing mkdir(%s)" % sys.argv[2]) 2027 mkdir(sys.argv[2], 0o0700)
2028 #-----------------------------------------------------------------------
2029 - def test_gmPaths():
2030 print("testing gmPaths()") 2031 print("-----------------") 2032 paths = gmPaths(wx=None, app_name='gnumed') 2033 print("user config dir:", paths.user_config_dir) 2034 print("system config dir:", paths.system_config_dir) 2035 print("local base dir:", paths.local_base_dir) 2036 print("system app data dir:", paths.system_app_data_dir) 2037 print("working directory :", paths.working_dir) 2038 print("temp directory :", paths.tmp_dir)
2039 #-----------------------------------------------------------------------
2040 - def test_none_if():
2041 print("testing none_if()") 2042 print("-----------------") 2043 tests = [ 2044 [None, None, None], 2045 ['a', 'a', None], 2046 ['a', 'b', 'a'], 2047 ['a', None, 'a'], 2048 [None, 'a', None], 2049 [1, 1, None], 2050 [1, 2, 1], 2051 [1, None, 1], 2052 [None, 1, None] 2053 ] 2054 2055 for test in tests: 2056 if none_if(value = test[0], none_equivalent = test[1]) != test[2]: 2057 print('ERROR: none_if(%s) returned [%s], expected [%s]' % (test[0], none_if(test[0], test[1]), test[2])) 2058 2059 return True
2060 #-----------------------------------------------------------------------
2061 - def test_bool2str():
2062 tests = [ 2063 [True, 'Yes', 'Yes', 'Yes'], 2064 [False, 'OK', 'not OK', 'not OK'] 2065 ] 2066 for test in tests: 2067 if bool2str(test[0], test[1], test[2]) != test[3]: 2068 print('ERROR: bool2str(%s, %s, %s) returned [%s], expected [%s]' % (test[0], test[1], test[2], bool2str(test[0], test[1], test[2]), test[3])) 2069 2070 return True
2071 #-----------------------------------------------------------------------
2072 - def test_bool2subst():
2073 2074 print(bool2subst(True, 'True', 'False', 'is None')) 2075 print(bool2subst(False, 'True', 'False', 'is None')) 2076 print(bool2subst(None, 'True', 'False', 'is None'))
2077 #-----------------------------------------------------------------------
2078 - def test_get_unique_filename():
2079 print(get_unique_filename()) 2080 print(get_unique_filename(prefix='test-')) 2081 print(get_unique_filename(suffix='tst')) 2082 print(get_unique_filename(prefix='test-', suffix='tst')) 2083 print(get_unique_filename(tmp_dir='/home/ncq/Archiv/'))
2084 #-----------------------------------------------------------------------
2085 - def test_size2str():
2086 print("testing size2str()") 2087 print("------------------") 2088 tests = [0, 1, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 100000000000, 1000000000000, 10000000000000] 2089 for test in tests: 2090 print(size2str(test))
2091 #-----------------------------------------------------------------------
2092 - def test_unwrap():
2093 2094 test = """ 2095 second line\n 2096 3rd starts with tab \n 2097 4th with a space \n 2098 2099 6th 2100 2101 """ 2102 print(unwrap(text = test, max_length = 25))
2103 #-----------------------------------------------------------------------
2104 - def test_wrap():
2105 test = 'line 1\nline 2\nline 3' 2106 2107 print("wrap 5-6-7 initial 0, subsequent 0") 2108 print(wrap(test, 5)) 2109 print() 2110 print(wrap(test, 6)) 2111 print() 2112 print(wrap(test, 7)) 2113 print("-------") 2114 input() 2115 print("wrap 5 initial 1-1-3, subsequent 1-3-1") 2116 print(wrap(test, 5, ' ', ' ')) 2117 print() 2118 print(wrap(test, 5, ' ', ' ')) 2119 print() 2120 print(wrap(test, 5, ' ', ' ')) 2121 print("-------") 2122 input() 2123 print("wrap 6 initial 1-1-3, subsequent 1-3-1") 2124 print(wrap(test, 6, ' ', ' ')) 2125 print() 2126 print(wrap(test, 6, ' ', ' ')) 2127 print() 2128 print(wrap(test, 6, ' ', ' ')) 2129 print("-------") 2130 input() 2131 print("wrap 7 initial 1-1-3, subsequent 1-3-1") 2132 print(wrap(test, 7, ' ', ' ')) 2133 print() 2134 print(wrap(test, 7, ' ', ' ')) 2135 print() 2136 print(wrap(test, 7, ' ', ' '))
2137 #-----------------------------------------------------------------------
2138 - def test_md5():
2139 print('md5 %s: %s' % (sys.argv[2], file2md5(sys.argv[2]))) 2140 print('chunked md5 %s: %s' % (sys.argv[2], file2chunked_md5(sys.argv[2])))
2141 #-----------------------------------------------------------------------
2142 - def test_unicode():
2143 print(u_link_symbol * 10)
2144 #-----------------------------------------------------------------------
2145 - def test_xml_escape():
2146 print(xml_escape_string('<')) 2147 print(xml_escape_string('>')) 2148 print(xml_escape_string('&'))
2149 #-----------------------------------------------------------------------
2150 - def test_tex_escape():
2151 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234'] 2152 tests.append(' '.join(tests)) 2153 for test in tests: 2154 print('%s:' % test, tex_escape_string(test))
2155 2156 #-----------------------------------------------------------------------
2157 - def test_rst2latex_snippet():
2158 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234'] 2159 tests.append(' '.join(tests)) 2160 tests.append('C:\Windows\Programme\System 32\lala.txt') 2161 tests.extend([ 2162 'should be identical', 2163 'text *some text* text', 2164 """A List 2165 ====== 2166 2167 1. 1 2168 2. 2 2169 2170 3. ist-list 2171 1. more 2172 2. noch was ü 2173 #. nummer x""" 2174 ]) 2175 for test in tests: 2176 print('==================================================') 2177 print('raw:') 2178 print(test) 2179 print('---------') 2180 print('ReST 2 LaTeX:') 2181 latex = rst2latex_snippet(test) 2182 print(latex) 2183 if latex.strip() == test.strip(): 2184 print('=> identical') 2185 print('---------') 2186 print('tex_escape_string:') 2187 print(tex_escape_string(test)) 2188 input()
2189 2190 #-----------------------------------------------------------------------
2191 - def test_strip_trailing_empty_lines():
2192 tests = [ 2193 'one line, no embedded line breaks ', 2194 'one line\nwith embedded\nline\nbreaks\n ' 2195 ] 2196 for test in tests: 2197 print('as list:') 2198 print(strip_trailing_empty_lines(text = test, eol='\n', return_list = True)) 2199 print('as string:') 2200 print('>>>%s<<<' % strip_trailing_empty_lines(text = test, eol='\n', return_list = False)) 2201 tests = [ 2202 ['list', 'without', 'empty', 'trailing', 'lines'], 2203 ['list', 'with', 'empty', 'trailing', 'lines', '', ' ', ''] 2204 ] 2205 for test in tests: 2206 print('as list:') 2207 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = True)) 2208 print('as string:') 2209 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = False))
2210 #-----------------------------------------------------------------------
2211 - def test_fname_stem():
2212 tests = [ 2213 r'abc.exe', 2214 r'\abc.exe', 2215 r'c:\abc.exe', 2216 r'c:\d\abc.exe', 2217 r'/home/ncq/tmp.txt', 2218 r'~/tmp.txt', 2219 r'./tmp.txt', 2220 r'./.././tmp.txt', 2221 r'tmp.txt' 2222 ] 2223 for t in tests: 2224 print("[%s] -> [%s]" % (t, fname_stem(t)))
2225 #-----------------------------------------------------------------------
2226 - def test_dir_is_empty():
2227 print(sys.argv[2], 'empty:', dir_is_empty(sys.argv[2]))
2228 2229 #-----------------------------------------------------------------------
2230 - def test_compare_dicts():
2231 d1 = {} 2232 d2 = {} 2233 d1[1] = 1 2234 d1[2] = 2 2235 d1[3] = 3 2236 # 4 2237 d1[5] = 5 2238 2239 d2[1] = 1 2240 d2[2] = None 2241 # 3 2242 d2[4] = 4 2243 2244 #compare_dict_likes(d1, d2) 2245 2246 d1 = {1: 1, 2: 2} 2247 d2 = {1: 1, 2: 2} 2248 2249 #compare_dict_likes(d1, d2, 'same1', 'same2') 2250 print(format_dict_like(d1, tabular = False)) 2251 print(format_dict_like(d1, tabular = True))
2252 #print(format_dict_like(d2)) 2253 2254 #-----------------------------------------------------------------------
2255 - def test_format_compare_dicts():
2256 d1 = {} 2257 d2 = {} 2258 d1[1] = 1 2259 d1[2] = 2 2260 d1[3] = 3 2261 # 4 2262 d1[5] = 5 2263 2264 d2[1] = 1 2265 d2[2] = None 2266 # 3 2267 d2[4] = 4 2268 2269 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2'))) 2270 2271 d1 = {1: 1, 2: 2} 2272 d2 = {1: 1, 2: 2} 2273 2274 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
2275 2276 #-----------------------------------------------------------------------
2277 - def test_rm_dir():
2278 rmdir('cx:\windows\system3__2xxxxxxxxxxxxx')
2279 2280 #-----------------------------------------------------------------------
2281 - def test_rm_dir_content():
2282 #print(rm_dir_content('cx:\windows\system3__2xxxxxxxxxxxxx')) 2283 print(rm_dir_content('/tmp/user/1000/tmp'))
2284 2285 #-----------------------------------------------------------------------
2286 - def test_strip_prefix():
2287 tests = [ 2288 ('', '', ''), 2289 ('a', 'a', ''), 2290 ('\.br\MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\', '\.br\\', 'MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\') 2291 ] 2292 for test in tests: 2293 text, prefix, expect = test 2294 result = strip_prefix(text, prefix) 2295 if result == expect: 2296 continue 2297 print('test failed:', test) 2298 print('result:', result)
2299 #-----------------------------------------------------------------------
2300 - def test_shorten_text():
2301 tst = [ 2302 ('123', 1), 2303 ('123', 2), 2304 ('123', 3), 2305 ('123', 4), 2306 ('', 1), 2307 ('1', 1), 2308 ('12', 1), 2309 ('', 2), 2310 ('1', 2), 2311 ('12', 2), 2312 ('123', 2) 2313 ] 2314 for txt, lng in tst: 2315 print('max', lng, 'of', txt, '=', shorten_text(txt, lng))
2316 #-----------------------------------------------------------------------
2317 - def test_fname_sanitize():
2318 tests = [ 2319 '/tmp/test.txt', 2320 '/tmp/ test.txt', 2321 '/tmp/ tes\\t.txt', 2322 'test' 2323 ] 2324 for test in tests: 2325 print (test, fname_sanitize(test))
2326 2327 #-----------------------------------------------------------------------
2328 - def test_create_qrcode():
2329 print(create_qrcode(text = sys.argv[2], filename=None, qr_filename=None, verbose = True))
2330 2331 #-----------------------------------------------------------------------
2332 - def test_enumerate_removable_partitions():
2333 parts = enumerate_removable_partitions() 2334 for part_name in parts: 2335 part = parts[part_name] 2336 print(part['device']) 2337 print(part['partition']) 2338 if part['is_mounted']: 2339 print('%s@%s: %s on %s by %s @ %s (FS=%s: %s free of %s total)' % ( 2340 part['type'], 2341 part['bus'], 2342 part['fs_label'], 2343 part['model'], 2344 part['vendor'], 2345 part['mountpoint'], 2346 part['fs_type'], 2347 part['bytes_free'], 2348 part['size_in_bytes'] 2349 )) 2350 else: 2351 print('%s@%s: %s on %s by %s (not mounted)' % ( 2352 part['type'], 2353 part['bus'], 2354 part['fs_label'], 2355 part['model'], 2356 part['vendor'] 2357 ))
2358 2359 #-----------------------------------------------------------------------
2360 - def test_enumerate_optical_writers():
2361 for writer in enumerate_optical_writers(): 2362 print('%s@%s: %s @ %s' % ( 2363 writer['type'], 2364 writer['bus'], 2365 writer['model'], 2366 writer['device'] 2367 ))
2368 2369 #-----------------------------------------------------------------------
2370 - def test_copy_tree_content():
2371 print(sys.argv[2], '->', sys.argv[3]) 2372 print(copy_tree_content(sys.argv[2], sys.argv[3]))
2373 2374 #-----------------------------------------------------------------------
2375 - def test_mk_sandbox_dir():
2376 print(mk_sandbox_dir(base_dir = '/tmp/abcd/efg/h'))
2377 2378 #-----------------------------------------------------------------------
2379 - def test_make_table_from_dicts():
2380 dicts = [ 2381 {'pkey': 1, 'value': 'a1'}, 2382 {'pkey': 2, 'value': 'b2'}, 2383 {'pkey': 3, 'value': 'c3'}, 2384 {'pkey': 4, 'value': 'd4'}, 2385 {'pkey': 5, 'value': 'd4'}, 2386 {'pkey': 5, 'value': 'c5'}, 2387 ] 2388 with open('x.txt', 'w', encoding = 'utf8') as f: 2389 f.write(dicts2table(dicts, left_margin=2, eol='\n', keys2ignore=None, show_only_changes=True, headers = ['d1', 'd2', 'd3', 'd4', 'd5', 'd6']))
2390 #print(dicts2table(dicts, left_margin=2, eol='\n', keys2ignore=None, show_only_changes=True, headers = ['d1', 'd2', 'd3', 'd4', 'd5', 'd6'])) 2391 2392 #----------------------------------------------------------------------- 2393 test_coalesce() 2394 #test_capitalize() 2395 #test_import_module() 2396 #test_mkdir() 2397 #test_gmPaths() 2398 #test_none_if() 2399 #test_bool2str() 2400 #test_bool2subst() 2401 #test_get_unique_filename() 2402 #test_size2str() 2403 #test_wrap() 2404 #test_input2decimal() 2405 #test_input2int() 2406 #test_unwrap() 2407 #test_md5() 2408 #test_unicode() 2409 #test_xml_escape() 2410 #test_strip_trailing_empty_lines() 2411 #test_fname_stem() 2412 #test_tex_escape() 2413 #test_rst2latex_snippet() 2414 #test_dir_is_empty() 2415 #test_compare_dicts() 2416 #test_rm_dir() 2417 #test_rm_dir_content() 2418 #test_strip_prefix() 2419 #test_shorten_text() 2420 #test_format_compare_dicts() 2421 #test_fname_sanitize() 2422 #test_create_qrcode() 2423 #test_enumerate_removable_partitions() 2424 #test_enumerate_optical_writers() 2425 #test_copy_tree_content() 2426 #test_mk_sandbox_dir() 2427 #test_make_table_from_dicts() 2428 2429 #=========================================================================== 2430