Package Gnumed :: Package pycommon :: Module gmTools
[frames] | no frames]

Source Code for Module Gnumed.pycommon.gmTools

   1  # -*- coding: utf-8 -*- 
   2   
   3   
   4  __doc__ = """GNUmed general tools.""" 
   5   
   6  #=========================================================================== 
   7  __author__ = "K. Hilbert <Karsten.Hilbert@gmx.net>" 
   8  __license__ = "GPL v2 or later (details at http://www.gnu.org)" 
   9   
  10  # std libs 
  11  import sys 
  12  import os 
  13  import os.path 
  14  import csv 
  15  import tempfile 
  16  import logging 
  17  import hashlib 
  18  import platform 
  19  import subprocess 
  20  import decimal 
  21  import getpass 
  22  import io 
  23  import functools 
  24  import json 
  25  import shutil 
  26  import zipfile 
  27  import datetime as pydt 
  28  import re as regex 
  29  import xml.sax.saxutils as xml_tools 
  30  # old: 
  31  import pickle, zlib 
  32  # docutils 
  33  du_core = None 
  34   
  35   
  36  # GNUmed libs 
  37  if __name__ == '__main__': 
  38          sys.path.insert(0, '../../') 
  39  from Gnumed.pycommon import gmBorg 
  40   
  41   
  42  _log = logging.getLogger('gm.tools') 
  43   
  44  # CAPitalization modes: 
  45  (       CAPS_NONE,                                      # don't touch it 
  46          CAPS_FIRST,                                     # CAP first char, leave rest as is 
  47          CAPS_ALLCAPS,                           # CAP all chars 
  48          CAPS_WORDS,                                     # CAP first char of every word 
  49          CAPS_NAMES,                                     # CAP in a way suitable for names (tries to be smart) 
  50          CAPS_FIRST_ONLY                         # CAP first char, lowercase the rest 
  51  ) = range(6) 
  52   
  53   
  54  u_currency_pound = '\u00A3'                             # Pound sign 
  55  u_currency_sign = '\u00A4'                                      # generic currency sign 
  56  u_currency_yen = '\u00A5'                                       # Yen sign 
  57  u_right_double_angle_quote = '\u00AB'           # << 
  58  u_registered_trademark = '\u00AE' 
  59  u_plus_minus = '\u00B1' 
  60  u_superscript_one = '\u00B9'                            # ^1 
  61  u_left_double_angle_quote = '\u00BB'            # >> 
  62  u_one_quarter = '\u00BC' 
  63  u_one_half = '\u00BD' 
  64  u_three_quarters = '\u00BE' 
  65  u_multiply = '\u00D7'                                           # x 
  66  u_greek_ALPHA = '\u0391' 
  67  u_greek_alpha = '\u03b1' 
  68  u_greek_OMEGA = '\u03A9' 
  69  u_greek_omega = '\u03c9' 
  70  u_dagger = '\u2020' 
  71  u_triangular_bullet = '\u2023'                                  # triangular bullet  (>) 
  72  u_ellipsis = '\u2026'                                                   # ... 
  73  u_euro = '\u20AC'                                                               # EURO sign 
  74  u_numero = '\u2116'                                                             # No. / # sign 
  75  u_down_left_arrow = '\u21B5'                                    # <-' 
  76  u_left_arrow = '\u2190'                                                 # <-- 
  77  u_up_arrow = '\u2191' 
  78  u_arrow2right = '\u2192'                                                # --> 
  79  u_down_arrow = '\u2193' 
  80  u_left_arrow_with_tail = '\u21a2'                               # <--< 
  81  u_arrow2right_from_bar = '\u21a6'                               # |-> 
  82  u_arrow2right_until_vertical_bar = '\u21e5'             # -->| 
  83  u_sum = '\u2211'                                                                # sigma 
  84  u_almost_equal_to = '\u2248'                                    # approximately / nearly / roughly 
  85  u_corresponds_to = '\u2258' 
  86  u_infinity = '\u221E' 
  87  u_arrow2right_until_vertical_bar2 = '\u2b72'    # -->| 
  88   
  89  u_diameter = '\u2300' 
  90  u_checkmark_crossed_out = '\u237B' 
  91  u_box_horiz_high = '\u23ba' 
  92  u_box_vert_left = '\u23b8' 
  93  u_box_vert_right = '\u23b9' 
  94   
  95  u_space_as_open_box = '\u2423' 
  96   
  97  u_box_horiz_single = '\u2500'                           # - 
  98  u_box_vert_light = '\u2502' 
  99  u_box_horiz_light_3dashes = '\u2504'            # ... 
 100  u_box_vert_light_4dashes = '\u2506' 
 101  u_box_horiz_4dashes = '\u2508'                          # .... 
 102  u_box_T_right = '\u251c'                                        # |- 
 103  u_box_T_left = '\u2524'                                         # -| 
 104  u_box_T_down = '\u252c' 
 105  u_box_T_up = '\u2534' 
 106  u_box_plus = '\u253c' 
 107  u_box_top_double = '\u2550' 
 108  u_box_top_left_double_single = '\u2552' 
 109  u_box_top_right_double_single = '\u2555' 
 110  u_box_top_left_arc = '\u256d' 
 111  u_box_top_right_arc = '\u256e' 
 112  u_box_bottom_right_arc = '\u256f' 
 113  u_box_bottom_left_arc = '\u2570' 
 114  u_box_horiz_light_heavy = '\u257c' 
 115  u_box_horiz_heavy_light = '\u257e' 
 116   
 117  u_skull_and_crossbones = '\u2620' 
 118  u_caduceus = '\u2624' 
 119  u_frowning_face = '\u2639' 
 120  u_smiling_face = '\u263a' 
 121  u_black_heart = '\u2665' 
 122  u_female = '\u2640' 
 123  u_male = '\u2642' 
 124  u_male_female = '\u26a5' 
 125  u_chain = '\u26d3' 
 126   
 127  u_checkmark_thin = '\u2713' 
 128  u_checkmark_thick = '\u2714' 
 129  u_heavy_greek_cross = '\u271a' 
 130  u_arrow2right_thick = '\u2794' 
 131  u_writing_hand = '\u270d' 
 132  u_pencil_1 = '\u270e' 
 133  u_pencil_2 = '\u270f' 
 134  u_pencil_3 = '\u2710' 
 135  u_latin_cross = '\u271d' 
 136   
 137  u_arrow2right_until_black_diamond = '\u291e'    # ->* 
 138   
 139  u_kanji_yen = '\u5186'                                                  # Yen kanji 
 140  u_replacement_character = '\ufffd' 
 141  u_link_symbol = '\u1f517' 
 142   
 143   
 144  _kB = 1024 
 145  _MB = 1024 * _kB 
 146  _GB = 1024 * _MB 
 147  _TB = 1024 * _GB 
 148  _PB = 1024 * _TB 
 149   
 150   
 151  _GM_TITLE_PREFIX = 'GMd' 
 152   
 153  #=========================================================================== 
154 -def handle_uncaught_exception_console(t, v, tb):
155 156 print(".========================================================") 157 print("| Unhandled exception caught !") 158 print("| Type :", t) 159 print("| Value:", v) 160 print("`========================================================") 161 _log.critical('unhandled exception caught', exc_info = (t,v,tb)) 162 sys.__excepthook__(t,v,tb)
163 164 #=========================================================================== 165 # path level operations 166 #---------------------------------------------------------------------------
167 -def mkdir(directory=None, mode=None):
168 """Create directory. 169 170 - creates parent dirs if necessary 171 - does not fail if directory exists 172 <mode>: numeric, say 0o0700 for "-rwx------" 173 """ 174 if os.path.isdir(directory): 175 if mode is None: 176 return True 177 178 changed = False 179 old_umask = os.umask(0) 180 try: 181 # does not WORK ! 182 #os.chmod(directory, mode, follow_symlinks = (os.chmod in os.supports_follow_symlinks)) # can't do better 183 os.chmod(directory, mode) 184 changed = True 185 finally: 186 os.umask(old_umask) 187 return changed 188 189 if mode is None: 190 os.makedirs(directory) 191 return True 192 193 old_umask = os.umask(0) 194 try: 195 os.makedirs(directory, mode) 196 finally: 197 os.umask(old_umask) 198 return True
199 200 #---------------------------------------------------------------------------
201 -def rmdir(directory):
202 #------------------------------- 203 def _on_rm_error(func, path, exc): 204 _log.error('error while shutil.rmtree(%s)', path, exc_info=exc) 205 return True
206 #------------------------------- 207 error_count = 0 208 try: 209 shutil.rmtree(directory, False, _on_rm_error) 210 except Exception: 211 _log.exception('cannot shutil.rmtree(%s)', directory) 212 error_count += 1 213 return error_count 214 215 #---------------------------------------------------------------------------
216 -def rm_dir_content(directory):
217 _log.debug('cleaning out [%s]', directory) 218 try: 219 items = os.listdir(directory) 220 except OSError: 221 return False 222 for item in items: 223 # attempt file/link removal and ignore (but log) errors 224 full_item = os.path.join(directory, item) 225 try: 226 os.remove(full_item) 227 except OSError: # as per the docs, this is a directory 228 _log.debug('[%s] seems to be a subdirectory', full_item) 229 errors = rmdir(full_item) 230 if errors > 0: 231 return False 232 except Exception: 233 _log.exception('cannot os.remove(%s) [a file or a link]', full_item) 234 return False 235 236 return True
237 238 #---------------------------------------------------------------------------
239 -def mk_sandbox_dir(prefix=None, base_dir=None):
240 if base_dir is None: 241 base_dir = gmPaths().tmp_dir 242 else: 243 if not os.path.isdir(base_dir): 244 mkdir(base_dir, mode = 0o0700) # (invoking user only) 245 if prefix is None: 246 prefix = 'sndbx-' 247 return tempfile.mkdtemp(prefix = prefix, suffix = '', dir = base_dir)
248 249 #---------------------------------------------------------------------------
250 -def parent_dir(directory):
251 return os.path.abspath(os.path.join(directory, '..'))
252 253 #---------------------------------------------------------------------------
254 -def dirname_stem(directory):
255 # /home/user/dir/ -> dir 256 # /home/user/dir -> dir 257 return os.path.basename(os.path.normpath(directory)) # normpath removes trailing slashes if any
258 259 #---------------------------------------------------------------------------
260 -def dir_is_empty(directory=None):
261 try: 262 empty = (len(os.listdir(directory)) == 0) 263 except OSError as exc: 264 if exc.errno != 2: # no such file 265 raise 266 empty = None 267 return empty
268 269 #---------------------------------------------------------------------------
270 -def copy_tree_content(directory, target_directory):
271 """Copy the *content* of <directory> *into* <target_directory> 272 which is created if need be. 273 """ 274 assert (directory is not None), 'source <directory> should not be None' 275 _log.debug('copying content of [%s] into [%s]', directory, target_directory) 276 try: 277 items = os.listdir(directory) 278 except OSError: 279 return None 280 281 for item in items: 282 full_item = os.path.join(directory, item) 283 if os.path.isdir(full_item): 284 target_subdir = os.path.join(target_directory, item) 285 try: 286 shutil.copytree(full_item, target_subdir) 287 except Exception: 288 _log.exception('cannot copy subdir [%s]', full_item) 289 return None 290 else: 291 try: 292 shutil.copy2(full_item, target_directory) 293 except Exception: 294 _log.exception('cannot copy file [%s]', full_item) 295 return None 296 297 return target_directory
298 299 #--------------------------------------------------------------------------- 300 #---------------------------------------------------------------------------
301 -class gmPaths(gmBorg.cBorg):
302 """This class provides the following paths: 303 304 .home_dir user home 305 .local_base_dir script installation dir 306 .working_dir current dir 307 .user_config_dir 308 .system_config_dir 309 .system_app_data_dir (not writable) 310 .tmp_dir instance-local 311 .user_tmp_dir user-local (NOT per instance) 312 .bytea_cache_dir caches downloaded BYTEA data 313 """
314 - def __init__(self, app_name=None, wx=None):
315 """Setup pathes. 316 317 <app_name> will default to (name of the script - .py) 318 """ 319 try: 320 self.already_inited 321 return 322 except AttributeError: 323 pass 324 325 self.init_paths(app_name=app_name, wx=wx) 326 self.already_inited = True
327 328 #-------------------------------------- 329 # public API 330 #--------------------------------------
331 - def init_paths(self, app_name=None, wx=None):
332 333 if wx is None: 334 _log.debug('wxPython not available') 335 _log.debug('detecting paths directly') 336 337 if app_name is None: 338 app_name, ext = os.path.splitext(os.path.basename(sys.argv[0])) 339 _log.info('app name detected as [%s]', app_name) 340 else: 341 _log.info('app name passed in as [%s]', app_name) 342 343 # the user home, doesn't work in Wine so work around that 344 self.__home_dir = None 345 346 # where the main script (the "binary") is installed 347 if getattr(sys, 'frozen', False): 348 _log.info('frozen app, installed into temporary path') 349 # this would find the path of *THIS* file 350 #self.local_base_dir = os.path.dirname(__file__) 351 # while this is documented on the web, the ${_MEIPASS2} does not exist 352 #self.local_base_dir = os.environ.get('_MEIPASS2') 353 # this is what Martin Zibricky <mzibr.public@gmail.com> told us to use 354 # when asking about this on pyinstaller@googlegroups.com 355 #self.local_base_dir = sys._MEIPASS 356 # however, we are --onedir, so we should look at sys.executable 357 # as per the pyinstaller manual 358 self.local_base_dir = os.path.dirname(sys.executable) 359 else: 360 self.local_base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) 361 362 # the current working dir at the OS 363 self.working_dir = os.path.abspath(os.curdir) 364 365 # user-specific config dir, usually below the home dir 366 mkdir(os.path.join(self.home_dir, '.%s' % app_name)) 367 self.user_config_dir = os.path.join(self.home_dir, '.%s' % app_name) 368 369 # system-wide config dir, usually below /etc/ under UN*X 370 try: 371 self.system_config_dir = os.path.join('/etc', app_name) 372 except ValueError: 373 #self.system_config_dir = self.local_base_dir 374 self.system_config_dir = self.user_config_dir 375 376 # system-wide application data dir 377 try: 378 self.system_app_data_dir = os.path.join(sys.prefix, 'share', app_name) 379 except ValueError: 380 self.system_app_data_dir = self.local_base_dir 381 382 # temporary directory 383 try: 384 self.__tmp_dir_already_set 385 _log.debug('temp dir already set') 386 except AttributeError: 387 _log.info('temp file prefix: %s', tempfile.gettempprefix()) 388 _log.info('initial (user level) temp dir: %s', tempfile.gettempdir()) 389 bytes_free = shutil.disk_usage(tempfile.gettempdir()).free 390 _log.info('free disk space for temp dir: %s (%s bytes)', size2str(size = bytes_free), bytes_free) 391 # $TMP/gnumed-$USER/ 392 self.user_tmp_dir = os.path.join(tempfile.gettempdir(), '%s-%s' % (app_name, getpass.getuser())) 393 mkdir(self.user_tmp_dir, 0o700) 394 _log.info('intermediate (app+user level) temp dir: %s', self.user_tmp_dir) 395 # $TMP/gnumed-$USER/g-$UNIQUE/ 396 tempfile.tempdir = self.user_tmp_dir # tell mkdtemp about intermediate dir 397 self.tmp_dir = tempfile.mkdtemp(prefix = 'g-') # will set tempfile.tempdir as side effect 398 _log.info('final (app instance level) temp dir: %s', tempfile.gettempdir()) 399 400 # BYTEA cache dir 401 cache_dir = os.path.join(self.user_tmp_dir, '.bytea_cache') 402 try: 403 stat = os.stat(cache_dir) 404 _log.warning('reusing BYTEA cache dir: %s', cache_dir) 405 _log.debug(stat) 406 except FileNotFoundError: 407 mkdir(cache_dir, mode = 0o0700) 408 self.bytea_cache_dir = cache_dir 409 410 self.__log_paths() 411 if wx is None: 412 return True 413 414 # retry with wxPython 415 _log.debug('re-detecting paths with wxPython') 416 417 std_paths = wx.StandardPaths.Get() 418 _log.info('wxPython app name is [%s]', wx.GetApp().GetAppName()) 419 420 # user-specific config dir, usually below the home dir 421 mkdir(os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name)) 422 self.user_config_dir = os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name) 423 424 # system-wide config dir, usually below /etc/ under UN*X 425 try: 426 tmp = std_paths.GetConfigDir() 427 if not tmp.endswith(app_name): 428 tmp = os.path.join(tmp, app_name) 429 self.system_config_dir = tmp 430 except ValueError: 431 # leave it at what it was from direct detection 432 pass 433 434 # system-wide application data dir 435 # Robin attests that the following doesn't always 436 # give sane values on Windows, so IFDEF it 437 if 'wxMSW' in wx.PlatformInfo: 438 _log.warning('this platform (wxMSW) sometimes returns a broken value for the system-wide application data dir') 439 else: 440 try: 441 self.system_app_data_dir = std_paths.GetDataDir() 442 except ValueError: 443 pass 444 445 self.__log_paths() 446 return True
447 448 #--------------------------------------
449 - def __log_paths(self):
450 _log.debug('sys.argv[0]: %s', sys.argv[0]) 451 _log.debug('sys.executable: %s', sys.executable) 452 _log.debug('sys._MEIPASS: %s', getattr(sys, '_MEIPASS', '<not found>')) 453 _log.debug('os.environ["_MEIPASS2"]: %s', os.environ.get('_MEIPASS2', '<not found>')) 454 _log.debug('__file__ : %s', __file__) 455 _log.debug('local application base dir: %s', self.local_base_dir) 456 _log.debug('current working dir: %s', self.working_dir) 457 _log.debug('user home dir: %s', self.home_dir) 458 _log.debug('user-specific config dir: %s', self.user_config_dir) 459 _log.debug('system-wide config dir: %s', self.system_config_dir) 460 _log.debug('system-wide application data dir: %s', self.system_app_data_dir) 461 _log.debug('temporary dir (user): %s', self.user_tmp_dir) 462 _log.debug('temporary dir (instance): %s', self.tmp_dir) 463 _log.debug('temporary dir (tempfile.tempdir): %s', tempfile.tempdir) 464 _log.debug('temporary dir (tempfile.gettempdir()): %s', tempfile.gettempdir()) 465 _log.debug('BYTEA cache dir: %s', self.bytea_cache_dir)
466 467 #-------------------------------------- 468 # properties 469 #--------------------------------------
470 - def _set_user_config_dir(self, path):
471 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 472 msg = '[%s:user_config_dir]: invalid path [%s]' % (self.__class__.__name__, path) 473 _log.error(msg) 474 raise ValueError(msg) 475 self.__user_config_dir = path
476
477 - def _get_user_config_dir(self):
478 return self.__user_config_dir
479 480 user_config_dir = property(_get_user_config_dir, _set_user_config_dir) 481 #--------------------------------------
482 - def _set_system_config_dir(self, path):
483 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 484 msg = '[%s:system_config_dir]: invalid path [%s]' % (self.__class__.__name__, path) 485 _log.error(msg) 486 raise ValueError(msg) 487 self.__system_config_dir = path
488
489 - def _get_system_config_dir(self):
490 return self.__system_config_dir
491 492 system_config_dir = property(_get_system_config_dir, _set_system_config_dir) 493 #--------------------------------------
494 - def _set_system_app_data_dir(self, path):
495 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 496 msg = '[%s:system_app_data_dir]: invalid path [%s]' % (self.__class__.__name__, path) 497 _log.error(msg) 498 raise ValueError(msg) 499 self.__system_app_data_dir = path
500
501 - def _get_system_app_data_dir(self):
502 return self.__system_app_data_dir
503 504 system_app_data_dir = property(_get_system_app_data_dir, _set_system_app_data_dir) 505 #--------------------------------------
506 - def _set_home_dir(self, path):
507 raise ValueError('invalid to set home dir')
508
509 - def _get_home_dir(self):
510 if self.__home_dir is not None: 511 return self.__home_dir 512 513 tmp = os.path.expanduser('~') 514 if tmp == '~': 515 _log.error('this platform does not expand ~ properly') 516 try: 517 tmp = os.environ['USERPROFILE'] 518 except KeyError: 519 _log.error('cannot access $USERPROFILE in environment') 520 521 if not ( 522 os.access(tmp, os.R_OK) 523 and 524 os.access(tmp, os.X_OK) 525 and 526 os.access(tmp, os.W_OK) 527 ): 528 msg = '[%s:home_dir]: invalid path [%s]' % (self.__class__.__name__, tmp) 529 _log.error(msg) 530 raise ValueError(msg) 531 532 self.__home_dir = tmp 533 return self.__home_dir
534 535 home_dir = property(_get_home_dir, _set_home_dir) 536 537 #--------------------------------------
538 - def _set_tmp_dir(self, path):
539 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 540 msg = '[%s:tmp_dir]: invalid path [%s]' % (self.__class__.__name__, path) 541 _log.error(msg) 542 raise ValueError(msg) 543 _log.debug('previous temp dir: %s', tempfile.gettempdir()) 544 self.__tmp_dir = path 545 tempfile.tempdir = self.__tmp_dir 546 _log.debug('new temp dir: %s', tempfile.gettempdir()) 547 self.__tmp_dir_already_set = True
548
549 - def _get_tmp_dir(self):
550 return self.__tmp_dir
551 552 tmp_dir = property(_get_tmp_dir, _set_tmp_dir)
553 554 #=========================================================================== 555 # file related tools 556 #---------------------------------------------------------------------------
557 -def recode_file(source_file=None, target_file=None, source_encoding='utf8', target_encoding=None, base_dir=None, error_mode='replace'):
558 if target_encoding is None: 559 return source_file 560 if target_encoding == source_encoding: 561 return source_file 562 if target_file is None: 563 target_file = get_unique_filename ( 564 prefix = '%s-%s_%s-' % (fname_stem(source_file), source_encoding, target_encoding), 565 suffix = fname_extension(source_file, '.txt'), 566 tmp_dir = base_dir 567 ) 568 569 _log.debug('[%s] -> [%s] (%s -> %s)', source_encoding, target_encoding, source_file, target_file) 570 571 in_file = io.open(source_file, mode = 'rt', encoding = source_encoding) 572 out_file = io.open(target_file, mode = 'wt', encoding = target_encoding, errors = error_mode) 573 for line in in_file: 574 out_file.write(line) 575 out_file.close() 576 in_file.close() 577 578 return target_file
579 580 #---------------------------------------------------------------------------
581 -def unzip_archive(archive_name, target_dir=None, remove_archive=False):
582 _log.debug('unzipping [%s] -> [%s]', archive_name, target_dir) 583 success = False 584 try: 585 with zipfile.ZipFile(archive_name) as archive: 586 archive.extractall(target_dir) 587 success = True 588 except Exception: 589 _log.exception('cannot unzip') 590 return False 591 if remove_archive: 592 remove_file(archive_name) 593 return success
594 595 #---------------------------------------------------------------------------
596 -def remove_file(filename, log_error=True, force=False):
597 if not os.path.lexists(filename): 598 return True 599 600 # attempt file removal and ignore (but log) errors 601 try: 602 os.remove(filename) 603 return True 604 605 except Exception: 606 if log_error: 607 _log.exception('cannot os.remove(%s)', filename) 608 609 if force: 610 tmp_name = get_unique_filename(tmp_dir = fname_dir(filename)) 611 _log.debug('attempting os.replace(%s -> %s)', filename, tmp_name) 612 try: 613 os.replace(filename, tmp_name) 614 return True 615 616 except Exception: 617 if log_error: 618 _log.exception('cannot os.remove(%s)', filename) 619 620 return False
621 622 #---------------------------------------------------------------------------
623 -def file2md5(filename=None, return_hex=True):
624 blocksize = 2**10 * 128 # 128k, since md5 uses 128 byte blocks 625 _log.debug('md5(%s): <%s> byte blocks', filename, blocksize) 626 f = io.open(filename, mode = 'rb') 627 md5 = hashlib.md5() 628 while True: 629 data = f.read(blocksize) 630 if not data: 631 break 632 md5.update(data) 633 f.close() 634 _log.debug('md5(%s): %s', filename, md5.hexdigest()) 635 if return_hex: 636 return md5.hexdigest() 637 638 return md5.digest()
639 640 #---------------------------------------------------------------------------
641 -def file2chunked_md5(filename=None, chunk_size=500*_MB):
642 _log.debug('chunked_md5(%s, chunk_size=%s bytes)', filename, chunk_size) 643 md5_concat = '' 644 f = open(filename, 'rb') 645 while True: 646 md5 = hashlib.md5() 647 data = f.read(chunk_size) 648 if not data: 649 break 650 md5.update(data) 651 md5_concat += md5.hexdigest() 652 f.close() 653 md5 = hashlib.md5() 654 md5.update(md5_concat) 655 hex_digest = md5.hexdigest() 656 _log.debug('md5("%s"): %s', md5_concat, hex_digest) 657 return hex_digest
658 659 #--------------------------------------------------------------------------- 660 default_csv_reader_rest_key = 'list_of_values_of_unknown_fields' 661
662 -def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
663 try: 664 is_dict_reader = kwargs['dict'] 665 del kwargs['dict'] 666 except KeyError: 667 is_dict_reader = False 668 669 if is_dict_reader: 670 kwargs['restkey'] = default_csv_reader_rest_key 671 return csv.DictReader(unicode_csv_data, dialect=dialect, **kwargs) 672 return csv.reader(unicode_csv_data, dialect=dialect, **kwargs)
673 674 675 676
677 -def old_unicode2charset_encoder(unicode_csv_data, encoding='utf-8'):
678 for line in unicode_csv_data: 679 yield line.encode(encoding)
680 681 #def utf_8_encoder(unicode_csv_data): 682 # for line in unicode_csv_data: 683 # yield line.encode('utf-8') 684
685 -def old_unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
686 687 # csv.py doesn't do Unicode; encode temporarily as UTF-8: 688 try: 689 is_dict_reader = kwargs['dict'] 690 del kwargs['dict'] 691 if is_dict_reader is not True: 692 raise KeyError 693 kwargs['restkey'] = default_csv_reader_rest_key 694 csv_reader = csv.DictReader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs) 695 except KeyError: 696 is_dict_reader = False 697 csv_reader = csv.reader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs) 698 699 for row in csv_reader: 700 # decode ENCODING back to Unicode, cell by cell: 701 if is_dict_reader: 702 for key in row.keys(): 703 if key == default_csv_reader_rest_key: 704 old_data = row[key] 705 new_data = [] 706 for val in old_data: 707 new_data.append(str(val, encoding)) 708 row[key] = new_data 709 if default_csv_reader_rest_key not in csv_reader.fieldnames: 710 csv_reader.fieldnames.append(default_csv_reader_rest_key) 711 else: 712 row[key] = str(row[key], encoding) 713 yield row 714 else: 715 yield [ str(cell, encoding) for cell in row ]
716 #yield [str(cell, 'utf-8') for cell in row] 717 718 #---------------------------------------------------------------------------
719 -def fname_sanitize(filename):
720 """Normalizes unicode, removes non-alpha characters, converts spaces to underscores.""" 721 722 dir_part, name_part = os.path.split(filename) 723 if name_part == '': 724 return filename 725 726 import unicodedata 727 name_part = unicodedata.normalize('NFKD', name_part) 728 # remove everything not in group [] 729 name_part = regex.sub ( 730 '[^.\w\s[\]()%§+-]', 731 '', 732 name_part, 733 flags = regex.UNICODE 734 ).strip() 735 # translate whitespace to underscore 736 name_part = regex.sub ( 737 '\s+', 738 '_', 739 name_part, 740 flags = regex.UNICODE 741 ) 742 return os.path.join(dir_part, name_part)
743 744 #---------------------------------------------------------------------------
745 -def fname_stem(filename):
746 """/home/user/dir/filename.ext -> filename""" 747 return os.path.splitext(os.path.basename(filename))[0]
748 749 #---------------------------------------------------------------------------
750 -def fname_stem_with_path(filename):
751 """/home/user/dir/filename.ext -> /home/user/dir/filename""" 752 return os.path.splitext(filename)[0]
753 754 #---------------------------------------------------------------------------
755 -def fname_extension(filename=None, fallback=None):
756 """ /home/user/dir/filename.ext -> .ext 757 '' or '.' -> fallback if any else '' 758 """ 759 ext = os.path.splitext(filename)[1] 760 if ext.strip() not in ['.', '']: 761 return ext 762 if fallback is None: 763 return '' 764 return fallback
765 766 #---------------------------------------------------------------------------
767 -def fname_dir(filename):
768 # /home/user/dir/filename.ext -> /home/user/dir 769 return os.path.split(filename)[0]
770 771 #---------------------------------------------------------------------------
772 -def fname_from_path(filename):
773 # /home/user/dir/filename.ext -> filename.ext 774 return os.path.split(filename)[1]
775 776 #---------------------------------------------------------------------------
777 -def get_unique_filename(prefix=None, suffix=None, tmp_dir=None, include_timestamp=False):
778 """This function has a race condition between 779 its file.close() 780 and actually 781 using the filename in callers. 782 783 The file will NOT exist after calling this function. 784 """ 785 if tmp_dir is None: 786 gmPaths() # setup tmp dir if necessary 787 else: 788 if ( 789 not os.access(tmp_dir, os.F_OK) 790 or 791 not os.access(tmp_dir, os.X_OK | os.W_OK) 792 ): 793 _log.warning('cannot os.access() temporary dir [%s], using system default', tmp_dir) 794 tmp_dir = None 795 796 if include_timestamp: 797 ts = pydt.datetime.now().strftime('%m%d-%H%M%S-') 798 else: 799 ts = '' 800 801 kwargs = { 802 'dir': tmp_dir, 803 # make sure file gets deleted as soon as 804 # .close()d so we can "safely" open it again 805 'delete': True 806 } 807 808 if prefix is None: 809 kwargs['prefix'] = 'gm-%s' % ts 810 else: 811 kwargs['prefix'] = prefix + ts 812 813 if suffix in [None, '']: 814 kwargs['suffix'] = '.tmp' 815 else: 816 if not suffix.startswith('.'): 817 suffix = '.' + suffix 818 kwargs['suffix'] = suffix 819 820 f = tempfile.NamedTemporaryFile(**kwargs) 821 filename = f.name 822 f.close() 823 824 return filename
825 826 #--------------------------------------------------------------------------- 843 844 #--------------------------------------------------------------------------- 868 869 #===========================================================================
870 -def import_module_from_directory(module_path=None, module_name=None, always_remove_path=False):
871 """Import a module from any location.""" 872 873 _log.debug('CWD: %s', os.getcwd()) 874 875 remove_path = always_remove_path or False 876 if module_path not in sys.path: 877 _log.info('appending to sys.path: [%s]' % module_path) 878 sys.path.append(module_path) 879 remove_path = True 880 881 _log.debug('will remove import path: %s', remove_path) 882 883 if module_name.endswith('.py'): 884 module_name = module_name[:-3] 885 886 try: 887 module = __import__(module_name) 888 except Exception: 889 _log.exception('cannot __import__() module [%s] from [%s]' % (module_name, module_path)) 890 while module_path in sys.path: 891 sys.path.remove(module_path) 892 raise 893 894 _log.info('imported module [%s] as [%s]' % (module_name, module)) 895 if remove_path: 896 while module_path in sys.path: 897 sys.path.remove(module_path) 898 899 return module
900 901 #=========================================================================== 902 # text related tools 903 #---------------------------------------------------------------------------
904 -def size2str(size=0, template='%s'):
905 if size == 1: 906 return template % _('1 Byte') 907 if size < 10 * _kB: 908 return template % _('%s Bytes') % size 909 if size < _MB: 910 return template % '%.1f kB' % (float(size) / _kB) 911 if size < _GB: 912 return template % '%.1f MB' % (float(size) / _MB) 913 if size < _TB: 914 return template % '%.1f GB' % (float(size) / _GB) 915 if size < _PB: 916 return template % '%.1f TB' % (float(size) / _TB) 917 return template % '%.1f PB' % (float(size) / _PB)
918 919 #---------------------------------------------------------------------------
920 -def bool2subst(boolean=None, true_return=True, false_return=False, none_return=None):
921 if boolean is None: 922 return none_return 923 if boolean: 924 return true_return 925 if not boolean: 926 return false_return 927 raise ValueError('bool2subst(): <boolean> arg must be either of True, False, None')
928 929 #---------------------------------------------------------------------------
930 -def bool2str(boolean=None, true_str='True', false_str='False'):
931 return bool2subst ( 932 boolean = bool(boolean), 933 true_return = true_str, 934 false_return = false_str 935 )
936 937 #---------------------------------------------------------------------------
938 -def none_if(value=None, none_equivalent=None, strip_string=False):
939 """Modelled after the SQL NULLIF function.""" 940 if value is None: 941 return None 942 943 if strip_string: 944 stripped = value.strip() 945 else: 946 stripped = value 947 if stripped == none_equivalent: 948 return None 949 950 return value
951 952 #---------------------------------------------------------------------------
953 -def coalesce(value2test=None, return_instead=None, template4value=None, template4instead=None, none_equivalents=None, function4value=None, value2return=None):
954 """Modelled after the SQL coalesce function. 955 956 To be used to simplify constructs like: 957 958 if value2test is None (or in none_equivalents): 959 value = (template4instead % return_instead) or return_instead 960 else: 961 value = (template4value % value2test) or value2test 962 print value 963 964 @param value2test: the value to be tested for <None> 965 966 @param return_instead: the value to be returned if <value2test> *is* None 967 968 @param template4value: if <value2test> is returned, replace the value into this template, must contain one <%s> 969 970 @param template4instead: if <return_instead> is returned, replace the value into this template, must contain one <%s> 971 972 @param value2return: a *value* to return if <value2test> is NOT None, AND there's no <template4value> 973 974 example: 975 function4value = ('strftime', '%Y-%m-%d') 976 977 Ideas: 978 - list of return_insteads: initial, [return_instead, template], [return_instead, template], [return_instead, template], template4value, ... 979 """ 980 if none_equivalents is None: 981 none_equivalents = [None] 982 983 if value2test in none_equivalents: 984 if template4instead is None: 985 return return_instead 986 return template4instead % return_instead 987 988 # at this point, value2test was not equivalent to None 989 990 # 1) explicit value to return supplied ? 991 if value2return is not None: 992 return value2return 993 994 value2return = value2test 995 # 2) function supplied to be applied to the value ? 996 if function4value is not None: 997 funcname, args = function4value 998 func = getattr(value2test, funcname) 999 value2return = func(args) 1000 1001 # 3) template supplied to be applied to the value ? 1002 if template4value is None: 1003 return value2return 1004 1005 try: 1006 return template4value % value2return 1007 except TypeError: 1008 # except (TypeError, ValueError): 1009 # this should go, actually, only needed because "old" calls 1010 # to coalesce will still abuse template4value as explicit value2return, 1011 # relying on the replacement to above to fail 1012 if hasattr(_log, 'log_stack_trace'): 1013 _log.log_stack_trace(message = 'deprecated use of <template4value> for <value2return>') 1014 else: 1015 _log.error('deprecated use of <template4value> for <value2return>') 1016 _log.error(locals()) 1017 return template4value
1018 1019 #---------------------------------------------------------------------------
1020 -def __cap_name(match_obj=None):
1021 val = match_obj.group(0).lower() 1022 if val in ['von', 'van', 'de', 'la', 'l', 'der', 'den']: # FIXME: this needs to expand, configurable ? 1023 return val 1024 buf = list(val) 1025 buf[0] = buf[0].upper() 1026 for part in ['mac', 'mc', 'de', 'la']: 1027 if len(val) > len(part) and val[:len(part)] == part: 1028 buf[len(part)] = buf[len(part)].upper() 1029 return ''.join(buf)
1030 1031 #---------------------------------------------------------------------------
1032 -def capitalize(text=None, mode=CAPS_NAMES):
1033 """Capitalize the first character but leave the rest alone. 1034 1035 Note that we must be careful about the locale, this may 1036 have issues ! However, for UTF strings it should just work. 1037 """ 1038 if (mode is None) or (mode == CAPS_NONE): 1039 return text 1040 1041 if len(text) == 0: 1042 return text 1043 1044 if mode == CAPS_FIRST: 1045 if len(text) == 1: 1046 return text[0].upper() 1047 return text[0].upper() + text[1:] 1048 1049 if mode == CAPS_ALLCAPS: 1050 return text.upper() 1051 1052 if mode == CAPS_FIRST_ONLY: 1053 # if len(text) == 1: 1054 # return text[0].upper() 1055 return text[0].upper() + text[1:].lower() 1056 1057 if mode == CAPS_WORDS: 1058 #return regex.sub(ur'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text) 1059 return regex.sub(r'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text) 1060 1061 if mode == CAPS_NAMES: 1062 #return regex.sub(r'\w+', __cap_name, text) 1063 return capitalize(text=text, mode=CAPS_FIRST) # until fixed 1064 1065 print("ERROR: invalid capitalization mode: [%s], leaving input as is" % mode) 1066 return text
1067 1068 #---------------------------------------------------------------------------
1069 -def input2decimal(initial=None):
1070 1071 if isinstance(initial, decimal.Decimal): 1072 return True, initial 1073 1074 val = initial 1075 1076 # float ? -> to string first 1077 if type(val) == type(float(1.4)): 1078 val = str(val) 1079 1080 # string ? -> "," to "." 1081 if isinstance(val, str): 1082 val = val.replace(',', '.', 1) 1083 val = val.strip() 1084 1085 try: 1086 d = decimal.Decimal(val) 1087 return True, d 1088 except (TypeError, decimal.InvalidOperation): 1089 return False, val
1090 1091 #---------------------------------------------------------------------------
1092 -def input2int(initial=None, minval=None, maxval=None):
1093 1094 val = initial 1095 1096 # string ? -> "," to "." 1097 if isinstance(val, str): 1098 val = val.replace(',', '.', 1) 1099 val = val.strip() 1100 1101 try: 1102 int_val = int(val) 1103 except (TypeError, ValueError): 1104 _log.exception('int(%s) failed', val) 1105 return False, initial 1106 1107 if minval is not None: 1108 if int_val < minval: 1109 _log.debug('%s < min (%s)', val, minval) 1110 return False, initial 1111 if maxval is not None: 1112 if int_val > maxval: 1113 _log.debug('%s > max (%s)', val, maxval) 1114 return False, initial 1115 1116 return True, int_val
1117 1118 #---------------------------------------------------------------------------
1119 -def strip_prefix(text, prefix, remove_repeats=False, remove_whitespace=False):
1120 if remove_whitespace: 1121 text = text.lstrip() 1122 if not text.startswith(prefix): 1123 return text 1124 1125 text = text.replace(prefix, '', 1) 1126 if not remove_repeats: 1127 if remove_whitespace: 1128 return text.lstrip() 1129 return text 1130 1131 return strip_prefix(text, prefix, remove_repeats = True, remove_whitespace = remove_whitespace)
1132 1133 #---------------------------------------------------------------------------
1134 -def decorate_window_title(title):
1135 if title.startswith(_GM_TITLE_PREFIX): 1136 return title 1137 return '%s: %s' % ( 1138 _GM_TITLE_PREFIX, 1139 title.lstrip() 1140 )
1141 1142 #---------------------------------------------------------------------------
1143 -def undecorate_window_title(title):
1144 return strip_prefix(title, _GM_TITLE_PREFIX + ':', remove_repeats = True, remove_whitespace = True)
1145 1146 #---------------------------------------------------------------------------
1147 -def strip_suffix(text, suffix, remove_repeats=False, remove_whitespace=False):
1148 suffix_len = len(suffix) 1149 if remove_repeats: 1150 if remove_whitespace: 1151 while text.rstrip().endswith(suffix): 1152 text = text.rstrip()[:-suffix_len].rstrip() 1153 return text 1154 while text.endswith(suffix): 1155 text = text[:-suffix_len] 1156 return text 1157 if remove_whitespace: 1158 return text.rstrip()[:-suffix_len].rstrip() 1159 return text[:-suffix_len]
1160 1161 #---------------------------------------------------------------------------
1162 -def strip_leading_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1163 if lines is None: 1164 lines = text.split(eol) 1165 1166 while True: 1167 if lines[0].strip(eol).strip() != '': 1168 break 1169 lines = lines[1:] 1170 1171 if return_list: 1172 return lines 1173 1174 return eol.join(lines)
1175 1176 #---------------------------------------------------------------------------
1177 -def strip_trailing_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1178 if lines is None: 1179 lines = text.split(eol) 1180 1181 while True: 1182 if lines[-1].strip(eol).strip() != '': 1183 break 1184 lines = lines[:-1] 1185 1186 if return_list: 1187 return lines 1188 1189 return eol.join(lines)
1190 1191 #---------------------------------------------------------------------------
1192 -def strip_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1193 return strip_trailing_empty_lines ( 1194 lines = strip_leading_empty_lines(lines = lines, text = text, eol = eol, return_list = True), 1195 text = None, 1196 eol = eol, 1197 return_list = return_list 1198 )
1199 1200 #---------------------------------------------------------------------------
1201 -def list2text(lines, initial_indent='', subsequent_indent='', eol='\n', strip_leading_empty_lines=True, strip_trailing_empty_lines=True, strip_trailing_whitespace=True, max_line_width=None):
1202 1203 if len(lines) == 0: 1204 return '' 1205 1206 if strip_leading_empty_lines: 1207 lines = strip_leading_empty_lines(lines = lines, eol = eol, return_list = True) 1208 1209 if strip_trailing_empty_lines: 1210 lines = strip_trailing_empty_lines(lines = lines, eol = eol, return_list = True) 1211 1212 if strip_trailing_whitespace: 1213 lines = [ l.rstrip() for l in lines ] 1214 1215 if max_line_width is not None: 1216 wrapped_lines = [] 1217 for l in lines: 1218 wrapped_lines.extend(wrap(l, max_line_width).split('\n')) 1219 lines = wrapped_lines 1220 1221 indented_lines = [initial_indent + lines[0]] 1222 indented_lines.extend([ subsequent_indent + l for l in lines[1:] ]) 1223 1224 return eol.join(indented_lines)
1225 1226 #---------------------------------------------------------------------------
1227 -def wrap(text=None, width=None, initial_indent='', subsequent_indent='', eol='\n'):
1228 """A word-wrap function that preserves existing line breaks 1229 and most spaces in the text. Expects that existing line 1230 breaks are posix newlines (\n). 1231 """ 1232 if width is None: 1233 return text 1234 1235 wrapped = initial_indent + functools.reduce ( 1236 lambda line, word, width=width: '%s%s%s' % ( 1237 line, 1238 ' \n'[(len(line) - line.rfind('\n') - 1 + len(word.split('\n',1)[0]) >= width)], 1239 word 1240 ), 1241 text.split(' ') 1242 ) 1243 if subsequent_indent != '': 1244 wrapped = ('\n%s' % subsequent_indent).join(wrapped.split('\n')) 1245 if eol != '\n': 1246 wrapped = wrapped.replace('\n', eol) 1247 return wrapped
1248 1249 #---------------------------------------------------------------------------
1250 -def unwrap(text=None, max_length=None, strip_whitespace=True, remove_empty_lines=True, line_separator = ' // '):
1251 1252 text = text.replace('\r', '') 1253 lines = text.split('\n') 1254 text = '' 1255 for line in lines: 1256 1257 if strip_whitespace: 1258 line = line.strip().strip('\t').strip() 1259 1260 if remove_empty_lines: 1261 if line == '': 1262 continue 1263 1264 text += ('%s%s' % (line, line_separator)) 1265 1266 text = text.rstrip(line_separator) 1267 1268 if max_length is not None: 1269 text = text[:max_length] 1270 1271 text = text.rstrip(line_separator) 1272 1273 return text
1274 1275 #---------------------------------------------------------------------------
1276 -def shorten_text(text=None, max_length=None):
1277 1278 if len(text) <= max_length: 1279 return text 1280 1281 return text[:max_length-1] + u_ellipsis
1282 1283 #---------------------------------------------------------------------------
1284 -def shorten_words_in_line(text=None, max_length=None, min_word_length=None, ignore_numbers=True, ellipsis=u_ellipsis):
1285 if text is None: 1286 return None 1287 if max_length is None: 1288 max_length = len(text) 1289 else: 1290 if len(text) <= max_length: 1291 return text 1292 old_words = regex.split('\s+', text, flags = regex.UNICODE) 1293 no_old_words = len(old_words) 1294 max_word_length = max(min_word_length, (max_length // no_old_words)) 1295 words = [] 1296 for word in old_words: 1297 if len(word) <= max_word_length: 1298 words.append(word) 1299 continue 1300 if ignore_numbers: 1301 tmp = word.replace('-', '').replace('+', '').replace('.', '').replace(',', '').replace('/', '').replace('&', '').replace('*', '') 1302 if tmp.isdigit(): 1303 words.append(word) 1304 continue 1305 words.append(word[:max_word_length] + ellipsis) 1306 return ' '.join(words)
1307 1308 #---------------------------------------------------------------------------
1309 -def xml_escape_string(text=None):
1310 """check for special XML characters and transform them""" 1311 return xml_tools.escape(text)
1312 1313 #---------------------------------------------------------------------------
1314 -def tex_escape_string(text=None, replace_known_unicode=True, replace_eol=False, keep_visual_eol=False):
1315 """Check for special TeX characters and transform them. 1316 1317 replace_eol: 1318 replaces "\n" with "\\newline" 1319 keep_visual_eol: 1320 replaces "\n" with "\\newline \n" such that 1321 both LaTeX will know to place a line break 1322 at this point as well as the visual formatting 1323 is preserved in the LaTeX source (think multi- 1324 row table cells) 1325 """ 1326 text = text.replace('\\', '\\textbackslash') # requires \usepackage{textcomp} in LaTeX source 1327 text = text.replace('^', '\\textasciicircum') 1328 text = text.replace('~', '\\textasciitilde') 1329 1330 text = text.replace('{', '\\{') 1331 text = text.replace('}', '\\}') 1332 text = text.replace('%', '\\%') 1333 text = text.replace('&', '\\&') 1334 text = text.replace('#', '\\#') 1335 text = text.replace('$', '\\$') 1336 text = text.replace('_', '\\_') 1337 if replace_eol: 1338 if keep_visual_eol: 1339 text = text.replace('\n', '\\newline \n') 1340 else: 1341 text = text.replace('\n', '\\newline ') 1342 1343 if replace_known_unicode: 1344 # this should NOT be replaced for Xe(La)Tex 1345 text = text.replace(u_euro, '\\EUR') # requires \usepackage{textcomp} in LaTeX source 1346 text = text.replace(u_sum, '$\\Sigma$') 1347 1348 return text
1349 1350 #---------------------------------------------------------------------------
1351 -def rst2latex_snippet(rst_text):
1352 global du_core 1353 if du_core is None: 1354 try: 1355 from docutils import core as du_core 1356 except ImportError: 1357 _log.warning('cannot turn ReST into LaTeX: docutils not installed') 1358 return tex_escape_string(text = rst_text) 1359 1360 parts = du_core.publish_parts ( 1361 source = rst_text.replace('\\', '\\\\'), 1362 source_path = '<internal>', 1363 writer_name = 'latex', 1364 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex', 1365 settings_overrides = { 1366 'input_encoding': 'unicode' # un-encoded unicode 1367 }, 1368 enable_exit_status = True # how to use ? 1369 ) 1370 return parts['body']
1371 1372 #---------------------------------------------------------------------------
1373 -def rst2html(rst_text, replace_eol=False, keep_visual_eol=False):
1374 global du_core 1375 if du_core is None: 1376 try: 1377 from docutils import core as du_core 1378 except ImportError: 1379 _log.warning('cannot turn ReST into HTML: docutils not installed') 1380 return html_escape_string(text = rst_text, replace_eol=False, keep_visual_eol=False) 1381 1382 parts = du_core.publish_parts ( 1383 source = rst_text.replace('\\', '\\\\'), 1384 source_path = '<internal>', 1385 writer_name = 'latex', 1386 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex', 1387 settings_overrides = { 1388 'input_encoding': 'unicode' # un-encoded unicode 1389 }, 1390 enable_exit_status = True # how to use ? 1391 ) 1392 return parts['body']
1393 1394 #---------------------------------------------------------------------------
1395 -def xetex_escape_string(text=None):
1396 # a web search did not reveal anything else for Xe(La)Tex 1397 # as opposed to LaTeX, except true unicode chars 1398 return tex_escape_string(text = text, replace_known_unicode = False)
1399 1400 #--------------------------------------------------------------------------- 1401 __html_escape_table = { 1402 "&": "&amp;", 1403 '"': "&quot;", 1404 "'": "&apos;", 1405 ">": "&gt;", 1406 "<": "&lt;", 1407 } 1408
1409 -def html_escape_string(text=None, replace_eol=False, keep_visual_eol=False):
1410 text = ''.join(__html_escape_table.get(char, char) for char in text) 1411 if replace_eol: 1412 if keep_visual_eol: 1413 text = text.replace('\n', '<br>\n') 1414 else: 1415 text = text.replace('\n', '<br>') 1416 return text
1417 1418 #---------------------------------------------------------------------------
1419 -def dict2json(obj):
1420 return json.dumps(obj, default = json_serialize)
1421 1422 #---------------------------------------------------------------------------
1423 -def json_serialize(obj):
1424 if isinstance(obj, pydt.datetime): 1425 return obj.isoformat() 1426 raise TypeError('cannot json_serialize(%s)' % type(obj))
1427 1428 #--------------------------------------------------------------------------- 1429 #---------------------------------------------------------------------------
1430 -def compare_dict_likes(d1, d2, title1=None, title2=None):
1431 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title1, '', '"%s" '), type(d1), coalesce(title2, '', '"%s" '), type(d2)) 1432 try: 1433 d1 = dict(d1) 1434 except TypeError: 1435 pass 1436 try: 1437 d2 = dict(d2) 1438 except TypeError: 1439 pass 1440 keys_d1 = frozenset(d1.keys()) 1441 keys_d2 = frozenset(d2.keys()) 1442 different = False 1443 if len(keys_d1) != len(keys_d2): 1444 _log.info('different number of keys: %s vs %s', len(keys_d1), len(keys_d2)) 1445 different = True 1446 for key in keys_d1: 1447 if key in keys_d2: 1448 if type(d1[key]) != type(d2[key]): 1449 _log.info('%25.25s: type(dict1) = %s = >>>%s<<<' % (key, type(d1[key]), d1[key])) 1450 _log.info('%25.25s type(dict2) = %s = >>>%s<<<' % ('', type(d2[key]), d2[key])) 1451 different = True 1452 continue 1453 if d1[key] == d2[key]: 1454 _log.info('%25.25s: both = >>>%s<<<' % (key, d1[key])) 1455 else: 1456 _log.info('%25.25s: dict1 = >>>%s<<<' % (key, d1[key])) 1457 _log.info('%25.25s dict2 = >>>%s<<<' % ('', d2[key])) 1458 different = True 1459 else: 1460 _log.info('%25.25s: %50.50s | <MISSING>' % (key, '>>>%s<<<' % d1[key])) 1461 different = True 1462 for key in keys_d2: 1463 if key in keys_d1: 1464 continue 1465 _log.info('%25.25s: %50.50s | %.50s' % (key, '<MISSING>', '>>>%s<<<' % d2[key])) 1466 different = True 1467 if different: 1468 _log.info('dict-likes appear to be different from each other') 1469 return False 1470 _log.info('dict-likes appear equal to each other') 1471 return True
1472 1473 #---------------------------------------------------------------------------
1474 -def format_dict_likes_comparison(d1, d2, title_left=None, title_right=None, left_margin=0, key_delim=' || ', data_delim=' | ', missing_string='=/=', difference_indicator='! ', ignore_diff_in_keys=None):
1475 1476 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title_left, '', '"%s" '), type(d1), coalesce(title_right, '', '"%s" '), type(d2)) 1477 append_type = False 1478 if None not in [title_left, title_right]: 1479 append_type = True 1480 type_left = type(d1) 1481 type_right = type(d2) 1482 if title_left is None: 1483 title_left = '%s' % type_left 1484 if title_right is None: 1485 title_right = '%s' % type_right 1486 1487 try: d1 = dict(d1) 1488 except TypeError: pass 1489 try: d2 = dict(d2) 1490 except TypeError: pass 1491 keys_d1 = d1.keys() 1492 keys_d2 = d2.keys() 1493 data = {} 1494 for key in keys_d1: 1495 data[key] = [d1[key], ' '] 1496 if key in d2: 1497 data[key][1] = d2[key] 1498 for key in keys_d2: 1499 if key in keys_d1: 1500 continue 1501 data[key] = [' ', d2[key]] 1502 max1 = max([ len('%s' % k) for k in keys_d1 ]) 1503 max2 = max([ len('%s' % k) for k in keys_d2 ]) 1504 max_len = max(max1, max2, len(_('<type>'))) 1505 max_key_len_str = '%' + '%s.%s' % (max_len, max_len) + 's' 1506 max1 = max([ len('%s' % d1[k]) for k in keys_d1 ]) 1507 max2 = max([ len('%s' % d2[k]) for k in keys_d2 ]) 1508 max_data_len = min(max(max1, max2), 100) 1509 max_data_len_str = '%' + '%s.%s' % (max_data_len, max_data_len) + 's' 1510 diff_indicator_len_str = '%' + '%s.%s' % (len(difference_indicator), len(difference_indicator)) + 's' 1511 line_template = (' ' * left_margin) + diff_indicator_len_str + max_key_len_str + key_delim + max_data_len_str + data_delim + '%s' 1512 1513 lines = [] 1514 # debugging: 1515 #lines.append(u' (40 regular spaces)') 1516 #lines.append((u' ' * 40) + u"(u' ' * 40)") 1517 #lines.append((u'%40.40s' % u'') + u"(u'%40.40s' % u'')") 1518 #lines.append((u'%40.40s' % u' ') + u"(u'%40.40s' % u' ')") 1519 #lines.append((u'%40.40s' % u'.') + u"(u'%40.40s' % u'.')") 1520 #lines.append(line_template) 1521 lines.append(line_template % ('', '', title_left, title_right)) 1522 if append_type: 1523 lines.append(line_template % ('', _('<type>'), type_left, type_right)) 1524 1525 if ignore_diff_in_keys is None: 1526 ignore_diff_in_keys = [] 1527 1528 for key in keys_d1: 1529 append_type = False 1530 txt_left_col = '%s' % d1[key] 1531 try: 1532 txt_right_col = '%s' % d2[key] 1533 if type(d1[key]) != type(d2[key]): 1534 append_type = True 1535 except KeyError: 1536 txt_right_col = missing_string 1537 lines.append(line_template % ( 1538 bool2subst ( 1539 ((txt_left_col == txt_right_col) or (key in ignore_diff_in_keys)), 1540 '', 1541 difference_indicator 1542 ), 1543 key, 1544 shorten_text(txt_left_col, max_data_len), 1545 shorten_text(txt_right_col, max_data_len) 1546 )) 1547 if append_type: 1548 lines.append(line_template % ( 1549 '', 1550 _('<type>'), 1551 shorten_text('%s' % type(d1[key]), max_data_len), 1552 shorten_text('%s' % type(d2[key]), max_data_len) 1553 )) 1554 1555 for key in keys_d2: 1556 if key in keys_d1: 1557 continue 1558 lines.append(line_template % ( 1559 bool2subst((key in ignore_diff_in_keys), '', difference_indicator), 1560 key, 1561 shorten_text(missing_string, max_data_len), 1562 shorten_text('%s' % d2[key], max_data_len) 1563 )) 1564 1565 return lines
1566 1567 #---------------------------------------------------------------------------
1568 -def format_dict_like(d, relevant_keys=None, template=None, missing_key_template='<[%(key)s] MISSING>', left_margin=0, tabular=False, value_delimiters=('>>>', '<<<'), eol='\n', values2ignore=None):
1569 if values2ignore is None: 1570 values2ignore = [] 1571 if template is not None: 1572 # all keys in template better exist in d 1573 try: 1574 return template % d 1575 except KeyError: 1576 # or else 1577 _log.exception('template contains %%()s key(s) which do not exist in data dict') 1578 # try to extend dict <d> to contain all required keys, 1579 # for that to work <relevant_keys> better list all 1580 # keys used in <template> 1581 if relevant_keys is not None: 1582 for key in relevant_keys: 1583 try: 1584 d[key] 1585 except KeyError: 1586 d[key] = missing_key_template % {'key': key} 1587 return template % d 1588 1589 if relevant_keys is None: 1590 relevant_keys = list(d.keys()) 1591 lines = [] 1592 if value_delimiters is None: 1593 delim_left = '' 1594 delim_right = '' 1595 else: 1596 delim_left, delim_right = value_delimiters 1597 if tabular: 1598 max_len = max([ len('%s' % k) for k in relevant_keys ]) 1599 max_len_str = '%s.%s' % (max_len, max_len) 1600 line_template = (' ' * left_margin) + '%' + max_len_str + ('s: %s%%s%s' % (delim_left, delim_right)) 1601 else: 1602 line_template = (' ' * left_margin) + '%%s: %s%%s%s' % (delim_left, delim_right) 1603 for key in relevant_keys: 1604 try: 1605 val = d[key] 1606 except KeyError: 1607 continue 1608 if val not in values2ignore: 1609 lines.append(line_template % (key, val)) 1610 if eol is None: 1611 return lines 1612 return eol.join(lines)
1613 1614 #---------------------------------------------------------------------------
1615 -def dicts2table(dict_list, left_margin=0, eol='\n', keys2ignore=None, column_labels=None, show_only_changes=False, equality_value='<=>', date_format=None): #, relevant_keys=None, template=None
1616 """Each dict in <dict_list> becomes a column. 1617 1618 - each key of dict becomes a row label, unless in keys2ignore 1619 1620 - each entry in the <column_labels> list becomes a column title 1621 """ 1622 keys2show = [] 1623 col_max_width = {} 1624 max_width_of_row_label_col = 0 1625 col_label_key = '__________#header#__________' 1626 if keys2ignore is None: 1627 keys2ignore = [] 1628 if column_labels is not None: 1629 keys2ignore.append(col_label_key) 1630 1631 # extract keys from all dicts and calculate column sizes 1632 for dict_idx in range(len(dict_list)): 1633 # convert potentially dict-*like* into dict 1634 d = dict(dict_list[dict_idx]) 1635 # add max-len column label row from <column_labels> list, if available 1636 if column_labels is not None: 1637 d[col_label_key] = max(column_labels[dict_idx].split('\n'), key = len) 1638 field_lengths = [] 1639 # loop over all keys in this dict 1640 for key in d.keys(): 1641 # ignore this key 1642 if key in keys2ignore: 1643 continue 1644 # remember length of value when displayed 1645 if isinstance(d[key], pydt.datetime): 1646 if date_format is None: 1647 field_lengths.append(len('%s' % d[key])) 1648 else: 1649 field_lengths.append(len(d[key].strftime(date_format))) 1650 else: 1651 field_lengths.append(len('%s' % d[key])) 1652 if key in keys2show: 1653 continue 1654 keys2show.append(key) 1655 max_width_of_row_label_col = max(max_width_of_row_label_col, len('%s' % key)) 1656 col_max_width[dict_idx] = max(field_lengths) 1657 1658 # pivot data into dict of lists per line 1659 lines = { k: [] for k in keys2show } 1660 prev_vals = {} 1661 for dict_idx in range(len(dict_list)): 1662 max_width_this_col = max(col_max_width[dict_idx], len(equality_value)) if show_only_changes else col_max_width[dict_idx] 1663 max_len_str = '%s.%s' % (max_width_this_col, max_width_this_col) 1664 field_template = ' %' + max_len_str + 's' 1665 d = dict_list[dict_idx] 1666 for key in keys2show: 1667 try: 1668 val = d[key] 1669 except KeyError: 1670 lines[key].append(field_template % _('<missing>')) 1671 continue 1672 if isinstance(val, pydt.datetime): 1673 if date_format is not None: 1674 val = val.strftime(date_format) 1675 lines[key].append(field_template % val) 1676 if show_only_changes: 1677 if key not in prev_vals: 1678 prev_vals[key] = '%s' % lines[key][-1] 1679 continue 1680 if lines[key][-1] != prev_vals[key]: 1681 prev_vals[key] = '%s' % lines[key][-1] 1682 continue 1683 lines[key][-1] = field_template % equality_value 1684 1685 # format data into table 1686 table_lines = [] 1687 max_len_str = '%s.%s' % (max_width_of_row_label_col, max_width_of_row_label_col) 1688 row_label_template = '%' + max_len_str + 's' 1689 for key in lines: 1690 # row label (= key) into first column 1691 line = (' ' * left_margin) + row_label_template % key + '|' 1692 # append list values as subsequent columns 1693 line += '|'.join(lines[key]) 1694 table_lines.append(line) 1695 1696 # insert lines with column labels (column headers) if any 1697 if column_labels is not None: 1698 # first column contains row labels, so no column label needed 1699 table_header_line_w_col_labels = (' ' * left_margin) + row_label_template % '' 1700 # second table header line: horizontal separator 1701 table_header_line_w_separator = (' ' * left_margin) + u_box_horiz_single * (max_width_of_row_label_col) 1702 max_col_label_widths = [ max(col_max_width[dict_idx], len(equality_value)) for dict_idx in range(len(dict_list)) ] 1703 for col_idx in range(len(column_labels)): 1704 max_len_str = '%s.%s' % (max_col_label_widths[col_idx], max_col_label_widths[col_idx]) 1705 col_label_template = '%' + max_len_str + 's' 1706 table_header_line_w_col_labels += '| ' 1707 table_header_line_w_col_labels += col_label_template % column_labels[col_idx] 1708 table_header_line_w_separator += '%s%s' % (u_box_plus, u_box_horiz_single) 1709 table_header_line_w_separator += u_box_horiz_single * max_col_label_widths[col_idx] 1710 table_lines.insert(0, table_header_line_w_separator) 1711 table_lines.insert(0, table_header_line_w_col_labels) 1712 1713 if eol is None: 1714 return table_lines 1715 1716 return ('|' + eol).join(table_lines) + '|' + eol 1717 1718 #---------------------------------------------------------------------------
1719 -def normalize_dict_like(d, required_keys, missing_key_template='<[%(key)s] MISSING>'):
1720 for key in required_keys: 1721 try: 1722 d[key] 1723 except KeyError: 1724 if missing_key_template is None: 1725 d[key] = None 1726 else: 1727 d[key] = missing_key_template % {'key': key} 1728 return d
1729 1730 #---------------------------------------------------------------------------
1731 -def enumerate_removable_partitions():
1732 try: 1733 import pyudev 1734 import psutil 1735 except ImportError: 1736 _log.error('pyudev and/or psutil not installed') 1737 return {} 1738 1739 removable_partitions = {} 1740 ctxt = pyudev.Context() 1741 removable_devices = [ dev for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk') if dev.attributes.get('removable') == b'1' ] 1742 all_mounted_partitions = { part.device: part for part in psutil.disk_partitions() } 1743 for device in removable_devices: 1744 _log.debug('removable device: %s', device.properties['ID_MODEL']) 1745 partitions_on_removable_device = { 1746 part.device_node: { 1747 'type': device.properties['ID_TYPE'], 1748 'bus': device.properties['ID_BUS'], 1749 'device': device.properties['DEVNAME'], 1750 'partition': part.properties['DEVNAME'], 1751 'vendor': part.properties['ID_VENDOR'], 1752 'model': part.properties['ID_MODEL'], 1753 'fs_label': part.properties['ID_FS_LABEL'], 1754 'is_mounted': False, 1755 'mountpoint': None, 1756 'fs_type': None, 1757 'size_in_bytes': -1, 1758 'bytes_free': 0 1759 } for part in ctxt.list_devices(subsystem='block', DEVTYPE='partition', parent=device) 1760 } 1761 for part in partitions_on_removable_device: 1762 try: 1763 partitions_on_removable_device[part]['mountpoint'] = all_mounted_partitions[part].mountpoint 1764 partitions_on_removable_device[part]['is_mounted'] = True 1765 partitions_on_removable_device[part]['fs_type'] = all_mounted_partitions[part].fstype 1766 du = shutil.disk_usage(all_mounted_partitions[part].mountpoint) 1767 partitions_on_removable_device[part]['size_in_bytes'] = du.total 1768 partitions_on_removable_device[part]['bytes_free'] = du.free 1769 except KeyError: 1770 pass # not mounted 1771 removable_partitions.update(partitions_on_removable_device) 1772 return removable_partitions
1773 1774 # debugging: 1775 #ctxt = pyudev.Context() 1776 #for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk'):# if dev.attributes.get('removable') == b'1': 1777 # for a in dev.attributes.available_attributes: 1778 # print(a, dev.attributes.get(a)) 1779 # for key, value in dev.items(): 1780 # print('{key}={value}'.format(key=key, value=value)) 1781 # print('---------------------------') 1782 1783 #---------------------------------------------------------------------------
1784 -def enumerate_optical_writers():
1785 try: 1786 import pyudev 1787 except ImportError: 1788 _log.error('pyudev not installed') 1789 return [] 1790 1791 optical_writers = [] 1792 ctxt = pyudev.Context() 1793 for dev in [ dev for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk') if dev.properties.get('ID_CDROM_CD_RW', None) == '1' ]: 1794 optical_writers.append ({ 1795 'type': dev.properties['ID_TYPE'], 1796 'bus': dev.properties['ID_BUS'], 1797 'device': dev.properties['DEVNAME'], 1798 'model': dev.properties['ID_MODEL'] 1799 }) 1800 return optical_writers
1801 1802 #--------------------------------------------------------------------------- 1803 #---------------------------------------------------------------------------
1804 -def prompted_input(prompt=None, default=None):
1805 """Obtains entry from standard input. 1806 1807 prompt: Prompt text to display in standard output 1808 default: Default value (for user to press enter only) 1809 CTRL-C: aborts and returns None 1810 """ 1811 if prompt is None: 1812 msg = '(CTRL-C aborts)' 1813 else: 1814 msg = '%s (CTRL-C aborts)' % prompt 1815 1816 if default is None: 1817 msg = msg + ': ' 1818 else: 1819 msg = '%s [%s]: ' % (msg, default) 1820 1821 try: 1822 usr_input = input(msg) 1823 except KeyboardInterrupt: 1824 return None 1825 1826 if usr_input == '': 1827 return default 1828 1829 return usr_input
1830 1831 #=========================================================================== 1832 # image handling tools 1833 #--------------------------------------------------------------------------- 1834 # builtin (ugly but tried and true) fallback icon 1835 __icon_serpent = \ 1836 """x\xdae\x8f\xb1\x0e\x83 \x10\x86w\x9f\xe2\x92\x1blb\xf2\x07\x96\xeaH:0\xd6\ 1837 \xc1\x85\xd5\x98N5\xa5\xef?\xf5N\xd0\x8a\xdcA\xc2\xf7qw\x84\xdb\xfa\xb5\xcd\ 1838 \xd4\xda;\xc9\x1a\xc8\xb6\xcd<\xb5\xa0\x85\x1e\xeb\xbc\xbc7b!\xf6\xdeHl\x1c\ 1839 \x94\x073\xec<*\xf7\xbe\xf7\x99\x9d\xb21~\xe7.\xf5\x1f\x1c\xd3\xbdVlL\xc2\ 1840 \xcf\xf8ye\xd0\x00\x90\x0etH \x84\x80B\xaa\x8a\x88\x85\xc4(U\x9d$\xfeR;\xc5J\ 1841 \xa6\x01\xbbt9\xceR\xc8\x81e_$\x98\xb9\x9c\xa9\x8d,y\xa9t\xc8\xcf\x152\xe0x\ 1842 \xe9$\xf5\x07\x95\x0cD\x95t:\xb1\x92\xae\x9cI\xa8~\x84\x1f\xe0\xa3ec""" 1843
1844 -def get_icon(wx=None):
1845 1846 paths = gmPaths(app_name = 'gnumed', wx = wx) 1847 1848 candidates = [ 1849 os.path.join(paths.system_app_data_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'), 1850 os.path.join(paths.local_base_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'), 1851 os.path.join(paths.system_app_data_dir, 'bitmaps', 'serpent.png'), 1852 os.path.join(paths.local_base_dir, 'bitmaps', 'serpent.png') 1853 ] 1854 1855 found_as = None 1856 for candidate in candidates: 1857 try: 1858 open(candidate, 'r').close() 1859 found_as = candidate 1860 break 1861 except IOError: 1862 _log.debug('icon not found in [%s]', candidate) 1863 1864 if found_as is None: 1865 _log.warning('no icon file found, falling back to builtin (ugly) icon') 1866 icon_bmp_data = wx.BitmapFromXPMData(pickle.loads(zlib.decompress(__icon_serpent))) 1867 icon.CopyFromBitmap(icon_bmp_data) 1868 else: 1869 _log.debug('icon found in [%s]', found_as) 1870 icon = wx.Icon() 1871 try: 1872 icon.LoadFile(found_as, wx.BITMAP_TYPE_ANY) #_PNG 1873 except AttributeError: 1874 _log.exception("this platform doesn't support wx.Icon().LoadFile()") 1875 1876 return icon
1877 1878 #---------------------------------------------------------------------------
1879 -def create_qrcode(text=None, filename=None, qr_filename=None, verbose=False):
1880 assert (not ((text is None) and (filename is None))), 'either <text> or <filename> must be specified' 1881 1882 try: 1883 import pyqrcode 1884 except ImportError: 1885 _log.exception('cannot import <pyqrcode>') 1886 return None 1887 if text is None: 1888 with io.open(filename, mode = 'rt', encoding = 'utf8') as input_file: 1889 text = input_file.read() 1890 if qr_filename is None: 1891 if filename is None: 1892 qr_filename = get_unique_filename(prefix = 'gm-qr-', suffix = '.png') 1893 else: 1894 qr_filename = get_unique_filename ( 1895 prefix = fname_stem(filename) + '-', 1896 suffix = fname_extension(filename) + '.png' 1897 ) 1898 _log.debug('[%s] -> [%s]', filename, qr_filename) 1899 qr = pyqrcode.create(text, encoding = 'utf8') 1900 if verbose: 1901 print('input file:', filename) 1902 print('output file:', qr_filename) 1903 print('text to encode:', text) 1904 print(qr.terminal()) 1905 qr.png(qr_filename, quiet_zone = 1) 1906 return qr_filename
1907 1908 #=========================================================================== 1909 # main 1910 #--------------------------------------------------------------------------- 1911 if __name__ == '__main__': 1912 1913 if len(sys.argv) < 2: 1914 sys.exit() 1915 1916 if sys.argv[1] != 'test': 1917 sys.exit() 1918 1919 # for testing: 1920 logging.basicConfig(level = logging.DEBUG) 1921 from Gnumed.pycommon import gmI18N 1922 gmI18N.activate_locale() 1923 gmI18N.install_domain() 1924 1925 #-----------------------------------------------------------------------
1926 - def test_input2decimal():
1927 1928 tests = [ 1929 [None, False], 1930 1931 ['', False], 1932 [' 0 ', True, 0], 1933 1934 [0, True, 0], 1935 [0.0, True, 0], 1936 [.0, True, 0], 1937 ['0', True, 0], 1938 ['0.0', True, 0], 1939 ['0,0', True, 0], 1940 ['00.0', True, 0], 1941 ['.0', True, 0], 1942 [',0', True, 0], 1943 1944 [0.1, True, decimal.Decimal('0.1')], 1945 [.01, True, decimal.Decimal('0.01')], 1946 ['0.1', True, decimal.Decimal('0.1')], 1947 ['0,1', True, decimal.Decimal('0.1')], 1948 ['00.1', True, decimal.Decimal('0.1')], 1949 ['.1', True, decimal.Decimal('0.1')], 1950 [',1', True, decimal.Decimal('0.1')], 1951 1952 [1, True, 1], 1953 [1.0, True, 1], 1954 ['1', True, 1], 1955 ['1.', True, 1], 1956 ['1,', True, 1], 1957 ['1.0', True, 1], 1958 ['1,0', True, 1], 1959 ['01.0', True, 1], 1960 ['01,0', True, 1], 1961 [' 01, ', True, 1], 1962 1963 [decimal.Decimal('1.1'), True, decimal.Decimal('1.1')] 1964 ] 1965 for test in tests: 1966 conversion_worked, result = input2decimal(initial = test[0]) 1967 1968 expected2work = test[1] 1969 1970 if conversion_worked: 1971 if expected2work: 1972 if result == test[2]: 1973 continue 1974 else: 1975 print("ERROR (conversion result wrong): >%s<, expected >%s<, got >%s<" % (test[0], test[2], result)) 1976 else: 1977 print("ERROR (conversion worked but was expected to fail): >%s<, got >%s<" % (test[0], result)) 1978 else: 1979 if not expected2work: 1980 continue 1981 else: 1982 print("ERROR (conversion failed but was expected to work): >%s<, expected >%s<" % (test[0], test[2]))
1983 #-----------------------------------------------------------------------
1984 - def test_input2int():
1985 print(input2int(0)) 1986 print(input2int('0')) 1987 print(input2int('0', 0, 0))
1988 #-----------------------------------------------------------------------
1989 - def test_coalesce():
1990 1991 val = None 1992 print(val, coalesce(val, 'is None', 'is not None')) 1993 val = 1 1994 print(val, coalesce(val, 'is None', 'is not None')) 1995 return 1996 1997 import datetime as dt 1998 print(coalesce(value2test = dt.datetime.now(), template4value = '-- %s --', function4value = ('strftime', '%Y-%m-%d'))) 1999 2000 print('testing coalesce()') 2001 print("------------------") 2002 tests = [ 2003 [None, 'something other than <None>', None, None, 'something other than <None>'], 2004 ['Captain', 'Mr.', '%s.'[:4], 'Mr.', 'Capt.'], 2005 ['value to test', 'test 3 failed', 'template with "%s" included', None, 'template with "value to test" included'], 2006 ['value to test', 'test 4 failed', 'template with value not included', None, 'template with value not included'], 2007 [None, 'initial value was None', 'template4value: %s', None, 'initial value was None'], 2008 [None, 'initial value was None', 'template4value: %%(abc)s', None, 'initial value was None'] 2009 ] 2010 passed = True 2011 for test in tests: 2012 result = coalesce ( 2013 value2test = test[0], 2014 return_instead = test[1], 2015 template4value = test[2], 2016 template4instead = test[3] 2017 ) 2018 if result != test[4]: 2019 print("ERROR") 2020 print("coalesce: (%s, %s, %s, %s)" % (test[0], test[1], test[2], test[3])) 2021 print("expected:", test[4]) 2022 print("received:", result) 2023 passed = False 2024 2025 if passed: 2026 print("passed") 2027 else: 2028 print("failed") 2029 return passed
2030 #-----------------------------------------------------------------------
2031 - def test_capitalize():
2032 print('testing capitalize() ...') 2033 success = True 2034 pairs = [ 2035 # [original, expected result, CAPS mode] 2036 ['Boot', 'Boot', CAPS_FIRST_ONLY], 2037 ['boot', 'Boot', CAPS_FIRST_ONLY], 2038 ['booT', 'Boot', CAPS_FIRST_ONLY], 2039 ['BoOt', 'Boot', CAPS_FIRST_ONLY], 2040 ['boots-Schau', 'Boots-Schau', CAPS_WORDS], 2041 ['boots-sChau', 'Boots-Schau', CAPS_WORDS], 2042 ['boot camp', 'Boot Camp', CAPS_WORDS], 2043 ['fahrner-Kampe', 'Fahrner-Kampe', CAPS_NAMES], 2044 ['häkkönen', 'Häkkönen', CAPS_NAMES], 2045 ['McBurney', 'McBurney', CAPS_NAMES], 2046 ['mcBurney', 'McBurney', CAPS_NAMES], 2047 ['blumberg', 'Blumberg', CAPS_NAMES], 2048 ['roVsing', 'RoVsing', CAPS_NAMES], 2049 ['Özdemir', 'Özdemir', CAPS_NAMES], 2050 ['özdemir', 'Özdemir', CAPS_NAMES], 2051 ] 2052 for pair in pairs: 2053 result = capitalize(pair[0], pair[2]) 2054 if result != pair[1]: 2055 success = False 2056 print('ERROR (caps mode %s): "%s" -> "%s", expected "%s"' % (pair[2], pair[0], result, pair[1])) 2057 2058 if success: 2059 print("... SUCCESS") 2060 2061 return success
2062 #-----------------------------------------------------------------------
2063 - def test_import_module():
2064 print("testing import_module_from_directory()") 2065 path = sys.argv[1] 2066 name = sys.argv[2] 2067 try: 2068 mod = import_module_from_directory(module_path = path, module_name = name) 2069 except Exception: 2070 print("module import failed, see log") 2071 return False 2072 2073 print("module import succeeded", mod) 2074 print(dir(mod)) 2075 return True
2076 #-----------------------------------------------------------------------
2077 - def test_mkdir():
2078 print("testing mkdir(%s)" % sys.argv[2]) 2079 mkdir(sys.argv[2], 0o0700)
2080 #-----------------------------------------------------------------------
2081 - def test_gmPaths():
2082 print("testing gmPaths()") 2083 print("-----------------") 2084 paths = gmPaths(wx=None, app_name='gnumed') 2085 print("user config dir:", paths.user_config_dir) 2086 print("system config dir:", paths.system_config_dir) 2087 print("local base dir:", paths.local_base_dir) 2088 print("system app data dir:", paths.system_app_data_dir) 2089 print("working directory :", paths.working_dir) 2090 print("temp directory :", paths.tmp_dir)
2091 #-----------------------------------------------------------------------
2092 - def test_none_if():
2093 print("testing none_if()") 2094 print("-----------------") 2095 tests = [ 2096 [None, None, None], 2097 ['a', 'a', None], 2098 ['a', 'b', 'a'], 2099 ['a', None, 'a'], 2100 [None, 'a', None], 2101 [1, 1, None], 2102 [1, 2, 1], 2103 [1, None, 1], 2104 [None, 1, None] 2105 ] 2106 2107 for test in tests: 2108 if none_if(value = test[0], none_equivalent = test[1]) != test[2]: 2109 print('ERROR: none_if(%s) returned [%s], expected [%s]' % (test[0], none_if(test[0], test[1]), test[2])) 2110 2111 return True
2112 #-----------------------------------------------------------------------
2113 - def test_bool2str():
2114 tests = [ 2115 [True, 'Yes', 'Yes', 'Yes'], 2116 [False, 'OK', 'not OK', 'not OK'] 2117 ] 2118 for test in tests: 2119 if bool2str(test[0], test[1], test[2]) != test[3]: 2120 print('ERROR: bool2str(%s, %s, %s) returned [%s], expected [%s]' % (test[0], test[1], test[2], bool2str(test[0], test[1], test[2]), test[3])) 2121 2122 return True
2123 #-----------------------------------------------------------------------
2124 - def test_bool2subst():
2125 2126 print(bool2subst(True, 'True', 'False', 'is None')) 2127 print(bool2subst(False, 'True', 'False', 'is None')) 2128 print(bool2subst(None, 'True', 'False', 'is None'))
2129 #-----------------------------------------------------------------------
2130 - def test_get_unique_filename():
2131 print(get_unique_filename()) 2132 print(get_unique_filename(prefix='test-')) 2133 print(get_unique_filename(suffix='tst')) 2134 print(get_unique_filename(prefix='test-', suffix='tst')) 2135 print(get_unique_filename(tmp_dir='/home/ncq/Archiv/'))
2136 #-----------------------------------------------------------------------
2137 - def test_size2str():
2138 print("testing size2str()") 2139 print("------------------") 2140 tests = [0, 1, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 100000000000, 1000000000000, 10000000000000] 2141 for test in tests: 2142 print(size2str(test))
2143 #-----------------------------------------------------------------------
2144 - def test_unwrap():
2145 2146 test = """ 2147 second line\n 2148 3rd starts with tab \n 2149 4th with a space \n 2150 2151 6th 2152 2153 """ 2154 print(unwrap(text = test, max_length = 25))
2155 #-----------------------------------------------------------------------
2156 - def test_wrap():
2157 test = 'line 1\nline 2\nline 3' 2158 2159 print("wrap 5-6-7 initial 0, subsequent 0") 2160 print(wrap(test, 5)) 2161 print() 2162 print(wrap(test, 6)) 2163 print() 2164 print(wrap(test, 7)) 2165 print("-------") 2166 input() 2167 print("wrap 5 initial 1-1-3, subsequent 1-3-1") 2168 print(wrap(test, 5, ' ', ' ')) 2169 print() 2170 print(wrap(test, 5, ' ', ' ')) 2171 print() 2172 print(wrap(test, 5, ' ', ' ')) 2173 print("-------") 2174 input() 2175 print("wrap 6 initial 1-1-3, subsequent 1-3-1") 2176 print(wrap(test, 6, ' ', ' ')) 2177 print() 2178 print(wrap(test, 6, ' ', ' ')) 2179 print() 2180 print(wrap(test, 6, ' ', ' ')) 2181 print("-------") 2182 input() 2183 print("wrap 7 initial 1-1-3, subsequent 1-3-1") 2184 print(wrap(test, 7, ' ', ' ')) 2185 print() 2186 print(wrap(test, 7, ' ', ' ')) 2187 print() 2188 print(wrap(test, 7, ' ', ' '))
2189 #-----------------------------------------------------------------------
2190 - def test_md5():
2191 print('md5 %s: %s' % (sys.argv[2], file2md5(sys.argv[2]))) 2192 print('chunked md5 %s: %s' % (sys.argv[2], file2chunked_md5(sys.argv[2])))
2193 #-----------------------------------------------------------------------
2194 - def test_unicode():
2195 print(u_link_symbol * 10)
2196 #-----------------------------------------------------------------------
2197 - def test_xml_escape():
2198 print(xml_escape_string('<')) 2199 print(xml_escape_string('>')) 2200 print(xml_escape_string('&'))
2201 #-----------------------------------------------------------------------
2202 - def test_tex_escape():
2203 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234'] 2204 tests.append(' '.join(tests)) 2205 for test in tests: 2206 print('%s:' % test, tex_escape_string(test))
2207 2208 #-----------------------------------------------------------------------
2209 - def test_rst2latex_snippet():
2210 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234'] 2211 tests.append(' '.join(tests)) 2212 tests.append('C:\Windows\Programme\System 32\lala.txt') 2213 tests.extend([ 2214 'should be identical', 2215 'text *some text* text', 2216 """A List 2217 ====== 2218 2219 1. 1 2220 2. 2 2221 2222 3. ist-list 2223 1. more 2224 2. noch was ü 2225 #. nummer x""" 2226 ]) 2227 for test in tests: 2228 print('==================================================') 2229 print('raw:') 2230 print(test) 2231 print('---------') 2232 print('ReST 2 LaTeX:') 2233 latex = rst2latex_snippet(test) 2234 print(latex) 2235 if latex.strip() == test.strip(): 2236 print('=> identical') 2237 print('---------') 2238 print('tex_escape_string:') 2239 print(tex_escape_string(test)) 2240 input()
2241 2242 #-----------------------------------------------------------------------
2243 - def test_strip_trailing_empty_lines():
2244 tests = [ 2245 'one line, no embedded line breaks ', 2246 'one line\nwith embedded\nline\nbreaks\n ' 2247 ] 2248 for test in tests: 2249 print('as list:') 2250 print(strip_trailing_empty_lines(text = test, eol='\n', return_list = True)) 2251 print('as string:') 2252 print('>>>%s<<<' % strip_trailing_empty_lines(text = test, eol='\n', return_list = False)) 2253 tests = [ 2254 ['list', 'without', 'empty', 'trailing', 'lines'], 2255 ['list', 'with', 'empty', 'trailing', 'lines', '', ' ', ''] 2256 ] 2257 for test in tests: 2258 print('as list:') 2259 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = True)) 2260 print('as string:') 2261 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = False))
2262 #-----------------------------------------------------------------------
2263 - def test_fname_stem():
2264 tests = [ 2265 r'abc.exe', 2266 r'\abc.exe', 2267 r'c:\abc.exe', 2268 r'c:\d\abc.exe', 2269 r'/home/ncq/tmp.txt', 2270 r'~/tmp.txt', 2271 r'./tmp.txt', 2272 r'./.././tmp.txt', 2273 r'tmp.txt' 2274 ] 2275 for t in tests: 2276 print("[%s] -> [%s]" % (t, fname_stem(t)))
2277 #-----------------------------------------------------------------------
2278 - def test_dir_is_empty():
2279 print(sys.argv[2], 'empty:', dir_is_empty(sys.argv[2]))
2280 2281 #-----------------------------------------------------------------------
2282 - def test_compare_dicts():
2283 d1 = {} 2284 d2 = {} 2285 d1[1] = 1 2286 d1[2] = 2 2287 d1[3] = 3 2288 # 4 2289 d1[5] = 5 2290 2291 d2[1] = 1 2292 d2[2] = None 2293 # 3 2294 d2[4] = 4 2295 2296 #compare_dict_likes(d1, d2) 2297 2298 d1 = {1: 1, 2: 2} 2299 d2 = {1: 1, 2: 2} 2300 2301 #compare_dict_likes(d1, d2, 'same1', 'same2') 2302 print(format_dict_like(d1, tabular = False)) 2303 print(format_dict_like(d1, tabular = True))
2304 #print(format_dict_like(d2)) 2305 2306 #-----------------------------------------------------------------------
2307 - def test_format_compare_dicts():
2308 d1 = {} 2309 d2 = {} 2310 d1[1] = 1 2311 d1[2] = 2 2312 d1[3] = 3 2313 # 4 2314 d1[5] = 5 2315 2316 d2[1] = 1 2317 d2[2] = None 2318 # 3 2319 d2[4] = 4 2320 2321 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2'))) 2322 2323 d1 = {1: 1, 2: 2} 2324 d2 = {1: 1, 2: 2} 2325 2326 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
2327 2328 #-----------------------------------------------------------------------
2329 - def test_rm_dir():
2330 rmdir('cx:\windows\system3__2xxxxxxxxxxxxx')
2331 2332 #-----------------------------------------------------------------------
2333 - def test_rm_dir_content():
2334 #print(rm_dir_content('cx:\windows\system3__2xxxxxxxxxxxxx')) 2335 print(rm_dir_content('/tmp/user/1000/tmp'))
2336 2337 #-----------------------------------------------------------------------
2338 - def test_strip_prefix():
2339 tests = [ 2340 ('', '', ''), 2341 ('a', 'a', ''), 2342 ('GMd: a window title', _GM_TITLE_PREFIX + ':', 'a window title'), 2343 ('\.br\MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\', '\.br\\', 'MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\') 2344 ] 2345 for test in tests: 2346 text, prefix, expect = test 2347 result = strip_prefix(text, prefix, remove_whitespace = True) 2348 if result == expect: 2349 continue 2350 print('test failed:', test) 2351 print('result:', result)
2352 2353 #-----------------------------------------------------------------------
2354 - def test_shorten_text():
2355 tst = [ 2356 ('123', 1), 2357 ('123', 2), 2358 ('123', 3), 2359 ('123', 4), 2360 ('', 1), 2361 ('1', 1), 2362 ('12', 1), 2363 ('', 2), 2364 ('1', 2), 2365 ('12', 2), 2366 ('123', 2) 2367 ] 2368 for txt, lng in tst: 2369 print('max', lng, 'of', txt, '=', shorten_text(txt, lng))
2370 #-----------------------------------------------------------------------
2371 - def test_fname_sanitize():
2372 tests = [ 2373 '/tmp/test.txt', 2374 '/tmp/ test.txt', 2375 '/tmp/ tes\\t.txt', 2376 'test' 2377 ] 2378 for test in tests: 2379 print (test, fname_sanitize(test))
2380 2381 #-----------------------------------------------------------------------
2382 - def test_create_qrcode():
2383 print(create_qrcode(text = sys.argv[2], filename=None, qr_filename=None, verbose = True))
2384 2385 #-----------------------------------------------------------------------
2386 - def test_enumerate_removable_partitions():
2387 parts = enumerate_removable_partitions() 2388 for part_name in parts: 2389 part = parts[part_name] 2390 print(part['device']) 2391 print(part['partition']) 2392 if part['is_mounted']: 2393 print('%s@%s: %s on %s by %s @ %s (FS=%s: %s free of %s total)' % ( 2394 part['type'], 2395 part['bus'], 2396 part['fs_label'], 2397 part['model'], 2398 part['vendor'], 2399 part['mountpoint'], 2400 part['fs_type'], 2401 part['bytes_free'], 2402 part['size_in_bytes'] 2403 )) 2404 else: 2405 print('%s@%s: %s on %s by %s (not mounted)' % ( 2406 part['type'], 2407 part['bus'], 2408 part['fs_label'], 2409 part['model'], 2410 part['vendor'] 2411 ))
2412 2413 #-----------------------------------------------------------------------
2414 - def test_enumerate_optical_writers():
2415 for writer in enumerate_optical_writers(): 2416 print('%s@%s: %s @ %s' % ( 2417 writer['type'], 2418 writer['bus'], 2419 writer['model'], 2420 writer['device'] 2421 ))
2422 2423 #-----------------------------------------------------------------------
2424 - def test_copy_tree_content():
2425 print(sys.argv[2], '->', sys.argv[3]) 2426 print(copy_tree_content(sys.argv[2], sys.argv[3]))
2427 2428 #-----------------------------------------------------------------------
2429 - def test_mk_sandbox_dir():
2430 print(mk_sandbox_dir(base_dir = '/tmp/abcd/efg/h'))
2431 2432 #-----------------------------------------------------------------------
2433 - def test_make_table_from_dicts():
2434 dicts = [ 2435 {'pkey': 1, 'value': 'a122'}, 2436 {'pkey': 2, 'value': 'b23'}, 2437 {'pkey': 3, 'value': 'c3'}, 2438 {'pkey': 4, 'value': 'd4ssssssssssss'}, 2439 {'pkey': 5, 'value': 'd4 asdfas '}, 2440 {'pkey': 5, 'value': 'c5---'}, 2441 ] 2442 with open('x.txt', 'w', encoding = 'utf8') as f: 2443 f.write(dicts2table(dicts, left_margin=2, eol='\n', keys2ignore=None, show_only_changes=True, column_labels = ['d1', 'd2', 'd3', 'd4', 'd5', 'd6']))
2444 #print(dicts2table(dicts, left_margin=2, eol='\n', keys2ignore=None, show_only_changes=True, column_labels = ['d1', 'd2', 'd3', 'd4', 'd5', 'd6'])) 2445 2446 #-----------------------------------------------------------------------
2447 - def test_decorate_window_title():
2448 for txt in globals(): 2449 print(decorate_window_title(txt))
2450 2451 #----------------------------------------------------------------------- 2452 #test_coalesce() 2453 #test_capitalize() 2454 #test_import_module() 2455 #test_mkdir() 2456 #test_gmPaths() 2457 #test_none_if() 2458 #test_bool2str() 2459 #test_bool2subst() 2460 #test_get_unique_filename() 2461 #test_size2str() 2462 #test_wrap() 2463 #test_input2decimal() 2464 #test_input2int() 2465 #test_unwrap() 2466 #test_md5() 2467 #test_unicode() 2468 #test_xml_escape() 2469 #test_strip_trailing_empty_lines() 2470 #test_fname_stem() 2471 #test_tex_escape() 2472 #test_rst2latex_snippet() 2473 #test_dir_is_empty() 2474 #test_compare_dicts() 2475 #test_rm_dir() 2476 #test_rm_dir_content() 2477 #test_strip_prefix() 2478 #test_shorten_text() 2479 #test_format_compare_dicts() 2480 #test_fname_sanitize() 2481 #test_create_qrcode() 2482 #test_enumerate_removable_partitions() 2483 #test_enumerate_optical_writers() 2484 #test_copy_tree_content() 2485 #test_mk_sandbox_dir() 2486 test_make_table_from_dicts() 2487 #test_decorate_window_title() 2488 2489 #=========================================================================== 2490