Package Gnumed :: Package pycommon :: Module gmTools
[frames] | no frames]

Source Code for Module Gnumed.pycommon.gmTools

   1  # -*- coding: utf-8 -*- 
   2   
   3   
   4  __doc__ = """GNUmed general tools.""" 
   5   
   6  #=========================================================================== 
   7  __author__ = "K. Hilbert <Karsten.Hilbert@gmx.net>" 
   8  __license__ = "GPL v2 or later (details at http://www.gnu.org)" 
   9   
  10  # std libs 
  11  import sys 
  12  import os 
  13  import os.path 
  14  import csv 
  15  import tempfile 
  16  import logging 
  17  import hashlib 
  18  import platform 
  19  import subprocess 
  20  import decimal 
  21  import getpass 
  22  import io 
  23  import functools 
  24  import json 
  25  import shutil 
  26  import zipfile 
  27  import datetime as pydt 
  28  import re as regex 
  29  import xml.sax.saxutils as xml_tools 
  30  # old: 
  31  import pickle, zlib 
  32  # docutils 
  33  du_core = None 
  34   
  35   
  36  # GNUmed libs 
  37  if __name__ == '__main__': 
  38          sys.path.insert(0, '../../') 
  39  from Gnumed.pycommon import gmBorg 
  40   
  41   
  42  _log = logging.getLogger('gm.tools') 
  43   
  44  # CAPitalization modes: 
  45  (       CAPS_NONE,                                      # don't touch it 
  46          CAPS_FIRST,                                     # CAP first char, leave rest as is 
  47          CAPS_ALLCAPS,                           # CAP all chars 
  48          CAPS_WORDS,                                     # CAP first char of every word 
  49          CAPS_NAMES,                                     # CAP in a way suitable for names (tries to be smart) 
  50          CAPS_FIRST_ONLY                         # CAP first char, lowercase the rest 
  51  ) = range(6) 
  52   
  53   
  54  u_currency_pound = '\u00A3'                             # Pound sign 
  55  u_currency_sign = '\u00A4'                                      # generic currency sign 
  56  u_currency_yen = '\u00A5'                                       # Yen sign 
  57  u_right_double_angle_quote = '\u00AB'           # << 
  58  u_registered_trademark = '\u00AE' 
  59  u_plus_minus = '\u00B1' 
  60  u_superscript_one = '\u00B9'                            # ^1 
  61  u_left_double_angle_quote = '\u00BB'            # >> 
  62  u_one_quarter = '\u00BC' 
  63  u_one_half = '\u00BD' 
  64  u_three_quarters = '\u00BE' 
  65  u_multiply = '\u00D7'                                           # x 
  66  u_greek_ALPHA = '\u0391' 
  67  u_greek_alpha = '\u03b1' 
  68  u_greek_OMEGA = '\u03A9' 
  69  u_greek_omega = '\u03c9' 
  70  u_dagger = '\u2020' 
  71  u_triangular_bullet = '\u2023'                                  # triangular bullet  (>) 
  72  u_ellipsis = '\u2026'                                                   # ... 
  73  u_euro = '\u20AC'                                                               # EURO sign 
  74  u_numero = '\u2116'                                                             # No. / # sign 
  75  u_down_left_arrow = '\u21B5'                                    # <-' 
  76  u_left_arrow = '\u2190'                                                 # <-- 
  77  u_up_arrow = '\u2191' 
  78  u_arrow2right = '\u2192'                                                # --> 
  79  u_down_arrow = '\u2193' 
  80  u_left_arrow_with_tail = '\u21a2'                               # <--< 
  81  u_arrow2right_from_bar = '\u21a6'                               # |-> 
  82  u_arrow2right_until_vertical_bar = '\u21e5'             # -->| 
  83  u_sum = '\u2211'                                                                # sigma 
  84  u_almost_equal_to = '\u2248'                                    # approximately / nearly / roughly 
  85  u_corresponds_to = '\u2258' 
  86  u_infinity = '\u221E' 
  87  u_arrow2right_until_vertical_bar2 = '\u2b72'    # -->| 
  88   
  89  u_diameter = '\u2300' 
  90  u_checkmark_crossed_out = '\u237B' 
  91  u_box_horiz_high = '\u23ba' 
  92  u_box_vert_left = '\u23b8' 
  93  u_box_vert_right = '\u23b9' 
  94   
  95  u_space_as_open_box = '\u2423' 
  96   
  97  u_box_horiz_single = '\u2500'                           # - 
  98  u_box_vert_light = '\u2502' 
  99  u_box_horiz_light_3dashes = '\u2504'            # ... 
 100  u_box_vert_light_4dashes = '\u2506' 
 101  u_box_horiz_4dashes = '\u2508'                          # .... 
 102  u_box_T_right = '\u251c'                                        # |- 
 103  u_box_T_left = '\u2524'                                         # -| 
 104  u_box_T_down = '\u252c' 
 105  u_box_T_up = '\u2534' 
 106  u_box_plus = '\u253c' 
 107  u_box_top_double = '\u2550' 
 108  u_box_top_left_double_single = '\u2552' 
 109  u_box_top_right_double_single = '\u2555' 
 110  u_box_top_left_arc = '\u256d' 
 111  u_box_top_right_arc = '\u256e' 
 112  u_box_bottom_right_arc = '\u256f' 
 113  u_box_bottom_left_arc = '\u2570' 
 114  u_box_horiz_light_heavy = '\u257c' 
 115  u_box_horiz_heavy_light = '\u257e' 
 116   
 117  u_skull_and_crossbones = '\u2620' 
 118  u_caduceus = '\u2624' 
 119  u_frowning_face = '\u2639' 
 120  u_smiling_face = '\u263a' 
 121  u_black_heart = '\u2665' 
 122  u_female = '\u2640' 
 123  u_male = '\u2642' 
 124  u_male_female = '\u26a5' 
 125  u_chain = '\u26d3' 
 126   
 127  u_checkmark_thin = '\u2713' 
 128  u_checkmark_thick = '\u2714' 
 129  u_heavy_greek_cross = '\u271a' 
 130  u_arrow2right_thick = '\u2794' 
 131  u_writing_hand = '\u270d' 
 132  u_pencil_1 = '\u270e' 
 133  u_pencil_2 = '\u270f' 
 134  u_pencil_3 = '\u2710' 
 135  u_latin_cross = '\u271d' 
 136   
 137  u_arrow2right_until_black_diamond = '\u291e'    # ->* 
 138   
 139  u_kanji_yen = '\u5186'                                                  # Yen kanji 
 140  u_replacement_character = '\ufffd' 
 141  u_link_symbol = '\u1f517' 
 142   
 143   
 144  _kB = 1024 
 145  _MB = 1024 * _kB 
 146  _GB = 1024 * _MB 
 147  _TB = 1024 * _GB 
 148  _PB = 1024 * _TB 
 149   
 150   
 151  _client_version = None 
 152   
 153   
 154  _GM_TITLE_PREFIX = 'GMd' 
 155   
 156  #=========================================================================== 
157 -def handle_uncaught_exception_console(t, v, tb):
158 159 print(".========================================================") 160 print("| Unhandled exception caught !") 161 print("| Type :", t) 162 print("| Value:", v) 163 print("`========================================================") 164 _log.critical('unhandled exception caught', exc_info = (t,v,tb)) 165 sys.__excepthook__(t,v,tb)
166 167 #=========================================================================== 168 # path level operations 169 #---------------------------------------------------------------------------
170 -def mkdir(directory=None, mode=None):
171 """Create directory. 172 173 - creates parent dirs if necessary 174 - does not fail if directory exists 175 <mode>: numeric, say 0o0700 for "-rwx------" 176 """ 177 if os.path.isdir(directory): 178 if mode is None: 179 return True 180 181 changed = False 182 old_umask = os.umask(0) 183 try: 184 # does not WORK ! 185 #os.chmod(directory, mode, follow_symlinks = (os.chmod in os.supports_follow_symlinks)) # can't do better 186 os.chmod(directory, mode) 187 changed = True 188 finally: 189 os.umask(old_umask) 190 return changed 191 192 if mode is None: 193 os.makedirs(directory) 194 return True 195 196 old_umask = os.umask(0) 197 try: 198 os.makedirs(directory, mode) 199 finally: 200 os.umask(old_umask) 201 return True
202 203 #---------------------------------------------------------------------------
204 -def create_directory_description_file(directory=None, readme=None, suffix=None):
205 assert (directory is not None), '<directory> must not be None' 206 207 README_fname = '.00-README.GNUmed' + coalesce(suffix, '.dir') 208 README_path = os.path.abspath(os.path.expanduser(os.path.join(directory, README_fname))) 209 _log.debug('%s', README_path) 210 if readme is None: 211 _log.debug('no README text, boilerplate only') 212 try: 213 README = open(README_path, mode = 'wt', encoding = 'utf8') 214 except Exception: 215 return False 216 217 line = 'GNUmed v%s -- %s' % (_client_version, pydt.datetime.now().strftime('%c')) 218 len_sep = len(line) 219 README.write(line) 220 README.write('\n') 221 line = README_path 222 len_sep = max(len_sep, len(line)) 223 README.write(line) 224 README.write('\n') 225 README.write('-' * len_sep) 226 README.write('\n') 227 README.write('\n') 228 README.write(readme) 229 README.write('\n') 230 README.close() 231 return True
232 233 #---------------------------------------------------------------------------
234 -def rmdir(directory):
235 #------------------------------- 236 def _on_rm_error(func, path, exc): 237 _log.error('error while shutil.rmtree(%s)', path, exc_info=exc) 238 return True
239 240 #------------------------------- 241 error_count = 0 242 try: 243 shutil.rmtree(directory, False, _on_rm_error) 244 except Exception: 245 _log.exception('cannot shutil.rmtree(%s)', directory) 246 error_count += 1 247 return error_count 248 249 #---------------------------------------------------------------------------
250 -def rm_dir_content(directory):
251 _log.debug('cleaning out [%s]', directory) 252 try: 253 items = os.listdir(directory) 254 except OSError: 255 return False 256 for item in items: 257 # attempt file/link removal and ignore (but log) errors 258 full_item = os.path.join(directory, item) 259 try: 260 os.remove(full_item) 261 except OSError: # as per the docs, this is a directory 262 _log.debug('[%s] seems to be a subdirectory', full_item) 263 errors = rmdir(full_item) 264 if errors > 0: 265 return False 266 except Exception: 267 _log.exception('cannot os.remove(%s) [a file or a link]', full_item) 268 return False 269 270 return True
271 272 #---------------------------------------------------------------------------
273 -def mk_sandbox_dir(prefix=None, base_dir=None):
274 if base_dir is None: 275 base_dir = gmPaths().tmp_dir 276 else: 277 if not os.path.isdir(base_dir): 278 mkdir(base_dir, mode = 0o0700) # (invoking user only) 279 if prefix is None: 280 prefix = 'sndbx-' 281 return tempfile.mkdtemp(prefix = prefix, suffix = '', dir = base_dir)
282 283 #---------------------------------------------------------------------------
284 -def parent_dir(directory):
285 return os.path.abspath(os.path.join(directory, '..'))
286 287 #---------------------------------------------------------------------------
288 -def dirname_stem(directory):
289 # /home/user/dir/ -> dir 290 # /home/user/dir -> dir 291 return os.path.basename(os.path.normpath(directory)) # normpath removes trailing slashes if any
292 293 #---------------------------------------------------------------------------
294 -def dir_is_empty(directory=None):
295 try: 296 empty = (len(os.listdir(directory)) == 0) 297 except OSError as exc: 298 if exc.errno != 2: # no such file 299 raise 300 empty = None 301 return empty
302 303 #---------------------------------------------------------------------------
304 -def copy_tree_content(directory, target_directory):
305 """Copy the *content* of <directory> *into* <target_directory> 306 which is created if need be. 307 """ 308 assert (directory is not None), 'source <directory> must not be None' 309 assert (target_directory is not None), '<target_directory> must not be None' 310 _log.debug('copying content of [%s] into [%s]', directory, target_directory) 311 try: 312 base_dir_items = os.listdir(directory) 313 except OSError: 314 _log.exception('cannot list dir [%s]', directory) 315 return None 316 317 for item in base_dir_items: 318 full_item = os.path.join(directory, item) 319 if os.path.isdir(full_item): 320 target_subdir = os.path.join(target_directory, item) 321 try: 322 shutil.copytree(full_item, target_subdir) 323 continue 324 except Exception: 325 _log.exception('cannot copy subdir [%s]', full_item) 326 return None 327 328 try: 329 shutil.copy2(full_item, target_directory) 330 except Exception: 331 _log.exception('cannot copy file [%s]', full_item) 332 return None 333 334 return target_directory
335 336 #--------------------------------------------------------------------------- 337 #---------------------------------------------------------------------------
338 -class gmPaths(gmBorg.cBorg):
339 """This class provides the following paths: 340 341 .home_dir user home 342 .local_base_dir script installation dir 343 .working_dir current dir 344 .user_config_dir 345 .system_config_dir 346 .system_app_data_dir (not writable) 347 .tmp_dir instance-local 348 .user_tmp_dir user-local (NOT per instance) 349 .bytea_cache_dir caches downloaded BYTEA data 350 """
351 - def __init__(self, app_name=None, wx=None):
352 """Setup pathes. 353 354 <app_name> will default to (name of the script - .py) 355 """ 356 try: 357 self.already_inited 358 return 359 except AttributeError: 360 pass 361 362 self.init_paths(app_name=app_name, wx=wx) 363 self.already_inited = True
364 365 #-------------------------------------- 366 # public API 367 #--------------------------------------
368 - def init_paths(self, app_name=None, wx=None):
369 370 if wx is None: 371 _log.debug('wxPython not available') 372 _log.debug('detecting paths directly') 373 374 if app_name is None: 375 app_name, ext = os.path.splitext(os.path.basename(sys.argv[0])) 376 _log.info('app name detected as [%s]', app_name) 377 else: 378 _log.info('app name passed in as [%s]', app_name) 379 380 # the user home, doesn't work in Wine so work around that 381 self.__home_dir = None 382 383 # where the main script (the "binary") is installed 384 if getattr(sys, 'frozen', False): 385 _log.info('frozen app, installed into temporary path') 386 # this would find the path of *THIS* file 387 #self.local_base_dir = os.path.dirname(__file__) 388 # while this is documented on the web, the ${_MEIPASS2} does not exist 389 #self.local_base_dir = os.environ.get('_MEIPASS2') 390 # this is what Martin Zibricky <mzibr.public@gmail.com> told us to use 391 # when asking about this on pyinstaller@googlegroups.com 392 #self.local_base_dir = sys._MEIPASS 393 # however, we are --onedir, so we should look at sys.executable 394 # as per the pyinstaller manual 395 self.local_base_dir = os.path.dirname(sys.executable) 396 else: 397 self.local_base_dir = os.path.abspath(os.path.dirname(sys.argv[0])) 398 399 # the current working dir at the OS 400 self.working_dir = os.path.abspath(os.curdir) 401 402 # user-specific config dir, usually below the home dir 403 mkdir(os.path.join(self.home_dir, '.%s' % app_name)) 404 self.user_config_dir = os.path.join(self.home_dir, '.%s' % app_name) 405 406 # system-wide config dir, usually below /etc/ under UN*X 407 try: 408 self.system_config_dir = os.path.join('/etc', app_name) 409 except ValueError: 410 #self.system_config_dir = self.local_base_dir 411 self.system_config_dir = self.user_config_dir 412 413 # system-wide application data dir 414 try: 415 self.system_app_data_dir = os.path.join(sys.prefix, 'share', app_name) 416 except ValueError: 417 self.system_app_data_dir = self.local_base_dir 418 419 # temporary directory 420 try: 421 self.__tmp_dir_already_set 422 _log.debug('temp dir already set') 423 except AttributeError: 424 _log.info('temp file prefix: %s', tempfile.gettempprefix()) 425 _log.info('initial (user level) temp dir: %s', tempfile.gettempdir()) 426 bytes_free = shutil.disk_usage(tempfile.gettempdir()).free 427 _log.info('free disk space for temp dir: %s (%s bytes)', size2str(size = bytes_free), bytes_free) 428 # $TMP/gnumed-$USER/ 429 self.user_tmp_dir = os.path.join(tempfile.gettempdir(), '%s-%s' % (app_name, getpass.getuser())) 430 mkdir(self.user_tmp_dir, 0o700) 431 _log.info('intermediate (app+user level) temp dir: %s', self.user_tmp_dir) 432 # $TMP/gnumed-$USER/g-$UNIQUE/ 433 tempfile.tempdir = self.user_tmp_dir # tell mkdtemp about intermediate dir 434 self.tmp_dir = tempfile.mkdtemp(prefix = 'g-') # will set tempfile.tempdir as side effect 435 _log.info('final (app instance level) temp dir: %s', tempfile.gettempdir()) 436 create_directory_description_file(directory = self.tmp_dir, readme = 'client instance tmp dir') 437 438 # BYTEA cache dir 439 cache_dir = os.path.join(self.user_tmp_dir, '.bytea_cache') 440 try: 441 stat = os.stat(cache_dir) 442 _log.warning('reusing BYTEA cache dir: %s', cache_dir) 443 _log.debug(stat) 444 except FileNotFoundError: 445 mkdir(cache_dir, mode = 0o0700) 446 self.bytea_cache_dir = cache_dir 447 create_directory_description_file(directory = self.bytea_cache_dir, readme = 'cache dir for BYTEA data') 448 449 self.__log_paths() 450 if wx is None: 451 return True 452 453 # retry with wxPython 454 _log.debug('re-detecting paths with wxPython') 455 456 std_paths = wx.StandardPaths.Get() 457 _log.info('wxPython app name is [%s]', wx.GetApp().GetAppName()) 458 459 # user-specific config dir, usually below the home dir 460 mkdir(os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name)) 461 self.user_config_dir = os.path.join(std_paths.GetUserConfigDir(), '.%s' % app_name) 462 463 # system-wide config dir, usually below /etc/ under UN*X 464 try: 465 tmp = std_paths.GetConfigDir() 466 if not tmp.endswith(app_name): 467 tmp = os.path.join(tmp, app_name) 468 self.system_config_dir = tmp 469 except ValueError: 470 # leave it at what it was from direct detection 471 pass 472 473 # system-wide application data dir 474 # Robin attests that the following doesn't always 475 # give sane values on Windows, so IFDEF it 476 if 'wxMSW' in wx.PlatformInfo: 477 _log.warning('this platform (wxMSW) sometimes returns a broken value for the system-wide application data dir') 478 else: 479 try: 480 self.system_app_data_dir = std_paths.GetDataDir() 481 except ValueError: 482 pass 483 484 self.__log_paths() 485 return True
486 487 #--------------------------------------
488 - def __log_paths(self):
489 _log.debug('sys.argv[0]: %s', sys.argv[0]) 490 _log.debug('sys.executable: %s', sys.executable) 491 _log.debug('sys._MEIPASS: %s', getattr(sys, '_MEIPASS', '<not found>')) 492 _log.debug('os.environ["_MEIPASS2"]: %s', os.environ.get('_MEIPASS2', '<not found>')) 493 _log.debug('__file__ : %s', __file__) 494 _log.debug('local application base dir: %s', self.local_base_dir) 495 _log.debug('current working dir: %s', self.working_dir) 496 _log.debug('user home dir: %s', self.home_dir) 497 _log.debug('user-specific config dir: %s', self.user_config_dir) 498 _log.debug('system-wide config dir: %s', self.system_config_dir) 499 _log.debug('system-wide application data dir: %s', self.system_app_data_dir) 500 _log.debug('temporary dir (user): %s', self.user_tmp_dir) 501 _log.debug('temporary dir (instance): %s', self.tmp_dir) 502 _log.debug('temporary dir (tempfile.tempdir): %s', tempfile.tempdir) 503 _log.debug('temporary dir (tempfile.gettempdir()): %s', tempfile.gettempdir()) 504 _log.debug('BYTEA cache dir: %s', self.bytea_cache_dir)
505 506 #-------------------------------------- 507 # properties 508 #--------------------------------------
509 - def _set_user_config_dir(self, path):
510 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 511 msg = '[%s:user_config_dir]: invalid path [%s]' % (self.__class__.__name__, path) 512 _log.error(msg) 513 raise ValueError(msg) 514 self.__user_config_dir = path
515
516 - def _get_user_config_dir(self):
517 return self.__user_config_dir
518 519 user_config_dir = property(_get_user_config_dir, _set_user_config_dir) 520 #--------------------------------------
521 - def _set_system_config_dir(self, path):
522 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 523 msg = '[%s:system_config_dir]: invalid path [%s]' % (self.__class__.__name__, path) 524 _log.error(msg) 525 raise ValueError(msg) 526 self.__system_config_dir = path
527
528 - def _get_system_config_dir(self):
529 return self.__system_config_dir
530 531 system_config_dir = property(_get_system_config_dir, _set_system_config_dir) 532 #--------------------------------------
533 - def _set_system_app_data_dir(self, path):
534 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 535 msg = '[%s:system_app_data_dir]: invalid path [%s]' % (self.__class__.__name__, path) 536 _log.error(msg) 537 raise ValueError(msg) 538 self.__system_app_data_dir = path
539
540 - def _get_system_app_data_dir(self):
541 return self.__system_app_data_dir
542 543 system_app_data_dir = property(_get_system_app_data_dir, _set_system_app_data_dir) 544 #--------------------------------------
545 - def _set_home_dir(self, path):
546 raise ValueError('invalid to set home dir')
547
548 - def _get_home_dir(self):
549 if self.__home_dir is not None: 550 return self.__home_dir 551 552 tmp = os.path.expanduser('~') 553 if tmp == '~': 554 _log.error('this platform does not expand ~ properly') 555 try: 556 tmp = os.environ['USERPROFILE'] 557 except KeyError: 558 _log.error('cannot access $USERPROFILE in environment') 559 560 if not ( 561 os.access(tmp, os.R_OK) 562 and 563 os.access(tmp, os.X_OK) 564 and 565 os.access(tmp, os.W_OK) 566 ): 567 msg = '[%s:home_dir]: invalid path [%s]' % (self.__class__.__name__, tmp) 568 _log.error(msg) 569 raise ValueError(msg) 570 571 self.__home_dir = tmp 572 return self.__home_dir
573 574 home_dir = property(_get_home_dir, _set_home_dir) 575 576 #--------------------------------------
577 - def _set_tmp_dir(self, path):
578 if not (os.access(path, os.R_OK) and os.access(path, os.X_OK)): 579 msg = '[%s:tmp_dir]: invalid path [%s]' % (self.__class__.__name__, path) 580 _log.error(msg) 581 raise ValueError(msg) 582 _log.debug('previous temp dir: %s', tempfile.gettempdir()) 583 self.__tmp_dir = path 584 tempfile.tempdir = self.__tmp_dir 585 _log.debug('new temp dir: %s', tempfile.gettempdir()) 586 self.__tmp_dir_already_set = True
587
588 - def _get_tmp_dir(self):
589 return self.__tmp_dir
590 591 tmp_dir = property(_get_tmp_dir, _set_tmp_dir)
592 593 #=========================================================================== 594 # file related tools 595 #---------------------------------------------------------------------------
596 -def recode_file(source_file=None, target_file=None, source_encoding='utf8', target_encoding=None, base_dir=None, error_mode='replace'):
597 if target_encoding is None: 598 return source_file 599 600 if target_encoding == source_encoding: 601 return source_file 602 603 if target_file is None: 604 target_file = get_unique_filename ( 605 prefix = '%s-%s_%s-' % (fname_stem(source_file), source_encoding, target_encoding), 606 suffix = fname_extension(source_file, '.txt'), 607 tmp_dir = base_dir 608 ) 609 _log.debug('[%s] -> [%s] (%s -> %s)', source_encoding, target_encoding, source_file, target_file) 610 in_file = io.open(source_file, mode = 'rt', encoding = source_encoding) 611 out_file = io.open(target_file, mode = 'wt', encoding = target_encoding, errors = error_mode) 612 for line in in_file: 613 out_file.write(line) 614 out_file.close() 615 in_file.close() 616 return target_file
617 618 #---------------------------------------------------------------------------
619 -def unzip_archive(archive_name, target_dir=None, remove_archive=False):
620 _log.debug('unzipping [%s] -> [%s]', archive_name, target_dir) 621 success = False 622 try: 623 with zipfile.ZipFile(archive_name) as archive: 624 archive.extractall(target_dir) 625 success = True 626 except Exception: 627 _log.exception('cannot unzip') 628 return False 629 if remove_archive: 630 remove_file(archive_name) 631 return success
632 633 #---------------------------------------------------------------------------
634 -def remove_file(filename, log_error=True, force=False):
635 if not os.path.lexists(filename): 636 return True 637 638 # attempt file removal and ignore (but log) errors 639 try: 640 os.remove(filename) 641 return True 642 643 except Exception: 644 if log_error: 645 _log.exception('cannot os.remove(%s)', filename) 646 647 if force: 648 tmp_name = get_unique_filename(tmp_dir = fname_dir(filename)) 649 _log.debug('attempting os.replace(%s -> %s)', filename, tmp_name) 650 try: 651 os.replace(filename, tmp_name) 652 return True 653 654 except Exception: 655 if log_error: 656 _log.exception('cannot os.replace(%s)', filename) 657 658 return False
659 660 #---------------------------------------------------------------------------
661 -def file2md5(filename=None, return_hex=True):
662 blocksize = 2**10 * 128 # 128k, since md5 uses 128 byte blocks 663 _log.debug('md5(%s): <%s> byte blocks', filename, blocksize) 664 f = io.open(filename, mode = 'rb') 665 md5 = hashlib.md5() 666 while True: 667 data = f.read(blocksize) 668 if not data: 669 break 670 md5.update(data) 671 f.close() 672 _log.debug('md5(%s): %s', filename, md5.hexdigest()) 673 if return_hex: 674 return md5.hexdigest() 675 676 return md5.digest()
677 678 #---------------------------------------------------------------------------
679 -def file2chunked_md5(filename=None, chunk_size=500*_MB):
680 _log.debug('chunked_md5(%s, chunk_size=%s bytes)', filename, chunk_size) 681 md5_concat = '' 682 f = open(filename, 'rb') 683 while True: 684 md5 = hashlib.md5() 685 data = f.read(chunk_size) 686 if not data: 687 break 688 md5.update(data) 689 md5_concat += md5.hexdigest() 690 f.close() 691 md5 = hashlib.md5() 692 md5.update(md5_concat) 693 hex_digest = md5.hexdigest() 694 _log.debug('md5("%s"): %s', md5_concat, hex_digest) 695 return hex_digest
696 697 #--------------------------------------------------------------------------- 698 default_csv_reader_rest_key = 'list_of_values_of_unknown_fields' 699
700 -def unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
701 try: 702 is_dict_reader = kwargs['dict'] 703 del kwargs['dict'] 704 except KeyError: 705 is_dict_reader = False 706 707 if is_dict_reader: 708 kwargs['restkey'] = default_csv_reader_rest_key 709 return csv.DictReader(unicode_csv_data, dialect=dialect, **kwargs) 710 return csv.reader(unicode_csv_data, dialect=dialect, **kwargs)
711 712 713 714
715 -def old_unicode2charset_encoder(unicode_csv_data, encoding='utf-8'):
716 for line in unicode_csv_data: 717 yield line.encode(encoding)
718 719 #def utf_8_encoder(unicode_csv_data): 720 # for line in unicode_csv_data: 721 # yield line.encode('utf-8') 722
723 -def old_unicode_csv_reader(unicode_csv_data, dialect=csv.excel, encoding='utf-8', **kwargs):
724 725 # csv.py doesn't do Unicode; encode temporarily as UTF-8: 726 try: 727 is_dict_reader = kwargs['dict'] 728 del kwargs['dict'] 729 if is_dict_reader is not True: 730 raise KeyError 731 kwargs['restkey'] = default_csv_reader_rest_key 732 csv_reader = csv.DictReader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs) 733 except KeyError: 734 is_dict_reader = False 735 csv_reader = csv.reader(unicode2charset_encoder(unicode_csv_data), dialect=dialect, **kwargs) 736 737 for row in csv_reader: 738 # decode ENCODING back to Unicode, cell by cell: 739 if is_dict_reader: 740 for key in row: 741 if key == default_csv_reader_rest_key: 742 old_data = row[key] 743 new_data = [] 744 for val in old_data: 745 new_data.append(str(val, encoding)) 746 row[key] = new_data 747 if default_csv_reader_rest_key not in csv_reader.fieldnames: 748 csv_reader.fieldnames.append(default_csv_reader_rest_key) 749 else: 750 row[key] = str(row[key], encoding) 751 yield row 752 else: 753 yield [ str(cell, encoding) for cell in row ]
754 #yield [str(cell, 'utf-8') for cell in row] 755 756 #---------------------------------------------------------------------------
757 -def fname_sanitize(filename):
758 """Normalizes unicode, removes non-alpha characters, converts spaces to underscores.""" 759 760 dir_part, name_part = os.path.split(filename) 761 if name_part == '': 762 return filename 763 764 import unicodedata 765 name_part = unicodedata.normalize('NFKD', name_part) 766 # remove everything not in group [] 767 name_part = regex.sub ( 768 '[^.\w\s[\]()%§+-]', 769 '', 770 name_part, 771 flags = regex.UNICODE 772 ).strip() 773 # translate whitespace to underscore 774 name_part = regex.sub ( 775 '\s+', 776 '_', 777 name_part, 778 flags = regex.UNICODE 779 ) 780 return os.path.join(dir_part, name_part)
781 782 #---------------------------------------------------------------------------
783 -def fname_stem(filename):
784 """/home/user/dir/filename.ext -> filename""" 785 return os.path.splitext(os.path.basename(filename))[0]
786 787 #---------------------------------------------------------------------------
788 -def fname_stem_with_path(filename):
789 """/home/user/dir/filename.ext -> /home/user/dir/filename""" 790 return os.path.splitext(filename)[0]
791 792 #---------------------------------------------------------------------------
793 -def fname_extension(filename=None, fallback=None):
794 """ /home/user/dir/filename.ext -> .ext 795 '' or '.' -> fallback if any else '' 796 """ 797 ext = os.path.splitext(filename)[1] 798 if ext.strip() not in ['.', '']: 799 return ext 800 if fallback is None: 801 return '' 802 return fallback
803 804 #---------------------------------------------------------------------------
805 -def fname_dir(filename):
806 # /home/user/dir/filename.ext -> /home/user/dir 807 return os.path.split(filename)[0]
808 809 #---------------------------------------------------------------------------
810 -def fname_from_path(filename):
811 # /home/user/dir/filename.ext -> filename.ext 812 return os.path.split(filename)[1]
813 814 #---------------------------------------------------------------------------
815 -def get_unique_filename(prefix=None, suffix=None, tmp_dir=None, include_timestamp=False):
816 """This function has a race condition between 817 its file.close() 818 and actually 819 using the filename in callers. 820 821 The file will NOT exist after calling this function. 822 """ 823 if tmp_dir is None: 824 gmPaths() # setup tmp dir if necessary 825 else: 826 if ( 827 not os.access(tmp_dir, os.F_OK) 828 or 829 not os.access(tmp_dir, os.X_OK | os.W_OK) 830 ): 831 _log.warning('cannot os.access() temporary dir [%s], using system default', tmp_dir) 832 tmp_dir = None 833 834 if include_timestamp: 835 ts = pydt.datetime.now().strftime('%m%d-%H%M%S-') 836 else: 837 ts = '' 838 839 kwargs = { 840 'dir': tmp_dir, 841 # make sure file gets deleted as soon as 842 # .close()d so we can "safely" open it again 843 'delete': True 844 } 845 846 if prefix is None: 847 kwargs['prefix'] = 'gm-%s' % ts 848 else: 849 kwargs['prefix'] = prefix + ts 850 851 if suffix in [None, '']: 852 kwargs['suffix'] = '.tmp' 853 else: 854 if not suffix.startswith('.'): 855 suffix = '.' + suffix 856 kwargs['suffix'] = suffix 857 858 f = tempfile.NamedTemporaryFile(**kwargs) 859 filename = f.name 860 f.close() 861 862 return filename
863 864 #--------------------------------------------------------------------------- 881 882 #--------------------------------------------------------------------------- 906 907 #===========================================================================
908 -def import_module_from_directory(module_path=None, module_name=None, always_remove_path=False):
909 """Import a module from any location.""" 910 911 _log.debug('CWD: %s', os.getcwd()) 912 913 remove_path = always_remove_path or False 914 if module_path not in sys.path: 915 _log.info('appending to sys.path: [%s]' % module_path) 916 sys.path.append(module_path) 917 remove_path = True 918 919 _log.debug('will remove import path: %s', remove_path) 920 921 if module_name.endswith('.py'): 922 module_name = module_name[:-3] 923 924 try: 925 module = __import__(module_name) 926 except Exception: 927 _log.exception('cannot __import__() module [%s] from [%s]' % (module_name, module_path)) 928 while module_path in sys.path: 929 sys.path.remove(module_path) 930 raise 931 932 _log.info('imported module [%s] as [%s]' % (module_name, module)) 933 if remove_path: 934 while module_path in sys.path: 935 sys.path.remove(module_path) 936 937 return module
938 939 #=========================================================================== 940 # text related tools 941 #---------------------------------------------------------------------------
942 -def size2str(size=0, template='%s'):
943 if size == 1: 944 return template % _('1 Byte') 945 if size < 10 * _kB: 946 return template % _('%s Bytes') % size 947 if size < _MB: 948 return template % '%.1f kB' % (float(size) / _kB) 949 if size < _GB: 950 return template % '%.1f MB' % (float(size) / _MB) 951 if size < _TB: 952 return template % '%.1f GB' % (float(size) / _GB) 953 if size < _PB: 954 return template % '%.1f TB' % (float(size) / _TB) 955 return template % '%.1f PB' % (float(size) / _PB)
956 957 #---------------------------------------------------------------------------
958 -def bool2subst(boolean=None, true_return=True, false_return=False, none_return=None):
959 if boolean is None: 960 return none_return 961 if boolean: 962 return true_return 963 if not boolean: 964 return false_return 965 raise ValueError('bool2subst(): <boolean> arg must be either of True, False, None')
966 967 #---------------------------------------------------------------------------
968 -def bool2str(boolean=None, true_str='True', false_str='False'):
969 return bool2subst ( 970 boolean = bool(boolean), 971 true_return = true_str, 972 false_return = false_str 973 )
974 975 #---------------------------------------------------------------------------
976 -def none_if(value=None, none_equivalent=None, strip_string=False):
977 """Modelled after the SQL NULLIF function.""" 978 if value is None: 979 return None 980 981 if strip_string: 982 stripped = value.strip() 983 else: 984 stripped = value 985 if stripped == none_equivalent: 986 return None 987 988 return value
989 990 #---------------------------------------------------------------------------
991 -def coalesce(value2test=None, return_instead=None, template4value=None, template4instead=None, none_equivalents=None, function4value=None, value2return=None):
992 """Modelled after the SQL coalesce function. 993 994 To be used to simplify constructs like: 995 996 if value2test is None (or in none_equivalents): 997 value = (template4instead % return_instead) or return_instead 998 else: 999 value = (template4value % value2test) or value2test 1000 print value 1001 1002 @param value2test: the value to be tested for <None> 1003 1004 @param return_instead: the value to be returned if <value2test> *is* None 1005 1006 @param template4value: if <value2test> is returned, replace the value into this template, must contain one <%s> 1007 1008 @param template4instead: if <return_instead> is returned, replace the value into this template, must contain one <%s> 1009 1010 @param value2return: a *value* to return if <value2test> is NOT None, AND there's no <template4value> 1011 1012 example: 1013 function4value = ('strftime', '%Y-%m-%d') 1014 1015 Ideas: 1016 - list of return_insteads: initial, [return_instead, template], [return_instead, template], [return_instead, template], template4value, ... 1017 """ 1018 if none_equivalents is None: 1019 none_equivalents = [None] 1020 1021 if value2test in none_equivalents: 1022 if template4instead is None: 1023 return return_instead 1024 return template4instead % return_instead 1025 1026 # at this point, value2test was not equivalent to None 1027 1028 # 1) explicit value to return supplied ? 1029 if value2return is not None: 1030 return value2return 1031 1032 value2return = value2test 1033 # 2) function supplied to be applied to the value ? 1034 if function4value is not None: 1035 funcname, args = function4value 1036 func = getattr(value2test, funcname) 1037 value2return = func(args) 1038 1039 # 3) template supplied to be applied to the value ? 1040 if template4value is None: 1041 return value2return 1042 1043 try: 1044 return template4value % value2return 1045 except TypeError: 1046 # except (TypeError, ValueError): 1047 # this should go, actually, only needed because "old" calls 1048 # to coalesce will still abuse template4value as explicit value2return, 1049 # relying on the replacement to above to fail 1050 if hasattr(_log, 'log_stack_trace'): 1051 _log.log_stack_trace(message = 'deprecated use of <template4value> for <value2return>') 1052 else: 1053 _log.error('deprecated use of <template4value> for <value2return>') 1054 _log.error(locals()) 1055 return template4value
1056 1057 #---------------------------------------------------------------------------
1058 -def __cap_name(match_obj=None):
1059 val = match_obj.group(0).lower() 1060 if val in ['von', 'van', 'de', 'la', 'l', 'der', 'den']: # FIXME: this needs to expand, configurable ? 1061 return val 1062 buf = list(val) 1063 buf[0] = buf[0].upper() 1064 for part in ['mac', 'mc', 'de', 'la']: 1065 if len(val) > len(part) and val[:len(part)] == part: 1066 buf[len(part)] = buf[len(part)].upper() 1067 return ''.join(buf)
1068 1069 #---------------------------------------------------------------------------
1070 -def capitalize(text=None, mode=CAPS_NAMES):
1071 """Capitalize the first character but leave the rest alone. 1072 1073 Note that we must be careful about the locale, this may 1074 have issues ! However, for UTF strings it should just work. 1075 """ 1076 if (mode is None) or (mode == CAPS_NONE): 1077 return text 1078 1079 if len(text) == 0: 1080 return text 1081 1082 if mode == CAPS_FIRST: 1083 if len(text) == 1: 1084 return text[0].upper() 1085 return text[0].upper() + text[1:] 1086 1087 if mode == CAPS_ALLCAPS: 1088 return text.upper() 1089 1090 if mode == CAPS_FIRST_ONLY: 1091 # if len(text) == 1: 1092 # return text[0].upper() 1093 return text[0].upper() + text[1:].lower() 1094 1095 if mode == CAPS_WORDS: 1096 #return regex.sub(ur'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text) 1097 return regex.sub(r'(\w)(\w+)', lambda x: x.group(1).upper() + x.group(2).lower(), text) 1098 1099 if mode == CAPS_NAMES: 1100 #return regex.sub(r'\w+', __cap_name, text) 1101 return capitalize(text=text, mode=CAPS_FIRST) # until fixed 1102 1103 print("ERROR: invalid capitalization mode: [%s], leaving input as is" % mode) 1104 return text
1105 1106 #---------------------------------------------------------------------------
1107 -def input2decimal(initial=None):
1108 1109 if isinstance(initial, decimal.Decimal): 1110 return True, initial 1111 1112 val = initial 1113 1114 # float ? -> to string first 1115 if type(val) == type(float(1.4)): 1116 val = str(val) 1117 1118 # string ? -> "," to "." 1119 if isinstance(val, str): 1120 val = val.replace(',', '.', 1) 1121 val = val.strip() 1122 1123 try: 1124 d = decimal.Decimal(val) 1125 return True, d 1126 except (TypeError, decimal.InvalidOperation): 1127 return False, val
1128 1129 #---------------------------------------------------------------------------
1130 -def input2int(initial=None, minval=None, maxval=None):
1131 1132 val = initial 1133 1134 # string ? -> "," to "." 1135 if isinstance(val, str): 1136 val = val.replace(',', '.', 1) 1137 val = val.strip() 1138 1139 try: 1140 int_val = int(val) 1141 except (TypeError, ValueError): 1142 _log.exception('int(%s) failed', val) 1143 return False, initial 1144 1145 if minval is not None: 1146 if int_val < minval: 1147 _log.debug('%s < min (%s)', val, minval) 1148 return False, initial 1149 if maxval is not None: 1150 if int_val > maxval: 1151 _log.debug('%s > max (%s)', val, maxval) 1152 return False, initial 1153 1154 return True, int_val
1155 1156 #---------------------------------------------------------------------------
1157 -def strip_prefix(text, prefix, remove_repeats=False, remove_whitespace=False):
1158 if remove_whitespace: 1159 text = text.lstrip() 1160 if not text.startswith(prefix): 1161 return text 1162 1163 text = text.replace(prefix, '', 1) 1164 if not remove_repeats: 1165 if remove_whitespace: 1166 return text.lstrip() 1167 return text 1168 1169 return strip_prefix(text, prefix, remove_repeats = True, remove_whitespace = remove_whitespace)
1170 1171 #---------------------------------------------------------------------------
1172 -def strip_suffix(text, suffix, remove_repeats=False, remove_whitespace=False):
1173 suffix_len = len(suffix) 1174 if remove_repeats: 1175 if remove_whitespace: 1176 while text.rstrip().endswith(suffix): 1177 text = text.rstrip()[:-suffix_len].rstrip() 1178 return text 1179 while text.endswith(suffix): 1180 text = text[:-suffix_len] 1181 return text 1182 if remove_whitespace: 1183 return text.rstrip()[:-suffix_len].rstrip() 1184 return text[:-suffix_len]
1185 1186 #---------------------------------------------------------------------------
1187 -def strip_leading_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1188 if lines is None: 1189 lines = text.split(eol) 1190 1191 while True: 1192 if lines[0].strip(eol).strip() != '': 1193 break 1194 lines = lines[1:] 1195 1196 if return_list: 1197 return lines 1198 1199 return eol.join(lines)
1200 1201 #---------------------------------------------------------------------------
1202 -def strip_trailing_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1203 if lines is None: 1204 lines = text.split(eol) 1205 1206 while True: 1207 if lines[-1].strip(eol).strip() != '': 1208 break 1209 lines = lines[:-1] 1210 1211 if return_list: 1212 return lines 1213 1214 return eol.join(lines)
1215 1216 #---------------------------------------------------------------------------
1217 -def strip_empty_lines(lines=None, text=None, eol='\n', return_list=True):
1218 return strip_trailing_empty_lines ( 1219 lines = strip_leading_empty_lines(lines = lines, text = text, eol = eol, return_list = True), 1220 text = None, 1221 eol = eol, 1222 return_list = return_list 1223 )
1224 1225 #---------------------------------------------------------------------------
1226 -def list2text(lines, initial_indent='', subsequent_indent='', eol='\n', strip_leading_empty_lines=True, strip_trailing_empty_lines=True, strip_trailing_whitespace=True, max_line_width=None):
1227 1228 if len(lines) == 0: 1229 return '' 1230 1231 if strip_leading_empty_lines: 1232 lines = strip_leading_empty_lines(lines = lines, eol = eol, return_list = True) 1233 1234 if strip_trailing_empty_lines: 1235 lines = strip_trailing_empty_lines(lines = lines, eol = eol, return_list = True) 1236 1237 if strip_trailing_whitespace: 1238 lines = [ l.rstrip() for l in lines ] 1239 1240 if max_line_width is not None: 1241 wrapped_lines = [] 1242 for l in lines: 1243 wrapped_lines.extend(wrap(l, max_line_width).split('\n')) 1244 lines = wrapped_lines 1245 1246 indented_lines = [initial_indent + lines[0]] 1247 indented_lines.extend([ subsequent_indent + l for l in lines[1:] ]) 1248 1249 return eol.join(indented_lines)
1250 1251 #---------------------------------------------------------------------------
1252 -def wrap(text=None, width=None, initial_indent='', subsequent_indent='', eol='\n'):
1253 """A word-wrap function that preserves existing line breaks 1254 and most spaces in the text. Expects that existing line 1255 breaks are posix newlines (\n). 1256 """ 1257 if width is None: 1258 return text 1259 1260 wrapped = initial_indent + functools.reduce ( 1261 lambda line, word, width=width: '%s%s%s' % ( 1262 line, 1263 ' \n'[(len(line) - line.rfind('\n') - 1 + len(word.split('\n',1)[0]) >= width)], 1264 word 1265 ), 1266 text.split(' ') 1267 ) 1268 if subsequent_indent != '': 1269 wrapped = ('\n%s' % subsequent_indent).join(wrapped.split('\n')) 1270 if eol != '\n': 1271 wrapped = wrapped.replace('\n', eol) 1272 return wrapped
1273 1274 #---------------------------------------------------------------------------
1275 -def unwrap(text=None, max_length=None, strip_whitespace=True, remove_empty_lines=True, line_separator = ' // '):
1276 1277 text = text.replace('\r', '') 1278 lines = text.split('\n') 1279 text = '' 1280 for line in lines: 1281 1282 if strip_whitespace: 1283 line = line.strip().strip('\t').strip() 1284 1285 if remove_empty_lines: 1286 if line == '': 1287 continue 1288 1289 text += ('%s%s' % (line, line_separator)) 1290 1291 text = text.rstrip(line_separator) 1292 1293 if max_length is not None: 1294 text = text[:max_length] 1295 1296 text = text.rstrip(line_separator) 1297 1298 return text
1299 1300 #---------------------------------------------------------------------------
1301 -def shorten_text(text=None, max_length=None):
1302 1303 if len(text) <= max_length: 1304 return text 1305 1306 return text[:max_length-1] + u_ellipsis
1307 1308 #---------------------------------------------------------------------------
1309 -def shorten_words_in_line(text=None, max_length=None, min_word_length=None, ignore_numbers=True, ellipsis=u_ellipsis):
1310 if text is None: 1311 return None 1312 if max_length is None: 1313 max_length = len(text) 1314 else: 1315 if len(text) <= max_length: 1316 return text 1317 old_words = regex.split('\s+', text, flags = regex.UNICODE) 1318 no_old_words = len(old_words) 1319 max_word_length = max(min_word_length, (max_length // no_old_words)) 1320 words = [] 1321 for word in old_words: 1322 if len(word) <= max_word_length: 1323 words.append(word) 1324 continue 1325 if ignore_numbers: 1326 tmp = word.replace('-', '').replace('+', '').replace('.', '').replace(',', '').replace('/', '').replace('&', '').replace('*', '') 1327 if tmp.isdigit(): 1328 words.append(word) 1329 continue 1330 words.append(word[:max_word_length] + ellipsis) 1331 return ' '.join(words)
1332 1333 #---------------------------------------------------------------------------
1334 -def xml_escape_string(text=None):
1335 """check for special XML characters and transform them""" 1336 return xml_tools.escape(text)
1337 1338 #---------------------------------------------------------------------------
1339 -def tex_escape_string(text=None, replace_known_unicode=True, replace_eol=False, keep_visual_eol=False):
1340 """Check for special TeX characters and transform them. 1341 1342 replace_eol: 1343 replaces "\n" with "\\newline" 1344 keep_visual_eol: 1345 replaces "\n" with "\\newline \n" such that 1346 both LaTeX will know to place a line break 1347 at this point as well as the visual formatting 1348 is preserved in the LaTeX source (think multi- 1349 row table cells) 1350 """ 1351 text = text.replace('\\', '\\textbackslash') # requires \usepackage{textcomp} in LaTeX source 1352 text = text.replace('^', '\\textasciicircum') 1353 text = text.replace('~', '\\textasciitilde') 1354 1355 text = text.replace('{', '\\{') 1356 text = text.replace('}', '\\}') 1357 text = text.replace('%', '\\%') 1358 text = text.replace('&', '\\&') 1359 text = text.replace('#', '\\#') 1360 text = text.replace('$', '\\$') 1361 text = text.replace('_', '\\_') 1362 if replace_eol: 1363 if keep_visual_eol: 1364 text = text.replace('\n', '\\newline \n') 1365 else: 1366 text = text.replace('\n', '\\newline ') 1367 1368 if replace_known_unicode: 1369 # this should NOT be replaced for Xe(La)Tex 1370 text = text.replace(u_euro, '\\EUR') # requires \usepackage{textcomp} in LaTeX source 1371 text = text.replace(u_sum, '$\\Sigma$') 1372 1373 return text
1374 1375 #---------------------------------------------------------------------------
1376 -def rst2latex_snippet(rst_text):
1377 global du_core 1378 if du_core is None: 1379 try: 1380 from docutils import core as du_core 1381 except ImportError: 1382 _log.warning('cannot turn ReST into LaTeX: docutils not installed') 1383 return tex_escape_string(text = rst_text) 1384 1385 parts = du_core.publish_parts ( 1386 source = rst_text.replace('\\', '\\\\'), 1387 source_path = '<internal>', 1388 writer_name = 'latex', 1389 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex', 1390 settings_overrides = { 1391 'input_encoding': 'unicode' # un-encoded unicode 1392 }, 1393 enable_exit_status = True # how to use ? 1394 ) 1395 return parts['body']
1396 1397 #---------------------------------------------------------------------------
1398 -def rst2html(rst_text, replace_eol=False, keep_visual_eol=False):
1399 global du_core 1400 if du_core is None: 1401 try: 1402 from docutils import core as du_core 1403 except ImportError: 1404 _log.warning('cannot turn ReST into HTML: docutils not installed') 1405 return html_escape_string(text = rst_text, replace_eol=False, keep_visual_eol=False) 1406 1407 parts = du_core.publish_parts ( 1408 source = rst_text.replace('\\', '\\\\'), 1409 source_path = '<internal>', 1410 writer_name = 'latex', 1411 #destination_path = '/path/to/LaTeX-template/for/calculating/relative/links/template.tex', 1412 settings_overrides = { 1413 'input_encoding': 'unicode' # un-encoded unicode 1414 }, 1415 enable_exit_status = True # how to use ? 1416 ) 1417 return parts['body']
1418 1419 #---------------------------------------------------------------------------
1420 -def xetex_escape_string(text=None):
1421 # a web search did not reveal anything else for Xe(La)Tex 1422 # as opposed to LaTeX, except true unicode chars 1423 return tex_escape_string(text = text, replace_known_unicode = False)
1424 1425 #--------------------------------------------------------------------------- 1426 __html_escape_table = { 1427 "&": "&amp;", 1428 '"': "&quot;", 1429 "'": "&apos;", 1430 ">": "&gt;", 1431 "<": "&lt;", 1432 } 1433
1434 -def html_escape_string(text=None, replace_eol=False, keep_visual_eol=False):
1435 text = ''.join(__html_escape_table.get(char, char) for char in text) 1436 if replace_eol: 1437 if keep_visual_eol: 1438 text = text.replace('\n', '<br>\n') 1439 else: 1440 text = text.replace('\n', '<br>') 1441 return text
1442 1443 #---------------------------------------------------------------------------
1444 -def dict2json(obj):
1445 return json.dumps(obj, default = json_serialize)
1446 1447 #---------------------------------------------------------------------------
1448 -def json_serialize(obj):
1449 if isinstance(obj, pydt.datetime): 1450 return obj.isoformat() 1451 raise TypeError('cannot json_serialize(%s)' % type(obj))
1452 1453 #--------------------------------------------------------------------------- 1454 #---------------------------------------------------------------------------
1455 -def compare_dict_likes(d1, d2, title1=None, title2=None):
1456 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title1, '', '"%s" '), type(d1), coalesce(title2, '', '"%s" '), type(d2)) 1457 try: 1458 d1 = dict(d1) 1459 except TypeError: 1460 pass 1461 try: 1462 d2 = dict(d2) 1463 except TypeError: 1464 pass 1465 keys_d1 = list(d1) 1466 keys_d2 = list(d2) 1467 different = False 1468 if len(keys_d1) != len(keys_d2): 1469 _log.info('different number of keys: %s vs %s', len(keys_d1), len(keys_d2)) 1470 different = True 1471 for key in keys_d1: 1472 if key in keys_d2: 1473 if type(d1[key]) != type(d2[key]): 1474 _log.info('%25.25s: type(dict1) = %s = >>>%s<<<' % (key, type(d1[key]), d1[key])) 1475 _log.info('%25.25s type(dict2) = %s = >>>%s<<<' % ('', type(d2[key]), d2[key])) 1476 different = True 1477 continue 1478 if d1[key] == d2[key]: 1479 _log.info('%25.25s: both = >>>%s<<<' % (key, d1[key])) 1480 else: 1481 _log.info('%25.25s: dict1 = >>>%s<<<' % (key, d1[key])) 1482 _log.info('%25.25s dict2 = >>>%s<<<' % ('', d2[key])) 1483 different = True 1484 else: 1485 _log.info('%25.25s: %50.50s | <MISSING>' % (key, '>>>%s<<<' % d1[key])) 1486 different = True 1487 for key in keys_d2: 1488 if key in keys_d1: 1489 continue 1490 _log.info('%25.25s: %50.50s | %.50s' % (key, '<MISSING>', '>>>%s<<<' % d2[key])) 1491 different = True 1492 if different: 1493 _log.info('dict-likes appear to be different from each other') 1494 return False 1495 _log.info('dict-likes appear equal to each other') 1496 return True
1497 1498 #---------------------------------------------------------------------------
1499 -def format_dict_likes_comparison(d1, d2, title_left=None, title_right=None, left_margin=0, key_delim=' || ', data_delim=' | ', missing_string='=/=', difference_indicator='! ', ignore_diff_in_keys=None):
1500 1501 _log.info('comparing dict-likes: %s[%s] vs %s[%s]', coalesce(title_left, '', '"%s" '), type(d1), coalesce(title_right, '', '"%s" '), type(d2)) 1502 append_type = False 1503 if None not in [title_left, title_right]: 1504 append_type = True 1505 type_left = type(d1) 1506 type_right = type(d2) 1507 if title_left is None: 1508 title_left = '%s' % type_left 1509 if title_right is None: 1510 title_right = '%s' % type_right 1511 1512 try: d1 = dict(d1) 1513 except TypeError: pass 1514 try: d2 = dict(d2) 1515 except TypeError: pass 1516 keys_d1 = list(d1) 1517 keys_d2 = list(d2) 1518 data = {} 1519 for key in keys_d1: 1520 data[key] = [d1[key], ' '] 1521 if key in d2: 1522 data[key][1] = d2[key] 1523 for key in keys_d2: 1524 if key in keys_d1: 1525 continue 1526 data[key] = [' ', d2[key]] 1527 max1 = max([ len('%s' % k) for k in keys_d1 ]) 1528 max2 = max([ len('%s' % k) for k in keys_d2 ]) 1529 max_len = max(max1, max2, len(_('<type>'))) 1530 max_key_len_str = '%' + '%s.%s' % (max_len, max_len) + 's' 1531 max1 = max([ len('%s' % d1[k]) for k in keys_d1 ]) 1532 max2 = max([ len('%s' % d2[k]) for k in keys_d2 ]) 1533 max_data_len = min(max(max1, max2), 100) 1534 max_data_len_str = '%' + '%s.%s' % (max_data_len, max_data_len) + 's' 1535 diff_indicator_len_str = '%' + '%s.%s' % (len(difference_indicator), len(difference_indicator)) + 's' 1536 line_template = (' ' * left_margin) + diff_indicator_len_str + max_key_len_str + key_delim + max_data_len_str + data_delim + '%s' 1537 1538 lines = [] 1539 # debugging: 1540 #lines.append(u' (40 regular spaces)') 1541 #lines.append((u' ' * 40) + u"(u' ' * 40)") 1542 #lines.append((u'%40.40s' % u'') + u"(u'%40.40s' % u'')") 1543 #lines.append((u'%40.40s' % u' ') + u"(u'%40.40s' % u' ')") 1544 #lines.append((u'%40.40s' % u'.') + u"(u'%40.40s' % u'.')") 1545 #lines.append(line_template) 1546 lines.append(line_template % ('', '', title_left, title_right)) 1547 if append_type: 1548 lines.append(line_template % ('', _('<type>'), type_left, type_right)) 1549 1550 if ignore_diff_in_keys is None: 1551 ignore_diff_in_keys = [] 1552 1553 for key in keys_d1: 1554 append_type = False 1555 txt_left_col = '%s' % d1[key] 1556 try: 1557 txt_right_col = '%s' % d2[key] 1558 if type(d1[key]) != type(d2[key]): 1559 append_type = True 1560 except KeyError: 1561 txt_right_col = missing_string 1562 lines.append(line_template % ( 1563 bool2subst ( 1564 ((txt_left_col == txt_right_col) or (key in ignore_diff_in_keys)), 1565 '', 1566 difference_indicator 1567 ), 1568 key, 1569 shorten_text(txt_left_col, max_data_len), 1570 shorten_text(txt_right_col, max_data_len) 1571 )) 1572 if append_type: 1573 lines.append(line_template % ( 1574 '', 1575 _('<type>'), 1576 shorten_text('%s' % type(d1[key]), max_data_len), 1577 shorten_text('%s' % type(d2[key]), max_data_len) 1578 )) 1579 1580 for key in keys_d2: 1581 if key in keys_d1: 1582 continue 1583 lines.append(line_template % ( 1584 bool2subst((key in ignore_diff_in_keys), '', difference_indicator), 1585 key, 1586 shorten_text(missing_string, max_data_len), 1587 shorten_text('%s' % d2[key], max_data_len) 1588 )) 1589 1590 return lines
1591 1592 #---------------------------------------------------------------------------
1593 -def format_dict_like(d, relevant_keys=None, template=None, missing_key_template='<[%(key)s] MISSING>', left_margin=0, tabular=False, value_delimiters=('>>>', '<<<'), eol='\n', values2ignore=None):
1594 if values2ignore is None: 1595 values2ignore = [] 1596 if template is not None: 1597 # all keys in template better exist in d 1598 try: 1599 return template % d 1600 except KeyError: 1601 # or else 1602 _log.exception('template contains %%()s key(s) which do not exist in data dict') 1603 # try to extend dict <d> to contain all required keys, 1604 # for that to work <relevant_keys> better list all 1605 # keys used in <template> 1606 if relevant_keys is not None: 1607 for key in relevant_keys: 1608 try: 1609 d[key] 1610 except KeyError: 1611 d[key] = missing_key_template % {'key': key} 1612 return template % d 1613 1614 if relevant_keys is None: 1615 relevant_keys = list(d) 1616 lines = [] 1617 if value_delimiters is None: 1618 delim_left = '' 1619 delim_right = '' 1620 else: 1621 delim_left, delim_right = value_delimiters 1622 if tabular: 1623 max_len = max([ len('%s' % k) for k in relevant_keys ]) 1624 max_len_str = '%s.%s' % (max_len, max_len) 1625 line_template = (' ' * left_margin) + '%' + max_len_str + ('s: %s%%s%s' % (delim_left, delim_right)) 1626 else: 1627 line_template = (' ' * left_margin) + '%%s: %s%%s%s' % (delim_left, delim_right) 1628 for key in relevant_keys: 1629 try: 1630 val = d[key] 1631 except KeyError: 1632 continue 1633 if val not in values2ignore: 1634 lines.append(line_template % (key, val)) 1635 if eol is None: 1636 return lines 1637 return eol.join(lines)
1638 1639 #---------------------------------------------------------------------------
1640 -def dicts2table(dict_list, left_margin=0, eol='\n', keys2ignore=None, column_labels=None, show_only_changes=False, equality_value='<=>', date_format=None): #, relevant_keys=None, template=None
1641 """Each dict in <dict_list> becomes a column. 1642 1643 - each key of dict becomes a row label, unless in keys2ignore 1644 1645 - each entry in the <column_labels> list becomes a column title 1646 """ 1647 keys2show = [] 1648 col_max_width = {} 1649 max_width_of_row_label_col = 0 1650 col_label_key = '__________#header#__________' 1651 if keys2ignore is None: 1652 keys2ignore = [] 1653 if column_labels is not None: 1654 keys2ignore.append(col_label_key) 1655 1656 # extract keys from all dicts and calculate column sizes 1657 for dict_idx in range(len(dict_list)): 1658 # convert potentially dict-*like* into dict 1659 d = dict(dict_list[dict_idx]) 1660 # add max-len column label row from <column_labels> list, if available 1661 if column_labels is not None: 1662 d[col_label_key] = max(column_labels[dict_idx].split('\n'), key = len) 1663 field_lengths = [] 1664 # loop over all keys in this dict 1665 for key in d: 1666 # ignore this key 1667 if key in keys2ignore: 1668 continue 1669 # remember length of value when displayed 1670 if isinstance(d[key], pydt.datetime): 1671 if date_format is None: 1672 field_lengths.append(len('%s' % d[key])) 1673 else: 1674 field_lengths.append(len(d[key].strftime(date_format))) 1675 else: 1676 field_lengths.append(len('%s' % d[key])) 1677 if key in keys2show: 1678 continue 1679 keys2show.append(key) 1680 max_width_of_row_label_col = max(max_width_of_row_label_col, len('%s' % key)) 1681 col_max_width[dict_idx] = max(field_lengths) 1682 1683 # pivot data into dict of lists per line 1684 lines = { k: [] for k in keys2show } 1685 prev_vals = {} 1686 for dict_idx in range(len(dict_list)): 1687 max_width_this_col = max(col_max_width[dict_idx], len(equality_value)) if show_only_changes else col_max_width[dict_idx] 1688 max_len_str = '%s.%s' % (max_width_this_col, max_width_this_col) 1689 field_template = ' %' + max_len_str + 's' 1690 d = dict_list[dict_idx] 1691 for key in keys2show: 1692 try: 1693 val = d[key] 1694 except KeyError: 1695 lines[key].append(field_template % _('<missing>')) 1696 continue 1697 if isinstance(val, pydt.datetime): 1698 if date_format is not None: 1699 val = val.strftime(date_format) 1700 lines[key].append(field_template % val) 1701 if show_only_changes: 1702 if key not in prev_vals: 1703 prev_vals[key] = '%s' % lines[key][-1] 1704 continue 1705 if lines[key][-1] != prev_vals[key]: 1706 prev_vals[key] = '%s' % lines[key][-1] 1707 continue 1708 lines[key][-1] = field_template % equality_value 1709 1710 # format data into table 1711 table_lines = [] 1712 max_len_str = '%s.%s' % (max_width_of_row_label_col, max_width_of_row_label_col) 1713 row_label_template = '%' + max_len_str + 's' 1714 for key in lines: 1715 # row label (= key) into first column 1716 line = (' ' * left_margin) + row_label_template % key + '|' 1717 # append list values as subsequent columns 1718 line += '|'.join(lines[key]) 1719 table_lines.append(line) 1720 1721 # insert lines with column labels (column headers) if any 1722 if column_labels is not None: 1723 # first column contains row labels, so no column label needed 1724 table_header_line_w_col_labels = (' ' * left_margin) + row_label_template % '' 1725 # second table header line: horizontal separator 1726 table_header_line_w_separator = (' ' * left_margin) + u_box_horiz_single * (max_width_of_row_label_col) 1727 max_col_label_widths = [ max(col_max_width[dict_idx], len(equality_value)) for dict_idx in range(len(dict_list)) ] 1728 for col_idx in range(len(column_labels)): 1729 max_len_str = '%s.%s' % (max_col_label_widths[col_idx], max_col_label_widths[col_idx]) 1730 col_label_template = '%' + max_len_str + 's' 1731 table_header_line_w_col_labels += '| ' 1732 table_header_line_w_col_labels += col_label_template % column_labels[col_idx] 1733 table_header_line_w_separator += '%s%s' % (u_box_plus, u_box_horiz_single) 1734 table_header_line_w_separator += u_box_horiz_single * max_col_label_widths[col_idx] 1735 table_lines.insert(0, table_header_line_w_separator) 1736 table_lines.insert(0, table_header_line_w_col_labels) 1737 1738 if eol is None: 1739 return table_lines 1740 1741 return ('|' + eol).join(table_lines) + '|' + eol 1742 1743 #---------------------------------------------------------------------------
1744 -def normalize_dict_like(d, required_keys, missing_key_template='<[%(key)s] MISSING>'):
1745 for key in required_keys: 1746 try: 1747 d[key] 1748 except KeyError: 1749 if missing_key_template is None: 1750 d[key] = None 1751 else: 1752 d[key] = missing_key_template % {'key': key} 1753 return d
1754 1755 #---------------------------------------------------------------------------
1756 -def enumerate_removable_partitions():
1757 try: 1758 import pyudev 1759 import psutil 1760 except ImportError: 1761 _log.error('pyudev and/or psutil not installed') 1762 return {} 1763 1764 removable_partitions = {} 1765 ctxt = pyudev.Context() 1766 removable_devices = [ dev for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk') if dev.attributes.get('removable') == b'1' ] 1767 all_mounted_partitions = { part.device: part for part in psutil.disk_partitions() } 1768 for device in removable_devices: 1769 _log.debug('removable device: %s', device.properties['ID_MODEL']) 1770 partitions_on_removable_device = { 1771 part.device_node: { 1772 'type': device.properties['ID_TYPE'], 1773 'bus': device.properties['ID_BUS'], 1774 'device': device.properties['DEVNAME'], 1775 'partition': part.properties['DEVNAME'], 1776 'vendor': part.properties['ID_VENDOR'], 1777 'model': part.properties['ID_MODEL'], 1778 'fs_label': part.properties['ID_FS_LABEL'], 1779 'is_mounted': False, 1780 'mountpoint': None, 1781 'fs_type': None, 1782 'size_in_bytes': -1, 1783 'bytes_free': 0 1784 } for part in ctxt.list_devices(subsystem='block', DEVTYPE='partition', parent=device) 1785 } 1786 for part in partitions_on_removable_device: 1787 try: 1788 partitions_on_removable_device[part]['mountpoint'] = all_mounted_partitions[part].mountpoint 1789 partitions_on_removable_device[part]['is_mounted'] = True 1790 partitions_on_removable_device[part]['fs_type'] = all_mounted_partitions[part].fstype 1791 du = shutil.disk_usage(all_mounted_partitions[part].mountpoint) 1792 partitions_on_removable_device[part]['size_in_bytes'] = du.total 1793 partitions_on_removable_device[part]['bytes_free'] = du.free 1794 except KeyError: 1795 pass # not mounted 1796 removable_partitions.update(partitions_on_removable_device) 1797 return removable_partitions
1798 1799 # debugging: 1800 #ctxt = pyudev.Context() 1801 #for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk'):# if dev.attributes.get('removable') == b'1': 1802 # for a in dev.attributes.available_attributes: 1803 # print(a, dev.attributes.get(a)) 1804 # for key, value in dev.items(): 1805 # print('{key}={value}'.format(key=key, value=value)) 1806 # print('---------------------------') 1807 1808 #---------------------------------------------------------------------------
1809 -def enumerate_optical_writers():
1810 try: 1811 import pyudev 1812 except ImportError: 1813 _log.error('pyudev not installed') 1814 return [] 1815 1816 optical_writers = [] 1817 ctxt = pyudev.Context() 1818 for dev in [ dev for dev in ctxt.list_devices(subsystem='block', DEVTYPE='disk') if dev.properties.get('ID_CDROM_CD_RW', None) == '1' ]: 1819 optical_writers.append ({ 1820 'type': dev.properties['ID_TYPE'], 1821 'bus': dev.properties['ID_BUS'], 1822 'device': dev.properties['DEVNAME'], 1823 'model': dev.properties['ID_MODEL'] 1824 }) 1825 return optical_writers
1826 1827 #--------------------------------------------------------------------------- 1828 #---------------------------------------------------------------------------
1829 -def prompted_input(prompt=None, default=None):
1830 """Obtains entry from standard input. 1831 1832 prompt: Prompt text to display in standard output 1833 default: Default value (for user to press enter only) 1834 CTRL-C: aborts and returns None 1835 """ 1836 if prompt is None: 1837 msg = '(CTRL-C aborts)' 1838 else: 1839 msg = '%s (CTRL-C aborts)' % prompt 1840 1841 if default is None: 1842 msg = msg + ': ' 1843 else: 1844 msg = '%s [%s]: ' % (msg, default) 1845 1846 try: 1847 usr_input = input(msg) 1848 except KeyboardInterrupt: 1849 return None 1850 1851 if usr_input == '': 1852 return default 1853 1854 return usr_input
1855 1856 #=========================================================================== 1857 # image handling tools 1858 #--------------------------------------------------------------------------- 1859 # builtin (ugly but tried and true) fallback icon 1860 __icon_serpent = \ 1861 """x\xdae\x8f\xb1\x0e\x83 \x10\x86w\x9f\xe2\x92\x1blb\xf2\x07\x96\xeaH:0\xd6\ 1862 \xc1\x85\xd5\x98N5\xa5\xef?\xf5N\xd0\x8a\xdcA\xc2\xf7qw\x84\xdb\xfa\xb5\xcd\ 1863 \xd4\xda;\xc9\x1a\xc8\xb6\xcd<\xb5\xa0\x85\x1e\xeb\xbc\xbc7b!\xf6\xdeHl\x1c\ 1864 \x94\x073\xec<*\xf7\xbe\xf7\x99\x9d\xb21~\xe7.\xf5\x1f\x1c\xd3\xbdVlL\xc2\ 1865 \xcf\xf8ye\xd0\x00\x90\x0etH \x84\x80B\xaa\x8a\x88\x85\xc4(U\x9d$\xfeR;\xc5J\ 1866 \xa6\x01\xbbt9\xceR\xc8\x81e_$\x98\xb9\x9c\xa9\x8d,y\xa9t\xc8\xcf\x152\xe0x\ 1867 \xe9$\xf5\x07\x95\x0cD\x95t:\xb1\x92\xae\x9cI\xa8~\x84\x1f\xe0\xa3ec""" 1868
1869 -def get_icon(wx=None):
1870 1871 paths = gmPaths(app_name = 'gnumed', wx = wx) 1872 1873 candidates = [ 1874 os.path.join(paths.system_app_data_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'), 1875 os.path.join(paths.local_base_dir, 'bitmaps', 'gm_icon-serpent_and_gnu.png'), 1876 os.path.join(paths.system_app_data_dir, 'bitmaps', 'serpent.png'), 1877 os.path.join(paths.local_base_dir, 'bitmaps', 'serpent.png') 1878 ] 1879 1880 found_as = None 1881 for candidate in candidates: 1882 try: 1883 open(candidate, 'r').close() 1884 found_as = candidate 1885 break 1886 except IOError: 1887 _log.debug('icon not found in [%s]', candidate) 1888 1889 if found_as is None: 1890 _log.warning('no icon file found, falling back to builtin (ugly) icon') 1891 icon_bmp_data = wx.BitmapFromXPMData(pickle.loads(zlib.decompress(__icon_serpent))) 1892 icon.CopyFromBitmap(icon_bmp_data) 1893 else: 1894 _log.debug('icon found in [%s]', found_as) 1895 icon = wx.Icon() 1896 try: 1897 icon.LoadFile(found_as, wx.BITMAP_TYPE_ANY) #_PNG 1898 except AttributeError: 1899 _log.exception("this platform doesn't support wx.Icon().LoadFile()") 1900 1901 return icon
1902 1903 #---------------------------------------------------------------------------
1904 -def create_qrcode(text=None, filename=None, qr_filename=None, verbose=False):
1905 assert (not ((text is None) and (filename is None))), 'either <text> or <filename> must be specified' 1906 1907 try: 1908 import pyqrcode 1909 except ImportError: 1910 _log.exception('cannot import <pyqrcode>') 1911 return None 1912 if text is None: 1913 with io.open(filename, mode = 'rt', encoding = 'utf8') as input_file: 1914 text = input_file.read() 1915 if qr_filename is None: 1916 if filename is None: 1917 qr_filename = get_unique_filename(prefix = 'gm-qr-', suffix = '.png') 1918 else: 1919 qr_filename = get_unique_filename ( 1920 prefix = fname_stem(filename) + '-', 1921 suffix = fname_extension(filename) + '.png' 1922 ) 1923 _log.debug('[%s] -> [%s]', filename, qr_filename) 1924 qr = pyqrcode.create(text, encoding = 'utf8') 1925 if verbose: 1926 print('input file:', filename) 1927 print('output file:', qr_filename) 1928 print('text to encode:', text) 1929 print(qr.terminal()) 1930 qr.png(qr_filename, quiet_zone = 1) 1931 return qr_filename
1932 1933 #=========================================================================== 1934 # main 1935 #--------------------------------------------------------------------------- 1936 if __name__ == '__main__': 1937 1938 if len(sys.argv) < 2: 1939 sys.exit() 1940 1941 if sys.argv[1] != 'test': 1942 sys.exit() 1943 1944 # for testing: 1945 logging.basicConfig(level = logging.DEBUG) 1946 from Gnumed.pycommon import gmI18N 1947 gmI18N.activate_locale() 1948 gmI18N.install_domain() 1949 1950 #-----------------------------------------------------------------------
1951 - def test_input2decimal():
1952 1953 tests = [ 1954 [None, False], 1955 1956 ['', False], 1957 [' 0 ', True, 0], 1958 1959 [0, True, 0], 1960 [0.0, True, 0], 1961 [.0, True, 0], 1962 ['0', True, 0], 1963 ['0.0', True, 0], 1964 ['0,0', True, 0], 1965 ['00.0', True, 0], 1966 ['.0', True, 0], 1967 [',0', True, 0], 1968 1969 [0.1, True, decimal.Decimal('0.1')], 1970 [.01, True, decimal.Decimal('0.01')], 1971 ['0.1', True, decimal.Decimal('0.1')], 1972 ['0,1', True, decimal.Decimal('0.1')], 1973 ['00.1', True, decimal.Decimal('0.1')], 1974 ['.1', True, decimal.Decimal('0.1')], 1975 [',1', True, decimal.Decimal('0.1')], 1976 1977 [1, True, 1], 1978 [1.0, True, 1], 1979 ['1', True, 1], 1980 ['1.', True, 1], 1981 ['1,', True, 1], 1982 ['1.0', True, 1], 1983 ['1,0', True, 1], 1984 ['01.0', True, 1], 1985 ['01,0', True, 1], 1986 [' 01, ', True, 1], 1987 1988 [decimal.Decimal('1.1'), True, decimal.Decimal('1.1')] 1989 ] 1990 for test in tests: 1991 conversion_worked, result = input2decimal(initial = test[0]) 1992 1993 expected2work = test[1] 1994 1995 if conversion_worked: 1996 if expected2work: 1997 if result == test[2]: 1998 continue 1999 else: 2000 print("ERROR (conversion result wrong): >%s<, expected >%s<, got >%s<" % (test[0], test[2], result)) 2001 else: 2002 print("ERROR (conversion worked but was expected to fail): >%s<, got >%s<" % (test[0], result)) 2003 else: 2004 if not expected2work: 2005 continue 2006 else: 2007 print("ERROR (conversion failed but was expected to work): >%s<, expected >%s<" % (test[0], test[2]))
2008 #-----------------------------------------------------------------------
2009 - def test_input2int():
2010 print(input2int(0)) 2011 print(input2int('0')) 2012 print(input2int('0', 0, 0))
2013 #-----------------------------------------------------------------------
2014 - def test_coalesce():
2015 2016 val = None 2017 print(val, coalesce(val, 'is None', 'is not None')) 2018 val = 1 2019 print(val, coalesce(val, 'is None', 'is not None')) 2020 return 2021 2022 import datetime as dt 2023 print(coalesce(value2test = dt.datetime.now(), template4value = '-- %s --', function4value = ('strftime', '%Y-%m-%d'))) 2024 2025 print('testing coalesce()') 2026 print("------------------") 2027 tests = [ 2028 [None, 'something other than <None>', None, None, 'something other than <None>'], 2029 ['Captain', 'Mr.', '%s.'[:4], 'Mr.', 'Capt.'], 2030 ['value to test', 'test 3 failed', 'template with "%s" included', None, 'template with "value to test" included'], 2031 ['value to test', 'test 4 failed', 'template with value not included', None, 'template with value not included'], 2032 [None, 'initial value was None', 'template4value: %s', None, 'initial value was None'], 2033 [None, 'initial value was None', 'template4value: %%(abc)s', None, 'initial value was None'] 2034 ] 2035 passed = True 2036 for test in tests: 2037 result = coalesce ( 2038 value2test = test[0], 2039 return_instead = test[1], 2040 template4value = test[2], 2041 template4instead = test[3] 2042 ) 2043 if result != test[4]: 2044 print("ERROR") 2045 print("coalesce: (%s, %s, %s, %s)" % (test[0], test[1], test[2], test[3])) 2046 print("expected:", test[4]) 2047 print("received:", result) 2048 passed = False 2049 2050 if passed: 2051 print("passed") 2052 else: 2053 print("failed") 2054 return passed
2055 #-----------------------------------------------------------------------
2056 - def test_capitalize():
2057 print('testing capitalize() ...') 2058 success = True 2059 pairs = [ 2060 # [original, expected result, CAPS mode] 2061 ['Boot', 'Boot', CAPS_FIRST_ONLY], 2062 ['boot', 'Boot', CAPS_FIRST_ONLY], 2063 ['booT', 'Boot', CAPS_FIRST_ONLY], 2064 ['BoOt', 'Boot', CAPS_FIRST_ONLY], 2065 ['boots-Schau', 'Boots-Schau', CAPS_WORDS], 2066 ['boots-sChau', 'Boots-Schau', CAPS_WORDS], 2067 ['boot camp', 'Boot Camp', CAPS_WORDS], 2068 ['fahrner-Kampe', 'Fahrner-Kampe', CAPS_NAMES], 2069 ['häkkönen', 'Häkkönen', CAPS_NAMES], 2070 ['McBurney', 'McBurney', CAPS_NAMES], 2071 ['mcBurney', 'McBurney', CAPS_NAMES], 2072 ['blumberg', 'Blumberg', CAPS_NAMES], 2073 ['roVsing', 'RoVsing', CAPS_NAMES], 2074 ['Özdemir', 'Özdemir', CAPS_NAMES], 2075 ['özdemir', 'Özdemir', CAPS_NAMES], 2076 ] 2077 for pair in pairs: 2078 result = capitalize(pair[0], pair[2]) 2079 if result != pair[1]: 2080 success = False 2081 print('ERROR (caps mode %s): "%s" -> "%s", expected "%s"' % (pair[2], pair[0], result, pair[1])) 2082 2083 if success: 2084 print("... SUCCESS") 2085 2086 return success
2087 #-----------------------------------------------------------------------
2088 - def test_import_module():
2089 print("testing import_module_from_directory()") 2090 path = sys.argv[1] 2091 name = sys.argv[2] 2092 try: 2093 mod = import_module_from_directory(module_path = path, module_name = name) 2094 except Exception: 2095 print("module import failed, see log") 2096 return False 2097 2098 print("module import succeeded", mod) 2099 print(dir(mod)) 2100 return True
2101 #-----------------------------------------------------------------------
2102 - def test_mkdir():
2103 print("testing mkdir(%s)" % sys.argv[2]) 2104 mkdir(sys.argv[2], 0o0700)
2105 #-----------------------------------------------------------------------
2106 - def test_gmPaths():
2107 print("testing gmPaths()") 2108 print("-----------------") 2109 paths = gmPaths(wx=None, app_name='gnumed') 2110 print("user config dir:", paths.user_config_dir) 2111 print("system config dir:", paths.system_config_dir) 2112 print("local base dir:", paths.local_base_dir) 2113 print("system app data dir:", paths.system_app_data_dir) 2114 print("working directory :", paths.working_dir) 2115 print("temp directory :", paths.tmp_dir)
2116 #-----------------------------------------------------------------------
2117 - def test_none_if():
2118 print("testing none_if()") 2119 print("-----------------") 2120 tests = [ 2121 [None, None, None], 2122 ['a', 'a', None], 2123 ['a', 'b', 'a'], 2124 ['a', None, 'a'], 2125 [None, 'a', None], 2126 [1, 1, None], 2127 [1, 2, 1], 2128 [1, None, 1], 2129 [None, 1, None] 2130 ] 2131 2132 for test in tests: 2133 if none_if(value = test[0], none_equivalent = test[1]) != test[2]: 2134 print('ERROR: none_if(%s) returned [%s], expected [%s]' % (test[0], none_if(test[0], test[1]), test[2])) 2135 2136 return True
2137 #-----------------------------------------------------------------------
2138 - def test_bool2str():
2139 tests = [ 2140 [True, 'Yes', 'Yes', 'Yes'], 2141 [False, 'OK', 'not OK', 'not OK'] 2142 ] 2143 for test in tests: 2144 if bool2str(test[0], test[1], test[2]) != test[3]: 2145 print('ERROR: bool2str(%s, %s, %s) returned [%s], expected [%s]' % (test[0], test[1], test[2], bool2str(test[0], test[1], test[2]), test[3])) 2146 2147 return True
2148 #-----------------------------------------------------------------------
2149 - def test_bool2subst():
2150 2151 print(bool2subst(True, 'True', 'False', 'is None')) 2152 print(bool2subst(False, 'True', 'False', 'is None')) 2153 print(bool2subst(None, 'True', 'False', 'is None'))
2154 #-----------------------------------------------------------------------
2155 - def test_get_unique_filename():
2156 print(get_unique_filename()) 2157 print(get_unique_filename(prefix='test-')) 2158 print(get_unique_filename(suffix='tst')) 2159 print(get_unique_filename(prefix='test-', suffix='tst')) 2160 print(get_unique_filename(tmp_dir='/home/ncq/Archiv/'))
2161 #-----------------------------------------------------------------------
2162 - def test_size2str():
2163 print("testing size2str()") 2164 print("------------------") 2165 tests = [0, 1, 1000, 10000, 100000, 1000000, 10000000, 100000000, 1000000000, 10000000000, 100000000000, 1000000000000, 10000000000000] 2166 for test in tests: 2167 print(size2str(test))
2168 #-----------------------------------------------------------------------
2169 - def test_unwrap():
2170 2171 test = """ 2172 second line\n 2173 3rd starts with tab \n 2174 4th with a space \n 2175 2176 6th 2177 2178 """ 2179 print(unwrap(text = test, max_length = 25))
2180 #-----------------------------------------------------------------------
2181 - def test_wrap():
2182 test = 'line 1\nline 2\nline 3' 2183 2184 print("wrap 5-6-7 initial 0, subsequent 0") 2185 print(wrap(test, 5)) 2186 print() 2187 print(wrap(test, 6)) 2188 print() 2189 print(wrap(test, 7)) 2190 print("-------") 2191 input() 2192 print("wrap 5 initial 1-1-3, subsequent 1-3-1") 2193 print(wrap(test, 5, ' ', ' ')) 2194 print() 2195 print(wrap(test, 5, ' ', ' ')) 2196 print() 2197 print(wrap(test, 5, ' ', ' ')) 2198 print("-------") 2199 input() 2200 print("wrap 6 initial 1-1-3, subsequent 1-3-1") 2201 print(wrap(test, 6, ' ', ' ')) 2202 print() 2203 print(wrap(test, 6, ' ', ' ')) 2204 print() 2205 print(wrap(test, 6, ' ', ' ')) 2206 print("-------") 2207 input() 2208 print("wrap 7 initial 1-1-3, subsequent 1-3-1") 2209 print(wrap(test, 7, ' ', ' ')) 2210 print() 2211 print(wrap(test, 7, ' ', ' ')) 2212 print() 2213 print(wrap(test, 7, ' ', ' '))
2214 #-----------------------------------------------------------------------
2215 - def test_md5():
2216 print('md5 %s: %s' % (sys.argv[2], file2md5(sys.argv[2]))) 2217 print('chunked md5 %s: %s' % (sys.argv[2], file2chunked_md5(sys.argv[2])))
2218 #-----------------------------------------------------------------------
2219 - def test_unicode():
2220 print(u_link_symbol * 10)
2221 #-----------------------------------------------------------------------
2222 - def test_xml_escape():
2223 print(xml_escape_string('<')) 2224 print(xml_escape_string('>')) 2225 print(xml_escape_string('&'))
2226 #-----------------------------------------------------------------------
2227 - def test_tex_escape():
2228 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234'] 2229 tests.append(' '.join(tests)) 2230 for test in tests: 2231 print('%s:' % test, tex_escape_string(test))
2232 2233 #-----------------------------------------------------------------------
2234 - def test_rst2latex_snippet():
2235 tests = ['\\', '^', '~', '{', '}', '%', '&', '#', '$', '_', u_euro, 'abc\ndef\n\n1234'] 2236 tests.append(' '.join(tests)) 2237 tests.append('C:\Windows\Programme\System 32\lala.txt') 2238 tests.extend([ 2239 'should be identical', 2240 'text *some text* text', 2241 """A List 2242 ====== 2243 2244 1. 1 2245 2. 2 2246 2247 3. ist-list 2248 1. more 2249 2. noch was ü 2250 #. nummer x""" 2251 ]) 2252 for test in tests: 2253 print('==================================================') 2254 print('raw:') 2255 print(test) 2256 print('---------') 2257 print('ReST 2 LaTeX:') 2258 latex = rst2latex_snippet(test) 2259 print(latex) 2260 if latex.strip() == test.strip(): 2261 print('=> identical') 2262 print('---------') 2263 print('tex_escape_string:') 2264 print(tex_escape_string(test)) 2265 input()
2266 2267 #-----------------------------------------------------------------------
2268 - def test_strip_trailing_empty_lines():
2269 tests = [ 2270 'one line, no embedded line breaks ', 2271 'one line\nwith embedded\nline\nbreaks\n ' 2272 ] 2273 for test in tests: 2274 print('as list:') 2275 print(strip_trailing_empty_lines(text = test, eol='\n', return_list = True)) 2276 print('as string:') 2277 print('>>>%s<<<' % strip_trailing_empty_lines(text = test, eol='\n', return_list = False)) 2278 tests = [ 2279 ['list', 'without', 'empty', 'trailing', 'lines'], 2280 ['list', 'with', 'empty', 'trailing', 'lines', '', ' ', ''] 2281 ] 2282 for test in tests: 2283 print('as list:') 2284 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = True)) 2285 print('as string:') 2286 print(strip_trailing_empty_lines(lines = test, eol = '\n', return_list = False))
2287 #-----------------------------------------------------------------------
2288 - def test_fname_stem():
2289 tests = [ 2290 r'abc.exe', 2291 r'\abc.exe', 2292 r'c:\abc.exe', 2293 r'c:\d\abc.exe', 2294 r'/home/ncq/tmp.txt', 2295 r'~/tmp.txt', 2296 r'./tmp.txt', 2297 r'./.././tmp.txt', 2298 r'tmp.txt' 2299 ] 2300 for t in tests: 2301 print("[%s] -> [%s]" % (t, fname_stem(t)))
2302 #-----------------------------------------------------------------------
2303 - def test_dir_is_empty():
2304 print(sys.argv[2], 'empty:', dir_is_empty(sys.argv[2]))
2305 2306 #-----------------------------------------------------------------------
2307 - def test_compare_dicts():
2308 d1 = {} 2309 d2 = {} 2310 d1[1] = 1 2311 d1[2] = 2 2312 d1[3] = 3 2313 # 4 2314 d1[5] = 5 2315 2316 d2[1] = 1 2317 d2[2] = None 2318 # 3 2319 d2[4] = 4 2320 2321 #compare_dict_likes(d1, d2) 2322 2323 d1 = {1: 1, 2: 2} 2324 d2 = {1: 1, 2: 2} 2325 2326 #compare_dict_likes(d1, d2, 'same1', 'same2') 2327 print(format_dict_like(d1, tabular = False)) 2328 print(format_dict_like(d1, tabular = True))
2329 #print(format_dict_like(d2)) 2330 2331 #-----------------------------------------------------------------------
2332 - def test_format_compare_dicts():
2333 d1 = {} 2334 d2 = {} 2335 d1[1] = 1 2336 d1[2] = 2 2337 d1[3] = 3 2338 # 4 2339 d1[5] = 5 2340 2341 d2[1] = 1 2342 d2[2] = None 2343 # 3 2344 d2[4] = 4 2345 2346 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2'))) 2347 2348 d1 = {1: 1, 2: 2} 2349 d2 = {1: 1, 2: 2} 2350 2351 print('\n'.join(format_dict_likes_comparison(d1, d2, 'd1', 'd2')))
2352 2353 #-----------------------------------------------------------------------
2354 - def test_rm_dir():
2355 rmdir('cx:\windows\system3__2xxxxxxxxxxxxx')
2356 2357 #-----------------------------------------------------------------------
2358 - def test_rm_dir_content():
2359 #print(rm_dir_content('cx:\windows\system3__2xxxxxxxxxxxxx')) 2360 print(rm_dir_content('/tmp/user/1000/tmp'))
2361 2362 #-----------------------------------------------------------------------
2363 - def test_strip_prefix():
2364 tests = [ 2365 ('', '', ''), 2366 ('a', 'a', ''), 2367 ('GMd: a window title', _GM_TITLE_PREFIX + ':', 'a window title'), 2368 ('\.br\MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\', '\.br\\', 'MICROCYTES+1\.br\SPHEROCYTES present\.br\POLYCHROMASIAmoderate\.br\\') 2369 ] 2370 for test in tests: 2371 text, prefix, expect = test 2372 result = strip_prefix(text, prefix, remove_whitespace = True) 2373 if result == expect: 2374 continue 2375 print('test failed:', test) 2376 print('result:', result)
2377 2378 #-----------------------------------------------------------------------
2379 - def test_shorten_text():
2380 tst = [ 2381 ('123', 1), 2382 ('123', 2), 2383 ('123', 3), 2384 ('123', 4), 2385 ('', 1), 2386 ('1', 1), 2387 ('12', 1), 2388 ('', 2), 2389 ('1', 2), 2390 ('12', 2), 2391 ('123', 2) 2392 ] 2393 for txt, lng in tst: 2394 print('max', lng, 'of', txt, '=', shorten_text(txt, lng))
2395 #-----------------------------------------------------------------------
2396 - def test_fname_sanitize():
2397 tests = [ 2398 '/tmp/test.txt', 2399 '/tmp/ test.txt', 2400 '/tmp/ tes\\t.txt', 2401 'test' 2402 ] 2403 for test in tests: 2404 print (test, fname_sanitize(test))
2405 2406 #-----------------------------------------------------------------------
2407 - def test_create_qrcode():
2408 print(create_qrcode(text = sys.argv[2], filename=None, qr_filename=None, verbose = True))
2409 2410 #-----------------------------------------------------------------------
2411 - def test_enumerate_removable_partitions():
2412 parts = enumerate_removable_partitions() 2413 for part_name in parts: 2414 part = parts[part_name] 2415 print(part['device']) 2416 print(part['partition']) 2417 if part['is_mounted']: 2418 print('%s@%s: %s on %s by %s @ %s (FS=%s: %s free of %s total)' % ( 2419 part['type'], 2420 part['bus'], 2421 part['fs_label'], 2422 part['model'], 2423 part['vendor'], 2424 part['mountpoint'], 2425 part['fs_type'], 2426 part['bytes_free'], 2427 part['size_in_bytes'] 2428 )) 2429 else: 2430 print('%s@%s: %s on %s by %s (not mounted)' % ( 2431 part['type'], 2432 part['bus'], 2433 part['fs_label'], 2434 part['model'], 2435 part['vendor'] 2436 ))
2437 2438 #-----------------------------------------------------------------------
2439 - def test_enumerate_optical_writers():
2440 for writer in enumerate_optical_writers(): 2441 print('%s@%s: %s @ %s' % ( 2442 writer['type'], 2443 writer['bus'], 2444 writer['model'], 2445 writer['device'] 2446 ))
2447 2448 #-----------------------------------------------------------------------
2449 - def test_copy_tree_content():
2450 print(sys.argv[2], '->', sys.argv[3]) 2451 print(copy_tree_content(sys.argv[2], sys.argv[3]))
2452 2453 #-----------------------------------------------------------------------
2454 - def test_mk_sandbox_dir():
2455 print(mk_sandbox_dir(base_dir = '/tmp/abcd/efg/h'))
2456 2457 #-----------------------------------------------------------------------
2458 - def test_make_table_from_dicts():
2459 dicts = [ 2460 {'pkey': 1, 'value': 'a122'}, 2461 {'pkey': 2, 'value': 'b23'}, 2462 {'pkey': 3, 'value': 'c3'}, 2463 {'pkey': 4, 'value': 'd4ssssssssssss'}, 2464 {'pkey': 5, 'value': 'd4 asdfas '}, 2465 {'pkey': 5, 'value': 'c5---'}, 2466 ] 2467 with open('x.txt', 'w', encoding = 'utf8') as f: 2468 f.write(dicts2table(dicts, left_margin=2, eol='\n', keys2ignore=None, show_only_changes=True, column_labels = ['d1', 'd2', 'd3', 'd4', 'd5', 'd6']))
2469 #print(dicts2table(dicts, left_margin=2, eol='\n', keys2ignore=None, show_only_changes=True, column_labels = ['d1', 'd2', 'd3', 'd4', 'd5', 'd6'])) 2470 2471 #-----------------------------------------------------------------------
2472 - def test_create_dir_desc_file():
2473 global _client_version 2474 _client_version = 'dev.test' 2475 print(create_directory_description_file ( 2476 directory = './', 2477 readme = 'test\ntest2\nsome more text', 2478 suffix = None 2479 ))
2480 2481 #----------------------------------------------------------------------- 2482 #test_coalesce() 2483 #test_capitalize() 2484 #test_import_module() 2485 #test_mkdir() 2486 #test_gmPaths() 2487 #test_none_if() 2488 #test_bool2str() 2489 #test_bool2subst() 2490 #test_get_unique_filename() 2491 #test_size2str() 2492 #test_wrap() 2493 #test_input2decimal() 2494 #test_input2int() 2495 #test_unwrap() 2496 #test_md5() 2497 #test_unicode() 2498 #test_xml_escape() 2499 #test_strip_trailing_empty_lines() 2500 #test_fname_stem() 2501 #test_tex_escape() 2502 #test_rst2latex_snippet() 2503 #test_dir_is_empty() 2504 #test_compare_dicts() 2505 #test_rm_dir() 2506 #test_rm_dir_content() 2507 #test_strip_prefix() 2508 #test_shorten_text() 2509 #test_format_compare_dicts() 2510 #test_fname_sanitize() 2511 #test_create_qrcode() 2512 #test_enumerate_removable_partitions() 2513 #test_enumerate_optical_writers() 2514 #test_copy_tree_content() 2515 #test_mk_sandbox_dir() 2516 #test_make_table_from_dicts() 2517 test_create_dir_desc_file() 2518 2519 #=========================================================================== 2520