Package Gnumed :: Package pycommon :: Module gmI18N
[frames] | no frames]

Source Code for Module Gnumed.pycommon.gmI18N

  1  __doc__ = """GNUmed client internationalization/localization. 
  2   
  3  All i18n/l10n issues should be handled through this modules. 
  4   
  5  Theory of operation: 
  6   
  7  To activate proper locale settings and translation services you need to 
  8   
  9  - import this module 
 10  - call activate_locale() 
 11  - call install_domain() 
 12   
 13  The translating method gettext.gettext() will then be 
 14  installed into the global (!) namespace as _(). Your own 
 15  modules thus need not do _anything_ (not even import gmI18N) 
 16  to have _() available to them for translating strings. You 
 17  need to make sure, however, that gmI18N is imported in your 
 18  main module before any of the modules using it. In order to 
 19  resolve circular references involving modules that 
 20  absolutely _have_ to be imported before this module you can 
 21  explicitly import gmI18N into them at the very beginning. 
 22   
 23  The text domain (i.e. the name of the message catalog file) 
 24  is derived from the name of the main executing script unless 
 25  explicitly passed to install_domain(). The language you 
 26  want to translate to is derived from environment variables 
 27  by the locale system unless explicitly passed to 
 28  install_domain(). 
 29   
 30  This module searches for message catalog files in 3 main locations: 
 31   
 32   - standard POSIX places (/usr/share/locale/ ...) 
 33   - below "${YOURAPPNAME_DIR}/po/" 
 34   - below "<directory of binary of your app>/../po/" 
 35   
 36  For DOS/Windows I don't know of standard places so probably 
 37  only the last option will work. I don't know a thing about 
 38  classic Mac behaviour. New Macs are POSIX, of course. 
 39   
 40  It will then try to install candidates and *verify* whether 
 41  the translation works by checking for the translation of a 
 42  tag within itself (this is similar to the self-compiling 
 43  compiler inserting a backdoor into its self-compiled 
 44  copies). 
 45   
 46  If none of this works it will fall back to making _() a noop. 
 47   
 48  @copyright: authors 
 49  """ 
 50  #=========================================================================== 
 51  __author__ = "H. Herb <hherb@gnumed.net>, I. Haywood <i.haywood@ugrad.unimelb.edu.au>, K. Hilbert <Karsten.Hilbert@gmx.net>" 
 52  __license__ = "GPL v2 or later (details at http://www.gnu.org)" 
 53   
 54   
 55  # stdlib 
 56  import sys 
 57  import os.path 
 58  import os 
 59  import locale 
 60  import gettext 
 61  import logging 
 62  import codecs 
 63  import re as regex 
 64   
 65  _log = logging.getLogger('gm.i18n') 
 66   
 67  system_locale = '' 
 68  system_locale_level = {} 
 69   
 70   
 71  _translate_original = lambda x:x 
 72  _substitutes_regex = regex.compile(r'%\(.+?\)s') 
 73   
 74  # ********************************************************** 
 75  # == do not remove this line =============================== 
 76  # it is needed to check for successful installation of 
 77  # the desired message catalog 
 78  # ********************************************************** 
 79  __orig_tag__ = 'Translate this or i18n into <en_EN> will not work properly !' 
 80  # ********************************************************** 
 81  # ********************************************************** 
 82   
 83  #=========================================================================== 
84 -def __split_locale_into_levels():
85 """Split locale into language, country and variant parts. 86 87 - we have observed the following formats in the wild: 88 - de_DE@euro 89 - ec_CA.UTF-8 90 - en_US:en 91 - German_Germany.1252 92 """ 93 _log.debug('splitting canonical locale [%s] into levels', system_locale) 94 95 global system_locale_level 96 system_locale_level['full'] = system_locale 97 # trim '@<variant>' part 98 system_locale_level['country'] = regex.split('@|:|\.', system_locale, 1)[0] 99 # trim '_<COUNTRY>@<variant>' part 100 system_locale_level['language'] = system_locale.split('_', 1)[0] 101 102 _log.debug('system locale levels: %s', system_locale_level)
103 104 #---------------------------------------------------------------------------
105 -def __log_locale_settings(message=None):
106 _setlocale_categories = {} 107 for category in 'LC_ALL LC_CTYPE LC_COLLATE LC_TIME LC_MONETARY LC_MESSAGES LC_NUMERIC'.split(): 108 try: 109 _setlocale_categories[category] = getattr(locale, category) 110 except: 111 _log.warning('this OS does not have locale.%s', category) 112 113 _getlocale_categories = {} 114 for category in 'LC_CTYPE LC_COLLATE LC_TIME LC_MONETARY LC_MESSAGES LC_NUMERIC'.split(): 115 try: 116 _getlocale_categories[category] = getattr(locale, category) 117 except: 118 pass 119 120 if message is not None: 121 _log.debug(message) 122 123 _log.debug('current locale settings:') 124 _log.debug('locale.getlocale(): %s' % str(locale.getlocale())) 125 for category in _getlocale_categories.keys(): 126 _log.debug('locale.getlocale(%s): %s' % (category, locale.getlocale(_getlocale_categories[category]))) 127 128 for category in _setlocale_categories.keys(): 129 _log.debug('(locale.setlocale(%s): %s)' % (category, locale.setlocale(_setlocale_categories[category]))) 130 131 try: 132 _log.debug('locale.getdefaultlocale() - default (user) locale: %s' % str(locale.getdefaultlocale())) 133 except ValueError: 134 _log.exception('the OS locale setup seems faulty') 135 136 _log.debug('encoding sanity check (also check "locale.nl_langinfo(CODESET)" below):') 137 pref_loc_enc = locale.getpreferredencoding(do_setlocale=False) 138 loc_enc = locale.getlocale()[1] 139 py_str_enc = sys.getdefaultencoding() 140 sys_fs_enc = sys.getfilesystemencoding() 141 _log.debug('sys.getdefaultencoding(): [%s]' % py_str_enc) 142 _log.debug('locale.getpreferredencoding(): [%s]' % pref_loc_enc) 143 _log.debug('locale.getlocale()[1]: [%s]' % loc_enc) 144 _log.debug('sys.getfilesystemencoding(): [%s]' % sys_fs_enc) 145 if loc_enc is not None: 146 loc_enc = loc_enc.upper() 147 loc_enc_compare = loc_enc.replace('-', '') 148 else: 149 loc_enc_compare = loc_enc 150 if pref_loc_enc.upper().replace('-', '') != loc_enc_compare: 151 _log.warning('encoding suggested by locale (%s) does not match encoding currently set in locale (%s)' % (pref_loc_enc, loc_enc)) 152 _log.warning('this might lead to encoding errors') 153 for enc in [pref_loc_enc, loc_enc, py_str_enc, sys_fs_enc]: 154 if enc is not None: 155 try: 156 codecs.lookup(enc) 157 _log.debug('<codecs> module CAN handle encoding [%s]' % enc) 158 except LookupError: 159 _log.warning('<codecs> module can NOT handle encoding [%s]' % enc) 160 _log.debug('on Linux you can determine a likely candidate for the encoding by running "locale charmap"') 161 162 _log.debug('locale related environment variables (${LANG} is typically used):') 163 for var in 'LANGUAGE LC_ALL LC_CTYPE LANG'.split(): 164 try: 165 _log.debug('${%s}=%s' % (var, os.environ[var])) 166 except KeyError: 167 _log.debug('${%s} not set' % (var)) 168 169 _log.debug('database of locale conventions:') 170 data = locale.localeconv() 171 for key in data.keys(): 172 if loc_enc is None: 173 _log.debug('locale.localeconv(%s): %s', key, data[key]) 174 else: 175 try: 176 _log.debug('locale.localeconv(%s): %s', key, str(data[key])) 177 except UnicodeDecodeError: 178 _log.debug('locale.localeconv(%s): %s', key, str(data[key], loc_enc)) 179 _nl_langinfo_categories = {} 180 for category in 'CODESET D_T_FMT D_FMT T_FMT T_FMT_AMPM RADIXCHAR THOUSEP YESEXPR NOEXPR CRNCYSTR ERA ERA_D_T_FMT ERA_D_FMT ALT_DIGITS'.split(): 181 try: 182 _nl_langinfo_categories[category] = getattr(locale, category) 183 except: 184 _log.warning('this OS does not support nl_langinfo category locale.%s' % category) 185 try: 186 for category in _nl_langinfo_categories.keys(): 187 if loc_enc is None: 188 _log.debug('locale.nl_langinfo(%s): %s' % (category, locale.nl_langinfo(_nl_langinfo_categories[category]))) 189 else: 190 try: 191 _log.debug('locale.nl_langinfo(%s): %s', category, str(locale.nl_langinfo(_nl_langinfo_categories[category]))) 192 except UnicodeDecodeError: 193 _log.debug('locale.nl_langinfo(%s): %s', category, str(locale.nl_langinfo(_nl_langinfo_categories[category]), loc_enc)) 194 except: 195 _log.exception('this OS does not support nl_langinfo') 196 197 _log.debug('gmI18N.get_encoding(): %s', get_encoding())
198 199 #--------------------------------------------------------------------------- 200 #def _translate_protected(term, strip_left=None, strip_right=None):
201 -def _translate_protected(term):
202 """This wraps _(). 203 204 It protects against translation errors such as a different number of "%s". 205 """ 206 translation = _translate_original(term) 207 208 # different number of %s substitutes ? 209 if translation.count('%s') != term.count('%s'): 210 _log.error('count("%s") mismatch, returning untranslated string') 211 _log.error('original : %s', term) 212 _log.error('translation: %s', translation) 213 return term 214 215 substitution_keys_in_original = set(_substitutes_regex.findall(term)) 216 substitution_keys_in_translation = set(_substitutes_regex.findall(translation)) 217 218 if not substitution_keys_in_translation.issubset(substitution_keys_in_original): 219 _log.error('"%(...)s" keys in translation not a subset of keys in original, returning untranslated string') 220 _log.error('original : %s', term) 221 _log.error('translation: %s', translation) 222 return term 223 224 # if strip_left is not None: 225 # translation = translation.lstrip(strip_left) 226 # 227 # if strip_right is not None: 228 # translation = translation.rstrip(strip_right) 229 230 return translation
231 232 #--------------------------------------------------------------------------- 233 # external API 234 #---------------------------------------------------------------------------
235 -def activate_locale():
236 """Get system locale from environment.""" 237 global system_locale 238 239 __log_locale_settings('unmodified startup locale settings (should be [C])') 240 241 # activate user-preferred locale 242 loc, enc = None, None 243 try: 244 # check whether already set 245 loc, loc_enc = locale.getlocale() 246 if loc is None: 247 loc = locale.setlocale(locale.LC_ALL, '') 248 _log.debug("activating user-default locale with <locale.setlocale(locale.LC_ALL, '')> returns: [%s]" % loc) 249 else: 250 _log.info('user-default locale already activated') 251 loc, loc_enc = locale.getlocale() 252 except AttributeError: 253 _log.exception('Windows does not support locale.LC_ALL') 254 except: 255 _log.exception('error activating user-default locale') 256 257 __log_locale_settings('locale settings after activating user-default locale') 258 259 # did we find any locale setting ? assume en_EN if not 260 if loc in [None, 'C']: 261 _log.error('the current system locale is still [None] or [C], assuming [en_EN]') 262 system_locale = "en_EN" 263 else: 264 system_locale = loc 265 266 # generate system locale levels 267 __split_locale_into_levels() 268 269 return True
270 271 #---------------------------------------------------------------------------
272 -def install_domain(domain=None, language=None, prefer_local_catalog=False):
273 """Install a text domain suitable for the main script.""" 274 275 # text domain directly specified ? 276 if domain is None: 277 _log.info('domain not specified, deriving from script name') 278 # get text domain from name of script 279 domain = os.path.splitext(os.path.basename(sys.argv[0]))[0] 280 _log.info('text domain is [%s]' % domain) 281 282 # http://www.opengroup.org/onlinepubs/009695399/basedefs/xbd_chap08.html 283 _log.debug('searching message catalog file for system locale [%s]' % system_locale) 284 285 _log.debug('checking process environment:') 286 for env_var in ['LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG']: 287 tmp = os.getenv(env_var) 288 if env_var is None: 289 _log.debug(' ${%s} not set' % env_var) 290 else: 291 _log.debug(' ${%s} = [%s]' % (env_var, tmp)) 292 293 # language codes to try 294 lang_candidates = [] 295 # first: explicit language or default system language 296 # language=None: unadulterated default language for user (locale.getlocale()[0] value) 297 # language != None: explicit language setting as passed in by the caller 298 lang_candidates.append(language) 299 if language is not None: 300 _log.info('explicit request for target language [%s]' % language) 301 # next: try default language for user if explicit language fails 302 lang_candidates.append(None) 303 304 # next try locale.getlocale()[0], if different (this can be strange on, say, Windows: Hungarian_Hungary) 305 if locale.getlocale()[0] not in lang_candidates: 306 lang_candidates.append(locale.getlocale()[0]) 307 308 # next try locale.get*default*locale()[0], if different 309 if locale.getdefaultlocale()[0] not in lang_candidates: 310 lang_candidates.append(locale.getdefaultlocale()[0]) 311 312 _log.debug('languages to try for translation: %s (None: implicit system default)', lang_candidates) 313 initial_lang = os.getenv('LANG') 314 _log.info('initial ${LANG} setting: %s', initial_lang) 315 316 # loop over language candidates 317 for lang_candidate in lang_candidates: 318 # setup baseline 319 _log.debug('resetting ${LANG} to initial user default [%s]', initial_lang) 320 if initial_lang is None: 321 del os.environ['LANG'] 322 lang2log = '$LANG=<>' 323 else: 324 os.environ['LANG'] = initial_lang 325 lang2log = '$LANG(default)=%s' % initial_lang 326 # setup candidate language 327 if lang_candidate is not None: 328 _log.info('explicitely overriding system locale language [%s] by setting ${LANG} to [%s]', initial_lang, lang_candidate) 329 os.environ['LANG'] = lang_candidate 330 lang2log = '$LANG(explicit)=%s' % lang_candidate 331 332 if __install_domain(domain = domain, prefer_local_catalog = prefer_local_catalog, language = lang2log): 333 return True 334 335 # install a dummy translation class 336 _log.warning("falling back to NullTranslations() class") 337 # this shouldn't fail 338 dummy = gettext.NullTranslations() 339 dummy.install() 340 return True
341 342 #---------------------------------------------------------------------------
343 -def __install_domain(domain, prefer_local_catalog, language='?'):
344 # <language> only used for logging 345 346 # search for message catalog 347 candidate_PO_dirs = [] 348 349 # - locally 350 if prefer_local_catalog: 351 _log.debug('prioritizing local message catalog') 352 # - one level above path to binary 353 # last resort for inferior operating systems such as DOS/Windows 354 # strip one directory level 355 # this is a rather neat trick :-) 356 loc_dir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..', 'po')) 357 _log.debug('looking one level above binary install directory: %s', loc_dir) 358 candidate_PO_dirs.append(loc_dir) 359 # - in path to binary 360 loc_dir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), 'po')) 361 _log.debug('looking in binary install directory: %s', loc_dir) 362 candidate_PO_dirs.append(loc_dir) 363 364 # - standard places 365 if os.name == 'posix': 366 _log.debug('system is POSIX, looking in standard locations (see Python Manual)') 367 # if this is reported to segfault/fail/except on some 368 # systems we may have to assume "sys.prefix/share/locale/" 369 candidate_PO_dirs.append(gettext.bindtextdomain(domain)) 370 else: 371 _log.debug('No use looking in standard POSIX locations - not a POSIX system.') 372 373 # - $(<script-name>_DIR)/ 374 env_key = "%s_DIR" % os.path.splitext(os.path.basename(sys.argv[0]))[0].upper() 375 _log.debug('looking at ${%s}' % env_key) 376 if env_key in os.environ: 377 loc_dir = os.path.abspath(os.path.join(os.environ[env_key], 'po')) 378 _log.debug('${%s} = "%s" -> [%s]' % (env_key, os.environ[env_key], loc_dir)) 379 candidate_PO_dirs.append(loc_dir) 380 else: 381 _log.info("${%s} not set" % env_key) 382 383 # - locally 384 if not prefer_local_catalog: 385 # - one level above path to binary 386 # last resort for inferior operating systems such as DOS/Windows 387 # strip one directory level 388 # this is a rather neat trick :-) 389 loc_dir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..', 'po')) 390 _log.debug('looking above binary install directory [%s]' % loc_dir) 391 candidate_PO_dirs.append(loc_dir) 392 # - in path to binary 393 loc_dir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), 'po' )) 394 _log.debug('looking in binary install directory [%s]' % loc_dir) 395 candidate_PO_dirs.append(loc_dir) 396 397 # now try to actually install it 398 for candidate_PO_dir in candidate_PO_dirs: 399 _log.debug('trying with (base=%s, %s, domain=%s)', candidate_PO_dir, language, domain) 400 _log.debug(' -> %s.mo', os.path.join(candidate_PO_dir, language, domain)) 401 if not os.path.exists(candidate_PO_dir): 402 continue 403 try: 404 gettext.install(domain, candidate_PO_dir) 405 except: 406 _log.exception('installing text domain [%s] failed from [%s]', domain, candidate_PO_dir) 407 continue 408 global _ 409 # does it translate ? 410 if _(__orig_tag__) == __orig_tag__: 411 _log.debug('does not translate: [%s] => [%s]', __orig_tag__, _(__orig_tag__)) 412 continue 413 else: 414 _log.debug('found msg catalog: [%s] => [%s]', __orig_tag__, _(__orig_tag__)) 415 import builtins 416 global _translate_original 417 _translate_original = builtins._ 418 builtins._ = _translate_protected 419 return True 420 421 return False
422 423 #=========================================================================== 424 _encoding_mismatch_already_logged = False 425 _current_encoding = None 426
427 -def get_encoding():
428 """Try to get a sane encoding. 429 430 On MaxOSX locale.setlocale(locale.LC_ALL, '') does not 431 have the desired effect, so that locale.getlocale()[1] 432 still returns None. So in that case try to fallback to 433 locale.getpreferredencoding(). 434 435 <sys.getdefaultencoding()> 436 - what Python itself uses to convert string <-> unicode 437 when no other encoding was specified 438 - ascii by default 439 - can be set in site.py and sitecustomize.py 440 <locale.getlocale()[1]> 441 - what the current locale is *actually* using 442 as the encoding for text conversion 443 <locale.getpreferredencoding()> 444 - what the current locale would *recommend* using 445 as the encoding for text conversion 446 """ 447 global _current_encoding 448 if _current_encoding is not None: 449 return _current_encoding 450 451 enc = sys.getdefaultencoding() 452 if enc != 'ascii': 453 _current_encoding = enc 454 return _current_encoding 455 456 enc = locale.getlocale()[1] 457 if enc is not None: 458 _current_encoding = enc 459 return _current_encoding 460 461 global _encoding_mismatch_already_logged 462 if not _encoding_mismatch_already_logged: 463 _log.debug('*actual* encoding of locale is None, using encoding *recommended* by locale') 464 _encoding_mismatch_already_logged = True 465 466 return locale.getpreferredencoding(do_setlocale=False)
467 468 #=========================================================================== 469 # Main 470 #--------------------------------------------------------------------------- 471 if __name__ == "__main__": 472 473 if len(sys.argv) == 1: 474 sys.exit() 475 476 if sys.argv[1] != 'test': 477 sys.exit() 478 479 logging.basicConfig(level = logging.DEBUG) 480 #----------------------------------------------------------------------
481 - def test_strcoll():
482 candidates = [ 483 # (u'a', u'a'), 484 # (u'a', u'b'), 485 # (u'1', u'1'), 486 # (u'1', u'2'), 487 # (u'A', u'A'), 488 # (u'a', u'A'), 489 ('\u270d', '\u270d'), 490 ('4', '\u270d' + '4'), 491 ('4.4', '\u270d' + '4.4'), 492 ('44', '\u270d' + '44'), 493 ('4', '\u270d' + '9'), 494 ('4', '\u270d' + '2'), 495 # (u'9', u'\u270d' + u'9'), 496 # (u'9', u'\u270d' + u'4'), 497 498 ] 499 for cands in candidates: 500 print(cands[0], '<vs>', cands[1], '=', locale.strcoll(cands[0], cands[1]))
501 # print(cands[1], u'<vs>', cands[0], '=', locale.strcoll(cands[1], cands[0])) 502 503 #---------------------------------------------------------------------- 504 print("======================================================================") 505 print("GNUmed i18n") 506 print("") 507 print("authors:", __author__) 508 print("license:", __license__) 509 print("======================================================================") 510 511 activate_locale() 512 print("system locale: ", system_locale, "; levels:", system_locale_level) 513 print("likely encoding:", get_encoding()) 514 515 if len(sys.argv) > 2: 516 install_domain(domain = sys.argv[2]) 517 else: 518 install_domain() 519 520 test_strcoll() 521 522 # ********************************************************************* # 523 # == do not remove this line ========================================== # 524 # it is needed to check for successful installation of # 525 # the desired message catalog # 526 # ********************************************************************* # 527 tmp = _('Translate this or i18n into <en_EN> will not work properly !') # 528 # ********************************************************************* # 529 # ********************************************************************* # 530