1 __doc__ = """GNUmed client internationalization/localization.
2
3 All i18n/l10n issues should be handled through this modules.
4
5 Theory of operation:
6
7 To activate proper locale settings and translation services you need to
8
9 - import this module
10 - call activate_locale()
11 - call install_domain()
12
13 The translating method gettext.gettext() will then be
14 installed into the global (!) namespace as _(). Your own
15 modules thus need not do _anything_ (not even import gmI18N)
16 to have _() available to them for translating strings. You
17 need to make sure, however, that gmI18N is imported in your
18 main module before any of the modules using it. In order to
19 resolve circular references involving modules that
20 absolutely _have_ to be imported before this module you can
21 explicitly import gmI18N into them at the very beginning.
22
23 The text domain (i.e. the name of the message catalog file)
24 is derived from the name of the main executing script unless
25 explicitly passed to install_domain(). The language you
26 want to translate to is derived from environment variables
27 by the locale system unless explicitly passed to
28 install_domain().
29
30 This module searches for message catalog files in 3 main locations:
31
32 - standard POSIX places (/usr/share/locale/ ...)
33 - below "${YOURAPPNAME_DIR}/po/"
34 - below "<directory of binary of your app>/../po/"
35
36 For DOS/Windows I don't know of standard places so probably
37 only the last option will work. I don't know a thing about
38 classic Mac behaviour. New Macs are POSIX, of course.
39
40 It will then try to install candidates and *verify* whether
41 the translation works by checking for the translation of a
42 tag within itself (this is similar to the self-compiling
43 compiler inserting a backdoor into its self-compiled
44 copies).
45
46 If none of this works it will fall back to making _() a noop.
47
48 @copyright: authors
49 """
50
51 __author__ = "H. Herb <hherb@gnumed.net>, I. Haywood <i.haywood@ugrad.unimelb.edu.au>, K. Hilbert <Karsten.Hilbert@gmx.net>"
52 __license__ = "GPL v2 or later (details at http://www.gnu.org)"
53
54
55
56 import sys
57 import os.path
58 import os
59 import locale
60 import gettext
61 import logging
62 import codecs
63 import re as regex
64
65 _log = logging.getLogger('gm.i18n')
66
67 system_locale = ''
68 system_locale_level = {}
69
70
71 _translate_original = lambda x:x
72 _substitutes_regex = regex.compile(r'%\(.+?\)s')
73
74
75
76
77
78
79 __orig_tag__ = 'Translate this or i18n into <en_EN> will not work properly !'
80
81
82
83
103
104
106 _setlocale_categories = {}
107 for category in 'LC_ALL LC_CTYPE LC_COLLATE LC_TIME LC_MONETARY LC_MESSAGES LC_NUMERIC'.split():
108 try:
109 _setlocale_categories[category] = getattr(locale, category)
110 except:
111 _log.warning('this OS does not have locale.%s', category)
112
113 _getlocale_categories = {}
114 for category in 'LC_CTYPE LC_COLLATE LC_TIME LC_MONETARY LC_MESSAGES LC_NUMERIC'.split():
115 try:
116 _getlocale_categories[category] = getattr(locale, category)
117 except:
118 pass
119
120 if message is not None:
121 _log.debug(message)
122
123 _log.debug('current locale settings:')
124 _log.debug('locale.getlocale(): %s' % str(locale.getlocale()))
125 for category in _getlocale_categories.keys():
126 _log.debug('locale.getlocale(%s): %s' % (category, locale.getlocale(_getlocale_categories[category])))
127
128 for category in _setlocale_categories.keys():
129 _log.debug('(locale.setlocale(%s): %s)' % (category, locale.setlocale(_setlocale_categories[category])))
130
131 try:
132 _log.debug('locale.getdefaultlocale() - default (user) locale: %s' % str(locale.getdefaultlocale()))
133 except ValueError:
134 _log.exception('the OS locale setup seems faulty')
135
136 _log.debug('encoding sanity check (also check "locale.nl_langinfo(CODESET)" below):')
137 pref_loc_enc = locale.getpreferredencoding(do_setlocale=False)
138 loc_enc = locale.getlocale()[1]
139 py_str_enc = sys.getdefaultencoding()
140 sys_fs_enc = sys.getfilesystemencoding()
141 _log.debug('sys.getdefaultencoding(): [%s]' % py_str_enc)
142 _log.debug('locale.getpreferredencoding(): [%s]' % pref_loc_enc)
143 _log.debug('locale.getlocale()[1]: [%s]' % loc_enc)
144 _log.debug('sys.getfilesystemencoding(): [%s]' % sys_fs_enc)
145 if loc_enc is not None:
146 loc_enc = loc_enc.upper()
147 loc_enc_compare = loc_enc.replace('-', '')
148 else:
149 loc_enc_compare = loc_enc
150 if pref_loc_enc.upper().replace('-', '') != loc_enc_compare:
151 _log.warning('encoding suggested by locale (%s) does not match encoding currently set in locale (%s)' % (pref_loc_enc, loc_enc))
152 _log.warning('this might lead to encoding errors')
153 for enc in [pref_loc_enc, loc_enc, py_str_enc, sys_fs_enc]:
154 if enc is not None:
155 try:
156 codecs.lookup(enc)
157 _log.debug('<codecs> module CAN handle encoding [%s]' % enc)
158 except LookupError:
159 _log.warning('<codecs> module can NOT handle encoding [%s]' % enc)
160 _log.debug('on Linux you can determine a likely candidate for the encoding by running "locale charmap"')
161
162 _log.debug('locale related environment variables (${LANG} is typically used):')
163 for var in 'LANGUAGE LC_ALL LC_CTYPE LANG'.split():
164 try:
165 _log.debug('${%s}=%s' % (var, os.environ[var]))
166 except KeyError:
167 _log.debug('${%s} not set' % (var))
168
169 _log.debug('database of locale conventions:')
170 data = locale.localeconv()
171 for key in data.keys():
172 if loc_enc is None:
173 _log.debug('locale.localeconv(%s): %s', key, data[key])
174 else:
175 try:
176 _log.debug('locale.localeconv(%s): %s', key, str(data[key]))
177 except UnicodeDecodeError:
178 _log.debug('locale.localeconv(%s): %s', key, str(data[key], loc_enc))
179 _nl_langinfo_categories = {}
180 for category in 'CODESET D_T_FMT D_FMT T_FMT T_FMT_AMPM RADIXCHAR THOUSEP YESEXPR NOEXPR CRNCYSTR ERA ERA_D_T_FMT ERA_D_FMT ALT_DIGITS'.split():
181 try:
182 _nl_langinfo_categories[category] = getattr(locale, category)
183 except:
184 _log.warning('this OS does not support nl_langinfo category locale.%s' % category)
185 try:
186 for category in _nl_langinfo_categories.keys():
187 if loc_enc is None:
188 _log.debug('locale.nl_langinfo(%s): %s' % (category, locale.nl_langinfo(_nl_langinfo_categories[category])))
189 else:
190 try:
191 _log.debug('locale.nl_langinfo(%s): %s', category, str(locale.nl_langinfo(_nl_langinfo_categories[category])))
192 except UnicodeDecodeError:
193 _log.debug('locale.nl_langinfo(%s): %s', category, str(locale.nl_langinfo(_nl_langinfo_categories[category]), loc_enc))
194 except:
195 _log.exception('this OS does not support nl_langinfo')
196
197 _log.debug('gmI18N.get_encoding(): %s', get_encoding())
198
199
200
202 """This wraps _().
203
204 It protects against translation errors such as a different number of "%s".
205 """
206 translation = _translate_original(term)
207
208
209 if translation.count('%s') != term.count('%s'):
210 _log.error('count("%s") mismatch, returning untranslated string')
211 _log.error('original : %s', term)
212 _log.error('translation: %s', translation)
213 return term
214
215 substitution_keys_in_original = set(_substitutes_regex.findall(term))
216 substitution_keys_in_translation = set(_substitutes_regex.findall(translation))
217
218 if not substitution_keys_in_translation.issubset(substitution_keys_in_original):
219 _log.error('"%(...)s" keys in translation not a subset of keys in original, returning untranslated string')
220 _log.error('original : %s', term)
221 _log.error('translation: %s', translation)
222 return term
223
224
225
226
227
228
229
230 return translation
231
232
233
234
236 """Get system locale from environment."""
237 global system_locale
238
239 __log_locale_settings('unmodified startup locale settings (should be [C])')
240
241
242 loc, enc = None, None
243 try:
244
245 loc, loc_enc = locale.getlocale()
246 if loc is None:
247 loc = locale.setlocale(locale.LC_ALL, '')
248 _log.debug("activating user-default locale with <locale.setlocale(locale.LC_ALL, '')> returns: [%s]" % loc)
249 else:
250 _log.info('user-default locale already activated')
251 loc, loc_enc = locale.getlocale()
252 except AttributeError:
253 _log.exception('Windows does not support locale.LC_ALL')
254 except:
255 _log.exception('error activating user-default locale')
256
257 __log_locale_settings('locale settings after activating user-default locale')
258
259
260 if loc in [None, 'C']:
261 _log.error('the current system locale is still [None] or [C], assuming [en_EN]')
262 system_locale = "en_EN"
263 else:
264 system_locale = loc
265
266
267 __split_locale_into_levels()
268
269 return True
270
271
272 -def install_domain(domain=None, language=None, prefer_local_catalog=False):
273 """Install a text domain suitable for the main script."""
274
275
276 if domain is None:
277 _log.info('domain not specified, deriving from script name')
278
279 domain = os.path.splitext(os.path.basename(sys.argv[0]))[0]
280 _log.info('text domain is [%s]' % domain)
281
282
283 _log.debug('searching message catalog file for system locale [%s]' % system_locale)
284
285 _log.debug('checking process environment:')
286 for env_var in ['LANGUAGE', 'LC_ALL', 'LC_MESSAGES', 'LANG']:
287 tmp = os.getenv(env_var)
288 if env_var is None:
289 _log.debug(' ${%s} not set' % env_var)
290 else:
291 _log.debug(' ${%s} = [%s]' % (env_var, tmp))
292
293
294 lang_candidates = []
295
296
297
298 lang_candidates.append(language)
299 if language is not None:
300 _log.info('explicit request for target language [%s]' % language)
301
302 lang_candidates.append(None)
303
304
305 if locale.getlocale()[0] not in lang_candidates:
306 lang_candidates.append(locale.getlocale()[0])
307
308
309 if locale.getdefaultlocale()[0] not in lang_candidates:
310 lang_candidates.append(locale.getdefaultlocale()[0])
311
312 _log.debug('languages to try for translation: %s (None: implicit system default)', lang_candidates)
313 initial_lang = os.getenv('LANG')
314 _log.info('initial ${LANG} setting: %s', initial_lang)
315
316
317 for lang_candidate in lang_candidates:
318
319 _log.debug('resetting ${LANG} to initial user default [%s]', initial_lang)
320 if initial_lang is None:
321 del os.environ['LANG']
322 lang2log = '$LANG=<>'
323 else:
324 os.environ['LANG'] = initial_lang
325 lang2log = '$LANG(default)=%s' % initial_lang
326
327 if lang_candidate is not None:
328 _log.info('explicitely overriding system locale language [%s] by setting ${LANG} to [%s]', initial_lang, lang_candidate)
329 os.environ['LANG'] = lang_candidate
330 lang2log = '$LANG(explicit)=%s' % lang_candidate
331
332 if __install_domain(domain = domain, prefer_local_catalog = prefer_local_catalog, language = lang2log):
333 return True
334
335
336 _log.warning("falling back to NullTranslations() class")
337
338 dummy = gettext.NullTranslations()
339 dummy.install()
340 return True
341
342
343 -def __install_domain(domain, prefer_local_catalog, language='?'):
344
345
346
347 candidate_PO_dirs = []
348
349
350 if prefer_local_catalog:
351 _log.debug('prioritizing local message catalog')
352
353
354
355
356 loc_dir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..', 'po'))
357 _log.debug('looking one level above binary install directory: %s', loc_dir)
358 candidate_PO_dirs.append(loc_dir)
359
360 loc_dir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), 'po'))
361 _log.debug('looking in binary install directory: %s', loc_dir)
362 candidate_PO_dirs.append(loc_dir)
363
364
365 if os.name == 'posix':
366 _log.debug('system is POSIX, looking in standard locations (see Python Manual)')
367
368
369 candidate_PO_dirs.append(gettext.bindtextdomain(domain))
370 else:
371 _log.debug('No use looking in standard POSIX locations - not a POSIX system.')
372
373
374 env_key = "%s_DIR" % os.path.splitext(os.path.basename(sys.argv[0]))[0].upper()
375 _log.debug('looking at ${%s}' % env_key)
376 if env_key in os.environ:
377 loc_dir = os.path.abspath(os.path.join(os.environ[env_key], 'po'))
378 _log.debug('${%s} = "%s" -> [%s]' % (env_key, os.environ[env_key], loc_dir))
379 candidate_PO_dirs.append(loc_dir)
380 else:
381 _log.info("${%s} not set" % env_key)
382
383
384 if not prefer_local_catalog:
385
386
387
388
389 loc_dir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), '..', 'po'))
390 _log.debug('looking above binary install directory [%s]' % loc_dir)
391 candidate_PO_dirs.append(loc_dir)
392
393 loc_dir = os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), 'po' ))
394 _log.debug('looking in binary install directory [%s]' % loc_dir)
395 candidate_PO_dirs.append(loc_dir)
396
397
398 for candidate_PO_dir in candidate_PO_dirs:
399 _log.debug('trying with (base=%s, %s, domain=%s)', candidate_PO_dir, language, domain)
400 _log.debug(' -> %s.mo', os.path.join(candidate_PO_dir, language, domain))
401 if not os.path.exists(candidate_PO_dir):
402 continue
403 try:
404 gettext.install(domain, candidate_PO_dir)
405 except:
406 _log.exception('installing text domain [%s] failed from [%s]', domain, candidate_PO_dir)
407 continue
408 global _
409
410 if _(__orig_tag__) == __orig_tag__:
411 _log.debug('does not translate: [%s] => [%s]', __orig_tag__, _(__orig_tag__))
412 continue
413 else:
414 _log.debug('found msg catalog: [%s] => [%s]', __orig_tag__, _(__orig_tag__))
415 import builtins
416 global _translate_original
417 _translate_original = builtins._
418 builtins._ = _translate_protected
419 return True
420
421 return False
422
423
424 _encoding_mismatch_already_logged = False
425 _current_encoding = None
426
428 """Try to get a sane encoding.
429
430 On MaxOSX locale.setlocale(locale.LC_ALL, '') does not
431 have the desired effect, so that locale.getlocale()[1]
432 still returns None. So in that case try to fallback to
433 locale.getpreferredencoding().
434
435 <sys.getdefaultencoding()>
436 - what Python itself uses to convert string <-> unicode
437 when no other encoding was specified
438 - ascii by default
439 - can be set in site.py and sitecustomize.py
440 <locale.getlocale()[1]>
441 - what the current locale is *actually* using
442 as the encoding for text conversion
443 <locale.getpreferredencoding()>
444 - what the current locale would *recommend* using
445 as the encoding for text conversion
446 """
447 global _current_encoding
448 if _current_encoding is not None:
449 return _current_encoding
450
451 enc = sys.getdefaultencoding()
452 if enc != 'ascii':
453 _current_encoding = enc
454 return _current_encoding
455
456 enc = locale.getlocale()[1]
457 if enc is not None:
458 _current_encoding = enc
459 return _current_encoding
460
461 global _encoding_mismatch_already_logged
462 if not _encoding_mismatch_already_logged:
463 _log.debug('*actual* encoding of locale is None, using encoding *recommended* by locale')
464 _encoding_mismatch_already_logged = True
465
466 return locale.getpreferredencoding(do_setlocale=False)
467
468
469
470
471 if __name__ == "__main__":
472
473 if len(sys.argv) == 1:
474 sys.exit()
475
476 if sys.argv[1] != 'test':
477 sys.exit()
478
479 logging.basicConfig(level = logging.DEBUG)
480
482 candidates = [
483
484
485
486
487
488
489 ('\u270d', '\u270d'),
490 ('4', '\u270d' + '4'),
491 ('4.4', '\u270d' + '4.4'),
492 ('44', '\u270d' + '44'),
493 ('4', '\u270d' + '9'),
494 ('4', '\u270d' + '2'),
495
496
497
498 ]
499 for cands in candidates:
500 print(cands[0], '<vs>', cands[1], '=', locale.strcoll(cands[0], cands[1]))
501
502
503
504 print("======================================================================")
505 print("GNUmed i18n")
506 print("")
507 print("authors:", __author__)
508 print("license:", __license__)
509 print("======================================================================")
510
511 activate_locale()
512 print("system locale: ", system_locale, "; levels:", system_locale_level)
513 print("likely encoding:", get_encoding())
514
515 if len(sys.argv) > 2:
516 install_domain(domain = sys.argv[2])
517 else:
518 install_domain()
519
520 test_strcoll()
521
522
523
524
525
526
527 tmp = _('Translate this or i18n into <en_EN> will not work properly !')
528
529
530