diff -r c7fdb0637d0b Lib/locale.py --- a/Lib/locale.py Fri Sep 13 11:46:24 2013 +0300 +++ b/Lib/locale.py Fri Sep 13 16:26:25 2013 +0300 @@ -336,6 +336,22 @@ # overridden below) _setlocale = setlocale +def _replace_encoding(code, encoding): + if '.' in code: + langname = code[:code.index('.')] + else: + langname = code + # Convert the encoding to a C lib compatible encoding string + norm_encoding = encodings.normalize_encoding(encoding) + #print('norm encoding: %r' % norm_encoding) + norm_encoding = encodings.aliases.aliases.get(norm_encoding, + norm_encoding) + #print('aliased encoding: %r' % norm_encoding) + encoding = locale_encoding_alias.get(norm_encoding, + norm_encoding) + #print('found encoding %r' % encoding) + return langname + '.' + encoding + def normalize(localename): """ Returns a normalized locale code for the given locale @@ -352,55 +368,71 @@ does. """ - # Normalize the locale name and extract the encoding - fullname = localename.lower() - if ':' in fullname: + # Normalize the locale name and extract the encoding and modifier + code = localename.lower() + if ':' in code: # ':' is sometimes used as encoding delimiter. - fullname = fullname.replace(':', '.') - if '.' in fullname: - langname, encoding = fullname.split('.')[:2] - fullname = langname + '.' + encoding + code = code.replace(':', '.') + if '@' in code: + code, modifier = code.split('@', 1) else: - langname = fullname + modifier = '' + if '.' in code: + langname, encoding = code.split('.')[:2] + else: + langname = code encoding = '' - # First lookup: fullname (possibly with encoding) - norm_encoding = encoding.replace('-', '') - norm_encoding = norm_encoding.replace('_', '') - lookup_name = langname + '.' + encoding + # First lookup: fullname (possibly with encoding and modifier) + lang_enc = langname + if encoding: + norm_encoding = encoding.replace('-', '') + norm_encoding = norm_encoding.replace('_', '') + lang_enc += '.' + norm_encoding + lookup_name = lang_enc + if modifier: + lookup_name += '@' + modifier code = locale_alias.get(lookup_name, None) if code is not None: return code - #print 'first lookup failed' + #print('first lookup failed') - # Second try: langname (without encoding) - code = locale_alias.get(langname, None) - if code is not None: - #print 'langname lookup succeeded' - if '.' in code: - langname, defenc = code.split('.') - else: - langname = code - defenc = '' - if encoding: - # Convert the encoding to a C lib compatible encoding string - norm_encoding = encodings.normalize_encoding(encoding) - #print 'norm encoding: %r' % norm_encoding - norm_encoding = encodings.aliases.aliases.get(norm_encoding, - norm_encoding) - #print 'aliased encoding: %r' % norm_encoding - encoding = locale_encoding_alias.get(norm_encoding, - norm_encoding) - else: - encoding = defenc - #print 'found encoding %r' % encoding - if encoding: - return langname + '.' + encoding - else: - return langname + if modifier: + # Second try: fullname without modifier (possibly with encoding) + code = locale_alias.get(lang_enc, None) + if code is not None: + #print('lookup without modifier succeeded') + if '@' not in code: + return code + '@' + modifier + if code.split('@', 1)[1].lower() == modifier: + return code + #print('second lookup failed') - else: - return localename + if encoding: + # Third try: langname (without encoding, possibly with modifier) + lookup_name = langname + if modifier: + lookup_name += '@' + modifier + code = locale_alias.get(lookup_name, None) + if code is not None: + #print('lookup without encoding succeeded') + if '@' not in code: + return _replace_encoding(code, encoding) + code, modifier = code.split('@', 1) + return _replace_encoding(code, encoding) + '@' + modifier + + if modifier: + # Fourth try: langname (without encoding and modifier) + code = locale_alias.get(langname, None) + if code is not None: + #print('lookup without modifier and encoding succeeded') + if '@' not in code: + return _replace_encoding(code, encoding) + '@' + modifier + code, defmod = code.split('@', 1) + if defmod.lower() == modifier: + return _replace_encoding(code, encoding) + '@' + defmod + + return localename def _parse_localename(localename): @@ -419,12 +451,16 @@ code = normalize(localename) if '@' in code: # Deal with locale modifiers - code, modifier = code.split('@') - if modifier == 'euro' and '.' not in code: - # Assume Latin-9 for @euro locales. This is bogus, - # since some systems may use other encodings for these - # locales. Also, we ignore other modifiers. - return code, 'iso-8859-15' + code, modifier = code.split('@', 1) + if '.' not in code: + if modifier == 'euro': + # Assume Latin-9 for @euro locales. This is bogus, + # since some systems may use other encodings for these + # locales. Also, we ignore other modifiers. + return code, 'iso-8859-15' + if '.' in modifier: + # Devanagari modifier placed before encoding. + return code, modifier.split('.')[1] if '.' in code: return tuple(code.split('.')[:2]) @@ -861,12 +897,12 @@ 'catalan': 'ca_ES.ISO8859-1', 'cextend': 'en_US.ISO8859-1', 'cextend.en': 'en_US.ISO8859-1', - 'chinese-s': 'zh_CN.eucCN', + 'chinese-s': 'zh_CN.gb2312', 'chinese-t': 'zh_TW.eucTW', 'croatian': 'hr_HR.ISO8859-2', 'cs': 'cs_CZ.ISO8859-2', 'cs_cs': 'cs_CZ.ISO8859-2', - 'cs_cs.iso88592': 'cs_CS.ISO8859-2', + 'cs_cs.iso88592': 'cs_CZ.ISO8859-2', 'cs_cz': 'cs_CZ.ISO8859-2', 'cs_cz.iso88592': 'cs_CZ.ISO8859-2', 'cy': 'cy_GB.ISO8859-1', @@ -1170,8 +1206,8 @@ 'he_il.cp1255': 'he_IL.CP1255', 'he_il.iso88598': 'he_IL.ISO8859-8', 'he_il.microsoftcp1255': 'he_IL.CP1255', - 'hebrew': 'iw_IL.ISO8859-8', - 'hebrew.iso88598': 'iw_IL.ISO8859-8', + 'hebrew': 'he_IL.ISO8859-8', + 'hebrew.iso88598': 'he_IL.ISO8859-8', 'hi': 'hi_IN.ISCII-DEV', 'hi_in': 'hi_IN.ISCII-DEV', 'hi_in.isciidev': 'hi_IN.ISCII-DEV', @@ -1427,10 +1463,10 @@ 'se_no': 'se_NO.UTF-8', 'serbocroatian': 'sr_RS.UTF-8@latin', 'sh': 'sr_RS.UTF-8@latin', - 'sh_ba.iso88592@bosnia': 'sr_CS.ISO8859-2', - 'sh_hr': 'sh_HR.ISO8859-2', + 'sh_ba.iso88592@bosnia': 'sr_RS.ISO8859-2', + 'sh_hr': 'hr_HR.ISO8859-2', 'sh_hr.iso88592': 'hr_HR.ISO8859-2', - 'sh_sp': 'sr_CS.ISO8859-2', + 'sh_sp': 'sr_RS.ISO8859-2', 'sh_yu': 'sr_RS.UTF-8@latin', 'si': 'si_LK.UTF-8', 'si_lk': 'si_LK.UTF-8', @@ -1445,8 +1481,8 @@ 'slovak': 'sk_SK.ISO8859-2', 'slovene': 'sl_SI.ISO8859-2', 'slovenian': 'sl_SI.ISO8859-2', - 'sp': 'sr_CS.ISO8859-5', - 'sp_yu': 'sr_CS.ISO8859-5', + 'sp': 'sr_RS.ISO8859-5', + 'sp_yu': 'sr_RS.ISO8859-5', 'spanish': 'es_ES.ISO8859-1', 'spanish.iso88591': 'es_ES.ISO8859-1', 'spanish_spain': 'es_ES.ISO8859-1', @@ -1459,23 +1495,24 @@ 'sr@latin': 'sr_RS.UTF-8@latin', 'sr@latn': 'sr_RS.UTF-8@latin', 'sr_cs': 'sr_RS.UTF-8', - 'sr_cs.iso88592': 'sr_CS.ISO8859-2', - 'sr_cs.iso88592@latn': 'sr_CS.ISO8859-2', - 'sr_cs.iso88595': 'sr_CS.ISO8859-5', + 'sr_cs.iso88592': 'sr_RS.ISO8859-2', + 'sr_cs.iso88592@latn': 'sr_RS.ISO8859-2', + 'sr_cs.iso88595': 'sr_RS.ISO8859-5', 'sr_cs.utf8@latn': 'sr_RS.UTF-8@latin', 'sr_cs@latn': 'sr_RS.UTF-8@latin', 'sr_me': 'sr_ME.UTF-8', 'sr_rs': 'sr_RS.UTF-8', + 'sr_rs.cp1251': 'sr_RS.CP1251', 'sr_rs.utf8@latn': 'sr_RS.UTF-8@latin', 'sr_rs@latin': 'sr_RS.UTF-8@latin', 'sr_rs@latn': 'sr_RS.UTF-8@latin', - 'sr_sp': 'sr_CS.ISO8859-2', + 'sr_sp': 'sr_RS.ISO8859-2', 'sr_yu': 'sr_RS.UTF-8@latin', - 'sr_yu.cp1251@cyrillic': 'sr_CS.CP1251', - 'sr_yu.iso88592': 'sr_CS.ISO8859-2', - 'sr_yu.iso88595': 'sr_CS.ISO8859-5', - 'sr_yu.iso88595@cyrillic': 'sr_CS.ISO8859-5', - 'sr_yu.microsoftcp1251@cyrillic': 'sr_CS.CP1251', + 'sr_yu.cp1251@cyrillic': 'sr_RS.CP1251', + 'sr_yu.iso88592': 'sr_RS.ISO8859-2', + 'sr_yu.iso88595': 'sr_RS.ISO8859-5', + 'sr_yu.iso88595@cyrillic': 'sr_RS.ISO8859-5', + 'sr_yu.microsoftcp1251@cyrillic': 'sr_RS.CP1251', 'sr_yu.utf8@cyrillic': 'sr_RS.UTF-8', 'sr_yu@cyrillic': 'sr_RS.UTF-8', 'ss': 'ss_ZA.ISO8859-1', @@ -1537,8 +1574,8 @@ 'uk_ua.iso88595': 'uk_UA.ISO8859-5', 'uk_ua.koi8u': 'uk_UA.KOI8-U', 'uk_ua.microsoftcp1251': 'uk_UA.CP1251', - 'univ': 'en_US.utf', - 'universal': 'en_US.utf', + 'univ': 'en_US.UTF-8', + 'universal': 'en_US.UTF-8', 'universal.utf8@ucs4': 'en_US.UTF-8', 'ur': 'ur_PK.CP1256', 'ur_pk': 'ur_PK.CP1256', @@ -1570,10 +1607,10 @@ 'yi_us': 'yi_US.CP1255', 'yi_us.cp1255': 'yi_US.CP1255', 'yi_us.microsoftcp1255': 'yi_US.CP1255', - 'zh': 'zh_CN.eucCN', + 'zh': 'zh_CN.gb2312', 'zh_cn': 'zh_CN.gb2312', 'zh_cn.big5': 'zh_TW.big5', - 'zh_cn.euc': 'zh_CN.eucCN', + 'zh_cn.euc': 'zh_CN.gb2312', 'zh_cn.gb18030': 'zh_CN.gb18030', 'zh_cn.gb2312': 'zh_CN.gb2312', 'zh_cn.gbk': 'zh_CN.gbk', diff -r c7fdb0637d0b Lib/test/test_locale.py --- a/Lib/test/test_locale.py Fri Sep 13 11:46:24 2013 +0300 +++ b/Lib/test/test_locale.py Fri Sep 13 16:26:25 2013 +0300 @@ -365,6 +365,74 @@ self.assertLess(locale.strxfrm('à'), locale.strxfrm('b')) +class NormalizeTest(unittest.TestCase): + def check(self, localename, expected): + self.assertEqual(locale.normalize(localename), expected, msg=localename) + + def test_locale_alias(self): + for localename, alias in locale.locale_alias.items(): + with self.subTest(locale=(localename, alias)): + self.check(localename, alias) + with self.subTest(locale=(localename, alias)): + self.check(alias, alias) + + def test_empty(self): + self.check('', '') + + def test_c(self): + self.check('c', 'C') + self.check('posix', 'C') + + def test_english(self): + self.check('en', 'en_US.ISO8859-1') + self.check('EN', 'en_US.ISO8859-1') + self.check('en_US', 'en_US.ISO8859-1') + self.check('en_us', 'en_US.ISO8859-1') + self.check('en_GB', 'en_GB.ISO8859-1') + self.check('en_US.UTF-8', 'en_US.UTF-8') + self.check('en_US.utf8', 'en_US.UTF-8') + self.check('en_US:UTF-8', 'en_US.UTF-8') + self.check('en_US.ISO8859-1', 'en_US.ISO8859-1') + self.check('en_US.US-ASCII', 'en_US.ISO8859-1') + self.check('english', 'en_EN.ISO8859-1') + + def test_hyphenated_encoding(self): + self.check('az_AZ.iso88599e', 'az_AZ.ISO8859-9E') + self.check('az_AZ.ISO8859-9E', 'az_AZ.ISO8859-9E') + self.check('tt_RU.koi8c', 'tt_RU.KOI8-C') + self.check('tt_RU.KOI8-C', 'tt_RU.KOI8-C') + self.check('lo_LA.cp1133', 'lo_LA.IBM-CP1133') + self.check('lo_LA.ibmcp1133', 'lo_LA.IBM-CP1133') + self.check('lo_LA.IBM-CP1133', 'lo_LA.IBM-CP1133') + self.check('uk_ua.microsoftcp1251', 'uk_UA.CP1251') + self.check('uk_ua.microsoft-cp1251', 'uk_UA.CP1251') + self.check('ka_ge.georgianacademy', 'ka_GE.GEORGIAN-ACADEMY') + self.check('ka_GE.GEORGIAN-ACADEMY', 'ka_GE.GEORGIAN-ACADEMY') + self.check('cs_CS.iso88592', 'cs_CZ.ISO8859-2') + self.check('cs_CS.ISO8859-2', 'cs_CZ.ISO8859-2') + self.check('cs_CZ.ISO88592', 'cs_CZ.ISO8859-2') + self.check('cs_CZ.ISO8859-2', 'cs_CZ.ISO8859-2') + + def test_euro_modifier(self): + self.check('de_DE@euro', 'de_DE.ISO8859-15') + self.check('en_US.ISO8859-15@euro', 'en_US.ISO8859-15') + + def test_latin_modifier(self): + self.check('be_BY.UTF-8@latin', 'be_BY.UTF-8@latin') + self.check('sr_RS.UTF-8@latin', 'sr_RS.UTF-8@latin') + + def test_valencia_modifier(self): + self.check('ca_ES.UTF-8@valencia', 'ca_ES.UTF-8@valencia') + self.check('ca_ES@valencia', 'ca_ES.ISO8859-1@valencia') + self.check('ca@valencia', 'ca_ES.ISO8859-1@valencia') + + def test_devanagari_modifier(self): + self.check('ks_in@devanagari', 'ks_IN@devanagari.UTF-8') + self.check('ks_IN@devanagari.UTF-8', 'ks_IN@devanagari.UTF-8') + self.check('sd', 'sd_IN@devanagari.UTF-8') + self.check('sd_IN@devanagari.UTF-8', 'sd_IN@devanagari.UTF-8') + + class TestMiscellaneous(unittest.TestCase): def test_getpreferredencoding(self): # Invoke getpreferredencoding to make sure it does not cause exceptions.