Index: babel/core.py =================================================================== --- babel/core.py (revision 411) +++ babel/core.py (working copy) @@ -223,7 +223,7 @@ def _data(self): if self.__data is None: - self.__data = localedata.load(str(self)) + self.__data = localedata.LocaleDataDict(localedata.load(str(self))) return self.__data _data = property(_data) @@ -326,7 +326,7 @@ Mapping of script codes to translated script names. >>> Locale('de', 'DE').variants['1901'] - u'alte deutsche Rechtschreibung' + u'Alte deutsche Rechtschreibung' :type: `dict` """) @@ -481,7 +481,7 @@ >>> Locale('en', 'US').time_zones['Europe/London']['long']['daylight'] u'British Summer Time' >>> Locale('en', 'US').time_zones['America/St_Johns']['city'] - u'St. John\u2019s' + u"St. John's" :type: `dict` """) Index: babel/localedata.py =================================================================== --- babel/localedata.py (revision 411) +++ babel/localedata.py (working copy) @@ -23,6 +23,7 @@ import threading except ImportError: import dummy_threading as threading +from UserDict import DictMixin __all__ = ['exists', 'load'] __docformat__ = 'restructuredtext en' @@ -31,6 +32,7 @@ _cache_lock = threading.RLock() _dirname = os.path.join(os.path.dirname(__file__), 'localedata') + def exists(name): """Check whether locale data is available for the given locale. @@ -42,6 +44,7 @@ return True return os.path.exists(os.path.join(_dirname, '%s.dat' % name)) + def list(): """Return a list of all locale identifiers for which locale data is available. @@ -54,6 +57,7 @@ os.path.splitext(filename) for filename in os.listdir(_dirname) ] if extension == '.dat' and stem != 'root'] + def load(name): """Load the locale data for the given locale. @@ -107,17 +111,79 @@ finally: _cache_lock.release() + def merge(dict1, dict2): """Merge the data from `dict2` into the `dict1` dictionary, making copies of nested dictionaries. + >>> d = {1: 'foo', 3: 'baz'} + >>> merge(d, {1: 'Foo', 2: 'Bar'}) + >>> d + {1: 'Foo', 2: 'Bar', 3: 'baz'} + + >>> d = {1: {1: 'FOO', 2: 'BAR'}, 2: 'foo'} + >>> merge(d, {1: {3: 'BAZ'}}) + >>> d + {1: {1: 'FOO', 2: 'BAR', 3: 'BAZ'}, 2: 'foo'} + :param dict1: the dictionary to merge into :param dict2: the dictionary containing the data that should be merged """ - for key, value in dict2.items(): - if value is not None: - if type(value) is dict: - dict1[key] = dict1.get(key, {}).copy() - merge(dict1[key], value) + for key, val2 in dict2.items(): + if val2 is not None: + if type(val2) is dict: + val1 = dict1.get(key, {}) + if type(val1) is Alias: + dict1[key] = (val1, val2) + else: + dict1[key] = val1.copy() + merge(dict1[key], val2) + elif type(val2) is Alias: + dict1[key] = (val2, dict1[key]) else: - dict1[key] = value + val1 = dict1.get(key) + if type(val1) is Alias: + dict1[key] = (val1, val2) + else: + dict1[key] = val2 + + +class Alias(object): + + def __init__(self, keys): + self.keys = keys + + def __repr__(self): + return '<%s %r>' % (type(self).__name__, self.keys) + + def resolve(self, data): + base = data + for key in self.keys: + data = data[key] + if type(data) is Alias: + data = data.resolve(base) + return data + + +class LocaleDataDict(DictMixin, dict): + + def __init__(self, data, base=None): + dict.__init__(self, data) + if base is None: + base = self + self.base = base + + def __getitem__(self, key): + val = dict.__getitem__(self, key) + if type(val) is Alias: # resolve an alias + val = val.resolve(self.base) + if type(val) is tuple: # Merge a partial dict with an alias + alias, others = val + val = alias.resolve(self.base).copy() + merge(val, others) + if type(val) is dict: # Return a nested alias-resolving dict + val = LocaleDataDict(val, base=self.base) + return val + + def copy(self): + return LocaleDataDict(dict.copy(self), base=self.base) Index: babel/tests/localedata.py =================================================================== --- babel/tests/localedata.py (revision 411) +++ babel/tests/localedata.py (working copy) @@ -16,9 +16,53 @@ from babel import localedata + +class MergeResolveTestCase(unittest.TestCase): + + def test_merge_items(self): + d = {1: 'foo', 3: 'baz'} + localedata.merge(d, {1: 'Foo', 2: 'Bar'}) + self.assertEqual({1: 'Foo', 2: 'Bar', 3: 'baz'}, d) + + def test_merge_nested_dict(self): + d1 = { + 'x': {'a': 1, 'b': 2, 'c': 3} + } + d2 = { + 'x': {'a': 1, 'b': 12, 'd': 14} + } + localedata.merge(d1, d2) + self.assertEqual({ + 'x': {'a': 1, 'b': 12, 'c': 3, 'd': 14} + }, d1) + + + def test_merge_with_alias_and_resolve(self): + alias = localedata.Alias('x') + d1 = { + 'x': {'a': 1, 'b': 2, 'c': 3}, + 'y': alias + } + d2 = { + 'x': {'a': 1, 'b': 12, 'd': 14}, + 'y': {'b': 22, 'e': 25} + } + localedata.merge(d1, d2) + self.assertEqual({ + 'x': {'a': 1, 'b': 12, 'c': 3, 'd': 14}, + 'y': (alias, {'b': 22, 'e': 25}) + }, d1) + d = localedata.LocaleDataDict(d1) + self.assertEqual({ + 'x': {'a': 1, 'b': 12, 'c': 3, 'd': 14}, + 'y': {'a': 1, 'b': 22, 'c': 3, 'd': 14, 'e': 25} + }, dict(d.items())) + + def suite(): suite = unittest.TestSuite() suite.addTest(doctest.DocTestSuite(localedata)) + suite.addTest(unittest.makeSuite(MergeResolveTestCase)) return suite if __name__ == '__main__': Index: babel/tests/dates.py =================================================================== --- babel/tests/dates.py (revision 411) +++ babel/tests/dates.py (working copy) @@ -29,6 +29,11 @@ fmt = dates.DateTimeFormat(d, locale='cs_CZ') self.assertEqual('1.', fmt['LLL']) + def test_abbreviated_month_alias(self): + d = date(2006, 3, 8) + fmt = dates.DateTimeFormat(d, locale='de_DE') + self.assertEqual(u'Mär', fmt['LLL']) + def test_week_of_year_first(self): d = date(2006, 1, 8) fmt = dates.DateTimeFormat(d, locale='de_DE') @@ -187,7 +192,7 @@ tz = timezone('Europe/Paris') t = time(15, 30, tzinfo=tz) fmt = dates.DateTimeFormat(t, locale='fr_FR') - self.assertEqual(u'Heure de l’Europe centrale', fmt['vvvv']) + self.assertEqual(u'heure d’Europe centrale', fmt['vvvv']) def test_hour_formatting(self): l = 'en_US' Index: babel/numbers.py =================================================================== --- babel/numbers.py (revision 411) +++ babel/numbers.py (working copy) @@ -165,9 +165,9 @@ >>> format_currency(1099.98, 'USD', locale='en_US') u'$1,099.98' >>> format_currency(1099.98, 'USD', locale='es_CO') - u'US$ 1.099,98' + u'US$\\xa01.099,98' >>> format_currency(1099.98, 'EUR', locale='de_DE') - u'1.099,98 \\u20ac' + u'1.099,98\\xa0\\u20ac' The pattern can also be specified explicitly: Index: babel/dates.py =================================================================== --- babel/dates.py (revision 411) +++ babel/dates.py (working copy) @@ -189,7 +189,7 @@ string is used for GMT: >>> get_timezone_gmt(dt, 'long', locale='fr_FR') - u'HMG-08:00' + u'UTC-08:00' :param datetime: the ``datetime`` object; if `None`, the current date and time in UTC is used Index: scripts/import_cldr.py =================================================================== --- scripts/import_cldr.py (revision 411) +++ scripts/import_cldr.py (working copy) @@ -16,6 +16,7 @@ from optparse import OptionParser import os import pickle +import re import sys try: from xml.etree.ElementTree import parse @@ -26,6 +27,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(sys.argv[0]), '..')) from babel import dates, numbers +from babel.localedata import Alias weekdays = {'mon': 0, 'tue': 1, 'wed': 2, 'thu': 3, 'fri': 4, 'sat': 5, 'sun': 6} @@ -36,6 +38,7 @@ def any(iterable): return filter(None, list(iterable)) + def _text(elem): buf = [elem.text or ''] for child in elem: @@ -43,6 +46,35 @@ buf.append(elem.tail or '') return u''.join(filter(None, buf)).strip() + +NAME_RE = re.compile(r"^\w+$") +TYPE_ATTR_RE = re.compile(r"^\w+\[@type='(.*?)'\]$") + +NAME_MAP = { + 'dateFormats': 'date_formats', + 'dateTimeFormats': 'datetime_formats', + 'eraAbbr': 'abbreviated', + 'eraNames': 'wide', + 'eraNarrow': 'narrow', + 'timeFormats': 'time_formats' +} + +def _translate_alias(ctxt, path): + parts = path.split('/') + keys = ctxt[:] + for part in parts: + if part == '..': + keys.pop() + else: + match = TYPE_ATTR_RE.match(part) + if match: + keys.append(match.group(1)) + else: + assert NAME_RE.match(part) + keys.append(NAME_MAP.get(part, part)) + return keys + + def main(): parser = OptionParser(usage='%prog path/to/cldr') options, args = parser.parse_args() @@ -109,6 +141,8 @@ stem, ext = os.path.splitext(filename) if ext != '.xml': continue + #if stem != 'root': + # break tree = parse(os.path.join(srcdir, 'main', filename)) data = {} @@ -227,88 +261,139 @@ months = data.setdefault('months', {}) for ctxt in calendar.findall('months/monthContext'): - ctxts = months.setdefault(ctxt.attrib['type'], {}) + ctxt_type = ctxt.attrib['type'] + ctxts = months.setdefault(ctxt_type, {}) for width in ctxt.findall('monthWidth'): - widths = ctxts.setdefault(width.attrib['type'], {}) - for elem in width.findall('month'): - if 'draft' in elem.attrib and int(elem.attrib['type']) in widths: - continue - widths[int(elem.attrib.get('type'))] = unicode(elem.text) + width_type = width.attrib['type'] + widths = ctxts.setdefault(width_type, {}) + for elem in width.getiterator(): + if elem.tag == 'month': + if 'draft' in elem.attrib and int(elem.attrib['type']) in widths: + continue + widths[int(elem.attrib.get('type'))] = unicode(elem.text) + elif elem.tag == 'alias': + assert elem.attrib['source'] == 'locale' + ctxts[width_type] = Alias( + _translate_alias(['months', ctxt_type, width_type], + elem.attrib['path']) + ) days = data.setdefault('days', {}) for ctxt in calendar.findall('days/dayContext'): - ctxts = days.setdefault(ctxt.attrib['type'], {}) + ctxt_type = ctxt.attrib['type'] + ctxts = days.setdefault(ctxt_type, {}) for width in ctxt.findall('dayWidth'): - widths = ctxts.setdefault(width.attrib['type'], {}) - for elem in width.findall('day'): - dtype = weekdays[elem.attrib['type']] - if 'draft' in elem.attrib and dtype in widths: - continue - widths[dtype] = unicode(elem.text) + width_type = width.attrib['type'] + widths = ctxts.setdefault(width_type, {}) + for elem in width.getiterator(): + if elem.tag == 'day': + dtype = weekdays[elem.attrib['type']] + if 'draft' in elem.attrib and dtype in widths: + continue + widths[dtype] = unicode(elem.text) + elif elem.tag == 'alias': + assert elem.attrib['source'] == 'locale' + ctxts[width_type] = Alias( + _translate_alias(['days', ctxt_type, width_type], + elem.attrib['path']) + ) quarters = data.setdefault('quarters', {}) for ctxt in calendar.findall('quarters/quarterContext'): + ctxt_type = ctxt.attrib['type'] ctxts = quarters.setdefault(ctxt.attrib['type'], {}) for width in ctxt.findall('quarterWidth'): - widths = ctxts.setdefault(width.attrib['type'], {}) - for elem in width.findall('quarter'): + width_type = width.attrib['type'] + widths = ctxts.setdefault(width_type, {}) + for elem in width.getiterator(): + if elem.tag == 'quarter': + if 'draft' in elem.attrib and int(elem.attrib['type']) in widths: + continue + widths[int(elem.attrib.get('type'))] = unicode(elem.text) + elif elem.tag == 'alias': + assert elem.attrib['source'] == 'locale' + ctxts[width_type] = Alias( + _translate_alias(['quarters', ctxt_type, width_type], + elem.attrib['path']) + ) + + eras = data.setdefault('eras', {}) + for width in calendar.findall('eras/*'): + width_type = NAME_MAP[width.tag] + widths = eras.setdefault(width_type, {}) + for elem in width.getiterator(): + if elem.tag == 'era': if 'draft' in elem.attrib and int(elem.attrib['type']) in widths: continue widths[int(elem.attrib.get('type'))] = unicode(elem.text) + elif elem.tag == 'alias': + assert elem.attrib['source'] == 'locale' + eras[width_type] = Alias( + _translate_alias(['eras', width_type], + elem.attrib['path']) + ) - eras = data.setdefault('eras', {}) - for width in calendar.findall('eras/*'): - ewidth = { - 'eraAbbr': 'abbreviated', - 'eraNames': 'wide', - 'eraNarrow': 'narrow', - }[width.tag] - widths = eras.setdefault(ewidth, {}) - for elem in width.findall('era'): - if 'draft' in elem.attrib and int(elem.attrib['type']) in widths: - continue - widths[int(elem.attrib.get('type'))] = unicode(elem.text) - # AM/PM periods = data.setdefault('periods', {}) for elem in calendar.findall('am'): - if 'draft' in elem.attrib and elem.tag in periods: + if ('draft' in elem.attrib or 'alt' in elem.attrib) and elem.tag in periods: continue periods[elem.tag] = unicode(elem.text) for elem in calendar.findall('pm'): - if 'draft' in elem.attrib and elem.tag in periods: + if ('draft' in elem.attrib or 'alt' in elem.attrib) and elem.tag in periods: continue periods[elem.tag] = unicode(elem.text) date_formats = data.setdefault('date_formats', {}) - for elem in calendar.findall('dateFormats/dateFormatLength'): - if 'draft' in elem.attrib and elem.attrib.get('type') in date_formats: - continue - try: - date_formats[elem.attrib.get('type')] = \ - dates.parse_pattern(unicode(elem.findtext('dateFormat/pattern'))) - except ValueError, e: - print>>sys.stderr, 'ERROR: %s' % e + for format in calendar.findall('dateFormats'): + for elem in format.getiterator(): + if elem.tag == 'dateFormatLength': + if 'draft' in elem.attrib and \ + elem.attrib.get('type') in date_formats: + continue + try: + date_formats[elem.attrib.get('type')] = \ + dates.parse_pattern(unicode(elem.findtext('dateFormat/pattern'))) + except ValueError, e: + print>>sys.stderr, 'ERROR: %s' % e + elif elem.tag == 'alias': + date_formats = Alias(_translate_alias( + ['date_formats'], elem.attrib['path']) + ) time_formats = data.setdefault('time_formats', {}) - for elem in calendar.findall('timeFormats/timeFormatLength'): - if 'draft' in elem.attrib and elem.attrib.get('type') in time_formats: - continue - try: - time_formats[elem.attrib.get('type')] = \ - dates.parse_pattern(unicode(elem.findtext('timeFormat/pattern'))) - except ValueError, e: - print>>sys.stderr, 'ERROR: %s' % e + for format in calendar.findall('timeFormats'): + for elem in format.getiterator(): + if elem.tag == 'timeFormatLength': + if 'draft' in elem.attrib and \ + elem.attrib.get('type') in time_formats: + continue + try: + time_formats[elem.attrib.get('type')] = \ + dates.parse_pattern(unicode(elem.findtext('timeFormat/pattern'))) + except ValueError, e: + print>>sys.stderr, 'ERROR: %s' % e + elif elem.tag == 'alias': + time_formats = Alias(_translate_alias( + ['time_formats'], elem.attrib['path']) + ) datetime_formats = data.setdefault('datetime_formats', {}) - for elem in calendar.findall('dateTimeFormats/dateTimeFormatLength'): - if 'draft' in elem.attrib and elem.attrib.get('type') in datetime_formats: - continue - try: - datetime_formats[elem.attrib.get('type')] = \ - unicode(elem.findtext('dateTimeFormat/pattern')) - except ValueError, e: - print>>sys.stderr, 'ERROR: %s' % e + for format in calendar.findall('dateTimeFormats'): + for elem in format.getiterator(): + if elem.tag == 'dateTimeFormatLength': + if 'draft' in elem.attrib and \ + elem.attrib.get('type') in datetime_formats: + continue + try: + datetime_formats[elem.attrib.get('type')] = \ + unicode(elem.findtext('dateTimeFormat/pattern')) + except ValueError, e: + print>>sys.stderr, 'ERROR: %s' % e + elif elem.tag == 'alias': + datetime_formats = Alias(_translate_alias( + ['datetime_formats'], elem.attrib['path']) + ) # @@ -360,5 +445,6 @@ finally: outfile.close() + if __name__ == '__main__': main()