Changeset 376

Timestamp:

06/12/08 18:26:52 (13 months ago)

Author:

aronacher

Message:

Added JavaScript extractor

Location:

trunk

Files:

• babel/messages/extract.py (modified) (1 diff)
• babel/messages/jslexer.py (added)
• babel/messages/tests/extract.py (modified) (2 diffs)
• ChangeLog (modified) (1 diff)
• setup.py (modified) (1 diff)

trunk/babel/messages/extract.py

@@ -429 +429 @@
         elif tok == NAME and value in keywords:
             funcname = value
+
+def extract_javascript(fileobj, keywords, comment_tags, options):
+    """Extract messages from JavaScript source code.
+
+    :param fileobj: the seekable, file-like object the messages should be
+                    extracted from
+    :param keywords: a list of keywords (i.e. function names) that should be
+                     recognized as translation functions
+    :param comment_tags: a list of translator tags to search for and include
+                         in the results
+    :param options: a dictionary of additional options (optional)
+    :return: an iterator over ``(lineno, funcname, message, comments)`` tuples
+    :rtype: ``iterator``
+    """
+    from babel.messages.jslexer import tokenize, unquote_string
+    funcname = message_lineno = None
+    messages = []
+    last_argument = None
+    translator_comments = []
+    encoding = options.get('encoding', 'utf-8')
+    last_token = None
+    call_stack = -1
+
+    for token in tokenize(fileobj.read().decode(encoding)):
+        if token.type == 'operator' and token.value == '(':
+            if funcname:
+                message_lineno = token.lineno
+                call_stack += 1
+
+        elif call_stack == -1 and token.type == 'linecomment':
+            value = token.value[2:].strip()
+            if translator_comments and \
+               translator_comments[-1][0] == token.lineno - 1:
+                translator_comments.append((token.lineno, value))
+                continue
+
+            for comment_tag in comment_tags:
+                if value.startswith(comment_tag):
+                    translator_comments.append((token.lineno, value.strip()))
+                    break
+
+        elif token.type == 'multilinecomment':
+            # only one multi-line comment may preceed a translation
+            translator_comments = []
+            value = token.value[2:-2].strip()
+            for comment_tag in comment_tags:
+                if value.startswith(comment_tag):
+                    lines = value.splitlines()
+                    if lines:
+                        lines[0] = lines[0].strip()
+                        lines[1:] = dedent('\n'.join(lines[1:])).splitlines()
+                        for offset, line in enumerate(lines):
+                            translator_comments.append((token.lineno + offset,
+                                                        line))
+                    break
+
+        elif funcname and call_stack == 0:
+            if token.type == 'operator' and token.value == ')':
+                if last_argument is not None:
+                    messages.append(last_argument)
+                if len(messages) > 1:
+                    messages = tuple(messages)
+                elif messages:
+                    messages = messages[0]
+                else:
+                    messages = None
+
+                # Comments don't apply unless they immediately preceed the
+                # message
+                if translator_comments and \
+                   translator_comments[-1][0] < message_lineno - 1:
+                    translator_comments = []
+
+                if messages is not None:
+                    yield (message_lineno, funcname, messages,
+                           [comment[1] for comment in translator_comments])
+
+                funcname = message_lineno = last_argument = None
+                translator_comments = []
+                messages = []
+                call_stack = -1
+
+            elif token.type == 'string':
+                last_argument = unquote_string(token.value)
+
+            elif token.type == 'operator' and token.value == ',':
+                if last_argument is not None:
+                    messages.append(last_argument)
+                    last_argument = None
+                else:
+                    messages.append(None)
+
+        elif call_stack > 0 and token.type == 'operator' \
+             and token.value == ')':
+            call_stack -= 1
+
+        elif funcname and call_stack == -1:
+            funcname = None
+
+        elif call_stack == -1 and token.type == 'name' and \
+             token.value in keywords and \
+             (last_token is None or last_token.type != 'name' or
+              last_token.value != 'function'):
+            funcname = token.value
+
+        last_token = token

trunk/babel/messages/tests/extract.py

@@ -322 +322 @@
                           u'a prefix too'], messages[1][2])
 
+class ExtractJavaScriptTestCase(unittest.TestCase):
+
+    def test_simple_extract(self):
+        buf = StringIO("""\
+msg1 = _('simple')
+msg2 = gettext('simple')
+msg3 = ngettext('s', 'p', 42)
+        """)
+        messages = \
+            list(extract.extract('javascript', buf, extract.DEFAULT_KEYWORDS,
+                                 [], {}))
+
+        self.assertEqual([(1, 'simple', []),
+                          (2, 'simple', []),
+                          (3, ('s', 'p'), [])], messages)
+
+    def test_various_calls(self):
+        buf = StringIO("""\
+msg1 = _(i18n_arg.replace(/"/, '"'))
+msg2 = ungettext(i18n_arg.replace(/"/, '"'), multi_arg.replace(/"/, '"'), 2)
+msg3 = ungettext("Babel", multi_arg.replace(/"/, '"'), 2)
+msg4 = ungettext(i18n_arg.replace(/"/, '"'), "Babels", 2)
+msg5 = ungettext('bunny', 'bunnies', parseInt(Math.random() * 2 + 1))
+msg6 = ungettext(arg0, 'bunnies', rparseInt(Math.random() * 2 + 1))
+msg7 = _(hello.there)
+msg8 = gettext('Rabbit')
+msg9 = dgettext('wiki', model.addPage())
+msg10 = dngettext(domain, 'Page', 'Pages', 3)
+""")
+        messages = \
+            list(extract.extract('javascript', buf, extract.DEFAULT_KEYWORDS, [],
+                                 {}))
+        self.assertEqual([(5, (u'bunny', u'bunnies'), []),
+                          (8, u'Rabbit', []),
+                          (10, (u'Page', u'Pages'), [])], messages)
+
+    def test_message_with_line_comment(self):
+        buf = StringIO("""\
+// NOTE: hello
+msg = _('Bonjour à tous')
+""")
+        messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
+        self.assertEqual(u'Bonjour à tous', messages[0][2])
+        self.assertEqual([u'NOTE: hello'], messages[0][3])
+
+    def test_message_with_multiline_comment(self):
+        buf = StringIO("""\
+/* NOTE: hello
+   and bonjour
+     and servus */
+msg = _('Bonjour à tous')
+""")
+        messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
+        self.assertEqual(u'Bonjour à tous', messages[0][2])
+        self.assertEqual([u'NOTE: hello', 'and bonjour', '  and servus'], messages[0][3])
+
+    def test_ignore_function_definitions(self):
+        buf = StringIO("""\
+function gettext(value) {
+    return translations[language][value] || value;
+}""")
+
+        messages = list(extract.extract_javascript(buf, ('gettext',), [], {}))
+        self.assertEqual(messages, [])
+
+    def test_misplaced_comments(self):
+        buf = StringIO("""\
+/* NOTE: this won't show up */
+foo()
+
+/* NOTE: this will */
+msg = _('Something')
+
+// NOTE: this will show up
+// too.
+msg = _('Something else')
+
+// NOTE: but this won't
+bar()
+
+_('no comment here')
+""")
+        messages = list(extract.extract_javascript(buf, ('_',), ['NOTE:'], {}))
+        self.assertEqual(u'Something', messages[0][2])
+        self.assertEqual([u'NOTE: this will'], messages[0][3])
+        self.assertEqual(u'Something else', messages[1][2])
+        self.assertEqual([u'NOTE: this will show up', 'too.'], messages[1][3])
+        self.assertEqual(u'no comment here', messages[2][2])
+        self.assertEqual([], messages[2][3])
+
 class ExtractTestCase(unittest.TestCase):
 
@@ -383 +473 @@
     suite.addTest(doctest.DocTestSuite(extract))
     suite.addTest(unittest.makeSuite(ExtractPythonTestCase))
+    suite.addTest(unittest.makeSuite(ExtractJavaScriptTestCase))
     suite.addTest(unittest.makeSuite(ExtractTestCase))
     return suite

trunk/ChangeLog

@@ -6 +6 @@
  * The stripping of the comment tags in comments is optional now and
    is done for each line in a comment.
-        
+ * a JavaScript extractor was added.
+
 
 Version 0.9.2

trunk/setup.py

@@ -76 +76 @@
     ignore = babel.messages.extract:extract_nothing
     python = babel.messages.extract:extract_python
+    javascript = babel.messages.extract:extract_javascript
     """,