Commits

Takayuki Shimizukawa committed 870a91c Merge

Merged in shimizukawa/sphinx-fix-i18n-fork/#955 (pull request #86)

Comments (0)

Files changed (10)

sphinx/environment.py

     Replace translatable nodes with their translated doctree.
     """
     default_priority = 0
+
     def apply(self):
         env = self.document.settings.env
         settings, source = self.document.settings, self.document['source']
             if not isinstance(patch, nodes.paragraph):
                 continue # skip for now
 
-            # copy text children
-            for i, child in enumerate(patch.children):
-                if isinstance(child, nodes.Text):
-                    child.parent = node
-                    node.children[i] = child
+            # auto-numbered foot note reference should use original 'ids'.
+            is_autonumber_footnote_ref = lambda node: \
+                    isinstance(node, nodes.footnote_reference) \
+                    and node.get('auto') == 1
+            old_foot_refs = node.traverse(is_autonumber_footnote_ref)
+            new_foot_refs = patch.traverse(is_autonumber_footnote_ref)
+            if len(old_foot_refs) != len(new_foot_refs):
+                env.warn_node('inconsistent footnote references in '
+                              'translated message', node)
+            for old, new in zip(old_foot_refs, new_foot_refs):
+                new['ids'] = old['ids']
+                self.document.autofootnote_refs.remove(old)
+                self.document.note_autofootnote_ref(new)
+
+            # reference should use original 'refname'.
+            # * reference target ".. _Python: ..." is not translatable.
+            # * section refname is not translatable.
+            # * inline reference "`Python <...>`_" has no 'refname'.
+            is_refnamed_ref = lambda node: \
+                    isinstance(node, nodes.reference) \
+                    and 'refname' in node
+            old_refs = node.traverse(is_refnamed_ref)
+            new_refs = patch.traverse(is_refnamed_ref)
+            applied_refname_map = {}
+            if len(old_refs) != len(new_refs):
+                env.warn_node('inconsistent references in '
+                              'translated message', node)
+            for new in new_refs:
+                if new['refname'] in applied_refname_map:
+                    # 2nd appearance of the reference
+                    new['refname'] = applied_refname_map[new['refname']]
+                elif old_refs:
+                    # 1st appearance of the reference in old_refs
+                    old = old_refs.pop(0)
+                    refname = old['refname']
+                    new['refname'] = refname
+                    applied_refname_map[new['refname']] = refname
+                else:
+                    # the reference is not found in old_refs
+                    applied_refname_map[new['refname']] = new['refname']
+
+                self.document.note_refname(new)
+
+            # update leaves
+            for child in patch.children:
+                child.parent = node
+            node.children = patch.children
 
 
 class SphinxStandaloneReader(standalone.Reader):
                 if 'orphan' in self.metadata[docname]:
                     continue
                 self.warn(docname, 'document isn\'t included in any toctree')
-

tests/root/contents.txt

    extensions
    versioning/index
    only
+   i18n/index
 
    Python <http://python.org/>
 

tests/root/i18n/external_links.po

+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2012, foof
+# This file is distributed under the same license as the foo package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: sphinx 1.0\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2012-11-22 08:28\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+msgid "i18n with external links"
+msgstr "EXTERNAL LINKS"
+
+msgid "External link to Python_."
+msgstr "EXTERNAL LINK TO Python_."
+
+msgid "Internal link to `i18n with external links`_."
+msgstr "`EXTERNAL LINKS`_ IS INTERNAL LINK."
+
+msgid "Inline link by `Sphinx <http://sphinx-doc.org>`_."
+msgstr "INLINE LINK BY `SPHINX <http://sphinx-doc.org>`_."
+
+msgid "Unnamed link__."
+msgstr "UNNAMED LINK__."

tests/root/i18n/external_links.txt

+:tocdepth: 2
+
+i18n with external links
+========================
+.. #1044 external-links-dont-work-in-localized-html
+
+* External link to Python_.
+* Internal link to `i18n with external links`_.
+* Inline link by `Sphinx <http://sphinx-doc.org>`_.
+* Unnamed link__.
+
+.. _Python: http://python.org
+.. __: http://google.com

tests/root/i18n/footnote.po

+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2012, foof
+# This file is distributed under the same license as the foo package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: sphinx 1.0\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2012-11-22 08:28\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+msgid "i18n with Footnote"
+msgstr "I18N WITH FOOTNOTE"
+
+msgid "[100]_ Contents [#]_ for `i18n with Footnote`_ [ref]_"
+msgstr "`I18N WITH FOOTNOTE`_ INCLUDE THIS CONTENTS [ref]_ [#]_ [100]_"
+
+msgid "This is a auto numbered footnote."
+msgstr "THIS IS A AUTO NUMBERED FOOTNOTE."
+
+msgid "This is a named footnote."
+msgstr "THIS IS A NAMED FOOTNOTE."
+
+msgid "This is a numbered footnote."
+msgstr "THIS IS A NUMBERED FOOTNOTE."
+

tests/root/i18n/footnote.txt

+:tocdepth: 2
+
+i18n with Footnote
+==================
+.. #955 cant-build-html-with-footnotes-when-using
+
+[100]_ Contents [#]_ for `i18n with Footnote`_ [ref]_
+
+.. [#] This is a auto numbered footnote.
+.. [ref] This is a named footnote.
+.. [100] This is a numbered footnote.

tests/root/i18n/index.txt

+.. toctree::
+   :maxdepth: 2
+   :numbered:
+
+   footnote
+   external_links
+   refs_inconsistency

tests/root/i18n/refs_inconsistency.po

+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2012, foof
+# This file is distributed under the same license as the foo package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: sphinx 1.0\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2012-12-05 08:28\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+msgid "i18n with refs inconsistency"
+msgstr "I18N WITH REFS INCONSISTENCY"
+
+msgid "[100]_ for [#]_ footnote [ref2]_."
+msgstr "FOR FOOTNOTE [ref2]_."
+
+msgid "for reference_."
+msgstr "reference_ FOR reference_."
+
+msgid "normal text."
+msgstr "ORPHAN REFERENCE: `I18N WITH REFS INCONSISTENCY`_."
+
+msgid "This is a auto numbered footnote."
+msgstr "THIS IS A AUTO NUMBERED FOOTNOTE."
+
+msgid "This is a named footnote."
+msgstr "THIS IS A NAMED FOOTNOTE."
+
+msgid "This is a numbered footnote."
+msgstr "THIS IS A NUMBERED FOOTNOTE."
+

tests/root/i18n/refs_inconsistency.txt

+:tocdepth: 2
+
+i18n with refs inconsistency
+=============================
+
+* [100]_ for [#]_ footnote [ref2]_.
+* for reference_.
+* normal text.
+
+.. [#] This is a auto numbered footnote.
+.. [ref2] This is a named footnote.
+.. [100] This is a numbered footnote.
+.. _reference: http://www.example.com

tests/test_intl.py

 """
 
 from subprocess import Popen, PIPE
+import re
+import os
+from StringIO import StringIO
 
 from util import *
 from util import SkipTest
 
 
+warnfile = StringIO()
+
+
 def setup_module():
     (test_root / 'xx' / 'LC_MESSAGES').makedirs()
     # Compile all required catalogs into binary format (*.mo).
-    for catalog in 'bom', 'subdir':
-        try:
-            p = Popen(['msgfmt', test_root / '%s.po' % catalog, '-o',
-                test_root / 'xx' / 'LC_MESSAGES' / '%s.mo' % catalog],
-                stdout=PIPE, stderr=PIPE)
-        except OSError:
-            # The test will fail the second time it's run if we don't
-            # tear down here. Not sure if there's a more idiomatic way
-            # of ensuring that teardown gets run in the event of an
-            # exception from the setup function.
-            teardown_module()
-            raise SkipTest  # most likely msgfmt was not found
-        else:
-            stdout, stderr = p.communicate()
-            if p.returncode != 0:
-                print stdout
-                print stderr
-                assert False, 'msgfmt exited with return code %s' % p.returncode
-            assert (test_root / 'xx' / 'LC_MESSAGES' / ('%s.mo' % catalog)
-                   ).isfile(), 'msgfmt failed'
+    for dirpath, dirs, files in os.walk(test_root):
+        dirpath = path(dirpath)
+        for f in [f for f in files if f.endswith('.po')]:
+            po = dirpath / f
+            mo = test_root / 'xx' / 'LC_MESSAGES' / (
+                    os.path.relpath(po[:-3], test_root) + '.mo')
+            if not mo.parent.exists():
+                mo.parent.makedirs()
+            try:
+                p = Popen(['msgfmt', po, '-o', mo],
+                    stdout=PIPE, stderr=PIPE)
+            except OSError:
+                # The test will fail the second time it's run if we don't
+                # tear down here. Not sure if there's a more idiomatic way
+                # of ensuring that teardown gets run in the event of an
+                # exception from the setup function.
+                teardown_module()
+                raise SkipTest  # most likely msgfmt was not found
+            else:
+                stdout, stderr = p.communicate()
+                if p.returncode != 0:
+                    print stdout
+                    print stderr
+                    assert False, 'msgfmt exited with return code %s' % p.returncode
+                assert mo.isfile(), 'msgfmt failed'
 
 
 def teardown_module():
     app.builder.build(['subdir/includes'])
     result = (app.outdir / 'subdir' / 'includes.txt').text(encoding='utf-8')
     assert result.startswith(u"\ntranslation\n***********\n\n")
+
+
+@with_app(buildername='html', cleanenv=True,
+          confoverrides={'language': 'xx', 'locale_dirs': ['.'],
+                         'gettext_compact': False})
+def test_i18n_footnote_break_refid(app):
+    """test for #955 cant-build-html-with-footnotes-when-using"""
+    app.builder.build(['i18n/footnote'])
+    result = (app.outdir / 'i18n' / 'footnote.html').text(encoding='utf-8')
+    # expect no error by build
+
+
+@with_app(buildername='text', cleanenv=True,
+          confoverrides={'language': 'xx', 'locale_dirs': ['.'],
+                         'gettext_compact': False})
+def test_i18n_footnote_regression(app):
+    """regression test for fix #955"""
+    app.builder.build(['i18n/footnote'])
+    result = (app.outdir / 'i18n' / 'footnote.txt').text(encoding='utf-8')
+    expect = (u"\nI18N WITH FOOTNOTE"
+              u"\n******************\n"  # underline matches new translation
+              u"\nI18N WITH FOOTNOTE INCLUDE THIS CONTENTS [ref] [1] [100]\n"
+              u"\n[1] THIS IS A AUTO NUMBERED FOOTNOTE.\n"
+              u"\n[ref] THIS IS A NAMED FOOTNOTE.\n"
+              u"\n[100] THIS IS A NUMBERED FOOTNOTE.\n")
+    assert result == expect
+
+
+@with_app(buildername='text', warning=warnfile, cleanenv=True,
+          confoverrides={'language': 'xx', 'locale_dirs': ['.'],
+                         'gettext_compact': False})
+def test_i18n_warn_for_number_of_references_inconsistency(app):
+    app.builddir.rmtree(True)
+    app.builder.build(['i18n/refs_inconsistency'])
+    result = (app.outdir / 'i18n' / 'refs_inconsistency.txt').text(encoding='utf-8')
+    expect = (u"\nI18N WITH REFS INCONSISTENCY"
+              u"\n****************************\n"
+              u"\n* FOR FOOTNOTE [ref2].\n"
+              u"\n* reference FOR reference.\n"
+              u"\n* ORPHAN REFERENCE: I18N WITH REFS INCONSISTENCY.\n"
+              u"\n[1] THIS IS A AUTO NUMBERED FOOTNOTE.\n"
+              u"\n[ref2] THIS IS A NAMED FOOTNOTE.\n"
+              u"\n[100] THIS IS A NUMBERED FOOTNOTE.\n")
+    assert result == expect
+
+    warnings = warnfile.getvalue().replace(os.sep, '/')
+    warning_fmt = u'.*/i18n/refs_inconsistency.txt:\\d+: ' \
+          u'WARNING: inconsistent %s in translated message\n'
+    expected_warning_expr = (
+        warning_fmt % 'footnote references' +
+        warning_fmt % 'references' +
+        warning_fmt % 'references')
+    assert re.search(expected_warning_expr, warnings)
+
+
+@with_app(buildername='html', cleanenv=True,
+          confoverrides={'language': 'xx', 'locale_dirs': ['.'],
+                         'gettext_compact': False})
+def test_i18n_link_to_undefined_reference(app):
+    app.builder.build(['i18n/refs_inconsistency'])
+    result = (app.outdir / 'i18n' / 'refs_inconsistency.html').text(encoding='utf-8')
+
+    expected_expr = """<a class="reference external" href="http://www.example.com">reference</a>"""
+    assert len(re.findall(expected_expr, result)) == 2
+
+    expected_expr = """<a class="reference internal" href="#reference">reference</a>"""
+    assert len(re.findall(expected_expr, result)) == 0
+
+    expected_expr = """<a class="reference internal" href="#i18n-with-refs-inconsistency">I18N WITH REFS INCONSISTENCY</a>"""
+    assert len(re.findall(expected_expr, result)) == 1
+
+
+@with_app(buildername='html', cleanenv=True,
+          confoverrides={'language': 'xx', 'locale_dirs': ['.'],
+                         'gettext_compact': False})
+def test_i18n_keep_external_links(app):
+    """regression test for #1044"""
+    app.builder.build(['i18n/external_links'])
+    result = (app.outdir / 'i18n' / 'external_links.html').text(encoding='utf-8')
+
+    # external link check
+    expect_line = u"""<li>EXTERNAL LINK TO <a class="reference external" href="http://python.org">Python</a>.</li>"""
+    matched = re.search('^<li>EXTERNAL LINK TO .*$', result, re.M)
+    matched_line = ''
+    if matched:
+        matched_line = matched.group()
+    assert expect_line == matched_line
+
+    # internal link check
+    expect_line = u"""<li><a class="reference internal" href="#i18n-with-external-links">EXTERNAL LINKS</a> IS INTERNAL LINK.</li>"""
+    matched = re.search('^<li><a .* IS INTERNAL LINK.</li>$', result, re.M)
+    matched_line = ''
+    if matched:
+        matched_line = matched.group()
+    assert expect_line == matched_line
+
+    # inline link check
+    expect_line = u"""<li>INLINE LINK BY <a class="reference external" href="http://sphinx-doc.org">SPHINX</a>.</li>"""
+    matched = re.search('^<li>INLINE LINK BY .*$', result, re.M)
+    matched_line = ''
+    if matched:
+        matched_line = matched.group()
+    assert expect_line == matched_line
+
+    # unnamed link check
+    expect_line = u"""<li>UNNAMED <a class="reference external" href="http://google.com">LINK</a>.</li>"""
+    matched = re.search('^<li>UNNAMED .*$', result, re.M)
+    matched_line = ''
+    if matched:
+        matched_line = matched.group()
+    assert expect_line == matched_line