[FIX] tools.translate: when loading entries from a PO file, use also …

…the POT.

On Launchpad, as commented on bug 933496, PO entries (and their comments) are
shared between series. This mean that e.g. the 7.0 series can have the wrong
`reference` comments (those beginning with #:) as they would be copied from say
the trunk series. Those `reference` comments are used to import translations
and look them up.

This patch adds a few lines of code to tools.translate so that targets
defined in the POT `reference` comments are used in addition to those from the
PO file.

Also adds a test module to validate the new behavior.

This patch stems from:
 - the 6.1 branch by Vo Minh Thu:
    https://code.launchpad.net/+branch/~openerp-dev/openobject-server/6.1-fix-po-targets-933496-vmt
 - the 7.0 port by Numerigraphe:
    https://code.launchpad.net/~numerigraphe-team/openobject-server/7.0-fix-po-targets-933496-vmt

openerp/tests/addons/test_translation_import/__init__.py

@@ -0,0 +1,2 @@
+# -*- coding: utf-8 -*-
+import models

openerp/tests/addons/test_translation_import/__openerp__.py

@@ -0,0 +1,15 @@
+# -*- coding: utf-8 -*-
+{
+    'name': 'test-translation-import',
+    'version': '0.1',
+    'category': 'Tests',
+    'description': """A module to test translation import.""",
+    'author': 'OpenERP SA',
+    'maintainer': 'OpenERP SA',
+    'website': 'http://www.openerp.com',
+    'depends': ['base'],
+    'data': ['view.xml'],
+    'test': ['tests.yml'],
+    'installable': True,
+    'auto_install': False,
+}

openerp/tests/addons/test_translation_import/i18n/fr.po

@@ -0,0 +1,52 @@
+# This is a test PO file, not a true french translation.
+# See the POT file for further information.
+msgid ""
+msgstr ""
+"Project-Id-Version: OpenERP Server 6.1\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2012-10-17 12:36+0000\n"
+"PO-Revision-Date: 2012-10-17 12:36+0000\n"
+"Last-Translator: <>\n"
+"Language-Team: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: \n"
+"Plural-Forms: \n"
+
+# Note: there is normally an additional line:
+#     #: code:addons/test_translation_import/models.py:17
+# This line is present in the POT and removed here to test the translation
+# import behavior.
+#. module: test_translation_import
+#: field:test.translation.import,name:0
+#, python-format
+msgid "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB"
+msgstr "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB in french"
+
+#. module: test_translation_import
+#: code:addons/test_translation_import/models.py:14
+#, python-format
+msgid "Ijkl"
+msgstr "Ijkl in french"
+
+#. module: test_translation_import
+#: model:ir.model,name:test_translation_import.model_test_translation_import
+msgid "test.translation.import"
+msgstr "test.translation.import in french"
+
+#. module: test_translation_import
+#: help:test.translation.import,name:0
+msgid "Efgh"
+msgstr "Efgh in french"
+
+#. module: test_translation_import
+#: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import
+#: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import
+msgid "Test translation import"
+msgstr "Test translation import in french"
+
+#. module: test_translation_import
+#: model:ir.ui.menu,name:test_translation_import.menu_test_translation
+msgid "Test translation"
+msgstr "Test translation in french"
+

openerp/tests/addons/test_translation_import/i18n/test_translation_import.pot

@@ -0,0 +1,58 @@
+# This is a test POT file, not a true template. It is manually maintained
+# to test the import translation behavior of OpenERP.
+#
+# In particular, the
+# `1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB` source is
+# given with two targets (the #: comments): `code` and `field`. The code one is
+# removed in the fr.po file. Still, the import should generate a database entry
+# for the `code` one. I.e. the targets defined in the POT must be added to the
+# targets defined in the PO file.  This was done to fix a bug, as reported by
+# lp:933496.
+#
+msgid ""
+msgstr ""
+"Project-Id-Version: OpenERP Server 6.1\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2012-10-17 12:36+0000\n"
+"PO-Revision-Date: 2012-10-17 12:36+0000\n"
+"Last-Translator: <>\n"
+"Language-Team: \n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: \n"
+"Plural-Forms: \n"
+
+#. module: test_translation_import
+#: code:addons/test_translation_import/models.py:17
+#: field:test.translation.import,name:0
+#, python-format
+msgid "1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB"
+msgstr ""
+
+#. module: test_translation_import
+#: code:addons/test_translation_import/models.py:14
+#, python-format
+msgid "Ijkl"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.model,name:test_translation_import.model_test_translation_import
+msgid "test.translation.import"
+msgstr ""
+
+#. module: test_translation_import
+#: help:test.translation.import,name:0
+msgid "Efgh"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.actions.act_window,name:test_translation_import.action_test_translation_import
+#: model:ir.ui.menu,name:test_translation_import.menu_test_translation_import
+msgid "Test translation import"
+msgstr ""
+
+#. module: test_translation_import
+#: model:ir.ui.menu,name:test_translation_import.menu_test_translation
+msgid "Test translation"
+msgstr ""
+

openerp/tests/addons/test_translation_import/models.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+import openerp
+from openerp.tools.translate import _
+
+class m(openerp.osv.orm.TransientModel):
+    """ A model to provide source strings.
+    """
+    _name = 'test.translation.import'
+
+    _columns = {
+        'name': openerp.osv.fields.char(
+            '1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB',
+            size=32, help='Efgh'),
+    }
+
+    _('Ijkl')
+
+    # With the name label above, this source string should be generated twice.
+    _('1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB')
+

openerp/tests/addons/test_translation_import/tests.yml

@@ -0,0 +1,13 @@
+-
+    Load the french translation.
+-
+    !python {model: ir.translation }: |
+        import openerp
+        openerp.tools.trans_load(cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False)
+-
+    Assert we have loaded the correct number of entries for the given source string.
+-
+    !python {model: ir.translation }: |
+        ids = self.search(cr, uid,
+            [('src', '=', '1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB')])
+        assert len(ids) == 2, "2 entries are expected, got %s instead." % len(ids)

openerp/tests/addons/test_translation_import/tests/__init__.py

@@ -0,0 +1,9 @@
+# -*- coding: utf-8 -*-
+import unittest2
+
+import test_term_count
+
+suite = [
+    test_term_count
+    ]
+

openerp/tests/addons/test_translation_import/tests/test_term_count.py

@@ -0,0 +1,16 @@
+# -*- coding: utf-8 -*-
+
+import openerp
+from openerp.tests import common
+
+class TestTermCount(common.TransactionCase):
+
+    def test_count_term(self):
+        """
+        Just make sure we have as many translation entries as we wanted.
+        """
+        openerp.tools.trans_load(self.cr, 'test_translation_import/i18n/fr.po', 'fr_FR', verbose=False)
+        ids = self.registry('ir.translation').search(self.cr, self.uid,
+            [('src', '=', '1XBUO5PUYH2RYZSA1FTLRYS8SPCNU1UYXMEYMM25ASV7JC2KTJZQESZYRV9L8CGB')])
+        self.assertEqual(len(ids), 2)
+

openerp/tests/addons/test_translation_import/view.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<openerp>
+    <data>
+
+        <record id="action_test_translation_import" model="ir.actions.act_window">
+            <field name="name">Test translation import</field>
+            <field name="type">ir.actions.act_window</field>
+            <field name="res_model">test.translation.import</field>
+            <field name="view_type">form</field>
+            <field name="view_mode">tree,form</field>
+            <field name="target">current</field>
+        </record>
+
+        <menuitem icon="STOCK_PREFERENCES" id="base.menu_tests" name="Tests"/>
+
+        <menuitem id="menu_test_translation" parent="base.menu_tests" name="Test translation"/>
+
+        <menuitem id="menu_test_translation_import"
+            name="Test translation import"
+            action="action_test_translation_import"
+            parent="menu_test_translation"/>
+    </data>
+</openerp>

openerp/tools/translate.py

@@ -313,6 +313,10 @@ def next(self):
                     if not line.startswith('module:'):
                         comments.append(line)
                 elif line.startswith('#:'):
+                    # Process the `reference` comments. Each line can specify
+                    # multiple targets (e.g. model, view, code, selection,
+                    # ...). For each target, we will return an additional
+                    # entry.
                     for lpart in line[2:].strip().split(' '):
                         trans_info = lpart.strip().split(':',2)
                         if trans_info and len(trans_info) == 2:
@@ -362,6 +366,9 @@ def next(self):
                 line = self.lines.pop(0).strip()
 
             if targets and not fuzzy:
+                # Use the first target for the current entry (returned at the
+                # end of this next() call), and keep the others to generate
+                # additional entries (returned the next next() calls).
                 trans_type, name, res_id = targets.pop(0)
                 for t, n, r in targets:
                     if t == trans_type == 'code': continue
@@ -945,6 +952,10 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
             # lets create the language with locale information
             lang_obj.load_lang(cr, SUPERUSER_ID, lang=lang, lang_name=lang_name)
 
+        # Parse also the POT: it will possibly provide additional targets.
+        # (Because the POT comments are correct on Launchpad but not the
+        # PO comments due to a Launchpad limitation. See LP bug 933496.)
+        pot_reader = []
 
         # now, the serious things: we read the language file
         fileobj.seek(0)
@@ -957,19 +968,42 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
         elif fileformat == 'po':
             reader = TinyPoFile(fileobj)
             f = ['type', 'name', 'res_id', 'src', 'value', 'comments']
+
+            # Make a reader for the POT file and be somewhat defensive for the
+            # stable branch.
+            if fileobj.name.endswith('.po'):
+                try:
+                    # Normally the path looks like /path/to/xxx/i18n/lang.po
+                    # and we try to find the corresponding
+                    # /path/to/xxx/i18n/xxx.pot file.
+                    head, _ = os.path.split(fileobj.name)
+                    head2, _ = os.path.split(head)
+                    head3, tail3 = os.path.split(head2)
+                    pot_handle = misc.file_open(os.path.join(head3, tail3, 'i18n', tail3 + '.pot'))
+                    pot_reader = TinyPoFile(pot_handle)
+                except:
+                    pass
+
         else:
             _logger.error('Bad file format: %s', fileformat)
             raise Exception(_('Bad file format'))
 
+        # Read the POT `reference` comments, and keep them indexed by source
+        # string.
+        pot_targets = {}
+        for type, name, res_id, src, _, comments in pot_reader:
+            if type is not None:
+                pot_targets.setdefault(src, {'value': None, 'targets': []})
+                pot_targets[src]['targets'].append((type, name, res_id))
+
         # read the rest of the file
-        line = 1
         irt_cursor = trans_obj._get_import_cursor(cr, SUPERUSER_ID, context=context)
 
-        for row in reader:
-            line += 1
+        def process_row(row):
+            """Process a single PO (or POT) entry."""
             # skip empty rows and rows where the translation field (=last fiefd) is empty
             #if (not row) or (not row[-1]):
-            #    continue
+            #    return
 
             # dictionary which holds values for this line of the csv file
             # {'lang': ..., 'type': ..., 'name': ..., 'res_id': ...,
@@ -979,9 +1013,17 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
             for i, field in enumerate(f):
                 dic[field] = row[i]
 
+            # Get the `reference` comments from the POT.
+            src = row[3]
+            if pot_reader and src in pot_targets:
+                pot_targets[src]['targets'] = filter(lambda x: x != row[:3], pot_targets[src]['targets'])
+                pot_targets[src]['value'] = row[4]
+                if not pot_targets[src]['targets']:
+                    del pot_targets[src]
+
             # This would skip terms that fail to specify a res_id
             if not dic.get('res_id'):
-                continue
+                return
 
             res_id = dic.pop('res_id')
             if res_id and isinstance(res_id, (int, long)) \
@@ -1002,6 +1044,21 @@ def trans_load_data(cr, fileobj, fileformat, lang, lang_name=None, verbose=True,
 
             irt_cursor.push(dic)
 
+        # First process the entries from the PO file (doing so also fills/removes
+        # the entries from the POT file).
+        for row in reader:
+            process_row(row)
+
+        # Then process the entries implied by the POT file (which is more
+        # correct w.r.t. the targets) if some of them remain.
+        pot_rows = []
+        for src in pot_targets:
+            value = pot_targets[src]['value']
+            for type, name, res_id in pot_targets[src]['targets']:
+                pot_rows.append((type, name, res_id, src, value, comments))
+        for row in pot_rows:
+            process_row(row)
+
         irt_cursor.finish()
         trans_obj.clear_caches()
         if verbose: