tools/test_pomerge.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from translate.tools import pomerge
   5 from translate.storage import factory
   6 from translate.storage import po
   7 from translate.storage import xliff
   8 from translate.misc import wStringIO
   9
  10 class TestPOMerge:
  11     xliffskeleton = '''<?xml version="1.0" ?>
  12 <xliff version="1.1" xmlns="urn:oasis:names:tc:xliff:document:1.1">
  13   <file original="filename.po" source-language="en-US" datatype="po">
  14     <body>
  15         %s
  16     </body>
  17   </file>
  18 </xliff>'''
  19
  20     def mergestore(self, templatesource, inputsource):
  21         """merges the sources of the given files and returns a new pofile object"""
  22         templatefile = wStringIO.StringIO(templatesource)
  23         inputfile = wStringIO.StringIO(inputsource)
  24         outputfile = wStringIO.StringIO()
  25         assert pomerge.mergestore(inputfile, outputfile, templatefile)
  26         outputpostring = outputfile.getvalue()
  27         outputpofile = po.pofile(outputpostring)
  28         return outputpofile
  29
  30     def mergexliff(self, templatesource, inputsource):
  31         """merges the sources of the given files and returns a new xlifffile object"""
  32         templatefile = wStringIO.StringIO(templatesource)
  33         inputfile = wStringIO.StringIO(inputsource)
  34         outputfile = wStringIO.StringIO()
  35         assert pomerge.mergestore(inputfile, outputfile, templatefile)
  36         outputxliffstring = outputfile.getvalue()
  37         print "Generated XML:"
  38         print outputxliffstring
  39         outputxlifffile = xliff.xlifffile(outputxliffstring)
  40         return outputxlifffile
  41
  42     def countunits(self, pofile):
  43         """returns the number of non-header items"""
  44         if pofile.units[0].isheader():
  45             return len(pofile.units) - 1
  46         else:
  47             return len(pofile.units)
  48
  49     def singleunit(self, pofile):
  50         """checks that the pofile contains a single non-header unit, and returns it"""
  51         assert self.countunits(pofile) == 1
  52         return pofile.units[-1]
  53
  54     def test_simplemerge(self):
  55         """checks that a simple po entry merges OK"""
  56         templatepo = '''#: simple.test\nmsgid "Simple String"\nmsgstr ""\n'''
  57         inputpo = '''#: simple.test\nmsgid "Simple String"\nmsgstr "Dimpled Ring"\n'''
  58         pofile = self.mergestore(templatepo, inputpo)
  59         pounit = self.singleunit(pofile)
  60         assert pounit.source == "Simple String"
  61         assert pounit.target == "Dimpled Ring"
  62
  63     def test_replacemerge(self):
  64         """checks that a simple po entry merges OK"""
  65         templatepo = '''#: simple.test\nmsgid "Simple String"\nmsgstr "Dimpled Ring"\n'''
  66         inputpo = '''#: simple.test\nmsgid "Simple String"\nmsgstr "Dimpled King"\n'''
  67         pofile = self.mergestore(templatepo, inputpo)
  68         pounit = self.singleunit(pofile)
  69         assert pounit.source == "Simple String"
  70         assert pounit.target == "Dimpled King"
  71
  72     def test_merging_locations(self):
  73         """check that locations on seperate lines are output in Gettext form of all on one line"""
  74         templatepo = '''#: location.c:1\n#: location.c:2\nmsgid "Simple String"\nmsgstr ""\n'''
  75         inputpo = '''#: location.c:1\n#: location.c:2\nmsgid "Simple String"\nmsgstr "Dimpled Ring"\n'''
  76         expectedpo = '''#: location.c:1%slocation.c:2\nmsgid "Simple String"\nmsgstr "Dimpled Ring"\n''' % po.lsep
  77         pofile = self.mergestore(templatepo, inputpo)
  78         print pofile
  79         assert str(pofile) == expectedpo
  80
  81     def test_reflowed_source_comments(self):
  82         """ensure that we don't duplicate source comments (locations) if they have been reflowed"""
  83         templatepo = '''#: newMenu.label\n#: newMenu.accesskey\nmsgid "&New"\nmsgstr ""\n'''
  84         newpo = '''#: newMenu.label newMenu.accesskey\nmsgid "&New"\nmsgstr "&Nuwe"\n'''
  85         expectedpo = '''#: newMenu.label%snewMenu.accesskey\nmsgid "&New"\nmsgstr "&Nuwe"\n''' % po.lsep
  86         pofile = self.mergestore(templatepo, newpo)
  87         pounit = self.singleunit(pofile)
  88         print pofile
  89         assert str(pofile) == expectedpo
  90
  91     def test_comments_with_blank_lines(self):
  92         """ensure that we don't loose empty newlines in comments"""
  93         templatepo = '''# # ***** BEGIN LICENSE BLOCK *****
  94 # Version: MPL 1.1/GPL 2.0/LGPL 2.1
  95 #
  96 # bla bla
  97 msgid "bla"
  98 msgstr "blabla"
  99 '''
 100         newpo = templatepo
 101         expectedpo =  templatepo
 102         pofile = self.mergestore(templatepo, newpo)
 103         pounit = self.singleunit(pofile)
 104         print pofile
 105         assert str(pofile) == expectedpo
 106
 107     def test_merge_dont_delete_unassociated_comments(self):
 108         """ensure that we do not delete comments in the PO file that are not assocaited with a message block"""
 109         templatepo = '''# Lonely comment\n\n# Translation comment\nmsgid "Bob"\nmsgstr "Toolmaker"\n'''
 110         mergepo = '''# Translation comment\nmsgid "Bob"\nmsgstr "Builder"\n'''
 111         expectedpo = '''# Lonely comment\n\n# Translation comment\nmsgid "Bob"\nmsgstr "Builder"\n'''
 112         pofile = self.mergestore(templatepo, mergepo)
 113 #        pounit = self.singleunit(pofile)
 114         print pofile
 115         assert str(pofile) == expectedpo
 116
 117     def test_preserve_format_trailing_newlines(self):
 118         """Test that we can merge messages correctly that end with a newline"""
 119         templatepo = '''msgid "Simple string\\n"\nmsgstr ""\n'''
 120         mergepo = '''msgid "Simple string\\n"\nmsgstr "Dimpled ring\\n"\n'''
 121         expectedpo = '''msgid "Simple string\\n"\nmsgstr "Dimpled ring\\n"\n'''
 122         pofile = self.mergestore(templatepo, mergepo)
 123         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 124         assert str(pofile) == expectedpo
 125
 126         templatepo = '''msgid ""\n"Simple string\\n"\nmsgstr ""\n'''
 127         mergepo = '''msgid ""\n"Simple string\\n"\nmsgstr ""\n"Dimpled ring\\n"\n'''
 128         expectedpo = '''msgid ""\n"Simple string\\n"\nmsgstr "Dimpled ring\\n"\n'''
 129         pofile = self.mergestore(templatepo, mergepo)
 130         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 131         assert str(pofile) == expectedpo
 132
 133     def test_preserve_format_minor_start_and_end_of_sentence_changes(self):
 134         """Test that we are not too fussy about large diffs for simple changes at the start or end of a sentence"""
 135         templatepo = '''msgid "Target type:"\nmsgstr "Doelsoort"\n\n'''
 136         mergepo = '''msgid "Target type:"\nmsgstr "Doelsoort:"\n'''
 137         expectedpo = mergepo
 138         pofile = self.mergestore(templatepo, mergepo)
 139         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 140         assert str(pofile) == expectedpo
 141
 142         templatepo = '''msgid "&Select"\nmsgstr "Kies"\n\n'''
 143         mergepo = '''msgid "&Select"\nmsgstr "&Kies"\n'''
 144         expectedpo = mergepo
 145         pofile = self.mergestore(templatepo, mergepo)
 146         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 147         assert str(pofile) == expectedpo
 148
 149         templatepo = '''msgid "en-us, en"\nmsgstr "en-us, en"\n'''
 150         mergepo = '''msgid "en-us, en"\nmsgstr "af-za, af, en-za, en-gb, en-us, en"\n'''
 151         expectedpo = mergepo
 152         pofile = self.mergestore(templatepo, mergepo)
 153         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 154         assert str(pofile) == expectedpo
 155
 156     def test_preserve_format_last_entry_in_a_file(self):
 157         """The last entry in a PO file is usualy not followed by an empty line.  Test that we preserve this"""
 158         templatepo = '''msgid "First"\nmsgstr ""\n\nmsgid "Second"\nmsgstr ""\n'''
 159         mergepo = '''msgid "First"\nmsgstr "Eerste"\n\nmsgid "Second"\nmsgstr "Tweede"\n'''
 160         expectedpo = '''msgid "First"\nmsgstr "Eerste"\n\nmsgid "Second"\nmsgstr "Tweede"\n'''
 161         pofile = self.mergestore(templatepo, mergepo)
 162         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 163         assert str(pofile) == expectedpo
 164
 165         templatepo = '''msgid "First"\nmsgstr ""\n\nmsgid "Second"\nmsgstr ""\n\n'''
 166         mergepo = '''msgid "First"\nmsgstr "Eerste"\n\nmsgid "Second"\nmsgstr "Tweede"\n'''
 167         expectedpo = '''msgid "First"\nmsgstr "Eerste"\n\nmsgid "Second"\nmsgstr "Tweede"\n'''
 168         pofile = self.mergestore(templatepo, mergepo)
 169         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 170         assert str(pofile) == expectedpo
 171
 172     def xtest_escape_tabs(self):
 173         """Ensure that input tabs are escaped in the output, like gettext does."""
 174
 175         # The strings below contains the tab character, not spaces.
 176         templatepo = '''msgid "First    Second"\nmsgstr ""\n\n'''
 177         mergepo = '''msgid "First       Second"\nmsgstr "Eerste Tweede"\n'''
 178         expectedpo = r'''imsgid "First\tSecond"
 179 msgstr "Eerste\tTweede"
 180 '''
 181         pofile = self.mergestore(templatepo, mergepo)
 182         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 183         assert str(pofile) == expectedpo
 184
 185     def test_preserve_comments_layout(self):
 186         """Ensure that when we merge with new '# (poconflict)' or other comments that we don't mess formating"""
 187         templatepo = '''#: filename\nmsgid "Desktop Background.bmp"\nmsgstr "Desktop Background.bmp"\n\n'''
 188         mergepo = '''# (pofilter) unchanged: please translate\n#: filename\nmsgid "Desktop Background.bmp"\nmsgstr "Desktop Background.bmp"\n'''
 189         expectedpo = mergepo
 190         pofile = self.mergestore(templatepo, mergepo)
 191         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 192         assert str(pofile) == expectedpo
 193
 194     def test_merge_dos2unix(self):
 195         """Test that merging a comment line with dos newlines doesn't add a new line"""
 196         templatepo = '''# User comment\n# (pofilter) Translate Toolkit comment\n#. Automatic comment\n#: location_comment.c:110\nmsgid "File"\nmsgstr "File"\n\n'''
 197         mergepo =  '''# User comment\r\n# (pofilter) Translate Toolkit comment\r\n#. Automatic comment\r\n#: location_comment.c:110\r\nmsgid "File"\r\nmsgstr "Ifayile"\r\n\r\n'''
 198         expectedpo = '''# User comment\n# (pofilter) Translate Toolkit comment\n#. Automatic comment\n#: location_comment.c:110\nmsgid "File"\nmsgstr "Ifayile"\n'''
 199         pofile = self.mergestore(templatepo, mergepo)
 200         assert str(pofile) == expectedpo
 201
 202         # Unassociated comment
 203         templatepo = '''# Lonely comment\n\n#: location_comment.c:110\nmsgid "Bob"\nmsgstr "Toolmaker"\n'''
 204         mergepo = '''# Lonely comment\r\n\r\n#: location_comment.c:110\r\nmsgid "Bob"\r\nmsgstr "Builder"\r\n\r\n'''
 205         expectedpo = '''# Lonely comment\n\n#: location_comment.c:110\nmsgid "Bob"\nmsgstr "Builder"\n'''
 206         pofile = self.mergestore(templatepo, mergepo)
 207         assert str(pofile) == expectedpo
 208
 209         # New comment
 210         templatepo = '''#: location_comment.c:110\nmsgid "File"\nmsgstr "File"\n\n'''
 211         mergepo =  '''# User comment\r\n# (pofilter) Translate Toolkit comment\r\n#: location_comment.c:110\r\nmsgid "File"\r\nmsgstr "Ifayile"\r\n\r\n'''
 212         expectedpo = '''# User comment\n# (pofilter) Translate Toolkit comment\n#: location_comment.c:110\nmsgid "File"\nmsgstr "Ifayile"\n'''
 213         pofile = self.mergestore(templatepo, mergepo)
 214         assert str(pofile) == expectedpo
 215
 216     def test_xliff_into_xliff(self):
 217         templatexliff = self.xliffskeleton % '''<trans-unit>
 218         <source>red</source>
 219         <target></target>
 220 </trans-unit>'''
 221         mergexliff = self.xliffskeleton % '''<trans-unit>
 222         <source>red</source>
 223         <target>rooi</target>
 224 </trans-unit>'''
 225         xlifffile = self.mergexliff(templatexliff, mergexliff)
 226         assert len(xlifffile.units) == 1
 227         unit = xlifffile.units[0]
 228         assert unit.source == "red"
 229         assert unit.target == "rooi"
 230
 231     def test_po_into_xliff(self):
 232         templatexliff = self.xliffskeleton % '''<trans-unit>
 233         <source>red</source>
 234         <target></target>
 235 </trans-unit>'''
 236         mergepo = 'msgid "red"\nmsgstr "rooi"'
 237         xlifffile = self.mergexliff(templatexliff, mergepo)
 238         assert len(xlifffile.units) == 1
 239         unit = xlifffile.units[0]
 240         assert unit.source == "red"
 241         assert unit.target == "rooi"
 242
 243     def test_xliff_into_po(self):
 244         templatepo = '# my comment\nmsgid "red"\nmsgstr ""'
 245         mergexliff = self.xliffskeleton % '''<trans-unit>
 246         <source>red</source>
 247         <target>rooi</target>
 248 </trans-unit>'''
 249         expectedpo = '# my comment\nmsgid "red"\nmsgstr "rooi"\n'
 250         pofile = self.mergestore(templatepo, mergexliff)
 251         assert str(pofile) == expectedpo
 252
 253     def test_merging_dont_merge_kde_comments_found_in_translation(self):
 254         """If we find a KDE comment in the translation (target) then do not merge it."""
 255
 256         templatepo = '''msgid "_: KDE comment\\n"\n"File"\nmsgstr "File"\n\n'''
 257         mergepo = '''msgid "_: KDE comment\\n"\n"File"\nmsgstr "_: KDE comment\\n"\n"Ifayile"\n\n'''
 258         expectedpo = '''msgid ""\n"_: KDE comment\\n"\n"File"\nmsgstr "Ifayile"\n'''
 259         pofile = self.mergestore(templatepo, mergepo)
 260         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 261         assert str(pofile) == expectedpo
 262
 263         # Translated kde comment.
 264         mergepo = '''msgid "_: KDE comment\\n"\n"File"\nmsgstr "_: KDE kommentaar\\n"\n"Ifayile"\n\n'''
 265         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 266         assert str(pofile) == expectedpo
 267
 268         # multiline KDE comment
 269         templatepo = '''msgid "_: KDE "\n"comment\\n"\n"File"\nmsgstr "File"\n\n'''
 270         mergepo = '''msgid "_: KDE "\n"comment\\n"\n"File"\nmsgstr "_: KDE "\n"comment\\n"\n"Ifayile"\n\n'''
 271         expectedpo = '''msgid ""\n"_: KDE comment\\n"\n"File"\nmsgstr "Ifayile"\n'''
 272         pofile = self.mergestore(templatepo, mergepo)
 273         print "Expected:\n%s\n\nMerged:\n%s" % (expectedpo, str(pofile))
 274         assert str(pofile) == expectedpo
 275
 276     def test_merging_untranslated_with_kde_disambiguation(self):
 277         """test merging untranslated messages that are the same except for KDE disambiguation"""
 278         templatepo = r'''#: sendMsgTitle
 279 #: sendMsgTitle.accesskey
 280 msgid "_: sendMsgTitle sendMsgTitle.accesskey\n"
 281 "Send Message"
 282 msgstr ""
 283
 284 #: sendMessageCheckWindowTitle
 285 #: sendMessageCheckWindowTitle.accesskey
 286 msgid "_: sendMessageCheckWindowTitle sendMessageCheckWindowTitle.accesskey\n"
 287 "Send Message"
 288 msgstr ""
 289 '''
 290         mergepo = r'''#: sendMsgTitle%ssendMsgTitle.accesskey
 291 msgid ""
 292 "_: sendMsgTitle sendMsgTitle.accesskey\n"
 293 "Send Message"
 294 msgstr "Stuur"
 295
 296 #: sendMessageCheckWindowTitle%ssendMessageCheckWindowTitle.accesskey
 297 msgid ""
 298 "_: sendMessageCheckWindowTitle sendMessageCheckWindowTitle.accesskey\n"
 299 "Send Message"
 300 msgstr "Stuur"
 301 ''' % (po.lsep, po.lsep)
 302         expectedpo = mergepo
 303         pofile = self.mergestore(templatepo, mergepo)
 304         print "Expected:\n%s\n---\nMerged:\n%s\n---" % (expectedpo, str(pofile))
 305         assert str(pofile) == expectedpo
 306
 307     def test_merging_header_entries(self):
 308         """Check that we do the right thing if we have header entries in the input PO."""
 309
 310         templatepo = r'''#, fuzzy
 311 msgid ""
 312 msgstr ""
 313 "Project-Id-Version: PACKAGE VERSION\n"
 314 "Report-Msgid-Bugs-To: new@example.com\n"
 315 "POT-Creation-Date: 2006-11-11 11:11+0000\n"
 316 "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
 317 "Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
 318 "Language-Team: LANGUAGE <LL@li.org>\n"
 319 "MIME-Version: 1.0\n"
 320 "Content-Type: text/plain; charset=UTF-8\n"
 321 "Content-Transfer-Encoding: 8bit\n"
 322 "Plural-Forms: nplurals=INTEGER; plural=EXPRESSION;\n"
 323 "X-Generator: Translate Toolkit 0.10rc2\n"
 324
 325 #: simple.test
 326 msgid "Simple String"
 327 msgstr ""
 328 '''
 329         mergepo = r'''msgid ""
 330 msgstr ""
 331 "Project-Id-Version: Pootle 0.10\n"
 332 "Report-Msgid-Bugs-To: old@example.com\n"
 333 "POT-Creation-Date: 2006-01-01 01:01+0100\n"
 334 "PO-Revision-Date: 2006-09-09 09:09+0900\n"
 335 "Last-Translator: Joe Translate <joe@example.com>\n"
 336 "Language-Team: Pig Latin <piglatin@example.com>\n"
 337 "MIME-Version: 1.0\n"
 338 "Content-Type: text/plain; charset=UTF-8\n"
 339 "Content-Transfer-Encoding: 8bit\n"
 340 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
 341 "X-Generator: Translate Toolkit 0.9\n"
 342
 343 #: simple.test
 344 msgid "Simple String"
 345 msgstr "Dimpled Ring"
 346 '''
 347         expectedpo = r'''msgid ""
 348 msgstr ""
 349 "Project-Id-Version: Pootle 0.10\n"
 350 "Report-Msgid-Bugs-To: new@example.com\n"
 351 "POT-Creation-Date: 2006-11-11 11:11+0000\n"
 352 "PO-Revision-Date: 2006-09-09 09:09+0900\n"
 353 "Last-Translator: Joe Translate <joe@example.com>\n"
 354 "Language-Team: Pig Latin <piglatin@example.com>\n"
 355 "MIME-Version: 1.0\n"
 356 "Content-Type: text/plain; charset=UTF-8\n"
 357 "Content-Transfer-Encoding: 8bit\n"
 358 "Plural-Forms: nplurals=2; plural=(n != 1);\n"
 359 "X-Generator: Translate Toolkit 0.10rc2\n"
 360
 361 #: simple.test
 362 msgid "Simple String"
 363 msgstr "Dimpled Ring"
 364 '''
 365         pofile = self.mergestore(templatepo, mergepo)
 366         print "Expected:\n%s\n---\nMerged:\n%s\n---" % (expectedpo, str(pofile))
 367         assert str(pofile) == expectedpo