storage/test_po.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from translate.storage import po
   5 from translate.storage import test_base
   6 from translate.misc import wStringIO
   7 from translate.misc.multistring import multistring
   8 from py.test import raises
   9
  10 def test_roundtrip_quoting():
  11     specials = ['Fish & chips', 'five < six', 'six > five',
  12                 'Use &nbsp;', 'Use &amp;nbsp;'
  13                 'A "solution"', "skop 'n bal", '"""', "'''",
  14                 '\n', '\t', '\r',
  15                 '\\n', '\\t', '\\r', '\\"', '\r\n', '\\r\\n', '\\']
  16     for special in specials:
  17         quoted_special = po.quoteforpo(special)
  18         unquoted_special = po.unquotefrompo(quoted_special)
  19         print "special: %r\nquoted: %r\nunquoted: %r\n" % (special, quoted_special, unquoted_special)
  20         assert special == unquoted_special
  21
  22 class TestPOUnit(test_base.TestTranslationUnit):
  23     UnitClass = po.pounit
  24     def test_istranslatable(self):
  25         """Tests for the correct behaviour of istranslatable()."""
  26         unit = self.UnitClass("Message")
  27         assert unit.istranslatable()
  28
  29         unit.source = ""
  30         assert not unit.istranslatable()
  31         # simulate a header
  32         unit.target = "PO-Revision-Date: 2006-02-09 23:33+0200\n"
  33         assert unit.isheader()
  34         assert not unit.istranslatable()
  35
  36         unit.source = "Message"
  37         unit.target = "Boodskap"
  38         unit.makeobsolete()
  39         assert not unit.istranslatable()
  40
  41     def test_adding_empty_note(self):
  42         unit = self.UnitClass("bla")
  43         assert not '#' in str(unit)
  44         for empty_string in [ "", " ", "\t", "\n" ]:
  45             unit.addnote(empty_string)
  46             assert not '#' in str(unit)
  47
  48     def test_markreview(self):
  49         """Tests if we can mark the unit to need review."""
  50         unit = self.unit
  51         # We have to explicitly set the target to nothing, otherwise xliff
  52         # tests will fail.
  53         # Can we make it default behavior for the UnitClass?
  54         unit.target = ""
  55
  56         unit.addnote("Test note 1", origin="translator")
  57         unit.addnote("Test note 2", origin="translator")
  58         original_notes = unit.getnotes(origin="translator")
  59
  60         assert not unit.isreview()
  61         unit.markreviewneeded()
  62         print unit.getnotes()
  63         assert unit.isreview()
  64         unit.markreviewneeded(False)
  65         assert not unit.isreview()
  66         assert unit.getnotes(origin="translator") == original_notes
  67         unit.markreviewneeded(explanation="Double check spelling.")
  68         assert unit.isreview()
  69         notes = unit.getnotes(origin="translator")
  70         assert notes.count("Double check spelling.") == 1
  71
  72     def test_errors(self):
  73         """Tests that we can add and retrieve error messages for a unit."""
  74         unit = self.unit
  75
  76         assert len(unit.geterrors()) == 0
  77         unit.adderror(errorname='test1', errortext='Test error message 1.')
  78         unit.adderror(errorname='test2', errortext='Test error message 2.')
  79         unit.adderror(errorname='test3', errortext='Test error message 3.')
  80         assert len(unit.geterrors()) == 3
  81         assert unit.geterrors()['test1'] == 'Test error message 1.'
  82         assert unit.geterrors()['test2'] == 'Test error message 2.'
  83         assert unit.geterrors()['test3'] == 'Test error message 3.'
  84         unit.adderror(errorname='test1', errortext='New error 1.')
  85         assert unit.geterrors()['test1'] == 'New error 1.'
  86
  87     def test_no_plural_settarget(self):
  88         """tests that target handling of file with no plural is correct"""
  89         # plain text, no plural test
  90         unit = self.UnitClass("Tree")
  91         unit.target = "ki"
  92         assert unit.target.strings == ["ki"]
  93         assert unit.source.strings == ["Tree"]
  94         assert unit.hasplural() == False
  95
  96         # plural test with multistring
  97         unit.setsource(["Tree", "Trees"])
  98         assert unit.source.strings == ["Tree", "Trees"]
  99         assert unit.hasplural()
 100         unit.target = multistring(["ki", "ni ki"])
 101         assert unit.target.strings == ["ki", "ni ki"]
 102
 103         # test of msgid with no plural and msgstr with plural
 104         unit = self.UnitClass("Tree")
 105         assert raises(ValueError, unit.settarget, [u"ki", u"ni ki"])
 106         assert unit.hasplural() == False
 107
 108     def test_wrapping_bug(self):
 109         """This tests for a wrapping bug that existed at some stage."""
 110         unit = self.UnitClass("")
 111         message = 'Projeke ya Pootle ka boyona e ho <a href="http://translate.sourceforge.net/">translate.sourceforge.net</a> moo o ka fumanang dintlha ka source code, di mailing list jwalo jwalo.'
 112         unit.target = message
 113         print unit.target
 114         assert unit.target == message
 115
 116     def test_extract_msgidcomments_from_text(self):
 117         """Test that KDE style comments are extracted correctly."""
 118         unit = self.UnitClass("test source")
 119
 120         kdetext = "_: Simple comment\nsimple text"
 121         assert unit._extract_msgidcomments(kdetext) == "Simple comment"
 122
 123     def test_isheader(self):
 124         """checks that we deal correctly with headers."""
 125         unit = self.UnitClass()
 126         unit.target = "PO-Revision-Date: 2006-02-09 23:33+0200\n"
 127         assert unit.isheader()
 128         unit.source = "Some English string"
 129         assert not unit.isheader()
 130         unit.source = u"Goeiemôre"
 131         assert not unit.isheader()
 132
 133 class TestPOFile(test_base.TestTranslationStore):
 134     StoreClass = po.pofile
 135     def poparse(self, posource):
 136         """helper that parses po source without requiring files"""
 137         dummyfile = wStringIO.StringIO(posource)
 138         pofile = self.StoreClass(dummyfile)
 139         return pofile
 140
 141     def poregen(self, posource):
 142         """helper that converts po source to pofile object and back"""
 143         return str(self.poparse(posource))
 144
 145     def pomerge(self, oldmessage, newmessage, authoritative):
 146         """helper that merges two messages"""
 147         oldpofile = self.poparse(oldmessage)
 148         oldunit = oldpofile.units[0]
 149         if newmessage:
 150             newpofile = self.poparse(newmessage)
 151             newunit = newpofile.units[0]
 152         else:
 153             newunit = oldpofile.UnitClass()
 154         oldunit.merge(newunit, authoritative=authoritative)
 155         print oldunit
 156         return str(oldunit)
 157
 158     def test_simpleentry(self):
 159         """checks that a simple po entry is parsed correctly"""
 160         posource = '#: test.c:100 test.c:101\nmsgid "test"\nmsgstr "rest"\n'
 161         pofile = self.poparse(posource)
 162         assert len(pofile.units) == 1
 163         thepo = pofile.units[0]
 164         assert thepo.getlocations() == ["test.c:100", "test.c:101"]
 165         assert thepo.source == "test"
 166         assert thepo.target == "rest"
 167
 168     def test_copy(self):
 169         """checks that we can copy all the needed PO fields"""
 170         posource = '''# TRANSLATOR-COMMENTS
 171 #. AUTOMATIC-COMMENTS
 172 #: REFERENCE...
 173 #, fuzzy
 174 msgctxt "CONTEXT"
 175 msgid "UNTRANSLATED-STRING"
 176 msgstr "TRANSLATED-STRING"'''
 177         pofile = self.poparse(posource)
 178         oldunit = pofile.units[0]
 179         newunit = oldunit.copy()
 180         assert newunit == oldunit
 181
 182     def test_parse_source_string(self):
 183         """parse a string"""
 184         posource = '#: test.c\nmsgid "test"\nmsgstr "rest"\n'
 185         pofile = self.poparse(posource)
 186         assert len(pofile.units) == 1
 187
 188     def test_parse_file(self):
 189         """test parsing a real file"""
 190         posource = '#: test.c\nmsgid "test"\nmsgstr "rest"\n'
 191         pofile = self.poparse(posource)
 192         assert len(pofile.units) == 1
 193
 194     def test_unicode(self):
 195         """check that the po class can handle Unicode characters"""
 196         posource = 'msgid ""\nmsgstr ""\n"Content-Type: text/plain; charset=UTF-8\\n"\n\n#: test.c\nmsgid "test"\nmsgstr "rest\xe2\x80\xa6"\n'
 197         pofile = self.poparse(posource)
 198         print pofile
 199         assert len(pofile.units) == 2
 200
 201     def test_plurals(self):
 202         posource = r'''msgid "Cow"
 203 msgid_plural "Cows"
 204 msgstr[0] "Koei"
 205 msgstr[1] "Koeie"
 206 '''
 207         pofile = self.poparse(posource)
 208         assert len(pofile.units) == 1
 209         unit = pofile.units[0]
 210         assert isinstance(unit.target, multistring)
 211         print unit.target.strings
 212         assert unit.target == "Koei"
 213         assert unit.target.strings == ["Koei", "Koeie"]
 214
 215         posource = r'''msgid "Skaap"
 216 msgid_plural "Skape"
 217 msgstr[0] "Sheep"
 218 '''
 219         pofile = self.poparse(posource)
 220         assert len(pofile.units) == 1
 221         unit = pofile.units[0]
 222         assert isinstance(unit.target, multistring)
 223         print unit.target.strings
 224         assert unit.target == "Sheep"
 225         assert unit.target.strings == ["Sheep"]
 226
 227     def test_plural_unicode(self):
 228         """tests that all parts of the multistring are unicode."""
 229         posource = r'''msgid "Ców"
 230 msgid_plural "Cóws"
 231 msgstr[0] "Kóei"
 232 msgstr[1] "Kóeie"
 233 '''
 234         pofile = self.poparse(posource)
 235         unit = pofile.units[0]
 236         assert isinstance(unit.source, multistring)
 237         assert isinstance(unit.source.strings[1], unicode)
 238
 239
 240     def wtest_kde_plurals(self):
 241         """Tests kde-style plurals. (Bug: 191)"""
 242         posource = '''msgid "_n Singular\n"
 243 "Plural"
 244 msgstr "Een\n"
 245 "Twee\n"
 246 "Drie"
 247 '''
 248         pofile = self.poparse(posource)
 249         assert len(pofile.units) == 1
 250         unit = pofile.units[0]
 251         assert unit.hasplural() == True
 252         assert isinstance(unit.source, multistring)
 253         print unit.source.strings
 254         assert unit.source == "Singular"
 255         assert unit.source.strings == ["Singular", "Plural"]
 256         assert isinstance(unit.target, multistring)
 257         print unit.target.strings
 258         assert unit.target == "Een"
 259         assert unit.target.strings == ["Een", "Twee", "Drie"]
 260
 261     def test_empty_lines_notes(self):
 262         """Tests that empty comment lines are preserved"""
 263         posource = r'''# License name
 264 #
 265 # license line 1
 266 # license line 2
 267 # license line 3
 268 msgid ""
 269 msgstr "POT-Creation-Date: 2006-03-08 17:30+0200\n"
 270 '''
 271         pofile = self.poparse(posource)
 272         assert str(pofile) == posource
 273
 274     def test_fuzzy(self):
 275         """checks that fuzzy functionality works as expected"""
 276         posource = '#, fuzzy\nmsgid "ball"\nmsgstr "bal"\n'
 277         expectednonfuzzy = 'msgid "ball"\nmsgstr "bal"\n'
 278         pofile = self.poparse(posource)
 279         print pofile
 280         assert pofile.units[0].isfuzzy()
 281         pofile.units[0].markfuzzy(False)
 282         assert not pofile.units[0].isfuzzy()
 283         assert str(pofile) == expectednonfuzzy
 284
 285         posource = '#, fuzzy, python-format\nmsgid "ball"\nmsgstr "bal"\n'
 286         expectednonfuzzy = '#, python-format\nmsgid "ball"\nmsgstr "bal"\n'
 287         pofile = self.poparse(posource)
 288         print pofile
 289         assert pofile.units[0].isfuzzy()
 290         pofile.units[0].markfuzzy(False)
 291         assert not pofile.units[0].isfuzzy()
 292         assert str(pofile) == expectednonfuzzy
 293
 294     def xtest_makeobsolete_untranslated(self):
 295         """Tests making an untranslated unit obsolete"""
 296         posource = '#. The automatic one\n#: test.c\nmsgid "test"\nmsgstr ""\n'
 297         pofile = self.poparse(posource)
 298         unit = pofile.units[0]
 299         assert not unit.isobsolete()
 300         unit.makeobsolete()
 301         assert str(unit) == ""
 302         # a better way might be for pomerge/pot2po to remove the unit
 303
 304     def test_merging_automaticcomments(self):
 305         """checks that new automatic comments override old ones"""
 306         oldsource = '#. old comment\n#: line:10\nmsgid "One"\nmsgstr "Een"\n'
 307         newsource = '#. new comment\n#: line:10\nmsgid "One"\nmsgstr ""\n'
 308         expected = '#. new comment\n#: line:10\nmsgid "One"\nmsgstr "Een"\n'
 309         assert self.pomerge(newsource, oldsource, authoritative=True) == expected
 310
 311     def test_malformed_units(self):
 312         """Test that we handle malformed units reasonably."""
 313         posource = 'msgid "thing\nmsgstr "ding"\nmsgid "Second thing"\nmsgstr "Tweede ding"\n'
 314         pofile = self.poparse(posource)
 315         assert len(pofile.units) == 2
 316
 317     def test_malformed_obsolete_units(self):
 318         """Test that we handle malformed obsolete units reasonably."""
 319         posource = '''msgid "thing
 320 msgstr "ding"
 321
 322 #~ msgid "Second thing"
 323 #~ msgstr "Tweede ding"
 324 #~ msgid "Third thing"
 325 #~ msgstr "Derde ding"
 326 '''
 327         pofile = self.poparse(posource)
 328         assert len(pofile.units) == 3
 329
 330     def test_uniforum_po(self):
 331         """Test that we handle Uniforum PO files."""
 332         posource = '''# File: ../somefile.cpp, line: 33
 333 msgid "thing"
 334 msgstr "ding"
 335 #
 336 # File: anotherfile.cpp, line: 34
 337 msgid "second"
 338 msgstr "tweede"
 339 '''
 340         pofile = self.poparse(posource)
 341         assert len(pofile.units) == 2
 342         # FIXME we still need to handle this correctly for proper Uniforum support if required
 343         #assert pofile.units[0].getlocations() == "File: somefile, line: 300"
 344         #assert pofile.units[1].getlocations() == "File: anotherfile, line: 200"
 345
 346     def test_obsolete(self):
 347         """Tests that obsolete messages work"""
 348         posource = '#~ msgid "Old thing"\n#~ msgstr "Ou ding"\n'
 349         pofile = self.poparse(posource)
 350         assert pofile.isempty()
 351         assert len(pofile.units) == 1
 352         unit = pofile.units[0]
 353         assert unit.isobsolete()
 354         assert str(pofile) == posource
 355
 356     def test_header_escapes(self):
 357         pofile = self.StoreClass()
 358         header = pofile.makeheader(**{"Report-Msgid-Bugs-To": r"http://qa.openoffice.org/issues/enter_bug.cgi?subcomponent=ui&comment=&short_desc=Localization%20issue%20in%20file%3A%20dbaccess\source\core\resource.oo&component=l10n&form_name=enter_issue"})
 359         pofile.addunit(header)
 360         filecontents = str(pofile)
 361         print filecontents
 362         # We need to make sure that the \r didn't get misrepresented as a
 363         # carriage return, but as a slash (escaped) followed by a normal 'r'
 364         assert r'\source\core\resource' in pofile.header().target
 365         assert r're\\resource' in filecontents
 366
 367     def test_makeobsolete(self):
 368         """Tests making a unit obsolete"""
 369         posource = '#. The automatic one\n#: test.c\nmsgid "test"\nmsgstr "rest"\n'
 370         poexpected = '#~ msgid "test"\n#~ msgstr "rest"\n'
 371         pofile = self.poparse(posource)
 372         print pofile
 373         unit = pofile.units[0]
 374         assert not unit.isobsolete()
 375         unit.makeobsolete()
 376         assert unit.isobsolete()
 377         print pofile
 378         assert str(unit) == poexpected
 379
 380     def test_makeobsolete_plural(self):
 381         """Tests making a plural unit obsolete"""
 382         posource = r'''msgid "Cow"
 383 msgid_plural "Cows"
 384 msgstr[0] "Koei"
 385 msgstr[1] "Koeie"
 386 '''
 387         poexpected = '''#~ msgid "Cow"
 388 #~ msgid_plural "Cows"
 389 #~ msgstr[0] "Koei"
 390 #~ msgstr[1] "Koeie"
 391 '''
 392         pofile = self.poparse(posource)
 393         print pofile
 394         unit = pofile.units[0]
 395         assert not unit.isobsolete()
 396         unit.makeobsolete()
 397         assert unit.isobsolete()
 398         print pofile
 399         assert str(unit) == poexpected
 400
 401
 402     def test_makeobsolete_msgctxt(self):
 403         """Tests making a unit with msgctxt obsolete"""
 404         posource = '#: test.c\nmsgctxt "Context"\nmsgid "test"\nmsgstr "rest"\n'
 405         poexpected = '#~ msgctxt "Context"\n#~ msgid "test"\n#~ msgstr "rest"\n'
 406         pofile = self.poparse(posource)
 407         print pofile
 408         unit = pofile.units[0]
 409         assert not unit.isobsolete()
 410         unit.makeobsolete()
 411         assert unit.isobsolete()
 412         print pofile
 413         assert str(unit) == poexpected
 414
 415     def test_makeobsolete_msgidcomments(self):
 416         """Tests making a unit with msgidcomments obsolete"""
 417         posource = '#: first.c\nmsgid ""\n"_: first.c\\n"\n"test"\nmsgstr "rest"\n\n#: second.c\nmsgid ""\n"_: second.c\\n"\n"test"\nmsgstr "rest"'
 418         poexpected = '#~ msgid ""\n#~ "_: first.c\\n"\n#~ "test"\n#~ msgstr "rest"\n'
 419         print "Source:\n%s" % posource
 420         print "Expected:\n%s" % poexpected
 421         pofile = self.poparse(posource)
 422         unit = pofile.units[0]
 423         assert not unit.isobsolete()
 424         unit.makeobsolete()
 425         assert unit.isobsolete()
 426         print "Result:\n%s" % pofile
 427         assert str(unit) == poexpected
 428
 429     def test_multiline_obsolete(self):
 430         """Tests for correct output of mulitline obsolete messages"""
 431         posource = '#~ msgid "Old thing\\n"\n#~ "Second old thing"\n#~ msgstr "Ou ding\\n"\n#~ "Tweede ou ding"\n'
 432         pofile = self.poparse(posource)
 433         assert pofile.isempty()
 434         assert len(pofile.units) == 1
 435         unit = pofile.units[0]
 436         assert unit.isobsolete()
 437         print str(pofile)
 438         print posource
 439         assert str(pofile) == posource
 440
 441     def test_merge_duplicates(self):
 442         """checks that merging duplicates works"""
 443         posource = '#: source1\nmsgid "test me"\nmsgstr ""\n\n#: source2\nmsgid "test me"\nmsgstr ""\n'
 444         pofile = self.poparse(posource)
 445         #assert len(pofile.units) == 2
 446         pofile.removeduplicates("merge")
 447         assert len(pofile.units) == 1
 448         assert pofile.units[0].getlocations() == ["source1", "source2"]
 449         print pofile
 450
 451     def test_merge_mixed_sources(self):
 452         """checks that merging works with different source location styles"""
 453         posource = '''
 454 #: source1
 455 #: source2
 456 msgid "test"
 457 msgstr ""
 458
 459 #: source1 source2
 460 msgid "test"
 461 msgstr ""
 462 '''
 463         pofile = self.poparse(posource)
 464         print str(pofile)
 465         pofile.removeduplicates("merge")
 466         print str(pofile)
 467         assert len(pofile.units) == 1
 468         assert pofile.units[0].getlocations() == ["source1", "source2"]
 469
 470     def test_parse_context(self):
 471         """Tests that msgctxt is parsed correctly and that it is accessible via the api methods."""
 472         posource = '''# Test comment
 473 #: source1
 474 msgctxt "noun"
 475 msgid "convert"
 476 msgstr "bekeerling"
 477
 478 # Test comment 2
 479 #: source2
 480 msgctxt "verb"
 481 msgid "convert"
 482 msgstr "omskakel"
 483 '''
 484         pofile = self.poparse(posource)
 485         unit = pofile.units[0]
 486
 487         assert unit.getcontext() == 'noun'
 488         assert unit.getnotes() == 'Test comment'
 489
 490         unit = pofile.units[1]
 491         assert unit.getcontext() == 'verb'
 492         assert unit.getnotes() == 'Test comment 2'
 493
 494     def test_parse_advanced_context(self):
 495         """Tests that some weird possible msgctxt scenarios are parsed correctly."""
 496         posource = r'''# Test multiline context
 497 #: source1
 498 msgctxt "Noun."
 499 " A person that changes his or her ways."
 500 msgid "convert"
 501 msgstr "bekeerling"
 502
 503 # Test quotes
 504 #: source2
 505 msgctxt "Verb. Converting from \"something\" to \"something else\"."
 506 msgid "convert"
 507 msgstr "omskakel"
 508
 509 # Test quotes, newlines and multiline.
 510 #: source3
 511 msgctxt "Verb.\nConverting from \"something\""
 512 " to \"something else\"."
 513 msgid "convert"
 514 msgstr "omskakel"
 515 '''
 516         pofile = self.poparse(posource)
 517         unit = pofile.units[0]
 518
 519         assert unit.getcontext() == 'Noun. A person that changes his or her ways.'
 520         assert unit.getnotes() == 'Test multiline context'
 521
 522         unit = pofile.units[1]
 523         assert unit.getcontext() == 'Verb. Converting from "something" to "something else".'
 524         assert unit.getnotes() == 'Test quotes'
 525
 526         unit = pofile.units[2]
 527         assert unit.getcontext() == 'Verb.\nConverting from "something" to "something else".'
 528         assert unit.getnotes() == 'Test quotes, newlines and multiline.'
 529
 530     def test_kde_context(self):
 531         """Tests that kde-style msgid comments can be retrieved via getcontext()."""
 532         posource = '''# Test comment
 533 #: source1
 534 msgid ""
 535 "_: Noun\\n"
 536 "convert"
 537 msgstr "bekeerling"
 538
 539 # Test comment 2
 540 #: source2
 541 msgid ""
 542 "_: Verb. _: "
 543 "The action of changing.\\n"
 544 "convert"
 545 msgstr "omskakel"
 546 '''
 547         pofile = self.poparse(posource)
 548         unit = pofile.units[0]
 549
 550         assert unit.getcontext() == 'Noun'
 551         assert unit.getnotes() == 'Test comment'
 552
 553         unit = pofile.units[1]
 554         assert unit.getcontext() == 'Verb. _: The action of changing.'
 555         assert unit.getnotes() == 'Test comment 2'
 556
 557     def test_id(self):
 558         """checks that ids work correctly"""
 559         posource = r'''
 560 msgid ""
 561 msgstr ""
 562 "PO-Revision-Date: 2006-02-09 23:33+0200\n"
 563 "MIME-Version: 1.0\n"
 564 "Content-Type: text/plain; charset=UTF-8\n"
 565 "Content-Transfer-Encoding: 8-bit\n"
 566
 567 msgid "plant"
 568 msgstr ""
 569
 570 msgid ""
 571 "_: Noun\n"
 572 "convert"
 573 msgstr "bekeerling"
 574
 575 msgctxt "verb"
 576 msgid ""
 577 "convert"
 578 msgstr "omskakel"
 579
 580 msgid "tree"
 581 msgid_plural "trees"
 582 msgstr[0] ""
 583 '''
 584         pofile = self.poparse(posource)
 585         assert pofile.units[0].getid() == ""
 586         assert pofile.units[1].getid() == "plant"
 587         assert pofile.units[2].getid() == "_: Noun\nconvert"
 588         assert pofile.units[3].getid() == "verb\04convert"
 589         # Gettext does not consider the plural to determine duplicates, only
 590         # the msgid. For generation of .mo files, we might want to use this
 591         # code to generate the entry for the hash table, but for now, it is
 592         # commented out for conformance to gettext.
 593 #        assert pofile.units[4].getid() == "tree\0trees"
 594