convert/test_dtd2po.py

   1 #!/usr/bin/env python
   2 # -*- coding: utf-8 -*-
   3
   4 from translate.convert import dtd2po
   5 from translate.convert import test_convert
   6 from translate.misc import wStringIO
   7 from translate.storage import po
   8 from translate.storage import dtd
   9
  10 class TestDTD2PO:
  11     def dtd2po(self, dtdsource, dtdtemplate=None):
  12         """helper that converts dtd source to po source without requiring files"""
  13         inputfile = wStringIO.StringIO(dtdsource)
  14         inputdtd = dtd.dtdfile(inputfile)
  15         convertor = dtd2po.dtd2po()
  16         if not dtdtemplate:
  17             outputpo = convertor.convertstore(inputdtd)
  18         else:
  19             templatefile = wStringIO.StringIO(dtdtemplate)
  20             templatedtd = dtd.dtdfile(templatefile)
  21             outputpo = convertor.mergestore(templatedtd, inputdtd)
  22         return outputpo
  23
  24     def convertdtd(self, dtdsource):
  25         """call the convertdtd, return the outputfile"""
  26         inputfile = wStringIO.StringIO(dtdsource)
  27         outputfile = wStringIO.StringIO()
  28         templatefile = None
  29         assert dtd2po.convertdtd(inputfile, outputfile, templatefile)
  30         return outputfile.getvalue()
  31
  32     def singleelement(self, pofile):
  33         """checks that the pofile contains a single non-header element, and returns it"""
  34         assert len(pofile.units) == 2
  35         assert pofile.units[0].isheader()
  36         print pofile.units[1]
  37         return pofile.units[1]
  38
  39     def countelements(self, pofile):
  40         """returns the number of non-header items"""
  41         if pofile.units[0].isheader():
  42             return len(pofile.units) - 1
  43         else:
  44             return len(pofile.units)
  45
  46     def test_simpleentity(self):
  47         """checks that a simple dtd entity definition converts properly to a po entry"""
  48         dtdsource = '<!ENTITY test.me "bananas for sale">\n'
  49         pofile = self.dtd2po(dtdsource)
  50         pounit = self.singleelement(pofile)
  51         assert pounit.source == "bananas for sale"
  52         assert pounit.target == ""
  53         # Now with a template language
  54         dtdtemplate = '<!ENTITY test.me "bananas for sale">\n'
  55         dtdtranslated = '<!ENTITY test.me "piesangs te koop">\n'
  56         pofile = self.dtd2po(dtdtranslated, dtdtemplate)
  57         pounit = self.singleelement(pofile)
  58         assert pounit.source == "bananas for sale"
  59         assert pounit.target == "piesangs te koop"
  60
  61     def test_convertdtd(self):
  62         """checks that the convertdtd function is working"""
  63         dtdsource = '<!ENTITY saveas.label "Save As...">\n'
  64         posource = self.convertdtd(dtdsource)
  65         pofile = po.pofile(wStringIO.StringIO(posource))
  66         unit = self.singleelement(pofile)
  67         assert unit.source == "Save As..."
  68         assert unit.target == ""
  69
  70     def test_apos(self):
  71         """apostrophe should not break a single-quoted entity definition, bug 69"""
  72         dtdsource = "<!ENTITY test.me 'bananas &apos; for sale'>\n"
  73         pofile = self.dtd2po(dtdsource)
  74         pounit = self.singleelement(pofile)
  75         assert pounit.source == "bananas ' for sale"
  76
  77     def test_quotes(self):
  78         """quotes should be handled in a single-quoted entity definition"""
  79         dtdsource = """<!ENTITY test.metoo '"Bananas" for sale'>\n"""
  80         pofile = self.dtd2po(dtdsource)
  81         pounit = self.singleelement(pofile)
  82         print str(pounit)
  83         assert pounit.source == '"Bananas" for sale'
  84
  85     def test_emptyentity(self):
  86         """checks that empty entity definitions survive into po file, bug 15"""
  87         dtdsource = '<!ENTITY credit.translation "">\n'
  88         pofile = self.dtd2po(dtdsource)
  89         pounit = self.singleelement(pofile)
  90         assert "credit.translation" in str(pounit)
  91
  92     def test_emptyentity_translated(self):
  93         """checks that if we translate an empty entity it makes it into the PO, bug 101"""
  94         dtdtemplate = '<!ENTITY credit.translation "">\n'
  95         dtdsource = '<!ENTITY credit.translation "Translators Names">\n'
  96         pofile = self.dtd2po(dtdsource, dtdtemplate)
  97         unit = self.singleelement(pofile)
  98         print unit
  99         assert "credit.translation" in str(unit)
 100         # We don't want this to simply be seen as a header:
 101         assert len(unit.getid()) != 0
 102         assert unit.target == "Translators Names"
 103
 104     def test_localisaton_note_simple(self):
 105         """test the simple localisation more becomes a #. comment"""
 106         dtdsource = '''<!-- LOCALIZATION NOTE (alwaysCheckDefault.height):
 107   There's some sort of bug which makes wrapping checkboxes not properly reflow,
 108   causing the bottom border of the groupbox to be cut off; set this
 109   appropriately if your localization causes this checkbox to wrap.
 110 -->
 111 <!ENTITY alwaysCheckDefault.height  "3em">
 112 '''
 113         pofile = self.dtd2po(dtdsource)
 114         posource = str(pofile)
 115         print posource
 116         assert posource.count('#.') == 5 # 1 Header extracted from, 3 comment lines, 1 autoinserted comment
 117
 118     def test_localisation_note_merge(self):
 119         """test that LOCALIZATION NOTES are added properly as #. comments and disambiguated with msgctxt entries"""
 120         dtdtemplate = '<!--LOCALIZATION NOTE (%s): Some note -->\n' + \
 121             '<!ENTITY %s "Source text">\n'
 122         dtdsource = dtdtemplate % ("note1.label", "note1.label") + dtdtemplate % ("note2.label", "note2.label")
 123         pofile = self.dtd2po(dtdsource)
 124         posource = str(pofile.units[1]) + str(pofile.units[2])
 125         print posource
 126         assert posource.count('#.') == 2
 127         assert posource.count('msgctxt') == 2
 128
 129     def test_donttranslate_simple(self):
 130         """check that we handle DONT_TRANSLATE messages properly"""
 131         dtdsource = '''<!-- LOCALIZATION NOTE (region.Altitude): DONT_TRANSLATE -->
 132 <!ENTITY region.Altitude "Very High">'''
 133         pofile = self.dtd2po(dtdsource)
 134         assert self.countelements(pofile) == 0
 135         dtdsource = '''<!-- LOCALIZATION NOTE (exampleOpenTag.label): DONT_TRANSLATE: they are text for HTML tagnames: "<i>" and "</i>" -->
 136 <!ENTITY exampleOpenTag.label "&lt;i&gt;">'''
 137         pofile = self.dtd2po(dtdsource)
 138         assert self.countelements(pofile) == 0
 139         dtdsource = '''<!-- LOCALIZATION NOTE (imapAdvanced.label): Do not translate "IMAP" -->
 140 <!ENTITY imapAdvanced.label "Advanced IMAP Server Settings">'''
 141         pofile = self.dtd2po(dtdsource)
 142         assert self.countelements(pofile) == 1
 143
 144     def test_donttranslate_label(self):
 145         """test strangeness when label entity is marked DONT_TRANSLATE and accesskey is not, bug 30"""
 146         dtdsource = '<!--LOCALIZATION NOTE (editorCheck.label): DONT_TRANSLATE -->\n' + \
 147             '<!ENTITY editorCheck.label "Composer">\n<!ENTITY editorCheck.accesskey "c">\n'
 148         pofile = self.dtd2po(dtdsource)
 149         posource = str(pofile)
 150         # we need to decided what we're going to do here - see the comments in bug 30
 151         # this tests the current implementation which is that the DONT_TRANSLATE string is removed, but the other remains
 152         assert 'editorCheck.label' not in posource
 153         assert 'editorCheck.accesskey' in posource
 154
 155     def test_donttranslate_onlyentity(self):
 156         """if the entity is itself just another entity then it shouldn't appear in the output PO file"""
 157         dtdsource = '''<!-- LOCALIZATION NOTE (mainWindow.title): DONT_TRANSLATE -->
 158 <!ENTITY mainWindow.title "&brandFullName;">'''
 159         pofile = self.dtd2po(dtdsource)
 160         assert self.countelements(pofile) == 0
 161
 162     def test_donttranslate_commentedout(self):
 163         """check that we don't process messages in <!-- comments -->: bug 102"""
 164         dtdsource = '''<!-- commenting out until bug 38906 is fixed
 165 <!ENTITY messagesHeader.label         "Messages"> -->'''
 166         pofile = self.dtd2po(dtdsource)
 167         assert self.countelements(pofile) == 0
 168
 169     def test_spaces_at_start_of_dtd_lines(self):
 170         """test that pretty print spaces at the start of subsequent DTD element lines are removed from the PO file, bug 79"""
 171         # Space at the end of the line
 172         dtdsource = '<!ENTITY  noupdatesfound.intro "First line then \n' + \
 173           '                                          next lines.">\n'
 174         pofile = self.dtd2po(dtdsource)
 175         pounit = self.singleelement(pofile)
 176         # We still need to decide how we handle line line breaks in the DTD entities.  It seems that we should actually
 177         # drop the line break but this has not been implemented yet.
 178         assert pounit.source == "First line then \nnext lines."
 179         # No space at the end of the line
 180         dtdsource = '<!ENTITY  noupdatesfound.intro "First line then\n' + \
 181           '                                          next lines.">\n'
 182         pofile = self.dtd2po(dtdsource)
 183         pounit = self.singleelement(pofile)
 184         assert pounit.source == "First line then \nnext lines."
 185
 186     def test_accesskeys_folding(self):
 187         """test that we fold accesskeys into message strings"""
 188         dtdsource_template = '<!ENTITY  fileSaveAs.%s "Save As...">\n<!ENTITY  fileSaveAs.%s "S">\n'
 189         lang_template = '<!ENTITY  fileSaveAs.%s "Gcina ka...">\n<!ENTITY  fileSaveAs.%s "G">\n'
 190         for label in ("label", "title"):
 191             for accesskey in ("accesskey", "accessKey", "akey"):
 192                 pofile = self.dtd2po(dtdsource_template % (label, accesskey))
 193                 pounit = self.singleelement(pofile)
 194                 assert pounit.source == "&Save As..."
 195                 # Test with template (bug 155)
 196                 pofile = self.dtd2po(lang_template % (label, accesskey), dtdsource_template % (label, accesskey))
 197                 pounit = self.singleelement(pofile)
 198                 assert pounit.source == "&Save As..."
 199                 assert pounit.target == "&Gcina ka..."
 200
 201     def test_accesskeys_mismatch(self):
 202         """check that we can handle accesskeys that don't match and thus can't be folded into the .label entry"""
 203         dtdsource = '<!ENTITY  fileSave.label "Save">\n' + \
 204            '<!ENTITY  fileSave.accesskey "z">\n'
 205         pofile = self.dtd2po(dtdsource)
 206         assert self.countelements(pofile) == 2
 207
 208     def test_carriage_return_in_multiline_dtd(self):
 209         """test that we create nice PO files when we find a \r\n in a multiline DTD element"""
 210         dtdsource = '<!ENTITY  noupdatesfound.intro "First line then \r\n' + \
 211           '                                          next lines.">\n'
 212         pofile = self.dtd2po(dtdsource)
 213         unit = self.singleelement(pofile)
 214         assert unit.source == "First line then \nnext lines."
 215
 216     def test_multiline_with_blankline(self):
 217         """test that we can process a multiline entity that has a blank line in it, bug 331"""
 218         dtdsource = '''
 219 <!ENTITY multiline.text "
 220 Some text
 221
 222 Some other text
 223 ">'''
 224         pofile = self.dtd2po(dtdsource)
 225         unit = self.singleelement(pofile)
 226         assert unit.source == "Some text \n \nSome other text"
 227
 228     def test_mulitline_closing_quotes(self):
 229         """test that we support various styles and spaces after closing quotes on multiline entities"""
 230         dtdsource = '''
 231 <!ENTITY pref.plural '<span>opsies</span><span
 232                       class="noWin">preferences</span>' >
 233 '''
 234         pofile = self.dtd2po(dtdsource)
 235         unit = self.singleelement(pofile)
 236         assert unit.source == '<span>opsies</span><span \nclass="noWin">preferences</span>'
 237
 238     def test_preserving_spaces(self):
 239         """test that we preserve space that appear at the start of the first line of a DTD entity"""
 240         # Space before first character
 241         dtdsource = '<!ENTITY mainWindow.titlemodifiermenuseparator " - ">'
 242         pofile = self.dtd2po(dtdsource)
 243         unit = self.singleelement(pofile)
 244         assert unit.source == " - "
 245         # Double line and spaces
 246         dtdsource = '<!ENTITY mainWindow.titlemodifiermenuseparator " - with a newline\n    and more text">'
 247         pofile = self.dtd2po(dtdsource)
 248         unit = self.singleelement(pofile)
 249         print repr(unit.source)
 250         assert unit.source == " - with a newline \nand more text"
 251
 252     def test_escaping_newline_tabs(self):
 253         """test that we handle all kinds of newline permutations"""
 254         dtdsource = '<!ENTITY  noupdatesfound.intro "A hard coded newline.\\nAnd tab\\t and a \\r carriage return.">\n'
 255         converter = dtd2po.dtd2po()
 256         thedtd = dtd.dtdunit()
 257         thedtd.parse(dtdsource)
 258         thepo = po.pounit()
 259         converter.convertstrings(thedtd, thepo)
 260         print thedtd
 261         print thepo.source
 262         # \n in a dtd should also appear as \n in the PO file
 263         assert thepo.source == r"A hard coded newline.\nAnd tab\t and a \r carriage return."
 264
 265     def test_abandoned_accelerator(self):
 266         """test that when a language DTD has an accelerator but the template DTD does not that we abandon the accelerator"""
 267         dtdtemplate = '<!ENTITY test.label "Test">\n'
 268         dtdlanguage = '<!ENTITY test.label "Toets">\n<!ENTITY test.accesskey "T">\n'
 269         pofile = self.dtd2po(dtdlanguage, dtdtemplate)
 270         unit = self.singleelement(pofile)
 271         assert unit.source == "Test"
 272         assert unit.target == "Toets"
 273
 274     def test_unassociable_accelerator(self):
 275         """test to see that we can handle accelerator keys that cannot be associated correctly"""
 276         dtdsource = '<!ENTITY  managecerts.button "Manage Certificates...">\n<!ENTITY  managecerts.accesskey "M">'
 277         pofile = self.dtd2po(dtdsource)
 278         assert pofile.units[1].source == "Manage Certificates..."
 279         assert pofile.units[2].source == "M"
 280         pofile = self.dtd2po(dtdsource, dtdsource)
 281         assert pofile.units[1].target == "Manage Certificates..."
 282         assert pofile.units[2].target == "M"
 283
 284     def test_changed_labels_and_accelerators(self):
 285         """test to ensure that when the template changes an entity name we can still manage the accelerators"""
 286         dtdtemplate = '''<!ENTITY  managecerts.caption      "Manage Certificates">
 287 <!ENTITY  managecerts.text         "Use the Certificate Manager to manage your personal certificates, as well as those of other people and certificate authorities.">
 288 <!ENTITY  managecerts.button       "Manage Certificates...">
 289 <!ENTITY  managecerts.accesskey    "M">'''
 290         dtdlanguage = '''<!ENTITY managecerts.label "ﺇﺩﺍﺭﺓ ﺎﻠﺸﻫﺍﺩﺎﺗ">
 291 <!ENTITY managecerts.text "ﺎﺴﺘﺧﺪﻣ ﻡﺪﻳﺭ ﺎﻠﺸﻫﺍﺩﺎﺗ ﻹﺩﺍﺭﺓ ﺶﻫﺍﺩﺎﺘﻛ ﺎﻠﺸﺨﺼﻳﺓ، ﺏﺍﻺﺿﺎﻓﺓ ﻞﺘﻠﻛ ﺎﻠﺧﺎﺻﺓ ﺏﺍﻶﺧﺮﻴﻧ ﻭ ﺲﻠﻃﺎﺗ ﺎﻠﺸﻫﺍﺩﺎﺗ.">
 292 <!ENTITY managecerts.button "ﺇﺩﺍﺭﺓ ﺎﻠﺸﻫﺍﺩﺎﺗ...">
 293 <!ENTITY managecerts.accesskey "ﺩ">'''
 294         pofile = self.dtd2po(dtdlanguage, dtdtemplate)
 295         print pofile
 296         assert pofile.units[3].source == "Manage Certificates..."
 297         assert pofile.units[3].target == u"ﺇﺩﺍﺭﺓ ﺎﻠﺸﻫﺍﺩﺎﺗ..."
 298         assert pofile.units[4].source == "M"
 299         assert pofile.units[4].target == u"ﺩ"
 300
 301     def wtest_accelerator_keys_not_in_sentence(self):
 302         """tests to ensure that we can manage accelerator keys that are not part of the transated sentence eg in Chinese"""
 303         dtdtemplate = '''<!ENTITY useAutoScroll.label             "Use autoscrolling">
 304 <!ENTITY useAutoScroll.accesskey         "a">'''
 305         dtdlanguage = '''<!ENTITY useAutoScroll.label             "使用自動捲動(Autoscrolling)">
 306 <!ENTITY useAutoScroll.accesskey         "a">'''
 307         pofile = self.dtd2po(dtdlanguage, dtdtemplate)
 308         print pofile
 309         assert pofile.units[1].target == "使用自動捲動(&Autoscrolling)"
 310         # We assume that accesskeys with no associated key should be done as follows "XXXX (&A)"
 311         # TODO - check that we can unfold this from PO -> DTD
 312         dtdlanguage = '''<!ENTITY useAutoScroll.label             "使用自動捲動">
 313 <!ENTITY useAutoScroll.accesskey         "a">'''
 314         pofile = self.dtd2po(dtdlanguage, dtdtemplate)
 315         print pofile
 316         assert pofile.units[1].target == "使用自動捲動 (&A)"
 317
 318     def test_exclude_entity_includes(self):
 319         """test that we don't turn an include into a translatable string"""
 320         dtdsource = '<!ENTITY % brandDTD SYSTEM "chrome://branding/locale/brand.dtd">'
 321         pofile = self.dtd2po(dtdsource)
 322         assert self.countelements(pofile) == 0
 323
 324     def test_linewraps(self):
 325         """check that redundant line wraps are removed from the po file"""
 326         dtdsource = '''<!ENTITY generic.longDesc "
 327 <p>Test me.</p>
 328 ">'''
 329         pofile = self.dtd2po(dtdsource)
 330         pounit = self.singleelement(pofile)
 331         assert pounit.source == "<p>Test me.</p>"
 332
 333     def test_merging_with_new_untranslated(self):
 334         """test that when we merge in new untranslated strings with existing translations we manage the encodings properly"""
 335         # This should probably be in test_po.py but was easier to do here
 336         dtdtemplate = '''<!ENTITY unreadFolders.label "Unread">\n<!ENTITY viewPickerUnread.label "Unread">\n<!ENTITY unreadColumn.label "Unread">'''
 337         dtdlanguage = '''<!ENTITY viewPickerUnread.label "Непрочетени">\n<!ENTITY unreadFolders.label "Непрочетени">'''
 338         pofile = self.dtd2po(dtdlanguage, dtdtemplate)
 339         print pofile
 340         assert pofile.units[1].source == "Unread"
 341
 342     def test_merge_without_template(self):
 343         """test that we we manage the case where we merge and their is no template file"""
 344         # If we supply a template file we should fail if the template file does not exist or is blank.  We should
 345         # not put the translation in as the source.
 346         # TODO: this test fails, since line 16 checks for "not dtdtemplate"
 347         #   instead of checking for "dtdtemplate is None". What is correct?
 348         dtdtemplate = ''
 349         dtdsource = '<!ENTITY no.template "Target">'
 350         pofile = self.dtd2po(dtdsource, dtdtemplate)
 351         print pofile
 352         assert self.countelements(pofile) == 0
 353
 354 class TestDTD2POCommand(test_convert.TestConvertCommand, TestDTD2PO):
 355     """Tests running actual dtd2po commands on files"""
 356     convertmodule = dtd2po
 357     defaultoptions = {"progress": "none"}
 358
 359     def test_help(self):
 360         """tests getting help"""
 361         options = test_convert.TestConvertCommand.test_help(self)
 362         options = self.help_check(options, "-P, --pot")
 363         options = self.help_check(options, "-t TEMPLATE, --template=TEMPLATE")
 364         options = self.help_check(options, "--duplicates=DUPLICATESTYLE", last=True)