chrome/test/chromedriver/test_expectations

   1 # Copyright (c) 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Test expectation list for WebDriver Java acceptance tests.
   6
   7 It is evaluated through Python.
   8 """
   9
  10 import fnmatch
  11
  12
  13 _REVISION_NEGATIVE_FILTER = {}
  14 _REVISION_NEGATIVE_FILTER['HEAD'] = [
  15     'AlertsTest.*',
  16     'AuthenticatedPageLoadingTest.*',
  17     'AvailableLogsTest.driverLogShouldBeEnabledByDefault',
  18     'BasicMouseInterfaceTest.testContextClick',
  19     'BasicMouseInterfaceTest.testMousePositionIsNotPreservedInActionsChain',
  20     'BasicMouseInterfaceTest.testMovingIntoAnImageEnclosedInALink',
  21     'BasicMouseInterfaceTest.testMovingMouseBackAndForthPastViewPort',
  22     'BasicMouseInterfaceTest.testMovingMousePastViewPort',
  23     'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
  24     # https://code.google.com/p/selenium/source/diff?spec=svn5e856ea3d0ee07db00aa41293f68ce2566c16888&r=a823e0a2ead8b5430a9a92a14452d0d47cb05f61&format=side&path=/java/client/test/org/openqa/selenium/ClickScrollingTest.java&old_path=/java/client/test/org/openqa/selenium/ClickScrollingTest.java&old=ee8ae7d15794146d24a744ec96d1192ff851f0fc
  25     'ClickScrollingTest.testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow',
  26     'ClickScrollingTest.testShouldNotBeAbleToClickElementThatIsOutOfViewInANonScrollableFrame',
  27     'ClickScrollingTest.testShouldNotScrollIfAlreadyScrolledAndElementIsInView',
  28     'ClickTest.testCanClickAnImageMapArea',
  29     # https://code.google.com/p/selenium/source/diff?spec=svn5e856ea3d0ee07db00aa41293f68ce2566c16888&r=9ec284bff77c06198af6097ca6703a48e1b0736b&format=side&path=/java/client/test/org/openqa/selenium/ClickTest.java&old_path=/java/client/test/org/openqa/selenium/ClickTest.java&old=5a0284c1a625ed8507f6d98857958e286a449bb7
  30     'ClickTest.testShouldBeAbleToClickOnAnElementInFrameGreaterThanTwoViewports',
  31     'CombinedInputActionsTest.testClickingOnFormElements',
  32     'CoordinatesTest.testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort',
  33     'CorrectEventFiringTest.testShouldFireEventsInTheRightOrder',
  34     'CorrectEventFiringTest.testShouldFireFocusEventWhenClicking',
  35     'DragAndDropTest.testDragTooFar',
  36     'ExecutingAsyncJavascriptTest.includesAlertTextInUnhandledAlertException',
  37     # https://code.google.com/p/selenium/source/diff?spec=svn5e856ea3d0ee07db00aa41293f68ce2566c16888&r=0e6a710cceed3037e8e44ae81579cf3ceca33fb6&format=side&path=/java/client/test/org/openqa/selenium/ExecutingAsyncJavascriptTest.java&old_path=/java/client/test/org/openqa/selenium/ExecutingAsyncJavascriptTest.java&old=ee8ae7d15794146d24a744ec96d1192ff851f0fc
  38     'ExecutingAsyncJavascriptTest.shouldCatchErrorsWithMessageAndStacktraceWhenExecutingInitialScript',
  39     'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScript',
  40     'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScriptWhichTimesOut',
  41     'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlert',
  42     'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlertWhichTimesOut',
  43     'ExecutingJavascriptTest.testShouldThrowExceptionIfExecutingOnNoPage',
  44     # https://code.google.com/p/selenium/source/diff?spec=svn5e856ea3d0ee07db00aa41293f68ce2566c16888&r=0e6a710cceed3037e8e44ae81579cf3ceca33fb6&format=side&path=/java/client/test/org/openqa/selenium/ExecutingJavascriptTest.java&old_path=/java/client/test/org/openqa/selenium/ExecutingJavascriptTest.java&old=bf0d4b7d040315e9311efaa3a91a2a59777608da
  45     'ExecutingJavascriptTest.testShouldThrowAnExceptionWithMessageAndStacktraceWhenTheJavascriptIsBad',
  46     'FormHandlingTest.handleFormWithJavascriptAction',
  47     'FormHandlingTest.testShouldNotBeAbleToSubmitAFormThatDoesNotExist',
  48     'FrameSwitchingTest.testShouldNotBeAbleToDoAnythingTheFrameIsDeletedFromUnderUs',
  49     'I18nTest.testShouldBeAbleToActivateIMEEngine',
  50     # Broken because AddWebStorage.java is broken.
  51     'LocalStorageTest.*',
  52     'LocationContextTest.*',
  53     'MiscTest.testShouldBeAbleToGetTheSourceOfAnXmlDocument',
  54     'PageLoadingTest.shouldBeAbleToDisableAcceptOfInsecureSslCertsWithRequiredCapability',
  55     'PageLoadingTest.testShouldNotWaitIndefinitelyIfAnExternalResourceFailsToLoad',
  56     'PageLoadingTest.testShouldTimeoutIfAPageTakesTooLongToLoad',
  57     # PerformanceLoggingTest is for firefox only; "performance" log deprecated.
  58     'PerformanceLoggingTest.*',
  59     'ProxySettingTest.canConfigureProxyWithRequiredCapability',
  60     'ProxySettingTest.requiredProxyCapabilityShouldHavePriority',
  61     'RenderedWebElementTest.testShouldHandleNonIntegerPositionAndSize',
  62     'SessionHandlingTest.callingQuitAfterClosingTheLastWindowIsANoOp',
  63     # Broken because AddWebStorage.java is broken.
  64     'SessionStorageTest.*',
  65     'SqlDatabaseTest.*',
  66     # https://code.google.com/p/selenium/source/diff?spec=svn6bbba240f54370803189ce28149072c59f5d5312&r=6bbba240f54370803189ce28149072c59f5d5312&format=side&path=/java/client/test/org/openqa/selenium/SvgDocumentTest.java
  67     'SvgDocumentTest.testClickOnSvgElement',
  68     # https://code.google.com/p/selenium/source/diff?spec=svn5e856ea3d0ee07db00aa41293f68ce2566c16888&r=859a1df06d56b4f2ba247f1dadbfceac8066efcb&format=side&path=/java/client/test/org/openqa/selenium/TakesScreenshotTest.java&old_path=/java/client/test/org/openqa/selenium/TakesScreenshotTest.java&old=cfea793d0410db2100a40daf88483b83d58af435
  69     'TakesScreenshotTest.testShouldCaptureScreenshot',
  70     'TakesScreenshotTest.testShouldCaptureScreenshotWithLongX',
  71     'TakesScreenshotTest.testShouldCaptureScreenshotWithLongY',
  72     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLong',
  73     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongX',
  74     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongY',
  75     'TextHandlingTest.testShouldNotReturnLtrMarks',
  76     'TextPagesTest.testShouldBeAbleToLoadASimplePageOfText',
  77     'TextPagesTest.testShouldThrowExceptionWhenAddingCookieToAPageThatIsNotHtml',
  78     'TypingTest.testNonPrintableCharactersShouldWorkWithContentEditableOrDesignModeSet',
  79     'TypingTest.testShouldBeAbleToTypeIntoContentEditableElementWithExistingValue',
  80     'TypingTest.testShouldNotTypeIntoElementsThatPreventKeyDownEvents',
  81     'TypingTest.testTypingIntoAnIFrameWithContentEditableOrDesignModeSet',
  82     'UnexpectedAlertBehaviorTest.*',
  83     # https://code.google.com/p/selenium/source/diff?spec=svn5e856ea3d0ee07db00aa41293f68ce2566c16888&r=e923c1389b85331d2c98aaeb6bbca88bc57821cd&format=side&path=/java/client/test/org/openqa/selenium/VisibilityTest.java&old_path=/java/client/test/org/openqa/selenium/VisibilityTest.java&old=ee8ae7d15794146d24a744ec96d1192ff851f0fc
  84     'VisibilityTest.testElementHiddenByOverflowXIsNotVisible',
  85     'VisibilityTest.testElementHiddenByOverflowYIsNotVisible',
  86     'VisibilityTest.tooSmallAWindowWithOverflowHiddenIsNotAProblem',
  87     'WindowTest.*',
  88 ]
  89 _REVISION_NEGATIVE_FILTER['28'] = [] + _REVISION_NEGATIVE_FILTER['HEAD']
  90 _REVISION_NEGATIVE_FILTER['27'] = [] + _REVISION_NEGATIVE_FILTER['HEAD']
  91 _REVISION_NEGATIVE_FILTER['26'] = [
  92     'UploadTest.testFileUploading',
  93     'CorrectEventFiringTest.testUploadingFileShouldFireOnChangeEvent',
  94     'FormHandlingTest.testShouldBeAbleToAlterTheContentsOfAFileUploadInputElement',
  95     'FormHandlingTest.testShouldBeAbleToUploadTheSameFileTwice',
  96     'FormHandlingTest.testShouldBeAbleToSendKeysToAFileUploadInputElementInAnXhtmlDocument',
  97 ] + _REVISION_NEGATIVE_FILTER['HEAD']
  98
  99 _OS_NEGATIVE_FILTER = {}
 100 _OS_NEGATIVE_FILTER['win'] = [
 101     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=282
 102     'PageLoadingTest.testShouldNotHangIfDocumentOpenCallIsNeverFollowedByDocumentCloseCall',
 103 ]
 104 _OS_NEGATIVE_FILTER['linux'] = []
 105 _OS_NEGATIVE_FILTER['mac'] = []
 106 _OS_NEGATIVE_FILTER['android'] = [
 107     'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
 108     'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView',
 109     'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame',
 110     'ClickTest.testCanClickOnAnElementWithTopSetToANegativeNumber',
 111     'ClickTest.testShouldOnlyFollowHrefOnce',
 112     'CombinedInputActionsTest.testCombiningShiftAndClickResultsInANewWindow',
 113     'CoordinatesTest.testShouldGetCoordinatesInViewPortOfAnElementInAFrame',
 114     'CoordinatesTest.testShouldGetCoordinatesInViewPortOfAnElementInANestedFrame',
 115     'CoordinatesTest.testShouldGetCoordinatesOfAHiddenElement',
 116     'CorrectEventFiringTest.testSendingKeysToAFocusedElementShouldNotBlurThatElement',
 117     'CorrectEventFiringTest.testSendingKeysToAnElementShouldCauseTheFocusEventToFire',
 118     'CorrectEventFiringTest.testSendingKeysToAnotherElementShouldCauseTheBlurEventToFire',
 119     'CorrectEventFiringTest.testShouldEmitClickEventWhenClickingOnATextInputElement',
 120     'ElementAttributeTest.testCanRetrieveTheCurrentValueOfATextFormField_emailInput',
 121     'ElementAttributeTest.testCanRetrieveTheCurrentValueOfATextFormField_textArea',
 122     'ElementAttributeTest.testCanRetrieveTheCurrentValueOfATextFormField_textInput',
 123     'ElementEqualityTest.testAnElementFoundInADifferentFrameViaJsShouldHaveSameId',
 124     'ElementFindingTest.testAnElementFoundInADifferentFrameIsStale',
 125     'ElementFindingTest.testAnElementFoundInADifferentFrameViaJsCanBeUsed',
 126     'ElementSelectingTest.testShouldBeAbleToToggleEnabledMultiSelectOption',
 127     'ExecutingAsyncJavascriptTest.shouldBeAbleToExecuteAsynchronousScripts',
 128     'ExecutingAsyncJavascriptTest.shouldNotTimeoutIfScriptCallsbackInsideAZeroTimeout',
 129     'FormHandlingTest.testSendingKeyboardEventsShouldAppendTextInInputs',
 130     'FormHandlingTest.testSendingKeyboardEventsShouldAppendTextInInputsWithExistingValue',
 131     'FormHandlingTest.testSendingKeyboardEventsShouldAppendTextinTextAreas',
 132     'FormHandlingTest.testSendKeysKeepsCapitalization',
 133     'FormHandlingTest.testShouldBeAbleToClearTextFromInputElements',
 134     'FormHandlingTest.testShouldBeAbleToClearTextFromTextAreas',
 135     'FormHandlingTest.testShouldBeAbleToEnterTextIntoATextAreaBySettingItsValue',
 136     'FormHandlingTest.testShouldEnterDataIntoFormFields',
 137     'FrameSwitchingTest.testClosingTheFinalBrowserWindowShouldNotCauseAnExceptionToBeThrown',
 138     'FrameSwitchingTest.testGetCurrentUrl',
 139     'FrameSwitchingTest.testShouldNotSwitchMagicallyToTheTopWindow',
 140     'I18nTest.testEnteringHebrewTextFromLeftToRight',
 141     'I18nTest.testEnteringHebrewTextFromRightToLeft',
 142     'I18nTest.testShouldBeAbleToInputJapanese',
 143     'ImplicitWaitTest.testShouldImplicitlyWaitForAnElementToBeVisibleBeforeInteracting',
 144     'JavascriptEnabledDriverTest.testChangeEventIsFiredAppropriatelyWhenFocusIsLost',
 145     'JavascriptEnabledDriverTest.testIssue80ClickShouldGenerateClickEvent',
 146     'JavascriptEnabledDriverTest.testShouldBeAbleToClickALinkThatClosesAWindow',
 147     'JavascriptEnabledDriverTest.testShouldFireOnChangeEventWhenSettingAnElementsValue',
 148     'OpacityTest.testShouldBeAbleToClickOnElementsWithOpacityZero',
 149     'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
 150     'SelectElementHandlingTest.testShouldBePossibleToDeselectASingleOptionFromASelectWhichAllowsMultipleChoices',
 151     'SelectElementTest.shouldAllowOptionsToBeDeselectedByIndex',
 152     'SelectElementTest.shouldAllowOptionsToBeDeselectedByReturnedValue',
 153     'SelectElementTest.shouldAllowUserToDeselectAllWhenSelectSupportsMultipleSelections',
 154     'SelectElementTest.shouldAllowUserToDeselectOptionsByVisibleText',
 155     'SessionHandlingTest.callingAnyOperationAfterClosingTheLastWindowShouldThrowAnException',
 156     'TakesScreenshotTest.testCaptureToBase64',
 157     'TakesScreenshotTest.testSaveScreenshotAsFile',
 158     'TextHandlingTest.testShouldBeAbleToEnterDatesAfterFillingInOtherValuesFirst',
 159     'TextHandlingTest.testShouldBeAbleToSetMoreThanOneLineOfTextInATextArea',
 160     'TextHandlingTest.testTextOfATextAreaShouldBeEqualToItsDefaultTextEvenAfterTyping',
 161     'TouchFlickTest.*',
 162     'TouchScrollTest.*',
 163     'TouchSingleTapTest.*',
 164     # Passes locally but not on bots??
 165     'TypingTest.testShouldFireKeyDownEvents',
 166     # Test is written using local files; doesn't work on Android.
 167     'UploadTest.testFileUploading',
 168     # Not applicable on Android.
 169     'WindowSwitchingTest.*',
 170 ]
 171
 172 def GetPassedJavaTestFilter(operating_system, chrome_version):
 173   """Returns the test filter for running all passing tests.
 174
 175   Args:
 176     operating_system: The operating system, one of 'linux', 'mac', 'win', or
 177         'android'.
 178     chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
 179
 180   Returns:
 181     Filter string, in Google Test (C++) format.
 182   """
 183   return '*-' + ':'.join(_OS_NEGATIVE_FILTER[operating_system] +
 184                          _REVISION_NEGATIVE_FILTER[chrome_version])
 185
 186 def ApplyJavaTestFilter(operating_system, chrome_version, tests):
 187   """Applies the test filter to the given list of tests.
 188
 189   Args:
 190     operating_system: The operating system, one of 'linux', 'mac', 'win', or
 191         'android'.
 192     chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
 193     test: list of test names to filter.
 194
 195   Returns:
 196     Set of passed test names.
 197   """
 198   filters = (_OS_NEGATIVE_FILTER[operating_system] +
 199              _REVISION_NEGATIVE_FILTER[chrome_version])
 200   passed = set(tests)
 201   for f in filters:
 202     passed.difference_update(set(t for t in tests if fnmatch.fnmatch(t, f)))
 203   return passed