chrome/test/chromedriver/test/test_expectations

   1 # Copyright 2013 The Chromium Authors. All rights reserved.
   2 # Use of this source code is governed by a BSD-style license that can be
   3 # found in the LICENSE file.
   4
   5 """Test expectation list for WebDriver Java acceptance tests.
   6
   7 It is evaluated through Python.
   8 """
   9
  10 import fnmatch
  11
  12
  13 _REVISION_NEGATIVE_FILTER = {}
  14 _REVISION_NEGATIVE_FILTER['HEAD'] = [
  15     'AlertsTest.testIncludesAlertTextInUnhandledAlertException',
  16     'AlertsTest.testSettingTheValueOfAnAlertThrows',
  17     'AlertsTest.testShouldGetTextOfAlertOpenedInSetTimeout',
  18     'AlertsTest.testShouldHandleAlertOnPageUnload',
  19     'AlertsTest.testShouldHandleAlertOnWindowClose',
  20     'AlertsTest.testShouldNotHandleAlertInAnotherWindow',
  21     'AlertsTest.testSwitchingToMissingAlertInAClosedWindowThrows',
  22     'AuthenticatedPageLoadingTest.*',
  23     'AvailableLogsTest.driverLogShouldBeEnabledByDefault',
  24     'BasicMouseInterfaceTest.testMousePositionIsNotPreservedInActionsChain',
  25     'BasicMouseInterfaceTest.testMovingIntoAnImageEnclosedInALink',
  26     'BasicMouseInterfaceTest.testMovingMouseBackAndForthPastViewPort',
  27     'BasicMouseInterfaceTest.testMovingMousePastViewPort',
  28     'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
  29     'ClickScrollingTest.testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow',
  30     'ClickScrollingTest.testShouldNotBeAbleToClickElementThatIsOutOfViewInANonScrollableFrame',
  31     'ClickScrollingTest.testShouldNotScrollIfAlreadyScrolledAndElementIsInView',
  32     'ClickTest.testShouldBeAbleToClickOnAnElementInFrameGreaterThanTwoViewports',
  33     'CombinedInputActionsTest.testClickingOnFormElements',
  34     'CoordinatesTest.testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort',
  35     'CorrectEventFiringTest.testShouldFireEventsInTheRightOrder',
  36     'CorrectEventFiringTest.testShouldFireFocusEventWhenClicking',
  37     'DragAndDropTest.testDragTooFar',
  38     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=374
  39     'ElementFindingTest.testShouldNotBeAbleToFindAnElementOnABlankPage',
  40     'ExecutingAsyncJavascriptTest.includesAlertTextInUnhandledAlertException',
  41     'ExecutingAsyncJavascriptTest.shouldCatchErrorsWithMessageAndStacktraceWhenExecutingInitialScript',
  42     'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScript',
  43     'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScriptWhichTimesOut',
  44     'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlert',
  45     'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlertWhichTimesOut',
  46     'ExecutingJavascriptTest.testShouldThrowExceptionIfExecutingOnNoPage',
  47     'ExecutingJavascriptTest.testShouldThrowAnExceptionWithMessageAndStacktraceWhenTheJavascriptIsBad',
  48     'FormHandlingTest.testShouldNotBeAbleToSubmitAFormThatDoesNotExist',
  49     'FrameSwitchingTest.testShouldNotBeAbleToDoAnythingTheFrameIsDeletedFromUnderUs',
  50     'I18nTest.testShouldBeAbleToActivateIMEEngine',
  51     # Broken because AddWebStorage.java is broken.
  52     'LocalStorageTest.*',
  53     'LocationContextTest.*',
  54     'MiscTest.testShouldBeAbleToGetTheSourceOfAnXmlDocument',
  55     'PageLoadingTest.shouldBeAbleToDisableAcceptOfInsecureSslCertsWithRequiredCapability',
  56     'PageLoadingTest.testShouldNotWaitIndefinitelyIfAnExternalResourceFailsToLoad',
  57     # PerformanceLoggingTest is for firefox only; "performance" log deprecated.
  58     'PerformanceLoggingTest.*',
  59     'ProxySettingTest.canConfigureProxyWithRequiredCapability',
  60     'ProxySettingTest.requiredProxyCapabilityShouldHavePriority',
  61     'RenderedWebElementTest.testShouldHandleNonIntegerPositionAndSize',
  62     # Broken because AddWebStorage.java is broken.
  63     'SessionStorageTest.*',
  64     'SqlDatabaseTest.*',
  65     'SvgDocumentTest.testClickOnSvgElement',
  66     'TakesScreenshotTest.testShouldCaptureScreenshot',
  67     'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePage',
  68     'TakesScreenshotTest.testShouldCaptureScreenshotWithLongX',
  69     'TakesScreenshotTest.testShouldCaptureScreenshotWithLongY',
  70     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLong',
  71     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongX',
  72     'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongY',
  73     'TextHandlingTest.testShouldNotReturnLtrMarks',
  74     'TextPagesTest.testShouldBeAbleToLoadASimplePageOfText',
  75     'TextPagesTest.testShouldThrowExceptionWhenAddingCookieToAPageThatIsNotHtml',
  76     'TypingTest.testGenerateKeyPressEventEvenWhenElementPreventsDefault',
  77     'TypingTest.testNonPrintableCharactersShouldWorkWithContentEditableOrDesignModeSet',
  78     'TypingTest.testShouldBeAbleToTypeIntoContentEditableElementWithExistingValue',
  79     'TypingTest.testShouldNotTypeIntoElementsThatPreventKeyDownEvents',
  80     'TypingTest.testTypingIntoAnIFrameWithContentEditableOrDesignModeSet',
  81     'UnexpectedAlertBehaviorTest.*',
  82     'VisibilityTest.testElementHiddenByOverflowXIsNotVisible',
  83     'VisibilityTest.testElementHiddenByOverflowYIsNotVisible',
  84     'VisibilityTest.tooSmallAWindowWithOverflowHiddenIsNotAProblem',
  85     'WindowTest.*',
  86     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528
  87     'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',
  88     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=653
  89     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
  90     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistory',
  91     # https://code.google.com/p/chromedriver/issues/detail?id=1117
  92     'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
  93     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=1113
  94     'FrameSwitchingTest.testShouldNotSwitchMagicallyToTheTopWindow',
  95     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=1151
  96     'DragAndDropTest.testDragAndDrop',
  97     'DragAndDropTest.testShouldAllowUsersToDragAndDropToElementsOffTheCurrentViewPort',
  98     'DragAndDropTest.testElementInDiv',
  99 ]
 100
 101
 102 _OS_NEGATIVE_FILTER = {}
 103 _OS_NEGATIVE_FILTER['win'] = [
 104     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=373
 105     'RenderedWebElementTest.testHoverPersists',
 106     'RenderedWebElementTest.canClickOnASuckerFishStyleMenu',
 107     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=416
 108     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching',
 109     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
 110 ]
 111 _OS_NEGATIVE_FILTER['linux'] = [
 112     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=416
 113     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
 114     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=1148
 115     'CombinedInputActionsTest.testCombiningShiftAndClickResultsInANewWindow',
 116     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=1150
 117     'BasicKeyboardInterfaceTest.testBasicKeyboardInputOnActiveElement',
 118 ]
 119 _OS_NEGATIVE_FILTER['mac'] = [
 120     # https://code.google.com/p/chromedriver/issues/detail?id=26
 121     'AlertsTest.testAlertShouldNotAllowAdditionalCommandsIfDismissed',
 122     'AlertsTest.testShouldAllowUsersToDismissAnAlertManually',
 123     'FormHandlingTest.handleFormWithJavascriptAction',
 124     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=354
 125     'AlertsTest.testShouldAllowUsersToAcceptAnAlertInAFrame',
 126     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=375
 127     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
 128     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=416
 129     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
 130     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=1149
 131     'ChromeDriverTests.testShouldAllowTheUserToSwitchToAnIFrameAndRemainFocusedOnIt',
 132 ]
 133 _OS_NEGATIVE_FILTER['android:chrome'] = [
 134     'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions',
 135     'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame',
 136     'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView',
 137     'ClickTest.testShouldOnlyFollowHrefOnce',
 138     'CombinedInputActionsTest.testCombiningShiftAndClickResultsInANewWindow',
 139     'ElementSelectingTest.testShouldBeAbleToToggleEnabledMultiSelectOption',
 140     'FrameSwitchingTest.testShouldBeAbleToClickInAFrameThatRewritesTopWindowLocation',
 141     'JavascriptEnabledDriverTest.testShouldBeAbleToClickALinkThatClosesAWindow',
 142     'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
 143     'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate',
 144     'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes',
 145     'PageLoadingTest.testShouldFollowMetaRedirects',
 146     'PageLoadingTest.testShouldWaitForDocumentToBeLoaded',
 147     'PerformanceLogTypeTest.pageLoadShouldProducePerformanceLogEntries',
 148     'PerformanceLogTypeTest.shouldBeAbleToEnablePerformanceLog',
 149     'SelectElementHandlingTest.testShouldBePossibleToDeselectASingleOptionFromASelectWhichAllowsMultipleChoices',
 150     'SelectElementTest.shouldAllowOptionsToBeDeselectedByIndex',
 151     'SelectElementTest.shouldAllowOptionsToBeDeselectedByReturnedValue',
 152     'SelectElementTest.shouldAllowUserToDeselectAllWhenSelectSupportsMultipleSelections',
 153     'SelectElementTest.shouldAllowUserToDeselectOptionsByVisibleText',
 154     'SessionHandlingTest.callingAnyOperationAfterClosingTheLastWindowShouldThrowAnException',
 155     'SessionHandlingTest.callingQuitAfterClosingTheLastWindowIsANoOp',
 156     'TakesScreenshotTest.testCaptureToBase64',
 157     'TakesScreenshotTest.testSaveScreenshotAsFile',
 158     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage',
 159     'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching',
 160     'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePageAfterSwitching',
 161
 162     # Alerts are not yet supported on Android.
 163     'AlertsTest.*',
 164
 165     # http://crbug.com/156390
 166     'DragAndDropTest.*',
 167
 168     # Scrolling touch events are not supported.
 169     'TouchScrollTest.*',
 170
 171     # These tests start multiple sessions, which is not supported on a single
 172     # Android device.
 173     "AvailableLogsTest.shouldBeAbleToEnableProfilerLog",
 174     "GetLogsTest.turningOffLogShouldMeanNoLogMessages",
 175     "SessionHandlingTest.callingAnyOperationAfterQuitShouldThrowAnException",
 176     "SessionHandlingTest.callingQuitMoreThanOnceOnASessionIsANoOp",
 177     # Test is written using local files; doesn't work on Android.
 178     'UploadTest.testFileUploading',
 179     # Not applicable on ChromeShell (doesn't support tabs).
 180     'WindowSwitchingTest.*',
 181
 182     # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528
 183     'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo',
 184
 185     # https://code.google.com/p/chromedriver/issues/detail?id=604
 186     'I18nTest.testShouldBeAbleToActivateIMEEngine',
 187
 188     # https://code.google.com/p/chromium/issues/detail?id=418590
 189     'ClearTest.testContentEditableAreaShouldClear',
 190     'ClickScrollingTest.testClickingOnAnchorScrollsPage',
 191     'JavascriptEnabledDriverTest.testChangeEventIsFiredAppropriatelyWhenFocusIsLost',
 192     'TypingTest.testShouldBeAbleToTypeIntoEmptyContentEditableElement',
 193
 194     # https://code.google.com/p/chromedriver/issues/detail?id=922
 195     'CorrectEventFiringTest.testShouldEmitOnClickEventsWhenSelectingElements',
 196     'CorrectEventFiringTest.testSendingKeysToAnotherElementShouldCauseTheBlurEventToFire',
 197
 198     # https://code.google.com/p/chromedriver/issues/detail?id=1176
 199     'ChromeDriverTests.clientLogShouldBeEnabledByDefault',
 200
 201     # https://code.google.com/p/chromedriver/issues/detail?id=1119
 202     'CombinedInputActionsTest.testCanClickOnLinksWithAnOffset',
 203     'CombinedInputActionsTest.testMouseMovementWorksWhenNavigatingToAnotherPage',
 204 ]
 205 _OS_NEGATIVE_FILTER['android:chrome_stable'] = (
 206     _OS_NEGATIVE_FILTER['android:chrome'])
 207 _OS_NEGATIVE_FILTER['android:chrome_beta'] = (
 208     _OS_NEGATIVE_FILTER['android:chrome'])
 209 _OS_NEGATIVE_FILTER['android:chromedriver_webview_shell'] = (
 210     _OS_NEGATIVE_FILTER['android:chrome'] + [
 211          # https://code.google.com/p/chromedriver/issues/detail?id=645
 212         'ClickScrollingTest.testShouldBeAbleToClickElementInAFrameThatIsOutOfView',
 213         'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInAFrameThatIsOutOfView',
 214         'FormHandlingTest.handleFormWithJavascriptAction',
 215         'TypingTest.testShouldReportKeyCodeOfArrowKeysUpDownEvents',
 216         'TypingTest.testShouldReportKeyCodeOfArrowKeys',
 217         'CombinedInputActionsTest.testCanClickOnLinks',
 218         'CombinedInputActionsTest.testCanClickOnLinksWithAnOffset',
 219         'CombinedInputActionsTest.testMouseMovementWorksWhenNavigatingToAnotherPage',
 220     ]
 221 )
 222
 223
 224 def _GetRevisionNegativeFilter(chrome_version):
 225   if chrome_version in _REVISION_NEGATIVE_FILTER:
 226     return _REVISION_NEGATIVE_FILTER[chrome_version]
 227   return _REVISION_NEGATIVE_FILTER['HEAD']
 228
 229
 230 def GetDisabledTestMatchers(operating_system, chrome_version):
 231   """Returns the list of disabled test matchers for the specific configuration.
 232
 233   Args:
 234     operating_system: The operating system, one of 'linux', 'mac', 'win', or
 235         'android'.
 236     chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
 237
 238   Returns:
 239     List of disabled test matchers, which may contain '*' wildcards.
 240   """
 241   return (_OS_NEGATIVE_FILTER[operating_system] +
 242           _GetRevisionNegativeFilter(chrome_version))[:]
 243
 244
 245 def ApplyJavaTestFilter(operating_system, chrome_version, tests):
 246   """Applies the test filter to the given list of tests.
 247
 248   Args:
 249     operating_system: The operating system, one of 'linux', 'mac', 'win', or
 250         'android'.
 251     chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'.
 252     test: list of test names to filter.
 253
 254   Returns:
 255     Set of passed test names.
 256   """
 257   filters = (_OS_NEGATIVE_FILTER[operating_system] +
 258              _GetRevisionNegativeFilter(chrome_version))
 259   passed = set(tests)
 260   for f in filters:
 261     passed.difference_update(set(t for t in tests if fnmatch.fnmatch(t, f)))
 262   return passed