| # Copyright 2013 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Test expectation list for WebDriver Java acceptance tests. |
| |
| It is evaluated through Python. |
| """ |
| |
| import fnmatch |
| |
| |
| _REVISION_NEGATIVE_FILTER = {} |
| _REVISION_NEGATIVE_FILTER['HEAD'] = [ |
| 'AlertsTest.testIncludesAlertTextInUnhandledAlertException', |
| 'AlertsTest.testSettingTheValueOfAnAlertThrows', |
| 'AlertsTest.testShouldGetTextOfAlertOpenedInSetTimeout', |
| 'AlertsTest.testShouldHandleAlertOnPageUnload', |
| 'AlertsTest.testShouldHandleAlertOnWindowClose', |
| 'AlertsTest.testShouldNotHandleAlertInAnotherWindow', |
| 'AuthenticatedPageLoadingTest.*', |
| 'AvailableLogsTest.driverLogShouldBeEnabledByDefault', |
| 'BasicMouseInterfaceTest.testMousePositionIsNotPreservedInActionsChain', |
| 'BasicMouseInterfaceTest.testMovingIntoAnImageEnclosedInALink', |
| 'BasicMouseInterfaceTest.testMovingMouseBackAndForthPastViewPort', |
| 'BasicMouseInterfaceTest.testMovingMousePastViewPort', |
| 'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions', |
| 'ClickScrollingTest.testShouldBeAbleToClickOnAnElementHiddenByDoubleOverflow', |
| 'ClickScrollingTest.testShouldNotBeAbleToClickElementThatIsOutOfViewInANonScrollableFrame', |
| 'ClickScrollingTest.testShouldNotScrollIfAlreadyScrolledAndElementIsInView', |
| 'ClickTest.testShouldBeAbleToClickOnAnElementInFrameGreaterThanTwoViewports', |
| 'CombinedInputActionsTest.testClickingOnFormElements', |
| 'CoordinatesTest.testShouldScrollPageAndGetCoordinatesOfAnElementThatIsOutOfViewPort', |
| 'CorrectEventFiringTest.testShouldFireEventsInTheRightOrder', |
| 'CorrectEventFiringTest.testShouldFireFocusEventWhenClicking', |
| 'DragAndDropTest.testDragTooFar', |
| 'ExecutingAsyncJavascriptTest.includesAlertTextInUnhandledAlertException', |
| 'ExecutingAsyncJavascriptTest.shouldCatchErrorsWithMessageAndStacktraceWhenExecutingInitialScript', |
| 'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScript', |
| 'ExecutingAsyncJavascriptTest.throwsIfAlertHappensDuringScriptWhichTimesOut', |
| 'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlert', |
| 'ExecutingAsyncJavascriptTest.throwsIfScriptTriggersAlertWhichTimesOut', |
| 'ExecutingJavascriptTest.testShouldThrowExceptionIfExecutingOnNoPage', |
| 'ExecutingJavascriptTest.testShouldThrowAnExceptionWithMessageAndStacktraceWhenTheJavascriptIsBad', |
| 'FormHandlingTest.testShouldNotBeAbleToSubmitAFormThatDoesNotExist', |
| 'FrameSwitchingTest.testShouldNotBeAbleToDoAnythingTheFrameIsDeletedFromUnderUs', |
| # https://code.google.com/p/chromedriver/issues/detail?id=1249 |
| 'FrameSwitchingTest.testGetCurrentUrl', |
| 'I18nTest.testShouldBeAbleToActivateIMEEngine', |
| # Broken because AddWebStorage.java is broken. |
| 'LocalStorageTest.*', |
| 'LocationContextTest.*', |
| 'MiscTest.testShouldBeAbleToGetTheSourceOfAnXmlDocument', |
| 'PageLoadingTest.shouldBeAbleToDisableAcceptOfInsecureSslCertsWithRequiredCapability', |
| 'PageLoadingTest.testShouldNotWaitIndefinitelyIfAnExternalResourceFailsToLoad', |
| # PerformanceLoggingTest is for firefox only; "performance" log deprecated. |
| 'PerformanceLoggingTest.*', |
| 'ProxySettingTest.canConfigureProxyWithRequiredCapability', |
| 'ProxySettingTest.requiredProxyCapabilityShouldHavePriority', |
| 'RenderedWebElementTest.testShouldHandleNonIntegerPositionAndSize', |
| # Broken because AddWebStorage.java is broken. |
| 'SessionStorageTest.*', |
| 'SqlDatabaseTest.*', |
| 'SvgDocumentTest.testClickOnSvgElement', |
| 'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePage', |
| 'TakesScreenshotTest.testShouldCaptureScreenshotWithLongX', |
| 'TakesScreenshotTest.testShouldCaptureScreenshotWithLongY', |
| 'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLong', |
| 'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongX', |
| 'TakesScreenshotTest.testShouldCaptureScreenshotWithTooLongY', |
| 'TextPagesTest.testShouldBeAbleToLoadASimplePageOfText', |
| 'TextPagesTest.testShouldThrowExceptionWhenAddingCookieToAPageThatIsNotHtml', |
| 'TypingTest.testGenerateKeyPressEventEvenWhenElementPreventsDefault', |
| 'TypingTest.testNonPrintableCharactersShouldWorkWithContentEditableOrDesignModeSet', |
| 'TypingTest.testShouldBeAbleToTypeIntoContentEditableElementWithExistingValue', |
| 'TypingTest.testShouldNotTypeIntoElementsThatPreventKeyDownEvents', |
| 'TypingTest.testTypingIntoAnIFrameWithContentEditableOrDesignModeSet', |
| 'UnexpectedAlertBehaviorTest.requiredUnhandledAlertCapabilityHasPriorityOverDesired', |
| 'UnexpectedAlertBehaviorTest.dismissUnhandledAlertsByDefault', |
| 'VisibilityTest.testElementHiddenByOverflowXIsNotVisible', |
| 'VisibilityTest.testElementHiddenByOverflowYIsNotVisible', |
| 'VisibilityTest.tooSmallAWindowWithOverflowHiddenIsNotAProblem', |
| 'WindowTest.testCanMaximizeTheWindowFromFrame', |
| 'WindowTest.testCanMaximizeTheWindowFromIframe', |
| 'WindowTest.testSetsTheSizeOfTheCurrentWindow', |
| 'WindowTest.testCanMaximizeTheWindow', |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528 |
| 'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo', |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=653 |
| 'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes', |
| 'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistory', |
| # https://code.google.com/p/chromedriver/issues/detail?id=1117 |
| 'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate', |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=1113 |
| 'FrameSwitchingTest.testShouldNotSwitchMagicallyToTheTopWindow', |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=1151 |
| 'DragAndDropTest.testDragAndDrop', |
| 'DragAndDropTest.testShouldAllowUsersToDragAndDropToElementsOffTheCurrentViewPort', |
| 'DragAndDropTest.testElementInDiv', |
| # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1503 |
| 'BasicMouseInterfaceTest.testDraggingElementWithMouseFiresEvents', |
| 'BasicMouseInterfaceTest.testDraggingElementWithMouseMovesItToAnotherList', |
| # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1674 |
| 'WindowSwitchingTest.testShouldBeAbleToIterateOverAllOpenWindows', |
| ] |
| _REVISION_NEGATIVE_FILTER['57'] = ( |
| _REVISION_NEGATIVE_FILTER['HEAD'] + [ |
| # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1625 |
| 'TakesScreenshotTest.*', |
| 'WindowTest.*', |
| ] |
| ) |
| |
| _OS_NEGATIVE_FILTER = {} |
| _OS_NEGATIVE_FILTER['win'] = [ |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=373 |
| 'RenderedWebElementTest.testHoverPersists', |
| 'RenderedWebElementTest.canClickOnASuckerFishStyleMenu', |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=416 |
| 'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching', |
| 'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage', |
| ] |
| _OS_NEGATIVE_FILTER['linux'] = [ |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=416 |
| 'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage', |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=1148 |
| 'CombinedInputActionsTest.testCombiningShiftAndClickResultsInANewWindow', |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=1150 |
| 'BasicKeyboardInterfaceTest.testBasicKeyboardInputOnActiveElement', |
| ] |
| _OS_NEGATIVE_FILTER['mac'] = [ |
| # https://code.google.com/p/chromedriver/issues/detail?id=26 |
| 'AlertsTest.testAlertShouldNotAllowAdditionalCommandsIfDismissed', |
| 'AlertsTest.testShouldAllowUsersToDismissAnAlertManually', |
| 'FormHandlingTest.handleFormWithJavascriptAction', |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=354 |
| 'AlertsTest.testShouldAllowUsersToAcceptAnAlertInAFrame', |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=375 |
| 'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes', |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=416 |
| 'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage', |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=1149 |
| 'ChromeDriverTests.testShouldAllowTheUserToSwitchToAnIFrameAndRemainFocusedOnIt', |
| ] |
| |
| _SPECIFIC_OS_REVISION_NEGATIVE_FILTER = {} |
| |
| _OS_NEGATIVE_FILTER['android:chrome'] = [ |
| 'ChromeOptionsFunctionalTest.canStartChromeWithCustomOptions', |
| 'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrame', |
| |
| 'ClickTest.testShouldOnlyFollowHrefOnce', |
| 'CombinedInputActionsTest.testCombiningShiftAndClickResultsInANewWindow', |
| 'ElementSelectingTest.testShouldBeAbleToToggleEnabledMultiSelectOption', |
| 'FrameSwitchingTest.testShouldBeAbleToClickInAFrameThatRewritesTopWindowLocation', |
| 'JavascriptEnabledDriverTest.testShouldBeAbleToClickALinkThatClosesAWindow', |
| 'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate', |
| 'PageLoadingTest.testShouldBeAbleToAccessPagesWithAnInsecureSslCertificate', |
| 'PageLoadingTest.testShouldBeAbleToNavigateBackInTheBrowserHistoryInPresenceOfIframes', |
| 'PageLoadingTest.testShouldFollowMetaRedirects', |
| 'PageLoadingTest.testShouldWaitForDocumentToBeLoaded', |
| 'PerformanceLogTypeTest.pageLoadShouldProducePerformanceLogEntries', |
| 'PerformanceLogTypeTest.shouldBeAbleToEnablePerformanceLog', |
| 'SelectElementHandlingTest.testShouldBePossibleToDeselectASingleOptionFromASelectWhichAllowsMultipleChoices', |
| 'SelectElementTest.shouldAllowOptionsToBeDeselectedByIndex', |
| 'SelectElementTest.shouldAllowOptionsToBeDeselectedByReturnedValue', |
| 'SelectElementTest.shouldAllowUserToDeselectAllWhenSelectSupportsMultipleSelections', |
| 'SelectElementTest.shouldAllowUserToDeselectOptionsByVisibleText', |
| 'SessionHandlingTest.callingAnyOperationAfterClosingTheLastWindowShouldThrowAnException', |
| 'SessionHandlingTest.callingQuitAfterClosingTheLastWindowIsANoOp', |
| 'TakesScreenshotTest.testCaptureToBase64', |
| 'TakesScreenshotTest.testSaveScreenshotAsFile', |
| 'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePage', |
| 'TakesScreenshotTest.testShouldCaptureScreenshotAtFramePageAfterSwitching', |
| 'TakesScreenshotTest.testShouldCaptureScreenshotAtIFramePageAfterSwitching', |
| |
| # Alerts are not yet supported on Android. |
| 'AlertsTest.*', |
| 'UnexpectedAlertBehaviorTest.*', |
| |
| # http://crbug.com/156390 |
| 'DragAndDropTest.*', |
| |
| # Scrolling touch events are not supported. |
| 'TouchScrollTest.*', |
| |
| # These tests start multiple sessions, which is not supported on a single |
| # Android device. |
| "AvailableLogsTest.shouldBeAbleToEnableProfilerLog", |
| "GetLogsTest.turningOffLogShouldMeanNoLogMessages", |
| "SessionHandlingTest.callingAnyOperationAfterQuitShouldThrowAnException", |
| "SessionHandlingTest.callingQuitMoreThanOnceOnASessionIsANoOp", |
| # Test is written using local files; doesn't work on Android. |
| 'UploadTest.testFileUploading', |
| |
| # Flaky: https://code.google.com/p/chromedriver/issues/detail?id=528 |
| 'PageLoadingTest.testShouldDoNothingIfThereIsNothingToGoBackTo', |
| |
| # https://code.google.com/p/chromedriver/issues/detail?id=604 |
| 'I18nTest.testShouldBeAbleToActivateIMEEngine', |
| |
| # https://code.google.com/p/chromium/issues/detail?id=418590 |
| 'ClearTest.testContentEditableAreaShouldClear', |
| 'ClickScrollingTest.testClickingOnAnchorScrollsPage', |
| 'JavascriptEnabledDriverTest.testChangeEventIsFiredAppropriatelyWhenFocusIsLost', |
| 'TypingTest.testShouldBeAbleToTypeIntoEmptyContentEditableElement', |
| |
| # https://code.google.com/p/chromedriver/issues/detail?id=922 |
| 'CorrectEventFiringTest.testShouldEmitOnClickEventsWhenSelectingElements', |
| 'CorrectEventFiringTest.testSendingKeysToAnotherElementShouldCauseTheBlurEventToFire', |
| |
| # https://code.google.com/p/chromedriver/issues/detail?id=1176 |
| 'ChromeDriverTests.clientLogShouldBeEnabledByDefault', |
| |
| # https://code.google.com/p/chromedriver/issues/detail?id=1119 |
| 'CombinedInputActionsTest.testCanClickOnLinksWithAnOffset', |
| 'CombinedInputActionsTest.testMouseMovementWorksWhenNavigatingToAnotherPage', |
| # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1005 |
| 'WindowTest.*', |
| |
| # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1478 |
| 'WindowSwitchingTest.*', |
| ] |
| _OS_NEGATIVE_FILTER['android:chrome_stable'] = ( |
| _OS_NEGATIVE_FILTER['android:chrome'] + [ |
| # https://bugs.chromium.org/p/chromedriver/issues/detail?id=998 |
| 'ImplicitWaitTest.testShouldImplicitlyWaitForASingleElement', |
| ] |
| ) |
| |
| _OS_NEGATIVE_FILTER['android:chromium'] = ( |
| _OS_NEGATIVE_FILTER['android:chrome'] + [ |
| 'CorrectEventFiringTest.testShouldFireClickEventWhenClicking', |
| 'CorrectEventFiringTest.testShouldFireMouseDownEventWhenClicking', |
| # https://bugs.chromium.org/p/chromedriver/issues/detail?id=998 |
| 'ImplicitWaitTest.testShouldImplicitlyWaitForASingleElement', |
| # https://bugs.chromium.org/p/chromedriver/issues/detail?id=922 |
| 'ClickTest.testCanClickOnAnElementWithTopSetToANegativeNumber', |
| 'CorrectEventFiringTest.testClickEventsShouldBubble', |
| 'VisibilityTest.testShouldModifyTheVisibilityOfAnElementDynamically', |
| 'CorrectEventFiringTest.testSendingKeysToAFocusedElementShouldNotBlurThatElement', |
| 'JavascriptEnabledDriverTest.testShouldBeAbleToClickOnSubmitButtons', |
| 'JavascriptEinabledDriverTest.testIssue80ClickShouldGenerateClickEvent', |
| 'CorrectEventFiringTest.testShouldFireMouseUpEventWhenClicking', |
| 'TypingTest.testShiftSelectionDeletes', |
| ] |
| ) |
| _OS_NEGATIVE_FILTER['android:chrome_beta'] = ( |
| _OS_NEGATIVE_FILTER['android:chrome'] + [ |
| # https://code.google.com/p/chromedriver/issues/detail?id=1220 |
| 'BasicMouseInterfaceTest.testDragAndDrop', |
| 'BasicMouseInterfaceTest.testDraggingElementWithMouseFiresEvents', |
| 'BasicMouseInterfaceTest.testDraggingElementWithMouseMovesItToAnotherList', |
| # https://bugs.chromium.org/p/chromedriver/issues/detail?id=998 |
| 'ImplicitWaitTest.testShouldImplicitlyWaitForASingleElement', |
| # https://bugs.chromium.org/p/chromedriver/issues/detail?id=922 |
| 'ClickTest.testCanClickOnAnElementWithTopSetToANegativeNumber', |
| 'CorrectEventFiringTest.testClickEventsShouldBubble', |
| 'CorrectEventFiringTest.testShouldFireMouseUpEventWhenClicking', |
| 'CorrectEventFiringTest.testShouldFireMouseDownEventWhenClicking', |
| 'ImplicitWaitTest.testShouldImplicitlyWaitUntilAtLeastOneElementIsFoundWhenSearchingForMany', |
| 'ImplicitWaitTest.testShouldImplicitlyWaitForAnElementToBeVisibleBeforeInteracting', |
| 'JavascriptEnabledDriverTest.testShouldBeAbleToClickOnSubmitButtons', |
| 'JavascriptEnabledDriverTest.testIssue80ClickShouldGenerateClickEvent', |
| 'TypingTest.testShiftSelectionDeletes', |
| 'VisibilityTest.testShouldModifyTheVisibilityOfAnElementDynamically', |
| ] |
| ) |
| _OS_NEGATIVE_FILTER['android:chromedriver_webview_shell'] = ( |
| _OS_NEGATIVE_FILTER['android:chrome'] + [ |
| # https://code.google.com/p/chromedriver/issues/detail?id=645 |
| 'ClickScrollingTest.testShouldBeAbleToClickElementInAFrameThatIsOutOfView', |
| 'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInAFrameThatIsOutOfView', |
| 'FormHandlingTest.handleFormWithJavascriptAction', |
| 'TypingTest.testShouldReportKeyCodeOfArrowKeysUpDownEvents', |
| 'TypingTest.testShouldReportKeyCodeOfArrowKeys', |
| # https://bugs.chromium.org/p/chromedriver/issues/detail?id=1006 |
| 'ClickScrollingTest.testShouldBeAbleToClickElementThatIsOutOfViewInANestedFrameThatIsOutOfView', |
| # Not applicable on ChromeDriverWebViewShell (doesn't support tabs). |
| 'WindowSwitchingTest.*', |
| |
| 'TakesScreenshotTest.testShouldCaptureScreenshot', |
| # TODO(gmanikpure): re-enable this test when we stop supporting |
| # WebView on KitKat. |
| 'ClickTest.testCanClickAnImageMapArea', |
| ] |
| ) |
| |
| |
| def _GetRevisionNegativeFilter(chrome_version): |
| if chrome_version in _REVISION_NEGATIVE_FILTER: |
| return _REVISION_NEGATIVE_FILTER[chrome_version] |
| return _REVISION_NEGATIVE_FILTER['HEAD'] |
| |
| def _GetSpecificOsRevisionNegativeFilter(operating_system, chrome_version): |
| os_version = operating_system + "_" + chrome_version |
| if os_version in _SPECIFIC_OS_REVISION_NEGATIVE_FILTER: |
| return _SPECIFIC_OS_REVISION_NEGATIVE_FILTER[os_version] |
| return [] |
| |
| |
| def GetDisabledTestMatchers(operating_system, chrome_version): |
| """Returns the list of disabled test matchers for the specific configuration. |
| |
| Args: |
| operating_system: The operating system, one of 'linux', 'mac', 'win', or |
| 'android'. |
| chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'. |
| |
| Returns: |
| List of disabled test matchers, which may contain '*' wildcards. |
| """ |
| return (_OS_NEGATIVE_FILTER[operating_system] + |
| _GetRevisionNegativeFilter(chrome_version) + |
| _GetSpecificOsRevisionNegativeFilter(operating_system, |
| chrome_version))[:] |
| |
| |
| def ApplyJavaTestFilter(operating_system, chrome_version, tests): |
| """Applies the test filter to the given list of tests. |
| |
| Args: |
| operating_system: The operating system, one of 'linux', 'mac', 'win', or |
| 'android'. |
| chrome_version: Chrome version to test against, e.g., 'HEAD' or '26'. |
| test: list of test names to filter. |
| |
| Returns: |
| Set of passed test names. |
| """ |
| filters = (_OS_NEGATIVE_FILTER[operating_system] + |
| _GetRevisionNegativeFilter(chrome_version) + |
| _GetSpecificOsRevisionNegativeFilter(operating_system, |
| chrome_version)) |
| passed = set(tests) |
| for f in filters: |
| passed.difference_update(set(t for t in tests if fnmatch.fnmatch(t, f))) |
| return passed |