test_framework
/
SeleniumBase
mirror of https://github.com/seleniumbase/SeleniumBase.git
3
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #301 from seleniumbase/automated-visual-testing 

Automated Visual Testing
This commit is contained in:
Michael Mintz 2019-04-01 03:19:46 -04:00 committed by GitHub
parent 4553a901f7 932ee2409f
commit 592dfbaf15
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 419 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.gitignore vendored
View File

@ -66,6 +66,9 @@ report.xml
# Tours
tours_exported
# Automated Visual Testing
visual_baseline
# Other
selenium-server-standalone.jar
proxy.zip

25
examples/visual_test.py Executable file
View File

@ -0,0 +1,25 @@
from seleniumbase import BaseCase
class AutomatedVisualTest(BaseCase):
def test_applitools_helloworld(self):
self.open('https://applitools.com/helloworld?diff1')
print('Creating baseline in "visual_baseline" folder...')
self.check_window(name="helloworld", baseline=True)
self.click('a[href="?diff1"]')
# Verify html tags match previous version
self.check_window(name="helloworld", level=1)
# Verify html tags + attributes match previous version
self.check_window(name="helloworld", level=2)
# Verify html tags + attributes + values match previous version
self.check_window(name="helloworld", level=3)
# Change the page enough for a Level-3 comparison to fail
self.click("button")
self.check_window(name="helloworld", level=1)
self.check_window(name="helloworld", level=2)
with self.assertRaises(Exception):
self.check_window(name="helloworld", level=3)
# Now that we know the exception was raised as expected,
# let's print out the comparison results by running in Level-0.
self.check_window(name="helloworld", level=0)

10
help_docs/method_summary.md
View File

@ -172,6 +172,14 @@ self.get_domain_url(url)
self.get_beautiful_soup(source=None)
self.get_unique_links()
self.get_link_status_code(link, allow_redirects=False, timeout=5)
self.assert_no_404_errors()
self.print_unique_links_with_status_codes()
self.safe_execute_script(script)
self.download_file(file_url, destination_folder=None)
@ -345,6 +353,8 @@ self.switch_to_window(window, timeout=settings.SMALL_TIMEOUT)
self.switch_to_default_window()
self.check_window(name="default", level=0, baseline=False)
self.save_screenshot(name, folder=None)
self.get_new_driver(browser=None, headless=None, servername=None, port=None,

6
requirements.txt
View File

@ -8,11 +8,11 @@ unittest2
selenium==3.141.0
requests==2.21.0
urllib3==1.24.1
pytest>=4.3.1
pytest>=4.4.0
pytest-cov>=2.6.1
pytest-html>=1.20.0
pytest-rerunfailures>=6.0
pytest-xdist>=1.26.1
pytest-rerunfailures>=7.0
pytest-xdist>=1.27.0
parameterized>=0.7.0
beautifulsoup4>=4.6.0
colorama==0.4.1

19
seleniumbase/core/visual_helper.py Executable file
View File

@ -0,0 +1,19 @@
import os
from seleniumbase.fixtures import constants
VISUAL_BASELINE_DIR = constants.VisualBaseline.STORAGE_FOLDER
abs_path = os.path.abspath('.')
visual_baseline_path = os.path.join(abs_path, VISUAL_BASELINE_DIR)
def get_visual_baseline_folder():
return visual_baseline_path
def visual_baseline_folder_setup():
""" Handle Logging """
if not os.path.exists(visual_baseline_path):
try:
os.makedirs(visual_baseline_path)
except Exception:
pass # Should only be reachable during multi-threaded runs

263
seleniumbase/fixtures/base_case.py
View File

@ -21,6 +21,8 @@ Page elements are given enough time to load before WebDriver acts on them.
Code becomes greatly simplified and easier to maintain.
"""
import codecs
import json
import logging
import math
import os
@ -38,6 +40,7 @@ from seleniumbase.core.testcase_manager import TestcaseManager
from seleniumbase.core import download_helper
from seleniumbase.core import log_helper
from seleniumbase.core import tour_helper
from seleniumbase.core import visual_helper
from seleniumbase.fixtures import constants
from seleniumbase.fixtures import js_utils
from seleniumbase.fixtures import page_actions
@ -71,8 +74,8 @@ class BaseCase(unittest.TestCase):
self.env = None # Add a shortened version of self.environment
self.__last_url_of_delayed_assert = "data:,"
self.__last_page_load_url = "data:,"
self.__page_check_count = 0
self.__page_check_failures = []
self.__delayed_assert_count = 0
self.__delayed_assert_failures = []
# Requires self._* instead of self.__* for external class use
self._html_report_extra = [] # (Used by pytest_plugin.py)
self._default_driver = None
@ -1622,6 +1625,43 @@ class BaseCase(unittest.TestCase):
soup = BeautifulSoup(source, "html.parser")
return soup
def get_unique_links(self):
""" Get all unique links in the html of the page source.
Page links include those obtained from:
"a"->"href", "img"->"src", "link"->"href", and "script"->"src". """
page_url = self.get_current_url()
soup = self.get_beautiful_soup(self.get_page_source())
links = page_utils._get_unique_links(page_url, soup)
return links
def get_link_status_code(self, link, allow_redirects=False, timeout=5):
""" Get the status code of a link.
If the timeout is exceeded, will return a 404.
For a list of available status codes, see:
https://en.wikipedia.org/wiki/List_of_HTTP_status_codes """
status_code = page_utils._get_link_status_code(
link, allow_redirects=allow_redirects, timeout=timeout)
return status_code
def assert_no_404_errors(self):
""" Assert no 404 errors from page links obtained from:
"a"->"href", "img"->"src", "link"->"href", and "script"->"src". """
links = self.get_unique_links()
for link in links:
status_code = str(self.get_link_status_code(link))
bad_link_str = 'Error: "%s" returned a 404!' % link
self.assert_not_equal(status_code, "404", bad_link_str)
def print_unique_links_with_status_codes(self):
""" Finds all unique links in the html of the page source
and then prints out those links with their status codes.
Format: ["link" -> "status_code"] (per line)
Page links include those obtained from:
"a"->"href", "img"->"src", "link"->"href", and "script"->"src". """
page_url = self.get_current_url()
soup = self.get_beautiful_soup(self.get_page_source())
page_utils._print_unique_links_with_status_codes(page_url, soup)
def safe_execute_script(self, script):
""" When executing a script that contains a jQuery command,
it's important that the jQuery library has been loaded first.
@ -1675,16 +1715,16 @@ class BaseCase(unittest.TestCase):
assert os.path.exists(self.get_path_of_downloaded_file(file))
def assert_true(self, expr, msg=None):
self.assertTrue(expr, msg=None)
self.assertTrue(expr, msg=msg)
def assert_false(self, expr, msg=None):
self.assertFalse(expr, msg=None)
self.assertFalse(expr, msg=msg)
def assert_equal(self, first, second, msg=None):
self.assertEqual(first, second, msg=None)
self.assertEqual(first, second, msg=msg)
def assert_not_equal(self, first, second, msg=None):
self.assertNotEqual(first, second, msg=None)
self.assertNotEqual(first, second, msg=msg)
def assert_no_js_errors(self):
""" Asserts that there are no JavaScript "SEVERE"-level page errors.
@ -2250,7 +2290,7 @@ class BaseCase(unittest.TestCase):
return True
# For backwards compatibility, earlier method names of the next
# four methods have remained even though they do the same thing,
# three methods have remained even though they do the same thing,
# with the exception of assert_*, which won't return the element,
# but like the others, will raise an exception if the call fails.
@ -2399,6 +2439,198 @@ class BaseCase(unittest.TestCase):
def switch_to_default_window(self):
self.switch_to_window(0)
def check_window(self, name="default", level=0, baseline=False):
""" *** Automated Visual Testing with SeleniumBase ***
The first time a test calls self.check_window() for a unique "name"
parameter provided, it will set a visual baseline, meaning that it
creates a folder, saves the URL to a file, saves the current window
screenshot to a file, and creates the following three files
with the listed data saved:
tags_level1.txt -> HTML tags from the window
tags_level2.txt -> HTML tags + attributes from the window
tags_level3.txt -> HTML tags + attributes/values from the window
Baseline folders are named based on the test name and the name
parameter passed to self.check_window(). The same test can store
multiple baseline folders.
If the baseline is being set/reset, the "level" doesn't matter.
After the first run of self.check_window(), it will compare the
HTML tags of the latest window to the one from the initial run.
Here's how the level system works:
* level=0 ->
DRY RUN ONLY - Will perform a comparison to the baseline, and
print out any differences that are found, but
won't fail the test even if differences exist.
* level=1 ->
HTML tags are compared to tags_level1.txt
* level=2 ->
HTML tags are compared to tags_level1.txt and
HTML tags/attributes are compared to tags_level2.txt
* level=3 ->
HTML tags are compared to tags_level1.txt and
HTML tags + attributes are compared to tags_level2.txt and
HTML tags + attributes/values are compared to tags_level3.txt
As shown, Level-3 is the most strict, Level-1 is the least strict.
If the comparisons from the latest window to the existing baseline
don't match, the current test will fail, except for Level-0 tests.
You can reset the visual baseline on the command line by using:
--visual_baseline
As long as "--visual_baseline" is used on the command line while
running tests, the self.check_window() method cannot fail because
it will rebuild the visual baseline rather than comparing the html
tags of the latest run to the existing baseline. If there are any
expected layout changes to a website that you're testing, you'll
need to reset the baseline to prevent unnecessary failures.
self.check_window() will fail with "Page Domain Mismatch Failure"
if the page domain doesn't match the domain of the baseline.
If you want to use self.check_window() to compare a web page to
a later version of itself from within the same test run, you can
add the parameter "baseline=True" to the first time you call
self.check_window() in a test to use that as the baseline. This
only makes sense if you're calling self.check_window() more than
once with the same name parameter in the same test.
Automated Visual Testing with self.check_window() is not very
effective for websites that have dynamic content that changes
the layout and structure of web pages. For those, you're much
better off using regular SeleniumBase functional testing.
Example usage:
self.check_window(name="testing", level=0)
self.check_window(name="xkcd_home", level=1)
self.check_window(name="github_page", level=2)
self.check_window(name="wikipedia_page", level=3)
"""
if level == "0":
level = 0
if level == "1":
level = 1
if level == "2":
level = 2
if level == "3":
level = 3
if level != 0 and level != 1 and level != 2 and level != 3:
raise Exception('Parameter "level" must be set to 0, 1, 2, or 3!')
module = self.__class__.__module__
if '.' in module and len(module.split('.')[-1]) > 1:
module = module.split('.')[-1]
test_id = "%s.%s" % (module, self._testMethodName)
if not name or len(name) < 1:
name = "default"
name = str(name)
visual_helper.visual_baseline_folder_setup()
baseline_dir = constants.VisualBaseline.STORAGE_FOLDER
visual_baseline_path = baseline_dir + "/" + test_id + "/" + name
page_url_file = visual_baseline_path + "/page_url.txt"
screenshot_file = visual_baseline_path + "/screenshot.png"
level_1_file = visual_baseline_path + "/tags_level_1.txt"
level_2_file = visual_baseline_path + "/tags_level_2.txt"
level_3_file = visual_baseline_path + "/tags_level_3.txt"
set_baseline = False
if baseline or self.visual_baseline:
set_baseline = True
if not os.path.exists(visual_baseline_path):
set_baseline = True
try:
os.makedirs(visual_baseline_path)
except Exception:
pass # Only reachable during multi-threaded test runs
if not os.path.exists(page_url_file):
set_baseline = True
if not os.path.exists(screenshot_file):
set_baseline = True
if not os.path.exists(level_1_file):
set_baseline = True
if not os.path.exists(level_2_file):
set_baseline = True
if not os.path.exists(level_3_file):
set_baseline = True
page_url = self.get_current_url()
soup = self.get_beautiful_soup()
html_tags = soup.body.find_all()
level_1 = [[tag.name] for tag in html_tags]
level_1 = json.loads(json.dumps(level_1)) # Tuples become lists
level_2 = [[tag.name, sorted(tag.attrs.keys())] for tag in html_tags]
level_2 = json.loads(json.dumps(level_2)) # Tuples become lists
level_3 = [[tag.name, sorted(tag.attrs.items())] for tag in html_tags]
level_3 = json.loads(json.dumps(level_3)) # Tuples become lists
if set_baseline:
self.save_screenshot("screenshot.png", visual_baseline_path)
out_file = codecs.open(page_url_file, "w+")
out_file.writelines(page_url)
out_file.close()
out_file = codecs.open(level_1_file, "w+")
out_file.writelines(json.dumps(level_1))
out_file.close()
out_file = codecs.open(level_2_file, "w+")
out_file.writelines(json.dumps(level_2))
out_file.close()
out_file = codecs.open(level_3_file, "w+")
out_file.writelines(json.dumps(level_3))
out_file.close()
if not set_baseline:
f = open(page_url_file, 'r')
page_url_data = f.read().strip()
f.close()
f = open(level_1_file, 'r')
level_1_data = json.loads(f.read())
f.close()
f = open(level_2_file, 'r')
level_2_data = json.loads(f.read())
f.close()
f = open(level_3_file, 'r')
level_3_data = json.loads(f.read())
f.close()
domain_fail = (
"Page Domain Mismatch Failure: "
"Current Page Domain doesn't match the Page Domain of the "
"Baseline! Can't compare two completely different sites! "
"Run with --visual_baseline to reset the baseline!")
level_1_failure = (
"\n\n*** Exception: <Level 1> Visual Diff Failure:\n"
"* HTML tags don't match the baseline!")
level_2_failure = (
"\n\n*** Exception: <Level 2> Visual Diff Failure:\n"
"* HTML tag attributes don't match the baseline!")
level_3_failure = (
"\n\n*** Exception: <Level 3> Visual Diff Failure:\n"
"* HTML tag attribute values don't match the baseline!")
page_domain = self.get_domain_url(page_url)
page_data_domain = self.get_domain_url(page_url_data)
unittest.TestCase.maxDiff = 1000
if level == 1 or level == 2 or level == 3:
self.assert_equal(page_domain, page_data_domain, domain_fail)
self.assert_equal(level_1, level_1_data, level_1_failure)
unittest.TestCase.maxDiff = None
if level == 2 or level == 3:
self.assert_equal(level_2, level_2_data, level_2_failure)
if level == 3:
self.assert_equal(level_3, level_3_data, level_3_failure)
if level == 0:
try:
unittest.TestCase.maxDiff = 1000
self.assert_equal(
page_domain, page_data_domain, domain_fail)
self.assert_equal(level_1, level_1_data, level_1_failure)
unittest.TestCase.maxDiff = None
self.assert_equal(level_2, level_2_data, level_2_failure)
self.assert_equal(level_3, level_3_data, level_3_failure)
except Exception as e:
print(e) # Level-0 Dry Run (Only print the differences)
def save_screenshot(self, name, folder=None):
""" The screenshot will be in PNG format. """
return page_actions.save_screenshot(self.driver, name, folder)
@ -2572,16 +2804,16 @@ class BaseCase(unittest.TestCase):
""" Add a delayed_assert failure into a list for future processing. """
current_url = self.driver.current_url
message = self.__get_exception_message()
self.__page_check_failures.append(
self.__delayed_assert_failures.append(
"CHECK #%s: (%s)\n %s" % (
self.__page_check_count, current_url, message))
self.__delayed_assert_count, current_url, message))
def delayed_assert_element(self, selector, by=By.CSS_SELECTOR,
timeout=settings.MINI_TIMEOUT):
""" A non-terminating assertion for an element on a page.
Failures will be saved until the process_delayed_asserts()
method is called from inside a test, likely at the end of it. """
self.__page_check_count += 1
self.__delayed_assert_count += 1
try:
url = self.get_current_url()
if url == self.__last_url_of_delayed_assert:
@ -2602,7 +2834,7 @@ class BaseCase(unittest.TestCase):
""" A non-terminating assertion for text from an element on a page.
Failures will be saved until the process_delayed_asserts()
method is called from inside a test, likely at the end of it. """
self.__page_check_count += 1
self.__delayed_assert_count += 1
try:
url = self.get_current_url()
if url == self.__last_url_of_delayed_assert:
@ -2629,12 +2861,12 @@ class BaseCase(unittest.TestCase):
the delayed asserts on a single html page so that the failure
screenshot matches the location of the delayed asserts.
If "print_only" is set to True, the exception won't get raised. """
if self.__page_check_failures:
if self.__delayed_assert_failures:
exception_output = ''
exception_output += "\n*** DELAYED ASSERTION FAILURES FOR: "
exception_output += "%s\n" % self.id()
all_failing_checks = self.__page_check_failures
self.__page_check_failures = []
all_failing_checks = self.__delayed_assert_failures
self.__delayed_assert_failures = []
for tb in all_failing_checks:
exception_output += "%s\n" % tb
if print_only:
@ -2851,6 +3083,7 @@ class BaseCase(unittest.TestCase):
self.verify_delay = sb_config.verify_delay
self.disable_csp = sb_config.disable_csp
self.save_screenshot_after_test = sb_config.save_screenshot
self.visual_baseline = sb_config.visual_baseline
self.timeout_multiplier = sb_config.timeout_multiplier
self.use_grid = False
if self.servername != "localhost":
@ -2980,7 +3213,7 @@ class BaseCase(unittest.TestCase):
has_exception = True
else:
has_exception = sys.exc_info()[1] is not None
if self.__page_check_failures:
if self.__delayed_assert_failures:
print(
"\nWhen using self.delayed_assert_*() methods in your tests, "
"remember to call self.process_delayed_asserts() afterwards. "

4
seleniumbase/fixtures/constants.py
View File

@ -19,6 +19,10 @@ class Files:
ARCHIVED_DOWNLOADS_FOLDER = "archived_files"
class VisualBaseline:
STORAGE_FOLDER = "visual_baseline"
class JQuery:
VER = "3.3.1"
# MIN_JS = "//cdnjs.cloudflare.com/ajax/libs/jquery/%s/jquery.min.js" % VER

85
seleniumbase/fixtures/page_utils.py
View File

@ -65,6 +65,91 @@ def is_valid_url(url):
return False
def _get_unique_links(page_url, soup):
"""
Returns all unique links.
Includes:
"a"->"href", "img"->"src", "link"->"href", and "script"->"src" links.
"""
prefix = 'http:'
if page_url.startswith('https:'):
prefix = 'https:'
simple_url = page_url.split('://')[1]
base_url = simple_url.split('/')[0]
full_base_url = prefix + "//" + base_url
raw_links = []
raw_unique_links = []
# Get "href" from all "a" tags
links = soup.find_all('a')
for link in links:
raw_links.append(link.get('href'))
# Get "src" from all "img" tags
img_links = soup.find_all('img')
for img_link in img_links:
raw_links.append(img_link.get('src'))
# Get "href" from all "link" tags
links = soup.find_all('link')
for link in links:
raw_links.append(link.get('href'))
# Get "src" from all "script" tags
img_links = soup.find_all('script')
for img_link in img_links:
raw_links.append(img_link.get('src'))
for link in raw_links:
if link not in raw_unique_links:
raw_unique_links.append(link)
unique_links = []
for link in raw_unique_links:
if link and len(link) > 1:
if link.startswith('//'):
link = prefix + link
elif link.startswith('/'):
link = full_base_url + link
elif link.startswith('#'):
link = full_base_url + link
else:
pass
unique_links.append(link)
return unique_links
def _get_link_status_code(link, allow_redirects=False, timeout=5):
""" Get the status code of a link.
If the timeout is exceeded, will return a 404.
For a list of available status codes, see:
https://en.wikipedia.org/wiki/List_of_HTTP_status_codes
"""
status_code = None
try:
response = requests.get(
link, allow_redirects=allow_redirects, timeout=timeout)
status_code = response.status_code
except Exception:
status_code = 404
return status_code
def _print_unique_links_with_status_codes(page_url, soup):
""" Finds all unique links in the html of the page source
and then prints out those links with their status codes.
Format: ["link" -> "status_code"] (per line)
Page links include those obtained from:
"a"->"href", "img"->"src", "link"->"href", and "script"->"src".
"""
links = _get_unique_links(page_url, soup)
for link in links:
status_code = _get_link_status_code(link)
print(link, " -> ", status_code)
def _download_file_to(file_url, destination_folder, new_file_name=None):
if new_file_name:
file_name = new_file_name

8
seleniumbase/plugins/pytest_plugin.py
View File

@ -173,6 +173,13 @@ def pytest_addoption(parser):
default=False,
help="""Take a screenshot on last page after the last step
of the test. (Added to the "latest_logs" folder.)""")
parser.addoption('--visual_baseline', action='store_true',
dest='visual_baseline',
default=False,
help="""Setting this resets the visual baseline for
Automated Visual Testing with SeleniumBase.
When a test calls self.check_window(), it will
rebuild its files in the visual_baseline folder.""")
parser.addoption('--timeout_multiplier', action='store',
dest='timeout_multiplier',
default=None,
@ -212,6 +219,7 @@ def pytest_configure(config):
sb_config.verify_delay = config.getoption('verify_delay')
sb_config.disable_csp = config.getoption('disable_csp')
sb_config.save_screenshot = config.getoption('save_screenshot')
sb_config.visual_baseline = config.getoption('visual_baseline')
sb_config.timeout_multiplier = config.getoption('timeout_multiplier')
if sb_config.with_testing_base:

10
seleniumbase/plugins/selenium_plugin.py
View File

@ -30,6 +30,7 @@ class SeleniumBrowser(Plugin):
self.options.verify_delay -- delay before MasterQA checks (--verify_delay)
self.options.disable_csp -- disable Content Security Policy (--disable_csp)
self.options.save_screenshot -- save screen after test (--save_screenshot)
self.options.visual_baseline -- set the visual baseline (--visual_baseline)
self.options.timeout_multiplier -- increase defaults (--timeout_multiplier)
"""
name = 'selenium' # Usage: --with-selenium
@ -144,6 +145,14 @@ class SeleniumBrowser(Plugin):
default=False,
help="""Take a screenshot on last page after the last step
of the test. (Added to the "latest_logs" folder.)""")
parser.add_option(
'--visual_baseline', action='store_true',
dest='visual_baseline',
default=False,
help="""Setting this resets the visual baseline for
Automated Visual Testing with SeleniumBase.
When a test calls self.check_window(), it will
rebuild its files in the visual_baseline folder.""")
parser.add_option(
'--timeout_multiplier', action='store',
dest='timeout_multiplier',
@ -176,6 +185,7 @@ class SeleniumBrowser(Plugin):
test.test.verify_delay = self.options.verify_delay # MasterQA
test.test.disable_csp = self.options.disable_csp
test.test.save_screenshot_after_test = self.options.save_screenshot
test.test.visual_baseline = self.options.visual_baseline
test.test.timeout_multiplier = self.options.timeout_multiplier
test.test.use_grid = False
if test.test.servername != "localhost":

8
setup.py
View File

@ -17,7 +17,7 @@ except IOError:
setup(
name='seleniumbase',
version='1.21.9',
version='1.22.0',
description='Reliable Browser Automation & Testing Framework',
long_description=long_description,
long_description_content_type='text/markdown',
@ -61,11 +61,11 @@ setup(
'selenium==3.141.0',
'requests==2.21.0', # Changing this may effect "urllib3"
'urllib3==1.24.1', # Keep this lib in sync with "requests"
'pytest>=4.3.1',
'pytest>=4.4.0',
'pytest-cov>=2.6.1',
'pytest-html>=1.20.0',
'pytest-rerunfailures>=6.0',
'pytest-xdist>=1.26.1',
'pytest-rerunfailures>=7.0',
'pytest-xdist>=1.27.0',
'parameterized>=0.7.0',
'beautifulsoup4>=4.6.0', # Keep at >=4.6.0 while using bs4
'colorama==0.4.1',