Merge pull request #91 from cbsiddharth/dev

Add blackbox testing scripts using pytest

.gitignore

@@ -44,7 +44,8 @@ tests/testsuite.dir/
 /python/.deps
 /python/.libs
 /python/setup.py
-/python/tdnf/__pycache__
+__pycache__/
+.pytest_cache/
 
 # Object files
 *.o

Dockerfile

@@ -6,6 +6,10 @@ RUN tdnf update  -q -y
 RUN tdnf remove  -q -y toybox
 RUN tdnf install -q -y build-essential autoconf automake make \
                        libtool sed curl-devel rpm-build popt-devel \
-                       libsolv-devel createrepo_c glib libxml2 findutils
+                       libsolv-devel createrepo_c glib libxml2 findutils \
+                       python3 python3-pip python3-setuptools
+
+# python build/test dependencies
+RUN pip3 install -q requests urllib3 pyOpenSSL pytest
 
 ENTRYPOINT ["./docker-entrypoint.sh"]

build-docker.sh

@@ -2,4 +2,3 @@
 
 docker build -t photon/tdnf-build .
 docker run --rm -it -v$(pwd):/build -w/build photon/tdnf-build
-

docker-entrypoint.sh

@@ -7,3 +7,7 @@ autoreconf
 ./configure
 make
 make check
+
+# pytests
+make install
+cd pytests && pytest --verbose

pytests/config.json

@@ -0,0 +1,54 @@
+{
+    "sglversion_pkgname": "gzip",
+    "mulversion_pkgname": "cloud-init",
+    "mulversion_lower": "18.3-4.ph3",
+    "mulPkgs": "etcd zip gzip",
+    "pkghassub": "nfs-utils",
+    "subpkg": "libtirpc",
+    "updateinfo_pkgname": "etcd",
+    "updateinfo_pkgversion": "3.0.9",
+    "installroot_pkgname": "gzip",
+    "installroot_pkgbin": "gzip",
+    "stress_test_times": 1000,
+    "check_local_pkg_url": "https://bintray.com/vmware/photon_release_3.0_x86_64/download_file?file_path=x86_64%2Fetcd-3.3.9-1.ph3.x86_64.rpm",
+    "test_list": [
+        "help",
+        "check-local",
+        "check-update",
+        "clean",
+        "erase",
+        "info",
+        "install",
+        "list",
+        "makecache",
+        "provides",
+        "remove",
+        "reinstall",
+        "repolist",
+        "search",
+        "updateinfo",
+        "downgrade",
+        "upgrade",
+        "upgrade-to",
+        "distro-sync",
+        "group",
+        "history",
+        "repository-packages",
+        "config",
+        "debugsolver",
+        "rpmverbosity",
+        "v",
+        "y",
+        "assumeno",
+        "version",
+        "installroot",
+        "nogpgcheck",
+        "releasever",
+        "setopt",
+        "refresh",
+        "cache",
+        "count",
+        "multiple packages",
+        "unexist"
+    ]
+}

pytests/conftest.py

@@ -0,0 +1,130 @@
+#
+# Copyright (C) 2019 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the GNU General Public License v2 (the "License");
+# you may not use this file except in compliance with the License. The terms
+# of the License are located in the COPYING file of this distribution.
+#
+#   Author: Siddharth Chandrasekaran <csiddharth@vmware.com>
+
+import pytest
+import json
+import subprocess
+import os
+import shutil
+import ssl
+import requests
+from urllib.parse import urlparse
+from OpenSSL.crypto import load_certificate, FILETYPE_PEM
+
+class JsonWrapper(object):
+
+    def __init__(self, filename):
+        self.filename = filename
+
+    def read(self):
+        with open(self.filename) as json_data:
+            self.data = json.load(json_data)
+        return self.data
+
+    def write(self, data):
+        self.data = data
+        with open(self.filename, 'wb') as outfile:
+            json.dump(data, outfile)
+
+
+class TestUtils(object):
+    def __init__(self):
+        cur_dir = os.path.dirname(os.path.realpath(__file__))
+        config_file = os.path.join(cur_dir, 'config.json')
+        self.config = JsonWrapper(config_file).read()
+
+    def check_package(self, package):
+        """ Check if a package exists """
+        ret = self.run([ 'tdnf', 'list', package ])
+        for line in ret['stdout']:
+            if package in line and '@System' in line:
+                return True
+        return False
+
+    def remove_package(self, package):
+        ret = self.run([ 'tdnf', 'erase', '-y', package ])
+        if ret['retval'] != 0:
+            return False
+        return True
+
+    def _requests_get(self, url, verify):
+        try:
+            r = requests.get(url, verify=verify, stream=True, timeout=5.0)
+        except:
+            return None
+        return r
+
+    def wget(self, url, out, enforce_https=True, fingerprint=None):
+        # Check URL
+        try:
+            u = urlparse(url)
+        except:
+            return False, "Failed to parse URL"
+        if not all([ u.scheme, u.netloc ]):
+            return False, 'Invalid URL'
+        if enforce_https:
+            if u.scheme != 'https':
+                return False, 'URL must be of secure origin (HTTPS)'
+        r = self._requests_get(url, True)
+        if r is None:
+            if fingerprint is None:
+                return False, "Unable to verify server certificate"
+            port = u.port
+            if port is None:
+                port = 443
+            try:
+                pem = ssl.get_server_certificate((u.netloc, port))
+                cert = load_certificate(FILETYPE_PEM, pem)
+                fp = cert.digest('sha1').decode()
+            except:
+                return False, "Failed to get server certificate"
+            if fingerprint != fp:
+                return False, "Server fingerprint did not match provided. Got: " + fp
+            # Download file without validation
+            r = self._requests_get(url, False)
+            if r is None:
+                return False, "Failed to download file"
+        r.raw.decode_content = True
+        with open(out, 'wb') as f:
+            shutil.copyfileobj(r.raw, f)
+
+        return True, None
+
+    def run(self, cmd):
+        use_shell = not isinstance(cmd, list)
+        process = subprocess.Popen(cmd, shell=use_shell,
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE)
+        process.wait()
+        out, err = process.communicate()
+        ret = {}
+        ret['stdout'] = []
+        ret['stdout'] = []
+        ret['retval'] = process.returncode
+        if out:
+            ret['stdout'] = out.decode().split('\n')
+        if err:
+            ret['strerr'] = err.decode().split('\n')
+
+        return ret
+
+def backup_config_files():
+    # Backup /etc/yum.repos.d/* and /etc/tdnf/tdnf.conf
+    pass
+
+def restore_config_files():
+    # Restore /etc/yum.repos.d/* and /etc/tdnf/tdnf.conf
+    pass
+
+@pytest.fixture(scope='session')
+def utils():
+    test_utils = TestUtils()
+    backup_config_files()
+    yield test_utils
+    restore_config_files()

pytests/tests/test_assumeno.py

@@ -0,0 +1,31 @@
+#
+# Copyright (C) 2019 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the GNU General Public License v2 (the "License");
+# you may not use this file except in compliance with the License. The terms
+# of the License are located in the COPYING file of this distribution.
+#
+#   Author: Siddharth Chandrasekaran <csiddharth@vmware.com>
+
+import pytest
+
+@pytest.fixture(scope='module', autouse=True)
+def setup_test(utils):
+    # remove sglversion_pkgname at the beginning of tests
+    pkgname = utils.config['sglversion_pkgname']
+    utils.run([ 'tdnf', 'erase', '-y', pkgname ])
+    yield
+    teardown_test(utils)
+
+def teardown_test(utils):
+    pass
+
+def test_assumeno_install(utils):
+    pkgname = utils.config['sglversion_pkgname']
+    utils.run([ 'tdnf', '--assumeno', 'install', pkgname ])
+    assert (utils.check_package(pkgname) == False)
+
+def test_assumeno_erase(utils):
+    pkgname = utils.config['sglversion_pkgname']
+    utils.run([ 'tdnf', '--assumeno',  'erase', pkgname ])
+    assert (utils.check_package(pkgname) == False)

pytests/tests/test_cache.py

@@ -0,0 +1,91 @@
+#
+# Copyright (C) 2019 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the GNU General Public License v2 (the "License");
+# you may not use this file except in compliance with the License. The terms
+# of the License are located in the COPYING file of this distribution.
+#
+#   Author: Siddharth Chandrasekaran <csiddharth@vmware.com>
+
+import os
+import pytest
+
+@pytest.fixture(scope='module', autouse=True)
+def setup_test(utils):
+    utils.run([ 'cp', '/etc/yum.repos.d/photon.repo', '/etc/yum.repos.d/photon.repo.bak' ])
+    utils.run([ 'cp', '/etc/yum.repos.d/photon-iso.repo', '/etc/yum.repos.d/photon-iso.repo.bak' ])
+    utils.run([ 'sed', '-i', 's/enabled=0/enabled=1/g', '/etc/yum.repos.d/photon.repo' ])
+    utils.run([ 'sed', '-i', 's/enabled=1/enabled=0/g', '/etc/yum.repos.d/photon-test.repo' ])
+    yield
+    teardown_test(utils)
+
+def teardown_test(utils):
+    utils.run([ 'cp', '/etc/yum.repos.d/photon.repo.bak', '/etc/yum.repos.d/photon.repo' ])
+    utils.run([ 'cp', '/etc/yum.repos.d/photon-iso.repo.bak', '/etc/yum.repos.d/photon-iso.repo' ])
+    utils.run([ 'sed', '-i', 's/enabled=1/enabled=0/g', '/etc/yum.repos.d/photon.repo' ])
+    utils.run([ 'sed', '-i', 's/enabled=0/enabled=1/g', '/etc/yum.repos.d/photon-test.repo' ])
+
+def clean_cache(utils):
+    utils.run([ 'rm', '-rf', '/var/cache/tdnf' ])
+
+def enable_cache(utils):
+    utils.run([ 'sed', '-i', '/keepcache/d', '/etc/tdnf/tdnf.conf' ])
+    utils.run([ 'sed', '-i', '$ a keepcache=true', '/etc/tdnf/tdnf.conf' ])
+
+def disable_cache(utils):
+    utils.run([ 'sed', '-i', '/keepcache/d', '/etc/tdnf/tdnf.conf' ])
+
+def check_package_in_cache(utils, pkgname):
+    ret = utils.run([ 'find', '/var/cache/tdnf', '-name', pkgname + '*.rpm' ])
+    if ret['stdout']:
+        return True
+    return False
+
+def test_install_without_cache(utils):
+    clean_cache(utils)
+    disable_cache(utils)
+
+    pkgname = utils.config["sglversion_pkgname"]
+    if utils.check_package(pkgname):
+        utils.remove_package(pkgname)
+
+    utils.run([ 'tdnf', 'install', '-y', '--nogpgcheck', pkgname ])
+
+    assert (check_package_in_cache(utils, pkgname) == False)
+
+def test_install_with_cache(utils):
+    clean_cache(utils)
+    enable_cache(utils)
+
+    pkgname = utils.config["sglversion_pkgname"]
+    if utils.check_package(pkgname):
+        utils.remove_package(pkgname)
+
+    utils.run([ 'tdnf', 'install', '-y', '--nogpgcheck', pkgname ])
+
+    assert (check_package_in_cache(utils, pkgname) == True)
+
+def test_install_with_keepcache_false(utils):
+    clean_cache(utils)
+    disable_cache(utils)
+    utils.run([ 'sed', '-i', '$ a keepcache=false', '/etc/tdnf/tdnf.conf' ])
+
+    pkgname = utils.config["sglversion_pkgname"]
+    if utils.check_package(pkgname):
+        utils.remove_package(pkgname)
+
+    utils.run([ 'tdnf', 'install', '-y', '--nogpgcheck', pkgname ])
+
+    assert (check_package_in_cache(utils, pkgname) == False)
+
+def test_disable_repo_make_cache(utils):
+    before = os.path.getmtime('/var/cache/tdnf/photon/lastrefresh')
+    utils.run([ 'tdnf', '--disablerepo=*', 'makecache' ])
+    after = os.path.getmtime('/var/cache/tdnf/photon/lastrefresh')
+    assert (before == after)
+
+def test_enable_repo_make_cache(utils):
+    before = os.path.getmtime('/var/cache/tdnf/photon/lastrefresh')
+    utils.run([ 'tdnf', '--disablerepo=*', '--enablerepo=photon', 'makecache' ])
+    after = os.path.getmtime('/var/cache/tdnf/photon/lastrefresh')
+    assert (before < after)

pytests/tests/test_check_local.py

@@ -0,0 +1,39 @@
+#
+# Copyright (C) 2019 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the GNU General Public License v2 (the "License");
+# you may not use this file except in compliance with the License. The terms
+# of the License are located in the COPYING file of this distribution.
+#
+#   Author: Siddharth Chandrasekaran <csiddharth@vmware.com>
+
+import os
+import tempfile
+import pytest
+
+@pytest.fixture(scope='module', autouse=True)
+def setup_test(utils):
+    yield
+    teardown_test(utils)
+
+def teardown_test(utils):
+    pass
+
+def test_check_local_no_args(utils):
+    ret = utils.run([ 'tdnf', 'check_local' ])
+    assert (ret['retval'] == 0)
+
+def test_check_local_empty_directory(utils):
+    temp_dir = tempfile.TemporaryDirectory()
+    ret = utils.run([ 'tdnf', 'check_local', temp_dir.name ])
+    assert (ret['retval'] == 0)
+
+def test_check_local_with_one_rpm(utils):
+    with tempfile.TemporaryDirectory() as tmpdir:
+        dest = os.path.join(tmpdir, 'test.rpm')
+        url = utils.config['check_local_pkg_url']
+        dl_status, _ = utils.wget(url, dest, enforce_https=False)
+        assert (dl_status == True)
+
+        ret = utils.run([ 'tdnf', 'check_local', tmpdir ])
+        assert (ret['retval'] == 0)

pytests/tests/test_check_update.py

@@ -0,0 +1,45 @@
+#
+# Copyright (C) 2019 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the GNU General Public License v2 (the "License");
+# you may not use this file except in compliance with the License. The terms
+# of the License are located in the COPYING file of this distribution.
+#
+#   Author: Siddharth Chandrasekaran <csiddharth@vmware.com>
+
+import pytest
+
+@pytest.fixture(scope='module', autouse=True)
+def setup_test(utils):
+    utils.run([ 'sed', '-i', 's/enabled=0/enabled=1/g', '/etc/yum.repos.d/photon-extras.repo' ])
+    yield
+    teardown_test(utils)
+
+def teardown_test(utils):
+    utils.run([ 'sed', '-i', 's/enabled=1/enabled=0/g', '/etc/yum.repos.d/photon-extras.repo' ])
+    pass
+
+def test_check_update_no_arg(utils):
+    ret = utils.run([ 'tdnf', 'check-update' ])
+    assert (ret['retval'] == 0)
+
+def test_check_update_invalid_args(utils):
+    ret = utils.run([ 'tdnf', 'check-update', 'abcd', '1234' ])
+    assert (ret['retval'] == 0)
+
+@pytest.mark.skip(reason='FIXME: chooses a suitbale package that works in Docker')
+def test_check_update_multi_version_package(utils):
+    package = utils.config["mulversion_pkgname"] + '-' + utils.config["mulversion_lower"]
+    ret = utils.run([ 'tdnf', 'install', '-y', '--nogpgcheck', package ])
+    assert (ret['retval'] == 0)
+
+    ret = utils.run([ 'tdnf', 'check-update', utils.config["mulversion_pkgname"] ])
+    assert (len(ret['stdout']) > 0)
+
+def test_check_update_single_version_package(utils):
+    package = utils.config["sglversion_pkgname"]
+    ret = utils.run([ 'tdnf', 'install', '-y', '--nogpgcheck', package ])
+    assert (ret['retval'] == 0)
+
+    ret = utils.run([ 'tdnf', 'check-update', package ])
+    assert (len(ret['stdout']) == 0)

pytests/tests/test_clean.py

@@ -0,0 +1,46 @@
+#
+# Copyright (C) 2019 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the GNU General Public License v2 (the "License");
+# you may not use this file except in compliance with the License. The terms
+# of the License are located in the COPYING file of this distribution.
+#
+#   Author: Siddharth Chandrasekaran <csiddharth@vmware.com>
+
+import pytest
+
+@pytest.fixture(scope='module', autouse=True)
+def setup_test(utils):
+    yield
+    teardown_test(utils)
+
+def teardown_test(utils):
+    pass
+
+def test_clean_no_args(utils):
+    ret = utils.run([ 'tdnf', 'clean' ])
+    assert (ret['retval'] == 135)
+
+def test_clean_invalid_arg(utils):
+    ret = utils.run([ 'tdnf', 'clean', 'abcde' ])
+    assert (ret['retval'] == 133)
+
+def test_clean_packages(utils):
+    ret = utils.run([ 'tdnf', 'clean', 'packages' ])
+    assert (ret['retval'] == 248)
+
+def test_clean_dbcache(utils):
+    ret = utils.run([ 'tdnf', 'clean', 'dbcache' ])
+    assert (ret['retval'] == 248)
+
+def test_clean_metadata(utils):
+    ret = utils.run([ 'tdnf', 'clean', 'metadata' ])
+    assert (ret['retval'] == 248)
+
+def test_clean_expire_cache(utils):
+    ret = utils.run([ 'tdnf', 'clean', 'expire-cache' ])
+    assert (ret['retval'] == 248)
+
+def test_clean_plugins(utils):
+    ret = utils.run([ 'tdnf', 'clean', 'plugins' ])
+    assert (ret['retval'] == 248)

pytests/tests/test_config.py

@@ -0,0 +1,51 @@
+#
+# Copyright (C) 2019 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the GNU General Public License v2 (the "License");
+# you may not use this file except in compliance with the License. The terms
+# of the License are located in the COPYING file of this distribution.
+#
+#   Author: Siddharth Chandrasekaran <csiddharth@vmware.com>
+
+import os
+import pytest
+
+@pytest.fixture(scope='module', autouse=True)
+def setup_test(utils):
+    utils.run([ 'cp', '/etc/yum.repos.d/photon.repo', '/etc/yum.repos.d/photon.repo.bak' ])
+    utils.run([ 'sed', '-i', 's/enabled=1/enabled=0/g', '/etc/yum.repos.d/photon.repo' ])
+    utils.run([ 'cp', '/etc/tdnf/tdnf.conf', '/etc/tdnf/mytdnf.conf' ])
+    utils.run([ 'sed', '-i', '/repodir/d', '/etc/tdnf/mytdnf.conf' ])
+    utils.run([ 'sed', '-i', '$ a repodir=/etc/myrepo', '/etc/tdnf/mytdnf.conf' ])
+    utils.run([ 'mkdir', '-p', '/etc/myrepo' ])
+    utils.run([ 'cp', '/etc/yum.repos.d/photon.repo', '/etc/myrepo/photon.repo' ])
+    utils.run([ 'sed', '-i', 's/enabled=0/enabled=1/g', '/etc/myrepo/photon.repo' ])
+    yield
+    teardown_test(utils)
+
+def teardown_test(utils):
+    utils.run([ 'cp', '/etc/yum.repos.d/photon.repo.bak', '/etc/yum.repos.d/photon.repo' ])
+    utils.run([ 'rm', '/etc/tdnf/mytdnf.conf' ])
+    utils.run([ 'rm', '-rf', '/etc/myrepo' ])
+
+def test_config_invalid(utils):
+    ret = utils.run([ 'tdnf', '--config', '/etc/tdnf/test123.conf', 'list', 'tdnf' ])
+    assert(ret['retval'] == 66)
+
+def test_config_list(utils):
+    ret = utils.run([ 'tdnf', '--config', '/etc/tdnf/mytdnf.conf', 'list', 'tdnf' ])
+    assert(ret['retval'] == 0)
+
+def test_config_list_with_disable_repos(utils):
+    ret = utils.run([ 'tdnf', '--disablerepo=*', '--config', '/etc/tdnf/mytdnf.conf', 'list', 'tdnf' ])
+    assert(ret['retval'] == 0)
+
+    for line in ret['stdout']:
+        if not line or '@System' in line:
+            continue
+        assert (False) # force fail test
+
+def test_config_invaid_repodir(utils):
+    utils.run([ 'sed', '-i', 's#repodir=/etc/myrepo#repodir=/etc/invalid#g', '/etc/tdnf/mytdnf.conf' ])
+    ret = utils.run([ 'tdnf', '--config', '/etc/tdnf/mytdnf.conf', 'list', 'tdnf' ])
+    assert(ret['retval'] == 45)

pytests/tests/test_count.py

@@ -0,0 +1,24 @@
+#
+# Copyright (C) 2019 VMware, Inc. All Rights Reserved.
+#
+# Licensed under the GNU General Public License v2 (the "License");
+# you may not use this file except in compliance with the License. The terms
+# of the License are located in the COPYING file of this distribution.
+#
+#   Author: Siddharth Chandrasekaran <csiddharth@vmware.com>
+
+import os
+import tempfile
+import pytest
+
+@pytest.fixture(scope='module', autouse=True)
+def setup_test(utils):
+    yield
+    teardown_test(utils)
+
+def teardown_test(utils):
+    pass
+
+def test_count(utils):
+    ret = utils.run([ 'tdnf', 'count' ])
+    assert(ret['retval'] == 0)