diff --git a/.gitignore b/.gitignore index 7e6fafbd2..1634ba0c8 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,10 @@ build/ dist/ .coverage plinth/tests/config_local.py -plinth/tests/coverage/report/ +htmlcov/ +functional_tests.test_plinth/ +functional_tests/test_plinth/ +geckodriver.log *.mo .vagrant/ .idea/ diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b90afd872..cc18ae2a3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,13 +17,12 @@ stages: run-unit-tests: stage: test script: - - apt-get install -y python3-coverage libjs-jquery libjs-jquery-isonscreen libjs-jquery-tablesorter libjs-jquery-throttle-debounce - adduser tester --gecos "First Last,RoomNumber,WorkPhone,HomePhone" --disabled-password - echo "tester:password" | chpasswd - cp -r . /home/tester/plinth - chown -R tester:tester /home/tester/plinth - - su -c "cd ~/plinth; python3 setup.py test_coverage" tester - - cp -r /home/tester/plinth/plinth/tests/coverage/report test-coverage-report + - su -c "cd ~/plinth; py.test-3 --cov=plinth --cov-report=html" tester + - cp -r /home/tester/plinth/htmlcov test-coverage-report artifacts: paths: diff --git a/HACKING.md b/HACKING.md index 9e012fd54..414d79c43 100644 --- a/HACKING.md +++ b/HACKING.md @@ -115,23 +115,32 @@ However, for some reason, you wish setup manually, the following tips will help: To run all the tests: ```bash -$ python3 setup.py test +$ py.test-3 ``` -To run a specific test function, test class or test module, use the `-s` option -with the fully qualified name. +Another way to run tests (not recommended): + +```bash +$ ./setup.py test +``` + +To run a specific test function, test class or test module, use pytest filtering +options. **Examples:** ```bash -# Run tests of a test module -$ python3 setup.py test -s plinth.tests.test_actions +# Run tests in a directory +$ py.test-3 plinth/tests + +# Run tests in a module +$ py.test-3 plinth/tests/test_actions.py # Run tests of one class in test module -$ python3 setup.py test -s plinth.tests.test_actions.TestActions +$ py.test-3 plinth/tests/test_actions.py::TestActions # Run one test in a class or module -$ python3 setup.py test -s plinth.tests.test_actions.TestActions.test_is_package_manager_busy +$ py.test-3 plinth/tests/test_actions.py::TestActions::test_is_package_manager_busy ``` ## Running the Test Coverage Analysis @@ -139,20 +148,86 @@ $ python3 setup.py test -s plinth.tests.test_actions.TestActions.test_is_package To run the coverage tool: ``` -$ python3 setup.py test_coverage +$ py.test-3 --cov=plinth ``` -Invoking this command generates a binary-format `.coverage` data file in -the top-level project directory which is recreated with each run, and -writes a set of HTML and other supporting files which comprise the -browsable coverage report to the `plinth/tests/coverage/report` directory. -`Index.html` presents the coverage summary, broken down by module. Data -columns can be sorted by clicking on the column header or by using mnemonic -hot-keys specified in the keyboard widget in the upper-right corner of the -page. Clicking on the name of a particular source file opens a page that -displays the contents of that file, with color-coding in the left margin to -indicate which statements or branches were executed via the tests (green) -and which statements or branches were not executed (red). +To collect HTML report: + +``` +$ py.test-3 --cov=plinth --cov-report=html +``` + +Invoking this command generates a HTML report to the `htmlcov` directory. +`index.html` presents the coverage summary, broken down by module. Data columns +can be sorted by clicking on the column header. Clicking on the name of a +particular source file opens a page that displays the contents of that file, +with color-coding in the left margin to indicate which statements or branches +were executed via the tests (green) and which statements or branches were not +executed (red). + +## Running Functional Tests + +### Install Dependencies + +``` +$ pip3 install splinter +$ pip3 install pytest-splinter +$ pip3 install pytest-bdd +$ sudo apt install xvfb # optional, to avoid opening browser windows +$ pip3 install pytest-xvfb # optional, to avoid opening browser windows +``` + +- Install the latest version of geckodriver. +It's usually a single binary which you can place at /usr/local/bin/geckodriver + +- Install the latest version of Mozilla Firefox. +Download and extract the latest version from the Firefox website and symlink the +binary named `firefox` to /usr/local/bin. + +Geckodriver will then use whatever version of Firefox you symlink as +/usr/local/bin/firefox. + +### Run FreedomBox Service + +*Warning*: Functional tests will change the configuration of the system + under test, including changing the hostname and users. Therefore you + should run the tests using FreedomBox running on a throw-away VM. + +The VM should have NAT port-forwarding enabled so that 4430 on the +host forwards to 443 on the guest. From where the tests are running, the web +interface of FreedomBox should be accessible at https://localhost:4430/. + +### Setup FreedomBox Service for tests + +Via Plinth, create a new user as follows: + +* Username: tester +* Password: testingtesting + +This step is optional if a fresh install of Plinth is being tested. Functional +tests will create the required user using FreedomBox's first boot process. + +### Run Functional Tests + +Run + +``` +$ py.test-3 --include-functional +``` + +The full test suite can take a long time to run (more than an hour). You can +also specify which tests to run, by tag or keyword: + +``` +$ py.test-3 -k essential --include-functional +``` + +If xvfb is installed and you still want to see browser windows, use the +`--no-xvfb` command-line argument. + +``` +$ py.test-3 --no-xvfb -k mediawiki --include-functional +``` ## Building the Documentation Separately diff --git a/conftest.py b/conftest.py new file mode 100644 index 000000000..b49b4a5c4 --- /dev/null +++ b/conftest.py @@ -0,0 +1,40 @@ +# +# This file is part of FreedomBox. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +""" +pytest configuration for all tests. +""" + +import pytest + + +def pytest_addoption(parser): + """Add a command line option to run functional tests.""" + parser.addoption('--include-functional', action='store_true', + default=False, help='Run functional tests also') + + +def pytest_collection_modifyitems(config, items): + """Filter out functional tests unless --include-functional arg is passed.""" + if config.getoption('--include-functional'): + # Option provided on command line, no filtering + return + + skip_functional = pytest.mark.skip( + reason='--include-functional not provided') + for item in items: + if 'functional' in item.keywords: + item.add_marker(skip_functional) diff --git a/debian/control b/debian/control index 2aa8473a0..ac1cd9973 100644 --- a/debian/control +++ b/debian/control @@ -24,7 +24,6 @@ Build-Depends: python3-bootstrapform, python3-cherrypy3, python3-configobj, - python3-coverage, python3-dbus, python3-django (>= 1.11), python3-django-axes (>= 3.0.3), @@ -33,6 +32,9 @@ Build-Depends: python3-gi, python3-psutil, python3-pytest, + python3-pytest-cov, + python3-pytest-django, + python3-pytest-runner, python3-requests, python3-ruamel.yaml, python3-setuptools, diff --git a/debian/copyright b/debian/copyright index f5b00934e..46a1a9464 100644 --- a/debian/copyright +++ b/debian/copyright @@ -117,10 +117,6 @@ Files: static/themes/default/lato/* Copyright: (c) 2010-2014, Łukasz Dziedzic License: OFL-1.1 -Files: plinth/tests/coverage/coverage.py -Copyright: 2009 Jeet Sukumaran and Mark T. Holder -License: GPL-3+ - Files: debian/* Copyright: 2013 Tzafrir Cohen 2013-2019 FreedomBox Authors diff --git a/functional_tests/.gitignore b/functional_tests/.gitignore deleted file mode 100644 index ceafd6dcc..000000000 --- a/functional_tests/.gitignore +++ /dev/null @@ -1,198 +0,0 @@ - -# Created by https://www.gitignore.io/api/vim,emacs,macos,python,vagrant - -### Emacs ### -# -*- mode: gitignore; -*- -*~ -\#*\# -/.emacs.desktop -/.emacs.desktop.lock -*.elc -auto-save-list -tramp -.\#* - -# Org-mode -.org-id-locations -*_archive - -# flymake-mode -*_flymake.* - -# eshell files -/eshell/history -/eshell/lastdir - -# elpa packages -/elpa/ - -# reftex files -*.rel - -# AUCTeX auto folder -/auto/ - -# cask packages -.cask/ -dist/ - -# Flycheck -flycheck_*.el - -# server auth directory -/server/ - -# projectiles files -.projectile - -# directory configuration -.dir-locals.el - -### macOS ### -*.DS_Store -.AppleDouble -.LSOverride - -# Icon must end with two \r -Icon - -# Thumbnails -._* - -# Files that might appear in the root of a volume -.DocumentRevisions-V100 -.fseventsd -.Spotlight-V100 -.TemporaryItems -.Trashes -.VolumeIcon.icns -.com.apple.timemachine.donotpresent - -# Directories potentially created on remote AFP share -.AppleDB -.AppleDesktop -Network Trash Folder -Temporary Items -.apdisk - -### Python ### -# Byte-compiled / optimized / DLL files -__pycache__/ -*.py[cod] -*$py.class - -# C extensions -*.so - -# Distribution / packaging -.Python -env/ -build/ -develop-eggs/ -downloads/ -eggs/ -.eggs/ -lib/ -lib64/ -parts/ -sdist/ -var/ -wheels/ -*.egg-info/ -.installed.cfg -*.egg - -# PyInstaller -# Usually these files are written by a python script from a template -# before PyInstaller builds the exe, so as to inject date/other infos into it. -*.manifest -*.spec - -# Installer logs -pip-log.txt -pip-delete-this-directory.txt - -# Unit test / coverage reports -htmlcov/ -.tox/ -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*,cover -.hypothesis/ - -# Translations -*.mo -*.pot - -# Django stuff: -*.log -local_settings.py - -# Flask stuff: -instance/ -.webassets-cache - -# Scrapy stuff: -.scrapy - -# Sphinx documentation -docs/_build/ - -# PyBuilder -target/ - -# Jupyter Notebook -.ipynb_checkpoints - -# pyenv -.python-version - -# celery beat schedule file -celerybeat-schedule - -# SageMath parsed files -*.sage.py - -# dotenv -.env - -# virtualenv -.venv -venv/ -ENV/ - -# Spyder project settings -.spyderproject -.spyproject - -# Rope project settings -.ropeproject - -# mkdocs documentation -/site - -### Vagrant ### -.vagrant/ -Vagrantfile - -### Vim ### -# swap -[._]*.s[a-v][a-z] -[._]*.sw[a-p] -[._]s[a-v][a-z] -[._]sw[a-p] -# session -Session.vim -# temporary -.netrwhist -# auto-generated tag files -tags - -# End of https://www.gitignore.io/api/vim,emacs,macos,python,vagrant - -test_plinth/ -geckodriver.log -.pytest_cache \ No newline at end of file diff --git a/functional_tests/README.md b/functional_tests/README.md deleted file mode 100644 index 63f2debdb..000000000 --- a/functional_tests/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# Install Dependencies - -``` -$ sudo apt install python3-pytest -$ pip3 install splinter -$ pip3 install pytest-splinter -$ pip3 install pytest-bdd -$ sudo apt install xvfb # optional, to avoid opening browser windows -$ pip3 install pytest-xvfb # optional, to avoid opening browser windows -``` - -- Install the latest version of geckodriver. -It's usually a single binary which you can place at /usr/local/bin/geckodriver - -- Install the latest version of Mozilla Firefox. -Download and extract the latest version from the Firefox website and symlink the binary named `firefox` to /usr/local/bin. - -Geckodriver will then use whatever version of Firefox you symlink as /usr/local/bin/firefox. - -# Run FreedomBox Service - -*Warning*: Functional tests will change the configuration of the system - under test, including changing the hostname and users. Therefore you - should run the tests using FreedomBox running on a throw-away VM. - -The VM should have NAT port-forwarding enabled so that 4430 on the -host forwards to 443 on the guest. From where the tests are running, the web -interface of FreedomBox should be accessible at https://localhost:4430/. - -# Setup FreedomBox Service for tests - -Via Plinth, create a new user as follows: - -* Username: tester -* Password: testingtesting - -This step is optional if a fresh install of Plinth is being -tested. Functional tests will create the required user using FreedomBox's -first boot process. - -# Run Functional Tests - -From the directory functional_tests, run - -``` -$ py.test -``` - -The full test suite can take a long time to run (over 15 minutes). You -can also specify which tests to run, by tag or keyword: - -``` -$ py.test -k essential -``` - -If xvfb is installed and you still want to see browser windows, use the -`--no-xvfb` command-line argument. - -``` -$ py.test --no-xvfb -k mediawiki -``` diff --git a/functional_tests/pytest.ini b/functional_tests/pytest.ini deleted file mode 100644 index 8826612a0..000000000 --- a/functional_tests/pytest.ini +++ /dev/null @@ -1,4 +0,0 @@ -; Ignore warnings temporary fix -; TODO:// Remove when https://github.com/pytest-dev/pytest-splinter/issues/112 is resolved -[pytest] -addopts = -p no:warnings \ No newline at end of file diff --git a/functional_tests/test_plinth.py b/functional_tests/test_plinth.py index bac46d939..1534a3291 100644 --- a/functional_tests/test_plinth.py +++ b/functional_tests/test_plinth.py @@ -16,15 +16,19 @@ # import pytest -from pytest_bdd import scenarios -from step_definitions.application import * -from step_definitions.interface import * -from step_definitions.service import * -from step_definitions.site import * -from step_definitions.system import * +try: + from pytest_bdd import scenarios +except ImportError: + pytestmark = pytest.mark.skip(reason='pytest_bdd is not installed') +else: + from step_definitions.application import * + from step_definitions.interface import * + from step_definitions.service import * + from step_definitions.site import * + from step_definitions.system import * -# Mark all tests are functional -pytestmark = pytest.mark.functional + # Mark all tests are functional + pytestmark = pytest.mark.functional -scenarios('features') + scenarios('features') diff --git a/manage.py b/manage.py new file mode 100644 index 000000000..7954c9da5 --- /dev/null +++ b/manage.py @@ -0,0 +1,25 @@ +# +# This file is part of FreedomBox. +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as +# published by the Free Software Foundation, either version 3 of the +# License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU Affero General Public License for more details. +# +# You should have received a copy of the GNU Affero General Public License +# along with this program. If not, see . +# +""" +Dummy file to help pytest-django path detection. + +pytest-django searches for a folder with manage.py and treats that as parent +directory for Django project. This folder is then added to Python path managed +in sys.path. This allows the Django setting module to be discovered as +plinth.tests.data.django_test_settings. pytest can then be invoked simply as +'py.test-3' instead of 'python3 -m pytest'. +""" diff --git a/plinth/tests/coverage/__init__.py b/plinth/tests/coverage/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/plinth/tests/coverage/coverage.py b/plinth/tests/coverage/coverage.py deleted file mode 100644 index 9dd130cd5..000000000 --- a/plinth/tests/coverage/coverage.py +++ /dev/null @@ -1,105 +0,0 @@ -#!/usr/bin/python3 -# -*- mode: python; mode: auto-fill; fill-column: 80 -*- -# -# This file is part of FreedomBox. -# -# Derived from code sample at: -# http://jeetworks.org/adding-test-code-coverage-analysis-to-a-python-projects-setup-command/ -# -# Copyright 2009 Jeet Sukumaran and Mark T. Holder. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . -# - -""" -Support for integration of code test coverage analysis with setuptools. -""" - -import glob -import setuptools -import shutil -import time - - -# Project directories with testable source code -SOURCE_DIRS = ['plinth'] + glob.glob('plinth/modules/*') - -# Files to exclude from coverage analysis and reporting -FILES_TO_OMIT = [ - 'plinth/tests/*.py', - 'plinth/modules/*/tests/*.py' -] - -# Location of coverage HTML report files -COVERAGE_REPORT_DIR = 'plinth/tests/coverage/report' - - -class CoverageCommand(setuptools.Command): - """ - Subclass of setuptools Command to perform code test coverage analysis. - """ - - description = 'Run test coverage analysis' - user_options = [ - ('test-module=', 't', 'Explicitly specify a single module to test') - ] - - def initialize_options(self): - """Initialize options to default values.""" - self.test_module = None - - def finalize_options(self): - pass - - def run(self): - """Main command implementation.""" - import coverage - - if self.distribution.install_requires: - self.distribution.fetch_build_eggs( - self.distribution.install_requires) - - if self.distribution.tests_require: - self.distribution.fetch_build_eggs( - self.distribution.tests_require) - - # Erase any existing HTML report files - try: - shutil.rmtree(COVERAGE_REPORT_DIR, True) - except Exception: - pass - - # Run the coverage analysis - cov = coverage.coverage(auto_data=True, config_file=True, - source=SOURCE_DIRS, omit=FILES_TO_OMIT) - cov.erase() # Erase existing coverage data file - cov.start() - # Invoke the Django test setup and test runner logic - from plinth.tests.runtests import run_tests - run_tests(pattern=self.test_module, return_to_caller=True) - cov.stop() - - # Generate an HTML report - html_report_title = 'FreedomBox -- Test Coverage as of ' + \ - time.strftime('%x %X %Z') - cov.html_report(directory=COVERAGE_REPORT_DIR, omit=FILES_TO_OMIT, - title=html_report_title) - - # Print a detailed console report with the overall coverage percentage - print() - cov.report(omit=FILES_TO_OMIT) - - # Print the location of the HTML report - print('\nThe HTML coverage report is located at {}.'.format( - COVERAGE_REPORT_DIR)) diff --git a/plinth/tests/runtests.py b/plinth/tests/runtests.py deleted file mode 100644 index 35196a867..000000000 --- a/plinth/tests/runtests.py +++ /dev/null @@ -1,53 +0,0 @@ -#!/usr/bin/python3 -# -# This file is part of FreedomBox. -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU Affero General Public License as -# published by the Free Software Foundation, either version 3 of the -# License, or (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU Affero General Public License for more details. -# -# You should have received a copy of the GNU Affero General Public License -# along with this program. If not, see . -# - -""" -Module for Django pre-test configuration and setup. -""" - -import os -import sys - -import django -from django.conf import settings -from django.test.utils import get_runner - - -def run_tests(pattern=None, return_to_caller=False): - """Set up the Django test environment and run the specified tests.""" - os.environ['DJANGO_SETTINGS_MODULE'] = \ - 'plinth.tests.data.django_test_settings' - django.setup() - test_runner_cls = get_runner(settings) - test_runner = test_runner_cls() - - if pattern is None: - pattern_list = [ - 'plinth/tests', - 'plinth/modules', - ] - else: - pattern_list = [pattern] - - failures = test_runner.run_tests(pattern_list) - if failures > 0 or not return_to_caller: - sys.exit(bool(failures)) - - -if __name__ == '__main__': - run_tests() diff --git a/plinth/tests/test_kvstore.py b/plinth/tests/test_kvstore.py index 51cec5eca..dab38a4be 100644 --- a/plinth/tests/test_kvstore.py +++ b/plinth/tests/test_kvstore.py @@ -19,11 +19,14 @@ Test module for key/value store. """ +import pytest + from django.test import TestCase from plinth import kvstore +@pytest.mark.django_db class KVStoreTestCase(TestCase): """Verify the behavior of the kvstore module.""" diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..d07567e8a --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +DJANGO_SETTINGS_MODULE = plinth.tests.data.django_test_settings +markers = functional diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 000000000..99ec86939 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,3 @@ +[aliases] +# When './setup.py test' is invoked, run './setup.py pytest' +test=pytest diff --git a/setup.py b/setup.py index 9c62b4665..494167661 100755 --- a/setup.py +++ b/setup.py @@ -35,7 +35,6 @@ import setuptools from setuptools.command.install import install from plinth import __version__ -from plinth.tests.coverage import coverage DIRECTORIES_TO_CREATE = [ '/var/lib/plinth', @@ -188,7 +187,6 @@ setuptools.setup( packages=find_packages(include=['plinth', 'plinth.*'], exclude=['*.templates']), scripts=['bin/plinth'], - test_suite='plinth.tests.runtests.run_tests', license='COPYING.md', classifiers=[ 'Development Status :: 3 - Alpha', @@ -205,7 +203,7 @@ setuptools.setup( 'Topic :: Internet :: WWW/HTTP :: WSGI :: Application', 'Topic :: System :: Systems Administration', ], - setup_requires=['setuptools-git'], + setup_requires=['pytest-runner', 'setuptools-git'], install_requires=[ 'cherrypy >= 3.0', 'configobj', @@ -219,7 +217,7 @@ setuptools.setup( 'requests', 'ruamel.yaml', ], - tests_require=['coverage >= 3.7'], + tests_require=['pytest', 'pytest-cov', 'pytest-django'], include_package_data=True, package_data={ 'plinth': [ @@ -286,7 +284,6 @@ setuptools.setup( 'clean': CustomClean, 'compile_translations': CompileTranslations, 'install_data': CustomInstallData, - 'test_coverage': coverage.CoverageCommand, 'update_translations': UpdateTranslations, }, )