#!/usr/bin/env python
# PySys System Test Framework, Copyright (C) 2006-2022 M.B. Grieve
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
"""
Writers that collect and report on code coverage data.
"""
__all__ = [
"PythonCoverageWriter",
]
import logging, sys, io, os, shlex
from pysys.constants import *
from pysys.writer.api import *
from pysys.writer.testoutput import CollectTestOutputWriter
from pysys.utils.fileutils import mkdir, deletedir, toLongPathSafe, fromLongPathSafe, pathexists
log = logging.getLogger('pysys.writer')
[docs]class PythonCoverageWriter(CollectTestOutputWriter):
"""Writer that collects Python code coverage files in a single directory and writes a coverage report during
runner cleanup. Requires the "coverage.py" library to be installed.
To enable this, run with ``-XcodeCoverage`` (or ``-XpythonCoverage``) and configure the ``destDir`` plugin property
in pysysproject.xml (e.g. to ``__coverage_python.${outDirName}``).
If coverage is generated, the directory containing all coverage files is published as an artifact named
"PythonCoverageDir". Optionally an archive of this directory can be generated by setting the
``destArchive`` property (see `CollectTestOutputWriter`), and published as "PythonCoverageArchive".
Note that to maintain compatibility with pre-1.6.0 projects, a PythonCoverageWriter instance will be _automatically_
added to the project if the ``pythonCoverageDir`` project property is set and none is explicitly configured; but
the automatic addition is deprecated so you should explicity add this writer to your project if you need it.
.. versionadded:: 1.6.0
The following properties can be set in the project configuration for this writer (and see also
`pysys.writer.testoutput.CollectTestOutputWriter` for inherited properties such as ``destArchive`` which produces
a .zip of the destDir and ``includeTestIf`` for limiting collection to only unit/smoke tests):
"""
# override CollectTestOutputWriter property values
destDir = u''
fileIncludesRegex = u'.*/[.]coverage[.]python.*' # executed against the path relative to the test root dir e.g. (pattern1|pattern2)
outputPattern = u'.coverage.python.@TESTID@_@FILENAME@.@FILENAME_EXT@.@UNIQUE@'
publishArtifactDirCategory = u'PythonCoverageDir'
publishArtifactArchiveCategory = u'PythonCoverageArchive'
pythonCoverageArgs = u''
"""
A string of command line arguments used to customize the ``coverage run`` and ``coverage html`` commands.
Use "..." double quotes around any arguments that contain spaces.
For example::
<property name="pythonCoverageArgs" value="--rcfile=${testRootDir}/python_coveragerc"/>
"""
includeCoverageFromPySysProcess = False
"""
Set this to True to enable measuring coverage for this process (i.e. PySys), rather than only child Python processes.
This is useful for testing PySys plugins.
.. versionadded:: 2.0
"""
__selfCoverage = None
def isEnabled(self, record=False, **kwargs):
if not (self.runner.getBoolProperty('pythonCoverage', default=self.runner.getBoolProperty('codeCoverage')) and self.destDir):
return False
try:
import coverage
assert coverage.__file__ != __file__, __file__ # just to make sure we're not getting confused with our own pysys coverage module
except ImportError:
# don't log higher than debug because this user may just be doing a --ci run with -XcodeCoverage for some
# other reason and may not even be intending to run with Python coverage
log.debug('Not enabling Python coverage because the coverage.py package is not installed')
return False
else:
return True
def setup(self, *args, **kwargs):
super(PythonCoverageWriter, self).setup(*args, **kwargs)
import coverage
if self.includeCoverageFromPySysProcess:
args = self.getCoverageArgsList()
assert len(args)==1 and args[0].startswith('--rcfile='), 'includeCoverageFromPySysProcess can only be used if pythonCoverageArgs is set to "--rcfile=XXXX"'
mkdir(self.destDir)
cov = coverage.Coverage(config_file=args[0][args[0].find('=')+1:], data_file=self.destDir+'/.coverage.pysys_parent')
log.debug('Enabling Python coverage for this process: %s', cov)
# These lines avoid unhelpful warnings, and also match what coverage.process_startup() does
cov._warn_preimported_source = False
cov._warn_unimported_source = False
cov._warn_no_data = False
cov.start()
self.__selfCoverage = cov
def getCoverageArgsList(self): # also used by startPython()
return shlex.split(self.pythonCoverageArgs.replace(u'\\',u'\\\\')) # need to escape windows \ else it gets removed; do this the same on all platforms for consistency
def cleanup(self, **kwargs):
if self.__selfCoverage is not None:
self.__selfCoverage.stop()
self.__selfCoverage.save()
coverageDestDir = self.destDir
assert os.path.isabs(coverageDestDir) # The base class is responsible for absolutizing this config property
if not pathexists(coverageDestDir):
log.info('No Python coverage files were generated.')
return
log.info('Preparing Python coverage report from %d files in: %s', len(os.listdir(coverageDestDir)), os.path.normpath(coverageDestDir))
coverageDestDir = os.path.normpath(fromLongPathSafe(coverageDestDir))
self.runner.startPython(['-m', 'coverage', 'combine'], abortOnError=True,
workingDir=coverageDestDir, stdouterr=coverageDestDir+'/python-coverage-combine',
disableCoverage=True, onError=lambda process:
'Failed to combine Python code coverage data: %s'%self.runner.getExprFromFile(process.stdout, '.+', returnNoneIfMissing=True)
or self.runner.logFileContents(process.stderr, maxLines=0))
# produces coverage.xml in a standard format that is useful to code coverage tools
self.runner.startPython(['-m', 'coverage', 'xml'], abortOnError=False,
workingDir=coverageDestDir, stdouterr=coverageDestDir+'/python-coverage-xml',
disableCoverage=True, onError=lambda process: self.runner.getExprFromFile(process.stdout, '.+', returnNoneIfMissing=True)
or self.runner.logFileContents(process.stderr, maxLines=0))
self.runner.startPython(['-m', 'coverage', 'html', '-d', toLongPathSafe(coverageDestDir+'/htmlcov')]+self.getCoverageArgsList(), abortOnError=False,
workingDir=coverageDestDir, stdouterr=coverageDestDir+'/python-coverage-html',
disableCoverage=True, onError=lambda process: self.runner.getExprFromFile(process.stdout, '.+', returnNoneIfMissing=True)
or self.runner.logFileContents(process.stderr, maxLines=0))
htmlcov = os.path.join(coverageDestDir, 'htmlcov', 'index.html')
if os.path.exists(htmlcov):
log.info('Python coverage HTML: %s', htmlcov)
# to avoid confusion, remove any zero byte out/err files from the above
for p in os.listdir(coverageDestDir):
p = os.path.join(coverageDestDir, p)
if p.endswith(('.out', '.err')) and os.path.getsize(p)==0:
os.remove(p)
self.archiveAndPublish()