#!/usr/bin/env python
# PySys System Test Framework, Copyright (C) 2006-2022 M.B. Grieve
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
"""
Writers for recording test results to Continuous Integration providers.
These writers only generate output if the
environment variables associated with their CI system are set.
If you wish to run PySys from a CI provider that's not listed here, for basic functionality all you need to
do is run the tests with the ``--ci`` option. Many CI providers (including Jenkins) will be able to read the
results of PySys tests if you configure them to look in the results directory generated by
`pysys.writer.outcomes.JUnitXMLResultsWriter`.
It may also be possible to configure some providers to automatically pick up archived test output from artifacts
from the `pysys.writer.testoutput.TestOutputArchiveWriter` directory.
If you want to implement additional functionality for your
CI provider, subclass `pysys.writer.api.BaseRecordResultsWriter` (using the classes in this module as an example), and
consider contributing your implementation back to the PySys project with a pull request.
"""
__all__ = ["GitHubActionsCIWriter", "TravisCIWriter", "stdoutPrint"]
import time, logging, sys, threading, os, re, collections
from pysys.constants import PrintLogs, OUTCOMES
from pysys.writer.api import BaseRecordResultsWriter, TestOutcomeSummaryGenerator, ArtifactPublisher, stripANSIEscapeCodes
from pysys.utils.logutils import ColorLogFormatter, stdoutPrint
log = logging.getLogger('pysys.writer')
github_log = logging.getLogger('pysys.writer.github')
[docs]class GitHubActionsCIWriter(BaseRecordResultsWriter, TestOutcomeSummaryGenerator, ArtifactPublisher):
"""
Writer for GitHub(R) Actions.
Produces annotations summarizing failures, adds grouping/folding of detailed test output, and
sets step output variables for any published artifacts (e.g. performance .csv files, archived test output etc)
which can be used to upload the artifacts when present. Step output variables for published artifacts are named
'artifact_CATEGORY' (for more details on artifact categories see `pysys.writer.api.ArtifactPublisher.publishArtifact`).
For example, if the step is given the id ``pysys`` then the directory containing archived test output for failures
can be uploaded using ``{{ steps.pysys.outputs.artifact_TestOutputArchiveDir }}``.
Be sure to include a unique run id (e.g. outdir) for each OS/job in the name of any uploaded artifacts so that
they do not overwrite each other when uploaded.
See the PySys sample projects on GitHub(R) for a workflow file you can copy into your own project to enable
running your PySys tests with GitHub(R) Actions.
Only enabled when running under GitHub Actions (specifically, if the ``GITHUB_ACTIONS=true`` environment variable is set).
"""
failureSummaryAnnotations = True
"""
Configures whether a single annotation is added with a summary of the number of failures and the outcome and reason
for all failures.
"""
failureTestLogAnnotations = True
"""
Configures whether annotations are added with the (run.log) log output for each of the first few test failures.
"""
maxAnnotations = 10
"""
Configures the maximum number of annotations generated by this invocation of PySys, to cope with GitHub limits.
This ensure we don't use up our entire allocation of annotations leaving no space for annotations from other
tools. Being aware of this limit also allows to us add a warning to the end of the last one to make clear no more
annotations will be shown even if there are more warnings.
"""
def isEnabled(self, **kwargs):
return os.getenv('GITHUB_ACTIONS','')=='true'
def outputGitHubCommand(self, cmd, value=u'', params={}):
# syntax is: ::workflow-command parameter1={data},parameter2={data}::{command value}
# escaping based on https://github.com/actions/toolkit/blob/master/packages/core/src/command.ts
toprint = u'::%s%s::%s'%(cmd,
(u' '+u','.join(u'%s=%s'%(k,v\
.replace('%', '%25')\
.replace('\r', '%0D')\
.replace('\n', '%0A')\
.replace(':', '%3A')\
.replace(',', '%2C')
) for k,v in params.items())) if params else u'', value\
.replace('%', '%25')\
.replace('\r', '%0D')\
.replace('\n', '%0A')
)
# since GitHub suppresses the actual commands written, it's useful to log this at debug
github_log.debug('GitHub Actions command %s', toprint)
stdoutPrint(toprint)
assert cmd != 'set-output', '::set-output is deprecated by GitHub; please call PySys publishArtifact() instead of outputGitHubCommand()'
def setup(self, numTests=0, cycles=1, xargs=None, threads=0, testoutdir=u'', runner=None, **kwargs):
super(GitHubActionsCIWriter, self).setup(numTests=numTests, cycles=cycles, xargs=xargs, threads=threads,
testoutdir=testoutdir, runner=runner, **kwargs)
self.__outputs = set()
self.remainingAnnotations = int(self.maxAnnotations)-2 # one is used up for the non-zero exit status and one is used for the summary
if str(self.failureTestLogAnnotations).lower()!='true': self.remainingAnnotations = 0
self.failureTestLogAnnotations = []
self.runner = runner
self.runid = os.path.basename(testoutdir)
if runner.printLogs is None:
# if setting was not overridden by user, default for CI is
# to only print failures since otherwise the output is too long
# and hard to find the logs of interest
runner.printLogs = PrintLogs.FAILURES
self.outputGitHubCommand(u'group', u'Logs for test run: %s' % self.runid)
# enable coloring automatically, since this CI provider supports it
runner.project.formatters.stdout.color = True
# in this provider, colors render more clearly with bright=False
ColorLogFormatter.configureANSIEscapeCodes(bright=False)
self.artifacts = {} # map of category:[paths]
def publishArtifact(self, path, category, **kwargs):
if self.artifacts is not None:
# normally we accumulate all the paths for each category ready to write out during cleanup
self.artifacts.setdefault(category, []).append(path)
else:
# if this writer's cleanup was already called, fallback to separate invocation (probably this will only
# happen for categories that publish just one path, so it'll work out OK)
log.debug('GitHubActionsCIWriter got publishArtifact after cleanup(), will publish anyway; category=%s path=%s', category, path)
self._publishToGitHub([path], category)
def _publishToGitHub(self, paths, category):
if not os.path.exists(paths[0]):
github_log.debug('GitHub Actions cannot set output parameter "%s" due to missing files: %s', category, paths)
return # auto-skip things that don't exist
name = 'artifact_'+category
val = ','.join(sorted(paths))
github_log.debug('GitHub Actions is setting output parameter %s=%s', name, val)
if not os.getenv('GITHUB_OUTPUT'):
github_log.warning('Cannot set GitHub Actions output "%s" since GITHUB_OUTPUT environment variable is not defined; available env vars=%s', name, ' '.join(sorted(os.environ.keys())))
return
with open(os.environ['GITHUB_OUTPUT'], 'a') as f: # use default encoding
f.write(f'{name}={val}\n')
f.flush()
self.__outputs.add(name)
def cleanup(self, **kwargs):
super(GitHubActionsCIWriter, self).cleanup(**kwargs)
# invoked after all tests but before summary is printed,
# a good place to close the folding detail section
self.outputGitHubCommand(u'endgroup')
# artifact publishing, mostly for use with uploading
# currently categories with multiple artifacts can't be used directly with the artifact action, but this may change in future
for category, paths in self.artifacts.items():
self._publishToGitHub(paths, category)
self.artifacts = None
if sum([self.outcomes[o] for o in OUTCOMES if o.isFailure()]):
self.outputGitHubCommand(u'group', u'(GitHub test failure annotations)')
if str(self.failureSummaryAnnotations).lower()=='true':
self.outputGitHubCommand(u'error', self.getSummaryText(),
# Slightly better than the default (".github") is to include the path to the project file
params={u'file':self.runner.project.projectFile.replace(u'\\',u'/')})
if self.failureTestLogAnnotations:
# Do them all in a group at the end since otherwise the annotation output gets mixed up with the test
# log output making it hard to understand
for a in self.failureTestLogAnnotations:
self.outputGitHubCommand(*a)
self.outputGitHubCommand(u'endgroup')
if self.__outputs:
github_log.info('GitHub output values: %s', ', '.join(sorted(self.__outputs)))
def processResult(self, testObj, cycle=0, testTime=0, testStart=0, runLogOutput=u'', **kwargs):
super(GitHubActionsCIWriter, self).processResult(testObj, cycle=cycle, testTime=testTime,
testStart=testStart, runLogOutput=runLogOutput, **kwargs)
if self.remainingAnnotations > 0 and testObj.getOutcome().isFailure():
# Currently, GitHub actions doesn't show the annotation against the source code unless the specified line
# number is one of the lines changes or surrounding context lines, but we do the best we can
m = re.search(u' \\[run\\.py:(\\d+)\\]', runLogOutput or u'')
lineno = m.group(1) if m else None
msg = stripANSIEscapeCodes(runLogOutput)
self.remainingAnnotations -= 1
if self.remainingAnnotations == 0: msg += '\n\n(annotation limit reached; for any additional test failures, see the detailed log)'
params = collections.OrderedDict()
file, lineno = testObj.getOutcomeLocation()
if file:
params[u'file'] = file.replace(u'\\',u'/')
if lineno: params[u'line'] = lineno
self.failureTestLogAnnotations.append([u'warning', msg, params])
[docs]class TravisCIWriter(BaseRecordResultsWriter):
"""
Writer for Travis CI(R).
Provides folding of test log details, and correct coloring of console output for Travis.
Only enabled when running under Travis (specifically,
if the ``TRAVIS=true`` environment variable is set).
"""
def isEnabled(self, **kwargs):
return os.getenv('TRAVIS','')=='true'
def setup(self, numTests=0, cycles=1, xargs=None, threads=0, testoutdir=u'', runner=None, **kwargs):
self.runid = os.path.basename(testoutdir)
if runner.printLogs is None:
# if setting was not overridden by user, default for Travis is
# to only print failures since otherwise the output is too long
# and hard to find the logs of interest
runner.printLogs = PrintLogs.FAILURES
stdoutPrint(u'travis_fold:start:PySys-%s'%self.runid.replace(' ', '-'))
# enable coloring automatically, since this CI provider supports it,
# but must explicitly disable bright colors since it doesn't yet support that
runner.project.formatters.stdout.color = True
ColorLogFormatter.configureANSIEscapeCodes(bright=False)
def cleanup(self, **kwargs):
# invoked after all tests but before summary is printed,
# a good place to close the folding detail section
stdoutPrint(u'travis_fold:end:PySys-%s'%self.runid.replace(' ', '-'))
def processResult(self, testObj, cycle=0, testTime=0, testStart=0, runLogOutput=u'', **kwargs):
# nothing to do for Travis as it doesn't collect results, we use the
# standard log printing mechanism
pass