346 lines
12 KiB
Python
346 lines
12 KiB
Python
|
#!/usr/bin/env python
|
||
|
|
# Copyright 2017 The PDFium Authors. All rights reserved.
|
||
|
|
# Use of this source code is governed by a BSD-style license that can be
|
||
|
|
# found in the LICENSE file.
|
||
|
|
"""Generates a coverage report for given tests.
|
||
|
|
|
||
|
|
Requires that 'use_clang_coverage = true' is set in args.gn.
|
||
|
|
Prefers that 'is_component_build = false' is set in args.gn.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import argparse
|
||
|
|
from collections import namedtuple
|
||
|
|
import fnmatch
|
||
|
|
import os
|
||
|
|
import pprint
|
||
|
|
import subprocess
|
||
|
|
import sys
|
||
|
|
|
||
|
|
# Add src dir to path to avoid having to set PYTHONPATH.
|
||
|
|
sys.path.append(
|
||
|
|
os.path.abspath(
|
||
|
|
os.path.join(
|
||
|
|
os.path.dirname(__file__), os.path.pardir, os.path.pardir,
|
||
|
|
os.path.pardir)))
|
||
|
|
|
||
|
|
from testing.tools.common import GetBooleanGnArg
|
||
|
|
|
||
|
|
# 'binary' is the file that is to be run for the test.
|
||
|
|
# 'use_test_runner' indicates if 'binary' depends on test_runner.py and thus
|
||
|
|
# requires special handling.
|
||
|
|
# 'opt_args' are optional arguments to pass to the test 'binary'.
|
||
|
|
TestSpec = namedtuple('TestSpec', 'binary, use_test_runner, opt_args')
|
||
|
|
|
||
|
|
# All of the coverage tests that the script knows how to run.
|
||
|
|
COVERAGE_TESTS = {
|
||
|
|
'pdfium_unittests':
|
||
|
|
TestSpec('pdfium_unittests', False, []),
|
||
|
|
'pdfium_embeddertests':
|
||
|
|
TestSpec('pdfium_embeddertests', False, []),
|
||
|
|
'corpus_tests':
|
||
|
|
TestSpec('run_corpus_tests.py', True, []),
|
||
|
|
'corpus_tests_javascript_disabled':
|
||
|
|
TestSpec('run_corpus_tests.py', True, ['--disable-javascript']),
|
||
|
|
'javascript_tests':
|
||
|
|
TestSpec('run_javascript_tests.py', True, []),
|
||
|
|
'javascript_tests_javascript_disabled':
|
||
|
|
TestSpec('run_javascript_tests.py', True, ['--disable-javascript']),
|
||
|
|
'pixel_tests':
|
||
|
|
TestSpec('run_pixel_tests.py', True, []),
|
||
|
|
'pixel_tests_javascript_disabled':
|
||
|
|
TestSpec('run_pixel_tests.py', True, ['--disable-javascript']),
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
class CoverageExecutor(object):
|
||
|
|
|
||
|
|
def __init__(self, parser, args):
|
||
|
|
"""Initialize executor based on the current script environment
|
||
|
|
|
||
|
|
Args:
|
||
|
|
parser: argparse.ArgumentParser for handling improper inputs.
|
||
|
|
args: Dictionary of arguments passed into the calling script.
|
||
|
|
"""
|
||
|
|
self.dry_run = args['dry_run']
|
||
|
|
self.verbose = args['verbose']
|
||
|
|
|
||
|
|
self.source_directory = args['source_directory']
|
||
|
|
if not os.path.isdir(self.source_directory):
|
||
|
|
parser.error("'%s' needs to be a directory" % self.source_directory)
|
||
|
|
|
||
|
|
self.llvm_directory = os.path.join(self.source_directory, 'third_party',
|
||
|
|
'llvm-build', 'Release+Asserts', 'bin')
|
||
|
|
if not os.path.isdir(self.llvm_directory):
|
||
|
|
parser.error("Cannot find LLVM bin directory , expected it to be in '%s'"
|
||
|
|
% self.llvm_directory)
|
||
|
|
|
||
|
|
self.build_directory = args['build_directory']
|
||
|
|
if not os.path.isdir(self.build_directory):
|
||
|
|
parser.error("'%s' needs to be a directory" % self.build_directory)
|
||
|
|
|
||
|
|
(self.coverage_tests,
|
||
|
|
self.build_targets) = self.calculate_coverage_tests(args)
|
||
|
|
if not self.coverage_tests:
|
||
|
|
parser.error(
|
||
|
|
'No valid tests in set to be run. This is likely due to bad command '
|
||
|
|
'line arguments')
|
||
|
|
|
||
|
|
if not GetBooleanGnArg('use_clang_coverage', self.build_directory,
|
||
|
|
self.verbose):
|
||
|
|
parser.error(
|
||
|
|
'use_clang_coverage does not appear to be set to true for build, but '
|
||
|
|
'is needed')
|
||
|
|
|
||
|
|
self.use_goma = GetBooleanGnArg('use_goma', self.build_directory,
|
||
|
|
self.verbose)
|
||
|
|
|
||
|
|
self.output_directory = args['output_directory']
|
||
|
|
if not os.path.exists(self.output_directory):
|
||
|
|
if not self.dry_run:
|
||
|
|
os.makedirs(self.output_directory)
|
||
|
|
elif not os.path.isdir(self.output_directory):
|
||
|
|
parser.error('%s exists, but is not a directory' % self.output_directory)
|
||
|
|
elif len(os.listdir(self.output_directory)) > 0:
|
||
|
|
parser.error('%s is not empty, cowardly refusing to continue' %
|
||
|
|
self.output_directory)
|
||
|
|
|
||
|
|
self.prof_data = os.path.join(self.output_directory, 'pdfium.profdata')
|
||
|
|
|
||
|
|
def check_output(self, args, dry_run=False, env=None):
|
||
|
|
"""Dry run aware wrapper of subprocess.check_output()"""
|
||
|
|
if dry_run:
|
||
|
|
print "Would have run '%s'" % ' '.join(args)
|
||
|
|
return ''
|
||
|
|
|
||
|
|
output = subprocess.check_output(args, env=env)
|
||
|
|
|
||
|
|
if self.verbose:
|
||
|
|
print "check_output(%s) returned '%s'" % (args, output)
|
||
|
|
return output
|
||
|
|
|
||
|
|
def call(self, args, dry_run=False, env=None):
|
||
|
|
"""Dry run aware wrapper of subprocess.call()"""
|
||
|
|
if dry_run:
|
||
|
|
print "Would have run '%s'" % ' '.join(args)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
output = subprocess.call(args, env=env)
|
||
|
|
|
||
|
|
if self.verbose:
|
||
|
|
print 'call(%s) returned %s' % (args, output)
|
||
|
|
return output
|
||
|
|
|
||
|
|
def call_silent(self, args, dry_run=False, env=None):
|
||
|
|
"""Dry run aware wrapper of subprocess.call() that eats output from call"""
|
||
|
|
if dry_run:
|
||
|
|
print "Would have run '%s'" % ' '.join(args)
|
||
|
|
return 0
|
||
|
|
|
||
|
|
with open(os.devnull, 'w') as f:
|
||
|
|
output = subprocess.call(args, env=env, stdout=f)
|
||
|
|
|
||
|
|
if self.verbose:
|
||
|
|
print 'call_silent(%s) returned %s' % (args, output)
|
||
|
|
return output
|
||
|
|
|
||
|
|
def calculate_coverage_tests(self, args):
|
||
|
|
"""Determine which tests should be run."""
|
||
|
|
testing_tools_directory = os.path.join(self.source_directory, 'testing',
|
||
|
|
'tools')
|
||
|
|
tests = args['tests'] if args['tests'] else COVERAGE_TESTS.keys()
|
||
|
|
coverage_tests = {}
|
||
|
|
build_targets = set()
|
||
|
|
for name in tests:
|
||
|
|
test_spec = COVERAGE_TESTS[name]
|
||
|
|
if test_spec.use_test_runner:
|
||
|
|
binary_path = os.path.join(testing_tools_directory, test_spec.binary)
|
||
|
|
build_targets.add('pdfium_diff')
|
||
|
|
build_targets.add('pdfium_test')
|
||
|
|
else:
|
||
|
|
binary_path = os.path.join(self.build_directory, test_spec.binary)
|
||
|
|
build_targets.add(name)
|
||
|
|
coverage_tests[name] = TestSpec(binary_path, test_spec.use_test_runner,
|
||
|
|
test_spec.opt_args)
|
||
|
|
|
||
|
|
build_targets = list(build_targets)
|
||
|
|
|
||
|
|
return coverage_tests, build_targets
|
||
|
|
|
||
|
|
def build_binaries(self):
|
||
|
|
"""Build all the binaries that are going to be needed for coverage
|
||
|
|
generation."""
|
||
|
|
call_args = ['ninja']
|
||
|
|
if self.use_goma:
|
||
|
|
call_args += ['-j', '250']
|
||
|
|
call_args += ['-C', self.build_directory]
|
||
|
|
call_args += self.build_targets
|
||
|
|
return self.call(call_args, dry_run=self.dry_run) == 0
|
||
|
|
|
||
|
|
def generate_coverage(self, name, spec):
|
||
|
|
"""Generate the coverage data for a test
|
||
|
|
|
||
|
|
Args:
|
||
|
|
name: Name associated with the test to be run. This is used as a label
|
||
|
|
in the coverage data, so should be unique across all of the tests
|
||
|
|
being run.
|
||
|
|
spec: Tuple containing the TestSpec.
|
||
|
|
"""
|
||
|
|
if self.verbose:
|
||
|
|
print "Generating coverage for test '%s', using data '%s'" % (name, spec)
|
||
|
|
if not os.path.exists(spec.binary):
|
||
|
|
print('Unable to generate coverage for %s, since it appears to not exist'
|
||
|
|
' @ %s') % (name, spec.binary)
|
||
|
|
return False
|
||
|
|
|
||
|
|
binary_args = [spec.binary]
|
||
|
|
if spec.opt_args:
|
||
|
|
binary_args.extend(spec.opt_args)
|
||
|
|
profile_pattern_string = '%8m'
|
||
|
|
expected_profraw_file = '%s.%s.profraw' % (name, profile_pattern_string)
|
||
|
|
expected_profraw_path = os.path.join(self.output_directory,
|
||
|
|
expected_profraw_file)
|
||
|
|
|
||
|
|
env = {
|
||
|
|
'LLVM_PROFILE_FILE': expected_profraw_path,
|
||
|
|
'PATH': os.getenv('PATH') + os.pathsep + self.llvm_directory
|
||
|
|
}
|
||
|
|
|
||
|
|
if spec.use_test_runner:
|
||
|
|
# Test runner performs multi-threading in the wrapper script, not the test
|
||
|
|
# binary, so need to limit the number of instances of the binary being run
|
||
|
|
# to the max value in LLVM_PROFILE_FILE, which is 8.
|
||
|
|
binary_args.extend(['-j', '8', '--build-dir', self.build_directory])
|
||
|
|
if self.call(binary_args, dry_run=self.dry_run, env=env) and self.verbose:
|
||
|
|
print('Running %s appears to have failed, which might affect '
|
||
|
|
'results') % spec.binary
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
def merge_raw_coverage_results(self):
|
||
|
|
"""Merge raw coverage data sets into one one file for report generation."""
|
||
|
|
llvm_profdata_bin = os.path.join(self.llvm_directory, 'llvm-profdata')
|
||
|
|
|
||
|
|
raw_data = []
|
||
|
|
raw_data_pattern = '*.profraw'
|
||
|
|
for file_name in os.listdir(self.output_directory):
|
||
|
|
if fnmatch.fnmatch(file_name, raw_data_pattern):
|
||
|
|
raw_data.append(os.path.join(self.output_directory, file_name))
|
||
|
|
|
||
|
|
return self.call(
|
||
|
|
[llvm_profdata_bin, 'merge', '-o', self.prof_data, '-sparse=true'] +
|
||
|
|
raw_data) == 0
|
||
|
|
|
||
|
|
def generate_html_report(self):
|
||
|
|
"""Generate HTML report by calling upstream coverage.py"""
|
||
|
|
coverage_bin = os.path.join(self.source_directory, 'tools', 'code_coverage',
|
||
|
|
'coverage.py')
|
||
|
|
report_directory = os.path.join(self.output_directory, 'HTML')
|
||
|
|
|
||
|
|
coverage_args = ['-p', self.prof_data]
|
||
|
|
coverage_args += ['-b', self.build_directory]
|
||
|
|
coverage_args += ['-o', report_directory]
|
||
|
|
coverage_args += self.build_targets
|
||
|
|
|
||
|
|
# Whitelist the directories of interest
|
||
|
|
coverage_args += ['-f', 'core']
|
||
|
|
coverage_args += ['-f', 'fpdfsdk']
|
||
|
|
coverage_args += ['-f', 'fxbarcode']
|
||
|
|
coverage_args += ['-f', 'fxjs']
|
||
|
|
coverage_args += ['-f', 'public']
|
||
|
|
coverage_args += ['-f', 'samples']
|
||
|
|
coverage_args += ['-f', 'xfa']
|
||
|
|
|
||
|
|
# Blacklist test files
|
||
|
|
coverage_args += ['-i', '.*test.*']
|
||
|
|
|
||
|
|
# Component view is only useful for Chromium
|
||
|
|
coverage_args += ['--no-component-view']
|
||
|
|
|
||
|
|
return self.call([coverage_bin] + coverage_args) == 0
|
||
|
|
|
||
|
|
def run(self):
|
||
|
|
"""Setup environment, execute the tests and generate coverage report"""
|
||
|
|
if not self.fetch_profiling_tools():
|
||
|
|
print 'Unable to fetch profiling tools'
|
||
|
|
return False
|
||
|
|
|
||
|
|
if not self.build_binaries():
|
||
|
|
print 'Failed to successfully build binaries'
|
||
|
|
return False
|
||
|
|
|
||
|
|
for name in self.coverage_tests.keys():
|
||
|
|
if not self.generate_coverage(name, self.coverage_tests[name]):
|
||
|
|
print 'Failed to successfully generate coverage data'
|
||
|
|
return False
|
||
|
|
|
||
|
|
if not self.merge_raw_coverage_results():
|
||
|
|
print 'Failed to successfully merge raw coverage results'
|
||
|
|
return False
|
||
|
|
|
||
|
|
if not self.generate_html_report():
|
||
|
|
print 'Failed to successfully generate HTML report'
|
||
|
|
return False
|
||
|
|
|
||
|
|
return True
|
||
|
|
|
||
|
|
def fetch_profiling_tools(self):
|
||
|
|
"""Call coverage.py with no args to ensure profiling tools are present."""
|
||
|
|
return self.call_silent(
|
||
|
|
os.path.join(self.source_directory, 'tools', 'code_coverage',
|
||
|
|
'coverage.py')) == 0
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
parser = argparse.ArgumentParser()
|
||
|
|
parser.formatter_class = argparse.RawDescriptionHelpFormatter
|
||
|
|
parser.description = 'Generates a coverage report for given tests.'
|
||
|
|
|
||
|
|
parser.add_argument(
|
||
|
|
'-s',
|
||
|
|
'--source_directory',
|
||
|
|
help='Location of PDFium source directory, defaults to CWD',
|
||
|
|
default=os.getcwd())
|
||
|
|
build_default = os.path.join('out', 'Coverage')
|
||
|
|
parser.add_argument(
|
||
|
|
'-b',
|
||
|
|
'--build_directory',
|
||
|
|
help=
|
||
|
|
'Location of PDFium build directory with coverage enabled, defaults to '
|
||
|
|
'%s under CWD' % build_default,
|
||
|
|
default=os.path.join(os.getcwd(), build_default))
|
||
|
|
output_default = 'coverage_report'
|
||
|
|
parser.add_argument(
|
||
|
|
'-o',
|
||
|
|
'--output_directory',
|
||
|
|
help='Location to write out coverage report to, defaults to %s under CWD '
|
||
|
|
% output_default,
|
||
|
|
default=os.path.join(os.getcwd(), output_default))
|
||
|
|
parser.add_argument(
|
||
|
|
'-n',
|
||
|
|
'--dry-run',
|
||
|
|
help='Output commands instead of executing them',
|
||
|
|
action='store_true')
|
||
|
|
parser.add_argument(
|
||
|
|
'-v',
|
||
|
|
'--verbose',
|
||
|
|
help='Output additional diagnostic information',
|
||
|
|
action='store_true')
|
||
|
|
parser.add_argument(
|
||
|
|
'tests',
|
||
|
|
help='Tests to be run, defaults to all. Valid entries are %s' %
|
||
|
|
COVERAGE_TESTS.keys(),
|
||
|
|
nargs='*')
|
||
|
|
|
||
|
|
args = vars(parser.parse_args())
|
||
|
|
if args['verbose']:
|
||
|
|
pprint.pprint(args)
|
||
|
|
|
||
|
|
executor = CoverageExecutor(parser, args)
|
||
|
|
if executor.run():
|
||
|
|
return 0
|
||
|
|
return 1
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == '__main__':
|
||
|
|
sys.exit(main())
|