377 lines
16 KiB
Python
377 lines
16 KiB
Python
# Copyright 2017 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
import os
|
|
import csv
|
|
import json
|
|
import time
|
|
import urllib
|
|
import urllib2
|
|
import logging
|
|
import httplib
|
|
|
|
import enterprise_longevity_helper
|
|
from autotest_lib.client.common_lib import error
|
|
from autotest_lib.client.common_lib.cros import tpm_utils
|
|
from autotest_lib.server import autotest
|
|
from autotest_lib.server import test
|
|
from autotest_lib.server.cros.multimedia import remote_facade_factory
|
|
|
|
|
|
STABILIZATION_DURATION = 60
|
|
MEASUREMENT_DURATION_SECONDS = 10
|
|
TMP_DIRECTORY = '/tmp/'
|
|
PERF_FILE_NAME_PREFIX = 'perf'
|
|
VERSION_PATTERN = r'^(\d+)\.(\d+)\.(\d+)$'
|
|
DASHBOARD_UPLOAD_URL = 'https://chromeperf.appspot.com/add_point'
|
|
EXPECTED_PARAMS = ['perf_capture_iterations', 'perf_capture_duration',
|
|
'sample_interval', 'metric_interval', 'test_type',
|
|
'kiosk_app_attributes']
|
|
|
|
|
|
class PerfUploadingError(Exception):
|
|
"""Exception raised in perf_uploader."""
|
|
pass
|
|
|
|
|
|
class enterprise_LongevityTrackerServer(test.test):
|
|
"""
|
|
Run Longevity Test: Collect performance data over long duration.
|
|
|
|
Run enterprise_KioskEnrollment and clear the TPM as necessary. After
|
|
enterprise enrollment is successful, collect and log cpu, memory, and
|
|
temperature data from the device under test.
|
|
|
|
"""
|
|
version = 1
|
|
|
|
|
|
def initialize(self):
|
|
self.temp_dir = os.path.split(self.tmpdir)[0]
|
|
|
|
|
|
#TODO(krishnargv@): Add a method to retrieve the version of the
|
|
# Kiosk app from its manifest.
|
|
def _initialize_test_variables(self):
|
|
"""Initialize test variables that will be uploaded to the dashboard."""
|
|
self.board_name = self.system_facade.get_current_board()
|
|
self.chromeos_version = self.system_facade.get_chromeos_release_version()
|
|
epoch_minutes = str(int(time.time() / 60))
|
|
self.point_id = enterprise_longevity_helper.get_point_id(
|
|
self.chromeos_version, epoch_minutes, VERSION_PATTERN)
|
|
self.test_suite_name = self.tagged_testname
|
|
self.perf_capture_duration = self.perf_params['perf_capture_duration']
|
|
self.sample_interval = self.perf_params['sample_interval']
|
|
self.metric_interval = self.perf_params['metric_interval']
|
|
self.perf_results = {'cpu': '0', 'mem': '0', 'temp': '0'}
|
|
|
|
|
|
def elapsed_time(self, mark_time):
|
|
"""
|
|
Get time elapsed since |mark_time|.
|
|
|
|
@param mark_time: point in time from which elapsed time is measured.
|
|
|
|
@returns time elapsed since the marked time.
|
|
|
|
"""
|
|
return time.time() - mark_time
|
|
|
|
|
|
#TODO(krishnargv): Replace _format_data_for_upload with a call to the
|
|
# _format_for_upload method of the perf_uploader.py
|
|
def _format_data_for_upload(self, chart_data):
|
|
"""
|
|
Collect chart data into an uploadable data JSON object.
|
|
|
|
@param chart_data: performance results formatted as chart data.
|
|
|
|
"""
|
|
perf_values = {
|
|
'format_version': '1.0',
|
|
'benchmark_name': self.test_suite_name,
|
|
'charts': chart_data,
|
|
}
|
|
#TODO(krishnargv): Add a method to capture the chrome_version.
|
|
dash_entry = {
|
|
'master': 'ChromeOS_Enterprise',
|
|
'bot': 'cros-%s' % self.board_name,
|
|
'point_id': self.point_id,
|
|
'versions': {
|
|
'cros_version': self.chromeos_version,
|
|
|
|
},
|
|
'supplemental': {
|
|
'default_rev': 'r_cros_version',
|
|
'kiosk_app_name': 'a_' + self.kiosk_app_name,
|
|
|
|
},
|
|
'chart_data': perf_values
|
|
}
|
|
return {'data': json.dumps(dash_entry)}
|
|
|
|
|
|
#TODO(krishnargv): Replace _send_to_dashboard with a call to the
|
|
# _send_to_dashboard method of the perf_uploader.py
|
|
def _send_to_dashboard(self, data_obj):
|
|
"""
|
|
Send formatted perf data to the perf dashboard.
|
|
|
|
@param data_obj: data object as returned by _format_data_for_upload().
|
|
|
|
@raises PerfUploadingError if an exception was raised when uploading.
|
|
|
|
"""
|
|
logging.debug('Data_obj to be uploaded: %s', data_obj)
|
|
encoded = urllib.urlencode(data_obj)
|
|
req = urllib2.Request(DASHBOARD_UPLOAD_URL, encoded)
|
|
try:
|
|
urllib2.urlopen(req)
|
|
except urllib2.HTTPError as e:
|
|
raise PerfUploadingError('HTTPError: %d %s for JSON %s\n' %
|
|
(e.code, e.msg, data_obj['data']))
|
|
except urllib2.URLError as e:
|
|
raise PerfUploadingError('URLError: %s for JSON %s\n' %
|
|
(str(e.reason), data_obj['data']))
|
|
except httplib.HTTPException:
|
|
raise PerfUploadingError('HTTPException for JSON %s\n' %
|
|
data_obj['data'])
|
|
|
|
|
|
def _write_perf_keyvals(self, perf_results):
|
|
"""
|
|
Write perf results to keyval file for AutoTest results.
|
|
|
|
@param perf_results: dict of attribute performance metrics.
|
|
|
|
"""
|
|
perf_keyval = {}
|
|
perf_keyval['cpu_usage'] = perf_results['cpu']
|
|
perf_keyval['memory_usage'] = perf_results['mem']
|
|
perf_keyval['temperature'] = perf_results['temp']
|
|
self.write_perf_keyval(perf_keyval)
|
|
|
|
|
|
def _write_perf_results(self, perf_results):
|
|
"""
|
|
Write perf results to results-chart.json file for Perf Dashboard.
|
|
|
|
@param perf_results: dict of attribute performance metrics.
|
|
|
|
"""
|
|
cpu_metric = perf_results['cpu']
|
|
mem_metric = perf_results['mem']
|
|
ec_metric = perf_results['temp']
|
|
self.output_perf_value(description='cpu_usage', value=cpu_metric,
|
|
units='percent', higher_is_better=False)
|
|
self.output_perf_value(description='mem_usage', value=mem_metric,
|
|
units='percent', higher_is_better=False)
|
|
self.output_perf_value(description='max_temp', value=ec_metric,
|
|
units='Celsius', higher_is_better=False)
|
|
|
|
|
|
def _record_perf_measurements(self, perf_values, perf_writer):
|
|
"""
|
|
Record attribute performance measurements, and write to file.
|
|
|
|
@param perf_values: dict of attribute performance values.
|
|
@param perf_writer: file to write performance measurements.
|
|
|
|
"""
|
|
# Get performance measurements.
|
|
cpu_usage = '%.3f' % enterprise_longevity_helper.get_cpu_usage(
|
|
self.system_facade, MEASUREMENT_DURATION_SECONDS)
|
|
mem_usage = '%.3f' % enterprise_longevity_helper.get_memory_usage(
|
|
self.system_facade)
|
|
max_temp = '%.3f' % enterprise_longevity_helper.get_temperature_data(
|
|
self.client, self.system_facade)
|
|
|
|
# Append measurements to attribute lists in perf values dictionary.
|
|
perf_values['cpu'].append(float(cpu_usage))
|
|
perf_values['mem'].append(float(mem_usage))
|
|
perf_values['temp'].append(float(max_temp))
|
|
|
|
# Write performance measurements to perf timestamped file.
|
|
time_stamp = time.strftime('%Y/%m/%d %H:%M:%S')
|
|
perf_writer.writerow([time_stamp, cpu_usage, mem_usage, max_temp])
|
|
logging.info('Time: %s, CPU: %r, Mem: %r, Temp: %r',
|
|
time_stamp, cpu_usage, mem_usage, max_temp)
|
|
|
|
|
|
def _setup_kiosk_app_on_dut(self, kiosk_app_attributes=None):
|
|
"""Enroll the DUT and setup a Kiosk app."""
|
|
info = self.client.host_info_store.get()
|
|
app_config_id = info.get_label_value('app_config_id')
|
|
if app_config_id and app_config_id.startswith(':'):
|
|
app_config_id = app_config_id[1:]
|
|
if kiosk_app_attributes:
|
|
kiosk_app_attributes = kiosk_app_attributes.rstrip()
|
|
self.kiosk_app_name, ext_id = kiosk_app_attributes.split(':')[:2]
|
|
|
|
tpm_utils.ClearTPMOwnerRequest(self.client)
|
|
logging.info("Enrolling the DUT to Kiosk mode")
|
|
autotest.Autotest(self.client).run_test(
|
|
'enterprise_KioskEnrollment',
|
|
kiosk_app_attributes=kiosk_app_attributes,
|
|
check_client_result=True)
|
|
|
|
#if self.kiosk_app_name == 'riseplayer':
|
|
# self.kiosk_facade.config_rise_player(ext_id, app_config_id)
|
|
|
|
|
|
def _run_perf_capture_cycle(self):
|
|
"""
|
|
Track performance of Chrome OS over a long period of time.
|
|
|
|
This method collects performance measurements, and calculates metrics
|
|
to upload to the performance dashboard. It creates two files to
|
|
collect and store performance values and results: perf_<timestamp>.csv
|
|
and perf_aggregated.csv.
|
|
|
|
At the start, it creates a unique perf timestamped file in the test's
|
|
temp_dir. As the cycle runs, it saves a time-stamped performance
|
|
value after each sample interval. Periodically, it calculates
|
|
the 90th percentile performance metrics from these values.
|
|
|
|
The perf_<timestamp> files on the device will survive multiple runs
|
|
of the longevity_Tracker by the server-side test, and will also
|
|
survive multiple runs of the server-side test.
|
|
|
|
At the end, it opens the perf aggregated file in the test's temp_dir,
|
|
and appends the contents of the perf timestamped file. It then
|
|
copies the perf aggregated file to the results directory as perf.csv.
|
|
This perf.csv file will be consumed by the AutoTest backend when the
|
|
server-side test ends.
|
|
|
|
Note that the perf_aggregated.csv file will grow larger with each run
|
|
of longevity_Tracker on the device by the server-side test.
|
|
|
|
This method will capture perf metrics every SAMPLE_INTERVAL secs, at
|
|
each METRIC_INTERVAL the 90 percentile of the collected metrics is
|
|
calculated and saved. The perf capture runs for PERF_CAPTURE_DURATION
|
|
secs. At the end of the PERF_CAPTURE_DURATION time interval the median
|
|
value of all 90th percentile metrics is returned.
|
|
|
|
@returns list of median performance metrics.
|
|
|
|
"""
|
|
test_start_time = time.time()
|
|
|
|
perf_values = {'cpu': [], 'mem': [], 'temp': []}
|
|
perf_metrics = {'cpu': [], 'mem': [], 'temp': []}
|
|
|
|
# Create perf_<timestamp> file and writer.
|
|
timestamp_fname = (PERF_FILE_NAME_PREFIX +
|
|
time.strftime('_%Y-%m-%d_%H-%M') + '.csv')
|
|
timestamp_fpath = os.path.join(self.temp_dir, timestamp_fname)
|
|
timestamp_file = enterprise_longevity_helper.open_perf_file(
|
|
timestamp_fpath)
|
|
timestamp_writer = csv.writer(timestamp_file)
|
|
|
|
# Align time of loop start with the sample interval.
|
|
test_elapsed_time = self.elapsed_time(test_start_time)
|
|
time.sleep(enterprise_longevity_helper.syncup_time(
|
|
test_elapsed_time, self.sample_interval))
|
|
test_elapsed_time = self.elapsed_time(test_start_time)
|
|
|
|
metric_start_time = time.time()
|
|
metric_prev_time = metric_start_time
|
|
|
|
metric_elapsed_prev_time = self.elapsed_time(metric_prev_time)
|
|
offset = enterprise_longevity_helper.modulo_time(
|
|
metric_elapsed_prev_time, self.metric_interval)
|
|
metric_timer = metric_elapsed_prev_time + offset
|
|
|
|
while self.elapsed_time(test_start_time) <= self.perf_capture_duration:
|
|
self._record_perf_measurements(perf_values, timestamp_writer)
|
|
|
|
# Periodically calculate and record 90th percentile metrics.
|
|
metric_elapsed_prev_time = self.elapsed_time(metric_prev_time)
|
|
metric_timer = metric_elapsed_prev_time + offset
|
|
if metric_timer >= self.metric_interval:
|
|
enterprise_longevity_helper.record_90th_metrics(
|
|
perf_values, perf_metrics)
|
|
perf_values = {'cpu': [], 'mem': [], 'temp': []}
|
|
|
|
# Set previous time to current time.
|
|
metric_prev_time = time.time()
|
|
metric_elapsed_prev_time = self.elapsed_time(metric_prev_time)
|
|
|
|
metric_elapsed_time = self.elapsed_time(metric_start_time)
|
|
offset = enterprise_longevity_helper.modulo_time(
|
|
metric_elapsed_time, self.metric_interval)
|
|
|
|
# Set the timer to time elapsed plus offset to next interval.
|
|
metric_timer = metric_elapsed_prev_time + offset
|
|
|
|
# Sync the loop time to the sample interval.
|
|
test_elapsed_time = self.elapsed_time(test_start_time)
|
|
time.sleep(enterprise_longevity_helper.syncup_time(
|
|
test_elapsed_time, self.sample_interval))
|
|
|
|
# Close perf timestamp file.
|
|
timestamp_file.close()
|
|
|
|
# Open perf timestamp file to read, and aggregated file to append.
|
|
timestamp_file = open(timestamp_fpath, 'r')
|
|
aggregated_fname = (PERF_FILE_NAME_PREFIX + '_aggregated.csv')
|
|
aggregated_fpath = os.path.join(self.temp_dir, aggregated_fname)
|
|
aggregated_file = enterprise_longevity_helper.open_perf_file(
|
|
aggregated_fpath)
|
|
|
|
# Append contents of perf timestamp file to perf aggregated file.
|
|
enterprise_longevity_helper.append_to_aggregated_file(
|
|
timestamp_file, aggregated_file)
|
|
timestamp_file.close()
|
|
aggregated_file.close()
|
|
|
|
# Copy perf aggregated file to test results directory.
|
|
enterprise_longevity_helper.copy_aggregated_to_resultsdir(
|
|
self.resultsdir, aggregated_fpath, 'perf.csv')
|
|
|
|
# Return median of each attribute performance metric.
|
|
logging.info("Perf_metrics: %r ", perf_metrics)
|
|
return enterprise_longevity_helper.get_median_metrics(perf_metrics)
|
|
|
|
|
|
def run_once(self, host=None, perf_params=None):
|
|
self.client = host
|
|
self.kiosk_app_name = None
|
|
self.perf_params = perf_params
|
|
logging.info('Perf params: %r', self.perf_params)
|
|
|
|
if not enterprise_longevity_helper.verify_perf_params(
|
|
EXPECTED_PARAMS, self.perf_params):
|
|
raise error.TestFail('Missing or incorrect perf_params in the'
|
|
' control file. Refer to the README.txt for'
|
|
' info on perf params.: %r'
|
|
%(self.perf_params))
|
|
|
|
factory = remote_facade_factory.RemoteFacadeFactory(
|
|
host, no_chrome=True)
|
|
self.system_facade = factory.create_system_facade()
|
|
self.kiosk_facade = factory.create_kiosk_facade()
|
|
|
|
self._setup_kiosk_app_on_dut(self.perf_params['kiosk_app_attributes'])
|
|
time.sleep(STABILIZATION_DURATION)
|
|
|
|
self._initialize_test_variables()
|
|
for iteration in range(self.perf_params['perf_capture_iterations']):
|
|
#TODO(krishnargv@): Add a method to verify that the Kiosk app is
|
|
# active and is running on the DUT.
|
|
logging.info("Running perf_capture Iteration: %d", iteration+1)
|
|
self.perf_results = self._run_perf_capture_cycle()
|
|
self._write_perf_keyvals(self.perf_results)
|
|
self._write_perf_results(self.perf_results)
|
|
|
|
# Post perf results directly to performance dashboard. You may view
|
|
# uploaded data at https://chromeperf.appspot.com/new_points,
|
|
# with test path pattern=ChromeOS_Enterprise/cros-*/longevity*/*
|
|
if perf_params['test_type'] == 'multiple_samples':
|
|
chart_data = enterprise_longevity_helper.read_perf_results(
|
|
self.resultsdir, 'results-chart.json')
|
|
data_obj = self._format_data_for_upload(chart_data)
|
|
self._send_to_dashboard(data_obj)
|
|
tpm_utils.ClearTPMOwnerRequest(self.client)
|