394 lines
16 KiB
Python
Executable File
394 lines
16 KiB
Python
Executable File
#!/usr/bin/env python
|
|
#
|
|
# Copyright 2013 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""Updates the Chrome reference builds.
|
|
|
|
Usage:
|
|
$ /path/to/update_reference_build.py
|
|
$ git commit -a
|
|
$ git cl upload
|
|
"""
|
|
|
|
import argparse
|
|
import collections
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib2
|
|
import zipfile
|
|
|
|
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'py_utils'))
|
|
|
|
from py_utils import cloud_storage
|
|
from dependency_manager import base_config
|
|
|
|
|
|
_CHROME_BINARIES_CONFIG = os.path.join(
|
|
os.path.dirname(os.path.abspath(__file__)), '..', '..', 'common',
|
|
'py_utils', 'py_utils', 'chrome_binaries.json')
|
|
|
|
_CHROME_GS_BUCKET = 'chrome-unsigned'
|
|
_CHROMIUM_GS_BUCKET = 'chromium-browser-snapshots'
|
|
|
|
# How many commit positions to search below and above omaha branch position to
|
|
# find closest chromium build snapshot. The value 10 is chosen because it looks
|
|
# more than sufficient from manual inspection of the bucket.
|
|
_CHROMIUM_SNAPSHOT_SEARCH_WINDOW = 10
|
|
|
|
# Remove a platform name from this list to disable updating it.
|
|
# Add one to enable updating it. (Must also update _PLATFORM_MAP.)
|
|
_PLATFORMS_TO_UPDATE = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64',
|
|
'android_k_armeabi-v7a', 'android_l_arm64-v8a',
|
|
'android_l_armeabi-v7a', 'android_n_armeabi-v7a',
|
|
'android_n_arm64-v8a', 'android_n_bundle_armeabi-v7a',
|
|
'android_n_bundle_arm64-v8a']
|
|
|
|
# Add platforms here if you also want to update chromium binary for it.
|
|
# Must add chromium_info for it in _PLATFORM_MAP.
|
|
_CHROMIUM_PLATFORMS = ['mac_x86_64', 'win_x86', 'win_AMD64', 'linux_x86_64']
|
|
|
|
# Remove a channel name from this list to disable updating it.
|
|
# Add one to enable updating it.
|
|
_CHANNELS_TO_UPDATE = ['stable', 'canary', 'dev']
|
|
|
|
|
|
# Omaha is Chrome's autoupdate server. It reports the current versions used
|
|
# by each platform on each channel.
|
|
_OMAHA_PLATFORMS = { 'stable': ['mac', 'linux', 'win', 'android'],
|
|
'dev': ['linux'], 'canary': ['mac', 'win']}
|
|
|
|
|
|
# All of the information we need to update each platform.
|
|
# omaha: name omaha uses for the platforms.
|
|
# zip_name: name of the zip file to be retrieved from cloud storage.
|
|
# gs_build: name of the Chrome build platform used in cloud storage.
|
|
# chromium_info: information needed to update chromium (optional).
|
|
# destination: Name of the folder to download the reference build to.
|
|
UpdateInfo = collections.namedtuple('UpdateInfo',
|
|
'omaha, gs_folder, gs_build, chromium_info, zip_name')
|
|
# build_dir: name of the build directory in _CHROMIUM_GS_BUCKET.
|
|
# zip_name: name of the zip file to be retrieved from cloud storage.
|
|
ChromiumInfo = collections.namedtuple('ChromiumInfo', 'build_dir, zip_name')
|
|
_PLATFORM_MAP = {'mac_x86_64': UpdateInfo(
|
|
omaha='mac',
|
|
gs_folder='desktop-*',
|
|
gs_build='mac64',
|
|
chromium_info=ChromiumInfo(
|
|
build_dir='Mac',
|
|
zip_name='chrome-mac.zip'),
|
|
zip_name='chrome-mac.zip'),
|
|
'win_x86': UpdateInfo(
|
|
omaha='win',
|
|
gs_folder='desktop-*',
|
|
gs_build='win-clang',
|
|
chromium_info=ChromiumInfo(
|
|
build_dir='Win',
|
|
zip_name='chrome-win.zip'),
|
|
zip_name='chrome-win-clang.zip'),
|
|
'win_AMD64': UpdateInfo(
|
|
omaha='win',
|
|
gs_folder='desktop-*',
|
|
gs_build='win64-clang',
|
|
chromium_info=ChromiumInfo(
|
|
build_dir='Win_x64',
|
|
zip_name='chrome-win.zip'),
|
|
zip_name='chrome-win64-clang.zip'),
|
|
'linux_x86_64': UpdateInfo(
|
|
omaha='linux',
|
|
gs_folder='desktop-*',
|
|
gs_build='linux64',
|
|
chromium_info=ChromiumInfo(
|
|
build_dir='Linux_x64',
|
|
zip_name='chrome-linux.zip'),
|
|
zip_name='chrome-linux64.zip'),
|
|
'android_k_armeabi-v7a': UpdateInfo(
|
|
omaha='android',
|
|
gs_folder='android-*',
|
|
gs_build='arm',
|
|
chromium_info=None,
|
|
zip_name='Chrome.apk'),
|
|
'android_l_arm64-v8a': UpdateInfo(
|
|
omaha='android',
|
|
gs_folder='android-*',
|
|
gs_build='arm_64',
|
|
chromium_info=None,
|
|
zip_name='ChromeModern.apk'),
|
|
'android_l_armeabi-v7a': UpdateInfo(
|
|
omaha='android',
|
|
gs_folder='android-*',
|
|
gs_build='arm',
|
|
chromium_info=None,
|
|
zip_name='Chrome.apk'),
|
|
'android_n_armeabi-v7a': UpdateInfo(
|
|
omaha='android',
|
|
gs_folder='android-*',
|
|
gs_build='arm',
|
|
chromium_info=None,
|
|
zip_name='Monochrome.apk'),
|
|
'android_n_arm64-v8a': UpdateInfo(
|
|
omaha='android',
|
|
gs_folder='android-*',
|
|
gs_build='arm_64',
|
|
chromium_info=None,
|
|
zip_name='Monochrome.apk'),
|
|
'android_n_bundle_armeabi-v7a': UpdateInfo(
|
|
omaha='android',
|
|
gs_folder='android-*',
|
|
gs_build='arm',
|
|
chromium_info=None,
|
|
zip_name='Monochrome.apks'),
|
|
'android_n_bundle_arm64-v8a': UpdateInfo(
|
|
omaha='android',
|
|
gs_folder='android-*',
|
|
gs_build='arm_64',
|
|
chromium_info=None,
|
|
zip_name='Monochrome.apks')
|
|
|
|
}
|
|
|
|
|
|
VersionInfo = collections.namedtuple('VersionInfo',
|
|
'version, branch_base_position')
|
|
|
|
|
|
def _ChannelVersionsMap(channel):
|
|
rows = _OmahaReportVersionInfo(channel)
|
|
omaha_versions_map = _OmahaVersionsMap(rows, channel)
|
|
channel_versions_map = {}
|
|
for platform in _PLATFORMS_TO_UPDATE:
|
|
omaha_platform = _PLATFORM_MAP[platform].omaha
|
|
if omaha_platform in omaha_versions_map:
|
|
channel_versions_map[platform] = omaha_versions_map[omaha_platform]
|
|
return channel_versions_map
|
|
|
|
|
|
def _OmahaReportVersionInfo(channel):
|
|
url ='https://omahaproxy.appspot.com/all?channel=%s' % channel
|
|
lines = urllib2.urlopen(url).readlines()
|
|
return [l.split(',') for l in lines]
|
|
|
|
|
|
def _OmahaVersionsMap(rows, channel):
|
|
platforms = _OMAHA_PLATFORMS.get(channel, [])
|
|
if (len(rows) < 1 or
|
|
rows[0][0:3] != ['os', 'channel', 'current_version'] or
|
|
rows[0][7] != 'branch_base_position'):
|
|
raise ValueError(
|
|
'Omaha report is not in the expected form: %s.' % rows)
|
|
versions_map = {}
|
|
for row in rows[1:]:
|
|
if row[1] != channel:
|
|
raise ValueError(
|
|
'Omaha report contains a line with the channel %s' % row[1])
|
|
if row[0] in platforms:
|
|
versions_map[row[0]] = VersionInfo(version=row[2],
|
|
branch_base_position=int(row[7]))
|
|
logging.warn('versions map: %s' % versions_map)
|
|
if not all(platform in versions_map for platform in platforms):
|
|
raise ValueError(
|
|
'Omaha report did not contain all desired platforms '
|
|
'for channel %s' % channel)
|
|
return versions_map
|
|
|
|
|
|
RemotePath = collections.namedtuple('RemotePath', 'bucket, path')
|
|
|
|
|
|
def _ResolveChromeRemotePath(platform_info, version_info):
|
|
# Path example: desktop-*/30.0.1595.0/precise32/chrome-precise32.zip
|
|
return RemotePath(bucket=_CHROME_GS_BUCKET,
|
|
path=('%s/%s/%s/%s' % (platform_info.gs_folder,
|
|
version_info.version,
|
|
platform_info.gs_build,
|
|
platform_info.zip_name)))
|
|
|
|
|
|
def _FindClosestChromiumSnapshot(base_position, build_dir):
|
|
"""Returns the closest chromium snapshot available in cloud storage.
|
|
|
|
Chromium snapshots are pulled from _CHROMIUM_BUILD_DIR in CHROMIUM_GS_BUCKET.
|
|
|
|
Continuous chromium snapshots do not always contain the exact release build.
|
|
This function queries the storage bucket and find the closest snapshot within
|
|
+/-_CHROMIUM_SNAPSHOT_SEARCH_WINDOW to find the closest build.
|
|
"""
|
|
min_position = base_position - _CHROMIUM_SNAPSHOT_SEARCH_WINDOW
|
|
max_position = base_position + _CHROMIUM_SNAPSHOT_SEARCH_WINDOW
|
|
|
|
# Getting the full list of objects in cloud storage bucket is prohibitively
|
|
# slow. It's faster to list objects with a prefix. Assuming we're looking at
|
|
# +/- 10 commit positions, for commit position 123456, we want to look at
|
|
# positions between 123446 an 123466. We do this by getting all snapshots
|
|
# with prefix 12344*, 12345*, and 12346*. This may get a few more snapshots
|
|
# that we intended, but that's fine since we take the min distance anyways.
|
|
min_position_prefix = min_position / 10;
|
|
max_position_prefix = max_position / 10;
|
|
|
|
available_positions = []
|
|
for position_prefix in range(min_position_prefix, max_position_prefix + 1):
|
|
query = '%s/%d*' % (build_dir, position_prefix)
|
|
try:
|
|
ls_results = cloud_storage.ListDirs(_CHROMIUM_GS_BUCKET, query)
|
|
except cloud_storage.NotFoundError:
|
|
# It's fine if there is no chromium snapshot available for one prefix.
|
|
# We will look at the rest of the prefixes.
|
|
continue
|
|
|
|
for entry in ls_results:
|
|
# entry looks like '/Linux_x64/${commit_position}/'.
|
|
position = int(entry.split('/')[2])
|
|
available_positions.append(position)
|
|
|
|
if len(available_positions) == 0:
|
|
raise ValueError('No chromium build found +/-%d commit positions of %d' %
|
|
(_CHROMIUM_SNAPSHOT_SEARCH_WINDOW, base_position))
|
|
|
|
distance_function = lambda position: abs(position - base_position)
|
|
min_distance_snapshot = min(available_positions, key=distance_function)
|
|
return min_distance_snapshot
|
|
|
|
|
|
def _ResolveChromiumRemotePath(channel, platform, version_info):
|
|
platform_info = _PLATFORM_MAP[platform]
|
|
branch_base_position = version_info.branch_base_position
|
|
omaha_version = version_info.version
|
|
build_dir = platform_info.chromium_info.build_dir
|
|
# Look through chromium-browser-snapshots for closest match.
|
|
closest_snapshot = _FindClosestChromiumSnapshot(
|
|
branch_base_position, build_dir)
|
|
if closest_snapshot != branch_base_position:
|
|
print ('Channel %s corresponds to commit position ' % channel +
|
|
'%d on %s, ' % (branch_base_position, platform) +
|
|
'but closest chromium snapshot available on ' +
|
|
'%s is %d' % (_CHROMIUM_GS_BUCKET, closest_snapshot))
|
|
return RemotePath(bucket=_CHROMIUM_GS_BUCKET,
|
|
path = ('%s/%s/%s' % (build_dir, closest_snapshot,
|
|
platform_info.chromium_info.zip_name)))
|
|
|
|
|
|
def _QueuePlatformUpdate(binary, platform, version_info, config, channel):
|
|
""" platform: the name of the platform for the browser to
|
|
be downloaded & updated from cloud storage. """
|
|
platform_info = _PLATFORM_MAP[platform]
|
|
|
|
if binary == 'chrome':
|
|
remote_path = _ResolveChromeRemotePath(platform_info, version_info)
|
|
elif binary == 'chromium':
|
|
remote_path = _ResolveChromiumRemotePath(channel, platform, version_info)
|
|
else:
|
|
raise ValueError('binary must be \'chrome\' or \'chromium\'')
|
|
|
|
if not cloud_storage.Exists(remote_path.bucket, remote_path.path):
|
|
cloud_storage_path = 'gs://%s/%s' % (remote_path.bucket, remote_path.path)
|
|
logging.warn('Failed to find %s build for version %s at path %s.' % (
|
|
platform, version_info.version, cloud_storage_path))
|
|
logging.warn('Skipping this update for this platform/channel.')
|
|
return
|
|
|
|
reference_builds_folder = os.path.join(
|
|
os.path.dirname(os.path.abspath(__file__)), 'chrome_telemetry_build',
|
|
'reference_builds', binary, channel)
|
|
if not os.path.exists(reference_builds_folder):
|
|
os.makedirs(reference_builds_folder)
|
|
local_dest_path = os.path.join(reference_builds_folder,
|
|
platform,
|
|
platform_info.zip_name)
|
|
cloud_storage.Get(remote_path.bucket, remote_path.path, local_dest_path)
|
|
_ModifyBuildIfNeeded(binary, local_dest_path, platform)
|
|
config.AddCloudStorageDependencyUpdateJob('%s_%s' % (binary, channel),
|
|
platform, local_dest_path, version=version_info.version,
|
|
execute_job=False)
|
|
|
|
|
|
def _ModifyBuildIfNeeded(binary, location, platform):
|
|
"""Hook to modify the build before saving it for Telemetry to use.
|
|
|
|
This can be used to remove various utilities that cause noise in a
|
|
test environment. Right now, it is just used to remove Keystone,
|
|
which is a tool used to autoupdate Chrome.
|
|
"""
|
|
if binary != 'chrome':
|
|
return
|
|
|
|
if platform == 'mac_x86_64':
|
|
_RemoveKeystoneFromBuild(location)
|
|
return
|
|
|
|
if 'mac' in platform:
|
|
raise NotImplementedError(
|
|
'Platform <%s> sounds like it is an OSX version. If so, we may need to '
|
|
'remove Keystone from it per crbug.com/932615. Please edit this script'
|
|
' and teach it what needs to be done :).')
|
|
|
|
|
|
def _RemoveKeystoneFromBuild(location):
|
|
"""Removes the Keystone autoupdate binary from the chrome mac zipfile."""
|
|
logging.info('Removing keystone from mac build at %s' % location)
|
|
temp_folder = tempfile.mkdtemp(prefix='RemoveKeystoneFromBuild')
|
|
try:
|
|
subprocess.check_call(['unzip', '-q', location, '-d', temp_folder])
|
|
keystone_folder = os.path.join(
|
|
temp_folder, 'chrome-mac', 'Google Chrome.app', 'Contents',
|
|
'Frameworks', 'Google Chrome Framework.framework', 'Frameworks',
|
|
'KeystoneRegistration.framework')
|
|
shutil.rmtree(keystone_folder)
|
|
os.remove(location)
|
|
subprocess.check_call(['zip', '--quiet', '--recurse-paths', '--symlinks',
|
|
location, 'chrome-mac'],
|
|
cwd=temp_folder)
|
|
finally:
|
|
shutil.rmtree(temp_folder)
|
|
|
|
|
|
def _NeedsUpdate(config, binary, channel, platform, version_info):
|
|
channel_version = version_info.version
|
|
print 'Checking %s (%s channel) on %s' % (binary, channel, platform)
|
|
current_version = config.GetVersion('%s_%s' % (binary, channel), platform)
|
|
print 'current: %s, channel: %s' % (current_version, channel_version)
|
|
if current_version and current_version == channel_version:
|
|
print 'Already up to date.'
|
|
return False
|
|
return True
|
|
|
|
|
|
def UpdateBuilds(args):
|
|
config = base_config.BaseConfig(_CHROME_BINARIES_CONFIG, writable=True)
|
|
for channel in _CHANNELS_TO_UPDATE:
|
|
channel_versions_map = _ChannelVersionsMap(channel)
|
|
for platform in channel_versions_map:
|
|
version_info = channel_versions_map.get(platform)
|
|
if args.update_chrome:
|
|
if _NeedsUpdate(config, 'chrome', channel, platform, version_info):
|
|
_QueuePlatformUpdate('chrome', platform, version_info, config,
|
|
channel)
|
|
if args.update_chromium and platform in _CHROMIUM_PLATFORMS:
|
|
if _NeedsUpdate(config, 'chromium', channel, platform, version_info):
|
|
_QueuePlatformUpdate('chromium', platform, version_info,
|
|
config, channel)
|
|
|
|
print 'Updating builds with downloaded binaries'
|
|
config.ExecuteUpdateJobs(force=True)
|
|
|
|
|
|
def main():
|
|
logging.getLogger().setLevel(logging.DEBUG)
|
|
parser = argparse.ArgumentParser(
|
|
description='Update reference binaries used by perf bots.')
|
|
parser.add_argument('--no-update-chrome', action='store_false',
|
|
dest='update_chrome', default=True,
|
|
help='do not update chrome binaries')
|
|
parser.add_argument('--no-update-chromium', action='store_false',
|
|
dest='update_chromium', default=True,
|
|
help='do not update chromium binaries')
|
|
args = parser.parse_args()
|
|
UpdateBuilds(args)
|
|
|
|
if __name__ == '__main__':
|
|
main()
|