141 lines
5.2 KiB
Python
141 lines
5.2 KiB
Python
# Copyright 2014 The Chromium Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
import base64
|
|
import io
|
|
import gzip
|
|
import os
|
|
import re
|
|
|
|
import six
|
|
|
|
from devil import devil_env
|
|
from devil.android import device_errors
|
|
from devil.utils import cmd_helper
|
|
|
|
MD5SUM_DEVICE_LIB_PATH = '/data/local/tmp/md5sum'
|
|
MD5SUM_DEVICE_BIN_PATH = MD5SUM_DEVICE_LIB_PATH + '/md5sum_bin'
|
|
|
|
_STARTS_WITH_CHECKSUM_RE = re.compile(r'^[0-9a-fA-F]{16}$')
|
|
|
|
# We need to cap how many paths we send to the md5_sum binaries at once because
|
|
# the ARG_MAX on Android devices is relatively small, typically 131072 bytes.
|
|
# However, the more paths we use per invocation, the lower the overhead of
|
|
# starting processes, so we want to maximize this number, but we can't compute
|
|
# it exactly as we don't know how well our paths will compress.
|
|
# 5000 is experimentally determined to be reasonable. 10000 fails, and 7500
|
|
# works with existing usage, so 5000 seems like a pretty safe compromise.
|
|
_MAX_PATHS_PER_INVOCATION = 5000
|
|
|
|
|
|
def CalculateHostMd5Sums(paths):
|
|
"""Calculates the MD5 sum value for all items in |paths|.
|
|
|
|
Directories are traversed recursively and the MD5 sum of each file found is
|
|
reported in the result.
|
|
|
|
Args:
|
|
paths: A list of host paths to md5sum.
|
|
Returns:
|
|
A dict mapping file paths to their respective md5sum checksums.
|
|
"""
|
|
if isinstance(paths, six.string_types):
|
|
paths = [paths]
|
|
paths = list(paths)
|
|
|
|
md5sum_bin_host_path = devil_env.config.FetchPath('md5sum_host')
|
|
if not os.path.exists(md5sum_bin_host_path):
|
|
raise IOError('File not built: %s' % md5sum_bin_host_path)
|
|
out = ""
|
|
for i in range(0, len(paths), _MAX_PATHS_PER_INVOCATION):
|
|
mem_file = io.BytesIO()
|
|
compressed = gzip.GzipFile(fileobj=mem_file, mode="wb")
|
|
data = ";".join(
|
|
[os.path.realpath(p) for p in paths[i:i+_MAX_PATHS_PER_INVOCATION]])
|
|
if six.PY3:
|
|
data = data.encode('utf-8')
|
|
compressed.write(data)
|
|
compressed.close()
|
|
compressed_paths = base64.b64encode(mem_file.getvalue())
|
|
out += cmd_helper.GetCmdOutput(
|
|
[md5sum_bin_host_path, "-gz", compressed_paths])
|
|
|
|
return dict(zip(paths, out.splitlines()))
|
|
|
|
|
|
def CalculateDeviceMd5Sums(paths, device):
|
|
"""Calculates the MD5 sum value for all items in |paths|.
|
|
|
|
Directories are traversed recursively and the MD5 sum of each file found is
|
|
reported in the result.
|
|
|
|
Args:
|
|
paths: A list of device paths to md5sum.
|
|
Returns:
|
|
A dict mapping file paths to their respective md5sum checksums.
|
|
"""
|
|
if not paths:
|
|
return {}
|
|
|
|
if isinstance(paths, six.string_types):
|
|
paths = [paths]
|
|
paths = list(paths)
|
|
|
|
md5sum_dist_path = devil_env.config.FetchPath('md5sum_device', device=device)
|
|
|
|
if os.path.isdir(md5sum_dist_path):
|
|
md5sum_dist_bin_path = os.path.join(md5sum_dist_path, 'md5sum_bin')
|
|
else:
|
|
md5sum_dist_bin_path = md5sum_dist_path
|
|
|
|
if not os.path.exists(md5sum_dist_path):
|
|
raise IOError('File not built: %s' % md5sum_dist_path)
|
|
md5sum_file_size = os.path.getsize(md5sum_dist_bin_path)
|
|
|
|
# For better performance, make the script as small as possible to try and
|
|
# avoid needing to write to an intermediary file (which RunShellCommand will
|
|
# do if necessary).
|
|
md5sum_script = 'a=%s;' % MD5SUM_DEVICE_BIN_PATH
|
|
# Check if the binary is missing or has changed (using its file size as an
|
|
# indicator), and trigger a (re-)push via the exit code.
|
|
md5sum_script += '! [[ $(ls -l $a) = *%d* ]]&&exit 2;' % md5sum_file_size
|
|
# Make sure it can find libbase.so
|
|
md5sum_script += 'export LD_LIBRARY_PATH=%s;' % MD5SUM_DEVICE_LIB_PATH
|
|
for i in range(0, len(paths), _MAX_PATHS_PER_INVOCATION):
|
|
mem_file = io.BytesIO()
|
|
compressed = gzip.GzipFile(fileobj=mem_file, mode="wb")
|
|
data = ";".join(paths[i:i+_MAX_PATHS_PER_INVOCATION])
|
|
if six.PY3:
|
|
data = data.encode('utf-8')
|
|
compressed.write(data)
|
|
compressed.close()
|
|
compressed_paths = base64.b64encode(mem_file.getvalue())
|
|
md5sum_script += '$a -gz %s;' % compressed_paths
|
|
try:
|
|
out = device.RunShellCommand(
|
|
md5sum_script, shell=True, check_return=True, large_output=True)
|
|
except device_errors.AdbShellCommandFailedError as e:
|
|
# Push the binary only if it is found to not exist
|
|
# (faster than checking up-front).
|
|
if e.status == 2:
|
|
# If files were previously pushed as root (adbd running as root), trying
|
|
# to re-push as non-root causes the push command to report success, but
|
|
# actually fail. So, wipe the directory first.
|
|
device.RunShellCommand(['rm', '-rf', MD5SUM_DEVICE_LIB_PATH],
|
|
as_root=True,
|
|
check_return=True)
|
|
if os.path.isdir(md5sum_dist_path):
|
|
device.adb.Push(md5sum_dist_path, MD5SUM_DEVICE_LIB_PATH)
|
|
else:
|
|
mkdir_cmd = 'a=%s;[[ -e $a ]] || mkdir $a' % MD5SUM_DEVICE_LIB_PATH
|
|
device.RunShellCommand(mkdir_cmd, shell=True, check_return=True)
|
|
device.adb.Push(md5sum_dist_bin_path, MD5SUM_DEVICE_BIN_PATH)
|
|
|
|
out = device.RunShellCommand(
|
|
md5sum_script, shell=True, check_return=True, large_output=True)
|
|
else:
|
|
raise
|
|
|
|
return dict(zip(paths, [l for l in out if _STARTS_WITH_CHECKSUM_RE.match(l)]))
|