453 lines
17 KiB
Python
453 lines
17 KiB
Python
# Lint as: python2, python3
|
|
# Copyright 2016 The Chromium OS Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""This module provides the utilities for avsync_probe's data processing.
|
|
|
|
We will get a lot of raw data from the avsync_probe.Capture(). One data per
|
|
millisecond.
|
|
AVSyncProbeDataParser will help to transform the raw data to more readable
|
|
formats. It also helps to calculate the audio/video sync timing if the
|
|
sound_interval_frames parameter is not None.
|
|
|
|
Example:
|
|
capture_data = avsync_probe.Capture(12)
|
|
parser = avsync_probe_utils.AVSyncProbeDataParser(self.resultsdir,
|
|
capture_data, 30)
|
|
|
|
# Use the following attributes to access data. They can be referenced in
|
|
# AVSyncProbeDataParser Class.
|
|
parser.video_duration_average
|
|
parser.video_duration_std
|
|
parser.sync_duration_averag
|
|
parser.sync_duration_std
|
|
parser.cumulative_frame_count
|
|
parser.dropped_frame_count
|
|
parser.corrupted_frame_count
|
|
parser.binarize_data
|
|
parser.audio_events
|
|
parser.video_events
|
|
|
|
"""
|
|
|
|
from __future__ import absolute_import
|
|
from __future__ import division
|
|
from __future__ import print_function
|
|
import collections
|
|
import logging
|
|
import math
|
|
import os
|
|
import sys
|
|
from six.moves import range
|
|
|
|
|
|
# Indices for binarize_data, audio_events and video_events.
|
|
TIME_INDEX = 0
|
|
VIDEO_INDEX = 1
|
|
AUDIO_INDEX = 2
|
|
# This index is used for video_events and audio_events.
|
|
# The slot contains the time difference to the previous event.
|
|
TIME_DIFF_INDEX = 3
|
|
|
|
# SyncResult namedtuple of audio and video frame.
|
|
# time_delay < 0 means that audio comes out first.
|
|
SyncResult = collections.namedtuple(
|
|
'SynResult', ['video_time', 'audio_time', 'time_delay'])
|
|
|
|
|
|
class GrayCode(object):
|
|
"""Converts bit patterns between binary and Gray code.
|
|
|
|
The bit patterns of Gray code values are packed into an int value.
|
|
For example, 4 is "110" in Gray code, which reads "6" when interpreted
|
|
as binary.
|
|
See "https://en.wikipedia.org/wiki/Gray_code"
|
|
|
|
"""
|
|
|
|
@staticmethod
|
|
def binary_to_gray(binary):
|
|
"""Binary code to gray code.
|
|
|
|
@param binary: Binary code.
|
|
@return: gray code.
|
|
|
|
"""
|
|
return binary ^ (binary >> 1)
|
|
|
|
@staticmethod
|
|
def gray_to_binary(gray):
|
|
"""Gray code to binary code.
|
|
|
|
@param gray: Gray code.
|
|
@return: binary code.
|
|
|
|
"""
|
|
result = gray
|
|
result ^= (result >> 16)
|
|
result ^= (result >> 8)
|
|
result ^= (result >> 4)
|
|
result ^= (result >> 2)
|
|
result ^= (result >> 1)
|
|
return result
|
|
|
|
|
|
class HysteresisSwitch(object):
|
|
"""
|
|
Iteratively binarizes input sequence using hysteresis comparator with a
|
|
pair of fixed thresholds.
|
|
|
|
Hysteresis means to use 2 different thresholds
|
|
for activating and de-activating output. It is often used for thresholding
|
|
time-series signal while reducing small noise in the input.
|
|
|
|
Note that the low threshold is exclusive but the high threshold is
|
|
inclusive.
|
|
When the same values were applied for the both, the object works as a
|
|
non-hysteresis switch.
|
|
(i.e. equivalent to the >= operator).
|
|
|
|
"""
|
|
|
|
def __init__(self, low_threshold, high_threshold, init_state):
|
|
"""Init HysteresisSwitch class.
|
|
|
|
@param low_threshold: The threshold value to deactivate the output.
|
|
The comparison is exclusive.
|
|
@param high_threshold: The threshold value to activate the output.
|
|
The comparison is inclusive.
|
|
@param init_state: True or False of the switch initial state.
|
|
|
|
"""
|
|
if low_threshold > high_threshold:
|
|
raise Exception('Low threshold %d exceeds the high threshold %d',
|
|
low_threshold, high_threshold)
|
|
self._low_threshold = low_threshold
|
|
self._high_threshold = high_threshold
|
|
self._last_state = init_state
|
|
|
|
def adjust_state(self, value):
|
|
"""Updates the state of the switch by the input value and returns the
|
|
result.
|
|
|
|
@param value: value for updating.
|
|
@return the state of the switch.
|
|
|
|
"""
|
|
if value < self._low_threshold:
|
|
self._last_state = False
|
|
|
|
if value >= self._high_threshold:
|
|
self._last_state = True
|
|
|
|
return self._last_state
|
|
|
|
|
|
class AVSyncProbeDataParser(object):
|
|
""" Digital information extraction from the raw sensor data sequence.
|
|
|
|
This class will transform the raw data to easier understand formats.
|
|
|
|
Attributes:
|
|
binarize_data: Transer the raw data to [Time, video code, is_audio].
|
|
video code is from 0-7 repeatedly.
|
|
video_events: Events of video frame.
|
|
audio_events: Events of when audio happens.
|
|
video_duration_average: (ms) The average duration during video frames.
|
|
video_duration_std: Standard deviation of the video_duration_average.
|
|
sync_duration_average: (ms) The average duration for audio/video sync.
|
|
sync_duration_std: Standard deviation of sync_duration_average.
|
|
cumulative_frame_count: Number of total video frames.
|
|
dropped_frame_count: Total dropped video frames.
|
|
corrupted_frame_count: Total corrupted video frames.
|
|
|
|
"""
|
|
# Thresholds for hysteresis binarization of input signals.
|
|
# Relative to the minumum (0.0) and maximum (1.0) values of the value range
|
|
# of each input signal.
|
|
_NORMALIZED_LOW_THRESHOLD = 0.6
|
|
_NORMALIZED_HIGH_THRESHOLD = 0.7
|
|
|
|
_VIDEO_CODE_CYCLE = (1 << 3)
|
|
|
|
def __init__(self, log_dir, capture_raw_data, video_fps,
|
|
sound_interval_frames=None):
|
|
"""Inits AVSyncProbeDataParser class.
|
|
|
|
@param log_dir: Directory for dumping each events' contents.
|
|
@param capture_raw_data: Raw data from avsync_probe device.
|
|
A list contains the list values of [timestamp, video0, video1,
|
|
video2, audio].
|
|
@param video_fps: Video frames per second. Used to know if the video
|
|
frame is dropoped or just corrupted.
|
|
@param sound_interval_frames: The period of sound (beep) in the number
|
|
of video frames. This class will help to calculate audio/video
|
|
sync stats if sound_interval_frames is not None.
|
|
|
|
"""
|
|
self.video_duration_average = None
|
|
self.video_duration_std = None
|
|
self.sync_duration_average = None
|
|
self.sync_duration_std = None
|
|
self.cumulative_frame_count = None
|
|
self.dropped_frame_count = None
|
|
|
|
self._log_dir = log_dir
|
|
self._raw_data = capture_raw_data
|
|
# Translate to millisecond for each video frame.
|
|
self._video_duration = 1000 // video_fps
|
|
self._sound_interval_frames = sound_interval_frames
|
|
self._log_list_data_to_file('raw.txt', capture_raw_data)
|
|
|
|
self.binarize_data = self._binarize_raw_data()
|
|
# we need to get audio events before remove video preamble frames.
|
|
# Because audio event may appear before the preamble frame, if we
|
|
# remove the preamble frames first, we will lost the audio event.
|
|
self.audio_events = self._detect_audio_events()
|
|
self._remove_video_preamble()
|
|
self.video_events = self._detect_video_events()
|
|
self._analyze_events()
|
|
self._calculate_statistics_report()
|
|
|
|
def _log_list_data_to_file(self, filename, data):
|
|
"""Log the list data to file.
|
|
|
|
It will log under self._log_dir directory.
|
|
|
|
@param filename: The file name.
|
|
@data: Data for logging.
|
|
|
|
"""
|
|
filepath = os.path.join(self._log_dir, filename)
|
|
with open(filepath, 'w') as f:
|
|
for v in data:
|
|
f.write('%s\n' % str(v))
|
|
|
|
def _get_hysteresis_switch(self, index):
|
|
"""Get HysteresisSwitch by the raw data.
|
|
|
|
@param index: The index of self._raw_data's element.
|
|
@return: HysteresisSwitch instance by the value of the raw data.
|
|
|
|
"""
|
|
max_value = max(x[index] for x in self._raw_data)
|
|
min_value = min(x[index] for x in self._raw_data)
|
|
scale = max_value - min_value
|
|
logging.info('index %d, max %d, min %d, scale %d', index, max_value,
|
|
min_value, scale)
|
|
return HysteresisSwitch(
|
|
min_value + scale * self._NORMALIZED_LOW_THRESHOLD,
|
|
min_value + scale * self._NORMALIZED_HIGH_THRESHOLD,
|
|
False)
|
|
|
|
def _binarize_raw_data(self):
|
|
"""Conducts adaptive thresholding and decoding embedded frame codes.
|
|
|
|
Sensors[0] is timestamp.
|
|
Sensors[1-3] are photo transistors, which outputs lower value for
|
|
brighter light(=white pixels on screen). These are used to detect black
|
|
and white pattern on the screen, and decoded as an integer code.
|
|
|
|
The final channel is for audio input, which outputs higher voltage for
|
|
larger sound volume. This will be used for detecting beep sounds added
|
|
to the video.
|
|
|
|
@return Decoded frame codes list for all the input frames. Each entry
|
|
contains [Timestamp, video code, is_audio].
|
|
|
|
"""
|
|
decoded_data = []
|
|
|
|
hystersis_switch = []
|
|
for i in range(5):
|
|
hystersis_switch.append(self._get_hysteresis_switch(i))
|
|
|
|
for data in self._raw_data:
|
|
code = 0
|
|
# Decode black-and-white pattern on video.
|
|
# There are 3 black or white boxes sensed by the sensors.
|
|
# Each square represents a single bit (white = 1, black = 0) coding
|
|
# an integer in Gray code.
|
|
for i in range(1, 4):
|
|
# Lower sensor value for brighter light(square painted white).
|
|
is_white = not hystersis_switch[i].adjust_state(data[i])
|
|
if is_white:
|
|
code |= (1 << (i - 1))
|
|
code = GrayCode.gray_to_binary(code)
|
|
# The final channel is sound signal. Higher sensor value for
|
|
# higher sound level.
|
|
sound = hystersis_switch[4].adjust_state(data[4])
|
|
decoded_data.append([data[0], code, sound])
|
|
|
|
self._log_list_data_to_file('binarize_raw.txt', decoded_data)
|
|
return decoded_data
|
|
|
|
def _remove_video_preamble(self):
|
|
"""Remove preamble video frames of self.binarize_data."""
|
|
# find preamble frame (code = 0)
|
|
index = next(i for i, v in enumerate(self.binarize_data)
|
|
if v[VIDEO_INDEX] == 0)
|
|
self.binarize_data = self.binarize_data[index:]
|
|
|
|
# skip preamble frame (code = 0)
|
|
index = next(i for i, v in enumerate(self.binarize_data)
|
|
if v[VIDEO_INDEX] != 0)
|
|
self.binarize_data = self.binarize_data[index:]
|
|
|
|
def _detect_events(self, detect_condition):
|
|
"""Detects events from the binarize data sequence by the
|
|
detect_condition.
|
|
|
|
@param detect_condition: callback function for checking event happens.
|
|
This API will pass index and element of binarize_data to the
|
|
callback function.
|
|
|
|
@return: The list of events. It's the same as the binarize_data and add
|
|
additional time_difference information.
|
|
|
|
"""
|
|
detected_events = []
|
|
previous_time = self.binarize_data[0][TIME_INDEX]
|
|
for i, v in enumerate(self.binarize_data):
|
|
if (detect_condition(i, v)):
|
|
time = v[TIME_INDEX]
|
|
time_difference = time - previous_time
|
|
# Copy a new instance here, because we will append time
|
|
# difference.
|
|
event = list(v)
|
|
event.append(time_difference)
|
|
detected_events.append(event)
|
|
previous_time = time
|
|
|
|
return detected_events
|
|
|
|
def _detect_audio_events(self):
|
|
"""Detects the audio start frame from the binarize data sequence.
|
|
|
|
@return: The list of Audio events. It's the same as the binarize_data
|
|
and add additional time_difference information.
|
|
|
|
"""
|
|
# Only check the first audio happen event.
|
|
detected_events = self._detect_events(
|
|
lambda i, v: (v[AUDIO_INDEX] and not
|
|
self.binarize_data[i - 1][AUDIO_INDEX]))
|
|
|
|
self._log_list_data_to_file('audio_events.txt', detected_events)
|
|
return detected_events
|
|
|
|
def _detect_video_events(self):
|
|
"""Detects the video frame from the binarize data sequence.
|
|
|
|
@return: The list of Video events. It's the same as the binarize_data
|
|
and add additional time_difference information.
|
|
|
|
"""
|
|
# remove duplicate frames. (frames in transition state.)
|
|
detected_events = self._detect_events(
|
|
lambda i, v: (v[VIDEO_INDEX] !=
|
|
self.binarize_data[i - 1][VIDEO_INDEX]))
|
|
|
|
self._log_list_data_to_file('video_events.txt', detected_events)
|
|
return detected_events
|
|
|
|
def _match_sync(self, video_time):
|
|
"""Match the audio/video sync timing.
|
|
|
|
This function will find the closest sound in the audio_events to the
|
|
video_time and returns a audio/video sync tuple.
|
|
|
|
@param video_time: the time of the video which have sound.
|
|
@return A SyncResult namedtuple containing:
|
|
- timestamp of the video frame which should have audio.
|
|
- timestamp of nearest audio frame.
|
|
- time delay between audio and video frame.
|
|
|
|
"""
|
|
closest_difference = sys.maxsize
|
|
audio_time = 0
|
|
for audio_event in self.audio_events:
|
|
difference = audio_event[TIME_INDEX] - video_time
|
|
if abs(difference) < abs(closest_difference):
|
|
closest_difference = difference
|
|
audio_time = audio_event[TIME_INDEX]
|
|
return SyncResult(video_time, audio_time, closest_difference)
|
|
|
|
def _calculate_statistics(self, data):
|
|
"""Calculate average and standard deviation of the list data.
|
|
|
|
@param data: The list of values to be calcualted.
|
|
@return: An tuple with (average, standard_deviation)
|
|
|
|
"""
|
|
if not data:
|
|
return (None, None)
|
|
|
|
total = sum(data)
|
|
average = total / len(data)
|
|
variance = sum((v - average)**2 for v in data) / len(data)
|
|
standard_deviation = math.sqrt(variance)
|
|
return (average, standard_deviation)
|
|
|
|
def _analyze_events(self):
|
|
"""Analyze audio/video events.
|
|
|
|
This function will analyze video frame status and audio/video sync
|
|
status.
|
|
|
|
"""
|
|
sound_interval_frames = self._sound_interval_frames
|
|
current_code = 0
|
|
cumulative_frame_count = 0
|
|
dropped_frame_count = 0
|
|
corrupted_frame_count = 0
|
|
sync_events = []
|
|
|
|
for v in self.video_events:
|
|
code = v[VIDEO_INDEX]
|
|
time = v[TIME_INDEX]
|
|
frame_diff = code - current_code
|
|
# Get difference of the codes. # The code is between 0 - 7.
|
|
if frame_diff < 0:
|
|
frame_diff += self._VIDEO_CODE_CYCLE
|
|
|
|
if frame_diff != 1:
|
|
# Check if we dropped frame or just got corrupted frame.
|
|
# Treat the frame as corrupted frame if the frame duration is
|
|
# less than 2 video frame duration.
|
|
if v[TIME_DIFF_INDEX] < 2 * self._video_duration:
|
|
logging.warn('Corrupted frame near %s', str(v))
|
|
# Correct the code.
|
|
code = current_code + 1
|
|
corrupted_frame_count += 1
|
|
frame_diff = 1
|
|
else:
|
|
logging.warn('Dropped frame near %s', str(v))
|
|
dropped_frame_count += (frame_diff - 1)
|
|
|
|
cumulative_frame_count += frame_diff
|
|
|
|
if sound_interval_frames is not None:
|
|
# This frame corresponds to a sound.
|
|
if cumulative_frame_count % sound_interval_frames == 1:
|
|
sync_events.append(self._match_sync(time))
|
|
|
|
current_code = code
|
|
self.cumulative_frame_count = cumulative_frame_count
|
|
self.dropped_frame_count = dropped_frame_count
|
|
self.corrupted_frame_count = corrupted_frame_count
|
|
self._sync_events = sync_events
|
|
self._log_list_data_to_file('sync.txt', sync_events)
|
|
|
|
def _calculate_statistics_report(self):
|
|
"""Calculates statistics report."""
|
|
video_duration_average, video_duration_std = self._calculate_statistics(
|
|
[v[TIME_DIFF_INDEX] for v in self.video_events])
|
|
sync_duration_average, sync_duration_std = self._calculate_statistics(
|
|
[v.time_delay for v in self._sync_events])
|
|
self.video_duration_average = video_duration_average
|
|
self.video_duration_std = video_duration_std
|
|
self.sync_duration_average = sync_duration_average
|
|
self.sync_duration_std = sync_duration_std
|