// Copyright 2019 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. #include "cast/streaming/receiver_session.h" #include #include #include #include #include "absl/strings/match.h" #include "absl/strings/numbers.h" #include "cast/common/channel/message_util.h" #include "cast/common/public/message_port.h" #include "cast/streaming/answer_messages.h" #include "cast/streaming/environment.h" #include "cast/streaming/message_fields.h" #include "cast/streaming/offer_messages.h" #include "cast/streaming/receiver.h" #include "cast/streaming/sender_message.h" #include "util/json/json_helpers.h" #include "util/osp_logging.h" namespace openscreen { namespace cast { namespace { template std::unique_ptr SelectStream( const std::vector& preferred_codecs, ReceiverSession::Client* client, const std::vector& offered_streams) { for (auto codec : preferred_codecs) { for (const Stream& offered_stream : offered_streams) { if (offered_stream.codec == codec && (offered_stream.stream.codec_parameter.empty() || client->SupportsCodecParameter( offered_stream.stream.codec_parameter))) { OSP_VLOG << "Selected " << CodecToString(codec) << " as codec for streaming"; return std::make_unique(offered_stream); } } } return nullptr; } MediaCapability ToCapability(AudioCodec codec) { switch (codec) { case AudioCodec::kAac: return MediaCapability::kAac; case AudioCodec::kOpus: return MediaCapability::kOpus; default: OSP_DLOG_FATAL << "Invalid audio codec: " << static_cast(codec); OSP_NOTREACHED(); } } MediaCapability ToCapability(VideoCodec codec) { switch (codec) { case VideoCodec::kVp8: return MediaCapability::kVp8; case VideoCodec::kVp9: return MediaCapability::kVp9; case VideoCodec::kH264: return MediaCapability::kH264; case VideoCodec::kHevc: return MediaCapability::kHevc; case VideoCodec::kAv1: return MediaCapability::kAv1; default: OSP_DLOG_FATAL << "Invalid video codec: " << static_cast(codec); OSP_NOTREACHED(); } } // Calculates whether any codecs present in |second| are not present in |first|. template bool IsMissingCodecs(const std::vector& first, const std::vector& second) { if (second.size() > first.size()) { return true; } for (auto codec : second) { if (std::find(first.begin(), first.end(), codec) == first.end()) { return true; } } return false; } // Calculates whether the limits defined by |first| are less restrictive than // those defined by |second|. // NOTE: These variables are intentionally passed by copy - the function will // mutate them. template bool HasLessRestrictiveLimits(std::vector first, std::vector second) { // Sort both vectors to allow for element-by-element comparison between the // two. All elements with |applies_to_all_codecs| set are sorted to the front. std::function sorter = [](const T& first, const T& second) { if (first.applies_to_all_codecs != second.applies_to_all_codecs) { return first.applies_to_all_codecs; } return static_cast(first.codec) < static_cast(second.codec); }; std::sort(first.begin(), first.end(), sorter); std::sort(second.begin(), second.end(), sorter); auto first_it = first.begin(); auto second_it = second.begin(); // |applies_to_all_codecs| is a special case, so handle that first. T fake_applies_to_all_codecs_struct; fake_applies_to_all_codecs_struct.applies_to_all_codecs = true; T* first_applies_to_all_codecs_struct = !first.empty() && first.front().applies_to_all_codecs ? &(*first_it++) : &fake_applies_to_all_codecs_struct; T* second_applies_to_all_codecs_struct = !second.empty() && second.front().applies_to_all_codecs ? &(*second_it++) : &fake_applies_to_all_codecs_struct; if (!first_applies_to_all_codecs_struct->IsSupersetOf( *second_applies_to_all_codecs_struct)) { return false; } // Now all elements of the vectors can be assumed to NOT have // |applies_to_all_codecs| set. So iterate through all codecs set in either // vector and check that the first has the less restrictive configuration set. while (first_it != first.end() || second_it != second.end()) { // Calculate the current codec to process, and whether each vector contains // an instance of this codec. decltype(T::codec) current_codec; bool use_first_fake = false; bool use_second_fake = false; if (first_it == first.end()) { current_codec = second_it->codec; use_first_fake = true; } else if (second_it == second.end()) { current_codec = first_it->codec; use_second_fake = true; } else { current_codec = std::min(first_it->codec, second_it->codec); use_first_fake = first_it->codec != current_codec; use_second_fake = second_it->codec != current_codec; } // Compare each vector's limit associated with this codec, or compare // against the default limits if no such codec limits are set. T fake_codecs_struct; fake_codecs_struct.codec = current_codec; T* first_codec_struct = use_first_fake ? &fake_codecs_struct : &(*first_it++); T* second_codec_struct = use_second_fake ? &fake_codecs_struct : &(*second_it++); OSP_DCHECK(!first_codec_struct->applies_to_all_codecs); OSP_DCHECK(!second_codec_struct->applies_to_all_codecs); if (!first_codec_struct->IsSupersetOf(*second_codec_struct)) { return false; } } return true; } } // namespace ReceiverSession::Client::~Client() = default; using RemotingPreferences = ReceiverSession::RemotingPreferences; using Preferences = ReceiverSession::Preferences; Preferences::Preferences() = default; Preferences::Preferences(std::vector video_codecs, std::vector audio_codecs) : video_codecs(std::move(video_codecs)), audio_codecs(std::move(audio_codecs)) {} Preferences::Preferences(std::vector video_codecs, std::vector audio_codecs, std::vector audio_limits, std::vector video_limits, std::unique_ptr description) : video_codecs(std::move(video_codecs)), audio_codecs(std::move(audio_codecs)), audio_limits(std::move(audio_limits)), video_limits(std::move(video_limits)), display_description(std::move(description)) {} Preferences::Preferences(Preferences&&) noexcept = default; Preferences& Preferences::operator=(Preferences&&) noexcept = default; Preferences::Preferences(const Preferences& other) { *this = other; } Preferences& Preferences::operator=(const Preferences& other) { video_codecs = other.video_codecs; audio_codecs = other.audio_codecs; audio_limits = other.audio_limits; video_limits = other.video_limits; if (other.display_description) { display_description = std::make_unique(*other.display_description); } if (other.remoting) { remoting = std::make_unique(*other.remoting); } return *this; } ReceiverSession::ReceiverSession(Client* const client, Environment* environment, MessagePort* message_port, Preferences preferences) : client_(client), environment_(environment), preferences_(std::move(preferences)), session_id_(MakeUniqueSessionId("streaming_receiver")), messenger_(message_port, session_id_, [this](Error error) { OSP_DLOG_WARN << "Got a session messenger error: " << error; client_->OnError(this, error); }), packet_router_(environment_) { OSP_DCHECK(client_); OSP_DCHECK(environment_); OSP_DCHECK(!std::any_of( preferences_.video_codecs.begin(), preferences_.video_codecs.end(), [](VideoCodec c) { return c == VideoCodec::kNotSpecified; })); OSP_DCHECK(!std::any_of( preferences_.audio_codecs.begin(), preferences_.audio_codecs.end(), [](AudioCodec c) { return c == AudioCodec::kNotSpecified; })); messenger_.SetHandler( SenderMessage::Type::kOffer, [this](SenderMessage message) { OnOffer(std::move(message)); }); messenger_.SetHandler(SenderMessage::Type::kGetCapabilities, [this](SenderMessage message) { OnCapabilitiesRequest(std::move(message)); }); messenger_.SetHandler(SenderMessage::Type::kRpc, [this](SenderMessage message) { this->OnRpcMessage(std::move(message)); }); environment_->SetSocketSubscriber(this); } ReceiverSession::~ReceiverSession() { ResetReceivers(Client::kEndOfSession); } void ReceiverSession::OnSocketReady() { if (pending_session_) { InitializeSession(*pending_session_); pending_session_.reset(); } } void ReceiverSession::OnSocketInvalid(Error error) { if (pending_session_) { SendErrorAnswerReply(pending_session_->sequence_number, "Failed to bind UDP socket"); pending_session_.reset(); } client_->OnError(this, Error(Error::Code::kSocketFailure, "The environment is invalid and should be replaced.")); } bool ReceiverSession::SessionProperties::IsValid() const { return (selected_audio || selected_video) && sequence_number >= 0; } void ReceiverSession::OnOffer(SenderMessage message) { // We just drop offers we can't respond to. Note that libcast senders will // always send a strictly positive sequence numbers, but zero is permitted // by the spec. if (message.sequence_number < 0) { OSP_DLOG_WARN << "Dropping offer with missing sequence number, can't respond"; return; } if (!message.valid) { SendErrorAnswerReply(message.sequence_number, "Failed to parse malformed OFFER"); client_->OnError(this, Error(Error::Code::kParameterInvalid, "Received invalid OFFER message")); return; } auto properties = std::make_unique(); properties->sequence_number = message.sequence_number; const Offer& offer = absl::get(message.body); if (offer.cast_mode == CastMode::kRemoting) { if (!preferences_.remoting) { SendErrorAnswerReply(message.sequence_number, "This receiver does not have remoting enabled."); return; } } properties->mode = offer.cast_mode; SelectStreams(offer, properties.get()); if (!properties->IsValid()) { SendErrorAnswerReply(message.sequence_number, "Failed to select any streams from OFFER"); return; } switch (environment_->socket_state()) { // If the environment is ready or in a bad state, we can respond // immediately. case Environment::SocketState::kInvalid: SendErrorAnswerReply(message.sequence_number, "UDP socket is closed, likely due to a bind error."); break; case Environment::SocketState::kReady: InitializeSession(*properties); break; // Else we need to store the properties we just created until we get a // ready or error event. case Environment::SocketState::kStarting: pending_session_ = std::move(properties); break; } } void ReceiverSession::OnCapabilitiesRequest(SenderMessage message) { if (message.sequence_number < 0) { OSP_DLOG_WARN << "Dropping offer with missing sequence number, can't respond"; return; } ReceiverMessage response{ ReceiverMessage::Type::kCapabilitiesResponse, message.sequence_number, true /* valid */ }; if (preferences_.remoting) { response.body = CreateRemotingCapabilityV2(); } else { response.valid = false; response.body = ReceiverError{static_cast(Error::Code::kRemotingNotSupported), "Remoting is not supported"}; } const Error result = messenger_.SendMessage(std::move(response)); if (!result.ok()) { client_->OnError(this, std::move(result)); } } void ReceiverSession::OnRpcMessage(SenderMessage message) { if (!message.valid) { OSP_DLOG_WARN << "Bad RPC message. This may or may not represent a serious problem."; return; } const auto& body = absl::get>(message.body); if (!rpc_messenger_) { OSP_DLOG_INFO << "Received an RPC message without having a messenger."; return; } rpc_messenger_->ProcessMessageFromRemote(body.data(), body.size()); } void ReceiverSession::SelectStreams(const Offer& offer, SessionProperties* properties) { if (offer.cast_mode == CastMode::kMirroring) { if (!offer.audio_streams.empty() && !preferences_.audio_codecs.empty()) { properties->selected_audio = SelectStream(preferences_.audio_codecs, client_, offer.audio_streams); } if (!offer.video_streams.empty() && !preferences_.video_codecs.empty()) { properties->selected_video = SelectStream(preferences_.video_codecs, client_, offer.video_streams); } } else { OSP_DCHECK(offer.cast_mode == CastMode::kRemoting); if (offer.audio_streams.size() == 1) { properties->selected_audio = std::make_unique(offer.audio_streams[0]); } if (offer.video_streams.size() == 1) { properties->selected_video = std::make_unique(offer.video_streams[0]); } } } void ReceiverSession::InitializeSession(const SessionProperties& properties) { Answer answer = ConstructAnswer(properties); if (!answer.IsValid()) { // If the answer message is invalid, there is no point in setting up a // negotiation because the sender won't be able to connect to it. SendErrorAnswerReply(properties.sequence_number, "Failed to construct an ANSWER message"); return; } // Only spawn receivers if we know we have a valid answer message. ConfiguredReceivers receivers = SpawnReceivers(properties); if (properties.mode == CastMode::kMirroring) { client_->OnNegotiated(this, std::move(receivers)); } else { // TODO(jophba): cleanup sequence number usage. rpc_messenger_ = std::make_unique([this](std::vector message) { Error error = this->messenger_.SendMessage( ReceiverMessage{ReceiverMessage::Type::kRpc, -1, true /* valid */, std::move(message)}); if (!error.ok()) { OSP_LOG_WARN << "Failed to send RPC message: " << error; } }); client_->OnRemotingNegotiated( this, RemotingNegotiation{std::move(receivers), rpc_messenger_.get()}); } const Error result = messenger_.SendMessage(ReceiverMessage{ ReceiverMessage::Type::kAnswer, properties.sequence_number, true /* valid */, std::move(answer)}); if (!result.ok()) { client_->OnError(this, std::move(result)); } } std::unique_ptr ReceiverSession::ConstructReceiver( const Stream& stream) { // Session config is currently only for mirroring. SessionConfig config = {stream.ssrc, stream.ssrc + 1, stream.rtp_timebase, stream.channels, stream.target_delay, stream.aes_key, stream.aes_iv_mask, /* is_pli_enabled */ true}; return std::make_unique(environment_, &packet_router_, std::move(config)); } ReceiverSession::ConfiguredReceivers ReceiverSession::SpawnReceivers( const SessionProperties& properties) { OSP_DCHECK(properties.IsValid()); ResetReceivers(Client::kRenegotiated); AudioCaptureConfig audio_config; if (properties.selected_audio) { current_audio_receiver_ = ConstructReceiver(properties.selected_audio->stream); audio_config = AudioCaptureConfig{properties.selected_audio->codec, properties.selected_audio->stream.channels, properties.selected_audio->bit_rate, properties.selected_audio->stream.rtp_timebase, properties.selected_audio->stream.target_delay, properties.selected_audio->stream.codec_parameter}; } VideoCaptureConfig video_config; if (properties.selected_video) { current_video_receiver_ = ConstructReceiver(properties.selected_video->stream); video_config = VideoCaptureConfig{properties.selected_video->codec, properties.selected_video->max_frame_rate, properties.selected_video->max_bit_rate, properties.selected_video->resolutions, properties.selected_video->stream.target_delay, properties.selected_video->stream.codec_parameter}; } return ConfiguredReceivers{ current_audio_receiver_.get(), std::move(audio_config), current_video_receiver_.get(), std::move(video_config)}; } void ReceiverSession::ResetReceivers(Client::ReceiversDestroyingReason reason) { if (current_video_receiver_ || current_audio_receiver_) { client_->OnReceiversDestroying(this, reason); current_audio_receiver_.reset(); current_video_receiver_.reset(); rpc_messenger_.reset(); } } Answer ReceiverSession::ConstructAnswer(const SessionProperties& properties) { OSP_DCHECK(properties.IsValid()); std::vector stream_indexes; std::vector stream_ssrcs; Constraints constraints; if (properties.selected_audio) { stream_indexes.push_back(properties.selected_audio->stream.index); stream_ssrcs.push_back(properties.selected_audio->stream.ssrc + 1); for (const auto& limit : preferences_.audio_limits) { if (limit.codec == properties.selected_audio->codec || limit.applies_to_all_codecs) { constraints.audio = AudioConstraints{ limit.max_sample_rate, limit.max_channels, limit.min_bit_rate, limit.max_bit_rate, limit.max_delay, }; break; } } } if (properties.selected_video) { stream_indexes.push_back(properties.selected_video->stream.index); stream_ssrcs.push_back(properties.selected_video->stream.ssrc + 1); for (const auto& limit : preferences_.video_limits) { if (limit.codec == properties.selected_video->codec || limit.applies_to_all_codecs) { constraints.video = VideoConstraints{ limit.max_pixels_per_second, absl::nullopt, /* min dimensions */ limit.max_dimensions, limit.min_bit_rate, limit.max_bit_rate, limit.max_delay, }; break; } } } absl::optional display; if (preferences_.display_description) { const auto* d = preferences_.display_description.get(); display = DisplayDescription{d->dimensions, absl::nullopt, d->can_scale_content ? AspectRatioConstraint::kVariable : AspectRatioConstraint::kFixed}; } // Only set the constraints in the answer if they are valid (meaning we // successfully found limits above). absl::optional answer_constraints; if (constraints.IsValid()) { answer_constraints = std::move(constraints); } return Answer{environment_->GetBoundLocalEndpoint().port, std::move(stream_indexes), std::move(stream_ssrcs), answer_constraints, std::move(display)}; } ReceiverCapability ReceiverSession::CreateRemotingCapabilityV2() { // If we don't support remoting, there is no reason to respond to // capability requests--they are not used for mirroring. OSP_DCHECK(preferences_.remoting); ReceiverCapability capability; capability.remoting_version = kSupportedRemotingVersion; for (const AudioCodec& codec : preferences_.audio_codecs) { capability.media_capabilities.push_back(ToCapability(codec)); } for (const VideoCodec& codec : preferences_.video_codecs) { capability.media_capabilities.push_back(ToCapability(codec)); } if (preferences_.remoting->supports_chrome_audio_codecs) { capability.media_capabilities.push_back(MediaCapability::kAudio); } if (preferences_.remoting->supports_4k) { capability.media_capabilities.push_back(MediaCapability::k4k); } return capability; } void ReceiverSession::SendErrorAnswerReply(int sequence_number, const char* message) { const Error error(Error::Code::kParseError, message); OSP_DLOG_WARN << message; const Error result = messenger_.SendMessage(ReceiverMessage{ ReceiverMessage::Type::kAnswer, sequence_number, false /* valid */, ReceiverError{static_cast(Error::Code::kParseError), message}}); if (!result.ok()) { client_->OnError(this, std::move(result)); } } bool ReceiverSession::VideoLimits::IsSupersetOf( const ReceiverSession::VideoLimits& second) const { return (applies_to_all_codecs == second.applies_to_all_codecs) && (applies_to_all_codecs || codec == second.codec) && (max_pixels_per_second >= second.max_pixels_per_second) && (min_bit_rate <= second.min_bit_rate) && (max_bit_rate >= second.max_bit_rate) && (max_delay >= second.max_delay) && (max_dimensions.IsSupersetOf(second.max_dimensions)); } bool ReceiverSession::AudioLimits::IsSupersetOf( const ReceiverSession::AudioLimits& second) const { return (applies_to_all_codecs == second.applies_to_all_codecs) && (applies_to_all_codecs || codec == second.codec) && (max_sample_rate >= second.max_sample_rate) && (max_channels >= second.max_channels) && (min_bit_rate <= second.min_bit_rate) && (max_bit_rate >= second.max_bit_rate) && (max_delay >= second.max_delay); } bool ReceiverSession::Display::IsSupersetOf( const ReceiverSession::Display& other) const { return dimensions.IsSupersetOf(other.dimensions) && (can_scale_content || !other.can_scale_content); } bool ReceiverSession::RemotingPreferences::IsSupersetOf( const ReceiverSession::RemotingPreferences& other) const { return (supports_chrome_audio_codecs || !other.supports_chrome_audio_codecs) && (supports_4k || !other.supports_4k); } bool ReceiverSession::Preferences::IsSupersetOf( const ReceiverSession::Preferences& other) const { // Check simple cases first. if ((!!display_description != !!other.display_description) || (display_description && !display_description->IsSupersetOf(*other.display_description))) { return false; } else if (other.remoting && (!remoting || !remoting->IsSupersetOf(*other.remoting))) { return false; } // Then check set codecs. if (IsMissingCodecs(video_codecs, other.video_codecs) || IsMissingCodecs(audio_codecs, other.audio_codecs)) { return false; } // Then check limits. Do this last because it's the most resource intensive to // check. return HasLessRestrictiveLimits(video_limits, other.video_limits) && HasLessRestrictiveLimits(audio_limits, other.audio_limits); } } // namespace cast } // namespace openscreen