Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/video/rtp_streams_synchronizer.h"
12 :
13 : #include "webrtc/base/checks.h"
14 : #include "webrtc/base/logging.h"
15 : #include "webrtc/base/timeutils.h"
16 : #include "webrtc/base/trace_event.h"
17 : #include "webrtc/modules/rtp_rtcp/include/rtp_receiver.h"
18 : #include "webrtc/modules/rtp_rtcp/include/rtp_rtcp.h"
19 : #include "webrtc/modules/video_coding/video_coding_impl.h"
20 : #include "webrtc/system_wrappers/include/clock.h"
21 : #include "webrtc/video/stream_synchronization.h"
22 : #include "webrtc/video_frame.h"
23 : #include "webrtc/voice_engine/include/voe_video_sync.h"
24 :
25 : namespace webrtc {
26 : namespace {
27 0 : bool UpdateMeasurements(StreamSynchronization::Measurements* stream,
28 : RtpRtcp* rtp_rtcp,
29 : RtpReceiver* receiver) {
30 0 : if (!receiver->Timestamp(&stream->latest_timestamp))
31 0 : return false;
32 0 : if (!receiver->LastReceivedTimeMs(&stream->latest_receive_time_ms))
33 0 : return false;
34 :
35 0 : uint32_t ntp_secs = 0;
36 0 : uint32_t ntp_frac = 0;
37 0 : uint32_t rtp_timestamp = 0;
38 0 : if (rtp_rtcp->RemoteNTP(&ntp_secs, &ntp_frac, nullptr, nullptr,
39 0 : &rtp_timestamp) != 0) {
40 0 : return false;
41 : }
42 :
43 0 : bool new_rtcp_sr = false;
44 0 : if (!stream->rtp_to_ntp.UpdateMeasurements(ntp_secs, ntp_frac, rtp_timestamp,
45 : &new_rtcp_sr)) {
46 0 : return false;
47 : }
48 :
49 0 : return true;
50 : }
51 : } // namespace
52 :
53 0 : RtpStreamsSynchronizer::RtpStreamsSynchronizer(
54 : vcm::VideoReceiver* video_receiver,
55 0 : RtpStreamReceiver* rtp_stream_receiver)
56 0 : : clock_(Clock::GetRealTimeClock()),
57 : video_receiver_(video_receiver),
58 0 : video_rtp_receiver_(rtp_stream_receiver->GetRtpReceiver()),
59 0 : video_rtp_rtcp_(rtp_stream_receiver->rtp_rtcp()),
60 : voe_channel_id_(-1),
61 : voe_sync_interface_(nullptr),
62 : audio_rtp_receiver_(nullptr),
63 : audio_rtp_rtcp_(nullptr),
64 : sync_(),
65 0 : last_sync_time_(rtc::TimeNanos()) {
66 0 : process_thread_checker_.DetachFromThread();
67 0 : }
68 :
69 0 : void RtpStreamsSynchronizer::ConfigureSync(int voe_channel_id,
70 : VoEVideoSync* voe_sync_interface) {
71 0 : if (voe_channel_id != -1)
72 0 : RTC_DCHECK(voe_sync_interface);
73 :
74 0 : rtc::CritScope lock(&crit_);
75 0 : if (voe_channel_id_ == voe_channel_id &&
76 0 : voe_sync_interface_ == voe_sync_interface) {
77 : // This prevents expensive no-ops.
78 0 : return;
79 : }
80 0 : voe_channel_id_ = voe_channel_id;
81 0 : voe_sync_interface_ = voe_sync_interface;
82 :
83 0 : audio_rtp_rtcp_ = nullptr;
84 0 : audio_rtp_receiver_ = nullptr;
85 0 : sync_.reset(nullptr);
86 :
87 0 : if (voe_channel_id_ != -1) {
88 0 : voe_sync_interface_->GetRtpRtcp(voe_channel_id_, &audio_rtp_rtcp_,
89 0 : &audio_rtp_receiver_);
90 0 : RTC_DCHECK(audio_rtp_rtcp_);
91 0 : RTC_DCHECK(audio_rtp_receiver_);
92 0 : sync_.reset(new StreamSynchronization(video_rtp_rtcp_->SSRC(),
93 0 : voe_channel_id_));
94 : }
95 : }
96 :
97 0 : int64_t RtpStreamsSynchronizer::TimeUntilNextProcess() {
98 0 : RTC_DCHECK_RUN_ON(&process_thread_checker_);
99 0 : const int64_t kSyncIntervalMs = 1000;
100 : return kSyncIntervalMs -
101 0 : (rtc::TimeNanos() - last_sync_time_) / rtc::kNumNanosecsPerMillisec;
102 : }
103 :
104 0 : void RtpStreamsSynchronizer::Process() {
105 0 : RTC_DCHECK_RUN_ON(&process_thread_checker_);
106 :
107 0 : const int current_video_delay_ms = video_receiver_->Delay();
108 0 : last_sync_time_ = rtc::TimeNanos();
109 :
110 0 : rtc::CritScope lock(&crit_);
111 0 : if (voe_channel_id_ == -1) {
112 0 : return;
113 : }
114 0 : RTC_DCHECK(voe_sync_interface_);
115 0 : RTC_DCHECK(sync_.get());
116 :
117 0 : int audio_jitter_buffer_delay_ms = 0;
118 0 : int playout_buffer_delay_ms = 0;
119 0 : int avsync_delay_ms = 0;
120 0 : if (voe_sync_interface_->GetDelayEstimate(voe_channel_id_,
121 : &audio_jitter_buffer_delay_ms,
122 : &playout_buffer_delay_ms,
123 0 : &avsync_delay_ms) != 0) {
124 0 : return;
125 : }
126 0 : const int current_audio_delay_ms = audio_jitter_buffer_delay_ms +
127 0 : playout_buffer_delay_ms;
128 :
129 0 : int64_t last_video_receive_ms = video_measurement_.latest_receive_time_ms;
130 0 : if (!UpdateMeasurements(&video_measurement_, video_rtp_rtcp_,
131 0 : video_rtp_receiver_)) {
132 0 : return;
133 : }
134 :
135 0 : if (!UpdateMeasurements(&audio_measurement_, audio_rtp_rtcp_,
136 : audio_rtp_receiver_)) {
137 0 : return;
138 : }
139 :
140 0 : if (last_video_receive_ms == video_measurement_.latest_receive_time_ms) {
141 : // No new video packet has been received since last update.
142 0 : return;
143 : }
144 :
145 : int relative_delay_ms;
146 : // Calculate how much later or earlier the audio stream is compared to video.
147 0 : if (!sync_->ComputeRelativeDelay(audio_measurement_, video_measurement_,
148 0 : &relative_delay_ms)) {
149 0 : return;
150 : }
151 :
152 0 : TRACE_COUNTER1("webrtc", "SyncCurrentVideoDelay", current_video_delay_ms);
153 0 : TRACE_COUNTER1("webrtc", "SyncCurrentAudioDelay", current_audio_delay_ms);
154 0 : TRACE_COUNTER1("webrtc", "SyncRelativeDelay", relative_delay_ms);
155 0 : int target_audio_delay_ms = 0;
156 0 : int target_video_delay_ms = current_video_delay_ms;
157 : // Calculate the necessary extra audio delay and desired total video
158 : // delay to get the streams in sync.
159 0 : if (!sync_->ComputeDelays(relative_delay_ms,
160 : current_audio_delay_ms,
161 : &target_audio_delay_ms,
162 : &target_video_delay_ms)) {
163 0 : return;
164 : }
165 :
166 0 : if (voe_sync_interface_->SetMinimumPlayoutDelay(
167 0 : voe_channel_id_, target_audio_delay_ms) == -1) {
168 0 : LOG(LS_ERROR) << "Error setting voice delay.";
169 : }
170 0 : video_receiver_->SetMinimumPlayoutDelay(target_video_delay_ms);
171 : }
172 :
173 0 : bool RtpStreamsSynchronizer::GetStreamSyncOffsetInMs(
174 : const VideoFrame& frame,
175 : int64_t* stream_offset_ms,
176 : double* estimated_freq_khz) const {
177 0 : rtc::CritScope lock(&crit_);
178 0 : if (voe_channel_id_ == -1)
179 0 : return false;
180 :
181 0 : uint32_t playout_timestamp = 0;
182 0 : if (voe_sync_interface_->GetPlayoutTimestamp(voe_channel_id_,
183 0 : playout_timestamp) != 0) {
184 0 : return false;
185 : }
186 :
187 : int64_t latest_audio_ntp;
188 0 : if (!audio_measurement_.rtp_to_ntp.Estimate(playout_timestamp,
189 : &latest_audio_ntp)) {
190 0 : return false;
191 : }
192 :
193 : int64_t latest_video_ntp;
194 0 : if (!video_measurement_.rtp_to_ntp.Estimate(frame.timestamp(),
195 : &latest_video_ntp)) {
196 0 : return false;
197 : }
198 :
199 : int64_t time_to_render_ms =
200 0 : frame.render_time_ms() - clock_->TimeInMilliseconds();
201 0 : if (time_to_render_ms > 0)
202 0 : latest_video_ntp += time_to_render_ms;
203 :
204 0 : *stream_offset_ms = latest_audio_ntp - latest_video_ntp;
205 0 : *estimated_freq_khz = video_measurement_.rtp_to_ntp.params().frequency_khz;
206 0 : return true;
207 : }
208 :
209 : } // namespace webrtc
|
