Line data Source code
1 : /*
2 : * Copyright (c) 2010 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #include "webrtc/media/base/videoadapter.h"
12 :
13 : #include <algorithm>
14 : #include <cmath>
15 : #include <cstdlib>
16 : #include <limits>
17 :
18 : #include "webrtc/base/arraysize.h"
19 : #include "webrtc/base/checks.h"
20 : #include "webrtc/base/logging.h"
21 : #include "webrtc/base/optional.h"
22 : #include "webrtc/media/base/mediaconstants.h"
23 : #include "webrtc/media/base/videocommon.h"
24 :
25 : namespace {
26 : struct Fraction {
27 : int numerator;
28 : int denominator;
29 : };
30 :
31 : // Round |value_to_round| to a multiple of |multiple|. Prefer rounding upwards,
32 : // but never more than |max_value|.
33 0 : int roundUp(int value_to_round, int multiple, int max_value) {
34 : const int rounded_value =
35 0 : (value_to_round + multiple - 1) / multiple * multiple;
36 0 : return rounded_value <= max_value ? rounded_value
37 0 : : (max_value / multiple * multiple);
38 : }
39 :
40 : // Generates a scale factor that makes |input_num_pixels| smaller or
41 : // larger than |target_num_pixels|, depending on the value of |step_up|.
42 0 : Fraction FindScale(int input_num_pixels, int target_num_pixels, bool step_up) {
43 : // This function only makes sense for a positive target.
44 0 : RTC_DCHECK_GT(target_num_pixels, 0);
45 0 : Fraction best_scale = Fraction{1, 1};
46 0 : Fraction last_scale = Fraction{1, 1};
47 : const float target_scale =
48 0 : sqrt(target_num_pixels / static_cast<float>(input_num_pixels));
49 0 : while (best_scale.numerator > (target_scale * best_scale.denominator)) {
50 0 : last_scale = best_scale;
51 0 : if (best_scale.numerator % 3 == 0 && best_scale.denominator % 2 == 0) {
52 : // Multiply by 2/3
53 0 : best_scale.numerator /= 3;
54 0 : best_scale.denominator /= 2;
55 : } else {
56 : // Multiply by 3/4
57 0 : best_scale.numerator *= 3;
58 0 : best_scale.denominator *= 4;
59 : }
60 : }
61 0 : if (step_up)
62 0 : return last_scale;
63 0 : return best_scale;
64 : }
65 : } // namespace
66 :
67 : namespace cricket {
68 :
69 0 : VideoAdapter::VideoAdapter(int required_resolution_alignment)
70 : : frames_in_(0),
71 : frames_out_(0),
72 : frames_scaled_(0),
73 : adaption_changes_(0),
74 : previous_width_(0),
75 : previous_height_(0),
76 : required_resolution_alignment_(required_resolution_alignment),
77 0 : resolution_request_max_pixel_count_(std::numeric_limits<int>::max()),
78 0 : step_up_(false) {}
79 :
80 0 : VideoAdapter::VideoAdapter() : VideoAdapter(1) {}
81 :
82 0 : VideoAdapter::~VideoAdapter() {}
83 :
84 0 : bool VideoAdapter::KeepFrame(int64_t in_timestamp_ns) {
85 0 : rtc::CritScope cs(&critical_section_);
86 0 : if (!requested_format_ || requested_format_->interval == 0)
87 0 : return true;
88 :
89 0 : if (next_frame_timestamp_ns_) {
90 : // Time until next frame should be outputted.
91 : const int64_t time_until_next_frame_ns =
92 0 : (*next_frame_timestamp_ns_ - in_timestamp_ns);
93 :
94 : // Continue if timestamp is withing expected range.
95 0 : if (std::abs(time_until_next_frame_ns) < 2 * requested_format_->interval) {
96 : // Drop if a frame shouldn't be outputted yet.
97 0 : if (time_until_next_frame_ns > 0)
98 0 : return false;
99 : // Time to output new frame.
100 0 : *next_frame_timestamp_ns_ += requested_format_->interval;
101 0 : return true;
102 : }
103 : }
104 :
105 : // First timestamp received or timestamp is way outside expected range, so
106 : // reset. Set first timestamp target to just half the interval to prefer
107 : // keeping frames in case of jitter.
108 : next_frame_timestamp_ns_ =
109 0 : rtc::Optional<int64_t>(in_timestamp_ns + requested_format_->interval / 2);
110 0 : return true;
111 : }
112 :
113 0 : bool VideoAdapter::AdaptFrameResolution(int in_width,
114 : int in_height,
115 : int64_t in_timestamp_ns,
116 : int* cropped_width,
117 : int* cropped_height,
118 : int* out_width,
119 : int* out_height) {
120 0 : rtc::CritScope cs(&critical_section_);
121 0 : ++frames_in_;
122 :
123 : // The max output pixel count is the minimum of the requests from
124 : // OnOutputFormatRequest and OnResolutionRequest.
125 0 : int max_pixel_count = resolution_request_max_pixel_count_;
126 0 : if (requested_format_) {
127 : // TODO(kthelgason): remove the - |step_up_| hack when we change how
128 : // resolution is requested from VideoSourceProxy.
129 : // This is required because we must not scale above the requested
130 : // format so we subtract one when scaling up.
131 0 : max_pixel_count = std::min(
132 0 : max_pixel_count, requested_format_->width * requested_format_->height -
133 0 : static_cast<int>(step_up_));
134 : }
135 0 : if (scale_) {
136 0 : if (max_pixel_count == std::numeric_limits<int>::max()) {
137 0 : max_pixel_count = in_width * in_height;
138 : }
139 : // approximates (width/scale_resolution_by_) * (height/scale_resolution_by_)
140 0 : max_pixel_count = (max_pixel_count / scale_resolution_by_) / scale_resolution_by_;
141 : }
142 :
143 : // Drop the input frame if necessary.
144 0 : if (max_pixel_count <= 0 || !KeepFrame(in_timestamp_ns)) {
145 : // Show VAdapt log every 90 frames dropped. (3 seconds)
146 0 : if ((frames_in_ - frames_out_) % 90 == 0) {
147 : // TODO(fbarchard): Reduce to LS_VERBOSE when adapter info is not needed
148 : // in default calls.
149 0 : LOG(LS_INFO) << "VAdapt Drop Frame: scaled " << frames_scaled_
150 0 : << " / out " << frames_out_
151 0 : << " / in " << frames_in_
152 0 : << " Changes: " << adaption_changes_
153 0 : << " Input: " << in_width
154 0 : << "x" << in_height
155 0 : << " timestamp: " << in_timestamp_ns
156 0 : << " Output: i"
157 0 : << (requested_format_ ? requested_format_->interval : 0);
158 : }
159 :
160 : // Drop frame.
161 0 : return false;
162 : }
163 :
164 : // Calculate how the input should be cropped.
165 0 : if (!requested_format_ ||
166 0 : requested_format_->width == 0 || requested_format_->height == 0) {
167 0 : *cropped_width = in_width;
168 0 : *cropped_height = in_height;
169 : } else {
170 : // Adjust |requested_format_| orientation to match input.
171 0 : if ((in_width > in_height) !=
172 0 : (requested_format_->width > requested_format_->height)) {
173 0 : std::swap(requested_format_->width, requested_format_->height);
174 : }
175 : const float requested_aspect =
176 0 : requested_format_->width /
177 0 : static_cast<float>(requested_format_->height);
178 0 : *cropped_width =
179 0 : std::min(in_width, static_cast<int>(in_height * requested_aspect));
180 0 : *cropped_height =
181 0 : std::min(in_height, static_cast<int>(in_width / requested_aspect));
182 : }
183 : const Fraction scale =
184 0 : FindScale(*cropped_width * *cropped_height, max_pixel_count, step_up_);
185 : // Adjust cropping slightly to get even integer output size and a perfect
186 : // scale factor. Make sure the resulting dimensions are aligned correctly
187 : // to be nice to hardware encoders.
188 0 : *cropped_width =
189 0 : roundUp(*cropped_width,
190 0 : scale.denominator * required_resolution_alignment_, in_width);
191 0 : *cropped_height =
192 0 : roundUp(*cropped_height,
193 0 : scale.denominator * required_resolution_alignment_, in_height);
194 0 : RTC_DCHECK_EQ(0, *cropped_width % scale.denominator);
195 0 : RTC_DCHECK_EQ(0, *cropped_height % scale.denominator);
196 :
197 : // Calculate final output size.
198 0 : *out_width = *cropped_width / scale.denominator * scale.numerator;
199 0 : *out_height = *cropped_height / scale.denominator * scale.numerator;
200 0 : RTC_DCHECK_EQ(0, *out_height % required_resolution_alignment_);
201 0 : RTC_DCHECK_EQ(0, *out_height % required_resolution_alignment_);
202 :
203 0 : ++frames_out_;
204 0 : if (scale.numerator != scale.denominator)
205 0 : ++frames_scaled_;
206 :
207 0 : if ((previous_width_ || scale_) && (previous_width_ != *out_width ||
208 0 : previous_height_ != *out_height)) {
209 0 : ++adaption_changes_;
210 0 : LOG(LS_INFO) << "Frame size changed: scaled " << frames_scaled_ << " / out "
211 0 : << frames_out_ << " / in " << frames_in_
212 0 : << " Changes: " << adaption_changes_ << " Input: " << in_width
213 0 : << "x" << in_height
214 0 : << " Scale: " << scale.numerator << "/" << scale.denominator
215 0 : << " Output: " << *out_width << "x" << *out_height << " i"
216 0 : << (requested_format_ ? requested_format_->interval : 0);
217 : }
218 :
219 0 : previous_width_ = *out_width;
220 0 : previous_height_ = *out_height;
221 :
222 0 : return true;
223 : }
224 :
225 0 : void VideoAdapter::OnOutputFormatRequest(const VideoFormat& format) {
226 0 : rtc::CritScope cs(&critical_section_);
227 0 : requested_format_ = rtc::Optional<VideoFormat>(format);
228 0 : next_frame_timestamp_ns_ = rtc::Optional<int64_t>();
229 0 : }
230 :
231 0 : void VideoAdapter::OnResolutionRequest(
232 : rtc::Optional<int> max_pixel_count,
233 : rtc::Optional<int> max_pixel_count_step_up) {
234 0 : rtc::CritScope cs(&critical_section_);
235 0 : resolution_request_max_pixel_count_ = max_pixel_count.value_or(
236 0 : max_pixel_count_step_up.value_or(std::numeric_limits<int>::max()));
237 0 : step_up_ = static_cast<bool>(max_pixel_count_step_up);
238 0 : }
239 :
240 0 : void VideoAdapter::OnScaleResolutionBy(
241 : rtc::Optional<float> scale_resolution_by) {
242 0 : rtc::CritScope cs(&critical_section_);
243 0 : scale_resolution_by_ = scale_resolution_by.value_or(1.0);
244 0 : RTC_DCHECK_GE(scale_resolution_by_, 1.0);
245 0 : scale_ = static_cast<bool>(scale_resolution_by);
246 0 : }
247 :
248 : } // namespace cricket
|
