Line data Source code
1 : /*
2 : * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 : *
4 : * Use of this source code is governed by a BSD-style license
5 : * that can be found in the LICENSE file in the root of the source
6 : * tree. An additional intellectual property rights grant can be found
7 : * in the file PATENTS. All contributing project authors may
8 : * be found in the AUTHORS file in the root of the source tree.
9 : */
10 :
11 : #ifndef WEBRTC_VIDEO_VIE_ENCODER_H_
12 : #define WEBRTC_VIDEO_VIE_ENCODER_H_
13 :
14 : #include <memory>
15 : #include <string>
16 : #include <vector>
17 :
18 : #include "webrtc/api/video/video_rotation.h"
19 : #include "webrtc/base/criticalsection.h"
20 : #include "webrtc/base/event.h"
21 : #include "webrtc/base/sequenced_task_checker.h"
22 : #include "webrtc/base/task_queue.h"
23 : #include "webrtc/call/call.h"
24 : #include "webrtc/common_types.h"
25 : #include "webrtc/common_video/include/video_bitrate_allocator.h"
26 : #include "webrtc/media/base/videosinkinterface.h"
27 : #include "webrtc/modules/video_coding/include/video_coding_defines.h"
28 : #include "webrtc/modules/video_coding/utility/quality_scaler.h"
29 : #include "webrtc/modules/video_coding/video_coding_impl.h"
30 : #include "webrtc/system_wrappers/include/atomic32.h"
31 : #include "webrtc/typedefs.h"
32 : #include "webrtc/video/overuse_frame_detector.h"
33 : #include "webrtc/video_encoder.h"
34 : #include "webrtc/video_send_stream.h"
35 :
36 : namespace webrtc {
37 :
38 : class ProcessThread;
39 : class SendStatisticsProxy;
40 : class VideoBitrateAllocationObserver;
41 :
42 : // VieEncoder represent a video encoder that accepts raw video frames as input
43 : // and produces an encoded bit stream.
44 : // Usage:
45 : // Instantiate.
46 : // Call SetSink.
47 : // Call SetSource.
48 : // Call ConfigureEncoder with the codec settings.
49 : // Call Stop() when done.
50 : class ViEEncoder : public rtc::VideoSinkInterface<VideoFrame>,
51 : public EncodedImageCallback,
52 : public VCMSendStatisticsCallback,
53 : public CPULoadStateObserver,
54 : public ScalingObserverInterface {
55 : public:
56 : // Interface for receiving encoded video frames and notifications about
57 : // configuration changes.
58 0 : class EncoderSink : public EncodedImageCallback {
59 : public:
60 : virtual void OnEncoderConfigurationChanged(
61 : std::vector<VideoStream> streams,
62 : int min_transmit_bitrate_bps) = 0;
63 : };
64 :
65 : // Downscale resolution at most 2 times for CPU reasons.
66 : static const int kMaxCpuDowngrades = 2;
67 :
68 : ViEEncoder(uint32_t number_of_cores,
69 : SendStatisticsProxy* stats_proxy,
70 : const VideoSendStream::Config::EncoderSettings& settings,
71 : rtc::VideoSinkInterface<VideoFrame>* pre_encode_callback,
72 : EncodedFrameObserver* encoder_timing);
73 : ~ViEEncoder();
74 : // RegisterProcessThread register |module_process_thread| with those objects
75 : // that use it. Registration has to happen on the thread where
76 : // |module_process_thread| was created (libjingle's worker thread).
77 : // TODO(perkj): Replace the use of |module_process_thread| with a TaskQueue.
78 : void RegisterProcessThread(ProcessThread* module_process_thread);
79 : void DeRegisterProcessThread();
80 :
81 : // Sets the source that will provide I420 video frames.
82 : // |degradation_preference| control whether or not resolution or frame rate
83 : // may be reduced.
84 : void SetSource(
85 : rtc::VideoSourceInterface<VideoFrame>* source,
86 : const VideoSendStream::DegradationPreference& degradation_preference);
87 :
88 : // Sets the |sink| that gets the encoded frames. |rotation_applied| means
89 : // that the source must support rotation. Only set |rotation_applied| if the
90 : // remote side does not support the rotation extension.
91 : void SetSink(EncoderSink* sink, bool rotation_applied);
92 :
93 : // TODO(perkj): Can we remove VideoCodec.startBitrate ?
94 : void SetStartBitrate(int start_bitrate_bps);
95 :
96 : void SetBitrateObserver(VideoBitrateAllocationObserver* bitrate_observer);
97 :
98 : void ConfigureEncoder(VideoEncoderConfig config,
99 : size_t max_data_payload_length,
100 : bool nack_enabled);
101 :
102 : // Permanently stop encoding. After this method has returned, it is
103 : // guaranteed that no encoded frames will be delivered to the sink.
104 : void Stop();
105 :
106 : void SendKeyFrame();
107 :
108 : // virtual to test EncoderStateFeedback with mocks.
109 : virtual void OnReceivedIntraFrameRequest(size_t stream_index);
110 : virtual void OnReceivedSLI(uint8_t picture_id);
111 : virtual void OnReceivedRPSI(uint64_t picture_id);
112 :
113 : void OnBitrateUpdated(uint32_t bitrate_bps,
114 : uint8_t fraction_lost,
115 : int64_t round_trip_time_ms);
116 :
117 : // CPULoadStateObserver interface
118 : void onLoadStateChanged(CPULoadState state) override;
119 :
120 : protected:
121 : // Used for testing. For example the |ScalingObserverInterface| methods must
122 : // be called on |encoder_queue_|.
123 : rtc::TaskQueue* encoder_queue() { return &encoder_queue_; }
124 :
125 : // webrtc::ScalingObserverInterface implementation.
126 : // These methods are protected for easier testing.
127 : void ScaleUp(ScaleReason reason) override;
128 : void ScaleDown(ScaleReason reason) override;
129 :
130 : private:
131 : class ConfigureEncoderTask;
132 : class EncodeTask;
133 : class VideoSourceProxy;
134 :
135 : class VideoFrameInfo {
136 : public:
137 0 : VideoFrameInfo(int width,
138 : int height,
139 : VideoRotation rotation,
140 : bool is_texture)
141 0 : : width(width),
142 : height(height),
143 : rotation(rotation),
144 0 : is_texture(is_texture) {}
145 : int width;
146 : int height;
147 : VideoRotation rotation;
148 : bool is_texture;
149 0 : int pixel_count() const { return width * height; }
150 : };
151 :
152 : void ConfigureEncoderOnTaskQueue(VideoEncoderConfig config,
153 : size_t max_data_payload_length,
154 : bool nack_enabled);
155 : void ReconfigureEncoder();
156 :
157 : // Implements VideoSinkInterface.
158 : void OnFrame(const VideoFrame& video_frame) override;
159 :
160 : // Implements VideoSendStatisticsCallback.
161 : void SendStatistics(uint32_t bit_rate,
162 : uint32_t frame_rate) override;
163 :
164 : void EncodeVideoFrame(const VideoFrame& frame,
165 : int64_t time_when_posted_in_ms);
166 :
167 : // Implements EncodedImageCallback.
168 : EncodedImageCallback::Result OnEncodedImage(
169 : const EncodedImage& encoded_image,
170 : const CodecSpecificInfo* codec_specific_info,
171 : const RTPFragmentationHeader* fragmentation) override;
172 :
173 : void OnDroppedFrame() override;
174 :
175 : bool EncoderPaused() const;
176 : void TraceFrameDropStart();
177 : void TraceFrameDropEnd();
178 :
179 : rtc::Event shutdown_event_;
180 :
181 : const uint32_t number_of_cores_;
182 :
183 : const std::unique_ptr<VideoSourceProxy> source_proxy_;
184 : EncoderSink* sink_;
185 : const VideoSendStream::Config::EncoderSettings settings_;
186 : const VideoCodecType codec_type_;
187 :
188 : vcm::VideoSender video_sender_ ACCESS_ON(&encoder_queue_);
189 : OveruseFrameDetector overuse_detector_ ACCESS_ON(&encoder_queue_);
190 : std::unique_ptr<QualityScaler> quality_scaler_ ACCESS_ON(&encoder_queue_);
191 :
192 : SendStatisticsProxy* const stats_proxy_;
193 : rtc::VideoSinkInterface<VideoFrame>* const pre_encode_callback_;
194 : ProcessThread* module_process_thread_;
195 : rtc::ThreadChecker module_process_thread_checker_;
196 : // |thread_checker_| checks that public methods that are related to lifetime
197 : // of ViEEncoder are called on the same thread.
198 : rtc::ThreadChecker thread_checker_;
199 :
200 : VideoEncoderConfig encoder_config_ ACCESS_ON(&encoder_queue_);
201 : std::unique_ptr<VideoBitrateAllocator> rate_allocator_
202 : ACCESS_ON(&encoder_queue_);
203 :
204 : // Set when ConfigureEncoder has been called in order to lazy reconfigure the
205 : // encoder on the next frame.
206 : bool pending_encoder_reconfiguration_ ACCESS_ON(&encoder_queue_);
207 : rtc::Optional<VideoFrameInfo> last_frame_info_ ACCESS_ON(&encoder_queue_);
208 : uint32_t encoder_start_bitrate_bps_ ACCESS_ON(&encoder_queue_);
209 : size_t max_data_payload_length_ ACCESS_ON(&encoder_queue_);
210 : bool nack_enabled_ ACCESS_ON(&encoder_queue_);
211 : uint32_t last_observed_bitrate_bps_ ACCESS_ON(&encoder_queue_);
212 : bool encoder_paused_and_dropped_frame_ ACCESS_ON(&encoder_queue_);
213 : bool has_received_sli_ ACCESS_ON(&encoder_queue_);
214 : uint8_t picture_id_sli_ ACCESS_ON(&encoder_queue_);
215 : bool has_received_rpsi_ ACCESS_ON(&encoder_queue_);
216 : uint64_t picture_id_rpsi_ ACCESS_ON(&encoder_queue_);
217 : Clock* const clock_;
218 : // Counters used for deciding if the video resolution is currently
219 : // restricted, and if so, why.
220 : std::vector<int> scale_counter_ ACCESS_ON(&encoder_queue_);
221 : // Set depending on degradation preferences
222 : bool scaling_enabled_ ACCESS_ON(&encoder_queue_) = false;
223 :
224 : // Pixel count last time the resolution was requested to be changed down.
225 : rtc::Optional<int> max_pixel_count_ ACCESS_ON(&encoder_queue_);
226 : // Pixel count last time the resolution was requested to be changed up.
227 : rtc::Optional<int> max_pixel_count_step_up_ ACCESS_ON(&encoder_queue_);
228 :
229 : rtc::RaceChecker incoming_frame_race_checker_
230 : GUARDED_BY(incoming_frame_race_checker_);
231 : Atomic32 posted_frames_waiting_for_encode_;
232 : // Used to make sure incoming time stamp is increasing for every frame.
233 : int64_t last_captured_timestamp_ GUARDED_BY(incoming_frame_race_checker_);
234 : // Delta used for translating between NTP and internal timestamps.
235 : const int64_t delta_ntp_internal_ms_ GUARDED_BY(incoming_frame_race_checker_);
236 :
237 : int64_t last_frame_log_ms_ GUARDED_BY(incoming_frame_race_checker_);
238 : int captured_frame_count_ ACCESS_ON(&encoder_queue_);
239 : int dropped_frame_count_ ACCESS_ON(&encoder_queue_);
240 :
241 : VideoBitrateAllocationObserver* bitrate_observer_ ACCESS_ON(&encoder_queue_);
242 : rtc::Optional<int64_t> last_parameters_update_ms_ ACCESS_ON(&encoder_queue_);
243 :
244 : // All public methods are proxied to |encoder_queue_|. It must must be
245 : // destroyed first to make sure no tasks are run that use other members.
246 : rtc::TaskQueue encoder_queue_;
247 :
248 : RTC_DISALLOW_COPY_AND_ASSIGN(ViEEncoder);
249 : };
250 :
251 : } // namespace webrtc
252 :
253 : #endif // WEBRTC_VIDEO_VIE_ENCODER_H_
|
