287 lines
10 KiB
C++
287 lines
10 KiB
C++
|
/*
|
||
|
|
* Copyright (c) 2018 The WebRTC project authors. All Rights Reserved.
|
||
|
|
*
|
||
|
|
* Use of this source code is governed by a BSD-style license
|
||
|
|
* that can be found in the LICENSE file in the root of the source
|
||
|
|
* tree. An additional intellectual property rights grant can be found
|
||
|
|
* in the file PATENTS. All contributing project authors may
|
||
|
|
* be found in the AUTHORS file in the root of the source tree.
|
||
|
|
*/
|
||
|
|
|
||
|
|
#include "video/video_quality_observer.h"
|
||
|
|
|
||
|
|
#include <algorithm>
|
||
|
|
#include <cmath>
|
||
|
|
#include <cstdint>
|
||
|
|
#include <string>
|
||
|
|
|
||
|
|
#include "rtc_base/logging.h"
|
||
|
|
#include "rtc_base/strings/string_builder.h"
|
||
|
|
#include "system_wrappers/include/metrics.h"
|
||
|
|
|
||
|
|
namespace webrtc {
|
||
|
|
const uint32_t VideoQualityObserver::kMinFrameSamplesToDetectFreeze = 5;
|
||
|
|
const uint32_t VideoQualityObserver::kMinIncreaseForFreezeMs = 150;
|
||
|
|
const uint32_t VideoQualityObserver::kAvgInterframeDelaysWindowSizeFrames = 30;
|
||
|
|
|
||
|
|
namespace {
|
||
|
|
constexpr int kMinVideoDurationMs = 3000;
|
||
|
|
constexpr int kMinRequiredSamples = 1;
|
||
|
|
constexpr int kPixelsInHighResolution =
|
||
|
|
960 * 540; // CPU-adapted HD still counts.
|
||
|
|
constexpr int kPixelsInMediumResolution = 640 * 360;
|
||
|
|
constexpr int kBlockyQpThresholdVp8 = 70;
|
||
|
|
constexpr int kBlockyQpThresholdVp9 = 180;
|
||
|
|
constexpr int kMaxNumCachedBlockyFrames = 100;
|
||
|
|
// TODO(ilnik): Add H264/HEVC thresholds.
|
||
|
|
} // namespace
|
||
|
|
|
||
|
|
VideoQualityObserver::VideoQualityObserver(VideoContentType content_type)
|
||
|
|
: last_frame_rendered_ms_(-1),
|
||
|
|
num_frames_rendered_(0),
|
||
|
|
first_frame_rendered_ms_(-1),
|
||
|
|
last_frame_pixels_(0),
|
||
|
|
is_last_frame_blocky_(false),
|
||
|
|
last_unfreeze_time_ms_(0),
|
||
|
|
render_interframe_delays_(kAvgInterframeDelaysWindowSizeFrames),
|
||
|
|
sum_squared_interframe_delays_secs_(0.0),
|
||
|
|
time_in_resolution_ms_(3, 0),
|
||
|
|
current_resolution_(Resolution::Low),
|
||
|
|
num_resolution_downgrades_(0),
|
||
|
|
time_in_blocky_video_ms_(0),
|
||
|
|
content_type_(content_type),
|
||
|
|
is_paused_(false) {}
|
||
|
|
|
||
|
|
void VideoQualityObserver::UpdateHistograms() {
|
||
|
|
// Don't report anything on an empty video stream.
|
||
|
|
if (num_frames_rendered_ == 0) {
|
||
|
|
return;
|
||
|
|
}
|
||
|
|
|
||
|
|
char log_stream_buf[2 * 1024];
|
||
|
|
rtc::SimpleStringBuilder log_stream(log_stream_buf);
|
||
|
|
|
||
|
|
if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
|
||
|
|
smooth_playback_durations_.Add(last_frame_rendered_ms_ -
|
||
|
|
last_unfreeze_time_ms_);
|
||
|
|
}
|
||
|
|
|
||
|
|
std::string uma_prefix = videocontenttypehelpers::IsScreenshare(content_type_)
|
||
|
|
? "WebRTC.Video.Screenshare"
|
||
|
|
: "WebRTC.Video";
|
||
|
|
|
||
|
|
auto mean_time_between_freezes =
|
||
|
|
smooth_playback_durations_.Avg(kMinRequiredSamples);
|
||
|
|
if (mean_time_between_freezes) {
|
||
|
|
RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanTimeBetweenFreezesMs",
|
||
|
|
*mean_time_between_freezes);
|
||
|
|
log_stream << uma_prefix << ".MeanTimeBetweenFreezesMs "
|
||
|
|
<< *mean_time_between_freezes << "\n";
|
||
|
|
}
|
||
|
|
auto avg_freeze_length = freezes_durations_.Avg(kMinRequiredSamples);
|
||
|
|
if (avg_freeze_length) {
|
||
|
|
RTC_HISTOGRAM_COUNTS_SPARSE_100000(uma_prefix + ".MeanFreezeDurationMs",
|
||
|
|
*avg_freeze_length);
|
||
|
|
log_stream << uma_prefix << ".MeanFreezeDurationMs " << *avg_freeze_length
|
||
|
|
<< "\n";
|
||
|
|
}
|
||
|
|
|
||
|
|
int64_t video_duration_ms =
|
||
|
|
last_frame_rendered_ms_ - first_frame_rendered_ms_;
|
||
|
|
|
||
|
|
if (video_duration_ms >= kMinVideoDurationMs) {
|
||
|
|
int time_spent_in_hd_percentage = static_cast<int>(
|
||
|
|
time_in_resolution_ms_[Resolution::High] * 100 / video_duration_ms);
|
||
|
|
RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInHdPercentage",
|
||
|
|
time_spent_in_hd_percentage);
|
||
|
|
log_stream << uma_prefix << ".TimeInHdPercentage "
|
||
|
|
<< time_spent_in_hd_percentage << "\n";
|
||
|
|
|
||
|
|
int time_with_blocky_video_percentage =
|
||
|
|
static_cast<int>(time_in_blocky_video_ms_ * 100 / video_duration_ms);
|
||
|
|
RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".TimeInBlockyVideoPercentage",
|
||
|
|
time_with_blocky_video_percentage);
|
||
|
|
log_stream << uma_prefix << ".TimeInBlockyVideoPercentage "
|
||
|
|
<< time_with_blocky_video_percentage << "\n";
|
||
|
|
|
||
|
|
int num_resolution_downgrades_per_minute =
|
||
|
|
num_resolution_downgrades_ * 60000 / video_duration_ms;
|
||
|
|
RTC_HISTOGRAM_COUNTS_SPARSE_100(
|
||
|
|
uma_prefix + ".NumberResolutionDownswitchesPerMinute",
|
||
|
|
num_resolution_downgrades_per_minute);
|
||
|
|
log_stream << uma_prefix << ".NumberResolutionDownswitchesPerMinute "
|
||
|
|
<< num_resolution_downgrades_per_minute << "\n";
|
||
|
|
|
||
|
|
int num_freezes_per_minute =
|
||
|
|
freezes_durations_.NumSamples() * 60000 / video_duration_ms;
|
||
|
|
RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".NumberFreezesPerMinute",
|
||
|
|
num_freezes_per_minute);
|
||
|
|
log_stream << uma_prefix << ".NumberFreezesPerMinute "
|
||
|
|
<< num_freezes_per_minute << "\n";
|
||
|
|
|
||
|
|
if (sum_squared_interframe_delays_secs_ > 0.0) {
|
||
|
|
int harmonic_framerate_fps = std::round(
|
||
|
|
video_duration_ms / (1000 * sum_squared_interframe_delays_secs_));
|
||
|
|
RTC_HISTOGRAM_COUNTS_SPARSE_100(uma_prefix + ".HarmonicFrameRate",
|
||
|
|
harmonic_framerate_fps);
|
||
|
|
log_stream << uma_prefix << ".HarmonicFrameRate "
|
||
|
|
<< harmonic_framerate_fps << "\n";
|
||
|
|
}
|
||
|
|
}
|
||
|
|
RTC_LOG(LS_INFO) << log_stream.str();
|
||
|
|
}
|
||
|
|
|
||
|
|
void VideoQualityObserver::OnRenderedFrame(const VideoFrame& frame,
|
||
|
|
int64_t now_ms) {
|
||
|
|
RTC_DCHECK_LE(last_frame_rendered_ms_, now_ms);
|
||
|
|
RTC_DCHECK_LE(last_unfreeze_time_ms_, now_ms);
|
||
|
|
|
||
|
|
if (num_frames_rendered_ == 0) {
|
||
|
|
first_frame_rendered_ms_ = last_unfreeze_time_ms_ = now_ms;
|
||
|
|
}
|
||
|
|
|
||
|
|
auto blocky_frame_it = blocky_frames_.find(frame.timestamp());
|
||
|
|
|
||
|
|
if (num_frames_rendered_ > 0) {
|
||
|
|
// Process inter-frame delay.
|
||
|
|
const int64_t interframe_delay_ms = now_ms - last_frame_rendered_ms_;
|
||
|
|
const double interframe_delays_secs = interframe_delay_ms / 1000.0;
|
||
|
|
|
||
|
|
// Sum of squared inter frame intervals is used to calculate the harmonic
|
||
|
|
// frame rate metric. The metric aims to reflect overall experience related
|
||
|
|
// to smoothness of video playback and includes both freezes and pauses.
|
||
|
|
sum_squared_interframe_delays_secs_ +=
|
||
|
|
interframe_delays_secs * interframe_delays_secs;
|
||
|
|
|
||
|
|
if (!is_paused_) {
|
||
|
|
render_interframe_delays_.AddSample(interframe_delay_ms);
|
||
|
|
|
||
|
|
bool was_freeze = false;
|
||
|
|
if (render_interframe_delays_.Size() >= kMinFrameSamplesToDetectFreeze) {
|
||
|
|
const absl::optional<int64_t> avg_interframe_delay =
|
||
|
|
render_interframe_delays_.GetAverageRoundedDown();
|
||
|
|
RTC_DCHECK(avg_interframe_delay);
|
||
|
|
was_freeze = interframe_delay_ms >=
|
||
|
|
std::max(3 * *avg_interframe_delay,
|
||
|
|
*avg_interframe_delay + kMinIncreaseForFreezeMs);
|
||
|
|
}
|
||
|
|
|
||
|
|
if (was_freeze) {
|
||
|
|
freezes_durations_.Add(interframe_delay_ms);
|
||
|
|
smooth_playback_durations_.Add(last_frame_rendered_ms_ -
|
||
|
|
last_unfreeze_time_ms_);
|
||
|
|
last_unfreeze_time_ms_ = now_ms;
|
||
|
|
} else {
|
||
|
|
// Count spatial metrics if there were no freeze.
|
||
|
|
time_in_resolution_ms_[current_resolution_] += interframe_delay_ms;
|
||
|
|
|
||
|
|
if (is_last_frame_blocky_) {
|
||
|
|
time_in_blocky_video_ms_ += interframe_delay_ms;
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
if (is_paused_) {
|
||
|
|
// If the stream was paused since the previous frame, do not count the
|
||
|
|
// pause toward smooth playback. Explicitly count the part before it and
|
||
|
|
// start the new smooth playback interval from this frame.
|
||
|
|
is_paused_ = false;
|
||
|
|
if (last_frame_rendered_ms_ > last_unfreeze_time_ms_) {
|
||
|
|
smooth_playback_durations_.Add(last_frame_rendered_ms_ -
|
||
|
|
last_unfreeze_time_ms_);
|
||
|
|
}
|
||
|
|
last_unfreeze_time_ms_ = now_ms;
|
||
|
|
|
||
|
|
if (num_frames_rendered_ > 0) {
|
||
|
|
pauses_durations_.Add(now_ms - last_frame_rendered_ms_);
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
int64_t pixels = frame.width() * frame.height();
|
||
|
|
if (pixels >= kPixelsInHighResolution) {
|
||
|
|
current_resolution_ = Resolution::High;
|
||
|
|
} else if (pixels >= kPixelsInMediumResolution) {
|
||
|
|
current_resolution_ = Resolution::Medium;
|
||
|
|
} else {
|
||
|
|
current_resolution_ = Resolution::Low;
|
||
|
|
}
|
||
|
|
|
||
|
|
if (pixels < last_frame_pixels_) {
|
||
|
|
++num_resolution_downgrades_;
|
||
|
|
}
|
||
|
|
|
||
|
|
last_frame_pixels_ = pixels;
|
||
|
|
last_frame_rendered_ms_ = now_ms;
|
||
|
|
|
||
|
|
is_last_frame_blocky_ = blocky_frame_it != blocky_frames_.end();
|
||
|
|
if (is_last_frame_blocky_) {
|
||
|
|
blocky_frames_.erase(blocky_frames_.begin(), ++blocky_frame_it);
|
||
|
|
}
|
||
|
|
|
||
|
|
++num_frames_rendered_;
|
||
|
|
}
|
||
|
|
|
||
|
|
void VideoQualityObserver::OnDecodedFrame(const VideoFrame& frame,
|
||
|
|
absl::optional<uint8_t> qp,
|
||
|
|
VideoCodecType codec) {
|
||
|
|
if (qp) {
|
||
|
|
absl::optional<int> qp_blocky_threshold;
|
||
|
|
// TODO(ilnik): add other codec types when we have QP for them.
|
||
|
|
switch (codec) {
|
||
|
|
case kVideoCodecVP8:
|
||
|
|
qp_blocky_threshold = kBlockyQpThresholdVp8;
|
||
|
|
break;
|
||
|
|
case kVideoCodecVP9:
|
||
|
|
qp_blocky_threshold = kBlockyQpThresholdVp9;
|
||
|
|
break;
|
||
|
|
default:
|
||
|
|
qp_blocky_threshold = absl::nullopt;
|
||
|
|
}
|
||
|
|
|
||
|
|
RTC_DCHECK(blocky_frames_.find(frame.timestamp()) == blocky_frames_.end());
|
||
|
|
|
||
|
|
if (qp_blocky_threshold && *qp > *qp_blocky_threshold) {
|
||
|
|
// Cache blocky frame. Its duration will be calculated in render callback.
|
||
|
|
if (blocky_frames_.size() > kMaxNumCachedBlockyFrames) {
|
||
|
|
RTC_LOG(LS_WARNING) << "Overflow of blocky frames cache.";
|
||
|
|
blocky_frames_.erase(
|
||
|
|
blocky_frames_.begin(),
|
||
|
|
std::next(blocky_frames_.begin(), kMaxNumCachedBlockyFrames / 2));
|
||
|
|
}
|
||
|
|
|
||
|
|
blocky_frames_.insert(frame.timestamp());
|
||
|
|
}
|
||
|
|
}
|
||
|
|
}
|
||
|
|
|
||
|
|
void VideoQualityObserver::OnStreamInactive() {
|
||
|
|
is_paused_ = true;
|
||
|
|
}
|
||
|
|
|
||
|
|
uint32_t VideoQualityObserver::NumFreezes() const {
|
||
|
|
return freezes_durations_.NumSamples();
|
||
|
|
}
|
||
|
|
|
||
|
|
uint32_t VideoQualityObserver::NumPauses() const {
|
||
|
|
return pauses_durations_.NumSamples();
|
||
|
|
}
|
||
|
|
|
||
|
|
uint32_t VideoQualityObserver::TotalFreezesDurationMs() const {
|
||
|
|
return freezes_durations_.Sum(kMinRequiredSamples).value_or(0);
|
||
|
|
}
|
||
|
|
|
||
|
|
uint32_t VideoQualityObserver::TotalPausesDurationMs() const {
|
||
|
|
return pauses_durations_.Sum(kMinRequiredSamples).value_or(0);
|
||
|
|
}
|
||
|
|
|
||
|
|
uint32_t VideoQualityObserver::TotalFramesDurationMs() const {
|
||
|
|
return last_frame_rendered_ms_ - first_frame_rendered_ms_;
|
||
|
|
}
|
||
|
|
|
||
|
|
double VideoQualityObserver::SumSquaredFrameDurationsSec() const {
|
||
|
|
return sum_squared_interframe_delays_secs_;
|
||
|
|
}
|
||
|
|
|
||
|
|
} // namespace webrtc
|